authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-10 17:37:33-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-12 13:14:51-08:00
log6744160211b2ab480e4c479e46440c76f00c1810
tree111370974ffe2f641aa9f4e1027c9522117f8a43
parent5c59a4623898cdb15b569f59fba49d48a7b9600f

zig libc: implement malloc

27 files changed, 198 insertions(+), 3546 deletions(-)

lib/c.zig+8-5
......@@ -47,19 +47,22 @@ pub fn errno(syscall_return_value: usize) c_int {
4747}
4848
4949comptime {
50 _ = @import("c/inttypes.zig");
5150 _ = @import("c/ctype.zig");
52 _ = @import("c/stdlib.zig");
51 _ = @import("c/inttypes.zig");
52 if (!builtin.target.isMinGW()) {
53 _ = @import("c/malloc.zig");
54 }
5355 _ = @import("c/math.zig");
56 _ = @import("c/stdlib.zig");
5457 _ = @import("c/string.zig");
5558 _ = @import("c/strings.zig");
56 _ = @import("c/wchar.zig");
5759
58 _ = @import("c/sys/mman.zig");
60 _ = @import("c/sys/capability.zig");
5961 _ = @import("c/sys/file.zig");
62 _ = @import("c/sys/mman.zig");
6063 _ = @import("c/sys/reboot.zig");
61 _ = @import("c/sys/capability.zig");
6264 _ = @import("c/sys/utsname.zig");
6365
6466 _ = @import("c/unistd.zig");
67 _ = @import("c/wchar.zig");
6568}
lib/c/malloc.zig created+182
......@@ -0,0 +1,182 @@
1//! Based on wrapping a stateless Zig Allocator implementation, apropriate for:
2//! - ReleaseFast and ReleaseSmall optimization modes, with multi-threading
3//! enabled.
4//! - WebAssembly in single-threaded mode.
5//!
6//! Because the libc APIs don't have client alignment and size tracking, in
7//! order to take advantage of Zig allocator implementations, additional
8//! metadata must be stored in the allocations.
9//!
10//! This implementation stores the metadata just before the pointer returned
11//! from `malloc`, just like many libc malloc implementations do, including
12//! musl. This has the downside of causing fragmentation for allocations with
13//! higher alignment, however most of that memory can be recovered by
14//! preemptively putting the gap onto the freelist.
15const builtin = @import("builtin");
16
17const std = @import("std");
18const assert = std.debug.assert;
19const Alignment = std.mem.Alignment;
20const alignment_bytes = @max(@alignOf(std.c.max_align_t), @sizeOf(Header));
21const alignment: Alignment = .fromByteUnits(alignment_bytes);
22
23const symbol = @import("../c.zig").symbol;
24
25comptime {
26 symbol(&malloc, "malloc");
27 symbol(&aligned_alloc, "aligned_alloc");
28 symbol(&posix_memalign, "posix_memalign");
29 symbol(&calloc, "calloc");
30 symbol(&realloc, "realloc");
31 symbol(&reallocarray, "reallocarray");
32 symbol(&free, "free");
33 symbol(&malloc_usable_size, "malloc_usable_size");
34
35 symbol(&valloc, "valloc");
36 symbol(&memalign, "memalign");
37}
38
39const no_context: *anyopaque = undefined;
40const no_ra: usize = undefined;
41const vtable = switch (builtin.cpu.arch) {
42 .wasm32, .wasm64 => std.heap.WasmAllocator.vtable,
43 else => std.heap.SmpAllocator.vtable,
44};
45
46/// Needed because libc memory allocators don't provide old alignment and size
47/// which are required by Zig memory allocators.
48const Header = packed struct(u64) {
49 alignment: Alignment,
50 /// Does not include the extra alignment bytes added.
51 size: Size,
52 padding: Padding = 0,
53
54 comptime {
55 assert(@sizeOf(Header) <= alignment_bytes);
56 }
57
58 const Size = @Int(.unsigned, @min(64 - @bitSizeOf(Alignment), @bitSizeOf(usize)));
59 const Padding = @Int(.unsigned, 64 - @bitSizeOf(Alignment) - @bitSizeOf(Size));
60
61 fn fromBase(base: [*]align(alignment_bytes) u8) *Header {
62 return @ptrCast(base - @sizeOf(Header));
63 }
64};
65
66fn malloc(n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
67 const size = std.math.cast(Header.Size, n) orelse return null;
68 const ptr: [*]align(alignment_bytes) u8 = @alignCast(
69 vtable.alloc(no_context, n + alignment_bytes, alignment, no_ra) orelse return null,
70 );
71 const base = ptr + alignment_bytes;
72 const header: *Header = .fromBase(base);
73 header.* = .{
74 .alignment = alignment,
75 .size = size,
76 };
77 return base;
78}
79
80fn aligned_alloc(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
81 const size = std.math.cast(Header.Size, n) orelse return null;
82 const max_align = alignment.max(.fromByteUnits(alloc_alignment));
83 const max_align_bytes = max_align.toByteUnits();
84 const ptr: [*]align(alignment_bytes) u8 = @alignCast(
85 vtable.alloc(no_context, n + max_align_bytes, max_align, no_ra) orelse return null,
86 );
87 const base: [*]align(alignment_bytes) u8 = @alignCast(ptr + max_align_bytes);
88 const header: *Header = .fromBase(base);
89 header.* = .{
90 .alignment = max_align,
91 .size = size,
92 };
93 return base;
94}
95
96fn calloc(elems: usize, len: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
97 const n = std.math.mul(usize, elems, len) catch return null;
98 const base = malloc(n) orelse return null;
99 @memset(base[0..n], 0);
100 return base;
101}
102
103fn realloc(opt_old_base: ?[*]align(alignment_bytes) u8, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
104 if (n == 0) {
105 free(opt_old_base);
106 return null;
107 }
108 const old_base = opt_old_base orelse return malloc(n);
109 const new_size = std.math.cast(Header.Size, n) orelse return null;
110 const old_header: *Header = .fromBase(old_base);
111 assert(old_header.padding == 0);
112 const old_size = old_header.size;
113 const old_alignment = old_header.alignment;
114 const old_alignment_bytes = old_alignment.toByteUnits();
115 const old_ptr = old_base - old_alignment_bytes;
116 const old_slice = old_ptr[0 .. old_size + old_alignment_bytes];
117 const new_base: [*]align(alignment_bytes) u8 = if (vtable.remap(
118 no_context,
119 old_slice,
120 old_alignment,
121 n + old_alignment_bytes,
122 no_ra,
123 )) |new_ptr| @alignCast(new_ptr + old_alignment_bytes) else b: {
124 const new_ptr: [*]align(alignment_bytes) u8 = @alignCast(
125 vtable.alloc(no_context, n + old_alignment_bytes, old_alignment, no_ra) orelse return null,
126 );
127 const new_base: [*]align(alignment_bytes) u8 = @alignCast(new_ptr + old_alignment_bytes);
128 const copy_len = @min(new_size, old_size);
129 @memcpy(new_base[0..copy_len], old_base[0..copy_len]);
130 vtable.free(no_context, old_slice, old_alignment, no_ra);
131 break :b new_base;
132 };
133 const new_header: *Header = .fromBase(new_base);
134 new_header.* = .{
135 .alignment = old_alignment,
136 .size = new_size,
137 };
138 return new_base;
139}
140
141fn reallocarray(opt_base: ?[*]align(alignment_bytes) u8, elems: usize, len: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
142 const n = std.math.mul(usize, elems, len) catch return null;
143 return realloc(opt_base, n);
144}
145
146fn free(opt_old_base: ?[*]align(alignment_bytes) u8) callconv(.c) void {
147 const old_base = opt_old_base orelse return;
148 const old_header: *Header = .fromBase(old_base);
149 assert(old_header.padding == 0);
150 const old_size = old_header.size;
151 const old_alignment = old_header.alignment;
152 const old_alignment_bytes = old_alignment.toByteUnits();
153 const old_ptr = old_base - old_alignment_bytes;
154 const old_slice = old_ptr[0 .. old_size + old_alignment_bytes];
155 vtable.free(no_context, old_slice, old_alignment, no_ra);
156}
157
158fn malloc_usable_size(opt_old_base: ?[*]align(alignment_bytes) u8) callconv(.c) usize {
159 const old_base = opt_old_base orelse return 0;
160 const old_header: *Header = .fromBase(old_base);
161 assert(old_header.padding == 0);
162 const old_size = old_header.size;
163 return old_size;
164}
165
166fn valloc(n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
167 return aligned_alloc(std.heap.pageSize(), n);
168}
169
170fn memalign(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
171 return aligned_alloc(alloc_alignment, n);
172}
173
174fn posix_memalign(result: *?[*]align(alignment_bytes) u8, alloc_alignment: usize, n: usize) callconv(.c) c_int {
175 if (alloc_alignment < @sizeOf(*anyopaque)) return @intFromEnum(std.c.E.INVAL);
176 if (n == 0) {
177 result.* = null;
178 } else {
179 result.* = aligned_alloc(alloc_alignment, n) orelse return @intFromEnum(std.c.E.NOMEM);
180 }
181 return 0;
182}
lib/libc/musl/src/malloc/calloc.c deleted-45
......@@ -1,45 +0,0 @@
1#include <stdlib.h>
2#include <stdint.h>
3#include <string.h>
4#include <errno.h>
5#include "dynlink.h"
6
7static size_t mal0_clear(char *p, size_t n)
8{
9 const size_t pagesz = 4096; /* arbitrary */
10 if (n < pagesz) return n;
11#ifdef __GNUC__
12 typedef uint64_t __attribute__((__may_alias__)) T;
13#else
14 typedef unsigned char T;
15#endif
16 char *pp = p + n;
17 size_t i = (uintptr_t)pp & (pagesz - 1);
18 for (;;) {
19 pp = memset(pp - i, 0, i);
20 if (pp - p < pagesz) return pp - p;
21 for (i = pagesz; i; i -= 2*sizeof(T), pp -= 2*sizeof(T))
22 if (((T *)pp)[-1] | ((T *)pp)[-2])
23 break;
24 }
25}
26
27static int allzerop(void *p)
28{
29 return 0;
30}
31weak_alias(allzerop, __malloc_allzerop);
32
33void *calloc(size_t m, size_t n)
34{
35 if (n && m > (size_t)-1/n) {
36 errno = ENOMEM;
37 return 0;
38 }
39 n *= m;
40 void *p = malloc(n);
41 if (!p || (!__malloc_replaced && __malloc_allzerop(p)))
42 return p;
43 n = mal0_clear(p, n);
44 return memset(p, 0, n);
45}
lib/libc/musl/src/malloc/free.c deleted-6
......@@ -1,6 +0,0 @@
1#include <stdlib.h>
2
3void free(void *p)
4{
5 __libc_free(p);
6}
lib/libc/musl/src/malloc/libc_calloc.c deleted-4
......@@ -1,4 +0,0 @@
1#define calloc __libc_calloc
2#define malloc __libc_malloc
3
4#include "calloc.c"
lib/libc/musl/src/malloc/lite_malloc.c deleted-118
......@@ -1,118 +0,0 @@
1#include <stdlib.h>
2#include <stdint.h>
3#include <limits.h>
4#include <errno.h>
5#include <sys/mman.h>
6#include "libc.h"
7#include "lock.h"
8#include "syscall.h"
9#include "fork_impl.h"
10
11#define ALIGN 16
12
13/* This function returns true if the interval [old,new]
14 * intersects the 'len'-sized interval below &libc.auxv
15 * (interpreted as the main-thread stack) or below &b
16 * (the current stack). It is used to defend against
17 * buggy brk implementations that can cross the stack. */
18
19static int traverses_stack_p(uintptr_t old, uintptr_t new)
20{
21 const uintptr_t len = 8<<20;
22 uintptr_t a, b;
23
24 b = (uintptr_t)libc.auxv;
25 a = b > len ? b-len : 0;
26 if (new>a && old<b) return 1;
27
28 b = (uintptr_t)&b;
29 a = b > len ? b-len : 0;
30 if (new>a && old<b) return 1;
31
32 return 0;
33}
34
35static volatile int lock[1];
36volatile int *const __bump_lockptr = lock;
37
38static void *__simple_malloc(size_t n)
39{
40 static uintptr_t brk, cur, end;
41 static unsigned mmap_step;
42 size_t align=1;
43 void *p;
44
45 if (n > SIZE_MAX/2) {
46 errno = ENOMEM;
47 return 0;
48 }
49
50 if (!n) n++;
51 while (align<n && align<ALIGN)
52 align += align;
53
54 LOCK(lock);
55
56 cur += -cur & align-1;
57
58 if (n > end-cur) {
59 size_t req = n - (end-cur) + PAGE_SIZE-1 & -PAGE_SIZE;
60
61 if (!cur) {
62 brk = __syscall(SYS_brk, 0);
63 brk += -brk & PAGE_SIZE-1;
64 cur = end = brk;
65 }
66
67 if (brk == end && req < SIZE_MAX-brk
68 && !traverses_stack_p(brk, brk+req)
69 && __syscall(SYS_brk, brk+req)==brk+req) {
70 brk = end += req;
71 } else {
72 int new_area = 0;
73 req = n + PAGE_SIZE-1 & -PAGE_SIZE;
74 /* Only make a new area rather than individual mmap
75 * if wasted space would be over 1/8 of the map. */
76 if (req-n > req/8) {
77 /* Geometric area size growth up to 64 pages,
78 * bounding waste by 1/8 of the area. */
79 size_t min = PAGE_SIZE<<(mmap_step/2);
80 if (min-n > end-cur) {
81 if (req < min) {
82 req = min;
83 if (mmap_step < 12)
84 mmap_step++;
85 }
86 new_area = 1;
87 }
88 }
89 void *mem = __mmap(0, req, PROT_READ|PROT_WRITE,
90 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
91 if (mem == MAP_FAILED || !new_area) {
92 UNLOCK(lock);
93 return mem==MAP_FAILED ? 0 : mem;
94 }
95 cur = (uintptr_t)mem;
96 end = cur + req;
97 }
98 }
99
100 p = (void *)cur;
101 cur += n;
102 UNLOCK(lock);
103 return p;
104}
105
106weak_alias(__simple_malloc, __libc_malloc_impl);
107
108void *__libc_malloc(size_t n)
109{
110 return __libc_malloc_impl(n);
111}
112
113static void *default_malloc(size_t n)
114{
115 return __libc_malloc_impl(n);
116}
117
118weak_alias(default_malloc, malloc);
lib/libc/musl/src/malloc/mallocng/aligned_alloc.c deleted-60
......@@ -1,60 +0,0 @@
1#include <stdlib.h>
2#include <errno.h>
3#include "meta.h"
4
5void *aligned_alloc(size_t align, size_t len)
6{
7 if ((align & -align) != align) {
8 errno = EINVAL;
9 return 0;
10 }
11
12 if (len > SIZE_MAX - align || align >= (1ULL<<31)*UNIT) {
13 errno = ENOMEM;
14 return 0;
15 }
16
17 if (DISABLE_ALIGNED_ALLOC) {
18 errno = ENOMEM;
19 return 0;
20 }
21
22 if (align <= UNIT) align = UNIT;
23
24 unsigned char *p = malloc(len + align - UNIT);
25 if (!p)
26 return 0;
27
28 struct meta *g = get_meta(p);
29 int idx = get_slot_index(p);
30 size_t stride = get_stride(g);
31 unsigned char *start = g->mem->storage + stride*idx;
32 unsigned char *end = g->mem->storage + stride*(idx+1) - IB;
33 size_t adj = -(uintptr_t)p & (align-1);
34
35 if (!adj) {
36 set_size(p, end, len);
37 return p;
38 }
39 p += adj;
40 uint32_t offset = (size_t)(p-g->mem->storage)/UNIT;
41 if (offset <= 0xffff) {
42 *(uint16_t *)(p-2) = offset;
43 p[-4] = 0;
44 } else {
45 // use a 32-bit offset if 16-bit doesn't fit. for this,
46 // 16-bit field must be zero, [-4] byte nonzero.
47 *(uint16_t *)(p-2) = 0;
48 *(uint32_t *)(p-8) = offset;
49 p[-4] = 1;
50 }
51 p[-3] = idx;
52 set_size(p, end, len);
53 // store offset to aligned enframing. this facilitates cycling
54 // offset and also iteration of heap for debugging/measurement.
55 // for extreme overalignment it won't fit but these are classless
56 // allocations anyway.
57 *(uint16_t *)(start - 2) = (size_t)(p-start)/UNIT;
58 start[-3] = 7<<5;
59 return p;
60}
lib/libc/musl/src/malloc/mallocng/donate.c deleted-39
......@@ -1,39 +0,0 @@
1#include <stdlib.h>
2#include <stdint.h>
3#include <limits.h>
4#include <string.h>
5#include <sys/mman.h>
6#include <errno.h>
7
8#include "meta.h"
9
10static void donate(unsigned char *base, size_t len)
11{
12 uintptr_t a = (uintptr_t)base;
13 uintptr_t b = a + len;
14 a += -a & (UNIT-1);
15 b -= b & (UNIT-1);
16 memset(base, 0, len);
17 for (int sc=47; sc>0 && b>a; sc-=4) {
18 if (b-a < (size_classes[sc]+1)*UNIT) continue;
19 struct meta *m = alloc_meta();
20 m->avail_mask = 0;
21 m->freed_mask = 1;
22 m->mem = (void *)a;
23 m->mem->meta = m;
24 m->last_idx = 0;
25 m->freeable = 0;
26 m->sizeclass = sc;
27 m->maplen = 0;
28 *((unsigned char *)m->mem+UNIT-4) = 0;
29 *((unsigned char *)m->mem+UNIT-3) = 255;
30 m->mem->storage[size_classes[sc]*UNIT-4] = 0;
31 queue(&ctx.active[sc], m);
32 a += (size_classes[sc]+1)*UNIT;
33 }
34}
35
36void __malloc_donate(char *start, char *end)
37{
38 donate((void *)start, end-start);
39}
lib/libc/musl/src/malloc/mallocng/free.c deleted-151
......@@ -1,151 +0,0 @@
1#define _BSD_SOURCE
2#include <stdlib.h>
3#include <sys/mman.h>
4
5#include "meta.h"
6
7struct mapinfo {
8 void *base;
9 size_t len;
10};
11
12static struct mapinfo nontrivial_free(struct meta *, int);
13
14static struct mapinfo free_group(struct meta *g)
15{
16 struct mapinfo mi = { 0 };
17 int sc = g->sizeclass;
18 if (sc < 48) {
19 ctx.usage_by_class[sc] -= g->last_idx+1;
20 }
21 if (g->maplen) {
22 step_seq();
23 record_seq(sc);
24 mi.base = g->mem;
25 mi.len = g->maplen*4096UL;
26 } else {
27 void *p = g->mem;
28 struct meta *m = get_meta(p);
29 int idx = get_slot_index(p);
30 g->mem->meta = 0;
31 // not checking size/reserved here; it's intentionally invalid
32 mi = nontrivial_free(m, idx);
33 }
34 free_meta(g);
35 return mi;
36}
37
38static int okay_to_free(struct meta *g)
39{
40 int sc = g->sizeclass;
41
42 if (!g->freeable) return 0;
43
44 // always free individual mmaps not suitable for reuse
45 if (sc >= 48 || get_stride(g) < UNIT*size_classes[sc])
46 return 1;
47
48 // always free groups allocated inside another group's slot
49 // since recreating them should not be expensive and they
50 // might be blocking freeing of a much larger group.
51 if (!g->maplen) return 1;
52
53 // if there is another non-full group, free this one to
54 // consolidate future allocations, reduce fragmentation.
55 if (g->next != g) return 1;
56
57 // free any group in a size class that's not bouncing
58 if (!is_bouncing(sc)) return 1;
59
60 size_t cnt = g->last_idx+1;
61 size_t usage = ctx.usage_by_class[sc];
62
63 // if usage is high enough that a larger count should be
64 // used, free the low-count group so a new one will be made.
65 if (9*cnt <= usage && cnt < 20)
66 return 1;
67
68 // otherwise, keep the last group in a bouncing class.
69 return 0;
70}
71
72static struct mapinfo nontrivial_free(struct meta *g, int i)
73{
74 uint32_t self = 1u<<i;
75 int sc = g->sizeclass;
76 uint32_t mask = g->freed_mask | g->avail_mask;
77
78 if (mask+self == (2u<<g->last_idx)-1 && okay_to_free(g)) {
79 // any multi-slot group is necessarily on an active list
80 // here, but single-slot groups might or might not be.
81 if (g->next) {
82 assert(sc < 48);
83 int activate_new = (ctx.active[sc]==g);
84 dequeue(&ctx.active[sc], g);
85 if (activate_new && ctx.active[sc])
86 activate_group(ctx.active[sc]);
87 }
88 return free_group(g);
89 } else if (!mask) {
90 assert(sc < 48);
91 // might still be active if there were no allocations
92 // after last available slot was taken.
93 if (ctx.active[sc] != g) {
94 queue(&ctx.active[sc], g);
95 }
96 }
97 a_or(&g->freed_mask, self);
98 return (struct mapinfo){ 0 };
99}
100
101void free(void *p)
102{
103 if (!p) return;
104
105 struct meta *g = get_meta(p);
106 int idx = get_slot_index(p);
107 size_t stride = get_stride(g);
108 unsigned char *start = g->mem->storage + stride*idx;
109 unsigned char *end = start + stride - IB;
110 get_nominal_size(p, end);
111 uint32_t self = 1u<<idx, all = (2u<<g->last_idx)-1;
112 ((unsigned char *)p)[-3] = 255;
113 // invalidate offset to group header, and cycle offset of
114 // used region within slot if current offset is zero.
115 *(uint16_t *)((char *)p-2) = 0;
116
117 // release any whole pages contained in the slot to be freed
118 // unless it's a single-slot group that will be unmapped.
119 if (((uintptr_t)(start-1) ^ (uintptr_t)end) >= 2*PGSZ && g->last_idx) {
120 unsigned char *base = start + (-(uintptr_t)start & (PGSZ-1));
121 size_t len = (end-base) & -PGSZ;
122 if (len && USE_MADV_FREE) {
123 int e = errno;
124 madvise(base, len, MADV_FREE);
125 errno = e;
126 }
127 }
128
129 // atomic free without locking if this is neither first or last slot
130 for (;;) {
131 uint32_t freed = g->freed_mask;
132 uint32_t avail = g->avail_mask;
133 uint32_t mask = freed | avail;
134 assert(!(mask&self));
135 if (!freed || mask+self==all) break;
136 if (!MT)
137 g->freed_mask = freed+self;
138 else if (a_cas(&g->freed_mask, freed, freed+self)!=freed)
139 continue;
140 return;
141 }
142
143 wrlock();
144 struct mapinfo mi = nontrivial_free(g, idx);
145 unlock();
146 if (mi.len) {
147 int e = errno;
148 munmap(mi.base, mi.len);
149 errno = e;
150 }
151}
lib/libc/musl/src/malloc/mallocng/glue.h deleted-95
......@@ -1,95 +0,0 @@
1#ifndef MALLOC_GLUE_H
2#define MALLOC_GLUE_H
3
4#include <stdint.h>
5#include <sys/mman.h>
6#include <pthread.h>
7#include <unistd.h>
8#include <elf.h>
9#include <string.h>
10#include "atomic.h"
11#include "syscall.h"
12#include "libc.h"
13#include "lock.h"
14#include "dynlink.h"
15
16// use macros to appropriately namespace these.
17#define size_classes __malloc_size_classes
18#define ctx __malloc_context
19#define alloc_meta __malloc_alloc_meta
20#define is_allzero __malloc_allzerop
21#define dump_heap __dump_heap
22
23#define malloc __libc_malloc_impl
24#define realloc __libc_realloc
25#define free __libc_free
26
27#define USE_MADV_FREE 0
28
29#if USE_REAL_ASSERT
30#include <assert.h>
31#else
32#undef assert
33#define assert(x) do { if (!(x)) a_crash(); } while(0)
34#endif
35
36#define brk(p) ((uintptr_t)__syscall(SYS_brk, p))
37
38#define mmap __mmap
39#define madvise __madvise
40#define mremap __mremap
41
42#define DISABLE_ALIGNED_ALLOC (__malloc_replaced && !__aligned_alloc_replaced)
43
44static inline uint64_t get_random_secret()
45{
46 uint64_t secret = (uintptr_t)&secret * 1103515245;
47 for (size_t i=0; libc.auxv[i]; i+=2)
48 if (libc.auxv[i]==AT_RANDOM)
49 memcpy(&secret, (char *)libc.auxv[i+1]+8, sizeof secret);
50 return secret;
51}
52
53#ifndef PAGESIZE
54#define PAGESIZE PAGE_SIZE
55#endif
56
57#define MT (libc.need_locks)
58
59#define RDLOCK_IS_EXCLUSIVE 1
60
61__attribute__((__visibility__("hidden")))
62extern int __malloc_lock[1];
63
64#define LOCK_OBJ_DEF \
65int __malloc_lock[1]; \
66void __malloc_atfork(int who) { malloc_atfork(who); }
67
68static inline void rdlock()
69{
70 if (MT) LOCK(__malloc_lock);
71}
72static inline void wrlock()
73{
74 if (MT) LOCK(__malloc_lock);
75}
76static inline void unlock()
77{
78 UNLOCK(__malloc_lock);
79}
80static inline void upgradelock()
81{
82}
83static inline void resetlock()
84{
85 __malloc_lock[0] = 0;
86}
87
88static inline void malloc_atfork(int who)
89{
90 if (who<0) rdlock();
91 else if (who>0) resetlock();
92 else unlock();
93}
94
95#endif
lib/libc/musl/src/malloc/mallocng/malloc.c deleted-387
......@@ -1,387 +0,0 @@
1#include <stdlib.h>
2#include <stdint.h>
3#include <limits.h>
4#include <string.h>
5#include <sys/mman.h>
6#include <errno.h>
7
8#include "meta.h"
9
10LOCK_OBJ_DEF;
11
12const uint16_t size_classes[] = {
13 1, 2, 3, 4, 5, 6, 7, 8,
14 9, 10, 12, 15,
15 18, 20, 25, 31,
16 36, 42, 50, 63,
17 72, 84, 102, 127,
18 146, 170, 204, 255,
19 292, 340, 409, 511,
20 584, 682, 818, 1023,
21 1169, 1364, 1637, 2047,
22 2340, 2730, 3276, 4095,
23 4680, 5460, 6552, 8191,
24};
25
26static const uint8_t small_cnt_tab[][3] = {
27 { 30, 30, 30 },
28 { 31, 15, 15 },
29 { 20, 10, 10 },
30 { 31, 15, 7 },
31 { 25, 12, 6 },
32 { 21, 10, 5 },
33 { 18, 8, 4 },
34 { 31, 15, 7 },
35 { 28, 14, 6 },
36};
37
38static const uint8_t med_cnt_tab[4] = { 28, 24, 20, 32 };
39
40struct malloc_context ctx = { 0 };
41
42struct meta *alloc_meta(void)
43{
44 struct meta *m;
45 unsigned char *p;
46 if (!ctx.init_done) {
47#ifndef PAGESIZE
48 ctx.pagesize = get_page_size();
49#endif
50 ctx.secret = get_random_secret();
51 ctx.init_done = 1;
52 }
53 size_t pagesize = PGSZ;
54 if (pagesize < 4096) pagesize = 4096;
55 if ((m = dequeue_head(&ctx.free_meta_head))) return m;
56 if (!ctx.avail_meta_count) {
57 int need_unprotect = 1;
58 if (!ctx.avail_meta_area_count && ctx.brk!=-1) {
59 uintptr_t new = ctx.brk + pagesize;
60 int need_guard = 0;
61 if (!ctx.brk) {
62 need_guard = 1;
63 ctx.brk = brk(0);
64 // some ancient kernels returned _ebss
65 // instead of next page as initial brk.
66 ctx.brk += -ctx.brk & (pagesize-1);
67 new = ctx.brk + 2*pagesize;
68 }
69 if (brk(new) != new) {
70 ctx.brk = -1;
71 } else {
72 if (need_guard) mmap((void *)ctx.brk, pagesize,
73 PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0);
74 ctx.brk = new;
75 ctx.avail_meta_areas = (void *)(new - pagesize);
76 ctx.avail_meta_area_count = pagesize>>12;
77 need_unprotect = 0;
78 }
79 }
80 if (!ctx.avail_meta_area_count) {
81 size_t n = 2UL << ctx.meta_alloc_shift;
82 p = mmap(0, n*pagesize, PROT_NONE,
83 MAP_PRIVATE|MAP_ANON, -1, 0);
84 if (p==MAP_FAILED) return 0;
85 ctx.avail_meta_areas = p + pagesize;
86 ctx.avail_meta_area_count = (n-1)*(pagesize>>12);
87 ctx.meta_alloc_shift++;
88 }
89 p = ctx.avail_meta_areas;
90 if ((uintptr_t)p & (pagesize-1)) need_unprotect = 0;
91 if (need_unprotect)
92 if (mprotect(p, pagesize, PROT_READ|PROT_WRITE)
93 && errno != ENOSYS)
94 return 0;
95 ctx.avail_meta_area_count--;
96 ctx.avail_meta_areas = p + 4096;
97 if (ctx.meta_area_tail) {
98 ctx.meta_area_tail->next = (void *)p;
99 } else {
100 ctx.meta_area_head = (void *)p;
101 }
102 ctx.meta_area_tail = (void *)p;
103 ctx.meta_area_tail->check = ctx.secret;
104 ctx.avail_meta_count = ctx.meta_area_tail->nslots
105 = (4096-sizeof(struct meta_area))/sizeof *m;
106 ctx.avail_meta = ctx.meta_area_tail->slots;
107 }
108 ctx.avail_meta_count--;
109 m = ctx.avail_meta++;
110 m->prev = m->next = 0;
111 return m;
112}
113
114static uint32_t try_avail(struct meta **pm)
115{
116 struct meta *m = *pm;
117 uint32_t first;
118 if (!m) return 0;
119 uint32_t mask = m->avail_mask;
120 if (!mask) {
121 if (!m) return 0;
122 if (!m->freed_mask) {
123 dequeue(pm, m);
124 m = *pm;
125 if (!m) return 0;
126 } else {
127 m = m->next;
128 *pm = m;
129 }
130
131 mask = m->freed_mask;
132
133 // skip fully-free group unless it's the only one
134 // or it's a permanently non-freeable group
135 if (mask == (2u<<m->last_idx)-1 && m->freeable) {
136 m = m->next;
137 *pm = m;
138 mask = m->freed_mask;
139 }
140
141 // activate more slots in a not-fully-active group
142 // if needed, but only as a last resort. prefer using
143 // any other group with free slots. this avoids
144 // touching & dirtying as-yet-unused pages.
145 if (!(mask & ((2u<<m->mem->active_idx)-1))) {
146 if (m->next != m) {
147 m = m->next;
148 *pm = m;
149 } else {
150 int cnt = m->mem->active_idx + 2;
151 int size = size_classes[m->sizeclass]*UNIT;
152 int span = UNIT + size*cnt;
153 // activate up to next 4k boundary
154 while ((span^(span+size-1)) < 4096) {
155 cnt++;
156 span += size;
157 }
158 if (cnt > m->last_idx+1)
159 cnt = m->last_idx+1;
160 m->mem->active_idx = cnt-1;
161 }
162 }
163 mask = activate_group(m);
164 assert(mask);
165 decay_bounces(m->sizeclass);
166 }
167 first = mask&-mask;
168 m->avail_mask = mask-first;
169 return first;
170}
171
172static int alloc_slot(int, size_t);
173
174static struct meta *alloc_group(int sc, size_t req)
175{
176 size_t size = UNIT*size_classes[sc];
177 int i = 0, cnt;
178 unsigned char *p;
179 struct meta *m = alloc_meta();
180 if (!m) return 0;
181 size_t usage = ctx.usage_by_class[sc];
182 size_t pagesize = PGSZ;
183 int active_idx;
184 if (sc < 9) {
185 while (i<2 && 4*small_cnt_tab[sc][i] > usage)
186 i++;
187 cnt = small_cnt_tab[sc][i];
188 } else {
189 // lookup max number of slots fitting in power-of-two size
190 // from a table, along with number of factors of two we
191 // can divide out without a remainder or reaching 1.
192 cnt = med_cnt_tab[sc&3];
193
194 // reduce cnt to avoid excessive eagar allocation.
195 while (!(cnt&1) && 4*cnt > usage)
196 cnt >>= 1;
197
198 // data structures don't support groups whose slot offsets
199 // in units don't fit in 16 bits.
200 while (size*cnt >= 65536*UNIT)
201 cnt >>= 1;
202 }
203
204 // If we selected a count of 1 above but it's not sufficient to use
205 // mmap, increase to 2. Then it might be; if not it will nest.
206 if (cnt==1 && size*cnt+UNIT <= pagesize/2) cnt = 2;
207
208 // All choices of size*cnt are "just below" a power of two, so anything
209 // larger than half the page size should be allocated as whole pages.
210 if (size*cnt+UNIT > pagesize/2) {
211 // check/update bounce counter to start/increase retention
212 // of freed maps, and inhibit use of low-count, odd-size
213 // small mappings and single-slot groups if activated.
214 int nosmall = is_bouncing(sc);
215 account_bounce(sc);
216 step_seq();
217
218 // since the following count reduction opportunities have
219 // an absolute memory usage cost, don't overdo them. count
220 // coarse usage as part of usage.
221 if (!(sc&1) && sc<32) usage += ctx.usage_by_class[sc+1];
222
223 // try to drop to a lower count if the one found above
224 // increases usage by more than 25%. these reduced counts
225 // roughly fill an integral number of pages, just not a
226 // power of two, limiting amount of unusable space.
227 if (4*cnt > usage && !nosmall) {
228 if (0);
229 else if ((sc&3)==1 && size*cnt>8*pagesize) cnt = 2;
230 else if ((sc&3)==2 && size*cnt>4*pagesize) cnt = 3;
231 else if ((sc&3)==0 && size*cnt>8*pagesize) cnt = 3;
232 else if ((sc&3)==0 && size*cnt>2*pagesize) cnt = 5;
233 }
234 size_t needed = size*cnt + UNIT;
235 needed += -needed & (pagesize-1);
236
237 // produce an individually-mmapped allocation if usage is low,
238 // bounce counter hasn't triggered, and either it saves memory
239 // or it avoids eagar slot allocation without wasting too much.
240 if (!nosmall && cnt<=7) {
241 req += IB + UNIT;
242 req += -req & (pagesize-1);
243 if (req<size+UNIT || (req>=4*pagesize && 2*cnt>usage)) {
244 cnt = 1;
245 needed = req;
246 }
247 }
248
249 p = mmap(0, needed, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0);
250 if (p==MAP_FAILED) {
251 free_meta(m);
252 return 0;
253 }
254 m->maplen = needed>>12;
255 ctx.mmap_counter++;
256 active_idx = (4096-UNIT)/size-1;
257 if (active_idx > cnt-1) active_idx = cnt-1;
258 if (active_idx < 0) active_idx = 0;
259 } else {
260 int j = size_to_class(UNIT+cnt*size-IB);
261 int idx = alloc_slot(j, UNIT+cnt*size-IB);
262 if (idx < 0) {
263 free_meta(m);
264 return 0;
265 }
266 struct meta *g = ctx.active[j];
267 p = enframe(g, idx, UNIT*size_classes[j]-IB, ctx.mmap_counter);
268 m->maplen = 0;
269 p[-3] = (p[-3]&31) | (6<<5);
270 for (int i=0; i<=cnt; i++)
271 p[UNIT+i*size-4] = 0;
272 active_idx = cnt-1;
273 }
274 ctx.usage_by_class[sc] += cnt;
275 m->avail_mask = (2u<<active_idx)-1;
276 m->freed_mask = (2u<<(cnt-1))-1 - m->avail_mask;
277 m->mem = (void *)p;
278 m->mem->meta = m;
279 m->mem->active_idx = active_idx;
280 m->last_idx = cnt-1;
281 m->freeable = 1;
282 m->sizeclass = sc;
283 return m;
284}
285
286static int alloc_slot(int sc, size_t req)
287{
288 uint32_t first = try_avail(&ctx.active[sc]);
289 if (first) return a_ctz_32(first);
290
291 struct meta *g = alloc_group(sc, req);
292 if (!g) return -1;
293
294 g->avail_mask--;
295 queue(&ctx.active[sc], g);
296 return 0;
297}
298
299void *malloc(size_t n)
300{
301 if (size_overflows(n)) return 0;
302 struct meta *g;
303 uint32_t mask, first;
304 int sc;
305 int idx;
306 int ctr;
307
308 if (n >= MMAP_THRESHOLD) {
309 size_t needed = n + IB + UNIT;
310 void *p = mmap(0, needed, PROT_READ|PROT_WRITE,
311 MAP_PRIVATE|MAP_ANON, -1, 0);
312 if (p==MAP_FAILED) return 0;
313 wrlock();
314 step_seq();
315 g = alloc_meta();
316 if (!g) {
317 unlock();
318 munmap(p, needed);
319 return 0;
320 }
321 g->mem = p;
322 g->mem->meta = g;
323 g->last_idx = 0;
324 g->freeable = 1;
325 g->sizeclass = 63;
326 g->maplen = (needed+4095)/4096;
327 g->avail_mask = g->freed_mask = 0;
328 // use a global counter to cycle offset in
329 // individually-mmapped allocations.
330 ctx.mmap_counter++;
331 idx = 0;
332 goto success;
333 }
334
335 sc = size_to_class(n);
336
337 rdlock();
338 g = ctx.active[sc];
339
340 // use coarse size classes initially when there are not yet
341 // any groups of desired size. this allows counts of 2 or 3
342 // to be allocated at first rather than having to start with
343 // 7 or 5, the min counts for even size classes.
344 if (!g && sc>=4 && sc<32 && sc!=6 && !(sc&1) && !ctx.usage_by_class[sc]) {
345 size_t usage = ctx.usage_by_class[sc|1];
346 // if a new group may be allocated, count it toward
347 // usage in deciding if we can use coarse class.
348 if (!ctx.active[sc|1] || (!ctx.active[sc|1]->avail_mask
349 && !ctx.active[sc|1]->freed_mask))
350 usage += 3;
351 if (usage <= 12)
352 sc |= 1;
353 g = ctx.active[sc];
354 }
355
356 for (;;) {
357 mask = g ? g->avail_mask : 0;
358 first = mask&-mask;
359 if (!first) break;
360 if (RDLOCK_IS_EXCLUSIVE || !MT)
361 g->avail_mask = mask-first;
362 else if (a_cas(&g->avail_mask, mask, mask-first)!=mask)
363 continue;
364 idx = a_ctz_32(first);
365 goto success;
366 }
367 upgradelock();
368
369 idx = alloc_slot(sc, n);
370 if (idx < 0) {
371 unlock();
372 return 0;
373 }
374 g = ctx.active[sc];
375
376success:
377 ctr = ctx.mmap_counter;
378 unlock();
379 return enframe(g, idx, n, ctr);
380}
381
382int is_allzero(void *p)
383{
384 struct meta *g = get_meta(p);
385 return g->sizeclass >= 48 ||
386 get_stride(g) < UNIT*size_classes[g->sizeclass];
387}
lib/libc/musl/src/malloc/mallocng/malloc_usable_size.c deleted-13
......@@ -1,13 +0,0 @@
1#include <stdlib.h>
2#include "meta.h"
3
4size_t malloc_usable_size(void *p)
5{
6 if (!p) return 0;
7 struct meta *g = get_meta(p);
8 int idx = get_slot_index(p);
9 size_t stride = get_stride(g);
10 unsigned char *start = g->mem->storage + stride*idx;
11 unsigned char *end = start + stride - IB;
12 return get_nominal_size(p, end);
13}
lib/libc/musl/src/malloc/mallocng/meta.h deleted-288
......@@ -1,288 +0,0 @@
1#ifndef MALLOC_META_H
2#define MALLOC_META_H
3
4#include <stdint.h>
5#include <errno.h>
6#include <limits.h>
7#include "glue.h"
8
9__attribute__((__visibility__("hidden")))
10extern const uint16_t size_classes[];
11
12#define MMAP_THRESHOLD 131052
13
14#define UNIT 16
15#define IB 4
16
17struct group {
18 struct meta *meta;
19 unsigned char active_idx:5;
20 char pad[UNIT - sizeof(struct meta *) - 1];
21 unsigned char storage[];
22};
23
24struct meta {
25 struct meta *prev, *next;
26 struct group *mem;
27 volatile int avail_mask, freed_mask;
28 uintptr_t last_idx:5;
29 uintptr_t freeable:1;
30 uintptr_t sizeclass:6;
31 uintptr_t maplen:8*sizeof(uintptr_t)-12;
32};
33
34struct meta_area {
35 uint64_t check;
36 struct meta_area *next;
37 int nslots;
38 struct meta slots[];
39};
40
41struct malloc_context {
42 uint64_t secret;
43#ifndef PAGESIZE
44 size_t pagesize;
45#endif
46 int init_done;
47 unsigned mmap_counter;
48 struct meta *free_meta_head;
49 struct meta *avail_meta;
50 size_t avail_meta_count, avail_meta_area_count, meta_alloc_shift;
51 struct meta_area *meta_area_head, *meta_area_tail;
52 unsigned char *avail_meta_areas;
53 struct meta *active[48];
54 size_t usage_by_class[48];
55 uint8_t unmap_seq[32], bounces[32];
56 uint8_t seq;
57 uintptr_t brk;
58};
59
60__attribute__((__visibility__("hidden")))
61extern struct malloc_context ctx;
62
63#ifdef PAGESIZE
64#define PGSZ PAGESIZE
65#else
66#define PGSZ ctx.pagesize
67#endif
68
69__attribute__((__visibility__("hidden")))
70struct meta *alloc_meta(void);
71
72__attribute__((__visibility__("hidden")))
73int is_allzero(void *);
74
75static inline void queue(struct meta **phead, struct meta *m)
76{
77 assert(!m->next);
78 assert(!m->prev);
79 if (*phead) {
80 struct meta *head = *phead;
81 m->next = head;
82 m->prev = head->prev;
83 m->next->prev = m->prev->next = m;
84 } else {
85 m->prev = m->next = m;
86 *phead = m;
87 }
88}
89
90static inline void dequeue(struct meta **phead, struct meta *m)
91{
92 if (m->next != m) {
93 m->prev->next = m->next;
94 m->next->prev = m->prev;
95 if (*phead == m) *phead = m->next;
96 } else {
97 *phead = 0;
98 }
99 m->prev = m->next = 0;
100}
101
102static inline struct meta *dequeue_head(struct meta **phead)
103{
104 struct meta *m = *phead;
105 if (m) dequeue(phead, m);
106 return m;
107}
108
109static inline void free_meta(struct meta *m)
110{
111 *m = (struct meta){0};
112 queue(&ctx.free_meta_head, m);
113}
114
115static inline uint32_t activate_group(struct meta *m)
116{
117 assert(!m->avail_mask);
118 uint32_t mask, act = (2u<<m->mem->active_idx)-1;
119 do mask = m->freed_mask;
120 while (a_cas(&m->freed_mask, mask, mask&~act)!=mask);
121 return m->avail_mask = mask & act;
122}
123
124static inline int get_slot_index(const unsigned char *p)
125{
126 return p[-3] & 31;
127}
128
129static inline struct meta *get_meta(const unsigned char *p)
130{
131 assert(!((uintptr_t)p & 15));
132 int offset = *(const uint16_t *)(p - 2);
133 int index = get_slot_index(p);
134 if (p[-4]) {
135 assert(!offset);
136 offset = *(uint32_t *)(p - 8);
137 assert(offset > 0xffff);
138 }
139 const struct group *base = (const void *)(p - UNIT*offset - UNIT);
140 const struct meta *meta = base->meta;
141 assert(meta->mem == base);
142 assert(index <= meta->last_idx);
143 assert(!(meta->avail_mask & (1u<<index)));
144 assert(!(meta->freed_mask & (1u<<index)));
145 const struct meta_area *area = (void *)((uintptr_t)meta & -4096);
146 assert(area->check == ctx.secret);
147 if (meta->sizeclass < 48) {
148 assert(offset >= size_classes[meta->sizeclass]*index);
149 assert(offset < size_classes[meta->sizeclass]*(index+1));
150 } else {
151 assert(meta->sizeclass == 63);
152 }
153 if (meta->maplen) {
154 assert(offset <= meta->maplen*4096UL/UNIT - 1);
155 }
156 return (struct meta *)meta;
157}
158
159static inline size_t get_nominal_size(const unsigned char *p, const unsigned char *end)
160{
161 size_t reserved = p[-3] >> 5;
162 if (reserved >= 5) {
163 assert(reserved == 5);
164 reserved = *(const uint32_t *)(end-4);
165 assert(reserved >= 5);
166 assert(!end[-5]);
167 }
168 assert(reserved <= end-p);
169 assert(!*(end-reserved));
170 // also check the slot's overflow byte
171 assert(!*end);
172 return end-reserved-p;
173}
174
175static inline size_t get_stride(const struct meta *g)
176{
177 if (!g->last_idx && g->maplen) {
178 return g->maplen*4096UL - UNIT;
179 } else {
180 return UNIT*size_classes[g->sizeclass];
181 }
182}
183
184static inline void set_size(unsigned char *p, unsigned char *end, size_t n)
185{
186 int reserved = end-p-n;
187 if (reserved) end[-reserved] = 0;
188 if (reserved >= 5) {
189 *(uint32_t *)(end-4) = reserved;
190 end[-5] = 0;
191 reserved = 5;
192 }
193 p[-3] = (p[-3]&31) + (reserved<<5);
194}
195
196static inline void *enframe(struct meta *g, int idx, size_t n, int ctr)
197{
198 size_t stride = get_stride(g);
199 size_t slack = (stride-IB-n)/UNIT;
200 unsigned char *p = g->mem->storage + stride*idx;
201 unsigned char *end = p+stride-IB;
202 // cycle offset within slot to increase interval to address
203 // reuse, facilitate trapping double-free.
204 int off = (p[-3] ? *(uint16_t *)(p-2) + 1 : ctr) & 255;
205 assert(!p[-4]);
206 if (off > slack) {
207 size_t m = slack;
208 m |= m>>1; m |= m>>2; m |= m>>4;
209 off &= m;
210 if (off > slack) off -= slack+1;
211 assert(off <= slack);
212 }
213 if (off) {
214 // store offset in unused header at offset zero
215 // if enframing at non-zero offset.
216 *(uint16_t *)(p-2) = off;
217 p[-3] = 7<<5;
218 p += UNIT*off;
219 // for nonzero offset there is no permanent check
220 // byte, so make one.
221 p[-4] = 0;
222 }
223 *(uint16_t *)(p-2) = (size_t)(p-g->mem->storage)/UNIT;
224 p[-3] = idx;
225 set_size(p, end, n);
226 return p;
227}
228
229static inline int size_to_class(size_t n)
230{
231 n = (n+IB-1)>>4;
232 if (n<10) return n;
233 n++;
234 int i = (28-a_clz_32(n))*4 + 8;
235 if (n>size_classes[i+1]) i+=2;
236 if (n>size_classes[i]) i++;
237 return i;
238}
239
240static inline int size_overflows(size_t n)
241{
242 if (n >= SIZE_MAX/2 - 4096) {
243 errno = ENOMEM;
244 return 1;
245 }
246 return 0;
247}
248
249static inline void step_seq(void)
250{
251 if (ctx.seq==255) {
252 for (int i=0; i<32; i++) ctx.unmap_seq[i] = 0;
253 ctx.seq = 1;
254 } else {
255 ctx.seq++;
256 }
257}
258
259static inline void record_seq(int sc)
260{
261 if (sc-7U < 32) ctx.unmap_seq[sc-7] = ctx.seq;
262}
263
264static inline void account_bounce(int sc)
265{
266 if (sc-7U < 32) {
267 int seq = ctx.unmap_seq[sc-7];
268 if (seq && ctx.seq-seq < 10) {
269 if (ctx.bounces[sc-7]+1 < 100)
270 ctx.bounces[sc-7]++;
271 else
272 ctx.bounces[sc-7] = 150;
273 }
274 }
275}
276
277static inline void decay_bounces(int sc)
278{
279 if (sc-7U < 32 && ctx.bounces[sc-7])
280 ctx.bounces[sc-7]--;
281}
282
283static inline int is_bouncing(int sc)
284{
285 return (sc-7U < 32 && ctx.bounces[sc-7] >= 100);
286}
287
288#endif
lib/libc/musl/src/malloc/mallocng/realloc.c deleted-51
......@@ -1,51 +0,0 @@
1#define _GNU_SOURCE
2#include <stdlib.h>
3#include <sys/mman.h>
4#include <string.h>
5#include "meta.h"
6
7void *realloc(void *p, size_t n)
8{
9 if (!p) return malloc(n);
10 if (size_overflows(n)) return 0;
11
12 struct meta *g = get_meta(p);
13 int idx = get_slot_index(p);
14 size_t stride = get_stride(g);
15 unsigned char *start = g->mem->storage + stride*idx;
16 unsigned char *end = start + stride - IB;
17 size_t old_size = get_nominal_size(p, end);
18 size_t avail_size = end-(unsigned char *)p;
19 void *new;
20
21 // only resize in-place if size class matches
22 if (n <= avail_size && n<MMAP_THRESHOLD
23 && size_to_class(n)+1 >= g->sizeclass) {
24 set_size(p, end, n);
25 return p;
26 }
27
28 // use mremap if old and new size are both mmap-worthy
29 if (g->sizeclass>=48 && n>=MMAP_THRESHOLD) {
30 assert(g->sizeclass==63);
31 size_t base = (unsigned char *)p-start;
32 size_t needed = (n + base + UNIT + IB + 4095) & -4096;
33 new = g->maplen*4096UL == needed ? g->mem :
34 mremap(g->mem, g->maplen*4096UL, needed, MREMAP_MAYMOVE);
35 if (new!=MAP_FAILED) {
36 g->mem = new;
37 g->maplen = needed/4096;
38 p = g->mem->storage + base;
39 end = g->mem->storage + (needed - UNIT) - IB;
40 *end = 0;
41 set_size(p, end, n);
42 return p;
43 }
44 }
45
46 new = malloc(n);
47 if (!new) return 0;
48 memcpy(new, p, n < old_size ? n : old_size);
49 free(p);
50 return new;
51}
lib/libc/musl/src/malloc/memalign.c deleted-7
......@@ -1,7 +0,0 @@
1#define _BSD_SOURCE
2#include <stdlib.h>
3
4void *memalign(size_t align, size_t len)
5{
6 return aligned_alloc(align, len);
7}
lib/libc/musl/src/malloc/oldmalloc/aligned_alloc.c deleted-53
......@@ -1,53 +0,0 @@
1#include <stdlib.h>
2#include <stdint.h>
3#include <errno.h>
4#include "malloc_impl.h"
5
6void *aligned_alloc(size_t align, size_t len)
7{
8 unsigned char *mem, *new;
9
10 if ((align & -align) != align) {
11 errno = EINVAL;
12 return 0;
13 }
14
15 if (len > SIZE_MAX - align ||
16 (__malloc_replaced && !__aligned_alloc_replaced)) {
17 errno = ENOMEM;
18 return 0;
19 }
20
21 if (align <= SIZE_ALIGN)
22 return malloc(len);
23
24 if (!(mem = malloc(len + align-1)))
25 return 0;
26
27 new = (void *)((uintptr_t)mem + align-1 & -align);
28 if (new == mem) return mem;
29
30 struct chunk *c = MEM_TO_CHUNK(mem);
31 struct chunk *n = MEM_TO_CHUNK(new);
32
33 if (IS_MMAPPED(c)) {
34 /* Apply difference between aligned and original
35 * address to the "extra" field of mmapped chunk. */
36 n->psize = c->psize + (new-mem);
37 n->csize = c->csize - (new-mem);
38 return new;
39 }
40
41 struct chunk *t = NEXT_CHUNK(c);
42
43 /* Split the allocated chunk into two chunks. The aligned part
44 * that will be used has the size in its footer reduced by the
45 * difference between the aligned and original addresses, and
46 * the resulting size copied to its header. A new header and
47 * footer are written for the split-off part to be freed. */
48 n->psize = c->csize = C_INUSE | (new-mem);
49 n->csize = t->psize -= new-mem;
50
51 __bin_chunk(c);
52 return new;
53}
lib/libc/musl/src/malloc/oldmalloc/malloc.c deleted-556
......@@ -1,556 +0,0 @@
1#define _GNU_SOURCE
2#include <stdlib.h>
3#include <string.h>
4#include <limits.h>
5#include <stdint.h>
6#include <errno.h>
7#include <sys/mman.h>
8#include "libc.h"
9#include "atomic.h"
10#include "pthread_impl.h"
11#include "malloc_impl.h"
12#include "fork_impl.h"
13
14#define malloc __libc_malloc_impl
15#define realloc __libc_realloc
16#define free __libc_free
17
18#if defined(__GNUC__) && defined(__PIC__)
19#define inline inline __attribute__((always_inline))
20#endif
21
22static struct {
23 volatile uint64_t binmap;
24 struct bin bins[64];
25 volatile int split_merge_lock[2];
26} mal;
27
28/* Synchronization tools */
29
30static inline void lock(volatile int *lk)
31{
32 int need_locks = libc.need_locks;
33 if (need_locks) {
34 while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1);
35 if (need_locks < 0) libc.need_locks = 0;
36 }
37}
38
39static inline void unlock(volatile int *lk)
40{
41 if (lk[0]) {
42 a_store(lk, 0);
43 if (lk[1]) __wake(lk, 1, 1);
44 }
45}
46
47static inline void lock_bin(int i)
48{
49 lock(mal.bins[i].lock);
50 if (!mal.bins[i].head)
51 mal.bins[i].head = mal.bins[i].tail = BIN_TO_CHUNK(i);
52}
53
54static inline void unlock_bin(int i)
55{
56 unlock(mal.bins[i].lock);
57}
58
59static int first_set(uint64_t x)
60{
61#if 1
62 return a_ctz_64(x);
63#else
64 static const char debruijn64[64] = {
65 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
66 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
67 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
68 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12
69 };
70 static const char debruijn32[32] = {
71 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
72 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
73 };
74 if (sizeof(long) < 8) {
75 uint32_t y = x;
76 if (!y) {
77 y = x>>32;
78 return 32 + debruijn32[(y&-y)*0x076be629 >> 27];
79 }
80 return debruijn32[(y&-y)*0x076be629 >> 27];
81 }
82 return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58];
83#endif
84}
85
86static const unsigned char bin_tab[60] = {
87 32,33,34,35,36,36,37,37,38,38,39,39,
88 40,40,40,40,41,41,41,41,42,42,42,42,43,43,43,43,
89 44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45,
90 46,46,46,46,46,46,46,46,47,47,47,47,47,47,47,47,
91};
92
93static int bin_index(size_t x)
94{
95 x = x / SIZE_ALIGN - 1;
96 if (x <= 32) return x;
97 if (x < 512) return bin_tab[x/8-4];
98 if (x > 0x1c00) return 63;
99 return bin_tab[x/128-4] + 16;
100}
101
102static int bin_index_up(size_t x)
103{
104 x = x / SIZE_ALIGN - 1;
105 if (x <= 32) return x;
106 x--;
107 if (x < 512) return bin_tab[x/8-4] + 1;
108 return bin_tab[x/128-4] + 17;
109}
110
111#if 0
112void __dump_heap(int x)
113{
114 struct chunk *c;
115 int i;
116 for (c = (void *)mal.heap; CHUNK_SIZE(c); c = NEXT_CHUNK(c))
117 fprintf(stderr, "base %p size %zu (%d) flags %d/%d\n",
118 c, CHUNK_SIZE(c), bin_index(CHUNK_SIZE(c)),
119 c->csize & 15,
120 NEXT_CHUNK(c)->psize & 15);
121 for (i=0; i<64; i++) {
122 if (mal.bins[i].head != BIN_TO_CHUNK(i) && mal.bins[i].head) {
123 fprintf(stderr, "bin %d: %p\n", i, mal.bins[i].head);
124 if (!(mal.binmap & 1ULL<<i))
125 fprintf(stderr, "missing from binmap!\n");
126 } else if (mal.binmap & 1ULL<<i)
127 fprintf(stderr, "binmap wrongly contains %d!\n", i);
128 }
129}
130#endif
131
132/* This function returns true if the interval [old,new]
133 * intersects the 'len'-sized interval below &libc.auxv
134 * (interpreted as the main-thread stack) or below &b
135 * (the current stack). It is used to defend against
136 * buggy brk implementations that can cross the stack. */
137
138static int traverses_stack_p(uintptr_t old, uintptr_t new)
139{
140 const uintptr_t len = 8<<20;
141 uintptr_t a, b;
142
143 b = (uintptr_t)libc.auxv;
144 a = b > len ? b-len : 0;
145 if (new>a && old<b) return 1;
146
147 b = (uintptr_t)&b;
148 a = b > len ? b-len : 0;
149 if (new>a && old<b) return 1;
150
151 return 0;
152}
153
154/* Expand the heap in-place if brk can be used, or otherwise via mmap,
155 * using an exponential lower bound on growth by mmap to make
156 * fragmentation asymptotically irrelevant. The size argument is both
157 * an input and an output, since the caller needs to know the size
158 * allocated, which will be larger than requested due to page alignment
159 * and mmap minimum size rules. The caller is responsible for locking
160 * to prevent concurrent calls. */
161
162static void *__expand_heap(size_t *pn)
163{
164 static uintptr_t brk;
165 static unsigned mmap_step;
166 size_t n = *pn;
167
168 if (n > SIZE_MAX/2 - PAGE_SIZE) {
169 errno = ENOMEM;
170 return 0;
171 }
172 n += -n & PAGE_SIZE-1;
173
174 if (!brk) {
175 brk = __syscall(SYS_brk, 0);
176 brk += -brk & PAGE_SIZE-1;
177 }
178
179 if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n)
180 && __syscall(SYS_brk, brk+n)==brk+n) {
181 *pn = n;
182 brk += n;
183 return (void *)(brk-n);
184 }
185
186 size_t min = (size_t)PAGE_SIZE << mmap_step/2;
187 if (n < min) n = min;
188 void *area = __mmap(0, n, PROT_READ|PROT_WRITE,
189 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
190 if (area == MAP_FAILED) return 0;
191 *pn = n;
192 mmap_step++;
193 return area;
194}
195
196static struct chunk *expand_heap(size_t n)
197{
198 static void *end;
199 void *p;
200 struct chunk *w;
201
202 /* The argument n already accounts for the caller's chunk
203 * overhead needs, but if the heap can't be extended in-place,
204 * we need room for an extra zero-sized sentinel chunk. */
205 n += SIZE_ALIGN;
206
207 p = __expand_heap(&n);
208 if (!p) return 0;
209
210 /* If not just expanding existing space, we need to make a
211 * new sentinel chunk below the allocated space. */
212 if (p != end) {
213 /* Valid/safe because of the prologue increment. */
214 n -= SIZE_ALIGN;
215 p = (char *)p + SIZE_ALIGN;
216 w = MEM_TO_CHUNK(p);
217 w->psize = 0 | C_INUSE;
218 }
219
220 /* Record new heap end and fill in footer. */
221 end = (char *)p + n;
222 w = MEM_TO_CHUNK(end);
223 w->psize = n | C_INUSE;
224 w->csize = 0 | C_INUSE;
225
226 /* Fill in header, which may be new or may be replacing a
227 * zero-size sentinel header at the old end-of-heap. */
228 w = MEM_TO_CHUNK(p);
229 w->csize = n | C_INUSE;
230
231 return w;
232}
233
234static int adjust_size(size_t *n)
235{
236 /* Result of pointer difference must fit in ptrdiff_t. */
237 if (*n-1 > PTRDIFF_MAX - SIZE_ALIGN - PAGE_SIZE) {
238 if (*n) {
239 errno = ENOMEM;
240 return -1;
241 } else {
242 *n = SIZE_ALIGN;
243 return 0;
244 }
245 }
246 *n = (*n + OVERHEAD + SIZE_ALIGN - 1) & SIZE_MASK;
247 return 0;
248}
249
250static void unbin(struct chunk *c, int i)
251{
252 if (c->prev == c->next)
253 a_and_64(&mal.binmap, ~(1ULL<<i));
254 c->prev->next = c->next;
255 c->next->prev = c->prev;
256 c->csize |= C_INUSE;
257 NEXT_CHUNK(c)->psize |= C_INUSE;
258}
259
260static void bin_chunk(struct chunk *self, int i)
261{
262 self->next = BIN_TO_CHUNK(i);
263 self->prev = mal.bins[i].tail;
264 self->next->prev = self;
265 self->prev->next = self;
266 if (self->prev == BIN_TO_CHUNK(i))
267 a_or_64(&mal.binmap, 1ULL<<i);
268}
269
270static void trim(struct chunk *self, size_t n)
271{
272 size_t n1 = CHUNK_SIZE(self);
273 struct chunk *next, *split;
274
275 if (n >= n1 - DONTCARE) return;
276
277 next = NEXT_CHUNK(self);
278 split = (void *)((char *)self + n);
279
280 split->psize = n | C_INUSE;
281 split->csize = n1-n;
282 next->psize = n1-n;
283 self->csize = n | C_INUSE;
284
285 int i = bin_index(n1-n);
286 lock_bin(i);
287
288 bin_chunk(split, i);
289
290 unlock_bin(i);
291}
292
293void *malloc(size_t n)
294{
295 struct chunk *c;
296 int i, j;
297 uint64_t mask;
298
299 if (adjust_size(&n) < 0) return 0;
300
301 if (n > MMAP_THRESHOLD) {
302 size_t len = n + OVERHEAD + PAGE_SIZE - 1 & -PAGE_SIZE;
303 char *base = __mmap(0, len, PROT_READ|PROT_WRITE,
304 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
305 if (base == (void *)-1) return 0;
306 c = (void *)(base + SIZE_ALIGN - OVERHEAD);
307 c->csize = len - (SIZE_ALIGN - OVERHEAD);
308 c->psize = SIZE_ALIGN - OVERHEAD;
309 return CHUNK_TO_MEM(c);
310 }
311
312 i = bin_index_up(n);
313 if (i<63 && (mal.binmap & (1ULL<<i))) {
314 lock_bin(i);
315 c = mal.bins[i].head;
316 if (c != BIN_TO_CHUNK(i) && CHUNK_SIZE(c)-n <= DONTCARE) {
317 unbin(c, i);
318 unlock_bin(i);
319 return CHUNK_TO_MEM(c);
320 }
321 unlock_bin(i);
322 }
323 lock(mal.split_merge_lock);
324 for (mask = mal.binmap & -(1ULL<<i); mask; mask -= (mask&-mask)) {
325 j = first_set(mask);
326 lock_bin(j);
327 c = mal.bins[j].head;
328 if (c != BIN_TO_CHUNK(j)) {
329 unbin(c, j);
330 unlock_bin(j);
331 break;
332 }
333 unlock_bin(j);
334 }
335 if (!mask) {
336 c = expand_heap(n);
337 if (!c) {
338 unlock(mal.split_merge_lock);
339 return 0;
340 }
341 }
342 trim(c, n);
343 unlock(mal.split_merge_lock);
344 return CHUNK_TO_MEM(c);
345}
346
347int __malloc_allzerop(void *p)
348{
349 return IS_MMAPPED(MEM_TO_CHUNK(p));
350}
351
352void *realloc(void *p, size_t n)
353{
354 struct chunk *self, *next;
355 size_t n0, n1;
356 void *new;
357
358 if (!p) return malloc(n);
359
360 if (adjust_size(&n) < 0) return 0;
361
362 self = MEM_TO_CHUNK(p);
363 n1 = n0 = CHUNK_SIZE(self);
364
365 if (n<=n0 && n0-n<=DONTCARE) return p;
366
367 if (IS_MMAPPED(self)) {
368 size_t extra = self->psize;
369 char *base = (char *)self - extra;
370 size_t oldlen = n0 + extra;
371 size_t newlen = n + extra;
372 /* Crash on realloc of freed chunk */
373 if (extra & 1) a_crash();
374 if (newlen < PAGE_SIZE && (new = malloc(n-OVERHEAD))) {
375 n0 = n;
376 goto copy_free_ret;
377 }
378 newlen = (newlen + PAGE_SIZE-1) & -PAGE_SIZE;
379 if (oldlen == newlen) return p;
380 base = __mremap(base, oldlen, newlen, MREMAP_MAYMOVE);
381 if (base == (void *)-1)
382 goto copy_realloc;
383 self = (void *)(base + extra);
384 self->csize = newlen - extra;
385 return CHUNK_TO_MEM(self);
386 }
387
388 next = NEXT_CHUNK(self);
389
390 /* Crash on corrupted footer (likely from buffer overflow) */
391 if (next->psize != self->csize) a_crash();
392
393 if (n < n0) {
394 int i = bin_index_up(n);
395 int j = bin_index(n0);
396 if (i<j && (mal.binmap & (1ULL << i)))
397 goto copy_realloc;
398 struct chunk *split = (void *)((char *)self + n);
399 self->csize = split->psize = n | C_INUSE;
400 split->csize = next->psize = n0-n | C_INUSE;
401 __bin_chunk(split);
402 return CHUNK_TO_MEM(self);
403 }
404
405 lock(mal.split_merge_lock);
406
407 size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next);
408 if (n0+nsize >= n) {
409 int i = bin_index(nsize);
410 lock_bin(i);
411 if (!(next->csize & C_INUSE)) {
412 unbin(next, i);
413 unlock_bin(i);
414 next = NEXT_CHUNK(next);
415 self->csize = next->psize = n0+nsize | C_INUSE;
416 trim(self, n);
417 unlock(mal.split_merge_lock);
418 return CHUNK_TO_MEM(self);
419 }
420 unlock_bin(i);
421 }
422 unlock(mal.split_merge_lock);
423
424copy_realloc:
425 /* As a last resort, allocate a new chunk and copy to it. */
426 new = malloc(n-OVERHEAD);
427 if (!new) return 0;
428copy_free_ret:
429 memcpy(new, p, (n<n0 ? n : n0) - OVERHEAD);
430 free(CHUNK_TO_MEM(self));
431 return new;
432}
433
434void __bin_chunk(struct chunk *self)
435{
436 struct chunk *next = NEXT_CHUNK(self);
437
438 /* Crash on corrupted footer (likely from buffer overflow) */
439 if (next->psize != self->csize) a_crash();
440
441 lock(mal.split_merge_lock);
442
443 size_t osize = CHUNK_SIZE(self), size = osize;
444
445 /* Since we hold split_merge_lock, only transition from free to
446 * in-use can race; in-use to free is impossible */
447 size_t psize = self->psize & C_INUSE ? 0 : CHUNK_PSIZE(self);
448 size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next);
449
450 if (psize) {
451 int i = bin_index(psize);
452 lock_bin(i);
453 if (!(self->psize & C_INUSE)) {
454 struct chunk *prev = PREV_CHUNK(self);
455 unbin(prev, i);
456 self = prev;
457 size += psize;
458 }
459 unlock_bin(i);
460 }
461 if (nsize) {
462 int i = bin_index(nsize);
463 lock_bin(i);
464 if (!(next->csize & C_INUSE)) {
465 unbin(next, i);
466 next = NEXT_CHUNK(next);
467 size += nsize;
468 }
469 unlock_bin(i);
470 }
471
472 int i = bin_index(size);
473 lock_bin(i);
474
475 self->csize = size;
476 next->psize = size;
477 bin_chunk(self, i);
478 unlock(mal.split_merge_lock);
479
480 /* Replace middle of large chunks with fresh zero pages */
481 if (size > RECLAIM && (size^(size-osize)) > size-osize) {
482 uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE;
483 uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE;
484 int e = errno;
485#if 1
486 __madvise((void *)a, b-a, MADV_DONTNEED);
487#else
488 __mmap((void *)a, b-a, PROT_READ|PROT_WRITE,
489 MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0);
490#endif
491 errno = e;
492 }
493
494 unlock_bin(i);
495}
496
497static void unmap_chunk(struct chunk *self)
498{
499 size_t extra = self->psize;
500 char *base = (char *)self - extra;
501 size_t len = CHUNK_SIZE(self) + extra;
502 /* Crash on double free */
503 if (extra & 1) a_crash();
504 int e = errno;
505 __munmap(base, len);
506 errno = e;
507}
508
509void free(void *p)
510{
511 if (!p) return;
512
513 struct chunk *self = MEM_TO_CHUNK(p);
514
515 if (IS_MMAPPED(self))
516 unmap_chunk(self);
517 else
518 __bin_chunk(self);
519}
520
521void __malloc_donate(char *start, char *end)
522{
523 size_t align_start_up = (SIZE_ALIGN-1) & (-(uintptr_t)start - OVERHEAD);
524 size_t align_end_down = (SIZE_ALIGN-1) & (uintptr_t)end;
525
526 /* Getting past this condition ensures that the padding for alignment
527 * and header overhead will not overflow and will leave a nonzero
528 * multiple of SIZE_ALIGN bytes between start and end. */
529 if (end - start <= OVERHEAD + align_start_up + align_end_down)
530 return;
531 start += align_start_up + OVERHEAD;
532 end -= align_end_down;
533
534 struct chunk *c = MEM_TO_CHUNK(start), *n = MEM_TO_CHUNK(end);
535 c->psize = n->csize = C_INUSE;
536 c->csize = n->psize = C_INUSE | (end-start);
537 __bin_chunk(c);
538}
539
540void __malloc_atfork(int who)
541{
542 if (who<0) {
543 lock(mal.split_merge_lock);
544 for (int i=0; i<64; i++)
545 lock(mal.bins[i].lock);
546 } else if (!who) {
547 for (int i=0; i<64; i++)
548 unlock(mal.bins[i].lock);
549 unlock(mal.split_merge_lock);
550 } else {
551 for (int i=0; i<64; i++)
552 mal.bins[i].lock[0] = mal.bins[i].lock[1] = 0;
553 mal.split_merge_lock[1] = 0;
554 mal.split_merge_lock[0] = 0;
555 }
556}
lib/libc/musl/src/malloc/oldmalloc/malloc_impl.h deleted-39
......@@ -1,39 +0,0 @@
1#ifndef MALLOC_IMPL_H
2#define MALLOC_IMPL_H
3
4#include <sys/mman.h>
5#include "dynlink.h"
6
7struct chunk {
8 size_t psize, csize;
9 struct chunk *next, *prev;
10};
11
12struct bin {
13 volatile int lock[2];
14 struct chunk *head;
15 struct chunk *tail;
16};
17
18#define SIZE_ALIGN (4*sizeof(size_t))
19#define SIZE_MASK (-SIZE_ALIGN)
20#define OVERHEAD (2*sizeof(size_t))
21#define MMAP_THRESHOLD (0x1c00*SIZE_ALIGN)
22#define DONTCARE 16
23#define RECLAIM 163840
24
25#define CHUNK_SIZE(c) ((c)->csize & -2)
26#define CHUNK_PSIZE(c) ((c)->psize & -2)
27#define PREV_CHUNK(c) ((struct chunk *)((char *)(c) - CHUNK_PSIZE(c)))
28#define NEXT_CHUNK(c) ((struct chunk *)((char *)(c) + CHUNK_SIZE(c)))
29#define MEM_TO_CHUNK(p) (struct chunk *)((char *)(p) - OVERHEAD)
30#define CHUNK_TO_MEM(c) (void *)((char *)(c) + OVERHEAD)
31#define BIN_TO_CHUNK(i) (MEM_TO_CHUNK(&mal.bins[i].head))
32
33#define C_INUSE ((size_t)1)
34
35#define IS_MMAPPED(c) !((c)->csize & (C_INUSE))
36
37hidden void __bin_chunk(struct chunk *);
38
39#endif
lib/libc/musl/src/malloc/oldmalloc/malloc_usable_size.c deleted-9
......@@ -1,9 +0,0 @@
1#include <malloc.h>
2#include "malloc_impl.h"
3
4hidden void *(*const __realloc_dep)(void *, size_t) = realloc;
5
6size_t malloc_usable_size(void *p)
7{
8 return p ? CHUNK_SIZE(MEM_TO_CHUNK(p)) - OVERHEAD : 0;
9}
lib/libc/musl/src/malloc/posix_memalign.c deleted-11
......@@ -1,11 +0,0 @@
1#include <stdlib.h>
2#include <errno.h>
3
4int posix_memalign(void **res, size_t align, size_t len)
5{
6 if (align < sizeof(void *)) return EINVAL;
7 void *mem = aligned_alloc(align, len);
8 if (!mem) return errno;
9 *res = mem;
10 return 0;
11}
lib/libc/musl/src/malloc/realloc.c deleted-6
......@@ -1,6 +0,0 @@
1#include <stdlib.h>
2
3void *realloc(void *p, size_t n)
4{
5 return __libc_realloc(p, n);
6}
lib/libc/musl/src/malloc/reallocarray.c deleted-13
......@@ -1,13 +0,0 @@
1#define _BSD_SOURCE
2#include <errno.h>
3#include <stdlib.h>
4
5void *reallocarray(void *ptr, size_t m, size_t n)
6{
7 if (n && m > -1 / n) {
8 errno = ENOMEM;
9 return 0;
10 }
11
12 return realloc(ptr, m * n);
13}
lib/libc/musl/src/malloc/replaced.c deleted-4
......@@ -1,4 +0,0 @@
1#include "dynlink.h"
2
3int __malloc_replaced;
4int __aligned_alloc_replaced;
lib/libc/wasi/emmalloc/emmalloc.c deleted-1540
......@@ -1,1540 +0,0 @@
1/*
2 * Copyright 2018 The Emscripten Authors. All rights reserved.
3 * Emscripten is available under two separate licenses, the MIT license and the
4 * University of Illinois/NCSA Open Source License. Both these licenses can be
5 * found in the LICENSE file.
6 *
7 * Simple minimalistic but efficient sbrk()-based malloc/free that works in
8 * singlethreaded and multithreaded builds.
9 *
10 * Assumptions:
11 *
12 * - sbrk() is used to claim new memory (sbrk handles geometric/linear
13 * - overallocation growth)
14 * - sbrk() can be used by other code outside emmalloc.
15 * - sbrk() is very fast in most cases (internal wasm call).
16 * - sbrk() returns pointers with an alignment of alignof(max_align_t)
17 *
18 * Invariants:
19 *
20 * - Per-allocation header overhead is 8 bytes, smallest allocated payload
21 * amount is 8 bytes, and a multiple of 4 bytes.
22 * - Acquired memory blocks are subdivided into disjoint regions that lie
23 * next to each other.
24 * - A region is either in used or free.
25 * Used regions may be adjacent, and a used and unused region
26 * may be adjacent, but not two unused ones - they would be
27 * merged.
28 * - Memory allocation takes constant time, unless the alloc needs to sbrk()
29 * or memory is very close to being exhausted.
30 *
31 * Debugging:
32 *
33 * - If not NDEBUG, runtime assert()s are in use.
34 * - If EMMALLOC_MEMVALIDATE is defined, a large amount of extra checks are done.
35 * - If EMMALLOC_VERBOSE is defined, a lot of operations are logged
36 * out, in addition to EMMALLOC_MEMVALIDATE.
37 * - Debugging and logging directly uses console.log via uses EM_ASM, not
38 * printf etc., to minimize any risk of debugging or logging depending on
39 * malloc.
40 */
41
42#include <stdalign.h>
43#include <stdbool.h>
44#include <stddef.h>
45#include <stdint.h>
46#include <unistd.h>
47#include <memory.h>
48#include <assert.h>
49#include <malloc.h>
50#include <limits.h>
51#include <stdlib.h>
52
53#ifdef __EMSCRIPTEN_TRACING__
54#include <emscripten/trace.h>
55#endif
56
57// Defind by the linker to have the address of the start of the heap.
58extern unsigned char __heap_base;
59extern unsigned char __heap_end;
60
61// Behavior of right shifting a signed integer is compiler implementation defined.
62static_assert((((int32_t)0x80000000U) >> 31) == -1, "This malloc implementation requires that right-shifting a signed integer produces a sign-extending (arithmetic) shift!");
63
64// Configuration: specifies the minimum alignment that malloc()ed memory outputs. Allocation requests with smaller alignment
65// than this will yield an allocation with this much alignment.
66#define MALLOC_ALIGNMENT alignof(max_align_t)
67static_assert(alignof(max_align_t) == 16, "max_align_t must be correct");
68
69#define EMMALLOC_EXPORT __attribute__((weak))
70
71#define MIN(x, y) ((x) < (y) ? (x) : (y))
72#define MAX(x, y) ((x) > (y) ? (x) : (y))
73
74#define NUM_FREE_BUCKETS 64
75#define BUCKET_BITMASK_T uint64_t
76
77// Dynamic memory is subdivided into regions, in the format
78
79// <size:uint32_t> ..... <size:uint32_t> | <size:uint32_t> ..... <size:uint32_t> | <size:uint32_t> ..... <size:uint32_t> | .....
80
81// That is, at the bottom and top end of each memory region, the size of that region is stored. That allows traversing the
82// memory regions backwards and forwards. Because each allocation must be at least a multiple of 4 bytes, the lowest two bits of
83// each size field is unused. Free regions are distinguished by used regions by having the FREE_REGION_FLAG bit present
84// in the size field. I.e. for free regions, the size field is odd, and for used regions, the size field reads even.
85#define FREE_REGION_FLAG 0x1u
86
87// Attempts to malloc() more than this many bytes would cause an overflow when calculating the size of a region,
88// therefore allocations larger than this are short-circuited immediately on entry.
89#define MAX_ALLOC_SIZE 0xFFFFFFC7u
90
91// A free region has the following structure:
92// <size:size_t> <prevptr> <nextptr> ... <size:size_t>
93
94typedef struct Region
95{
96 size_t size;
97 // Use a circular doubly linked list to represent free region data.
98 struct Region *prev, *next;
99 // ... N bytes of free data
100 size_t _at_the_end_of_this_struct_size; // do not dereference, this is present for convenient struct sizeof() computation only
101} Region;
102
103// Each memory block starts with a RootRegion at the beginning.
104// The RootRegion specifies the size of the region block, and forms a linked
105// list of all RootRegions in the program, starting with `listOfAllRegions`
106// below.
107typedef struct RootRegion
108{
109 uint32_t size;
110 struct RootRegion *next;
111 uint8_t* endPtr;
112} RootRegion;
113
114#if defined(__EMSCRIPTEN_PTHREADS__)
115// In multithreaded builds, use a simple global spinlock strategy to acquire/release access to the memory allocator.
116static volatile uint8_t multithreadingLock = 0;
117#define MALLOC_ACQUIRE() while(__sync_lock_test_and_set(&multithreadingLock, 1)) { while(multithreadingLock) { /*nop*/ } }
118#define MALLOC_RELEASE() __sync_lock_release(&multithreadingLock)
119// Test code to ensure we have tight malloc acquire/release guards in place.
120#define ASSERT_MALLOC_IS_ACQUIRED() assert(multithreadingLock == 1)
121#else
122// In singlethreaded builds, no need for locking.
123#define MALLOC_ACQUIRE() ((void)0)
124#define MALLOC_RELEASE() ((void)0)
125#define ASSERT_MALLOC_IS_ACQUIRED() ((void)0)
126#endif
127
128#define IS_POWER_OF_2(val) (((val) & ((val)-1)) == 0)
129#define ALIGN_UP(ptr, alignment) ((uint8_t*)((((uintptr_t)(ptr)) + ((alignment)-1)) & ~((alignment)-1)))
130#define HAS_ALIGNMENT(ptr, alignment) ((((uintptr_t)(ptr)) & ((alignment)-1)) == 0)
131
132static_assert(IS_POWER_OF_2(MALLOC_ALIGNMENT), "MALLOC_ALIGNMENT must be a power of two value!");
133static_assert(MALLOC_ALIGNMENT >= 4, "Smallest possible MALLOC_ALIGNMENT if 4!");
134
135// A region that contains as payload a single forward linked list of pointers to
136// root regions of each disjoint region blocks.
137static RootRegion *listOfAllRegions = NULL;
138
139// For each of the buckets, maintain a linked list head node. The head node for each
140// free region is a sentinel node that does not actually represent any free space, but
141// the sentinel is used to avoid awkward testing against (if node == freeRegionHeadNode)
142// when adding and removing elements from the linked list, i.e. we are guaranteed that
143// the sentinel node is always fixed and there, and the actual free region list elements
144// start at freeRegionBuckets[i].next each.
145static Region freeRegionBuckets[NUM_FREE_BUCKETS] = {
146 { .prev = &freeRegionBuckets[0], .next = &freeRegionBuckets[0] },
147 { .prev = &freeRegionBuckets[1], .next = &freeRegionBuckets[1] },
148 { .prev = &freeRegionBuckets[2], .next = &freeRegionBuckets[2] },
149 { .prev = &freeRegionBuckets[3], .next = &freeRegionBuckets[3] },
150 { .prev = &freeRegionBuckets[4], .next = &freeRegionBuckets[4] },
151 { .prev = &freeRegionBuckets[5], .next = &freeRegionBuckets[5] },
152 { .prev = &freeRegionBuckets[6], .next = &freeRegionBuckets[6] },
153 { .prev = &freeRegionBuckets[7], .next = &freeRegionBuckets[7] },
154 { .prev = &freeRegionBuckets[8], .next = &freeRegionBuckets[8] },
155 { .prev = &freeRegionBuckets[9], .next = &freeRegionBuckets[9] },
156 { .prev = &freeRegionBuckets[10], .next = &freeRegionBuckets[10] },
157 { .prev = &freeRegionBuckets[11], .next = &freeRegionBuckets[11] },
158 { .prev = &freeRegionBuckets[12], .next = &freeRegionBuckets[12] },
159 { .prev = &freeRegionBuckets[13], .next = &freeRegionBuckets[13] },
160 { .prev = &freeRegionBuckets[14], .next = &freeRegionBuckets[14] },
161 { .prev = &freeRegionBuckets[15], .next = &freeRegionBuckets[15] },
162 { .prev = &freeRegionBuckets[16], .next = &freeRegionBuckets[16] },
163 { .prev = &freeRegionBuckets[17], .next = &freeRegionBuckets[17] },
164 { .prev = &freeRegionBuckets[18], .next = &freeRegionBuckets[18] },
165 { .prev = &freeRegionBuckets[19], .next = &freeRegionBuckets[19] },
166 { .prev = &freeRegionBuckets[20], .next = &freeRegionBuckets[20] },
167 { .prev = &freeRegionBuckets[21], .next = &freeRegionBuckets[21] },
168 { .prev = &freeRegionBuckets[22], .next = &freeRegionBuckets[22] },
169 { .prev = &freeRegionBuckets[23], .next = &freeRegionBuckets[23] },
170 { .prev = &freeRegionBuckets[24], .next = &freeRegionBuckets[24] },
171 { .prev = &freeRegionBuckets[25], .next = &freeRegionBuckets[25] },
172 { .prev = &freeRegionBuckets[26], .next = &freeRegionBuckets[26] },
173 { .prev = &freeRegionBuckets[27], .next = &freeRegionBuckets[27] },
174 { .prev = &freeRegionBuckets[28], .next = &freeRegionBuckets[28] },
175 { .prev = &freeRegionBuckets[29], .next = &freeRegionBuckets[29] },
176 { .prev = &freeRegionBuckets[30], .next = &freeRegionBuckets[30] },
177 { .prev = &freeRegionBuckets[31], .next = &freeRegionBuckets[31] },
178 { .prev = &freeRegionBuckets[32], .next = &freeRegionBuckets[32] },
179 { .prev = &freeRegionBuckets[33], .next = &freeRegionBuckets[33] },
180 { .prev = &freeRegionBuckets[34], .next = &freeRegionBuckets[34] },
181 { .prev = &freeRegionBuckets[35], .next = &freeRegionBuckets[35] },
182 { .prev = &freeRegionBuckets[36], .next = &freeRegionBuckets[36] },
183 { .prev = &freeRegionBuckets[37], .next = &freeRegionBuckets[37] },
184 { .prev = &freeRegionBuckets[38], .next = &freeRegionBuckets[38] },
185 { .prev = &freeRegionBuckets[39], .next = &freeRegionBuckets[39] },
186 { .prev = &freeRegionBuckets[40], .next = &freeRegionBuckets[40] },
187 { .prev = &freeRegionBuckets[41], .next = &freeRegionBuckets[41] },
188 { .prev = &freeRegionBuckets[42], .next = &freeRegionBuckets[42] },
189 { .prev = &freeRegionBuckets[43], .next = &freeRegionBuckets[43] },
190 { .prev = &freeRegionBuckets[44], .next = &freeRegionBuckets[44] },
191 { .prev = &freeRegionBuckets[45], .next = &freeRegionBuckets[45] },
192 { .prev = &freeRegionBuckets[46], .next = &freeRegionBuckets[46] },
193 { .prev = &freeRegionBuckets[47], .next = &freeRegionBuckets[47] },
194 { .prev = &freeRegionBuckets[48], .next = &freeRegionBuckets[48] },
195 { .prev = &freeRegionBuckets[49], .next = &freeRegionBuckets[49] },
196 { .prev = &freeRegionBuckets[50], .next = &freeRegionBuckets[50] },
197 { .prev = &freeRegionBuckets[51], .next = &freeRegionBuckets[51] },
198 { .prev = &freeRegionBuckets[52], .next = &freeRegionBuckets[52] },
199 { .prev = &freeRegionBuckets[53], .next = &freeRegionBuckets[53] },
200 { .prev = &freeRegionBuckets[54], .next = &freeRegionBuckets[54] },
201 { .prev = &freeRegionBuckets[55], .next = &freeRegionBuckets[55] },
202 { .prev = &freeRegionBuckets[56], .next = &freeRegionBuckets[56] },
203 { .prev = &freeRegionBuckets[57], .next = &freeRegionBuckets[57] },
204 { .prev = &freeRegionBuckets[58], .next = &freeRegionBuckets[58] },
205 { .prev = &freeRegionBuckets[59], .next = &freeRegionBuckets[59] },
206 { .prev = &freeRegionBuckets[60], .next = &freeRegionBuckets[60] },
207 { .prev = &freeRegionBuckets[61], .next = &freeRegionBuckets[61] },
208 { .prev = &freeRegionBuckets[62], .next = &freeRegionBuckets[62] },
209 { .prev = &freeRegionBuckets[63], .next = &freeRegionBuckets[63] },
210};
211
212// A bitmask that tracks the population status for each of the 64 distinct memory regions:
213// a zero at bit position i means that the free list bucket i is empty. This bitmask is
214// used to avoid redundant scanning of the 64 different free region buckets: instead by
215// looking at the bitmask we can find in constant time an index to a free region bucket
216// that contains free memory of desired size.
217static BUCKET_BITMASK_T freeRegionBucketsUsed = 0;
218
219// Amount of bytes taken up by allocation header data
220#define REGION_HEADER_SIZE (2*sizeof(size_t))
221
222// Smallest allocation size that is possible is 2*pointer size, since payload of each region must at least contain space
223// to store the free region linked list prev and next pointers. An allocation size smaller than this will be rounded up
224// to this size.
225#define SMALLEST_ALLOCATION_SIZE (2*sizeof(void*))
226
227/* Subdivide regions of free space into distinct circular doubly linked lists, where each linked list
228represents a range of free space blocks. The following function compute_free_list_bucket() converts
229an allocation size to the bucket index that should be looked at. The buckets are grouped as follows:
230
231 Bucket 0: [8, 15], range size=8
232 Bucket 1: [16, 23], range size=8
233 Bucket 2: [24, 31], range size=8
234 Bucket 3: [32, 39], range size=8
235 Bucket 4: [40, 47], range size=8
236 Bucket 5: [48, 55], range size=8
237 Bucket 6: [56, 63], range size=8
238 Bucket 7: [64, 71], range size=8
239 Bucket 8: [72, 79], range size=8
240 Bucket 9: [80, 87], range size=8
241 Bucket 10: [88, 95], range size=8
242 Bucket 11: [96, 103], range size=8
243 Bucket 12: [104, 111], range size=8
244 Bucket 13: [112, 119], range size=8
245 Bucket 14: [120, 159], range size=40
246 Bucket 15: [160, 191], range size=32
247 Bucket 16: [192, 223], range size=32
248 Bucket 17: [224, 255], range size=32
249 Bucket 18: [256, 319], range size=64
250 Bucket 19: [320, 383], range size=64
251 Bucket 20: [384, 447], range size=64
252 Bucket 21: [448, 511], range size=64
253 Bucket 22: [512, 639], range size=128
254 Bucket 23: [640, 767], range size=128
255 Bucket 24: [768, 895], range size=128
256 Bucket 25: [896, 1023], range size=128
257 Bucket 26: [1024, 1279], range size=256
258 Bucket 27: [1280, 1535], range size=256
259 Bucket 28: [1536, 1791], range size=256
260 Bucket 29: [1792, 2047], range size=256
261 Bucket 30: [2048, 2559], range size=512
262 Bucket 31: [2560, 3071], range size=512
263 Bucket 32: [3072, 3583], range size=512
264 Bucket 33: [3584, 6143], range size=2560
265 Bucket 34: [6144, 8191], range size=2048
266 Bucket 35: [8192, 12287], range size=4096
267 Bucket 36: [12288, 16383], range size=4096
268 Bucket 37: [16384, 24575], range size=8192
269 Bucket 38: [24576, 32767], range size=8192
270 Bucket 39: [32768, 49151], range size=16384
271 Bucket 40: [49152, 65535], range size=16384
272 Bucket 41: [65536, 98303], range size=32768
273 Bucket 42: [98304, 131071], range size=32768
274 Bucket 43: [131072, 196607], range size=65536
275 Bucket 44: [196608, 262143], range size=65536
276 Bucket 45: [262144, 393215], range size=131072
277 Bucket 46: [393216, 524287], range size=131072
278 Bucket 47: [524288, 786431], range size=262144
279 Bucket 48: [786432, 1048575], range size=262144
280 Bucket 49: [1048576, 1572863], range size=524288
281 Bucket 50: [1572864, 2097151], range size=524288
282 Bucket 51: [2097152, 3145727], range size=1048576
283 Bucket 52: [3145728, 4194303], range size=1048576
284 Bucket 53: [4194304, 6291455], range size=2097152
285 Bucket 54: [6291456, 8388607], range size=2097152
286 Bucket 55: [8388608, 12582911], range size=4194304
287 Bucket 56: [12582912, 16777215], range size=4194304
288 Bucket 57: [16777216, 25165823], range size=8388608
289 Bucket 58: [25165824, 33554431], range size=8388608
290 Bucket 59: [33554432, 50331647], range size=16777216
291 Bucket 60: [50331648, 67108863], range size=16777216
292 Bucket 61: [67108864, 100663295], range size=33554432
293 Bucket 62: [100663296, 134217727], range size=33554432
294 Bucket 63: 134217728 bytes and larger. */
295static_assert(NUM_FREE_BUCKETS == 64, "Following function is tailored specifically for NUM_FREE_BUCKETS == 64 case");
296static int compute_free_list_bucket(size_t allocSize)
297{
298 if (allocSize < 128) return (allocSize >> 3) - 1;
299 int clz = __builtin_clz(allocSize);
300 int bucketIndex = (clz > 19) ? 110 - (clz<<2) + ((allocSize >> (29-clz)) ^ 4) : MIN(71 - (clz<<1) + ((allocSize >> (30-clz)) ^ 2), NUM_FREE_BUCKETS-1);
301 assert(bucketIndex >= 0);
302 assert(bucketIndex < NUM_FREE_BUCKETS);
303 return bucketIndex;
304}
305
306#define DECODE_CEILING_SIZE(size) ((size_t)((size) & ~FREE_REGION_FLAG))
307
308static Region *prev_region(Region *region)
309{
310 size_t prevRegionSize = ((size_t*)region)[-1];
311 prevRegionSize = DECODE_CEILING_SIZE(prevRegionSize);
312 return (Region*)((uint8_t*)region - prevRegionSize);
313}
314
315static Region *next_region(Region *region)
316{
317 return (Region*)((uint8_t*)region + region->size);
318}
319
320static size_t region_ceiling_size(Region *region)
321{
322 return ((size_t*)((uint8_t*)region + region->size))[-1];
323}
324
325static bool region_is_free(Region *r)
326{
327 return region_ceiling_size(r) & FREE_REGION_FLAG;
328}
329
330static bool region_is_in_use(Region *r)
331{
332 return r->size == region_ceiling_size(r);
333}
334
335static size_t size_of_region_from_ceiling(Region *r)
336{
337 size_t size = region_ceiling_size(r);
338 return DECODE_CEILING_SIZE(size);
339}
340
341static bool debug_region_is_consistent(Region *r)
342{
343 assert(r);
344 size_t sizeAtBottom = r->size;
345 size_t sizeAtCeiling = size_of_region_from_ceiling(r);
346 return sizeAtBottom == sizeAtCeiling;
347}
348
349static uint8_t *region_payload_start_ptr(Region *region)
350{
351 return (uint8_t*)region + sizeof(size_t);
352}
353
354static uint8_t *region_payload_end_ptr(Region *region)
355{
356 return (uint8_t*)region + region->size - sizeof(size_t);
357}
358
359static void create_used_region(void *ptr, size_t size)
360{
361 assert(ptr);
362 assert(HAS_ALIGNMENT(ptr, sizeof(size_t)));
363 assert(HAS_ALIGNMENT(size, sizeof(size_t)));
364 assert(size >= sizeof(Region));
365 *(size_t*)ptr = size;
366 ((size_t*)ptr)[(size/sizeof(size_t))-1] = size;
367}
368
369static void create_free_region(void *ptr, size_t size)
370{
371 assert(ptr);
372 assert(HAS_ALIGNMENT(ptr, sizeof(size_t)));
373 assert(HAS_ALIGNMENT(size, sizeof(size_t)));
374 assert(size >= sizeof(Region));
375 Region *freeRegion = (Region*)ptr;
376 freeRegion->size = size;
377 ((size_t*)ptr)[(size/sizeof(size_t))-1] = size | FREE_REGION_FLAG;
378}
379
380static void prepend_to_free_list(Region *region, Region *prependTo)
381{
382 assert(region);
383 assert(prependTo);
384 // N.b. the region we are prepending to is always the sentinel node,
385 // which represents a dummy node that is technically not a free node, so
386 // region_is_free(prependTo) does not hold.
387 assert(region_is_free((Region*)region));
388 region->next = prependTo;
389 region->prev = prependTo->prev;
390 assert(region->prev);
391 prependTo->prev = region;
392 region->prev->next = region;
393}
394
395static void unlink_from_free_list(Region *region)
396{
397 assert(region);
398 assert(region_is_free((Region*)region));
399 assert(region->prev);
400 assert(region->next);
401 region->prev->next = region->next;
402 region->next->prev = region->prev;
403}
404
405static void link_to_free_list(Region *freeRegion)
406{
407 assert(freeRegion);
408 assert(freeRegion->size >= sizeof(Region));
409 int bucketIndex = compute_free_list_bucket(freeRegion->size-REGION_HEADER_SIZE);
410 Region *freeListHead = freeRegionBuckets + bucketIndex;
411 freeRegion->prev = freeListHead;
412 freeRegion->next = freeListHead->next;
413 assert(freeRegion->next);
414 freeListHead->next = freeRegion;
415 freeRegion->next->prev = freeRegion;
416 freeRegionBucketsUsed |= ((BUCKET_BITMASK_T)1) << bucketIndex;
417}
418
419#if 0
420static void dump_memory_regions()
421{
422 ASSERT_MALLOC_IS_ACQUIRED();
423 RootRegion *root = listOfAllRegions;
424 MAIN_THREAD_ASYNC_EM_ASM(console.log('All memory regions:'));
425 while(root)
426 {
427 Region *r = (Region*)root;
428 assert(debug_region_is_consistent(r));
429 uint8_t *lastRegionEnd = root->endPtr;
430 MAIN_THREAD_ASYNC_EM_ASM(console.log('Region block 0x'+($0>>>0).toString(16)+' - 0x'+($1>>>0).toString(16)+ ' ('+($2>>>0)+' bytes):'),
431 r, lastRegionEnd, lastRegionEnd-(uint8_t*)r);
432 while((uint8_t*)r < lastRegionEnd)
433 {
434 MAIN_THREAD_ASYNC_EM_ASM(console.log('Region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+')'),
435 r, r->size, region_ceiling_size(r) == r->size);
436
437 assert(debug_region_is_consistent(r));
438 size_t sizeFromCeiling = size_of_region_from_ceiling(r);
439 if (sizeFromCeiling != r->size)
440 MAIN_THREAD_ASYNC_EM_ASM(console.log('Corrupt region! Size marker at the end of the region does not match: '+($0>>>0)), sizeFromCeiling);
441 if (r->size == 0)
442 break;
443 r = next_region(r);
444 }
445 root = root->next;
446 MAIN_THREAD_ASYNC_EM_ASM(console.log(""));
447 }
448 MAIN_THREAD_ASYNC_EM_ASM(console.log('Free regions:'));
449 for(int i = 0; i < NUM_FREE_BUCKETS; ++i)
450 {
451 Region *prev = &freeRegionBuckets[i];
452 Region *fr = freeRegionBuckets[i].next;
453 while(fr != &freeRegionBuckets[i])
454 {
455 MAIN_THREAD_ASYNC_EM_ASM(console.log('In bucket '+$0+', free region 0x'+($1>>>0).toString(16)+', size: ' + ($2>>>0) + ' (size at ceiling: '+($3>>>0)+'), prev: 0x' + ($4>>>0).toString(16) + ', next: 0x' + ($5>>>0).toString(16)),
456 i, fr, fr->size, size_of_region_from_ceiling(fr), fr->prev, fr->next);
457 assert(debug_region_is_consistent(fr));
458 assert(region_is_free(fr));
459 assert(fr->prev == prev);
460 prev = fr;
461 assert(fr->next != fr);
462 assert(fr->prev != fr);
463 fr = fr->next;
464 }
465 }
466 MAIN_THREAD_ASYNC_EM_ASM(console.log('Free bucket index map: ' + ($0>>>0).toString(2) + ' ' + ($1>>>0).toString(2)), (uint32_t)(freeRegionBucketsUsed >> 32), (uint32_t)freeRegionBucketsUsed);
467 MAIN_THREAD_ASYNC_EM_ASM(console.log(""));
468}
469
470void emmalloc_dump_memory_regions()
471{
472 MALLOC_ACQUIRE();
473 dump_memory_regions();
474 MALLOC_RELEASE();
475}
476
477static int validate_memory_regions()
478{
479 ASSERT_MALLOC_IS_ACQUIRED();
480 RootRegion *root = listOfAllRegions;
481 while(root)
482 {
483 Region *r = (Region*)root;
484 if (!debug_region_is_consistent(r))
485 {
486 MAIN_THREAD_ASYNC_EM_ASM(console.error('Used region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+') is corrupt (size markers in the beginning and at the end of the region do not match!)'),
487 r, r->size, region_ceiling_size(r) == r->size);
488 return 1;
489 }
490 uint8_t *lastRegionEnd = root->endPtr;
491 while((uint8_t*)r < lastRegionEnd)
492 {
493 if (!debug_region_is_consistent(r))
494 {
495 MAIN_THREAD_ASYNC_EM_ASM(console.error('Used region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+') is corrupt (size markers in the beginning and at the end of the region do not match!)'),
496 r, r->size, region_ceiling_size(r) == r->size);
497 return 1;
498 }
499 if (r->size == 0)
500 break;
501 r = next_region(r);
502 }
503 root = root->next;
504 }
505 for(int i = 0; i < NUM_FREE_BUCKETS; ++i)
506 {
507 Region *prev = &freeRegionBuckets[i];
508 Region *fr = freeRegionBuckets[i].next;
509 while(fr != &freeRegionBuckets[i])
510 {
511 if (!debug_region_is_consistent(fr) || !region_is_free(fr) || fr->prev != prev || fr->next == fr || fr->prev == fr)
512 {
513 MAIN_THREAD_ASYNC_EM_ASM(console.log('In bucket '+$0+', free region 0x'+($1>>>0).toString(16)+', size: ' + ($2>>>0) + ' (size at ceiling: '+($3>>>0)+'), prev: 0x' + ($4>>>0).toString(16) + ', next: 0x' + ($5>>>0).toString(16) + ' is corrupt!'),
514 i, fr, fr->size, size_of_region_from_ceiling(fr), fr->prev, fr->next);
515 return 1;
516 }
517 prev = fr;
518 fr = fr->next;
519 }
520 }
521 return 0;
522}
523
524int emmalloc_validate_memory_regions()
525{
526 MALLOC_ACQUIRE();
527 int memoryError = validate_memory_regions();
528 MALLOC_RELEASE();
529 return memoryError;
530}
531#endif
532
533static bool claim_more_memory(size_t numBytes)
534{
535#ifdef EMMALLOC_VERBOSE
536 MAIN_THREAD_ASYNC_EM_ASM(console.log('claim_more_memory(numBytes='+($0>>>0)+ ')'), numBytes);
537#endif
538
539#ifdef EMMALLOC_MEMVALIDATE
540 validate_memory_regions();
541#endif
542
543 uint8_t *startPtr;
544 uint8_t *endPtr;
545 do {
546 // If this is the first time we're called, see if we can use
547 // the initial heap memory set up by wasm-ld.
548 if (!listOfAllRegions) {
549 unsigned char *heap_base = &__heap_base;
550 unsigned char *heap_end = &__heap_end;
551 if (heap_end < heap_base) {
552 __builtin_trap();
553 }
554 if (numBytes <= (size_t)(heap_end - heap_base)) {
555 startPtr = heap_base;
556 endPtr = heap_end;
557 break;
558 }
559 }
560
561 // Round numBytes up to the nearest page size.
562 numBytes = (numBytes + (PAGE_SIZE-1)) & -PAGE_SIZE;
563
564 // Claim memory via sbrk
565 startPtr = (uint8_t*)sbrk(numBytes);
566 if ((intptr_t)startPtr == -1)
567 {
568#ifdef EMMALLOC_VERBOSE
569 MAIN_THREAD_ASYNC_EM_ASM(console.error('claim_more_memory: sbrk failed!'));
570#endif
571 return false;
572 }
573#ifdef EMMALLOC_VERBOSE
574 MAIN_THREAD_ASYNC_EM_ASM(console.log('claim_more_memory: claimed 0x' + ($0>>>0).toString(16) + ' - 0x' + ($1>>>0).toString(16) + ' (' + ($2>>>0) + ' bytes) via sbrk()'), startPtr, startPtr + numBytes, numBytes);
575#endif
576 assert(HAS_ALIGNMENT(startPtr, alignof(size_t)));
577 endPtr = startPtr + numBytes;
578 } while (0);
579
580 // Create a sentinel region at the end of the new heap block
581 Region *endSentinelRegion = (Region*)(endPtr - sizeof(Region));
582 create_used_region(endSentinelRegion, sizeof(Region));
583
584 // If we are the sole user of sbrk(), it will feed us continuous/consecutive memory addresses - take advantage
585 // of that if so: instead of creating two disjoint memory regions blocks, expand the previous one to a larger size.
586 uint8_t *previousSbrkEndAddress = listOfAllRegions ? listOfAllRegions->endPtr : 0;
587 if (startPtr == previousSbrkEndAddress)
588 {
589 Region *prevEndSentinel = prev_region((Region*)startPtr);
590 assert(debug_region_is_consistent(prevEndSentinel));
591 assert(region_is_in_use(prevEndSentinel));
592 Region *prevRegion = prev_region(prevEndSentinel);
593 assert(debug_region_is_consistent(prevRegion));
594
595 listOfAllRegions->endPtr = endPtr;
596
597 // Two scenarios, either the last region of the previous block was in use, in which case we need to create
598 // a new free region in the newly allocated space; or it was free, in which case we can extend that region
599 // to cover a larger size.
600 if (region_is_free(prevRegion))
601 {
602 size_t newFreeRegionSize = (uint8_t*)endSentinelRegion - (uint8_t*)prevRegion;
603 unlink_from_free_list(prevRegion);
604 create_free_region(prevRegion, newFreeRegionSize);
605 link_to_free_list(prevRegion);
606 return true;
607 }
608 // else: last region of the previous block was in use. Since we are joining two consecutive sbrk() blocks,
609 // we can swallow the end sentinel of the previous block away.
610 startPtr -= sizeof(Region);
611 }
612 else
613 {
614 // Create a root region at the start of the heap block
615 create_used_region(startPtr, sizeof(Region));
616
617 // Dynamic heap start region:
618 RootRegion *newRegionBlock = (RootRegion*)startPtr;
619 newRegionBlock->next = listOfAllRegions; // Pointer to next region block head
620 newRegionBlock->endPtr = endPtr; // Pointer to the end address of this region block
621 listOfAllRegions = newRegionBlock;
622 startPtr += sizeof(Region);
623 }
624
625 // Create a new memory region for the new claimed free space.
626 create_free_region(startPtr, (uint8_t*)endSentinelRegion - startPtr);
627 link_to_free_list((Region*)startPtr);
628 return true;
629}
630
631#if 0
632// Initialize emmalloc during static initialization.
633// See system/lib/README.md for static constructor ordering.
634__attribute__((constructor(47)))
635static void initialize_emmalloc_heap()
636{
637 // Initialize circular doubly linked lists representing free space
638 // Never useful to unroll this for loop, just takes up code size.
639#pragma clang loop unroll(disable)
640 for(int i = 0; i < NUM_FREE_BUCKETS; ++i)
641 freeRegionBuckets[i].prev = freeRegionBuckets[i].next = &freeRegionBuckets[i];
642
643#ifdef EMMALLOC_VERBOSE
644 MAIN_THREAD_ASYNC_EM_ASM(console.log('initialize_emmalloc_heap()'));
645#endif
646
647 // Start with a tiny dynamic region.
648 claim_more_memory(3*sizeof(Region));
649}
650
651void emmalloc_blank_slate_from_orbit()
652{
653 MALLOC_ACQUIRE();
654 listOfAllRegions = NULL;
655 freeRegionBucketsUsed = 0;
656 initialize_emmalloc_heap();
657 MALLOC_RELEASE();
658}
659#endif
660
661static void *attempt_allocate(Region *freeRegion, size_t alignment, size_t size)
662{
663 ASSERT_MALLOC_IS_ACQUIRED();
664 assert(freeRegion);
665 // Look at the next potential free region to allocate into.
666 // First, we should check if the free region has enough of payload bytes contained
667 // in it to accommodate the new allocation. This check needs to take account the
668 // requested allocation alignment, so the payload memory area needs to be rounded
669 // upwards to the desired alignment.
670 uint8_t *payloadStartPtr = region_payload_start_ptr(freeRegion);
671 uint8_t *payloadStartPtrAligned = ALIGN_UP(payloadStartPtr, alignment);
672 uint8_t *payloadEndPtr = region_payload_end_ptr(freeRegion);
673
674 // Do we have enough free space, taking into account alignment?
675 if (payloadStartPtrAligned + size > payloadEndPtr)
676 return NULL;
677
678 // We have enough free space, so the memory allocation will be made into this region. Remove this free region
679 // from the list of free regions: whatever slop remains will be later added back to the free region pool.
680 unlink_from_free_list(freeRegion);
681
682 // Before we proceed further, fix up the boundary of this region and the region that precedes this one,
683 // so that the boundary between the two regions happens at a right spot for the payload to be aligned.
684 if (payloadStartPtr != payloadStartPtrAligned)
685 {
686 Region *prevRegion = prev_region((Region*)freeRegion);
687 // We never have two free regions adjacent to each other, so the region before this free
688 // region should be in use.
689 assert(region_is_in_use(prevRegion));
690 size_t regionBoundaryBumpAmount = payloadStartPtrAligned - payloadStartPtr;
691 size_t newThisRegionSize = freeRegion->size - regionBoundaryBumpAmount;
692 create_used_region(prevRegion, prevRegion->size + regionBoundaryBumpAmount);
693 freeRegion = (Region *)((uint8_t*)freeRegion + regionBoundaryBumpAmount);
694 freeRegion->size = newThisRegionSize;
695 }
696 // Next, we need to decide whether this region is so large that it should be split into two regions,
697 // one representing the newly used memory area, and at the high end a remaining leftover free area.
698 // This splitting to two is done always if there is enough space for the high end to fit a region.
699 // Carve 'size' bytes of payload off this region. So,
700 // [sz prev next sz]
701 // becomes
702 // [sz payload sz] [sz prev next sz]
703 if (sizeof(Region) + REGION_HEADER_SIZE + size <= freeRegion->size)
704 {
705 // There is enough space to keep a free region at the end of the carved out block
706 // -> construct the new block
707 Region *newFreeRegion = (Region *)((uint8_t*)freeRegion + REGION_HEADER_SIZE + size);
708 create_free_region(newFreeRegion, freeRegion->size - size - REGION_HEADER_SIZE);
709 link_to_free_list(newFreeRegion);
710
711 // Recreate the resized Region under its new size.
712 create_used_region(freeRegion, size + REGION_HEADER_SIZE);
713 }
714 else
715 {
716 // There is not enough space to split the free memory region into used+free parts, so consume the whole
717 // region as used memory, not leaving a free memory region behind.
718 // Initialize the free region as used by resetting the ceiling size to the same value as the size at bottom.
719 ((size_t*)((uint8_t*)freeRegion + freeRegion->size))[-1] = freeRegion->size;
720 }
721
722#ifdef __EMSCRIPTEN_TRACING__
723 emscripten_trace_record_allocation(freeRegion, freeRegion->size);
724#endif
725
726#ifdef EMMALLOC_VERBOSE
727 MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_allocate - succeeded allocating memory, region ptr=0x' + ($0>>>0).toString(16) + ', align=' + $1 + ', payload size=' + ($2>>>0) + ' bytes)'), freeRegion, alignment, size);
728#endif
729
730 return (uint8_t*)freeRegion + sizeof(size_t);
731}
732
733static size_t validate_alloc_alignment(size_t alignment)
734{
735 // Cannot perform allocations that are less than 4 byte aligned, because the Region
736 // control structures need to be aligned. Also round up to minimum outputted alignment.
737 alignment = MAX(alignment, MALLOC_ALIGNMENT);
738 // Arbitrary upper limit on alignment - very likely a programming bug if alignment is higher than this.
739 assert(alignment <= 1024*1024);
740 return alignment;
741}
742
743static size_t validate_alloc_size(size_t size)
744{
745 assert(size + REGION_HEADER_SIZE > size);
746
747 // Allocation sizes must be a multiple of pointer sizes, and at least 2*sizeof(pointer).
748 size_t validatedSize = size > SMALLEST_ALLOCATION_SIZE ? (size_t)ALIGN_UP(size, sizeof(Region*)) : SMALLEST_ALLOCATION_SIZE;
749 assert(validatedSize >= size); // 32-bit wraparound should not occur, too large sizes should be stopped before
750
751 return validatedSize;
752}
753
754static void *allocate_memory(size_t alignment, size_t size)
755{
756 ASSERT_MALLOC_IS_ACQUIRED();
757
758#ifdef EMMALLOC_VERBOSE
759 MAIN_THREAD_ASYNC_EM_ASM(console.log('allocate_memory(align=' + $0 + ', size=' + ($1>>>0) + ' bytes)'), alignment, size);
760#endif
761
762#ifdef EMMALLOC_MEMVALIDATE
763 validate_memory_regions();
764#endif
765
766 if (!IS_POWER_OF_2(alignment))
767 {
768#ifdef EMMALLOC_VERBOSE
769 MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: alignment not power of 2!'));
770#endif
771 return 0;
772 }
773
774 if (size > MAX_ALLOC_SIZE)
775 {
776#ifdef EMMALLOC_VERBOSE
777 MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size);
778#endif
779 return 0;
780 }
781
782 alignment = validate_alloc_alignment(alignment);
783 size = validate_alloc_size(size);
784
785 // Attempt to allocate memory starting from smallest bucket that can contain the required amount of memory.
786 // Under normal alignment conditions this should always be the first or second bucket we look at, but if
787 // performing an allocation with complex alignment, we may need to look at multiple buckets.
788 int bucketIndex = compute_free_list_bucket(size);
789 BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed >> bucketIndex;
790
791 // Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap.
792 while(bucketMask)
793 {
794 BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask);
795 bucketIndex += indexAdd;
796 bucketMask >>= indexAdd;
797 assert(bucketIndex >= 0);
798 assert(bucketIndex <= NUM_FREE_BUCKETS-1);
799 assert(freeRegionBucketsUsed & (((BUCKET_BITMASK_T)1) << bucketIndex));
800
801 Region *freeRegion = freeRegionBuckets[bucketIndex].next;
802 assert(freeRegion);
803 if (freeRegion != &freeRegionBuckets[bucketIndex])
804 {
805 void *ptr = attempt_allocate(freeRegion, alignment, size);
806 if (ptr)
807 return ptr;
808
809 // We were not able to allocate from the first region found in this bucket, so penalize
810 // the region by cycling it to the end of the doubly circular linked list. (constant time)
811 // This provides a randomized guarantee that when performing allocations of size k to a
812 // bucket of [k-something, k+something] range, we will not always attempt to satisfy the
813 // allocation from the same available region at the front of the list, but we try each
814 // region in turn.
815 unlink_from_free_list(freeRegion);
816 prepend_to_free_list(freeRegion, &freeRegionBuckets[bucketIndex]);
817 // But do not stick around to attempt to look at other regions in this bucket - move
818 // to search the next populated bucket index if this did not fit. This gives a practical
819 // "allocation in constant time" guarantee, since the next higher bucket will only have
820 // regions that are all of strictly larger size than the requested allocation. Only if
821 // there is a difficult alignment requirement we may fail to perform the allocation from
822 // a region in the next bucket, and if so, we keep trying higher buckets until one of them
823 // works.
824 ++bucketIndex;
825 bucketMask >>= 1;
826 }
827 else
828 {
829 // This bucket was not populated after all with any regions,
830 // but we just had a stale bit set to mark a populated bucket.
831 // Reset the bit to update latest status so that we do not
832 // redundantly look at this bucket again.
833 freeRegionBucketsUsed &= ~(((BUCKET_BITMASK_T)1) << bucketIndex);
834 bucketMask ^= 1;
835 }
836 // Instead of recomputing bucketMask from scratch at the end of each loop, it is updated as we go,
837 // to avoid undefined behavior with (x >> 32)/(x >> 64) when bucketIndex reaches 32/64, (the shift would comes out as a no-op instead of 0).
838
839 assert((bucketIndex == NUM_FREE_BUCKETS && bucketMask == 0) || (bucketMask == freeRegionBucketsUsed >> bucketIndex));
840 }
841
842 // None of the buckets were able to accommodate an allocation. If this happens we are almost out of memory.
843 // The largest bucket might contain some suitable regions, but we only looked at one region in that bucket, so
844 // as a last resort, loop through more free regions in the bucket that represents the largest allocations available.
845 // But only if the bucket representing largest allocations available is not any of the first thirty buckets,
846 // these represent allocatable areas less than <1024 bytes - which could be a lot of scrap.
847 // In such case, prefer to sbrk() in more memory right away.
848 int largestBucketIndex = NUM_FREE_BUCKETS - 1 - __builtin_clzll(freeRegionBucketsUsed);
849 // freeRegion will be null if there is absolutely no memory left. (all buckets are 100% used)
850 Region *freeRegion = freeRegionBucketsUsed ? freeRegionBuckets[largestBucketIndex].next : 0;
851 if (freeRegionBucketsUsed >> 30)
852 {
853 // Look only at a constant number of regions in this bucket max, to avoid bad worst case behavior.
854 // If this many regions cannot find free space, we give up and prefer to sbrk() more instead.
855 const int maxRegionsToTryBeforeGivingUp = 99;
856 int numTriesLeft = maxRegionsToTryBeforeGivingUp;
857 while(freeRegion != &freeRegionBuckets[largestBucketIndex] && numTriesLeft-- > 0)
858 {
859 void *ptr = attempt_allocate(freeRegion, alignment, size);
860 if (ptr)
861 return ptr;
862 freeRegion = freeRegion->next;
863 }
864 }
865
866 // We were unable to find a free memory region. Must sbrk() in more memory!
867 size_t numBytesToClaim = size+sizeof(Region)*3;
868 assert(numBytesToClaim > size); // 32-bit wraparound should not happen here, allocation size has been validated above!
869 bool success = claim_more_memory(numBytesToClaim);
870 if (success)
871 return allocate_memory(alignment, size); // Recurse back to itself to try again
872
873 // also sbrk() failed, we are really really constrained :( As a last resort, go back to looking at the
874 // bucket we already looked at above, continuing where the above search left off - perhaps there are
875 // regions we overlooked the first time that might be able to satisfy the allocation.
876 if (freeRegion)
877 {
878 while(freeRegion != &freeRegionBuckets[largestBucketIndex])
879 {
880 void *ptr = attempt_allocate(freeRegion, alignment, size);
881 if (ptr)
882 return ptr;
883 freeRegion = freeRegion->next;
884 }
885 }
886
887#ifdef EMMALLOC_VERBOSE
888 MAIN_THREAD_ASYNC_EM_ASM(console.log('Could not find a free memory block!'));
889#endif
890
891 return 0;
892}
893
894static
895void *emmalloc_memalign(size_t alignment, size_t size)
896{
897 MALLOC_ACQUIRE();
898 void *ptr = allocate_memory(alignment, size);
899 MALLOC_RELEASE();
900 return ptr;
901}
902
903#if 0
904void * EMMALLOC_EXPORT memalign(size_t alignment, size_t size)
905{
906 return emmalloc_memalign(alignment, size);
907}
908#endif
909
910void * EMMALLOC_EXPORT aligned_alloc(size_t alignment, size_t size)
911{
912 if ((alignment % sizeof(void *) != 0) || (size % alignment) != 0)
913 return 0;
914 return emmalloc_memalign(alignment, size);
915}
916
917static
918void *emmalloc_malloc(size_t size)
919{
920 return emmalloc_memalign(MALLOC_ALIGNMENT, size);
921}
922
923void * EMMALLOC_EXPORT malloc(size_t size)
924{
925 return emmalloc_malloc(size);
926}
927
928static
929size_t emmalloc_usable_size(void *ptr)
930{
931 if (!ptr)
932 return 0;
933
934 uint8_t *regionStartPtr = (uint8_t*)ptr - sizeof(size_t);
935 Region *region = (Region*)(regionStartPtr);
936 assert(HAS_ALIGNMENT(region, sizeof(size_t)));
937
938 MALLOC_ACQUIRE();
939
940 size_t size = region->size;
941 assert(size >= sizeof(Region));
942 assert(region_is_in_use(region));
943
944 MALLOC_RELEASE();
945
946 return size - REGION_HEADER_SIZE;
947}
948
949size_t EMMALLOC_EXPORT malloc_usable_size(void *ptr)
950{
951 return emmalloc_usable_size(ptr);
952}
953
954static
955void emmalloc_free(void *ptr)
956{
957#ifdef EMMALLOC_MEMVALIDATE
958 emmalloc_validate_memory_regions();
959#endif
960
961 if (!ptr)
962 return;
963
964#ifdef EMMALLOC_VERBOSE
965 MAIN_THREAD_ASYNC_EM_ASM(console.log('free(ptr=0x'+($0>>>0).toString(16)+')'), ptr);
966#endif
967
968 uint8_t *regionStartPtr = (uint8_t*)ptr - sizeof(size_t);
969 Region *region = (Region*)(regionStartPtr);
970 assert(HAS_ALIGNMENT(region, sizeof(size_t)));
971
972 MALLOC_ACQUIRE();
973
974 size_t size = region->size;
975#ifdef EMMALLOC_VERBOSE
976 if (size < sizeof(Region) || !region_is_in_use(region))
977 {
978 if (debug_region_is_consistent(region))
979 // LLVM wasm backend bug: cannot use MAIN_THREAD_ASYNC_EM_ASM() here, that generates internal compiler error
980 // Reproducible by running e.g. other.test_alloc_3GB
981 EM_ASM(console.error('Double free at region ptr 0x' + ($0>>>0).toString(16) + ', region->size: 0x' + ($1>>>0).toString(16) + ', region->sizeAtCeiling: 0x' + ($2>>>0).toString(16) + ')'), region, size, region_ceiling_size(region));
982 else
983 MAIN_THREAD_ASYNC_EM_ASM(console.error('Corrupt region at region ptr 0x' + ($0>>>0).toString(16) + ' region->size: 0x' + ($1>>>0).toString(16) + ', region->sizeAtCeiling: 0x' + ($2>>>0).toString(16) + ')'), region, size, region_ceiling_size(region));
984 }
985#endif
986 assert(size >= sizeof(Region));
987 assert(region_is_in_use(region));
988
989#ifdef __EMSCRIPTEN_TRACING__
990 emscripten_trace_record_free(region);
991#endif
992
993 // Check merging with left side
994 size_t prevRegionSizeField = ((size_t*)region)[-1];
995 size_t prevRegionSize = prevRegionSizeField & ~FREE_REGION_FLAG;
996 if (prevRegionSizeField != prevRegionSize) // Previous region is free?
997 {
998 Region *prevRegion = (Region*)((uint8_t*)region - prevRegionSize);
999 assert(debug_region_is_consistent(prevRegion));
1000 unlink_from_free_list(prevRegion);
1001 regionStartPtr = (uint8_t*)prevRegion;
1002 size += prevRegionSize;
1003 }
1004
1005 // Check merging with right side
1006 Region *nextRegion = next_region(region);
1007 assert(debug_region_is_consistent(nextRegion));
1008 size_t sizeAtEnd = *(size_t*)region_payload_end_ptr(nextRegion);
1009 if (nextRegion->size != sizeAtEnd)
1010 {
1011 unlink_from_free_list(nextRegion);
1012 size += nextRegion->size;
1013 }
1014
1015 create_free_region(regionStartPtr, size);
1016 link_to_free_list((Region*)regionStartPtr);
1017
1018 MALLOC_RELEASE();
1019
1020#ifdef EMMALLOC_MEMVALIDATE
1021 emmalloc_validate_memory_regions();
1022#endif
1023}
1024
1025void EMMALLOC_EXPORT free(void *ptr)
1026{
1027 emmalloc_free(ptr);
1028}
1029
1030// Can be called to attempt to increase or decrease the size of the given region
1031// to a new size (in-place). Returns 1 if resize succeeds, and 0 on failure.
1032static int attempt_region_resize(Region *region, size_t size)
1033{
1034 ASSERT_MALLOC_IS_ACQUIRED();
1035 assert(size > 0);
1036 assert(HAS_ALIGNMENT(size, sizeof(size_t)));
1037
1038#ifdef EMMALLOC_VERBOSE
1039 MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_region_resize(region=0x' + ($0>>>0).toString(16) + ', size=' + ($1>>>0) + ' bytes)'), region, size);
1040#endif
1041
1042 // First attempt to resize this region, if the next region that follows this one
1043 // is a free region.
1044 Region *nextRegion = next_region(region);
1045 uint8_t *nextRegionEndPtr = (uint8_t*)nextRegion + nextRegion->size;
1046 size_t sizeAtCeiling = ((size_t*)nextRegionEndPtr)[-1];
1047 if (nextRegion->size != sizeAtCeiling) // Next region is free?
1048 {
1049 assert(region_is_free(nextRegion));
1050 uint8_t *newNextRegionStartPtr = (uint8_t*)region + size;
1051 assert(HAS_ALIGNMENT(newNextRegionStartPtr, sizeof(size_t)));
1052 // Next region does not shrink to too small size?
1053 if (newNextRegionStartPtr + sizeof(Region) <= nextRegionEndPtr)
1054 {
1055 unlink_from_free_list(nextRegion);
1056 create_free_region(newNextRegionStartPtr, nextRegionEndPtr - newNextRegionStartPtr);
1057 link_to_free_list((Region*)newNextRegionStartPtr);
1058 create_used_region(region, newNextRegionStartPtr - (uint8_t*)region);
1059 return 1;
1060 }
1061 // If we remove the next region altogether, allocation is satisfied?
1062 if (newNextRegionStartPtr <= nextRegionEndPtr)
1063 {
1064 unlink_from_free_list(nextRegion);
1065 create_used_region(region, region->size + nextRegion->size);
1066 return 1;
1067 }
1068 }
1069 else
1070 {
1071 // Next region is an used region - we cannot change its starting address. However if we are shrinking the
1072 // size of this region, we can create a new free region between this and the next used region.
1073 if (size + sizeof(Region) <= region->size)
1074 {
1075 size_t freeRegionSize = region->size - size;
1076 create_used_region(region, size);
1077 Region *freeRegion = (Region *)((uint8_t*)region + size);
1078 create_free_region(freeRegion, freeRegionSize);
1079 link_to_free_list(freeRegion);
1080 return 1;
1081 }
1082 else if (size <= region->size)
1083 {
1084 // Caller was asking to shrink the size, but due to not being able to fit a full Region in the shrunk
1085 // area, we cannot actually do anything. This occurs if the shrink amount is really small. In such case,
1086 // just call it success without doing any work.
1087 return 1;
1088 }
1089 }
1090#ifdef EMMALLOC_VERBOSE
1091 MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_region_resize failed.'));
1092#endif
1093 return 0;
1094}
1095
1096static int acquire_and_attempt_region_resize(Region *region, size_t size)
1097{
1098 MALLOC_ACQUIRE();
1099 int success = attempt_region_resize(region, size);
1100 MALLOC_RELEASE();
1101 return success;
1102}
1103
1104static
1105void *emmalloc_aligned_realloc(void *ptr, size_t alignment, size_t size)
1106{
1107#ifdef EMMALLOC_VERBOSE
1108 MAIN_THREAD_ASYNC_EM_ASM(console.log('aligned_realloc(ptr=0x' + ($0>>>0).toString(16) + ', alignment=' + $1 + ', size=' + ($2>>>0)), ptr, alignment, size);
1109#endif
1110
1111 if (!ptr)
1112 return emmalloc_memalign(alignment, size);
1113
1114 if (size == 0)
1115 {
1116 free(ptr);
1117 return 0;
1118 }
1119
1120 if (size > MAX_ALLOC_SIZE)
1121 {
1122#ifdef EMMALLOC_VERBOSE
1123 MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size);
1124#endif
1125 return 0;
1126 }
1127
1128 assert(IS_POWER_OF_2(alignment));
1129 // aligned_realloc() cannot be used to ask to change the alignment of a pointer.
1130 assert(HAS_ALIGNMENT(ptr, alignment));
1131 size = validate_alloc_size(size);
1132
1133 // Calculate the region start address of the original allocation
1134 Region *region = (Region*)((uint8_t*)ptr - sizeof(size_t));
1135
1136 // First attempt to resize the given region to avoid having to copy memory around
1137 if (acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE))
1138 {
1139#ifdef __EMSCRIPTEN_TRACING__
1140 emscripten_trace_record_reallocation(ptr, ptr, size);
1141#endif
1142 return ptr;
1143 }
1144
1145 // If resize failed, we must allocate a new region, copy the data over, and then
1146 // free the old region.
1147 void *newptr = emmalloc_memalign(alignment, size);
1148 if (newptr)
1149 {
1150 memcpy(newptr, ptr, MIN(size, region->size - REGION_HEADER_SIZE));
1151 free(ptr);
1152 }
1153 // N.B. If there is not enough memory, the old memory block should not be freed and
1154 // null pointer is returned.
1155 return newptr;
1156}
1157
1158#if 0
1159void * EMMALLOC_EXPORT aligned_realloc(void *ptr, size_t alignment, size_t size)
1160{
1161 return emmalloc_aligned_realloc(ptr, alignment, size);
1162}
1163#endif
1164
1165#if 0
1166// realloc_try() is like realloc(), but only attempts to try to resize the existing memory
1167// area. If resizing the existing memory area fails, then realloc_try() will return 0
1168// (the original memory block is not freed or modified). If resizing succeeds, previous
1169// memory contents will be valid up to min(old length, new length) bytes.
1170void *emmalloc_realloc_try(void *ptr, size_t size)
1171{
1172 if (!ptr)
1173 return 0;
1174
1175 if (size == 0)
1176 {
1177 free(ptr);
1178 return 0;
1179 }
1180
1181 if (size > MAX_ALLOC_SIZE)
1182 {
1183#ifdef EMMALLOC_VERBOSE
1184 MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size);
1185#endif
1186 return 0;
1187 }
1188
1189 size = validate_alloc_size(size);
1190
1191 // Calculate the region start address of the original allocation
1192 Region *region = (Region*)((uint8_t*)ptr - sizeof(size_t));
1193
1194 // Attempt to resize the given region to avoid having to copy memory around
1195 int success = acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE);
1196#ifdef __EMSCRIPTEN_TRACING__
1197 if (success)
1198 emscripten_trace_record_reallocation(ptr, ptr, size);
1199#endif
1200 return success ? ptr : 0;
1201}
1202
1203// emmalloc_aligned_realloc_uninitialized() is like aligned_realloc(), but old memory contents
1204// will be undefined after reallocation. (old memory is not preserved in any case)
1205void *emmalloc_aligned_realloc_uninitialized(void *ptr, size_t alignment, size_t size)
1206{
1207 if (!ptr)
1208 return emmalloc_memalign(alignment, size);
1209
1210 if (size == 0)
1211 {
1212 free(ptr);
1213 return 0;
1214 }
1215
1216 if (size > MAX_ALLOC_SIZE)
1217 {
1218#ifdef EMMALLOC_VERBOSE
1219 MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size);
1220#endif
1221 return 0;
1222 }
1223
1224 size = validate_alloc_size(size);
1225
1226 // Calculate the region start address of the original allocation
1227 Region *region = (Region*)((uint8_t*)ptr - sizeof(size_t));
1228
1229 // First attempt to resize the given region to avoid having to copy memory around
1230 if (acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE))
1231 {
1232#ifdef __EMSCRIPTEN_TRACING__
1233 emscripten_trace_record_reallocation(ptr, ptr, size);
1234#endif
1235 return ptr;
1236 }
1237
1238 // If resize failed, drop the old region and allocate a new region. Memory is not
1239 // copied over
1240 free(ptr);
1241 return emmalloc_memalign(alignment, size);
1242}
1243#endif
1244
1245static
1246void *emmalloc_realloc(void *ptr, size_t size)
1247{
1248 return emmalloc_aligned_realloc(ptr, MALLOC_ALIGNMENT, size);
1249}
1250
1251void * EMMALLOC_EXPORT realloc(void *ptr, size_t size)
1252{
1253 return emmalloc_realloc(ptr, size);
1254}
1255
1256#if 0
1257// realloc_uninitialized() is like realloc(), but old memory contents
1258// will be undefined after reallocation. (old memory is not preserved in any case)
1259void *emmalloc_realloc_uninitialized(void *ptr, size_t size)
1260{
1261 return emmalloc_aligned_realloc_uninitialized(ptr, MALLOC_ALIGNMENT, size);
1262}
1263#endif
1264
1265static
1266int emmalloc_posix_memalign(void **memptr, size_t alignment, size_t size)
1267{
1268 assert(memptr);
1269 if (alignment % sizeof(void *) != 0)
1270 return 22/* EINVAL*/;
1271 *memptr = emmalloc_memalign(alignment, size);
1272 return *memptr ? 0 : 12/*ENOMEM*/;
1273}
1274
1275int EMMALLOC_EXPORT posix_memalign(void **memptr, size_t alignment, size_t size)
1276{
1277 return emmalloc_posix_memalign(memptr, alignment, size);
1278}
1279
1280static
1281void *emmalloc_calloc(size_t num, size_t size)
1282{
1283 size_t bytes = num*size;
1284 void *ptr = emmalloc_memalign(MALLOC_ALIGNMENT, bytes);
1285 if (ptr)
1286 memset(ptr, 0, bytes);
1287 return ptr;
1288}
1289
1290void * EMMALLOC_EXPORT calloc(size_t num, size_t size)
1291{
1292 return emmalloc_calloc(num, size);
1293}
1294
1295#if 0
1296static int count_linked_list_size(Region *list)
1297{
1298 int size = 1;
1299 for(Region *i = list->next; i != list; list = list->next)
1300 ++size;
1301 return size;
1302}
1303
1304static size_t count_linked_list_space(Region *list)
1305{
1306 size_t space = 0;
1307 for(Region *i = list->next; i != list; list = list->next)
1308 space += region_payload_end_ptr(i) - region_payload_start_ptr(i);
1309 return space;
1310}
1311
1312struct mallinfo emmalloc_mallinfo()
1313{
1314 MALLOC_ACQUIRE();
1315
1316 struct mallinfo info;
1317 // Non-mmapped space allocated (bytes): For emmalloc,
1318 // let's define this as the difference between heap size and dynamic top end.
1319 info.arena = emscripten_get_heap_size() - (size_t)sbrk(0);
1320 // Number of "ordinary" blocks. Let's define this as the number of highest
1321 // size blocks. (subtract one from each, since there is a sentinel node in each list)
1322 info.ordblks = count_linked_list_size(&freeRegionBuckets[NUM_FREE_BUCKETS-1])-1;
1323 // Number of free "fastbin" blocks. For emmalloc, define this as the number
1324 // of blocks that are not in the largest pristine block.
1325 info.smblks = 0;
1326 // The total number of bytes in free "fastbin" blocks.
1327 info.fsmblks = 0;
1328 for(int i = 0; i < NUM_FREE_BUCKETS-1; ++i)
1329 {
1330 info.smblks += count_linked_list_size(&freeRegionBuckets[i])-1;
1331 info.fsmblks += count_linked_list_space(&freeRegionBuckets[i]);
1332 }
1333
1334 info.hblks = 0; // Number of mmapped regions: always 0. (no mmap support)
1335 info.hblkhd = 0; // Amount of bytes in mmapped regions: always 0. (no mmap support)
1336
1337 // Walk through all the heap blocks to report the following data:
1338 // The "highwater mark" for allocated space—that is, the maximum amount of
1339 // space that was ever allocated. Emmalloc does not want to pay code to
1340 // track this, so this is only reported from current allocation data, and
1341 // may not be accurate.
1342 info.usmblks = 0;
1343 info.uordblks = 0; // The total number of bytes used by in-use allocations.
1344 info.fordblks = 0; // The total number of bytes in free blocks.
1345 // The total amount of releasable free space at the top of the heap.
1346 // This is the maximum number of bytes that could ideally be released by malloc_trim(3).
1347 Region *lastActualRegion = prev_region((Region*)(listOfAllRegions->endPtr - sizeof(Region)));
1348 info.keepcost = region_is_free(lastActualRegion) ? lastActualRegion->size : 0;
1349
1350 RootRegion *root = listOfAllRegions;
1351 while(root)
1352 {
1353 Region *r = (Region*)root;
1354 assert(debug_region_is_consistent(r));
1355 uint8_t *lastRegionEnd = root->endPtr;
1356 while((uint8_t*)r < lastRegionEnd)
1357 {
1358 assert(debug_region_is_consistent(r));
1359
1360 if (region_is_free(r))
1361 {
1362 // Count only the payload of the free block towards free memory.
1363 info.fordblks += region_payload_end_ptr(r) - region_payload_start_ptr(r);
1364 // But the header data of the free block goes towards used memory.
1365 info.uordblks += REGION_HEADER_SIZE;
1366 }
1367 else
1368 {
1369 info.uordblks += r->size;
1370 }
1371 // Update approximate watermark data
1372 info.usmblks = MAX(info.usmblks, (intptr_t)r + r->size);
1373
1374 if (r->size == 0)
1375 break;
1376 r = next_region(r);
1377 }
1378 root = root->next;
1379 }
1380
1381 MALLOC_RELEASE();
1382 return info;
1383}
1384
1385struct mallinfo EMMALLOC_EXPORT mallinfo()
1386{
1387 return emmalloc_mallinfo();
1388}
1389
1390// Note! This function is not fully multithreadin safe: while this function is running, other threads should not be
1391// allowed to call sbrk()!
1392static int trim_dynamic_heap_reservation(size_t pad)
1393{
1394 ASSERT_MALLOC_IS_ACQUIRED();
1395
1396 if (!listOfAllRegions)
1397 return 0; // emmalloc is not controlling any dynamic memory at all - cannot release memory.
1398 uint8_t *previousSbrkEndAddress = listOfAllRegions->endPtr;
1399 assert(sbrk(0) == previousSbrkEndAddress);
1400 size_t lastMemoryRegionSize = ((size_t*)previousSbrkEndAddress)[-1];
1401 assert(lastMemoryRegionSize == 16); // // The last memory region should be a sentinel node of exactly 16 bytes in size.
1402 Region *endSentinelRegion = (Region*)(previousSbrkEndAddress - sizeof(Region));
1403 Region *lastActualRegion = prev_region(endSentinelRegion);
1404
1405 // Round padding up to multiple of 4 bytes to keep sbrk() and memory region alignment intact.
1406 // Also have at least 8 bytes of payload so that we can form a full free region.
1407 size_t newRegionSize = (size_t)ALIGN_UP(pad, 4);
1408 if (pad > 0)
1409 newRegionSize += sizeof(Region) - (newRegionSize - pad);
1410
1411 if (!region_is_free(lastActualRegion) || lastActualRegion->size <= newRegionSize)
1412 return 0; // Last actual region is in use, or caller desired to leave more free memory intact than there is.
1413
1414 // This many bytes will be shrunk away.
1415 size_t shrinkAmount = lastActualRegion->size - newRegionSize;
1416 assert(HAS_ALIGNMENT(shrinkAmount, 4));
1417
1418 unlink_from_free_list(lastActualRegion);
1419 // If pad == 0, we should delete the last free region altogether. If pad > 0,
1420 // shrink the last free region to the desired size.
1421 if (newRegionSize > 0)
1422 {
1423 create_free_region(lastActualRegion, newRegionSize);
1424 link_to_free_list(lastActualRegion);
1425 }
1426
1427 // Recreate the sentinel region at the end of the last free region
1428 endSentinelRegion = (Region*)((uint8_t*)lastActualRegion + newRegionSize);
1429 create_used_region(endSentinelRegion, sizeof(Region));
1430
1431 // And update the size field of the whole region block.
1432 listOfAllRegions->endPtr = (uint8_t*)endSentinelRegion + sizeof(Region);
1433
1434 // Finally call sbrk() to shrink the memory area.
1435 void *oldSbrk = sbrk(-(intptr_t)shrinkAmount);
1436 assert((intptr_t)oldSbrk != -1); // Shrinking with sbrk() should never fail.
1437 assert(oldSbrk == previousSbrkEndAddress); // Another thread should not have raced to increase sbrk() on us!
1438
1439 // All successful, and we actually trimmed memory!
1440 return 1;
1441}
1442
1443int emmalloc_trim(size_t pad)
1444{
1445 MALLOC_ACQUIRE();
1446 int success = trim_dynamic_heap_reservation(pad);
1447 MALLOC_RELEASE();
1448 return success;
1449}
1450
1451int EMMALLOC_EXPORT malloc_trim(size_t pad)
1452{
1453 return emmalloc_trim(pad);
1454}
1455
1456size_t emmalloc_dynamic_heap_size()
1457{
1458 size_t dynamicHeapSize = 0;
1459
1460 MALLOC_ACQUIRE();
1461 RootRegion *root = listOfAllRegions;
1462 while(root)
1463 {
1464 dynamicHeapSize += root->endPtr - (uint8_t*)root;
1465 root = root->next;
1466 }
1467 MALLOC_RELEASE();
1468 return dynamicHeapSize;
1469}
1470
1471size_t emmalloc_free_dynamic_memory()
1472{
1473 size_t freeDynamicMemory = 0;
1474
1475 int bucketIndex = 0;
1476
1477 MALLOC_ACQUIRE();
1478 BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed;
1479
1480 // Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap.
1481 while(bucketMask)
1482 {
1483 BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask);
1484 bucketIndex += indexAdd;
1485 bucketMask >>= indexAdd;
1486 for(Region *freeRegion = freeRegionBuckets[bucketIndex].next;
1487 freeRegion != &freeRegionBuckets[bucketIndex];
1488 freeRegion = freeRegion->next)
1489 {
1490 freeDynamicMemory += freeRegion->size - REGION_HEADER_SIZE;
1491 }
1492 ++bucketIndex;
1493 bucketMask >>= 1;
1494 }
1495 MALLOC_RELEASE();
1496 return freeDynamicMemory;
1497}
1498
1499size_t emmalloc_compute_free_dynamic_memory_fragmentation_map(size_t freeMemorySizeMap[32])
1500{
1501 memset((void*)freeMemorySizeMap, 0, sizeof(freeMemorySizeMap[0])*32);
1502
1503 size_t numFreeMemoryRegions = 0;
1504 int bucketIndex = 0;
1505 MALLOC_ACQUIRE();
1506 BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed;
1507
1508 // Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap.
1509 while(bucketMask)
1510 {
1511 BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask);
1512 bucketIndex += indexAdd;
1513 bucketMask >>= indexAdd;
1514 for(Region *freeRegion = freeRegionBuckets[bucketIndex].next;
1515 freeRegion != &freeRegionBuckets[bucketIndex];
1516 freeRegion = freeRegion->next)
1517 {
1518 ++numFreeMemoryRegions;
1519 size_t freeDynamicMemory = freeRegion->size - REGION_HEADER_SIZE;
1520 if (freeDynamicMemory > 0)
1521 ++freeMemorySizeMap[31-__builtin_clz(freeDynamicMemory)];
1522 else
1523 ++freeMemorySizeMap[0];
1524 }
1525 ++bucketIndex;
1526 bucketMask >>= 1;
1527 }
1528 MALLOC_RELEASE();
1529 return numFreeMemoryRegions;
1530}
1531
1532size_t emmalloc_unclaimed_heap_memory(void) {
1533 return emscripten_get_heap_max() - (size_t)sbrk(0);
1534}
1535#endif
1536
1537// Define these to satisfy musl references.
1538void *__libc_malloc(size_t) __attribute__((alias("malloc")));
1539void __libc_free(void *) __attribute__((alias("free")));
1540void *__libc_calloc(size_t nmemb, size_t size) __attribute__((alias("calloc")));
lib/std/heap/SmpAllocator.zig+7-6
......@@ -26,6 +26,7 @@
2626//! By limiting the thread-local metadata array to the same number as the CPU
2727//! count, ensures that as threads are created and destroyed, they cycle
2828//! through the full set of freelists.
29const SmpAllocator = @This();
2930
3031const builtin = @import("builtin");
3132
......@@ -34,7 +35,7 @@ const assert = std.debug.assert;
3435const mem = std.mem;
3536const math = std.math;
3637const Allocator = std.mem.Allocator;
37const SmpAllocator = @This();
38const Alignment = std.mem.Alignment;
3839const PageAllocator = std.heap.PageAllocator;
3940
4041cpu_count: u32,
......@@ -114,7 +115,7 @@ comptime {
114115 assert(!builtin.single_threaded); // you're holding it wrong
115116}
116117
117fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
118fn alloc(context: *anyopaque, len: usize, alignment: Alignment, ra: usize) ?[*]u8 {
118119 _ = context;
119120 _ = ra;
120121 const class = sizeClassIndex(len, alignment);
......@@ -172,7 +173,7 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?
172173 }
173174}
174175
175fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) bool {
176fn resize(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ra: usize) bool {
176177 _ = context;
177178 _ = ra;
178179 const class = sizeClassIndex(memory.len, alignment);
......@@ -184,7 +185,7 @@ fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len:
184185 return new_class == class;
185186}
186187
187fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) ?[*]u8 {
188fn remap(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ra: usize) ?[*]u8 {
188189 _ = context;
189190 _ = ra;
190191 const class = sizeClassIndex(memory.len, alignment);
......@@ -196,7 +197,7 @@ fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: u
196197 return if (new_class == class) memory.ptr else null;
197198}
198199
199fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize) void {
200fn free(context: *anyopaque, memory: []u8, alignment: Alignment, ra: usize) void {
200201 _ = context;
201202 _ = ra;
202203 const class = sizeClassIndex(memory.len, alignment);
......@@ -214,7 +215,7 @@ fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize)
214215 t.frees[class] = @intFromPtr(node);
215216}
216217
217fn sizeClassIndex(len: usize, alignment: mem.Alignment) usize {
218fn sizeClassIndex(len: usize, alignment: Alignment) usize {
218219 return @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment), min_class) - min_class;
219220}
220221
src/libs/musl.zig+1-20
......@@ -352,8 +352,7 @@ const Ext = enum {
352352fn addSrcFile(arena: Allocator, source_table: *std.StringArrayHashMap(Ext), file_path: []const u8) !void {
353353 const ext: Ext = ext: {
354354 if (mem.endsWith(u8, file_path, ".c")) {
355 if (mem.startsWith(u8, file_path, "musl/src/malloc/") or
356 mem.startsWith(u8, file_path, "musl/src/string/") or
355 if (mem.startsWith(u8, file_path, "musl/src/string/") or
357356 mem.startsWith(u8, file_path, "musl/src/internal/"))
358357 {
359358 break :ext .o3;
......@@ -786,24 +785,6 @@ const src_files = [_][]const u8{
786785 "musl/src/locale/uselocale.c",
787786 "musl/src/locale/wcscoll.c",
788787 "musl/src/locale/wcsxfrm.c",
789 "musl/src/malloc/calloc.c",
790 "musl/src/malloc/free.c",
791 "musl/src/malloc/libc_calloc.c",
792 "musl/src/malloc/lite_malloc.c",
793 "musl/src/malloc/mallocng/aligned_alloc.c",
794 "musl/src/malloc/mallocng/donate.c",
795 "musl/src/malloc/mallocng/free.c",
796 "musl/src/malloc/mallocng/malloc.c",
797 "musl/src/malloc/mallocng/malloc_usable_size.c",
798 "musl/src/malloc/mallocng/realloc.c",
799 "musl/src/malloc/memalign.c",
800 "musl/src/malloc/oldmalloc/aligned_alloc.c",
801 "musl/src/malloc/oldmalloc/malloc.c",
802 "musl/src/malloc/oldmalloc/malloc_usable_size.c",
803 "musl/src/malloc/posix_memalign.c",
804 "musl/src/malloc/reallocarray.c",
805 "musl/src/malloc/realloc.c",
806 "musl/src/malloc/replaced.c",
807788 "musl/src/math/aarch64/fma.c",
808789 "musl/src/math/aarch64/fmaf.c",
809790 "musl/src/math/aarch64/llrint.c",
src/libs/wasi_libc.zig-20
......@@ -77,22 +77,6 @@ pub fn buildCrtFile(comp: *Compilation, crt_file: CrtFile, prog_node: std.Progre
7777 .libc_a => {
7878 var libc_sources = std.array_list.Managed(Compilation.CSourceFile).init(arena);
7979
80 {
81 // Compile emmalloc.
82 var args = std.array_list.Managed([]const u8).init(arena);
83 try addCCArgs(comp, arena, &args, .{ .want_O3 = true, .no_strict_aliasing = true });
84
85 for (emmalloc_src_files) |file_path| {
86 try libc_sources.append(.{
87 .src_path = try comp.dirs.zig_lib.join(arena, &.{
88 "libc", try sanitize(arena, file_path),
89 }),
90 .extra_flags = args.items,
91 .owner = undefined,
92 });
93 }
94 }
95
9680 {
9781 // Compile libc-bottom-half.
9882 var args = std.array_list.Managed([]const u8).init(arena);
......@@ -472,10 +456,6 @@ fn addLibcTopHalfIncludes(
472456 });
473457}
474458
475const emmalloc_src_files = [_][]const u8{
476 "wasi/emmalloc/emmalloc.c",
477};
478
479459const libc_bottom_half_src_files = [_][]const u8{
480460 "wasi/libc-bottom-half/cloudlibc/src/libc/dirent/closedir.c",
481461 "wasi/libc-bottom-half/cloudlibc/src/libc/dirent/dirfd.c",