Commit 4a3adaaa23faa1ae22fa72389850f04ece2570ca | zig

build.zig+3-18

 @@ -483,7 +483,7 @@ pub fn build(b: *std.Build) !void {
         .skip_linux = skip_linux,
         .skip_llvm = skip_llvm,
         .skip_libc = skip_libc,
-        .max_rss = 3_300_000_000,
+        .max_rss = 3_500_000_000,
     }));
     test_modules_step.dependOn(tests.addModuleTests(b, .{
 @@ -518,7 +518,7 @@ pub fn build(b: *std.Build) !void {
         .test_extra_targets = test_extra_targets,
         .root_src = "lib/c.zig",
         .name = "zigc",
-        .desc = "Run the zigc tests",
+        .desc = "Run the zig libc implementation unit tests",
         .optimize_modes = optimization_modes,
         .include_paths = &.{},
         .skip_single_threaded = true,
 @@ -560,22 +560,7 @@ pub fn build(b: *std.Build) !void {
         .skip_linux = skip_linux,
         .skip_llvm = skip_llvm,
         .skip_libc = skip_libc,
-        .max_rss = switch (b.graph.host.result.os.tag) {
+        .max_rss = 8_500_000_000,
-            .freebsd => switch (b.graph.host.result.cpu.arch) {
-                .x86_64 => 3_756_422_348,
-                else => 3_800_000_000,
-            },
-            .linux => 6_800_000_000,
-            .macos => switch (b.graph.host.result.cpu.arch) {
-                .aarch64 => 8_273_795_481,
-                else => 8_300_000_000,
-            },
-            .windows => switch (b.graph.host.result.cpu.arch) {
-                .x86_64 => 3_750_236_160,
-                else => 3_800_000_000,
-            },
-            else => 8_300_000_000,
-        },
     }));
     const unit_tests_step = b.step("test-unit", "Run the compiler source unit tests");

lib/c.zig+33-15

...	@@ -15,17 +15,32 @@ pub const panic = if (builtin.is_test)	...	@@ -15,17 +15,32 @@ pub const panic = if (builtin.is_test)
15	else	15	else
16	std.debug.no_panic;	16	std.debug.no_panic;
17		17
18	/// It is incorrect to make this conditional on `builtin.is_test`, because it is possible that	18	/// It is possible that this libc is being linked into a different test
19	/// libzigc is being linked into a different test compilation, as opposed to being tested itself.	19	/// compilation, as opposed to being tested itself. In such case,
20	pub const linkage: std.builtin.GlobalLinkage = .strong;	20	/// `builtin.link_libc` will be `true` along with `builtin.is_test`.
21		21	///
22	/// Determines the symbol's visibility to other objects.	22	/// When we don't have a complete libc, `builtin.link_libc` will be `false` and
23	/// For WebAssembly this allows the symbol to be resolved to other modules, but will not	23	/// we will be missing externally provided symbols, such as `_errno` from
24	/// export it to the host runtime.	24	/// ucrtbase.dll. In such case, we must avoid analyzing otherwise exported
25	pub const visibility: std.builtin.SymbolVisibility = .hidden;	25	/// functions because it would cause undefined symbol usage.
26		26	///
		27	/// Unfortunately such logic cannot be automatically done in this function body
		28	/// since `func` will always be analyzed by the time we get here, so `comptime`
		29	/// blocks will need to each check for `builtin.link_libc` and skip exports
		30	/// when the exported functions have libc dependencies not provided by this
		31	/// compilation unit.
27	pub inline fn symbol(comptime func: *const anyopaque, comptime name: []const u8) void {	32	pub inline fn symbol(comptime func: *const anyopaque, comptime name: []const u8) void {
28	@export(func, .{ .name = name, .linkage = linkage, .visibility = visibility });	33	@export(func, .{
		34	.name = name,
		35	// Normally, libc goes into a static archive, making all symbols
		36	// overridable. However, Zig supports including the libc functions as part
		37	// of the Zig Compilation Unit, so to support this use case we make all
		38	// symbols weak.
		39	.linkage = .weak,
		40	// For WebAssembly, hidden visibility allows the symbol to be resolved to
		41	// other modules, but will not export it to the host runtime.
		42	.visibility = .hidden,
		43	});
29	}	44	}
30		45
31	/// Given a low-level syscall return value, sets errno and returns `-1`, or on	46	/// Given a low-level syscall return value, sets errno and returns `-1`, or on
...	@@ -47,19 +62,22 @@ pub fn errno(syscall_return_value: usize) c_int {	...	@@ -47,19 +62,22 @@ pub fn errno(syscall_return_value: usize) c_int {
47	}	62	}
48		63
49	comptime {	64	comptime {
50	_ = @import("c/inttypes.zig");
51	_ = @import("c/ctype.zig");	65	_ = @import("c/ctype.zig");
52	_ = @import("c/stdlib.zig");	66	_ = @import("c/inttypes.zig");
		67	if (!builtin.target.isMinGW()) {
		68	_ = @import("c/malloc.zig");
		69	}
53	_ = @import("c/math.zig");	70	_ = @import("c/math.zig");
		71	_ = @import("c/stdlib.zig");
54	_ = @import("c/string.zig");	72	_ = @import("c/string.zig");
55	_ = @import("c/strings.zig");	73	_ = @import("c/strings.zig");
56	_ = @import("c/wchar.zig");
57		74
58	_ = @import("c/sys/mman.zig");	75	_ = @import("c/sys/capability.zig");
59	_ = @import("c/sys/file.zig");	76	_ = @import("c/sys/file.zig");
		77	_ = @import("c/sys/mman.zig");
60	_ = @import("c/sys/reboot.zig");	78	_ = @import("c/sys/reboot.zig");
61	_ = @import("c/sys/capability.zig");
62	_ = @import("c/sys/utsname.zig");	79	_ = @import("c/sys/utsname.zig");
63		80
64	_ = @import("c/unistd.zig");	81	_ = @import("c/unistd.zig");
		82	_ = @import("c/wchar.zig");
65	}	83	}

lib/c/malloc.zig created+195

...	@@ -0,0 +1,195 @@
1	//! Based on wrapping a stateless Zig Allocator implementation, appropriate for:
2	//! - ReleaseFast and ReleaseSmall optimization modes, with multi-threading
3	//! enabled.
4	//! - WebAssembly or Linux in single-threaded release modes.
5	//!
6	//! Because the libc APIs don't have client alignment and size tracking, in
7	//! order to take advantage of Zig allocator implementations, additional
8	//! metadata must be stored in the allocations.
9	//!
10	//! This implementation stores the metadata just before the pointer returned
11	//! from `malloc`, just like many libc malloc implementations do, including
12	//! musl. This has the downside of causing fragmentation for allocations with
13	//! higher alignment, however most of that memory can be recovered by
14	//! preemptively putting the gap onto the freelist.
15	const builtin = @import("builtin");
16
17	const std = @import("std");
18	const assert = std.debug.assert;
19	const Alignment = std.mem.Alignment;
20	const alignment_bytes = @max(@alignOf(std.c.max_align_t), @sizeOf(Header));
21	const alignment: Alignment = .fromByteUnits(alignment_bytes);
22
23	const symbol = @import("../c.zig").symbol;
24
25	comptime {
26	// Dependency on external errno location.
27	if (builtin.link_libc) {
28	symbol(&malloc, "malloc");
29	symbol(&aligned_alloc, "aligned_alloc");
30	symbol(&posix_memalign, "posix_memalign");
31	symbol(&calloc, "calloc");
32	symbol(&realloc, "realloc");
33	symbol(&reallocarray, "reallocarray");
34	symbol(&free, "free");
35	symbol(&malloc_usable_size, "malloc_usable_size");
36
37	symbol(&valloc, "valloc");
38	symbol(&memalign, "memalign");
39	}
40	}
41
42	const no_context: *anyopaque = undefined;
43	const no_ra: usize = undefined;
44	const vtable = switch (builtin.cpu.arch) {
45	.wasm32, .wasm64 => std.heap.WasmAllocator.vtable,
46	else => if (builtin.single_threaded) std.heap.BrkAllocator.vtable else std.heap.SmpAllocator.vtable,
47	};
48
49	/// Needed because libc memory allocators don't provide old alignment and size
50	/// which are required by Zig memory allocators.
51	const Header = packed struct(u64) {
52	alignment: Alignment,
53	/// Does not include the extra alignment bytes added.
54	size: Size,
55	padding: Padding = 0,
56
57	comptime {
58	assert(@sizeOf(Header) <= alignment_bytes);
59	}
60
61	const Size = @Int(.unsigned, @min(64 - @bitSizeOf(Alignment), @bitSizeOf(usize)));
62	const Padding = @Int(.unsigned, 64 - @bitSizeOf(Alignment) - @bitSizeOf(Size));
63
64	fn fromBase(base: []align(alignment_bytes) u8) Header {
65	return @ptrCast(base - @sizeOf(Header));
66	}
67	};
68
69	fn malloc(n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
70	const size = std.math.cast(Header.Size, n) orelse return nomem();
71	const ptr: [*]align(alignment_bytes) u8 = @alignCast(
72	vtable.alloc(no_context, n + alignment_bytes, alignment, no_ra) orelse return nomem(),
73	);
74	const base = ptr + alignment_bytes;
75	const header: *Header = .fromBase(base);
76	header.* = .{
77	.alignment = alignment,
78	.size = size,
79	};
80	return base;
81	}
82
83	fn aligned_alloc(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
84	return aligned_alloc_inner(alloc_alignment, n) orelse return nomem();
85	}
86
87	/// Avoids setting errno so it can be called by `posix_memalign`.
88	fn aligned_alloc_inner(alloc_alignment: usize, n: usize) ?[*]align(alignment_bytes) u8 {
89	const size = std.math.cast(Header.Size, n) orelse return null;
90	const max_align = alignment.max(.fromByteUnits(alloc_alignment));
91	const max_align_bytes = max_align.toByteUnits();
92	const ptr: [*]align(alignment_bytes) u8 = @alignCast(
93	vtable.alloc(no_context, n + max_align_bytes, max_align, no_ra) orelse return null,
94	);
95	const base: [*]align(alignment_bytes) u8 = @alignCast(ptr + max_align_bytes);
96	const header: *Header = .fromBase(base);
97	header.* = .{
98	.alignment = max_align,
99	.size = size,
100	};
101	return base;
102	}
103
104	fn calloc(elems: usize, len: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
105	const n = std.math.mul(usize, elems, len) catch return nomem();
106	const base = malloc(n) orelse return null;
107	@memset(base[0..n], 0);
108	return base;
109	}
110
111	fn realloc(opt_old_base: ?[]align(alignment_bytes) u8, n: usize) callconv(.c) ?[]align(alignment_bytes) u8 {
112	if (n == 0) {
113	free(opt_old_base);
114	return null;
115	}
116	const old_base = opt_old_base orelse return malloc(n);
117	const new_size = std.math.cast(Header.Size, n) orelse return nomem();
118	const old_header: *Header = .fromBase(old_base);
119	assert(old_header.padding == 0);
120	const old_size = old_header.size;
121	const old_alignment = old_header.alignment;
122	const old_alignment_bytes = old_alignment.toByteUnits();
123	const old_ptr = old_base - old_alignment_bytes;
124	const old_slice = old_ptr[0 .. old_size + old_alignment_bytes];
125	const new_base: [*]align(alignment_bytes) u8 = if (vtable.remap(
126	no_context,
127	old_slice,
128	old_alignment,
129	n + old_alignment_bytes,
130	no_ra,
131	)) \|new_ptr\| @alignCast(new_ptr + old_alignment_bytes) else b: {
132	const new_ptr: [*]align(alignment_bytes) u8 = @alignCast(
133	vtable.alloc(no_context, n + old_alignment_bytes, old_alignment, no_ra) orelse
134	return nomem(),
135	);
136	const new_base: [*]align(alignment_bytes) u8 = @alignCast(new_ptr + old_alignment_bytes);
137	const copy_len = @min(new_size, old_size);
138	@memcpy(new_base[0..copy_len], old_base[0..copy_len]);
139	vtable.free(no_context, old_slice, old_alignment, no_ra);
140	break :b new_base;
141	};
142	const new_header: *Header = .fromBase(new_base);
143	new_header.* = .{
144	.alignment = old_alignment,
145	.size = new_size,
146	};
147	return new_base;
148	}
149
150	fn reallocarray(opt_base: ?[]align(alignment_bytes) u8, elems: usize, len: usize) callconv(.c) ?[]align(alignment_bytes) u8 {
151	const n = std.math.mul(usize, elems, len) catch return nomem();
152	return realloc(opt_base, n);
153	}
154
155	fn free(opt_old_base: ?[*]align(alignment_bytes) u8) callconv(.c) void {
156	const old_base = opt_old_base orelse return;
157	const old_header: *Header = .fromBase(old_base);
158	assert(old_header.padding == 0);
159	const old_size = old_header.size;
160	const old_alignment = old_header.alignment;
161	const old_alignment_bytes = old_alignment.toByteUnits();
162	const old_ptr = old_base - old_alignment_bytes;
163	const old_slice = old_ptr[0 .. old_size + old_alignment_bytes];
164	vtable.free(no_context, old_slice, old_alignment, no_ra);
165	}
166
167	fn malloc_usable_size(opt_old_base: ?[*]align(alignment_bytes) u8) callconv(.c) usize {
168	const old_base = opt_old_base orelse return 0;
169	const old_header: *Header = .fromBase(old_base);
170	assert(old_header.padding == 0);
171	const old_size = old_header.size;
172	return old_size;
173	}
174
175	fn valloc(n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
176	return aligned_alloc(std.heap.pageSize(), n);
177	}
178
179	fn memalign(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 {
180	return aligned_alloc(alloc_alignment, n);
181	}
182
183	fn posix_memalign(result: ?[]align(alignment_bytes) u8, alloc_alignment: usize, n: usize) callconv(.c) c_int {
184	if (alloc_alignment < @sizeOf(*anyopaque)) return @intFromEnum(std.c.E.INVAL);
185	result.* = aligned_alloc_inner(alloc_alignment, n) orelse return @intFromEnum(std.c.E.NOMEM);
186	return 0;
187	}
188
189	/// Libc memory allocation functions must set errno in addition to returning
190	/// `null`.
191	fn nomem() ?[*]align(alignment_bytes) u8 {
192	@branchHint(.cold);
193	std.c._errno().* = @intFromEnum(std.c.E.NOMEM);
194	return null;
195	}

lib/libc/musl/src/aio/aio.c-5

 @@ -11,11 +11,6 @@
 #include "pthread_impl.h"
 #include "aio_impl.h"
-#define malloc __libc_malloc
-#define calloc __libc_calloc
-#define realloc __libc_realloc
-#define free __libc_free
 /* The following is a threads-based implementation of AIO with minimal
  * dependence on implementation details. Most synchronization is
  * performed with pthread primitives, but atomics and futex operations

lib/libc/musl/src/exit/atexit.c-5

 @@ -4,11 +4,6 @@
 #include "lock.h"
 #include "fork_impl.h"
-#define malloc __libc_malloc
-#define calloc __libc_calloc
-#define realloc undef
-#define free undef
 /* Ensure that at least 32 atexit handlers can be registered without malloc */
 #define COUNT 32

lib/libc/musl/src/include/stdlib.h-6

...	@@ -10,10 +10,4 @@ hidden int __ptsname_r(int, char *, size_t);	...	@@ -10,10 +10,4 @@ hidden int __ptsname_r(int, char *, size_t);
10	hidden char __randname(char );	10	hidden char __randname(char );
11	hidden void __qsort_r (void , size_t, size_t, int ()(const void , const void , void ), void );	11	hidden void __qsort_r (void , size_t, size_t, int ()(const void , const void , void ), void );
12		12
13	hidden void *__libc_malloc(size_t);
14	hidden void *__libc_malloc_impl(size_t);
15	hidden void *__libc_calloc(size_t, size_t);
16	hidden void __libc_realloc(void , size_t);
17	hidden void __libc_free(void *);
18
19	#endif	13	#endif

lib/libc/musl/src/ldso/dlerror.c-5

 @@ -5,11 +5,6 @@
 #include "dynlink.h"
 #include "atomic.h"
-#define malloc __libc_malloc
-#define calloc __libc_calloc
-#define realloc __libc_realloc
-#define free __libc_free
 char *dlerror()
 {
 	pthread_t self = __pthread_self();

lib/libc/musl/src/locale/dcngettext.c-5

 @@ -12,11 +12,6 @@
 #include "lock.h"
 #include "fork_impl.h"
-#define malloc __libc_malloc
-#define calloc __libc_calloc
-#define realloc undef
-#define free undef
 struct binding {
 	struct binding *next;
 	int dirlen;

lib/libc/musl/src/locale/duplocale.c-5

 @@ -3,11 +3,6 @@
 #include "locale_impl.h"
 #include "libc.h"
-#define malloc __libc_malloc
-#define calloc undef
-#define realloc undef
-#define free undef
 locale_t __duplocale(locale_t old)
 {
 	locale_t new = malloc(sizeof *new);

lib/libc/musl/src/locale/freelocale.c-5

 @@ -1,11 +1,6 @@
 #include <stdlib.h>
 #include "locale_impl.h"
-#define malloc undef
-#define calloc undef
-#define realloc undef
-#define free __libc_free
 void freelocale(locale_t l)
 {
 	if (__loc_is_allocated(l)) free(l);

lib/libc/musl/src/locale/locale_map.c-5

 @@ -7,11 +7,6 @@
 #include "lock.h"
 #include "fork_impl.h"
-#define malloc __libc_malloc
-#define calloc undef
-#define realloc undef
-#define free undef
 const char *__lctrans_impl(const char *msg, const struct __locale_map *lm)
 {
 	const char *trans = 0;

lib/libc/musl/src/locale/newlocale.c-5

 @@ -4,11 +4,6 @@
 #include "locale_impl.h"
 #include "lock.h"
-#define malloc __libc_malloc
-#define calloc undef
-#define realloc undef
-#define free undef
 static int default_locale_init_done;
 static struct __locale_struct default_locale, default_ctype_locale;

lib/libc/musl/src/malloc/calloc.c deleted-45

...	@@ -1,45 +0,0 @@
1	#include <stdlib.h>
2	#include <stdint.h>
3	#include <string.h>
4	#include <errno.h>
5	#include "dynlink.h"
6
7	static size_t mal0_clear(char *p, size_t n)
8	{
9	const size_t pagesz = 4096; /* arbitrary */
10	if (n < pagesz) return n;
11	#ifdef __GNUC__
12	typedef uint64_t __attribute__((__may_alias__)) T;
13	#else
14	typedef unsigned char T;
15	#endif
16	char *pp = p + n;
17	size_t i = (uintptr_t)pp & (pagesz - 1);
18	for (;;) {
19	pp = memset(pp - i, 0, i);
20	if (pp - p < pagesz) return pp - p;
21	for (i = pagesz; i; i -= 2sizeof(T), pp -= 2sizeof(T))
22	if (((T )pp)[-1] \| ((T )pp)[-2])
23	break;
24	}
25	}
26
27	static int allzerop(void *p)
28	{
29	return 0;
30	}
31	weak_alias(allzerop, __malloc_allzerop);
32
33	void *calloc(size_t m, size_t n)
34	{
35	if (n && m > (size_t)-1/n) {
36	errno = ENOMEM;
37	return 0;
38	}
39	n *= m;
40	void *p = malloc(n);
41	if (!p \|\| (!__malloc_replaced && __malloc_allzerop(p)))
42	return p;
43	n = mal0_clear(p, n);
44	return memset(p, 0, n);
45	}

lib/libc/musl/src/malloc/free.c deleted-6

...	@@ -1,6 +0,0 @@
1	#include <stdlib.h>
2
3	void free(void *p)
4	{
5	__libc_free(p);
6	}

lib/libc/musl/src/malloc/libc_calloc.c deleted-4

...	@@ -1,4 +0,0 @@
1	#define calloc __libc_calloc
2	#define malloc __libc_malloc
3
4	#include "calloc.c"

lib/libc/musl/src/malloc/lite_malloc.c deleted-118

...	@@ -1,118 +0,0 @@
1	#include <stdlib.h>
2	#include <stdint.h>
3	#include <limits.h>
4	#include <errno.h>
5	#include <sys/mman.h>
6	#include "libc.h"
7	#include "lock.h"
8	#include "syscall.h"
9	#include "fork_impl.h"
10
11	#define ALIGN 16
12
13	/* This function returns true if the interval [old,new]
14	* intersects the 'len'-sized interval below &libc.auxv
15	* (interpreted as the main-thread stack) or below &b
16	* (the current stack). It is used to defend against
17	* buggy brk implementations that can cross the stack. */
18
19	static int traverses_stack_p(uintptr_t old, uintptr_t new)
20	{
21	const uintptr_t len = 8<<20;
22	uintptr_t a, b;
23
24	b = (uintptr_t)libc.auxv;
25	a = b > len ? b-len : 0;
26	if (new>a && old<b) return 1;
27
28	b = (uintptr_t)&b;
29	a = b > len ? b-len : 0;
30	if (new>a && old<b) return 1;
31
32	return 0;
33	}
34
35	static volatile int lock[1];
36	volatile int *const __bump_lockptr = lock;
37
38	static void *__simple_malloc(size_t n)
39	{
40	static uintptr_t brk, cur, end;
41	static unsigned mmap_step;
42	size_t align=1;
43	void *p;
44
45	if (n > SIZE_MAX/2) {
46	errno = ENOMEM;
47	return 0;
48	}
49
50	if (!n) n++;
51	while (align<n && align<ALIGN)
52	align += align;
53
54	LOCK(lock);
55
56	cur += -cur & align-1;
57
58	if (n > end-cur) {
59	size_t req = n - (end-cur) + PAGE_SIZE-1 & -PAGE_SIZE;
60
61	if (!cur) {
62	brk = __syscall(SYS_brk, 0);
63	brk += -brk & PAGE_SIZE-1;
64	cur = end = brk;
65	}
66
67	if (brk == end && req < SIZE_MAX-brk
68	&& !traverses_stack_p(brk, brk+req)
69	&& __syscall(SYS_brk, brk+req)==brk+req) {
70	brk = end += req;
71	} else {
72	int new_area = 0;
73	req = n + PAGE_SIZE-1 & -PAGE_SIZE;
74	/* Only make a new area rather than individual mmap
75	* if wasted space would be over 1/8 of the map. */
76	if (req-n > req/8) {
77	/* Geometric area size growth up to 64 pages,
78	* bounding waste by 1/8 of the area. */
79	size_t min = PAGE_SIZE<<(mmap_step/2);
80	if (min-n > end-cur) {
81	if (req < min) {
82	req = min;
83	if (mmap_step < 12)
84	mmap_step++;
85	}
86	new_area = 1;
87	}
88	}
89	void *mem = __mmap(0, req, PROT_READ\|PROT_WRITE,
90	MAP_PRIVATE\|MAP_ANONYMOUS, -1, 0);
91	if (mem == MAP_FAILED \|\| !new_area) {
92	UNLOCK(lock);
93	return mem==MAP_FAILED ? 0 : mem;
94	}
95	cur = (uintptr_t)mem;
96	end = cur + req;
97	}
98	}
99
100	p = (void *)cur;
101	cur += n;
102	UNLOCK(lock);
103	return p;
104	}
105
106	weak_alias(__simple_malloc, __libc_malloc_impl);
107
108	void *__libc_malloc(size_t n)
109	{
110	return __libc_malloc_impl(n);
111	}
112
113	static void *default_malloc(size_t n)
114	{
115	return __libc_malloc_impl(n);
116	}
117
118	weak_alias(default_malloc, malloc);

lib/libc/musl/src/malloc/mallocng/aligned_alloc.c deleted-60

...	@@ -1,60 +0,0 @@
1	#include <stdlib.h>
2	#include <errno.h>
3	#include "meta.h"
4
5	void *aligned_alloc(size_t align, size_t len)
6	{
7	if ((align & -align) != align) {
8	errno = EINVAL;
9	return 0;
10	}
11
12	if (len > SIZE_MAX - align \|\| align >= (1ULL<<31)*UNIT) {
13	errno = ENOMEM;
14	return 0;
15	}
16
17	if (DISABLE_ALIGNED_ALLOC) {
18	errno = ENOMEM;
19	return 0;
20	}
21
22	if (align <= UNIT) align = UNIT;
23
24	unsigned char *p = malloc(len + align - UNIT);
25	if (!p)
26	return 0;
27
28	struct meta *g = get_meta(p);
29	int idx = get_slot_index(p);
30	size_t stride = get_stride(g);
31	unsigned char start = g->mem->storage + strideidx;
32	unsigned char end = g->mem->storage + stride(idx+1) - IB;
33	size_t adj = -(uintptr_t)p & (align-1);
34
35	if (!adj) {
36	set_size(p, end, len);
37	return p;
38	}
39	p += adj;
40	uint32_t offset = (size_t)(p-g->mem->storage)/UNIT;
41	if (offset <= 0xffff) {
42	(uint16_t )(p-2) = offset;
43	p[-4] = 0;
44	} else {
45	// use a 32-bit offset if 16-bit doesn't fit. for this,
46	// 16-bit field must be zero, [-4] byte nonzero.
47	(uint16_t )(p-2) = 0;
48	(uint32_t )(p-8) = offset;
49	p[-4] = 1;
50	}
51	p[-3] = idx;
52	set_size(p, end, len);
53	// store offset to aligned enframing. this facilitates cycling
54	// offset and also iteration of heap for debugging/measurement.
55	// for extreme overalignment it won't fit but these are classless
56	// allocations anyway.
57	(uint16_t )(start - 2) = (size_t)(p-start)/UNIT;
58	start[-3] = 7<<5;
59	return p;
60	}

lib/libc/musl/src/malloc/mallocng/donate.c deleted-39

...	@@ -1,39 +0,0 @@
1	#include <stdlib.h>
2	#include <stdint.h>
3	#include <limits.h>
4	#include <string.h>
5	#include <sys/mman.h>
6	#include <errno.h>
7
8	#include "meta.h"
9
10	static void donate(unsigned char *base, size_t len)
11	{
12	uintptr_t a = (uintptr_t)base;
13	uintptr_t b = a + len;
14	a += -a & (UNIT-1);
15	b -= b & (UNIT-1);
16	memset(base, 0, len);
17	for (int sc=47; sc>0 && b>a; sc-=4) {
18	if (b-a < (size_classes[sc]+1)*UNIT) continue;
19	struct meta *m = alloc_meta();
20	m->avail_mask = 0;
21	m->freed_mask = 1;
22	m->mem = (void *)a;
23	m->mem->meta = m;
24	m->last_idx = 0;
25	m->freeable = 0;
26	m->sizeclass = sc;
27	m->maplen = 0;
28	((unsigned char )m->mem+UNIT-4) = 0;
29	((unsigned char )m->mem+UNIT-3) = 255;
30	m->mem->storage[size_classes[sc]*UNIT-4] = 0;
31	queue(&ctx.active[sc], m);
32	a += (size_classes[sc]+1)*UNIT;
33	}
34	}
35
36	void __malloc_donate(char start, char end)
37	{
38	donate((void *)start, end-start);
39	}

lib/libc/musl/src/malloc/mallocng/free.c deleted-151

...	@@ -1,151 +0,0 @@
1	#define _BSD_SOURCE
2	#include <stdlib.h>
3	#include <sys/mman.h>
4
5	#include "meta.h"
6
7	struct mapinfo {
8	void *base;
9	size_t len;
10	};
11
12	static struct mapinfo nontrivial_free(struct meta *, int);
13
14	static struct mapinfo free_group(struct meta *g)
15	{
16	struct mapinfo mi = { 0 };
17	int sc = g->sizeclass;
18	if (sc < 48) {
19	ctx.usage_by_class[sc] -= g->last_idx+1;
20	}
21	if (g->maplen) {
22	step_seq();
23	record_seq(sc);
24	mi.base = g->mem;
25	mi.len = g->maplen*4096UL;
26	} else {
27	void *p = g->mem;
28	struct meta *m = get_meta(p);
29	int idx = get_slot_index(p);
30	g->mem->meta = 0;
31	// not checking size/reserved here; it's intentionally invalid
32	mi = nontrivial_free(m, idx);
33	}
34	free_meta(g);
35	return mi;
36	}
37
38	static int okay_to_free(struct meta *g)
39	{
40	int sc = g->sizeclass;
41
42	if (!g->freeable) return 0;
43
44	// always free individual mmaps not suitable for reuse
45	if (sc >= 48 \|\| get_stride(g) < UNIT*size_classes[sc])
46	return 1;
47
48	// always free groups allocated inside another group's slot
49	// since recreating them should not be expensive and they
50	// might be blocking freeing of a much larger group.
51	if (!g->maplen) return 1;
52
53	// if there is another non-full group, free this one to
54	// consolidate future allocations, reduce fragmentation.
55	if (g->next != g) return 1;
56
57	// free any group in a size class that's not bouncing
58	if (!is_bouncing(sc)) return 1;
59
60	size_t cnt = g->last_idx+1;
61	size_t usage = ctx.usage_by_class[sc];
62
63	// if usage is high enough that a larger count should be
64	// used, free the low-count group so a new one will be made.
65	if (9*cnt <= usage && cnt < 20)
66	return 1;
67
68	// otherwise, keep the last group in a bouncing class.
69	return 0;
70	}
71
72	static struct mapinfo nontrivial_free(struct meta *g, int i)
73	{
74	uint32_t self = 1u<<i;
75	int sc = g->sizeclass;
76	uint32_t mask = g->freed_mask \| g->avail_mask;
77
78	if (mask+self == (2u<<g->last_idx)-1 && okay_to_free(g)) {
79	// any multi-slot group is necessarily on an active list
80	// here, but single-slot groups might or might not be.
81	if (g->next) {
82	assert(sc < 48);
83	int activate_new = (ctx.active[sc]==g);
84	dequeue(&ctx.active[sc], g);
85	if (activate_new && ctx.active[sc])
86	activate_group(ctx.active[sc]);
87	}
88	return free_group(g);
89	} else if (!mask) {
90	assert(sc < 48);
91	// might still be active if there were no allocations
92	// after last available slot was taken.
93	if (ctx.active[sc] != g) {
94	queue(&ctx.active[sc], g);
95	}
96	}
97	a_or(&g->freed_mask, self);
98	return (struct mapinfo){ 0 };
99	}
100
101	void free(void *p)
102	{
103	if (!p) return;
104
105	struct meta *g = get_meta(p);
106	int idx = get_slot_index(p);
107	size_t stride = get_stride(g);
108	unsigned char start = g->mem->storage + strideidx;
109	unsigned char *end = start + stride - IB;
110	get_nominal_size(p, end);
111	uint32_t self = 1u<<idx, all = (2u<<g->last_idx)-1;
112	((unsigned char *)p)[-3] = 255;
113	// invalidate offset to group header, and cycle offset of
114	// used region within slot if current offset is zero.
115	(uint16_t )((char *)p-2) = 0;
116
117	// release any whole pages contained in the slot to be freed
118	// unless it's a single-slot group that will be unmapped.
119	if (((uintptr_t)(start-1) ^ (uintptr_t)end) >= 2*PGSZ && g->last_idx) {
120	unsigned char *base = start + (-(uintptr_t)start & (PGSZ-1));
121	size_t len = (end-base) & -PGSZ;
122	if (len && USE_MADV_FREE) {
123	int e = errno;
124	madvise(base, len, MADV_FREE);
125	errno = e;
126	}
127	}
128
129	// atomic free without locking if this is neither first or last slot
130	for (;;) {
131	uint32_t freed = g->freed_mask;
132	uint32_t avail = g->avail_mask;
133	uint32_t mask = freed \| avail;
134	assert(!(mask&self));
135	if (!freed \|\| mask+self==all) break;
136	if (!MT)
137	g->freed_mask = freed+self;
138	else if (a_cas(&g->freed_mask, freed, freed+self)!=freed)
139	continue;
140	return;
141	}
142
143	wrlock();
144	struct mapinfo mi = nontrivial_free(g, idx);
145	unlock();
146	if (mi.len) {
147	int e = errno;
148	munmap(mi.base, mi.len);
149	errno = e;
150	}
151	}

lib/libc/musl/src/malloc/mallocng/glue.h deleted-95

...	@@ -1,95 +0,0 @@
1	#ifndef MALLOC_GLUE_H
2	#define MALLOC_GLUE_H
3
4	#include <stdint.h>
5	#include <sys/mman.h>
6	#include <pthread.h>
7	#include <unistd.h>
8	#include <elf.h>
9	#include <string.h>
10	#include "atomic.h"
11	#include "syscall.h"
12	#include "libc.h"
13	#include "lock.h"
14	#include "dynlink.h"
15
16	// use macros to appropriately namespace these.
17	#define size_classes __malloc_size_classes
18	#define ctx __malloc_context
19	#define alloc_meta __malloc_alloc_meta
20	#define is_allzero __malloc_allzerop
21	#define dump_heap __dump_heap
22
23	#define malloc __libc_malloc_impl
24	#define realloc __libc_realloc
25	#define free __libc_free
26
27	#define USE_MADV_FREE 0
28
29	#if USE_REAL_ASSERT
30	#include <assert.h>
31	#else
32	#undef assert
33	#define assert(x) do { if (!(x)) a_crash(); } while(0)
34	#endif
35
36	#define brk(p) ((uintptr_t)__syscall(SYS_brk, p))
37
38	#define mmap __mmap
39	#define madvise __madvise
40	#define mremap __mremap
41
42	#define DISABLE_ALIGNED_ALLOC (__malloc_replaced && !__aligned_alloc_replaced)
43
44	static inline uint64_t get_random_secret()
45	{
46	uint64_t secret = (uintptr_t)&secret * 1103515245;
47	for (size_t i=0; libc.auxv[i]; i+=2)
48	if (libc.auxv[i]==AT_RANDOM)
49	memcpy(&secret, (char *)libc.auxv[i+1]+8, sizeof secret);
50	return secret;
51	}
52
53	#ifndef PAGESIZE
54	#define PAGESIZE PAGE_SIZE
55	#endif
56
57	#define MT (libc.need_locks)
58
59	#define RDLOCK_IS_EXCLUSIVE 1
60
61	__attribute__((__visibility__("hidden")))
62	extern int __malloc_lock[1];
63
64	#define LOCK_OBJ_DEF \
65	int __malloc_lock[1]; \
66	void __malloc_atfork(int who) { malloc_atfork(who); }
67
68	static inline void rdlock()
69	{
70	if (MT) LOCK(__malloc_lock);
71	}
72	static inline void wrlock()
73	{
74	if (MT) LOCK(__malloc_lock);
75	}
76	static inline void unlock()
77	{
78	UNLOCK(__malloc_lock);
79	}
80	static inline void upgradelock()
81	{
82	}
83	static inline void resetlock()
84	{
85	__malloc_lock[0] = 0;
86	}
87
88	static inline void malloc_atfork(int who)
89	{
90	if (who<0) rdlock();
91	else if (who>0) resetlock();
92	else unlock();
93	}
94
95	#endif

lib/libc/musl/src/malloc/mallocng/malloc.c deleted-387

...	@@ -1,387 +0,0 @@
1	#include <stdlib.h>
2	#include <stdint.h>
3	#include <limits.h>
4	#include <string.h>
5	#include <sys/mman.h>
6	#include <errno.h>
7
8	#include "meta.h"
9
10	LOCK_OBJ_DEF;
11
12	const uint16_t size_classes[] = {
13	1, 2, 3, 4, 5, 6, 7, 8,
14	9, 10, 12, 15,
15	18, 20, 25, 31,
16	36, 42, 50, 63,
17	72, 84, 102, 127,
18	146, 170, 204, 255,
19	292, 340, 409, 511,
20	584, 682, 818, 1023,
21	1169, 1364, 1637, 2047,
22	2340, 2730, 3276, 4095,
23	4680, 5460, 6552, 8191,
24	};
25
26	static const uint8_t small_cnt_tab[][3] = {
27	{ 30, 30, 30 },
28	{ 31, 15, 15 },
29	{ 20, 10, 10 },
30	{ 31, 15, 7 },
31	{ 25, 12, 6 },
32	{ 21, 10, 5 },
33	{ 18, 8, 4 },
34	{ 31, 15, 7 },
35	{ 28, 14, 6 },
36	};
37
38	static const uint8_t med_cnt_tab[4] = { 28, 24, 20, 32 };
39
40	struct malloc_context ctx = { 0 };
41
42	struct meta *alloc_meta(void)
43	{
44	struct meta *m;
45	unsigned char *p;
46	if (!ctx.init_done) {
47	#ifndef PAGESIZE
48	ctx.pagesize = get_page_size();
49	#endif
50	ctx.secret = get_random_secret();
51	ctx.init_done = 1;
52	}
53	size_t pagesize = PGSZ;
54	if (pagesize < 4096) pagesize = 4096;
55	if ((m = dequeue_head(&ctx.free_meta_head))) return m;
56	if (!ctx.avail_meta_count) {
57	int need_unprotect = 1;
58	if (!ctx.avail_meta_area_count && ctx.brk!=-1) {
59	uintptr_t new = ctx.brk + pagesize;
60	int need_guard = 0;
61	if (!ctx.brk) {
62	need_guard = 1;
63	ctx.brk = brk(0);
64	// some ancient kernels returned _ebss
65	// instead of next page as initial brk.
66	ctx.brk += -ctx.brk & (pagesize-1);
67	new = ctx.brk + 2*pagesize;
68	}
69	if (brk(new) != new) {
70	ctx.brk = -1;
71	} else {
72	if (need_guard) mmap((void *)ctx.brk, pagesize,
73	PROT_NONE, MAP_ANON\|MAP_PRIVATE\|MAP_FIXED, -1, 0);
74	ctx.brk = new;
75	ctx.avail_meta_areas = (void *)(new - pagesize);
76	ctx.avail_meta_area_count = pagesize>>12;
77	need_unprotect = 0;
78	}
79	}
80	if (!ctx.avail_meta_area_count) {
81	size_t n = 2UL << ctx.meta_alloc_shift;
82	p = mmap(0, n*pagesize, PROT_NONE,
83	MAP_PRIVATE\|MAP_ANON, -1, 0);
84	if (p==MAP_FAILED) return 0;
85	ctx.avail_meta_areas = p + pagesize;
86	ctx.avail_meta_area_count = (n-1)*(pagesize>>12);
87	ctx.meta_alloc_shift++;
88	}
89	p = ctx.avail_meta_areas;
90	if ((uintptr_t)p & (pagesize-1)) need_unprotect = 0;
91	if (need_unprotect)
92	if (mprotect(p, pagesize, PROT_READ\|PROT_WRITE)
93	&& errno != ENOSYS)
94	return 0;
95	ctx.avail_meta_area_count--;
96	ctx.avail_meta_areas = p + 4096;
97	if (ctx.meta_area_tail) {
98	ctx.meta_area_tail->next = (void *)p;
99	} else {
100	ctx.meta_area_head = (void *)p;
101	}
102	ctx.meta_area_tail = (void *)p;
103	ctx.meta_area_tail->check = ctx.secret;
104	ctx.avail_meta_count = ctx.meta_area_tail->nslots
105	= (4096-sizeof(struct meta_area))/sizeof *m;
106	ctx.avail_meta = ctx.meta_area_tail->slots;
107	}
108	ctx.avail_meta_count--;
109	m = ctx.avail_meta++;
110	m->prev = m->next = 0;
111	return m;
112	}
113
114	static uint32_t try_avail(struct meta **pm)
115	{
116	struct meta m = pm;
117	uint32_t first;
118	if (!m) return 0;
119	uint32_t mask = m->avail_mask;
120	if (!mask) {
121	if (!m) return 0;
122	if (!m->freed_mask) {
123	dequeue(pm, m);
124	m = *pm;
125	if (!m) return 0;
126	} else {
127	m = m->next;
128	*pm = m;
129	}
130
131	mask = m->freed_mask;
132
133	// skip fully-free group unless it's the only one
134	// or it's a permanently non-freeable group
135	if (mask == (2u<<m->last_idx)-1 && m->freeable) {
136	m = m->next;
137	*pm = m;
138	mask = m->freed_mask;
139	}
140
141	// activate more slots in a not-fully-active group
142	// if needed, but only as a last resort. prefer using
143	// any other group with free slots. this avoids
144	// touching & dirtying as-yet-unused pages.
145	if (!(mask & ((2u<<m->mem->active_idx)-1))) {
146	if (m->next != m) {
147	m = m->next;
148	*pm = m;
149	} else {
150	int cnt = m->mem->active_idx + 2;
151	int size = size_classes[m->sizeclass]*UNIT;
152	int span = UNIT + size*cnt;
153	// activate up to next 4k boundary
154	while ((span^(span+size-1)) < 4096) {
155	cnt++;
156	span += size;
157	}
158	if (cnt > m->last_idx+1)
159	cnt = m->last_idx+1;
160	m->mem->active_idx = cnt-1;
161	}
162	}
163	mask = activate_group(m);
164	assert(mask);
165	decay_bounces(m->sizeclass);
166	}
167	first = mask&-mask;
168	m->avail_mask = mask-first;
169	return first;
170	}
171
172	static int alloc_slot(int, size_t);
173
174	static struct meta *alloc_group(int sc, size_t req)
175	{
176	size_t size = UNIT*size_classes[sc];
177	int i = 0, cnt;
178	unsigned char *p;
179	struct meta *m = alloc_meta();
180	if (!m) return 0;
181	size_t usage = ctx.usage_by_class[sc];
182	size_t pagesize = PGSZ;
183	int active_idx;
184	if (sc < 9) {
185	while (i<2 && 4*small_cnt_tab[sc][i] > usage)
186	i++;
187	cnt = small_cnt_tab[sc][i];
188	} else {
189	// lookup max number of slots fitting in power-of-two size
190	// from a table, along with number of factors of two we
191	// can divide out without a remainder or reaching 1.
192	cnt = med_cnt_tab[sc&3];
193
194	// reduce cnt to avoid excessive eagar allocation.
195	while (!(cnt&1) && 4*cnt > usage)
196	cnt >>= 1;
197
198	// data structures don't support groups whose slot offsets
199	// in units don't fit in 16 bits.
200	while (sizecnt >= 65536UNIT)
201	cnt >>= 1;
202	}
203
204	// If we selected a count of 1 above but it's not sufficient to use
205	// mmap, increase to 2. Then it might be; if not it will nest.
206	if (cnt==1 && size*cnt+UNIT <= pagesize/2) cnt = 2;
207
208	// All choices of size*cnt are "just below" a power of two, so anything
209	// larger than half the page size should be allocated as whole pages.
210	if (size*cnt+UNIT > pagesize/2) {
211	// check/update bounce counter to start/increase retention
212	// of freed maps, and inhibit use of low-count, odd-size
213	// small mappings and single-slot groups if activated.
214	int nosmall = is_bouncing(sc);
215	account_bounce(sc);
216	step_seq();
217
218	// since the following count reduction opportunities have
219	// an absolute memory usage cost, don't overdo them. count
220	// coarse usage as part of usage.
221	if (!(sc&1) && sc<32) usage += ctx.usage_by_class[sc+1];
222
223	// try to drop to a lower count if the one found above
224	// increases usage by more than 25%. these reduced counts
225	// roughly fill an integral number of pages, just not a
226	// power of two, limiting amount of unusable space.
227	if (4*cnt > usage && !nosmall) {
228	if (0);
229	else if ((sc&3)==1 && sizecnt>8pagesize) cnt = 2;
230	else if ((sc&3)==2 && sizecnt>4pagesize) cnt = 3;
231	else if ((sc&3)==0 && sizecnt>8pagesize) cnt = 3;
232	else if ((sc&3)==0 && sizecnt>2pagesize) cnt = 5;
233	}
234	size_t needed = size*cnt + UNIT;
235	needed += -needed & (pagesize-1);
236
237	// produce an individually-mmapped allocation if usage is low,
238	// bounce counter hasn't triggered, and either it saves memory
239	// or it avoids eagar slot allocation without wasting too much.
240	if (!nosmall && cnt<=7) {
241	req += IB + UNIT;
242	req += -req & (pagesize-1);
243	if (req<size+UNIT \|\| (req>=4pagesize && 2cnt>usage)) {
244	cnt = 1;
245	needed = req;
246	}
247	}
248
249	p = mmap(0, needed, PROT_READ\|PROT_WRITE, MAP_PRIVATE\|MAP_ANON, -1, 0);
250	if (p==MAP_FAILED) {
251	free_meta(m);
252	return 0;
253	}
254	m->maplen = needed>>12;
255	ctx.mmap_counter++;
256	active_idx = (4096-UNIT)/size-1;
257	if (active_idx > cnt-1) active_idx = cnt-1;
258	if (active_idx < 0) active_idx = 0;
259	} else {
260	int j = size_to_class(UNIT+cnt*size-IB);
261	int idx = alloc_slot(j, UNIT+cnt*size-IB);
262	if (idx < 0) {
263	free_meta(m);
264	return 0;
265	}
266	struct meta *g = ctx.active[j];
267	p = enframe(g, idx, UNIT*size_classes[j]-IB, ctx.mmap_counter);
268	m->maplen = 0;
269	p[-3] = (p[-3]&31) \| (6<<5);
270	for (int i=0; i<=cnt; i++)
271	p[UNIT+i*size-4] = 0;
272	active_idx = cnt-1;
273	}
274	ctx.usage_by_class[sc] += cnt;
275	m->avail_mask = (2u<<active_idx)-1;
276	m->freed_mask = (2u<<(cnt-1))-1 - m->avail_mask;
277	m->mem = (void *)p;
278	m->mem->meta = m;
279	m->mem->active_idx = active_idx;
280	m->last_idx = cnt-1;
281	m->freeable = 1;
282	m->sizeclass = sc;
283	return m;
284	}
285
286	static int alloc_slot(int sc, size_t req)
287	{
288	uint32_t first = try_avail(&ctx.active[sc]);
289	if (first) return a_ctz_32(first);
290
291	struct meta *g = alloc_group(sc, req);
292	if (!g) return -1;
293
294	g->avail_mask--;
295	queue(&ctx.active[sc], g);
296	return 0;
297	}
298
299	void *malloc(size_t n)
300	{
301	if (size_overflows(n)) return 0;
302	struct meta *g;
303	uint32_t mask, first;
304	int sc;
305	int idx;
306	int ctr;
307
308	if (n >= MMAP_THRESHOLD) {
309	size_t needed = n + IB + UNIT;
310	void *p = mmap(0, needed, PROT_READ\|PROT_WRITE,
311	MAP_PRIVATE\|MAP_ANON, -1, 0);
312	if (p==MAP_FAILED) return 0;
313	wrlock();
314	step_seq();
315	g = alloc_meta();
316	if (!g) {
317	unlock();
318	munmap(p, needed);
319	return 0;
320	}
321	g->mem = p;
322	g->mem->meta = g;
323	g->last_idx = 0;
324	g->freeable = 1;
325	g->sizeclass = 63;
326	g->maplen = (needed+4095)/4096;
327	g->avail_mask = g->freed_mask = 0;
328	// use a global counter to cycle offset in
329	// individually-mmapped allocations.
330	ctx.mmap_counter++;
331	idx = 0;
332	goto success;
333	}
334
335	sc = size_to_class(n);
336
337	rdlock();
338	g = ctx.active[sc];
339
340	// use coarse size classes initially when there are not yet
341	// any groups of desired size. this allows counts of 2 or 3
342	// to be allocated at first rather than having to start with
343	// 7 or 5, the min counts for even size classes.
344	if (!g && sc>=4 && sc<32 && sc!=6 && !(sc&1) && !ctx.usage_by_class[sc]) {
345	size_t usage = ctx.usage_by_class[sc\|1];
346	// if a new group may be allocated, count it toward
347	// usage in deciding if we can use coarse class.
348	if (!ctx.active[sc\|1] \|\| (!ctx.active[sc\|1]->avail_mask
349	&& !ctx.active[sc\|1]->freed_mask))
350	usage += 3;
351	if (usage <= 12)
352	sc \|= 1;
353	g = ctx.active[sc];
354	}
355
356	for (;;) {
357	mask = g ? g->avail_mask : 0;
358	first = mask&-mask;
359	if (!first) break;
360	if (RDLOCK_IS_EXCLUSIVE \|\| !MT)
361	g->avail_mask = mask-first;
362	else if (a_cas(&g->avail_mask, mask, mask-first)!=mask)
363	continue;
364	idx = a_ctz_32(first);
365	goto success;
366	}
367	upgradelock();
368
369	idx = alloc_slot(sc, n);
370	if (idx < 0) {
371	unlock();
372	return 0;
373	}
374	g = ctx.active[sc];
375
376	success:
377	ctr = ctx.mmap_counter;
378	unlock();
379	return enframe(g, idx, n, ctr);
380	}
381
382	int is_allzero(void *p)
383	{
384	struct meta *g = get_meta(p);
385	return g->sizeclass >= 48 \|\|
386	get_stride(g) < UNIT*size_classes[g->sizeclass];
387	}

lib/libc/musl/src/malloc/mallocng/malloc_usable_size.c deleted-13

...	@@ -1,13 +0,0 @@
1	#include <stdlib.h>
2	#include "meta.h"
3
4	size_t malloc_usable_size(void *p)
5	{
6	if (!p) return 0;
7	struct meta *g = get_meta(p);
8	int idx = get_slot_index(p);
9	size_t stride = get_stride(g);
10	unsigned char start = g->mem->storage + strideidx;
11	unsigned char *end = start + stride - IB;
12	return get_nominal_size(p, end);
13	}

lib/libc/musl/src/malloc/mallocng/meta.h deleted-288

...	@@ -1,288 +0,0 @@
1	#ifndef MALLOC_META_H
2	#define MALLOC_META_H
3
4	#include <stdint.h>
5	#include <errno.h>
6	#include <limits.h>
7	#include "glue.h"
8
9	__attribute__((__visibility__("hidden")))
10	extern const uint16_t size_classes[];
11
12	#define MMAP_THRESHOLD 131052
13
14	#define UNIT 16
15	#define IB 4
16
17	struct group {
18	struct meta *meta;
19	unsigned char active_idx:5;
20	char pad[UNIT - sizeof(struct meta *) - 1];
21	unsigned char storage[];
22	};
23
24	struct meta {
25	struct meta prev, next;
26	struct group *mem;
27	volatile int avail_mask, freed_mask;
28	uintptr_t last_idx:5;
29	uintptr_t freeable:1;
30	uintptr_t sizeclass:6;
31	uintptr_t maplen:8*sizeof(uintptr_t)-12;
32	};
33
34	struct meta_area {
35	uint64_t check;
36	struct meta_area *next;
37	int nslots;
38	struct meta slots[];
39	};
40
41	struct malloc_context {
42	uint64_t secret;
43	#ifndef PAGESIZE
44	size_t pagesize;
45	#endif
46	int init_done;
47	unsigned mmap_counter;
48	struct meta *free_meta_head;
49	struct meta *avail_meta;
50	size_t avail_meta_count, avail_meta_area_count, meta_alloc_shift;
51	struct meta_area meta_area_head, meta_area_tail;
52	unsigned char *avail_meta_areas;
53	struct meta *active[48];
54	size_t usage_by_class[48];
55	uint8_t unmap_seq[32], bounces[32];
56	uint8_t seq;
57	uintptr_t brk;
58	};
59
60	__attribute__((__visibility__("hidden")))
61	extern struct malloc_context ctx;
62
63	#ifdef PAGESIZE
64	#define PGSZ PAGESIZE
65	#else
66	#define PGSZ ctx.pagesize
67	#endif
68
69	__attribute__((__visibility__("hidden")))
70	struct meta *alloc_meta(void);
71
72	__attribute__((__visibility__("hidden")))
73	int is_allzero(void *);
74
75	static inline void queue(struct meta *phead, struct meta m)
76	{
77	assert(!m->next);
78	assert(!m->prev);
79	if (*phead) {
80	struct meta head = phead;
81	m->next = head;
82	m->prev = head->prev;
83	m->next->prev = m->prev->next = m;
84	} else {
85	m->prev = m->next = m;
86	*phead = m;
87	}
88	}
89
90	static inline void dequeue(struct meta *phead, struct meta m)
91	{
92	if (m->next != m) {
93	m->prev->next = m->next;
94	m->next->prev = m->prev;
95	if (phead == m) phead = m->next;
96	} else {
97	*phead = 0;
98	}
99	m->prev = m->next = 0;
100	}
101
102	static inline struct meta dequeue_head(struct meta *phead)
103	{
104	struct meta m = phead;
105	if (m) dequeue(phead, m);
106	return m;
107	}
108
109	static inline void free_meta(struct meta *m)
110	{
111	*m = (struct meta){0};
112	queue(&ctx.free_meta_head, m);
113	}
114
115	static inline uint32_t activate_group(struct meta *m)
116	{
117	assert(!m->avail_mask);
118	uint32_t mask, act = (2u<<m->mem->active_idx)-1;
119	do mask = m->freed_mask;
120	while (a_cas(&m->freed_mask, mask, mask&~act)!=mask);
121	return m->avail_mask = mask & act;
122	}
123
124	static inline int get_slot_index(const unsigned char *p)
125	{
126	return p[-3] & 31;
127	}
128
129	static inline struct meta get_meta(const unsigned char p)
130	{
131	assert(!((uintptr_t)p & 15));
132	int offset = (const uint16_t )(p - 2);
133	int index = get_slot_index(p);
134	if (p[-4]) {
135	assert(!offset);
136	offset = (uint32_t )(p - 8);
137	assert(offset > 0xffff);
138	}
139	const struct group base = (const void )(p - UNIT*offset - UNIT);
140	const struct meta *meta = base->meta;
141	assert(meta->mem == base);
142	assert(index <= meta->last_idx);
143	assert(!(meta->avail_mask & (1u<<index)));
144	assert(!(meta->freed_mask & (1u<<index)));
145	const struct meta_area area = (void )((uintptr_t)meta & -4096);
146	assert(area->check == ctx.secret);
147	if (meta->sizeclass < 48) {
148	assert(offset >= size_classes[meta->sizeclass]*index);
149	assert(offset < size_classes[meta->sizeclass]*(index+1));
150	} else {
151	assert(meta->sizeclass == 63);
152	}
153	if (meta->maplen) {
154	assert(offset <= meta->maplen*4096UL/UNIT - 1);
155	}
156	return (struct meta *)meta;
157	}
158
159	static inline size_t get_nominal_size(const unsigned char p, const unsigned char end)
160	{
161	size_t reserved = p[-3] >> 5;
162	if (reserved >= 5) {
163	assert(reserved == 5);
164	reserved = (const uint32_t )(end-4);
165	assert(reserved >= 5);
166	assert(!end[-5]);
167	}
168	assert(reserved <= end-p);
169	assert(!*(end-reserved));
170	// also check the slot's overflow byte
171	assert(!*end);
172	return end-reserved-p;
173	}
174
175	static inline size_t get_stride(const struct meta *g)
176	{
177	if (!g->last_idx && g->maplen) {
178	return g->maplen*4096UL - UNIT;
179	} else {
180	return UNIT*size_classes[g->sizeclass];
181	}
182	}
183
184	static inline void set_size(unsigned char p, unsigned char end, size_t n)
185	{
186	int reserved = end-p-n;
187	if (reserved) end[-reserved] = 0;
188	if (reserved >= 5) {
189	(uint32_t )(end-4) = reserved;
190	end[-5] = 0;
191	reserved = 5;
192	}
193	p[-3] = (p[-3]&31) + (reserved<<5);
194	}
195
196	static inline void enframe(struct meta g, int idx, size_t n, int ctr)
197	{
198	size_t stride = get_stride(g);
199	size_t slack = (stride-IB-n)/UNIT;
200	unsigned char p = g->mem->storage + strideidx;
201	unsigned char *end = p+stride-IB;
202	// cycle offset within slot to increase interval to address
203	// reuse, facilitate trapping double-free.
204	int off = (p[-3] ? (uint16_t )(p-2) + 1 : ctr) & 255;
205	assert(!p[-4]);
206	if (off > slack) {
207	size_t m = slack;
208	m \|= m>>1; m \|= m>>2; m \|= m>>4;
209	off &= m;
210	if (off > slack) off -= slack+1;
211	assert(off <= slack);
212	}
213	if (off) {
214	// store offset in unused header at offset zero
215	// if enframing at non-zero offset.
216	(uint16_t )(p-2) = off;
217	p[-3] = 7<<5;
218	p += UNIT*off;
219	// for nonzero offset there is no permanent check
220	// byte, so make one.
221	p[-4] = 0;
222	}
223	(uint16_t )(p-2) = (size_t)(p-g->mem->storage)/UNIT;
224	p[-3] = idx;
225	set_size(p, end, n);
226	return p;
227	}
228
229	static inline int size_to_class(size_t n)
230	{
231	n = (n+IB-1)>>4;
232	if (n<10) return n;
233	n++;
234	int i = (28-a_clz_32(n))*4 + 8;
235	if (n>size_classes[i+1]) i+=2;
236	if (n>size_classes[i]) i++;
237	return i;
238	}
239
240	static inline int size_overflows(size_t n)
241	{
242	if (n >= SIZE_MAX/2 - 4096) {
243	errno = ENOMEM;
244	return 1;
245	}
246	return 0;
247	}
248
249	static inline void step_seq(void)
250	{
251	if (ctx.seq==255) {
252	for (int i=0; i<32; i++) ctx.unmap_seq[i] = 0;
253	ctx.seq = 1;
254	} else {
255	ctx.seq++;
256	}
257	}
258
259	static inline void record_seq(int sc)
260	{
261	if (sc-7U < 32) ctx.unmap_seq[sc-7] = ctx.seq;
262	}
263
264	static inline void account_bounce(int sc)
265	{
266	if (sc-7U < 32) {
267	int seq = ctx.unmap_seq[sc-7];
268	if (seq && ctx.seq-seq < 10) {
269	if (ctx.bounces[sc-7]+1 < 100)
270	ctx.bounces[sc-7]++;
271	else
272	ctx.bounces[sc-7] = 150;
273	}
274	}
275	}
276
277	static inline void decay_bounces(int sc)
278	{
279	if (sc-7U < 32 && ctx.bounces[sc-7])
280	ctx.bounces[sc-7]--;
281	}
282
283	static inline int is_bouncing(int sc)
284	{
285	return (sc-7U < 32 && ctx.bounces[sc-7] >= 100);
286	}
287
288	#endif

lib/libc/musl/src/malloc/mallocng/realloc.c deleted-51

...	@@ -1,51 +0,0 @@
1	#define _GNU_SOURCE
2	#include <stdlib.h>
3	#include <sys/mman.h>
4	#include <string.h>
5	#include "meta.h"
6
7	void realloc(void p, size_t n)
8	{
9	if (!p) return malloc(n);
10	if (size_overflows(n)) return 0;
11
12	struct meta *g = get_meta(p);
13	int idx = get_slot_index(p);
14	size_t stride = get_stride(g);
15	unsigned char start = g->mem->storage + strideidx;
16	unsigned char *end = start + stride - IB;
17	size_t old_size = get_nominal_size(p, end);
18	size_t avail_size = end-(unsigned char *)p;
19	void *new;
20
21	// only resize in-place if size class matches
22	if (n <= avail_size && n<MMAP_THRESHOLD
23	&& size_to_class(n)+1 >= g->sizeclass) {
24	set_size(p, end, n);
25	return p;
26	}
27
28	// use mremap if old and new size are both mmap-worthy
29	if (g->sizeclass>=48 && n>=MMAP_THRESHOLD) {
30	assert(g->sizeclass==63);
31	size_t base = (unsigned char *)p-start;
32	size_t needed = (n + base + UNIT + IB + 4095) & -4096;
33	new = g->maplen*4096UL == needed ? g->mem :
34	mremap(g->mem, g->maplen*4096UL, needed, MREMAP_MAYMOVE);
35	if (new!=MAP_FAILED) {
36	g->mem = new;
37	g->maplen = needed/4096;
38	p = g->mem->storage + base;
39	end = g->mem->storage + (needed - UNIT) - IB;
40	*end = 0;
41	set_size(p, end, n);
42	return p;
43	}
44	}
45
46	new = malloc(n);
47	if (!new) return 0;
48	memcpy(new, p, n < old_size ? n : old_size);
49	free(p);
50	return new;
51	}

lib/libc/musl/src/malloc/memalign.c deleted-7

...	@@ -1,7 +0,0 @@
1	#define _BSD_SOURCE
2	#include <stdlib.h>
3
4	void *memalign(size_t align, size_t len)
5	{
6	return aligned_alloc(align, len);
7	}

lib/libc/musl/src/malloc/oldmalloc/aligned_alloc.c deleted-53

...	@@ -1,53 +0,0 @@
1	#include <stdlib.h>
2	#include <stdint.h>
3	#include <errno.h>
4	#include "malloc_impl.h"
5
6	void *aligned_alloc(size_t align, size_t len)
7	{
8	unsigned char mem, new;
9
10	if ((align & -align) != align) {
11	errno = EINVAL;
12	return 0;
13	}
14
15	if (len > SIZE_MAX - align \|\|
16	(__malloc_replaced && !__aligned_alloc_replaced)) {
17	errno = ENOMEM;
18	return 0;
19	}
20
21	if (align <= SIZE_ALIGN)
22	return malloc(len);
23
24	if (!(mem = malloc(len + align-1)))
25	return 0;
26
27	new = (void *)((uintptr_t)mem + align-1 & -align);
28	if (new == mem) return mem;
29
30	struct chunk *c = MEM_TO_CHUNK(mem);
31	struct chunk *n = MEM_TO_CHUNK(new);
32
33	if (IS_MMAPPED(c)) {
34	/* Apply difference between aligned and original
35	* address to the "extra" field of mmapped chunk. */
36	n->psize = c->psize + (new-mem);
37	n->csize = c->csize - (new-mem);
38	return new;
39	}
40
41	struct chunk *t = NEXT_CHUNK(c);
42
43	/* Split the allocated chunk into two chunks. The aligned part
44	* that will be used has the size in its footer reduced by the
45	* difference between the aligned and original addresses, and
46	* the resulting size copied to its header. A new header and
47	* footer are written for the split-off part to be freed. */
48	n->psize = c->csize = C_INUSE \| (new-mem);
49	n->csize = t->psize -= new-mem;
50
51	__bin_chunk(c);
52	return new;
53	}

lib/libc/musl/src/malloc/oldmalloc/malloc.c deleted-556

...	@@ -1,556 +0,0 @@
1	#define _GNU_SOURCE
2	#include <stdlib.h>
3	#include <string.h>
4	#include <limits.h>
5	#include <stdint.h>
6	#include <errno.h>
7	#include <sys/mman.h>
8	#include "libc.h"
9	#include "atomic.h"
10	#include "pthread_impl.h"
11	#include "malloc_impl.h"
12	#include "fork_impl.h"
13
14	#define malloc __libc_malloc_impl
15	#define realloc __libc_realloc
16	#define free __libc_free
17
18	#if defined(__GNUC__) && defined(__PIC__)
19	#define inline inline __attribute__((always_inline))
20	#endif
21
22	static struct {
23	volatile uint64_t binmap;
24	struct bin bins[64];
25	volatile int split_merge_lock[2];
26	} mal;
27
28	/* Synchronization tools */
29
30	static inline void lock(volatile int *lk)
31	{
32	int need_locks = libc.need_locks;
33	if (need_locks) {
34	while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1);
35	if (need_locks < 0) libc.need_locks = 0;
36	}
37	}
38
39	static inline void unlock(volatile int *lk)
40	{
41	if (lk[0]) {
42	a_store(lk, 0);
43	if (lk[1]) __wake(lk, 1, 1);
44	}
45	}
46
47	static inline void lock_bin(int i)
48	{
49	lock(mal.bins[i].lock);
50	if (!mal.bins[i].head)
51	mal.bins[i].head = mal.bins[i].tail = BIN_TO_CHUNK(i);
52	}
53
54	static inline void unlock_bin(int i)
55	{
56	unlock(mal.bins[i].lock);
57	}
58
59	static int first_set(uint64_t x)
60	{
61	#if 1
62	return a_ctz_64(x);
63	#else
64	static const char debruijn64[64] = {
65	0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28,
66	62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11,
67	63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10,
68	51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12
69	};
70	static const char debruijn32[32] = {
71	0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13,
72	31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14
73	};
74	if (sizeof(long) < 8) {
75	uint32_t y = x;
76	if (!y) {
77	y = x>>32;
78	return 32 + debruijn32[(y&-y)*0x076be629 >> 27];
79	}
80	return debruijn32[(y&-y)*0x076be629 >> 27];
81	}
82	return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58];
83	#endif
84	}
85
86	static const unsigned char bin_tab[60] = {
87	32,33,34,35,36,36,37,37,38,38,39,39,
88	40,40,40,40,41,41,41,41,42,42,42,42,43,43,43,43,
89	44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45,
90	46,46,46,46,46,46,46,46,47,47,47,47,47,47,47,47,
91	};
92
93	static int bin_index(size_t x)
94	{
95	x = x / SIZE_ALIGN - 1;
96	if (x <= 32) return x;
97	if (x < 512) return bin_tab[x/8-4];
98	if (x > 0x1c00) return 63;
99	return bin_tab[x/128-4] + 16;
100	}
101
102	static int bin_index_up(size_t x)
103	{
104	x = x / SIZE_ALIGN - 1;
105	if (x <= 32) return x;
106	x--;
107	if (x < 512) return bin_tab[x/8-4] + 1;
108	return bin_tab[x/128-4] + 17;
109	}
110
111	#if 0
112	void __dump_heap(int x)
113	{
114	struct chunk *c;
115	int i;
116	for (c = (void *)mal.heap; CHUNK_SIZE(c); c = NEXT_CHUNK(c))
117	fprintf(stderr, "base %p size %zu (%d) flags %d/%d\n",
118	c, CHUNK_SIZE(c), bin_index(CHUNK_SIZE(c)),
119	c->csize & 15,
120	NEXT_CHUNK(c)->psize & 15);
121	for (i=0; i<64; i++) {
122	if (mal.bins[i].head != BIN_TO_CHUNK(i) && mal.bins[i].head) {
123	fprintf(stderr, "bin %d: %p\n", i, mal.bins[i].head);
124	if (!(mal.binmap & 1ULL<<i))
125	fprintf(stderr, "missing from binmap!\n");
126	} else if (mal.binmap & 1ULL<<i)
127	fprintf(stderr, "binmap wrongly contains %d!\n", i);
128	}
129	}
130	#endif
131
132	/* This function returns true if the interval [old,new]
133	* intersects the 'len'-sized interval below &libc.auxv
134	* (interpreted as the main-thread stack) or below &b
135	* (the current stack). It is used to defend against
136	* buggy brk implementations that can cross the stack. */
137
138	static int traverses_stack_p(uintptr_t old, uintptr_t new)
139	{
140	const uintptr_t len = 8<<20;
141	uintptr_t a, b;
142
143	b = (uintptr_t)libc.auxv;
144	a = b > len ? b-len : 0;
145	if (new>a && old<b) return 1;
146
147	b = (uintptr_t)&b;
148	a = b > len ? b-len : 0;
149	if (new>a && old<b) return 1;
150
151	return 0;
152	}
153
154	/* Expand the heap in-place if brk can be used, or otherwise via mmap,
155	* using an exponential lower bound on growth by mmap to make
156	* fragmentation asymptotically irrelevant. The size argument is both
157	* an input and an output, since the caller needs to know the size
158	* allocated, which will be larger than requested due to page alignment
159	* and mmap minimum size rules. The caller is responsible for locking
160	* to prevent concurrent calls. */
161
162	static void __expand_heap(size_t pn)
163	{
164	static uintptr_t brk;
165	static unsigned mmap_step;
166	size_t n = *pn;
167
168	if (n > SIZE_MAX/2 - PAGE_SIZE) {
169	errno = ENOMEM;
170	return 0;
171	}
172	n += -n & PAGE_SIZE-1;
173
174	if (!brk) {
175	brk = __syscall(SYS_brk, 0);
176	brk += -brk & PAGE_SIZE-1;
177	}
178
179	if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n)
180	&& __syscall(SYS_brk, brk+n)==brk+n) {
181	*pn = n;
182	brk += n;
183	return (void *)(brk-n);
184	}
185
186	size_t min = (size_t)PAGE_SIZE << mmap_step/2;
187	if (n < min) n = min;
188	void *area = __mmap(0, n, PROT_READ\|PROT_WRITE,
189	MAP_PRIVATE\|MAP_ANONYMOUS, -1, 0);
190	if (area == MAP_FAILED) return 0;
191	*pn = n;
192	mmap_step++;
193	return area;
194	}
195
196	static struct chunk *expand_heap(size_t n)
197	{
198	static void *end;
199	void *p;
200	struct chunk *w;
201
202	/* The argument n already accounts for the caller's chunk
203	* overhead needs, but if the heap can't be extended in-place,
204	* we need room for an extra zero-sized sentinel chunk. */
205	n += SIZE_ALIGN;
206
207	p = __expand_heap(&n);
208	if (!p) return 0;
209
210	/* If not just expanding existing space, we need to make a
211	* new sentinel chunk below the allocated space. */
212	if (p != end) {
213	/* Valid/safe because of the prologue increment. */
214	n -= SIZE_ALIGN;
215	p = (char *)p + SIZE_ALIGN;
216	w = MEM_TO_CHUNK(p);
217	w->psize = 0 \| C_INUSE;
218	}
219
220	/* Record new heap end and fill in footer. */
221	end = (char *)p + n;
222	w = MEM_TO_CHUNK(end);
223	w->psize = n \| C_INUSE;
224	w->csize = 0 \| C_INUSE;
225
226	/* Fill in header, which may be new or may be replacing a
227	* zero-size sentinel header at the old end-of-heap. */
228	w = MEM_TO_CHUNK(p);
229	w->csize = n \| C_INUSE;
230
231	return w;
232	}
233
234	static int adjust_size(size_t *n)
235	{
236	/* Result of pointer difference must fit in ptrdiff_t. */
237	if (*n-1 > PTRDIFF_MAX - SIZE_ALIGN - PAGE_SIZE) {
238	if (*n) {
239	errno = ENOMEM;
240	return -1;
241	} else {
242	*n = SIZE_ALIGN;
243	return 0;
244	}
245	}
246	n = (n + OVERHEAD + SIZE_ALIGN - 1) & SIZE_MASK;
247	return 0;
248	}
249
250	static void unbin(struct chunk *c, int i)
251	{
252	if (c->prev == c->next)
253	a_and_64(&mal.binmap, ~(1ULL<<i));
254	c->prev->next = c->next;
255	c->next->prev = c->prev;
256	c->csize \|= C_INUSE;
257	NEXT_CHUNK(c)->psize \|= C_INUSE;
258	}
259
260	static void bin_chunk(struct chunk *self, int i)
261	{
262	self->next = BIN_TO_CHUNK(i);
263	self->prev = mal.bins[i].tail;
264	self->next->prev = self;
265	self->prev->next = self;
266	if (self->prev == BIN_TO_CHUNK(i))
267	a_or_64(&mal.binmap, 1ULL<<i);
268	}
269
270	static void trim(struct chunk *self, size_t n)
271	{
272	size_t n1 = CHUNK_SIZE(self);
273	struct chunk next, split;
274
275	if (n >= n1 - DONTCARE) return;
276
277	next = NEXT_CHUNK(self);
278	split = (void )((char )self + n);
279
280	split->psize = n \| C_INUSE;
281	split->csize = n1-n;
282	next->psize = n1-n;
283	self->csize = n \| C_INUSE;
284
285	int i = bin_index(n1-n);
286	lock_bin(i);
287
288	bin_chunk(split, i);
289
290	unlock_bin(i);
291	}
292
293	void *malloc(size_t n)
294	{
295	struct chunk *c;
296	int i, j;
297	uint64_t mask;
298
299	if (adjust_size(&n) < 0) return 0;
300
301	if (n > MMAP_THRESHOLD) {
302	size_t len = n + OVERHEAD + PAGE_SIZE - 1 & -PAGE_SIZE;
303	char *base = __mmap(0, len, PROT_READ\|PROT_WRITE,
304	MAP_PRIVATE\|MAP_ANONYMOUS, -1, 0);
305	if (base == (void *)-1) return 0;
306	c = (void *)(base + SIZE_ALIGN - OVERHEAD);
307	c->csize = len - (SIZE_ALIGN - OVERHEAD);
308	c->psize = SIZE_ALIGN - OVERHEAD;
309	return CHUNK_TO_MEM(c);
310	}
311
312	i = bin_index_up(n);
313	if (i<63 && (mal.binmap & (1ULL<<i))) {
314	lock_bin(i);
315	c = mal.bins[i].head;
316	if (c != BIN_TO_CHUNK(i) && CHUNK_SIZE(c)-n <= DONTCARE) {
317	unbin(c, i);
318	unlock_bin(i);
319	return CHUNK_TO_MEM(c);
320	}
321	unlock_bin(i);
322	}
323	lock(mal.split_merge_lock);
324	for (mask = mal.binmap & -(1ULL<<i); mask; mask -= (mask&-mask)) {
325	j = first_set(mask);
326	lock_bin(j);
327	c = mal.bins[j].head;
328	if (c != BIN_TO_CHUNK(j)) {
329	unbin(c, j);
330	unlock_bin(j);
331	break;
332	}
333	unlock_bin(j);
334	}
335	if (!mask) {
336	c = expand_heap(n);
337	if (!c) {
338	unlock(mal.split_merge_lock);
339	return 0;
340	}
341	}
342	trim(c, n);
343	unlock(mal.split_merge_lock);
344	return CHUNK_TO_MEM(c);
345	}
346
347	int __malloc_allzerop(void *p)
348	{
349	return IS_MMAPPED(MEM_TO_CHUNK(p));
350	}
351
352	void realloc(void p, size_t n)
353	{
354	struct chunk self, next;
355	size_t n0, n1;
356	void *new;
357
358	if (!p) return malloc(n);
359
360	if (adjust_size(&n) < 0) return 0;
361
362	self = MEM_TO_CHUNK(p);
363	n1 = n0 = CHUNK_SIZE(self);
364
365	if (n<=n0 && n0-n<=DONTCARE) return p;
366
367	if (IS_MMAPPED(self)) {
368	size_t extra = self->psize;
369	char base = (char )self - extra;
370	size_t oldlen = n0 + extra;
371	size_t newlen = n + extra;
372	/* Crash on realloc of freed chunk */
373	if (extra & 1) a_crash();
374	if (newlen < PAGE_SIZE && (new = malloc(n-OVERHEAD))) {
375	n0 = n;
376	goto copy_free_ret;
377	}
378	newlen = (newlen + PAGE_SIZE-1) & -PAGE_SIZE;
379	if (oldlen == newlen) return p;
380	base = __mremap(base, oldlen, newlen, MREMAP_MAYMOVE);
381	if (base == (void *)-1)
382	goto copy_realloc;
383	self = (void *)(base + extra);
384	self->csize = newlen - extra;
385	return CHUNK_TO_MEM(self);
386	}
387
388	next = NEXT_CHUNK(self);
389
390	/* Crash on corrupted footer (likely from buffer overflow) */
391	if (next->psize != self->csize) a_crash();
392
393	if (n < n0) {
394	int i = bin_index_up(n);
395	int j = bin_index(n0);
396	if (i<j && (mal.binmap & (1ULL << i)))
397	goto copy_realloc;
398	struct chunk split = (void )((char *)self + n);
399	self->csize = split->psize = n \| C_INUSE;
400	split->csize = next->psize = n0-n \| C_INUSE;
401	__bin_chunk(split);
402	return CHUNK_TO_MEM(self);
403	}
404
405	lock(mal.split_merge_lock);
406
407	size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next);
408	if (n0+nsize >= n) {
409	int i = bin_index(nsize);
410	lock_bin(i);
411	if (!(next->csize & C_INUSE)) {
412	unbin(next, i);
413	unlock_bin(i);
414	next = NEXT_CHUNK(next);
415	self->csize = next->psize = n0+nsize \| C_INUSE;
416	trim(self, n);
417	unlock(mal.split_merge_lock);
418	return CHUNK_TO_MEM(self);
419	}
420	unlock_bin(i);
421	}
422	unlock(mal.split_merge_lock);
423
424	copy_realloc:
425	/* As a last resort, allocate a new chunk and copy to it. */
426	new = malloc(n-OVERHEAD);
427	if (!new) return 0;
428	copy_free_ret:
429	memcpy(new, p, (n<n0 ? n : n0) - OVERHEAD);
430	free(CHUNK_TO_MEM(self));
431	return new;
432	}
433
434	void __bin_chunk(struct chunk *self)
435	{
436	struct chunk *next = NEXT_CHUNK(self);
437
438	/* Crash on corrupted footer (likely from buffer overflow) */
439	if (next->psize != self->csize) a_crash();
440
441	lock(mal.split_merge_lock);
442
443	size_t osize = CHUNK_SIZE(self), size = osize;
444
445	/* Since we hold split_merge_lock, only transition from free to
446	* in-use can race; in-use to free is impossible */
447	size_t psize = self->psize & C_INUSE ? 0 : CHUNK_PSIZE(self);
448	size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next);
449
450	if (psize) {
451	int i = bin_index(psize);
452	lock_bin(i);
453	if (!(self->psize & C_INUSE)) {
454	struct chunk *prev = PREV_CHUNK(self);
455	unbin(prev, i);
456	self = prev;
457	size += psize;
458	}
459	unlock_bin(i);
460	}
461	if (nsize) {
462	int i = bin_index(nsize);
463	lock_bin(i);
464	if (!(next->csize & C_INUSE)) {
465	unbin(next, i);
466	next = NEXT_CHUNK(next);
467	size += nsize;
468	}
469	unlock_bin(i);
470	}
471
472	int i = bin_index(size);
473	lock_bin(i);
474
475	self->csize = size;
476	next->psize = size;
477	bin_chunk(self, i);
478	unlock(mal.split_merge_lock);
479
480	/* Replace middle of large chunks with fresh zero pages */
481	if (size > RECLAIM && (size^(size-osize)) > size-osize) {
482	uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE;
483	uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE;
484	int e = errno;
485	#if 1
486	__madvise((void *)a, b-a, MADV_DONTNEED);
487	#else
488	__mmap((void *)a, b-a, PROT_READ\|PROT_WRITE,
489	MAP_PRIVATE\|MAP_ANONYMOUS\|MAP_FIXED, -1, 0);
490	#endif
491	errno = e;
492	}
493
494	unlock_bin(i);
495	}
496
497	static void unmap_chunk(struct chunk *self)
498	{
499	size_t extra = self->psize;
500	char base = (char )self - extra;
501	size_t len = CHUNK_SIZE(self) + extra;
502	/* Crash on double free */
503	if (extra & 1) a_crash();
504	int e = errno;
505	__munmap(base, len);
506	errno = e;
507	}
508
509	void free(void *p)
510	{
511	if (!p) return;
512
513	struct chunk *self = MEM_TO_CHUNK(p);
514
515	if (IS_MMAPPED(self))
516	unmap_chunk(self);
517	else
518	__bin_chunk(self);
519	}
520
521	void __malloc_donate(char start, char end)
522	{
523	size_t align_start_up = (SIZE_ALIGN-1) & (-(uintptr_t)start - OVERHEAD);
524	size_t align_end_down = (SIZE_ALIGN-1) & (uintptr_t)end;
525
526	/* Getting past this condition ensures that the padding for alignment
527	* and header overhead will not overflow and will leave a nonzero
528	* multiple of SIZE_ALIGN bytes between start and end. */
529	if (end - start <= OVERHEAD + align_start_up + align_end_down)
530	return;
531	start += align_start_up + OVERHEAD;
532	end -= align_end_down;
533
534	struct chunk c = MEM_TO_CHUNK(start), n = MEM_TO_CHUNK(end);
535	c->psize = n->csize = C_INUSE;
536	c->csize = n->psize = C_INUSE \| (end-start);
537	__bin_chunk(c);
538	}
539
540	void __malloc_atfork(int who)
541	{
542	if (who<0) {
543	lock(mal.split_merge_lock);
544	for (int i=0; i<64; i++)
545	lock(mal.bins[i].lock);
546	} else if (!who) {
547	for (int i=0; i<64; i++)
548	unlock(mal.bins[i].lock);
549	unlock(mal.split_merge_lock);
550	} else {
551	for (int i=0; i<64; i++)
552	mal.bins[i].lock[0] = mal.bins[i].lock[1] = 0;
553	mal.split_merge_lock[1] = 0;
554	mal.split_merge_lock[0] = 0;
555	}
556	}

lib/libc/musl/src/malloc/oldmalloc/malloc_impl.h deleted-39

...	@@ -1,39 +0,0 @@
1	#ifndef MALLOC_IMPL_H
2	#define MALLOC_IMPL_H
3
4	#include <sys/mman.h>
5	#include "dynlink.h"
6
7	struct chunk {
8	size_t psize, csize;
9	struct chunk next, prev;
10	};
11
12	struct bin {
13	volatile int lock[2];
14	struct chunk *head;
15	struct chunk *tail;
16	};
17
18	#define SIZE_ALIGN (4*sizeof(size_t))
19	#define SIZE_MASK (-SIZE_ALIGN)
20	#define OVERHEAD (2*sizeof(size_t))
21	#define MMAP_THRESHOLD (0x1c00*SIZE_ALIGN)
22	#define DONTCARE 16
23	#define RECLAIM 163840
24
25	#define CHUNK_SIZE(c) ((c)->csize & -2)
26	#define CHUNK_PSIZE(c) ((c)->psize & -2)
27	#define PREV_CHUNK(c) ((struct chunk )((char )(c) - CHUNK_PSIZE(c)))
28	#define NEXT_CHUNK(c) ((struct chunk )((char )(c) + CHUNK_SIZE(c)))
29	#define MEM_TO_CHUNK(p) (struct chunk )((char )(p) - OVERHEAD)
30	#define CHUNK_TO_MEM(c) (void )((char )(c) + OVERHEAD)
31	#define BIN_TO_CHUNK(i) (MEM_TO_CHUNK(&mal.bins[i].head))
32
33	#define C_INUSE ((size_t)1)
34
35	#define IS_MMAPPED(c) !((c)->csize & (C_INUSE))
36
37	hidden void __bin_chunk(struct chunk *);
38
39	#endif

lib/libc/musl/src/malloc/oldmalloc/malloc_usable_size.c deleted-9

...	@@ -1,9 +0,0 @@
1	#include <malloc.h>
2	#include "malloc_impl.h"
3
4	hidden void (const __realloc_dep)(void *, size_t) = realloc;
5
6	size_t malloc_usable_size(void *p)
7	{
8	return p ? CHUNK_SIZE(MEM_TO_CHUNK(p)) - OVERHEAD : 0;
9	}

lib/libc/musl/src/malloc/posix_memalign.c deleted-11

...	@@ -1,11 +0,0 @@
1	#include <stdlib.h>
2	#include <errno.h>
3
4	int posix_memalign(void **res, size_t align, size_t len)
5	{
6	if (align < sizeof(void *)) return EINVAL;
7	void *mem = aligned_alloc(align, len);
8	if (!mem) return errno;
9	*res = mem;
10	return 0;
11	}

lib/libc/musl/src/malloc/realloc.c deleted-6

...	@@ -1,6 +0,0 @@
1	#include <stdlib.h>
2
3	void realloc(void p, size_t n)
4	{
5	return __libc_realloc(p, n);
6	}

lib/libc/musl/src/malloc/reallocarray.c deleted-13

...	@@ -1,13 +0,0 @@
1	#define _BSD_SOURCE
2	#include <errno.h>
3	#include <stdlib.h>
4
5	void reallocarray(void ptr, size_t m, size_t n)
6	{
7	if (n && m > -1 / n) {
8	errno = ENOMEM;
9	return 0;
10	}
11
12	return realloc(ptr, m * n);
13	}

lib/libc/musl/src/malloc/replaced.c deleted-4

...	@@ -1,4 +0,0 @@
1	#include "dynlink.h"
2
3	int __malloc_replaced;
4	int __aligned_alloc_replaced;

lib/libc/musl/src/process/fdop.h-5

...	@@ -10,8 +10,3 @@ struct fdop {	...	@@ -10,8 +10,3 @@ struct fdop {
10	mode_t mode;	10	mode_t mode;
11	char path[];	11	char path[];
12	};	12	};
13
14	#define malloc __libc_malloc
15	#define calloc __libc_calloc
16	#define realloc undef
17	#define free __libc_free

lib/libc/musl/src/thread/pthread_atfork.c-5

 @@ -3,11 +3,6 @@
 #include "libc.h"
 #include "lock.h"
-#define malloc __libc_malloc
-#define calloc undef
-#define realloc undef
-#define free undef
 static struct atfork_funcs {
 	void (*prepare)(void);
 	void (*parent)(void);

lib/libc/musl/src/thread/sem_open.c-5

 @@ -14,11 +14,6 @@
 #include "lock.h"
 #include "fork_impl.h"
-#define malloc __libc_malloc
-#define calloc __libc_calloc
-#define realloc undef
-#define free undef
 static struct {
 	ino_t ino;
 	sem_t *sem;

lib/libc/musl/src/time/__tz.c-5

 @@ -9,11 +9,6 @@
 #include "lock.h"
 #include "fork_impl.h"
-#define malloc __libc_malloc
-#define calloc undef
-#define realloc undef
-#define free undef
 long  __timezone = 0;
 int   __daylight = 0;
 char *__tzname[2] = { 0, 0 };

lib/libc/wasi/emmalloc/emmalloc.c deleted-1540

...	@@ -1,1540 +0,0 @@
1	/*
2	* Copyright 2018 The Emscripten Authors. All rights reserved.
3	* Emscripten is available under two separate licenses, the MIT license and the
4	* University of Illinois/NCSA Open Source License. Both these licenses can be
5	* found in the LICENSE file.
6	*
7	* Simple minimalistic but efficient sbrk()-based malloc/free that works in
8	* singlethreaded and multithreaded builds.
9	*
10	* Assumptions:
11	*
12	* - sbrk() is used to claim new memory (sbrk handles geometric/linear
13	* - overallocation growth)
14	* - sbrk() can be used by other code outside emmalloc.
15	* - sbrk() is very fast in most cases (internal wasm call).
16	* - sbrk() returns pointers with an alignment of alignof(max_align_t)
17	*
18	* Invariants:
19	*
20	* - Per-allocation header overhead is 8 bytes, smallest allocated payload
21	* amount is 8 bytes, and a multiple of 4 bytes.
22	* - Acquired memory blocks are subdivided into disjoint regions that lie
23	* next to each other.
24	* - A region is either in used or free.
25	* Used regions may be adjacent, and a used and unused region
26	* may be adjacent, but not two unused ones - they would be
27	* merged.
28	* - Memory allocation takes constant time, unless the alloc needs to sbrk()
29	* or memory is very close to being exhausted.
30	*
31	* Debugging:
32	*
33	* - If not NDEBUG, runtime assert()s are in use.
34	* - If EMMALLOC_MEMVALIDATE is defined, a large amount of extra checks are done.
35	* - If EMMALLOC_VERBOSE is defined, a lot of operations are logged
36	* out, in addition to EMMALLOC_MEMVALIDATE.
37	* - Debugging and logging directly uses console.log via uses EM_ASM, not
38	* printf etc., to minimize any risk of debugging or logging depending on
39	* malloc.
40	*/
41
42	#include <stdalign.h>
43	#include <stdbool.h>
44	#include <stddef.h>
45	#include <stdint.h>
46	#include <unistd.h>
47	#include <memory.h>
48	#include <assert.h>
49	#include <malloc.h>
50	#include <limits.h>
51	#include <stdlib.h>
52
53	#ifdef __EMSCRIPTEN_TRACING__
54	#include <emscripten/trace.h>
55	#endif
56
57	// Defind by the linker to have the address of the start of the heap.
58	extern unsigned char __heap_base;
59	extern unsigned char __heap_end;
60
61	// Behavior of right shifting a signed integer is compiler implementation defined.
62	static_assert((((int32_t)0x80000000U) >> 31) == -1, "This malloc implementation requires that right-shifting a signed integer produces a sign-extending (arithmetic) shift!");
63
64	// Configuration: specifies the minimum alignment that malloc()ed memory outputs. Allocation requests with smaller alignment
65	// than this will yield an allocation with this much alignment.
66	#define MALLOC_ALIGNMENT alignof(max_align_t)
67	static_assert(alignof(max_align_t) == 16, "max_align_t must be correct");
68
69	#define EMMALLOC_EXPORT __attribute__((weak))
70
71	#define MIN(x, y) ((x) < (y) ? (x) : (y))
72	#define MAX(x, y) ((x) > (y) ? (x) : (y))
73
74	#define NUM_FREE_BUCKETS 64
75	#define BUCKET_BITMASK_T uint64_t
76
77	// Dynamic memory is subdivided into regions, in the format
78
79	// <size:uint32_t> ..... <size:uint32_t> \| <size:uint32_t> ..... <size:uint32_t> \| <size:uint32_t> ..... <size:uint32_t> \| .....
80
81	// That is, at the bottom and top end of each memory region, the size of that region is stored. That allows traversing the
82	// memory regions backwards and forwards. Because each allocation must be at least a multiple of 4 bytes, the lowest two bits of
83	// each size field is unused. Free regions are distinguished by used regions by having the FREE_REGION_FLAG bit present
84	// in the size field. I.e. for free regions, the size field is odd, and for used regions, the size field reads even.
85	#define FREE_REGION_FLAG 0x1u
86
87	// Attempts to malloc() more than this many bytes would cause an overflow when calculating the size of a region,
88	// therefore allocations larger than this are short-circuited immediately on entry.
89	#define MAX_ALLOC_SIZE 0xFFFFFFC7u
90
91	// A free region has the following structure:
92	// <size:size_t> <prevptr> <nextptr> ... <size:size_t>
93
94	typedef struct Region
95	{
96	size_t size;
97	// Use a circular doubly linked list to represent free region data.
98	struct Region prev, next;
99	// ... N bytes of free data
100	size_t _at_the_end_of_this_struct_size; // do not dereference, this is present for convenient struct sizeof() computation only
101	} Region;
102
103	// Each memory block starts with a RootRegion at the beginning.
104	// The RootRegion specifies the size of the region block, and forms a linked
105	// list of all RootRegions in the program, starting with `listOfAllRegions`
106	// below.
107	typedef struct RootRegion
108	{
109	uint32_t size;
110	struct RootRegion *next;
111	uint8_t* endPtr;
112	} RootRegion;
113
114	#if defined(__EMSCRIPTEN_PTHREADS__)
115	// In multithreaded builds, use a simple global spinlock strategy to acquire/release access to the memory allocator.
116	static volatile uint8_t multithreadingLock = 0;
117	#define MALLOC_ACQUIRE() while(__sync_lock_test_and_set(&multithreadingLock, 1)) { while(multithreadingLock) { /nop/ } }
118	#define MALLOC_RELEASE() __sync_lock_release(&multithreadingLock)
119	// Test code to ensure we have tight malloc acquire/release guards in place.
120	#define ASSERT_MALLOC_IS_ACQUIRED() assert(multithreadingLock == 1)
121	#else
122	// In singlethreaded builds, no need for locking.
123	#define MALLOC_ACQUIRE() ((void)0)
124	#define MALLOC_RELEASE() ((void)0)
125	#define ASSERT_MALLOC_IS_ACQUIRED() ((void)0)
126	#endif
127
128	#define IS_POWER_OF_2(val) (((val) & ((val)-1)) == 0)
129	#define ALIGN_UP(ptr, alignment) ((uint8_t*)((((uintptr_t)(ptr)) + ((alignment)-1)) & ~((alignment)-1)))
130	#define HAS_ALIGNMENT(ptr, alignment) ((((uintptr_t)(ptr)) & ((alignment)-1)) == 0)
131
132	static_assert(IS_POWER_OF_2(MALLOC_ALIGNMENT), "MALLOC_ALIGNMENT must be a power of two value!");
133	static_assert(MALLOC_ALIGNMENT >= 4, "Smallest possible MALLOC_ALIGNMENT if 4!");
134
135	// A region that contains as payload a single forward linked list of pointers to
136	// root regions of each disjoint region blocks.
137	static RootRegion *listOfAllRegions = NULL;
138
139	// For each of the buckets, maintain a linked list head node. The head node for each
140	// free region is a sentinel node that does not actually represent any free space, but
141	// the sentinel is used to avoid awkward testing against (if node == freeRegionHeadNode)
142	// when adding and removing elements from the linked list, i.e. we are guaranteed that
143	// the sentinel node is always fixed and there, and the actual free region list elements
144	// start at freeRegionBuckets[i].next each.
145	static Region freeRegionBuckets[NUM_FREE_BUCKETS] = {
146	{ .prev = &freeRegionBuckets[0], .next = &freeRegionBuckets[0] },
147	{ .prev = &freeRegionBuckets[1], .next = &freeRegionBuckets[1] },
148	{ .prev = &freeRegionBuckets[2], .next = &freeRegionBuckets[2] },
149	{ .prev = &freeRegionBuckets[3], .next = &freeRegionBuckets[3] },
150	{ .prev = &freeRegionBuckets[4], .next = &freeRegionBuckets[4] },
151	{ .prev = &freeRegionBuckets[5], .next = &freeRegionBuckets[5] },
152	{ .prev = &freeRegionBuckets[6], .next = &freeRegionBuckets[6] },
153	{ .prev = &freeRegionBuckets[7], .next = &freeRegionBuckets[7] },
154	{ .prev = &freeRegionBuckets[8], .next = &freeRegionBuckets[8] },
155	{ .prev = &freeRegionBuckets[9], .next = &freeRegionBuckets[9] },
156	{ .prev = &freeRegionBuckets[10], .next = &freeRegionBuckets[10] },
157	{ .prev = &freeRegionBuckets[11], .next = &freeRegionBuckets[11] },
158	{ .prev = &freeRegionBuckets[12], .next = &freeRegionBuckets[12] },
159	{ .prev = &freeRegionBuckets[13], .next = &freeRegionBuckets[13] },
160	{ .prev = &freeRegionBuckets[14], .next = &freeRegionBuckets[14] },
161	{ .prev = &freeRegionBuckets[15], .next = &freeRegionBuckets[15] },
162	{ .prev = &freeRegionBuckets[16], .next = &freeRegionBuckets[16] },
163	{ .prev = &freeRegionBuckets[17], .next = &freeRegionBuckets[17] },
164	{ .prev = &freeRegionBuckets[18], .next = &freeRegionBuckets[18] },
165	{ .prev = &freeRegionBuckets[19], .next = &freeRegionBuckets[19] },
166	{ .prev = &freeRegionBuckets[20], .next = &freeRegionBuckets[20] },
167	{ .prev = &freeRegionBuckets[21], .next = &freeRegionBuckets[21] },
168	{ .prev = &freeRegionBuckets[22], .next = &freeRegionBuckets[22] },
169	{ .prev = &freeRegionBuckets[23], .next = &freeRegionBuckets[23] },
170	{ .prev = &freeRegionBuckets[24], .next = &freeRegionBuckets[24] },
171	{ .prev = &freeRegionBuckets[25], .next = &freeRegionBuckets[25] },
172	{ .prev = &freeRegionBuckets[26], .next = &freeRegionBuckets[26] },
173	{ .prev = &freeRegionBuckets[27], .next = &freeRegionBuckets[27] },
174	{ .prev = &freeRegionBuckets[28], .next = &freeRegionBuckets[28] },
175	{ .prev = &freeRegionBuckets[29], .next = &freeRegionBuckets[29] },
176	{ .prev = &freeRegionBuckets[30], .next = &freeRegionBuckets[30] },
177	{ .prev = &freeRegionBuckets[31], .next = &freeRegionBuckets[31] },
178	{ .prev = &freeRegionBuckets[32], .next = &freeRegionBuckets[32] },
179	{ .prev = &freeRegionBuckets[33], .next = &freeRegionBuckets[33] },
180	{ .prev = &freeRegionBuckets[34], .next = &freeRegionBuckets[34] },
181	{ .prev = &freeRegionBuckets[35], .next = &freeRegionBuckets[35] },
182	{ .prev = &freeRegionBuckets[36], .next = &freeRegionBuckets[36] },
183	{ .prev = &freeRegionBuckets[37], .next = &freeRegionBuckets[37] },
184	{ .prev = &freeRegionBuckets[38], .next = &freeRegionBuckets[38] },
185	{ .prev = &freeRegionBuckets[39], .next = &freeRegionBuckets[39] },
186	{ .prev = &freeRegionBuckets[40], .next = &freeRegionBuckets[40] },
187	{ .prev = &freeRegionBuckets[41], .next = &freeRegionBuckets[41] },
188	{ .prev = &freeRegionBuckets[42], .next = &freeRegionBuckets[42] },
189	{ .prev = &freeRegionBuckets[43], .next = &freeRegionBuckets[43] },
190	{ .prev = &freeRegionBuckets[44], .next = &freeRegionBuckets[44] },
191	{ .prev = &freeRegionBuckets[45], .next = &freeRegionBuckets[45] },
192	{ .prev = &freeRegionBuckets[46], .next = &freeRegionBuckets[46] },
193	{ .prev = &freeRegionBuckets[47], .next = &freeRegionBuckets[47] },
194	{ .prev = &freeRegionBuckets[48], .next = &freeRegionBuckets[48] },
195	{ .prev = &freeRegionBuckets[49], .next = &freeRegionBuckets[49] },
196	{ .prev = &freeRegionBuckets[50], .next = &freeRegionBuckets[50] },
197	{ .prev = &freeRegionBuckets[51], .next = &freeRegionBuckets[51] },
198	{ .prev = &freeRegionBuckets[52], .next = &freeRegionBuckets[52] },
199	{ .prev = &freeRegionBuckets[53], .next = &freeRegionBuckets[53] },
200	{ .prev = &freeRegionBuckets[54], .next = &freeRegionBuckets[54] },
201	{ .prev = &freeRegionBuckets[55], .next = &freeRegionBuckets[55] },
202	{ .prev = &freeRegionBuckets[56], .next = &freeRegionBuckets[56] },
203	{ .prev = &freeRegionBuckets[57], .next = &freeRegionBuckets[57] },
204	{ .prev = &freeRegionBuckets[58], .next = &freeRegionBuckets[58] },
205	{ .prev = &freeRegionBuckets[59], .next = &freeRegionBuckets[59] },
206	{ .prev = &freeRegionBuckets[60], .next = &freeRegionBuckets[60] },
207	{ .prev = &freeRegionBuckets[61], .next = &freeRegionBuckets[61] },
208	{ .prev = &freeRegionBuckets[62], .next = &freeRegionBuckets[62] },
209	{ .prev = &freeRegionBuckets[63], .next = &freeRegionBuckets[63] },
210	};
211
212	// A bitmask that tracks the population status for each of the 64 distinct memory regions:
213	// a zero at bit position i means that the free list bucket i is empty. This bitmask is
214	// used to avoid redundant scanning of the 64 different free region buckets: instead by
215	// looking at the bitmask we can find in constant time an index to a free region bucket
216	// that contains free memory of desired size.
217	static BUCKET_BITMASK_T freeRegionBucketsUsed = 0;
218
219	// Amount of bytes taken up by allocation header data
220	#define REGION_HEADER_SIZE (2*sizeof(size_t))
221
222	// Smallest allocation size that is possible is 2*pointer size, since payload of each region must at least contain space
223	// to store the free region linked list prev and next pointers. An allocation size smaller than this will be rounded up
224	// to this size.
225	#define SMALLEST_ALLOCATION_SIZE (2sizeof(void))
226
227	/* Subdivide regions of free space into distinct circular doubly linked lists, where each linked list
228	represents a range of free space blocks. The following function compute_free_list_bucket() converts
229	an allocation size to the bucket index that should be looked at. The buckets are grouped as follows:
230
231	Bucket 0: [8, 15], range size=8
232	Bucket 1: [16, 23], range size=8
233	Bucket 2: [24, 31], range size=8
234	Bucket 3: [32, 39], range size=8
235	Bucket 4: [40, 47], range size=8
236	Bucket 5: [48, 55], range size=8
237	Bucket 6: [56, 63], range size=8
238	Bucket 7: [64, 71], range size=8
239	Bucket 8: [72, 79], range size=8
240	Bucket 9: [80, 87], range size=8
241	Bucket 10: [88, 95], range size=8
242	Bucket 11: [96, 103], range size=8
243	Bucket 12: [104, 111], range size=8
244	Bucket 13: [112, 119], range size=8
245	Bucket 14: [120, 159], range size=40
246	Bucket 15: [160, 191], range size=32
247	Bucket 16: [192, 223], range size=32
248	Bucket 17: [224, 255], range size=32
249	Bucket 18: [256, 319], range size=64
250	Bucket 19: [320, 383], range size=64
251	Bucket 20: [384, 447], range size=64
252	Bucket 21: [448, 511], range size=64
253	Bucket 22: [512, 639], range size=128
254	Bucket 23: [640, 767], range size=128
255	Bucket 24: [768, 895], range size=128
256	Bucket 25: [896, 1023], range size=128
257	Bucket 26: [1024, 1279], range size=256
258	Bucket 27: [1280, 1535], range size=256
259	Bucket 28: [1536, 1791], range size=256
260	Bucket 29: [1792, 2047], range size=256
261	Bucket 30: [2048, 2559], range size=512
262	Bucket 31: [2560, 3071], range size=512
263	Bucket 32: [3072, 3583], range size=512
264	Bucket 33: [3584, 6143], range size=2560
265	Bucket 34: [6144, 8191], range size=2048
266	Bucket 35: [8192, 12287], range size=4096
267	Bucket 36: [12288, 16383], range size=4096
268	Bucket 37: [16384, 24575], range size=8192
269	Bucket 38: [24576, 32767], range size=8192
270	Bucket 39: [32768, 49151], range size=16384
271	Bucket 40: [49152, 65535], range size=16384
272	Bucket 41: [65536, 98303], range size=32768
273	Bucket 42: [98304, 131071], range size=32768
274	Bucket 43: [131072, 196607], range size=65536
275	Bucket 44: [196608, 262143], range size=65536
276	Bucket 45: [262144, 393215], range size=131072
277	Bucket 46: [393216, 524287], range size=131072
278	Bucket 47: [524288, 786431], range size=262144
279	Bucket 48: [786432, 1048575], range size=262144
280	Bucket 49: [1048576, 1572863], range size=524288
281	Bucket 50: [1572864, 2097151], range size=524288
282	Bucket 51: [2097152, 3145727], range size=1048576
283	Bucket 52: [3145728, 4194303], range size=1048576
284	Bucket 53: [4194304, 6291455], range size=2097152
285	Bucket 54: [6291456, 8388607], range size=2097152
286	Bucket 55: [8388608, 12582911], range size=4194304
287	Bucket 56: [12582912, 16777215], range size=4194304
288	Bucket 57: [16777216, 25165823], range size=8388608
289	Bucket 58: [25165824, 33554431], range size=8388608
290	Bucket 59: [33554432, 50331647], range size=16777216
291	Bucket 60: [50331648, 67108863], range size=16777216
292	Bucket 61: [67108864, 100663295], range size=33554432
293	Bucket 62: [100663296, 134217727], range size=33554432
294	Bucket 63: 134217728 bytes and larger. */
295	static_assert(NUM_FREE_BUCKETS == 64, "Following function is tailored specifically for NUM_FREE_BUCKETS == 64 case");
296	static int compute_free_list_bucket(size_t allocSize)
297	{
298	if (allocSize < 128) return (allocSize >> 3) - 1;
299	int clz = __builtin_clz(allocSize);
300	int bucketIndex = (clz > 19) ? 110 - (clz<<2) + ((allocSize >> (29-clz)) ^ 4) : MIN(71 - (clz<<1) + ((allocSize >> (30-clz)) ^ 2), NUM_FREE_BUCKETS-1);
301	assert(bucketIndex >= 0);
302	assert(bucketIndex < NUM_FREE_BUCKETS);
303	return bucketIndex;
304	}
305
306	#define DECODE_CEILING_SIZE(size) ((size_t)((size) & ~FREE_REGION_FLAG))
307
308	static Region prev_region(Region region)
309	{
310	size_t prevRegionSize = ((size_t*)region)[-1];
311	prevRegionSize = DECODE_CEILING_SIZE(prevRegionSize);
312	return (Region)((uint8_t)region - prevRegionSize);
313	}
314
315	static Region next_region(Region region)
316	{
317	return (Region)((uint8_t)region + region->size);
318	}
319
320	static size_t region_ceiling_size(Region *region)
321	{
322	return ((size_t)((uint8_t)region + region->size))[-1];
323	}
324
325	static bool region_is_free(Region *r)
326	{
327	return region_ceiling_size(r) & FREE_REGION_FLAG;
328	}
329
330	static bool region_is_in_use(Region *r)
331	{
332	return r->size == region_ceiling_size(r);
333	}
334
335	static size_t size_of_region_from_ceiling(Region *r)
336	{
337	size_t size = region_ceiling_size(r);
338	return DECODE_CEILING_SIZE(size);
339	}
340
341	static bool debug_region_is_consistent(Region *r)
342	{
343	assert(r);
344	size_t sizeAtBottom = r->size;
345	size_t sizeAtCeiling = size_of_region_from_ceiling(r);
346	return sizeAtBottom == sizeAtCeiling;
347	}
348
349	static uint8_t region_payload_start_ptr(Region region)
350	{
351	return (uint8_t*)region + sizeof(size_t);
352	}
353
354	static uint8_t region_payload_end_ptr(Region region)
355	{
356	return (uint8_t*)region + region->size - sizeof(size_t);
357	}
358
359	static void create_used_region(void *ptr, size_t size)
360	{
361	assert(ptr);
362	assert(HAS_ALIGNMENT(ptr, sizeof(size_t)));
363	assert(HAS_ALIGNMENT(size, sizeof(size_t)));
364	assert(size >= sizeof(Region));
365	(size_t)ptr = size;
366	((size_t*)ptr)[(size/sizeof(size_t))-1] = size;
367	}
368
369	static void create_free_region(void *ptr, size_t size)
370	{
371	assert(ptr);
372	assert(HAS_ALIGNMENT(ptr, sizeof(size_t)));
373	assert(HAS_ALIGNMENT(size, sizeof(size_t)));
374	assert(size >= sizeof(Region));
375	Region freeRegion = (Region)ptr;
376	freeRegion->size = size;
377	((size_t*)ptr)[(size/sizeof(size_t))-1] = size \| FREE_REGION_FLAG;
378	}
379
380	static void prepend_to_free_list(Region region, Region prependTo)
381	{
382	assert(region);
383	assert(prependTo);
384	// N.b. the region we are prepending to is always the sentinel node,
385	// which represents a dummy node that is technically not a free node, so
386	// region_is_free(prependTo) does not hold.
387	assert(region_is_free((Region*)region));
388	region->next = prependTo;
389	region->prev = prependTo->prev;
390	assert(region->prev);
391	prependTo->prev = region;
392	region->prev->next = region;
393	}
394
395	static void unlink_from_free_list(Region *region)
396	{
397	assert(region);
398	assert(region_is_free((Region*)region));
399	assert(region->prev);
400	assert(region->next);
401	region->prev->next = region->next;
402	region->next->prev = region->prev;
403	}
404
405	static void link_to_free_list(Region *freeRegion)
406	{
407	assert(freeRegion);
408	assert(freeRegion->size >= sizeof(Region));
409	int bucketIndex = compute_free_list_bucket(freeRegion->size-REGION_HEADER_SIZE);
410	Region *freeListHead = freeRegionBuckets + bucketIndex;
411	freeRegion->prev = freeListHead;
412	freeRegion->next = freeListHead->next;
413	assert(freeRegion->next);
414	freeListHead->next = freeRegion;
415	freeRegion->next->prev = freeRegion;
416	freeRegionBucketsUsed \|= ((BUCKET_BITMASK_T)1) << bucketIndex;
417	}
418
419	#if 0
420	static void dump_memory_regions()
421	{
422	ASSERT_MALLOC_IS_ACQUIRED();
423	RootRegion *root = listOfAllRegions;
424	MAIN_THREAD_ASYNC_EM_ASM(console.log('All memory regions:'));
425	while(root)
426	{
427	Region r = (Region)root;
428	assert(debug_region_is_consistent(r));
429	uint8_t *lastRegionEnd = root->endPtr;
430	MAIN_THREAD_ASYNC_EM_ASM(console.log('Region block 0x'+($0>>>0).toString(16)+' - 0x'+($1>>>0).toString(16)+ ' ('+($2>>>0)+' bytes):'),
431	r, lastRegionEnd, lastRegionEnd-(uint8_t*)r);
432	while((uint8_t*)r < lastRegionEnd)
433	{
434	MAIN_THREAD_ASYNC_EM_ASM(console.log('Region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+')'),
435	r, r->size, region_ceiling_size(r) == r->size);
436
437	assert(debug_region_is_consistent(r));
438	size_t sizeFromCeiling = size_of_region_from_ceiling(r);
439	if (sizeFromCeiling != r->size)
440	MAIN_THREAD_ASYNC_EM_ASM(console.log('Corrupt region! Size marker at the end of the region does not match: '+($0>>>0)), sizeFromCeiling);
441	if (r->size == 0)
442	break;
443	r = next_region(r);
444	}
445	root = root->next;
446	MAIN_THREAD_ASYNC_EM_ASM(console.log(""));
447	}
448	MAIN_THREAD_ASYNC_EM_ASM(console.log('Free regions:'));
449	for(int i = 0; i < NUM_FREE_BUCKETS; ++i)
450	{
451	Region *prev = &freeRegionBuckets[i];
452	Region *fr = freeRegionBuckets[i].next;
453	while(fr != &freeRegionBuckets[i])
454	{
455	MAIN_THREAD_ASYNC_EM_ASM(console.log('In bucket '+$0+', free region 0x'+($1>>>0).toString(16)+', size: ' + ($2>>>0) + ' (size at ceiling: '+($3>>>0)+'), prev: 0x' + ($4>>>0).toString(16) + ', next: 0x' + ($5>>>0).toString(16)),
456	i, fr, fr->size, size_of_region_from_ceiling(fr), fr->prev, fr->next);
457	assert(debug_region_is_consistent(fr));
458	assert(region_is_free(fr));
459	assert(fr->prev == prev);
460	prev = fr;
461	assert(fr->next != fr);
462	assert(fr->prev != fr);
463	fr = fr->next;
464	}
465	}
466	MAIN_THREAD_ASYNC_EM_ASM(console.log('Free bucket index map: ' + ($0>>>0).toString(2) + ' ' + ($1>>>0).toString(2)), (uint32_t)(freeRegionBucketsUsed >> 32), (uint32_t)freeRegionBucketsUsed);
467	MAIN_THREAD_ASYNC_EM_ASM(console.log(""));
468	}
469
470	void emmalloc_dump_memory_regions()
471	{
472	MALLOC_ACQUIRE();
473	dump_memory_regions();
474	MALLOC_RELEASE();
475	}
476
477	static int validate_memory_regions()
478	{
479	ASSERT_MALLOC_IS_ACQUIRED();
480	RootRegion *root = listOfAllRegions;
481	while(root)
482	{
483	Region r = (Region)root;
484	if (!debug_region_is_consistent(r))
485	{
486	MAIN_THREAD_ASYNC_EM_ASM(console.error('Used region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+') is corrupt (size markers in the beginning and at the end of the region do not match!)'),
487	r, r->size, region_ceiling_size(r) == r->size);
488	return 1;
489	}
490	uint8_t *lastRegionEnd = root->endPtr;
491	while((uint8_t*)r < lastRegionEnd)
492	{
493	if (!debug_region_is_consistent(r))
494	{
495	MAIN_THREAD_ASYNC_EM_ASM(console.error('Used region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+') is corrupt (size markers in the beginning and at the end of the region do not match!)'),
496	r, r->size, region_ceiling_size(r) == r->size);
497	return 1;
498	}
499	if (r->size == 0)
500	break;
501	r = next_region(r);
502	}
503	root = root->next;
504	}
505	for(int i = 0; i < NUM_FREE_BUCKETS; ++i)
506	{
507	Region *prev = &freeRegionBuckets[i];
508	Region *fr = freeRegionBuckets[i].next;
509	while(fr != &freeRegionBuckets[i])
510	{
511	if (!debug_region_is_consistent(fr) \|\| !region_is_free(fr) \|\| fr->prev != prev \|\| fr->next == fr \|\| fr->prev == fr)
512	{
513	MAIN_THREAD_ASYNC_EM_ASM(console.log('In bucket '+$0+', free region 0x'+($1>>>0).toString(16)+', size: ' + ($2>>>0) + ' (size at ceiling: '+($3>>>0)+'), prev: 0x' + ($4>>>0).toString(16) + ', next: 0x' + ($5>>>0).toString(16) + ' is corrupt!'),
514	i, fr, fr->size, size_of_region_from_ceiling(fr), fr->prev, fr->next);
515	return 1;
516	}
517	prev = fr;
518	fr = fr->next;
519	}
520	}
521	return 0;
522	}
523
524	int emmalloc_validate_memory_regions()
525	{
526	MALLOC_ACQUIRE();
527	int memoryError = validate_memory_regions();
528	MALLOC_RELEASE();
529	return memoryError;
530	}
531	#endif
532
533	static bool claim_more_memory(size_t numBytes)
534	{
535	#ifdef EMMALLOC_VERBOSE
536	MAIN_THREAD_ASYNC_EM_ASM(console.log('claim_more_memory(numBytes='+($0>>>0)+ ')'), numBytes);
537	#endif
538
539	#ifdef EMMALLOC_MEMVALIDATE
540	validate_memory_regions();
541	#endif
542
543	uint8_t *startPtr;
544	uint8_t *endPtr;
545	do {
546	// If this is the first time we're called, see if we can use
547	// the initial heap memory set up by wasm-ld.
548	if (!listOfAllRegions) {
549	unsigned char *heap_base = &__heap_base;
550	unsigned char *heap_end = &__heap_end;
551	if (heap_end < heap_base) {
552	__builtin_trap();
553	}
554	if (numBytes <= (size_t)(heap_end - heap_base)) {
555	startPtr = heap_base;
556	endPtr = heap_end;
557	break;
558	}
559	}
560
561	// Round numBytes up to the nearest page size.
562	numBytes = (numBytes + (PAGE_SIZE-1)) & -PAGE_SIZE;
563
564	// Claim memory via sbrk
565	startPtr = (uint8_t*)sbrk(numBytes);
566	if ((intptr_t)startPtr == -1)
567	{
568	#ifdef EMMALLOC_VERBOSE
569	MAIN_THREAD_ASYNC_EM_ASM(console.error('claim_more_memory: sbrk failed!'));
570	#endif
571	return false;
572	}
573	#ifdef EMMALLOC_VERBOSE
574	MAIN_THREAD_ASYNC_EM_ASM(console.log('claim_more_memory: claimed 0x' + ($0>>>0).toString(16) + ' - 0x' + ($1>>>0).toString(16) + ' (' + ($2>>>0) + ' bytes) via sbrk()'), startPtr, startPtr + numBytes, numBytes);
575	#endif
576	assert(HAS_ALIGNMENT(startPtr, alignof(size_t)));
577	endPtr = startPtr + numBytes;
578	} while (0);
579
580	// Create a sentinel region at the end of the new heap block
581	Region endSentinelRegion = (Region)(endPtr - sizeof(Region));
582	create_used_region(endSentinelRegion, sizeof(Region));
583
584	// If we are the sole user of sbrk(), it will feed us continuous/consecutive memory addresses - take advantage
585	// of that if so: instead of creating two disjoint memory regions blocks, expand the previous one to a larger size.
586	uint8_t *previousSbrkEndAddress = listOfAllRegions ? listOfAllRegions->endPtr : 0;
587	if (startPtr == previousSbrkEndAddress)
588	{
589	Region prevEndSentinel = prev_region((Region)startPtr);
590	assert(debug_region_is_consistent(prevEndSentinel));
591	assert(region_is_in_use(prevEndSentinel));
592	Region *prevRegion = prev_region(prevEndSentinel);
593	assert(debug_region_is_consistent(prevRegion));
594
595	listOfAllRegions->endPtr = endPtr;
596
597	// Two scenarios, either the last region of the previous block was in use, in which case we need to create
598	// a new free region in the newly allocated space; or it was free, in which case we can extend that region
599	// to cover a larger size.
600	if (region_is_free(prevRegion))
601	{
602	size_t newFreeRegionSize = (uint8_t)endSentinelRegion - (uint8_t)prevRegion;
603	unlink_from_free_list(prevRegion);
604	create_free_region(prevRegion, newFreeRegionSize);
605	link_to_free_list(prevRegion);
606	return true;
607	}
608	// else: last region of the previous block was in use. Since we are joining two consecutive sbrk() blocks,
609	// we can swallow the end sentinel of the previous block away.
610	startPtr -= sizeof(Region);
611	}
612	else
613	{
614	// Create a root region at the start of the heap block
615	create_used_region(startPtr, sizeof(Region));
616
617	// Dynamic heap start region:
618	RootRegion newRegionBlock = (RootRegion)startPtr;
619	newRegionBlock->next = listOfAllRegions; // Pointer to next region block head
620	newRegionBlock->endPtr = endPtr; // Pointer to the end address of this region block
621	listOfAllRegions = newRegionBlock;
622	startPtr += sizeof(Region);
623	}
624
625	// Create a new memory region for the new claimed free space.
626	create_free_region(startPtr, (uint8_t*)endSentinelRegion - startPtr);
627	link_to_free_list((Region*)startPtr);
628	return true;
629	}
630
631	#if 0
632	// Initialize emmalloc during static initialization.
633	// See system/lib/README.md for static constructor ordering.
634	__attribute__((constructor(47)))
635	static void initialize_emmalloc_heap()
636	{
637	// Initialize circular doubly linked lists representing free space
638	// Never useful to unroll this for loop, just takes up code size.
639	#pragma clang loop unroll(disable)
640	for(int i = 0; i < NUM_FREE_BUCKETS; ++i)
641	freeRegionBuckets[i].prev = freeRegionBuckets[i].next = &freeRegionBuckets[i];
642
643	#ifdef EMMALLOC_VERBOSE
644	MAIN_THREAD_ASYNC_EM_ASM(console.log('initialize_emmalloc_heap()'));
645	#endif
646
647	// Start with a tiny dynamic region.
648	claim_more_memory(3*sizeof(Region));
649	}
650
651	void emmalloc_blank_slate_from_orbit()
652	{
653	MALLOC_ACQUIRE();
654	listOfAllRegions = NULL;
655	freeRegionBucketsUsed = 0;
656	initialize_emmalloc_heap();
657	MALLOC_RELEASE();
658	}
659	#endif
660
661	static void attempt_allocate(Region freeRegion, size_t alignment, size_t size)
662	{
663	ASSERT_MALLOC_IS_ACQUIRED();
664	assert(freeRegion);
665	// Look at the next potential free region to allocate into.
666	// First, we should check if the free region has enough of payload bytes contained
667	// in it to accommodate the new allocation. This check needs to take account the
668	// requested allocation alignment, so the payload memory area needs to be rounded
669	// upwards to the desired alignment.
670	uint8_t *payloadStartPtr = region_payload_start_ptr(freeRegion);
671	uint8_t *payloadStartPtrAligned = ALIGN_UP(payloadStartPtr, alignment);
672	uint8_t *payloadEndPtr = region_payload_end_ptr(freeRegion);
673
674	// Do we have enough free space, taking into account alignment?
675	if (payloadStartPtrAligned + size > payloadEndPtr)
676	return NULL;
677
678	// We have enough free space, so the memory allocation will be made into this region. Remove this free region
679	// from the list of free regions: whatever slop remains will be later added back to the free region pool.
680	unlink_from_free_list(freeRegion);
681
682	// Before we proceed further, fix up the boundary of this region and the region that precedes this one,
683	// so that the boundary between the two regions happens at a right spot for the payload to be aligned.
684	if (payloadStartPtr != payloadStartPtrAligned)
685	{
686	Region prevRegion = prev_region((Region)freeRegion);
687	// We never have two free regions adjacent to each other, so the region before this free
688	// region should be in use.
689	assert(region_is_in_use(prevRegion));
690	size_t regionBoundaryBumpAmount = payloadStartPtrAligned - payloadStartPtr;
691	size_t newThisRegionSize = freeRegion->size - regionBoundaryBumpAmount;
692	create_used_region(prevRegion, prevRegion->size + regionBoundaryBumpAmount);
693	freeRegion = (Region )((uint8_t)freeRegion + regionBoundaryBumpAmount);
694	freeRegion->size = newThisRegionSize;
695	}
696	// Next, we need to decide whether this region is so large that it should be split into two regions,
697	// one representing the newly used memory area, and at the high end a remaining leftover free area.
698	// This splitting to two is done always if there is enough space for the high end to fit a region.
699	// Carve 'size' bytes of payload off this region. So,
700	// [sz prev next sz]
701	// becomes
702	// [sz payload sz] [sz prev next sz]
703	if (sizeof(Region) + REGION_HEADER_SIZE + size <= freeRegion->size)
704	{
705	// There is enough space to keep a free region at the end of the carved out block
706	// -> construct the new block
707	Region newFreeRegion = (Region )((uint8_t*)freeRegion + REGION_HEADER_SIZE + size);
708	create_free_region(newFreeRegion, freeRegion->size - size - REGION_HEADER_SIZE);
709	link_to_free_list(newFreeRegion);
710
711	// Recreate the resized Region under its new size.
712	create_used_region(freeRegion, size + REGION_HEADER_SIZE);
713	}
714	else
715	{
716	// There is not enough space to split the free memory region into used+free parts, so consume the whole
717	// region as used memory, not leaving a free memory region behind.
718	// Initialize the free region as used by resetting the ceiling size to the same value as the size at bottom.
719	((size_t)((uint8_t)freeRegion + freeRegion->size))[-1] = freeRegion->size;
720	}
721
722	#ifdef __EMSCRIPTEN_TRACING__
723	emscripten_trace_record_allocation(freeRegion, freeRegion->size);
724	#endif
725
726	#ifdef EMMALLOC_VERBOSE
727	MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_allocate - succeeded allocating memory, region ptr=0x' + ($0>>>0).toString(16) + ', align=' + $1 + ', payload size=' + ($2>>>0) + ' bytes)'), freeRegion, alignment, size);
728	#endif
729
730	return (uint8_t*)freeRegion + sizeof(size_t);
731	}
732
733	static size_t validate_alloc_alignment(size_t alignment)
734	{
735	// Cannot perform allocations that are less than 4 byte aligned, because the Region
736	// control structures need to be aligned. Also round up to minimum outputted alignment.
737	alignment = MAX(alignment, MALLOC_ALIGNMENT);
738	// Arbitrary upper limit on alignment - very likely a programming bug if alignment is higher than this.
739	assert(alignment <= 1024*1024);
740	return alignment;
741	}
742
743	static size_t validate_alloc_size(size_t size)
744	{
745	assert(size + REGION_HEADER_SIZE > size);
746
747	// Allocation sizes must be a multiple of pointer sizes, and at least 2*sizeof(pointer).
748	size_t validatedSize = size > SMALLEST_ALLOCATION_SIZE ? (size_t)ALIGN_UP(size, sizeof(Region*)) : SMALLEST_ALLOCATION_SIZE;
749	assert(validatedSize >= size); // 32-bit wraparound should not occur, too large sizes should be stopped before
750
751	return validatedSize;
752	}
753
754	static void *allocate_memory(size_t alignment, size_t size)
755	{
756	ASSERT_MALLOC_IS_ACQUIRED();
757
758	#ifdef EMMALLOC_VERBOSE
759	MAIN_THREAD_ASYNC_EM_ASM(console.log('allocate_memory(align=' + $0 + ', size=' + ($1>>>0) + ' bytes)'), alignment, size);
760	#endif
761
762	#ifdef EMMALLOC_MEMVALIDATE
763	validate_memory_regions();
764	#endif
765
766	if (!IS_POWER_OF_2(alignment))
767	{
768	#ifdef EMMALLOC_VERBOSE
769	MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: alignment not power of 2!'));
770	#endif
771	return 0;
772	}
773
774	if (size > MAX_ALLOC_SIZE)
775	{
776	#ifdef EMMALLOC_VERBOSE
777	MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size);
778	#endif
779	return 0;
780	}
781
782	alignment = validate_alloc_alignment(alignment);
783	size = validate_alloc_size(size);
784
785	// Attempt to allocate memory starting from smallest bucket that can contain the required amount of memory.
786	// Under normal alignment conditions this should always be the first or second bucket we look at, but if
787	// performing an allocation with complex alignment, we may need to look at multiple buckets.
788	int bucketIndex = compute_free_list_bucket(size);
789	BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed >> bucketIndex;
790
791	// Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap.
792	while(bucketMask)
793	{
794	BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask);
795	bucketIndex += indexAdd;
796	bucketMask >>= indexAdd;
797	assert(bucketIndex >= 0);
798	assert(bucketIndex <= NUM_FREE_BUCKETS-1);
799	assert(freeRegionBucketsUsed & (((BUCKET_BITMASK_T)1) << bucketIndex));
800
801	Region *freeRegion = freeRegionBuckets[bucketIndex].next;
802	assert(freeRegion);
803	if (freeRegion != &freeRegionBuckets[bucketIndex])
804	{
805	void *ptr = attempt_allocate(freeRegion, alignment, size);
806	if (ptr)
807	return ptr;
808
809	// We were not able to allocate from the first region found in this bucket, so penalize
810	// the region by cycling it to the end of the doubly circular linked list. (constant time)
811	// This provides a randomized guarantee that when performing allocations of size k to a
812	// bucket of [k-something, k+something] range, we will not always attempt to satisfy the
813	// allocation from the same available region at the front of the list, but we try each
814	// region in turn.
815	unlink_from_free_list(freeRegion);
816	prepend_to_free_list(freeRegion, &freeRegionBuckets[bucketIndex]);
817	// But do not stick around to attempt to look at other regions in this bucket - move
818	// to search the next populated bucket index if this did not fit. This gives a practical
819	// "allocation in constant time" guarantee, since the next higher bucket will only have
820	// regions that are all of strictly larger size than the requested allocation. Only if
821	// there is a difficult alignment requirement we may fail to perform the allocation from
822	// a region in the next bucket, and if so, we keep trying higher buckets until one of them
823	// works.
824	++bucketIndex;
825	bucketMask >>= 1;
826	}
827	else
828	{
829	// This bucket was not populated after all with any regions,
830	// but we just had a stale bit set to mark a populated bucket.
831	// Reset the bit to update latest status so that we do not
832	// redundantly look at this bucket again.
833	freeRegionBucketsUsed &= ~(((BUCKET_BITMASK_T)1) << bucketIndex);
834	bucketMask ^= 1;
835	}
836	// Instead of recomputing bucketMask from scratch at the end of each loop, it is updated as we go,
837	// to avoid undefined behavior with (x >> 32)/(x >> 64) when bucketIndex reaches 32/64, (the shift would comes out as a no-op instead of 0).
838
839	assert((bucketIndex == NUM_FREE_BUCKETS && bucketMask == 0) \|\| (bucketMask == freeRegionBucketsUsed >> bucketIndex));
840	}
841
842	// None of the buckets were able to accommodate an allocation. If this happens we are almost out of memory.
843	// The largest bucket might contain some suitable regions, but we only looked at one region in that bucket, so
844	// as a last resort, loop through more free regions in the bucket that represents the largest allocations available.
845	// But only if the bucket representing largest allocations available is not any of the first thirty buckets,
846	// these represent allocatable areas less than <1024 bytes - which could be a lot of scrap.
847	// In such case, prefer to sbrk() in more memory right away.
848	int largestBucketIndex = NUM_FREE_BUCKETS - 1 - __builtin_clzll(freeRegionBucketsUsed);
849	// freeRegion will be null if there is absolutely no memory left. (all buckets are 100% used)
850	Region *freeRegion = freeRegionBucketsUsed ? freeRegionBuckets[largestBucketIndex].next : 0;
851	if (freeRegionBucketsUsed >> 30)
852	{
853	// Look only at a constant number of regions in this bucket max, to avoid bad worst case behavior.
854	// If this many regions cannot find free space, we give up and prefer to sbrk() more instead.
855	const int maxRegionsToTryBeforeGivingUp = 99;
856	int numTriesLeft = maxRegionsToTryBeforeGivingUp;
857	while(freeRegion != &freeRegionBuckets[largestBucketIndex] && numTriesLeft-- > 0)
858	{
859	void *ptr = attempt_allocate(freeRegion, alignment, size);
860	if (ptr)
861	return ptr;
862	freeRegion = freeRegion->next;
863	}
864	}
865
866	// We were unable to find a free memory region. Must sbrk() in more memory!
867	size_t numBytesToClaim = size+sizeof(Region)*3;
868	assert(numBytesToClaim > size); // 32-bit wraparound should not happen here, allocation size has been validated above!
869	bool success = claim_more_memory(numBytesToClaim);
870	if (success)
871	return allocate_memory(alignment, size); // Recurse back to itself to try again
872
873	// also sbrk() failed, we are really really constrained :( As a last resort, go back to looking at the
874	// bucket we already looked at above, continuing where the above search left off - perhaps there are
875	// regions we overlooked the first time that might be able to satisfy the allocation.
876	if (freeRegion)
877	{
878	while(freeRegion != &freeRegionBuckets[largestBucketIndex])
879	{
880	void *ptr = attempt_allocate(freeRegion, alignment, size);
881	if (ptr)
882	return ptr;
883	freeRegion = freeRegion->next;
884	}
885	}
886
887	#ifdef EMMALLOC_VERBOSE
888	MAIN_THREAD_ASYNC_EM_ASM(console.log('Could not find a free memory block!'));
889	#endif
890
891	return 0;
892	}
893
894	static
895	void *emmalloc_memalign(size_t alignment, size_t size)
896	{
897	MALLOC_ACQUIRE();
898	void *ptr = allocate_memory(alignment, size);
899	MALLOC_RELEASE();
900	return ptr;
901	}
902
903	#if 0
904	void * EMMALLOC_EXPORT memalign(size_t alignment, size_t size)
905	{
906	return emmalloc_memalign(alignment, size);
907	}
908	#endif
909
910	void * EMMALLOC_EXPORT aligned_alloc(size_t alignment, size_t size)
911	{
912	if ((alignment % sizeof(void *) != 0) \|\| (size % alignment) != 0)
913	return 0;
914	return emmalloc_memalign(alignment, size);
915	}
916
917	static
918	void *emmalloc_malloc(size_t size)
919	{
920	return emmalloc_memalign(MALLOC_ALIGNMENT, size);
921	}
922
923	void * EMMALLOC_EXPORT malloc(size_t size)
924	{
925	return emmalloc_malloc(size);
926	}
927
928	static
929	size_t emmalloc_usable_size(void *ptr)
930	{
931	if (!ptr)
932	return 0;
933
934	uint8_t regionStartPtr = (uint8_t)ptr - sizeof(size_t);
935	Region region = (Region)(regionStartPtr);
936	assert(HAS_ALIGNMENT(region, sizeof(size_t)));
937
938	MALLOC_ACQUIRE();
939
940	size_t size = region->size;
941	assert(size >= sizeof(Region));
942	assert(region_is_in_use(region));
943
944	MALLOC_RELEASE();
945
946	return size - REGION_HEADER_SIZE;
947	}
948
949	size_t EMMALLOC_EXPORT malloc_usable_size(void *ptr)
950	{
951	return emmalloc_usable_size(ptr);
952	}
953
954	static
955	void emmalloc_free(void *ptr)
956	{
957	#ifdef EMMALLOC_MEMVALIDATE
958	emmalloc_validate_memory_regions();
959	#endif
960
961	if (!ptr)
962	return;
963
964	#ifdef EMMALLOC_VERBOSE
965	MAIN_THREAD_ASYNC_EM_ASM(console.log('free(ptr=0x'+($0>>>0).toString(16)+')'), ptr);
966	#endif
967
968	uint8_t regionStartPtr = (uint8_t)ptr - sizeof(size_t);
969	Region region = (Region)(regionStartPtr);
970	assert(HAS_ALIGNMENT(region, sizeof(size_t)));
971
972	MALLOC_ACQUIRE();
973
974	size_t size = region->size;
975	#ifdef EMMALLOC_VERBOSE
976	if (size < sizeof(Region) \|\| !region_is_in_use(region))
977	{
978	if (debug_region_is_consistent(region))
979	// LLVM wasm backend bug: cannot use MAIN_THREAD_ASYNC_EM_ASM() here, that generates internal compiler error
980	// Reproducible by running e.g. other.test_alloc_3GB
981	EM_ASM(console.error('Double free at region ptr 0x' + ($0>>>0).toString(16) + ', region->size: 0x' + ($1>>>0).toString(16) + ', region->sizeAtCeiling: 0x' + ($2>>>0).toString(16) + ')'), region, size, region_ceiling_size(region));
982	else
983	MAIN_THREAD_ASYNC_EM_ASM(console.error('Corrupt region at region ptr 0x' + ($0>>>0).toString(16) + ' region->size: 0x' + ($1>>>0).toString(16) + ', region->sizeAtCeiling: 0x' + ($2>>>0).toString(16) + ')'), region, size, region_ceiling_size(region));
984	}
985	#endif
986	assert(size >= sizeof(Region));
987	assert(region_is_in_use(region));
988
989	#ifdef __EMSCRIPTEN_TRACING__
990	emscripten_trace_record_free(region);
991	#endif
992
993	// Check merging with left side
994	size_t prevRegionSizeField = ((size_t*)region)[-1];
995	size_t prevRegionSize = prevRegionSizeField & ~FREE_REGION_FLAG;
996	if (prevRegionSizeField != prevRegionSize) // Previous region is free?
997	{
998	Region prevRegion = (Region)((uint8_t*)region - prevRegionSize);
999	assert(debug_region_is_consistent(prevRegion));
1000	unlink_from_free_list(prevRegion);
1001	regionStartPtr = (uint8_t*)prevRegion;
1002	size += prevRegionSize;
1003	}
1004
1005	// Check merging with right side
1006	Region *nextRegion = next_region(region);
1007	assert(debug_region_is_consistent(nextRegion));
1008	size_t sizeAtEnd = (size_t)region_payload_end_ptr(nextRegion);
1009	if (nextRegion->size != sizeAtEnd)
1010	{
1011	unlink_from_free_list(nextRegion);
1012	size += nextRegion->size;
1013	}
1014
1015	create_free_region(regionStartPtr, size);
1016	link_to_free_list((Region*)regionStartPtr);
1017
1018	MALLOC_RELEASE();
1019
1020	#ifdef EMMALLOC_MEMVALIDATE
1021	emmalloc_validate_memory_regions();
1022	#endif
1023	}
1024
1025	void EMMALLOC_EXPORT free(void *ptr)
1026	{
1027	emmalloc_free(ptr);
1028	}
1029
1030	// Can be called to attempt to increase or decrease the size of the given region
1031	// to a new size (in-place). Returns 1 if resize succeeds, and 0 on failure.
1032	static int attempt_region_resize(Region *region, size_t size)
1033	{
1034	ASSERT_MALLOC_IS_ACQUIRED();
1035	assert(size > 0);
1036	assert(HAS_ALIGNMENT(size, sizeof(size_t)));
1037
1038	#ifdef EMMALLOC_VERBOSE
1039	MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_region_resize(region=0x' + ($0>>>0).toString(16) + ', size=' + ($1>>>0) + ' bytes)'), region, size);
1040	#endif
1041
1042	// First attempt to resize this region, if the next region that follows this one
1043	// is a free region.
1044	Region *nextRegion = next_region(region);
1045	uint8_t nextRegionEndPtr = (uint8_t)nextRegion + nextRegion->size;
1046	size_t sizeAtCeiling = ((size_t*)nextRegionEndPtr)[-1];
1047	if (nextRegion->size != sizeAtCeiling) // Next region is free?
1048	{
1049	assert(region_is_free(nextRegion));
1050	uint8_t newNextRegionStartPtr = (uint8_t)region + size;
1051	assert(HAS_ALIGNMENT(newNextRegionStartPtr, sizeof(size_t)));
1052	// Next region does not shrink to too small size?
1053	if (newNextRegionStartPtr + sizeof(Region) <= nextRegionEndPtr)
1054	{
1055	unlink_from_free_list(nextRegion);
1056	create_free_region(newNextRegionStartPtr, nextRegionEndPtr - newNextRegionStartPtr);
1057	link_to_free_list((Region*)newNextRegionStartPtr);
1058	create_used_region(region, newNextRegionStartPtr - (uint8_t*)region);
1059	return 1;
1060	}
1061	// If we remove the next region altogether, allocation is satisfied?
1062	if (newNextRegionStartPtr <= nextRegionEndPtr)
1063	{
1064	unlink_from_free_list(nextRegion);
1065	create_used_region(region, region->size + nextRegion->size);
1066	return 1;
1067	}
1068	}
1069	else
1070	{
1071	// Next region is an used region - we cannot change its starting address. However if we are shrinking the
1072	// size of this region, we can create a new free region between this and the next used region.
1073	if (size + sizeof(Region) <= region->size)
1074	{
1075	size_t freeRegionSize = region->size - size;
1076	create_used_region(region, size);
1077	Region freeRegion = (Region )((uint8_t*)region + size);
1078	create_free_region(freeRegion, freeRegionSize);
1079	link_to_free_list(freeRegion);
1080	return 1;
1081	}
1082	else if (size <= region->size)
1083	{
1084	// Caller was asking to shrink the size, but due to not being able to fit a full Region in the shrunk
1085	// area, we cannot actually do anything. This occurs if the shrink amount is really small. In such case,
1086	// just call it success without doing any work.
1087	return 1;
1088	}
1089	}
1090	#ifdef EMMALLOC_VERBOSE
1091	MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_region_resize failed.'));
1092	#endif
1093	return 0;
1094	}
1095
1096	static int acquire_and_attempt_region_resize(Region *region, size_t size)
1097	{
1098	MALLOC_ACQUIRE();
1099	int success = attempt_region_resize(region, size);
1100	MALLOC_RELEASE();
1101	return success;
1102	}
1103
1104	static
1105	void emmalloc_aligned_realloc(void ptr, size_t alignment, size_t size)
1106	{
1107	#ifdef EMMALLOC_VERBOSE
1108	MAIN_THREAD_ASYNC_EM_ASM(console.log('aligned_realloc(ptr=0x' + ($0>>>0).toString(16) + ', alignment=' + $1 + ', size=' + ($2>>>0)), ptr, alignment, size);
1109	#endif
1110
1111	if (!ptr)
1112	return emmalloc_memalign(alignment, size);
1113
1114	if (size == 0)
1115	{
1116	free(ptr);
1117	return 0;
1118	}
1119
1120	if (size > MAX_ALLOC_SIZE)
1121	{
1122	#ifdef EMMALLOC_VERBOSE
1123	MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size);
1124	#endif
1125	return 0;
1126	}
1127
1128	assert(IS_POWER_OF_2(alignment));
1129	// aligned_realloc() cannot be used to ask to change the alignment of a pointer.
1130	assert(HAS_ALIGNMENT(ptr, alignment));
1131	size = validate_alloc_size(size);
1132
1133	// Calculate the region start address of the original allocation
1134	Region region = (Region)((uint8_t*)ptr - sizeof(size_t));
1135
1136	// First attempt to resize the given region to avoid having to copy memory around
1137	if (acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE))
1138	{
1139	#ifdef __EMSCRIPTEN_TRACING__
1140	emscripten_trace_record_reallocation(ptr, ptr, size);
1141	#endif
1142	return ptr;
1143	}
1144
1145	// If resize failed, we must allocate a new region, copy the data over, and then
1146	// free the old region.
1147	void *newptr = emmalloc_memalign(alignment, size);
1148	if (newptr)
1149	{
1150	memcpy(newptr, ptr, MIN(size, region->size - REGION_HEADER_SIZE));
1151	free(ptr);
1152	}
1153	// N.B. If there is not enough memory, the old memory block should not be freed and
1154	// null pointer is returned.
1155	return newptr;
1156	}
1157
1158	#if 0
1159	void * EMMALLOC_EXPORT aligned_realloc(void *ptr, size_t alignment, size_t size)
1160	{
1161	return emmalloc_aligned_realloc(ptr, alignment, size);
1162	}
1163	#endif
1164
1165	#if 0
1166	// realloc_try() is like realloc(), but only attempts to try to resize the existing memory
1167	// area. If resizing the existing memory area fails, then realloc_try() will return 0
1168	// (the original memory block is not freed or modified). If resizing succeeds, previous
1169	// memory contents will be valid up to min(old length, new length) bytes.
1170	void emmalloc_realloc_try(void ptr, size_t size)
1171	{
1172	if (!ptr)
1173	return 0;
1174
1175	if (size == 0)
1176	{
1177	free(ptr);
1178	return 0;
1179	}
1180
1181	if (size > MAX_ALLOC_SIZE)
1182	{
1183	#ifdef EMMALLOC_VERBOSE
1184	MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size);
1185	#endif
1186	return 0;
1187	}
1188
1189	size = validate_alloc_size(size);
1190
1191	// Calculate the region start address of the original allocation
1192	Region region = (Region)((uint8_t*)ptr - sizeof(size_t));
1193
1194	// Attempt to resize the given region to avoid having to copy memory around
1195	int success = acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE);
1196	#ifdef __EMSCRIPTEN_TRACING__
1197	if (success)
1198	emscripten_trace_record_reallocation(ptr, ptr, size);
1199	#endif
1200	return success ? ptr : 0;
1201	}
1202
1203	// emmalloc_aligned_realloc_uninitialized() is like aligned_realloc(), but old memory contents
1204	// will be undefined after reallocation. (old memory is not preserved in any case)
1205	void emmalloc_aligned_realloc_uninitialized(void ptr, size_t alignment, size_t size)
1206	{
1207	if (!ptr)
1208	return emmalloc_memalign(alignment, size);
1209
1210	if (size == 0)
1211	{
1212	free(ptr);
1213	return 0;
1214	}
1215
1216	if (size > MAX_ALLOC_SIZE)
1217	{
1218	#ifdef EMMALLOC_VERBOSE
1219	MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size);
1220	#endif
1221	return 0;
1222	}
1223
1224	size = validate_alloc_size(size);
1225
1226	// Calculate the region start address of the original allocation
1227	Region region = (Region)((uint8_t*)ptr - sizeof(size_t));
1228
1229	// First attempt to resize the given region to avoid having to copy memory around
1230	if (acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE))
1231	{
1232	#ifdef __EMSCRIPTEN_TRACING__
1233	emscripten_trace_record_reallocation(ptr, ptr, size);
1234	#endif
1235	return ptr;
1236	}
1237
1238	// If resize failed, drop the old region and allocate a new region. Memory is not
1239	// copied over
1240	free(ptr);
1241	return emmalloc_memalign(alignment, size);
1242	}
1243	#endif
1244
1245	static
1246	void emmalloc_realloc(void ptr, size_t size)
1247	{
1248	return emmalloc_aligned_realloc(ptr, MALLOC_ALIGNMENT, size);
1249	}
1250
1251	void * EMMALLOC_EXPORT realloc(void *ptr, size_t size)
1252	{
1253	return emmalloc_realloc(ptr, size);
1254	}
1255
1256	#if 0
1257	// realloc_uninitialized() is like realloc(), but old memory contents
1258	// will be undefined after reallocation. (old memory is not preserved in any case)
1259	void emmalloc_realloc_uninitialized(void ptr, size_t size)
1260	{
1261	return emmalloc_aligned_realloc_uninitialized(ptr, MALLOC_ALIGNMENT, size);
1262	}
1263	#endif
1264
1265	static
1266	int emmalloc_posix_memalign(void **memptr, size_t alignment, size_t size)
1267	{
1268	assert(memptr);
1269	if (alignment % sizeof(void *) != 0)
1270	return 22/* EINVAL*/;
1271	*memptr = emmalloc_memalign(alignment, size);
1272	return memptr ? 0 : 12/ENOMEM*/;
1273	}
1274
1275	int EMMALLOC_EXPORT posix_memalign(void **memptr, size_t alignment, size_t size)
1276	{
1277	return emmalloc_posix_memalign(memptr, alignment, size);
1278	}
1279
1280	static
1281	void *emmalloc_calloc(size_t num, size_t size)
1282	{
1283	size_t bytes = num*size;
1284	void *ptr = emmalloc_memalign(MALLOC_ALIGNMENT, bytes);
1285	if (ptr)
1286	memset(ptr, 0, bytes);
1287	return ptr;
1288	}
1289
1290	void * EMMALLOC_EXPORT calloc(size_t num, size_t size)
1291	{
1292	return emmalloc_calloc(num, size);
1293	}
1294
1295	#if 0
1296	static int count_linked_list_size(Region *list)
1297	{
1298	int size = 1;
1299	for(Region *i = list->next; i != list; list = list->next)
1300	++size;
1301	return size;
1302	}
1303
1304	static size_t count_linked_list_space(Region *list)
1305	{
1306	size_t space = 0;
1307	for(Region *i = list->next; i != list; list = list->next)
1308	space += region_payload_end_ptr(i) - region_payload_start_ptr(i);
1309	return space;
1310	}
1311
1312	struct mallinfo emmalloc_mallinfo()
1313	{
1314	MALLOC_ACQUIRE();
1315
1316	struct mallinfo info;
1317	// Non-mmapped space allocated (bytes): For emmalloc,
1318	// let's define this as the difference between heap size and dynamic top end.
1319	info.arena = emscripten_get_heap_size() - (size_t)sbrk(0);
1320	// Number of "ordinary" blocks. Let's define this as the number of highest
1321	// size blocks. (subtract one from each, since there is a sentinel node in each list)
1322	info.ordblks = count_linked_list_size(&freeRegionBuckets[NUM_FREE_BUCKETS-1])-1;
1323	// Number of free "fastbin" blocks. For emmalloc, define this as the number
1324	// of blocks that are not in the largest pristine block.
1325	info.smblks = 0;
1326	// The total number of bytes in free "fastbin" blocks.
1327	info.fsmblks = 0;
1328	for(int i = 0; i < NUM_FREE_BUCKETS-1; ++i)
1329	{
1330	info.smblks += count_linked_list_size(&freeRegionBuckets[i])-1;
1331	info.fsmblks += count_linked_list_space(&freeRegionBuckets[i]);
1332	}
1333
1334	info.hblks = 0; // Number of mmapped regions: always 0. (no mmap support)
1335	info.hblkhd = 0; // Amount of bytes in mmapped regions: always 0. (no mmap support)
1336
1337	// Walk through all the heap blocks to report the following data:
1338	// The "highwater mark" for allocated space—that is, the maximum amount of
1339	// space that was ever allocated. Emmalloc does not want to pay code to
1340	// track this, so this is only reported from current allocation data, and
1341	// may not be accurate.
1342	info.usmblks = 0;
1343	info.uordblks = 0; // The total number of bytes used by in-use allocations.
1344	info.fordblks = 0; // The total number of bytes in free blocks.
1345	// The total amount of releasable free space at the top of the heap.
1346	// This is the maximum number of bytes that could ideally be released by malloc_trim(3).
1347	Region lastActualRegion = prev_region((Region)(listOfAllRegions->endPtr - sizeof(Region)));
1348	info.keepcost = region_is_free(lastActualRegion) ? lastActualRegion->size : 0;
1349
1350	RootRegion *root = listOfAllRegions;
1351	while(root)
1352	{
1353	Region r = (Region)root;
1354	assert(debug_region_is_consistent(r));
1355	uint8_t *lastRegionEnd = root->endPtr;
1356	while((uint8_t*)r < lastRegionEnd)
1357	{
1358	assert(debug_region_is_consistent(r));
1359
1360	if (region_is_free(r))
1361	{
1362	// Count only the payload of the free block towards free memory.
1363	info.fordblks += region_payload_end_ptr(r) - region_payload_start_ptr(r);
1364	// But the header data of the free block goes towards used memory.
1365	info.uordblks += REGION_HEADER_SIZE;
1366	}
1367	else
1368	{
1369	info.uordblks += r->size;
1370	}
1371	// Update approximate watermark data
1372	info.usmblks = MAX(info.usmblks, (intptr_t)r + r->size);
1373
1374	if (r->size == 0)
1375	break;
1376	r = next_region(r);
1377	}
1378	root = root->next;
1379	}
1380
1381	MALLOC_RELEASE();
1382	return info;
1383	}
1384
1385	struct mallinfo EMMALLOC_EXPORT mallinfo()
1386	{
1387	return emmalloc_mallinfo();
1388	}
1389
1390	// Note! This function is not fully multithreadin safe: while this function is running, other threads should not be
1391	// allowed to call sbrk()!
1392	static int trim_dynamic_heap_reservation(size_t pad)
1393	{
1394	ASSERT_MALLOC_IS_ACQUIRED();
1395
1396	if (!listOfAllRegions)
1397	return 0; // emmalloc is not controlling any dynamic memory at all - cannot release memory.
1398	uint8_t *previousSbrkEndAddress = listOfAllRegions->endPtr;
1399	assert(sbrk(0) == previousSbrkEndAddress);
1400	size_t lastMemoryRegionSize = ((size_t*)previousSbrkEndAddress)[-1];
1401	assert(lastMemoryRegionSize == 16); // // The last memory region should be a sentinel node of exactly 16 bytes in size.
1402	Region endSentinelRegion = (Region)(previousSbrkEndAddress - sizeof(Region));
1403	Region *lastActualRegion = prev_region(endSentinelRegion);
1404
1405	// Round padding up to multiple of 4 bytes to keep sbrk() and memory region alignment intact.
1406	// Also have at least 8 bytes of payload so that we can form a full free region.
1407	size_t newRegionSize = (size_t)ALIGN_UP(pad, 4);
1408	if (pad > 0)
1409	newRegionSize += sizeof(Region) - (newRegionSize - pad);
1410
1411	if (!region_is_free(lastActualRegion) \|\| lastActualRegion->size <= newRegionSize)
1412	return 0; // Last actual region is in use, or caller desired to leave more free memory intact than there is.
1413
1414	// This many bytes will be shrunk away.
1415	size_t shrinkAmount = lastActualRegion->size - newRegionSize;
1416	assert(HAS_ALIGNMENT(shrinkAmount, 4));
1417
1418	unlink_from_free_list(lastActualRegion);
1419	// If pad == 0, we should delete the last free region altogether. If pad > 0,
1420	// shrink the last free region to the desired size.
1421	if (newRegionSize > 0)
1422	{
1423	create_free_region(lastActualRegion, newRegionSize);
1424	link_to_free_list(lastActualRegion);
1425	}
1426
1427	// Recreate the sentinel region at the end of the last free region
1428	endSentinelRegion = (Region)((uint8_t)lastActualRegion + newRegionSize);
1429	create_used_region(endSentinelRegion, sizeof(Region));
1430
1431	// And update the size field of the whole region block.
1432	listOfAllRegions->endPtr = (uint8_t*)endSentinelRegion + sizeof(Region);
1433
1434	// Finally call sbrk() to shrink the memory area.
1435	void *oldSbrk = sbrk(-(intptr_t)shrinkAmount);
1436	assert((intptr_t)oldSbrk != -1); // Shrinking with sbrk() should never fail.
1437	assert(oldSbrk == previousSbrkEndAddress); // Another thread should not have raced to increase sbrk() on us!
1438
1439	// All successful, and we actually trimmed memory!
1440	return 1;
1441	}
1442
1443	int emmalloc_trim(size_t pad)
1444	{
1445	MALLOC_ACQUIRE();
1446	int success = trim_dynamic_heap_reservation(pad);
1447	MALLOC_RELEASE();
1448	return success;
1449	}
1450
1451	int EMMALLOC_EXPORT malloc_trim(size_t pad)
1452	{
1453	return emmalloc_trim(pad);
1454	}
1455
1456	size_t emmalloc_dynamic_heap_size()
1457	{
1458	size_t dynamicHeapSize = 0;
1459
1460	MALLOC_ACQUIRE();
1461	RootRegion *root = listOfAllRegions;
1462	while(root)
1463	{
1464	dynamicHeapSize += root->endPtr - (uint8_t*)root;
1465	root = root->next;
1466	}
1467	MALLOC_RELEASE();
1468	return dynamicHeapSize;
1469	}
1470
1471	size_t emmalloc_free_dynamic_memory()
1472	{
1473	size_t freeDynamicMemory = 0;
1474
1475	int bucketIndex = 0;
1476
1477	MALLOC_ACQUIRE();
1478	BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed;
1479
1480	// Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap.
1481	while(bucketMask)
1482	{
1483	BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask);
1484	bucketIndex += indexAdd;
1485	bucketMask >>= indexAdd;
1486	for(Region *freeRegion = freeRegionBuckets[bucketIndex].next;
1487	freeRegion != &freeRegionBuckets[bucketIndex];
1488	freeRegion = freeRegion->next)
1489	{
1490	freeDynamicMemory += freeRegion->size - REGION_HEADER_SIZE;
1491	}
1492	++bucketIndex;
1493	bucketMask >>= 1;
1494	}
1495	MALLOC_RELEASE();
1496	return freeDynamicMemory;
1497	}
1498
1499	size_t emmalloc_compute_free_dynamic_memory_fragmentation_map(size_t freeMemorySizeMap[32])
1500	{
1501	memset((void)freeMemorySizeMap, 0, sizeof(freeMemorySizeMap[0])32);
1502
1503	size_t numFreeMemoryRegions = 0;
1504	int bucketIndex = 0;
1505	MALLOC_ACQUIRE();
1506	BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed;
1507
1508	// Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap.
1509	while(bucketMask)
1510	{
1511	BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask);
1512	bucketIndex += indexAdd;
1513	bucketMask >>= indexAdd;
1514	for(Region *freeRegion = freeRegionBuckets[bucketIndex].next;
1515	freeRegion != &freeRegionBuckets[bucketIndex];
1516	freeRegion = freeRegion->next)
1517	{
1518	++numFreeMemoryRegions;
1519	size_t freeDynamicMemory = freeRegion->size - REGION_HEADER_SIZE;
1520	if (freeDynamicMemory > 0)
1521	++freeMemorySizeMap[31-__builtin_clz(freeDynamicMemory)];
1522	else
1523	++freeMemorySizeMap[0];
1524	}
1525	++bucketIndex;
1526	bucketMask >>= 1;
1527	}
1528	MALLOC_RELEASE();
1529	return numFreeMemoryRegions;
1530	}
1531
1532	size_t emmalloc_unclaimed_heap_memory(void) {
1533	return emscripten_get_heap_max() - (size_t)sbrk(0);
1534	}
1535	#endif
1536
1537	// Define these to satisfy musl references.
1538	void *__libc_malloc(size_t) __attribute__((alias("malloc")));
1539	void __libc_free(void *) __attribute__((alias("free")));
1540	void *__libc_calloc(size_t nmemb, size_t size) __attribute__((alias("calloc")));

lib/libc/wasi/libc-top-half/musl/src/exit/atexit.c-5

 @@ -4,11 +4,6 @@
 #include "lock.h"
 #include "fork_impl.h"
-#define malloc __libc_malloc
-#define calloc __libc_calloc
-#define realloc undef
-#define free undef
 /* Ensure that at least 32 atexit handlers can be registered without malloc */
 #define COUNT 32

lib/libc/wasi/libc-top-half/musl/src/include/stdlib.h-6

...	@@ -10,10 +10,4 @@ hidden int __ptsname_r(int, char *, size_t);	...	@@ -10,10 +10,4 @@ hidden int __ptsname_r(int, char *, size_t);
10	hidden char __randname(char );	10	hidden char __randname(char );
11	hidden void __qsort_r (void , size_t, size_t, int ()(const void , const void , void ), void );	11	hidden void __qsort_r (void , size_t, size_t, int ()(const void , const void , void ), void );
12		12
13	hidden void *__libc_malloc(size_t);
14	hidden void *__libc_malloc_impl(size_t);
15	hidden void *__libc_calloc(size_t, size_t);
16	hidden void __libc_realloc(void , size_t);
17	hidden void __libc_free(void *);
18
19	#endif	13	#endif

lib/libc/wasi/libc-top-half/musl/src/locale/locale_map.c-5

 @@ -9,11 +9,6 @@
 #include "lock.h"
 #include "fork_impl.h"
-#define malloc __libc_malloc
-#define calloc undef
-#define realloc undef
-#define free undef
 const char *__lctrans_impl(const char *msg, const struct __locale_map *lm)
 {
 	const char *trans = 0;

lib/libc/wasi/libc-top-half/musl/src/locale/newlocale.c-5

 @@ -6,11 +6,6 @@
 #include "locale_impl.h"
 #include "lock.h"
-#define malloc __libc_malloc
-#define calloc undef
-#define realloc undef
-#define free undef
 static int default_locale_init_done;
 static struct __locale_struct default_locale, default_ctype_locale;

lib/libc/wasi/libc-top-half/musl/src/time/__tz.c-5

 @@ -11,11 +11,6 @@
 #include "lock.h"
 #include "fork_impl.h"
-#define malloc __libc_malloc
-#define calloc undef
-#define realloc undef
-#define free undef
 #ifdef __wasilibc_unmodified_upstream // timezone data
 long  __timezone = 0;
 int   __daylight = 0;

lib/std/heap.zig+15-14

 @@ -13,9 +13,9 @@ pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator;
 pub const SmpAllocator = @import("heap/SmpAllocator.zig");
 pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig");
 pub const PageAllocator = @import("heap/PageAllocator.zig");
-pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator;
 pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig");
-pub const WasmAllocator = @import("heap/WasmAllocator.zig");
+pub const WasmAllocator = if (builtin.single_threaded) BrkAllocator else @compileError("unimplemented");
+pub const BrkAllocator = @import("heap/BrkAllocator.zig");
 pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config;
 pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator;
 @@ -356,9 +356,6 @@ pub const page_allocator: Allocator = if (@hasDecl(root, "os") and
 else if (builtin.target.cpu.arch.isWasm()) .{
     .ptr = undefined,
     .vtable = &WasmAllocator.vtable,
-} else if (builtin.target.os.tag == .plan9) .{
-    .ptr = undefined,
-    .vtable = &SbrkAllocator(std.os.plan9.sbrk).vtable,
 } else .{
     .ptr = undefined,
     .vtable = &PageAllocator.vtable,
 @@ -369,16 +366,18 @@ pub const smp_allocator: Allocator = .{
     .vtable = &SmpAllocator.vtable,
 };
-/// This allocator is fast, small, and specific to WebAssembly. In the future,
+/// This allocator is fast, small, and specific to WebAssembly.
-/// this will be the implementation automatically selected by
-/// `GeneralPurposeAllocator` when compiling in `ReleaseSmall` mode for wasm32
-/// and wasm64 architectures.
-/// Until then, it is available here to play with.
 pub const wasm_allocator: Allocator = .{
     .ptr = undefined,
     .vtable = &WasmAllocator.vtable,
 };
+/// Supports single-threaded WebAssembly and Linux.
+pub const brk_allocator: Allocator = .{
+    .ptr = undefined,
+    .vtable = &BrkAllocator.vtable,
+};
 /// Returns a `StackFallbackAllocator` allocating using either a
 /// `FixedBufferAllocator` on an array of size `size` and falling back to
 /// `fallback_allocator` if that fails.
 @@ -1014,9 +1013,11 @@ test {
     _ = GeneralPurposeAllocator;
     _ = FixedBufferAllocator;
     _ = ThreadSafeAllocator;
-    _ = SbrkAllocator;
+    if (builtin.single_threaded) {
-    if (builtin.target.cpu.arch.isWasm()) {
+        if (builtin.cpu.arch.isWasm() or (builtin.os.tag == .linux and !builtin.link_libc)) {
-        _ = WasmAllocator;
+            _ = brk_allocator;
+        }
+    } else {
+        _ = smp_allocator;
     }
-    if (!builtin.single_threaded) _ = smp_allocator;
 }

lib/std/heap/BrkAllocator.zig created+348

...	@@ -0,0 +1,348 @@
1	//! Supports single-threaded targets that have a sbrk-like primitive which includes
2	//! Linux and WebAssembly.
3	//!
4	//! On Linux, assumes exclusive access to the brk syscall.
5	const BrkAllocator = @This();
6	const builtin = @import("builtin");
7
8	const std = @import("../std.zig");
9	const Allocator = std.mem.Allocator;
10	const Alignment = std.mem.Alignment;
11	const assert = std.debug.assert;
12	const math = std.math;
13
14	comptime {
15	if (!builtin.single_threaded) @compileError("unsupported");
16	}
17
18	next_addrs: [size_class_count]usize = @splat(0),
19	/// For each size class, points to the freed pointer.
20	frees: [size_class_count]usize = @splat(0),
21	/// For each big size class, points to the freed pointer.
22	big_frees: [big_size_class_count]usize = @splat(0),
23	prev_brk: usize = 0,
24
25	var global: BrkAllocator = .{};
26
27	pub const vtable: Allocator.VTable = .{
28	.alloc = alloc,
29	.resize = resize,
30	.remap = remap,
31	.free = free,
32	};
33
34	pub const Error = Allocator.Error;
35
36	const max_usize = math.maxInt(usize);
37	const ushift = math.Log2Int(usize);
38	const bigpage_size: comptime_int = @max(64 * 1024, std.heap.page_size_max);
39	const bigpage_count = max_usize / bigpage_size;
40
41	/// Because of storing free list pointers, the minimum size class is 3.
42	const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize)));
43	const size_class_count = math.log2(bigpage_size) - min_class;
44	/// 0 - 1 bigpage
45	/// 1 - 2 bigpages
46	/// 2 - 4 bigpages
47	/// etc.
48	const big_size_class_count = math.log2(bigpage_count);
49
50	fn alloc(ctx: anyopaque, len: usize, alignment: Alignment, return_address: usize) ?[]u8 {
51	_ = ctx;
52	_ = return_address;
53	// Make room for the freelist next pointer.
54	const actual_len = @max(len +\| @sizeOf(usize), alignment.toByteUnits());
55	const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
56	const class = math.log2(slot_size) - min_class;
57	if (class < size_class_count) {
58	const addr = a: {
59	const top_free_ptr = global.frees[class];
60	if (top_free_ptr != 0) {
61	const node: *usize = @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize)));
62	global.frees[class] = node.*;
63	break :a top_free_ptr;
64	}
65
66	const next_addr = global.next_addrs[class];
67	if (next_addr % bigpage_size == 0) {
68	const addr = allocBigPages(1);
69	if (addr == 0) return null;
70	//std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
71	// slot_size, class, addr,
72	//});
73	global.next_addrs[class] = addr + slot_size;
74	break :a addr;
75	} else {
76	global.next_addrs[class] = next_addr + slot_size;
77	break :a next_addr;
78	}
79	};
80	return @ptrFromInt(addr);
81	}
82	const bigpages_needed = bigPagesNeeded(actual_len);
83	return @ptrFromInt(allocBigPages(bigpages_needed));
84	}
85
86	fn resize(
87	ctx: *anyopaque,
88	buf: []u8,
89	alignment: Alignment,
90	new_len: usize,
91	return_address: usize,
92	) bool {
93	_ = ctx;
94	_ = return_address;
95	// We don't want to move anything from one size class to another, but we
96	// can recover bytes in between powers of two.
97	const buf_align = alignment.toByteUnits();
98	const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
99	const new_actual_len = @max(new_len +\| @sizeOf(usize), buf_align);
100	const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
101	const old_small_class = math.log2(old_small_slot_size) - min_class;
102	if (old_small_class < size_class_count) {
103	const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return false;
104	return old_small_slot_size == new_small_slot_size;
105	} else {
106	const old_bigpages_needed = bigPagesNeeded(old_actual_len);
107	const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed);
108	const new_bigpages_needed = bigPagesNeeded(new_actual_len);
109	const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return false;
110	return old_big_slot_pages == new_big_slot_pages;
111	}
112	}
113
114	fn remap(
115	context: *anyopaque,
116	memory: []u8,
117	alignment: Alignment,
118	new_len: usize,
119	return_address: usize,
120	) ?[*]u8 {
121	return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
122	}
123
124	fn free(
125	ctx: *anyopaque,
126	buf: []u8,
127	alignment: Alignment,
128	return_address: usize,
129	) void {
130	_ = ctx;
131	_ = return_address;
132	const buf_align = alignment.toByteUnits();
133	const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
134	const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
135	const class = math.log2(slot_size) - min_class;
136	const addr = @intFromPtr(buf.ptr);
137	if (class < size_class_count) {
138	const node: *usize = @ptrFromInt(addr + (slot_size - @sizeOf(usize)));
139	node.* = global.frees[class];
140	global.frees[class] = addr;
141	} else {
142	const bigpages_needed = bigPagesNeeded(actual_len);
143	const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed);
144	const big_slot_size_bytes = pow2_pages * bigpage_size;
145	const node: *usize = @ptrFromInt(addr + (big_slot_size_bytes - @sizeOf(usize)));
146	const big_class = math.log2(pow2_pages);
147	node.* = global.big_frees[big_class];
148	global.big_frees[big_class] = addr;
149	}
150	}
151
152	inline fn bigPagesNeeded(byte_count: usize) usize {
153	return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size;
154	}
155
156	fn allocBigPages(n: usize) usize {
157	const pow2_pages = math.ceilPowerOfTwoAssert(usize, n);
158	const slot_size_bytes = pow2_pages * bigpage_size;
159	const class = math.log2(pow2_pages);
160
161	const top_free_ptr = global.big_frees[class];
162	if (top_free_ptr != 0) {
163	const node: *usize = @ptrFromInt(top_free_ptr + (slot_size_bytes - @sizeOf(usize)));
164	global.big_frees[class] = node.*;
165	return top_free_ptr;
166	}
167
168	if (builtin.cpu.arch.isWasm()) {
169	comptime assert(std.heap.page_size_max == std.heap.page_size_min);
170	const page_size = std.heap.page_size_max;
171	const pages_per_bigpage = bigpage_size / page_size;
172	const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage);
173	if (page_index == -1) return 0;
174	return @as(usize, @intCast(page_index)) * page_size;
175	} else if (builtin.os.tag == .linux) {
176	const prev_brk = global.prev_brk;
177	const start_brk = if (prev_brk == 0)
178	std.mem.alignForward(usize, std.os.linux.brk(0), bigpage_size)
179	else
180	prev_brk;
181	const end_brk = start_brk + pow2_pages * bigpage_size;
182	const new_prev_brk = std.os.linux.brk(end_brk);
183	global.prev_brk = new_prev_brk;
184	if (new_prev_brk != end_brk) return 0;
185	return start_brk;
186	} else {
187	@compileError("no sbrk-like OS primitive available");
188	}
189	}
190
191	const test_ally: Allocator = .{
192	.ptr = undefined,
193	.vtable = &vtable,
194	};
195
196	test "small allocations - free in same order" {
197	var list: [513]*u64 = undefined;
198
199	var i: usize = 0;
200	while (i < 513) : (i += 1) {
201	const ptr = try test_ally.create(u64);
202	list[i] = ptr;
203	}
204
205	for (list) \|ptr\| {
206	test_ally.destroy(ptr);
207	}
208	}
209
210	test "small allocations - free in reverse order" {
211	var list: [513]*u64 = undefined;
212
213	var i: usize = 0;
214	while (i < 513) : (i += 1) {
215	const ptr = try test_ally.create(u64);
216	list[i] = ptr;
217	}
218
219	i = list.len;
220	while (i > 0) {
221	i -= 1;
222	const ptr = list[i];
223	test_ally.destroy(ptr);
224	}
225	}
226
227	test "large allocations" {
228	const ptr1 = try test_ally.alloc(u64, 42768);
229	const ptr2 = try test_ally.alloc(u64, 52768);
230	test_ally.free(ptr1);
231	const ptr3 = try test_ally.alloc(u64, 62768);
232	test_ally.free(ptr3);
233	test_ally.free(ptr2);
234	}
235
236	test "very large allocation" {
237	try std.testing.expectError(error.OutOfMemory, test_ally.alloc(u8, math.maxInt(usize)));
238	}
239
240	test "realloc" {
241	var slice = try test_ally.alignedAlloc(u8, .of(u32), 1);
242	defer test_ally.free(slice);
243	slice[0] = 0x12;
244
245	// This reallocation should keep its pointer address.
246	const old_slice = slice;
247	slice = try test_ally.realloc(slice, 2);
248	try std.testing.expect(old_slice.ptr == slice.ptr);
249	try std.testing.expect(slice[0] == 0x12);
250	slice[1] = 0x34;
251
252	// This requires upgrading to a larger size class
253	slice = try test_ally.realloc(slice, 17);
254	try std.testing.expect(slice[0] == 0x12);
255	try std.testing.expect(slice[1] == 0x34);
256	}
257
258	test "shrink" {
259	var slice = try test_ally.alloc(u8, 20);
260	defer test_ally.free(slice);
261
262	@memset(slice, 0x11);
263
264	try std.testing.expect(test_ally.resize(slice, 17));
265	slice = slice[0..17];
266
267	for (slice) \|b\| {
268	try std.testing.expect(b == 0x11);
269	}
270
271	try std.testing.expect(test_ally.resize(slice, 16));
272	slice = slice[0..16];
273
274	for (slice) \|b\| {
275	try std.testing.expect(b == 0x11);
276	}
277	}
278
279	test "large object - grow" {
280	if (builtin.os.tag == .linux) return error.SkipZigTest;
281
282	var slice1 = try test_ally.alloc(u8, bigpage_size * 2 - 20);
283	defer test_ally.free(slice1);
284
285	const old = slice1;
286	slice1 = try test_ally.realloc(slice1, bigpage_size * 2 - 10);
287	try std.testing.expectEqual(slice1.ptr, old.ptr);
288
289	slice1 = try test_ally.realloc(slice1, bigpage_size * 2);
290	slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1);
291	}
292
293	test "realloc small object to large object" {
294	var slice = try test_ally.alloc(u8, 70);
295	defer test_ally.free(slice);
296	slice[0] = 0x12;
297	slice[60] = 0x34;
298
299	// This requires upgrading to a large object
300	const large_object_size = bigpage_size * 2 + 50;
301	slice = try test_ally.realloc(slice, large_object_size);
302	try std.testing.expect(slice[0] == 0x12);
303	try std.testing.expect(slice[60] == 0x34);
304	}
305
306	test "shrink large object to large object" {
307	var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50);
308	defer test_ally.free(slice);
309	slice[0] = 0x12;
310	slice[60] = 0x34;
311
312	try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1));
313	slice = slice[0 .. bigpage_size * 2 + 1];
314	try std.testing.expect(slice[0] == 0x12);
315	try std.testing.expect(slice[60] == 0x34);
316
317	try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1));
318	try std.testing.expect(slice[0] == 0x12);
319	try std.testing.expect(slice[60] == 0x34);
320
321	slice = try test_ally.realloc(slice, bigpage_size * 2);
322	try std.testing.expect(slice[0] == 0x12);
323	try std.testing.expect(slice[60] == 0x34);
324	}
325
326	test "realloc large object to small object" {
327	var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50);
328	defer test_ally.free(slice);
329	slice[0] = 0x12;
330	slice[16] = 0x34;
331
332	slice = try test_ally.realloc(slice, 19);
333	try std.testing.expect(slice[0] == 0x12);
334	try std.testing.expect(slice[16] == 0x34);
335	}
336
337	test "objects of size 1024 and 2048" {
338	const slice = try test_ally.alloc(u8, 1025);
339	const slice2 = try test_ally.alloc(u8, 3000);
340
341	test_ally.free(slice);
342	test_ally.free(slice2);
343	}
344
345	test "standard allocator tests" {
346	try std.heap.testAllocator(test_ally);
347	try std.heap.testAllocatorAligned(test_ally);
348	}

lib/std/heap/PageAllocator.zig+19-22

 @@ -1,19 +1,17 @@
-const std = @import("../std.zig");
 const builtin = @import("builtin");
+const native_os = builtin.os.tag;
+const std = @import("../std.zig");
 const Allocator = std.mem.Allocator;
+const Alignment = std.mem.Alignment;
 const mem = std.mem;
 const maxInt = std.math.maxInt;
 const assert = std.debug.assert;
-const native_os = builtin.os.tag;
 const windows = std.os.windows;
-const ntdll = windows.ntdll;
+const ntdll = std.os.windows.ntdll;
 const posix = std.posix;
 const page_size_min = std.heap.page_size_min;
-const SUCCESS = @import("../os/windows/ntstatus.zig").NTSTATUS.SUCCESS;
-const MEM_RESERVE_PLACEHOLDER = windows.MEM_RESERVE_PLACEHOLDER;
-const MEM_PRESERVE_PLACEHOLDER = windows.MEM_PRESERVE_PLACEHOLDER;
 pub const vtable: Allocator.VTable = .{
     .alloc = alloc,
     .resize = resize,
 @@ -21,7 +19,7 @@ pub const vtable: Allocator.VTable = .{
     .free = free,
 };
-pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 {
+pub fn map(n: usize, alignment: Alignment) ?[*]u8 {
     const page_size = std.heap.pageSize();
     if (n >= maxInt(usize) - page_size) return null;
     const alignment_bytes = alignment.toByteUnits();
 @@ -33,11 +31,11 @@ pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 {
         const current_process = windows.GetCurrentProcess();
         var status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true, .RESERVE = true }, .{ .READWRITE = true });
-        if (status == SUCCESS and mem.isAligned(@intFromPtr(base_addr), alignment_bytes)) {
+        if (status == .SUCCESS and mem.isAligned(@intFromPtr(base_addr), alignment_bytes)) {
             return @ptrCast(base_addr);
         }
-        if (status == SUCCESS) {
+        if (status == .SUCCESS) {
             var region_size: windows.SIZE_T = 0;
             _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&base_addr), &region_size, .{ .RELEASE = true });
         }
 @@ -50,7 +48,7 @@ pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 {
         status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .RESERVE = true, .RESERVE_PLACEHOLDER = true }, .{ .NOACCESS = true });
-        if (status != SUCCESS) return null;
+        if (status != .SUCCESS) return null;
         const placeholder_addr = @intFromPtr(base_addr);
         const aligned_addr = mem.alignForward(usize, placeholder_addr, alignment_bytes);
 @@ -75,7 +73,7 @@ pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 {
         status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true }, .{ .READWRITE = true });
-        if (status == SUCCESS) {
+        if (status == .SUCCESS) {
             return @ptrCast(base_addr);
         }
 @@ -116,31 +114,29 @@ pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 {
     return result_ptr;
 }
-fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
+fn alloc(context: *anyopaque, n: usize, alignment: Alignment, ra: usize) ?[*]u8 {
     _ = context;
     _ = ra;
     assert(n > 0);
     return map(n, alignment);
 }
-fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, return_address: usize) bool {
+fn resize(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, return_address: usize) bool {
     _ = context;
-    _ = alignment;
     _ = return_address;
-    return realloc(memory, new_len, false) != null;
+    return realloc(memory, alignment, new_len, false) != null;
 }
-fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, return_address: usize) ?[*]u8 {
+fn remap(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, return_address: usize) ?[*]u8 {
     _ = context;
-    _ = alignment;
     _ = return_address;
-    return realloc(memory, new_len, true);
+    return realloc(memory, alignment, new_len, true);
 }
-fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, return_address: usize) void {
+fn free(context: *anyopaque, memory: []u8, alignment: Alignment, return_address: usize) void {
     _ = context;
-    _ = alignment;
     _ = return_address;
+    _ = alignment;
     return unmap(@alignCast(memory));
 }
 @@ -155,9 +151,10 @@ pub fn unmap(memory: []align(page_size_min) u8) void {
     }
 }
-pub fn realloc(uncasted_memory: []u8, new_len: usize, may_move: bool) ?[*]u8 {
+pub fn realloc(uncasted_memory: []u8, alignment: Alignment, new_len: usize, may_move: bool) ?[*]u8 {
     const memory: []align(page_size_min) u8 = @alignCast(uncasted_memory);
     const page_size = std.heap.pageSize();
+    if (alignment.toByteUnits() > page_size) return null;
     const new_size_aligned = mem.alignForward(usize, new_len, page_size);
     if (native_os == .windows) {

lib/std/heap/SmpAllocator.zig+9-8

 @@ -26,6 +26,7 @@
 //! By limiting the thread-local metadata array to the same number as the CPU
 //! count, ensures that as threads are created and destroyed, they cycle
 //! through the full set of freelists.
+const SmpAllocator = @This();
 const builtin = @import("builtin");
 @@ -34,7 +35,7 @@ const assert = std.debug.assert;
 const mem = std.mem;
 const math = std.math;
 const Allocator = std.mem.Allocator;
-const SmpAllocator = @This();
+const Alignment = std.mem.Alignment;
 const PageAllocator = std.heap.PageAllocator;
 cpu_count: u32,
 @@ -114,7 +115,7 @@ comptime {
     assert(!builtin.single_threaded); // you're holding it wrong
 }
-fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 {
+fn alloc(context: *anyopaque, len: usize, alignment: Alignment, ra: usize) ?[*]u8 {
     _ = context;
     _ = ra;
     const class = sizeClassIndex(len, alignment);
 @@ -172,31 +173,31 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?
     }
 }
-fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) bool {
+fn resize(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ra: usize) bool {
     _ = context;
     _ = ra;
     const class = sizeClassIndex(memory.len, alignment);
     const new_class = sizeClassIndex(new_len, alignment);
     if (class >= size_class_count) {
         if (new_class < size_class_count) return false;
-        return PageAllocator.realloc(memory, new_len, false) != null;
+        return PageAllocator.realloc(memory, alignment, new_len, false) != null;
     }
     return new_class == class;
 }
-fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) ?[*]u8 {
+fn remap(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ra: usize) ?[*]u8 {
     _ = context;
     _ = ra;
     const class = sizeClassIndex(memory.len, alignment);
     const new_class = sizeClassIndex(new_len, alignment);
     if (class >= size_class_count) {
         if (new_class < size_class_count) return null;
-        return PageAllocator.realloc(memory, new_len, true);
+        return PageAllocator.realloc(memory, alignment, new_len, true);
     }
     return if (new_class == class) memory.ptr else null;
 }
-fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize) void {
+fn free(context: *anyopaque, memory: []u8, alignment: Alignment, ra: usize) void {
     _ = context;
     _ = ra;
     const class = sizeClassIndex(memory.len, alignment);
 @@ -214,7 +215,7 @@ fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize)
     t.frees[class] = @intFromPtr(node);
 }
-fn sizeClassIndex(len: usize, alignment: mem.Alignment) usize {
+fn sizeClassIndex(len: usize, alignment: Alignment) usize {
     return @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment), min_class) - min_class;
 }

lib/std/heap/WasmAllocator.zig deleted-326

...	@@ -1,326 +0,0 @@
1	const std = @import("../std.zig");
2	const builtin = @import("builtin");
3	const Allocator = std.mem.Allocator;
4	const mem = std.mem;
5	const assert = std.debug.assert;
6	const wasm = std.wasm;
7	const math = std.math;
8
9	comptime {
10	if (!builtin.target.cpu.arch.isWasm()) {
11	@compileError("only available for wasm32 arch");
12	}
13	if (!builtin.single_threaded) {
14	@compileError("TODO implement support for multi-threaded wasm");
15	}
16	}
17
18	pub const vtable: Allocator.VTable = .{
19	.alloc = alloc,
20	.resize = resize,
21	.remap = remap,
22	.free = free,
23	};
24
25	pub const Error = Allocator.Error;
26
27	const max_usize = math.maxInt(usize);
28	const ushift = math.Log2Int(usize);
29	const bigpage_size = 64 * 1024;
30	const pages_per_bigpage = bigpage_size / wasm.page_size;
31	const bigpage_count = max_usize / bigpage_size;
32
33	/// Because of storing free list pointers, the minimum size class is 3.
34	const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize)));
35	const size_class_count = math.log2(bigpage_size) - min_class;
36	/// 0 - 1 bigpage
37	/// 1 - 2 bigpages
38	/// 2 - 4 bigpages
39	/// etc.
40	const big_size_class_count = math.log2(bigpage_count);
41
42	var next_addrs: [size_class_count]usize = @splat(0);
43	/// For each size class, points to the freed pointer.
44	var frees: [size_class_count]usize = @splat(0);
45	/// For each big size class, points to the freed pointer.
46	var big_frees: [big_size_class_count]usize = @splat(0);
47
48	fn alloc(ctx: anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[]u8 {
49	_ = ctx;
50	_ = return_address;
51	// Make room for the freelist next pointer.
52	const actual_len = @max(len +\| @sizeOf(usize), alignment.toByteUnits());
53	const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
54	const class = math.log2(slot_size) - min_class;
55	if (class < size_class_count) {
56	const addr = a: {
57	const top_free_ptr = frees[class];
58	if (top_free_ptr != 0) {
59	const node: *usize = @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize)));
60	frees[class] = node.*;
61	break :a top_free_ptr;
62	}
63
64	const next_addr = next_addrs[class];
65	if (next_addr % wasm.page_size == 0) {
66	const addr = allocBigPages(1);
67	if (addr == 0) return null;
68	//std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
69	// slot_size, class, addr,
70	//});
71	next_addrs[class] = addr + slot_size;
72	break :a addr;
73	} else {
74	next_addrs[class] = next_addr + slot_size;
75	break :a next_addr;
76	}
77	};
78	return @ptrFromInt(addr);
79	}
80	const bigpages_needed = bigPagesNeeded(actual_len);
81	return @ptrFromInt(allocBigPages(bigpages_needed));
82	}
83
84	fn resize(
85	ctx: *anyopaque,
86	buf: []u8,
87	alignment: mem.Alignment,
88	new_len: usize,
89	return_address: usize,
90	) bool {
91	_ = ctx;
92	_ = return_address;
93	// We don't want to move anything from one size class to another, but we
94	// can recover bytes in between powers of two.
95	const buf_align = alignment.toByteUnits();
96	const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
97	const new_actual_len = @max(new_len +\| @sizeOf(usize), buf_align);
98	const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
99	const old_small_class = math.log2(old_small_slot_size) - min_class;
100	if (old_small_class < size_class_count) {
101	const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return false;
102	return old_small_slot_size == new_small_slot_size;
103	} else {
104	const old_bigpages_needed = bigPagesNeeded(old_actual_len);
105	const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed);
106	const new_bigpages_needed = bigPagesNeeded(new_actual_len);
107	const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return false;
108	return old_big_slot_pages == new_big_slot_pages;
109	}
110	}
111
112	fn remap(
113	context: *anyopaque,
114	memory: []u8,
115	alignment: mem.Alignment,
116	new_len: usize,
117	return_address: usize,
118	) ?[*]u8 {
119	return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
120	}
121
122	fn free(
123	ctx: *anyopaque,
124	buf: []u8,
125	alignment: mem.Alignment,
126	return_address: usize,
127	) void {
128	_ = ctx;
129	_ = return_address;
130	const buf_align = alignment.toByteUnits();
131	const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
132	const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
133	const class = math.log2(slot_size) - min_class;
134	const addr = @intFromPtr(buf.ptr);
135	if (class < size_class_count) {
136	const node: *usize = @ptrFromInt(addr + (slot_size - @sizeOf(usize)));
137	node.* = frees[class];
138	frees[class] = addr;
139	} else {
140	const bigpages_needed = bigPagesNeeded(actual_len);
141	const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed);
142	const big_slot_size_bytes = pow2_pages * bigpage_size;
143	const node: *usize = @ptrFromInt(addr + (big_slot_size_bytes - @sizeOf(usize)));
144	const big_class = math.log2(pow2_pages);
145	node.* = big_frees[big_class];
146	big_frees[big_class] = addr;
147	}
148	}
149
150	inline fn bigPagesNeeded(byte_count: usize) usize {
151	return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size;
152	}
153
154	fn allocBigPages(n: usize) usize {
155	const pow2_pages = math.ceilPowerOfTwoAssert(usize, n);
156	const slot_size_bytes = pow2_pages * bigpage_size;
157	const class = math.log2(pow2_pages);
158
159	const top_free_ptr = big_frees[class];
160	if (top_free_ptr != 0) {
161	const node: *usize = @ptrFromInt(top_free_ptr + (slot_size_bytes - @sizeOf(usize)));
162	big_frees[class] = node.*;
163	return top_free_ptr;
164	}
165
166	const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage);
167	if (page_index == -1) return 0;
168	return @as(usize, @intCast(page_index)) * wasm.page_size;
169	}
170
171	const test_ally: Allocator = .{
172	.ptr = undefined,
173	.vtable = &vtable,
174	};
175
176	test "small allocations - free in same order" {
177	var list: [513]*u64 = undefined;
178
179	var i: usize = 0;
180	while (i < 513) : (i += 1) {
181	const ptr = try test_ally.create(u64);
182	list[i] = ptr;
183	}
184
185	for (list) \|ptr\| {
186	test_ally.destroy(ptr);
187	}
188	}
189
190	test "small allocations - free in reverse order" {
191	var list: [513]*u64 = undefined;
192
193	var i: usize = 0;
194	while (i < 513) : (i += 1) {
195	const ptr = try test_ally.create(u64);
196	list[i] = ptr;
197	}
198
199	i = list.len;
200	while (i > 0) {
201	i -= 1;
202	const ptr = list[i];
203	test_ally.destroy(ptr);
204	}
205	}
206
207	test "large allocations" {
208	const ptr1 = try test_ally.alloc(u64, 42768);
209	const ptr2 = try test_ally.alloc(u64, 52768);
210	test_ally.free(ptr1);
211	const ptr3 = try test_ally.alloc(u64, 62768);
212	test_ally.free(ptr3);
213	test_ally.free(ptr2);
214	}
215
216	test "very large allocation" {
217	try std.testing.expectError(error.OutOfMemory, test_ally.alloc(u8, math.maxInt(usize)));
218	}
219
220	test "realloc" {
221	var slice = try test_ally.alignedAlloc(u8, .of(u32), 1);
222	defer test_ally.free(slice);
223	slice[0] = 0x12;
224
225	// This reallocation should keep its pointer address.
226	const old_slice = slice;
227	slice = try test_ally.realloc(slice, 2);
228	try std.testing.expect(old_slice.ptr == slice.ptr);
229	try std.testing.expect(slice[0] == 0x12);
230	slice[1] = 0x34;
231
232	// This requires upgrading to a larger size class
233	slice = try test_ally.realloc(slice, 17);
234	try std.testing.expect(slice[0] == 0x12);
235	try std.testing.expect(slice[1] == 0x34);
236	}
237
238	test "shrink" {
239	var slice = try test_ally.alloc(u8, 20);
240	defer test_ally.free(slice);
241
242	@memset(slice, 0x11);
243
244	try std.testing.expect(test_ally.resize(slice, 17));
245	slice = slice[0..17];
246
247	for (slice) \|b\| {
248	try std.testing.expect(b == 0x11);
249	}
250
251	try std.testing.expect(test_ally.resize(slice, 16));
252	slice = slice[0..16];
253
254	for (slice) \|b\| {
255	try std.testing.expect(b == 0x11);
256	}
257	}
258
259	test "large object - grow" {
260	var slice1 = try test_ally.alloc(u8, bigpage_size * 2 - 20);
261	defer test_ally.free(slice1);
262
263	const old = slice1;
264	slice1 = try test_ally.realloc(slice1, bigpage_size * 2 - 10);
265	try std.testing.expect(slice1.ptr == old.ptr);
266
267	slice1 = try test_ally.realloc(slice1, bigpage_size * 2);
268	slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1);
269	}
270
271	test "realloc small object to large object" {
272	var slice = try test_ally.alloc(u8, 70);
273	defer test_ally.free(slice);
274	slice[0] = 0x12;
275	slice[60] = 0x34;
276
277	// This requires upgrading to a large object
278	const large_object_size = bigpage_size * 2 + 50;
279	slice = try test_ally.realloc(slice, large_object_size);
280	try std.testing.expect(slice[0] == 0x12);
281	try std.testing.expect(slice[60] == 0x34);
282	}
283
284	test "shrink large object to large object" {
285	var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50);
286	defer test_ally.free(slice);
287	slice[0] = 0x12;
288	slice[60] = 0x34;
289
290	try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1));
291	slice = slice[0 .. bigpage_size * 2 + 1];
292	try std.testing.expect(slice[0] == 0x12);
293	try std.testing.expect(slice[60] == 0x34);
294
295	try std.testing.expect(test_ally.resize(slice, bigpage_size * 2 + 1));
296	try std.testing.expect(slice[0] == 0x12);
297	try std.testing.expect(slice[60] == 0x34);
298
299	slice = try test_ally.realloc(slice, bigpage_size * 2);
300	try std.testing.expect(slice[0] == 0x12);
301	try std.testing.expect(slice[60] == 0x34);
302	}
303
304	test "realloc large object to small object" {
305	var slice = try test_ally.alloc(u8, bigpage_size * 2 + 50);
306	defer test_ally.free(slice);
307	slice[0] = 0x12;
308	slice[16] = 0x34;
309
310	slice = try test_ally.realloc(slice, 19);
311	try std.testing.expect(slice[0] == 0x12);
312	try std.testing.expect(slice[16] == 0x34);
313	}
314
315	test "objects of size 1024 and 2048" {
316	const slice = try test_ally.alloc(u8, 1025);
317	const slice2 = try test_ally.alloc(u8, 3000);
318
319	test_ally.free(slice);
320	test_ally.free(slice2);
321	}
322
323	test "standard allocator tests" {
324	try std.heap.testAllocator(test_ally);
325	try std.heap.testAllocatorAligned(test_ally);
326	}

lib/std/heap/sbrk_allocator.zig deleted-180

...	@@ -1,180 +0,0 @@
1	const builtin = @import("builtin");
2
3	const std = @import("../std.zig");
4	const Io = std.Io;
5	const math = std.math;
6	const Allocator = std.mem.Allocator;
7	const mem = std.mem;
8	const heap = std.heap;
9	const assert = std.debug.assert;
10
11	pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type {
12	return struct {
13	pub const vtable: Allocator.VTable = .{
14	.alloc = alloc,
15	.resize = resize,
16	.remap = remap,
17	.free = free,
18	};
19
20	pub const Error = Allocator.Error;
21
22	const max_usize = math.maxInt(usize);
23	const ushift = math.Log2Int(usize);
24	const bigpage_size = 64 * 1024;
25	const pages_per_bigpage = bigpage_size / heap.pageSize();
26	const bigpage_count = max_usize / bigpage_size;
27
28	/// Because of storing free list pointers, the minimum size class is 3.
29	const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize)));
30	const size_class_count = math.log2(bigpage_size) - min_class;
31	/// 0 - 1 bigpage
32	/// 1 - 2 bigpages
33	/// 2 - 4 bigpages
34	/// etc.
35	const big_size_class_count = math.log2(bigpage_count);
36
37	var next_addrs = [1]usize{0} ** size_class_count;
38	/// For each size class, points to the freed pointer.
39	var frees = [1]usize{0} ** size_class_count;
40	/// For each big size class, points to the freed pointer.
41	var big_frees = [1]usize{0} ** big_size_class_count;
42
43	// TODO don't do the naive locking strategy
44	var mutex: Io.Mutex = .{};
45	fn alloc(ctx: anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[]u8 {
46	_ = ctx;
47	_ = return_address;
48	Io.Threaded.mutexLock(&mutex);
49	defer Io.Threaded.mutexUnlock(&mutex);
50	// Make room for the freelist next pointer.
51	const actual_len = @max(len +\| @sizeOf(usize), alignment.toByteUnits());
52	const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null;
53	const class = math.log2(slot_size) - min_class;
54	if (class < size_class_count) {
55	const addr = a: {
56	const top_free_ptr = frees[class];
57	if (top_free_ptr != 0) {
58	const node = @as(*usize, @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize))));
59	frees[class] = node.*;
60	break :a top_free_ptr;
61	}
62
63	const next_addr = next_addrs[class];
64	if (next_addr % heap.pageSize() == 0) {
65	const addr = allocBigPages(1);
66	if (addr == 0) return null;
67	//std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{
68	// slot_size, class, addr,
69	//});
70	next_addrs[class] = addr + slot_size;
71	break :a addr;
72	} else {
73	next_addrs[class] = next_addr + slot_size;
74	break :a next_addr;
75	}
76	};
77	return @as([*]u8, @ptrFromInt(addr));
78	}
79	const bigpages_needed = bigPagesNeeded(actual_len);
80	const addr = allocBigPages(bigpages_needed);
81	return @as([*]u8, @ptrFromInt(addr));
82	}
83
84	fn resize(
85	ctx: *anyopaque,
86	buf: []u8,
87	alignment: mem.Alignment,
88	new_len: usize,
89	return_address: usize,
90	) bool {
91	_ = ctx;
92	_ = return_address;
93	Io.Threaded.mutexLock(&mutex);
94	defer Io.Threaded.mutexUnlock(&mutex);
95	// We don't want to move anything from one size class to another, but we
96	// can recover bytes in between powers of two.
97	const buf_align = alignment.toByteUnits();
98	const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align);
99	const new_actual_len = @max(new_len +\| @sizeOf(usize), buf_align);
100	const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len);
101	const old_small_class = math.log2(old_small_slot_size) - min_class;
102	if (old_small_class < size_class_count) {
103	const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return false;
104	return old_small_slot_size == new_small_slot_size;
105	} else {
106	const old_bigpages_needed = bigPagesNeeded(old_actual_len);
107	const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed);
108	const new_bigpages_needed = bigPagesNeeded(new_actual_len);
109	const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return false;
110	return old_big_slot_pages == new_big_slot_pages;
111	}
112	}
113
114	fn remap(
115	context: *anyopaque,
116	memory: []u8,
117	alignment: mem.Alignment,
118	new_len: usize,
119	return_address: usize,
120	) ?[*]u8 {
121	return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null;
122	}
123
124	fn free(
125	ctx: *anyopaque,
126	buf: []u8,
127	alignment: mem.Alignment,
128	return_address: usize,
129	) void {
130	_ = ctx;
131	_ = return_address;
132	Io.Threaded.mutexLock(&mutex);
133	defer Io.Threaded.mutexUnlock(&mutex);
134	const buf_align = alignment.toByteUnits();
135	const actual_len = @max(buf.len + @sizeOf(usize), buf_align);
136	const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len);
137	const class = math.log2(slot_size) - min_class;
138	const addr = @intFromPtr(buf.ptr);
139	if (class < size_class_count) {
140	const node = @as(*usize, @ptrFromInt(addr + (slot_size - @sizeOf(usize))));
141	node.* = frees[class];
142	frees[class] = addr;
143	} else {
144	const bigpages_needed = bigPagesNeeded(actual_len);
145	const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed);
146	const big_slot_size_bytes = pow2_pages * bigpage_size;
147	const node = @as(*usize, @ptrFromInt(addr + (big_slot_size_bytes - @sizeOf(usize))));
148	const big_class = math.log2(pow2_pages);
149	node.* = big_frees[big_class];
150	big_frees[big_class] = addr;
151	}
152	}
153
154	inline fn bigPagesNeeded(byte_count: usize) usize {
155	return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size;
156	}
157
158	fn allocBigPages(n: usize) usize {
159	const pow2_pages = math.ceilPowerOfTwoAssert(usize, n);
160	const slot_size_bytes = pow2_pages * bigpage_size;
161	const class = math.log2(pow2_pages);
162
163	const top_free_ptr = big_frees[class];
164	if (top_free_ptr != 0) {
165	const node = @as(*usize, @ptrFromInt(top_free_ptr + (slot_size_bytes - @sizeOf(usize))));
166	big_frees[class] = node.*;
167	return top_free_ptr;
168	}
169	return sbrk(pow2_pages * pages_per_bigpage * heap.pageSize());
170	}
171	};
172	}
173
174	test SbrkAllocator {
175	_ = SbrkAllocator(struct {
176	fn sbrk(_: usize) usize {
177	return 0;
178	}
179	}.sbrk);
180	}

lib/std/os/linux.zig+4

 @@ -594,6 +594,10 @@ pub fn errno(r: usize) E {
     return @enumFromInt(int);
 }
+pub fn brk(addr: usize) usize {
+    return syscall1(.brk, addr);
+}
 pub fn dup(old: i32) usize {
     return syscall1(.dup, @as(usize, @bitCast(@as(isize, old))));
 }

src/libs/musl.zig+1-20

...	@@ -352,8 +352,7 @@ const Ext = enum {	...	@@ -352,8 +352,7 @@ const Ext = enum {
352	fn addSrcFile(arena: Allocator, source_table: *std.StringArrayHashMap(Ext), file_path: []const u8) !void {	352	fn addSrcFile(arena: Allocator, source_table: *std.StringArrayHashMap(Ext), file_path: []const u8) !void {
353	const ext: Ext = ext: {	353	const ext: Ext = ext: {
354	if (mem.endsWith(u8, file_path, ".c")) {	354	if (mem.endsWith(u8, file_path, ".c")) {
355	if (mem.startsWith(u8, file_path, "musl/src/malloc/") or	355	if (mem.startsWith(u8, file_path, "musl/src/string/") or
356	mem.startsWith(u8, file_path, "musl/src/string/") or
357	mem.startsWith(u8, file_path, "musl/src/internal/"))	356	mem.startsWith(u8, file_path, "musl/src/internal/"))
358	{	357	{
359	break :ext .o3;	358	break :ext .o3;
...	@@ -786,24 +785,6 @@ const src_files = [_][]const u8{	...	@@ -786,24 +785,6 @@ const src_files = [_][]const u8{
786	"musl/src/locale/uselocale.c",	785	"musl/src/locale/uselocale.c",
787	"musl/src/locale/wcscoll.c",	786	"musl/src/locale/wcscoll.c",
788	"musl/src/locale/wcsxfrm.c",	787	"musl/src/locale/wcsxfrm.c",
789	"musl/src/malloc/calloc.c",
790	"musl/src/malloc/free.c",
791	"musl/src/malloc/libc_calloc.c",
792	"musl/src/malloc/lite_malloc.c",
793	"musl/src/malloc/mallocng/aligned_alloc.c",
794	"musl/src/malloc/mallocng/donate.c",
795	"musl/src/malloc/mallocng/free.c",
796	"musl/src/malloc/mallocng/malloc.c",
797	"musl/src/malloc/mallocng/malloc_usable_size.c",
798	"musl/src/malloc/mallocng/realloc.c",
799	"musl/src/malloc/memalign.c",
800	"musl/src/malloc/oldmalloc/aligned_alloc.c",
801	"musl/src/malloc/oldmalloc/malloc.c",
802	"musl/src/malloc/oldmalloc/malloc_usable_size.c",
803	"musl/src/malloc/posix_memalign.c",
804	"musl/src/malloc/reallocarray.c",
805	"musl/src/malloc/realloc.c",
806	"musl/src/malloc/replaced.c",
807	"musl/src/math/aarch64/fma.c",	788	"musl/src/math/aarch64/fma.c",
808	"musl/src/math/aarch64/fmaf.c",	789	"musl/src/math/aarch64/fmaf.c",
809	"musl/src/math/aarch64/llrint.c",	790	"musl/src/math/aarch64/llrint.c",

src/libs/wasi_libc.zig-20

...	@@ -77,22 +77,6 @@ pub fn buildCrtFile(comp: *Compilation, crt_file: CrtFile, prog_node: std.Progre	...	@@ -77,22 +77,6 @@ pub fn buildCrtFile(comp: *Compilation, crt_file: CrtFile, prog_node: std.Progre
77	.libc_a => {	77	.libc_a => {
78	var libc_sources = std.array_list.Managed(Compilation.CSourceFile).init(arena);	78	var libc_sources = std.array_list.Managed(Compilation.CSourceFile).init(arena);
79		79
80	{
81	// Compile emmalloc.
82	var args = std.array_list.Managed([]const u8).init(arena);
83	try addCCArgs(comp, arena, &args, .{ .want_O3 = true, .no_strict_aliasing = true });
84
85	for (emmalloc_src_files) \|file_path\| {
86	try libc_sources.append(.{
87	.src_path = try comp.dirs.zig_lib.join(arena, &.{
88	"libc", try sanitize(arena, file_path),
89	}),
90	.extra_flags = args.items,
91	.owner = undefined,
92	});
93	}
94	}
95
96	{	80	{
97	// Compile libc-bottom-half.	81	// Compile libc-bottom-half.
98	var args = std.array_list.Managed([]const u8).init(arena);	82	var args = std.array_list.Managed([]const u8).init(arena);
...	@@ -472,10 +456,6 @@ fn addLibcTopHalfIncludes(	...	@@ -472,10 +456,6 @@ fn addLibcTopHalfIncludes(
472	});	456	});
473	}	457	}
474		458
475	const emmalloc_src_files = [_][]const u8{
476	"wasi/emmalloc/emmalloc.c",
477	};
478
479	const libc_bottom_half_src_files = [_][]const u8{	459	const libc_bottom_half_src_files = [_][]const u8{
480	"wasi/libc-bottom-half/cloudlibc/src/libc/dirent/closedir.c",	460	"wasi/libc-bottom-half/cloudlibc/src/libc/dirent/closedir.c",
481	"wasi/libc-bottom-half/cloudlibc/src/libc/dirent/dirfd.c",	461	"wasi/libc-bottom-half/cloudlibc/src/libc/dirent/dirfd.c",

test/src/Libc.zig+3-1

 @@ -60,7 +60,9 @@ pub fn addTarget(libc: *const Libc, target: std.Build.ResolvedTarget) void {
             .link_libc = true,
         });
-        var libtest_c_source_files: []const []const u8 = &.{ "print.c", "rand.c", "mtest.c", "setrlim.c", "memfill.c", "vmfill.c", "fdfill.c", "utf8.c" };
+        var libtest_c_source_files: []const []const u8 = &.{
+            "print.c", "rand.c", "mtest.c", "setrlim.c", "memfill.c", "vmfill.c", "fdfill.c", "utf8.c",
+        };
         libtest_mod.addCSourceFiles(.{
             .root = common,
             .files = libtest_c_source_files[0..if (target.result.isMuslLibC()) 8 else 3],

author	Andrew Kelley <andrew@ziglang.org> 2026-02-13 06:30:42+01:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-02-13 06:30:42+01:00
log	4a3adaaa23faa1ae22fa72389850f04ece2570ca
tree	0ca92f641532c2b1e09381c1cff7b840cd0aeacd
parent	37109fa4ef5beea9c4ec1ea78e4fcc02df34be3c
parent	0957761d5c26ef23c04333120974323109a38f93

Merge pull request 'zig libc: implement malloc' (#31177) from libc-malloc into master