From 0a61eca22c5628b0a56d875246fc4ef007a3bad9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Feb 2026 13:53:26 -0800 Subject: [PATCH 01/12] bump max_rss for module tests --- build.zig | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/build.zig b/build.zig index bd1fd2d046ad4470ace4946605f1f17e4960693a..5eba8f82db9fe1567cf81dae1495499b8e627bf3 100644 --- a/build.zig +++ b/build.zig @@ -560,22 +560,7 @@ pub fn build(b: *std.Build) !void { .skip_linux = skip_linux, .skip_llvm = skip_llvm, .skip_libc = skip_libc, - .max_rss = switch (b.graph.host.result.os.tag) { - .freebsd => switch (b.graph.host.result.cpu.arch) { - .x86_64 => 3_756_422_348, - else => 3_800_000_000, - }, - .linux => 6_800_000_000, - .macos => switch (b.graph.host.result.cpu.arch) { - .aarch64 => 8_273_795_481, - else => 8_300_000_000, - }, - .windows => switch (b.graph.host.result.cpu.arch) { - .x86_64 => 3_750_236_160, - else => 3_800_000_000, - }, - else => 8_300_000_000, - }, + .max_rss = 8_500_000_000, })); const unit_tests_step = b.step("test-unit", "Run the compiler source unit tests"); -- 2.54.0 From 5c59a4623898cdb15b569f59fba49d48a7b9600f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Feb 2026 19:41:37 -0800 Subject: [PATCH 02/12] std.heap.PageAllocator: fix not respecting alignments in remap and resize, alignments larger than page size were incorrectly ignored. --- lib/std/heap/PageAllocator.zig | 41 ++++++++++++++++------------------ lib/std/heap/SmpAllocator.zig | 4 ++-- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/lib/std/heap/PageAllocator.zig b/lib/std/heap/PageAllocator.zig index 4ca3ec944fc5f47ebb61dba0360afdf21c0b76ce..bed9be54493cd11a01458b3a53eceacdb4cfaa3d 100644 --- a/lib/std/heap/PageAllocator.zig +++ b/lib/std/heap/PageAllocator.zig @@ -1,19 +1,17 @@ -const std = @import("../std.zig"); const builtin = @import("builtin"); +const native_os = builtin.os.tag; + +const std = @import("../std.zig"); const Allocator = std.mem.Allocator; +const Alignment = std.mem.Alignment; const mem = std.mem; const maxInt = std.math.maxInt; const assert = std.debug.assert; -const native_os = builtin.os.tag; const windows = std.os.windows; -const ntdll = windows.ntdll; +const ntdll = std.os.windows.ntdll; const posix = std.posix; const page_size_min = std.heap.page_size_min; -const SUCCESS = @import("../os/windows/ntstatus.zig").NTSTATUS.SUCCESS; -const MEM_RESERVE_PLACEHOLDER = windows.MEM_RESERVE_PLACEHOLDER; -const MEM_PRESERVE_PLACEHOLDER = windows.MEM_PRESERVE_PLACEHOLDER; - pub const vtable: Allocator.VTable = .{ .alloc = alloc, .resize = resize, @@ -21,7 +19,7 @@ pub const vtable: Allocator.VTable = .{ .free = free, }; -pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 { +pub fn map(n: usize, alignment: Alignment) ?[*]u8 { const page_size = std.heap.pageSize(); if (n >= maxInt(usize) - page_size) return null; const alignment_bytes = alignment.toByteUnits(); @@ -33,11 +31,11 @@ pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 { const current_process = windows.GetCurrentProcess(); var status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true, .RESERVE = true }, .{ .READWRITE = true }); - if (status == SUCCESS and mem.isAligned(@intFromPtr(base_addr), alignment_bytes)) { + if (status == .SUCCESS and mem.isAligned(@intFromPtr(base_addr), alignment_bytes)) { return @ptrCast(base_addr); } - if (status == SUCCESS) { + if (status == .SUCCESS) { var region_size: windows.SIZE_T = 0; _ = ntdll.NtFreeVirtualMemory(current_process, @ptrCast(&base_addr), ®ion_size, .{ .RELEASE = true }); } @@ -50,7 +48,7 @@ pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 { status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .RESERVE = true, .RESERVE_PLACEHOLDER = true }, .{ .NOACCESS = true }); - if (status != SUCCESS) return null; + if (status != .SUCCESS) return null; const placeholder_addr = @intFromPtr(base_addr); const aligned_addr = mem.alignForward(usize, placeholder_addr, alignment_bytes); @@ -75,7 +73,7 @@ pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 { status = ntdll.NtAllocateVirtualMemory(current_process, @ptrCast(&base_addr), 0, &size, .{ .COMMIT = true }, .{ .READWRITE = true }); - if (status == SUCCESS) { + if (status == .SUCCESS) { return @ptrCast(base_addr); } @@ -116,31 +114,29 @@ pub fn map(n: usize, alignment: mem.Alignment) ?[*]u8 { return result_ptr; } -fn alloc(context: *anyopaque, n: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { +fn alloc(context: *anyopaque, n: usize, alignment: Alignment, ra: usize) ?[*]u8 { _ = context; _ = ra; assert(n > 0); return map(n, alignment); } -fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, return_address: usize) bool { +fn resize(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, return_address: usize) bool { _ = context; - _ = alignment; _ = return_address; - return realloc(memory, new_len, false) != null; + return realloc(memory, alignment, new_len, false) != null; } -fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, return_address: usize) ?[*]u8 { +fn remap(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, return_address: usize) ?[*]u8 { _ = context; - _ = alignment; _ = return_address; - return realloc(memory, new_len, true); + return realloc(memory, alignment, new_len, true); } -fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, return_address: usize) void { +fn free(context: *anyopaque, memory: []u8, alignment: Alignment, return_address: usize) void { _ = context; - _ = alignment; _ = return_address; + _ = alignment; return unmap(@alignCast(memory)); } @@ -155,9 +151,10 @@ pub fn unmap(memory: []align(page_size_min) u8) void { } } -pub fn realloc(uncasted_memory: []u8, new_len: usize, may_move: bool) ?[*]u8 { +pub fn realloc(uncasted_memory: []u8, alignment: Alignment, new_len: usize, may_move: bool) ?[*]u8 { const memory: []align(page_size_min) u8 = @alignCast(uncasted_memory); const page_size = std.heap.pageSize(); + if (alignment.toByteUnits() > page_size) return null; const new_size_aligned = mem.alignForward(usize, new_len, page_size); if (native_os == .windows) { diff --git a/lib/std/heap/SmpAllocator.zig b/lib/std/heap/SmpAllocator.zig index e51e4975b68dce33c3c9d2e78691f8b9bec32159..50380d29b534e5c7d41a32c0c9d74af4efad3f08 100644 --- a/lib/std/heap/SmpAllocator.zig +++ b/lib/std/heap/SmpAllocator.zig @@ -179,7 +179,7 @@ fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: const new_class = sizeClassIndex(new_len, alignment); if (class >= size_class_count) { if (new_class < size_class_count) return false; - return PageAllocator.realloc(memory, new_len, false) != null; + return PageAllocator.realloc(memory, alignment, new_len, false) != null; } return new_class == class; } @@ -191,7 +191,7 @@ fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: u const new_class = sizeClassIndex(new_len, alignment); if (class >= size_class_count) { if (new_class < size_class_count) return null; - return PageAllocator.realloc(memory, new_len, true); + return PageAllocator.realloc(memory, alignment, new_len, true); } return if (new_class == class) memory.ptr else null; } -- 2.54.0 From 6744160211b2ab480e4c479e46440c76f00c1810 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Feb 2026 17:37:33 -0800 Subject: [PATCH 03/12] zig libc: implement malloc --- lib/c.zig | 13 +- lib/c/malloc.zig | 182 ++ lib/libc/musl/src/malloc/calloc.c | 45 - lib/libc/musl/src/malloc/free.c | 6 - lib/libc/musl/src/malloc/libc_calloc.c | 4 - lib/libc/musl/src/malloc/lite_malloc.c | 118 -- .../musl/src/malloc/mallocng/aligned_alloc.c | 60 - lib/libc/musl/src/malloc/mallocng/donate.c | 39 - lib/libc/musl/src/malloc/mallocng/free.c | 151 -- lib/libc/musl/src/malloc/mallocng/glue.h | 95 - lib/libc/musl/src/malloc/mallocng/malloc.c | 387 ----- .../src/malloc/mallocng/malloc_usable_size.c | 13 - lib/libc/musl/src/malloc/mallocng/meta.h | 288 --- lib/libc/musl/src/malloc/mallocng/realloc.c | 51 - lib/libc/musl/src/malloc/memalign.c | 7 - .../musl/src/malloc/oldmalloc/aligned_alloc.c | 53 - lib/libc/musl/src/malloc/oldmalloc/malloc.c | 556 ------ .../musl/src/malloc/oldmalloc/malloc_impl.h | 39 - .../src/malloc/oldmalloc/malloc_usable_size.c | 9 - lib/libc/musl/src/malloc/posix_memalign.c | 11 - lib/libc/musl/src/malloc/realloc.c | 6 - lib/libc/musl/src/malloc/reallocarray.c | 13 - lib/libc/musl/src/malloc/replaced.c | 4 - lib/libc/wasi/emmalloc/emmalloc.c | 1540 ----------------- lib/std/heap/SmpAllocator.zig | 13 +- src/libs/musl.zig | 21 +- src/libs/wasi_libc.zig | 20 - 27 files changed, 198 insertions(+), 3546 deletions(-) create mode 100644 lib/c/malloc.zig delete mode 100644 lib/libc/musl/src/malloc/calloc.c delete mode 100644 lib/libc/musl/src/malloc/free.c delete mode 100644 lib/libc/musl/src/malloc/libc_calloc.c delete mode 100644 lib/libc/musl/src/malloc/lite_malloc.c delete mode 100644 lib/libc/musl/src/malloc/mallocng/aligned_alloc.c delete mode 100644 lib/libc/musl/src/malloc/mallocng/donate.c delete mode 100644 lib/libc/musl/src/malloc/mallocng/free.c delete mode 100644 lib/libc/musl/src/malloc/mallocng/glue.h delete mode 100644 lib/libc/musl/src/malloc/mallocng/malloc.c delete mode 100644 lib/libc/musl/src/malloc/mallocng/malloc_usable_size.c delete mode 100644 lib/libc/musl/src/malloc/mallocng/meta.h delete mode 100644 lib/libc/musl/src/malloc/mallocng/realloc.c delete mode 100644 lib/libc/musl/src/malloc/memalign.c delete mode 100644 lib/libc/musl/src/malloc/oldmalloc/aligned_alloc.c delete mode 100644 lib/libc/musl/src/malloc/oldmalloc/malloc.c delete mode 100644 lib/libc/musl/src/malloc/oldmalloc/malloc_impl.h delete mode 100644 lib/libc/musl/src/malloc/oldmalloc/malloc_usable_size.c delete mode 100644 lib/libc/musl/src/malloc/posix_memalign.c delete mode 100644 lib/libc/musl/src/malloc/realloc.c delete mode 100644 lib/libc/musl/src/malloc/reallocarray.c delete mode 100644 lib/libc/musl/src/malloc/replaced.c delete mode 100644 lib/libc/wasi/emmalloc/emmalloc.c diff --git a/lib/c.zig b/lib/c.zig index c25293b84dc9424e8373cd325ccb12ba875eb618..8b181dc1184b6504085de5adaf2ce04f639ec25c 100644 --- a/lib/c.zig +++ b/lib/c.zig @@ -47,19 +47,22 @@ pub fn errno(syscall_return_value: usize) c_int { } comptime { - _ = @import("c/inttypes.zig"); _ = @import("c/ctype.zig"); - _ = @import("c/stdlib.zig"); + _ = @import("c/inttypes.zig"); + if (!builtin.target.isMinGW()) { + _ = @import("c/malloc.zig"); + } _ = @import("c/math.zig"); + _ = @import("c/stdlib.zig"); _ = @import("c/string.zig"); _ = @import("c/strings.zig"); - _ = @import("c/wchar.zig"); - _ = @import("c/sys/mman.zig"); + _ = @import("c/sys/capability.zig"); _ = @import("c/sys/file.zig"); + _ = @import("c/sys/mman.zig"); _ = @import("c/sys/reboot.zig"); - _ = @import("c/sys/capability.zig"); _ = @import("c/sys/utsname.zig"); _ = @import("c/unistd.zig"); + _ = @import("c/wchar.zig"); } diff --git a/lib/c/malloc.zig b/lib/c/malloc.zig new file mode 100644 index 0000000000000000000000000000000000000000..72e7621b61057fdb1c31737c0af3c0b0262d53c0 --- /dev/null +++ b/lib/c/malloc.zig @@ -0,0 +1,182 @@ +//! Based on wrapping a stateless Zig Allocator implementation, apropriate for: +//! - ReleaseFast and ReleaseSmall optimization modes, with multi-threading +//! enabled. +//! - WebAssembly in single-threaded mode. +//! +//! Because the libc APIs don't have client alignment and size tracking, in +//! order to take advantage of Zig allocator implementations, additional +//! metadata must be stored in the allocations. +//! +//! This implementation stores the metadata just before the pointer returned +//! from `malloc`, just like many libc malloc implementations do, including +//! musl. This has the downside of causing fragmentation for allocations with +//! higher alignment, however most of that memory can be recovered by +//! preemptively putting the gap onto the freelist. +const builtin = @import("builtin"); + +const std = @import("std"); +const assert = std.debug.assert; +const Alignment = std.mem.Alignment; +const alignment_bytes = @max(@alignOf(std.c.max_align_t), @sizeOf(Header)); +const alignment: Alignment = .fromByteUnits(alignment_bytes); + +const symbol = @import("../c.zig").symbol; + +comptime { + symbol(&malloc, "malloc"); + symbol(&aligned_alloc, "aligned_alloc"); + symbol(&posix_memalign, "posix_memalign"); + symbol(&calloc, "calloc"); + symbol(&realloc, "realloc"); + symbol(&reallocarray, "reallocarray"); + symbol(&free, "free"); + symbol(&malloc_usable_size, "malloc_usable_size"); + + symbol(&valloc, "valloc"); + symbol(&memalign, "memalign"); +} + +const no_context: *anyopaque = undefined; +const no_ra: usize = undefined; +const vtable = switch (builtin.cpu.arch) { + .wasm32, .wasm64 => std.heap.WasmAllocator.vtable, + else => std.heap.SmpAllocator.vtable, +}; + +/// Needed because libc memory allocators don't provide old alignment and size +/// which are required by Zig memory allocators. +const Header = packed struct(u64) { + alignment: Alignment, + /// Does not include the extra alignment bytes added. + size: Size, + padding: Padding = 0, + + comptime { + assert(@sizeOf(Header) <= alignment_bytes); + } + + const Size = @Int(.unsigned, @min(64 - @bitSizeOf(Alignment), @bitSizeOf(usize))); + const Padding = @Int(.unsigned, 64 - @bitSizeOf(Alignment) - @bitSizeOf(Size)); + + fn fromBase(base: [*]align(alignment_bytes) u8) *Header { + return @ptrCast(base - @sizeOf(Header)); + } +}; + +fn malloc(n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + const size = std.math.cast(Header.Size, n) orelse return null; + const ptr: [*]align(alignment_bytes) u8 = @alignCast( + vtable.alloc(no_context, n + alignment_bytes, alignment, no_ra) orelse return null, + ); + const base = ptr + alignment_bytes; + const header: *Header = .fromBase(base); + header.* = .{ + .alignment = alignment, + .size = size, + }; + return base; +} + +fn aligned_alloc(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + const size = std.math.cast(Header.Size, n) orelse return null; + const max_align = alignment.max(.fromByteUnits(alloc_alignment)); + const max_align_bytes = max_align.toByteUnits(); + const ptr: [*]align(alignment_bytes) u8 = @alignCast( + vtable.alloc(no_context, n + max_align_bytes, max_align, no_ra) orelse return null, + ); + const base: [*]align(alignment_bytes) u8 = @alignCast(ptr + max_align_bytes); + const header: *Header = .fromBase(base); + header.* = .{ + .alignment = max_align, + .size = size, + }; + return base; +} + +fn calloc(elems: usize, len: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + const n = std.math.mul(usize, elems, len) catch return null; + const base = malloc(n) orelse return null; + @memset(base[0..n], 0); + return base; +} + +fn realloc(opt_old_base: ?[*]align(alignment_bytes) u8, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + if (n == 0) { + free(opt_old_base); + return null; + } + const old_base = opt_old_base orelse return malloc(n); + const new_size = std.math.cast(Header.Size, n) orelse return null; + const old_header: *Header = .fromBase(old_base); + assert(old_header.padding == 0); + const old_size = old_header.size; + const old_alignment = old_header.alignment; + const old_alignment_bytes = old_alignment.toByteUnits(); + const old_ptr = old_base - old_alignment_bytes; + const old_slice = old_ptr[0 .. old_size + old_alignment_bytes]; + const new_base: [*]align(alignment_bytes) u8 = if (vtable.remap( + no_context, + old_slice, + old_alignment, + n + old_alignment_bytes, + no_ra, + )) |new_ptr| @alignCast(new_ptr + old_alignment_bytes) else b: { + const new_ptr: [*]align(alignment_bytes) u8 = @alignCast( + vtable.alloc(no_context, n + old_alignment_bytes, old_alignment, no_ra) orelse return null, + ); + const new_base: [*]align(alignment_bytes) u8 = @alignCast(new_ptr + old_alignment_bytes); + const copy_len = @min(new_size, old_size); + @memcpy(new_base[0..copy_len], old_base[0..copy_len]); + vtable.free(no_context, old_slice, old_alignment, no_ra); + break :b new_base; + }; + const new_header: *Header = .fromBase(new_base); + new_header.* = .{ + .alignment = old_alignment, + .size = new_size, + }; + return new_base; +} + +fn reallocarray(opt_base: ?[*]align(alignment_bytes) u8, elems: usize, len: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + const n = std.math.mul(usize, elems, len) catch return null; + return realloc(opt_base, n); +} + +fn free(opt_old_base: ?[*]align(alignment_bytes) u8) callconv(.c) void { + const old_base = opt_old_base orelse return; + const old_header: *Header = .fromBase(old_base); + assert(old_header.padding == 0); + const old_size = old_header.size; + const old_alignment = old_header.alignment; + const old_alignment_bytes = old_alignment.toByteUnits(); + const old_ptr = old_base - old_alignment_bytes; + const old_slice = old_ptr[0 .. old_size + old_alignment_bytes]; + vtable.free(no_context, old_slice, old_alignment, no_ra); +} + +fn malloc_usable_size(opt_old_base: ?[*]align(alignment_bytes) u8) callconv(.c) usize { + const old_base = opt_old_base orelse return 0; + const old_header: *Header = .fromBase(old_base); + assert(old_header.padding == 0); + const old_size = old_header.size; + return old_size; +} + +fn valloc(n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + return aligned_alloc(std.heap.pageSize(), n); +} + +fn memalign(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + return aligned_alloc(alloc_alignment, n); +} + +fn posix_memalign(result: *?[*]align(alignment_bytes) u8, alloc_alignment: usize, n: usize) callconv(.c) c_int { + if (alloc_alignment < @sizeOf(*anyopaque)) return @intFromEnum(std.c.E.INVAL); + if (n == 0) { + result.* = null; + } else { + result.* = aligned_alloc(alloc_alignment, n) orelse return @intFromEnum(std.c.E.NOMEM); + } + return 0; +} diff --git a/lib/libc/musl/src/malloc/calloc.c b/lib/libc/musl/src/malloc/calloc.c deleted file mode 100644 index bf6bddca3fd81aca3b246391e34ea9afbc95ed3e..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/calloc.c +++ /dev/null @@ -1,45 +0,0 @@ -#include -#include -#include -#include -#include "dynlink.h" - -static size_t mal0_clear(char *p, size_t n) -{ - const size_t pagesz = 4096; /* arbitrary */ - if (n < pagesz) return n; -#ifdef __GNUC__ - typedef uint64_t __attribute__((__may_alias__)) T; -#else - typedef unsigned char T; -#endif - char *pp = p + n; - size_t i = (uintptr_t)pp & (pagesz - 1); - for (;;) { - pp = memset(pp - i, 0, i); - if (pp - p < pagesz) return pp - p; - for (i = pagesz; i; i -= 2*sizeof(T), pp -= 2*sizeof(T)) - if (((T *)pp)[-1] | ((T *)pp)[-2]) - break; - } -} - -static int allzerop(void *p) -{ - return 0; -} -weak_alias(allzerop, __malloc_allzerop); - -void *calloc(size_t m, size_t n) -{ - if (n && m > (size_t)-1/n) { - errno = ENOMEM; - return 0; - } - n *= m; - void *p = malloc(n); - if (!p || (!__malloc_replaced && __malloc_allzerop(p))) - return p; - n = mal0_clear(p, n); - return memset(p, 0, n); -} diff --git a/lib/libc/musl/src/malloc/free.c b/lib/libc/musl/src/malloc/free.c deleted file mode 100644 index 3944f7b28f3c3265cb092cf984c69e89f62f435a..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/free.c +++ /dev/null @@ -1,6 +0,0 @@ -#include - -void free(void *p) -{ - __libc_free(p); -} diff --git a/lib/libc/musl/src/malloc/libc_calloc.c b/lib/libc/musl/src/malloc/libc_calloc.c deleted file mode 100644 index d25eabea47f563d8d1dc5b93d55bc976cb64e1cf..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/libc_calloc.c +++ /dev/null @@ -1,4 +0,0 @@ -#define calloc __libc_calloc -#define malloc __libc_malloc - -#include "calloc.c" diff --git a/lib/libc/musl/src/malloc/lite_malloc.c b/lib/libc/musl/src/malloc/lite_malloc.c deleted file mode 100644 index 43a988fbb8adcc3c6469c21b9a4ad13018f2758b..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/lite_malloc.c +++ /dev/null @@ -1,118 +0,0 @@ -#include -#include -#include -#include -#include -#include "libc.h" -#include "lock.h" -#include "syscall.h" -#include "fork_impl.h" - -#define ALIGN 16 - -/* This function returns true if the interval [old,new] - * intersects the 'len'-sized interval below &libc.auxv - * (interpreted as the main-thread stack) or below &b - * (the current stack). It is used to defend against - * buggy brk implementations that can cross the stack. */ - -static int traverses_stack_p(uintptr_t old, uintptr_t new) -{ - const uintptr_t len = 8<<20; - uintptr_t a, b; - - b = (uintptr_t)libc.auxv; - a = b > len ? b-len : 0; - if (new>a && old len ? b-len : 0; - if (new>a && old SIZE_MAX/2) { - errno = ENOMEM; - return 0; - } - - if (!n) n++; - while (align end-cur) { - size_t req = n - (end-cur) + PAGE_SIZE-1 & -PAGE_SIZE; - - if (!cur) { - brk = __syscall(SYS_brk, 0); - brk += -brk & PAGE_SIZE-1; - cur = end = brk; - } - - if (brk == end && req < SIZE_MAX-brk - && !traverses_stack_p(brk, brk+req) - && __syscall(SYS_brk, brk+req)==brk+req) { - brk = end += req; - } else { - int new_area = 0; - req = n + PAGE_SIZE-1 & -PAGE_SIZE; - /* Only make a new area rather than individual mmap - * if wasted space would be over 1/8 of the map. */ - if (req-n > req/8) { - /* Geometric area size growth up to 64 pages, - * bounding waste by 1/8 of the area. */ - size_t min = PAGE_SIZE<<(mmap_step/2); - if (min-n > end-cur) { - if (req < min) { - req = min; - if (mmap_step < 12) - mmap_step++; - } - new_area = 1; - } - } - void *mem = __mmap(0, req, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED || !new_area) { - UNLOCK(lock); - return mem==MAP_FAILED ? 0 : mem; - } - cur = (uintptr_t)mem; - end = cur + req; - } - } - - p = (void *)cur; - cur += n; - UNLOCK(lock); - return p; -} - -weak_alias(__simple_malloc, __libc_malloc_impl); - -void *__libc_malloc(size_t n) -{ - return __libc_malloc_impl(n); -} - -static void *default_malloc(size_t n) -{ - return __libc_malloc_impl(n); -} - -weak_alias(default_malloc, malloc); diff --git a/lib/libc/musl/src/malloc/mallocng/aligned_alloc.c b/lib/libc/musl/src/malloc/mallocng/aligned_alloc.c deleted file mode 100644 index e0862a83ae00444734417d97dea6b4a679e557df..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/mallocng/aligned_alloc.c +++ /dev/null @@ -1,60 +0,0 @@ -#include -#include -#include "meta.h" - -void *aligned_alloc(size_t align, size_t len) -{ - if ((align & -align) != align) { - errno = EINVAL; - return 0; - } - - if (len > SIZE_MAX - align || align >= (1ULL<<31)*UNIT) { - errno = ENOMEM; - return 0; - } - - if (DISABLE_ALIGNED_ALLOC) { - errno = ENOMEM; - return 0; - } - - if (align <= UNIT) align = UNIT; - - unsigned char *p = malloc(len + align - UNIT); - if (!p) - return 0; - - struct meta *g = get_meta(p); - int idx = get_slot_index(p); - size_t stride = get_stride(g); - unsigned char *start = g->mem->storage + stride*idx; - unsigned char *end = g->mem->storage + stride*(idx+1) - IB; - size_t adj = -(uintptr_t)p & (align-1); - - if (!adj) { - set_size(p, end, len); - return p; - } - p += adj; - uint32_t offset = (size_t)(p-g->mem->storage)/UNIT; - if (offset <= 0xffff) { - *(uint16_t *)(p-2) = offset; - p[-4] = 0; - } else { - // use a 32-bit offset if 16-bit doesn't fit. for this, - // 16-bit field must be zero, [-4] byte nonzero. - *(uint16_t *)(p-2) = 0; - *(uint32_t *)(p-8) = offset; - p[-4] = 1; - } - p[-3] = idx; - set_size(p, end, len); - // store offset to aligned enframing. this facilitates cycling - // offset and also iteration of heap for debugging/measurement. - // for extreme overalignment it won't fit but these are classless - // allocations anyway. - *(uint16_t *)(start - 2) = (size_t)(p-start)/UNIT; - start[-3] = 7<<5; - return p; -} diff --git a/lib/libc/musl/src/malloc/mallocng/donate.c b/lib/libc/musl/src/malloc/mallocng/donate.c deleted file mode 100644 index 41d850f357e332f2ec24c198385d39258df4b921..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/mallocng/donate.c +++ /dev/null @@ -1,39 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "meta.h" - -static void donate(unsigned char *base, size_t len) -{ - uintptr_t a = (uintptr_t)base; - uintptr_t b = a + len; - a += -a & (UNIT-1); - b -= b & (UNIT-1); - memset(base, 0, len); - for (int sc=47; sc>0 && b>a; sc-=4) { - if (b-a < (size_classes[sc]+1)*UNIT) continue; - struct meta *m = alloc_meta(); - m->avail_mask = 0; - m->freed_mask = 1; - m->mem = (void *)a; - m->mem->meta = m; - m->last_idx = 0; - m->freeable = 0; - m->sizeclass = sc; - m->maplen = 0; - *((unsigned char *)m->mem+UNIT-4) = 0; - *((unsigned char *)m->mem+UNIT-3) = 255; - m->mem->storage[size_classes[sc]*UNIT-4] = 0; - queue(&ctx.active[sc], m); - a += (size_classes[sc]+1)*UNIT; - } -} - -void __malloc_donate(char *start, char *end) -{ - donate((void *)start, end-start); -} diff --git a/lib/libc/musl/src/malloc/mallocng/free.c b/lib/libc/musl/src/malloc/mallocng/free.c deleted file mode 100644 index 43f32aade86b62b99baac87435ee5fb5ef43080c..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/mallocng/free.c +++ /dev/null @@ -1,151 +0,0 @@ -#define _BSD_SOURCE -#include -#include - -#include "meta.h" - -struct mapinfo { - void *base; - size_t len; -}; - -static struct mapinfo nontrivial_free(struct meta *, int); - -static struct mapinfo free_group(struct meta *g) -{ - struct mapinfo mi = { 0 }; - int sc = g->sizeclass; - if (sc < 48) { - ctx.usage_by_class[sc] -= g->last_idx+1; - } - if (g->maplen) { - step_seq(); - record_seq(sc); - mi.base = g->mem; - mi.len = g->maplen*4096UL; - } else { - void *p = g->mem; - struct meta *m = get_meta(p); - int idx = get_slot_index(p); - g->mem->meta = 0; - // not checking size/reserved here; it's intentionally invalid - mi = nontrivial_free(m, idx); - } - free_meta(g); - return mi; -} - -static int okay_to_free(struct meta *g) -{ - int sc = g->sizeclass; - - if (!g->freeable) return 0; - - // always free individual mmaps not suitable for reuse - if (sc >= 48 || get_stride(g) < UNIT*size_classes[sc]) - return 1; - - // always free groups allocated inside another group's slot - // since recreating them should not be expensive and they - // might be blocking freeing of a much larger group. - if (!g->maplen) return 1; - - // if there is another non-full group, free this one to - // consolidate future allocations, reduce fragmentation. - if (g->next != g) return 1; - - // free any group in a size class that's not bouncing - if (!is_bouncing(sc)) return 1; - - size_t cnt = g->last_idx+1; - size_t usage = ctx.usage_by_class[sc]; - - // if usage is high enough that a larger count should be - // used, free the low-count group so a new one will be made. - if (9*cnt <= usage && cnt < 20) - return 1; - - // otherwise, keep the last group in a bouncing class. - return 0; -} - -static struct mapinfo nontrivial_free(struct meta *g, int i) -{ - uint32_t self = 1u<sizeclass; - uint32_t mask = g->freed_mask | g->avail_mask; - - if (mask+self == (2u<last_idx)-1 && okay_to_free(g)) { - // any multi-slot group is necessarily on an active list - // here, but single-slot groups might or might not be. - if (g->next) { - assert(sc < 48); - int activate_new = (ctx.active[sc]==g); - dequeue(&ctx.active[sc], g); - if (activate_new && ctx.active[sc]) - activate_group(ctx.active[sc]); - } - return free_group(g); - } else if (!mask) { - assert(sc < 48); - // might still be active if there were no allocations - // after last available slot was taken. - if (ctx.active[sc] != g) { - queue(&ctx.active[sc], g); - } - } - a_or(&g->freed_mask, self); - return (struct mapinfo){ 0 }; -} - -void free(void *p) -{ - if (!p) return; - - struct meta *g = get_meta(p); - int idx = get_slot_index(p); - size_t stride = get_stride(g); - unsigned char *start = g->mem->storage + stride*idx; - unsigned char *end = start + stride - IB; - get_nominal_size(p, end); - uint32_t self = 1u<last_idx)-1; - ((unsigned char *)p)[-3] = 255; - // invalidate offset to group header, and cycle offset of - // used region within slot if current offset is zero. - *(uint16_t *)((char *)p-2) = 0; - - // release any whole pages contained in the slot to be freed - // unless it's a single-slot group that will be unmapped. - if (((uintptr_t)(start-1) ^ (uintptr_t)end) >= 2*PGSZ && g->last_idx) { - unsigned char *base = start + (-(uintptr_t)start & (PGSZ-1)); - size_t len = (end-base) & -PGSZ; - if (len && USE_MADV_FREE) { - int e = errno; - madvise(base, len, MADV_FREE); - errno = e; - } - } - - // atomic free without locking if this is neither first or last slot - for (;;) { - uint32_t freed = g->freed_mask; - uint32_t avail = g->avail_mask; - uint32_t mask = freed | avail; - assert(!(mask&self)); - if (!freed || mask+self==all) break; - if (!MT) - g->freed_mask = freed+self; - else if (a_cas(&g->freed_mask, freed, freed+self)!=freed) - continue; - return; - } - - wrlock(); - struct mapinfo mi = nontrivial_free(g, idx); - unlock(); - if (mi.len) { - int e = errno; - munmap(mi.base, mi.len); - errno = e; - } -} diff --git a/lib/libc/musl/src/malloc/mallocng/glue.h b/lib/libc/musl/src/malloc/mallocng/glue.h deleted file mode 100644 index 77f4c812b20a77bea56596d0b0bb82626cc34853..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/mallocng/glue.h +++ /dev/null @@ -1,95 +0,0 @@ -#ifndef MALLOC_GLUE_H -#define MALLOC_GLUE_H - -#include -#include -#include -#include -#include -#include -#include "atomic.h" -#include "syscall.h" -#include "libc.h" -#include "lock.h" -#include "dynlink.h" - -// use macros to appropriately namespace these. -#define size_classes __malloc_size_classes -#define ctx __malloc_context -#define alloc_meta __malloc_alloc_meta -#define is_allzero __malloc_allzerop -#define dump_heap __dump_heap - -#define malloc __libc_malloc_impl -#define realloc __libc_realloc -#define free __libc_free - -#define USE_MADV_FREE 0 - -#if USE_REAL_ASSERT -#include -#else -#undef assert -#define assert(x) do { if (!(x)) a_crash(); } while(0) -#endif - -#define brk(p) ((uintptr_t)__syscall(SYS_brk, p)) - -#define mmap __mmap -#define madvise __madvise -#define mremap __mremap - -#define DISABLE_ALIGNED_ALLOC (__malloc_replaced && !__aligned_alloc_replaced) - -static inline uint64_t get_random_secret() -{ - uint64_t secret = (uintptr_t)&secret * 1103515245; - for (size_t i=0; libc.auxv[i]; i+=2) - if (libc.auxv[i]==AT_RANDOM) - memcpy(&secret, (char *)libc.auxv[i+1]+8, sizeof secret); - return secret; -} - -#ifndef PAGESIZE -#define PAGESIZE PAGE_SIZE -#endif - -#define MT (libc.need_locks) - -#define RDLOCK_IS_EXCLUSIVE 1 - -__attribute__((__visibility__("hidden"))) -extern int __malloc_lock[1]; - -#define LOCK_OBJ_DEF \ -int __malloc_lock[1]; \ -void __malloc_atfork(int who) { malloc_atfork(who); } - -static inline void rdlock() -{ - if (MT) LOCK(__malloc_lock); -} -static inline void wrlock() -{ - if (MT) LOCK(__malloc_lock); -} -static inline void unlock() -{ - UNLOCK(__malloc_lock); -} -static inline void upgradelock() -{ -} -static inline void resetlock() -{ - __malloc_lock[0] = 0; -} - -static inline void malloc_atfork(int who) -{ - if (who<0) rdlock(); - else if (who>0) resetlock(); - else unlock(); -} - -#endif diff --git a/lib/libc/musl/src/malloc/mallocng/malloc.c b/lib/libc/musl/src/malloc/mallocng/malloc.c deleted file mode 100644 index d695ab8ec98216d3420e41b820f951c9474adca4..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/mallocng/malloc.c +++ /dev/null @@ -1,387 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "meta.h" - -LOCK_OBJ_DEF; - -const uint16_t size_classes[] = { - 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 12, 15, - 18, 20, 25, 31, - 36, 42, 50, 63, - 72, 84, 102, 127, - 146, 170, 204, 255, - 292, 340, 409, 511, - 584, 682, 818, 1023, - 1169, 1364, 1637, 2047, - 2340, 2730, 3276, 4095, - 4680, 5460, 6552, 8191, -}; - -static const uint8_t small_cnt_tab[][3] = { - { 30, 30, 30 }, - { 31, 15, 15 }, - { 20, 10, 10 }, - { 31, 15, 7 }, - { 25, 12, 6 }, - { 21, 10, 5 }, - { 18, 8, 4 }, - { 31, 15, 7 }, - { 28, 14, 6 }, -}; - -static const uint8_t med_cnt_tab[4] = { 28, 24, 20, 32 }; - -struct malloc_context ctx = { 0 }; - -struct meta *alloc_meta(void) -{ - struct meta *m; - unsigned char *p; - if (!ctx.init_done) { -#ifndef PAGESIZE - ctx.pagesize = get_page_size(); -#endif - ctx.secret = get_random_secret(); - ctx.init_done = 1; - } - size_t pagesize = PGSZ; - if (pagesize < 4096) pagesize = 4096; - if ((m = dequeue_head(&ctx.free_meta_head))) return m; - if (!ctx.avail_meta_count) { - int need_unprotect = 1; - if (!ctx.avail_meta_area_count && ctx.brk!=-1) { - uintptr_t new = ctx.brk + pagesize; - int need_guard = 0; - if (!ctx.brk) { - need_guard = 1; - ctx.brk = brk(0); - // some ancient kernels returned _ebss - // instead of next page as initial brk. - ctx.brk += -ctx.brk & (pagesize-1); - new = ctx.brk + 2*pagesize; - } - if (brk(new) != new) { - ctx.brk = -1; - } else { - if (need_guard) mmap((void *)ctx.brk, pagesize, - PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0); - ctx.brk = new; - ctx.avail_meta_areas = (void *)(new - pagesize); - ctx.avail_meta_area_count = pagesize>>12; - need_unprotect = 0; - } - } - if (!ctx.avail_meta_area_count) { - size_t n = 2UL << ctx.meta_alloc_shift; - p = mmap(0, n*pagesize, PROT_NONE, - MAP_PRIVATE|MAP_ANON, -1, 0); - if (p==MAP_FAILED) return 0; - ctx.avail_meta_areas = p + pagesize; - ctx.avail_meta_area_count = (n-1)*(pagesize>>12); - ctx.meta_alloc_shift++; - } - p = ctx.avail_meta_areas; - if ((uintptr_t)p & (pagesize-1)) need_unprotect = 0; - if (need_unprotect) - if (mprotect(p, pagesize, PROT_READ|PROT_WRITE) - && errno != ENOSYS) - return 0; - ctx.avail_meta_area_count--; - ctx.avail_meta_areas = p + 4096; - if (ctx.meta_area_tail) { - ctx.meta_area_tail->next = (void *)p; - } else { - ctx.meta_area_head = (void *)p; - } - ctx.meta_area_tail = (void *)p; - ctx.meta_area_tail->check = ctx.secret; - ctx.avail_meta_count = ctx.meta_area_tail->nslots - = (4096-sizeof(struct meta_area))/sizeof *m; - ctx.avail_meta = ctx.meta_area_tail->slots; - } - ctx.avail_meta_count--; - m = ctx.avail_meta++; - m->prev = m->next = 0; - return m; -} - -static uint32_t try_avail(struct meta **pm) -{ - struct meta *m = *pm; - uint32_t first; - if (!m) return 0; - uint32_t mask = m->avail_mask; - if (!mask) { - if (!m) return 0; - if (!m->freed_mask) { - dequeue(pm, m); - m = *pm; - if (!m) return 0; - } else { - m = m->next; - *pm = m; - } - - mask = m->freed_mask; - - // skip fully-free group unless it's the only one - // or it's a permanently non-freeable group - if (mask == (2u<last_idx)-1 && m->freeable) { - m = m->next; - *pm = m; - mask = m->freed_mask; - } - - // activate more slots in a not-fully-active group - // if needed, but only as a last resort. prefer using - // any other group with free slots. this avoids - // touching & dirtying as-yet-unused pages. - if (!(mask & ((2u<mem->active_idx)-1))) { - if (m->next != m) { - m = m->next; - *pm = m; - } else { - int cnt = m->mem->active_idx + 2; - int size = size_classes[m->sizeclass]*UNIT; - int span = UNIT + size*cnt; - // activate up to next 4k boundary - while ((span^(span+size-1)) < 4096) { - cnt++; - span += size; - } - if (cnt > m->last_idx+1) - cnt = m->last_idx+1; - m->mem->active_idx = cnt-1; - } - } - mask = activate_group(m); - assert(mask); - decay_bounces(m->sizeclass); - } - first = mask&-mask; - m->avail_mask = mask-first; - return first; -} - -static int alloc_slot(int, size_t); - -static struct meta *alloc_group(int sc, size_t req) -{ - size_t size = UNIT*size_classes[sc]; - int i = 0, cnt; - unsigned char *p; - struct meta *m = alloc_meta(); - if (!m) return 0; - size_t usage = ctx.usage_by_class[sc]; - size_t pagesize = PGSZ; - int active_idx; - if (sc < 9) { - while (i<2 && 4*small_cnt_tab[sc][i] > usage) - i++; - cnt = small_cnt_tab[sc][i]; - } else { - // lookup max number of slots fitting in power-of-two size - // from a table, along with number of factors of two we - // can divide out without a remainder or reaching 1. - cnt = med_cnt_tab[sc&3]; - - // reduce cnt to avoid excessive eagar allocation. - while (!(cnt&1) && 4*cnt > usage) - cnt >>= 1; - - // data structures don't support groups whose slot offsets - // in units don't fit in 16 bits. - while (size*cnt >= 65536*UNIT) - cnt >>= 1; - } - - // If we selected a count of 1 above but it's not sufficient to use - // mmap, increase to 2. Then it might be; if not it will nest. - if (cnt==1 && size*cnt+UNIT <= pagesize/2) cnt = 2; - - // All choices of size*cnt are "just below" a power of two, so anything - // larger than half the page size should be allocated as whole pages. - if (size*cnt+UNIT > pagesize/2) { - // check/update bounce counter to start/increase retention - // of freed maps, and inhibit use of low-count, odd-size - // small mappings and single-slot groups if activated. - int nosmall = is_bouncing(sc); - account_bounce(sc); - step_seq(); - - // since the following count reduction opportunities have - // an absolute memory usage cost, don't overdo them. count - // coarse usage as part of usage. - if (!(sc&1) && sc<32) usage += ctx.usage_by_class[sc+1]; - - // try to drop to a lower count if the one found above - // increases usage by more than 25%. these reduced counts - // roughly fill an integral number of pages, just not a - // power of two, limiting amount of unusable space. - if (4*cnt > usage && !nosmall) { - if (0); - else if ((sc&3)==1 && size*cnt>8*pagesize) cnt = 2; - else if ((sc&3)==2 && size*cnt>4*pagesize) cnt = 3; - else if ((sc&3)==0 && size*cnt>8*pagesize) cnt = 3; - else if ((sc&3)==0 && size*cnt>2*pagesize) cnt = 5; - } - size_t needed = size*cnt + UNIT; - needed += -needed & (pagesize-1); - - // produce an individually-mmapped allocation if usage is low, - // bounce counter hasn't triggered, and either it saves memory - // or it avoids eagar slot allocation without wasting too much. - if (!nosmall && cnt<=7) { - req += IB + UNIT; - req += -req & (pagesize-1); - if (req=4*pagesize && 2*cnt>usage)) { - cnt = 1; - needed = req; - } - } - - p = mmap(0, needed, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0); - if (p==MAP_FAILED) { - free_meta(m); - return 0; - } - m->maplen = needed>>12; - ctx.mmap_counter++; - active_idx = (4096-UNIT)/size-1; - if (active_idx > cnt-1) active_idx = cnt-1; - if (active_idx < 0) active_idx = 0; - } else { - int j = size_to_class(UNIT+cnt*size-IB); - int idx = alloc_slot(j, UNIT+cnt*size-IB); - if (idx < 0) { - free_meta(m); - return 0; - } - struct meta *g = ctx.active[j]; - p = enframe(g, idx, UNIT*size_classes[j]-IB, ctx.mmap_counter); - m->maplen = 0; - p[-3] = (p[-3]&31) | (6<<5); - for (int i=0; i<=cnt; i++) - p[UNIT+i*size-4] = 0; - active_idx = cnt-1; - } - ctx.usage_by_class[sc] += cnt; - m->avail_mask = (2u<freed_mask = (2u<<(cnt-1))-1 - m->avail_mask; - m->mem = (void *)p; - m->mem->meta = m; - m->mem->active_idx = active_idx; - m->last_idx = cnt-1; - m->freeable = 1; - m->sizeclass = sc; - return m; -} - -static int alloc_slot(int sc, size_t req) -{ - uint32_t first = try_avail(&ctx.active[sc]); - if (first) return a_ctz_32(first); - - struct meta *g = alloc_group(sc, req); - if (!g) return -1; - - g->avail_mask--; - queue(&ctx.active[sc], g); - return 0; -} - -void *malloc(size_t n) -{ - if (size_overflows(n)) return 0; - struct meta *g; - uint32_t mask, first; - int sc; - int idx; - int ctr; - - if (n >= MMAP_THRESHOLD) { - size_t needed = n + IB + UNIT; - void *p = mmap(0, needed, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANON, -1, 0); - if (p==MAP_FAILED) return 0; - wrlock(); - step_seq(); - g = alloc_meta(); - if (!g) { - unlock(); - munmap(p, needed); - return 0; - } - g->mem = p; - g->mem->meta = g; - g->last_idx = 0; - g->freeable = 1; - g->sizeclass = 63; - g->maplen = (needed+4095)/4096; - g->avail_mask = g->freed_mask = 0; - // use a global counter to cycle offset in - // individually-mmapped allocations. - ctx.mmap_counter++; - idx = 0; - goto success; - } - - sc = size_to_class(n); - - rdlock(); - g = ctx.active[sc]; - - // use coarse size classes initially when there are not yet - // any groups of desired size. this allows counts of 2 or 3 - // to be allocated at first rather than having to start with - // 7 or 5, the min counts for even size classes. - if (!g && sc>=4 && sc<32 && sc!=6 && !(sc&1) && !ctx.usage_by_class[sc]) { - size_t usage = ctx.usage_by_class[sc|1]; - // if a new group may be allocated, count it toward - // usage in deciding if we can use coarse class. - if (!ctx.active[sc|1] || (!ctx.active[sc|1]->avail_mask - && !ctx.active[sc|1]->freed_mask)) - usage += 3; - if (usage <= 12) - sc |= 1; - g = ctx.active[sc]; - } - - for (;;) { - mask = g ? g->avail_mask : 0; - first = mask&-mask; - if (!first) break; - if (RDLOCK_IS_EXCLUSIVE || !MT) - g->avail_mask = mask-first; - else if (a_cas(&g->avail_mask, mask, mask-first)!=mask) - continue; - idx = a_ctz_32(first); - goto success; - } - upgradelock(); - - idx = alloc_slot(sc, n); - if (idx < 0) { - unlock(); - return 0; - } - g = ctx.active[sc]; - -success: - ctr = ctx.mmap_counter; - unlock(); - return enframe(g, idx, n, ctr); -} - -int is_allzero(void *p) -{ - struct meta *g = get_meta(p); - return g->sizeclass >= 48 || - get_stride(g) < UNIT*size_classes[g->sizeclass]; -} diff --git a/lib/libc/musl/src/malloc/mallocng/malloc_usable_size.c b/lib/libc/musl/src/malloc/mallocng/malloc_usable_size.c deleted file mode 100644 index ce6a960c6f5835d5fd7f16818a139e5d9991edb4..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/mallocng/malloc_usable_size.c +++ /dev/null @@ -1,13 +0,0 @@ -#include -#include "meta.h" - -size_t malloc_usable_size(void *p) -{ - if (!p) return 0; - struct meta *g = get_meta(p); - int idx = get_slot_index(p); - size_t stride = get_stride(g); - unsigned char *start = g->mem->storage + stride*idx; - unsigned char *end = start + stride - IB; - return get_nominal_size(p, end); -} diff --git a/lib/libc/musl/src/malloc/mallocng/meta.h b/lib/libc/musl/src/malloc/mallocng/meta.h deleted file mode 100644 index 61ec53f9a589941c5edb29da0f6fa9342faf42ab..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/mallocng/meta.h +++ /dev/null @@ -1,288 +0,0 @@ -#ifndef MALLOC_META_H -#define MALLOC_META_H - -#include -#include -#include -#include "glue.h" - -__attribute__((__visibility__("hidden"))) -extern const uint16_t size_classes[]; - -#define MMAP_THRESHOLD 131052 - -#define UNIT 16 -#define IB 4 - -struct group { - struct meta *meta; - unsigned char active_idx:5; - char pad[UNIT - sizeof(struct meta *) - 1]; - unsigned char storage[]; -}; - -struct meta { - struct meta *prev, *next; - struct group *mem; - volatile int avail_mask, freed_mask; - uintptr_t last_idx:5; - uintptr_t freeable:1; - uintptr_t sizeclass:6; - uintptr_t maplen:8*sizeof(uintptr_t)-12; -}; - -struct meta_area { - uint64_t check; - struct meta_area *next; - int nslots; - struct meta slots[]; -}; - -struct malloc_context { - uint64_t secret; -#ifndef PAGESIZE - size_t pagesize; -#endif - int init_done; - unsigned mmap_counter; - struct meta *free_meta_head; - struct meta *avail_meta; - size_t avail_meta_count, avail_meta_area_count, meta_alloc_shift; - struct meta_area *meta_area_head, *meta_area_tail; - unsigned char *avail_meta_areas; - struct meta *active[48]; - size_t usage_by_class[48]; - uint8_t unmap_seq[32], bounces[32]; - uint8_t seq; - uintptr_t brk; -}; - -__attribute__((__visibility__("hidden"))) -extern struct malloc_context ctx; - -#ifdef PAGESIZE -#define PGSZ PAGESIZE -#else -#define PGSZ ctx.pagesize -#endif - -__attribute__((__visibility__("hidden"))) -struct meta *alloc_meta(void); - -__attribute__((__visibility__("hidden"))) -int is_allzero(void *); - -static inline void queue(struct meta **phead, struct meta *m) -{ - assert(!m->next); - assert(!m->prev); - if (*phead) { - struct meta *head = *phead; - m->next = head; - m->prev = head->prev; - m->next->prev = m->prev->next = m; - } else { - m->prev = m->next = m; - *phead = m; - } -} - -static inline void dequeue(struct meta **phead, struct meta *m) -{ - if (m->next != m) { - m->prev->next = m->next; - m->next->prev = m->prev; - if (*phead == m) *phead = m->next; - } else { - *phead = 0; - } - m->prev = m->next = 0; -} - -static inline struct meta *dequeue_head(struct meta **phead) -{ - struct meta *m = *phead; - if (m) dequeue(phead, m); - return m; -} - -static inline void free_meta(struct meta *m) -{ - *m = (struct meta){0}; - queue(&ctx.free_meta_head, m); -} - -static inline uint32_t activate_group(struct meta *m) -{ - assert(!m->avail_mask); - uint32_t mask, act = (2u<mem->active_idx)-1; - do mask = m->freed_mask; - while (a_cas(&m->freed_mask, mask, mask&~act)!=mask); - return m->avail_mask = mask & act; -} - -static inline int get_slot_index(const unsigned char *p) -{ - return p[-3] & 31; -} - -static inline struct meta *get_meta(const unsigned char *p) -{ - assert(!((uintptr_t)p & 15)); - int offset = *(const uint16_t *)(p - 2); - int index = get_slot_index(p); - if (p[-4]) { - assert(!offset); - offset = *(uint32_t *)(p - 8); - assert(offset > 0xffff); - } - const struct group *base = (const void *)(p - UNIT*offset - UNIT); - const struct meta *meta = base->meta; - assert(meta->mem == base); - assert(index <= meta->last_idx); - assert(!(meta->avail_mask & (1u<freed_mask & (1u<check == ctx.secret); - if (meta->sizeclass < 48) { - assert(offset >= size_classes[meta->sizeclass]*index); - assert(offset < size_classes[meta->sizeclass]*(index+1)); - } else { - assert(meta->sizeclass == 63); - } - if (meta->maplen) { - assert(offset <= meta->maplen*4096UL/UNIT - 1); - } - return (struct meta *)meta; -} - -static inline size_t get_nominal_size(const unsigned char *p, const unsigned char *end) -{ - size_t reserved = p[-3] >> 5; - if (reserved >= 5) { - assert(reserved == 5); - reserved = *(const uint32_t *)(end-4); - assert(reserved >= 5); - assert(!end[-5]); - } - assert(reserved <= end-p); - assert(!*(end-reserved)); - // also check the slot's overflow byte - assert(!*end); - return end-reserved-p; -} - -static inline size_t get_stride(const struct meta *g) -{ - if (!g->last_idx && g->maplen) { - return g->maplen*4096UL - UNIT; - } else { - return UNIT*size_classes[g->sizeclass]; - } -} - -static inline void set_size(unsigned char *p, unsigned char *end, size_t n) -{ - int reserved = end-p-n; - if (reserved) end[-reserved] = 0; - if (reserved >= 5) { - *(uint32_t *)(end-4) = reserved; - end[-5] = 0; - reserved = 5; - } - p[-3] = (p[-3]&31) + (reserved<<5); -} - -static inline void *enframe(struct meta *g, int idx, size_t n, int ctr) -{ - size_t stride = get_stride(g); - size_t slack = (stride-IB-n)/UNIT; - unsigned char *p = g->mem->storage + stride*idx; - unsigned char *end = p+stride-IB; - // cycle offset within slot to increase interval to address - // reuse, facilitate trapping double-free. - int off = (p[-3] ? *(uint16_t *)(p-2) + 1 : ctr) & 255; - assert(!p[-4]); - if (off > slack) { - size_t m = slack; - m |= m>>1; m |= m>>2; m |= m>>4; - off &= m; - if (off > slack) off -= slack+1; - assert(off <= slack); - } - if (off) { - // store offset in unused header at offset zero - // if enframing at non-zero offset. - *(uint16_t *)(p-2) = off; - p[-3] = 7<<5; - p += UNIT*off; - // for nonzero offset there is no permanent check - // byte, so make one. - p[-4] = 0; - } - *(uint16_t *)(p-2) = (size_t)(p-g->mem->storage)/UNIT; - p[-3] = idx; - set_size(p, end, n); - return p; -} - -static inline int size_to_class(size_t n) -{ - n = (n+IB-1)>>4; - if (n<10) return n; - n++; - int i = (28-a_clz_32(n))*4 + 8; - if (n>size_classes[i+1]) i+=2; - if (n>size_classes[i]) i++; - return i; -} - -static inline int size_overflows(size_t n) -{ - if (n >= SIZE_MAX/2 - 4096) { - errno = ENOMEM; - return 1; - } - return 0; -} - -static inline void step_seq(void) -{ - if (ctx.seq==255) { - for (int i=0; i<32; i++) ctx.unmap_seq[i] = 0; - ctx.seq = 1; - } else { - ctx.seq++; - } -} - -static inline void record_seq(int sc) -{ - if (sc-7U < 32) ctx.unmap_seq[sc-7] = ctx.seq; -} - -static inline void account_bounce(int sc) -{ - if (sc-7U < 32) { - int seq = ctx.unmap_seq[sc-7]; - if (seq && ctx.seq-seq < 10) { - if (ctx.bounces[sc-7]+1 < 100) - ctx.bounces[sc-7]++; - else - ctx.bounces[sc-7] = 150; - } - } -} - -static inline void decay_bounces(int sc) -{ - if (sc-7U < 32 && ctx.bounces[sc-7]) - ctx.bounces[sc-7]--; -} - -static inline int is_bouncing(int sc) -{ - return (sc-7U < 32 && ctx.bounces[sc-7] >= 100); -} - -#endif diff --git a/lib/libc/musl/src/malloc/mallocng/realloc.c b/lib/libc/musl/src/malloc/mallocng/realloc.c deleted file mode 100644 index 18769f42d83ae71dabea1e7caf117fe4115af139..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/mallocng/realloc.c +++ /dev/null @@ -1,51 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include "meta.h" - -void *realloc(void *p, size_t n) -{ - if (!p) return malloc(n); - if (size_overflows(n)) return 0; - - struct meta *g = get_meta(p); - int idx = get_slot_index(p); - size_t stride = get_stride(g); - unsigned char *start = g->mem->storage + stride*idx; - unsigned char *end = start + stride - IB; - size_t old_size = get_nominal_size(p, end); - size_t avail_size = end-(unsigned char *)p; - void *new; - - // only resize in-place if size class matches - if (n <= avail_size && n= g->sizeclass) { - set_size(p, end, n); - return p; - } - - // use mremap if old and new size are both mmap-worthy - if (g->sizeclass>=48 && n>=MMAP_THRESHOLD) { - assert(g->sizeclass==63); - size_t base = (unsigned char *)p-start; - size_t needed = (n + base + UNIT + IB + 4095) & -4096; - new = g->maplen*4096UL == needed ? g->mem : - mremap(g->mem, g->maplen*4096UL, needed, MREMAP_MAYMOVE); - if (new!=MAP_FAILED) { - g->mem = new; - g->maplen = needed/4096; - p = g->mem->storage + base; - end = g->mem->storage + (needed - UNIT) - IB; - *end = 0; - set_size(p, end, n); - return p; - } - } - - new = malloc(n); - if (!new) return 0; - memcpy(new, p, n < old_size ? n : old_size); - free(p); - return new; -} diff --git a/lib/libc/musl/src/malloc/memalign.c b/lib/libc/musl/src/malloc/memalign.c deleted file mode 100644 index 32cd87d812f7f4228d1f85932f7a7dbb75777de7..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/memalign.c +++ /dev/null @@ -1,7 +0,0 @@ -#define _BSD_SOURCE -#include - -void *memalign(size_t align, size_t len) -{ - return aligned_alloc(align, len); -} diff --git a/lib/libc/musl/src/malloc/oldmalloc/aligned_alloc.c b/lib/libc/musl/src/malloc/oldmalloc/aligned_alloc.c deleted file mode 100644 index 4adca3b4f6b7704b3cb034c4a087eaf4e08c2749..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/oldmalloc/aligned_alloc.c +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include -#include -#include "malloc_impl.h" - -void *aligned_alloc(size_t align, size_t len) -{ - unsigned char *mem, *new; - - if ((align & -align) != align) { - errno = EINVAL; - return 0; - } - - if (len > SIZE_MAX - align || - (__malloc_replaced && !__aligned_alloc_replaced)) { - errno = ENOMEM; - return 0; - } - - if (align <= SIZE_ALIGN) - return malloc(len); - - if (!(mem = malloc(len + align-1))) - return 0; - - new = (void *)((uintptr_t)mem + align-1 & -align); - if (new == mem) return mem; - - struct chunk *c = MEM_TO_CHUNK(mem); - struct chunk *n = MEM_TO_CHUNK(new); - - if (IS_MMAPPED(c)) { - /* Apply difference between aligned and original - * address to the "extra" field of mmapped chunk. */ - n->psize = c->psize + (new-mem); - n->csize = c->csize - (new-mem); - return new; - } - - struct chunk *t = NEXT_CHUNK(c); - - /* Split the allocated chunk into two chunks. The aligned part - * that will be used has the size in its footer reduced by the - * difference between the aligned and original addresses, and - * the resulting size copied to its header. A new header and - * footer are written for the split-off part to be freed. */ - n->psize = c->csize = C_INUSE | (new-mem); - n->csize = t->psize -= new-mem; - - __bin_chunk(c); - return new; -} diff --git a/lib/libc/musl/src/malloc/oldmalloc/malloc.c b/lib/libc/musl/src/malloc/oldmalloc/malloc.c deleted file mode 100644 index 25d00d44dea770a7f78ac61a15aeda9308057b2a..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/oldmalloc/malloc.c +++ /dev/null @@ -1,556 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include "libc.h" -#include "atomic.h" -#include "pthread_impl.h" -#include "malloc_impl.h" -#include "fork_impl.h" - -#define malloc __libc_malloc_impl -#define realloc __libc_realloc -#define free __libc_free - -#if defined(__GNUC__) && defined(__PIC__) -#define inline inline __attribute__((always_inline)) -#endif - -static struct { - volatile uint64_t binmap; - struct bin bins[64]; - volatile int split_merge_lock[2]; -} mal; - -/* Synchronization tools */ - -static inline void lock(volatile int *lk) -{ - int need_locks = libc.need_locks; - if (need_locks) { - while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1); - if (need_locks < 0) libc.need_locks = 0; - } -} - -static inline void unlock(volatile int *lk) -{ - if (lk[0]) { - a_store(lk, 0); - if (lk[1]) __wake(lk, 1, 1); - } -} - -static inline void lock_bin(int i) -{ - lock(mal.bins[i].lock); - if (!mal.bins[i].head) - mal.bins[i].head = mal.bins[i].tail = BIN_TO_CHUNK(i); -} - -static inline void unlock_bin(int i) -{ - unlock(mal.bins[i].lock); -} - -static int first_set(uint64_t x) -{ -#if 1 - return a_ctz_64(x); -#else - static const char debruijn64[64] = { - 0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, - 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, - 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, - 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12 - }; - static const char debruijn32[32] = { - 0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, - 31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14 - }; - if (sizeof(long) < 8) { - uint32_t y = x; - if (!y) { - y = x>>32; - return 32 + debruijn32[(y&-y)*0x076be629 >> 27]; - } - return debruijn32[(y&-y)*0x076be629 >> 27]; - } - return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58]; -#endif -} - -static const unsigned char bin_tab[60] = { - 32,33,34,35,36,36,37,37,38,38,39,39, - 40,40,40,40,41,41,41,41,42,42,42,42,43,43,43,43, - 44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45, - 46,46,46,46,46,46,46,46,47,47,47,47,47,47,47,47, -}; - -static int bin_index(size_t x) -{ - x = x / SIZE_ALIGN - 1; - if (x <= 32) return x; - if (x < 512) return bin_tab[x/8-4]; - if (x > 0x1c00) return 63; - return bin_tab[x/128-4] + 16; -} - -static int bin_index_up(size_t x) -{ - x = x / SIZE_ALIGN - 1; - if (x <= 32) return x; - x--; - if (x < 512) return bin_tab[x/8-4] + 1; - return bin_tab[x/128-4] + 17; -} - -#if 0 -void __dump_heap(int x) -{ - struct chunk *c; - int i; - for (c = (void *)mal.heap; CHUNK_SIZE(c); c = NEXT_CHUNK(c)) - fprintf(stderr, "base %p size %zu (%d) flags %d/%d\n", - c, CHUNK_SIZE(c), bin_index(CHUNK_SIZE(c)), - c->csize & 15, - NEXT_CHUNK(c)->psize & 15); - for (i=0; i<64; i++) { - if (mal.bins[i].head != BIN_TO_CHUNK(i) && mal.bins[i].head) { - fprintf(stderr, "bin %d: %p\n", i, mal.bins[i].head); - if (!(mal.binmap & 1ULL< len ? b-len : 0; - if (new>a && old len ? b-len : 0; - if (new>a && old SIZE_MAX/2 - PAGE_SIZE) { - errno = ENOMEM; - return 0; - } - n += -n & PAGE_SIZE-1; - - if (!brk) { - brk = __syscall(SYS_brk, 0); - brk += -brk & PAGE_SIZE-1; - } - - if (n < SIZE_MAX-brk && !traverses_stack_p(brk, brk+n) - && __syscall(SYS_brk, brk+n)==brk+n) { - *pn = n; - brk += n; - return (void *)(brk-n); - } - - size_t min = (size_t)PAGE_SIZE << mmap_step/2; - if (n < min) n = min; - void *area = __mmap(0, n, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (area == MAP_FAILED) return 0; - *pn = n; - mmap_step++; - return area; -} - -static struct chunk *expand_heap(size_t n) -{ - static void *end; - void *p; - struct chunk *w; - - /* The argument n already accounts for the caller's chunk - * overhead needs, but if the heap can't be extended in-place, - * we need room for an extra zero-sized sentinel chunk. */ - n += SIZE_ALIGN; - - p = __expand_heap(&n); - if (!p) return 0; - - /* If not just expanding existing space, we need to make a - * new sentinel chunk below the allocated space. */ - if (p != end) { - /* Valid/safe because of the prologue increment. */ - n -= SIZE_ALIGN; - p = (char *)p + SIZE_ALIGN; - w = MEM_TO_CHUNK(p); - w->psize = 0 | C_INUSE; - } - - /* Record new heap end and fill in footer. */ - end = (char *)p + n; - w = MEM_TO_CHUNK(end); - w->psize = n | C_INUSE; - w->csize = 0 | C_INUSE; - - /* Fill in header, which may be new or may be replacing a - * zero-size sentinel header at the old end-of-heap. */ - w = MEM_TO_CHUNK(p); - w->csize = n | C_INUSE; - - return w; -} - -static int adjust_size(size_t *n) -{ - /* Result of pointer difference must fit in ptrdiff_t. */ - if (*n-1 > PTRDIFF_MAX - SIZE_ALIGN - PAGE_SIZE) { - if (*n) { - errno = ENOMEM; - return -1; - } else { - *n = SIZE_ALIGN; - return 0; - } - } - *n = (*n + OVERHEAD + SIZE_ALIGN - 1) & SIZE_MASK; - return 0; -} - -static void unbin(struct chunk *c, int i) -{ - if (c->prev == c->next) - a_and_64(&mal.binmap, ~(1ULL<prev->next = c->next; - c->next->prev = c->prev; - c->csize |= C_INUSE; - NEXT_CHUNK(c)->psize |= C_INUSE; -} - -static void bin_chunk(struct chunk *self, int i) -{ - self->next = BIN_TO_CHUNK(i); - self->prev = mal.bins[i].tail; - self->next->prev = self; - self->prev->next = self; - if (self->prev == BIN_TO_CHUNK(i)) - a_or_64(&mal.binmap, 1ULL<= n1 - DONTCARE) return; - - next = NEXT_CHUNK(self); - split = (void *)((char *)self + n); - - split->psize = n | C_INUSE; - split->csize = n1-n; - next->psize = n1-n; - self->csize = n | C_INUSE; - - int i = bin_index(n1-n); - lock_bin(i); - - bin_chunk(split, i); - - unlock_bin(i); -} - -void *malloc(size_t n) -{ - struct chunk *c; - int i, j; - uint64_t mask; - - if (adjust_size(&n) < 0) return 0; - - if (n > MMAP_THRESHOLD) { - size_t len = n + OVERHEAD + PAGE_SIZE - 1 & -PAGE_SIZE; - char *base = __mmap(0, len, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (base == (void *)-1) return 0; - c = (void *)(base + SIZE_ALIGN - OVERHEAD); - c->csize = len - (SIZE_ALIGN - OVERHEAD); - c->psize = SIZE_ALIGN - OVERHEAD; - return CHUNK_TO_MEM(c); - } - - i = bin_index_up(n); - if (i<63 && (mal.binmap & (1ULL<psize; - char *base = (char *)self - extra; - size_t oldlen = n0 + extra; - size_t newlen = n + extra; - /* Crash on realloc of freed chunk */ - if (extra & 1) a_crash(); - if (newlen < PAGE_SIZE && (new = malloc(n-OVERHEAD))) { - n0 = n; - goto copy_free_ret; - } - newlen = (newlen + PAGE_SIZE-1) & -PAGE_SIZE; - if (oldlen == newlen) return p; - base = __mremap(base, oldlen, newlen, MREMAP_MAYMOVE); - if (base == (void *)-1) - goto copy_realloc; - self = (void *)(base + extra); - self->csize = newlen - extra; - return CHUNK_TO_MEM(self); - } - - next = NEXT_CHUNK(self); - - /* Crash on corrupted footer (likely from buffer overflow) */ - if (next->psize != self->csize) a_crash(); - - if (n < n0) { - int i = bin_index_up(n); - int j = bin_index(n0); - if (icsize = split->psize = n | C_INUSE; - split->csize = next->psize = n0-n | C_INUSE; - __bin_chunk(split); - return CHUNK_TO_MEM(self); - } - - lock(mal.split_merge_lock); - - size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next); - if (n0+nsize >= n) { - int i = bin_index(nsize); - lock_bin(i); - if (!(next->csize & C_INUSE)) { - unbin(next, i); - unlock_bin(i); - next = NEXT_CHUNK(next); - self->csize = next->psize = n0+nsize | C_INUSE; - trim(self, n); - unlock(mal.split_merge_lock); - return CHUNK_TO_MEM(self); - } - unlock_bin(i); - } - unlock(mal.split_merge_lock); - -copy_realloc: - /* As a last resort, allocate a new chunk and copy to it. */ - new = malloc(n-OVERHEAD); - if (!new) return 0; -copy_free_ret: - memcpy(new, p, (npsize != self->csize) a_crash(); - - lock(mal.split_merge_lock); - - size_t osize = CHUNK_SIZE(self), size = osize; - - /* Since we hold split_merge_lock, only transition from free to - * in-use can race; in-use to free is impossible */ - size_t psize = self->psize & C_INUSE ? 0 : CHUNK_PSIZE(self); - size_t nsize = next->csize & C_INUSE ? 0 : CHUNK_SIZE(next); - - if (psize) { - int i = bin_index(psize); - lock_bin(i); - if (!(self->psize & C_INUSE)) { - struct chunk *prev = PREV_CHUNK(self); - unbin(prev, i); - self = prev; - size += psize; - } - unlock_bin(i); - } - if (nsize) { - int i = bin_index(nsize); - lock_bin(i); - if (!(next->csize & C_INUSE)) { - unbin(next, i); - next = NEXT_CHUNK(next); - size += nsize; - } - unlock_bin(i); - } - - int i = bin_index(size); - lock_bin(i); - - self->csize = size; - next->psize = size; - bin_chunk(self, i); - unlock(mal.split_merge_lock); - - /* Replace middle of large chunks with fresh zero pages */ - if (size > RECLAIM && (size^(size-osize)) > size-osize) { - uintptr_t a = (uintptr_t)self + SIZE_ALIGN+PAGE_SIZE-1 & -PAGE_SIZE; - uintptr_t b = (uintptr_t)next - SIZE_ALIGN & -PAGE_SIZE; - int e = errno; -#if 1 - __madvise((void *)a, b-a, MADV_DONTNEED); -#else - __mmap((void *)a, b-a, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0); -#endif - errno = e; - } - - unlock_bin(i); -} - -static void unmap_chunk(struct chunk *self) -{ - size_t extra = self->psize; - char *base = (char *)self - extra; - size_t len = CHUNK_SIZE(self) + extra; - /* Crash on double free */ - if (extra & 1) a_crash(); - int e = errno; - __munmap(base, len); - errno = e; -} - -void free(void *p) -{ - if (!p) return; - - struct chunk *self = MEM_TO_CHUNK(p); - - if (IS_MMAPPED(self)) - unmap_chunk(self); - else - __bin_chunk(self); -} - -void __malloc_donate(char *start, char *end) -{ - size_t align_start_up = (SIZE_ALIGN-1) & (-(uintptr_t)start - OVERHEAD); - size_t align_end_down = (SIZE_ALIGN-1) & (uintptr_t)end; - - /* Getting past this condition ensures that the padding for alignment - * and header overhead will not overflow and will leave a nonzero - * multiple of SIZE_ALIGN bytes between start and end. */ - if (end - start <= OVERHEAD + align_start_up + align_end_down) - return; - start += align_start_up + OVERHEAD; - end -= align_end_down; - - struct chunk *c = MEM_TO_CHUNK(start), *n = MEM_TO_CHUNK(end); - c->psize = n->csize = C_INUSE; - c->csize = n->psize = C_INUSE | (end-start); - __bin_chunk(c); -} - -void __malloc_atfork(int who) -{ - if (who<0) { - lock(mal.split_merge_lock); - for (int i=0; i<64; i++) - lock(mal.bins[i].lock); - } else if (!who) { - for (int i=0; i<64; i++) - unlock(mal.bins[i].lock); - unlock(mal.split_merge_lock); - } else { - for (int i=0; i<64; i++) - mal.bins[i].lock[0] = mal.bins[i].lock[1] = 0; - mal.split_merge_lock[1] = 0; - mal.split_merge_lock[0] = 0; - } -} diff --git a/lib/libc/musl/src/malloc/oldmalloc/malloc_impl.h b/lib/libc/musl/src/malloc/oldmalloc/malloc_impl.h deleted file mode 100644 index e1cf4774c1e40d44d338200e95f8028b8083ce53..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/oldmalloc/malloc_impl.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef MALLOC_IMPL_H -#define MALLOC_IMPL_H - -#include -#include "dynlink.h" - -struct chunk { - size_t psize, csize; - struct chunk *next, *prev; -}; - -struct bin { - volatile int lock[2]; - struct chunk *head; - struct chunk *tail; -}; - -#define SIZE_ALIGN (4*sizeof(size_t)) -#define SIZE_MASK (-SIZE_ALIGN) -#define OVERHEAD (2*sizeof(size_t)) -#define MMAP_THRESHOLD (0x1c00*SIZE_ALIGN) -#define DONTCARE 16 -#define RECLAIM 163840 - -#define CHUNK_SIZE(c) ((c)->csize & -2) -#define CHUNK_PSIZE(c) ((c)->psize & -2) -#define PREV_CHUNK(c) ((struct chunk *)((char *)(c) - CHUNK_PSIZE(c))) -#define NEXT_CHUNK(c) ((struct chunk *)((char *)(c) + CHUNK_SIZE(c))) -#define MEM_TO_CHUNK(p) (struct chunk *)((char *)(p) - OVERHEAD) -#define CHUNK_TO_MEM(c) (void *)((char *)(c) + OVERHEAD) -#define BIN_TO_CHUNK(i) (MEM_TO_CHUNK(&mal.bins[i].head)) - -#define C_INUSE ((size_t)1) - -#define IS_MMAPPED(c) !((c)->csize & (C_INUSE)) - -hidden void __bin_chunk(struct chunk *); - -#endif diff --git a/lib/libc/musl/src/malloc/oldmalloc/malloc_usable_size.c b/lib/libc/musl/src/malloc/oldmalloc/malloc_usable_size.c deleted file mode 100644 index 672b518ad0ef15d9b3ad19a0538c6490aa88ca63..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/oldmalloc/malloc_usable_size.c +++ /dev/null @@ -1,9 +0,0 @@ -#include -#include "malloc_impl.h" - -hidden void *(*const __realloc_dep)(void *, size_t) = realloc; - -size_t malloc_usable_size(void *p) -{ - return p ? CHUNK_SIZE(MEM_TO_CHUNK(p)) - OVERHEAD : 0; -} diff --git a/lib/libc/musl/src/malloc/posix_memalign.c b/lib/libc/musl/src/malloc/posix_memalign.c deleted file mode 100644 index ad4d8f473015e378ad1025363cd68507ec539f18..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/posix_memalign.c +++ /dev/null @@ -1,11 +0,0 @@ -#include -#include - -int posix_memalign(void **res, size_t align, size_t len) -{ - if (align < sizeof(void *)) return EINVAL; - void *mem = aligned_alloc(align, len); - if (!mem) return errno; - *res = mem; - return 0; -} diff --git a/lib/libc/musl/src/malloc/realloc.c b/lib/libc/musl/src/malloc/realloc.c deleted file mode 100644 index fb0e8b7c47869c5ad259f430132867d3744a4441..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/realloc.c +++ /dev/null @@ -1,6 +0,0 @@ -#include - -void *realloc(void *p, size_t n) -{ - return __libc_realloc(p, n); -} diff --git a/lib/libc/musl/src/malloc/reallocarray.c b/lib/libc/musl/src/malloc/reallocarray.c deleted file mode 100644 index 4a6ebe460469b69a6e860f6f1ad643c40c35fbeb..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/reallocarray.c +++ /dev/null @@ -1,13 +0,0 @@ -#define _BSD_SOURCE -#include -#include - -void *reallocarray(void *ptr, size_t m, size_t n) -{ - if (n && m > -1 / n) { - errno = ENOMEM; - return 0; - } - - return realloc(ptr, m * n); -} diff --git a/lib/libc/musl/src/malloc/replaced.c b/lib/libc/musl/src/malloc/replaced.c deleted file mode 100644 index 07fce61ec08d67f39fce99e659719042ad17f356..0000000000000000000000000000000000000000 --- a/lib/libc/musl/src/malloc/replaced.c +++ /dev/null @@ -1,4 +0,0 @@ -#include "dynlink.h" - -int __malloc_replaced; -int __aligned_alloc_replaced; diff --git a/lib/libc/wasi/emmalloc/emmalloc.c b/lib/libc/wasi/emmalloc/emmalloc.c deleted file mode 100644 index e97ef44dc058772c640a2be5889513dce73ef592..0000000000000000000000000000000000000000 --- a/lib/libc/wasi/emmalloc/emmalloc.c +++ /dev/null @@ -1,1540 +0,0 @@ -/* - * Copyright 2018 The Emscripten Authors. All rights reserved. - * Emscripten is available under two separate licenses, the MIT license and the - * University of Illinois/NCSA Open Source License. Both these licenses can be - * found in the LICENSE file. - * - * Simple minimalistic but efficient sbrk()-based malloc/free that works in - * singlethreaded and multithreaded builds. - * - * Assumptions: - * - * - sbrk() is used to claim new memory (sbrk handles geometric/linear - * - overallocation growth) - * - sbrk() can be used by other code outside emmalloc. - * - sbrk() is very fast in most cases (internal wasm call). - * - sbrk() returns pointers with an alignment of alignof(max_align_t) - * - * Invariants: - * - * - Per-allocation header overhead is 8 bytes, smallest allocated payload - * amount is 8 bytes, and a multiple of 4 bytes. - * - Acquired memory blocks are subdivided into disjoint regions that lie - * next to each other. - * - A region is either in used or free. - * Used regions may be adjacent, and a used and unused region - * may be adjacent, but not two unused ones - they would be - * merged. - * - Memory allocation takes constant time, unless the alloc needs to sbrk() - * or memory is very close to being exhausted. - * - * Debugging: - * - * - If not NDEBUG, runtime assert()s are in use. - * - If EMMALLOC_MEMVALIDATE is defined, a large amount of extra checks are done. - * - If EMMALLOC_VERBOSE is defined, a lot of operations are logged - * out, in addition to EMMALLOC_MEMVALIDATE. - * - Debugging and logging directly uses console.log via uses EM_ASM, not - * printf etc., to minimize any risk of debugging or logging depending on - * malloc. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __EMSCRIPTEN_TRACING__ -#include -#endif - -// Defind by the linker to have the address of the start of the heap. -extern unsigned char __heap_base; -extern unsigned char __heap_end; - -// Behavior of right shifting a signed integer is compiler implementation defined. -static_assert((((int32_t)0x80000000U) >> 31) == -1, "This malloc implementation requires that right-shifting a signed integer produces a sign-extending (arithmetic) shift!"); - -// Configuration: specifies the minimum alignment that malloc()ed memory outputs. Allocation requests with smaller alignment -// than this will yield an allocation with this much alignment. -#define MALLOC_ALIGNMENT alignof(max_align_t) -static_assert(alignof(max_align_t) == 16, "max_align_t must be correct"); - -#define EMMALLOC_EXPORT __attribute__((weak)) - -#define MIN(x, y) ((x) < (y) ? (x) : (y)) -#define MAX(x, y) ((x) > (y) ? (x) : (y)) - -#define NUM_FREE_BUCKETS 64 -#define BUCKET_BITMASK_T uint64_t - -// Dynamic memory is subdivided into regions, in the format - -// ..... | ..... | ..... | ..... - -// That is, at the bottom and top end of each memory region, the size of that region is stored. That allows traversing the -// memory regions backwards and forwards. Because each allocation must be at least a multiple of 4 bytes, the lowest two bits of -// each size field is unused. Free regions are distinguished by used regions by having the FREE_REGION_FLAG bit present -// in the size field. I.e. for free regions, the size field is odd, and for used regions, the size field reads even. -#define FREE_REGION_FLAG 0x1u - -// Attempts to malloc() more than this many bytes would cause an overflow when calculating the size of a region, -// therefore allocations larger than this are short-circuited immediately on entry. -#define MAX_ALLOC_SIZE 0xFFFFFFC7u - -// A free region has the following structure: -// ... - -typedef struct Region -{ - size_t size; - // Use a circular doubly linked list to represent free region data. - struct Region *prev, *next; - // ... N bytes of free data - size_t _at_the_end_of_this_struct_size; // do not dereference, this is present for convenient struct sizeof() computation only -} Region; - -// Each memory block starts with a RootRegion at the beginning. -// The RootRegion specifies the size of the region block, and forms a linked -// list of all RootRegions in the program, starting with `listOfAllRegions` -// below. -typedef struct RootRegion -{ - uint32_t size; - struct RootRegion *next; - uint8_t* endPtr; -} RootRegion; - -#if defined(__EMSCRIPTEN_PTHREADS__) -// In multithreaded builds, use a simple global spinlock strategy to acquire/release access to the memory allocator. -static volatile uint8_t multithreadingLock = 0; -#define MALLOC_ACQUIRE() while(__sync_lock_test_and_set(&multithreadingLock, 1)) { while(multithreadingLock) { /*nop*/ } } -#define MALLOC_RELEASE() __sync_lock_release(&multithreadingLock) -// Test code to ensure we have tight malloc acquire/release guards in place. -#define ASSERT_MALLOC_IS_ACQUIRED() assert(multithreadingLock == 1) -#else -// In singlethreaded builds, no need for locking. -#define MALLOC_ACQUIRE() ((void)0) -#define MALLOC_RELEASE() ((void)0) -#define ASSERT_MALLOC_IS_ACQUIRED() ((void)0) -#endif - -#define IS_POWER_OF_2(val) (((val) & ((val)-1)) == 0) -#define ALIGN_UP(ptr, alignment) ((uint8_t*)((((uintptr_t)(ptr)) + ((alignment)-1)) & ~((alignment)-1))) -#define HAS_ALIGNMENT(ptr, alignment) ((((uintptr_t)(ptr)) & ((alignment)-1)) == 0) - -static_assert(IS_POWER_OF_2(MALLOC_ALIGNMENT), "MALLOC_ALIGNMENT must be a power of two value!"); -static_assert(MALLOC_ALIGNMENT >= 4, "Smallest possible MALLOC_ALIGNMENT if 4!"); - -// A region that contains as payload a single forward linked list of pointers to -// root regions of each disjoint region blocks. -static RootRegion *listOfAllRegions = NULL; - -// For each of the buckets, maintain a linked list head node. The head node for each -// free region is a sentinel node that does not actually represent any free space, but -// the sentinel is used to avoid awkward testing against (if node == freeRegionHeadNode) -// when adding and removing elements from the linked list, i.e. we are guaranteed that -// the sentinel node is always fixed and there, and the actual free region list elements -// start at freeRegionBuckets[i].next each. -static Region freeRegionBuckets[NUM_FREE_BUCKETS] = { - { .prev = &freeRegionBuckets[0], .next = &freeRegionBuckets[0] }, - { .prev = &freeRegionBuckets[1], .next = &freeRegionBuckets[1] }, - { .prev = &freeRegionBuckets[2], .next = &freeRegionBuckets[2] }, - { .prev = &freeRegionBuckets[3], .next = &freeRegionBuckets[3] }, - { .prev = &freeRegionBuckets[4], .next = &freeRegionBuckets[4] }, - { .prev = &freeRegionBuckets[5], .next = &freeRegionBuckets[5] }, - { .prev = &freeRegionBuckets[6], .next = &freeRegionBuckets[6] }, - { .prev = &freeRegionBuckets[7], .next = &freeRegionBuckets[7] }, - { .prev = &freeRegionBuckets[8], .next = &freeRegionBuckets[8] }, - { .prev = &freeRegionBuckets[9], .next = &freeRegionBuckets[9] }, - { .prev = &freeRegionBuckets[10], .next = &freeRegionBuckets[10] }, - { .prev = &freeRegionBuckets[11], .next = &freeRegionBuckets[11] }, - { .prev = &freeRegionBuckets[12], .next = &freeRegionBuckets[12] }, - { .prev = &freeRegionBuckets[13], .next = &freeRegionBuckets[13] }, - { .prev = &freeRegionBuckets[14], .next = &freeRegionBuckets[14] }, - { .prev = &freeRegionBuckets[15], .next = &freeRegionBuckets[15] }, - { .prev = &freeRegionBuckets[16], .next = &freeRegionBuckets[16] }, - { .prev = &freeRegionBuckets[17], .next = &freeRegionBuckets[17] }, - { .prev = &freeRegionBuckets[18], .next = &freeRegionBuckets[18] }, - { .prev = &freeRegionBuckets[19], .next = &freeRegionBuckets[19] }, - { .prev = &freeRegionBuckets[20], .next = &freeRegionBuckets[20] }, - { .prev = &freeRegionBuckets[21], .next = &freeRegionBuckets[21] }, - { .prev = &freeRegionBuckets[22], .next = &freeRegionBuckets[22] }, - { .prev = &freeRegionBuckets[23], .next = &freeRegionBuckets[23] }, - { .prev = &freeRegionBuckets[24], .next = &freeRegionBuckets[24] }, - { .prev = &freeRegionBuckets[25], .next = &freeRegionBuckets[25] }, - { .prev = &freeRegionBuckets[26], .next = &freeRegionBuckets[26] }, - { .prev = &freeRegionBuckets[27], .next = &freeRegionBuckets[27] }, - { .prev = &freeRegionBuckets[28], .next = &freeRegionBuckets[28] }, - { .prev = &freeRegionBuckets[29], .next = &freeRegionBuckets[29] }, - { .prev = &freeRegionBuckets[30], .next = &freeRegionBuckets[30] }, - { .prev = &freeRegionBuckets[31], .next = &freeRegionBuckets[31] }, - { .prev = &freeRegionBuckets[32], .next = &freeRegionBuckets[32] }, - { .prev = &freeRegionBuckets[33], .next = &freeRegionBuckets[33] }, - { .prev = &freeRegionBuckets[34], .next = &freeRegionBuckets[34] }, - { .prev = &freeRegionBuckets[35], .next = &freeRegionBuckets[35] }, - { .prev = &freeRegionBuckets[36], .next = &freeRegionBuckets[36] }, - { .prev = &freeRegionBuckets[37], .next = &freeRegionBuckets[37] }, - { .prev = &freeRegionBuckets[38], .next = &freeRegionBuckets[38] }, - { .prev = &freeRegionBuckets[39], .next = &freeRegionBuckets[39] }, - { .prev = &freeRegionBuckets[40], .next = &freeRegionBuckets[40] }, - { .prev = &freeRegionBuckets[41], .next = &freeRegionBuckets[41] }, - { .prev = &freeRegionBuckets[42], .next = &freeRegionBuckets[42] }, - { .prev = &freeRegionBuckets[43], .next = &freeRegionBuckets[43] }, - { .prev = &freeRegionBuckets[44], .next = &freeRegionBuckets[44] }, - { .prev = &freeRegionBuckets[45], .next = &freeRegionBuckets[45] }, - { .prev = &freeRegionBuckets[46], .next = &freeRegionBuckets[46] }, - { .prev = &freeRegionBuckets[47], .next = &freeRegionBuckets[47] }, - { .prev = &freeRegionBuckets[48], .next = &freeRegionBuckets[48] }, - { .prev = &freeRegionBuckets[49], .next = &freeRegionBuckets[49] }, - { .prev = &freeRegionBuckets[50], .next = &freeRegionBuckets[50] }, - { .prev = &freeRegionBuckets[51], .next = &freeRegionBuckets[51] }, - { .prev = &freeRegionBuckets[52], .next = &freeRegionBuckets[52] }, - { .prev = &freeRegionBuckets[53], .next = &freeRegionBuckets[53] }, - { .prev = &freeRegionBuckets[54], .next = &freeRegionBuckets[54] }, - { .prev = &freeRegionBuckets[55], .next = &freeRegionBuckets[55] }, - { .prev = &freeRegionBuckets[56], .next = &freeRegionBuckets[56] }, - { .prev = &freeRegionBuckets[57], .next = &freeRegionBuckets[57] }, - { .prev = &freeRegionBuckets[58], .next = &freeRegionBuckets[58] }, - { .prev = &freeRegionBuckets[59], .next = &freeRegionBuckets[59] }, - { .prev = &freeRegionBuckets[60], .next = &freeRegionBuckets[60] }, - { .prev = &freeRegionBuckets[61], .next = &freeRegionBuckets[61] }, - { .prev = &freeRegionBuckets[62], .next = &freeRegionBuckets[62] }, - { .prev = &freeRegionBuckets[63], .next = &freeRegionBuckets[63] }, -}; - -// A bitmask that tracks the population status for each of the 64 distinct memory regions: -// a zero at bit position i means that the free list bucket i is empty. This bitmask is -// used to avoid redundant scanning of the 64 different free region buckets: instead by -// looking at the bitmask we can find in constant time an index to a free region bucket -// that contains free memory of desired size. -static BUCKET_BITMASK_T freeRegionBucketsUsed = 0; - -// Amount of bytes taken up by allocation header data -#define REGION_HEADER_SIZE (2*sizeof(size_t)) - -// Smallest allocation size that is possible is 2*pointer size, since payload of each region must at least contain space -// to store the free region linked list prev and next pointers. An allocation size smaller than this will be rounded up -// to this size. -#define SMALLEST_ALLOCATION_SIZE (2*sizeof(void*)) - -/* Subdivide regions of free space into distinct circular doubly linked lists, where each linked list -represents a range of free space blocks. The following function compute_free_list_bucket() converts -an allocation size to the bucket index that should be looked at. The buckets are grouped as follows: - - Bucket 0: [8, 15], range size=8 - Bucket 1: [16, 23], range size=8 - Bucket 2: [24, 31], range size=8 - Bucket 3: [32, 39], range size=8 - Bucket 4: [40, 47], range size=8 - Bucket 5: [48, 55], range size=8 - Bucket 6: [56, 63], range size=8 - Bucket 7: [64, 71], range size=8 - Bucket 8: [72, 79], range size=8 - Bucket 9: [80, 87], range size=8 - Bucket 10: [88, 95], range size=8 - Bucket 11: [96, 103], range size=8 - Bucket 12: [104, 111], range size=8 - Bucket 13: [112, 119], range size=8 - Bucket 14: [120, 159], range size=40 - Bucket 15: [160, 191], range size=32 - Bucket 16: [192, 223], range size=32 - Bucket 17: [224, 255], range size=32 - Bucket 18: [256, 319], range size=64 - Bucket 19: [320, 383], range size=64 - Bucket 20: [384, 447], range size=64 - Bucket 21: [448, 511], range size=64 - Bucket 22: [512, 639], range size=128 - Bucket 23: [640, 767], range size=128 - Bucket 24: [768, 895], range size=128 - Bucket 25: [896, 1023], range size=128 - Bucket 26: [1024, 1279], range size=256 - Bucket 27: [1280, 1535], range size=256 - Bucket 28: [1536, 1791], range size=256 - Bucket 29: [1792, 2047], range size=256 - Bucket 30: [2048, 2559], range size=512 - Bucket 31: [2560, 3071], range size=512 - Bucket 32: [3072, 3583], range size=512 - Bucket 33: [3584, 6143], range size=2560 - Bucket 34: [6144, 8191], range size=2048 - Bucket 35: [8192, 12287], range size=4096 - Bucket 36: [12288, 16383], range size=4096 - Bucket 37: [16384, 24575], range size=8192 - Bucket 38: [24576, 32767], range size=8192 - Bucket 39: [32768, 49151], range size=16384 - Bucket 40: [49152, 65535], range size=16384 - Bucket 41: [65536, 98303], range size=32768 - Bucket 42: [98304, 131071], range size=32768 - Bucket 43: [131072, 196607], range size=65536 - Bucket 44: [196608, 262143], range size=65536 - Bucket 45: [262144, 393215], range size=131072 - Bucket 46: [393216, 524287], range size=131072 - Bucket 47: [524288, 786431], range size=262144 - Bucket 48: [786432, 1048575], range size=262144 - Bucket 49: [1048576, 1572863], range size=524288 - Bucket 50: [1572864, 2097151], range size=524288 - Bucket 51: [2097152, 3145727], range size=1048576 - Bucket 52: [3145728, 4194303], range size=1048576 - Bucket 53: [4194304, 6291455], range size=2097152 - Bucket 54: [6291456, 8388607], range size=2097152 - Bucket 55: [8388608, 12582911], range size=4194304 - Bucket 56: [12582912, 16777215], range size=4194304 - Bucket 57: [16777216, 25165823], range size=8388608 - Bucket 58: [25165824, 33554431], range size=8388608 - Bucket 59: [33554432, 50331647], range size=16777216 - Bucket 60: [50331648, 67108863], range size=16777216 - Bucket 61: [67108864, 100663295], range size=33554432 - Bucket 62: [100663296, 134217727], range size=33554432 - Bucket 63: 134217728 bytes and larger. */ -static_assert(NUM_FREE_BUCKETS == 64, "Following function is tailored specifically for NUM_FREE_BUCKETS == 64 case"); -static int compute_free_list_bucket(size_t allocSize) -{ - if (allocSize < 128) return (allocSize >> 3) - 1; - int clz = __builtin_clz(allocSize); - int bucketIndex = (clz > 19) ? 110 - (clz<<2) + ((allocSize >> (29-clz)) ^ 4) : MIN(71 - (clz<<1) + ((allocSize >> (30-clz)) ^ 2), NUM_FREE_BUCKETS-1); - assert(bucketIndex >= 0); - assert(bucketIndex < NUM_FREE_BUCKETS); - return bucketIndex; -} - -#define DECODE_CEILING_SIZE(size) ((size_t)((size) & ~FREE_REGION_FLAG)) - -static Region *prev_region(Region *region) -{ - size_t prevRegionSize = ((size_t*)region)[-1]; - prevRegionSize = DECODE_CEILING_SIZE(prevRegionSize); - return (Region*)((uint8_t*)region - prevRegionSize); -} - -static Region *next_region(Region *region) -{ - return (Region*)((uint8_t*)region + region->size); -} - -static size_t region_ceiling_size(Region *region) -{ - return ((size_t*)((uint8_t*)region + region->size))[-1]; -} - -static bool region_is_free(Region *r) -{ - return region_ceiling_size(r) & FREE_REGION_FLAG; -} - -static bool region_is_in_use(Region *r) -{ - return r->size == region_ceiling_size(r); -} - -static size_t size_of_region_from_ceiling(Region *r) -{ - size_t size = region_ceiling_size(r); - return DECODE_CEILING_SIZE(size); -} - -static bool debug_region_is_consistent(Region *r) -{ - assert(r); - size_t sizeAtBottom = r->size; - size_t sizeAtCeiling = size_of_region_from_ceiling(r); - return sizeAtBottom == sizeAtCeiling; -} - -static uint8_t *region_payload_start_ptr(Region *region) -{ - return (uint8_t*)region + sizeof(size_t); -} - -static uint8_t *region_payload_end_ptr(Region *region) -{ - return (uint8_t*)region + region->size - sizeof(size_t); -} - -static void create_used_region(void *ptr, size_t size) -{ - assert(ptr); - assert(HAS_ALIGNMENT(ptr, sizeof(size_t))); - assert(HAS_ALIGNMENT(size, sizeof(size_t))); - assert(size >= sizeof(Region)); - *(size_t*)ptr = size; - ((size_t*)ptr)[(size/sizeof(size_t))-1] = size; -} - -static void create_free_region(void *ptr, size_t size) -{ - assert(ptr); - assert(HAS_ALIGNMENT(ptr, sizeof(size_t))); - assert(HAS_ALIGNMENT(size, sizeof(size_t))); - assert(size >= sizeof(Region)); - Region *freeRegion = (Region*)ptr; - freeRegion->size = size; - ((size_t*)ptr)[(size/sizeof(size_t))-1] = size | FREE_REGION_FLAG; -} - -static void prepend_to_free_list(Region *region, Region *prependTo) -{ - assert(region); - assert(prependTo); - // N.b. the region we are prepending to is always the sentinel node, - // which represents a dummy node that is technically not a free node, so - // region_is_free(prependTo) does not hold. - assert(region_is_free((Region*)region)); - region->next = prependTo; - region->prev = prependTo->prev; - assert(region->prev); - prependTo->prev = region; - region->prev->next = region; -} - -static void unlink_from_free_list(Region *region) -{ - assert(region); - assert(region_is_free((Region*)region)); - assert(region->prev); - assert(region->next); - region->prev->next = region->next; - region->next->prev = region->prev; -} - -static void link_to_free_list(Region *freeRegion) -{ - assert(freeRegion); - assert(freeRegion->size >= sizeof(Region)); - int bucketIndex = compute_free_list_bucket(freeRegion->size-REGION_HEADER_SIZE); - Region *freeListHead = freeRegionBuckets + bucketIndex; - freeRegion->prev = freeListHead; - freeRegion->next = freeListHead->next; - assert(freeRegion->next); - freeListHead->next = freeRegion; - freeRegion->next->prev = freeRegion; - freeRegionBucketsUsed |= ((BUCKET_BITMASK_T)1) << bucketIndex; -} - -#if 0 -static void dump_memory_regions() -{ - ASSERT_MALLOC_IS_ACQUIRED(); - RootRegion *root = listOfAllRegions; - MAIN_THREAD_ASYNC_EM_ASM(console.log('All memory regions:')); - while(root) - { - Region *r = (Region*)root; - assert(debug_region_is_consistent(r)); - uint8_t *lastRegionEnd = root->endPtr; - MAIN_THREAD_ASYNC_EM_ASM(console.log('Region block 0x'+($0>>>0).toString(16)+' - 0x'+($1>>>0).toString(16)+ ' ('+($2>>>0)+' bytes):'), - r, lastRegionEnd, lastRegionEnd-(uint8_t*)r); - while((uint8_t*)r < lastRegionEnd) - { - MAIN_THREAD_ASYNC_EM_ASM(console.log('Region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+')'), - r, r->size, region_ceiling_size(r) == r->size); - - assert(debug_region_is_consistent(r)); - size_t sizeFromCeiling = size_of_region_from_ceiling(r); - if (sizeFromCeiling != r->size) - MAIN_THREAD_ASYNC_EM_ASM(console.log('Corrupt region! Size marker at the end of the region does not match: '+($0>>>0)), sizeFromCeiling); - if (r->size == 0) - break; - r = next_region(r); - } - root = root->next; - MAIN_THREAD_ASYNC_EM_ASM(console.log("")); - } - MAIN_THREAD_ASYNC_EM_ASM(console.log('Free regions:')); - for(int i = 0; i < NUM_FREE_BUCKETS; ++i) - { - Region *prev = &freeRegionBuckets[i]; - Region *fr = freeRegionBuckets[i].next; - while(fr != &freeRegionBuckets[i]) - { - MAIN_THREAD_ASYNC_EM_ASM(console.log('In bucket '+$0+', free region 0x'+($1>>>0).toString(16)+', size: ' + ($2>>>0) + ' (size at ceiling: '+($3>>>0)+'), prev: 0x' + ($4>>>0).toString(16) + ', next: 0x' + ($5>>>0).toString(16)), - i, fr, fr->size, size_of_region_from_ceiling(fr), fr->prev, fr->next); - assert(debug_region_is_consistent(fr)); - assert(region_is_free(fr)); - assert(fr->prev == prev); - prev = fr; - assert(fr->next != fr); - assert(fr->prev != fr); - fr = fr->next; - } - } - MAIN_THREAD_ASYNC_EM_ASM(console.log('Free bucket index map: ' + ($0>>>0).toString(2) + ' ' + ($1>>>0).toString(2)), (uint32_t)(freeRegionBucketsUsed >> 32), (uint32_t)freeRegionBucketsUsed); - MAIN_THREAD_ASYNC_EM_ASM(console.log("")); -} - -void emmalloc_dump_memory_regions() -{ - MALLOC_ACQUIRE(); - dump_memory_regions(); - MALLOC_RELEASE(); -} - -static int validate_memory_regions() -{ - ASSERT_MALLOC_IS_ACQUIRED(); - RootRegion *root = listOfAllRegions; - while(root) - { - Region *r = (Region*)root; - if (!debug_region_is_consistent(r)) - { - MAIN_THREAD_ASYNC_EM_ASM(console.error('Used region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+') is corrupt (size markers in the beginning and at the end of the region do not match!)'), - r, r->size, region_ceiling_size(r) == r->size); - return 1; - } - uint8_t *lastRegionEnd = root->endPtr; - while((uint8_t*)r < lastRegionEnd) - { - if (!debug_region_is_consistent(r)) - { - MAIN_THREAD_ASYNC_EM_ASM(console.error('Used region 0x'+($0>>>0).toString(16)+', size: '+($1>>>0)+' ('+($2?"used":"--FREE--")+') is corrupt (size markers in the beginning and at the end of the region do not match!)'), - r, r->size, region_ceiling_size(r) == r->size); - return 1; - } - if (r->size == 0) - break; - r = next_region(r); - } - root = root->next; - } - for(int i = 0; i < NUM_FREE_BUCKETS; ++i) - { - Region *prev = &freeRegionBuckets[i]; - Region *fr = freeRegionBuckets[i].next; - while(fr != &freeRegionBuckets[i]) - { - if (!debug_region_is_consistent(fr) || !region_is_free(fr) || fr->prev != prev || fr->next == fr || fr->prev == fr) - { - MAIN_THREAD_ASYNC_EM_ASM(console.log('In bucket '+$0+', free region 0x'+($1>>>0).toString(16)+', size: ' + ($2>>>0) + ' (size at ceiling: '+($3>>>0)+'), prev: 0x' + ($4>>>0).toString(16) + ', next: 0x' + ($5>>>0).toString(16) + ' is corrupt!'), - i, fr, fr->size, size_of_region_from_ceiling(fr), fr->prev, fr->next); - return 1; - } - prev = fr; - fr = fr->next; - } - } - return 0; -} - -int emmalloc_validate_memory_regions() -{ - MALLOC_ACQUIRE(); - int memoryError = validate_memory_regions(); - MALLOC_RELEASE(); - return memoryError; -} -#endif - -static bool claim_more_memory(size_t numBytes) -{ -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('claim_more_memory(numBytes='+($0>>>0)+ ')'), numBytes); -#endif - -#ifdef EMMALLOC_MEMVALIDATE - validate_memory_regions(); -#endif - - uint8_t *startPtr; - uint8_t *endPtr; - do { - // If this is the first time we're called, see if we can use - // the initial heap memory set up by wasm-ld. - if (!listOfAllRegions) { - unsigned char *heap_base = &__heap_base; - unsigned char *heap_end = &__heap_end; - if (heap_end < heap_base) { - __builtin_trap(); - } - if (numBytes <= (size_t)(heap_end - heap_base)) { - startPtr = heap_base; - endPtr = heap_end; - break; - } - } - - // Round numBytes up to the nearest page size. - numBytes = (numBytes + (PAGE_SIZE-1)) & -PAGE_SIZE; - - // Claim memory via sbrk - startPtr = (uint8_t*)sbrk(numBytes); - if ((intptr_t)startPtr == -1) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.error('claim_more_memory: sbrk failed!')); -#endif - return false; - } -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('claim_more_memory: claimed 0x' + ($0>>>0).toString(16) + ' - 0x' + ($1>>>0).toString(16) + ' (' + ($2>>>0) + ' bytes) via sbrk()'), startPtr, startPtr + numBytes, numBytes); -#endif - assert(HAS_ALIGNMENT(startPtr, alignof(size_t))); - endPtr = startPtr + numBytes; - } while (0); - - // Create a sentinel region at the end of the new heap block - Region *endSentinelRegion = (Region*)(endPtr - sizeof(Region)); - create_used_region(endSentinelRegion, sizeof(Region)); - - // If we are the sole user of sbrk(), it will feed us continuous/consecutive memory addresses - take advantage - // of that if so: instead of creating two disjoint memory regions blocks, expand the previous one to a larger size. - uint8_t *previousSbrkEndAddress = listOfAllRegions ? listOfAllRegions->endPtr : 0; - if (startPtr == previousSbrkEndAddress) - { - Region *prevEndSentinel = prev_region((Region*)startPtr); - assert(debug_region_is_consistent(prevEndSentinel)); - assert(region_is_in_use(prevEndSentinel)); - Region *prevRegion = prev_region(prevEndSentinel); - assert(debug_region_is_consistent(prevRegion)); - - listOfAllRegions->endPtr = endPtr; - - // Two scenarios, either the last region of the previous block was in use, in which case we need to create - // a new free region in the newly allocated space; or it was free, in which case we can extend that region - // to cover a larger size. - if (region_is_free(prevRegion)) - { - size_t newFreeRegionSize = (uint8_t*)endSentinelRegion - (uint8_t*)prevRegion; - unlink_from_free_list(prevRegion); - create_free_region(prevRegion, newFreeRegionSize); - link_to_free_list(prevRegion); - return true; - } - // else: last region of the previous block was in use. Since we are joining two consecutive sbrk() blocks, - // we can swallow the end sentinel of the previous block away. - startPtr -= sizeof(Region); - } - else - { - // Create a root region at the start of the heap block - create_used_region(startPtr, sizeof(Region)); - - // Dynamic heap start region: - RootRegion *newRegionBlock = (RootRegion*)startPtr; - newRegionBlock->next = listOfAllRegions; // Pointer to next region block head - newRegionBlock->endPtr = endPtr; // Pointer to the end address of this region block - listOfAllRegions = newRegionBlock; - startPtr += sizeof(Region); - } - - // Create a new memory region for the new claimed free space. - create_free_region(startPtr, (uint8_t*)endSentinelRegion - startPtr); - link_to_free_list((Region*)startPtr); - return true; -} - -#if 0 -// Initialize emmalloc during static initialization. -// See system/lib/README.md for static constructor ordering. -__attribute__((constructor(47))) -static void initialize_emmalloc_heap() -{ - // Initialize circular doubly linked lists representing free space - // Never useful to unroll this for loop, just takes up code size. -#pragma clang loop unroll(disable) - for(int i = 0; i < NUM_FREE_BUCKETS; ++i) - freeRegionBuckets[i].prev = freeRegionBuckets[i].next = &freeRegionBuckets[i]; - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('initialize_emmalloc_heap()')); -#endif - - // Start with a tiny dynamic region. - claim_more_memory(3*sizeof(Region)); -} - -void emmalloc_blank_slate_from_orbit() -{ - MALLOC_ACQUIRE(); - listOfAllRegions = NULL; - freeRegionBucketsUsed = 0; - initialize_emmalloc_heap(); - MALLOC_RELEASE(); -} -#endif - -static void *attempt_allocate(Region *freeRegion, size_t alignment, size_t size) -{ - ASSERT_MALLOC_IS_ACQUIRED(); - assert(freeRegion); - // Look at the next potential free region to allocate into. - // First, we should check if the free region has enough of payload bytes contained - // in it to accommodate the new allocation. This check needs to take account the - // requested allocation alignment, so the payload memory area needs to be rounded - // upwards to the desired alignment. - uint8_t *payloadStartPtr = region_payload_start_ptr(freeRegion); - uint8_t *payloadStartPtrAligned = ALIGN_UP(payloadStartPtr, alignment); - uint8_t *payloadEndPtr = region_payload_end_ptr(freeRegion); - - // Do we have enough free space, taking into account alignment? - if (payloadStartPtrAligned + size > payloadEndPtr) - return NULL; - - // We have enough free space, so the memory allocation will be made into this region. Remove this free region - // from the list of free regions: whatever slop remains will be later added back to the free region pool. - unlink_from_free_list(freeRegion); - - // Before we proceed further, fix up the boundary of this region and the region that precedes this one, - // so that the boundary between the two regions happens at a right spot for the payload to be aligned. - if (payloadStartPtr != payloadStartPtrAligned) - { - Region *prevRegion = prev_region((Region*)freeRegion); - // We never have two free regions adjacent to each other, so the region before this free - // region should be in use. - assert(region_is_in_use(prevRegion)); - size_t regionBoundaryBumpAmount = payloadStartPtrAligned - payloadStartPtr; - size_t newThisRegionSize = freeRegion->size - regionBoundaryBumpAmount; - create_used_region(prevRegion, prevRegion->size + regionBoundaryBumpAmount); - freeRegion = (Region *)((uint8_t*)freeRegion + regionBoundaryBumpAmount); - freeRegion->size = newThisRegionSize; - } - // Next, we need to decide whether this region is so large that it should be split into two regions, - // one representing the newly used memory area, and at the high end a remaining leftover free area. - // This splitting to two is done always if there is enough space for the high end to fit a region. - // Carve 'size' bytes of payload off this region. So, - // [sz prev next sz] - // becomes - // [sz payload sz] [sz prev next sz] - if (sizeof(Region) + REGION_HEADER_SIZE + size <= freeRegion->size) - { - // There is enough space to keep a free region at the end of the carved out block - // -> construct the new block - Region *newFreeRegion = (Region *)((uint8_t*)freeRegion + REGION_HEADER_SIZE + size); - create_free_region(newFreeRegion, freeRegion->size - size - REGION_HEADER_SIZE); - link_to_free_list(newFreeRegion); - - // Recreate the resized Region under its new size. - create_used_region(freeRegion, size + REGION_HEADER_SIZE); - } - else - { - // There is not enough space to split the free memory region into used+free parts, so consume the whole - // region as used memory, not leaving a free memory region behind. - // Initialize the free region as used by resetting the ceiling size to the same value as the size at bottom. - ((size_t*)((uint8_t*)freeRegion + freeRegion->size))[-1] = freeRegion->size; - } - -#ifdef __EMSCRIPTEN_TRACING__ - emscripten_trace_record_allocation(freeRegion, freeRegion->size); -#endif - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_allocate - succeeded allocating memory, region ptr=0x' + ($0>>>0).toString(16) + ', align=' + $1 + ', payload size=' + ($2>>>0) + ' bytes)'), freeRegion, alignment, size); -#endif - - return (uint8_t*)freeRegion + sizeof(size_t); -} - -static size_t validate_alloc_alignment(size_t alignment) -{ - // Cannot perform allocations that are less than 4 byte aligned, because the Region - // control structures need to be aligned. Also round up to minimum outputted alignment. - alignment = MAX(alignment, MALLOC_ALIGNMENT); - // Arbitrary upper limit on alignment - very likely a programming bug if alignment is higher than this. - assert(alignment <= 1024*1024); - return alignment; -} - -static size_t validate_alloc_size(size_t size) -{ - assert(size + REGION_HEADER_SIZE > size); - - // Allocation sizes must be a multiple of pointer sizes, and at least 2*sizeof(pointer). - size_t validatedSize = size > SMALLEST_ALLOCATION_SIZE ? (size_t)ALIGN_UP(size, sizeof(Region*)) : SMALLEST_ALLOCATION_SIZE; - assert(validatedSize >= size); // 32-bit wraparound should not occur, too large sizes should be stopped before - - return validatedSize; -} - -static void *allocate_memory(size_t alignment, size_t size) -{ - ASSERT_MALLOC_IS_ACQUIRED(); - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('allocate_memory(align=' + $0 + ', size=' + ($1>>>0) + ' bytes)'), alignment, size); -#endif - -#ifdef EMMALLOC_MEMVALIDATE - validate_memory_regions(); -#endif - - if (!IS_POWER_OF_2(alignment)) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: alignment not power of 2!')); -#endif - return 0; - } - - if (size > MAX_ALLOC_SIZE) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size); -#endif - return 0; - } - - alignment = validate_alloc_alignment(alignment); - size = validate_alloc_size(size); - - // Attempt to allocate memory starting from smallest bucket that can contain the required amount of memory. - // Under normal alignment conditions this should always be the first or second bucket we look at, but if - // performing an allocation with complex alignment, we may need to look at multiple buckets. - int bucketIndex = compute_free_list_bucket(size); - BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed >> bucketIndex; - - // Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap. - while(bucketMask) - { - BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask); - bucketIndex += indexAdd; - bucketMask >>= indexAdd; - assert(bucketIndex >= 0); - assert(bucketIndex <= NUM_FREE_BUCKETS-1); - assert(freeRegionBucketsUsed & (((BUCKET_BITMASK_T)1) << bucketIndex)); - - Region *freeRegion = freeRegionBuckets[bucketIndex].next; - assert(freeRegion); - if (freeRegion != &freeRegionBuckets[bucketIndex]) - { - void *ptr = attempt_allocate(freeRegion, alignment, size); - if (ptr) - return ptr; - - // We were not able to allocate from the first region found in this bucket, so penalize - // the region by cycling it to the end of the doubly circular linked list. (constant time) - // This provides a randomized guarantee that when performing allocations of size k to a - // bucket of [k-something, k+something] range, we will not always attempt to satisfy the - // allocation from the same available region at the front of the list, but we try each - // region in turn. - unlink_from_free_list(freeRegion); - prepend_to_free_list(freeRegion, &freeRegionBuckets[bucketIndex]); - // But do not stick around to attempt to look at other regions in this bucket - move - // to search the next populated bucket index if this did not fit. This gives a practical - // "allocation in constant time" guarantee, since the next higher bucket will only have - // regions that are all of strictly larger size than the requested allocation. Only if - // there is a difficult alignment requirement we may fail to perform the allocation from - // a region in the next bucket, and if so, we keep trying higher buckets until one of them - // works. - ++bucketIndex; - bucketMask >>= 1; - } - else - { - // This bucket was not populated after all with any regions, - // but we just had a stale bit set to mark a populated bucket. - // Reset the bit to update latest status so that we do not - // redundantly look at this bucket again. - freeRegionBucketsUsed &= ~(((BUCKET_BITMASK_T)1) << bucketIndex); - bucketMask ^= 1; - } - // Instead of recomputing bucketMask from scratch at the end of each loop, it is updated as we go, - // to avoid undefined behavior with (x >> 32)/(x >> 64) when bucketIndex reaches 32/64, (the shift would comes out as a no-op instead of 0). - - assert((bucketIndex == NUM_FREE_BUCKETS && bucketMask == 0) || (bucketMask == freeRegionBucketsUsed >> bucketIndex)); - } - - // None of the buckets were able to accommodate an allocation. If this happens we are almost out of memory. - // The largest bucket might contain some suitable regions, but we only looked at one region in that bucket, so - // as a last resort, loop through more free regions in the bucket that represents the largest allocations available. - // But only if the bucket representing largest allocations available is not any of the first thirty buckets, - // these represent allocatable areas less than <1024 bytes - which could be a lot of scrap. - // In such case, prefer to sbrk() in more memory right away. - int largestBucketIndex = NUM_FREE_BUCKETS - 1 - __builtin_clzll(freeRegionBucketsUsed); - // freeRegion will be null if there is absolutely no memory left. (all buckets are 100% used) - Region *freeRegion = freeRegionBucketsUsed ? freeRegionBuckets[largestBucketIndex].next : 0; - if (freeRegionBucketsUsed >> 30) - { - // Look only at a constant number of regions in this bucket max, to avoid bad worst case behavior. - // If this many regions cannot find free space, we give up and prefer to sbrk() more instead. - const int maxRegionsToTryBeforeGivingUp = 99; - int numTriesLeft = maxRegionsToTryBeforeGivingUp; - while(freeRegion != &freeRegionBuckets[largestBucketIndex] && numTriesLeft-- > 0) - { - void *ptr = attempt_allocate(freeRegion, alignment, size); - if (ptr) - return ptr; - freeRegion = freeRegion->next; - } - } - - // We were unable to find a free memory region. Must sbrk() in more memory! - size_t numBytesToClaim = size+sizeof(Region)*3; - assert(numBytesToClaim > size); // 32-bit wraparound should not happen here, allocation size has been validated above! - bool success = claim_more_memory(numBytesToClaim); - if (success) - return allocate_memory(alignment, size); // Recurse back to itself to try again - - // also sbrk() failed, we are really really constrained :( As a last resort, go back to looking at the - // bucket we already looked at above, continuing where the above search left off - perhaps there are - // regions we overlooked the first time that might be able to satisfy the allocation. - if (freeRegion) - { - while(freeRegion != &freeRegionBuckets[largestBucketIndex]) - { - void *ptr = attempt_allocate(freeRegion, alignment, size); - if (ptr) - return ptr; - freeRegion = freeRegion->next; - } - } - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Could not find a free memory block!')); -#endif - - return 0; -} - -static -void *emmalloc_memalign(size_t alignment, size_t size) -{ - MALLOC_ACQUIRE(); - void *ptr = allocate_memory(alignment, size); - MALLOC_RELEASE(); - return ptr; -} - -#if 0 -void * EMMALLOC_EXPORT memalign(size_t alignment, size_t size) -{ - return emmalloc_memalign(alignment, size); -} -#endif - -void * EMMALLOC_EXPORT aligned_alloc(size_t alignment, size_t size) -{ - if ((alignment % sizeof(void *) != 0) || (size % alignment) != 0) - return 0; - return emmalloc_memalign(alignment, size); -} - -static -void *emmalloc_malloc(size_t size) -{ - return emmalloc_memalign(MALLOC_ALIGNMENT, size); -} - -void * EMMALLOC_EXPORT malloc(size_t size) -{ - return emmalloc_malloc(size); -} - -static -size_t emmalloc_usable_size(void *ptr) -{ - if (!ptr) - return 0; - - uint8_t *regionStartPtr = (uint8_t*)ptr - sizeof(size_t); - Region *region = (Region*)(regionStartPtr); - assert(HAS_ALIGNMENT(region, sizeof(size_t))); - - MALLOC_ACQUIRE(); - - size_t size = region->size; - assert(size >= sizeof(Region)); - assert(region_is_in_use(region)); - - MALLOC_RELEASE(); - - return size - REGION_HEADER_SIZE; -} - -size_t EMMALLOC_EXPORT malloc_usable_size(void *ptr) -{ - return emmalloc_usable_size(ptr); -} - -static -void emmalloc_free(void *ptr) -{ -#ifdef EMMALLOC_MEMVALIDATE - emmalloc_validate_memory_regions(); -#endif - - if (!ptr) - return; - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('free(ptr=0x'+($0>>>0).toString(16)+')'), ptr); -#endif - - uint8_t *regionStartPtr = (uint8_t*)ptr - sizeof(size_t); - Region *region = (Region*)(regionStartPtr); - assert(HAS_ALIGNMENT(region, sizeof(size_t))); - - MALLOC_ACQUIRE(); - - size_t size = region->size; -#ifdef EMMALLOC_VERBOSE - if (size < sizeof(Region) || !region_is_in_use(region)) - { - if (debug_region_is_consistent(region)) - // LLVM wasm backend bug: cannot use MAIN_THREAD_ASYNC_EM_ASM() here, that generates internal compiler error - // Reproducible by running e.g. other.test_alloc_3GB - EM_ASM(console.error('Double free at region ptr 0x' + ($0>>>0).toString(16) + ', region->size: 0x' + ($1>>>0).toString(16) + ', region->sizeAtCeiling: 0x' + ($2>>>0).toString(16) + ')'), region, size, region_ceiling_size(region)); - else - MAIN_THREAD_ASYNC_EM_ASM(console.error('Corrupt region at region ptr 0x' + ($0>>>0).toString(16) + ' region->size: 0x' + ($1>>>0).toString(16) + ', region->sizeAtCeiling: 0x' + ($2>>>0).toString(16) + ')'), region, size, region_ceiling_size(region)); - } -#endif - assert(size >= sizeof(Region)); - assert(region_is_in_use(region)); - -#ifdef __EMSCRIPTEN_TRACING__ - emscripten_trace_record_free(region); -#endif - - // Check merging with left side - size_t prevRegionSizeField = ((size_t*)region)[-1]; - size_t prevRegionSize = prevRegionSizeField & ~FREE_REGION_FLAG; - if (prevRegionSizeField != prevRegionSize) // Previous region is free? - { - Region *prevRegion = (Region*)((uint8_t*)region - prevRegionSize); - assert(debug_region_is_consistent(prevRegion)); - unlink_from_free_list(prevRegion); - regionStartPtr = (uint8_t*)prevRegion; - size += prevRegionSize; - } - - // Check merging with right side - Region *nextRegion = next_region(region); - assert(debug_region_is_consistent(nextRegion)); - size_t sizeAtEnd = *(size_t*)region_payload_end_ptr(nextRegion); - if (nextRegion->size != sizeAtEnd) - { - unlink_from_free_list(nextRegion); - size += nextRegion->size; - } - - create_free_region(regionStartPtr, size); - link_to_free_list((Region*)regionStartPtr); - - MALLOC_RELEASE(); - -#ifdef EMMALLOC_MEMVALIDATE - emmalloc_validate_memory_regions(); -#endif -} - -void EMMALLOC_EXPORT free(void *ptr) -{ - emmalloc_free(ptr); -} - -// Can be called to attempt to increase or decrease the size of the given region -// to a new size (in-place). Returns 1 if resize succeeds, and 0 on failure. -static int attempt_region_resize(Region *region, size_t size) -{ - ASSERT_MALLOC_IS_ACQUIRED(); - assert(size > 0); - assert(HAS_ALIGNMENT(size, sizeof(size_t))); - -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_region_resize(region=0x' + ($0>>>0).toString(16) + ', size=' + ($1>>>0) + ' bytes)'), region, size); -#endif - - // First attempt to resize this region, if the next region that follows this one - // is a free region. - Region *nextRegion = next_region(region); - uint8_t *nextRegionEndPtr = (uint8_t*)nextRegion + nextRegion->size; - size_t sizeAtCeiling = ((size_t*)nextRegionEndPtr)[-1]; - if (nextRegion->size != sizeAtCeiling) // Next region is free? - { - assert(region_is_free(nextRegion)); - uint8_t *newNextRegionStartPtr = (uint8_t*)region + size; - assert(HAS_ALIGNMENT(newNextRegionStartPtr, sizeof(size_t))); - // Next region does not shrink to too small size? - if (newNextRegionStartPtr + sizeof(Region) <= nextRegionEndPtr) - { - unlink_from_free_list(nextRegion); - create_free_region(newNextRegionStartPtr, nextRegionEndPtr - newNextRegionStartPtr); - link_to_free_list((Region*)newNextRegionStartPtr); - create_used_region(region, newNextRegionStartPtr - (uint8_t*)region); - return 1; - } - // If we remove the next region altogether, allocation is satisfied? - if (newNextRegionStartPtr <= nextRegionEndPtr) - { - unlink_from_free_list(nextRegion); - create_used_region(region, region->size + nextRegion->size); - return 1; - } - } - else - { - // Next region is an used region - we cannot change its starting address. However if we are shrinking the - // size of this region, we can create a new free region between this and the next used region. - if (size + sizeof(Region) <= region->size) - { - size_t freeRegionSize = region->size - size; - create_used_region(region, size); - Region *freeRegion = (Region *)((uint8_t*)region + size); - create_free_region(freeRegion, freeRegionSize); - link_to_free_list(freeRegion); - return 1; - } - else if (size <= region->size) - { - // Caller was asking to shrink the size, but due to not being able to fit a full Region in the shrunk - // area, we cannot actually do anything. This occurs if the shrink amount is really small. In such case, - // just call it success without doing any work. - return 1; - } - } -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('attempt_region_resize failed.')); -#endif - return 0; -} - -static int acquire_and_attempt_region_resize(Region *region, size_t size) -{ - MALLOC_ACQUIRE(); - int success = attempt_region_resize(region, size); - MALLOC_RELEASE(); - return success; -} - -static -void *emmalloc_aligned_realloc(void *ptr, size_t alignment, size_t size) -{ -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('aligned_realloc(ptr=0x' + ($0>>>0).toString(16) + ', alignment=' + $1 + ', size=' + ($2>>>0)), ptr, alignment, size); -#endif - - if (!ptr) - return emmalloc_memalign(alignment, size); - - if (size == 0) - { - free(ptr); - return 0; - } - - if (size > MAX_ALLOC_SIZE) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size); -#endif - return 0; - } - - assert(IS_POWER_OF_2(alignment)); - // aligned_realloc() cannot be used to ask to change the alignment of a pointer. - assert(HAS_ALIGNMENT(ptr, alignment)); - size = validate_alloc_size(size); - - // Calculate the region start address of the original allocation - Region *region = (Region*)((uint8_t*)ptr - sizeof(size_t)); - - // First attempt to resize the given region to avoid having to copy memory around - if (acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE)) - { -#ifdef __EMSCRIPTEN_TRACING__ - emscripten_trace_record_reallocation(ptr, ptr, size); -#endif - return ptr; - } - - // If resize failed, we must allocate a new region, copy the data over, and then - // free the old region. - void *newptr = emmalloc_memalign(alignment, size); - if (newptr) - { - memcpy(newptr, ptr, MIN(size, region->size - REGION_HEADER_SIZE)); - free(ptr); - } - // N.B. If there is not enough memory, the old memory block should not be freed and - // null pointer is returned. - return newptr; -} - -#if 0 -void * EMMALLOC_EXPORT aligned_realloc(void *ptr, size_t alignment, size_t size) -{ - return emmalloc_aligned_realloc(ptr, alignment, size); -} -#endif - -#if 0 -// realloc_try() is like realloc(), but only attempts to try to resize the existing memory -// area. If resizing the existing memory area fails, then realloc_try() will return 0 -// (the original memory block is not freed or modified). If resizing succeeds, previous -// memory contents will be valid up to min(old length, new length) bytes. -void *emmalloc_realloc_try(void *ptr, size_t size) -{ - if (!ptr) - return 0; - - if (size == 0) - { - free(ptr); - return 0; - } - - if (size > MAX_ALLOC_SIZE) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size); -#endif - return 0; - } - - size = validate_alloc_size(size); - - // Calculate the region start address of the original allocation - Region *region = (Region*)((uint8_t*)ptr - sizeof(size_t)); - - // Attempt to resize the given region to avoid having to copy memory around - int success = acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE); -#ifdef __EMSCRIPTEN_TRACING__ - if (success) - emscripten_trace_record_reallocation(ptr, ptr, size); -#endif - return success ? ptr : 0; -} - -// emmalloc_aligned_realloc_uninitialized() is like aligned_realloc(), but old memory contents -// will be undefined after reallocation. (old memory is not preserved in any case) -void *emmalloc_aligned_realloc_uninitialized(void *ptr, size_t alignment, size_t size) -{ - if (!ptr) - return emmalloc_memalign(alignment, size); - - if (size == 0) - { - free(ptr); - return 0; - } - - if (size > MAX_ALLOC_SIZE) - { -#ifdef EMMALLOC_VERBOSE - MAIN_THREAD_ASYNC_EM_ASM(console.log('Allocation failed: attempted allocation size is too large: ' + ($0 >>> 0) + 'bytes! (negative integer wraparound?)'), size); -#endif - return 0; - } - - size = validate_alloc_size(size); - - // Calculate the region start address of the original allocation - Region *region = (Region*)((uint8_t*)ptr - sizeof(size_t)); - - // First attempt to resize the given region to avoid having to copy memory around - if (acquire_and_attempt_region_resize(region, size + REGION_HEADER_SIZE)) - { -#ifdef __EMSCRIPTEN_TRACING__ - emscripten_trace_record_reallocation(ptr, ptr, size); -#endif - return ptr; - } - - // If resize failed, drop the old region and allocate a new region. Memory is not - // copied over - free(ptr); - return emmalloc_memalign(alignment, size); -} -#endif - -static -void *emmalloc_realloc(void *ptr, size_t size) -{ - return emmalloc_aligned_realloc(ptr, MALLOC_ALIGNMENT, size); -} - -void * EMMALLOC_EXPORT realloc(void *ptr, size_t size) -{ - return emmalloc_realloc(ptr, size); -} - -#if 0 -// realloc_uninitialized() is like realloc(), but old memory contents -// will be undefined after reallocation. (old memory is not preserved in any case) -void *emmalloc_realloc_uninitialized(void *ptr, size_t size) -{ - return emmalloc_aligned_realloc_uninitialized(ptr, MALLOC_ALIGNMENT, size); -} -#endif - -static -int emmalloc_posix_memalign(void **memptr, size_t alignment, size_t size) -{ - assert(memptr); - if (alignment % sizeof(void *) != 0) - return 22/* EINVAL*/; - *memptr = emmalloc_memalign(alignment, size); - return *memptr ? 0 : 12/*ENOMEM*/; -} - -int EMMALLOC_EXPORT posix_memalign(void **memptr, size_t alignment, size_t size) -{ - return emmalloc_posix_memalign(memptr, alignment, size); -} - -static -void *emmalloc_calloc(size_t num, size_t size) -{ - size_t bytes = num*size; - void *ptr = emmalloc_memalign(MALLOC_ALIGNMENT, bytes); - if (ptr) - memset(ptr, 0, bytes); - return ptr; -} - -void * EMMALLOC_EXPORT calloc(size_t num, size_t size) -{ - return emmalloc_calloc(num, size); -} - -#if 0 -static int count_linked_list_size(Region *list) -{ - int size = 1; - for(Region *i = list->next; i != list; list = list->next) - ++size; - return size; -} - -static size_t count_linked_list_space(Region *list) -{ - size_t space = 0; - for(Region *i = list->next; i != list; list = list->next) - space += region_payload_end_ptr(i) - region_payload_start_ptr(i); - return space; -} - -struct mallinfo emmalloc_mallinfo() -{ - MALLOC_ACQUIRE(); - - struct mallinfo info; - // Non-mmapped space allocated (bytes): For emmalloc, - // let's define this as the difference between heap size and dynamic top end. - info.arena = emscripten_get_heap_size() - (size_t)sbrk(0); - // Number of "ordinary" blocks. Let's define this as the number of highest - // size blocks. (subtract one from each, since there is a sentinel node in each list) - info.ordblks = count_linked_list_size(&freeRegionBuckets[NUM_FREE_BUCKETS-1])-1; - // Number of free "fastbin" blocks. For emmalloc, define this as the number - // of blocks that are not in the largest pristine block. - info.smblks = 0; - // The total number of bytes in free "fastbin" blocks. - info.fsmblks = 0; - for(int i = 0; i < NUM_FREE_BUCKETS-1; ++i) - { - info.smblks += count_linked_list_size(&freeRegionBuckets[i])-1; - info.fsmblks += count_linked_list_space(&freeRegionBuckets[i]); - } - - info.hblks = 0; // Number of mmapped regions: always 0. (no mmap support) - info.hblkhd = 0; // Amount of bytes in mmapped regions: always 0. (no mmap support) - - // Walk through all the heap blocks to report the following data: - // The "highwater mark" for allocated space—that is, the maximum amount of - // space that was ever allocated. Emmalloc does not want to pay code to - // track this, so this is only reported from current allocation data, and - // may not be accurate. - info.usmblks = 0; - info.uordblks = 0; // The total number of bytes used by in-use allocations. - info.fordblks = 0; // The total number of bytes in free blocks. - // The total amount of releasable free space at the top of the heap. - // This is the maximum number of bytes that could ideally be released by malloc_trim(3). - Region *lastActualRegion = prev_region((Region*)(listOfAllRegions->endPtr - sizeof(Region))); - info.keepcost = region_is_free(lastActualRegion) ? lastActualRegion->size : 0; - - RootRegion *root = listOfAllRegions; - while(root) - { - Region *r = (Region*)root; - assert(debug_region_is_consistent(r)); - uint8_t *lastRegionEnd = root->endPtr; - while((uint8_t*)r < lastRegionEnd) - { - assert(debug_region_is_consistent(r)); - - if (region_is_free(r)) - { - // Count only the payload of the free block towards free memory. - info.fordblks += region_payload_end_ptr(r) - region_payload_start_ptr(r); - // But the header data of the free block goes towards used memory. - info.uordblks += REGION_HEADER_SIZE; - } - else - { - info.uordblks += r->size; - } - // Update approximate watermark data - info.usmblks = MAX(info.usmblks, (intptr_t)r + r->size); - - if (r->size == 0) - break; - r = next_region(r); - } - root = root->next; - } - - MALLOC_RELEASE(); - return info; -} - -struct mallinfo EMMALLOC_EXPORT mallinfo() -{ - return emmalloc_mallinfo(); -} - -// Note! This function is not fully multithreadin safe: while this function is running, other threads should not be -// allowed to call sbrk()! -static int trim_dynamic_heap_reservation(size_t pad) -{ - ASSERT_MALLOC_IS_ACQUIRED(); - - if (!listOfAllRegions) - return 0; // emmalloc is not controlling any dynamic memory at all - cannot release memory. - uint8_t *previousSbrkEndAddress = listOfAllRegions->endPtr; - assert(sbrk(0) == previousSbrkEndAddress); - size_t lastMemoryRegionSize = ((size_t*)previousSbrkEndAddress)[-1]; - assert(lastMemoryRegionSize == 16); // // The last memory region should be a sentinel node of exactly 16 bytes in size. - Region *endSentinelRegion = (Region*)(previousSbrkEndAddress - sizeof(Region)); - Region *lastActualRegion = prev_region(endSentinelRegion); - - // Round padding up to multiple of 4 bytes to keep sbrk() and memory region alignment intact. - // Also have at least 8 bytes of payload so that we can form a full free region. - size_t newRegionSize = (size_t)ALIGN_UP(pad, 4); - if (pad > 0) - newRegionSize += sizeof(Region) - (newRegionSize - pad); - - if (!region_is_free(lastActualRegion) || lastActualRegion->size <= newRegionSize) - return 0; // Last actual region is in use, or caller desired to leave more free memory intact than there is. - - // This many bytes will be shrunk away. - size_t shrinkAmount = lastActualRegion->size - newRegionSize; - assert(HAS_ALIGNMENT(shrinkAmount, 4)); - - unlink_from_free_list(lastActualRegion); - // If pad == 0, we should delete the last free region altogether. If pad > 0, - // shrink the last free region to the desired size. - if (newRegionSize > 0) - { - create_free_region(lastActualRegion, newRegionSize); - link_to_free_list(lastActualRegion); - } - - // Recreate the sentinel region at the end of the last free region - endSentinelRegion = (Region*)((uint8_t*)lastActualRegion + newRegionSize); - create_used_region(endSentinelRegion, sizeof(Region)); - - // And update the size field of the whole region block. - listOfAllRegions->endPtr = (uint8_t*)endSentinelRegion + sizeof(Region); - - // Finally call sbrk() to shrink the memory area. - void *oldSbrk = sbrk(-(intptr_t)shrinkAmount); - assert((intptr_t)oldSbrk != -1); // Shrinking with sbrk() should never fail. - assert(oldSbrk == previousSbrkEndAddress); // Another thread should not have raced to increase sbrk() on us! - - // All successful, and we actually trimmed memory! - return 1; -} - -int emmalloc_trim(size_t pad) -{ - MALLOC_ACQUIRE(); - int success = trim_dynamic_heap_reservation(pad); - MALLOC_RELEASE(); - return success; -} - -int EMMALLOC_EXPORT malloc_trim(size_t pad) -{ - return emmalloc_trim(pad); -} - -size_t emmalloc_dynamic_heap_size() -{ - size_t dynamicHeapSize = 0; - - MALLOC_ACQUIRE(); - RootRegion *root = listOfAllRegions; - while(root) - { - dynamicHeapSize += root->endPtr - (uint8_t*)root; - root = root->next; - } - MALLOC_RELEASE(); - return dynamicHeapSize; -} - -size_t emmalloc_free_dynamic_memory() -{ - size_t freeDynamicMemory = 0; - - int bucketIndex = 0; - - MALLOC_ACQUIRE(); - BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed; - - // Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap. - while(bucketMask) - { - BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask); - bucketIndex += indexAdd; - bucketMask >>= indexAdd; - for(Region *freeRegion = freeRegionBuckets[bucketIndex].next; - freeRegion != &freeRegionBuckets[bucketIndex]; - freeRegion = freeRegion->next) - { - freeDynamicMemory += freeRegion->size - REGION_HEADER_SIZE; - } - ++bucketIndex; - bucketMask >>= 1; - } - MALLOC_RELEASE(); - return freeDynamicMemory; -} - -size_t emmalloc_compute_free_dynamic_memory_fragmentation_map(size_t freeMemorySizeMap[32]) -{ - memset((void*)freeMemorySizeMap, 0, sizeof(freeMemorySizeMap[0])*32); - - size_t numFreeMemoryRegions = 0; - int bucketIndex = 0; - MALLOC_ACQUIRE(); - BUCKET_BITMASK_T bucketMask = freeRegionBucketsUsed; - - // Loop through each bucket that has free regions in it, based on bits set in freeRegionBucketsUsed bitmap. - while(bucketMask) - { - BUCKET_BITMASK_T indexAdd = __builtin_ctzll(bucketMask); - bucketIndex += indexAdd; - bucketMask >>= indexAdd; - for(Region *freeRegion = freeRegionBuckets[bucketIndex].next; - freeRegion != &freeRegionBuckets[bucketIndex]; - freeRegion = freeRegion->next) - { - ++numFreeMemoryRegions; - size_t freeDynamicMemory = freeRegion->size - REGION_HEADER_SIZE; - if (freeDynamicMemory > 0) - ++freeMemorySizeMap[31-__builtin_clz(freeDynamicMemory)]; - else - ++freeMemorySizeMap[0]; - } - ++bucketIndex; - bucketMask >>= 1; - } - MALLOC_RELEASE(); - return numFreeMemoryRegions; -} - -size_t emmalloc_unclaimed_heap_memory(void) { - return emscripten_get_heap_max() - (size_t)sbrk(0); -} -#endif - -// Define these to satisfy musl references. -void *__libc_malloc(size_t) __attribute__((alias("malloc"))); -void __libc_free(void *) __attribute__((alias("free"))); -void *__libc_calloc(size_t nmemb, size_t size) __attribute__((alias("calloc"))); diff --git a/lib/std/heap/SmpAllocator.zig b/lib/std/heap/SmpAllocator.zig index 50380d29b534e5c7d41a32c0c9d74af4efad3f08..5982026cbca00834fe7d7be5e099a9a964c17ecc 100644 --- a/lib/std/heap/SmpAllocator.zig +++ b/lib/std/heap/SmpAllocator.zig @@ -26,6 +26,7 @@ //! By limiting the thread-local metadata array to the same number as the CPU //! count, ensures that as threads are created and destroyed, they cycle //! through the full set of freelists. +const SmpAllocator = @This(); const builtin = @import("builtin"); @@ -34,7 +35,7 @@ const assert = std.debug.assert; const mem = std.mem; const math = std.math; const Allocator = std.mem.Allocator; -const SmpAllocator = @This(); +const Alignment = std.mem.Alignment; const PageAllocator = std.heap.PageAllocator; cpu_count: u32, @@ -114,7 +115,7 @@ comptime { assert(!builtin.single_threaded); // you're holding it wrong } -fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ?[*]u8 { +fn alloc(context: *anyopaque, len: usize, alignment: Alignment, ra: usize) ?[*]u8 { _ = context; _ = ra; const class = sizeClassIndex(len, alignment); @@ -172,7 +173,7 @@ fn alloc(context: *anyopaque, len: usize, alignment: mem.Alignment, ra: usize) ? } } -fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) bool { +fn resize(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ra: usize) bool { _ = context; _ = ra; const class = sizeClassIndex(memory.len, alignment); @@ -184,7 +185,7 @@ fn resize(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: return new_class == class; } -fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: usize, ra: usize) ?[*]u8 { +fn remap(context: *anyopaque, memory: []u8, alignment: Alignment, new_len: usize, ra: usize) ?[*]u8 { _ = context; _ = ra; const class = sizeClassIndex(memory.len, alignment); @@ -196,7 +197,7 @@ fn remap(context: *anyopaque, memory: []u8, alignment: mem.Alignment, new_len: u return if (new_class == class) memory.ptr else null; } -fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize) void { +fn free(context: *anyopaque, memory: []u8, alignment: Alignment, ra: usize) void { _ = context; _ = ra; const class = sizeClassIndex(memory.len, alignment); @@ -214,7 +215,7 @@ fn free(context: *anyopaque, memory: []u8, alignment: mem.Alignment, ra: usize) t.frees[class] = @intFromPtr(node); } -fn sizeClassIndex(len: usize, alignment: mem.Alignment) usize { +fn sizeClassIndex(len: usize, alignment: Alignment) usize { return @max(@bitSizeOf(usize) - @clz(len - 1), @intFromEnum(alignment), min_class) - min_class; } diff --git a/src/libs/musl.zig b/src/libs/musl.zig index 8cdc4a966f2bdad68f5623543d651255df9fcd98..54fa1aed853c53dd213666ad5184eee881814868 100644 --- a/src/libs/musl.zig +++ b/src/libs/musl.zig @@ -352,8 +352,7 @@ const Ext = enum { fn addSrcFile(arena: Allocator, source_table: *std.StringArrayHashMap(Ext), file_path: []const u8) !void { const ext: Ext = ext: { if (mem.endsWith(u8, file_path, ".c")) { - if (mem.startsWith(u8, file_path, "musl/src/malloc/") or - mem.startsWith(u8, file_path, "musl/src/string/") or + if (mem.startsWith(u8, file_path, "musl/src/string/") or mem.startsWith(u8, file_path, "musl/src/internal/")) { break :ext .o3; @@ -786,24 +785,6 @@ const src_files = [_][]const u8{ "musl/src/locale/uselocale.c", "musl/src/locale/wcscoll.c", "musl/src/locale/wcsxfrm.c", - "musl/src/malloc/calloc.c", - "musl/src/malloc/free.c", - "musl/src/malloc/libc_calloc.c", - "musl/src/malloc/lite_malloc.c", - "musl/src/malloc/mallocng/aligned_alloc.c", - "musl/src/malloc/mallocng/donate.c", - "musl/src/malloc/mallocng/free.c", - "musl/src/malloc/mallocng/malloc.c", - "musl/src/malloc/mallocng/malloc_usable_size.c", - "musl/src/malloc/mallocng/realloc.c", - "musl/src/malloc/memalign.c", - "musl/src/malloc/oldmalloc/aligned_alloc.c", - "musl/src/malloc/oldmalloc/malloc.c", - "musl/src/malloc/oldmalloc/malloc_usable_size.c", - "musl/src/malloc/posix_memalign.c", - "musl/src/malloc/reallocarray.c", - "musl/src/malloc/realloc.c", - "musl/src/malloc/replaced.c", "musl/src/math/aarch64/fma.c", "musl/src/math/aarch64/fmaf.c", "musl/src/math/aarch64/llrint.c", diff --git a/src/libs/wasi_libc.zig b/src/libs/wasi_libc.zig index ca6231bf53344375ac78b809dcfe749b5a482b7c..2809f5c9f31ecb17c031bf1c55ef4406c988d7fc 100644 --- a/src/libs/wasi_libc.zig +++ b/src/libs/wasi_libc.zig @@ -77,22 +77,6 @@ pub fn buildCrtFile(comp: *Compilation, crt_file: CrtFile, prog_node: std.Progre .libc_a => { var libc_sources = std.array_list.Managed(Compilation.CSourceFile).init(arena); - { - // Compile emmalloc. - var args = std.array_list.Managed([]const u8).init(arena); - try addCCArgs(comp, arena, &args, .{ .want_O3 = true, .no_strict_aliasing = true }); - - for (emmalloc_src_files) |file_path| { - try libc_sources.append(.{ - .src_path = try comp.dirs.zig_lib.join(arena, &.{ - "libc", try sanitize(arena, file_path), - }), - .extra_flags = args.items, - .owner = undefined, - }); - } - } - { // Compile libc-bottom-half. var args = std.array_list.Managed([]const u8).init(arena); @@ -472,10 +456,6 @@ fn addLibcTopHalfIncludes( }); } -const emmalloc_src_files = [_][]const u8{ - "wasi/emmalloc/emmalloc.c", -}; - const libc_bottom_half_src_files = [_][]const u8{ "wasi/libc-bottom-half/cloudlibc/src/libc/dirent/closedir.c", "wasi/libc-bottom-half/cloudlibc/src/libc/dirent/dirfd.c", -- 2.54.0 From 6ccabbd4e558dd0d56c471408c4cf29e0992dc1c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Feb 2026 19:28:43 -0800 Subject: [PATCH 04/12] std: brk allocator for single-threaded mode --- lib/c/malloc.zig | 6 +- lib/std/heap.zig | 29 +-- .../{WasmAllocator.zig => BrkAllocator.zig} | 96 ++++++---- lib/std/heap/sbrk_allocator.zig | 180 ------------------ lib/std/os/linux.zig | 4 + 5 files changed, 82 insertions(+), 233 deletions(-) rename lib/std/heap/{WasmAllocator.zig => BrkAllocator.zig} (79%) delete mode 100644 lib/std/heap/sbrk_allocator.zig diff --git a/lib/c/malloc.zig b/lib/c/malloc.zig index 72e7621b61057fdb1c31737c0af3c0b0262d53c0..4b8ba77b5ecef1d569c54f362285f0a647b2b596 100644 --- a/lib/c/malloc.zig +++ b/lib/c/malloc.zig @@ -1,7 +1,7 @@ -//! Based on wrapping a stateless Zig Allocator implementation, apropriate for: +//! Based on wrapping a stateless Zig Allocator implementation, appropriate for: //! - ReleaseFast and ReleaseSmall optimization modes, with multi-threading //! enabled. -//! - WebAssembly in single-threaded mode. +//! - WebAssembly or Linux in single-threaded release modes. //! //! Because the libc APIs don't have client alignment and size tracking, in //! order to take advantage of Zig allocator implementations, additional @@ -40,7 +40,7 @@ const no_context: *anyopaque = undefined; const no_ra: usize = undefined; const vtable = switch (builtin.cpu.arch) { .wasm32, .wasm64 => std.heap.WasmAllocator.vtable, - else => std.heap.SmpAllocator.vtable, + else => if (builtin.single_threaded) std.heap.BrkAllocator.vtable else std.heap.SmpAllocator.vtable, }; /// Needed because libc memory allocators don't provide old alignment and size diff --git a/lib/std/heap.zig b/lib/std/heap.zig index ba7ef0aabf4c77ef905aa14836ab0e91102b0ba1..562d0846efde69bf0d58b926896b6fc8100b3db2 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -13,9 +13,9 @@ pub const ArenaAllocator = @import("heap/arena_allocator.zig").ArenaAllocator; pub const SmpAllocator = @import("heap/SmpAllocator.zig"); pub const FixedBufferAllocator = @import("heap/FixedBufferAllocator.zig"); pub const PageAllocator = @import("heap/PageAllocator.zig"); -pub const SbrkAllocator = @import("heap/sbrk_allocator.zig").SbrkAllocator; pub const ThreadSafeAllocator = @import("heap/ThreadSafeAllocator.zig"); -pub const WasmAllocator = @import("heap/WasmAllocator.zig"); +pub const WasmAllocator = if (builtin.single_threaded) BrkAllocator else @compileError("unimplemented"); +pub const BrkAllocator = @import("heap/BrkAllocator.zig"); pub const DebugAllocatorConfig = @import("heap/debug_allocator.zig").Config; pub const DebugAllocator = @import("heap/debug_allocator.zig").DebugAllocator; @@ -356,9 +356,6 @@ pub const page_allocator: Allocator = if (@hasDecl(root, "os") and else if (builtin.target.cpu.arch.isWasm()) .{ .ptr = undefined, .vtable = &WasmAllocator.vtable, -} else if (builtin.target.os.tag == .plan9) .{ - .ptr = undefined, - .vtable = &SbrkAllocator(std.os.plan9.sbrk).vtable, } else .{ .ptr = undefined, .vtable = &PageAllocator.vtable, @@ -369,16 +366,18 @@ pub const smp_allocator: Allocator = .{ .vtable = &SmpAllocator.vtable, }; -/// This allocator is fast, small, and specific to WebAssembly. In the future, -/// this will be the implementation automatically selected by -/// `GeneralPurposeAllocator` when compiling in `ReleaseSmall` mode for wasm32 -/// and wasm64 architectures. -/// Until then, it is available here to play with. +/// This allocator is fast, small, and specific to WebAssembly. pub const wasm_allocator: Allocator = .{ .ptr = undefined, .vtable = &WasmAllocator.vtable, }; +/// Supports single-threaded WebAssembly and Linux. +pub const brk_allocator: Allocator = .{ + .ptr = undefined, + .vtable = &BrkAllocator.vtable, +}; + /// Returns a `StackFallbackAllocator` allocating using either a /// `FixedBufferAllocator` on an array of size `size` and falling back to /// `fallback_allocator` if that fails. @@ -1014,9 +1013,11 @@ test { _ = GeneralPurposeAllocator; _ = FixedBufferAllocator; _ = ThreadSafeAllocator; - _ = SbrkAllocator; - if (builtin.target.cpu.arch.isWasm()) { - _ = WasmAllocator; + if (builtin.single_threaded) { + if (builtin.cpu.arch.isWasm() or (builtin.os.tag == .linux and !builtin.link_libc)) { + _ = brk_allocator; + } + } else { + _ = smp_allocator; } - if (!builtin.single_threaded) _ = smp_allocator; } diff --git a/lib/std/heap/WasmAllocator.zig b/lib/std/heap/BrkAllocator.zig similarity index 79% rename from lib/std/heap/WasmAllocator.zig rename to lib/std/heap/BrkAllocator.zig index 2045d7c30373742e15afe2bb800a97f86be0d847..91b7230c9d30423d53afb3b6946d9c940990716e 100644 --- a/lib/std/heap/WasmAllocator.zig +++ b/lib/std/heap/BrkAllocator.zig @@ -1,20 +1,30 @@ -const std = @import("../std.zig"); +//! Supports single-threaded targets that have a sbrk-like primitive which includes +//! Linux and WebAssembly. +//! +//! On Linux, assumes exclusive access to the brk syscall. +const BrkAllocator = @This(); const builtin = @import("builtin"); + +const std = @import("../std.zig"); const Allocator = std.mem.Allocator; -const mem = std.mem; +const Alignment = std.mem.Alignment; const assert = std.debug.assert; -const wasm = std.wasm; const math = std.math; +const page_size_max = std.heap.page_size_max; comptime { - if (!builtin.target.cpu.arch.isWasm()) { - @compileError("only available for wasm32 arch"); - } - if (!builtin.single_threaded) { - @compileError("TODO implement support for multi-threaded wasm"); - } + if (!builtin.single_threaded) @compileError("unsupported"); } +next_addrs: [size_class_count]usize = @splat(0), +/// For each size class, points to the freed pointer. +frees: [size_class_count]usize = @splat(0), +/// For each big size class, points to the freed pointer. +big_frees: [big_size_class_count]usize = @splat(0), +prev_brk: usize = 0, + +var global: BrkAllocator = .{}; + pub const vtable: Allocator.VTable = .{ .alloc = alloc, .resize = resize, @@ -27,9 +37,13 @@ pub const Error = Allocator.Error; const max_usize = math.maxInt(usize); const ushift = math.Log2Int(usize); const bigpage_size = 64 * 1024; -const pages_per_bigpage = bigpage_size / wasm.page_size; +const pages_per_bigpage = bigpage_size / page_size_max; const bigpage_count = max_usize / bigpage_size; +comptime { + assert(bigpage_size >= page_size_max); +} + /// Because of storing free list pointers, the minimum size class is 3. const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize))); const size_class_count = math.log2(bigpage_size) - min_class; @@ -39,13 +53,7 @@ const size_class_count = math.log2(bigpage_size) - min_class; /// etc. const big_size_class_count = math.log2(bigpage_count); -var next_addrs: [size_class_count]usize = @splat(0); -/// For each size class, points to the freed pointer. -var frees: [size_class_count]usize = @splat(0); -/// For each big size class, points to the freed pointer. -var big_frees: [big_size_class_count]usize = @splat(0); - -fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[*]u8 { +fn alloc(ctx: *anyopaque, len: usize, alignment: Alignment, return_address: usize) ?[*]u8 { _ = ctx; _ = return_address; // Make room for the freelist next pointer. @@ -54,24 +62,24 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: const class = math.log2(slot_size) - min_class; if (class < size_class_count) { const addr = a: { - const top_free_ptr = frees[class]; + const top_free_ptr = global.frees[class]; if (top_free_ptr != 0) { const node: *usize = @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize))); - frees[class] = node.*; + global.frees[class] = node.*; break :a top_free_ptr; } - const next_addr = next_addrs[class]; - if (next_addr % wasm.page_size == 0) { + const next_addr = global.next_addrs[class]; + if (next_addr % page_size_max == 0) { const addr = allocBigPages(1); if (addr == 0) return null; //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ // slot_size, class, addr, //}); - next_addrs[class] = addr + slot_size; + global.next_addrs[class] = addr + slot_size; break :a addr; } else { - next_addrs[class] = next_addr + slot_size; + global.next_addrs[class] = next_addr + slot_size; break :a next_addr; } }; @@ -84,7 +92,7 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: fn resize( ctx: *anyopaque, buf: []u8, - alignment: mem.Alignment, + alignment: Alignment, new_len: usize, return_address: usize, ) bool { @@ -112,7 +120,7 @@ fn resize( fn remap( context: *anyopaque, memory: []u8, - alignment: mem.Alignment, + alignment: Alignment, new_len: usize, return_address: usize, ) ?[*]u8 { @@ -122,7 +130,7 @@ fn remap( fn free( ctx: *anyopaque, buf: []u8, - alignment: mem.Alignment, + alignment: Alignment, return_address: usize, ) void { _ = ctx; @@ -134,16 +142,16 @@ fn free( const addr = @intFromPtr(buf.ptr); if (class < size_class_count) { const node: *usize = @ptrFromInt(addr + (slot_size - @sizeOf(usize))); - node.* = frees[class]; - frees[class] = addr; + node.* = global.frees[class]; + global.frees[class] = addr; } else { const bigpages_needed = bigPagesNeeded(actual_len); const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed); const big_slot_size_bytes = pow2_pages * bigpage_size; const node: *usize = @ptrFromInt(addr + (big_slot_size_bytes - @sizeOf(usize))); const big_class = math.log2(pow2_pages); - node.* = big_frees[big_class]; - big_frees[big_class] = addr; + node.* = global.big_frees[big_class]; + global.big_frees[big_class] = addr; } } @@ -156,16 +164,30 @@ fn allocBigPages(n: usize) usize { const slot_size_bytes = pow2_pages * bigpage_size; const class = math.log2(pow2_pages); - const top_free_ptr = big_frees[class]; + const top_free_ptr = global.big_frees[class]; if (top_free_ptr != 0) { const node: *usize = @ptrFromInt(top_free_ptr + (slot_size_bytes - @sizeOf(usize))); - big_frees[class] = node.*; + global.big_frees[class] = node.*; return top_free_ptr; } - const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage); - if (page_index == -1) return 0; - return @as(usize, @intCast(page_index)) * wasm.page_size; + if (builtin.cpu.arch.isWasm()) { + const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage); + if (page_index == -1) return 0; + return @as(usize, @intCast(page_index)) * page_size_max; + } else if (builtin.os.tag == .linux) { + const start_brk = s: { + const start_brk = global.prev_brk; + break :s if (start_brk == 0) std.os.linux.brk(0) else start_brk; + }; + const end_brk = start_brk + pow2_pages * pages_per_bigpage * page_size_max; + const new_prev_brk = std.os.linux.brk(end_brk); + global.prev_brk = new_prev_brk; + if (new_prev_brk != end_brk) return 0; + return start_brk; + } else { + @compileError("no sbrk-like OS primitive available"); + } } const test_ally: Allocator = .{ @@ -257,12 +279,14 @@ test "shrink" { } test "large object - grow" { + if (builtin.os.tag == .linux) return error.SkipZigTest; + var slice1 = try test_ally.alloc(u8, bigpage_size * 2 - 20); defer test_ally.free(slice1); const old = slice1; slice1 = try test_ally.realloc(slice1, bigpage_size * 2 - 10); - try std.testing.expect(slice1.ptr == old.ptr); + try std.testing.expectEqual(slice1.ptr, old.ptr); slice1 = try test_ally.realloc(slice1, bigpage_size * 2); slice1 = try test_ally.realloc(slice1, bigpage_size * 2 + 1); diff --git a/lib/std/heap/sbrk_allocator.zig b/lib/std/heap/sbrk_allocator.zig deleted file mode 100644 index c4006d332239c38bc9fbeb7f770f81fd5f77bb22..0000000000000000000000000000000000000000 --- a/lib/std/heap/sbrk_allocator.zig +++ /dev/null @@ -1,180 +0,0 @@ -const builtin = @import("builtin"); - -const std = @import("../std.zig"); -const Io = std.Io; -const math = std.math; -const Allocator = std.mem.Allocator; -const mem = std.mem; -const heap = std.heap; -const assert = std.debug.assert; - -pub fn SbrkAllocator(comptime sbrk: *const fn (n: usize) usize) type { - return struct { - pub const vtable: Allocator.VTable = .{ - .alloc = alloc, - .resize = resize, - .remap = remap, - .free = free, - }; - - pub const Error = Allocator.Error; - - const max_usize = math.maxInt(usize); - const ushift = math.Log2Int(usize); - const bigpage_size = 64 * 1024; - const pages_per_bigpage = bigpage_size / heap.pageSize(); - const bigpage_count = max_usize / bigpage_size; - - /// Because of storing free list pointers, the minimum size class is 3. - const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize))); - const size_class_count = math.log2(bigpage_size) - min_class; - /// 0 - 1 bigpage - /// 1 - 2 bigpages - /// 2 - 4 bigpages - /// etc. - const big_size_class_count = math.log2(bigpage_count); - - var next_addrs = [1]usize{0} ** size_class_count; - /// For each size class, points to the freed pointer. - var frees = [1]usize{0} ** size_class_count; - /// For each big size class, points to the freed pointer. - var big_frees = [1]usize{0} ** big_size_class_count; - - // TODO don't do the naive locking strategy - var mutex: Io.Mutex = .{}; - fn alloc(ctx: *anyopaque, len: usize, alignment: mem.Alignment, return_address: usize) ?[*]u8 { - _ = ctx; - _ = return_address; - Io.Threaded.mutexLock(&mutex); - defer Io.Threaded.mutexUnlock(&mutex); - // Make room for the freelist next pointer. - const actual_len = @max(len +| @sizeOf(usize), alignment.toByteUnits()); - const slot_size = math.ceilPowerOfTwo(usize, actual_len) catch return null; - const class = math.log2(slot_size) - min_class; - if (class < size_class_count) { - const addr = a: { - const top_free_ptr = frees[class]; - if (top_free_ptr != 0) { - const node = @as(*usize, @ptrFromInt(top_free_ptr + (slot_size - @sizeOf(usize)))); - frees[class] = node.*; - break :a top_free_ptr; - } - - const next_addr = next_addrs[class]; - if (next_addr % heap.pageSize() == 0) { - const addr = allocBigPages(1); - if (addr == 0) return null; - //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ - // slot_size, class, addr, - //}); - next_addrs[class] = addr + slot_size; - break :a addr; - } else { - next_addrs[class] = next_addr + slot_size; - break :a next_addr; - } - }; - return @as([*]u8, @ptrFromInt(addr)); - } - const bigpages_needed = bigPagesNeeded(actual_len); - const addr = allocBigPages(bigpages_needed); - return @as([*]u8, @ptrFromInt(addr)); - } - - fn resize( - ctx: *anyopaque, - buf: []u8, - alignment: mem.Alignment, - new_len: usize, - return_address: usize, - ) bool { - _ = ctx; - _ = return_address; - Io.Threaded.mutexLock(&mutex); - defer Io.Threaded.mutexUnlock(&mutex); - // We don't want to move anything from one size class to another, but we - // can recover bytes in between powers of two. - const buf_align = alignment.toByteUnits(); - const old_actual_len = @max(buf.len + @sizeOf(usize), buf_align); - const new_actual_len = @max(new_len +| @sizeOf(usize), buf_align); - const old_small_slot_size = math.ceilPowerOfTwoAssert(usize, old_actual_len); - const old_small_class = math.log2(old_small_slot_size) - min_class; - if (old_small_class < size_class_count) { - const new_small_slot_size = math.ceilPowerOfTwo(usize, new_actual_len) catch return false; - return old_small_slot_size == new_small_slot_size; - } else { - const old_bigpages_needed = bigPagesNeeded(old_actual_len); - const old_big_slot_pages = math.ceilPowerOfTwoAssert(usize, old_bigpages_needed); - const new_bigpages_needed = bigPagesNeeded(new_actual_len); - const new_big_slot_pages = math.ceilPowerOfTwo(usize, new_bigpages_needed) catch return false; - return old_big_slot_pages == new_big_slot_pages; - } - } - - fn remap( - context: *anyopaque, - memory: []u8, - alignment: mem.Alignment, - new_len: usize, - return_address: usize, - ) ?[*]u8 { - return if (resize(context, memory, alignment, new_len, return_address)) memory.ptr else null; - } - - fn free( - ctx: *anyopaque, - buf: []u8, - alignment: mem.Alignment, - return_address: usize, - ) void { - _ = ctx; - _ = return_address; - Io.Threaded.mutexLock(&mutex); - defer Io.Threaded.mutexUnlock(&mutex); - const buf_align = alignment.toByteUnits(); - const actual_len = @max(buf.len + @sizeOf(usize), buf_align); - const slot_size = math.ceilPowerOfTwoAssert(usize, actual_len); - const class = math.log2(slot_size) - min_class; - const addr = @intFromPtr(buf.ptr); - if (class < size_class_count) { - const node = @as(*usize, @ptrFromInt(addr + (slot_size - @sizeOf(usize)))); - node.* = frees[class]; - frees[class] = addr; - } else { - const bigpages_needed = bigPagesNeeded(actual_len); - const pow2_pages = math.ceilPowerOfTwoAssert(usize, bigpages_needed); - const big_slot_size_bytes = pow2_pages * bigpage_size; - const node = @as(*usize, @ptrFromInt(addr + (big_slot_size_bytes - @sizeOf(usize)))); - const big_class = math.log2(pow2_pages); - node.* = big_frees[big_class]; - big_frees[big_class] = addr; - } - } - - inline fn bigPagesNeeded(byte_count: usize) usize { - return (byte_count + (bigpage_size + (@sizeOf(usize) - 1))) / bigpage_size; - } - - fn allocBigPages(n: usize) usize { - const pow2_pages = math.ceilPowerOfTwoAssert(usize, n); - const slot_size_bytes = pow2_pages * bigpage_size; - const class = math.log2(pow2_pages); - - const top_free_ptr = big_frees[class]; - if (top_free_ptr != 0) { - const node = @as(*usize, @ptrFromInt(top_free_ptr + (slot_size_bytes - @sizeOf(usize)))); - big_frees[class] = node.*; - return top_free_ptr; - } - return sbrk(pow2_pages * pages_per_bigpage * heap.pageSize()); - } - }; -} - -test SbrkAllocator { - _ = SbrkAllocator(struct { - fn sbrk(_: usize) usize { - return 0; - } - }.sbrk); -} diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 32dd627e09ebdb29e7871b13f4d0ac3a1fe94fb2..8cce6c1dae8d06646450d865a8d6b932f6254710 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -594,6 +594,10 @@ pub fn errno(r: usize) E { return @enumFromInt(int); } +pub fn brk(addr: usize) usize { + return syscall1(.brk, addr); +} + pub fn dup(old: i32) usize { return syscall1(.dup, @as(usize, @bitCast(@as(isize, old)))); } -- 2.54.0 From e2fccd8dd5c09ed805c000a6c26c7d1eb1e7d6c1 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Feb 2026 19:52:01 -0800 Subject: [PATCH 05/12] musl: remove macro redefinitions of malloc --- lib/libc/musl/src/aio/aio.c | 5 ----- lib/libc/musl/src/exit/atexit.c | 5 ----- lib/libc/musl/src/include/stdlib.h | 6 ------ lib/libc/musl/src/ldso/dlerror.c | 5 ----- lib/libc/musl/src/locale/dcngettext.c | 5 ----- lib/libc/musl/src/locale/duplocale.c | 5 ----- lib/libc/musl/src/locale/freelocale.c | 5 ----- lib/libc/musl/src/locale/locale_map.c | 5 ----- lib/libc/musl/src/locale/newlocale.c | 5 ----- lib/libc/musl/src/process/fdop.h | 5 ----- lib/libc/musl/src/thread/pthread_atfork.c | 5 ----- lib/libc/musl/src/thread/sem_open.c | 5 ----- lib/libc/musl/src/time/__tz.c | 5 ----- 13 files changed, 66 deletions(-) diff --git a/lib/libc/musl/src/aio/aio.c b/lib/libc/musl/src/aio/aio.c index d7e063bf93ca82fb5a6708a4bad7f8e1ead9e752..5c586260c33e79e71834deb71e38f046e1882ced 100644 --- a/lib/libc/musl/src/aio/aio.c +++ b/lib/libc/musl/src/aio/aio.c @@ -11,11 +11,6 @@ #include "pthread_impl.h" #include "aio_impl.h" -#define malloc __libc_malloc -#define calloc __libc_calloc -#define realloc __libc_realloc -#define free __libc_free - /* The following is a threads-based implementation of AIO with minimal * dependence on implementation details. Most synchronization is * performed with pthread primitives, but atomics and futex operations diff --git a/lib/libc/musl/src/exit/atexit.c b/lib/libc/musl/src/exit/atexit.c index 854e9fddbe559218805261c7623dbaa7304569d1..2804a1d7ad31971be2bda9447f76b271f51da30b 100644 --- a/lib/libc/musl/src/exit/atexit.c +++ b/lib/libc/musl/src/exit/atexit.c @@ -4,11 +4,6 @@ #include "lock.h" #include "fork_impl.h" -#define malloc __libc_malloc -#define calloc __libc_calloc -#define realloc undef -#define free undef - /* Ensure that at least 32 atexit handlers can be registered without malloc */ #define COUNT 32 diff --git a/lib/libc/musl/src/include/stdlib.h b/lib/libc/musl/src/include/stdlib.h index 812b04de2fd6b6efb1576a72eea4fa16007fc2ad..1599691fbeec4d037674c309a03dccec1e12db36 100644 --- a/lib/libc/musl/src/include/stdlib.h +++ b/lib/libc/musl/src/include/stdlib.h @@ -10,10 +10,4 @@ hidden int __ptsname_r(int, char *, size_t); hidden char *__randname(char *); hidden void __qsort_r (void *, size_t, size_t, int (*)(const void *, const void *, void *), void *); -hidden void *__libc_malloc(size_t); -hidden void *__libc_malloc_impl(size_t); -hidden void *__libc_calloc(size_t, size_t); -hidden void *__libc_realloc(void *, size_t); -hidden void __libc_free(void *); - #endif diff --git a/lib/libc/musl/src/ldso/dlerror.c b/lib/libc/musl/src/ldso/dlerror.c index dae0f3a9b24a93d01882fdafd4ed91cb29416375..141df3ab134988e03a570a70828e37400c2ec87c 100644 --- a/lib/libc/musl/src/ldso/dlerror.c +++ b/lib/libc/musl/src/ldso/dlerror.c @@ -5,11 +5,6 @@ #include "dynlink.h" #include "atomic.h" -#define malloc __libc_malloc -#define calloc __libc_calloc -#define realloc __libc_realloc -#define free __libc_free - char *dlerror() { pthread_t self = __pthread_self(); diff --git a/lib/libc/musl/src/locale/dcngettext.c b/lib/libc/musl/src/locale/dcngettext.c index 0b53286db753648d68ddef5b2905091bf39c4b9f..67e551058cb03aa77d34828ed7ca0ef47a043a57 100644 --- a/lib/libc/musl/src/locale/dcngettext.c +++ b/lib/libc/musl/src/locale/dcngettext.c @@ -12,11 +12,6 @@ #include "lock.h" #include "fork_impl.h" -#define malloc __libc_malloc -#define calloc __libc_calloc -#define realloc undef -#define free undef - struct binding { struct binding *next; int dirlen; diff --git a/lib/libc/musl/src/locale/duplocale.c b/lib/libc/musl/src/locale/duplocale.c index 5ce33ae6ded0acc40a93706bf39db10c3775f2b3..030b64cb0e706cdf29c7104526e86b6ee08d7adb 100644 --- a/lib/libc/musl/src/locale/duplocale.c +++ b/lib/libc/musl/src/locale/duplocale.c @@ -3,11 +3,6 @@ #include "locale_impl.h" #include "libc.h" -#define malloc __libc_malloc -#define calloc undef -#define realloc undef -#define free undef - locale_t __duplocale(locale_t old) { locale_t new = malloc(sizeof *new); diff --git a/lib/libc/musl/src/locale/freelocale.c b/lib/libc/musl/src/locale/freelocale.c index 385d12069d5733d8a291ceb6c870d842befed953..802b8bfe1c72458d8def7c505df0c022d026b1f0 100644 --- a/lib/libc/musl/src/locale/freelocale.c +++ b/lib/libc/musl/src/locale/freelocale.c @@ -1,11 +1,6 @@ #include #include "locale_impl.h" -#define malloc undef -#define calloc undef -#define realloc undef -#define free __libc_free - void freelocale(locale_t l) { if (__loc_is_allocated(l)) free(l); diff --git a/lib/libc/musl/src/locale/locale_map.c b/lib/libc/musl/src/locale/locale_map.c index da61f7fc032c5d010cd6409589aeb2e0ca61c84f..a8c9e7f24427920941ae0fe2dc3d24bf0af0f443 100644 --- a/lib/libc/musl/src/locale/locale_map.c +++ b/lib/libc/musl/src/locale/locale_map.c @@ -7,11 +7,6 @@ #include "lock.h" #include "fork_impl.h" -#define malloc __libc_malloc -#define calloc undef -#define realloc undef -#define free undef - const char *__lctrans_impl(const char *msg, const struct __locale_map *lm) { const char *trans = 0; diff --git a/lib/libc/musl/src/locale/newlocale.c b/lib/libc/musl/src/locale/newlocale.c index 9ac3cd386f99df95ba6e8f5a32a91b9c402db10d..12ae87d679e987293756808d5b05bb0846f86861 100644 --- a/lib/libc/musl/src/locale/newlocale.c +++ b/lib/libc/musl/src/locale/newlocale.c @@ -4,11 +4,6 @@ #include "locale_impl.h" #include "lock.h" -#define malloc __libc_malloc -#define calloc undef -#define realloc undef -#define free undef - static int default_locale_init_done; static struct __locale_struct default_locale, default_ctype_locale; diff --git a/lib/libc/musl/src/process/fdop.h b/lib/libc/musl/src/process/fdop.h index 7cf733b21dc14ce2708fac19fa46dea87fcdfa9b..5adf1443874854c6a7a33c4b641ff18c152a0f38 100644 --- a/lib/libc/musl/src/process/fdop.h +++ b/lib/libc/musl/src/process/fdop.h @@ -10,8 +10,3 @@ struct fdop { mode_t mode; char path[]; }; - -#define malloc __libc_malloc -#define calloc __libc_calloc -#define realloc undef -#define free __libc_free diff --git a/lib/libc/musl/src/thread/pthread_atfork.c b/lib/libc/musl/src/thread/pthread_atfork.c index 26d325438cec6ae63cc814766de557cc7a6a89c3..6d348ac8b73f8f3bc02dba246c303ee459cebe51 100644 --- a/lib/libc/musl/src/thread/pthread_atfork.c +++ b/lib/libc/musl/src/thread/pthread_atfork.c @@ -3,11 +3,6 @@ #include "libc.h" #include "lock.h" -#define malloc __libc_malloc -#define calloc undef -#define realloc undef -#define free undef - static struct atfork_funcs { void (*prepare)(void); void (*parent)(void); diff --git a/lib/libc/musl/src/thread/sem_open.c b/lib/libc/musl/src/thread/sem_open.c index 0ad29de96cc1667f3a6d629f5da4104b56ede9d6..c3d2f6d3c0b2f97d64c3e72dd68c00062655c527 100644 --- a/lib/libc/musl/src/thread/sem_open.c +++ b/lib/libc/musl/src/thread/sem_open.c @@ -14,11 +14,6 @@ #include "lock.h" #include "fork_impl.h" -#define malloc __libc_malloc -#define calloc __libc_calloc -#define realloc undef -#define free undef - static struct { ino_t ino; sem_t *sem; diff --git a/lib/libc/musl/src/time/__tz.c b/lib/libc/musl/src/time/__tz.c index c34b3eb755a54b38efa8f8e4b39e0bfb1e23b406..ff51bd9802244cbac936e449f284455994151c24 100644 --- a/lib/libc/musl/src/time/__tz.c +++ b/lib/libc/musl/src/time/__tz.c @@ -9,11 +9,6 @@ #include "lock.h" #include "fork_impl.h" -#define malloc __libc_malloc -#define calloc undef -#define realloc undef -#define free undef - long __timezone = 0; int __daylight = 0; char *__tzname[2] = { 0, 0 }; -- 2.54.0 From f5d5dbd1668b9fc7347b017563e2e951f3156c1e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Feb 2026 23:48:49 -0800 Subject: [PATCH 06/12] wasi libc: remove malloc defines --- lib/libc/wasi/libc-top-half/musl/src/exit/atexit.c | 5 ----- lib/libc/wasi/libc-top-half/musl/src/include/stdlib.h | 6 ------ lib/libc/wasi/libc-top-half/musl/src/locale/locale_map.c | 5 ----- lib/libc/wasi/libc-top-half/musl/src/locale/newlocale.c | 5 ----- lib/libc/wasi/libc-top-half/musl/src/time/__tz.c | 5 ----- 5 files changed, 26 deletions(-) diff --git a/lib/libc/wasi/libc-top-half/musl/src/exit/atexit.c b/lib/libc/wasi/libc-top-half/musl/src/exit/atexit.c index 155292d0ea5fed4390c0e6c86dce40032d6df470..30f897b7eae8478a67cebc7c0b804ffcc78ced73 100644 --- a/lib/libc/wasi/libc-top-half/musl/src/exit/atexit.c +++ b/lib/libc/wasi/libc-top-half/musl/src/exit/atexit.c @@ -4,11 +4,6 @@ #include "lock.h" #include "fork_impl.h" -#define malloc __libc_malloc -#define calloc __libc_calloc -#define realloc undef -#define free undef - /* Ensure that at least 32 atexit handlers can be registered without malloc */ #define COUNT 32 diff --git a/lib/libc/wasi/libc-top-half/musl/src/include/stdlib.h b/lib/libc/wasi/libc-top-half/musl/src/include/stdlib.h index 812b04de2fd6b6efb1576a72eea4fa16007fc2ad..1599691fbeec4d037674c309a03dccec1e12db36 100644 --- a/lib/libc/wasi/libc-top-half/musl/src/include/stdlib.h +++ b/lib/libc/wasi/libc-top-half/musl/src/include/stdlib.h @@ -10,10 +10,4 @@ hidden int __ptsname_r(int, char *, size_t); hidden char *__randname(char *); hidden void __qsort_r (void *, size_t, size_t, int (*)(const void *, const void *, void *), void *); -hidden void *__libc_malloc(size_t); -hidden void *__libc_malloc_impl(size_t); -hidden void *__libc_calloc(size_t, size_t); -hidden void *__libc_realloc(void *, size_t); -hidden void __libc_free(void *); - #endif diff --git a/lib/libc/wasi/libc-top-half/musl/src/locale/locale_map.c b/lib/libc/wasi/libc-top-half/musl/src/locale/locale_map.c index ebfb158f4a32fc6c25d04f609c04e4cab647a287..b7463fa54f177655215057db0ed348dd17f44464 100644 --- a/lib/libc/wasi/libc-top-half/musl/src/locale/locale_map.c +++ b/lib/libc/wasi/libc-top-half/musl/src/locale/locale_map.c @@ -9,11 +9,6 @@ #include "lock.h" #include "fork_impl.h" -#define malloc __libc_malloc -#define calloc undef -#define realloc undef -#define free undef - const char *__lctrans_impl(const char *msg, const struct __locale_map *lm) { const char *trans = 0; diff --git a/lib/libc/wasi/libc-top-half/musl/src/locale/newlocale.c b/lib/libc/wasi/libc-top-half/musl/src/locale/newlocale.c index 11c778557a484f16142467ecf07f5d24cc049972..154682de971f6b65acd420b6639b692f1b7a33eb 100644 --- a/lib/libc/wasi/libc-top-half/musl/src/locale/newlocale.c +++ b/lib/libc/wasi/libc-top-half/musl/src/locale/newlocale.c @@ -6,11 +6,6 @@ #include "locale_impl.h" #include "lock.h" -#define malloc __libc_malloc -#define calloc undef -#define realloc undef -#define free undef - static int default_locale_init_done; static struct __locale_struct default_locale, default_ctype_locale; diff --git a/lib/libc/wasi/libc-top-half/musl/src/time/__tz.c b/lib/libc/wasi/libc-top-half/musl/src/time/__tz.c index 7efb9a975716fff8f3b63dc0f4ca2f7965aadfd9..ee05eb58f8444241bdaad5b7ae47fb3a83c9fa3f 100644 --- a/lib/libc/wasi/libc-top-half/musl/src/time/__tz.c +++ b/lib/libc/wasi/libc-top-half/musl/src/time/__tz.c @@ -11,11 +11,6 @@ #include "lock.h" #include "fork_impl.h" -#define malloc __libc_malloc -#define calloc undef -#define realloc undef -#define free undef - #ifdef __wasilibc_unmodified_upstream // timezone data long __timezone = 0; int __daylight = 0; -- 2.54.0 From 21829a5b4ab0f21864e6e84fe585da365bab50c2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Feb 2026 23:48:04 -0800 Subject: [PATCH 07/12] zig libc: export all symbols weak Normally, libc goes into a static archive, making all symbols overrideable. However, Zig supports including the libc functions as part of the Zig Compilation Unit, so to support this use case we make all symbols weak. --- lib/c.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/c.zig b/lib/c.zig index 8b181dc1184b6504085de5adaf2ce04f639ec25c..b9f2a27eb7065ca5023f9d45c3e9e20fe30dbe99 100644 --- a/lib/c.zig +++ b/lib/c.zig @@ -15,17 +15,17 @@ pub const panic = if (builtin.is_test) else std.debug.no_panic; -/// It is incorrect to make this conditional on `builtin.is_test`, because it is possible that -/// libzigc is being linked into a different test compilation, as opposed to being tested itself. -pub const linkage: std.builtin.GlobalLinkage = .strong; - /// Determines the symbol's visibility to other objects. /// For WebAssembly this allows the symbol to be resolved to other modules, but will not /// export it to the host runtime. pub const visibility: std.builtin.SymbolVisibility = .hidden; pub inline fn symbol(comptime func: *const anyopaque, comptime name: []const u8) void { - @export(func, .{ .name = name, .linkage = linkage, .visibility = visibility }); + // Normally, libc goes into a static archive, making all symbols + // overridable. However, Zig supports including the libc functions as part + // of the Zig Compilation Unit, so to support this use case we make all + // symbols weak. + @export(func, .{ .name = name, .linkage = .weak, .visibility = visibility }); } /// Given a low-level syscall return value, sets errno and returns `-1`, or on -- 2.54.0 From 0de7668c013c5d3e095f0e436dba1b22a6a01f14 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Feb 2026 23:49:09 -0800 Subject: [PATCH 08/12] test harness: refmt for readability --- test/src/Libc.zig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/src/Libc.zig b/test/src/Libc.zig index 423d7856d1e581d13155830df176dead3f95bcf9..c6cea0d57133e3f55c24d115fdc76585e066a102 100644 --- a/test/src/Libc.zig +++ b/test/src/Libc.zig @@ -60,7 +60,9 @@ pub fn addTarget(libc: *const Libc, target: std.Build.ResolvedTarget) void { .link_libc = true, }); - var libtest_c_source_files: []const []const u8 = &.{ "print.c", "rand.c", "mtest.c", "setrlim.c", "memfill.c", "vmfill.c", "fdfill.c", "utf8.c" }; + var libtest_c_source_files: []const []const u8 = &.{ + "print.c", "rand.c", "mtest.c", "setrlim.c", "memfill.c", "vmfill.c", "fdfill.c", "utf8.c", + }; libtest_mod.addCSourceFiles(.{ .root = common, .files = libtest_c_source_files[0..if (target.result.isMuslLibC()) 8 else 3], -- 2.54.0 From ec02571a30e2975039cef5cb92467082ea5de5c0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Feb 2026 23:49:21 -0800 Subject: [PATCH 09/12] zig libc malloc: set errno when returning null --- lib/c/malloc.zig | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/lib/c/malloc.zig b/lib/c/malloc.zig index 4b8ba77b5ecef1d569c54f362285f0a647b2b596..5ddae3776754882ad722b63543ae1285a55ee268 100644 --- a/lib/c/malloc.zig +++ b/lib/c/malloc.zig @@ -64,9 +64,9 @@ const Header = packed struct(u64) { }; fn malloc(n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { - const size = std.math.cast(Header.Size, n) orelse return null; + const size = std.math.cast(Header.Size, n) orelse return nomem(); const ptr: [*]align(alignment_bytes) u8 = @alignCast( - vtable.alloc(no_context, n + alignment_bytes, alignment, no_ra) orelse return null, + vtable.alloc(no_context, n + alignment_bytes, alignment, no_ra) orelse return nomem(), ); const base = ptr + alignment_bytes; const header: *Header = .fromBase(base); @@ -78,6 +78,11 @@ fn malloc(n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { } fn aligned_alloc(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { + return aligned_alloc_inner(alloc_alignment, n) orelse return nomem(); +} + +/// Avoids setting errno so it can be called by `posix_memalign`. +fn aligned_alloc_inner(alloc_alignment: usize, n: usize) ?[*]align(alignment_bytes) u8 { const size = std.math.cast(Header.Size, n) orelse return null; const max_align = alignment.max(.fromByteUnits(alloc_alignment)); const max_align_bytes = max_align.toByteUnits(); @@ -94,7 +99,7 @@ fn aligned_alloc(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignm } fn calloc(elems: usize, len: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { - const n = std.math.mul(usize, elems, len) catch return null; + const n = std.math.mul(usize, elems, len) catch return nomem(); const base = malloc(n) orelse return null; @memset(base[0..n], 0); return base; @@ -106,7 +111,7 @@ fn realloc(opt_old_base: ?[*]align(alignment_bytes) u8, n: usize) callconv(.c) ? return null; } const old_base = opt_old_base orelse return malloc(n); - const new_size = std.math.cast(Header.Size, n) orelse return null; + const new_size = std.math.cast(Header.Size, n) orelse return nomem(); const old_header: *Header = .fromBase(old_base); assert(old_header.padding == 0); const old_size = old_header.size; @@ -122,7 +127,8 @@ fn realloc(opt_old_base: ?[*]align(alignment_bytes) u8, n: usize) callconv(.c) ? no_ra, )) |new_ptr| @alignCast(new_ptr + old_alignment_bytes) else b: { const new_ptr: [*]align(alignment_bytes) u8 = @alignCast( - vtable.alloc(no_context, n + old_alignment_bytes, old_alignment, no_ra) orelse return null, + vtable.alloc(no_context, n + old_alignment_bytes, old_alignment, no_ra) orelse + return nomem(), ); const new_base: [*]align(alignment_bytes) u8 = @alignCast(new_ptr + old_alignment_bytes); const copy_len = @min(new_size, old_size); @@ -139,7 +145,7 @@ fn realloc(opt_old_base: ?[*]align(alignment_bytes) u8, n: usize) callconv(.c) ? } fn reallocarray(opt_base: ?[*]align(alignment_bytes) u8, elems: usize, len: usize) callconv(.c) ?[*]align(alignment_bytes) u8 { - const n = std.math.mul(usize, elems, len) catch return null; + const n = std.math.mul(usize, elems, len) catch return nomem(); return realloc(opt_base, n); } @@ -173,10 +179,14 @@ fn memalign(alloc_alignment: usize, n: usize) callconv(.c) ?[*]align(alignment_b fn posix_memalign(result: *?[*]align(alignment_bytes) u8, alloc_alignment: usize, n: usize) callconv(.c) c_int { if (alloc_alignment < @sizeOf(*anyopaque)) return @intFromEnum(std.c.E.INVAL); - if (n == 0) { - result.* = null; - } else { - result.* = aligned_alloc(alloc_alignment, n) orelse return @intFromEnum(std.c.E.NOMEM); - } + result.* = aligned_alloc_inner(alloc_alignment, n) orelse return @intFromEnum(std.c.E.NOMEM); return 0; } + +/// Libc memory allocation functions must set errno in addition to returning +/// `null`. +fn nomem() ?[*]align(alignment_bytes) u8 { + @branchHint(.cold); + std.c._errno().* = @intFromEnum(std.c.E.NOMEM); + return null; +} -- 2.54.0 From 6d52678a6cfee1bc05d995bef8d963a5921ed8ea Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 12 Feb 2026 13:13:48 -0800 Subject: [PATCH 10/12] zig libc malloc: skip export when unit testing These functions can only be exported when external libc components are available due to the errno location dependency. Note that even when zig libc is complete, on Windows, errno location will always be external (in ucrtbase.dll). --- build.zig | 2 +- lib/c.zig | 35 +++++++++++++++++++++++++---------- lib/c/malloc.zig | 23 +++++++++++++---------- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/build.zig b/build.zig index 5eba8f82db9fe1567cf81dae1495499b8e627bf3..ed3c764a24e566b12321711da6178db3616b958e 100644 --- a/build.zig +++ b/build.zig @@ -518,7 +518,7 @@ pub fn build(b: *std.Build) !void { .test_extra_targets = test_extra_targets, .root_src = "lib/c.zig", .name = "zigc", - .desc = "Run the zigc tests", + .desc = "Run the zig libc implementation unit tests", .optimize_modes = optimization_modes, .include_paths = &.{}, .skip_single_threaded = true, diff --git a/lib/c.zig b/lib/c.zig index b9f2a27eb7065ca5023f9d45c3e9e20fe30dbe99..2e21581588be05490b621a9525de0019e7c3fa42 100644 --- a/lib/c.zig +++ b/lib/c.zig @@ -15,17 +15,32 @@ pub const panic = if (builtin.is_test) else std.debug.no_panic; -/// Determines the symbol's visibility to other objects. -/// For WebAssembly this allows the symbol to be resolved to other modules, but will not -/// export it to the host runtime. -pub const visibility: std.builtin.SymbolVisibility = .hidden; - +/// It is possible that this libc is being linked into a different test +/// compilation, as opposed to being tested itself. In such case, +/// `builtin.link_libc` will be `true` along with `builtin.is_test`. +/// +/// When we don't have a complete libc, `builtin.link_libc` will be `false` and +/// we will be missing externally provided symbols, such as `_errno` from +/// ucrtbase.dll. In such case, we must avoid analyzing otherwise exported +/// functions because it would cause undefined symbol usage. +/// +/// Unfortunately such logic cannot be automatically done in this function body +/// since `func` will always be analyzed by the time we get here, so `comptime` +/// blocks will need to each check for `builtin.link_libc` and skip exports +/// when the exported functions have libc dependencies not provided by this +/// compilation unit. pub inline fn symbol(comptime func: *const anyopaque, comptime name: []const u8) void { - // Normally, libc goes into a static archive, making all symbols - // overridable. However, Zig supports including the libc functions as part - // of the Zig Compilation Unit, so to support this use case we make all - // symbols weak. - @export(func, .{ .name = name, .linkage = .weak, .visibility = visibility }); + @export(func, .{ + .name = name, + // Normally, libc goes into a static archive, making all symbols + // overridable. However, Zig supports including the libc functions as part + // of the Zig Compilation Unit, so to support this use case we make all + // symbols weak. + .linkage = .weak, + // For WebAssembly, hidden visibility allows the symbol to be resolved to + // other modules, but will not export it to the host runtime. + .visibility = .hidden, + }); } /// Given a low-level syscall return value, sets errno and returns `-1`, or on diff --git a/lib/c/malloc.zig b/lib/c/malloc.zig index 5ddae3776754882ad722b63543ae1285a55ee268..4ef795c278716763b40507431c5513979f0bf2d3 100644 --- a/lib/c/malloc.zig +++ b/lib/c/malloc.zig @@ -23,17 +23,20 @@ const alignment: Alignment = .fromByteUnits(alignment_bytes); const symbol = @import("../c.zig").symbol; comptime { - symbol(&malloc, "malloc"); - symbol(&aligned_alloc, "aligned_alloc"); - symbol(&posix_memalign, "posix_memalign"); - symbol(&calloc, "calloc"); - symbol(&realloc, "realloc"); - symbol(&reallocarray, "reallocarray"); - symbol(&free, "free"); - symbol(&malloc_usable_size, "malloc_usable_size"); + // Dependency on external errno location. + if (builtin.link_libc) { + symbol(&malloc, "malloc"); + symbol(&aligned_alloc, "aligned_alloc"); + symbol(&posix_memalign, "posix_memalign"); + symbol(&calloc, "calloc"); + symbol(&realloc, "realloc"); + symbol(&reallocarray, "reallocarray"); + symbol(&free, "free"); + symbol(&malloc_usable_size, "malloc_usable_size"); - symbol(&valloc, "valloc"); - symbol(&memalign, "memalign"); + symbol(&valloc, "valloc"); + symbol(&memalign, "memalign"); + } } const no_context: *anyopaque = undefined; -- 2.54.0 From 6296924214fe8f0c08c7fcf30ecce5974133d1b4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 12 Feb 2026 15:00:22 -0800 Subject: [PATCH 11/12] build: bump behavior max_rss --- build.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.zig b/build.zig index ed3c764a24e566b12321711da6178db3616b958e..b2227b0cb81fb8e74f83b2c3107751c7756e3640 100644 --- a/build.zig +++ b/build.zig @@ -483,7 +483,7 @@ pub fn build(b: *std.Build) !void { .skip_linux = skip_linux, .skip_llvm = skip_llvm, .skip_libc = skip_libc, - .max_rss = 3_300_000_000, + .max_rss = 3_500_000_000, })); test_modules_step.dependOn(tests.addModuleTests(b, .{ -- 2.54.0 From 0957761d5c26ef23c04333120974323109a38f93 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 12 Feb 2026 14:58:49 -0800 Subject: [PATCH 12/12] std.heap.BrkAllocator: fix incorrect assumptions --- lib/std/heap/BrkAllocator.zig | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/std/heap/BrkAllocator.zig b/lib/std/heap/BrkAllocator.zig index 91b7230c9d30423d53afb3b6946d9c940990716e..1f0c88635f945be4ec5708901e4b7b3ccb21bfb3 100644 --- a/lib/std/heap/BrkAllocator.zig +++ b/lib/std/heap/BrkAllocator.zig @@ -10,7 +10,6 @@ const Allocator = std.mem.Allocator; const Alignment = std.mem.Alignment; const assert = std.debug.assert; const math = std.math; -const page_size_max = std.heap.page_size_max; comptime { if (!builtin.single_threaded) @compileError("unsupported"); @@ -36,14 +35,9 @@ pub const Error = Allocator.Error; const max_usize = math.maxInt(usize); const ushift = math.Log2Int(usize); -const bigpage_size = 64 * 1024; -const pages_per_bigpage = bigpage_size / page_size_max; +const bigpage_size: comptime_int = @max(64 * 1024, std.heap.page_size_max); const bigpage_count = max_usize / bigpage_size; -comptime { - assert(bigpage_size >= page_size_max); -} - /// Because of storing free list pointers, the minimum size class is 3. const min_class = math.log2(math.ceilPowerOfTwoAssert(usize, 1 + @sizeOf(usize))); const size_class_count = math.log2(bigpage_size) - min_class; @@ -70,7 +64,7 @@ fn alloc(ctx: *anyopaque, len: usize, alignment: Alignment, return_address: usiz } const next_addr = global.next_addrs[class]; - if (next_addr % page_size_max == 0) { + if (next_addr % bigpage_size == 0) { const addr = allocBigPages(1); if (addr == 0) return null; //std.debug.print("allocated fresh slot_size={d} class={d} addr=0x{x}\n", .{ @@ -172,15 +166,19 @@ fn allocBigPages(n: usize) usize { } if (builtin.cpu.arch.isWasm()) { + comptime assert(std.heap.page_size_max == std.heap.page_size_min); + const page_size = std.heap.page_size_max; + const pages_per_bigpage = bigpage_size / page_size; const page_index = @wasmMemoryGrow(0, pow2_pages * pages_per_bigpage); if (page_index == -1) return 0; - return @as(usize, @intCast(page_index)) * page_size_max; + return @as(usize, @intCast(page_index)) * page_size; } else if (builtin.os.tag == .linux) { - const start_brk = s: { - const start_brk = global.prev_brk; - break :s if (start_brk == 0) std.os.linux.brk(0) else start_brk; - }; - const end_brk = start_brk + pow2_pages * pages_per_bigpage * page_size_max; + const prev_brk = global.prev_brk; + const start_brk = if (prev_brk == 0) + std.mem.alignForward(usize, std.os.linux.brk(0), bigpage_size) + else + prev_brk; + const end_brk = start_brk + pow2_pages * bigpage_size; const new_prev_brk = std.os.linux.brk(end_brk); global.prev_brk = new_prev_brk; if (new_prev_brk != end_brk) return 0; -- 2.54.0