authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-01-26 22:58:01+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-01-26 22:58:01+01:00
log5eb55ba866484b7933c279006b63d8c2b1eb5856
tree802d678e727a056013ae4981ffafdf4c2f8b5508
parent8d4b8ab6a5de33836acbef9d53b5ed47c15839e4
parent499ba5d55c00b9d32691dc9ff49db49ba6bbded6

Merge pull request 'compiler: use Io.MemoryMap' (#30961) from use-mmap into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/30961

8 files changed, 106 insertions(+), 222 deletions(-)

lib/std/Io.zig+1-1
...@@ -656,7 +656,7 @@ pub const VTable = struct {...@@ -656,7 +656,7 @@ pub const VTable = struct {
656656
657 fileMemoryMapCreate: *const fn (?*anyopaque, File, File.MemoryMap.CreateOptions) File.MemoryMap.CreateError!File.MemoryMap,657 fileMemoryMapCreate: *const fn (?*anyopaque, File, File.MemoryMap.CreateOptions) File.MemoryMap.CreateError!File.MemoryMap,
658 fileMemoryMapDestroy: *const fn (?*anyopaque, *File.MemoryMap) void,658 fileMemoryMapDestroy: *const fn (?*anyopaque, *File.MemoryMap) void,
659 fileMemoryMapSetLength: *const fn (?*anyopaque, *File.MemoryMap, File.MemoryMap.CreateOptions) File.MemoryMap.SetLengthError!void,659 fileMemoryMapSetLength: *const fn (?*anyopaque, *File.MemoryMap, usize) File.MemoryMap.SetLengthError!void,
660 fileMemoryMapRead: *const fn (?*anyopaque, *File.MemoryMap) File.ReadPositionalError!void,660 fileMemoryMapRead: *const fn (?*anyopaque, *File.MemoryMap) File.ReadPositionalError!void,
661 fileMemoryMapWrite: *const fn (?*anyopaque, *File.MemoryMap) File.WritePositionalError!void,661 fileMemoryMapWrite: *const fn (?*anyopaque, *File.MemoryMap) File.WritePositionalError!void,
662662
lib/std/Io/File/MemoryMap.zig+5-11
...@@ -74,6 +74,9 @@ pub fn destroy(mm: *MemoryMap, io: Io) void {...@@ -74,6 +74,9 @@ pub fn destroy(mm: *MemoryMap, io: Io) void {
74}74}
7575
76pub const SetLengthError = error{76pub const SetLengthError = error{
77 /// Changing the mapping length could not be done atomically. Caller must
78 /// use `destroy` and `create` to resize the mapping.
79 OperationUnsupported,
77 /// One of the following:80 /// One of the following:
78 /// * The `File.Kind` is not `file`.81 /// * The `File.Kind` is not `file`.
79 /// * The file is not open for reading and read access protections enabled.82 /// * The file is not open for reading and read access protections enabled.
...@@ -91,17 +94,8 @@ pub const SetLengthError = error{...@@ -91,17 +94,8 @@ pub const SetLengthError = error{
91/// of the file after calling this is unspecified until `write` is called.94/// of the file after calling this is unspecified until `write` is called.
92///95///
93/// May change the pointer address of `memory`.96/// May change the pointer address of `memory`.
94///97pub fn setLength(mm: *MemoryMap, io: Io, new_len: usize) SetLengthError!void {
95/// `options` is needed because the mapping may need to be destroyed and98 return io.vtable.fileMemoryMapSetLength(io.userdata, mm, new_len);
96/// re-created. All the same options must be provided except for `len` which is
97/// the new length.
98///
99/// This operation cannot be completed atomically on all operating systems.
100/// When this function fails, the `MemoryMap` may be left in an unmapped state,
101/// which can be detected by checking if `memory.len` is zero. In such case it
102/// is safe to call `destroy` which will have no effect.
103pub fn setLength(mm: *MemoryMap, io: Io, options: CreateOptions) SetLengthError!void {
104 return io.vtable.fileMemoryMapSetLength(io.userdata, mm, options);
105}99}
106100
107/// Synchronizes the contents of `memory` from `file`.101/// Synchronizes the contents of `memory` from `file`.
lib/std/Io/Threaded.zig+3-33
...@@ -16597,27 +16597,21 @@ fn fileMemoryMapDestroy(userdata: ?*anyopaque, mm: *File.MemoryMap) void {...@@ -16597,27 +16597,21 @@ fn fileMemoryMapDestroy(userdata: ?*anyopaque, mm: *File.MemoryMap) void {
16597fn fileMemoryMapSetLength(16597fn fileMemoryMapSetLength(
16598 userdata: ?*anyopaque,16598 userdata: ?*anyopaque,
16599 mm: *File.MemoryMap,16599 mm: *File.MemoryMap,
16600 options: File.MemoryMap.CreateOptions,16600 new_len: usize,
16601) File.MemoryMap.SetLengthError!void {16601) File.MemoryMap.SetLengthError!void {
16602 const t: *Threaded = @ptrCast(@alignCast(userdata));16602 const t: *Threaded = @ptrCast(@alignCast(userdata));
16603 const page_size = std.heap.pageSize();16603 const page_size = std.heap.pageSize();
16604 const alignment: Alignment = .fromByteUnits(page_size);16604 const alignment: Alignment = .fromByteUnits(page_size);
16605 const page_align = std.heap.page_size_min;16605 const page_align = std.heap.page_size_min;
16606 const old_memory = mm.memory;16606 const old_memory = mm.memory;
16607 const new_len = options.len;
1660816607
16609 if (mm.section) |section| {16608 if (mm.section) |section| {
16609 _ = section;
16610 if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {16610 if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
16611 mm.memory.len = new_len;16611 mm.memory.len = new_len;
16612 return;16612 return;
16613 }16613 }
16614 switch (native_os) {16614 switch (native_os) {
16615 .windows => {
16616 _ = windows.ntdll.NtUnmapViewOfSection(windows.current_process, old_memory.ptr);
16617 windows.CloseHandle(section);
16618 mm.section = windows.INVALID_HANDLE_VALUE;
16619 mm.memory = &.{};
16620 },
16621 .wasi => unreachable,16615 .wasi => unreachable,
16622 .linux => {16616 .linux => {
16623 const flags: posix.MREMAP = .{ .MAYMOVE = true };16617 const flags: posix.MREMAP = .{ .MAYMOVE = true };
...@@ -16647,31 +16641,7 @@ fn fileMemoryMapSetLength(...@@ -16647,31 +16641,7 @@ fn fileMemoryMapSetLength(
16647 mm.memory = new_memory;16641 mm.memory = new_memory;
16648 return;16642 return;
16649 },16643 },
16650 else => {16644 else => return error.OperationUnsupported,
16651 switch (posix.errno(posix.system.munmap(old_memory.ptr, old_memory.len))) {
16652 .SUCCESS => {},
16653 else => |e| {
16654 if (builtin.mode == .Debug) std.log.err("failed to unmap {d} bytes at {*}: {t}", .{
16655 old_memory.len, old_memory.ptr, e,
16656 });
16657 // munmap must be infallible, or we cannot design reliable software.
16658 return error.Unexpected;
16659 },
16660 }
16661 mm.memory = &.{};
16662 },
16663 }
16664 if (createFileMap(mm.file, options.protection, mm.offset, options.populate, new_len)) |result| {
16665 mm.* = result;
16666 return;
16667 } else |err| switch (err) {
16668 error.OperationUnsupported,
16669 error.Unseekable,
16670 error.SectionOversize,
16671 error.MappingAlreadyExists,
16672 error.FileLockConflict,
16673 => return error.Unexpected, // It worked before on the same open file.
16674 else => |e| return e,
16675 }16645 }
16676 } else {16646 } else {
16677 const gpa = t.allocator;16647 const gpa = t.allocator;
lib/std/Io/Threaded/test.zig+8-1
...@@ -260,7 +260,14 @@ test "memory mapping fallback" {...@@ -260,7 +260,14 @@ test "memory mapping fallback" {
260260
261 try testing.expectEqualStrings("this9is9my", mm.memory);261 try testing.expectEqualStrings("this9is9my", mm.memory);
262262
263 try mm.setLength(io, .{ .len = "this9is9my data123".len });263 const new_len = "this9is9my data123".len;
264 mm.setLength(io, new_len) catch |err| switch (err) {
265 error.OperationUnsupported => {
266 mm.destroy(io);
267 mm = try file.createMemoryMap(io, .{ .len = new_len });
268 },
269 else => |e| return e,
270 };
264 try mm.read(io);271 try mm.read(io);
265272
266 try testing.expectEqualStrings("this9is9my data123", mm.memory);273 try testing.expectEqualStrings("this9is9my data123", mm.memory);
lib/std/Io/test.zig+8-3
...@@ -643,9 +643,14 @@ test "memory mapping" {...@@ -643,9 +643,14 @@ test "memory mapping" {
643 try expectEqualStrings("this9is9my", mm.memory);643 try expectEqualStrings("this9is9my", mm.memory);
644644
645 // Cross a page boundary to require an actual remap.645 // Cross a page boundary to require an actual remap.
646 try mm.setLength(io, .{646 const new_len = std.heap.pageSize() * 2;
647 .len = std.heap.pageSize() * 2,647 mm.setLength(io, new_len) catch |err| switch (err) {
648 });648 error.OperationUnsupported => {
649 mm.destroy(io);
650 mm = try file.createMemoryMap(io, .{ .len = new_len });
651 },
652 else => |e| return e,
653 };
649 try mm.read(io);654 try mm.read(io);
650655
651 try expectEqualStrings("this9is9my data123\x00\x00", mm.memory[0.."this9is9my data123\x00\x00".len]);656 try expectEqualStrings("this9is9my data123\x00\x00", mm.memory[0.."this9is9my data123\x00\x00".len]);
src/link.zig+5-5
...@@ -651,10 +651,10 @@ pub const File = struct {...@@ -651,10 +651,10 @@ pub const File = struct {
651 &coff.mf651 &coff.mf
652 else652 else
653 unreachable;653 unreachable;
654 mf.file = try base.emit.root_dir.handle.openFile(io, base.emit.sub_path, .{654 mf.memory_map.file = try base.emit.root_dir.handle.openFile(io, base.emit.sub_path, .{
655 .mode = .read_write,655 .mode = .read_write,
656 });656 });
657 base.file = mf.file;657 base.file = mf.memory_map.file;
658 try mf.ensureTotalCapacity(@intCast(mf.nodes.items[0].location().resolve(mf)[1]));658 try mf.ensureTotalCapacity(@intCast(mf.nodes.items[0].location().resolve(mf)[1]));
659 },659 },
660 .c, .spirv => dev.checkAny(&.{ .c_linker, .spirv_linker }),660 .c, .spirv => dev.checkAny(&.{ .c_linker, .spirv_linker }),
...@@ -729,9 +729,9 @@ pub const File = struct {...@@ -729,9 +729,9 @@ pub const File = struct {
729 else729 else
730 unreachable;730 unreachable;
731 mf.unmap();731 mf.unmap();
732 assert(mf.file.handle == f.handle);732 assert(mf.memory_map.file.handle == f.handle);
733 mf.file.close(io);733 mf.memory_map.file.close(io);
734 mf.file = undefined;734 mf.memory_map.file = undefined;
735 base.file = null;735 base.file = null;
736 },736 },
737 .c, .spirv => dev.checkAny(&.{ .c_linker, .spirv_linker }),737 .c, .spirv => dev.checkAny(&.{ .c_linker, .spirv_linker }),
src/link/Elf2.zig+10-5
...@@ -1691,10 +1691,10 @@ fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {...@@ -1691,10 +1691,10 @@ fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {
1691}1691}
16921692
1693pub fn identClass(elf: *const Elf) std.elf.CLASS {1693pub fn identClass(elf: *const Elf) std.elf.CLASS {
1694 return @enumFromInt(elf.mf.contents[std.elf.EI.CLASS]);1694 return @enumFromInt(elf.mf.memory_map.memory[std.elf.EI.CLASS]);
1695}1695}
1696pub fn identData(elf: *const Elf) std.elf.DATA {1696pub fn identData(elf: *const Elf) std.elf.DATA {
1697 return @enumFromInt(elf.mf.contents[std.elf.EI.DATA]);1697 return @enumFromInt(elf.mf.memory_map.memory[std.elf.EI.DATA]);
1698}1698}
16991699
1700pub fn targetEndian(elf: *const Elf) std.builtin.Endian {1700pub fn targetEndian(elf: *const Elf) std.builtin.Endian {
...@@ -2102,7 +2102,7 @@ fn loadObject(...@@ -2102,7 +2102,7 @@ fn loadObject(
2102 log.debug("loadObject({f}{f})", .{ path.fmtEscapeString(), fmtMemberString(member) });2102 log.debug("loadObject({f}{f})", .{ path.fmtEscapeString(), fmtMemberString(member) });
2103 const ident = try r.peek(std.elf.EI.OSABI);2103 const ident = try r.peek(std.elf.EI.OSABI);
2104 if (!std.mem.eql(u8, ident[0..std.elf.MAGIC.len], std.elf.MAGIC)) return error.BadMagic;2104 if (!std.mem.eql(u8, ident[0..std.elf.MAGIC.len], std.elf.MAGIC)) return error.BadMagic;
2105 if (!std.mem.eql(u8, ident[std.elf.MAGIC.len..], elf.mf.contents[std.elf.MAGIC.len..ident.len]))2105 if (!std.mem.eql(u8, ident[std.elf.MAGIC.len..], elf.mf.memory_map.memory[std.elf.MAGIC.len..ident.len]))
2106 return diags.failParse(path, "bad ident", .{});2106 return diags.failParse(path, "bad ident", .{});
2107 try elf.symtab.ensureUnusedCapacity(gpa, 1);2107 try elf.symtab.ensureUnusedCapacity(gpa, 1);
2108 try elf.inputs.ensureUnusedCapacity(gpa, 1);2108 try elf.inputs.ensureUnusedCapacity(gpa, 1);
...@@ -2341,7 +2341,7 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) !void {...@@ -2341,7 +2341,7 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) !void {
2341 log.debug("loadDso({f})", .{path.fmtEscapeString()});2341 log.debug("loadDso({f})", .{path.fmtEscapeString()});
2342 const ident = try r.peek(std.elf.EI.NIDENT);2342 const ident = try r.peek(std.elf.EI.NIDENT);
2343 if (!std.mem.eql(u8, ident[0..std.elf.MAGIC.len], std.elf.MAGIC)) return error.BadMagic;2343 if (!std.mem.eql(u8, ident[0..std.elf.MAGIC.len], std.elf.MAGIC)) return error.BadMagic;
2344 if (!std.mem.eql(u8, ident[std.elf.MAGIC.len..], elf.mf.contents[std.elf.MAGIC.len..ident.len]))2344 if (!std.mem.eql(u8, ident[std.elf.MAGIC.len..], elf.mf.memory_map.memory[std.elf.MAGIC.len..ident.len]))
2345 return diags.failParse(path, "bad ident", .{});2345 return diags.failParse(path, "bad ident", .{});
2346 const target_endian = elf.targetEndian();2346 const target_endian = elf.targetEndian();
2347 switch (elf.identClass()) {2347 switch (elf.identClass()) {
...@@ -3090,9 +3090,14 @@ pub fn flush(...@@ -3090,9 +3090,14 @@ pub fn flush(
3090 tid: Zcu.PerThread.Id,3090 tid: Zcu.PerThread.Id,
3091 prog_node: std.Progress.Node,3091 prog_node: std.Progress.Node,
3092) !void {3092) !void {
3093 const comp = elf.base.comp;
3093 _ = arena;3094 _ = arena;
3094 _ = prog_node;3095 _ = prog_node;
3095 while (try elf.idle(tid)) {}3096 while (try elf.idle(tid)) {}
3097 elf.mf.flush() catch |err| switch (err) {
3098 error.Canceled => |e| return e,
3099 else => |e| return comp.link_diags.fail("flush write failed: {t}", .{e}),
3100 };
3096}3101}
30973102
3098pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) !bool {3103pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) !bool {
...@@ -3839,7 +3844,7 @@ pub fn printNode(...@@ -3839,7 +3844,7 @@ pub fn printNode(
3839 const line_len = 0x10;3844 const line_len = 0x10;
3840 var line_it = std.mem.window(3845 var line_it = std.mem.window(
3841 u8,3846 u8,
3842 elf.mf.contents[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)],3847 elf.mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)],
3843 line_len,3848 line_len,
3844 line_len,3849 line_len,
3845 );3850 );
src/link/MappedFile.zig+66-163
...@@ -11,15 +11,13 @@ const linux = std.os.linux;...@@ -11,15 +11,13 @@ const linux = std.os.linux;
11const windows = std.os.windows;11const windows = std.os.windows;
1212
13io: Io,13io: Io,
14file: std.Io.File,
15flags: packed struct {14flags: packed struct {
16 block_size: std.mem.Alignment,15 block_size: std.mem.Alignment,
17 copy_file_range_unsupported: bool,16 copy_file_range_unsupported: bool,
18 fallocate_punch_hole_unsupported: bool,17 fallocate_punch_hole_unsupported: bool,
19 fallocate_insert_range_unsupported: bool,18 fallocate_insert_range_unsupported: bool,
20},19},
21section: if (is_windows) windows.HANDLE else void,20memory_map: Io.File.MemoryMap,
22contents: []align(std.heap.page_size_min) u8,
23nodes: std.ArrayList(Node),21nodes: std.ArrayList(Node),
24free_ni: Node.Index,22free_ni: Node.Index,
25large: std.ArrayList(u64),23large: std.ArrayList(u64),
...@@ -29,26 +27,20 @@ writers: std.SinglyLinkedList,...@@ -29,26 +27,20 @@ writers: std.SinglyLinkedList,
2927
30pub const growth_factor = 4;28pub const growth_factor = 4;
3129
32pub const Error = std.posix.MMapError || std.posix.MRemapError || Io.File.LengthError || error{30pub const Error = error{
33 NotFile,31 NotFile,
34 SystemResources,32} || Io.File.MemoryMap.CreateError || Io.File.MemoryMap.SetLengthError || Io.File.WritePositionalError;
35 IsDir,
36 Unseekable,
37 NoSpaceLeft,
3833
39 InputOutput,34pub fn init(file: Io.File, gpa: std.mem.Allocator, io: Io) !MappedFile {
40 FileTooBig,
41 FileBusy,
42 NonResizable,
43};
44
45pub fn init(file: std.Io.File, gpa: std.mem.Allocator, io: Io) !MappedFile {
46 var mf: MappedFile = .{35 var mf: MappedFile = .{
47 .io = io,36 .io = io,
48 .file = file,
49 .flags = undefined,37 .flags = undefined,
50 .section = if (is_windows) windows.INVALID_HANDLE_VALUE else {},38 .memory_map = .{
51 .contents = &.{},39 .file = file,
40 .memory = &.{},
41 .offset = 0,
42 .section = null,
43 },
52 .nodes = .empty,44 .nodes = .empty,
53 .free_ni = .none,45 .free_ni = .none,
54 .large = .empty,46 .large = .empty,
...@@ -58,61 +50,9 @@ pub fn init(file: std.Io.File, gpa: std.mem.Allocator, io: Io) !MappedFile {...@@ -58,61 +50,9 @@ pub fn init(file: std.Io.File, gpa: std.mem.Allocator, io: Io) !MappedFile {
58 };50 };
59 errdefer mf.deinit(gpa);51 errdefer mf.deinit(gpa);
60 const size: u64, const block_size = stat: {52 const size: u64, const block_size = stat: {
61 if (is_windows) {53 const stat = try file.stat(io);
62 var sbi: windows.SYSTEM_BASIC_INFORMATION = undefined;54 if (stat.kind != .file) return error.PathAlreadyExists;
63 break :stat .{55 break :stat .{ stat.size, @max(std.heap.pageSize(), stat.block_size) };
64 try windows.GetFileSizeEx(file.handle),
65 switch (windows.ntdll.NtQuerySystemInformation(
66 .SystemBasicInformation,
67 &sbi,
68 @sizeOf(windows.SYSTEM_BASIC_INFORMATION),
69 null,
70 )) {
71 .SUCCESS => @max(sbi.PageSize, sbi.AllocationGranularity),
72 else => std.heap.page_size_max,
73 },
74 };
75 }
76 if (is_linux) {
77 const use_c = std.c.versionCheck(if (builtin.abi.isAndroid())
78 .{ .major = 30, .minor = 0, .patch = 0 }
79 else
80 .{ .major = 2, .minor = 28, .patch = 0 });
81 const sys = if (use_c) std.c else std.os.linux;
82 while (true) {
83 var statx = std.mem.zeroes(linux.Statx);
84 const rc = sys.statx(
85 mf.file.handle,
86 "",
87 std.posix.AT.EMPTY_PATH,
88 .{ .TYPE = true, .SIZE = true, .BLOCKS = true },
89 &statx,
90 );
91 switch (sys.errno(rc)) {
92 .SUCCESS => {
93 assert(statx.mask.TYPE);
94 assert(statx.mask.SIZE);
95 assert(statx.mask.BLOCKS);
96 if (!std.posix.S.ISREG(statx.mode)) return error.PathAlreadyExists;
97 break :stat .{ statx.size, @max(std.heap.pageSize(), statx.blksize) };
98 },
99 .INTR => continue,
100 .ACCES => return error.AccessDenied,
101 .BADF => if (std.debug.runtime_safety) unreachable else return error.Unexpected,
102 .FAULT => if (std.debug.runtime_safety) unreachable else return error.Unexpected,
103 .INVAL => if (std.debug.runtime_safety) unreachable else return error.Unexpected,
104 .LOOP => return error.SymLinkLoop,
105 .NAMETOOLONG => return error.NameTooLong,
106 .NOENT => return error.FileNotFound,
107 .NOTDIR => return error.FileNotFound,
108 .NOMEM => return error.SystemResources,
109 else => |err| return std.posix.unexpectedErrno(err),
110 }
111 }
112 }
113 const stat = try std.posix.fstat(mf.file.handle);
114 if (!std.posix.S.ISREG(stat.mode)) return error.PathAlreadyExists;
115 break :stat .{ @bitCast(stat.size), @max(std.heap.pageSize(), stat.blksize) };
116 };56 };
117 mf.flags = .{57 mf.flags = .{
118 .block_size = .fromByteUnits(std.math.ceilPowerOfTwoAssert(usize, block_size)),58 .block_size = .fromByteUnits(std.math.ceilPowerOfTwoAssert(usize, block_size)),
...@@ -348,12 +288,12 @@ pub const Node = extern struct {...@@ -348,12 +288,12 @@ pub const Node = extern struct {
348288
349 pub fn slice(ni: Node.Index, mf: *const MappedFile) []u8 {289 pub fn slice(ni: Node.Index, mf: *const MappedFile) []u8 {
350 const file_loc = ni.fileLocation(mf, true);290 const file_loc = ni.fileLocation(mf, true);
351 return mf.contents[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)];291 return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)];
352 }292 }
353293
354 pub fn sliceConst(ni: Node.Index, mf: *const MappedFile) []const u8 {294 pub fn sliceConst(ni: Node.Index, mf: *const MappedFile) []const u8 {
355 const file_loc = ni.fileLocation(mf, false);295 const file_loc = ni.fileLocation(mf, false);
356 return mf.contents[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)];296 return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)];
357 }297 }
358298
359 pub fn resize(ni: Node.Index, mf: *MappedFile, gpa: std.mem.Allocator, size: u64) !void {299 pub fn resize(ni: Node.Index, mf: *MappedFile, gpa: std.mem.Allocator, size: u64) !void {
...@@ -396,7 +336,7 @@ pub const Node = extern struct {...@@ -396,7 +336,7 @@ pub const Node = extern struct {
396 mf: *MappedFile,336 mf: *MappedFile,
397 writer_node: std.SinglyLinkedList.Node,337 writer_node: std.SinglyLinkedList.Node,
398 ni: Node.Index,338 ni: Node.Index,
399 interface: std.Io.Writer,339 interface: Io.Writer,
400 err: ?Error,340 err: ?Error,
401341
402 pub fn deinit(w: *Writer) void {342 pub fn deinit(w: *Writer) void {
...@@ -404,18 +344,18 @@ pub const Node = extern struct {...@@ -404,18 +344,18 @@ pub const Node = extern struct {
404 w.* = undefined;344 w.* = undefined;
405 }345 }
406346
407 const vtable: std.Io.Writer.VTable = .{347 const vtable: Io.Writer.VTable = .{
408 .drain = drain,348 .drain = drain,
409 .sendFile = sendFile,349 .sendFile = sendFile,
410 .flush = std.Io.Writer.noopFlush,350 .flush = Io.Writer.noopFlush,
411 .rebase = growingRebase,351 .rebase = growingRebase,
412 };352 };
413353
414 fn drain(354 fn drain(
415 interface: *std.Io.Writer,355 interface: *Io.Writer,
416 data: []const []const u8,356 data: []const []const u8,
417 splat: usize,357 splat: usize,
418 ) std.Io.Writer.Error!usize {358 ) Io.Writer.Error!usize {
419 const pattern = data[data.len - 1];359 const pattern = data[data.len - 1];
420 const splat_len = pattern.len * splat;360 const splat_len = pattern.len * splat;
421 const start_len = interface.end;361 const start_len = interface.end;
...@@ -442,10 +382,10 @@ pub const Node = extern struct {...@@ -442,10 +382,10 @@ pub const Node = extern struct {
442 }382 }
443383
444 fn sendFile(384 fn sendFile(
445 interface: *std.Io.Writer,385 interface: *Io.Writer,
446 file_reader: *std.Io.File.Reader,386 file_reader: *Io.File.Reader,
447 limit: std.Io.Limit,387 limit: Io.Limit,
448 ) std.Io.Writer.FileError!usize {388 ) Io.Writer.FileError!usize {
449 if (limit == .nothing) return 0;389 if (limit == .nothing) return 0;
450 const pos = file_reader.logicalPos();390 const pos = file_reader.logicalPos();
451 const additional = if (file_reader.getSize()) |size| size - pos else |_| std.atomic.cache_line;391 const additional = if (file_reader.getSize()) |size| size - pos else |_| std.atomic.cache_line;
...@@ -489,10 +429,10 @@ pub const Node = extern struct {...@@ -489,10 +429,10 @@ pub const Node = extern struct {
489 }429 }
490430
491 fn growingRebase(431 fn growingRebase(
492 interface: *std.Io.Writer,432 interface: *Io.Writer,
493 preserve: usize,433 preserve: usize,
494 unused_capacity: usize,434 unused_capacity: usize,
495 ) std.Io.Writer.Error!void {435 ) Io.Writer.Error!void {
496 _ = preserve;436 _ = preserve;
497 const total_capacity = interface.end + unused_capacity;437 const total_capacity = interface.end + unused_capacity;
498 if (interface.buffer.len >= total_capacity) return;438 if (interface.buffer.len >= total_capacity) return;
...@@ -661,7 +601,8 @@ fn resizeNode(mf: *MappedFile, gpa: std.mem.Allocator, ni: Node.Index, requested...@@ -661,7 +601,8 @@ fn resizeNode(mf: *MappedFile, gpa: std.mem.Allocator, ni: Node.Index, requested
661 // Resize the entire file601 // Resize the entire file
662 if (ni == Node.Index.root) {602 if (ni == Node.Index.root) {
663 try mf.ensureCapacityForSetLocation(gpa);603 try mf.ensureCapacityForSetLocation(gpa);
664 try mf.file.setLength(io, new_size);604 try mf.memory_map.write(io);
605 try mf.memory_map.file.setLength(io, new_size);
665 try mf.ensureTotalCapacity(@intCast(new_size));606 try mf.ensureTotalCapacity(@intCast(new_size));
666 ni.setLocationAssumeCapacity(mf, old_offset, new_size);607 ni.setLocationAssumeCapacity(mf, old_offset, new_size);
667 return;608 return;
...@@ -685,6 +626,7 @@ fn resizeNode(mf: *MappedFile, gpa: std.mem.Allocator, ni: Node.Index, requested...@@ -685,6 +626,7 @@ fn resizeNode(mf: *MappedFile, gpa: std.mem.Allocator, ni: Node.Index, requested
685 if (is_linux and !mf.flags.fallocate_insert_range_unsupported and626 if (is_linux and !mf.flags.fallocate_insert_range_unsupported and
686 node.flags.alignment.order(mf.flags.block_size).compare(.gte))627 node.flags.alignment.order(mf.flags.block_size).compare(.gte))
687 insert_range: {628 insert_range: {
629 try mf.memory_map.write(io);
688 // Ask the filesystem driver to insert extents into the file without copying any data630 // Ask the filesystem driver to insert extents into the file without copying any data
689 const last_offset, const last_size = parent.last.location(mf).resolve(mf);631 const last_offset, const last_size = parent.last.location(mf).resolve(mf);
690 const last_end = last_offset + last_size;632 const last_end = last_offset + last_size;
...@@ -696,12 +638,12 @@ fn resizeNode(mf: *MappedFile, gpa: std.mem.Allocator, ni: Node.Index, requested...@@ -696,12 +638,12 @@ fn resizeNode(mf: *MappedFile, gpa: std.mem.Allocator, ni: Node.Index, requested
696 _, const file_size = Node.Index.root.location(mf).resolve(mf);638 _, const file_size = Node.Index.root.location(mf).resolve(mf);
697 while (true) switch (linux.errno(switch (std.math.order(range_file_offset, file_size)) {639 while (true) switch (linux.errno(switch (std.math.order(range_file_offset, file_size)) {
698 .lt => linux.fallocate(640 .lt => linux.fallocate(
699 mf.file.handle,641 mf.memory_map.file.handle,
700 linux.FALLOC.FL_INSERT_RANGE,642 linux.FALLOC.FL_INSERT_RANGE,
701 @intCast(range_file_offset),643 @intCast(range_file_offset),
702 @intCast(range_size),644 @intCast(range_size),
703 ),645 ),
704 .eq => linux.ftruncate(mf.file.handle, @intCast(range_file_offset + range_size)),646 .eq => linux.ftruncate(mf.memory_map.file.handle, @intCast(range_file_offset + range_size)),
705 .gt => unreachable,647 .gt => unreachable,
706 })) {648 })) {
707 .SUCCESS => {649 .SUCCESS => {
...@@ -908,7 +850,7 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size:...@@ -908,7 +850,7 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size:
908 if (is_linux and !mf.flags.fallocate_punch_hole_unsupported and850 if (is_linux and !mf.flags.fallocate_punch_hole_unsupported and
909 size >= mf.flags.block_size.toByteUnits() * 2 - 1) while (true)851 size >= mf.flags.block_size.toByteUnits() * 2 - 1) while (true)
910 switch (linux.errno(linux.fallocate(852 switch (linux.errno(linux.fallocate(
911 mf.file.handle,853 mf.memory_map.file.handle,
912 linux.FALLOC.FL_PUNCH_HOLE | linux.FALLOC.FL_KEEP_SIZE,854 linux.FALLOC.FL_PUNCH_HOLE | linux.FALLOC.FL_KEEP_SIZE,
913 @intCast(old_file_offset),855 @intCast(old_file_offset),
914 @intCast(size),856 @intCast(size),
...@@ -928,24 +870,26 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size:...@@ -928,24 +870,26 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size:
928 .TXTBSY => return error.FileBusy,870 .TXTBSY => return error.FileBusy,
929 else => |e| return std.posix.unexpectedErrno(e),871 else => |e| return std.posix.unexpectedErrno(e),
930 };872 };
931 @memset(mf.contents[@intCast(old_file_offset)..][0..@intCast(size)], 0);873 @memset(mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], 0);
932}874}
933875
934fn copyRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) !void {876fn copyRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) !void {
935 const copy_size = try mf.copyFileRange(mf.file, old_file_offset, new_file_offset, size);877 const copy_size = try mf.copyFileRange(mf.memory_map.file, old_file_offset, new_file_offset, size);
936 if (copy_size < size) @memcpy(878 if (copy_size < size) @memcpy(
937 mf.contents[@intCast(new_file_offset + copy_size)..][0..@intCast(size - copy_size)],879 mf.memory_map.memory[@intCast(new_file_offset + copy_size)..][0..@intCast(size - copy_size)],
938 mf.contents[@intCast(old_file_offset + copy_size)..][0..@intCast(size - copy_size)],880 mf.memory_map.memory[@intCast(old_file_offset + copy_size)..][0..@intCast(size - copy_size)],
939 );881 );
940}882}
941883
942fn copyFileRange(884fn copyFileRange(
943 mf: *MappedFile,885 mf: *MappedFile,
944 old_file: std.Io.File,886 old_file: Io.File,
945 old_file_offset: u64,887 old_file_offset: u64,
946 new_file_offset: u64,888 new_file_offset: u64,
947 size: u64,889 size: u64,
948) !u64 {890) !u64 {
891 const io = mf.io;
892 try mf.memory_map.write(io);
949 var remaining_size = size;893 var remaining_size = size;
950 if (is_linux and !mf.flags.copy_file_range_unsupported) {894 if (is_linux and !mf.flags.copy_file_range_unsupported) {
951 var old_file_offset_mut: i64 = @intCast(old_file_offset);895 var old_file_offset_mut: i64 = @intCast(old_file_offset);
...@@ -954,7 +898,7 @@ fn copyFileRange(...@@ -954,7 +898,7 @@ fn copyFileRange(
954 const copy_len = linux.copy_file_range(898 const copy_len = linux.copy_file_range(
955 old_file.handle,899 old_file.handle,
956 &old_file_offset_mut,900 &old_file_offset_mut,
957 mf.file.handle,901 mf.memory_map.file.handle,
958 &new_file_offset_mut,902 &new_file_offset_mut,
959 @intCast(remaining_size),903 @intCast(remaining_size),
960 0,904 0,
...@@ -990,82 +934,41 @@ fn ensureCapacityForSetLocation(mf: *MappedFile, gpa: std.mem.Allocator) !void {...@@ -990,82 +934,41 @@ fn ensureCapacityForSetLocation(mf: *MappedFile, gpa: std.mem.Allocator) !void {
990}934}
991935
992pub fn ensureTotalCapacity(mf: *MappedFile, new_capacity: usize) !void {936pub fn ensureTotalCapacity(mf: *MappedFile, new_capacity: usize) !void {
993 if (mf.contents.len >= new_capacity) return;937 if (mf.memory_map.memory.len >= new_capacity) return;
994 try mf.ensureTotalCapacityPrecise(new_capacity +| new_capacity / growth_factor);938 try mf.ensureTotalCapacityPrecise(new_capacity +| new_capacity / growth_factor);
995}939}
996940
997pub fn ensureTotalCapacityPrecise(mf: *MappedFile, new_capacity: usize) !void {941pub fn ensureTotalCapacityPrecise(mf: *MappedFile, new_capacity: usize) !void {
998 if (mf.contents.len >= new_capacity) return;942 if (mf.memory_map.memory.len >= new_capacity) return;
943 const io = mf.io;
999 const aligned_capacity = mf.flags.block_size.forward(new_capacity);944 const aligned_capacity = mf.flags.block_size.forward(new_capacity);
1000 if (!is_linux) mf.unmap() else if (mf.contents.len > 0) {945
1001 mf.contents = try std.posix.mremap(946 if (mf.memory_map.memory.len > 0) {
1002 mf.contents.ptr,947 if (mf.memory_map.setLength(io, aligned_capacity)) |_| {
1003 mf.contents.len,948 return;
1004 aligned_capacity,949 } else |err| switch (err) {
1005 .{ .MAYMOVE = true },950 error.OperationUnsupported => {},
1006 null,951 else => |e| return e,
1007 );
1008 return;
1009 }
1010 if (is_windows) {
1011 if (mf.section == windows.INVALID_HANDLE_VALUE) switch (windows.ntdll.NtCreateSection(
1012 &mf.section,
1013 .{
1014 .SPECIFIC = .{ .SECTION = .{
1015 .QUERY = true,
1016 .MAP_WRITE = true,
1017 .MAP_READ = true,
1018 .EXTEND_SIZE = true,
1019 } },
1020 .STANDARD = .{ .RIGHTS = .REQUIRED },
1021 },
1022 null,
1023 @constCast(&@as(i64, @intCast(aligned_capacity))),
1024 .{ .READWRITE = true },
1025 .{ .COMMIT = true },
1026 mf.file.handle,
1027 )) {
1028 .SUCCESS => {},
1029 else => return error.MemoryMappingNotSupported,
1030 };
1031 var contents_ptr: ?[*]align(std.heap.page_size_min) u8 = null;
1032 var contents_len = aligned_capacity;
1033 switch (windows.ntdll.NtMapViewOfSection(
1034 mf.section,
1035 windows.GetCurrentProcess(),
1036 @ptrCast(&contents_ptr),
1037 null,
1038 0,
1039 null,
1040 &contents_len,
1041 .Unmap,
1042 .{},
1043 .{ .READWRITE = true },
1044 )) {
1045 .SUCCESS => mf.contents = contents_ptr.?[0..contents_len],
1046 else => return error.MemoryMappingNotSupported,
1047 }952 }
1048 } else mf.contents = try std.posix.mmap(953 unmap(mf);
1049 null,954 }
1050 aligned_capacity,955
1051 .{ .READ = true, .WRITE = true },956 const file = mf.memory_map.file;
1052 .{ .TYPE = if (is_linux) .SHARED_VALIDATE else .SHARED },957 mf.memory_map = try .create(io, file, .{ .len = aligned_capacity });
1053 mf.file.handle,
1054 0,
1055 );
1056}958}
1057959
1058pub fn unmap(mf: *MappedFile) void {960pub fn unmap(mf: *MappedFile) void {
1059 if (mf.contents.len == 0) return;961 if (mf.memory_map.memory.len == 0) return;
1060 if (is_windows)962 const io = mf.io;
1061 _ = windows.ntdll.NtUnmapViewOfSection(windows.GetCurrentProcess(), mf.contents.ptr)963 const file = mf.memory_map.file;
1062 else964 mf.memory_map.destroy(io);
1063 std.posix.munmap(mf.contents);965 mf.memory_map.memory = &.{};
1064 mf.contents = &.{};966 mf.memory_map.file = file;
1065 if (is_windows and mf.section != windows.INVALID_HANDLE_VALUE) {967}
1066 windows.CloseHandle(mf.section);968
1067 mf.section = windows.INVALID_HANDLE_VALUE;969pub fn flush(mf: *MappedFile) Io.File.WritePositionalError!void {
1068 }970 const io = mf.io;
971 try mf.memory_map.write(io);
1069}972}
1070973
1071fn verify(mf: *MappedFile) void {974fn verify(mf: *MappedFile) void {