authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-01-14 19:16:09-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-01-15 14:18:20-08:00
log482189843220d4bd21ab4fdb737c20ee57580769
treed5cd417a1ca03a04e51ce7505924c2fe71566780
parentbed7bc37c43afce335610867aedbcd506ade65f4

std.Io.File.MemoryMap API tuning

- remove file_size parameter from MemoryMap.write - remove requirement for mapping length to be aligned - align allocated fallback memory - add unit test for std.Io.Threaded.disable_memory_mapping = true - add unit test for MemoryMap.setLength

6 files changed, 145 insertions(+), 74 deletions(-)

lib/std/Io.zig+1-1
...@@ -658,7 +658,7 @@ pub const VTable = struct {...@@ -658,7 +658,7 @@ pub const VTable = struct {
658 fileMemoryMapDestroy: *const fn (?*anyopaque, *File.MemoryMap) void,658 fileMemoryMapDestroy: *const fn (?*anyopaque, *File.MemoryMap) void,
659 fileMemoryMapSetLength: *const fn (?*anyopaque, *File.MemoryMap, n: usize) File.MemoryMap.SetLengthError!void,659 fileMemoryMapSetLength: *const fn (?*anyopaque, *File.MemoryMap, n: usize) File.MemoryMap.SetLengthError!void,
660 fileMemoryMapRead: *const fn (?*anyopaque, *File.MemoryMap) File.ReadPositionalError!void,660 fileMemoryMapRead: *const fn (?*anyopaque, *File.MemoryMap) File.ReadPositionalError!void,
661 fileMemoryMapWrite: *const fn (?*anyopaque, *File.MemoryMap, file_size: u64) File.WritePositionalError!void,661 fileMemoryMapWrite: *const fn (?*anyopaque, *File.MemoryMap) File.WritePositionalError!void,
662662
663 processExecutableOpen: *const fn (?*anyopaque, File.OpenFlags) std.process.OpenExecutableError!File,663 processExecutableOpen: *const fn (?*anyopaque, File.OpenFlags) std.process.OpenExecutableError!File,
664 processExecutablePath: *const fn (?*anyopaque, buffer: []u8) std.process.ExecutablePathError!usize,664 processExecutablePath: *const fn (?*anyopaque, buffer: []u8) std.process.ExecutablePathError!usize,
lib/std/Io/File.zig+1-5
...@@ -68,11 +68,7 @@ pub const Stat = struct {...@@ -68,11 +68,7 @@ pub const Stat = struct {
68 ctime: Io.Timestamp,68 ctime: Io.Timestamp,
69 /// Smallest chunk length in bytes appropriate for optimal I/O. This will69 /// Smallest chunk length in bytes appropriate for optimal I/O. This will
70 /// be set to `1` for operating systems or file systems that do not70 /// be set to `1` for operating systems or file systems that do not
71 /// recognize this concept. Not always a power of two. When creating a71 /// recognize this concept. Not always a power of two.
72 /// `MemoryMap`, the mapping length must be a multiple of this value.
73 ///
74 /// On Windows, this is whichever is larger: PageSize or
75 /// AllocationGranularity.
76 block_size: BlockSize,72 block_size: BlockSize,
77};73};
7874
lib/std/Io/File/MemoryMap.zig+15-14
...@@ -13,8 +13,8 @@ file: File,...@@ -13,8 +13,8 @@ file: File,
13/// Byte index inside `file` where `memory` starts. Page-aligned.13/// Byte index inside `file` where `memory` starts. Page-aligned.
14offset: u64,14offset: u64,
15/// Memory that may or may not remain consistent with file contents. Use `read`15/// Memory that may or may not remain consistent with file contents. Use `read`
16/// and `write` to ensure synchronization points. No minimum alignment on the16/// and `write` to ensure synchronization points. Pointer is page-aligned but
17/// pointer is guaranteed, but the length is page-aligned.17/// length is not.
18memory: []u8,18memory: []u8,
19/// Tells whether it is memory-mapped or file operations. On Windows this also19/// Tells whether it is memory-mapped or file operations. On Windows this also
20/// has a section handle.20/// has a section handle.
...@@ -37,11 +37,13 @@ pub const CreateError = error{...@@ -37,11 +37,13 @@ pub const CreateError = error{
37} || Allocator.Error || File.ReadPositionalError;37} || Allocator.Error || File.ReadPositionalError;
3838
39pub const CreateOptions = struct {39pub const CreateOptions = struct {
40 /// Size of the mapping, in bytes. If this is longer than the file size, it40 /// Size of the mapping, in bytes. If this is longer than the file size,
41 /// will be filled with zeroes.41 /// `memory` beyond the file end will be filled with zeroes and it is
42 /// unspecified whether, after calling `write`, the file length will be
43 /// set to `len` or remain unchanged.
42 ///44 ///
43 /// Asserted to be a multiple of page size which can be obtained via45 /// This value has no minimum alignment requirement, but may gain
44 /// `std.heap.pageSize`.46 /// efficiency benefits from being a multiple of `File.Stat.block_size`.
45 len: usize,47 len: usize,
46 /// When this has read set to false, bytes that are not modified before a48 /// When this has read set to false, bytes that are not modified before a
47 /// sync may have the original file contents, or may be set to zero.49 /// sync may have the original file contents, or may be set to zero.
...@@ -81,10 +83,9 @@ pub fn setLength(...@@ -81,10 +83,9 @@ pub fn setLength(
81 mm: *MemoryMap,83 mm: *MemoryMap,
82 io: Io,84 io: Io,
83 /// New size of the mapping, in bytes. If this is longer than the file85 /// New size of the mapping, in bytes. If this is longer than the file
84 /// size, it will be filled with zeroes. Asserted to be a multiple of page86 /// size, it will be filled with zeroes. No alignment requirement.
85 /// size which can be obtained with `std.heap.pageSize`.
86 new_length: usize,87 new_length: usize,
87) File.SetLengthError!void {88) SetLengthError!void {
88 return io.vtable.fileMemoryMapSetLength(io.userdata, mm, new_length);89 return io.vtable.fileMemoryMapSetLength(io.userdata, mm, new_length);
89}90}
9091
...@@ -95,9 +96,9 @@ pub fn read(mm: *MemoryMap, io: Io) File.ReadPositionalError!void {...@@ -95,9 +96,9 @@ pub fn read(mm: *MemoryMap, io: Io) File.ReadPositionalError!void {
9596
96/// Synchronizes the contents of `memory` to `file`.97/// Synchronizes the contents of `memory` to `file`.
97///98///
98/// Size of the mapping may be longer than the file size, so the `file_size`99/// If `memory.len` is greater than file size, the bytes beyond the end of the
99/// argument is used to avoid writing too many bytes. If `file_size` is not100/// file may be dropped, or they may be written, extending the size of the
100/// handy, use `File.length` to get it.101/// file.
101pub fn write(mm: *MemoryMap, io: Io, file_size: u64) File.WritePositionalError!void {102pub fn write(mm: *MemoryMap, io: Io) File.WritePositionalError!void {
102 return io.vtable.fileMemoryMapWrite(io.userdata, mm, file_size);103 return io.vtable.fileMemoryMapWrite(io.userdata, mm);
103}104}
lib/std/Io/Threaded.zig+50-48
...@@ -16172,8 +16172,6 @@ fn fileMemoryMapCreate(...@@ -16172,8 +16172,6 @@ fn fileMemoryMapCreate(
16172 const offset = options.offset;16172 const offset = options.offset;
16173 const len = options.len;16173 const len = options.len;
1617416174
16175 assert(std.mem.isAligned(len, std.heap.page_size_min));
16176
16177 if (!t.disable_memory_mapping) {16175 if (!t.disable_memory_mapping) {
16178 if (createFileMap(file, options.protection, offset, options.populate, len)) |result| {16176 if (createFileMap(file, options.protection, offset, options.populate, len)) |result| {
16179 return result;16177 return result;
...@@ -16187,12 +16185,13 @@ fn fileMemoryMapCreate(...@@ -16187,12 +16185,13 @@ fn fileMemoryMapCreate(
16187 }16185 }
1618816186
16189 const gpa = t.allocator;16187 const gpa = t.allocator;
16188 const page_size = std.heap.pageSize();
16189 const alignment: Alignment = .fromByteUnits(page_size);
16190 const memory = m: {16190 const memory = m: {
16191 const ptr = gpa.rawAlloc(len, .@"1", @returnAddress()) orelse16191 const ptr = gpa.rawAlloc(len, alignment, @returnAddress()) orelse return error.OutOfMemory;
16192 return error.OutOfMemory;
16193 break :m ptr[0..len];16192 break :m ptr[0..len];
16194 };16193 };
16195 errdefer gpa.rawFree(memory, .@"1", @returnAddress());16194 errdefer gpa.rawFree(memory, alignment, @returnAddress());
1619616195
16197 if (!options.undefined_contents) try mmSyncRead(file, memory, offset);16196 if (!options.undefined_contents) try mmSyncRead(file, memory, offset);
1619816197
...@@ -16363,7 +16362,7 @@ fn fileMemoryMapDestroy(userdata: ?*anyopaque, mm: *File.MemoryMap) void {...@@ -16363,7 +16362,7 @@ fn fileMemoryMapDestroy(userdata: ?*anyopaque, mm: *File.MemoryMap) void {
16363 }16362 }
16364 } else {16363 } else {
16365 const gpa = t.allocator;16364 const gpa = t.allocator;
16366 gpa.rawFree(memory, .@"1", @returnAddress());16365 gpa.rawFree(memory, .fromByteUnits(std.heap.pageSize()), @returnAddress());
16367 }16366 }
16368 mm.* = undefined;16367 mm.* = undefined;
16369}16368}
...@@ -16374,46 +16373,54 @@ fn fileMemoryMapSetLength(...@@ -16374,46 +16373,54 @@ fn fileMemoryMapSetLength(
16374 new_len: usize,16373 new_len: usize,
16375) File.MemoryMap.SetLengthError!void {16374) File.MemoryMap.SetLengthError!void {
16376 const t: *Threaded = @ptrCast(@alignCast(userdata));16375 const t: *Threaded = @ptrCast(@alignCast(userdata));
16377 assert(std.mem.isAligned(new_len, std.heap.page_size_min));16376 const page_size = std.heap.pageSize();
16378 if (mm.section) |section| switch (native_os) {16377 const alignment: Alignment = .fromByteUnits(page_size);
16379 .windows => {16378
16380 _ = section;16379 if (mm.section) |section| {
16381 @panic("TODO");16380 if (alignment.forward(new_len) == alignment.forward(mm.memory.len)) {
16382 },16381 mm.memory.len = new_len;
16383 .wasi => unreachable,16382 return;
16384 else => {16383 }
16385 const flags: posix.MREMAP = .{ .MAYMOVE = true };16384 switch (native_os) {
16386 const addr_hint: ?[*]const u8 = null;16385 .windows => {
16387 const new_memory = while (true) {16386 _ = section;
16388 const syscall: Syscall = try .start();16387 @panic("TODO");
16389 const rc = posix.system.mremap(mm.memory.ptr, mm.memory.len, new_len, flags, addr_hint);16388 },
16390 syscall.finish();16389 .wasi => unreachable,
16391 const err: posix.E = if (builtin.link_libc) e: {16390 else => {
16392 if (rc != std.c.MAP_FAILED) break @as([*]u8, @ptrCast(@alignCast(rc)))[0..new_len];16391 const flags: posix.MREMAP = .{ .MAYMOVE = true };
16393 break :e @enumFromInt(posix.system._errno().*);16392 const addr_hint: ?[*]const u8 = null;
16394 } else e: {16393 const new_memory = while (true) {
16395 const err = posix.errno(rc);16394 const syscall: Syscall = try .start();
16396 if (err == .SUCCESS) break @as([*]u8, @ptrFromInt(rc))[0..new_len];16395 const rc = posix.system.mremap(mm.memory.ptr, mm.memory.len, new_len, flags, addr_hint);
16397 break :e err;16396 syscall.finish();
16397 const err: posix.E = if (builtin.link_libc) e: {
16398 if (rc != std.c.MAP_FAILED) break @as([*]u8, @ptrCast(@alignCast(rc)))[0..new_len];
16399 break :e @enumFromInt(posix.system._errno().*);
16400 } else e: {
16401 const err = posix.errno(rc);
16402 if (err == .SUCCESS) break @as([*]u8, @ptrFromInt(rc))[0..new_len];
16403 break :e err;
16404 };
16405 switch (err) {
16406 .SUCCESS => unreachable,
16407 .INTR => continue,
16408 .AGAIN => return error.LockedMemoryLimitExceeded,
16409 .NOMEM => return error.OutOfMemory,
16410 .INVAL => return errnoBug(err),
16411 .FAULT => return errnoBug(err),
16412 else => return posix.unexpectedErrno(err),
16413 }
16398 };16414 };
16399 switch (err) {16415 mm.memory = new_memory;
16400 .SUCCESS => unreachable,16416 },
16401 .INTR => continue,16417 }
16402 .AGAIN => return error.LockedMemoryLimitExceeded,
16403 .NOMEM => return error.OutOfMemory,
16404 .INVAL => return errnoBug(err),
16405 .FAULT => return errnoBug(err),
16406 else => return posix.unexpectedErrno(err),
16407 }
16408 };
16409 mm.memory = new_memory;
16410 },
16411 } else {16418 } else {
16412 const gpa = t.allocator;16419 const gpa = t.allocator;
16413 if (gpa.rawRemap(mm.memory, .@"1", new_len, @returnAddress())) |new_ptr| {16420 if (gpa.rawRemap(mm.memory, alignment, new_len, @returnAddress())) |new_ptr| {
16414 mm.memory = new_ptr[0..new_len];16421 mm.memory = new_ptr[0..new_len];
16415 } else {16422 } else {
16416 const new_ptr = gpa.rawAlloc(new_len, .@"1", @returnAddress()) orelse16423 const new_ptr = gpa.rawAlloc(new_len, alignment, @returnAddress()) orelse
16417 return error.OutOfMemory;16424 return error.OutOfMemory;
16418 const copy_len = @min(new_len, mm.memory.len);16425 const copy_len = @min(new_len, mm.memory.len);
16419 @memcpy(new_ptr[0..copy_len], mm.memory[0..copy_len]);16426 @memcpy(new_ptr[0..copy_len], mm.memory[0..copy_len]);
...@@ -16429,16 +16436,11 @@ fn fileMemoryMapRead(userdata: ?*anyopaque, mm: *File.MemoryMap) File.ReadPositi...@@ -16429,16 +16436,11 @@ fn fileMemoryMapRead(userdata: ?*anyopaque, mm: *File.MemoryMap) File.ReadPositi
16429 return mmSyncRead(mm.file, mm.memory, mm.offset);16436 return mmSyncRead(mm.file, mm.memory, mm.offset);
16430}16437}
1643116438
16432fn fileMemoryMapWrite(16439fn fileMemoryMapWrite(userdata: ?*anyopaque, mm: *File.MemoryMap) File.WritePositionalError!void {
16433 userdata: ?*anyopaque,
16434 mm: *File.MemoryMap,
16435 file_size: u64,
16436) File.WritePositionalError!void {
16437 const t: *Threaded = @ptrCast(@alignCast(userdata));16440 const t: *Threaded = @ptrCast(@alignCast(userdata));
16438 _ = t;16441 _ = t;
16439 if (mm.section != null) return;16442 if (mm.section != null) return;
16440 const offset = mm.offset;16443 return mmSyncWrite(mm.file, mm.memory, mm.offset);
16441 return mmSyncWrite(mm.file, mm.memory[0..@intCast(file_size - offset)], offset);
16442}16444}
1644316445
16444fn mmSyncRead(file: File, memory: []u8, offset: u64) File.ReadPositionalError!void {16446fn mmSyncRead(file: File, memory: []u8, offset: u64) File.ReadPositionalError!void {
lib/std/Io/Threaded/test.zig+57
...@@ -204,3 +204,60 @@ test "cancel blocked read from pipe" {...@@ -204,3 +204,60 @@ test "cancel blocked read from pipe" {
204 try io.sleep(.fromMilliseconds(10), .awake);204 try io.sleep(.fromMilliseconds(10), .awake);
205 try future.cancel(io);205 try future.cancel(io);
206}206}
207
208test "memory mapping fallback" {
209 var threaded: std.Io.Threaded = .init(std.testing.allocator, .{
210 .argv0 = .empty,
211 .environ = .empty,
212 .disable_memory_mapping = true,
213 });
214 defer threaded.deinit();
215 const io = threaded.io();
216
217 var tmp = testing.tmpDir(.{});
218 defer tmp.cleanup();
219
220 try tmp.dir.writeFile(io, .{
221 .sub_path = "blah.txt",
222 .data = "this is my data123",
223 });
224
225 {
226 var file = try tmp.dir.openFile(io, "blah.txt", .{ .mode = .read_write });
227 defer file.close(io);
228
229 // The `Io.File.MemoryMap` API does not specify what happens if we supply a
230 // length greater than file size, but this is testing specifically std.Io.Threaded
231 // with disable_memory_mapping = true.
232 var mm = try file.createMemoryMap(io, .{ .len = "this is my data123".len + 3 });
233 defer mm.destroy(io);
234
235 try testing.expectEqualStrings("this is my data123\x00\x00\x00", mm.memory);
236 mm.memory[4] = '9';
237 mm.memory[7] = '9';
238
239 try mm.write(io);
240 }
241
242 var buffer: [100]u8 = undefined;
243 const updated_contents = try tmp.dir.readFile(io, "blah.txt", &buffer);
244 try testing.expectEqualStrings("this9is9my data123\x00\x00\x00", updated_contents);
245
246 {
247 var file = try tmp.dir.openFile(io, "blah.txt", .{ .mode = .read_only });
248 defer file.close(io);
249
250 var mm = try file.createMemoryMap(io, .{
251 .len = "this9is9my".len,
252 .protection = .{ .read = true },
253 });
254 defer mm.destroy(io);
255
256 try testing.expectEqualStrings("this9is9my", mm.memory);
257
258 try mm.setLength(io, "this9is9my data123".len);
259 try mm.read(io);
260
261 try testing.expectEqualStrings("this9is9my data123", mm.memory);
262 }
263}
lib/std/Io/test.zig+21-6
...@@ -608,20 +608,35 @@ test "memory mapping" {...@@ -608,20 +608,35 @@ test "memory mapping" {
608 var file = try tmp.dir.openFile(io, "blah.txt", .{ .mode = .read_write });608 var file = try tmp.dir.openFile(io, "blah.txt", .{ .mode = .read_write });
609 defer file.close(io);609 defer file.close(io);
610610
611 const stat = try file.stat(io);611 var mm = try file.createMemoryMap(io, .{ .len = "this is my data123".len });
612 const aligned_len = std.mem.alignForward(usize, @intCast(stat.size), std.heap.pageSize());
613
614 var mm = try file.createMemoryMap(io, .{ .len = aligned_len });
615 defer mm.destroy(io);612 defer mm.destroy(io);
616613
617 try expectEqualStrings("this is my data123", std.mem.sliceTo(mm.memory, 0));614 try expectEqualStrings("this is my data123", mm.memory);
618 mm.memory[4] = '9';615 mm.memory[4] = '9';
619 mm.memory[7] = '9';616 mm.memory[7] = '9';
620617
621 try mm.write(io, stat.size);618 try mm.write(io);
622 }619 }
623620
624 var buffer: [100]u8 = undefined;621 var buffer: [100]u8 = undefined;
625 const updated_contents = try tmp.dir.readFile(io, "blah.txt", &buffer);622 const updated_contents = try tmp.dir.readFile(io, "blah.txt", &buffer);
626 try expectEqualStrings("this9is9my data123", updated_contents);623 try expectEqualStrings("this9is9my data123", updated_contents);
624
625 {
626 var file = try tmp.dir.openFile(io, "blah.txt", .{ .mode = .read_only });
627 defer file.close(io);
628
629 var mm = try file.createMemoryMap(io, .{
630 .len = "this9is9my".len,
631 .protection = .{ .read = true },
632 });
633 defer mm.destroy(io);
634
635 try expectEqualStrings("this9is9my", mm.memory);
636
637 try mm.setLength(io, "this9is9my data123".len);
638 try mm.read(io);
639
640 try expectEqualStrings("this9is9my data123", mm.memory);
641 }
627}642}