| ... | ... | @@ -57,6 +57,7 @@ use_sendfile: UseSendfile = .default, |
| 57 | 57 | use_copy_file_range: UseCopyFileRange = .default, |
| 58 | 58 | use_fcopyfile: UseFcopyfile = .default, |
| 59 | 59 | use_fchmodat2: UseFchmodat2 = .default, |
| 60 | disable_memory_mapping: bool, |
| 60 | 61 | |
| 61 | 62 | stderr_writer: File.Writer = .{ |
| 62 | 63 | .io = undefined, |
| ... | ... | @@ -75,6 +76,13 @@ random_file: RandomFile = .{}, |
| 75 | 76 | |
| 76 | 77 | csprng: Csprng = .{}, |
| 77 | 78 | |
| 79 | system_basic_information: SystemBasicInformation = .{}, |
| 80 | |
| 81 | const SystemBasicInformation = if (!is_windows) struct {} else struct { |
| 82 | buffer: windows.SYSTEM_BASIC_INFORMATION = undefined, |
| 83 | initialized: std.atomic.Value(bool) = .{ .raw = false }, |
| 84 | }; |
| 85 | |
| 78 | 86 | pub const Csprng = struct { |
| 79 | 87 | rng: std.Random.DefaultCsprng = .{ |
| 80 | 88 | .state = undefined, |
| ... | ... | @@ -1220,6 +1228,8 @@ pub const InitOptions = struct { |
| 1220 | 1228 | /// * `processExecutablePath` on OpenBSD and Haiku (observes "PATH"). |
| 1221 | 1229 | /// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath` |
| 1222 | 1230 | environ: process.Environ, |
| 1231 | /// If set to `true`, `File.MemoryMap` APIs will always take the fallback path. |
| 1232 | disable_memory_mapping: bool = false, |
| 1223 | 1233 | }; |
| 1224 | 1234 | |
| 1225 | 1235 | /// Related: |
| ... | ... | @@ -1247,6 +1257,7 @@ pub fn init( |
| 1247 | 1257 | .argv0 = options.argv0, |
| 1248 | 1258 | .environ = .{ .process_environ = options.environ }, |
| 1249 | 1259 | .worker_threads = init_single_threaded.worker_threads, |
| 1260 | .disable_memory_mapping = options.disable_memory_mapping, |
| 1250 | 1261 | }; |
| 1251 | 1262 | |
| 1252 | 1263 | const cpu_count = std.Thread.getCpuCount(); |
| ... | ... | @@ -1263,6 +1274,7 @@ pub fn init( |
| 1263 | 1274 | .argv0 = options.argv0, |
| 1264 | 1275 | .environ = .{ .process_environ = options.environ }, |
| 1265 | 1276 | .worker_threads = .init(null), |
| 1277 | .disable_memory_mapping = options.disable_memory_mapping, |
| 1266 | 1278 | }; |
| 1267 | 1279 | |
| 1268 | 1280 | if (posix.Sigaction != void) { |
| ... | ... | @@ -1299,6 +1311,7 @@ pub const init_single_threaded: Threaded = .{ |
| 1299 | 1311 | .argv0 = .empty, |
| 1300 | 1312 | .environ = .{}, |
| 1301 | 1313 | .worker_threads = .init(null), |
| 1314 | .disable_memory_mapping = false, |
| 1302 | 1315 | }; |
| 1303 | 1316 | |
| 1304 | 1317 | var global_single_threaded_instance: Threaded = .init_single_threaded; |
| ... | ... | @@ -2935,7 +2948,11 @@ fn fileStatLinux(userdata: ?*anyopaque, file: File) File.StatError!File.Stat { |
| 2935 | 2948 | |
| 2936 | 2949 | fn fileStatWindows(userdata: ?*anyopaque, file: File) File.StatError!File.Stat { |
| 2937 | 2950 | const t: *Threaded = @ptrCast(@alignCast(userdata)); |
| 2938 | | _ = t; |
| 2951 | |
| 2952 | const block_size: u32 = if (t.systemBasicInformation()) |sbi| |
| 2953 | @intCast(@max(sbi.PageSize, sbi.AllocationGranularity)) |
| 2954 | else |
| 2955 | std.heap.page_size_max; |
| 2939 | 2956 | |
| 2940 | 2957 | var io_status_block: windows.IO_STATUS_BLOCK = undefined; |
| 2941 | 2958 | var info: windows.FILE.ALL_INFORMATION = undefined; |
| ... | ... | @@ -2997,10 +3014,31 @@ fn fileStatWindows(userdata: ?*anyopaque, file: File) File.StatError!File.Stat { |
| 2997 | 3014 | .atime = windows.fromSysTime(info.BasicInformation.LastAccessTime), |
| 2998 | 3015 | .mtime = windows.fromSysTime(info.BasicInformation.LastWriteTime), |
| 2999 | 3016 | .ctime = windows.fromSysTime(info.BasicInformation.ChangeTime), |
| 3000 | | .nlink = 0, |
| 3017 | .nlink = info.StandardInformation.NumberOfLinks, |
| 3018 | .block_size = block_size, |
| 3001 | 3019 | }; |
| 3002 | 3020 | } |
| 3003 | 3021 | |
| 3022 | fn systemBasicInformation(t: *Threaded) ?*const windows.SYSTEM_BASIC_INFORMATION { |
| 3023 | if (!t.system_basic_information.initialized.load(.acquire)) { |
| 3024 | t.mutex.lock(); |
| 3025 | defer t.mutex.unlock(); |
| 3026 | |
| 3027 | switch (windows.ntdll.NtQuerySystemInformation( |
| 3028 | .SystemBasicInformation, |
| 3029 | &t.system_basic_information.buffer, |
| 3030 | @sizeOf(windows.SYSTEM_BASIC_INFORMATION), |
| 3031 | null, |
| 3032 | )) { |
| 3033 | .SUCCESS => {}, |
| 3034 | else => return null, |
| 3035 | } |
| 3036 | |
| 3037 | t.system_basic_information.initialized.store(true, .release); |
| 3038 | } |
| 3039 | return &t.system_basic_information.buffer; |
| 3040 | } |
| 3041 | |
| 3004 | 3042 | fn fileStatWasi(userdata: ?*anyopaque, file: File) File.StatError!File.Stat { |
| 3005 | 3043 | if (builtin.link_libc) return fileStatPosix(userdata, file); |
| 3006 | 3044 | |
| ... | ... | @@ -8008,10 +8046,10 @@ fn fileReadStreamingWindows(userdata: ?*anyopaque, file: File, data: []const []u |
| 8008 | 8046 | syscall.finish(); |
| 8009 | 8047 | return 0; |
| 8010 | 8048 | }, |
| 8011 | | .NETNAME_DELETED => return syscall.fail(error.ConnectionResetByPeer), |
| 8049 | .NETNAME_DELETED => if (is_debug) unreachable else return error.Unexpected, |
| 8012 | 8050 | .LOCK_VIOLATION => return syscall.fail(error.LockViolation), |
| 8013 | 8051 | .ACCESS_DENIED => return syscall.fail(error.AccessDenied), |
| 8014 | | .INVALID_HANDLE => return syscall.fail(error.NotOpenForReading), |
| 8052 | .INVALID_HANDLE => if (is_debug) unreachable else return error.Unexpected, |
| 8015 | 8053 | // TODO: Determine if INVALID_FUNCTION is possible in more scenarios than just passing |
| 8016 | 8054 | // a handle to a directory. |
| 8017 | 8055 | .INVALID_FUNCTION => return syscall.fail(error.IsDir), |
| ... | ... | @@ -8158,10 +8196,10 @@ fn fileReadPositionalWindows(userdata: ?*anyopaque, file: File, data: []const [] |
| 8158 | 8196 | syscall.finish(); |
| 8159 | 8197 | return 0; |
| 8160 | 8198 | }, |
| 8161 | | .NETNAME_DELETED => return syscall.fail(error.ConnectionResetByPeer), |
| 8199 | .NETNAME_DELETED => if (is_debug) unreachable else return error.Unexpected, |
| 8162 | 8200 | .LOCK_VIOLATION => return syscall.fail(error.LockViolation), |
| 8163 | 8201 | .ACCESS_DENIED => return syscall.fail(error.AccessDenied), |
| 8164 | | .INVALID_HANDLE => return syscall.fail(error.NotOpenForReading), |
| 8202 | .INVALID_HANDLE => if (is_debug) unreachable else return error.Unexpected, |
| 8165 | 8203 | // TODO: Determine if INVALID_FUNCTION is possible in more scenarios than just passing |
| 8166 | 8204 | // a handle to a directory. |
| 8167 | 8205 | .INVALID_FUNCTION => return syscall.fail(error.IsDir), |
| ... | ... | @@ -8842,7 +8880,7 @@ fn writeFilePositionalWindows( |
| 8842 | 8880 | .NOT_ENOUGH_MEMORY => return syscall.fail(error.SystemResources), |
| 8843 | 8881 | .NOT_ENOUGH_QUOTA => return syscall.fail(error.SystemResources), |
| 8844 | 8882 | .NO_DATA => return syscall.fail(error.BrokenPipe), |
| 8845 | | .INVALID_HANDLE => return syscall.fail(error.NotOpenForWriting), |
| 8883 | .INVALID_HANDLE => if (is_debug) unreachable else return error.Unexpected, // use after free |
| 8846 | 8884 | .LOCK_VIOLATION => return syscall.fail(error.LockViolation), |
| 8847 | 8885 | .ACCESS_DENIED => return syscall.fail(error.AccessDenied), |
| 8848 | 8886 | .WORKING_SET_QUOTA => return syscall.fail(error.SystemResources), |
| ... | ... | @@ -12470,6 +12508,7 @@ const linux_statx_request: std.os.linux.STATX = .{ |
| 12470 | 12508 | .INO = true, |
| 12471 | 12509 | .SIZE = true, |
| 12472 | 12510 | .NLINK = true, |
| 12511 | .BLOCKS = true, |
| 12473 | 12512 | }; |
| 12474 | 12513 | |
| 12475 | 12514 | const linux_statx_check: std.os.linux.STATX = .{ |
| ... | ... | @@ -12481,6 +12520,7 @@ const linux_statx_check: std.os.linux.STATX = .{ |
| 12481 | 12520 | .INO = true, |
| 12482 | 12521 | .SIZE = true, |
| 12483 | 12522 | .NLINK = true, |
| 12523 | .BLOCKS = false, |
| 12484 | 12524 | }; |
| 12485 | 12525 | |
| 12486 | 12526 | fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Stat { |
| ... | ... | @@ -12499,6 +12539,7 @@ fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Stat { |
| 12499 | 12539 | }, |
| 12500 | 12540 | .mtime = .{ .nanoseconds = @intCast(@as(i128, stx.mtime.sec) * std.time.ns_per_s + stx.mtime.nsec) }, |
| 12501 | 12541 | .ctime = .{ .nanoseconds = @intCast(@as(i128, stx.ctime.sec) * std.time.ns_per_s + stx.ctime.nsec) }, |
| 12542 | .block_size = if (stx.mask.BLOCKS) stx.blksize else 1, |
| 12502 | 12543 | }; |
| 12503 | 12544 | } |
| 12504 | 12545 | |
| ... | ... | @@ -12547,6 +12588,7 @@ fn statFromPosix(st: *const posix.Stat) File.Stat { |
| 12547 | 12588 | .atime = timestampFromPosix(&atime), |
| 12548 | 12589 | .mtime = timestampFromPosix(&mtime), |
| 12549 | 12590 | .ctime = timestampFromPosix(&ctime), |
| 12591 | .block_size = st.blksize, |
| 12550 | 12592 | }; |
| 12551 | 12593 | } |
| 12552 | 12594 | |
| ... | ... | @@ -12568,6 +12610,7 @@ fn statFromWasi(st: *const std.os.wasi.filestat_t) File.Stat { |
| 12568 | 12610 | .atime = .fromNanoseconds(st.atim), |
| 12569 | 12611 | .mtime = .fromNanoseconds(st.mtim), |
| 12570 | 12612 | .ctime = .fromNanoseconds(st.ctim), |
| 12613 | .block_size = 1, |
| 12571 | 12614 | }; |
| 12572 | 12615 | } |
| 12573 | 12616 | |
| ... | ... | @@ -16127,28 +16170,29 @@ fn fileMemoryMapCreate( |
| 16127 | 16170 | ) File.MemoryMap.CreateError!File.MemoryMap { |
| 16128 | 16171 | const t: *Threaded = @ptrCast(@alignCast(userdata)); |
| 16129 | 16172 | const offset = options.offset; |
| 16173 | const len = options.len; |
| 16130 | 16174 | |
| 16131 | | const page_size = std.heap.pageSize(); |
| 16132 | | const aligned_len: usize = options.len.?; // TODO query if necessary |
| 16175 | assert(std.mem.isAligned(len, std.heap.page_size_min)); |
| 16133 | 16176 | |
| 16134 | | if (createFileMap(file, options.protection, offset, options.populate, aligned_len)) |result| { |
| 16135 | | return result; |
| 16136 | | } else |err| switch (err) { |
| 16137 | | error.Unseekable, error.Canceled => |e| return e, |
| 16138 | | else => { |
| 16139 | | if (builtin.mode == .Debug) |
| 16140 | | std.log.warn("memory mapping failed with {t}, falling back to file operations", .{err}); |
| 16141 | | }, |
| 16177 | if (!t.disable_memory_mapping) { |
| 16178 | if (createFileMap(file, options.protection, offset, options.populate, len)) |result| { |
| 16179 | return result; |
| 16180 | } else |err| switch (err) { |
| 16181 | error.Unseekable, error.Canceled, error.AccessDenied => |e| return e, |
| 16182 | else => { |
| 16183 | if (builtin.mode == .Debug) |
| 16184 | std.log.warn("memory mapping failed with {t}, falling back to file operations", .{err}); |
| 16185 | }, |
| 16186 | } |
| 16142 | 16187 | } |
| 16143 | 16188 | |
| 16144 | 16189 | const gpa = t.allocator; |
| 16145 | | const alignment: Alignment = .fromByteUnits(page_size); |
| 16146 | 16190 | const memory = m: { |
| 16147 | | const ptr = gpa.rawAlloc(aligned_len, alignment, @returnAddress()) orelse |
| 16191 | const ptr = gpa.rawAlloc(len, .@"1", @returnAddress()) orelse |
| 16148 | 16192 | return error.OutOfMemory; |
| 16149 | | break :m ptr[0..aligned_len]; |
| 16193 | break :m ptr[0..len]; |
| 16150 | 16194 | }; |
| 16151 | | errdefer gpa.rawFree(memory, alignment, @returnAddress()); |
| 16195 | errdefer gpa.rawFree(memory, .@"1", @returnAddress()); |
| 16152 | 16196 | |
| 16153 | 16197 | if (!options.undefined_contents) try mmSyncRead(file, memory, offset); |
| 16154 | 16198 | |
| ... | ... | @@ -16180,12 +16224,13 @@ const CreateFileMapError = error{ |
| 16180 | 16224 | OutOfMemory, |
| 16181 | 16225 | MappingAlreadyExists, |
| 16182 | 16226 | Unseekable, |
| 16227 | FileLockConflict, |
| 16183 | 16228 | } || Io.Cancelable || Io.UnexpectedError; |
| 16184 | 16229 | |
| 16185 | 16230 | fn createFileMap( |
| 16186 | 16231 | file: File, |
| 16187 | 16232 | protection: std.process.MemoryProtection, |
| 16188 | | offset: usize, |
| 16233 | offset: u64, |
| 16189 | 16234 | populate: bool, |
| 16190 | 16235 | aligned_len: usize, |
| 16191 | 16236 | ) CreateFileMapError!File.MemoryMap { |
| ... | ... | @@ -16212,7 +16257,7 @@ fn createFileMap( |
| 16212 | 16257 | file.handle, |
| 16213 | 16258 | )) { |
| 16214 | 16259 | .SUCCESS => {}, |
| 16215 | | .FILE_LOCK_CONFLICT => return error.FileLocked, |
| 16260 | .FILE_LOCK_CONFLICT => return error.FileLockConflict, |
| 16216 | 16261 | .INVALID_FILE_FOR_SECTION => return error.OperationUnsupported, |
| 16217 | 16262 | else => |status| return windows.unexpectedStatus(status), |
| 16218 | 16263 | } |
| ... | ... | @@ -16318,9 +16363,7 @@ fn fileMemoryMapDestroy(userdata: ?*anyopaque, mm: *File.MemoryMap) void { |
| 16318 | 16363 | } |
| 16319 | 16364 | } else { |
| 16320 | 16365 | const gpa = t.allocator; |
| 16321 | | const page_size = std.heap.pageSize(); |
| 16322 | | const alignment: Alignment = .fromByteUnits(page_size); |
| 16323 | | gpa.rawFree(memory, alignment, @returnAddress()); |
| 16366 | gpa.rawFree(memory, .@"1", @returnAddress()); |
| 16324 | 16367 | } |
| 16325 | 16368 | mm.* = undefined; |
| 16326 | 16369 | } |
| ... | ... | @@ -16331,6 +16374,7 @@ fn fileMemoryMapSetLength( |
| 16331 | 16374 | new_len: usize, |
| 16332 | 16375 | ) File.MemoryMap.SetLengthError!void { |
| 16333 | 16376 | const t: *Threaded = @ptrCast(@alignCast(userdata)); |
| 16377 | assert(std.mem.isAligned(new_len, std.heap.page_size_min)); |
| 16334 | 16378 | if (mm.section) |section| switch (native_os) { |
| 16335 | 16379 | .windows => { |
| 16336 | 16380 | _ = section; |
| ... | ... | @@ -16366,12 +16410,10 @@ fn fileMemoryMapSetLength( |
| 16366 | 16410 | }, |
| 16367 | 16411 | } else { |
| 16368 | 16412 | const gpa = t.allocator; |
| 16369 | | const page_size = std.heap.pageSize(); |
| 16370 | | const alignment: Alignment = .fromByteUnits(page_size); |
| 16371 | | if (gpa.rawRemap(mm.memory, alignment, new_len, @returnAddress())) |new_ptr| { |
| 16413 | if (gpa.rawRemap(mm.memory, .@"1", new_len, @returnAddress())) |new_ptr| { |
| 16372 | 16414 | mm.memory = new_ptr[0..new_len]; |
| 16373 | 16415 | } else { |
| 16374 | | const new_ptr = gpa.rawAlloc(new_len, alignment, @returnAddress()) orelse |
| 16416 | const new_ptr = gpa.rawAlloc(new_len, .@"1", @returnAddress()) orelse |
| 16375 | 16417 | return error.OutOfMemory; |
| 16376 | 16418 | const copy_len = @min(new_len, mm.memory.len); |
| 16377 | 16419 | @memcpy(new_ptr[0..copy_len], mm.memory[0..copy_len]); |
| ... | ... | @@ -16387,11 +16429,16 @@ fn fileMemoryMapRead(userdata: ?*anyopaque, mm: *File.MemoryMap) File.ReadPositi |
| 16387 | 16429 | return mmSyncRead(mm.file, mm.memory, mm.offset); |
| 16388 | 16430 | } |
| 16389 | 16431 | |
| 16390 | | fn fileMemoryMapWrite(userdata: ?*anyopaque, mm: *File.MemoryMap) File.WritePositionalError!void { |
| 16432 | fn fileMemoryMapWrite( |
| 16433 | userdata: ?*anyopaque, |
| 16434 | mm: *File.MemoryMap, |
| 16435 | file_size: u64, |
| 16436 | ) File.WritePositionalError!void { |
| 16391 | 16437 | const t: *Threaded = @ptrCast(@alignCast(userdata)); |
| 16392 | 16438 | _ = t; |
| 16393 | 16439 | if (mm.section != null) return; |
| 16394 | | return mmSyncWrite(mm.file, mm.memory, mm.offset); |
| 16440 | const offset = mm.offset; |
| 16441 | return mmSyncWrite(mm.file, mm.memory[0..@intCast(file_size - offset)], offset); |
| 16395 | 16442 | } |
| 16396 | 16443 | |
| 16397 | 16444 | fn mmSyncRead(file: File, memory: []u8, offset: u64) File.ReadPositionalError!void { |