authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-10-08 20:14:47-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-10-08 20:14:47-04:00
log76a195473dac059a842fed2a6ba581ca99947d2b
treea70b527370932d85aaa00199b99b6738ded73a8b
parentb02341d6f58e0b8a87fc2ab589dcfd85e5dc96cd
parent03762da2af8753ecf7f4bc2005dd00d570c51ae7
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6516 from LemonBoy/fastfilecopy

std: Make file copy ops use zero-copy mechanisms

3 files changed, 76 insertions(+), 12 deletions(-)

lib/std/c/darwin.zig+8
...@@ -18,6 +18,14 @@ pub extern "c" fn _dyld_get_image_header(image_index: u32) ?*mach_header;...@@ -18,6 +18,14 @@ pub extern "c" fn _dyld_get_image_header(image_index: u32) ?*mach_header;
18pub extern "c" fn _dyld_get_image_vmaddr_slide(image_index: u32) usize;18pub extern "c" fn _dyld_get_image_vmaddr_slide(image_index: u32) usize;
19pub extern "c" fn _dyld_get_image_name(image_index: u32) [*:0]const u8;19pub extern "c" fn _dyld_get_image_name(image_index: u32) [*:0]const u8;
2020
21pub const COPYFILE_ACL = 1 << 0;
22pub const COPYFILE_STAT = 1 << 1;
23pub const COPYFILE_XATTR = 1 << 2;
24pub const COPYFILE_DATA = 1 << 3;
25
26pub const copyfile_state_t = *@Type(.Opaque);
27pub extern "c" fn fcopyfile(from: fd_t, to: fd_t, state: ?copyfile_state_t, flags: u32) c_int;
28
21pub extern "c" fn @"realpath$DARWIN_EXTSN"(noalias file_name: [*:0]const u8, noalias resolved_name: [*]u8) ?[*:0]u8;29pub extern "c" fn @"realpath$DARWIN_EXTSN"(noalias file_name: [*:0]const u8, noalias resolved_name: [*]u8) ?[*:0]u8;
2230
23pub extern "c" fn __getdirentries64(fd: c_int, buf_ptr: [*]u8, buf_len: usize, basep: *i64) isize;31pub extern "c" fn __getdirentries64(fd: c_int, buf_ptr: [*]u8, buf_len: usize, basep: *i64) isize;
lib/std/fs.zig+48-1
...@@ -1823,7 +1823,7 @@ pub const Dir = struct {...@@ -1823,7 +1823,7 @@ pub const Dir = struct {
1823 var atomic_file = try dest_dir.atomicFile(dest_path, .{ .mode = mode });1823 var atomic_file = try dest_dir.atomicFile(dest_path, .{ .mode = mode });
1824 defer atomic_file.deinit();1824 defer atomic_file.deinit();
18251825
1826 try atomic_file.file.writeFileAll(in_file, .{ .in_len = size });1826 try copy_file(in_file.handle, atomic_file.file.handle);
1827 return atomic_file.finish();1827 return atomic_file.finish();
1828 }1828 }
18291829
...@@ -2271,6 +2271,53 @@ pub fn realpathAlloc(allocator: *Allocator, pathname: []const u8) ![]u8 {...@@ -2271,6 +2271,53 @@ pub fn realpathAlloc(allocator: *Allocator, pathname: []const u8) ![]u8 {
2271 return allocator.dupe(u8, try os.realpath(pathname, &buf));2271 return allocator.dupe(u8, try os.realpath(pathname, &buf));
2272}2272}
22732273
2274const CopyFileError = error{SystemResources} || os.CopyFileRangeError || os.SendFileError;
2275
2276// Transfer all the data between two file descriptors in the most efficient way.
2277// The copy starts at offset 0, the initial offsets are preserved.
2278// No metadata is transferred over.
2279fn copy_file(fd_in: os.fd_t, fd_out: os.fd_t) CopyFileError!void {
2280 if (comptime std.Target.current.isDarwin()) {
2281 const rc = os.system.fcopyfile(fd_in, fd_out, null, os.system.COPYFILE_DATA);
2282 switch (os.errno(rc)) {
2283 0 => return,
2284 os.EINVAL => unreachable,
2285 os.ENOMEM => return error.SystemResources,
2286 // The source file is not a directory, symbolic link, or regular file.
2287 // Try with the fallback path before giving up.
2288 os.ENOTSUP => {},
2289 else => |err| return os.unexpectedErrno(err),
2290 }
2291 }
2292
2293 if (std.Target.current.os.tag == .linux) {
2294 // Try copy_file_range first as that works at the FS level and is the
2295 // most efficient method (if available).
2296 var offset: u64 = 0;
2297 cfr_loop: while (true) {
2298 // The kernel checks the u64 value `offset+count` for overflow, use
2299 // a 32 bit value so that the syscall won't return EINVAL except for
2300 // impossibly large files (> 2^64-1 - 2^32-1).
2301 const amt = try os.copy_file_range(fd_in, offset, fd_out, offset, math.maxInt(u32), 0);
2302 // Terminate when no data was copied
2303 if (amt == 0) break :cfr_loop;
2304 offset += amt;
2305 }
2306 return;
2307 }
2308
2309 // Sendfile is a zero-copy mechanism iff the OS supports it, otherwise the
2310 // fallback code will copy the contents chunk by chunk.
2311 const empty_iovec = [0]os.iovec_const{};
2312 var offset: u64 = 0;
2313 sendfile_loop: while (true) {
2314 const amt = try os.sendfile(fd_out, fd_in, offset, 0, &empty_iovec, &empty_iovec, 0);
2315 // Terminate when no data was copied
2316 if (amt == 0) break :sendfile_loop;
2317 offset += amt;
2318 }
2319}
2320
2274test "" {2321test "" {
2275 if (builtin.os.tag != .wasi) {2322 if (builtin.os.tag != .wasi) {
2276 _ = makeDirAbsolute;2323 _ = makeDirAbsolute;
lib/std/os.zig+20-11
...@@ -4945,6 +4945,9 @@ pub fn sendfile(...@@ -4945,6 +4945,9 @@ pub fn sendfile(
4945pub const CopyFileRangeError = error{4945pub const CopyFileRangeError = error{
4946 FileTooBig,4946 FileTooBig,
4947 InputOutput,4947 InputOutput,
4948 /// `fd_in` is not open for reading; or `fd_out` is not open for writing;
4949 /// or the `O_APPEND` flag is set for `fd_out`.
4950 FilesOpenedWithWrongFlags,
4948 IsDir,4951 IsDir,
4949 OutOfMemory,4952 OutOfMemory,
4950 NoSpaceLeft,4953 NoSpaceLeft,
...@@ -4953,6 +4956,11 @@ pub const CopyFileRangeError = error{...@@ -4953,6 +4956,11 @@ pub const CopyFileRangeError = error{
4953 FileBusy,4956 FileBusy,
4954} || PReadError || PWriteError || UnexpectedError;4957} || PReadError || PWriteError || UnexpectedError;
49554958
4959var has_copy_file_range_syscall = init: {
4960 const kernel_has_syscall = std.Target.current.os.isAtLeast(.linux, .{ .major = 4, .minor = 5 }) orelse true;
4961 break :init std.atomic.Int(bool).init(kernel_has_syscall);
4962};
4963
4956/// Transfer data between file descriptors at specified offsets.4964/// Transfer data between file descriptors at specified offsets.
4957/// Returns the number of bytes written, which can less than requested.4965/// Returns the number of bytes written, which can less than requested.
4958///4966///
...@@ -4981,22 +4989,18 @@ pub const CopyFileRangeError = error{...@@ -4981,22 +4989,18 @@ pub const CopyFileRangeError = error{
4981pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len: usize, flags: u32) CopyFileRangeError!usize {4989pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len: usize, flags: u32) CopyFileRangeError!usize {
4982 const use_c = std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 }).ok;4990 const use_c = std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 }).ok;
49834991
4984 // TODO support for other systems than linux4992 if (std.Target.current.os.tag == .linux and
4985 const try_syscall = comptime std.Target.current.os.isAtLeast(.linux, .{ .major = 4, .minor = 5 }) != false;4993 (use_c or has_copy_file_range_syscall.get()))
49864994 {
4987 if (use_c or try_syscall) {
4988 const sys = if (use_c) std.c else linux;4995 const sys = if (use_c) std.c else linux;
49894996
4990 var off_in_copy = @bitCast(i64, off_in);4997 var off_in_copy = @bitCast(i64, off_in);
4991 var off_out_copy = @bitCast(i64, off_out);4998 var off_out_copy = @bitCast(i64, off_out);
49924999
4993 const rc = sys.copy_file_range(fd_in, &off_in_copy, fd_out, &off_out_copy, len, flags);5000 const rc = sys.copy_file_range(fd_in, &off_in_copy, fd_out, &off_out_copy, len, flags);
4994
4995 // TODO avoid wasting a syscall every time if kernel is too old and returns ENOSYS https://github.com/ziglang/zig/issues/1018
4996
4997 switch (sys.getErrno(rc)) {5001 switch (sys.getErrno(rc)) {
4998 0 => return @intCast(usize, rc),5002 0 => return @intCast(usize, rc),
4999 EBADF => unreachable,5003 EBADF => return error.FilesOpenedWithWrongFlags,
5000 EFBIG => return error.FileTooBig,5004 EFBIG => return error.FileTooBig,
5001 EIO => return error.InputOutput,5005 EIO => return error.InputOutput,
5002 EISDIR => return error.IsDir,5006 EISDIR => return error.IsDir,
...@@ -5005,9 +5009,14 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len...@@ -5005,9 +5009,14 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len
5005 EOVERFLOW => return error.Unseekable,5009 EOVERFLOW => return error.Unseekable,
5006 EPERM => return error.PermissionDenied,5010 EPERM => return error.PermissionDenied,
5007 ETXTBSY => return error.FileBusy,5011 ETXTBSY => return error.FileBusy,
5008 EINVAL => {}, // these may not be regular files, try fallback5012 // these may not be regular files, try fallback
5009 EXDEV => {}, // support for cross-filesystem copy added in Linux 5.3, use fallback5013 EINVAL => {},
5010 ENOSYS => {}, // syscall added in Linux 4.5, use fallback5014 // support for cross-filesystem copy added in Linux 5.3, use fallback
5015 EXDEV => {},
5016 // syscall added in Linux 4.5, use fallback
5017 ENOSYS => {
5018 has_copy_file_range_syscall.set(false);
5019 },
5011 else => |err| return unexpectedErrno(err),5020 else => |err| return unexpectedErrno(err),
5012 }5021 }
5013 }5022 }