authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-10-03 12:31:17+02:00
committergravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-10-03 12:31:17+02:00
log0f248e0988b24bf4fcdaadd0e47127462206711e
treea90c54f9c5ada84504d1c6775a7e35a8827e1a66
parentb358c281718a2f6437612fb73590a4992242cf52

std: Make file copy ops use zero-copy mechanisms

Use copy_file_range on Linux (if available), fcopyfile on Darwin, sendfile on *BSDs (and on Linux kernels without copy_file_range).

3 files changed, 102 insertions(+), 11 deletions(-)

lib/std/c/darwin.zig+10
...@@ -18,6 +18,16 @@ pub extern "c" fn _dyld_get_image_header(image_index: u32) ?*mach_header;...@@ -18,6 +18,16 @@ pub extern "c" fn _dyld_get_image_header(image_index: u32) ?*mach_header;
18pub extern "c" fn _dyld_get_image_vmaddr_slide(image_index: u32) usize;18pub extern "c" fn _dyld_get_image_vmaddr_slide(image_index: u32) usize;
19pub extern "c" fn _dyld_get_image_name(image_index: u32) [*:0]const u8;19pub extern "c" fn _dyld_get_image_name(image_index: u32) [*:0]const u8;
2020
21pub const COPYFILE_ACL = 1 << 0;
22pub const COPYFILE_STAT = 1 << 1;
23pub const COPYFILE_XATTR = 1 << 2;
24pub const COPYFILE_DATA = 1 << 3;
25
26pub const copyfile_state_t = *@Type(.Opaque);
27pub extern "c" fn copyfile_state_alloc() copyfile_state_t;
28pub extern "c" fn copyfile_state_free(state: copyfile_state_t) c_int;
29pub extern "c" fn fcopyfile(from: fd_t, to: fd_t, state: ?copyfile_state_t, flags: u32) c_int;
30
21pub extern "c" fn @"realpath$DARWIN_EXTSN"(noalias file_name: [*:0]const u8, noalias resolved_name: [*]u8) ?[*:0]u8;31pub extern "c" fn @"realpath$DARWIN_EXTSN"(noalias file_name: [*:0]const u8, noalias resolved_name: [*]u8) ?[*:0]u8;
2232
23pub extern "c" fn __getdirentries64(fd: c_int, buf_ptr: [*]u8, buf_len: usize, basep: *i64) isize;33pub extern "c" fn __getdirentries64(fd: c_int, buf_ptr: [*]u8, buf_len: usize, basep: *i64) isize;
lib/std/fs.zig+1-1
...@@ -1823,7 +1823,7 @@ pub const Dir = struct {...@@ -1823,7 +1823,7 @@ pub const Dir = struct {
1823 var atomic_file = try dest_dir.atomicFile(dest_path, .{ .mode = mode });1823 var atomic_file = try dest_dir.atomicFile(dest_path, .{ .mode = mode });
1824 defer atomic_file.deinit();1824 defer atomic_file.deinit();
18251825
1826 try atomic_file.file.writeFileAll(in_file, .{ .in_len = size });1826 try os.copy_file(in_file.handle, atomic_file.file.handle, .{ .file_size = size });
1827 return atomic_file.finish();1827 return atomic_file.finish();
1828 }1828 }
18291829
lib/std/os.zig+91-10
...@@ -4981,19 +4981,15 @@ pub const CopyFileRangeError = error{...@@ -4981,19 +4981,15 @@ pub const CopyFileRangeError = error{
4981pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len: usize, flags: u32) CopyFileRangeError!usize {4981pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len: usize, flags: u32) CopyFileRangeError!usize {
4982 const use_c = std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 }).ok;4982 const use_c = std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 }).ok;
49834983
4984 // TODO support for other systems than linux4984 if (std.Target.current.os.tag == .linux and
4985 const try_syscall = comptime std.Target.current.os.isAtLeast(.linux, .{ .major = 4, .minor = 5 }) != false;4985 (use_c or has_copy_file_range_syscall.get() != 0))
49864986 {
4987 if (use_c or try_syscall) {
4988 const sys = if (use_c) std.c else linux;4987 const sys = if (use_c) std.c else linux;
49894988
4990 var off_in_copy = @bitCast(i64, off_in);4989 var off_in_copy = @bitCast(i64, off_in);
4991 var off_out_copy = @bitCast(i64, off_out);4990 var off_out_copy = @bitCast(i64, off_out);
49924991
4993 const rc = sys.copy_file_range(fd_in, &off_in_copy, fd_out, &off_out_copy, len, flags);4992 const rc = sys.copy_file_range(fd_in, &off_in_copy, fd_out, &off_out_copy, len, flags);
4994
4995 // TODO avoid wasting a syscall every time if kernel is too old and returns ENOSYS https://github.com/ziglang/zig/issues/1018
4996
4997 switch (sys.getErrno(rc)) {4993 switch (sys.getErrno(rc)) {
4998 0 => return @intCast(usize, rc),4994 0 => return @intCast(usize, rc),
4999 EBADF => unreachable,4995 EBADF => unreachable,
...@@ -5005,9 +5001,14 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len...@@ -5005,9 +5001,14 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len
5005 EOVERFLOW => return error.Unseekable,5001 EOVERFLOW => return error.Unseekable,
5006 EPERM => return error.PermissionDenied,5002 EPERM => return error.PermissionDenied,
5007 ETXTBSY => return error.FileBusy,5003 ETXTBSY => return error.FileBusy,
5008 EINVAL => {}, // these may not be regular files, try fallback5004 // these may not be regular files, try fallback
5009 EXDEV => {}, // support for cross-filesystem copy added in Linux 5.3, use fallback5005 EINVAL => {},
5010 ENOSYS => {}, // syscall added in Linux 4.5, use fallback5006 // support for cross-filesystem copy added in Linux 5.3, use fallback
5007 EXDEV => {},
5008 // syscall added in Linux 4.5, use fallback
5009 ENOSYS => {
5010 has_copy_file_range_syscall.set(0);
5011 },
5011 else => |err| return unexpectedErrno(err),5012 else => |err| return unexpectedErrno(err),
5012 }5013 }
5013 }5014 }
...@@ -5021,6 +5022,86 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len...@@ -5021,6 +5022,86 @@ pub fn copy_file_range(fd_in: fd_t, off_in: u64, fd_out: fd_t, off_out: u64, len
5021 return pwrite(fd_out, buf[0..amt_read], off_out);5022 return pwrite(fd_out, buf[0..amt_read], off_out);
5022}5023}
50235024
5025var has_copy_file_range_syscall = std.atomic.Int(u1).init(1);
5026
5027pub const CopyFileOptions = struct {
5028 /// Size in bytes of the source files, if available saves a call to stat().
5029 file_size: ?u64 = null,
5030};
5031
5032pub const CopyFileError = error{
5033 SystemResources,
5034 FileTooBig,
5035 InputOutput,
5036 IsDir,
5037 OutOfMemory,
5038 NoSpaceLeft,
5039 Unseekable,
5040 PermissionDenied,
5041 FileBusy,
5042} || FStatError || SendFileError;
5043
5044/// Transfer all the data between two file descriptors in the most efficient way.
5045/// No metadata is transferred over.
5046pub fn copy_file(fd_in: fd_t, fd_out: fd_t, options: CopyFileOptions) CopyFileError!void {
5047 if (comptime std.Target.current.isDarwin()) {
5048 const rc = system.fcopyfile(fd_in, fd_out, null, system.COPYFILE_DATA);
5049 switch (errno(rc)) {
5050 0 => return,
5051 EINVAL => unreachable,
5052 ENOMEM => return error.SystemResources,
5053 // The source file was not a directory, symbolic link, or regular file.
5054 // Try with the fallback path before giving up.
5055 ENOTSUP => {},
5056 else => |err| return unexpectedErrno(err),
5057 }
5058 }
5059
5060 const src_file_size = options.file_size orelse
5061 @bitCast(u64, (try fstat(fd_in)).size);
5062 var remaining = src_file_size;
5063
5064 if (std.Target.current.os.tag == .linux) {
5065 // Try copy_file_range first as that works at the FS level and is the
5066 // most efficient method (if available).
5067 if (has_copy_file_range_syscall.get() != 0) {
5068 cfr_loop: while (remaining > 0) {
5069 const copy_amt = math.cast(usize, remaining) catch math.maxInt(usize);
5070 const rc = linux.copy_file_range(fd_in, null, fd_out, null, copy_amt, 0);
5071 switch (errno(rc)) {
5072 0 => {},
5073 EBADF => unreachable,
5074 EFBIG => return error.FileTooBig,
5075 EIO => return error.InputOutput,
5076 EISDIR => return error.IsDir,
5077 ENOMEM => return error.OutOfMemory,
5078 ENOSPC => return error.NoSpaceLeft,
5079 EOVERFLOW => return error.Unseekable,
5080 EPERM => return error.PermissionDenied,
5081 ETXTBSY => return error.FileBusy,
5082 // these may not be regular files, try fallback
5083 EINVAL => break :cfr_loop,
5084 // support for cross-filesystem copy added in Linux 5.3, use fallback
5085 EXDEV => break :cfr_loop,
5086 // syscall added in Linux 4.5, use fallback
5087 ENOSYS => {
5088 has_copy_file_range_syscall.set(0);
5089 break :cfr_loop;
5090 },
5091 else => |err| return unexpectedErrno(err),
5092 }
5093 remaining -= rc;
5094 }
5095 return;
5096 }
5097 }
5098
5099 // Sendfile is a zero-copy mechanism iff the OS supports it, otherwise the
5100 // fallback code will copy the contents chunk by chunk.
5101 const empty_iovec = [0]iovec_const{};
5102 _ = try sendfile(fd_out, fd_in, 0, remaining, &empty_iovec, &empty_iovec, 0);
5103}
5104
5024pub const PollError = error{5105pub const PollError = error{
5025 /// The kernel had no space to allocate file descriptor tables.5106 /// The kernel had no space to allocate file descriptor tables.
5026 SystemResources,5107 SystemResources,