| author | |
| committer | |
| log | 96cbdd145d6ec5bf2e3ed80743c99ab4b2cdff68 |
| tree | c1b7478f66a390d50bcedb9577fbb93a54ecfff9 |
| parent | b35c55e2373ace674cd1eec7f5086b805d1c8256 |
it wasn't accounting for both writer and reader buffering3 files changed, 128 insertions(+), 299 deletions(-)
lib/std/c.zig+2-2| ... | @@ -10497,9 +10497,9 @@ pub const sysconf = switch (native_os) { | ... | @@ -10497,9 +10497,9 @@ pub const sysconf = switch (native_os) { |
| 10497 | 10497 | ||
| 10498 | pub const sf_hdtr = switch (native_os) { | 10498 | pub const sf_hdtr = switch (native_os) { |
| 10499 | .freebsd, .macos, .ios, .tvos, .watchos, .visionos => extern struct { | 10499 | .freebsd, .macos, .ios, .tvos, .watchos, .visionos => extern struct { |
| 10500 | headers: [*]const iovec_const, | 10500 | headers: ?[*]const iovec_const, |
| 10501 | hdr_cnt: c_int, | 10501 | hdr_cnt: c_int, |
| 10502 | trailers: [*]const iovec_const, | 10502 | trailers: ?[*]const iovec_const, |
| 10503 | trl_cnt: c_int, | 10503 | trl_cnt: c_int, |
| 10504 | }, | 10504 | }, |
| 10505 | else => void, | 10505 | else => void, |
lib/std/fs/File.zig+126-8| ... | @@ -1435,7 +1435,7 @@ pub const Reader = struct { | ... | @@ -1435,7 +1435,7 @@ pub const Reader = struct { |
| 1435 | } | 1435 | } |
| 1436 | return 0; | 1436 | return 0; |
| 1437 | }; | 1437 | }; |
| 1438 | const n = @min(size - pos, std.math.maxInt(i64), @intFromEnum(limit)); | 1438 | const n = @min(size - pos, maxInt(i64), @intFromEnum(limit)); |
| 1439 | file.seekBy(n) catch |err| { | 1439 | file.seekBy(n) catch |err| { |
| 1440 | r.seek_err = err; | 1440 | r.seek_err = err; |
| 1441 | return 0; | 1441 | return 0; |
| ... | @@ -1726,18 +1726,123 @@ pub const Writer = struct { | ... | @@ -1726,18 +1726,123 @@ pub const Writer = struct { |
| 1726 | file_reader: *Reader, | 1726 | file_reader: *Reader, |
| 1727 | limit: std.io.Limit, | 1727 | limit: std.io.Limit, |
| 1728 | ) std.io.Writer.FileError!usize { | 1728 | ) std.io.Writer.FileError!usize { |
| 1729 | const reader_buffered = file_reader.interface.buffered(); | ||
| 1730 | if (reader_buffered.len >= @intFromEnum(limit)) | ||
| 1731 | return sendFileBuffered(io_w, file_reader, reader_buffered); | ||
| 1732 | const writer_buffered = io_w.buffered(); | ||
| 1733 | const file_limit = @intFromEnum(limit) - reader_buffered.len; | ||
| 1729 | const w: *Writer = @alignCast(@fieldParentPtr("interface", io_w)); | 1734 | const w: *Writer = @alignCast(@fieldParentPtr("interface", io_w)); |
| 1730 | const out_fd = w.file.handle; | 1735 | const out_fd = w.file.handle; |
| 1731 | const in_fd = file_reader.file.handle; | 1736 | const in_fd = file_reader.file.handle; |
| 1732 | // TODO try using copy_file_range on FreeBSD | 1737 | |
| 1733 | // TODO try using sendfile on macOS | 1738 | if (native_os == .freebsd and w.mode == .streaming) sf: { |
| 1734 | // TODO try using sendfile on FreeBSD | 1739 | // Try using sendfile on FreeBSD. |
| 1740 | if (w.sendfile_err != null) break :sf; | ||
| 1741 | const offset = std.math.cast(std.c.off_t, file_reader.pos) orelse break :sf; | ||
| 1742 | var hdtr_data: std.c.sf_hdtr = undefined; | ||
| 1743 | var headers: [2]posix.iovec_const = undefined; | ||
| 1744 | var headers_i: u8 = 0; | ||
| 1745 | if (writer_buffered.len != 0) { | ||
| 1746 | headers[headers_i] = .{ .base = writer_buffered.ptr, .len = writer_buffered.len }; | ||
| 1747 | headers_i += 1; | ||
| 1748 | } | ||
| 1749 | if (reader_buffered.len != 0) { | ||
| 1750 | headers[headers_i] = .{ .base = reader_buffered.ptr, .len = reader_buffered.len }; | ||
| 1751 | headers_i += 1; | ||
| 1752 | } | ||
| 1753 | const hdtr: ?*std.c.sf_hdtr = if (headers_i == 0) null else b: { | ||
| 1754 | hdtr_data = .{ | ||
| 1755 | .headers = &headers, | ||
| 1756 | .hdr_cnt = headers_i, | ||
| 1757 | .trailers = null, | ||
| 1758 | .trl_cnt = 0, | ||
| 1759 | }; | ||
| 1760 | break :b &hdtr_data; | ||
| 1761 | }; | ||
| 1762 | var sbytes: std.c.off_t = undefined; | ||
| 1763 | const nbytes: usize = @min(file_limit, maxInt(usize)); | ||
| 1764 | const flags = 0; | ||
| 1765 | switch (posix.errno(std.c.sendfile(in_fd, out_fd, offset, nbytes, hdtr, &sbytes, flags))) { | ||
| 1766 | .SUCCESS, .INTR => {}, | ||
| 1767 | .INVAL, .OPNOTSUPP, .NOTSOCK, .NOSYS => w.sendfile_err = error.UnsupportedOperation, | ||
| 1768 | .BADF => if (builtin.mode == .Debug) @panic("race condition") else { | ||
| 1769 | w.sendfile_err = error.Unexpected; | ||
| 1770 | }, | ||
| 1771 | .FAULT => if (builtin.mode == .Debug) @panic("segmentation fault") else { | ||
| 1772 | w.sendfile_err = error.Unexpected; | ||
| 1773 | }, | ||
| 1774 | .NOTCONN => w.sendfile_err = error.BrokenPipe, | ||
| 1775 | .AGAIN, .BUSY => if (sbytes == 0) { | ||
| 1776 | w.sendfile_err = error.WouldBlock; | ||
| 1777 | }, | ||
| 1778 | .IO => w.sendfile_err = error.InputOutput, | ||
| 1779 | .PIPE => w.sendfile_err = error.BrokenPipe, | ||
| 1780 | .NOBUFS => w.sendfile_err = error.SystemResources, | ||
| 1781 | else => |err| w.sendfile_err = posix.unexpectedErrno(err), | ||
| 1782 | } | ||
| 1783 | const consumed = io_w.consume(@bitCast(sbytes)); | ||
| 1784 | file_reader.seekTo(file_reader.pos + consumed) catch return error.ReadFailed; | ||
| 1785 | return consumed; | ||
| 1786 | } | ||
| 1787 | |||
| 1788 | if (native_os.isDarwin() and w.mode == .streaming) sf: { | ||
| 1789 | // Try using sendfile on macOS. | ||
| 1790 | if (w.sendfile_err != null) break :sf; | ||
| 1791 | const offset = std.math.cast(std.c.off_t, file_reader.pos) orelse break :sf; | ||
| 1792 | var hdtr_data: std.c.sf_hdtr = undefined; | ||
| 1793 | var headers: [2]posix.iovec_const = undefined; | ||
| 1794 | var headers_i: u8 = 0; | ||
| 1795 | if (writer_buffered.len != 0) { | ||
| 1796 | headers[headers_i] = .{ .base = writer_buffered.ptr, .len = writer_buffered.len }; | ||
| 1797 | headers_i += 1; | ||
| 1798 | } | ||
| 1799 | if (reader_buffered.len != 0) { | ||
| 1800 | headers[headers_i] = .{ .base = reader_buffered.ptr, .len = reader_buffered.len }; | ||
| 1801 | headers_i += 1; | ||
| 1802 | } | ||
| 1803 | const hdtr: ?*std.c.sf_hdtr = if (headers_i == 0) null else b: { | ||
| 1804 | hdtr_data = .{ | ||
| 1805 | .headers = &headers, | ||
| 1806 | .hdr_cnt = headers_i, | ||
| 1807 | .trailers = null, | ||
| 1808 | .trl_cnt = 0, | ||
| 1809 | }; | ||
| 1810 | break :b &hdtr_data; | ||
| 1811 | }; | ||
| 1812 | const max_count = maxInt(i32); // Avoid EINVAL. | ||
| 1813 | var sbytes: std.c.off_t = @min(file_limit, max_count); | ||
| 1814 | const flags = 0; | ||
| 1815 | switch (posix.errno(std.c.sendfile(in_fd, out_fd, offset, &sbytes, hdtr, flags))) { | ||
| 1816 | .SUCCESS, .INTR => {}, | ||
| 1817 | .OPNOTSUPP, .NOTSOCK, .NOSYS => w.sendfile_err = error.UnsupportedOperation, | ||
| 1818 | .BADF => if (builtin.mode == .Debug) @panic("race condition") else { | ||
| 1819 | w.sendfile_err = error.Unexpected; | ||
| 1820 | }, | ||
| 1821 | .FAULT => if (builtin.mode == .Debug) @panic("segmentation fault") else { | ||
| 1822 | w.sendfile_err = error.Unexpected; | ||
| 1823 | }, | ||
| 1824 | .INVAL => if (builtin.mode == .Debug) @panic("invalid API usage") else { | ||
| 1825 | w.sendfile_err = error.Unexpected; | ||
| 1826 | }, | ||
| 1827 | .NOTCONN => w.sendfile_err = error.BrokenPipe, | ||
| 1828 | .AGAIN => if (sbytes == 0) { | ||
| 1829 | w.sendfile_err = error.WouldBlock; | ||
| 1830 | }, | ||
| 1831 | .IO => w.sendfile_err = error.InputOutput, | ||
| 1832 | .PIPE => w.sendfile_err = error.BrokenPipe, | ||
| 1833 | else => |err| w.sendfile_err = posix.unexpectedErrno(err), | ||
| 1834 | } | ||
| 1835 | const consumed = io_w.consume(@bitCast(sbytes)); | ||
| 1836 | file_reader.seekTo(file_reader.pos + consumed) catch return error.ReadFailed; | ||
| 1837 | return consumed; | ||
| 1838 | } | ||
| 1839 | |||
| 1735 | if (native_os == .linux and w.mode == .streaming) sf: { | 1840 | if (native_os == .linux and w.mode == .streaming) sf: { |
| 1736 | // Try using sendfile on Linux. | 1841 | // Try using sendfile on Linux. |
| 1737 | if (w.sendfile_err != null) break :sf; | 1842 | if (w.sendfile_err != null) break :sf; |
| 1738 | // Linux sendfile does not support headers. | 1843 | // Linux sendfile does not support headers. |
| 1739 | const buffered = limit.slice(file_reader.interface.buffer); | 1844 | if (writer_buffered.len != 0 or reader_buffered.len != 0) |
| 1740 | if (io_w.end != 0 or buffered.len != 0) return drain(io_w, &.{buffered}, 1); | 1845 | return sendFileBuffered(io_w, file_reader, reader_buffered); |
| 1741 | const max_count = 0x7ffff000; // Avoid EINVAL. | 1846 | const max_count = 0x7ffff000; // Avoid EINVAL. |
| 1742 | var off: std.os.linux.off_t = undefined; | 1847 | var off: std.os.linux.off_t = undefined; |
| 1743 | const off_ptr: ?*std.os.linux.off_t, const count: usize = switch (file_reader.mode) { | 1848 | const off_ptr: ?*std.os.linux.off_t, const count: usize = switch (file_reader.mode) { |
| ... | @@ -1784,6 +1889,7 @@ pub const Writer = struct { | ... | @@ -1784,6 +1889,7 @@ pub const Writer = struct { |
| 1784 | w.pos += n; | 1889 | w.pos += n; |
| 1785 | return n; | 1890 | return n; |
| 1786 | } | 1891 | } |
| 1892 | |||
| 1787 | const copy_file_range = switch (native_os) { | 1893 | const copy_file_range = switch (native_os) { |
| 1788 | .freebsd => std.os.freebsd.copy_file_range, | 1894 | .freebsd => std.os.freebsd.copy_file_range, |
| 1789 | .linux => if (std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 })) std.os.linux.wrapped.copy_file_range else {}, | 1895 | .linux => if (std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 })) std.os.linux.wrapped.copy_file_range else {}, |
| ... | @@ -1791,8 +1897,8 @@ pub const Writer = struct { | ... | @@ -1791,8 +1897,8 @@ pub const Writer = struct { |
| 1791 | }; | 1897 | }; |
| 1792 | if (@TypeOf(copy_file_range) != void) cfr: { | 1898 | if (@TypeOf(copy_file_range) != void) cfr: { |
| 1793 | if (w.copy_file_range_err != null) break :cfr; | 1899 | if (w.copy_file_range_err != null) break :cfr; |
| 1794 | const buffered = limit.slice(file_reader.interface.buffer); | 1900 | if (writer_buffered.len != 0 or reader_buffered.len != 0) |
| 1795 | if (io_w.end != 0 or buffered.len != 0) return drain(io_w, &.{buffered}, 1); | 1901 | return sendFileBuffered(io_w, file_reader, reader_buffered); |
| 1796 | var off_in: i64 = undefined; | 1902 | var off_in: i64 = undefined; |
| 1797 | var off_out: i64 = undefined; | 1903 | var off_out: i64 = undefined; |
| 1798 | const off_in_ptr: ?*i64 = switch (file_reader.mode) { | 1904 | const off_in_ptr: ?*i64 = switch (file_reader.mode) { |
| ... | @@ -1832,6 +1938,8 @@ pub const Writer = struct { | ... | @@ -1832,6 +1938,8 @@ pub const Writer = struct { |
| 1832 | if (w.pos != 0) break :fcf; | 1938 | if (w.pos != 0) break :fcf; |
| 1833 | if (limit != .unlimited) break :fcf; | 1939 | if (limit != .unlimited) break :fcf; |
| 1834 | const size = file_reader.getSize() catch break :fcf; | 1940 | const size = file_reader.getSize() catch break :fcf; |
| 1941 | if (writer_buffered.len != 0 or reader_buffered.len != 0) | ||
| 1942 | return sendFileBuffered(io_w, file_reader, reader_buffered); | ||
| 1835 | const rc = std.c.fcopyfile(in_fd, out_fd, null, .{ .DATA = true }); | 1943 | const rc = std.c.fcopyfile(in_fd, out_fd, null, .{ .DATA = true }); |
| 1836 | switch (posix.errno(rc)) { | 1944 | switch (posix.errno(rc)) { |
| 1837 | .SUCCESS => {}, | 1945 | .SUCCESS => {}, |
| ... | @@ -1860,6 +1968,16 @@ pub const Writer = struct { | ... | @@ -1860,6 +1968,16 @@ pub const Writer = struct { |
| 1860 | return error.Unimplemented; | 1968 | return error.Unimplemented; |
| 1861 | } | 1969 | } |
| 1862 | 1970 | ||
| 1971 | fn sendFileBuffered( | ||
| 1972 | io_w: *std.io.Writer, | ||
| 1973 | file_reader: *Reader, | ||
| 1974 | reader_buffered: []const u8, | ||
| 1975 | ) std.io.Writer.FileError!usize { | ||
| 1976 | const n = try drain(io_w, &.{reader_buffered}, 1); | ||
| 1977 | file_reader.seekTo(file_reader.pos + n) catch return error.ReadFailed; | ||
| 1978 | return n; | ||
| 1979 | } | ||
| 1980 | |||
| 1863 | pub fn seekTo(w: *Writer, offset: u64) SeekError!void { | 1981 | pub fn seekTo(w: *Writer, offset: u64) SeekError!void { |
| 1864 | switch (w.mode) { | 1982 | switch (w.mode) { |
| 1865 | .positional, .positional_reading => { | 1983 | .positional, .positional_reading => { |
lib/std/posix.zig-289| ... | @@ -6326,295 +6326,6 @@ pub fn send( | ... | @@ -6326,295 +6326,6 @@ pub fn send( |
| 6326 | }; | 6326 | }; |
| 6327 | } | 6327 | } |
| 6328 | 6328 | ||
| 6329 | pub const SendFileError = PReadError || WriteError || SendError; | ||
| 6330 | |||
| 6331 | /// Transfer data between file descriptors, with optional headers and trailers. | ||
| 6332 | /// | ||
| 6333 | /// Returns the number of bytes written, which can be zero. | ||
| 6334 | /// | ||
| 6335 | /// The `sendfile` call copies `in_len` bytes from one file descriptor to another. When possible, | ||
| 6336 | /// this is done within the operating system kernel, which can provide better performance | ||
| 6337 | /// characteristics than transferring data from kernel to user space and back, such as with | ||
| 6338 | /// `read` and `write` calls. When `in_len` is `0`, it means to copy until the end of the input file has been | ||
| 6339 | /// reached. Note, however, that partial writes are still possible in this case. | ||
| 6340 | /// | ||
| 6341 | /// `in_fd` must be a file descriptor opened for reading, and `out_fd` must be a file descriptor | ||
| 6342 | /// opened for writing. They may be any kind of file descriptor; however, if `in_fd` is not a regular | ||
| 6343 | /// file system file, it may cause this function to fall back to calling `read` and `write`, in which case | ||
| 6344 | /// atomicity guarantees no longer apply. | ||
| 6345 | /// | ||
| 6346 | /// Copying begins reading at `in_offset`. The input file descriptor seek position is ignored and not updated. | ||
| 6347 | /// If the output file descriptor has a seek position, it is updated as bytes are written. When | ||
| 6348 | /// `in_offset` is past the end of the input file, it successfully reads 0 bytes. | ||
| 6349 | /// | ||
| 6350 | /// `flags` has different meanings per operating system; refer to the respective man pages. | ||
| 6351 | /// | ||
| 6352 | /// These systems support atomically sending everything, including headers and trailers: | ||
| 6353 | /// * macOS | ||
| 6354 | /// * FreeBSD | ||
| 6355 | /// | ||
| 6356 | /// These systems support in-kernel data copying, but headers and trailers are not sent atomically: | ||
| 6357 | /// * Linux | ||
| 6358 | /// | ||
| 6359 | /// Other systems fall back to calling `read` / `write`. | ||
| 6360 | /// | ||
| 6361 | /// Linux has a limit on how many bytes may be transferred in one `sendfile` call, which is `0x7ffff000` | ||
| 6362 | /// on both 64-bit and 32-bit systems. This is due to using a signed C int as the return value, as | ||
| 6363 | /// well as stuffing the errno codes into the last `4096` values. This is noted on the `sendfile` man page. | ||
| 6364 | /// The limit on Darwin is `0x7fffffff`, trying to write more than that returns EINVAL. | ||
| 6365 | /// The corresponding POSIX limit on this is `maxInt(isize)`. | ||
| 6366 | pub fn sendfile( | ||
| 6367 | out_fd: fd_t, | ||
| 6368 | in_fd: fd_t, | ||
| 6369 | in_offset: u64, | ||
| 6370 | in_len: u64, | ||
| 6371 | headers: []const iovec_const, | ||
| 6372 | trailers: []const iovec_const, | ||
| 6373 | flags: u32, | ||
| 6374 | ) SendFileError!usize { | ||
| 6375 | var header_done = false; | ||
| 6376 | var total_written: usize = 0; | ||
| 6377 | |||
| 6378 | // Prevents EOVERFLOW. | ||
| 6379 | const size_t = std.meta.Int(.unsigned, @typeInfo(usize).int.bits - 1); | ||
| 6380 | const max_count = switch (native_os) { | ||
| 6381 | .linux => 0x7ffff000, | ||
| 6382 | .macos, .ios, .watchos, .tvos, .visionos => maxInt(i32), | ||
| 6383 | else => maxInt(size_t), | ||
| 6384 | }; | ||
| 6385 | |||
| 6386 | switch (native_os) { | ||
| 6387 | .linux => sf: { | ||
| 6388 | if (headers.len != 0) { | ||
| 6389 | const amt = try writev(out_fd, headers); | ||
| 6390 | total_written += amt; | ||
| 6391 | if (amt < count_iovec_bytes(headers)) return total_written; | ||
| 6392 | header_done = true; | ||
| 6393 | } | ||
| 6394 | |||
| 6395 | // Here we match BSD behavior, making a zero count value send as many bytes as possible. | ||
| 6396 | const adjusted_count = if (in_len == 0) max_count else @min(in_len, max_count); | ||
| 6397 | |||
| 6398 | const sendfile_sym = if (lfs64_abi) system.sendfile64 else system.sendfile; | ||
| 6399 | while (true) { | ||
| 6400 | var offset: off_t = @bitCast(in_offset); | ||
| 6401 | const rc = sendfile_sym(out_fd, in_fd, &offset, adjusted_count); | ||
| 6402 | switch (errno(rc)) { | ||
| 6403 | .SUCCESS => { | ||
| 6404 | const amt: usize = @bitCast(rc); | ||
| 6405 | total_written += amt; | ||
| 6406 | if (in_len == 0 and amt == 0) { | ||
| 6407 | // We have detected EOF from `in_fd`. | ||
| 6408 | break; | ||
| 6409 | } else if (amt < in_len) { | ||
| 6410 | return total_written; | ||
| 6411 | } else { | ||
| 6412 | break; | ||
| 6413 | } | ||
| 6414 | }, | ||
| 6415 | |||
| 6416 | .BADF => unreachable, // Always a race condition. | ||
| 6417 | .FAULT => unreachable, // Segmentation fault. | ||
| 6418 | .OVERFLOW => unreachable, // We avoid passing too large of a `count`. | ||
| 6419 | .NOTCONN => return error.BrokenPipe, // `out_fd` is an unconnected socket | ||
| 6420 | |||
| 6421 | .INVAL => { | ||
| 6422 | // EINVAL could be any of the following situations: | ||
| 6423 | // * Descriptor is not valid or locked | ||
| 6424 | // * an mmap(2)-like operation is not available for in_fd | ||
| 6425 | // * count is negative | ||
| 6426 | // * out_fd has the APPEND flag set | ||
| 6427 | // Because of the "mmap(2)-like operation" possibility, we fall back to doing read/write | ||
| 6428 | // manually. | ||
| 6429 | break :sf; | ||
| 6430 | }, | ||
| 6431 | .AGAIN => return error.WouldBlock, | ||
| 6432 | .IO => return error.InputOutput, | ||
| 6433 | .PIPE => return error.BrokenPipe, | ||
| 6434 | .NOMEM => return error.SystemResources, | ||
| 6435 | .NXIO => return error.Unseekable, | ||
| 6436 | .SPIPE => return error.Unseekable, | ||
| 6437 | else => |err| { | ||
| 6438 | unexpectedErrno(err) catch {}; | ||
| 6439 | break :sf; | ||
| 6440 | }, | ||
| 6441 | } | ||
| 6442 | } | ||
| 6443 | |||
| 6444 | if (trailers.len != 0) { | ||
| 6445 | total_written += try writev(out_fd, trailers); | ||
| 6446 | } | ||
| 6447 | |||
| 6448 | return total_written; | ||
| 6449 | }, | ||
| 6450 | .freebsd => sf: { | ||
| 6451 | var hdtr_data: std.c.sf_hdtr = undefined; | ||
| 6452 | var hdtr: ?*std.c.sf_hdtr = null; | ||
| 6453 | if (headers.len != 0 or trailers.len != 0) { | ||
| 6454 | // Here we carefully avoid `@intCast` by returning partial writes when | ||
| 6455 | // too many io vectors are provided. | ||
| 6456 | const hdr_cnt = cast(u31, headers.len) orelse maxInt(u31); | ||
| 6457 | if (headers.len > hdr_cnt) return writev(out_fd, headers); | ||
| 6458 | |||
| 6459 | const trl_cnt = cast(u31, trailers.len) orelse maxInt(u31); | ||
| 6460 | |||
| 6461 | hdtr_data = std.c.sf_hdtr{ | ||
| 6462 | .headers = headers.ptr, | ||
| 6463 | .hdr_cnt = hdr_cnt, | ||
| 6464 | .trailers = trailers.ptr, | ||
| 6465 | .trl_cnt = trl_cnt, | ||
| 6466 | }; | ||
| 6467 | hdtr = &hdtr_data; | ||
| 6468 | } | ||
| 6469 | |||
| 6470 | while (true) { | ||
| 6471 | var sbytes: off_t = undefined; | ||
| 6472 | const err = errno(system.sendfile(in_fd, out_fd, @bitCast(in_offset), @min(in_len, max_count), hdtr, &sbytes, flags)); | ||
| 6473 | const amt: usize = @bitCast(sbytes); | ||
| 6474 | switch (err) { | ||
| 6475 | .SUCCESS => return amt, | ||
| 6476 | |||
| 6477 | .BADF => unreachable, // Always a race condition. | ||
| 6478 | .FAULT => unreachable, // Segmentation fault. | ||
| 6479 | .NOTCONN => return error.BrokenPipe, // `out_fd` is an unconnected socket | ||
| 6480 | |||
| 6481 | .INVAL, .OPNOTSUPP, .NOTSOCK, .NOSYS => { | ||
| 6482 | // EINVAL could be any of the following situations: | ||
| 6483 | // * The fd argument is not a regular file. | ||
| 6484 | // * The s argument is not a SOCK.STREAM type socket. | ||
| 6485 | // * The offset argument is negative. | ||
| 6486 | // Because of some of these possibilities, we fall back to doing read/write | ||
| 6487 | // manually, the same as ENOSYS. | ||
| 6488 | break :sf; | ||
| 6489 | }, | ||
| 6490 | |||
| 6491 | .INTR => if (amt != 0) return amt else continue, | ||
| 6492 | |||
| 6493 | .AGAIN => if (amt != 0) { | ||
| 6494 | return amt; | ||
| 6495 | } else { | ||
| 6496 | return error.WouldBlock; | ||
| 6497 | }, | ||
| 6498 | |||
| 6499 | .BUSY => if (amt != 0) { | ||
| 6500 | return amt; | ||
| 6501 | } else { | ||
| 6502 | return error.WouldBlock; | ||
| 6503 | }, | ||
| 6504 | |||
| 6505 | .IO => return error.InputOutput, | ||
| 6506 | .NOBUFS => return error.SystemResources, | ||
| 6507 | .PIPE => return error.BrokenPipe, | ||
| 6508 | |||
| 6509 | else => { | ||
| 6510 | unexpectedErrno(err) catch {}; | ||
| 6511 | if (amt != 0) { | ||
| 6512 | return amt; | ||
| 6513 | } else { | ||
| 6514 | break :sf; | ||
| 6515 | } | ||
| 6516 | }, | ||
| 6517 | } | ||
| 6518 | } | ||
| 6519 | }, | ||
| 6520 | .macos, .ios, .tvos, .watchos, .visionos => sf: { | ||
| 6521 | var hdtr_data: std.c.sf_hdtr = undefined; | ||
| 6522 | var hdtr: ?*std.c.sf_hdtr = null; | ||
| 6523 | if (headers.len != 0 or trailers.len != 0) { | ||
| 6524 | // Here we carefully avoid `@intCast` by returning partial writes when | ||
| 6525 | // too many io vectors are provided. | ||
| 6526 | const hdr_cnt = cast(u31, headers.len) orelse maxInt(u31); | ||
| 6527 | if (headers.len > hdr_cnt) return writev(out_fd, headers); | ||
| 6528 | |||
| 6529 | const trl_cnt = cast(u31, trailers.len) orelse maxInt(u31); | ||
| 6530 | |||
| 6531 | hdtr_data = std.c.sf_hdtr{ | ||
| 6532 | .headers = headers.ptr, | ||
| 6533 | .hdr_cnt = hdr_cnt, | ||
| 6534 | .trailers = trailers.ptr, | ||
| 6535 | .trl_cnt = trl_cnt, | ||
| 6536 | }; | ||
| 6537 | hdtr = &hdtr_data; | ||
| 6538 | } | ||
| 6539 | |||
| 6540 | while (true) { | ||
| 6541 | var sbytes: off_t = @min(in_len, max_count); | ||
| 6542 | const err = errno(system.sendfile(in_fd, out_fd, @bitCast(in_offset), &sbytes, hdtr, flags)); | ||
| 6543 | const amt: usize = @bitCast(sbytes); | ||
| 6544 | switch (err) { | ||
| 6545 | .SUCCESS => return amt, | ||
| 6546 | |||
| 6547 | .BADF => unreachable, // Always a race condition. | ||
| 6548 | .FAULT => unreachable, // Segmentation fault. | ||
| 6549 | .INVAL => unreachable, | ||
| 6550 | .NOTCONN => return error.BrokenPipe, // `out_fd` is an unconnected socket | ||
| 6551 | |||
| 6552 | .OPNOTSUPP, .NOTSOCK, .NOSYS => break :sf, | ||
| 6553 | |||
| 6554 | .INTR => if (amt != 0) return amt else continue, | ||
| 6555 | |||
| 6556 | .AGAIN => if (amt != 0) { | ||
| 6557 | return amt; | ||
| 6558 | } else { | ||
| 6559 | return error.WouldBlock; | ||
| 6560 | }, | ||
| 6561 | |||
| 6562 | .IO => return error.InputOutput, | ||
| 6563 | .PIPE => return error.BrokenPipe, | ||
| 6564 | |||
| 6565 | else => { | ||
| 6566 | unexpectedErrno(err) catch {}; | ||
| 6567 | if (amt != 0) { | ||
| 6568 | return amt; | ||
| 6569 | } else { | ||
| 6570 | break :sf; | ||
| 6571 | } | ||
| 6572 | }, | ||
| 6573 | } | ||
| 6574 | } | ||
| 6575 | }, | ||
| 6576 | else => {}, // fall back to read/write | ||
| 6577 | } | ||
| 6578 | |||
| 6579 | if (headers.len != 0 and !header_done) { | ||
| 6580 | const amt = try writev(out_fd, headers); | ||
| 6581 | total_written += amt; | ||
| 6582 | if (amt < count_iovec_bytes(headers)) return total_written; | ||
| 6583 | } | ||
| 6584 | |||
| 6585 | rw: { | ||
| 6586 | var buf: [8 * 4096]u8 = undefined; | ||
| 6587 | // Here we match BSD behavior, making a zero count value send as many bytes as possible. | ||
| 6588 | const adjusted_count = if (in_len == 0) buf.len else @min(buf.len, in_len); | ||
| 6589 | const amt_read = try pread(in_fd, buf[0..adjusted_count], in_offset); | ||
| 6590 | if (amt_read == 0) { | ||
| 6591 | if (in_len == 0) { | ||
| 6592 | // We have detected EOF from `in_fd`. | ||
| 6593 | break :rw; | ||
| 6594 | } else { | ||
| 6595 | return total_written; | ||
| 6596 | } | ||
| 6597 | } | ||
| 6598 | const amt_written = try write(out_fd, buf[0..amt_read]); | ||
| 6599 | total_written += amt_written; | ||
| 6600 | if (amt_written < in_len or in_len == 0) return total_written; | ||
| 6601 | } | ||
| 6602 | |||
| 6603 | if (trailers.len != 0) { | ||
| 6604 | total_written += try writev(out_fd, trailers); | ||
| 6605 | } | ||
| 6606 | |||
| 6607 | return total_written; | ||
| 6608 | } | ||
| 6609 | |||
| 6610 | fn count_iovec_bytes(iovs: []const iovec_const) usize { | ||
| 6611 | var count: usize = 0; | ||
| 6612 | for (iovs) |iov| { | ||
| 6613 | count += iov.len; | ||
| 6614 | } | ||
| 6615 | return count; | ||
| 6616 | } | ||
| 6617 | |||
| 6618 | pub const CopyFileRangeError = error{ | 6329 | pub const CopyFileRangeError = error{ |
| 6619 | FileTooBig, | 6330 | FileTooBig, |
| 6620 | InputOutput, | 6331 | InputOutput, |