authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-10-09 02:30:31-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-10-09 02:30:31-07:00
log529aa9f270863cfb832bbc65d67c54527126a382
tree0a3b1fa91cf1d528b0c6cb080a4cc7ef76851559
parent0bdd1b5274ceb6ca4145fc880a3d78276801c3fe
parentd83d79c032cf72d26a20068c9f5172b8dba56df8
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #25512 from ziglang/sendfile-fixes

std.Io: Writer and Reader bug fixes related to sendFile, delimiters, Limited, and seeking

10 files changed, 503 insertions(+), 307 deletions(-)

build.zig+7-2
...@@ -81,12 +81,13 @@ pub fn build(b: *std.Build) !void {...@@ -81,12 +81,13 @@ pub fn build(b: *std.Build) !void {
81 docs_step.dependOn(langref_step);81 docs_step.dependOn(langref_step);
82 docs_step.dependOn(std_docs_step);82 docs_step.dependOn(std_docs_step);
8383
84 const test_default_only = b.option(bool, "test-default-only", "Limit test matrix to exactly one target configuration") orelse false;
84 const skip_debug = b.option(bool, "skip-debug", "Main test suite skips debug builds") orelse false;85 const skip_debug = b.option(bool, "skip-debug", "Main test suite skips debug builds") orelse false;
85 const skip_release = b.option(bool, "skip-release", "Main test suite skips release builds") orelse false;86 const skip_release = b.option(bool, "skip-release", "Main test suite skips release builds") orelse test_default_only;
86 const skip_release_small = b.option(bool, "skip-release-small", "Main test suite skips release-small builds") orelse skip_release;87 const skip_release_small = b.option(bool, "skip-release-small", "Main test suite skips release-small builds") orelse skip_release;
87 const skip_release_fast = b.option(bool, "skip-release-fast", "Main test suite skips release-fast builds") orelse skip_release;88 const skip_release_fast = b.option(bool, "skip-release-fast", "Main test suite skips release-fast builds") orelse skip_release;
88 const skip_release_safe = b.option(bool, "skip-release-safe", "Main test suite skips release-safe builds") orelse skip_release;89 const skip_release_safe = b.option(bool, "skip-release-safe", "Main test suite skips release-safe builds") orelse skip_release;
89 const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse false;90 const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse test_default_only;
90 const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;91 const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;
91 const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false;92 const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false;
92 const skip_compile_errors = b.option(bool, "skip-compile-errors", "Main test suite skips compile error tests") orelse false;93 const skip_compile_errors = b.option(bool, "skip-compile-errors", "Main test suite skips compile error tests") orelse false;
...@@ -449,6 +450,7 @@ pub fn build(b: *std.Build) !void {...@@ -449,6 +450,7 @@ pub fn build(b: *std.Build) !void {
449 .include_paths = &.{},450 .include_paths = &.{},
450 .skip_single_threaded = skip_single_threaded,451 .skip_single_threaded = skip_single_threaded,
451 .skip_non_native = skip_non_native,452 .skip_non_native = skip_non_native,
453 .test_default_only = test_default_only,
452 .skip_freebsd = skip_freebsd,454 .skip_freebsd = skip_freebsd,
453 .skip_netbsd = skip_netbsd,455 .skip_netbsd = skip_netbsd,
454 .skip_windows = skip_windows,456 .skip_windows = skip_windows,
...@@ -471,6 +473,7 @@ pub fn build(b: *std.Build) !void {...@@ -471,6 +473,7 @@ pub fn build(b: *std.Build) !void {
471 .include_paths = &.{},473 .include_paths = &.{},
472 .skip_single_threaded = true,474 .skip_single_threaded = true,
473 .skip_non_native = skip_non_native,475 .skip_non_native = skip_non_native,
476 .test_default_only = test_default_only,
474 .skip_freebsd = skip_freebsd,477 .skip_freebsd = skip_freebsd,
475 .skip_netbsd = skip_netbsd,478 .skip_netbsd = skip_netbsd,
476 .skip_windows = skip_windows,479 .skip_windows = skip_windows,
...@@ -492,6 +495,7 @@ pub fn build(b: *std.Build) !void {...@@ -492,6 +495,7 @@ pub fn build(b: *std.Build) !void {
492 .include_paths = &.{},495 .include_paths = &.{},
493 .skip_single_threaded = true,496 .skip_single_threaded = true,
494 .skip_non_native = skip_non_native,497 .skip_non_native = skip_non_native,
498 .test_default_only = test_default_only,
495 .skip_freebsd = skip_freebsd,499 .skip_freebsd = skip_freebsd,
496 .skip_netbsd = skip_netbsd,500 .skip_netbsd = skip_netbsd,
497 .skip_windows = skip_windows,501 .skip_windows = skip_windows,
...@@ -513,6 +517,7 @@ pub fn build(b: *std.Build) !void {...@@ -513,6 +517,7 @@ pub fn build(b: *std.Build) !void {
513 .include_paths = &.{},517 .include_paths = &.{},
514 .skip_single_threaded = skip_single_threaded,518 .skip_single_threaded = skip_single_threaded,
515 .skip_non_native = skip_non_native,519 .skip_non_native = skip_non_native,
520 .test_default_only = test_default_only,
516 .skip_freebsd = skip_freebsd,521 .skip_freebsd = skip_freebsd,
517 .skip_netbsd = skip_netbsd,522 .skip_netbsd = skip_netbsd,
518 .skip_windows = skip_windows,523 .skip_windows = skip_windows,
lib/std/Io/Reader.zig+72-37
...@@ -481,7 +481,6 @@ pub fn readVecAll(r: *Reader, data: [][]u8) Error!void {...@@ -481,7 +481,6 @@ pub fn readVecAll(r: *Reader, data: [][]u8) Error!void {
481/// is returned instead.481/// is returned instead.
482///482///
483/// See also:483/// See also:
484/// * `peek`
485/// * `toss`484/// * `toss`
486pub fn peek(r: *Reader, n: usize) Error![]u8 {485pub fn peek(r: *Reader, n: usize) Error![]u8 {
487 try r.fill(n);486 try r.fill(n);
...@@ -732,7 +731,7 @@ pub const DelimiterError = error{...@@ -732,7 +731,7 @@ pub const DelimiterError = error{
732};731};
733732
734/// Returns a slice of the next bytes of buffered data from the stream until733/// Returns a slice of the next bytes of buffered data from the stream until
735/// `sentinel` is found, advancing the seek position.734/// `sentinel` is found, advancing the seek position past the sentinel.
736///735///
737/// Returned slice has a sentinel.736/// Returned slice has a sentinel.
738///737///
...@@ -765,7 +764,7 @@ pub fn peekSentinel(r: *Reader, comptime sentinel: u8) DelimiterError![:sentinel...@@ -765,7 +764,7 @@ pub fn peekSentinel(r: *Reader, comptime sentinel: u8) DelimiterError![:sentinel
765}764}
766765
767/// Returns a slice of the next bytes of buffered data from the stream until766/// Returns a slice of the next bytes of buffered data from the stream until
768/// `delimiter` is found, advancing the seek position.767/// `delimiter` is found, advancing the seek position past the delimiter.
769///768///
770/// Returned slice includes the delimiter as the last byte.769/// Returned slice includes the delimiter as the last byte.
771///770///
...@@ -793,32 +792,42 @@ pub fn takeDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {...@@ -793,32 +792,42 @@ pub fn takeDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
793/// * `peekDelimiterExclusive`792/// * `peekDelimiterExclusive`
794/// * `takeDelimiterInclusive`793/// * `takeDelimiterInclusive`
795pub fn peekDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {794pub fn peekDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
796 const buffer = r.buffer[0..r.end];795 {
797 const seek = r.seek;796 const contents = r.buffer[0..r.end];
798 if (std.mem.indexOfScalarPos(u8, buffer, seek, delimiter)) |delimiter_index| {797 const seek = r.seek;
799 @branchHint(.likely);798 if (std.mem.findScalarPos(u8, contents, seek, delimiter)) |end| {
800 return buffer[seek .. delimiter_index + 1];799 @branchHint(.likely);
800 return contents[seek .. end + 1];
801 }
801 }802 }
802 // TODO take a parameter for max search length rather than relying on buffer capacity803 while (true) {
803 try rebase(r, r.buffer.len);804 const content_len = r.end - r.seek;
804 while (r.buffer.len - r.end != 0) {805 if (r.buffer.len - content_len == 0) break;
805 const existing_buffered_len = r.end - r.seek;806 try fillMore(r);
806 const end_cap = r.buffer[r.end..];807 const seek = r.seek;
807 var writer: Writer = .fixed(end_cap);808 const contents = r.buffer[0..r.end];
808 const n = r.vtable.stream(r, &writer, .limited(end_cap.len)) catch |err| switch (err) {809 if (std.mem.findScalarPos(u8, contents, seek + content_len, delimiter)) |end| {
809 error.WriteFailed => unreachable,810 return contents[seek .. end + 1];
810 else => |e| return e,
811 };
812 r.end += n;
813 if (std.mem.indexOfScalarPos(u8, r.buffer[0..r.end], r.seek + existing_buffered_len, delimiter)) |delimiter_index| {
814 return r.buffer[r.seek .. delimiter_index + 1];
815 }811 }
816 }812 }
817 return error.StreamTooLong;813 // It might or might not be end of stream. There is no more buffer space
814 // left to disambiguate. If `StreamTooLong` was added to `RebaseError` then
815 // this logic could be replaced by removing the exit condition from the
816 // above while loop. That error code would represent when `buffer` capacity
817 // is too small for an operation, replacing the current use of asserts.
818 var failing_writer = Writer.failing;
819 while (r.vtable.stream(r, &failing_writer, .limited(1))) |n| {
820 assert(n == 0);
821 } else |err| switch (err) {
822 error.WriteFailed => return error.StreamTooLong,
823 error.ReadFailed => |e| return e,
824 error.EndOfStream => |e| return e,
825 }
818}826}
819827
820/// Returns a slice of the next bytes of buffered data from the stream until828/// Returns a slice of the next bytes of buffered data from the stream until
821/// `delimiter` is found, advancing the seek position up to the delimiter.829/// `delimiter` is found, advancing the seek position up to (but not past)
830/// the delimiter.
822///831///
823/// Returned slice excludes the delimiter. End-of-stream is treated equivalent832/// Returned slice excludes the delimiter. End-of-stream is treated equivalent
824/// to a delimiter, unless it would result in a length 0 return value, in which833/// to a delimiter, unless it would result in a length 0 return value, in which
...@@ -832,20 +841,13 @@ pub fn peekDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {...@@ -832,20 +841,13 @@ pub fn peekDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
832/// Invalidates previously returned values from `peek`.841/// Invalidates previously returned values from `peek`.
833///842///
834/// See also:843/// See also:
844/// * `takeDelimiter`
835/// * `takeDelimiterInclusive`845/// * `takeDelimiterInclusive`
836/// * `peekDelimiterExclusive`846/// * `peekDelimiterExclusive`
837pub fn takeDelimiterExclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {847pub fn takeDelimiterExclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
838 const result = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) {848 const result = try r.peekDelimiterExclusive(delimiter);
839 error.EndOfStream => {
840 const remaining = r.buffer[r.seek..r.end];
841 if (remaining.len == 0) return error.EndOfStream;
842 r.toss(remaining.len);
843 return remaining;
844 },
845 else => |e| return e,
846 };
847 r.toss(result.len);849 r.toss(result.len);
848 return result[0 .. result.len - 1];850 return result;
849}851}
850852
851/// Returns a slice of the next bytes of buffered data from the stream until853/// Returns a slice of the next bytes of buffered data from the stream until
...@@ -866,7 +868,7 @@ pub fn takeDelimiterExclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {...@@ -866,7 +868,7 @@ pub fn takeDelimiterExclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
866/// * `takeDelimiterInclusive`868/// * `takeDelimiterInclusive`
867/// * `takeDelimiterExclusive`869/// * `takeDelimiterExclusive`
868pub fn takeDelimiter(r: *Reader, delimiter: u8) error{ ReadFailed, StreamTooLong }!?[]u8 {870pub fn takeDelimiter(r: *Reader, delimiter: u8) error{ ReadFailed, StreamTooLong }!?[]u8 {
869 const result = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) {871 const inclusive = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) {
870 error.EndOfStream => {872 error.EndOfStream => {
871 const remaining = r.buffer[r.seek..r.end];873 const remaining = r.buffer[r.seek..r.end];
872 if (remaining.len == 0) return null;874 if (remaining.len == 0) return null;
...@@ -875,8 +877,8 @@ pub fn takeDelimiter(r: *Reader, delimiter: u8) error{ ReadFailed, StreamTooLong...@@ -875,8 +877,8 @@ pub fn takeDelimiter(r: *Reader, delimiter: u8) error{ ReadFailed, StreamTooLong
875 },877 },
876 else => |e| return e,878 else => |e| return e,
877 };879 };
878 r.toss(result.len + 1);880 r.toss(inclusive.len);
879 return result[0 .. result.len - 1];881 return inclusive[0 .. inclusive.len - 1];
880}882}
881883
882/// Returns a slice of the next bytes of buffered data from the stream until884/// Returns a slice of the next bytes of buffered data from the stream until
...@@ -1403,6 +1405,9 @@ test peekSentinel {...@@ -1403,6 +1405,9 @@ test peekSentinel {
1403 var r: Reader = .fixed("ab\nc");1405 var r: Reader = .fixed("ab\nc");
1404 try testing.expectEqualStrings("ab", try r.peekSentinel('\n'));1406 try testing.expectEqualStrings("ab", try r.peekSentinel('\n'));
1405 try testing.expectEqualStrings("ab", try r.peekSentinel('\n'));1407 try testing.expectEqualStrings("ab", try r.peekSentinel('\n'));
1408 r.toss(3);
1409 try testing.expectError(error.EndOfStream, r.peekSentinel('\n'));
1410 try testing.expectEqualStrings("c", try r.peek(1));
1406}1411}
14071412
1408test takeDelimiterInclusive {1413test takeDelimiterInclusive {
...@@ -1417,22 +1422,52 @@ test peekDelimiterInclusive {...@@ -1417,22 +1422,52 @@ test peekDelimiterInclusive {
1417 try testing.expectEqualStrings("ab\n", try r.peekDelimiterInclusive('\n'));1422 try testing.expectEqualStrings("ab\n", try r.peekDelimiterInclusive('\n'));
1418 r.toss(3);1423 r.toss(3);
1419 try testing.expectError(error.EndOfStream, r.peekDelimiterInclusive('\n'));1424 try testing.expectError(error.EndOfStream, r.peekDelimiterInclusive('\n'));
1425 try testing.expectEqualStrings("c", try r.peek(1));
1420}1426}
14211427
1422test takeDelimiterExclusive {1428test takeDelimiterExclusive {
1423 var r: Reader = .fixed("ab\nc");1429 var r: Reader = .fixed("ab\nc");
1430
1424 try testing.expectEqualStrings("ab", try r.takeDelimiterExclusive('\n'));1431 try testing.expectEqualStrings("ab", try r.takeDelimiterExclusive('\n'));
1432 try testing.expectEqualStrings("", try r.takeDelimiterExclusive('\n'));
1433 try testing.expectEqualStrings("", try r.takeDelimiterExclusive('\n'));
1434 try testing.expectEqualStrings("\n", try r.take(1));
1435
1425 try testing.expectEqualStrings("c", try r.takeDelimiterExclusive('\n'));1436 try testing.expectEqualStrings("c", try r.takeDelimiterExclusive('\n'));
1426 try testing.expectError(error.EndOfStream, r.takeDelimiterExclusive('\n'));1437 try testing.expectError(error.EndOfStream, r.takeDelimiterExclusive('\n'));
1427}1438}
14281439
1429test peekDelimiterExclusive {1440test peekDelimiterExclusive {
1430 var r: Reader = .fixed("ab\nc");1441 var r: Reader = .fixed("ab\nc");
1442
1431 try testing.expectEqualStrings("ab", try r.peekDelimiterExclusive('\n'));1443 try testing.expectEqualStrings("ab", try r.peekDelimiterExclusive('\n'));
1432 try testing.expectEqualStrings("ab", try r.peekDelimiterExclusive('\n'));1444 try testing.expectEqualStrings("ab", try r.peekDelimiterExclusive('\n'));
1433 r.toss(3);1445 r.toss(2);
1446 try testing.expectEqualStrings("", try r.peekDelimiterExclusive('\n'));
1447 try testing.expectEqualStrings("\n", try r.take(1));
1448
1434 try testing.expectEqualStrings("c", try r.peekDelimiterExclusive('\n'));1449 try testing.expectEqualStrings("c", try r.peekDelimiterExclusive('\n'));
1435 try testing.expectEqualStrings("c", try r.peekDelimiterExclusive('\n'));1450 try testing.expectEqualStrings("c", try r.peekDelimiterExclusive('\n'));
1451 r.toss(1);
1452 try testing.expectError(error.EndOfStream, r.peekDelimiterExclusive('\n'));
1453}
1454
1455test takeDelimiter {
1456 var r: Reader = .fixed("ab\nc\n\nd");
1457 try testing.expectEqualStrings("ab", (try r.takeDelimiter('\n')).?);
1458 try testing.expectEqualStrings("c", (try r.takeDelimiter('\n')).?);
1459 try testing.expectEqualStrings("", (try r.takeDelimiter('\n')).?);
1460 try testing.expectEqualStrings("d", (try r.takeDelimiter('\n')).?);
1461 try testing.expectEqual(null, try r.takeDelimiter('\n'));
1462 try testing.expectEqual(null, try r.takeDelimiter('\n'));
1463
1464 r = .fixed("ab\nc\n\nd\n"); // one trailing newline does not affect behavior
1465 try testing.expectEqualStrings("ab", (try r.takeDelimiter('\n')).?);
1466 try testing.expectEqualStrings("c", (try r.takeDelimiter('\n')).?);
1467 try testing.expectEqualStrings("", (try r.takeDelimiter('\n')).?);
1468 try testing.expectEqualStrings("d", (try r.takeDelimiter('\n')).?);
1469 try testing.expectEqual(null, try r.takeDelimiter('\n'));
1470 try testing.expectEqual(null, try r.takeDelimiter('\n'));
1436}1471}
14371472
1438test streamDelimiter {1473test streamDelimiter {
lib/std/Io/Reader/Limited.zig+44
...@@ -27,6 +27,7 @@ pub fn init(reader: *Reader, limit: Limit, buffer: []u8) Limited {...@@ -27,6 +27,7 @@ pub fn init(reader: *Reader, limit: Limit, buffer: []u8) Limited {
2727
28fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {28fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {
29 const l: *Limited = @fieldParentPtr("interface", r);29 const l: *Limited = @fieldParentPtr("interface", r);
30 if (l.remaining == .nothing) return error.EndOfStream;
30 const combined_limit = limit.min(l.remaining);31 const combined_limit = limit.min(l.remaining);
31 const n = try l.unlimited.stream(w, combined_limit);32 const n = try l.unlimited.stream(w, combined_limit);
32 l.remaining = l.remaining.subtract(n).?;33 l.remaining = l.remaining.subtract(n).?;
...@@ -51,8 +52,51 @@ test stream {...@@ -51,8 +52,51 @@ test stream {
5152
52fn discard(r: *Reader, limit: Limit) Reader.Error!usize {53fn discard(r: *Reader, limit: Limit) Reader.Error!usize {
53 const l: *Limited = @fieldParentPtr("interface", r);54 const l: *Limited = @fieldParentPtr("interface", r);
55 if (l.remaining == .nothing) return error.EndOfStream;
54 const combined_limit = limit.min(l.remaining);56 const combined_limit = limit.min(l.remaining);
55 const n = try l.unlimited.discard(combined_limit);57 const n = try l.unlimited.discard(combined_limit);
56 l.remaining = l.remaining.subtract(n).?;58 l.remaining = l.remaining.subtract(n).?;
57 return n;59 return n;
58}60}
61
62test "end of stream, read, hit limit exactly" {
63 var f: Reader = .fixed("i'm dying");
64 var l = f.limited(.limited(4), &.{});
65 const r = &l.interface;
66
67 var buf: [2]u8 = undefined;
68 try r.readSliceAll(&buf);
69 try r.readSliceAll(&buf);
70 try std.testing.expectError(error.EndOfStream, l.interface.readSliceAll(&buf));
71}
72
73test "end of stream, read, hit limit after partial read" {
74 var f: Reader = .fixed("i'm dying");
75 var l = f.limited(.limited(5), &.{});
76 const r = &l.interface;
77
78 var buf: [2]u8 = undefined;
79 try r.readSliceAll(&buf);
80 try r.readSliceAll(&buf);
81 try std.testing.expectError(error.EndOfStream, l.interface.readSliceAll(&buf));
82}
83
84test "end of stream, discard, hit limit exactly" {
85 var f: Reader = .fixed("i'm dying");
86 var l = f.limited(.limited(4), &.{});
87 const r = &l.interface;
88
89 try r.discardAll(2);
90 try r.discardAll(2);
91 try std.testing.expectError(error.EndOfStream, l.interface.discardAll(2));
92}
93
94test "end of stream, discard, hit limit after partial read" {
95 var f: Reader = .fixed("i'm dying");
96 var l = f.limited(.limited(5), &.{});
97 const r = &l.interface;
98
99 try r.discardAll(2);
100 try r.discardAll(2);
101 try std.testing.expectError(error.EndOfStream, l.interface.discardAll(2));
102}
lib/std/Io/Writer.zig+31-6
...@@ -923,10 +923,12 @@ pub fn sendFileHeader(...@@ -923,10 +923,12 @@ pub fn sendFileHeader(
923 return n;923 return n;
924}924}
925925
926/// Asserts nonzero buffer capacity.926/// Asserts nonzero buffer capacity and nonzero `limit`.
927pub fn sendFileReading(w: *Writer, file_reader: *File.Reader, limit: Limit) FileReadingError!usize {927pub fn sendFileReading(w: *Writer, file_reader: *File.Reader, limit: Limit) FileReadingError!usize {
928 assert(limit != .nothing);
928 const dest = limit.slice(try w.writableSliceGreedy(1));929 const dest = limit.slice(try w.writableSliceGreedy(1));
929 const n = try file_reader.read(dest);930 const n = try file_reader.interface.readSliceShort(dest);
931 if (n == 0) return error.EndOfStream;
930 w.advance(n);932 w.advance(n);
931 return n;933 return n;
932}934}
...@@ -2778,7 +2780,8 @@ pub const Allocating = struct {...@@ -2778,7 +2780,8 @@ pub const Allocating = struct {
2778 if (additional == 0) return error.EndOfStream;2780 if (additional == 0) return error.EndOfStream;
2779 a.ensureUnusedCapacity(limit.minInt64(additional)) catch return error.WriteFailed;2781 a.ensureUnusedCapacity(limit.minInt64(additional)) catch return error.WriteFailed;
2780 const dest = limit.slice(a.writer.buffer[a.writer.end..]);2782 const dest = limit.slice(a.writer.buffer[a.writer.end..]);
2781 const n = try file_reader.read(dest);2783 const n = try file_reader.interface.readSliceShort(dest);
2784 if (n == 0) return error.EndOfStream;
2782 a.writer.end += n;2785 a.writer.end += n;
2783 return n;2786 return n;
2784 }2787 }
...@@ -2849,18 +2852,40 @@ test "allocating sendFile" {...@@ -2849,18 +2852,40 @@ test "allocating sendFile" {
28492852
2850 const file = try tmp_dir.dir.createFile("input.txt", .{ .read = true });2853 const file = try tmp_dir.dir.createFile("input.txt", .{ .read = true });
2851 defer file.close();2854 defer file.close();
2852 var r_buffer: [256]u8 = undefined;2855 var r_buffer: [2]u8 = undefined;
2853 var file_writer: std.fs.File.Writer = .init(file, &r_buffer);2856 var file_writer: std.fs.File.Writer = .init(file, &r_buffer);
2854 try file_writer.interface.writeByte('h');2857 try file_writer.interface.writeAll("abcd");
2855 try file_writer.interface.flush();2858 try file_writer.interface.flush();
28562859
2857 var file_reader = file_writer.moveToReader();2860 var file_reader = file_writer.moveToReader();
2858 try file_reader.seekTo(0);2861 try file_reader.seekTo(0);
2862 try file_reader.interface.fill(2);
28592863
2860 var allocating: Writer.Allocating = .init(testing.allocator);2864 var allocating: Writer.Allocating = .init(testing.allocator);
2861 defer allocating.deinit();2865 defer allocating.deinit();
2866 try allocating.ensureUnusedCapacity(1);
2867 try testing.expectEqual(4, allocating.writer.sendFileAll(&file_reader, .unlimited));
2868 try testing.expectEqualStrings("abcd", allocating.writer.buffered());
2869}
2870
2871test sendFileReading {
2872 var tmp_dir = testing.tmpDir(.{});
2873 defer tmp_dir.cleanup();
2874
2875 const file = try tmp_dir.dir.createFile("input.txt", .{ .read = true });
2876 defer file.close();
2877 var r_buffer: [2]u8 = undefined;
2878 var file_writer: std.fs.File.Writer = .init(file, &r_buffer);
2879 try file_writer.interface.writeAll("abcd");
2880 try file_writer.interface.flush();
28622881
2863 _ = try file_reader.interface.streamRemaining(&allocating.writer);2882 var file_reader = file_writer.moveToReader();
2883 try file_reader.seekTo(0);
2884 try file_reader.interface.fill(2);
2885
2886 var w_buffer: [1]u8 = undefined;
2887 var discarding: Writer.Discarding = .init(&w_buffer);
2888 try testing.expectEqual(4, discarding.writer.sendFileReadingAll(&file_reader, .unlimited));
2864}2889}
28652890
2866test writeStruct {2891test writeStruct {
lib/std/fs/File.zig+103-103
...@@ -1189,7 +1189,7 @@ pub const Reader = struct {...@@ -1189,7 +1189,7 @@ pub const Reader = struct {
1189 pub fn seekBy(r: *Reader, offset: i64) Reader.SeekError!void {1189 pub fn seekBy(r: *Reader, offset: i64) Reader.SeekError!void {
1190 switch (r.mode) {1190 switch (r.mode) {
1191 .positional, .positional_reading => {1191 .positional, .positional_reading => {
1192 setPosAdjustingBuffer(r, @intCast(@as(i64, @intCast(r.pos)) + offset));1192 setLogicalPos(r, @intCast(@as(i64, @intCast(logicalPos(r))) + offset));
1193 },1193 },
1194 .streaming, .streaming_reading => {1194 .streaming, .streaming_reading => {
1195 if (posix.SEEK == void) {1195 if (posix.SEEK == void) {
...@@ -1198,7 +1198,7 @@ pub const Reader = struct {...@@ -1198,7 +1198,7 @@ pub const Reader = struct {
1198 }1198 }
1199 const seek_err = r.seek_err orelse e: {1199 const seek_err = r.seek_err orelse e: {
1200 if (posix.lseek_CUR(r.file.handle, offset)) |_| {1200 if (posix.lseek_CUR(r.file.handle, offset)) |_| {
1201 setPosAdjustingBuffer(r, @intCast(@as(i64, @intCast(r.pos)) + offset));1201 setLogicalPos(r, @intCast(@as(i64, @intCast(logicalPos(r))) + offset));
1202 return;1202 return;
1203 } else |err| {1203 } else |err| {
1204 r.seek_err = err;1204 r.seek_err = err;
...@@ -1222,16 +1222,16 @@ pub const Reader = struct {...@@ -1222,16 +1222,16 @@ pub const Reader = struct {
1222 pub fn seekTo(r: *Reader, offset: u64) Reader.SeekError!void {1222 pub fn seekTo(r: *Reader, offset: u64) Reader.SeekError!void {
1223 switch (r.mode) {1223 switch (r.mode) {
1224 .positional, .positional_reading => {1224 .positional, .positional_reading => {
1225 setPosAdjustingBuffer(r, offset);1225 setLogicalPos(r, offset);
1226 },1226 },
1227 .streaming, .streaming_reading => {1227 .streaming, .streaming_reading => {
1228 if (offset >= r.pos) return Reader.seekBy(r, @intCast(offset - r.pos));1228 if (offset >= r.pos) return Reader.seekBy(r, @intCast(offset - logicalPos(r)));
1229 if (r.seek_err) |err| return err;1229 if (r.seek_err) |err| return err;
1230 posix.lseek_SET(r.file.handle, offset) catch |err| {1230 posix.lseek_SET(r.file.handle, offset) catch |err| {
1231 r.seek_err = err;1231 r.seek_err = err;
1232 return err;1232 return err;
1233 };1233 };
1234 setPosAdjustingBuffer(r, offset);1234 setLogicalPos(r, offset);
1235 },1235 },
1236 .failure => return r.seek_err.?,1236 .failure => return r.seek_err.?,
1237 }1237 }
...@@ -1241,7 +1241,7 @@ pub const Reader = struct {...@@ -1241,7 +1241,7 @@ pub const Reader = struct {
1241 return r.pos - r.interface.bufferedLen();1241 return r.pos - r.interface.bufferedLen();
1242 }1242 }
12431243
1244 fn setPosAdjustingBuffer(r: *Reader, offset: u64) void {1244 fn setLogicalPos(r: *Reader, offset: u64) void {
1245 const logical_pos = logicalPos(r);1245 const logical_pos = logicalPos(r);
1246 if (offset < logical_pos or offset >= r.pos) {1246 if (offset < logical_pos or offset >= r.pos) {
1247 r.interface.seek = 0;1247 r.interface.seek = 0;
...@@ -1269,13 +1269,15 @@ pub const Reader = struct {...@@ -1269,13 +1269,15 @@ pub const Reader = struct {
1269 },1269 },
1270 .positional_reading => {1270 .positional_reading => {
1271 const dest = limit.slice(try w.writableSliceGreedy(1));1271 const dest = limit.slice(try w.writableSliceGreedy(1));
1272 const n = try readPositional(r, dest);1272 var data: [1][]u8 = .{dest};
1273 const n = try readVecPositional(r, &data);
1273 w.advance(n);1274 w.advance(n);
1274 return n;1275 return n;
1275 },1276 },
1276 .streaming_reading => {1277 .streaming_reading => {
1277 const dest = limit.slice(try w.writableSliceGreedy(1));1278 const dest = limit.slice(try w.writableSliceGreedy(1));
1278 const n = try readStreaming(r, dest);1279 var data: [1][]u8 = .{dest};
1280 const n = try readVecStreaming(r, &data);
1279 w.advance(n);1281 w.advance(n);
1280 return n;1282 return n;
1281 },1283 },
...@@ -1286,92 +1288,98 @@ pub const Reader = struct {...@@ -1286,92 +1288,98 @@ pub const Reader = struct {
1286 fn readVec(io_reader: *std.Io.Reader, data: [][]u8) std.Io.Reader.Error!usize {1288 fn readVec(io_reader: *std.Io.Reader, data: [][]u8) std.Io.Reader.Error!usize {
1287 const r: *Reader = @alignCast(@fieldParentPtr("interface", io_reader));1289 const r: *Reader = @alignCast(@fieldParentPtr("interface", io_reader));
1288 switch (r.mode) {1290 switch (r.mode) {
1289 .positional, .positional_reading => {1291 .positional, .positional_reading => return readVecPositional(r, data),
1290 if (is_windows) {1292 .streaming, .streaming_reading => return readVecStreaming(r, data),
1291 // Unfortunately, `ReadFileScatter` cannot be used since it1293 .failure => return error.ReadFailed,
1292 // requires page alignment.1294 }
1293 if (io_reader.seek == io_reader.end) {1295 }
1294 io_reader.seek = 0;1296
1295 io_reader.end = 0;1297 fn readVecPositional(r: *Reader, data: [][]u8) std.Io.Reader.Error!usize {
1296 }1298 const io_reader = &r.interface;
1297 const first = data[0];1299 if (is_windows) {
1298 if (first.len >= io_reader.buffer.len - io_reader.end) {1300 // Unfortunately, `ReadFileScatter` cannot be used since it
1299 return readPositional(r, first);1301 // requires page alignment.
1300 } else {1302 if (io_reader.seek == io_reader.end) {
1301 io_reader.end += try readPositional(r, io_reader.buffer[io_reader.end..]);1303 io_reader.seek = 0;
1302 return 0;1304 io_reader.end = 0;
1303 }1305 }
1304 }1306 const first = data[0];
1305 var iovecs_buffer: [max_buffers_len]posix.iovec = undefined;1307 if (first.len >= io_reader.buffer.len - io_reader.end) {
1306 const dest_n, const data_size = try io_reader.writableVectorPosix(&iovecs_buffer, data);1308 return readPositional(r, first);
1307 const dest = iovecs_buffer[0..dest_n];1309 } else {
1308 assert(dest[0].len > 0);1310 io_reader.end += try readPositional(r, io_reader.buffer[io_reader.end..]);
1309 const n = posix.preadv(r.file.handle, dest, r.pos) catch |err| switch (err) {1311 return 0;
1310 error.Unseekable => {1312 }
1311 r.mode = r.mode.toStreaming();1313 }
1312 const pos = r.pos;1314 var iovecs_buffer: [max_buffers_len]posix.iovec = undefined;
1313 if (pos != 0) {1315 const dest_n, const data_size = try io_reader.writableVectorPosix(&iovecs_buffer, data);
1314 r.pos = 0;1316 const dest = iovecs_buffer[0..dest_n];
1315 r.seekBy(@intCast(pos)) catch {1317 assert(dest[0].len > 0);
1316 r.mode = .failure;1318 const n = posix.preadv(r.file.handle, dest, r.pos) catch |err| switch (err) {
1317 return error.ReadFailed;1319 error.Unseekable => {
1318 };1320 r.mode = r.mode.toStreaming();
1319 }1321 const pos = r.pos;
1320 return 0;1322 if (pos != 0) {
1321 },1323 r.pos = 0;
1322 else => |e| {1324 r.seekBy(@intCast(pos)) catch {
1323 r.err = e;1325 r.mode = .failure;
1324 return error.ReadFailed;1326 return error.ReadFailed;
1325 },1327 };
1326 };
1327 if (n == 0) {
1328 r.size = r.pos;
1329 return error.EndOfStream;
1330 }
1331 r.pos += n;
1332 if (n > data_size) {
1333 io_reader.end += n - data_size;
1334 return data_size;
1335 }1328 }
1336 return n;1329 return 0;
1337 },1330 },
1338 .streaming, .streaming_reading => {1331 else => |e| {
1339 if (is_windows) {1332 r.err = e;
1340 // Unfortunately, `ReadFileScatter` cannot be used since it1333 return error.ReadFailed;
1341 // requires page alignment.
1342 if (io_reader.seek == io_reader.end) {
1343 io_reader.seek = 0;
1344 io_reader.end = 0;
1345 }
1346 const first = data[0];
1347 if (first.len >= io_reader.buffer.len - io_reader.end) {
1348 return readStreaming(r, first);
1349 } else {
1350 io_reader.end += try readStreaming(r, io_reader.buffer[io_reader.end..]);
1351 return 0;
1352 }
1353 }
1354 var iovecs_buffer: [max_buffers_len]posix.iovec = undefined;
1355 const dest_n, const data_size = try io_reader.writableVectorPosix(&iovecs_buffer, data);
1356 const dest = iovecs_buffer[0..dest_n];
1357 assert(dest[0].len > 0);
1358 const n = posix.readv(r.file.handle, dest) catch |err| {
1359 r.err = err;
1360 return error.ReadFailed;
1361 };
1362 if (n == 0) {
1363 r.size = r.pos;
1364 return error.EndOfStream;
1365 }
1366 r.pos += n;
1367 if (n > data_size) {
1368 io_reader.end += n - data_size;
1369 return data_size;
1370 }
1371 return n;
1372 },1334 },
1373 .failure => return error.ReadFailed,1335 };
1336 if (n == 0) {
1337 r.size = r.pos;
1338 return error.EndOfStream;
1339 }
1340 r.pos += n;
1341 if (n > data_size) {
1342 io_reader.end += n - data_size;
1343 return data_size;
1374 }1344 }
1345 return n;
1346 }
1347
1348 fn readVecStreaming(r: *Reader, data: [][]u8) std.Io.Reader.Error!usize {
1349 const io_reader = &r.interface;
1350 if (is_windows) {
1351 // Unfortunately, `ReadFileScatter` cannot be used since it
1352 // requires page alignment.
1353 if (io_reader.seek == io_reader.end) {
1354 io_reader.seek = 0;
1355 io_reader.end = 0;
1356 }
1357 const first = data[0];
1358 if (first.len >= io_reader.buffer.len - io_reader.end) {
1359 return readStreaming(r, first);
1360 } else {
1361 io_reader.end += try readStreaming(r, io_reader.buffer[io_reader.end..]);
1362 return 0;
1363 }
1364 }
1365 var iovecs_buffer: [max_buffers_len]posix.iovec = undefined;
1366 const dest_n, const data_size = try io_reader.writableVectorPosix(&iovecs_buffer, data);
1367 const dest = iovecs_buffer[0..dest_n];
1368 assert(dest[0].len > 0);
1369 const n = posix.readv(r.file.handle, dest) catch |err| {
1370 r.err = err;
1371 return error.ReadFailed;
1372 };
1373 if (n == 0) {
1374 r.size = r.pos;
1375 return error.EndOfStream;
1376 }
1377 r.pos += n;
1378 if (n > data_size) {
1379 io_reader.end += n - data_size;
1380 return data_size;
1381 }
1382 return n;
1375 }1383 }
13761384
1377 fn discard(io_reader: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize {1385 fn discard(io_reader: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize {
...@@ -1440,7 +1448,7 @@ pub const Reader = struct {...@@ -1440,7 +1448,7 @@ pub const Reader = struct {
1440 }1448 }
1441 }1449 }
14421450
1443 pub fn readPositional(r: *Reader, dest: []u8) std.Io.Reader.Error!usize {1451 fn readPositional(r: *Reader, dest: []u8) std.Io.Reader.Error!usize {
1444 const n = r.file.pread(dest, r.pos) catch |err| switch (err) {1452 const n = r.file.pread(dest, r.pos) catch |err| switch (err) {
1445 error.Unseekable => {1453 error.Unseekable => {
1446 r.mode = r.mode.toStreaming();1454 r.mode = r.mode.toStreaming();
...@@ -1467,7 +1475,7 @@ pub const Reader = struct {...@@ -1467,7 +1475,7 @@ pub const Reader = struct {
1467 return n;1475 return n;
1468 }1476 }
14691477
1470 pub fn readStreaming(r: *Reader, dest: []u8) std.Io.Reader.Error!usize {1478 fn readStreaming(r: *Reader, dest: []u8) std.Io.Reader.Error!usize {
1471 const n = r.file.read(dest) catch |err| {1479 const n = r.file.read(dest) catch |err| {
1472 r.err = err;1480 r.err = err;
1473 return error.ReadFailed;1481 return error.ReadFailed;
...@@ -1480,14 +1488,6 @@ pub const Reader = struct {...@@ -1480,14 +1488,6 @@ pub const Reader = struct {
1480 return n;1488 return n;
1481 }1489 }
14821490
1483 pub fn read(r: *Reader, dest: []u8) std.Io.Reader.Error!usize {
1484 switch (r.mode) {
1485 .positional, .positional_reading => return readPositional(r, dest),
1486 .streaming, .streaming_reading => return readStreaming(r, dest),
1487 .failure => return error.ReadFailed,
1488 }
1489 }
1490
1491 pub fn atEnd(r: *Reader) bool {1491 pub fn atEnd(r: *Reader) bool {
1492 // Even if stat fails, size is set when end is encountered.1492 // Even if stat fails, size is set when end is encountered.
1493 const size = r.size orelse return false;1493 const size = r.size orelse return false;
...@@ -1732,7 +1732,7 @@ pub const Writer = struct {...@@ -1732,7 +1732,7 @@ pub const Writer = struct {
1732 ) std.Io.Writer.FileError!usize {1732 ) std.Io.Writer.FileError!usize {
1733 const reader_buffered = file_reader.interface.buffered();1733 const reader_buffered = file_reader.interface.buffered();
1734 if (reader_buffered.len >= @intFromEnum(limit))1734 if (reader_buffered.len >= @intFromEnum(limit))
1735 return sendFileBuffered(io_w, file_reader, reader_buffered);1735 return sendFileBuffered(io_w, file_reader, limit.slice(reader_buffered));
1736 const writer_buffered = io_w.buffered();1736 const writer_buffered = io_w.buffered();
1737 const file_limit = @intFromEnum(limit) - reader_buffered.len;1737 const file_limit = @intFromEnum(limit) - reader_buffered.len;
1738 const w: *Writer = @alignCast(@fieldParentPtr("interface", io_w));1738 const w: *Writer = @alignCast(@fieldParentPtr("interface", io_w));
...@@ -1804,7 +1804,7 @@ pub const Writer = struct {...@@ -1804,7 +1804,7 @@ pub const Writer = struct {
1804 return error.EndOfStream;1804 return error.EndOfStream;
1805 }1805 }
1806 const consumed = io_w.consume(@intCast(sbytes));1806 const consumed = io_w.consume(@intCast(sbytes));
1807 file_reader.seekTo(file_reader.pos + consumed) catch return error.ReadFailed;1807 file_reader.seekBy(@intCast(consumed)) catch return error.ReadFailed;
1808 return consumed;1808 return consumed;
1809 }1809 }
18101810
...@@ -1865,7 +1865,7 @@ pub const Writer = struct {...@@ -1865,7 +1865,7 @@ pub const Writer = struct {
1865 return error.EndOfStream;1865 return error.EndOfStream;
1866 }1866 }
1867 const consumed = io_w.consume(@bitCast(len));1867 const consumed = io_w.consume(@bitCast(len));
1868 file_reader.seekTo(file_reader.pos + consumed) catch return error.ReadFailed;1868 file_reader.seekBy(@intCast(consumed)) catch return error.ReadFailed;
1869 return consumed;1869 return consumed;
1870 }1870 }
18711871
...@@ -1998,7 +1998,7 @@ pub const Writer = struct {...@@ -1998,7 +1998,7 @@ pub const Writer = struct {
1998 reader_buffered: []const u8,1998 reader_buffered: []const u8,
1999 ) std.Io.Writer.FileError!usize {1999 ) std.Io.Writer.FileError!usize {
2000 const n = try drain(io_w, &.{reader_buffered}, 1);2000 const n = try drain(io_w, &.{reader_buffered}, 1);
2001 file_reader.seekTo(file_reader.pos + n) catch return error.ReadFailed;2001 file_reader.seekBy(@intCast(n)) catch return error.ReadFailed;
2002 return n;2002 return n;
2003 }2003 }
20042004
lib/std/fs/test.zig+66
...@@ -1525,6 +1525,41 @@ test "sendfile" {...@@ -1525,6 +1525,41 @@ test "sendfile" {
1525 try testing.expectEqualStrings("header1\nsecond header\nine1\nsecontrailer1\nsecond trailer\n", written_buf[0..amt]);1525 try testing.expectEqualStrings("header1\nsecond header\nine1\nsecontrailer1\nsecond trailer\n", written_buf[0..amt]);
1526}1526}
15271527
1528test "sendfile with buffered data" {
1529 var tmp = tmpDir(.{});
1530 defer tmp.cleanup();
1531
1532 try tmp.dir.makePath("os_test_tmp");
1533
1534 var dir = try tmp.dir.openDir("os_test_tmp", .{});
1535 defer dir.close();
1536
1537 var src_file = try dir.createFile("sendfile1.txt", .{ .read = true });
1538 defer src_file.close();
1539
1540 try src_file.writeAll("AAAABBBB");
1541
1542 var dest_file = try dir.createFile("sendfile2.txt", .{ .read = true });
1543 defer dest_file.close();
1544
1545 var src_buffer: [32]u8 = undefined;
1546 var file_reader = src_file.reader(&src_buffer);
1547
1548 try file_reader.seekTo(0);
1549 try file_reader.interface.fill(8);
1550
1551 var fallback_buffer: [32]u8 = undefined;
1552 var file_writer = dest_file.writer(&fallback_buffer);
1553
1554 try std.testing.expectEqual(4, try file_writer.interface.sendFileAll(&file_reader, .limited(4)));
1555
1556 var written_buf: [8]u8 = undefined;
1557 const amt = try dest_file.preadAll(&written_buf, 0);
1558
1559 try std.testing.expectEqual(4, amt);
1560 try std.testing.expectEqualSlices(u8, "AAAA", written_buf[0..amt]);
1561}
1562
1528test "copyRangeAll" {1563test "copyRangeAll" {
1529 var tmp = tmpDir(.{});1564 var tmp = tmpDir(.{});
1530 defer tmp.cleanup();1565 defer tmp.cleanup();
...@@ -2250,3 +2285,34 @@ test "seekTo flushes buffered data" {...@@ -2250,3 +2285,34 @@ test "seekTo flushes buffered data" {
2250 try file_reader.interface.readSliceAll(&buf);2285 try file_reader.interface.readSliceAll(&buf);
2251 try std.testing.expectEqualStrings(contents, &buf);2286 try std.testing.expectEqualStrings(contents, &buf);
2252}2287}
2288
2289test "File.Writer sendfile with buffered contents" {
2290 var tmp_dir = testing.tmpDir(.{});
2291 defer tmp_dir.cleanup();
2292
2293 {
2294 try tmp_dir.dir.writeFile(.{ .sub_path = "a", .data = "bcd" });
2295 const in = try tmp_dir.dir.openFile("a", .{});
2296 defer in.close();
2297 const out = try tmp_dir.dir.createFile("b", .{});
2298 defer out.close();
2299
2300 var in_buf: [2]u8 = undefined;
2301 var in_r = in.reader(&in_buf);
2302 _ = try in_r.getSize(); // Catch seeks past end by populating size
2303 try in_r.interface.fill(2);
2304
2305 var out_buf: [1]u8 = undefined;
2306 var out_w = out.writerStreaming(&out_buf);
2307 try out_w.interface.writeByte('a');
2308 try testing.expectEqual(3, try out_w.interface.sendFileAll(&in_r, .unlimited));
2309 try out_w.interface.flush();
2310 }
2311
2312 var check = try tmp_dir.dir.openFile("b", .{});
2313 defer check.close();
2314 var check_buf: [4]u8 = undefined;
2315 var check_r = check.reader(&check_buf);
2316 try testing.expectEqualStrings("abcd", try check_r.interface.take(4));
2317 try testing.expectError(error.EndOfStream, check_r.interface.takeByte());
2318}
lib/std/net.zig+3-2
...@@ -1396,7 +1396,7 @@ fn parseHosts(...@@ -1396,7 +1396,7 @@ fn parseHosts(
1396 br: *Io.Reader,1396 br: *Io.Reader,
1397) error{ OutOfMemory, ReadFailed }!void {1397) error{ OutOfMemory, ReadFailed }!void {
1398 while (true) {1398 while (true) {
1399 const line = br.takeDelimiterExclusive('\n') catch |err| switch (err) {1399 const line = br.takeDelimiter('\n') catch |err| switch (err) {
1400 error.StreamTooLong => {1400 error.StreamTooLong => {
1401 // Skip lines that are too long.1401 // Skip lines that are too long.
1402 _ = br.discardDelimiterInclusive('\n') catch |e| switch (e) {1402 _ = br.discardDelimiterInclusive('\n') catch |e| switch (e) {
...@@ -1406,7 +1406,8 @@ fn parseHosts(...@@ -1406,7 +1406,8 @@ fn parseHosts(
1406 continue;1406 continue;
1407 },1407 },
1408 error.ReadFailed => return error.ReadFailed,1408 error.ReadFailed => return error.ReadFailed,
1409 error.EndOfStream => break,1409 } orelse {
1410 break; // end of stream
1410 };1411 };
1411 var split_it = mem.splitScalar(u8, line, '#');1412 var split_it = mem.splitScalar(u8, line, '#');
1412 const no_comment_line = split_it.first();1413 const no_comment_line = split_it.first();
lib/std/zig/system/linux.zig+1-4
...@@ -359,14 +359,11 @@ fn CpuinfoParser(comptime impl: anytype) type {...@@ -359,14 +359,11 @@ fn CpuinfoParser(comptime impl: anytype) type {
359 return struct {359 return struct {
360 fn parse(arch: Target.Cpu.Arch, reader: *std.Io.Reader) !?Target.Cpu {360 fn parse(arch: Target.Cpu.Arch, reader: *std.Io.Reader) !?Target.Cpu {
361 var obj: impl = .{};361 var obj: impl = .{};
362 while (reader.takeDelimiterExclusive('\n')) |line| {362 while (try reader.takeDelimiter('\n')) |line| {
363 const colon_pos = mem.indexOfScalar(u8, line, ':') orelse continue;363 const colon_pos = mem.indexOfScalar(u8, line, ':') orelse continue;
364 const key = mem.trimEnd(u8, line[0..colon_pos], " \t");364 const key = mem.trimEnd(u8, line[0..colon_pos], " \t");
365 const value = mem.trimStart(u8, line[colon_pos + 1 ..], " \t");365 const value = mem.trimStart(u8, line[colon_pos + 1 ..], " \t");
366 if (!try obj.line_hook(key, value)) break;366 if (!try obj.line_hook(key, value)) break;
367 } else |err| switch (err) {
368 error.EndOfStream => {},
369 else => |e| return e,
370 }367 }
371 return obj.finalize(arch);368 return obj.finalize(arch);
372 }369 }
src/link/MappedFile.zig+2-1
...@@ -411,7 +411,8 @@ pub const Node = extern struct {...@@ -411,7 +411,8 @@ pub const Node = extern struct {
411 .failure,411 .failure,
412 => {412 => {
413 const dest = limit.slice(interface.unusedCapacitySlice());413 const dest = limit.slice(interface.unusedCapacitySlice());
414 const n = try file_reader.read(dest);414 const n = try file_reader.interface.readSliceShort(dest);
415 if (n == 0) return error.EndOfStream;
415 interface.end += n;416 interface.end += n;
416 return n;417 return n;
417 },418 },
test/tests.zig+174-152
...@@ -44,7 +44,7 @@ const test_targets = blk: {...@@ -44,7 +44,7 @@ const test_targets = blk: {
44 break :blk [_]TestTarget{44 break :blk [_]TestTarget{
45 // Native Targets45 // Native Targets
4646
47 .{},47 .{}, // 0 index must be all defaults
48 .{48 .{
49 .link_libc = true,49 .link_libc = true,
50 },50 },
...@@ -2224,6 +2224,7 @@ const ModuleTestOptions = struct {...@@ -2224,6 +2224,7 @@ const ModuleTestOptions = struct {
2224 desc: []const u8,2224 desc: []const u8,
2225 optimize_modes: []const OptimizeMode,2225 optimize_modes: []const OptimizeMode,
2226 include_paths: []const []const u8,2226 include_paths: []const []const u8,
2227 test_default_only: bool,
2227 skip_single_threaded: bool,2228 skip_single_threaded: bool,
2228 skip_non_native: bool,2229 skip_non_native: bool,
2229 skip_freebsd: bool,2230 skip_freebsd: bool,
...@@ -2235,13 +2236,21 @@ const ModuleTestOptions = struct {...@@ -2235,13 +2236,21 @@ const ModuleTestOptions = struct {
2235 skip_libc: bool,2236 skip_libc: bool,
2236 max_rss: usize = 0,2237 max_rss: usize = 0,
2237 no_builtin: bool = false,2238 no_builtin: bool = false,
2238 build_options: ?*std.Build.Step.Options = null,2239 build_options: ?*Step.Options = null,
2239};2240};
22402241
2241pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {2242pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
2242 const step = b.step(b.fmt("test-{s}", .{options.name}), options.desc);2243 const step = b.step(b.fmt("test-{s}", .{options.name}), options.desc);
22432244
2244 for_targets: for (test_targets) |test_target| {2245 if (options.test_default_only) {
2246 const test_target = &test_targets[0];
2247 const resolved_target = b.resolveTargetQuery(test_target.target);
2248 const triple_txt = resolved_target.query.zigTriple(b.allocator) catch @panic("OOM");
2249 addOneModuleTest(b, step, test_target, &resolved_target, triple_txt, options);
2250 return step;
2251 }
2252
2253 for_targets: for (&test_targets) |*test_target| {
2245 if (test_target.skip_modules.len > 0) {2254 if (test_target.skip_modules.len > 0) {
2246 for (test_target.skip_modules) |skip_mod| {2255 for (test_target.skip_modules) |skip_mod| {
2247 if (std.mem.eql(u8, options.name, skip_mod)) continue :for_targets;2256 if (std.mem.eql(u8, options.name, skip_mod)) continue :for_targets;
...@@ -2306,167 +2315,180 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {...@@ -2306,167 +2315,180 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
2306 } else false;2315 } else false;
2307 if (!want_this_mode) continue;2316 if (!want_this_mode) continue;
23082317
2309 const libc_suffix = if (test_target.link_libc == true) "-libc" else "";2318 addOneModuleTest(b, step, test_target, &resolved_target, triple_txt, options);
2310 const model_txt = target.cpu.model.name;2319 }
2320 return step;
2321}
23112322
2312 // wasm32-wasi builds need more RAM, idk why2323fn addOneModuleTest(
2313 const max_rss = if (target.os.tag == .wasi)2324 b: *std.Build,
2314 options.max_rss * 22325 step: *Step,
2315 else2326 test_target: *const TestTarget,
2316 options.max_rss;2327 resolved_target: *const std.Build.ResolvedTarget,
2328 triple_txt: []const u8,
2329 options: ModuleTestOptions,
2330) void {
2331 const target = &resolved_target.result;
2332 const libc_suffix = if (test_target.link_libc == true) "-libc" else "";
2333 const model_txt = target.cpu.model.name;
2334
2335 // wasm32-wasi builds need more RAM, idk why
2336 const max_rss = if (target.os.tag == .wasi)
2337 options.max_rss * 2
2338 else
2339 options.max_rss;
2340
2341 const these_tests = b.addTest(.{
2342 .root_module = b.createModule(.{
2343 .root_source_file = b.path(options.root_src),
2344 .optimize = test_target.optimize_mode,
2345 .target = resolved_target.*,
2346 .link_libc = test_target.link_libc,
2347 .pic = test_target.pic,
2348 .strip = test_target.strip,
2349 .single_threaded = test_target.single_threaded,
2350 }),
2351 .max_rss = max_rss,
2352 .filters = options.test_filters,
2353 .use_llvm = test_target.use_llvm,
2354 .use_lld = test_target.use_lld,
2355 .zig_lib_dir = b.path("lib"),
2356 });
2357 these_tests.linkage = test_target.linkage;
2358 if (options.no_builtin) these_tests.root_module.no_builtin = false;
2359 if (options.build_options) |build_options| {
2360 these_tests.root_module.addOptions("build_options", build_options);
2361 }
2362 const single_threaded_suffix = if (test_target.single_threaded == true) "-single" else "";
2363 const backend_suffix = if (test_target.use_llvm == true)
2364 "-llvm"
2365 else if (target.ofmt == std.Target.ObjectFormat.c)
2366 "-cbe"
2367 else if (test_target.use_llvm == false)
2368 "-selfhosted"
2369 else
2370 "";
2371 const use_lld = if (test_target.use_lld == false) "-no-lld" else "";
2372 const linkage_name = if (test_target.linkage) |linkage| switch (linkage) {
2373 inline else => |t| "-" ++ @tagName(t),
2374 } else "";
2375 const use_pic = if (test_target.pic == true) "-pic" else "";
2376
2377 for (options.include_paths) |include_path| these_tests.root_module.addIncludePath(b.path(include_path));
2378
2379 const qualified_name = b.fmt("{s}-{s}-{s}-{t}{s}{s}{s}{s}{s}{s}", .{
2380 options.name,
2381 triple_txt,
2382 model_txt,
2383 test_target.optimize_mode,
2384 libc_suffix,
2385 single_threaded_suffix,
2386 backend_suffix,
2387 use_lld,
2388 linkage_name,
2389 use_pic,
2390 });
23172391
2318 const these_tests = b.addTest(.{2392 if (target.ofmt == std.Target.ObjectFormat.c) {
2319 .root_module = b.createModule(.{2393 var altered_query = test_target.target;
2320 .root_source_file = b.path(options.root_src),2394 altered_query.ofmt = null;
2321 .optimize = test_target.optimize_mode,2395
2322 .target = resolved_target,2396 const compile_c = b.createModule(.{
2323 .link_libc = test_target.link_libc,2397 .root_source_file = null,
2324 .pic = test_target.pic,2398 .link_libc = test_target.link_libc,
2325 .strip = test_target.strip,2399 .target = b.resolveTargetQuery(altered_query),
2326 .single_threaded = test_target.single_threaded,
2327 }),
2328 .max_rss = max_rss,
2329 .filters = options.test_filters,
2330 .use_llvm = test_target.use_llvm,
2331 .use_lld = test_target.use_lld,
2332 .zig_lib_dir = b.path("lib"),
2333 });2400 });
2334 these_tests.linkage = test_target.linkage;2401 const compile_c_exe = b.addExecutable(.{
2335 if (options.no_builtin) these_tests.root_module.no_builtin = false;2402 .name = qualified_name,
2336 if (options.build_options) |build_options| {2403 .root_module = compile_c,
2337 these_tests.root_module.addOptions("build_options", build_options);2404 .zig_lib_dir = b.path("lib"),
2338 }
2339 const single_threaded_suffix = if (test_target.single_threaded == true) "-single" else "";
2340 const backend_suffix = if (test_target.use_llvm == true)
2341 "-llvm"
2342 else if (target.ofmt == std.Target.ObjectFormat.c)
2343 "-cbe"
2344 else if (test_target.use_llvm == false)
2345 "-selfhosted"
2346 else
2347 "";
2348 const use_lld = if (test_target.use_lld == false) "-no-lld" else "";
2349 const linkage_name = if (test_target.linkage) |linkage| switch (linkage) {
2350 inline else => |t| "-" ++ @tagName(t),
2351 } else "";
2352 const use_pic = if (test_target.pic == true) "-pic" else "";
2353
2354 for (options.include_paths) |include_path| these_tests.root_module.addIncludePath(b.path(include_path));
2355
2356 const qualified_name = b.fmt("{s}-{s}-{s}-{s}{s}{s}{s}{s}{s}{s}", .{
2357 options.name,
2358 triple_txt,
2359 model_txt,
2360 @tagName(test_target.optimize_mode),
2361 libc_suffix,
2362 single_threaded_suffix,
2363 backend_suffix,
2364 use_lld,
2365 linkage_name,
2366 use_pic,
2367 });2405 });
23682406
2369 if (target.ofmt == std.Target.ObjectFormat.c) {2407 compile_c.addCSourceFile(.{
2370 var altered_query = test_target.target;2408 .file = these_tests.getEmittedBin(),
2371 altered_query.ofmt = null;2409 .flags = &.{
23722410 // Tracking issue for making the C backend generate C89 compatible code:
2373 const compile_c = b.createModule(.{2411 // https://github.com/ziglang/zig/issues/19468
2374 .root_source_file = null,2412 "-std=c99",
2375 .link_libc = test_target.link_libc,2413 "-Werror",
2376 .target = b.resolveTargetQuery(altered_query),2414
2377 });2415 "-Wall",
2378 const compile_c_exe = b.addExecutable(.{2416 "-Wembedded-directive",
2379 .name = qualified_name,2417 "-Wempty-translation-unit",
2380 .root_module = compile_c,2418 "-Wextra",
2381 .zig_lib_dir = b.path("lib"),2419 "-Wgnu",
2382 });2420 "-Winvalid-utf8",
23832421 "-Wkeyword-macro",
2384 compile_c.addCSourceFile(.{2422 "-Woverlength-strings",
2385 .file = these_tests.getEmittedBin(),2423
2386 .flags = &.{2424 // Tracking issue for making the C backend generate code
2387 // Tracking issue for making the C backend generate C89 compatible code:2425 // that does not trigger warnings:
2388 // https://github.com/ziglang/zig/issues/194682426 // https://github.com/ziglang/zig/issues/19467
2389 "-std=c99",2427
2390 "-Werror",2428 // spotted everywhere
23912429 "-Wno-builtin-requires-header",
2392 "-Wall",2430
2393 "-Wembedded-directive",2431 // spotted on linux
2394 "-Wempty-translation-unit",2432 "-Wno-braced-scalar-init",
2395 "-Wextra",2433 "-Wno-excess-initializers",
2396 "-Wgnu",2434 "-Wno-incompatible-pointer-types-discards-qualifiers",
2397 "-Winvalid-utf8",2435 "-Wno-unused",
2398 "-Wkeyword-macro",2436 "-Wno-unused-parameter",
2399 "-Woverlength-strings",2437
24002438 // spotted on darwin
2401 // Tracking issue for making the C backend generate code2439 "-Wno-incompatible-pointer-types",
2402 // that does not trigger warnings:2440
2403 // https://github.com/ziglang/zig/issues/194672441 // https://github.com/llvm/llvm-project/issues/153314
24042442 "-Wno-unterminated-string-initialization",
2405 // spotted everywhere2443
2406 "-Wno-builtin-requires-header",2444 // In both Zig and C it is legal to return a pointer to a
24072445 // local. The C backend lowers such thing directly, so the
2408 // spotted on linux2446 // corresponding warning in C must be disabled.
2409 "-Wno-braced-scalar-init",2447 "-Wno-return-stack-address",
2410 "-Wno-excess-initializers",2448 },
2411 "-Wno-incompatible-pointer-types-discards-qualifiers",2449 });
2412 "-Wno-unused",2450 compile_c.addIncludePath(b.path("lib")); // for zig.h
2413 "-Wno-unused-parameter",2451 if (target.os.tag == .windows) {
24142452 if (true) {
2415 // spotted on darwin2453 // Unfortunately this requires about 8G of RAM for clang to compile
2416 "-Wno-incompatible-pointer-types",2454 // and our Windows CI runners do not have this much.
24172455 // TODO This is not an appropriate way to work around this problem.
2418 // https://github.com/llvm/llvm-project/issues/1533142456 step.dependOn(&these_tests.step);
2419 "-Wno-unterminated-string-initialization",2457 return;
24202458 }
2421 // In both Zig and C it is legal to return a pointer to a2459 if (test_target.link_libc == false) {
2422 // local. The C backend lowers such thing directly, so the2460 compile_c_exe.subsystem = .Console;
2423 // corresponding warning in C must be disabled.2461 compile_c.linkSystemLibrary("kernel32", .{});
2424 "-Wno-return-stack-address",2462 compile_c.linkSystemLibrary("ntdll", .{});
2425 },2463 }
2426 });2464 if (mem.eql(u8, options.name, "std")) {
2427 compile_c.addIncludePath(b.path("lib")); // for zig.h
2428 if (target.os.tag == .windows) {
2429 if (true) {
2430 // Unfortunately this requires about 8G of RAM for clang to compile
2431 // and our Windows CI runners do not have this much.
2432 step.dependOn(&these_tests.step);
2433 continue;
2434 }
2435 if (test_target.link_libc == false) {2465 if (test_target.link_libc == false) {
2436 compile_c_exe.subsystem = .Console;2466 compile_c.linkSystemLibrary("shell32", .{});
2437 compile_c.linkSystemLibrary("kernel32", .{});2467 compile_c.linkSystemLibrary("advapi32", .{});
2438 compile_c.linkSystemLibrary("ntdll", .{});
2439 }
2440 if (mem.eql(u8, options.name, "std")) {
2441 if (test_target.link_libc == false) {
2442 compile_c.linkSystemLibrary("shell32", .{});
2443 compile_c.linkSystemLibrary("advapi32", .{});
2444 }
2445 compile_c.linkSystemLibrary("crypt32", .{});
2446 compile_c.linkSystemLibrary("ws2_32", .{});
2447 compile_c.linkSystemLibrary("ole32", .{});
2448 }2468 }
2469 compile_c.linkSystemLibrary("crypt32", .{});
2470 compile_c.linkSystemLibrary("ws2_32", .{});
2471 compile_c.linkSystemLibrary("ole32", .{});
2449 }2472 }
2473 }
24502474
2451 const run = b.addRunArtifact(compile_c_exe);2475 const run = b.addRunArtifact(compile_c_exe);
2452 run.skip_foreign_checks = true;2476 run.skip_foreign_checks = true;
2453 run.enableTestRunnerMode();2477 run.enableTestRunnerMode();
2454 run.setName(b.fmt("run test {s}", .{qualified_name}));2478 run.setName(b.fmt("run test {s}", .{qualified_name}));
24552479
2456 step.dependOn(&run.step);2480 step.dependOn(&run.step);
2457 } else if (target.cpu.arch.isSpirV()) {2481 } else if (target.cpu.arch.isSpirV()) {
2458 // Don't run spirv binaries2482 // Don't run spirv binaries
2459 _ = these_tests.getEmittedBin();2483 _ = these_tests.getEmittedBin();
2460 step.dependOn(&these_tests.step);2484 step.dependOn(&these_tests.step);
2461 } else {2485 } else {
2462 const run = b.addRunArtifact(these_tests);2486 const run = b.addRunArtifact(these_tests);
2463 run.skip_foreign_checks = true;2487 run.skip_foreign_checks = true;
2464 run.setName(b.fmt("run test {s}", .{qualified_name}));2488 run.setName(b.fmt("run test {s}", .{qualified_name}));
24652489
2466 step.dependOn(&run.step);2490 step.dependOn(&run.step);
2467 }
2468 }2491 }
2469 return step;
2470}2492}
24712493
2472pub fn wouldUseLlvm(use_llvm: ?bool, query: std.Target.Query, optimize_mode: OptimizeMode) bool {2494pub fn wouldUseLlvm(use_llvm: ?bool, query: std.Target.Query, optimize_mode: OptimizeMode) bool {