authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-09-06 10:23:41+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-10-08 17:00:15-07:00
logbf58b4e41990973a72ef49163abe63d41c966e23
tree43cacbdf9ba9d128dc7f73b3c667c9c3b684a621
parent889942a8b74d563e51923d9d14a45e58445aa38f

std.Io.Reader: fix delimiter bugs

Fix `takeDelimiter` and `takeDelimiterExclusive` tossing too many bytes (#25132) Also add/improve test coverage for all delimiter and sentinel methods, update usages of `takeDelimiterExclusive` to not rely on the fixed bug, tweak a handful of doc comments, and slightly simplify some logic. I have not fixed #24950 in this commit because I am a little less certain about the appropriate solution there. Resolves: #25132 Co-authored-by: Andrew Kelley <andrew@ziglang.org>

3 files changed, 48 insertions(+), 24 deletions(-)

lib/std/Io/Reader.zig+44-18
......@@ -481,7 +481,6 @@ pub fn readVecAll(r: *Reader, data: [][]u8) Error!void {
481481/// is returned instead.
482482///
483483/// See also:
484/// * `peek`
485484/// * `toss`
486485pub fn peek(r: *Reader, n: usize) Error![]u8 {
487486 try r.fill(n);
......@@ -732,7 +731,7 @@ pub const DelimiterError = error{
732731};
733732
734733/// Returns a slice of the next bytes of buffered data from the stream until
735/// `sentinel` is found, advancing the seek position.
734/// `sentinel` is found, advancing the seek position past the sentinel.
736735///
737736/// Returned slice has a sentinel.
738737///
......@@ -765,7 +764,7 @@ pub fn peekSentinel(r: *Reader, comptime sentinel: u8) DelimiterError![:sentinel
765764}
766765
767766/// Returns a slice of the next bytes of buffered data from the stream until
768/// `delimiter` is found, advancing the seek position.
767/// `delimiter` is found, advancing the seek position past the delimiter.
769768///
770769/// Returned slice includes the delimiter as the last byte.
771770///
......@@ -818,7 +817,8 @@ pub fn peekDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
818817}
819818
820819/// Returns a slice of the next bytes of buffered data from the stream until
821/// `delimiter` is found, advancing the seek position up to the delimiter.
820/// `delimiter` is found, advancing the seek position up to (but not past)
821/// the delimiter.
822822///
823823/// Returned slice excludes the delimiter. End-of-stream is treated equivalent
824824/// to a delimiter, unless it would result in a length 0 return value, in which
......@@ -832,20 +832,13 @@ pub fn peekDelimiterInclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
832832/// Invalidates previously returned values from `peek`.
833833///
834834/// See also:
835/// * `takeDelimiter`
835836/// * `takeDelimiterInclusive`
836837/// * `peekDelimiterExclusive`
837838pub fn takeDelimiterExclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
838 const result = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) {
839 error.EndOfStream => {
840 const remaining = r.buffer[r.seek..r.end];
841 if (remaining.len == 0) return error.EndOfStream;
842 r.toss(remaining.len);
843 return remaining;
844 },
845 else => |e| return e,
846 };
839 const result = try r.peekDelimiterExclusive(delimiter);
847840 r.toss(result.len);
848 return result[0 .. result.len - 1];
841 return result;
849842}
850843
851844/// Returns a slice of the next bytes of buffered data from the stream until
......@@ -866,7 +859,7 @@ pub fn takeDelimiterExclusive(r: *Reader, delimiter: u8) DelimiterError![]u8 {
866859/// * `takeDelimiterInclusive`
867860/// * `takeDelimiterExclusive`
868861pub fn takeDelimiter(r: *Reader, delimiter: u8) error{ ReadFailed, StreamTooLong }!?[]u8 {
869 const result = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) {
862 const inclusive = r.peekDelimiterInclusive(delimiter) catch |err| switch (err) {
870863 error.EndOfStream => {
871864 const remaining = r.buffer[r.seek..r.end];
872865 if (remaining.len == 0) return null;
......@@ -875,8 +868,8 @@ pub fn takeDelimiter(r: *Reader, delimiter: u8) error{ ReadFailed, StreamTooLong
875868 },
876869 else => |e| return e,
877870 };
878 r.toss(result.len + 1);
879 return result[0 .. result.len - 1];
871 r.toss(inclusive.len);
872 return inclusive[0 .. inclusive.len - 1];
880873}
881874
882875/// Returns a slice of the next bytes of buffered data from the stream until
......@@ -1403,6 +1396,9 @@ test peekSentinel {
14031396 var r: Reader = .fixed("ab\nc");
14041397 try testing.expectEqualStrings("ab", try r.peekSentinel('\n'));
14051398 try testing.expectEqualStrings("ab", try r.peekSentinel('\n'));
1399 r.toss(3);
1400 try testing.expectError(error.EndOfStream, r.peekSentinel('\n'));
1401 try testing.expectEqualStrings("c", try r.peek(1));
14061402}
14071403
14081404test takeDelimiterInclusive {
......@@ -1417,22 +1413,52 @@ test peekDelimiterInclusive {
14171413 try testing.expectEqualStrings("ab\n", try r.peekDelimiterInclusive('\n'));
14181414 r.toss(3);
14191415 try testing.expectError(error.EndOfStream, r.peekDelimiterInclusive('\n'));
1416 try testing.expectEqualStrings("c", try r.peek(1));
14201417}
14211418
14221419test takeDelimiterExclusive {
14231420 var r: Reader = .fixed("ab\nc");
1421
14241422 try testing.expectEqualStrings("ab", try r.takeDelimiterExclusive('\n'));
1423 try testing.expectEqualStrings("", try r.takeDelimiterExclusive('\n'));
1424 try testing.expectEqualStrings("", try r.takeDelimiterExclusive('\n'));
1425 try testing.expectEqualStrings("\n", try r.take(1));
1426
14251427 try testing.expectEqualStrings("c", try r.takeDelimiterExclusive('\n'));
14261428 try testing.expectError(error.EndOfStream, r.takeDelimiterExclusive('\n'));
14271429}
14281430
14291431test peekDelimiterExclusive {
14301432 var r: Reader = .fixed("ab\nc");
1433
14311434 try testing.expectEqualStrings("ab", try r.peekDelimiterExclusive('\n'));
14321435 try testing.expectEqualStrings("ab", try r.peekDelimiterExclusive('\n'));
1433 r.toss(3);
1436 r.toss(2);
1437 try testing.expectEqualStrings("", try r.peekDelimiterExclusive('\n'));
1438 try testing.expectEqualStrings("\n", try r.take(1));
1439
14341440 try testing.expectEqualStrings("c", try r.peekDelimiterExclusive('\n'));
14351441 try testing.expectEqualStrings("c", try r.peekDelimiterExclusive('\n'));
1442 r.toss(1);
1443 try testing.expectError(error.EndOfStream, r.peekDelimiterExclusive('\n'));
1444}
1445
1446test takeDelimiter {
1447 var r: Reader = .fixed("ab\nc\n\nd");
1448 try testing.expectEqualStrings("ab", (try r.takeDelimiter('\n')).?);
1449 try testing.expectEqualStrings("c", (try r.takeDelimiter('\n')).?);
1450 try testing.expectEqualStrings("", (try r.takeDelimiter('\n')).?);
1451 try testing.expectEqualStrings("d", (try r.takeDelimiter('\n')).?);
1452 try testing.expectEqual(null, try r.takeDelimiter('\n'));
1453 try testing.expectEqual(null, try r.takeDelimiter('\n'));
1454
1455 r = .fixed("ab\nc\n\nd\n"); // one trailing newline does not affect behavior
1456 try testing.expectEqualStrings("ab", (try r.takeDelimiter('\n')).?);
1457 try testing.expectEqualStrings("c", (try r.takeDelimiter('\n')).?);
1458 try testing.expectEqualStrings("", (try r.takeDelimiter('\n')).?);
1459 try testing.expectEqualStrings("d", (try r.takeDelimiter('\n')).?);
1460 try testing.expectEqual(null, try r.takeDelimiter('\n'));
1461 try testing.expectEqual(null, try r.takeDelimiter('\n'));
14361462}
14371463
14381464test streamDelimiter {
lib/std/net.zig+3-2
......@@ -1396,7 +1396,7 @@ fn parseHosts(
13961396 br: *Io.Reader,
13971397) error{ OutOfMemory, ReadFailed }!void {
13981398 while (true) {
1399 const line = br.takeDelimiterExclusive('\n') catch |err| switch (err) {
1399 const line = br.takeDelimiter('\n') catch |err| switch (err) {
14001400 error.StreamTooLong => {
14011401 // Skip lines that are too long.
14021402 _ = br.discardDelimiterInclusive('\n') catch |e| switch (e) {
......@@ -1406,7 +1406,8 @@ fn parseHosts(
14061406 continue;
14071407 },
14081408 error.ReadFailed => return error.ReadFailed,
1409 error.EndOfStream => break,
1409 } orelse {
1410 break; // end of stream
14101411 };
14111412 var split_it = mem.splitScalar(u8, line, '#');
14121413 const no_comment_line = split_it.first();
lib/std/zig/system/linux.zig+1-4
......@@ -359,14 +359,11 @@ fn CpuinfoParser(comptime impl: anytype) type {
359359 return struct {
360360 fn parse(arch: Target.Cpu.Arch, reader: *std.Io.Reader) !?Target.Cpu {
361361 var obj: impl = .{};
362 while (reader.takeDelimiterExclusive('\n')) |line| {
362 while (try reader.takeDelimiter('\n')) |line| {
363363 const colon_pos = mem.indexOfScalar(u8, line, ':') orelse continue;
364364 const key = mem.trimEnd(u8, line[0..colon_pos], " \t");
365365 const value = mem.trimStart(u8, line[colon_pos + 1 ..], " \t");
366366 if (!try obj.line_hook(key, value)) break;
367 } else |err| switch (err) {
368 error.EndOfStream => {},
369 else => |e| return e,
370367 }
371368 return obj.finalize(arch);
372369 }