authorgravatar for git@bfiedler.chBen Fiedler <git@bfiedler.ch> 2022-06-24 17:24:44+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2022-06-28 21:45:02+02:00
log76546b3f8e19b11f0c88259b1b3e5fd14cfbde31
treebf4b76051728ea72a9a28617668efe9200a8d663
parentbb2929ba083d3c5d86cae1d83cba0abd43ffa3d5

std.mem: add peek() to TokenIterator(T)

1 files changed, 17 insertions(+), 3 deletions(-)

lib/std/mem.zig+17-3
......@@ -1608,6 +1608,7 @@ pub fn tokenize(comptime T: type, buffer: []const T, delimiter_bytes: []const T)
16081608test "tokenize" {
16091609 var it = tokenize(u8, " abc def ghi ", " ");
16101610 try testing.expect(eql(u8, it.next().?, "abc"));
1611 try testing.expect(eql(u8, it.peek().?, "def"));
16111612 try testing.expect(eql(u8, it.next().?, "def"));
16121613 try testing.expect(eql(u8, it.next().?, "ghi"));
16131614 try testing.expect(it.next() == null);
......@@ -1626,9 +1627,11 @@ test "tokenize" {
16261627
16271628 it = tokenize(u8, "|", "|");
16281629 try testing.expect(it.next() == null);
1630 try testing.expect(it.peek() == null);
16291631
16301632 it = tokenize(u8, "", "|");
16311633 try testing.expect(it.next() == null);
1634 try testing.expect(it.peek() == null);
16321635
16331636 it = tokenize(u8, "hello", "");
16341637 try testing.expect(eql(u8, it.next().?, "hello"));
......@@ -1650,11 +1653,13 @@ test "tokenize" {
16501653test "tokenize (multibyte)" {
16511654 var it = tokenize(u8, "a|b,c/d e", " /,|");
16521655 try testing.expect(eql(u8, it.next().?, "a"));
1656 try testing.expect(eql(u8, it.peek().?, "b"));
16531657 try testing.expect(eql(u8, it.next().?, "b"));
16541658 try testing.expect(eql(u8, it.next().?, "c"));
16551659 try testing.expect(eql(u8, it.next().?, "d"));
16561660 try testing.expect(eql(u8, it.next().?, "e"));
16571661 try testing.expect(it.next() == null);
1662 try testing.expect(it.peek() == null);
16581663
16591664 var it16 = tokenize(
16601665 u16,
......@@ -1778,8 +1783,17 @@ pub fn TokenIterator(comptime T: type) type {
17781783
17791784 const Self = @This();
17801785
1781 /// Returns a slice of the next token, or null if tokenization is complete.
1786 /// Returns a slice of the current token, or null if tokenization is
1787 /// complete, and advances to the next token.
17821788 pub fn next(self: *Self) ?[]const T {
1789 const result = self.peek() orelse return null;
1790 self.index += result.len;
1791 return result;
1792 }
1793
1794 /// Returns a slice of the current token, or null if tokenization is
1795 /// complete. Does not advance to the next token.
1796 pub fn peek(self: *Self) ?[]const T {
17831797 // move to beginning of token
17841798 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
17851799 const start = self.index;
......@@ -1788,8 +1802,8 @@ pub fn TokenIterator(comptime T: type) type {
17881802 }
17891803
17901804 // move to end of token
1791 while (self.index < self.buffer.len and !self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1792 const end = self.index;
1805 var end = start;
1806 while (end < self.buffer.len and !self.isSplitByte(self.buffer[end])) : (end += 1) {}
17931807
17941808 return self.buffer[start..end];
17951809 }