authorgravatar for git@bfiedler.chBen Fiedler <git@bfiedler.ch> 2022-06-24 17:24:44+02:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2022-06-28 21:45:02+02:00
log76546b3f8e19b11f0c88259b1b3e5fd14cfbde31
treebf4b76051728ea72a9a28617668efe9200a8d663
parentbb2929ba083d3c5d86cae1d83cba0abd43ffa3d5

std.mem: add peek() to TokenIterator(T)

1 files changed, 17 insertions(+), 3 deletions(-)

lib/std/mem.zig+17-3
...@@ -1608,6 +1608,7 @@ pub fn tokenize(comptime T: type, buffer: []const T, delimiter_bytes: []const T)...@@ -1608,6 +1608,7 @@ pub fn tokenize(comptime T: type, buffer: []const T, delimiter_bytes: []const T)
1608test "tokenize" {1608test "tokenize" {
1609 var it = tokenize(u8, " abc def ghi ", " ");1609 var it = tokenize(u8, " abc def ghi ", " ");
1610 try testing.expect(eql(u8, it.next().?, "abc"));1610 try testing.expect(eql(u8, it.next().?, "abc"));
1611 try testing.expect(eql(u8, it.peek().?, "def"));
1611 try testing.expect(eql(u8, it.next().?, "def"));1612 try testing.expect(eql(u8, it.next().?, "def"));
1612 try testing.expect(eql(u8, it.next().?, "ghi"));1613 try testing.expect(eql(u8, it.next().?, "ghi"));
1613 try testing.expect(it.next() == null);1614 try testing.expect(it.next() == null);
...@@ -1626,9 +1627,11 @@ test "tokenize" {...@@ -1626,9 +1627,11 @@ test "tokenize" {
16261627
1627 it = tokenize(u8, "|", "|");1628 it = tokenize(u8, "|", "|");
1628 try testing.expect(it.next() == null);1629 try testing.expect(it.next() == null);
1630 try testing.expect(it.peek() == null);
16291631
1630 it = tokenize(u8, "", "|");1632 it = tokenize(u8, "", "|");
1631 try testing.expect(it.next() == null);1633 try testing.expect(it.next() == null);
1634 try testing.expect(it.peek() == null);
16321635
1633 it = tokenize(u8, "hello", "");1636 it = tokenize(u8, "hello", "");
1634 try testing.expect(eql(u8, it.next().?, "hello"));1637 try testing.expect(eql(u8, it.next().?, "hello"));
...@@ -1650,11 +1653,13 @@ test "tokenize" {...@@ -1650,11 +1653,13 @@ test "tokenize" {
1650test "tokenize (multibyte)" {1653test "tokenize (multibyte)" {
1651 var it = tokenize(u8, "a|b,c/d e", " /,|");1654 var it = tokenize(u8, "a|b,c/d e", " /,|");
1652 try testing.expect(eql(u8, it.next().?, "a"));1655 try testing.expect(eql(u8, it.next().?, "a"));
1656 try testing.expect(eql(u8, it.peek().?, "b"));
1653 try testing.expect(eql(u8, it.next().?, "b"));1657 try testing.expect(eql(u8, it.next().?, "b"));
1654 try testing.expect(eql(u8, it.next().?, "c"));1658 try testing.expect(eql(u8, it.next().?, "c"));
1655 try testing.expect(eql(u8, it.next().?, "d"));1659 try testing.expect(eql(u8, it.next().?, "d"));
1656 try testing.expect(eql(u8, it.next().?, "e"));1660 try testing.expect(eql(u8, it.next().?, "e"));
1657 try testing.expect(it.next() == null);1661 try testing.expect(it.next() == null);
1662 try testing.expect(it.peek() == null);
16581663
1659 var it16 = tokenize(1664 var it16 = tokenize(
1660 u16,1665 u16,
...@@ -1778,8 +1783,17 @@ pub fn TokenIterator(comptime T: type) type {...@@ -1778,8 +1783,17 @@ pub fn TokenIterator(comptime T: type) type {
17781783
1779 const Self = @This();1784 const Self = @This();
17801785
1781 /// Returns a slice of the next token, or null if tokenization is complete.1786 /// Returns a slice of the current token, or null if tokenization is
1787 /// complete, and advances to the next token.
1782 pub fn next(self: *Self) ?[]const T {1788 pub fn next(self: *Self) ?[]const T {
1789 const result = self.peek() orelse return null;
1790 self.index += result.len;
1791 return result;
1792 }
1793
1794 /// Returns a slice of the current token, or null if tokenization is
1795 /// complete. Does not advance to the next token.
1796 pub fn peek(self: *Self) ?[]const T {
1783 // move to beginning of token1797 // move to beginning of token
1784 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}1798 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1785 const start = self.index;1799 const start = self.index;
...@@ -1788,8 +1802,8 @@ pub fn TokenIterator(comptime T: type) type {...@@ -1788,8 +1802,8 @@ pub fn TokenIterator(comptime T: type) type {
1788 }1802 }
17891803
1790 // move to end of token1804 // move to end of token
1791 while (self.index < self.buffer.len and !self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}1805 var end = start;
1792 const end = self.index;1806 while (end < self.buffer.len and !self.isSplitByte(self.buffer[end])) : (end += 1) {}
17931807
1794 return self.buffer[start..end];1808 return self.buffer[start..end];
1795 }1809 }