authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2021-08-06 01:53:07-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2021-08-06 01:53:07-07:00
log05fd20dc104b3654ce9c5d7a22a2bff66a940dba
treec4e88a28618da033cbad7832a7a86d57159b9fc4
parentea7bdeb67d474526732b117992971603e4065f98

Make mem.split and mem.tokenize generic instead of assuming u8

This allows these functions to work on slices of u16, etc

1 files changed, 125 insertions(+), 77 deletions(-)

lib/std/mem.zig+125-77
......@@ -1575,8 +1575,8 @@ test "bswapAllFields" {
15751575/// If `delimiter_bytes` does not exist in buffer,
15761576/// the iterator will return `buffer`, null, in that order.
15771577/// See also the related function `split`.
1578pub fn tokenize(buffer: []const u8, delimiter_bytes: []const u8) TokenIterator {
1579 return TokenIterator{
1578pub fn tokenize(comptime T: type, buffer: []const T, delimiter_bytes: []const T) TokenIterator(T) {
1579 return .{
15801580 .index = 0,
15811581 .buffer = buffer,
15821582 .delimiter_bytes = delimiter_bytes,
......@@ -1584,51 +1584,71 @@ pub fn tokenize(buffer: []const u8, delimiter_bytes: []const u8) TokenIterator {
15841584}
15851585
15861586test "mem.tokenize" {
1587 var it = tokenize(" abc def ghi ", " ");
1587 var it = tokenize(u8, " abc def ghi ", " ");
15881588 try testing.expect(eql(u8, it.next().?, "abc"));
15891589 try testing.expect(eql(u8, it.next().?, "def"));
15901590 try testing.expect(eql(u8, it.next().?, "ghi"));
15911591 try testing.expect(it.next() == null);
15921592
1593 it = tokenize("..\\bob", "\\");
1593 it = tokenize(u8, "..\\bob", "\\");
15941594 try testing.expect(eql(u8, it.next().?, ".."));
15951595 try testing.expect(eql(u8, "..", "..\\bob"[0..it.index]));
15961596 try testing.expect(eql(u8, it.next().?, "bob"));
15971597 try testing.expect(it.next() == null);
15981598
1599 it = tokenize("//a/b", "/");
1599 it = tokenize(u8, "//a/b", "/");
16001600 try testing.expect(eql(u8, it.next().?, "a"));
16011601 try testing.expect(eql(u8, it.next().?, "b"));
16021602 try testing.expect(eql(u8, "//a/b", "//a/b"[0..it.index]));
16031603 try testing.expect(it.next() == null);
16041604
1605 it = tokenize("|", "|");
1605 it = tokenize(u8, "|", "|");
16061606 try testing.expect(it.next() == null);
16071607
1608 it = tokenize("", "|");
1608 it = tokenize(u8, "", "|");
16091609 try testing.expect(it.next() == null);
16101610
1611 it = tokenize("hello", "");
1611 it = tokenize(u8, "hello", "");
16121612 try testing.expect(eql(u8, it.next().?, "hello"));
16131613 try testing.expect(it.next() == null);
16141614
1615 it = tokenize("hello", " ");
1615 it = tokenize(u8, "hello", " ");
16161616 try testing.expect(eql(u8, it.next().?, "hello"));
16171617 try testing.expect(it.next() == null);
1618
1619 var it16 = tokenize(
1620 u16,
1621 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
1622 std.unicode.utf8ToUtf16LeStringLiteral(" "),
1623 );
1624 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));
1625 try testing.expect(it16.next() == null);
16181626}
16191627
16201628test "mem.tokenize (multibyte)" {
1621 var it = tokenize("a|b,c/d e", " /,|");
1629 var it = tokenize(u8, "a|b,c/d e", " /,|");
16221630 try testing.expect(eql(u8, it.next().?, "a"));
16231631 try testing.expect(eql(u8, it.next().?, "b"));
16241632 try testing.expect(eql(u8, it.next().?, "c"));
16251633 try testing.expect(eql(u8, it.next().?, "d"));
16261634 try testing.expect(eql(u8, it.next().?, "e"));
16271635 try testing.expect(it.next() == null);
1636
1637 var it16 = tokenize(
1638 u16,
1639 std.unicode.utf8ToUtf16LeStringLiteral("a|b,c/d e"),
1640 std.unicode.utf8ToUtf16LeStringLiteral(" /,|"),
1641 );
1642 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a")));
1643 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b")));
1644 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c")));
1645 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d")));
1646 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("e")));
1647 try testing.expect(it16.next() == null);
16281648}
16291649
16301650test "mem.tokenize (reset)" {
1631 var it = tokenize(" abc def ghi ", " ");
1651 var it = tokenize(u8, " abc def ghi ", " ");
16321652 try testing.expect(eql(u8, it.next().?, "abc"));
16331653 try testing.expect(eql(u8, it.next().?, "def"));
16341654 try testing.expect(eql(u8, it.next().?, "ghi"));
......@@ -1649,9 +1669,9 @@ test "mem.tokenize (reset)" {
16491669/// the iterator will return `buffer`, null, in that order.
16501670/// The delimiter length must not be zero.
16511671/// See also the related function `tokenize`.
1652pub fn split(buffer: []const u8, delimiter: []const u8) SplitIterator {
1672pub fn split(comptime T: type, buffer: []const T, delimiter: []const T) SplitIterator(T) {
16531673 assert(delimiter.len != 0);
1654 return SplitIterator{
1674 return .{
16551675 .index = 0,
16561676 .buffer = buffer,
16571677 .delimiter = delimiter,
......@@ -1661,35 +1681,55 @@ pub fn split(buffer: []const u8, delimiter: []const u8) SplitIterator {
16611681pub const separate = @compileError("deprecated: renamed to split (behavior remains unchanged)");
16621682
16631683test "mem.split" {
1664 var it = split("abc|def||ghi", "|");
1684 var it = split(u8, "abc|def||ghi", "|");
16651685 try testing.expect(eql(u8, it.next().?, "abc"));
16661686 try testing.expect(eql(u8, it.next().?, "def"));
16671687 try testing.expect(eql(u8, it.next().?, ""));
16681688 try testing.expect(eql(u8, it.next().?, "ghi"));
16691689 try testing.expect(it.next() == null);
16701690
1671 it = split("", "|");
1691 it = split(u8, "", "|");
16721692 try testing.expect(eql(u8, it.next().?, ""));
16731693 try testing.expect(it.next() == null);
16741694
1675 it = split("|", "|");
1695 it = split(u8, "|", "|");
16761696 try testing.expect(eql(u8, it.next().?, ""));
16771697 try testing.expect(eql(u8, it.next().?, ""));
16781698 try testing.expect(it.next() == null);
16791699
1680 it = split("hello", " ");
1700 it = split(u8, "hello", " ");
16811701 try testing.expect(eql(u8, it.next().?, "hello"));
16821702 try testing.expect(it.next() == null);
1703
1704 var it16 = split(
1705 u16,
1706 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
1707 std.unicode.utf8ToUtf16LeStringLiteral(" "),
1708 );
1709 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));
1710 try testing.expect(it16.next() == null);
16831711}
16841712
16851713test "mem.split (multibyte)" {
1686 var it = split("a, b ,, c, d, e", ", ");
1714 var it = split(u8, "a, b ,, c, d, e", ", ");
16871715 try testing.expect(eql(u8, it.next().?, "a"));
16881716 try testing.expect(eql(u8, it.next().?, "b ,"));
16891717 try testing.expect(eql(u8, it.next().?, "c"));
16901718 try testing.expect(eql(u8, it.next().?, "d"));
16911719 try testing.expect(eql(u8, it.next().?, "e"));
16921720 try testing.expect(it.next() == null);
1721
1722 var it16 = split(
1723 u16,
1724 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),
1725 std.unicode.utf8ToUtf16LeStringLiteral(", "),
1726 );
1727 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a")));
1728 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b ,")));
1729 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c")));
1730 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d")));
1731 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("e")));
1732 try testing.expect(it16.next() == null);
16931733}
16941734
16951735pub fn startsWith(comptime T: type, haystack: []const T, needle: []const T) bool {
......@@ -1710,75 +1750,83 @@ test "mem.endsWith" {
17101750 try testing.expect(!endsWith(u8, "Bob", "Bo"));
17111751}
17121752
1713pub const TokenIterator = struct {
1714 buffer: []const u8,
1715 delimiter_bytes: []const u8,
1716 index: usize,
1753pub fn TokenIterator(comptime T: type) type {
1754 return struct {
1755 buffer: []const T,
1756 delimiter_bytes: []const T,
1757 index: usize,
17171758
1718 /// Returns a slice of the next token, or null if tokenization is complete.
1719 pub fn next(self: *TokenIterator) ?[]const u8 {
1720 // move to beginning of token
1721 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1722 const start = self.index;
1723 if (start == self.buffer.len) {
1724 return null;
1725 }
1759 const Self = @This();
17261760
1727 // move to end of token
1728 while (self.index < self.buffer.len and !self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1729 const end = self.index;
1761 /// Returns a slice of the next token, or null if tokenization is complete.
1762 pub fn next(self: *Self) ?[]const T {
1763 // move to beginning of token
1764 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1765 const start = self.index;
1766 if (start == self.buffer.len) {
1767 return null;
1768 }
17301769
1731 return self.buffer[start..end];
1732 }
1770 // move to end of token
1771 while (self.index < self.buffer.len and !self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1772 const end = self.index;
17331773
1734 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1735 pub fn rest(self: TokenIterator) []const u8 {
1736 // move to beginning of token
1737 var index: usize = self.index;
1738 while (index < self.buffer.len and self.isSplitByte(self.buffer[index])) : (index += 1) {}
1739 return self.buffer[index..];
1740 }
1774 return self.buffer[start..end];
1775 }
17411776
1742 /// Resets the iterator to the initial token.
1743 pub fn reset(self: *TokenIterator) void {
1744 self.index = 0;
1745 }
1777 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1778 pub fn rest(self: Self) []const T {
1779 // move to beginning of token
1780 var index: usize = self.index;
1781 while (index < self.buffer.len and self.isSplitByte(self.buffer[index])) : (index += 1) {}
1782 return self.buffer[index..];
1783 }
1784
1785 /// Resets the iterator to the initial token.
1786 pub fn reset(self: *Self) void {
1787 self.index = 0;
1788 }
17461789
1747 fn isSplitByte(self: TokenIterator, byte: u8) bool {
1748 for (self.delimiter_bytes) |delimiter_byte| {
1749 if (byte == delimiter_byte) {
1750 return true;
1790 fn isSplitByte(self: Self, byte: T) bool {
1791 for (self.delimiter_bytes) |delimiter_byte| {
1792 if (byte == delimiter_byte) {
1793 return true;
1794 }
17511795 }
1796 return false;
17521797 }
1753 return false;
1754 }
1755};
1798 };
1799}
17561800
1757pub const SplitIterator = struct {
1758 buffer: []const u8,
1759 index: ?usize,
1760 delimiter: []const u8,
1761
1762 /// Returns a slice of the next field, or null if splitting is complete.
1763 pub fn next(self: *SplitIterator) ?[]const u8 {
1764 const start = self.index orelse return null;
1765 const end = if (indexOfPos(u8, self.buffer, start, self.delimiter)) |delim_start| blk: {
1766 self.index = delim_start + self.delimiter.len;
1767 break :blk delim_start;
1768 } else blk: {
1769 self.index = null;
1770 break :blk self.buffer.len;
1771 };
1772 return self.buffer[start..end];
1773 }
1801pub fn SplitIterator(comptime T: type) type {
1802 return struct {
1803 buffer: []const T,
1804 index: ?usize,
1805 delimiter: []const T,
17741806
1775 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1776 pub fn rest(self: SplitIterator) []const u8 {
1777 const end = self.buffer.len;
1778 const start = self.index orelse end;
1779 return self.buffer[start..end];
1780 }
1781};
1807 const Self = @This();
1808
1809 /// Returns a slice of the next field, or null if splitting is complete.
1810 pub fn next(self: *Self) ?[]const T {
1811 const start = self.index orelse return null;
1812 const end = if (indexOfPos(T, self.buffer, start, self.delimiter)) |delim_start| blk: {
1813 self.index = delim_start + self.delimiter.len;
1814 break :blk delim_start;
1815 } else blk: {
1816 self.index = null;
1817 break :blk self.buffer.len;
1818 };
1819 return self.buffer[start..end];
1820 }
1821
1822 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1823 pub fn rest(self: Self) []const T {
1824 const end = self.buffer.len;
1825 const start = self.index orelse end;
1826 return self.buffer[start..end];
1827 }
1828 };
1829}
17821830
17831831/// Naively combines a series of slices with a separator.
17841832/// Allocates memory for the result, which must be freed by the caller.