authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-05-04 18:19:31-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2023-05-13 13:43:50-07:00
logbda645d911fd79c5aba1f93c0877c7eb915ed709
treeb7b62c37df5b7533aeef900a6899cec2c2c5447e
parentbc17b38788e173afefc6a4770e5b0610dbe44670

std.mem: Split `split` and `splitBackwards` into 3 versions by delimiter type: full, any, and scalar

This allows users to choose which version they need for their particular use case, as the previous default (now the 'full' version) was (1) not always the desired type of delimiter and (2) performed worse than the scalar version if the delimiter was a single item.

1 files changed, 290 insertions(+), 56 deletions(-)

lib/std/mem.zig+290-56
...@@ -2012,18 +2012,23 @@ test "tokenize (reset)" {...@@ -2012,18 +2012,23 @@ test "tokenize (reset)" {
2012 try testing.expect(it.next() == null);2012 try testing.expect(it.next() == null);
2013}2013}
20142014
2015/// Deprecated: use `splitFull`, `splitAny`, or `splitScalar`
2016pub const split = splitFull;
2017
2015/// Returns an iterator that iterates over the slices of `buffer` that2018/// Returns an iterator that iterates over the slices of `buffer` that
2016/// are separated by bytes in `delimiter`.2019/// are separated by the byte sequence in `delimiter`.
2017///2020///
2018/// `split(u8, "abc|def||ghi", "|")` will return slices2021/// `splitFull(u8, "abc||def||||ghi", "||")` will return slices
2019/// for "abc", "def", "", "ghi", null, in that order.2022/// for "abc", "def", "", "ghi", null, in that order.
2020///2023///
2021/// If `delimiter` does not exist in buffer,2024/// If `delimiter` does not exist in buffer,
2022/// the iterator will return `buffer`, null, in that order.2025/// the iterator will return `buffer`, null, in that order.
2023/// The delimiter length must not be zero.2026/// The delimiter length must not be zero.
2024///2027///
2025/// See also: `tokenize` and `splitBackwards`.2028/// See also: `splitAny`, `splitScalar`, `splitBackwardsFull`,
2026pub fn split(comptime T: type, buffer: []const T, delimiter: []const T) SplitIterator(T) {2029/// `splitBackwardsAny`,`splitBackwardsScalar`, and
2030/// `tokenize`.
2031pub fn splitFull(comptime T: type, buffer: []const T, delimiter: []const T) SplitIterator(T, .full) {
2027 assert(delimiter.len != 0);2032 assert(delimiter.len != 0);
2028 return .{2033 return .{
2029 .index = 0,2034 .index = 0,
...@@ -2032,8 +2037,48 @@ pub fn split(comptime T: type, buffer: []const T, delimiter: []const T) SplitIte...@@ -2032,8 +2037,48 @@ pub fn split(comptime T: type, buffer: []const T, delimiter: []const T) SplitIte
2032 };2037 };
2033}2038}
20342039
2035test "split" {2040/// Returns an iterator that iterates over the slices of `buffer` that
2036 var it = split(u8, "abc|def||ghi", "|");2041/// are separated by any item in `delimiters`.
2042///
2043/// `splitAny(u8, "abc,def||ghi", "|,")` will return slices
2044/// for "abc", "def", "", "ghi", null, in that order.
2045///
2046/// If none of `delimiters` exist in buffer,
2047/// the iterator will return `buffer`, null, in that order.
2048///
2049/// See also: `splitFull`, `splitScalar`, `splitBackwardsFull`,
2050/// `splitBackwardsAny`,`splitBackwardsScalar`, and
2051/// `tokenize`.
2052pub fn splitAny(comptime T: type, buffer: []const T, delimiters: []const T) SplitIterator(T, .any) {
2053 return .{
2054 .index = 0,
2055 .buffer = buffer,
2056 .delimiter = delimiters,
2057 };
2058}
2059
2060/// Returns an iterator that iterates over the slices of `buffer` that
2061/// are separated by `delimiter`.
2062///
2063/// `splitScalar(u8, "abc|def||ghi", '|')` will return slices
2064/// for "abc", "def", "", "ghi", null, in that order.
2065///
2066/// If `delimiter` does not exist in buffer,
2067/// the iterator will return `buffer`, null, in that order.
2068///
2069/// See also: `splitFull`, `splitAny`, `splitBackwardsFull`,
2070/// `splitBackwardsAny`,`splitBackwardsScalar`, and
2071/// `tokenize`.
2072pub fn splitScalar(comptime T: type, buffer: []const T, delimiter: T) SplitIterator(T, .scalar) {
2073 return .{
2074 .index = 0,
2075 .buffer = buffer,
2076 .delimiter = delimiter,
2077 };
2078}
2079
2080test "splitScalar" {
2081 var it = splitScalar(u8, "abc|def||ghi", '|');
2037 try testing.expectEqualSlices(u8, it.rest(), "abc|def||ghi");2082 try testing.expectEqualSlices(u8, it.rest(), "abc|def||ghi");
2038 try testing.expectEqualSlices(u8, it.first(), "abc");2083 try testing.expectEqualSlices(u8, it.first(), "abc");
20392084
...@@ -2049,30 +2094,30 @@ test "split" {...@@ -2049,30 +2094,30 @@ test "split" {
2049 try testing.expectEqualSlices(u8, it.rest(), "");2094 try testing.expectEqualSlices(u8, it.rest(), "");
2050 try testing.expect(it.next() == null);2095 try testing.expect(it.next() == null);
20512096
2052 it = split(u8, "", "|");2097 it = splitScalar(u8, "", '|');
2053 try testing.expectEqualSlices(u8, it.first(), "");2098 try testing.expectEqualSlices(u8, it.first(), "");
2054 try testing.expect(it.next() == null);2099 try testing.expect(it.next() == null);
20552100
2056 it = split(u8, "|", "|");2101 it = splitScalar(u8, "|", '|');
2057 try testing.expectEqualSlices(u8, it.first(), "");2102 try testing.expectEqualSlices(u8, it.first(), "");
2058 try testing.expectEqualSlices(u8, it.next().?, "");2103 try testing.expectEqualSlices(u8, it.next().?, "");
2059 try testing.expect(it.next() == null);2104 try testing.expect(it.next() == null);
20602105
2061 it = split(u8, "hello", " ");2106 it = splitScalar(u8, "hello", ' ');
2062 try testing.expectEqualSlices(u8, it.first(), "hello");2107 try testing.expectEqualSlices(u8, it.first(), "hello");
2063 try testing.expect(it.next() == null);2108 try testing.expect(it.next() == null);
20642109
2065 var it16 = split(2110 var it16 = splitScalar(
2066 u16,2111 u16,
2067 std.unicode.utf8ToUtf16LeStringLiteral("hello"),2112 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
2068 std.unicode.utf8ToUtf16LeStringLiteral(" "),2113 ' ',
2069 );2114 );
2070 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("hello"));2115 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("hello"));
2071 try testing.expect(it16.next() == null);2116 try testing.expect(it16.next() == null);
2072}2117}
20732118
2074test "split (multibyte)" {2119test "splitFull (multibyte)" {
2075 var it = split(u8, "a, b ,, c, d, e", ", ");2120 var it = splitFull(u8, "a, b ,, c, d, e", ", ");
2076 try testing.expectEqualSlices(u8, it.first(), "a");2121 try testing.expectEqualSlices(u8, it.first(), "a");
2077 try testing.expectEqualSlices(u8, it.rest(), "b ,, c, d, e");2122 try testing.expectEqualSlices(u8, it.rest(), "b ,, c, d, e");
2078 try testing.expectEqualSlices(u8, it.next().?, "b ,");2123 try testing.expectEqualSlices(u8, it.next().?, "b ,");
...@@ -2081,7 +2126,7 @@ test "split (multibyte)" {...@@ -2081,7 +2126,7 @@ test "split (multibyte)" {
2081 try testing.expectEqualSlices(u8, it.next().?, "e");2126 try testing.expectEqualSlices(u8, it.next().?, "e");
2082 try testing.expect(it.next() == null);2127 try testing.expect(it.next() == null);
20832128
2084 var it16 = split(2129 var it16 = splitFull(
2085 u16,2130 u16,
2086 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),2131 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),
2087 std.unicode.utf8ToUtf16LeStringLiteral(", "),2132 std.unicode.utf8ToUtf16LeStringLiteral(", "),
...@@ -2094,42 +2139,144 @@ test "split (multibyte)" {...@@ -2094,42 +2139,144 @@ test "split (multibyte)" {
2094 try testing.expect(it16.next() == null);2139 try testing.expect(it16.next() == null);
2095}2140}
20962141
2142test "splitAny" {
2143 var it = splitAny(u8, "a,b, c d e", ", ");
2144 try testing.expectEqualSlices(u8, it.first(), "a");
2145 try testing.expectEqualSlices(u8, it.rest(), "b, c d e");
2146 try testing.expectEqualSlices(u8, it.next().?, "b");
2147 try testing.expectEqualSlices(u8, it.next().?, "");
2148 try testing.expectEqualSlices(u8, it.next().?, "c");
2149 try testing.expectEqualSlices(u8, it.next().?, "d");
2150 try testing.expectEqualSlices(u8, it.next().?, "e");
2151 try testing.expect(it.next() == null);
2152
2153 it = splitAny(u8, "hello", "");
2154 try testing.expect(eql(u8, it.next().?, "hello"));
2155 try testing.expect(it.next() == null);
2156
2157 var it16 = splitAny(
2158 u16,
2159 std.unicode.utf8ToUtf16LeStringLiteral("a,b, c d e"),
2160 std.unicode.utf8ToUtf16LeStringLiteral(", "),
2161 );
2162 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("a"));
2163 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b"));
2164 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral(""));
2165 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c"));
2166 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d"));
2167 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("e"));
2168 try testing.expect(it16.next() == null);
2169}
2170
2097test "split (reset)" {2171test "split (reset)" {
2098 var it = split(u8, "abc def ghi", " ");2172 {
2099 try testing.expect(eql(u8, it.first(), "abc"));2173 var it = splitFull(u8, "abc def ghi", " ");
2100 try testing.expect(eql(u8, it.next().?, "def"));2174 try testing.expect(eql(u8, it.first(), "abc"));
2101 try testing.expect(eql(u8, it.next().?, "ghi"));2175 try testing.expect(eql(u8, it.next().?, "def"));
2176 try testing.expect(eql(u8, it.next().?, "ghi"));
21022177
2103 it.reset();2178 it.reset();
21042179
2105 try testing.expect(eql(u8, it.first(), "abc"));2180 try testing.expect(eql(u8, it.first(), "abc"));
2106 try testing.expect(eql(u8, it.next().?, "def"));2181 try testing.expect(eql(u8, it.next().?, "def"));
2107 try testing.expect(eql(u8, it.next().?, "ghi"));2182 try testing.expect(eql(u8, it.next().?, "ghi"));
2108 try testing.expect(it.next() == null);2183 try testing.expect(it.next() == null);
2184 }
2185 {
2186 var it = splitAny(u8, "abc def,ghi", " ,");
2187 try testing.expect(eql(u8, it.first(), "abc"));
2188 try testing.expect(eql(u8, it.next().?, "def"));
2189 try testing.expect(eql(u8, it.next().?, "ghi"));
2190
2191 it.reset();
2192
2193 try testing.expect(eql(u8, it.first(), "abc"));
2194 try testing.expect(eql(u8, it.next().?, "def"));
2195 try testing.expect(eql(u8, it.next().?, "ghi"));
2196 try testing.expect(it.next() == null);
2197 }
2198 {
2199 var it = splitScalar(u8, "abc def ghi", ' ');
2200 try testing.expect(eql(u8, it.first(), "abc"));
2201 try testing.expect(eql(u8, it.next().?, "def"));
2202 try testing.expect(eql(u8, it.next().?, "ghi"));
2203
2204 it.reset();
2205
2206 try testing.expect(eql(u8, it.first(), "abc"));
2207 try testing.expect(eql(u8, it.next().?, "def"));
2208 try testing.expect(eql(u8, it.next().?, "ghi"));
2209 try testing.expect(it.next() == null);
2210 }
2109}2211}
21102212
2111/// Returns an iterator that iterates backwards over the slices of `buffer`2213/// Deprecated: use `splitBackwardsFull`, `splitBackwardsAny`, or `splitBackwardsScalar`
2112/// that are separated by bytes in `delimiter`.2214pub const splitBackwards = splitBackwardsFull;
2215
2216/// Returns an iterator that iterates backwards over the slices of `buffer` that
2217/// are separated by the sequence in `delimiter`.
2113///2218///
2114/// `splitBackwards(u8, "abc|def||ghi", "|")` will return slices2219/// `splitBackwardsFull(u8, "abc||def||||ghi", "||")` will return slices
2115/// for "ghi", "", "def", "abc", null, in that order.2220/// for "ghi", "", "def", "abc", null, in that order.
2116///2221///
2117/// If `delimiter` does not exist in buffer,2222/// If `delimiter` does not exist in buffer,
2118/// the iterator will return `buffer`, null, in that order.2223/// the iterator will return `buffer`, null, in that order.
2119/// The delimiter length must not be zero.2224/// The delimiter length must not be zero.
2120///2225///
2121/// See also: `tokenize` and `split`.2226/// See also: `splitBackwardsAny`, `splitBackwardsScalar`,
2122pub fn splitBackwards(comptime T: type, buffer: []const T, delimiter: []const T) SplitBackwardsIterator(T) {2227/// `splitFull`, `splitAny`,`splitScalar`, and
2228/// `tokenize`.
2229pub fn splitBackwardsFull(comptime T: type, buffer: []const T, delimiter: []const T) SplitBackwardsIterator(T, .full) {
2123 assert(delimiter.len != 0);2230 assert(delimiter.len != 0);
2124 return SplitBackwardsIterator(T){2231 return .{
2125 .index = buffer.len,2232 .index = buffer.len,
2126 .buffer = buffer,2233 .buffer = buffer,
2127 .delimiter = delimiter,2234 .delimiter = delimiter,
2128 };2235 };
2129}2236}
21302237
2131test "splitBackwards" {2238/// Returns an iterator that iterates backwards over the slices of `buffer` that
2132 var it = splitBackwards(u8, "abc|def||ghi", "|");2239/// are separated by any item in `delimiters`.
2240///
2241/// `splitBackwardsAny(u8, "abc,def||ghi", "|,")` will return slices
2242/// for "ghi", "", "def", "abc", null, in that order.
2243///
2244/// If none of `delimiters` exist in buffer,
2245/// the iterator will return `buffer`, null, in that order.
2246///
2247/// See also: `splitBackwardsFull`, `splitBackwardsScalar`,
2248/// `splitFull`, `splitAny`,`splitScalar`, and
2249/// `tokenize`.
2250pub fn splitBackwardsAny(comptime T: type, buffer: []const T, delimiters: []const T) SplitBackwardsIterator(T, .any) {
2251 return .{
2252 .index = buffer.len,
2253 .buffer = buffer,
2254 .delimiter = delimiters,
2255 };
2256}
2257
2258/// Returns an iterator that iterates backwards over the slices of `buffer` that
2259/// are separated by `delimiter`.
2260///
2261/// `splitBackwardsScalar(u8, "abc|def||ghi", '|')` will return slices
2262/// for "ghi", "", "def", "abc", null, in that order.
2263///
2264/// If `delimiter` does not exist in buffer,
2265/// the iterator will return `buffer`, null, in that order.
2266///
2267/// See also: `splitBackwardsFull`, `splitBackwardsAny`,
2268/// `splitFull`, `splitAny`,`splitScalar`, and
2269/// `tokenize`.
2270pub fn splitBackwardsScalar(comptime T: type, buffer: []const T, delimiter: T) SplitBackwardsIterator(T, .scalar) {
2271 return .{
2272 .index = buffer.len,
2273 .buffer = buffer,
2274 .delimiter = delimiter,
2275 };
2276}
2277
2278test "splitBackwardsScalar" {
2279 var it = splitBackwardsScalar(u8, "abc|def||ghi", '|');
2133 try testing.expectEqualSlices(u8, it.rest(), "abc|def||ghi");2280 try testing.expectEqualSlices(u8, it.rest(), "abc|def||ghi");
2134 try testing.expectEqualSlices(u8, it.first(), "ghi");2281 try testing.expectEqualSlices(u8, it.first(), "ghi");
21352282
...@@ -2145,30 +2292,30 @@ test "splitBackwards" {...@@ -2145,30 +2292,30 @@ test "splitBackwards" {
2145 try testing.expectEqualSlices(u8, it.rest(), "");2292 try testing.expectEqualSlices(u8, it.rest(), "");
2146 try testing.expect(it.next() == null);2293 try testing.expect(it.next() == null);
21472294
2148 it = splitBackwards(u8, "", "|");2295 it = splitBackwardsScalar(u8, "", '|');
2149 try testing.expectEqualSlices(u8, it.first(), "");2296 try testing.expectEqualSlices(u8, it.first(), "");
2150 try testing.expect(it.next() == null);2297 try testing.expect(it.next() == null);
21512298
2152 it = splitBackwards(u8, "|", "|");2299 it = splitBackwardsScalar(u8, "|", '|');
2153 try testing.expectEqualSlices(u8, it.first(), "");2300 try testing.expectEqualSlices(u8, it.first(), "");
2154 try testing.expectEqualSlices(u8, it.next().?, "");2301 try testing.expectEqualSlices(u8, it.next().?, "");
2155 try testing.expect(it.next() == null);2302 try testing.expect(it.next() == null);
21562303
2157 it = splitBackwards(u8, "hello", " ");2304 it = splitBackwardsScalar(u8, "hello", ' ');
2158 try testing.expectEqualSlices(u8, it.first(), "hello");2305 try testing.expectEqualSlices(u8, it.first(), "hello");
2159 try testing.expect(it.next() == null);2306 try testing.expect(it.next() == null);
21602307
2161 var it16 = splitBackwards(2308 var it16 = splitBackwardsScalar(
2162 u16,2309 u16,
2163 std.unicode.utf8ToUtf16LeStringLiteral("hello"),2310 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
2164 std.unicode.utf8ToUtf16LeStringLiteral(" "),2311 ' ',
2165 );2312 );
2166 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("hello"));2313 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("hello"));
2167 try testing.expect(it16.next() == null);2314 try testing.expect(it16.next() == null);
2168}2315}
21692316
2170test "splitBackwards (multibyte)" {2317test "splitBackwardsFull (multibyte)" {
2171 var it = splitBackwards(u8, "a, b ,, c, d, e", ", ");2318 var it = splitBackwardsFull(u8, "a, b ,, c, d, e", ", ");
2172 try testing.expectEqualSlices(u8, it.rest(), "a, b ,, c, d, e");2319 try testing.expectEqualSlices(u8, it.rest(), "a, b ,, c, d, e");
2173 try testing.expectEqualSlices(u8, it.first(), "e");2320 try testing.expectEqualSlices(u8, it.first(), "e");
21742321
...@@ -2187,7 +2334,7 @@ test "splitBackwards (multibyte)" {...@@ -2187,7 +2334,7 @@ test "splitBackwards (multibyte)" {
2187 try testing.expectEqualSlices(u8, it.rest(), "");2334 try testing.expectEqualSlices(u8, it.rest(), "");
2188 try testing.expect(it.next() == null);2335 try testing.expect(it.next() == null);
21892336
2190 var it16 = splitBackwards(2337 var it16 = splitBackwardsFull(
2191 u16,2338 u16,
2192 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),2339 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),
2193 std.unicode.utf8ToUtf16LeStringLiteral(", "),2340 std.unicode.utf8ToUtf16LeStringLiteral(", "),
...@@ -2200,18 +2347,83 @@ test "splitBackwards (multibyte)" {...@@ -2200,18 +2347,83 @@ test "splitBackwards (multibyte)" {
2200 try testing.expect(it16.next() == null);2347 try testing.expect(it16.next() == null);
2201}2348}
22022349
2203test "splitBackwards (reset)" {2350test "splitBackwardsAny" {
2204 var it = splitBackwards(u8, "abc def ghi", " ");2351 var it = splitBackwardsAny(u8, "a,b, c d e", ", ");
2205 try testing.expect(eql(u8, it.first(), "ghi"));2352 try testing.expectEqualSlices(u8, it.rest(), "a,b, c d e");
2206 try testing.expect(eql(u8, it.next().?, "def"));2353 try testing.expectEqualSlices(u8, it.first(), "e");
2207 try testing.expect(eql(u8, it.next().?, "abc"));
22082354
2209 it.reset();2355 try testing.expectEqualSlices(u8, it.rest(), "a,b, c d");
2356 try testing.expectEqualSlices(u8, it.next().?, "d");
22102357
2211 try testing.expect(eql(u8, it.first(), "ghi"));2358 try testing.expectEqualSlices(u8, it.rest(), "a,b, c");
2212 try testing.expect(eql(u8, it.next().?, "def"));2359 try testing.expectEqualSlices(u8, it.next().?, "c");
2213 try testing.expect(eql(u8, it.next().?, "abc"));2360
2361 try testing.expectEqualSlices(u8, it.rest(), "a,b,");
2362 try testing.expectEqualSlices(u8, it.next().?, "");
2363
2364 try testing.expectEqualSlices(u8, it.rest(), "a,b");
2365 try testing.expectEqualSlices(u8, it.next().?, "b");
2366
2367 try testing.expectEqualSlices(u8, it.rest(), "a");
2368 try testing.expectEqualSlices(u8, it.next().?, "a");
2369
2370 try testing.expectEqualSlices(u8, it.rest(), "");
2214 try testing.expect(it.next() == null);2371 try testing.expect(it.next() == null);
2372
2373 var it16 = splitBackwardsAny(
2374 u16,
2375 std.unicode.utf8ToUtf16LeStringLiteral("a,b, c d e"),
2376 std.unicode.utf8ToUtf16LeStringLiteral(", "),
2377 );
2378 try testing.expectEqualSlices(u16, it16.first(), std.unicode.utf8ToUtf16LeStringLiteral("e"));
2379 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d"));
2380 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c"));
2381 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral(""));
2382 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b"));
2383 try testing.expectEqualSlices(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a"));
2384 try testing.expect(it16.next() == null);
2385}
2386
2387test "splitBackwards (reset)" {
2388 {
2389 var it = splitBackwardsFull(u8, "abc def ghi", " ");
2390 try testing.expect(eql(u8, it.first(), "ghi"));
2391 try testing.expect(eql(u8, it.next().?, "def"));
2392 try testing.expect(eql(u8, it.next().?, "abc"));
2393
2394 it.reset();
2395
2396 try testing.expect(eql(u8, it.first(), "ghi"));
2397 try testing.expect(eql(u8, it.next().?, "def"));
2398 try testing.expect(eql(u8, it.next().?, "abc"));
2399 try testing.expect(it.next() == null);
2400 }
2401 {
2402 var it = splitBackwardsAny(u8, "abc def,ghi", " ,");
2403 try testing.expect(eql(u8, it.first(), "ghi"));
2404 try testing.expect(eql(u8, it.next().?, "def"));
2405 try testing.expect(eql(u8, it.next().?, "abc"));
2406
2407 it.reset();
2408
2409 try testing.expect(eql(u8, it.first(), "ghi"));
2410 try testing.expect(eql(u8, it.next().?, "def"));
2411 try testing.expect(eql(u8, it.next().?, "abc"));
2412 try testing.expect(it.next() == null);
2413 }
2414 {
2415 var it = splitBackwardsScalar(u8, "abc def ghi", ' ');
2416 try testing.expect(eql(u8, it.first(), "ghi"));
2417 try testing.expect(eql(u8, it.next().?, "def"));
2418 try testing.expect(eql(u8, it.next().?, "abc"));
2419
2420 it.reset();
2421
2422 try testing.expect(eql(u8, it.first(), "ghi"));
2423 try testing.expect(eql(u8, it.next().?, "def"));
2424 try testing.expect(eql(u8, it.next().?, "abc"));
2425 try testing.expect(it.next() == null);
2426 }
2215}2427}
22162428
2217/// Returns an iterator with a sliding window of slices for `buffer`.2429/// Returns an iterator with a sliding window of slices for `buffer`.
...@@ -2382,6 +2594,8 @@ test "endsWith" {...@@ -2382,6 +2594,8 @@ test "endsWith" {
2382 try testing.expect(!endsWith(u8, "Bob", "Bo"));2594 try testing.expect(!endsWith(u8, "Bob", "Bo"));
2383}2595}
23842596
2597pub const DelimiterType = enum { full, any, scalar };
2598
2385pub fn TokenIterator(comptime T: type) type {2599pub fn TokenIterator(comptime T: type) type {
2386 return struct {2600 return struct {
2387 buffer: []const T,2601 buffer: []const T,
...@@ -2439,11 +2653,14 @@ pub fn TokenIterator(comptime T: type) type {...@@ -2439,11 +2653,14 @@ pub fn TokenIterator(comptime T: type) type {
2439 };2653 };
2440}2654}
24412655
2442pub fn SplitIterator(comptime T: type) type {2656pub fn SplitIterator(comptime T: type, comptime delimiter_type: DelimiterType) type {
2443 return struct {2657 return struct {
2444 buffer: []const T,2658 buffer: []const T,
2445 index: ?usize,2659 index: ?usize,
2446 delimiter: []const T,2660 delimiter: switch (delimiter_type) {
2661 .full, .any => []const T,
2662 .scalar => T,
2663 },
24472664
2448 const Self = @This();2665 const Self = @This();
24492666
...@@ -2457,8 +2674,15 @@ pub fn SplitIterator(comptime T: type) type {...@@ -2457,8 +2674,15 @@ pub fn SplitIterator(comptime T: type) type {
2457 /// Returns a slice of the next field, or null if splitting is complete.2674 /// Returns a slice of the next field, or null if splitting is complete.
2458 pub fn next(self: *Self) ?[]const T {2675 pub fn next(self: *Self) ?[]const T {
2459 const start = self.index orelse return null;2676 const start = self.index orelse return null;
2460 const end = if (indexOfPos(T, self.buffer, start, self.delimiter)) |delim_start| blk: {2677 const end = if (switch (delimiter_type) {
2461 self.index = delim_start + self.delimiter.len;2678 .full => indexOfPos(T, self.buffer, start, self.delimiter),
2679 .any => indexOfAnyPos(T, self.buffer, start, self.delimiter),
2680 .scalar => indexOfScalarPos(T, self.buffer, start, self.delimiter),
2681 }) |delim_start| blk: {
2682 self.index = delim_start + switch (delimiter_type) {
2683 .full => self.delimiter.len,
2684 .any, .scalar => 1,
2685 };
2462 break :blk delim_start;2686 break :blk delim_start;
2463 } else blk: {2687 } else blk: {
2464 self.index = null;2688 self.index = null;
...@@ -2481,11 +2705,14 @@ pub fn SplitIterator(comptime T: type) type {...@@ -2481,11 +2705,14 @@ pub fn SplitIterator(comptime T: type) type {
2481 };2705 };
2482}2706}
24832707
2484pub fn SplitBackwardsIterator(comptime T: type) type {2708pub fn SplitBackwardsIterator(comptime T: type, comptime delimiter_type: DelimiterType) type {
2485 return struct {2709 return struct {
2486 buffer: []const T,2710 buffer: []const T,
2487 index: ?usize,2711 index: ?usize,
2488 delimiter: []const T,2712 delimiter: switch (delimiter_type) {
2713 .full, .any => []const T,
2714 .scalar => T,
2715 },
24892716
2490 const Self = @This();2717 const Self = @This();
24912718
...@@ -2499,9 +2726,16 @@ pub fn SplitBackwardsIterator(comptime T: type) type {...@@ -2499,9 +2726,16 @@ pub fn SplitBackwardsIterator(comptime T: type) type {
2499 /// Returns a slice of the next field, or null if splitting is complete.2726 /// Returns a slice of the next field, or null if splitting is complete.
2500 pub fn next(self: *Self) ?[]const T {2727 pub fn next(self: *Self) ?[]const T {
2501 const end = self.index orelse return null;2728 const end = self.index orelse return null;
2502 const start = if (lastIndexOf(T, self.buffer[0..end], self.delimiter)) |delim_start| blk: {2729 const start = if (switch (delimiter_type) {
2730 .full => lastIndexOf(T, self.buffer[0..end], self.delimiter),
2731 .any => lastIndexOfAny(T, self.buffer[0..end], self.delimiter),
2732 .scalar => lastIndexOfScalar(T, self.buffer[0..end], self.delimiter),
2733 }) |delim_start| blk: {
2503 self.index = delim_start;2734 self.index = delim_start;
2504 break :blk delim_start + self.delimiter.len;2735 break :blk delim_start + switch (delimiter_type) {
2736 .full => self.delimiter.len,
2737 .any, .scalar => 1,
2738 };
2505 } else blk: {2739 } else blk: {
2506 self.index = null;2740 self.index = null;
2507 break :blk 0;2741 break :blk 0;