authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-08-07 11:14:45-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-08-07 11:14:45-07:00
logd94252496e6622189fca72ad6d6b17db0dcb2e03
treebe8281fdcdb48d4e1ec700666f6c8259191eb5df
parent259f3458a162120288eb80dea4e55cd4ed9cf4c5
parentd31352ee85d633876877d87b813cd3611aa17d88
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #9531 from squeek502/split-tokenize-generic

Make mem.split and mem.tokenize generic instead of assuming u8

25 files changed, 201 insertions(+), 153 deletions(-)

build.zig+5-5
......@@ -187,7 +187,7 @@ pub fn build(b: *Builder) !void {
187187 },
188188 2 => {
189189 // Untagged development build (e.g. 0.8.0-684-gbbe2cca1a).
190 var it = mem.split(git_describe, "-");
190 var it = mem.split(u8, git_describe, "-");
191191 const tagged_ancestor = it.next() orelse unreachable;
192192 const commit_height = it.next() orelse unreachable;
193193 const commit_id = it.next() orelse unreachable;
......@@ -479,7 +479,7 @@ fn addCxxKnownPath(
479479 ctx.cxx_compiler,
480480 b.fmt("-print-file-name={s}", .{objname}),
481481 });
482 const path_unpadded = mem.tokenize(path_padded, "\r\n").next().?;
482 const path_unpadded = mem.tokenize(u8, path_padded, "\r\n").next().?;
483483 if (mem.eql(u8, path_unpadded, objname)) {
484484 if (errtxt) |msg| {
485485 warn("{s}", .{msg});
......@@ -502,7 +502,7 @@ fn addCxxKnownPath(
502502}
503503
504504fn addCMakeLibraryList(exe: *std.build.LibExeObjStep, list: []const u8) void {
505 var it = mem.tokenize(list, ";");
505 var it = mem.tokenize(u8, list, ";");
506506 while (it.next()) |lib| {
507507 if (mem.startsWith(u8, lib, "-l")) {
508508 exe.linkSystemLibrary(lib["-l".len..]);
......@@ -596,11 +596,11 @@ fn findAndParseConfigH(b: *Builder, config_h_path_option: ?[]const u8) ?CMakeCon
596596 },
597597 };
598598
599 var lines_it = mem.tokenize(config_h_text, "\r\n");
599 var lines_it = mem.tokenize(u8, config_h_text, "\r\n");
600600 while (lines_it.next()) |line| {
601601 inline for (mappings) |mapping| {
602602 if (mem.startsWith(u8, line, mapping.prefix)) {
603 var it = mem.split(line, "\"");
603 var it = mem.split(u8, line, "\"");
604604 _ = it.next().?; // skip the stuff before the quote
605605 const quoted = it.next().?; // the stuff inside the quote
606606 @field(ctx, mapping.field) = toNativePathSep(b, quoted);
lib/std/SemanticVersion.zig+5-5
......@@ -48,8 +48,8 @@ pub fn order(lhs: Version, rhs: Version) std.math.Order {
4848 if (lhs.pre == null and rhs.pre != null) return .gt;
4949
5050 // Iterate over pre-release identifiers until a difference is found.
51 var lhs_pre_it = std.mem.split(lhs.pre.?, ".");
52 var rhs_pre_it = std.mem.split(rhs.pre.?, ".");
51 var lhs_pre_it = std.mem.split(u8, lhs.pre.?, ".");
52 var rhs_pre_it = std.mem.split(u8, rhs.pre.?, ".");
5353 while (true) {
5454 const next_lid = lhs_pre_it.next();
5555 const next_rid = rhs_pre_it.next();
......@@ -92,7 +92,7 @@ pub fn parse(text: []const u8) !Version {
9292 // Parse the required major, minor, and patch numbers.
9393 const extra_index = std.mem.indexOfAny(u8, text, "-+");
9494 const required = text[0..(extra_index orelse text.len)];
95 var it = std.mem.split(required, ".");
95 var it = std.mem.split(u8, required, ".");
9696 var ver = Version{
9797 .major = try parseNum(it.next() orelse return error.InvalidVersion),
9898 .minor = try parseNum(it.next() orelse return error.InvalidVersion),
......@@ -114,7 +114,7 @@ pub fn parse(text: []const u8) !Version {
114114 // Check validity of optional pre-release identifiers.
115115 // See: https://semver.org/#spec-item-9
116116 if (ver.pre) |pre| {
117 it = std.mem.split(pre, ".");
117 it = std.mem.split(u8, pre, ".");
118118 while (it.next()) |id| {
119119 // Identifiers MUST NOT be empty.
120120 if (id.len == 0) return error.InvalidVersion;
......@@ -133,7 +133,7 @@ pub fn parse(text: []const u8) !Version {
133133 // Check validity of optional build metadata identifiers.
134134 // See: https://semver.org/#spec-item-10
135135 if (ver.build) |build| {
136 it = std.mem.split(build, ".");
136 it = std.mem.split(u8, build, ".");
137137 while (it.next()) |id| {
138138 // Identifiers MUST NOT be empty.
139139 if (id.len == 0) return error.InvalidVersion;
lib/std/build.zig+4-4
......@@ -1085,7 +1085,7 @@ pub const Builder = struct {
10851085 if (fs.path.isAbsolute(name)) {
10861086 return name;
10871087 }
1088 var it = mem.tokenize(PATH, &[_]u8{fs.path.delimiter});
1088 var it = mem.tokenize(u8, PATH, &[_]u8{fs.path.delimiter});
10891089 while (it.next()) |path| {
10901090 const full_path = try fs.path.join(self.allocator, &[_][]const u8{
10911091 path,
......@@ -1211,10 +1211,10 @@ pub const Builder = struct {
12111211 const stdout = try self.execAllowFail(&[_][]const u8{ "pkg-config", "--list-all" }, out_code, .Ignore);
12121212 var list = ArrayList(PkgConfigPkg).init(self.allocator);
12131213 errdefer list.deinit();
1214 var line_it = mem.tokenize(stdout, "\r\n");
1214 var line_it = mem.tokenize(u8, stdout, "\r\n");
12151215 while (line_it.next()) |line| {
12161216 if (mem.trim(u8, line, " \t").len == 0) continue;
1217 var tok_it = mem.tokenize(line, " \t");
1217 var tok_it = mem.tokenize(u8, line, " \t");
12181218 try list.append(PkgConfigPkg{
12191219 .name = tok_it.next() orelse return error.PkgConfigInvalidOutput,
12201220 .desc = tok_it.rest(),
......@@ -1872,7 +1872,7 @@ pub const LibExeObjStep = struct {
18721872 error.FileNotFound => return error.PkgConfigNotInstalled,
18731873 else => return err,
18741874 };
1875 var it = mem.tokenize(stdout, " \r\n\t");
1875 var it = mem.tokenize(u8, stdout, " \r\n\t");
18761876 while (it.next()) |tok| {
18771877 if (mem.eql(u8, tok, "-I")) {
18781878 const dir = it.next() orelse return error.PkgConfigInvalidOutput;
lib/std/builtin.zig+1-1
......@@ -509,7 +509,7 @@ pub const Version = struct {
509509 // found no digits or '.' before unexpected character
510510 if (end == 0) return error.InvalidVersion;
511511
512 var it = std.mem.split(text[0..end], ".");
512 var it = std.mem.split(u8, text[0..end], ".");
513513 // substring is not empty, first call will succeed
514514 const major = it.next().?;
515515 if (major.len == 0) return error.InvalidVersion;
lib/std/child_process.zig+2-2
......@@ -836,12 +836,12 @@ pub const ChildProcess = struct {
836836
837837 const app_name = self.argv[0];
838838
839 var it = mem.tokenize(PATH, ";");
839 var it = mem.tokenize(u8, PATH, ";");
840840 retry: while (it.next()) |search_path| {
841841 const path_no_ext = try fs.path.join(self.allocator, &[_][]const u8{ search_path, app_name });
842842 defer self.allocator.free(path_no_ext);
843843
844 var ext_it = mem.tokenize(PATHEXT, ";");
844 var ext_it = mem.tokenize(u8, PATHEXT, ";");
845845 while (ext_it.next()) |app_ext| {
846846 const joined_path = try mem.concat(self.allocator, u8, &[_][]const u8{ path_no_ext, app_ext });
847847 defer self.allocator.free(joined_path);
lib/std/fs.zig+1-1
......@@ -2455,7 +2455,7 @@ pub fn selfExePath(out_buffer: []u8) SelfExePathError![]u8 {
24552455 } else if (argv0.len != 0) {
24562456 // argv[0] is not empty (and not a path): search it inside PATH
24572457 const PATH = std.os.getenvZ("PATH") orelse return error.FileNotFound;
2458 var path_it = mem.tokenize(PATH, &[_]u8{path.delimiter});
2458 var path_it = mem.tokenize(u8, PATH, &[_]u8{path.delimiter});
24592459 while (path_it.next()) |a_path| {
24602460 var resolved_path_buf: [MAX_PATH_BYTES - 1:0]u8 = undefined;
24612461 const resolved_path = std.fmt.bufPrintZ(&resolved_path_buf, "{s}/{s}", .{
lib/std/fs/path.zig+13-13
......@@ -345,7 +345,7 @@ pub fn windowsParsePath(path: []const u8) WindowsPath {
345345 return relative_path;
346346 }
347347
348 var it = mem.tokenize(path, &[_]u8{this_sep});
348 var it = mem.tokenize(u8, path, &[_]u8{this_sep});
349349 _ = (it.next() orelse return relative_path);
350350 _ = (it.next() orelse return relative_path);
351351 return WindowsPath{
......@@ -407,8 +407,8 @@ fn networkShareServersEql(ns1: []const u8, ns2: []const u8) bool {
407407 const sep1 = ns1[0];
408408 const sep2 = ns2[0];
409409
410 var it1 = mem.tokenize(ns1, &[_]u8{sep1});
411 var it2 = mem.tokenize(ns2, &[_]u8{sep2});
410 var it1 = mem.tokenize(u8, ns1, &[_]u8{sep1});
411 var it2 = mem.tokenize(u8, ns2, &[_]u8{sep2});
412412
413413 // TODO ASCII is wrong, we actually need full unicode support to compare paths.
414414 return asciiEqlIgnoreCase(it1.next().?, it2.next().?);
......@@ -428,8 +428,8 @@ fn compareDiskDesignators(kind: WindowsPath.Kind, p1: []const u8, p2: []const u8
428428 const sep1 = p1[0];
429429 const sep2 = p2[0];
430430
431 var it1 = mem.tokenize(p1, &[_]u8{sep1});
432 var it2 = mem.tokenize(p2, &[_]u8{sep2});
431 var it1 = mem.tokenize(u8, p1, &[_]u8{sep1});
432 var it2 = mem.tokenize(u8, p2, &[_]u8{sep2});
433433
434434 // TODO ASCII is wrong, we actually need full unicode support to compare paths.
435435 return asciiEqlIgnoreCase(it1.next().?, it2.next().?) and asciiEqlIgnoreCase(it1.next().?, it2.next().?);
......@@ -551,7 +551,7 @@ pub fn resolveWindows(allocator: *Allocator, paths: []const []const u8) ![]u8 {
551551 },
552552 WindowsPath.Kind.NetworkShare => {
553553 result = try allocator.alloc(u8, max_size);
554 var it = mem.tokenize(paths[first_index], "/\\");
554 var it = mem.tokenize(u8, paths[first_index], "/\\");
555555 const server_name = it.next().?;
556556 const other_name = it.next().?;
557557
......@@ -618,7 +618,7 @@ pub fn resolveWindows(allocator: *Allocator, paths: []const []const u8) ![]u8 {
618618 if (!correct_disk_designator) {
619619 continue;
620620 }
621 var it = mem.tokenize(p[parsed.disk_designator.len..], "/\\");
621 var it = mem.tokenize(u8, p[parsed.disk_designator.len..], "/\\");
622622 while (it.next()) |component| {
623623 if (mem.eql(u8, component, ".")) {
624624 continue;
......@@ -687,7 +687,7 @@ pub fn resolvePosix(allocator: *Allocator, paths: []const []const u8) ![]u8 {
687687 errdefer allocator.free(result);
688688
689689 for (paths[first_index..]) |p| {
690 var it = mem.tokenize(p, "/");
690 var it = mem.tokenize(u8, p, "/");
691691 while (it.next()) |component| {
692692 if (mem.eql(u8, component, ".")) {
693693 continue;
......@@ -1101,8 +1101,8 @@ pub fn relativeWindows(allocator: *Allocator, from: []const u8, to: []const u8)
11011101 return resolved_to;
11021102 }
11031103
1104 var from_it = mem.tokenize(resolved_from, "/\\");
1105 var to_it = mem.tokenize(resolved_to, "/\\");
1104 var from_it = mem.tokenize(u8, resolved_from, "/\\");
1105 var to_it = mem.tokenize(u8, resolved_to, "/\\");
11061106 while (true) {
11071107 const from_component = from_it.next() orelse return allocator.dupe(u8, to_it.rest());
11081108 const to_rest = to_it.rest();
......@@ -1131,7 +1131,7 @@ pub fn relativeWindows(allocator: *Allocator, from: []const u8, to: []const u8)
11311131 // shave off the trailing slash
11321132 result_index -= 1;
11331133
1134 var rest_it = mem.tokenize(to_rest, "/\\");
1134 var rest_it = mem.tokenize(u8, to_rest, "/\\");
11351135 while (rest_it.next()) |to_component| {
11361136 result[result_index] = '\\';
11371137 result_index += 1;
......@@ -1152,8 +1152,8 @@ pub fn relativePosix(allocator: *Allocator, from: []const u8, to: []const u8) ![
11521152 const resolved_to = try resolvePosix(allocator, &[_][]const u8{to});
11531153 defer allocator.free(resolved_to);
11541154
1155 var from_it = mem.tokenize(resolved_from, "/");
1156 var to_it = mem.tokenize(resolved_to, "/");
1155 var from_it = mem.tokenize(u8, resolved_from, "/");
1156 var to_it = mem.tokenize(u8, resolved_to, "/");
11571157 while (true) {
11581158 const from_component = from_it.next() orelse return allocator.dupe(u8, to_it.rest());
11591159 const to_rest = to_it.rest();
lib/std/mem.zig+125-77
......@@ -1575,8 +1575,8 @@ test "bswapAllFields" {
15751575/// If `delimiter_bytes` does not exist in buffer,
15761576/// the iterator will return `buffer`, null, in that order.
15771577/// See also the related function `split`.
1578pub fn tokenize(buffer: []const u8, delimiter_bytes: []const u8) TokenIterator {
1579 return TokenIterator{
1578pub fn tokenize(comptime T: type, buffer: []const T, delimiter_bytes: []const T) TokenIterator(T) {
1579 return .{
15801580 .index = 0,
15811581 .buffer = buffer,
15821582 .delimiter_bytes = delimiter_bytes,
......@@ -1584,51 +1584,71 @@ pub fn tokenize(buffer: []const u8, delimiter_bytes: []const u8) TokenIterator {
15841584}
15851585
15861586test "mem.tokenize" {
1587 var it = tokenize(" abc def ghi ", " ");
1587 var it = tokenize(u8, " abc def ghi ", " ");
15881588 try testing.expect(eql(u8, it.next().?, "abc"));
15891589 try testing.expect(eql(u8, it.next().?, "def"));
15901590 try testing.expect(eql(u8, it.next().?, "ghi"));
15911591 try testing.expect(it.next() == null);
15921592
1593 it = tokenize("..\\bob", "\\");
1593 it = tokenize(u8, "..\\bob", "\\");
15941594 try testing.expect(eql(u8, it.next().?, ".."));
15951595 try testing.expect(eql(u8, "..", "..\\bob"[0..it.index]));
15961596 try testing.expect(eql(u8, it.next().?, "bob"));
15971597 try testing.expect(it.next() == null);
15981598
1599 it = tokenize("//a/b", "/");
1599 it = tokenize(u8, "//a/b", "/");
16001600 try testing.expect(eql(u8, it.next().?, "a"));
16011601 try testing.expect(eql(u8, it.next().?, "b"));
16021602 try testing.expect(eql(u8, "//a/b", "//a/b"[0..it.index]));
16031603 try testing.expect(it.next() == null);
16041604
1605 it = tokenize("|", "|");
1605 it = tokenize(u8, "|", "|");
16061606 try testing.expect(it.next() == null);
16071607
1608 it = tokenize("", "|");
1608 it = tokenize(u8, "", "|");
16091609 try testing.expect(it.next() == null);
16101610
1611 it = tokenize("hello", "");
1611 it = tokenize(u8, "hello", "");
16121612 try testing.expect(eql(u8, it.next().?, "hello"));
16131613 try testing.expect(it.next() == null);
16141614
1615 it = tokenize("hello", " ");
1615 it = tokenize(u8, "hello", " ");
16161616 try testing.expect(eql(u8, it.next().?, "hello"));
16171617 try testing.expect(it.next() == null);
1618
1619 var it16 = tokenize(
1620 u16,
1621 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
1622 std.unicode.utf8ToUtf16LeStringLiteral(" "),
1623 );
1624 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));
1625 try testing.expect(it16.next() == null);
16181626}
16191627
16201628test "mem.tokenize (multibyte)" {
1621 var it = tokenize("a|b,c/d e", " /,|");
1629 var it = tokenize(u8, "a|b,c/d e", " /,|");
16221630 try testing.expect(eql(u8, it.next().?, "a"));
16231631 try testing.expect(eql(u8, it.next().?, "b"));
16241632 try testing.expect(eql(u8, it.next().?, "c"));
16251633 try testing.expect(eql(u8, it.next().?, "d"));
16261634 try testing.expect(eql(u8, it.next().?, "e"));
16271635 try testing.expect(it.next() == null);
1636
1637 var it16 = tokenize(
1638 u16,
1639 std.unicode.utf8ToUtf16LeStringLiteral("a|b,c/d e"),
1640 std.unicode.utf8ToUtf16LeStringLiteral(" /,|"),
1641 );
1642 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a")));
1643 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b")));
1644 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c")));
1645 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d")));
1646 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("e")));
1647 try testing.expect(it16.next() == null);
16281648}
16291649
16301650test "mem.tokenize (reset)" {
1631 var it = tokenize(" abc def ghi ", " ");
1651 var it = tokenize(u8, " abc def ghi ", " ");
16321652 try testing.expect(eql(u8, it.next().?, "abc"));
16331653 try testing.expect(eql(u8, it.next().?, "def"));
16341654 try testing.expect(eql(u8, it.next().?, "ghi"));
......@@ -1649,9 +1669,9 @@ test "mem.tokenize (reset)" {
16491669/// the iterator will return `buffer`, null, in that order.
16501670/// The delimiter length must not be zero.
16511671/// See also the related function `tokenize`.
1652pub fn split(buffer: []const u8, delimiter: []const u8) SplitIterator {
1672pub fn split(comptime T: type, buffer: []const T, delimiter: []const T) SplitIterator(T) {
16531673 assert(delimiter.len != 0);
1654 return SplitIterator{
1674 return .{
16551675 .index = 0,
16561676 .buffer = buffer,
16571677 .delimiter = delimiter,
......@@ -1661,35 +1681,55 @@ pub fn split(buffer: []const u8, delimiter: []const u8) SplitIterator {
16611681pub const separate = @compileError("deprecated: renamed to split (behavior remains unchanged)");
16621682
16631683test "mem.split" {
1664 var it = split("abc|def||ghi", "|");
1684 var it = split(u8, "abc|def||ghi", "|");
16651685 try testing.expect(eql(u8, it.next().?, "abc"));
16661686 try testing.expect(eql(u8, it.next().?, "def"));
16671687 try testing.expect(eql(u8, it.next().?, ""));
16681688 try testing.expect(eql(u8, it.next().?, "ghi"));
16691689 try testing.expect(it.next() == null);
16701690
1671 it = split("", "|");
1691 it = split(u8, "", "|");
16721692 try testing.expect(eql(u8, it.next().?, ""));
16731693 try testing.expect(it.next() == null);
16741694
1675 it = split("|", "|");
1695 it = split(u8, "|", "|");
16761696 try testing.expect(eql(u8, it.next().?, ""));
16771697 try testing.expect(eql(u8, it.next().?, ""));
16781698 try testing.expect(it.next() == null);
16791699
1680 it = split("hello", " ");
1700 it = split(u8, "hello", " ");
16811701 try testing.expect(eql(u8, it.next().?, "hello"));
16821702 try testing.expect(it.next() == null);
1703
1704 var it16 = split(
1705 u16,
1706 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
1707 std.unicode.utf8ToUtf16LeStringLiteral(" "),
1708 );
1709 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));
1710 try testing.expect(it16.next() == null);
16831711}
16841712
16851713test "mem.split (multibyte)" {
1686 var it = split("a, b ,, c, d, e", ", ");
1714 var it = split(u8, "a, b ,, c, d, e", ", ");
16871715 try testing.expect(eql(u8, it.next().?, "a"));
16881716 try testing.expect(eql(u8, it.next().?, "b ,"));
16891717 try testing.expect(eql(u8, it.next().?, "c"));
16901718 try testing.expect(eql(u8, it.next().?, "d"));
16911719 try testing.expect(eql(u8, it.next().?, "e"));
16921720 try testing.expect(it.next() == null);
1721
1722 var it16 = split(
1723 u16,
1724 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),
1725 std.unicode.utf8ToUtf16LeStringLiteral(", "),
1726 );
1727 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a")));
1728 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b ,")));
1729 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c")));
1730 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d")));
1731 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("e")));
1732 try testing.expect(it16.next() == null);
16931733}
16941734
16951735pub fn startsWith(comptime T: type, haystack: []const T, needle: []const T) bool {
......@@ -1710,75 +1750,83 @@ test "mem.endsWith" {
17101750 try testing.expect(!endsWith(u8, "Bob", "Bo"));
17111751}
17121752
1713pub const TokenIterator = struct {
1714 buffer: []const u8,
1715 delimiter_bytes: []const u8,
1716 index: usize,
1753pub fn TokenIterator(comptime T: type) type {
1754 return struct {
1755 buffer: []const T,
1756 delimiter_bytes: []const T,
1757 index: usize,
17171758
1718 /// Returns a slice of the next token, or null if tokenization is complete.
1719 pub fn next(self: *TokenIterator) ?[]const u8 {
1720 // move to beginning of token
1721 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1722 const start = self.index;
1723 if (start == self.buffer.len) {
1724 return null;
1725 }
1759 const Self = @This();
17261760
1727 // move to end of token
1728 while (self.index < self.buffer.len and !self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1729 const end = self.index;
1761 /// Returns a slice of the next token, or null if tokenization is complete.
1762 pub fn next(self: *Self) ?[]const T {
1763 // move to beginning of token
1764 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1765 const start = self.index;
1766 if (start == self.buffer.len) {
1767 return null;
1768 }
17301769
1731 return self.buffer[start..end];
1732 }
1770 // move to end of token
1771 while (self.index < self.buffer.len and !self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1772 const end = self.index;
17331773
1734 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1735 pub fn rest(self: TokenIterator) []const u8 {
1736 // move to beginning of token
1737 var index: usize = self.index;
1738 while (index < self.buffer.len and self.isSplitByte(self.buffer[index])) : (index += 1) {}
1739 return self.buffer[index..];
1740 }
1774 return self.buffer[start..end];
1775 }
17411776
1742 /// Resets the iterator to the initial token.
1743 pub fn reset(self: *TokenIterator) void {
1744 self.index = 0;
1745 }
1777 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1778 pub fn rest(self: Self) []const T {
1779 // move to beginning of token
1780 var index: usize = self.index;
1781 while (index < self.buffer.len and self.isSplitByte(self.buffer[index])) : (index += 1) {}
1782 return self.buffer[index..];
1783 }
1784
1785 /// Resets the iterator to the initial token.
1786 pub fn reset(self: *Self) void {
1787 self.index = 0;
1788 }
17461789
1747 fn isSplitByte(self: TokenIterator, byte: u8) bool {
1748 for (self.delimiter_bytes) |delimiter_byte| {
1749 if (byte == delimiter_byte) {
1750 return true;
1790 fn isSplitByte(self: Self, byte: T) bool {
1791 for (self.delimiter_bytes) |delimiter_byte| {
1792 if (byte == delimiter_byte) {
1793 return true;
1794 }
17511795 }
1796 return false;
17521797 }
1753 return false;
1754 }
1755};
1798 };
1799}
17561800
1757pub const SplitIterator = struct {
1758 buffer: []const u8,
1759 index: ?usize,
1760 delimiter: []const u8,
1761
1762 /// Returns a slice of the next field, or null if splitting is complete.
1763 pub fn next(self: *SplitIterator) ?[]const u8 {
1764 const start = self.index orelse return null;
1765 const end = if (indexOfPos(u8, self.buffer, start, self.delimiter)) |delim_start| blk: {
1766 self.index = delim_start + self.delimiter.len;
1767 break :blk delim_start;
1768 } else blk: {
1769 self.index = null;
1770 break :blk self.buffer.len;
1771 };
1772 return self.buffer[start..end];
1773 }
1801pub fn SplitIterator(comptime T: type) type {
1802 return struct {
1803 buffer: []const T,
1804 index: ?usize,
1805 delimiter: []const T,
17741806
1775 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1776 pub fn rest(self: SplitIterator) []const u8 {
1777 const end = self.buffer.len;
1778 const start = self.index orelse end;
1779 return self.buffer[start..end];
1780 }
1781};
1807 const Self = @This();
1808
1809 /// Returns a slice of the next field, or null if splitting is complete.
1810 pub fn next(self: *Self) ?[]const T {
1811 const start = self.index orelse return null;
1812 const end = if (indexOfPos(T, self.buffer, start, self.delimiter)) |delim_start| blk: {
1813 self.index = delim_start + self.delimiter.len;
1814 break :blk delim_start;
1815 } else blk: {
1816 self.index = null;
1817 break :blk self.buffer.len;
1818 };
1819 return self.buffer[start..end];
1820 }
1821
1822 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1823 pub fn rest(self: Self) []const T {
1824 const end = self.buffer.len;
1825 const start = self.index orelse end;
1826 return self.buffer[start..end];
1827 }
1828 };
1829}
17821830
17831831/// Naively combines a series of slices with a separator.
17841832/// Allocates memory for the result, which must be freed by the caller.
lib/std/net.zig+6-6
......@@ -1130,9 +1130,9 @@ fn linuxLookupNameFromHosts(
11301130 },
11311131 else => |e| return e,
11321132 }) |line| {
1133 const no_comment_line = mem.split(line, "#").next().?;
1133 const no_comment_line = mem.split(u8, line, "#").next().?;
11341134
1135 var line_it = mem.tokenize(no_comment_line, " \t");
1135 var line_it = mem.tokenize(u8, no_comment_line, " \t");
11361136 const ip_text = line_it.next() orelse continue;
11371137 var first_name_text: ?[]const u8 = null;
11381138 while (line_it.next()) |name_text| {
......@@ -1211,7 +1211,7 @@ fn linuxLookupNameFromDnsSearch(
12111211 mem.copy(u8, canon.items, canon_name);
12121212 try canon.append('.');
12131213
1214 var tok_it = mem.tokenize(search, " \t");
1214 var tok_it = mem.tokenize(u8, search, " \t");
12151215 while (tok_it.next()) |tok| {
12161216 canon.shrinkRetainingCapacity(canon_name.len + 1);
12171217 try canon.appendSlice(tok);
......@@ -1328,13 +1328,13 @@ fn getResolvConf(allocator: *mem.Allocator, rc: *ResolvConf) !void {
13281328 },
13291329 else => |e| return e,
13301330 }) |line| {
1331 const no_comment_line = mem.split(line, "#").next().?;
1332 var line_it = mem.tokenize(no_comment_line, " \t");
1331 const no_comment_line = mem.split(u8, line, "#").next().?;
1332 var line_it = mem.tokenize(u8, no_comment_line, " \t");
13331333
13341334 const token = line_it.next() orelse continue;
13351335 if (mem.eql(u8, token, "options")) {
13361336 while (line_it.next()) |sub_tok| {
1337 var colon_it = mem.split(sub_tok, ":");
1337 var colon_it = mem.split(u8, sub_tok, ":");
13381338 const name = colon_it.next().?;
13391339 const value_txt = colon_it.next() orelse continue;
13401340 const value = std.fmt.parseInt(u8, value_txt, 10) catch |err| switch (err) {
lib/std/os.zig+1-1
......@@ -1378,7 +1378,7 @@ pub fn execvpeZ_expandArg0(
13781378 // Use of MAX_PATH_BYTES here is valid as the path_buf will be passed
13791379 // directly to the operating system in execveZ.
13801380 var path_buf: [MAX_PATH_BYTES]u8 = undefined;
1381 var it = mem.tokenize(PATH, ":");
1381 var it = mem.tokenize(u8, PATH, ":");
13821382 var seen_eacces = false;
13831383 var err: ExecveError = undefined;
13841384
lib/std/process.zig+1-1
......@@ -109,7 +109,7 @@ pub fn getEnvMap(allocator: *Allocator) !BufMap {
109109
110110 for (environ) |env| {
111111 const pair = mem.spanZ(env);
112 var parts = mem.split(pair, "=");
112 var parts = mem.split(u8, pair, "=");
113113 const key = parts.next().?;
114114 const value = parts.next().?;
115115 try result.put(key, value);
lib/std/zig/cross_target.zig+5-5
......@@ -233,7 +233,7 @@ pub const CrossTarget = struct {
233233 .dynamic_linker = DynamicLinker.init(args.dynamic_linker),
234234 };
235235
236 var it = mem.split(args.arch_os_abi, "-");
236 var it = mem.split(u8, args.arch_os_abi, "-");
237237 const arch_name = it.next().?;
238238 const arch_is_native = mem.eql(u8, arch_name, "native");
239239 if (!arch_is_native) {
......@@ -251,7 +251,7 @@ pub const CrossTarget = struct {
251251
252252 const opt_abi_text = it.next();
253253 if (opt_abi_text) |abi_text| {
254 var abi_it = mem.split(abi_text, ".");
254 var abi_it = mem.split(u8, abi_text, ".");
255255 const abi = std.meta.stringToEnum(Target.Abi, abi_it.next().?) orelse
256256 return error.UnknownApplicationBinaryInterface;
257257 result.abi = abi;
......@@ -699,7 +699,7 @@ pub const CrossTarget = struct {
699699 }
700700
701701 fn parseOs(result: *CrossTarget, diags: *ParseOptions.Diagnostics, text: []const u8) !void {
702 var it = mem.split(text, ".");
702 var it = mem.split(u8, text, ".");
703703 const os_name = it.next().?;
704704 diags.os_name = os_name;
705705 const os_is_native = mem.eql(u8, os_name, "native");
......@@ -757,7 +757,7 @@ pub const CrossTarget = struct {
757757 .linux,
758758 .dragonfly,
759759 => {
760 var range_it = mem.split(version_text, "...");
760 var range_it = mem.split(u8, version_text, "...");
761761
762762 const min_text = range_it.next().?;
763763 const min_ver = SemVer.parse(min_text) catch |err| switch (err) {
......@@ -777,7 +777,7 @@ pub const CrossTarget = struct {
777777 },
778778
779779 .windows => {
780 var range_it = mem.split(version_text, "...");
780 var range_it = mem.split(u8, version_text, "...");
781781
782782 const min_text = range_it.next().?;
783783 const min_ver = std.meta.stringToEnum(Target.Os.WindowsVersion, min_text) orelse
lib/std/zig/system.zig+3-3
......@@ -44,7 +44,7 @@ pub const NativePaths = struct {
4444 defer allocator.free(nix_cflags_compile);
4545
4646 is_nix = true;
47 var it = mem.tokenize(nix_cflags_compile, " ");
47 var it = mem.tokenize(u8, nix_cflags_compile, " ");
4848 while (true) {
4949 const word = it.next() orelse break;
5050 if (mem.eql(u8, word, "-isystem")) {
......@@ -69,7 +69,7 @@ pub const NativePaths = struct {
6969 defer allocator.free(nix_ldflags);
7070
7171 is_nix = true;
72 var it = mem.tokenize(nix_ldflags, " ");
72 var it = mem.tokenize(u8, nix_ldflags, " ");
7373 while (true) {
7474 const word = it.next() orelse break;
7575 if (mem.eql(u8, word, "-rpath")) {
......@@ -839,7 +839,7 @@ pub const NativeTargetInfo = struct {
839839 error.Overflow => return error.InvalidElfFile,
840840 };
841841 const rpath_list = mem.spanZ(std.meta.assumeSentinel(strtab[rpoff_usize..].ptr, 0));
842 var it = mem.tokenize(rpath_list, ":");
842 var it = mem.tokenize(u8, rpath_list, ":");
843843 while (it.next()) |rpath| {
844844 var dir = fs.cwd().openDir(rpath, .{}) catch |err| switch (err) {
845845 error.NameTooLong => unreachable,
src/Cache.zig+2-2
......@@ -356,7 +356,7 @@ pub const Manifest = struct {
356356
357357 const input_file_count = self.files.items.len;
358358 var any_file_changed = false;
359 var line_iter = mem.tokenize(file_contents, "\n");
359 var line_iter = mem.tokenize(u8, file_contents, "\n");
360360 var idx: usize = 0;
361361 while (line_iter.next()) |line| {
362362 defer idx += 1;
......@@ -373,7 +373,7 @@ pub const Manifest = struct {
373373 break :blk new;
374374 };
375375
376 var iter = mem.tokenize(line, " ");
376 var iter = mem.tokenize(u8, line, " ");
377377 const size = iter.next() orelse return error.InvalidFormat;
378378 const inode = iter.next() orelse return error.InvalidFormat;
379379 const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
src/Compilation.zig+2-2
......@@ -3341,7 +3341,7 @@ pub fn hasSharedLibraryExt(filename: []const u8) bool {
33413341 return true;
33423342 }
33433343 // Look for .so.X, .so.X.Y, .so.X.Y.Z
3344 var it = mem.split(filename, ".");
3344 var it = mem.split(u8, filename, ".");
33453345 _ = it.next().?;
33463346 var so_txt = it.next() orelse return false;
33473347 while (!mem.eql(u8, so_txt, "so")) {
......@@ -4086,7 +4086,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
40864086 };
40874087
40884088 if (directory.handle.readFileAlloc(comp.gpa, libs_txt_basename, 10 * 1024 * 1024)) |libs_txt| {
4089 var it = mem.tokenize(libs_txt, "\n");
4089 var it = mem.tokenize(u8, libs_txt, "\n");
40904090 while (it.next()) |lib_name| {
40914091 try comp.stage1AddLinkLib(lib_name);
40924092 }
src/codegen.zig+1-1
......@@ -3656,7 +3656,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
36563656 }
36573657
36583658 {
3659 var iter = std.mem.tokenize(asm_source, "\n\r");
3659 var iter = std.mem.tokenize(u8, asm_source, "\n\r");
36603660 while (iter.next()) |ins| {
36613661 if (mem.eql(u8, ins, "syscall")) {
36623662 try self.code.appendSlice(&[_]u8{ 0x0f, 0x05 });
src/glibc.zig+7-7
......@@ -107,7 +107,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
107107 defer gpa.free(abi_txt_contents);
108108
109109 {
110 var it = mem.tokenize(vers_txt_contents, "\r\n");
110 var it = mem.tokenize(u8, vers_txt_contents, "\r\n");
111111 var line_i: usize = 1;
112112 while (it.next()) |line| : (line_i += 1) {
113113 const prefix = "GLIBC_";
......@@ -124,10 +124,10 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
124124 }
125125 }
126126 {
127 var file_it = mem.tokenize(fns_txt_contents, "\r\n");
127 var file_it = mem.tokenize(u8, fns_txt_contents, "\r\n");
128128 var line_i: usize = 1;
129129 while (file_it.next()) |line| : (line_i += 1) {
130 var line_it = mem.tokenize(line, " ");
130 var line_it = mem.tokenize(u8, line, " ");
131131 const fn_name = line_it.next() orelse {
132132 std.log.err("fns.txt:{d}: expected function name", .{line_i});
133133 return error.ZigInstallationCorrupt;
......@@ -147,7 +147,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
147147 }
148148 }
149149 {
150 var file_it = mem.split(abi_txt_contents, "\n");
150 var file_it = mem.split(u8, abi_txt_contents, "\n");
151151 var line_i: usize = 0;
152152 while (true) {
153153 const ver_list_base: []VerList = blk: {
......@@ -155,9 +155,9 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
155155 if (line.len == 0) break;
156156 line_i += 1;
157157 const ver_list_base = try arena.alloc(VerList, all_functions.items.len);
158 var line_it = mem.tokenize(line, " ");
158 var line_it = mem.tokenize(u8, line, " ");
159159 while (line_it.next()) |target_string| {
160 var component_it = mem.tokenize(target_string, "-");
160 var component_it = mem.tokenize(u8, target_string, "-");
161161 const arch_name = component_it.next() orelse {
162162 std.log.err("abi.txt:{d}: expected arch name", .{line_i});
163163 return error.ZigInstallationCorrupt;
......@@ -203,7 +203,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
203203 .versions = undefined,
204204 .len = 0,
205205 };
206 var line_it = mem.tokenize(line, " ");
206 var line_it = mem.tokenize(u8, line, " ");
207207 while (line_it.next()) |version_index_string| {
208208 if (ver_list.len >= ver_list.versions.len) {
209209 // If this happens with legit data, increase the array len in the type.
src/libc_installation.zig+5-5
......@@ -60,10 +60,10 @@ pub const LibCInstallation = struct {
6060 const contents = try std.fs.cwd().readFileAlloc(allocator, libc_file, std.math.maxInt(usize));
6161 defer allocator.free(contents);
6262
63 var it = std.mem.tokenize(contents, "\n");
63 var it = std.mem.tokenize(u8, contents, "\n");
6464 while (it.next()) |line| {
6565 if (line.len == 0 or line[0] == '#') continue;
66 var line_it = std.mem.split(line, "=");
66 var line_it = std.mem.split(u8, line, "=");
6767 const name = line_it.next() orelse {
6868 log.err("missing equal sign after field name\n", .{});
6969 return error.ParseError;
......@@ -298,7 +298,7 @@ pub const LibCInstallation = struct {
298298 },
299299 }
300300
301 var it = std.mem.tokenize(exec_res.stderr, "\n\r");
301 var it = std.mem.tokenize(u8, exec_res.stderr, "\n\r");
302302 var search_paths = std.ArrayList([]const u8).init(allocator);
303303 defer search_paths.deinit();
304304 while (it.next()) |line| {
......@@ -616,7 +616,7 @@ fn ccPrintFileName(args: CCPrintFileNameOptions) ![:0]u8 {
616616 },
617617 }
618618
619 var it = std.mem.tokenize(exec_res.stdout, "\n\r");
619 var it = std.mem.tokenize(u8, exec_res.stdout, "\n\r");
620620 const line = it.next() orelse return error.LibCRuntimeNotFound;
621621 // When this command fails, it returns exit code 0 and duplicates the input file name.
622622 // So we detect failure by checking if the output matches exactly the input.
......@@ -695,7 +695,7 @@ fn appendCcExe(args: *std.ArrayList([]const u8), skip_cc_env_var: bool) !void {
695695 return;
696696 };
697697 // Respect space-separated flags to the C compiler.
698 var it = std.mem.tokenize(cc_env_var, " ");
698 var it = std.mem.tokenize(u8, cc_env_var, " ");
699699 while (it.next()) |arg| {
700700 try args.append(arg);
701701 }
src/link/MachO/Dylib.zig+1-1
......@@ -106,7 +106,7 @@ pub const Id = struct {
106106 var out: u32 = 0;
107107 var values: [3][]const u8 = undefined;
108108
109 var split = mem.split(string, ".");
109 var split = mem.split(u8, string, ".");
110110 var count: u4 = 0;
111111 while (split.next()) |value| {
112112 if (count > 2) {
src/main.zig+2-2
......@@ -1200,7 +1200,7 @@ fn buildOutputType(
12001200 },
12011201 .rdynamic => rdynamic = true,
12021202 .wl => {
1203 var split_it = mem.split(it.only_arg, ",");
1203 var split_it = mem.split(u8, it.only_arg, ",");
12041204 while (split_it.next()) |linker_arg| {
12051205 // Handle nested-joined args like `-Wl,-rpath=foo`.
12061206 // Must be prefixed with 1 or 2 dashes.
......@@ -3655,7 +3655,7 @@ pub const ClangArgIterator = struct {
36553655 defer allocator.free(resp_contents);
36563656 // TODO is there a specification for this file format? Let's find it and make this parsing more robust
36573657 // at the very least I'm guessing this needs to handle quotes and `#` comments.
3658 var it = mem.tokenize(resp_contents, " \t\r\n");
3658 var it = mem.tokenize(u8, resp_contents, " \t\r\n");
36593659 var resp_arg_list = std.ArrayList([]const u8).init(allocator);
36603660 defer resp_arg_list.deinit();
36613661 {
src/test.zig+2-2
......@@ -228,7 +228,7 @@ pub const TestContext = struct {
228228 continue;
229229 }
230230 // example: "file.zig:1:2: error: bad thing happened"
231 var it = std.mem.split(err_msg_line, ":");
231 var it = std.mem.split(u8, err_msg_line, ":");
232232 const src_path = it.next() orelse @panic("missing colon");
233233 const line_text = it.next() orelse @panic("missing line");
234234 const col_text = it.next() orelse @panic("missing column");
......@@ -779,7 +779,7 @@ pub const TestContext = struct {
779779 }
780780 var ok = true;
781781 if (case.expect_exact) {
782 var err_iter = std.mem.split(result.stderr, "\n");
782 var err_iter = std.mem.split(u8, result.stderr, "\n");
783783 var i: usize = 0;
784784 ok = while (err_iter.next()) |line| : (i += 1) {
785785 if (i >= case_error_list.len) break false;
test/behavior/bugs/6456.zig+1-1
......@@ -13,7 +13,7 @@ test "issue 6456" {
1313 comptime {
1414 var fields: []const StructField = &[0]StructField{};
1515
16 var it = std.mem.tokenize(text, "\n");
16 var it = std.mem.tokenize(u8, text, "\n");
1717 while (it.next()) |name| {
1818 fields = fields ++ &[_]StructField{StructField{
1919 .alignment = 0,
test/tests.zig+1-1
......@@ -768,7 +768,7 @@ pub const StackTracesContext = struct {
768768 var buf = ArrayList(u8).init(b.allocator);
769769 defer buf.deinit();
770770 if (stderr.len != 0 and stderr[stderr.len - 1] == '\n') stderr = stderr[0 .. stderr.len - 1];
771 var it = mem.split(stderr, "\n");
771 var it = mem.split(u8, stderr, "\n");
772772 process_lines: while (it.next()) |line| {
773773 if (line.len == 0) continue;
774774
tools/update_glibc.zig+4-4
......@@ -188,9 +188,9 @@ pub fn main() !void {
188188 std.debug.warn("unable to open {s}: {}\n", .{ abi_list_filename, err });
189189 std.process.exit(1);
190190 };
191 var lines_it = std.mem.tokenize(contents, "\n");
191 var lines_it = std.mem.tokenize(u8, contents, "\n");
192192 while (lines_it.next()) |line| {
193 var tok_it = std.mem.tokenize(line, " ");
193 var tok_it = std.mem.tokenize(u8, line, " ");
194194 const ver = tok_it.next().?;
195195 const name = tok_it.next().?;
196196 const category = tok_it.next().?;
......@@ -319,8 +319,8 @@ pub fn strCmpLessThan(context: void, a: []const u8, b: []const u8) bool {
319319pub fn versionLessThan(context: void, a: []const u8, b: []const u8) bool {
320320 _ = context;
321321 const sep_chars = "GLIBC_.";
322 var a_tokens = std.mem.tokenize(a, sep_chars);
323 var b_tokens = std.mem.tokenize(b, sep_chars);
322 var a_tokens = std.mem.tokenize(u8, a, sep_chars);
323 var b_tokens = std.mem.tokenize(u8, b, sep_chars);
324324
325325 while (true) {
326326 const a_next = a_tokens.next();
tools/update_spirv_features.zig+1-1
......@@ -19,7 +19,7 @@ const Version = struct {
1919 minor: u32,
2020
2121 fn parse(str: []const u8) !Version {
22 var it = std.mem.split(str, ".");
22 var it = std.mem.split(u8, str, ".");
2323
2424 const major = it.next() orelse return error.InvalidVersion;
2525 const minor = it.next() orelse return error.InvalidVersion;