authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-08-07 11:14:45-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2021-08-07 11:14:45-07:00
logd94252496e6622189fca72ad6d6b17db0dcb2e03
treebe8281fdcdb48d4e1ec700666f6c8259191eb5df
parent259f3458a162120288eb80dea4e55cd4ed9cf4c5
parentd31352ee85d633876877d87b813cd3611aa17d88
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #9531 from squeek502/split-tokenize-generic

Make mem.split and mem.tokenize generic instead of assuming u8

25 files changed, 201 insertions(+), 153 deletions(-)

build.zig+5-5
...@@ -187,7 +187,7 @@ pub fn build(b: *Builder) !void {...@@ -187,7 +187,7 @@ pub fn build(b: *Builder) !void {
187 },187 },
188 2 => {188 2 => {
189 // Untagged development build (e.g. 0.8.0-684-gbbe2cca1a).189 // Untagged development build (e.g. 0.8.0-684-gbbe2cca1a).
190 var it = mem.split(git_describe, "-");190 var it = mem.split(u8, git_describe, "-");
191 const tagged_ancestor = it.next() orelse unreachable;191 const tagged_ancestor = it.next() orelse unreachable;
192 const commit_height = it.next() orelse unreachable;192 const commit_height = it.next() orelse unreachable;
193 const commit_id = it.next() orelse unreachable;193 const commit_id = it.next() orelse unreachable;
...@@ -479,7 +479,7 @@ fn addCxxKnownPath(...@@ -479,7 +479,7 @@ fn addCxxKnownPath(
479 ctx.cxx_compiler,479 ctx.cxx_compiler,
480 b.fmt("-print-file-name={s}", .{objname}),480 b.fmt("-print-file-name={s}", .{objname}),
481 });481 });
482 const path_unpadded = mem.tokenize(path_padded, "\r\n").next().?;482 const path_unpadded = mem.tokenize(u8, path_padded, "\r\n").next().?;
483 if (mem.eql(u8, path_unpadded, objname)) {483 if (mem.eql(u8, path_unpadded, objname)) {
484 if (errtxt) |msg| {484 if (errtxt) |msg| {
485 warn("{s}", .{msg});485 warn("{s}", .{msg});
...@@ -502,7 +502,7 @@ fn addCxxKnownPath(...@@ -502,7 +502,7 @@ fn addCxxKnownPath(
502}502}
503503
504fn addCMakeLibraryList(exe: *std.build.LibExeObjStep, list: []const u8) void {504fn addCMakeLibraryList(exe: *std.build.LibExeObjStep, list: []const u8) void {
505 var it = mem.tokenize(list, ";");505 var it = mem.tokenize(u8, list, ";");
506 while (it.next()) |lib| {506 while (it.next()) |lib| {
507 if (mem.startsWith(u8, lib, "-l")) {507 if (mem.startsWith(u8, lib, "-l")) {
508 exe.linkSystemLibrary(lib["-l".len..]);508 exe.linkSystemLibrary(lib["-l".len..]);
...@@ -596,11 +596,11 @@ fn findAndParseConfigH(b: *Builder, config_h_path_option: ?[]const u8) ?CMakeCon...@@ -596,11 +596,11 @@ fn findAndParseConfigH(b: *Builder, config_h_path_option: ?[]const u8) ?CMakeCon
596 },596 },
597 };597 };
598598
599 var lines_it = mem.tokenize(config_h_text, "\r\n");599 var lines_it = mem.tokenize(u8, config_h_text, "\r\n");
600 while (lines_it.next()) |line| {600 while (lines_it.next()) |line| {
601 inline for (mappings) |mapping| {601 inline for (mappings) |mapping| {
602 if (mem.startsWith(u8, line, mapping.prefix)) {602 if (mem.startsWith(u8, line, mapping.prefix)) {
603 var it = mem.split(line, "\"");603 var it = mem.split(u8, line, "\"");
604 _ = it.next().?; // skip the stuff before the quote604 _ = it.next().?; // skip the stuff before the quote
605 const quoted = it.next().?; // the stuff inside the quote605 const quoted = it.next().?; // the stuff inside the quote
606 @field(ctx, mapping.field) = toNativePathSep(b, quoted);606 @field(ctx, mapping.field) = toNativePathSep(b, quoted);
lib/std/SemanticVersion.zig+5-5
...@@ -48,8 +48,8 @@ pub fn order(lhs: Version, rhs: Version) std.math.Order {...@@ -48,8 +48,8 @@ pub fn order(lhs: Version, rhs: Version) std.math.Order {
48 if (lhs.pre == null and rhs.pre != null) return .gt;48 if (lhs.pre == null and rhs.pre != null) return .gt;
4949
50 // Iterate over pre-release identifiers until a difference is found.50 // Iterate over pre-release identifiers until a difference is found.
51 var lhs_pre_it = std.mem.split(lhs.pre.?, ".");51 var lhs_pre_it = std.mem.split(u8, lhs.pre.?, ".");
52 var rhs_pre_it = std.mem.split(rhs.pre.?, ".");52 var rhs_pre_it = std.mem.split(u8, rhs.pre.?, ".");
53 while (true) {53 while (true) {
54 const next_lid = lhs_pre_it.next();54 const next_lid = lhs_pre_it.next();
55 const next_rid = rhs_pre_it.next();55 const next_rid = rhs_pre_it.next();
...@@ -92,7 +92,7 @@ pub fn parse(text: []const u8) !Version {...@@ -92,7 +92,7 @@ pub fn parse(text: []const u8) !Version {
92 // Parse the required major, minor, and patch numbers.92 // Parse the required major, minor, and patch numbers.
93 const extra_index = std.mem.indexOfAny(u8, text, "-+");93 const extra_index = std.mem.indexOfAny(u8, text, "-+");
94 const required = text[0..(extra_index orelse text.len)];94 const required = text[0..(extra_index orelse text.len)];
95 var it = std.mem.split(required, ".");95 var it = std.mem.split(u8, required, ".");
96 var ver = Version{96 var ver = Version{
97 .major = try parseNum(it.next() orelse return error.InvalidVersion),97 .major = try parseNum(it.next() orelse return error.InvalidVersion),
98 .minor = try parseNum(it.next() orelse return error.InvalidVersion),98 .minor = try parseNum(it.next() orelse return error.InvalidVersion),
...@@ -114,7 +114,7 @@ pub fn parse(text: []const u8) !Version {...@@ -114,7 +114,7 @@ pub fn parse(text: []const u8) !Version {
114 // Check validity of optional pre-release identifiers.114 // Check validity of optional pre-release identifiers.
115 // See: https://semver.org/#spec-item-9115 // See: https://semver.org/#spec-item-9
116 if (ver.pre) |pre| {116 if (ver.pre) |pre| {
117 it = std.mem.split(pre, ".");117 it = std.mem.split(u8, pre, ".");
118 while (it.next()) |id| {118 while (it.next()) |id| {
119 // Identifiers MUST NOT be empty.119 // Identifiers MUST NOT be empty.
120 if (id.len == 0) return error.InvalidVersion;120 if (id.len == 0) return error.InvalidVersion;
...@@ -133,7 +133,7 @@ pub fn parse(text: []const u8) !Version {...@@ -133,7 +133,7 @@ pub fn parse(text: []const u8) !Version {
133 // Check validity of optional build metadata identifiers.133 // Check validity of optional build metadata identifiers.
134 // See: https://semver.org/#spec-item-10134 // See: https://semver.org/#spec-item-10
135 if (ver.build) |build| {135 if (ver.build) |build| {
136 it = std.mem.split(build, ".");136 it = std.mem.split(u8, build, ".");
137 while (it.next()) |id| {137 while (it.next()) |id| {
138 // Identifiers MUST NOT be empty.138 // Identifiers MUST NOT be empty.
139 if (id.len == 0) return error.InvalidVersion;139 if (id.len == 0) return error.InvalidVersion;
lib/std/build.zig+4-4
...@@ -1085,7 +1085,7 @@ pub const Builder = struct {...@@ -1085,7 +1085,7 @@ pub const Builder = struct {
1085 if (fs.path.isAbsolute(name)) {1085 if (fs.path.isAbsolute(name)) {
1086 return name;1086 return name;
1087 }1087 }
1088 var it = mem.tokenize(PATH, &[_]u8{fs.path.delimiter});1088 var it = mem.tokenize(u8, PATH, &[_]u8{fs.path.delimiter});
1089 while (it.next()) |path| {1089 while (it.next()) |path| {
1090 const full_path = try fs.path.join(self.allocator, &[_][]const u8{1090 const full_path = try fs.path.join(self.allocator, &[_][]const u8{
1091 path,1091 path,
...@@ -1211,10 +1211,10 @@ pub const Builder = struct {...@@ -1211,10 +1211,10 @@ pub const Builder = struct {
1211 const stdout = try self.execAllowFail(&[_][]const u8{ "pkg-config", "--list-all" }, out_code, .Ignore);1211 const stdout = try self.execAllowFail(&[_][]const u8{ "pkg-config", "--list-all" }, out_code, .Ignore);
1212 var list = ArrayList(PkgConfigPkg).init(self.allocator);1212 var list = ArrayList(PkgConfigPkg).init(self.allocator);
1213 errdefer list.deinit();1213 errdefer list.deinit();
1214 var line_it = mem.tokenize(stdout, "\r\n");1214 var line_it = mem.tokenize(u8, stdout, "\r\n");
1215 while (line_it.next()) |line| {1215 while (line_it.next()) |line| {
1216 if (mem.trim(u8, line, " \t").len == 0) continue;1216 if (mem.trim(u8, line, " \t").len == 0) continue;
1217 var tok_it = mem.tokenize(line, " \t");1217 var tok_it = mem.tokenize(u8, line, " \t");
1218 try list.append(PkgConfigPkg{1218 try list.append(PkgConfigPkg{
1219 .name = tok_it.next() orelse return error.PkgConfigInvalidOutput,1219 .name = tok_it.next() orelse return error.PkgConfigInvalidOutput,
1220 .desc = tok_it.rest(),1220 .desc = tok_it.rest(),
...@@ -1872,7 +1872,7 @@ pub const LibExeObjStep = struct {...@@ -1872,7 +1872,7 @@ pub const LibExeObjStep = struct {
1872 error.FileNotFound => return error.PkgConfigNotInstalled,1872 error.FileNotFound => return error.PkgConfigNotInstalled,
1873 else => return err,1873 else => return err,
1874 };1874 };
1875 var it = mem.tokenize(stdout, " \r\n\t");1875 var it = mem.tokenize(u8, stdout, " \r\n\t");
1876 while (it.next()) |tok| {1876 while (it.next()) |tok| {
1877 if (mem.eql(u8, tok, "-I")) {1877 if (mem.eql(u8, tok, "-I")) {
1878 const dir = it.next() orelse return error.PkgConfigInvalidOutput;1878 const dir = it.next() orelse return error.PkgConfigInvalidOutput;
lib/std/builtin.zig+1-1
...@@ -509,7 +509,7 @@ pub const Version = struct {...@@ -509,7 +509,7 @@ pub const Version = struct {
509 // found no digits or '.' before unexpected character509 // found no digits or '.' before unexpected character
510 if (end == 0) return error.InvalidVersion;510 if (end == 0) return error.InvalidVersion;
511511
512 var it = std.mem.split(text[0..end], ".");512 var it = std.mem.split(u8, text[0..end], ".");
513 // substring is not empty, first call will succeed513 // substring is not empty, first call will succeed
514 const major = it.next().?;514 const major = it.next().?;
515 if (major.len == 0) return error.InvalidVersion;515 if (major.len == 0) return error.InvalidVersion;
lib/std/child_process.zig+2-2
...@@ -836,12 +836,12 @@ pub const ChildProcess = struct {...@@ -836,12 +836,12 @@ pub const ChildProcess = struct {
836836
837 const app_name = self.argv[0];837 const app_name = self.argv[0];
838838
839 var it = mem.tokenize(PATH, ";");839 var it = mem.tokenize(u8, PATH, ";");
840 retry: while (it.next()) |search_path| {840 retry: while (it.next()) |search_path| {
841 const path_no_ext = try fs.path.join(self.allocator, &[_][]const u8{ search_path, app_name });841 const path_no_ext = try fs.path.join(self.allocator, &[_][]const u8{ search_path, app_name });
842 defer self.allocator.free(path_no_ext);842 defer self.allocator.free(path_no_ext);
843843
844 var ext_it = mem.tokenize(PATHEXT, ";");844 var ext_it = mem.tokenize(u8, PATHEXT, ";");
845 while (ext_it.next()) |app_ext| {845 while (ext_it.next()) |app_ext| {
846 const joined_path = try mem.concat(self.allocator, u8, &[_][]const u8{ path_no_ext, app_ext });846 const joined_path = try mem.concat(self.allocator, u8, &[_][]const u8{ path_no_ext, app_ext });
847 defer self.allocator.free(joined_path);847 defer self.allocator.free(joined_path);
lib/std/fs.zig+1-1
...@@ -2455,7 +2455,7 @@ pub fn selfExePath(out_buffer: []u8) SelfExePathError![]u8 {...@@ -2455,7 +2455,7 @@ pub fn selfExePath(out_buffer: []u8) SelfExePathError![]u8 {
2455 } else if (argv0.len != 0) {2455 } else if (argv0.len != 0) {
2456 // argv[0] is not empty (and not a path): search it inside PATH2456 // argv[0] is not empty (and not a path): search it inside PATH
2457 const PATH = std.os.getenvZ("PATH") orelse return error.FileNotFound;2457 const PATH = std.os.getenvZ("PATH") orelse return error.FileNotFound;
2458 var path_it = mem.tokenize(PATH, &[_]u8{path.delimiter});2458 var path_it = mem.tokenize(u8, PATH, &[_]u8{path.delimiter});
2459 while (path_it.next()) |a_path| {2459 while (path_it.next()) |a_path| {
2460 var resolved_path_buf: [MAX_PATH_BYTES - 1:0]u8 = undefined;2460 var resolved_path_buf: [MAX_PATH_BYTES - 1:0]u8 = undefined;
2461 const resolved_path = std.fmt.bufPrintZ(&resolved_path_buf, "{s}/{s}", .{2461 const resolved_path = std.fmt.bufPrintZ(&resolved_path_buf, "{s}/{s}", .{
lib/std/fs/path.zig+13-13
...@@ -345,7 +345,7 @@ pub fn windowsParsePath(path: []const u8) WindowsPath {...@@ -345,7 +345,7 @@ pub fn windowsParsePath(path: []const u8) WindowsPath {
345 return relative_path;345 return relative_path;
346 }346 }
347347
348 var it = mem.tokenize(path, &[_]u8{this_sep});348 var it = mem.tokenize(u8, path, &[_]u8{this_sep});
349 _ = (it.next() orelse return relative_path);349 _ = (it.next() orelse return relative_path);
350 _ = (it.next() orelse return relative_path);350 _ = (it.next() orelse return relative_path);
351 return WindowsPath{351 return WindowsPath{
...@@ -407,8 +407,8 @@ fn networkShareServersEql(ns1: []const u8, ns2: []const u8) bool {...@@ -407,8 +407,8 @@ fn networkShareServersEql(ns1: []const u8, ns2: []const u8) bool {
407 const sep1 = ns1[0];407 const sep1 = ns1[0];
408 const sep2 = ns2[0];408 const sep2 = ns2[0];
409409
410 var it1 = mem.tokenize(ns1, &[_]u8{sep1});410 var it1 = mem.tokenize(u8, ns1, &[_]u8{sep1});
411 var it2 = mem.tokenize(ns2, &[_]u8{sep2});411 var it2 = mem.tokenize(u8, ns2, &[_]u8{sep2});
412412
413 // TODO ASCII is wrong, we actually need full unicode support to compare paths.413 // TODO ASCII is wrong, we actually need full unicode support to compare paths.
414 return asciiEqlIgnoreCase(it1.next().?, it2.next().?);414 return asciiEqlIgnoreCase(it1.next().?, it2.next().?);
...@@ -428,8 +428,8 @@ fn compareDiskDesignators(kind: WindowsPath.Kind, p1: []const u8, p2: []const u8...@@ -428,8 +428,8 @@ fn compareDiskDesignators(kind: WindowsPath.Kind, p1: []const u8, p2: []const u8
428 const sep1 = p1[0];428 const sep1 = p1[0];
429 const sep2 = p2[0];429 const sep2 = p2[0];
430430
431 var it1 = mem.tokenize(p1, &[_]u8{sep1});431 var it1 = mem.tokenize(u8, p1, &[_]u8{sep1});
432 var it2 = mem.tokenize(p2, &[_]u8{sep2});432 var it2 = mem.tokenize(u8, p2, &[_]u8{sep2});
433433
434 // TODO ASCII is wrong, we actually need full unicode support to compare paths.434 // TODO ASCII is wrong, we actually need full unicode support to compare paths.
435 return asciiEqlIgnoreCase(it1.next().?, it2.next().?) and asciiEqlIgnoreCase(it1.next().?, it2.next().?);435 return asciiEqlIgnoreCase(it1.next().?, it2.next().?) and asciiEqlIgnoreCase(it1.next().?, it2.next().?);
...@@ -551,7 +551,7 @@ pub fn resolveWindows(allocator: *Allocator, paths: []const []const u8) ![]u8 {...@@ -551,7 +551,7 @@ pub fn resolveWindows(allocator: *Allocator, paths: []const []const u8) ![]u8 {
551 },551 },
552 WindowsPath.Kind.NetworkShare => {552 WindowsPath.Kind.NetworkShare => {
553 result = try allocator.alloc(u8, max_size);553 result = try allocator.alloc(u8, max_size);
554 var it = mem.tokenize(paths[first_index], "/\\");554 var it = mem.tokenize(u8, paths[first_index], "/\\");
555 const server_name = it.next().?;555 const server_name = it.next().?;
556 const other_name = it.next().?;556 const other_name = it.next().?;
557557
...@@ -618,7 +618,7 @@ pub fn resolveWindows(allocator: *Allocator, paths: []const []const u8) ![]u8 {...@@ -618,7 +618,7 @@ pub fn resolveWindows(allocator: *Allocator, paths: []const []const u8) ![]u8 {
618 if (!correct_disk_designator) {618 if (!correct_disk_designator) {
619 continue;619 continue;
620 }620 }
621 var it = mem.tokenize(p[parsed.disk_designator.len..], "/\\");621 var it = mem.tokenize(u8, p[parsed.disk_designator.len..], "/\\");
622 while (it.next()) |component| {622 while (it.next()) |component| {
623 if (mem.eql(u8, component, ".")) {623 if (mem.eql(u8, component, ".")) {
624 continue;624 continue;
...@@ -687,7 +687,7 @@ pub fn resolvePosix(allocator: *Allocator, paths: []const []const u8) ![]u8 {...@@ -687,7 +687,7 @@ pub fn resolvePosix(allocator: *Allocator, paths: []const []const u8) ![]u8 {
687 errdefer allocator.free(result);687 errdefer allocator.free(result);
688688
689 for (paths[first_index..]) |p| {689 for (paths[first_index..]) |p| {
690 var it = mem.tokenize(p, "/");690 var it = mem.tokenize(u8, p, "/");
691 while (it.next()) |component| {691 while (it.next()) |component| {
692 if (mem.eql(u8, component, ".")) {692 if (mem.eql(u8, component, ".")) {
693 continue;693 continue;
...@@ -1101,8 +1101,8 @@ pub fn relativeWindows(allocator: *Allocator, from: []const u8, to: []const u8)...@@ -1101,8 +1101,8 @@ pub fn relativeWindows(allocator: *Allocator, from: []const u8, to: []const u8)
1101 return resolved_to;1101 return resolved_to;
1102 }1102 }
11031103
1104 var from_it = mem.tokenize(resolved_from, "/\\");1104 var from_it = mem.tokenize(u8, resolved_from, "/\\");
1105 var to_it = mem.tokenize(resolved_to, "/\\");1105 var to_it = mem.tokenize(u8, resolved_to, "/\\");
1106 while (true) {1106 while (true) {
1107 const from_component = from_it.next() orelse return allocator.dupe(u8, to_it.rest());1107 const from_component = from_it.next() orelse return allocator.dupe(u8, to_it.rest());
1108 const to_rest = to_it.rest();1108 const to_rest = to_it.rest();
...@@ -1131,7 +1131,7 @@ pub fn relativeWindows(allocator: *Allocator, from: []const u8, to: []const u8)...@@ -1131,7 +1131,7 @@ pub fn relativeWindows(allocator: *Allocator, from: []const u8, to: []const u8)
1131 // shave off the trailing slash1131 // shave off the trailing slash
1132 result_index -= 1;1132 result_index -= 1;
11331133
1134 var rest_it = mem.tokenize(to_rest, "/\\");1134 var rest_it = mem.tokenize(u8, to_rest, "/\\");
1135 while (rest_it.next()) |to_component| {1135 while (rest_it.next()) |to_component| {
1136 result[result_index] = '\\';1136 result[result_index] = '\\';
1137 result_index += 1;1137 result_index += 1;
...@@ -1152,8 +1152,8 @@ pub fn relativePosix(allocator: *Allocator, from: []const u8, to: []const u8) ![...@@ -1152,8 +1152,8 @@ pub fn relativePosix(allocator: *Allocator, from: []const u8, to: []const u8) ![
1152 const resolved_to = try resolvePosix(allocator, &[_][]const u8{to});1152 const resolved_to = try resolvePosix(allocator, &[_][]const u8{to});
1153 defer allocator.free(resolved_to);1153 defer allocator.free(resolved_to);
11541154
1155 var from_it = mem.tokenize(resolved_from, "/");1155 var from_it = mem.tokenize(u8, resolved_from, "/");
1156 var to_it = mem.tokenize(resolved_to, "/");1156 var to_it = mem.tokenize(u8, resolved_to, "/");
1157 while (true) {1157 while (true) {
1158 const from_component = from_it.next() orelse return allocator.dupe(u8, to_it.rest());1158 const from_component = from_it.next() orelse return allocator.dupe(u8, to_it.rest());
1159 const to_rest = to_it.rest();1159 const to_rest = to_it.rest();
lib/std/mem.zig+125-77
...@@ -1575,8 +1575,8 @@ test "bswapAllFields" {...@@ -1575,8 +1575,8 @@ test "bswapAllFields" {
1575/// If `delimiter_bytes` does not exist in buffer,1575/// If `delimiter_bytes` does not exist in buffer,
1576/// the iterator will return `buffer`, null, in that order.1576/// the iterator will return `buffer`, null, in that order.
1577/// See also the related function `split`.1577/// See also the related function `split`.
1578pub fn tokenize(buffer: []const u8, delimiter_bytes: []const u8) TokenIterator {1578pub fn tokenize(comptime T: type, buffer: []const T, delimiter_bytes: []const T) TokenIterator(T) {
1579 return TokenIterator{1579 return .{
1580 .index = 0,1580 .index = 0,
1581 .buffer = buffer,1581 .buffer = buffer,
1582 .delimiter_bytes = delimiter_bytes,1582 .delimiter_bytes = delimiter_bytes,
...@@ -1584,51 +1584,71 @@ pub fn tokenize(buffer: []const u8, delimiter_bytes: []const u8) TokenIterator {...@@ -1584,51 +1584,71 @@ pub fn tokenize(buffer: []const u8, delimiter_bytes: []const u8) TokenIterator {
1584}1584}
15851585
1586test "mem.tokenize" {1586test "mem.tokenize" {
1587 var it = tokenize(" abc def ghi ", " ");1587 var it = tokenize(u8, " abc def ghi ", " ");
1588 try testing.expect(eql(u8, it.next().?, "abc"));1588 try testing.expect(eql(u8, it.next().?, "abc"));
1589 try testing.expect(eql(u8, it.next().?, "def"));1589 try testing.expect(eql(u8, it.next().?, "def"));
1590 try testing.expect(eql(u8, it.next().?, "ghi"));1590 try testing.expect(eql(u8, it.next().?, "ghi"));
1591 try testing.expect(it.next() == null);1591 try testing.expect(it.next() == null);
15921592
1593 it = tokenize("..\\bob", "\\");1593 it = tokenize(u8, "..\\bob", "\\");
1594 try testing.expect(eql(u8, it.next().?, ".."));1594 try testing.expect(eql(u8, it.next().?, ".."));
1595 try testing.expect(eql(u8, "..", "..\\bob"[0..it.index]));1595 try testing.expect(eql(u8, "..", "..\\bob"[0..it.index]));
1596 try testing.expect(eql(u8, it.next().?, "bob"));1596 try testing.expect(eql(u8, it.next().?, "bob"));
1597 try testing.expect(it.next() == null);1597 try testing.expect(it.next() == null);
15981598
1599 it = tokenize("//a/b", "/");1599 it = tokenize(u8, "//a/b", "/");
1600 try testing.expect(eql(u8, it.next().?, "a"));1600 try testing.expect(eql(u8, it.next().?, "a"));
1601 try testing.expect(eql(u8, it.next().?, "b"));1601 try testing.expect(eql(u8, it.next().?, "b"));
1602 try testing.expect(eql(u8, "//a/b", "//a/b"[0..it.index]));1602 try testing.expect(eql(u8, "//a/b", "//a/b"[0..it.index]));
1603 try testing.expect(it.next() == null);1603 try testing.expect(it.next() == null);
16041604
1605 it = tokenize("|", "|");1605 it = tokenize(u8, "|", "|");
1606 try testing.expect(it.next() == null);1606 try testing.expect(it.next() == null);
16071607
1608 it = tokenize("", "|");1608 it = tokenize(u8, "", "|");
1609 try testing.expect(it.next() == null);1609 try testing.expect(it.next() == null);
16101610
1611 it = tokenize("hello", "");1611 it = tokenize(u8, "hello", "");
1612 try testing.expect(eql(u8, it.next().?, "hello"));1612 try testing.expect(eql(u8, it.next().?, "hello"));
1613 try testing.expect(it.next() == null);1613 try testing.expect(it.next() == null);
16141614
1615 it = tokenize("hello", " ");1615 it = tokenize(u8, "hello", " ");
1616 try testing.expect(eql(u8, it.next().?, "hello"));1616 try testing.expect(eql(u8, it.next().?, "hello"));
1617 try testing.expect(it.next() == null);1617 try testing.expect(it.next() == null);
1618
1619 var it16 = tokenize(
1620 u16,
1621 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
1622 std.unicode.utf8ToUtf16LeStringLiteral(" "),
1623 );
1624 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));
1625 try testing.expect(it16.next() == null);
1618}1626}
16191627
1620test "mem.tokenize (multibyte)" {1628test "mem.tokenize (multibyte)" {
1621 var it = tokenize("a|b,c/d e", " /,|");1629 var it = tokenize(u8, "a|b,c/d e", " /,|");
1622 try testing.expect(eql(u8, it.next().?, "a"));1630 try testing.expect(eql(u8, it.next().?, "a"));
1623 try testing.expect(eql(u8, it.next().?, "b"));1631 try testing.expect(eql(u8, it.next().?, "b"));
1624 try testing.expect(eql(u8, it.next().?, "c"));1632 try testing.expect(eql(u8, it.next().?, "c"));
1625 try testing.expect(eql(u8, it.next().?, "d"));1633 try testing.expect(eql(u8, it.next().?, "d"));
1626 try testing.expect(eql(u8, it.next().?, "e"));1634 try testing.expect(eql(u8, it.next().?, "e"));
1627 try testing.expect(it.next() == null);1635 try testing.expect(it.next() == null);
1636
1637 var it16 = tokenize(
1638 u16,
1639 std.unicode.utf8ToUtf16LeStringLiteral("a|b,c/d e"),
1640 std.unicode.utf8ToUtf16LeStringLiteral(" /,|"),
1641 );
1642 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a")));
1643 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b")));
1644 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c")));
1645 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d")));
1646 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("e")));
1647 try testing.expect(it16.next() == null);
1628}1648}
16291649
1630test "mem.tokenize (reset)" {1650test "mem.tokenize (reset)" {
1631 var it = tokenize(" abc def ghi ", " ");1651 var it = tokenize(u8, " abc def ghi ", " ");
1632 try testing.expect(eql(u8, it.next().?, "abc"));1652 try testing.expect(eql(u8, it.next().?, "abc"));
1633 try testing.expect(eql(u8, it.next().?, "def"));1653 try testing.expect(eql(u8, it.next().?, "def"));
1634 try testing.expect(eql(u8, it.next().?, "ghi"));1654 try testing.expect(eql(u8, it.next().?, "ghi"));
...@@ -1649,9 +1669,9 @@ test "mem.tokenize (reset)" {...@@ -1649,9 +1669,9 @@ test "mem.tokenize (reset)" {
1649/// the iterator will return `buffer`, null, in that order.1669/// the iterator will return `buffer`, null, in that order.
1650/// The delimiter length must not be zero.1670/// The delimiter length must not be zero.
1651/// See also the related function `tokenize`.1671/// See also the related function `tokenize`.
1652pub fn split(buffer: []const u8, delimiter: []const u8) SplitIterator {1672pub fn split(comptime T: type, buffer: []const T, delimiter: []const T) SplitIterator(T) {
1653 assert(delimiter.len != 0);1673 assert(delimiter.len != 0);
1654 return SplitIterator{1674 return .{
1655 .index = 0,1675 .index = 0,
1656 .buffer = buffer,1676 .buffer = buffer,
1657 .delimiter = delimiter,1677 .delimiter = delimiter,
...@@ -1661,35 +1681,55 @@ pub fn split(buffer: []const u8, delimiter: []const u8) SplitIterator {...@@ -1661,35 +1681,55 @@ pub fn split(buffer: []const u8, delimiter: []const u8) SplitIterator {
1661pub const separate = @compileError("deprecated: renamed to split (behavior remains unchanged)");1681pub const separate = @compileError("deprecated: renamed to split (behavior remains unchanged)");
16621682
1663test "mem.split" {1683test "mem.split" {
1664 var it = split("abc|def||ghi", "|");1684 var it = split(u8, "abc|def||ghi", "|");
1665 try testing.expect(eql(u8, it.next().?, "abc"));1685 try testing.expect(eql(u8, it.next().?, "abc"));
1666 try testing.expect(eql(u8, it.next().?, "def"));1686 try testing.expect(eql(u8, it.next().?, "def"));
1667 try testing.expect(eql(u8, it.next().?, ""));1687 try testing.expect(eql(u8, it.next().?, ""));
1668 try testing.expect(eql(u8, it.next().?, "ghi"));1688 try testing.expect(eql(u8, it.next().?, "ghi"));
1669 try testing.expect(it.next() == null);1689 try testing.expect(it.next() == null);
16701690
1671 it = split("", "|");1691 it = split(u8, "", "|");
1672 try testing.expect(eql(u8, it.next().?, ""));1692 try testing.expect(eql(u8, it.next().?, ""));
1673 try testing.expect(it.next() == null);1693 try testing.expect(it.next() == null);
16741694
1675 it = split("|", "|");1695 it = split(u8, "|", "|");
1676 try testing.expect(eql(u8, it.next().?, ""));1696 try testing.expect(eql(u8, it.next().?, ""));
1677 try testing.expect(eql(u8, it.next().?, ""));1697 try testing.expect(eql(u8, it.next().?, ""));
1678 try testing.expect(it.next() == null);1698 try testing.expect(it.next() == null);
16791699
1680 it = split("hello", " ");1700 it = split(u8, "hello", " ");
1681 try testing.expect(eql(u8, it.next().?, "hello"));1701 try testing.expect(eql(u8, it.next().?, "hello"));
1682 try testing.expect(it.next() == null);1702 try testing.expect(it.next() == null);
1703
1704 var it16 = split(
1705 u16,
1706 std.unicode.utf8ToUtf16LeStringLiteral("hello"),
1707 std.unicode.utf8ToUtf16LeStringLiteral(" "),
1708 );
1709 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("hello")));
1710 try testing.expect(it16.next() == null);
1683}1711}
16841712
1685test "mem.split (multibyte)" {1713test "mem.split (multibyte)" {
1686 var it = split("a, b ,, c, d, e", ", ");1714 var it = split(u8, "a, b ,, c, d, e", ", ");
1687 try testing.expect(eql(u8, it.next().?, "a"));1715 try testing.expect(eql(u8, it.next().?, "a"));
1688 try testing.expect(eql(u8, it.next().?, "b ,"));1716 try testing.expect(eql(u8, it.next().?, "b ,"));
1689 try testing.expect(eql(u8, it.next().?, "c"));1717 try testing.expect(eql(u8, it.next().?, "c"));
1690 try testing.expect(eql(u8, it.next().?, "d"));1718 try testing.expect(eql(u8, it.next().?, "d"));
1691 try testing.expect(eql(u8, it.next().?, "e"));1719 try testing.expect(eql(u8, it.next().?, "e"));
1692 try testing.expect(it.next() == null);1720 try testing.expect(it.next() == null);
1721
1722 var it16 = split(
1723 u16,
1724 std.unicode.utf8ToUtf16LeStringLiteral("a, b ,, c, d, e"),
1725 std.unicode.utf8ToUtf16LeStringLiteral(", "),
1726 );
1727 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("a")));
1728 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("b ,")));
1729 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("c")));
1730 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("d")));
1731 try testing.expect(eql(u16, it16.next().?, std.unicode.utf8ToUtf16LeStringLiteral("e")));
1732 try testing.expect(it16.next() == null);
1693}1733}
16941734
1695pub fn startsWith(comptime T: type, haystack: []const T, needle: []const T) bool {1735pub fn startsWith(comptime T: type, haystack: []const T, needle: []const T) bool {
...@@ -1710,75 +1750,83 @@ test "mem.endsWith" {...@@ -1710,75 +1750,83 @@ test "mem.endsWith" {
1710 try testing.expect(!endsWith(u8, "Bob", "Bo"));1750 try testing.expect(!endsWith(u8, "Bob", "Bo"));
1711}1751}
17121752
1713pub const TokenIterator = struct {1753pub fn TokenIterator(comptime T: type) type {
1714 buffer: []const u8,1754 return struct {
1715 delimiter_bytes: []const u8,1755 buffer: []const T,
1716 index: usize,1756 delimiter_bytes: []const T,
1757 index: usize,
17171758
1718 /// Returns a slice of the next token, or null if tokenization is complete.1759 const Self = @This();
1719 pub fn next(self: *TokenIterator) ?[]const u8 {
1720 // move to beginning of token
1721 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1722 const start = self.index;
1723 if (start == self.buffer.len) {
1724 return null;
1725 }
17261760
1727 // move to end of token1761 /// Returns a slice of the next token, or null if tokenization is complete.
1728 while (self.index < self.buffer.len and !self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}1762 pub fn next(self: *Self) ?[]const T {
1729 const end = self.index;1763 // move to beginning of token
1764 while (self.index < self.buffer.len and self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1765 const start = self.index;
1766 if (start == self.buffer.len) {
1767 return null;
1768 }
17301769
1731 return self.buffer[start..end];1770 // move to end of token
1732 }1771 while (self.index < self.buffer.len and !self.isSplitByte(self.buffer[self.index])) : (self.index += 1) {}
1772 const end = self.index;
17331773
1734 /// Returns a slice of the remaining bytes. Does not affect iterator state.1774 return self.buffer[start..end];
1735 pub fn rest(self: TokenIterator) []const u8 {1775 }
1736 // move to beginning of token
1737 var index: usize = self.index;
1738 while (index < self.buffer.len and self.isSplitByte(self.buffer[index])) : (index += 1) {}
1739 return self.buffer[index..];
1740 }
17411776
1742 /// Resets the iterator to the initial token.1777 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1743 pub fn reset(self: *TokenIterator) void {1778 pub fn rest(self: Self) []const T {
1744 self.index = 0;1779 // move to beginning of token
1745 }1780 var index: usize = self.index;
1781 while (index < self.buffer.len and self.isSplitByte(self.buffer[index])) : (index += 1) {}
1782 return self.buffer[index..];
1783 }
1784
1785 /// Resets the iterator to the initial token.
1786 pub fn reset(self: *Self) void {
1787 self.index = 0;
1788 }
17461789
1747 fn isSplitByte(self: TokenIterator, byte: u8) bool {1790 fn isSplitByte(self: Self, byte: T) bool {
1748 for (self.delimiter_bytes) |delimiter_byte| {1791 for (self.delimiter_bytes) |delimiter_byte| {
1749 if (byte == delimiter_byte) {1792 if (byte == delimiter_byte) {
1750 return true;1793 return true;
1794 }
1751 }1795 }
1796 return false;
1752 }1797 }
1753 return false;1798 };
1754 }1799}
1755};
17561800
1757pub const SplitIterator = struct {1801pub fn SplitIterator(comptime T: type) type {
1758 buffer: []const u8,1802 return struct {
1759 index: ?usize,1803 buffer: []const T,
1760 delimiter: []const u8,1804 index: ?usize,
17611805 delimiter: []const T,
1762 /// Returns a slice of the next field, or null if splitting is complete.
1763 pub fn next(self: *SplitIterator) ?[]const u8 {
1764 const start = self.index orelse return null;
1765 const end = if (indexOfPos(u8, self.buffer, start, self.delimiter)) |delim_start| blk: {
1766 self.index = delim_start + self.delimiter.len;
1767 break :blk delim_start;
1768 } else blk: {
1769 self.index = null;
1770 break :blk self.buffer.len;
1771 };
1772 return self.buffer[start..end];
1773 }
17741806
1775 /// Returns a slice of the remaining bytes. Does not affect iterator state.1807 const Self = @This();
1776 pub fn rest(self: SplitIterator) []const u8 {1808
1777 const end = self.buffer.len;1809 /// Returns a slice of the next field, or null if splitting is complete.
1778 const start = self.index orelse end;1810 pub fn next(self: *Self) ?[]const T {
1779 return self.buffer[start..end];1811 const start = self.index orelse return null;
1780 }1812 const end = if (indexOfPos(T, self.buffer, start, self.delimiter)) |delim_start| blk: {
1781};1813 self.index = delim_start + self.delimiter.len;
1814 break :blk delim_start;
1815 } else blk: {
1816 self.index = null;
1817 break :blk self.buffer.len;
1818 };
1819 return self.buffer[start..end];
1820 }
1821
1822 /// Returns a slice of the remaining bytes. Does not affect iterator state.
1823 pub fn rest(self: Self) []const T {
1824 const end = self.buffer.len;
1825 const start = self.index orelse end;
1826 return self.buffer[start..end];
1827 }
1828 };
1829}
17821830
1783/// Naively combines a series of slices with a separator.1831/// Naively combines a series of slices with a separator.
1784/// Allocates memory for the result, which must be freed by the caller.1832/// Allocates memory for the result, which must be freed by the caller.
lib/std/net.zig+6-6
...@@ -1130,9 +1130,9 @@ fn linuxLookupNameFromHosts(...@@ -1130,9 +1130,9 @@ fn linuxLookupNameFromHosts(
1130 },1130 },
1131 else => |e| return e,1131 else => |e| return e,
1132 }) |line| {1132 }) |line| {
1133 const no_comment_line = mem.split(line, "#").next().?;1133 const no_comment_line = mem.split(u8, line, "#").next().?;
11341134
1135 var line_it = mem.tokenize(no_comment_line, " \t");1135 var line_it = mem.tokenize(u8, no_comment_line, " \t");
1136 const ip_text = line_it.next() orelse continue;1136 const ip_text = line_it.next() orelse continue;
1137 var first_name_text: ?[]const u8 = null;1137 var first_name_text: ?[]const u8 = null;
1138 while (line_it.next()) |name_text| {1138 while (line_it.next()) |name_text| {
...@@ -1211,7 +1211,7 @@ fn linuxLookupNameFromDnsSearch(...@@ -1211,7 +1211,7 @@ fn linuxLookupNameFromDnsSearch(
1211 mem.copy(u8, canon.items, canon_name);1211 mem.copy(u8, canon.items, canon_name);
1212 try canon.append('.');1212 try canon.append('.');
12131213
1214 var tok_it = mem.tokenize(search, " \t");1214 var tok_it = mem.tokenize(u8, search, " \t");
1215 while (tok_it.next()) |tok| {1215 while (tok_it.next()) |tok| {
1216 canon.shrinkRetainingCapacity(canon_name.len + 1);1216 canon.shrinkRetainingCapacity(canon_name.len + 1);
1217 try canon.appendSlice(tok);1217 try canon.appendSlice(tok);
...@@ -1328,13 +1328,13 @@ fn getResolvConf(allocator: *mem.Allocator, rc: *ResolvConf) !void {...@@ -1328,13 +1328,13 @@ fn getResolvConf(allocator: *mem.Allocator, rc: *ResolvConf) !void {
1328 },1328 },
1329 else => |e| return e,1329 else => |e| return e,
1330 }) |line| {1330 }) |line| {
1331 const no_comment_line = mem.split(line, "#").next().?;1331 const no_comment_line = mem.split(u8, line, "#").next().?;
1332 var line_it = mem.tokenize(no_comment_line, " \t");1332 var line_it = mem.tokenize(u8, no_comment_line, " \t");
13331333
1334 const token = line_it.next() orelse continue;1334 const token = line_it.next() orelse continue;
1335 if (mem.eql(u8, token, "options")) {1335 if (mem.eql(u8, token, "options")) {
1336 while (line_it.next()) |sub_tok| {1336 while (line_it.next()) |sub_tok| {
1337 var colon_it = mem.split(sub_tok, ":");1337 var colon_it = mem.split(u8, sub_tok, ":");
1338 const name = colon_it.next().?;1338 const name = colon_it.next().?;
1339 const value_txt = colon_it.next() orelse continue;1339 const value_txt = colon_it.next() orelse continue;
1340 const value = std.fmt.parseInt(u8, value_txt, 10) catch |err| switch (err) {1340 const value = std.fmt.parseInt(u8, value_txt, 10) catch |err| switch (err) {
lib/std/os.zig+1-1
...@@ -1378,7 +1378,7 @@ pub fn execvpeZ_expandArg0(...@@ -1378,7 +1378,7 @@ pub fn execvpeZ_expandArg0(
1378 // Use of MAX_PATH_BYTES here is valid as the path_buf will be passed1378 // Use of MAX_PATH_BYTES here is valid as the path_buf will be passed
1379 // directly to the operating system in execveZ.1379 // directly to the operating system in execveZ.
1380 var path_buf: [MAX_PATH_BYTES]u8 = undefined;1380 var path_buf: [MAX_PATH_BYTES]u8 = undefined;
1381 var it = mem.tokenize(PATH, ":");1381 var it = mem.tokenize(u8, PATH, ":");
1382 var seen_eacces = false;1382 var seen_eacces = false;
1383 var err: ExecveError = undefined;1383 var err: ExecveError = undefined;
13841384
lib/std/process.zig+1-1
...@@ -109,7 +109,7 @@ pub fn getEnvMap(allocator: *Allocator) !BufMap {...@@ -109,7 +109,7 @@ pub fn getEnvMap(allocator: *Allocator) !BufMap {
109109
110 for (environ) |env| {110 for (environ) |env| {
111 const pair = mem.spanZ(env);111 const pair = mem.spanZ(env);
112 var parts = mem.split(pair, "=");112 var parts = mem.split(u8, pair, "=");
113 const key = parts.next().?;113 const key = parts.next().?;
114 const value = parts.next().?;114 const value = parts.next().?;
115 try result.put(key, value);115 try result.put(key, value);
lib/std/zig/cross_target.zig+5-5
...@@ -233,7 +233,7 @@ pub const CrossTarget = struct {...@@ -233,7 +233,7 @@ pub const CrossTarget = struct {
233 .dynamic_linker = DynamicLinker.init(args.dynamic_linker),233 .dynamic_linker = DynamicLinker.init(args.dynamic_linker),
234 };234 };
235235
236 var it = mem.split(args.arch_os_abi, "-");236 var it = mem.split(u8, args.arch_os_abi, "-");
237 const arch_name = it.next().?;237 const arch_name = it.next().?;
238 const arch_is_native = mem.eql(u8, arch_name, "native");238 const arch_is_native = mem.eql(u8, arch_name, "native");
239 if (!arch_is_native) {239 if (!arch_is_native) {
...@@ -251,7 +251,7 @@ pub const CrossTarget = struct {...@@ -251,7 +251,7 @@ pub const CrossTarget = struct {
251251
252 const opt_abi_text = it.next();252 const opt_abi_text = it.next();
253 if (opt_abi_text) |abi_text| {253 if (opt_abi_text) |abi_text| {
254 var abi_it = mem.split(abi_text, ".");254 var abi_it = mem.split(u8, abi_text, ".");
255 const abi = std.meta.stringToEnum(Target.Abi, abi_it.next().?) orelse255 const abi = std.meta.stringToEnum(Target.Abi, abi_it.next().?) orelse
256 return error.UnknownApplicationBinaryInterface;256 return error.UnknownApplicationBinaryInterface;
257 result.abi = abi;257 result.abi = abi;
...@@ -699,7 +699,7 @@ pub const CrossTarget = struct {...@@ -699,7 +699,7 @@ pub const CrossTarget = struct {
699 }699 }
700700
701 fn parseOs(result: *CrossTarget, diags: *ParseOptions.Diagnostics, text: []const u8) !void {701 fn parseOs(result: *CrossTarget, diags: *ParseOptions.Diagnostics, text: []const u8) !void {
702 var it = mem.split(text, ".");702 var it = mem.split(u8, text, ".");
703 const os_name = it.next().?;703 const os_name = it.next().?;
704 diags.os_name = os_name;704 diags.os_name = os_name;
705 const os_is_native = mem.eql(u8, os_name, "native");705 const os_is_native = mem.eql(u8, os_name, "native");
...@@ -757,7 +757,7 @@ pub const CrossTarget = struct {...@@ -757,7 +757,7 @@ pub const CrossTarget = struct {
757 .linux,757 .linux,
758 .dragonfly,758 .dragonfly,
759 => {759 => {
760 var range_it = mem.split(version_text, "...");760 var range_it = mem.split(u8, version_text, "...");
761761
762 const min_text = range_it.next().?;762 const min_text = range_it.next().?;
763 const min_ver = SemVer.parse(min_text) catch |err| switch (err) {763 const min_ver = SemVer.parse(min_text) catch |err| switch (err) {
...@@ -777,7 +777,7 @@ pub const CrossTarget = struct {...@@ -777,7 +777,7 @@ pub const CrossTarget = struct {
777 },777 },
778778
779 .windows => {779 .windows => {
780 var range_it = mem.split(version_text, "...");780 var range_it = mem.split(u8, version_text, "...");
781781
782 const min_text = range_it.next().?;782 const min_text = range_it.next().?;
783 const min_ver = std.meta.stringToEnum(Target.Os.WindowsVersion, min_text) orelse783 const min_ver = std.meta.stringToEnum(Target.Os.WindowsVersion, min_text) orelse
lib/std/zig/system.zig+3-3
...@@ -44,7 +44,7 @@ pub const NativePaths = struct {...@@ -44,7 +44,7 @@ pub const NativePaths = struct {
44 defer allocator.free(nix_cflags_compile);44 defer allocator.free(nix_cflags_compile);
4545
46 is_nix = true;46 is_nix = true;
47 var it = mem.tokenize(nix_cflags_compile, " ");47 var it = mem.tokenize(u8, nix_cflags_compile, " ");
48 while (true) {48 while (true) {
49 const word = it.next() orelse break;49 const word = it.next() orelse break;
50 if (mem.eql(u8, word, "-isystem")) {50 if (mem.eql(u8, word, "-isystem")) {
...@@ -69,7 +69,7 @@ pub const NativePaths = struct {...@@ -69,7 +69,7 @@ pub const NativePaths = struct {
69 defer allocator.free(nix_ldflags);69 defer allocator.free(nix_ldflags);
7070
71 is_nix = true;71 is_nix = true;
72 var it = mem.tokenize(nix_ldflags, " ");72 var it = mem.tokenize(u8, nix_ldflags, " ");
73 while (true) {73 while (true) {
74 const word = it.next() orelse break;74 const word = it.next() orelse break;
75 if (mem.eql(u8, word, "-rpath")) {75 if (mem.eql(u8, word, "-rpath")) {
...@@ -839,7 +839,7 @@ pub const NativeTargetInfo = struct {...@@ -839,7 +839,7 @@ pub const NativeTargetInfo = struct {
839 error.Overflow => return error.InvalidElfFile,839 error.Overflow => return error.InvalidElfFile,
840 };840 };
841 const rpath_list = mem.spanZ(std.meta.assumeSentinel(strtab[rpoff_usize..].ptr, 0));841 const rpath_list = mem.spanZ(std.meta.assumeSentinel(strtab[rpoff_usize..].ptr, 0));
842 var it = mem.tokenize(rpath_list, ":");842 var it = mem.tokenize(u8, rpath_list, ":");
843 while (it.next()) |rpath| {843 while (it.next()) |rpath| {
844 var dir = fs.cwd().openDir(rpath, .{}) catch |err| switch (err) {844 var dir = fs.cwd().openDir(rpath, .{}) catch |err| switch (err) {
845 error.NameTooLong => unreachable,845 error.NameTooLong => unreachable,
src/Cache.zig+2-2
...@@ -356,7 +356,7 @@ pub const Manifest = struct {...@@ -356,7 +356,7 @@ pub const Manifest = struct {
356356
357 const input_file_count = self.files.items.len;357 const input_file_count = self.files.items.len;
358 var any_file_changed = false;358 var any_file_changed = false;
359 var line_iter = mem.tokenize(file_contents, "\n");359 var line_iter = mem.tokenize(u8, file_contents, "\n");
360 var idx: usize = 0;360 var idx: usize = 0;
361 while (line_iter.next()) |line| {361 while (line_iter.next()) |line| {
362 defer idx += 1;362 defer idx += 1;
...@@ -373,7 +373,7 @@ pub const Manifest = struct {...@@ -373,7 +373,7 @@ pub const Manifest = struct {
373 break :blk new;373 break :blk new;
374 };374 };
375375
376 var iter = mem.tokenize(line, " ");376 var iter = mem.tokenize(u8, line, " ");
377 const size = iter.next() orelse return error.InvalidFormat;377 const size = iter.next() orelse return error.InvalidFormat;
378 const inode = iter.next() orelse return error.InvalidFormat;378 const inode = iter.next() orelse return error.InvalidFormat;
379 const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;379 const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
src/Compilation.zig+2-2
...@@ -3341,7 +3341,7 @@ pub fn hasSharedLibraryExt(filename: []const u8) bool {...@@ -3341,7 +3341,7 @@ pub fn hasSharedLibraryExt(filename: []const u8) bool {
3341 return true;3341 return true;
3342 }3342 }
3343 // Look for .so.X, .so.X.Y, .so.X.Y.Z3343 // Look for .so.X, .so.X.Y, .so.X.Y.Z
3344 var it = mem.split(filename, ".");3344 var it = mem.split(u8, filename, ".");
3345 _ = it.next().?;3345 _ = it.next().?;
3346 var so_txt = it.next() orelse return false;3346 var so_txt = it.next() orelse return false;
3347 while (!mem.eql(u8, so_txt, "so")) {3347 while (!mem.eql(u8, so_txt, "so")) {
...@@ -4086,7 +4086,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node...@@ -4086,7 +4086,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
4086 };4086 };
40874087
4088 if (directory.handle.readFileAlloc(comp.gpa, libs_txt_basename, 10 * 1024 * 1024)) |libs_txt| {4088 if (directory.handle.readFileAlloc(comp.gpa, libs_txt_basename, 10 * 1024 * 1024)) |libs_txt| {
4089 var it = mem.tokenize(libs_txt, "\n");4089 var it = mem.tokenize(u8, libs_txt, "\n");
4090 while (it.next()) |lib_name| {4090 while (it.next()) |lib_name| {
4091 try comp.stage1AddLinkLib(lib_name);4091 try comp.stage1AddLinkLib(lib_name);
4092 }4092 }
src/codegen.zig+1-1
...@@ -3656,7 +3656,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {...@@ -3656,7 +3656,7 @@ fn Function(comptime arch: std.Target.Cpu.Arch) type {
3656 }3656 }
36573657
3658 {3658 {
3659 var iter = std.mem.tokenize(asm_source, "\n\r");3659 var iter = std.mem.tokenize(u8, asm_source, "\n\r");
3660 while (iter.next()) |ins| {3660 while (iter.next()) |ins| {
3661 if (mem.eql(u8, ins, "syscall")) {3661 if (mem.eql(u8, ins, "syscall")) {
3662 try self.code.appendSlice(&[_]u8{ 0x0f, 0x05 });3662 try self.code.appendSlice(&[_]u8{ 0x0f, 0x05 });
src/glibc.zig+7-7
...@@ -107,7 +107,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!...@@ -107,7 +107,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
107 defer gpa.free(abi_txt_contents);107 defer gpa.free(abi_txt_contents);
108108
109 {109 {
110 var it = mem.tokenize(vers_txt_contents, "\r\n");110 var it = mem.tokenize(u8, vers_txt_contents, "\r\n");
111 var line_i: usize = 1;111 var line_i: usize = 1;
112 while (it.next()) |line| : (line_i += 1) {112 while (it.next()) |line| : (line_i += 1) {
113 const prefix = "GLIBC_";113 const prefix = "GLIBC_";
...@@ -124,10 +124,10 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!...@@ -124,10 +124,10 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
124 }124 }
125 }125 }
126 {126 {
127 var file_it = mem.tokenize(fns_txt_contents, "\r\n");127 var file_it = mem.tokenize(u8, fns_txt_contents, "\r\n");
128 var line_i: usize = 1;128 var line_i: usize = 1;
129 while (file_it.next()) |line| : (line_i += 1) {129 while (file_it.next()) |line| : (line_i += 1) {
130 var line_it = mem.tokenize(line, " ");130 var line_it = mem.tokenize(u8, line, " ");
131 const fn_name = line_it.next() orelse {131 const fn_name = line_it.next() orelse {
132 std.log.err("fns.txt:{d}: expected function name", .{line_i});132 std.log.err("fns.txt:{d}: expected function name", .{line_i});
133 return error.ZigInstallationCorrupt;133 return error.ZigInstallationCorrupt;
...@@ -147,7 +147,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!...@@ -147,7 +147,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
147 }147 }
148 }148 }
149 {149 {
150 var file_it = mem.split(abi_txt_contents, "\n");150 var file_it = mem.split(u8, abi_txt_contents, "\n");
151 var line_i: usize = 0;151 var line_i: usize = 0;
152 while (true) {152 while (true) {
153 const ver_list_base: []VerList = blk: {153 const ver_list_base: []VerList = blk: {
...@@ -155,9 +155,9 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!...@@ -155,9 +155,9 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
155 if (line.len == 0) break;155 if (line.len == 0) break;
156 line_i += 1;156 line_i += 1;
157 const ver_list_base = try arena.alloc(VerList, all_functions.items.len);157 const ver_list_base = try arena.alloc(VerList, all_functions.items.len);
158 var line_it = mem.tokenize(line, " ");158 var line_it = mem.tokenize(u8, line, " ");
159 while (line_it.next()) |target_string| {159 while (line_it.next()) |target_string| {
160 var component_it = mem.tokenize(target_string, "-");160 var component_it = mem.tokenize(u8, target_string, "-");
161 const arch_name = component_it.next() orelse {161 const arch_name = component_it.next() orelse {
162 std.log.err("abi.txt:{d}: expected arch name", .{line_i});162 std.log.err("abi.txt:{d}: expected arch name", .{line_i});
163 return error.ZigInstallationCorrupt;163 return error.ZigInstallationCorrupt;
...@@ -203,7 +203,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!...@@ -203,7 +203,7 @@ pub fn loadMetaData(gpa: *Allocator, zig_lib_dir: std.fs.Dir) LoadMetaDataError!
203 .versions = undefined,203 .versions = undefined,
204 .len = 0,204 .len = 0,
205 };205 };
206 var line_it = mem.tokenize(line, " ");206 var line_it = mem.tokenize(u8, line, " ");
207 while (line_it.next()) |version_index_string| {207 while (line_it.next()) |version_index_string| {
208 if (ver_list.len >= ver_list.versions.len) {208 if (ver_list.len >= ver_list.versions.len) {
209 // If this happens with legit data, increase the array len in the type.209 // If this happens with legit data, increase the array len in the type.
src/libc_installation.zig+5-5
...@@ -60,10 +60,10 @@ pub const LibCInstallation = struct {...@@ -60,10 +60,10 @@ pub const LibCInstallation = struct {
60 const contents = try std.fs.cwd().readFileAlloc(allocator, libc_file, std.math.maxInt(usize));60 const contents = try std.fs.cwd().readFileAlloc(allocator, libc_file, std.math.maxInt(usize));
61 defer allocator.free(contents);61 defer allocator.free(contents);
6262
63 var it = std.mem.tokenize(contents, "\n");63 var it = std.mem.tokenize(u8, contents, "\n");
64 while (it.next()) |line| {64 while (it.next()) |line| {
65 if (line.len == 0 or line[0] == '#') continue;65 if (line.len == 0 or line[0] == '#') continue;
66 var line_it = std.mem.split(line, "=");66 var line_it = std.mem.split(u8, line, "=");
67 const name = line_it.next() orelse {67 const name = line_it.next() orelse {
68 log.err("missing equal sign after field name\n", .{});68 log.err("missing equal sign after field name\n", .{});
69 return error.ParseError;69 return error.ParseError;
...@@ -298,7 +298,7 @@ pub const LibCInstallation = struct {...@@ -298,7 +298,7 @@ pub const LibCInstallation = struct {
298 },298 },
299 }299 }
300300
301 var it = std.mem.tokenize(exec_res.stderr, "\n\r");301 var it = std.mem.tokenize(u8, exec_res.stderr, "\n\r");
302 var search_paths = std.ArrayList([]const u8).init(allocator);302 var search_paths = std.ArrayList([]const u8).init(allocator);
303 defer search_paths.deinit();303 defer search_paths.deinit();
304 while (it.next()) |line| {304 while (it.next()) |line| {
...@@ -616,7 +616,7 @@ fn ccPrintFileName(args: CCPrintFileNameOptions) ![:0]u8 {...@@ -616,7 +616,7 @@ fn ccPrintFileName(args: CCPrintFileNameOptions) ![:0]u8 {
616 },616 },
617 }617 }
618618
619 var it = std.mem.tokenize(exec_res.stdout, "\n\r");619 var it = std.mem.tokenize(u8, exec_res.stdout, "\n\r");
620 const line = it.next() orelse return error.LibCRuntimeNotFound;620 const line = it.next() orelse return error.LibCRuntimeNotFound;
621 // When this command fails, it returns exit code 0 and duplicates the input file name.621 // When this command fails, it returns exit code 0 and duplicates the input file name.
622 // So we detect failure by checking if the output matches exactly the input.622 // So we detect failure by checking if the output matches exactly the input.
...@@ -695,7 +695,7 @@ fn appendCcExe(args: *std.ArrayList([]const u8), skip_cc_env_var: bool) !void {...@@ -695,7 +695,7 @@ fn appendCcExe(args: *std.ArrayList([]const u8), skip_cc_env_var: bool) !void {
695 return;695 return;
696 };696 };
697 // Respect space-separated flags to the C compiler.697 // Respect space-separated flags to the C compiler.
698 var it = std.mem.tokenize(cc_env_var, " ");698 var it = std.mem.tokenize(u8, cc_env_var, " ");
699 while (it.next()) |arg| {699 while (it.next()) |arg| {
700 try args.append(arg);700 try args.append(arg);
701 }701 }
src/link/MachO/Dylib.zig+1-1
...@@ -106,7 +106,7 @@ pub const Id = struct {...@@ -106,7 +106,7 @@ pub const Id = struct {
106 var out: u32 = 0;106 var out: u32 = 0;
107 var values: [3][]const u8 = undefined;107 var values: [3][]const u8 = undefined;
108108
109 var split = mem.split(string, ".");109 var split = mem.split(u8, string, ".");
110 var count: u4 = 0;110 var count: u4 = 0;
111 while (split.next()) |value| {111 while (split.next()) |value| {
112 if (count > 2) {112 if (count > 2) {
src/main.zig+2-2
...@@ -1200,7 +1200,7 @@ fn buildOutputType(...@@ -1200,7 +1200,7 @@ fn buildOutputType(
1200 },1200 },
1201 .rdynamic => rdynamic = true,1201 .rdynamic => rdynamic = true,
1202 .wl => {1202 .wl => {
1203 var split_it = mem.split(it.only_arg, ",");1203 var split_it = mem.split(u8, it.only_arg, ",");
1204 while (split_it.next()) |linker_arg| {1204 while (split_it.next()) |linker_arg| {
1205 // Handle nested-joined args like `-Wl,-rpath=foo`.1205 // Handle nested-joined args like `-Wl,-rpath=foo`.
1206 // Must be prefixed with 1 or 2 dashes.1206 // Must be prefixed with 1 or 2 dashes.
...@@ -3655,7 +3655,7 @@ pub const ClangArgIterator = struct {...@@ -3655,7 +3655,7 @@ pub const ClangArgIterator = struct {
3655 defer allocator.free(resp_contents);3655 defer allocator.free(resp_contents);
3656 // TODO is there a specification for this file format? Let's find it and make this parsing more robust3656 // TODO is there a specification for this file format? Let's find it and make this parsing more robust
3657 // at the very least I'm guessing this needs to handle quotes and `#` comments.3657 // at the very least I'm guessing this needs to handle quotes and `#` comments.
3658 var it = mem.tokenize(resp_contents, " \t\r\n");3658 var it = mem.tokenize(u8, resp_contents, " \t\r\n");
3659 var resp_arg_list = std.ArrayList([]const u8).init(allocator);3659 var resp_arg_list = std.ArrayList([]const u8).init(allocator);
3660 defer resp_arg_list.deinit();3660 defer resp_arg_list.deinit();
3661 {3661 {
src/test.zig+2-2
...@@ -228,7 +228,7 @@ pub const TestContext = struct {...@@ -228,7 +228,7 @@ pub const TestContext = struct {
228 continue;228 continue;
229 }229 }
230 // example: "file.zig:1:2: error: bad thing happened"230 // example: "file.zig:1:2: error: bad thing happened"
231 var it = std.mem.split(err_msg_line, ":");231 var it = std.mem.split(u8, err_msg_line, ":");
232 const src_path = it.next() orelse @panic("missing colon");232 const src_path = it.next() orelse @panic("missing colon");
233 const line_text = it.next() orelse @panic("missing line");233 const line_text = it.next() orelse @panic("missing line");
234 const col_text = it.next() orelse @panic("missing column");234 const col_text = it.next() orelse @panic("missing column");
...@@ -779,7 +779,7 @@ pub const TestContext = struct {...@@ -779,7 +779,7 @@ pub const TestContext = struct {
779 }779 }
780 var ok = true;780 var ok = true;
781 if (case.expect_exact) {781 if (case.expect_exact) {
782 var err_iter = std.mem.split(result.stderr, "\n");782 var err_iter = std.mem.split(u8, result.stderr, "\n");
783 var i: usize = 0;783 var i: usize = 0;
784 ok = while (err_iter.next()) |line| : (i += 1) {784 ok = while (err_iter.next()) |line| : (i += 1) {
785 if (i >= case_error_list.len) break false;785 if (i >= case_error_list.len) break false;
test/behavior/bugs/6456.zig+1-1
...@@ -13,7 +13,7 @@ test "issue 6456" {...@@ -13,7 +13,7 @@ test "issue 6456" {
13 comptime {13 comptime {
14 var fields: []const StructField = &[0]StructField{};14 var fields: []const StructField = &[0]StructField{};
1515
16 var it = std.mem.tokenize(text, "\n");16 var it = std.mem.tokenize(u8, text, "\n");
17 while (it.next()) |name| {17 while (it.next()) |name| {
18 fields = fields ++ &[_]StructField{StructField{18 fields = fields ++ &[_]StructField{StructField{
19 .alignment = 0,19 .alignment = 0,
test/tests.zig+1-1
...@@ -768,7 +768,7 @@ pub const StackTracesContext = struct {...@@ -768,7 +768,7 @@ pub const StackTracesContext = struct {
768 var buf = ArrayList(u8).init(b.allocator);768 var buf = ArrayList(u8).init(b.allocator);
769 defer buf.deinit();769 defer buf.deinit();
770 if (stderr.len != 0 and stderr[stderr.len - 1] == '\n') stderr = stderr[0 .. stderr.len - 1];770 if (stderr.len != 0 and stderr[stderr.len - 1] == '\n') stderr = stderr[0 .. stderr.len - 1];
771 var it = mem.split(stderr, "\n");771 var it = mem.split(u8, stderr, "\n");
772 process_lines: while (it.next()) |line| {772 process_lines: while (it.next()) |line| {
773 if (line.len == 0) continue;773 if (line.len == 0) continue;
774774
tools/update_glibc.zig+4-4
...@@ -188,9 +188,9 @@ pub fn main() !void {...@@ -188,9 +188,9 @@ pub fn main() !void {
188 std.debug.warn("unable to open {s}: {}\n", .{ abi_list_filename, err });188 std.debug.warn("unable to open {s}: {}\n", .{ abi_list_filename, err });
189 std.process.exit(1);189 std.process.exit(1);
190 };190 };
191 var lines_it = std.mem.tokenize(contents, "\n");191 var lines_it = std.mem.tokenize(u8, contents, "\n");
192 while (lines_it.next()) |line| {192 while (lines_it.next()) |line| {
193 var tok_it = std.mem.tokenize(line, " ");193 var tok_it = std.mem.tokenize(u8, line, " ");
194 const ver = tok_it.next().?;194 const ver = tok_it.next().?;
195 const name = tok_it.next().?;195 const name = tok_it.next().?;
196 const category = tok_it.next().?;196 const category = tok_it.next().?;
...@@ -319,8 +319,8 @@ pub fn strCmpLessThan(context: void, a: []const u8, b: []const u8) bool {...@@ -319,8 +319,8 @@ pub fn strCmpLessThan(context: void, a: []const u8, b: []const u8) bool {
319pub fn versionLessThan(context: void, a: []const u8, b: []const u8) bool {319pub fn versionLessThan(context: void, a: []const u8, b: []const u8) bool {
320 _ = context;320 _ = context;
321 const sep_chars = "GLIBC_.";321 const sep_chars = "GLIBC_.";
322 var a_tokens = std.mem.tokenize(a, sep_chars);322 var a_tokens = std.mem.tokenize(u8, a, sep_chars);
323 var b_tokens = std.mem.tokenize(b, sep_chars);323 var b_tokens = std.mem.tokenize(u8, b, sep_chars);
324324
325 while (true) {325 while (true) {
326 const a_next = a_tokens.next();326 const a_next = a_tokens.next();
tools/update_spirv_features.zig+1-1
...@@ -19,7 +19,7 @@ const Version = struct {...@@ -19,7 +19,7 @@ const Version = struct {
19 minor: u32,19 minor: u32,
2020
21 fn parse(str: []const u8) !Version {21 fn parse(str: []const u8) !Version {
22 var it = std.mem.split(str, ".");22 var it = std.mem.split(u8, str, ".");
2323
24 const major = it.next() orelse return error.InvalidVersion;24 const major = it.next() orelse return error.InvalidVersion;
25 const minor = it.next() orelse return error.InvalidVersion;25 const minor = it.next() orelse return error.InvalidVersion;