Commit 841b38aae8913972d12b08cf3e0e497be305efb1 | zig

Line comments do not produce actual tokens so they need special
handling for null bytes.

Closes #14346

1 files changed, 10 insertions(+), 1 deletions(-)

lib/std/zig/tokenizer.zig+10-1

...	...	@@ -1151,7 +1151,13 @@ pub const Tokenizer = struct {
1151	1151	},
1152	1152	},
1153	1153	.line_comment => switch (c) {
1154		0 => break,
	1154	0 => {
	1155	if (self.index != self.buffer.len) {
	1156	result.tag = .invalid;
	1157	self.index += 1;
	1158	}
	1159	break;
	1160	},
1155	1161	'\n' => {
1156	1162	state = .start;
1157	1163	result.loc.start = self.index + 1;
...	...	@@ -1865,6 +1871,9 @@ test "null byte before eof" {
1865	1871	try testTokenize("//\x00", &.{.invalid});
1866	1872	try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });
1867	1873	try testTokenize("\x00", &.{.invalid});
	1874	try testTokenize("// NUL\x00\n", &.{.invalid});
	1875	try testTokenize("///\x00\n", &.{ .doc_comment, .invalid });
	1876	try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
1868	1877	}
1869	1878
1870	1879	fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {