authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2023-01-17 20:39:19+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2023-01-17 20:39:19+02:00
log841b38aae8913972d12b08cf3e0e497be305efb1
tree67e2ae4b70664b16c5504f2ae6ed0123191a8dfc
parentfc066992d912186da08ac9250b3772103aafbbe8

tokenizer: detect null as non-first byte of a line comment

Line comments do not produce actual tokens so they need special handling for null bytes. Closes #14346

1 files changed, 10 insertions(+), 1 deletions(-)

lib/std/zig/tokenizer.zig+10-1
...@@ -1151,7 +1151,13 @@ pub const Tokenizer = struct {...@@ -1151,7 +1151,13 @@ pub const Tokenizer = struct {
1151 },1151 },
1152 },1152 },
1153 .line_comment => switch (c) {1153 .line_comment => switch (c) {
1154 0 => break,1154 0 => {
1155 if (self.index != self.buffer.len) {
1156 result.tag = .invalid;
1157 self.index += 1;
1158 }
1159 break;
1160 },
1155 '\n' => {1161 '\n' => {
1156 state = .start;1162 state = .start;
1157 result.loc.start = self.index + 1;1163 result.loc.start = self.index + 1;
...@@ -1865,6 +1871,9 @@ test "null byte before eof" {...@@ -1865,6 +1871,9 @@ test "null byte before eof" {
1865 try testTokenize("//\x00", &.{.invalid});1871 try testTokenize("//\x00", &.{.invalid});
1866 try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });1872 try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });
1867 try testTokenize("\x00", &.{.invalid});1873 try testTokenize("\x00", &.{.invalid});
1874 try testTokenize("// NUL\x00\n", &.{.invalid});
1875 try testTokenize("///\x00\n", &.{ .doc_comment, .invalid });
1876 try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
1868}1877}
18691878
1870fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {1879fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {