authorgravatar for stojkovic.igor@gmail.comIgor Stojković <stojkovic.igor@gmail.com> 2024-09-23 22:06:30+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-09-23 13:06:30-07:00
log0676c04681ad7104faa7634c781df386e839d17b
tree820e9619b6449516c4ff4cba6cb6db6edc216034
parent943176bbfcf3125451fa64e082ee357c28e70413
signaturebadge-check Signed by PGP key B5690EEEBB952194

tokenizer: fix 0 byte following invalid (#21482)

closes #21481

1 files changed, 3 insertions(+), 0 deletions(-)

lib/std/zig/tokenizer.zig+3
...@@ -535,6 +535,8 @@ pub const Tokenizer = struct {...@@ -535,6 +535,8 @@ pub const Tokenizer = struct {
535 switch (self.buffer[self.index]) {535 switch (self.buffer[self.index]) {
536 0 => if (self.index == self.buffer.len) {536 0 => if (self.index == self.buffer.len) {
537 result.tag = .invalid;537 result.tag = .invalid;
538 } else {
539 continue :state .invalid;
538 },540 },
539 '\n' => result.tag = .invalid,541 '\n' => result.tag = .invalid,
540 else => continue :state .invalid,542 else => continue :state .invalid,
...@@ -1250,6 +1252,7 @@ test "invalid token characters" {...@@ -1250,6 +1252,7 @@ test "invalid token characters" {
12501252
1251test "invalid literal/comment characters" {1253test "invalid literal/comment characters" {
1252 try testTokenize("\"\x00\"", &.{.invalid});1254 try testTokenize("\"\x00\"", &.{.invalid});
1255 try testTokenize("`\x00`", &.{.invalid});
1253 try testTokenize("//\x00", &.{.invalid});1256 try testTokenize("//\x00", &.{.invalid});
1254 try testTokenize("//\x1f", &.{.invalid});1257 try testTokenize("//\x1f", &.{.invalid});
1255 try testTokenize("//\x7f", &.{.invalid});1258 try testTokenize("//\x7f", &.{.invalid});