authorgravatar for stojkovic.igor@gmail.comIgor Stojković <stojkovic.igor@gmail.com> 2024-09-23 22:06:30+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-09-23 13:06:30-07:00
log0676c04681ad7104faa7634c781df386e839d17b
tree820e9619b6449516c4ff4cba6cb6db6edc216034
parent943176bbfcf3125451fa64e082ee357c28e70413
signaturebadge-check Signed by PGP key B5690EEEBB952194

tokenizer: fix 0 byte following invalid (#21482)

closes #21481

1 files changed, 3 insertions(+), 0 deletions(-)

lib/std/zig/tokenizer.zig+3
......@@ -535,6 +535,8 @@ pub const Tokenizer = struct {
535535 switch (self.buffer[self.index]) {
536536 0 => if (self.index == self.buffer.len) {
537537 result.tag = .invalid;
538 } else {
539 continue :state .invalid;
538540 },
539541 '\n' => result.tag = .invalid,
540542 else => continue :state .invalid,
......@@ -1250,6 +1252,7 @@ test "invalid token characters" {
12501252
12511253test "invalid literal/comment characters" {
12521254 try testTokenize("\"\x00\"", &.{.invalid});
1255 try testTokenize("`\x00`", &.{.invalid});
12531256 try testTokenize("//\x00", &.{.invalid});
12541257 try testTokenize("//\x1f", &.{.invalid});
12551258 try testTokenize("//\x7f", &.{.invalid});