authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-12-07 21:43:01+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-12-08 00:16:30+02:00
log6039554b26b1aee6502063d9130907369c4ffd38
treec80027762a1249d5bf572908ea11da5bf45072c1
parent653c0bae0c473c52716dad35ec8f7f9511114a56

tokenizer: detect null bytes before EOF

Closes #13811

1 files changed, 17 insertions(+), 1 deletions(-)

lib/std/zig/tokenizer.zig+17-1
......@@ -425,7 +425,16 @@ pub const Tokenizer = struct {
425425 const c = self.buffer[self.index];
426426 switch (state) {
427427 .start => switch (c) {
428 0 => break,
428 0 => {
429 if (self.index != self.buffer.len) {
430 result.tag = .invalid;
431 result.loc.start = self.index;
432 self.index += 1;
433 result.loc.end = self.index;
434 return result;
435 }
436 break;
437 },
429438 ' ', '\n', '\t', '\r' => {
430439 result.loc.start = self.index + 1;
431440 },
......@@ -1851,6 +1860,13 @@ test "saturating operators" {
18511860 try testTokenize("-|=", &.{.minus_pipe_equal});
18521861}
18531862
1863test "null byte before eof" {
1864 try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });
1865 try testTokenize("//\x00", &.{.invalid});
1866 try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });
1867 try testTokenize("\x00", &.{.invalid});
1868}
1869
18541870fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
18551871 var tokenizer = Tokenizer.init(source);
18561872 for (expected_token_tags) |expected_token_tag| {