authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-12-07 21:43:01+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-12-08 00:16:30+02:00
log6039554b26b1aee6502063d9130907369c4ffd38
treec80027762a1249d5bf572908ea11da5bf45072c1
parent653c0bae0c473c52716dad35ec8f7f9511114a56

tokenizer: detect null bytes before EOF

Closes #13811

1 files changed, 17 insertions(+), 1 deletions(-)

lib/std/zig/tokenizer.zig+17-1
...@@ -425,7 +425,16 @@ pub const Tokenizer = struct {...@@ -425,7 +425,16 @@ pub const Tokenizer = struct {
425 const c = self.buffer[self.index];425 const c = self.buffer[self.index];
426 switch (state) {426 switch (state) {
427 .start => switch (c) {427 .start => switch (c) {
428 0 => break,428 0 => {
429 if (self.index != self.buffer.len) {
430 result.tag = .invalid;
431 result.loc.start = self.index;
432 self.index += 1;
433 result.loc.end = self.index;
434 return result;
435 }
436 break;
437 },
429 ' ', '\n', '\t', '\r' => {438 ' ', '\n', '\t', '\r' => {
430 result.loc.start = self.index + 1;439 result.loc.start = self.index + 1;
431 },440 },
...@@ -1851,6 +1860,13 @@ test "saturating operators" {...@@ -1851,6 +1860,13 @@ test "saturating operators" {
1851 try testTokenize("-|=", &.{.minus_pipe_equal});1860 try testTokenize("-|=", &.{.minus_pipe_equal});
1852}1861}
18531862
1863test "null byte before eof" {
1864 try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });
1865 try testTokenize("//\x00", &.{.invalid});
1866 try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });
1867 try testTokenize("\x00", &.{.invalid});
1868}
1869
1854fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {1870fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
1855 var tokenizer = Tokenizer.init(source);1871 var tokenizer = Tokenizer.init(source);
1856 for (expected_token_tags) |expected_token_tag| {1872 for (expected_token_tags) |expected_token_tag| {