authorgravatar for goon.pri.low@gmail.comKendall Condon <goon.pri.low@gmail.com> 2024-07-09 11:20:04-04:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2024-07-15 11:31:19+03:00
logc50f3003874d561aa696feb71f2089562307cdca
tree63990eafe12e6fa5045851d865d389fbe0386fc6
parent9d38e82b5c0f28ea6a2d8d31ebd73b6e2a8aad26

Tokenizer bug fixes and improvements

Fixes many error messages corresponding to invalid bytes displaying the wrong byte. Additionaly improves handling of UTF-8 in some places.

5 files changed, 214 insertions(+), 155 deletions(-)

lib/std/zig/Ast.zig+1-2
......@@ -188,9 +188,8 @@ pub fn tokenSlice(tree: Ast, token_index: TokenIndex) []const u8 {
188188 var tokenizer: std.zig.Tokenizer = .{
189189 .buffer = tree.source,
190190 .index = token_starts[token_index],
191 .pending_invalid_token = null,
192191 };
193 const token = tokenizer.findTagAtCurrentIndex(token_tag);
192 const token = tokenizer.next();
194193 assert(token.tag == token_tag);
195194 return tree.source[token.loc.start..token.loc.end];
196195}
lib/std/zig/AstGen.zig+4-4
......@@ -13824,10 +13824,10 @@ fn lowerAstErrors(astgen: *AstGen) !void {
1382413824 var notes: std.ArrayListUnmanaged(u32) = .{};
1382513825 defer notes.deinit(gpa);
1382613826
13827 if (token_tags[parse_err.token + @intFromBool(parse_err.token_is_prev)] == .invalid) {
13828 const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);
13829 const bad_off: u32 = @intCast(tree.tokenSlice(parse_err.token + @intFromBool(parse_err.token_is_prev)).len);
13830 const byte_abs = token_starts[parse_err.token + @intFromBool(parse_err.token_is_prev)] + bad_off;
13827 const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);
13828 if (token_tags[tok] == .invalid) {
13829 const bad_off: u32 = @intCast(tree.tokenSlice(tok).len);
13830 const byte_abs = token_starts[tok] + bad_off;
1383113831 try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{
1383213832 std.zig.fmtEscapes(tree.source[byte_abs..][0..1]),
1383313833 }));
lib/std/zig/tokenizer.zig+160-147
......@@ -337,7 +337,6 @@ pub const Token = struct {
337337pub const Tokenizer = struct {
338338 buffer: [:0]const u8,
339339 index: usize,
340 pending_invalid_token: ?Token,
341340
342341 /// For debugging purposes
343342 pub fn dump(self: *Tokenizer, token: *const Token) void {
......@@ -350,7 +349,6 @@ pub const Tokenizer = struct {
350349 return Tokenizer{
351350 .buffer = buffer,
352351 .index = src_start,
353 .pending_invalid_token = null,
354352 };
355353 }
356354
......@@ -366,8 +364,6 @@ pub const Tokenizer = struct {
366364 char_literal_hex_escape,
367365 char_literal_unicode_escape_saw_u,
368366 char_literal_unicode_escape,
369 char_literal_unicode_invalid,
370 char_literal_unicode,
371367 char_literal_end,
372368 backslash,
373369 equal,
......@@ -406,43 +402,7 @@ pub const Tokenizer = struct {
406402 saw_at_sign,
407403 };
408404
409 /// This is a workaround to the fact that the tokenizer can queue up
410 /// 'pending_invalid_token's when parsing literals, which means that we need
411 /// to scan from the start of the current line to find a matching tag - just
412 /// in case it was an invalid character generated during literal
413 /// tokenization. Ideally this processing of this would be pushed to the AST
414 /// parser or another later stage, both to give more useful error messages
415 /// with that extra context and in order to be able to remove this
416 /// workaround.
417 pub fn findTagAtCurrentIndex(self: *Tokenizer, tag: Token.Tag) Token {
418 if (tag == .invalid) {
419 const target_index = self.index;
420 var starting_index = target_index;
421 while (starting_index > 0) {
422 if (self.buffer[starting_index] == '\n') {
423 break;
424 }
425 starting_index -= 1;
426 }
427
428 self.index = starting_index;
429 while (self.index <= target_index or self.pending_invalid_token != null) {
430 const result = self.next();
431 if (result.loc.start == target_index and result.tag == tag) {
432 return result;
433 }
434 }
435 unreachable;
436 } else {
437 return self.next();
438 }
439 }
440
441405 pub fn next(self: *Tokenizer) Token {
442 if (self.pending_invalid_token) |token| {
443 self.pending_invalid_token = null;
444 return token;
445 }
446406 var state: State = .start;
447407 var result = Token{
448408 .tag = .eof,
......@@ -452,7 +412,6 @@ pub const Tokenizer = struct {
452412 },
453413 };
454414 var seen_escape_digits: usize = undefined;
455 var remaining_code_units: usize = undefined;
456415 while (true) : (self.index += 1) {
457416 const c = self.buffer[self.index];
458417 switch (state) {
......@@ -460,9 +419,8 @@ pub const Tokenizer = struct {
460419 0 => {
461420 if (self.index != self.buffer.len) {
462421 result.tag = .invalid;
463 result.loc.start = self.index;
464 self.index += 1;
465422 result.loc.end = self.index;
423 self.index += 1;
466424 return result;
467425 }
468426 break;
......@@ -589,7 +547,7 @@ pub const Tokenizer = struct {
589547 else => {
590548 result.tag = .invalid;
591549 result.loc.end = self.index;
592 self.index += 1;
550 self.index += std.unicode.utf8ByteSequenceLength(c) catch 1;
593551 return result;
594552 },
595553 },
......@@ -762,6 +720,14 @@ pub const Tokenizer = struct {
762720 },
763721 },
764722 .string_literal => switch (c) {
723 0, '\n' => {
724 result.tag = .invalid;
725 result.loc.end = self.index;
726 if (self.index != self.buffer.len) {
727 self.index += 1;
728 }
729 return result;
730 },
765731 '\\' => {
766732 state = .string_literal_backslash;
767733 },
......@@ -769,68 +735,75 @@ pub const Tokenizer = struct {
769735 self.index += 1;
770736 break;
771737 },
772 0 => {
773 if (self.index == self.buffer.len) {
738 else => {
739 if (self.invalidCharacterLength()) |len| {
774740 result.tag = .invalid;
775 break;
776 } else {
777 self.checkLiteralCharacter();
741 result.loc.end = self.index;
742 self.index += len;
743 return result;
778744 }
745
746 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
779747 },
780 '\n' => {
781 result.tag = .invalid;
782 break;
783 },
784 else => self.checkLiteralCharacter(),
785748 },
786749
787750 .string_literal_backslash => switch (c) {
788751 0, '\n' => {
789752 result.tag = .invalid;
790 break;
753 result.loc.end = self.index;
754 if (self.index != self.buffer.len) {
755 self.index += 1;
756 }
757 return result;
791758 },
792759 else => {
793760 state = .string_literal;
761
762 if (self.invalidCharacterLength()) |len| {
763 result.tag = .invalid;
764 result.loc.end = self.index;
765 self.index += len;
766 return result;
767 }
768
769 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
794770 },
795771 },
796772
797773 .char_literal => switch (c) {
798 0 => {
774 0, '\n', '\'' => {
799775 result.tag = .invalid;
800 break;
776 result.loc.end = self.index;
777 if (self.index != self.buffer.len) {
778 self.index += 1;
779 }
780 return result;
801781 },
802782 '\\' => {
803783 state = .char_literal_backslash;
804784 },
805 '\'', 0x80...0xbf, 0xf8...0xff => {
806 result.tag = .invalid;
807 break;
808 },
809 0xc0...0xdf => { // 110xxxxx
810 remaining_code_units = 1;
811 state = .char_literal_unicode;
812 },
813 0xe0...0xef => { // 1110xxxx
814 remaining_code_units = 2;
815 state = .char_literal_unicode;
816 },
817 0xf0...0xf7 => { // 11110xxx
818 remaining_code_units = 3;
819 state = .char_literal_unicode;
820 },
821 '\n' => {
822 result.tag = .invalid;
823 break;
824 },
825785 else => {
826786 state = .char_literal_end;
787
788 if (self.invalidCharacterLength()) |len| {
789 result.tag = .invalid;
790 result.loc.end = self.index;
791 self.index += len;
792 return result;
793 }
794
795 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
827796 },
828797 },
829798
830799 .char_literal_backslash => switch (c) {
831800 0, '\n' => {
832801 result.tag = .invalid;
833 break;
802 result.loc.end = self.index;
803 if (self.index != self.buffer.len) {
804 self.index += 1;
805 }
806 return result;
834807 },
835808 'x' => {
836809 state = .char_literal_hex_escape;
......@@ -841,6 +814,15 @@ pub const Tokenizer = struct {
841814 },
842815 else => {
843816 state = .char_literal_end;
817
818 if (self.invalidCharacterLength()) |len| {
819 result.tag = .invalid;
820 result.loc.end = self.index;
821 self.index += len;
822 return result;
823 }
824
825 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
844826 },
845827 },
846828
......@@ -858,42 +840,26 @@ pub const Tokenizer = struct {
858840 },
859841
860842 .char_literal_unicode_escape_saw_u => switch (c) {
861 0 => {
862 result.tag = .invalid;
863 break;
864 },
865843 '{' => {
866844 state = .char_literal_unicode_escape;
867845 },
868846 else => {
869847 result.tag = .invalid;
870 state = .char_literal_unicode_invalid;
848 break;
871849 },
872850 },
873851
874852 .char_literal_unicode_escape => switch (c) {
875 0 => {
876 result.tag = .invalid;
877 break;
878 },
879853 '0'...'9', 'a'...'f', 'A'...'F' => {},
880854 '}' => {
881855 state = .char_literal_end; // too many/few digits handled later
882856 },
883857 else => {
884858 result.tag = .invalid;
885 state = .char_literal_unicode_invalid;
859 break;
886860 },
887861 },
888862
889 .char_literal_unicode_invalid => switch (c) {
890 // Keep consuming characters until an obvious stopping point.
891 // This consolidates e.g. `u{0ab1Q}` into a single invalid token
892 // instead of creating the tokens `u{0ab1`, `Q`, `}`
893 '0'...'9', 'a'...'z', 'A'...'Z', '}' => {},
894 else => break,
895 },
896
897863 .char_literal_end => switch (c) {
898864 '\'' => {
899865 result.tag = .char_literal;
......@@ -906,27 +872,31 @@ pub const Tokenizer = struct {
906872 },
907873 },
908874
909 .char_literal_unicode => switch (c) {
910 0x80...0xbf => {
911 remaining_code_units -= 1;
912 if (remaining_code_units == 0) {
913 state = .char_literal_end;
875 .multiline_string_literal_line => switch (c) {
876 0 => {
877 if (self.index != self.buffer.len) {
878 result.tag = .invalid;
879 result.loc.end = self.index;
880 self.index += 1;
881 return result;
914882 }
915 },
916 else => {
917 result.tag = .invalid;
918883 break;
919884 },
920 },
921
922 .multiline_string_literal_line => switch (c) {
923 0 => break,
924885 '\n' => {
925886 self.index += 1;
926887 break;
927888 },
928889 '\t' => {},
929 else => self.checkLiteralCharacter(),
890 else => {
891 if (self.invalidCharacterLength()) |len| {
892 result.tag = .invalid;
893 result.loc.end = self.index;
894 self.index += len;
895 return result;
896 }
897
898 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
899 },
930900 },
931901
932902 .bang => switch (c) {
......@@ -1144,7 +1114,9 @@ pub const Tokenizer = struct {
11441114 0 => {
11451115 if (self.index != self.buffer.len) {
11461116 result.tag = .invalid;
1117 result.loc.end = self.index;
11471118 self.index += 1;
1119 return result;
11481120 }
11491121 break;
11501122 },
......@@ -1159,17 +1131,37 @@ pub const Tokenizer = struct {
11591131 state = .start;
11601132 result.loc.start = self.index + 1;
11611133 },
1162 '\t' => state = .line_comment,
1134 '\t' => {
1135 state = .line_comment;
1136 },
11631137 else => {
11641138 state = .line_comment;
1165 self.checkLiteralCharacter();
1139
1140 if (self.invalidCharacterLength()) |len| {
1141 result.tag = .invalid;
1142 result.loc.end = self.index;
1143 self.index += len;
1144 return result;
1145 }
1146
1147 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
11661148 },
11671149 },
11681150 .doc_comment_start => switch (c) {
11691151 '/' => {
11701152 state = .line_comment;
11711153 },
1172 0, '\n' => {
1154 0 => {
1155 if (self.index != self.buffer.len) {
1156 result.tag = .invalid;
1157 result.loc.end = self.index;
1158 self.index += 1;
1159 return result;
1160 }
1161 result.tag = .doc_comment;
1162 break;
1163 },
1164 '\n' => {
11731165 result.tag = .doc_comment;
11741166 break;
11751167 },
......@@ -1180,14 +1172,24 @@ pub const Tokenizer = struct {
11801172 else => {
11811173 state = .doc_comment;
11821174 result.tag = .doc_comment;
1183 self.checkLiteralCharacter();
1175
1176 if (self.invalidCharacterLength()) |len| {
1177 result.tag = .invalid;
1178 result.loc.end = self.index;
1179 self.index += len;
1180 return result;
1181 }
1182
1183 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
11841184 },
11851185 },
11861186 .line_comment => switch (c) {
11871187 0 => {
11881188 if (self.index != self.buffer.len) {
11891189 result.tag = .invalid;
1190 result.loc.end = self.index;
11901191 self.index += 1;
1192 return result;
11911193 }
11921194 break;
11931195 },
......@@ -1196,12 +1198,30 @@ pub const Tokenizer = struct {
11961198 result.loc.start = self.index + 1;
11971199 },
11981200 '\t' => {},
1199 else => self.checkLiteralCharacter(),
1201 else => {
1202 if (self.invalidCharacterLength()) |len| {
1203 result.tag = .invalid;
1204 result.loc.end = self.index;
1205 self.index += len;
1206 return result;
1207 }
1208
1209 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1210 },
12001211 },
12011212 .doc_comment => switch (c) {
12021213 0, '\n' => break,
12031214 '\t' => {},
1204 else => self.checkLiteralCharacter(),
1215 else => {
1216 if (self.invalidCharacterLength()) |len| {
1217 result.tag = .invalid;
1218 result.loc.end = self.index;
1219 self.index += len;
1220 return result;
1221 }
1222
1223 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1224 },
12051225 },
12061226 .int => switch (c) {
12071227 '.' => state = .int_period,
......@@ -1244,10 +1264,6 @@ pub const Tokenizer = struct {
12441264 }
12451265
12461266 if (result.tag == .eof) {
1247 if (self.pending_invalid_token) |token| {
1248 self.pending_invalid_token = null;
1249 return token;
1250 }
12511267 result.loc.start = self.index;
12521268 }
12531269
......@@ -1255,27 +1271,14 @@ pub const Tokenizer = struct {
12551271 return result;
12561272 }
12571273
1258 fn checkLiteralCharacter(self: *Tokenizer) void {
1259 if (self.pending_invalid_token != null) return;
1260 const invalid_length = self.getInvalidCharacterLength();
1261 if (invalid_length == 0) return;
1262 self.pending_invalid_token = .{
1263 .tag = .invalid,
1264 .loc = .{
1265 .start = self.index,
1266 .end = self.index + invalid_length,
1267 },
1268 };
1269 }
1270
1271 fn getInvalidCharacterLength(self: *Tokenizer) u3 {
1274 fn invalidCharacterLength(self: *Tokenizer) ?u3 {
12721275 const c0 = self.buffer[self.index];
12731276 if (std.ascii.isAscii(c0)) {
12741277 if (c0 == '\r') {
12751278 if (self.index + 1 < self.buffer.len and self.buffer[self.index + 1] == '\n') {
12761279 // Carriage returns are *only* allowed just before a linefeed as part of a CRLF pair, otherwise
12771280 // they constitute an illegal byte!
1278 return 0;
1281 return null;
12791282 } else {
12801283 return 1;
12811284 }
......@@ -1285,7 +1288,7 @@ pub const Tokenizer = struct {
12851288 return 1;
12861289 }
12871290 // looks fine to me.
1288 return 0;
1291 return null;
12891292 } else {
12901293 // check utf8-encoded character.
12911294 const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
......@@ -1308,8 +1311,7 @@ pub const Tokenizer = struct {
13081311 },
13091312 else => unreachable,
13101313 }
1311 self.index += length - 1;
1312 return 0;
1314 return null;
13131315 }
13141316 }
13151317};
......@@ -1394,27 +1396,37 @@ test "code point literal with unicode escapes" {
13941396 // Invalid unicode escapes
13951397 try testTokenize(
13961398 \\'\u'
1397 , &.{.invalid});
1399 , &.{ .invalid, .invalid });
13981400 try testTokenize(
13991401 \\'\u{{'
1400 , &.{ .invalid, .invalid });
1402 , &.{ .invalid, .l_brace, .invalid });
14011403 try testTokenize(
14021404 \\'\u{}'
14031405 , &.{.char_literal});
14041406 try testTokenize(
14051407 \\'\u{s}'
1406 , &.{ .invalid, .invalid });
1408 , &.{
1409 .invalid,
1410 .identifier,
1411 .r_brace,
1412 .invalid,
1413 });
14071414 try testTokenize(
14081415 \\'\u{2z}'
1409 , &.{ .invalid, .invalid });
1416 , &.{
1417 .invalid,
1418 .identifier,
1419 .r_brace,
1420 .invalid,
1421 });
14101422 try testTokenize(
14111423 \\'\u{4a'
1412 , &.{.invalid});
1424 , &.{ .invalid, .invalid }); // 4a is valid
14131425
14141426 // Test old-style unicode literals
14151427 try testTokenize(
14161428 \\'\u0333'
1417 , &.{ .invalid, .invalid });
1429 , &.{ .invalid, .number_literal, .invalid });
14181430 try testTokenize(
14191431 \\'\U0333'
14201432 , &.{ .invalid, .number_literal, .invalid });
......@@ -1453,13 +1465,14 @@ test "invalid token characters" {
14531465 try testTokenize("`", &.{.invalid});
14541466 try testTokenize("'c", &.{.invalid});
14551467 try testTokenize("'", &.{.invalid});
1456 try testTokenize("''", &.{ .invalid, .invalid });
1468 try testTokenize("''", &.{.invalid});
1469 try testTokenize("'\n'", &.{ .invalid, .invalid });
14571470}
14581471
14591472test "invalid literal/comment characters" {
14601473 try testTokenize("\"\x00\"", &.{
1461 .string_literal,
14621474 .invalid,
1475 .invalid, // Incomplete string literal starting after invalid
14631476 });
14641477 try testTokenize("//\x00", &.{
14651478 .invalid,
......@@ -1910,10 +1923,10 @@ test "saturating operators" {
19101923test "null byte before eof" {
19111924 try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });
19121925 try testTokenize("//\x00", &.{.invalid});
1913 try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });
1926 try testTokenize("\\\\\x00", &.{.invalid});
19141927 try testTokenize("\x00", &.{.invalid});
19151928 try testTokenize("// NUL\x00\n", &.{.invalid});
1916 try testTokenize("///\x00\n", &.{ .doc_comment, .invalid });
1929 try testTokenize("///\x00\n", &.{.invalid});
19171930 try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
19181931}
19191932
test/cases/compile_errors/invalid_unicode_escape.zig created+11
......@@ -0,0 +1,11 @@
1export fn entry() void {
2 const a = '\u{12z34}';
3}
4
5// error
6// backend=stage2
7// target=native
8//
9// :2:15: error: expected expression, found 'invalid bytes'
10// :2:21: note: invalid byte: 'z'
11
test/compile_errors.zig+38-2
......@@ -42,8 +42,8 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
4242 const case = ctx.obj("isolated carriage return in multiline string literal", b.graph.host);
4343
4444 case.addError("const foo = \\\\\test\r\r rogue carriage return\n;", &[_][]const u8{
45 ":1:19: error: expected ';' after declaration",
46 ":1:20: note: invalid byte: '\\r'",
45 ":1:13: error: expected expression, found 'invalid bytes'",
46 ":1:19: note: invalid byte: '\\r'",
4747 });
4848 }
4949
......@@ -217,4 +217,40 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
217217 \\pub fn anytypeFunction(_: anytype) void {}
218218 );
219219 }
220
221 {
222 const case = ctx.obj("invalid byte in string", b.graph.host);
223
224 case.addError("_ = \"\x01Q\";", &[_][]const u8{
225 ":1:5: error: expected expression, found 'invalid bytes'",
226 ":1:6: note: invalid byte: '\\x01'",
227 });
228 }
229
230 {
231 const case = ctx.obj("invalid byte in comment", b.graph.host);
232
233 case.addError("//\x01Q", &[_][]const u8{
234 ":1:1: error: expected type expression, found 'invalid bytes'",
235 ":1:3: note: invalid byte: '\\x01'",
236 });
237 }
238
239 {
240 const case = ctx.obj("control character in character literal", b.graph.host);
241
242 case.addError("const c = '\x01';", &[_][]const u8{
243 ":1:11: error: expected expression, found 'invalid bytes'",
244 ":1:12: note: invalid byte: '\\x01'",
245 });
246 }
247
248 {
249 const case = ctx.obj("invalid byte at start of token", b.graph.host);
250
251 case.addError("x = \x00Q", &[_][]const u8{
252 ":1:5: error: expected expression, found 'invalid bytes'",
253 ":1:5: note: invalid byte: '\\x00'",
254 });
255 }
220256}