authorgravatar for goon.pri.low@gmail.comKendall Condon <goon.pri.low@gmail.com> 2024-07-09 11:20:04-04:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2024-07-15 11:31:19+03:00
logc50f3003874d561aa696feb71f2089562307cdca
tree63990eafe12e6fa5045851d865d389fbe0386fc6
parent9d38e82b5c0f28ea6a2d8d31ebd73b6e2a8aad26

Tokenizer bug fixes and improvements

Fixes many error messages corresponding to invalid bytes displaying the wrong byte. Additionaly improves handling of UTF-8 in some places.

5 files changed, 214 insertions(+), 155 deletions(-)

lib/std/zig/Ast.zig+1-2
...@@ -188,9 +188,8 @@ pub fn tokenSlice(tree: Ast, token_index: TokenIndex) []const u8 {...@@ -188,9 +188,8 @@ pub fn tokenSlice(tree: Ast, token_index: TokenIndex) []const u8 {
188 var tokenizer: std.zig.Tokenizer = .{188 var tokenizer: std.zig.Tokenizer = .{
189 .buffer = tree.source,189 .buffer = tree.source,
190 .index = token_starts[token_index],190 .index = token_starts[token_index],
191 .pending_invalid_token = null,
192 };191 };
193 const token = tokenizer.findTagAtCurrentIndex(token_tag);192 const token = tokenizer.next();
194 assert(token.tag == token_tag);193 assert(token.tag == token_tag);
195 return tree.source[token.loc.start..token.loc.end];194 return tree.source[token.loc.start..token.loc.end];
196}195}
lib/std/zig/AstGen.zig+4-4
...@@ -13824,10 +13824,10 @@ fn lowerAstErrors(astgen: *AstGen) !void {...@@ -13824,10 +13824,10 @@ fn lowerAstErrors(astgen: *AstGen) !void {
13824 var notes: std.ArrayListUnmanaged(u32) = .{};13824 var notes: std.ArrayListUnmanaged(u32) = .{};
13825 defer notes.deinit(gpa);13825 defer notes.deinit(gpa);
1382613826
13827 if (token_tags[parse_err.token + @intFromBool(parse_err.token_is_prev)] == .invalid) {13827 const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);
13828 const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);13828 if (token_tags[tok] == .invalid) {
13829 const bad_off: u32 = @intCast(tree.tokenSlice(parse_err.token + @intFromBool(parse_err.token_is_prev)).len);13829 const bad_off: u32 = @intCast(tree.tokenSlice(tok).len);
13830 const byte_abs = token_starts[parse_err.token + @intFromBool(parse_err.token_is_prev)] + bad_off;13830 const byte_abs = token_starts[tok] + bad_off;
13831 try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{13831 try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{
13832 std.zig.fmtEscapes(tree.source[byte_abs..][0..1]),13832 std.zig.fmtEscapes(tree.source[byte_abs..][0..1]),
13833 }));13833 }));
lib/std/zig/tokenizer.zig+160-147
...@@ -337,7 +337,6 @@ pub const Token = struct {...@@ -337,7 +337,6 @@ pub const Token = struct {
337pub const Tokenizer = struct {337pub const Tokenizer = struct {
338 buffer: [:0]const u8,338 buffer: [:0]const u8,
339 index: usize,339 index: usize,
340 pending_invalid_token: ?Token,
341340
342 /// For debugging purposes341 /// For debugging purposes
343 pub fn dump(self: *Tokenizer, token: *const Token) void {342 pub fn dump(self: *Tokenizer, token: *const Token) void {
...@@ -350,7 +349,6 @@ pub const Tokenizer = struct {...@@ -350,7 +349,6 @@ pub const Tokenizer = struct {
350 return Tokenizer{349 return Tokenizer{
351 .buffer = buffer,350 .buffer = buffer,
352 .index = src_start,351 .index = src_start,
353 .pending_invalid_token = null,
354 };352 };
355 }353 }
356354
...@@ -366,8 +364,6 @@ pub const Tokenizer = struct {...@@ -366,8 +364,6 @@ pub const Tokenizer = struct {
366 char_literal_hex_escape,364 char_literal_hex_escape,
367 char_literal_unicode_escape_saw_u,365 char_literal_unicode_escape_saw_u,
368 char_literal_unicode_escape,366 char_literal_unicode_escape,
369 char_literal_unicode_invalid,
370 char_literal_unicode,
371 char_literal_end,367 char_literal_end,
372 backslash,368 backslash,
373 equal,369 equal,
...@@ -406,43 +402,7 @@ pub const Tokenizer = struct {...@@ -406,43 +402,7 @@ pub const Tokenizer = struct {
406 saw_at_sign,402 saw_at_sign,
407 };403 };
408404
409 /// This is a workaround to the fact that the tokenizer can queue up
410 /// 'pending_invalid_token's when parsing literals, which means that we need
411 /// to scan from the start of the current line to find a matching tag - just
412 /// in case it was an invalid character generated during literal
413 /// tokenization. Ideally this processing of this would be pushed to the AST
414 /// parser or another later stage, both to give more useful error messages
415 /// with that extra context and in order to be able to remove this
416 /// workaround.
417 pub fn findTagAtCurrentIndex(self: *Tokenizer, tag: Token.Tag) Token {
418 if (tag == .invalid) {
419 const target_index = self.index;
420 var starting_index = target_index;
421 while (starting_index > 0) {
422 if (self.buffer[starting_index] == '\n') {
423 break;
424 }
425 starting_index -= 1;
426 }
427
428 self.index = starting_index;
429 while (self.index <= target_index or self.pending_invalid_token != null) {
430 const result = self.next();
431 if (result.loc.start == target_index and result.tag == tag) {
432 return result;
433 }
434 }
435 unreachable;
436 } else {
437 return self.next();
438 }
439 }
440
441 pub fn next(self: *Tokenizer) Token {405 pub fn next(self: *Tokenizer) Token {
442 if (self.pending_invalid_token) |token| {
443 self.pending_invalid_token = null;
444 return token;
445 }
446 var state: State = .start;406 var state: State = .start;
447 var result = Token{407 var result = Token{
448 .tag = .eof,408 .tag = .eof,
...@@ -452,7 +412,6 @@ pub const Tokenizer = struct {...@@ -452,7 +412,6 @@ pub const Tokenizer = struct {
452 },412 },
453 };413 };
454 var seen_escape_digits: usize = undefined;414 var seen_escape_digits: usize = undefined;
455 var remaining_code_units: usize = undefined;
456 while (true) : (self.index += 1) {415 while (true) : (self.index += 1) {
457 const c = self.buffer[self.index];416 const c = self.buffer[self.index];
458 switch (state) {417 switch (state) {
...@@ -460,9 +419,8 @@ pub const Tokenizer = struct {...@@ -460,9 +419,8 @@ pub const Tokenizer = struct {
460 0 => {419 0 => {
461 if (self.index != self.buffer.len) {420 if (self.index != self.buffer.len) {
462 result.tag = .invalid;421 result.tag = .invalid;
463 result.loc.start = self.index;
464 self.index += 1;
465 result.loc.end = self.index;422 result.loc.end = self.index;
423 self.index += 1;
466 return result;424 return result;
467 }425 }
468 break;426 break;
...@@ -589,7 +547,7 @@ pub const Tokenizer = struct {...@@ -589,7 +547,7 @@ pub const Tokenizer = struct {
589 else => {547 else => {
590 result.tag = .invalid;548 result.tag = .invalid;
591 result.loc.end = self.index;549 result.loc.end = self.index;
592 self.index += 1;550 self.index += std.unicode.utf8ByteSequenceLength(c) catch 1;
593 return result;551 return result;
594 },552 },
595 },553 },
...@@ -762,6 +720,14 @@ pub const Tokenizer = struct {...@@ -762,6 +720,14 @@ pub const Tokenizer = struct {
762 },720 },
763 },721 },
764 .string_literal => switch (c) {722 .string_literal => switch (c) {
723 0, '\n' => {
724 result.tag = .invalid;
725 result.loc.end = self.index;
726 if (self.index != self.buffer.len) {
727 self.index += 1;
728 }
729 return result;
730 },
765 '\\' => {731 '\\' => {
766 state = .string_literal_backslash;732 state = .string_literal_backslash;
767 },733 },
...@@ -769,68 +735,75 @@ pub const Tokenizer = struct {...@@ -769,68 +735,75 @@ pub const Tokenizer = struct {
769 self.index += 1;735 self.index += 1;
770 break;736 break;
771 },737 },
772 0 => {738 else => {
773 if (self.index == self.buffer.len) {739 if (self.invalidCharacterLength()) |len| {
774 result.tag = .invalid;740 result.tag = .invalid;
775 break;741 result.loc.end = self.index;
776 } else {742 self.index += len;
777 self.checkLiteralCharacter();743 return result;
778 }744 }
745
746 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
779 },747 },
780 '\n' => {
781 result.tag = .invalid;
782 break;
783 },
784 else => self.checkLiteralCharacter(),
785 },748 },
786749
787 .string_literal_backslash => switch (c) {750 .string_literal_backslash => switch (c) {
788 0, '\n' => {751 0, '\n' => {
789 result.tag = .invalid;752 result.tag = .invalid;
790 break;753 result.loc.end = self.index;
754 if (self.index != self.buffer.len) {
755 self.index += 1;
756 }
757 return result;
791 },758 },
792 else => {759 else => {
793 state = .string_literal;760 state = .string_literal;
761
762 if (self.invalidCharacterLength()) |len| {
763 result.tag = .invalid;
764 result.loc.end = self.index;
765 self.index += len;
766 return result;
767 }
768
769 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
794 },770 },
795 },771 },
796772
797 .char_literal => switch (c) {773 .char_literal => switch (c) {
798 0 => {774 0, '\n', '\'' => {
799 result.tag = .invalid;775 result.tag = .invalid;
800 break;776 result.loc.end = self.index;
777 if (self.index != self.buffer.len) {
778 self.index += 1;
779 }
780 return result;
801 },781 },
802 '\\' => {782 '\\' => {
803 state = .char_literal_backslash;783 state = .char_literal_backslash;
804 },784 },
805 '\'', 0x80...0xbf, 0xf8...0xff => {
806 result.tag = .invalid;
807 break;
808 },
809 0xc0...0xdf => { // 110xxxxx
810 remaining_code_units = 1;
811 state = .char_literal_unicode;
812 },
813 0xe0...0xef => { // 1110xxxx
814 remaining_code_units = 2;
815 state = .char_literal_unicode;
816 },
817 0xf0...0xf7 => { // 11110xxx
818 remaining_code_units = 3;
819 state = .char_literal_unicode;
820 },
821 '\n' => {
822 result.tag = .invalid;
823 break;
824 },
825 else => {785 else => {
826 state = .char_literal_end;786 state = .char_literal_end;
787
788 if (self.invalidCharacterLength()) |len| {
789 result.tag = .invalid;
790 result.loc.end = self.index;
791 self.index += len;
792 return result;
793 }
794
795 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
827 },796 },
828 },797 },
829798
830 .char_literal_backslash => switch (c) {799 .char_literal_backslash => switch (c) {
831 0, '\n' => {800 0, '\n' => {
832 result.tag = .invalid;801 result.tag = .invalid;
833 break;802 result.loc.end = self.index;
803 if (self.index != self.buffer.len) {
804 self.index += 1;
805 }
806 return result;
834 },807 },
835 'x' => {808 'x' => {
836 state = .char_literal_hex_escape;809 state = .char_literal_hex_escape;
...@@ -841,6 +814,15 @@ pub const Tokenizer = struct {...@@ -841,6 +814,15 @@ pub const Tokenizer = struct {
841 },814 },
842 else => {815 else => {
843 state = .char_literal_end;816 state = .char_literal_end;
817
818 if (self.invalidCharacterLength()) |len| {
819 result.tag = .invalid;
820 result.loc.end = self.index;
821 self.index += len;
822 return result;
823 }
824
825 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
844 },826 },
845 },827 },
846828
...@@ -858,42 +840,26 @@ pub const Tokenizer = struct {...@@ -858,42 +840,26 @@ pub const Tokenizer = struct {
858 },840 },
859841
860 .char_literal_unicode_escape_saw_u => switch (c) {842 .char_literal_unicode_escape_saw_u => switch (c) {
861 0 => {
862 result.tag = .invalid;
863 break;
864 },
865 '{' => {843 '{' => {
866 state = .char_literal_unicode_escape;844 state = .char_literal_unicode_escape;
867 },845 },
868 else => {846 else => {
869 result.tag = .invalid;847 result.tag = .invalid;
870 state = .char_literal_unicode_invalid;848 break;
871 },849 },
872 },850 },
873851
874 .char_literal_unicode_escape => switch (c) {852 .char_literal_unicode_escape => switch (c) {
875 0 => {
876 result.tag = .invalid;
877 break;
878 },
879 '0'...'9', 'a'...'f', 'A'...'F' => {},853 '0'...'9', 'a'...'f', 'A'...'F' => {},
880 '}' => {854 '}' => {
881 state = .char_literal_end; // too many/few digits handled later855 state = .char_literal_end; // too many/few digits handled later
882 },856 },
883 else => {857 else => {
884 result.tag = .invalid;858 result.tag = .invalid;
885 state = .char_literal_unicode_invalid;859 break;
886 },860 },
887 },861 },
888862
889 .char_literal_unicode_invalid => switch (c) {
890 // Keep consuming characters until an obvious stopping point.
891 // This consolidates e.g. `u{0ab1Q}` into a single invalid token
892 // instead of creating the tokens `u{0ab1`, `Q`, `}`
893 '0'...'9', 'a'...'z', 'A'...'Z', '}' => {},
894 else => break,
895 },
896
897 .char_literal_end => switch (c) {863 .char_literal_end => switch (c) {
898 '\'' => {864 '\'' => {
899 result.tag = .char_literal;865 result.tag = .char_literal;
...@@ -906,27 +872,31 @@ pub const Tokenizer = struct {...@@ -906,27 +872,31 @@ pub const Tokenizer = struct {
906 },872 },
907 },873 },
908874
909 .char_literal_unicode => switch (c) {875 .multiline_string_literal_line => switch (c) {
910 0x80...0xbf => {876 0 => {
911 remaining_code_units -= 1;877 if (self.index != self.buffer.len) {
912 if (remaining_code_units == 0) {878 result.tag = .invalid;
913 state = .char_literal_end;879 result.loc.end = self.index;
880 self.index += 1;
881 return result;
914 }882 }
915 },
916 else => {
917 result.tag = .invalid;
918 break;883 break;
919 },884 },
920 },
921
922 .multiline_string_literal_line => switch (c) {
923 0 => break,
924 '\n' => {885 '\n' => {
925 self.index += 1;886 self.index += 1;
926 break;887 break;
927 },888 },
928 '\t' => {},889 '\t' => {},
929 else => self.checkLiteralCharacter(),890 else => {
891 if (self.invalidCharacterLength()) |len| {
892 result.tag = .invalid;
893 result.loc.end = self.index;
894 self.index += len;
895 return result;
896 }
897
898 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
899 },
930 },900 },
931901
932 .bang => switch (c) {902 .bang => switch (c) {
...@@ -1144,7 +1114,9 @@ pub const Tokenizer = struct {...@@ -1144,7 +1114,9 @@ pub const Tokenizer = struct {
1144 0 => {1114 0 => {
1145 if (self.index != self.buffer.len) {1115 if (self.index != self.buffer.len) {
1146 result.tag = .invalid;1116 result.tag = .invalid;
1117 result.loc.end = self.index;
1147 self.index += 1;1118 self.index += 1;
1119 return result;
1148 }1120 }
1149 break;1121 break;
1150 },1122 },
...@@ -1159,17 +1131,37 @@ pub const Tokenizer = struct {...@@ -1159,17 +1131,37 @@ pub const Tokenizer = struct {
1159 state = .start;1131 state = .start;
1160 result.loc.start = self.index + 1;1132 result.loc.start = self.index + 1;
1161 },1133 },
1162 '\t' => state = .line_comment,1134 '\t' => {
1135 state = .line_comment;
1136 },
1163 else => {1137 else => {
1164 state = .line_comment;1138 state = .line_comment;
1165 self.checkLiteralCharacter();1139
1140 if (self.invalidCharacterLength()) |len| {
1141 result.tag = .invalid;
1142 result.loc.end = self.index;
1143 self.index += len;
1144 return result;
1145 }
1146
1147 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1166 },1148 },
1167 },1149 },
1168 .doc_comment_start => switch (c) {1150 .doc_comment_start => switch (c) {
1169 '/' => {1151 '/' => {
1170 state = .line_comment;1152 state = .line_comment;
1171 },1153 },
1172 0, '\n' => {1154 0 => {
1155 if (self.index != self.buffer.len) {
1156 result.tag = .invalid;
1157 result.loc.end = self.index;
1158 self.index += 1;
1159 return result;
1160 }
1161 result.tag = .doc_comment;
1162 break;
1163 },
1164 '\n' => {
1173 result.tag = .doc_comment;1165 result.tag = .doc_comment;
1174 break;1166 break;
1175 },1167 },
...@@ -1180,14 +1172,24 @@ pub const Tokenizer = struct {...@@ -1180,14 +1172,24 @@ pub const Tokenizer = struct {
1180 else => {1172 else => {
1181 state = .doc_comment;1173 state = .doc_comment;
1182 result.tag = .doc_comment;1174 result.tag = .doc_comment;
1183 self.checkLiteralCharacter();1175
1176 if (self.invalidCharacterLength()) |len| {
1177 result.tag = .invalid;
1178 result.loc.end = self.index;
1179 self.index += len;
1180 return result;
1181 }
1182
1183 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1184 },1184 },
1185 },1185 },
1186 .line_comment => switch (c) {1186 .line_comment => switch (c) {
1187 0 => {1187 0 => {
1188 if (self.index != self.buffer.len) {1188 if (self.index != self.buffer.len) {
1189 result.tag = .invalid;1189 result.tag = .invalid;
1190 result.loc.end = self.index;
1190 self.index += 1;1191 self.index += 1;
1192 return result;
1191 }1193 }
1192 break;1194 break;
1193 },1195 },
...@@ -1196,12 +1198,30 @@ pub const Tokenizer = struct {...@@ -1196,12 +1198,30 @@ pub const Tokenizer = struct {
1196 result.loc.start = self.index + 1;1198 result.loc.start = self.index + 1;
1197 },1199 },
1198 '\t' => {},1200 '\t' => {},
1199 else => self.checkLiteralCharacter(),1201 else => {
1202 if (self.invalidCharacterLength()) |len| {
1203 result.tag = .invalid;
1204 result.loc.end = self.index;
1205 self.index += len;
1206 return result;
1207 }
1208
1209 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1210 },
1200 },1211 },
1201 .doc_comment => switch (c) {1212 .doc_comment => switch (c) {
1202 0, '\n' => break,1213 0, '\n' => break,
1203 '\t' => {},1214 '\t' => {},
1204 else => self.checkLiteralCharacter(),1215 else => {
1216 if (self.invalidCharacterLength()) |len| {
1217 result.tag = .invalid;
1218 result.loc.end = self.index;
1219 self.index += len;
1220 return result;
1221 }
1222
1223 self.index += (std.unicode.utf8ByteSequenceLength(c) catch unreachable) - 1;
1224 },
1205 },1225 },
1206 .int => switch (c) {1226 .int => switch (c) {
1207 '.' => state = .int_period,1227 '.' => state = .int_period,
...@@ -1244,10 +1264,6 @@ pub const Tokenizer = struct {...@@ -1244,10 +1264,6 @@ pub const Tokenizer = struct {
1244 }1264 }
12451265
1246 if (result.tag == .eof) {1266 if (result.tag == .eof) {
1247 if (self.pending_invalid_token) |token| {
1248 self.pending_invalid_token = null;
1249 return token;
1250 }
1251 result.loc.start = self.index;1267 result.loc.start = self.index;
1252 }1268 }
12531269
...@@ -1255,27 +1271,14 @@ pub const Tokenizer = struct {...@@ -1255,27 +1271,14 @@ pub const Tokenizer = struct {
1255 return result;1271 return result;
1256 }1272 }
12571273
1258 fn checkLiteralCharacter(self: *Tokenizer) void {1274 fn invalidCharacterLength(self: *Tokenizer) ?u3 {
1259 if (self.pending_invalid_token != null) return;
1260 const invalid_length = self.getInvalidCharacterLength();
1261 if (invalid_length == 0) return;
1262 self.pending_invalid_token = .{
1263 .tag = .invalid,
1264 .loc = .{
1265 .start = self.index,
1266 .end = self.index + invalid_length,
1267 },
1268 };
1269 }
1270
1271 fn getInvalidCharacterLength(self: *Tokenizer) u3 {
1272 const c0 = self.buffer[self.index];1275 const c0 = self.buffer[self.index];
1273 if (std.ascii.isAscii(c0)) {1276 if (std.ascii.isAscii(c0)) {
1274 if (c0 == '\r') {1277 if (c0 == '\r') {
1275 if (self.index + 1 < self.buffer.len and self.buffer[self.index + 1] == '\n') {1278 if (self.index + 1 < self.buffer.len and self.buffer[self.index + 1] == '\n') {
1276 // Carriage returns are *only* allowed just before a linefeed as part of a CRLF pair, otherwise1279 // Carriage returns are *only* allowed just before a linefeed as part of a CRLF pair, otherwise
1277 // they constitute an illegal byte!1280 // they constitute an illegal byte!
1278 return 0;1281 return null;
1279 } else {1282 } else {
1280 return 1;1283 return 1;
1281 }1284 }
...@@ -1285,7 +1288,7 @@ pub const Tokenizer = struct {...@@ -1285,7 +1288,7 @@ pub const Tokenizer = struct {
1285 return 1;1288 return 1;
1286 }1289 }
1287 // looks fine to me.1290 // looks fine to me.
1288 return 0;1291 return null;
1289 } else {1292 } else {
1290 // check utf8-encoded character.1293 // check utf8-encoded character.
1291 const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;1294 const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
...@@ -1308,8 +1311,7 @@ pub const Tokenizer = struct {...@@ -1308,8 +1311,7 @@ pub const Tokenizer = struct {
1308 },1311 },
1309 else => unreachable,1312 else => unreachable,
1310 }1313 }
1311 self.index += length - 1;1314 return null;
1312 return 0;
1313 }1315 }
1314 }1316 }
1315};1317};
...@@ -1394,27 +1396,37 @@ test "code point literal with unicode escapes" {...@@ -1394,27 +1396,37 @@ test "code point literal with unicode escapes" {
1394 // Invalid unicode escapes1396 // Invalid unicode escapes
1395 try testTokenize(1397 try testTokenize(
1396 \\'\u'1398 \\'\u'
1397 , &.{.invalid});1399 , &.{ .invalid, .invalid });
1398 try testTokenize(1400 try testTokenize(
1399 \\'\u{{'1401 \\'\u{{'
1400 , &.{ .invalid, .invalid });1402 , &.{ .invalid, .l_brace, .invalid });
1401 try testTokenize(1403 try testTokenize(
1402 \\'\u{}'1404 \\'\u{}'
1403 , &.{.char_literal});1405 , &.{.char_literal});
1404 try testTokenize(1406 try testTokenize(
1405 \\'\u{s}'1407 \\'\u{s}'
1406 , &.{ .invalid, .invalid });1408 , &.{
1409 .invalid,
1410 .identifier,
1411 .r_brace,
1412 .invalid,
1413 });
1407 try testTokenize(1414 try testTokenize(
1408 \\'\u{2z}'1415 \\'\u{2z}'
1409 , &.{ .invalid, .invalid });1416 , &.{
1417 .invalid,
1418 .identifier,
1419 .r_brace,
1420 .invalid,
1421 });
1410 try testTokenize(1422 try testTokenize(
1411 \\'\u{4a'1423 \\'\u{4a'
1412 , &.{.invalid});1424 , &.{ .invalid, .invalid }); // 4a is valid
14131425
1414 // Test old-style unicode literals1426 // Test old-style unicode literals
1415 try testTokenize(1427 try testTokenize(
1416 \\'\u0333'1428 \\'\u0333'
1417 , &.{ .invalid, .invalid });1429 , &.{ .invalid, .number_literal, .invalid });
1418 try testTokenize(1430 try testTokenize(
1419 \\'\U0333'1431 \\'\U0333'
1420 , &.{ .invalid, .number_literal, .invalid });1432 , &.{ .invalid, .number_literal, .invalid });
...@@ -1453,13 +1465,14 @@ test "invalid token characters" {...@@ -1453,13 +1465,14 @@ test "invalid token characters" {
1453 try testTokenize("`", &.{.invalid});1465 try testTokenize("`", &.{.invalid});
1454 try testTokenize("'c", &.{.invalid});1466 try testTokenize("'c", &.{.invalid});
1455 try testTokenize("'", &.{.invalid});1467 try testTokenize("'", &.{.invalid});
1456 try testTokenize("''", &.{ .invalid, .invalid });1468 try testTokenize("''", &.{.invalid});
1469 try testTokenize("'\n'", &.{ .invalid, .invalid });
1457}1470}
14581471
1459test "invalid literal/comment characters" {1472test "invalid literal/comment characters" {
1460 try testTokenize("\"\x00\"", &.{1473 try testTokenize("\"\x00\"", &.{
1461 .string_literal,
1462 .invalid,1474 .invalid,
1475 .invalid, // Incomplete string literal starting after invalid
1463 });1476 });
1464 try testTokenize("//\x00", &.{1477 try testTokenize("//\x00", &.{
1465 .invalid,1478 .invalid,
...@@ -1910,10 +1923,10 @@ test "saturating operators" {...@@ -1910,10 +1923,10 @@ test "saturating operators" {
1910test "null byte before eof" {1923test "null byte before eof" {
1911 try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });1924 try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });
1912 try testTokenize("//\x00", &.{.invalid});1925 try testTokenize("//\x00", &.{.invalid});
1913 try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });1926 try testTokenize("\\\\\x00", &.{.invalid});
1914 try testTokenize("\x00", &.{.invalid});1927 try testTokenize("\x00", &.{.invalid});
1915 try testTokenize("// NUL\x00\n", &.{.invalid});1928 try testTokenize("// NUL\x00\n", &.{.invalid});
1916 try testTokenize("///\x00\n", &.{ .doc_comment, .invalid });1929 try testTokenize("///\x00\n", &.{.invalid});
1917 try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });1930 try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
1918}1931}
19191932
test/cases/compile_errors/invalid_unicode_escape.zig created+11
...@@ -0,0 +1,11 @@
1export fn entry() void {
2 const a = '\u{12z34}';
3}
4
5// error
6// backend=stage2
7// target=native
8//
9// :2:15: error: expected expression, found 'invalid bytes'
10// :2:21: note: invalid byte: 'z'
11
test/compile_errors.zig+38-2
...@@ -42,8 +42,8 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {...@@ -42,8 +42,8 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
42 const case = ctx.obj("isolated carriage return in multiline string literal", b.graph.host);42 const case = ctx.obj("isolated carriage return in multiline string literal", b.graph.host);
4343
44 case.addError("const foo = \\\\\test\r\r rogue carriage return\n;", &[_][]const u8{44 case.addError("const foo = \\\\\test\r\r rogue carriage return\n;", &[_][]const u8{
45 ":1:19: error: expected ';' after declaration",45 ":1:13: error: expected expression, found 'invalid bytes'",
46 ":1:20: note: invalid byte: '\\r'",46 ":1:19: note: invalid byte: '\\r'",
47 });47 });
48 }48 }
4949
...@@ -217,4 +217,40 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {...@@ -217,4 +217,40 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
217 \\pub fn anytypeFunction(_: anytype) void {}217 \\pub fn anytypeFunction(_: anytype) void {}
218 );218 );
219 }219 }
220
221 {
222 const case = ctx.obj("invalid byte in string", b.graph.host);
223
224 case.addError("_ = \"\x01Q\";", &[_][]const u8{
225 ":1:5: error: expected expression, found 'invalid bytes'",
226 ":1:6: note: invalid byte: '\\x01'",
227 });
228 }
229
230 {
231 const case = ctx.obj("invalid byte in comment", b.graph.host);
232
233 case.addError("//\x01Q", &[_][]const u8{
234 ":1:1: error: expected type expression, found 'invalid bytes'",
235 ":1:3: note: invalid byte: '\\x01'",
236 });
237 }
238
239 {
240 const case = ctx.obj("control character in character literal", b.graph.host);
241
242 case.addError("const c = '\x01';", &[_][]const u8{
243 ":1:11: error: expected expression, found 'invalid bytes'",
244 ":1:12: note: invalid byte: '\\x01'",
245 });
246 }
247
248 {
249 const case = ctx.obj("invalid byte at start of token", b.graph.host);
250
251 case.addError("x = \x00Q", &[_][]const u8{
252 ":1:5: error: expected expression, found 'invalid bytes'",
253 ":1:5: note: invalid byte: '\\x00'",
254 });
255 }
220}256}