authorgravatar for codroid@gmail.comhryx <codroid@gmail.com> 2019-07-04 14:48:23-07:00
committergravatar for codroid@gmail.comhryx <codroid@gmail.com> 2019-07-04 14:48:23-07:00
log8365a7aab49938ff77228b72388301f562287415
tree4006d200eb1d65ace14e47e71217e4ca80cb6654
parent47addd87ac5e97e30cc591f9d7ac7fe1ea6af573
signaturelock-open Commit is signed but in an unrecognized format.

Unicode escapes: stage2 tokenizer and parser test

2 files changed, 101 insertions(+), 14 deletions(-)

std/zig/parser_test.zig+1-1
......@@ -68,7 +68,7 @@ test "zig fmt: enum literal inside array literal" {
6868
6969test "zig fmt: character literal larger than u8" {
7070 try testCanonical(
71 \\const x = '\U01f4a9';
71 \\const x = '\u{01f4a9}';
7272 \\
7373 );
7474}
std/zig/tokenizer.zig+100-13
......@@ -240,6 +240,9 @@ pub const Tokenizer = struct {
240240 CharLiteral,
241241 CharLiteralBackslash,
242242 CharLiteralHexEscape,
243 CharLiteralUnicodeEscapeSawU,
244 CharLiteralUnicodeEscape,
245 CharLiteralUnicodeInvalid,
243246 CharLiteralEnd,
244247 Backslash,
245248 Equal,
......@@ -296,7 +299,6 @@ pub const Tokenizer = struct {
296299 .end = undefined,
297300 };
298301 var seen_escape_digits: usize = undefined;
299 var expected_escape_digits: usize = undefined;
300302 while (self.index < self.buffer.len) : (self.index += 1) {
301303 const c = self.buffer[self.index];
302304 switch (state) {
......@@ -664,17 +666,9 @@ pub const Tokenizer = struct {
664666 'x' => {
665667 state = State.CharLiteralHexEscape;
666668 seen_escape_digits = 0;
667 expected_escape_digits = 2;
668669 },
669670 'u' => {
670 state = State.CharLiteralHexEscape;
671 seen_escape_digits = 0;
672 expected_escape_digits = 4;
673 },
674 'U' => {
675 state = State.CharLiteralHexEscape;
676 seen_escape_digits = 0;
677 expected_escape_digits = 6;
671 state = State.CharLiteralUnicodeEscapeSawU;
678672 },
679673 else => {
680674 state = State.CharLiteralEnd;
......@@ -682,9 +676,9 @@ pub const Tokenizer = struct {
682676 },
683677
684678 State.CharLiteralHexEscape => switch (c) {
685 '0'...'9', 'a'...'z', 'A'...'F' => {
679 '0'...'9', 'a'...'f', 'A'...'F' => {
686680 seen_escape_digits += 1;
687 if (seen_escape_digits == expected_escape_digits) {
681 if (seen_escape_digits == 2) {
688682 state = State.CharLiteralEnd;
689683 }
690684 },
......@@ -694,6 +688,43 @@ pub const Tokenizer = struct {
694688 },
695689 },
696690
691 State.CharLiteralUnicodeEscapeSawU => switch (c) {
692 '{' => {
693 state = State.CharLiteralUnicodeEscape;
694 seen_escape_digits = 0;
695 },
696 else => {
697 result.id = Token.Id.Invalid;
698 state = State.CharLiteralUnicodeInvalid;
699 },
700 },
701
702 State.CharLiteralUnicodeEscape => switch (c) {
703 '0'...'9', 'a'...'f', 'A'...'F' => {
704 seen_escape_digits += 1;
705 },
706 '}' => {
707 if (seen_escape_digits == 0) {
708 result.id = Token.Id.Invalid;
709 state = State.CharLiteralUnicodeInvalid;
710 } else {
711 state = State.CharLiteralEnd;
712 }
713 },
714 else => {
715 result.id = Token.Id.Invalid;
716 state = State.CharLiteralUnicodeInvalid;
717 },
718 },
719
720 State.CharLiteralUnicodeInvalid => switch (c) {
721 // Keep consuming characters until an obvious stopping point.
722 // This consolidates e.g. `u{0ab1Q}` into a single invalid token
723 // instead of creating the tokens `u{0ab1`, `Q`, `}`
724 '0'...'9', 'a'...'z', 'A'...'Z', '}' => {},
725 else => break,
726 },
727
697728 State.CharLiteralEnd => switch (c) {
698729 '\'' => {
699730 result.id = Token.Id.CharLiteral;
......@@ -1055,6 +1086,9 @@ pub const Tokenizer = struct {
10551086 State.CharLiteral,
10561087 State.CharLiteralBackslash,
10571088 State.CharLiteralHexEscape,
1089 State.CharLiteralUnicodeEscapeSawU,
1090 State.CharLiteralUnicodeEscape,
1091 State.CharLiteralUnicodeInvalid,
10581092 State.CharLiteralEnd,
10591093 State.StringLiteralBackslash,
10601094 State.LBracketStar,
......@@ -1208,7 +1242,60 @@ test "tokenizer - unknown length pointer and then c pointer" {
12081242test "tokenizer - char literal with hex escape" {
12091243 testTokenize(
12101244 \\'\x1b'
1211 , [_]Token.Id{Token.Id.CharLiteral});
1245 , [_]Token.Id{.CharLiteral});
1246 testTokenize(
1247 \\'\x1'
1248 , [_]Token.Id{ .Invalid, .Invalid });
1249}
1250
1251test "tokenizer - char literal with unicode escapes" {
1252 // Valid unicode escapes
1253 testTokenize(
1254 \\'\u{3}'
1255 , [_]Token.Id{.CharLiteral});
1256 testTokenize(
1257 \\'\u{01}'
1258 , [_]Token.Id{.CharLiteral});
1259 testTokenize(
1260 \\'\u{2a}'
1261 , [_]Token.Id{.CharLiteral});
1262 testTokenize(
1263 \\'\u{3f9}'
1264 , [_]Token.Id{.CharLiteral});
1265 testTokenize(
1266 \\'\u{6E09aBc1523}'
1267 , [_]Token.Id{.CharLiteral});
1268 testTokenize(
1269 \\"\u{440}"
1270 , [_]Token.Id{.StringLiteral});
1271
1272 // Invalid unicode escapes
1273 testTokenize(
1274 \\'\u'
1275 , [_]Token.Id{.Invalid});
1276 testTokenize(
1277 \\'\u{{'
1278 , [_]Token.Id{ .Invalid, .Invalid });
1279 testTokenize(
1280 \\'\u{}'
1281 , [_]Token.Id{ .Invalid, .Invalid });
1282 testTokenize(
1283 \\'\u{s}'
1284 , [_]Token.Id{ .Invalid, .Invalid });
1285 testTokenize(
1286 \\'\u{2z}'
1287 , [_]Token.Id{ .Invalid, .Invalid });
1288 testTokenize(
1289 \\'\u{4a'
1290 , [_]Token.Id{.Invalid});
1291
1292 // Test old-style unicode literals
1293 testTokenize(
1294 \\'\u0333'
1295 , [_]Token.Id{ .Invalid, .Invalid });
1296 testTokenize(
1297 \\'\U0333'
1298 , [_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });
12121299}
12131300
12141301test "tokenizer - float literal e exponent" {