authorgravatar for codroid@gmail.comhryx <codroid@gmail.com> 2019-07-04 14:48:23-07:00
committergravatar for codroid@gmail.comhryx <codroid@gmail.com> 2019-07-04 14:48:23-07:00
log8365a7aab49938ff77228b72388301f562287415
tree4006d200eb1d65ace14e47e71217e4ca80cb6654
parent47addd87ac5e97e30cc591f9d7ac7fe1ea6af573
signaturelock-open Commit is signed but in an unrecognized format.

Unicode escapes: stage2 tokenizer and parser test

2 files changed, 101 insertions(+), 14 deletions(-)

std/zig/parser_test.zig+1-1
...@@ -68,7 +68,7 @@ test "zig fmt: enum literal inside array literal" {...@@ -68,7 +68,7 @@ test "zig fmt: enum literal inside array literal" {
6868
69test "zig fmt: character literal larger than u8" {69test "zig fmt: character literal larger than u8" {
70 try testCanonical(70 try testCanonical(
71 \\const x = '\U01f4a9';71 \\const x = '\u{01f4a9}';
72 \\72 \\
73 );73 );
74}74}
std/zig/tokenizer.zig+100-13
...@@ -240,6 +240,9 @@ pub const Tokenizer = struct {...@@ -240,6 +240,9 @@ pub const Tokenizer = struct {
240 CharLiteral,240 CharLiteral,
241 CharLiteralBackslash,241 CharLiteralBackslash,
242 CharLiteralHexEscape,242 CharLiteralHexEscape,
243 CharLiteralUnicodeEscapeSawU,
244 CharLiteralUnicodeEscape,
245 CharLiteralUnicodeInvalid,
243 CharLiteralEnd,246 CharLiteralEnd,
244 Backslash,247 Backslash,
245 Equal,248 Equal,
...@@ -296,7 +299,6 @@ pub const Tokenizer = struct {...@@ -296,7 +299,6 @@ pub const Tokenizer = struct {
296 .end = undefined,299 .end = undefined,
297 };300 };
298 var seen_escape_digits: usize = undefined;301 var seen_escape_digits: usize = undefined;
299 var expected_escape_digits: usize = undefined;
300 while (self.index < self.buffer.len) : (self.index += 1) {302 while (self.index < self.buffer.len) : (self.index += 1) {
301 const c = self.buffer[self.index];303 const c = self.buffer[self.index];
302 switch (state) {304 switch (state) {
...@@ -664,17 +666,9 @@ pub const Tokenizer = struct {...@@ -664,17 +666,9 @@ pub const Tokenizer = struct {
664 'x' => {666 'x' => {
665 state = State.CharLiteralHexEscape;667 state = State.CharLiteralHexEscape;
666 seen_escape_digits = 0;668 seen_escape_digits = 0;
667 expected_escape_digits = 2;
668 },669 },
669 'u' => {670 'u' => {
670 state = State.CharLiteralHexEscape;671 state = State.CharLiteralUnicodeEscapeSawU;
671 seen_escape_digits = 0;
672 expected_escape_digits = 4;
673 },
674 'U' => {
675 state = State.CharLiteralHexEscape;
676 seen_escape_digits = 0;
677 expected_escape_digits = 6;
678 },672 },
679 else => {673 else => {
680 state = State.CharLiteralEnd;674 state = State.CharLiteralEnd;
...@@ -682,9 +676,9 @@ pub const Tokenizer = struct {...@@ -682,9 +676,9 @@ pub const Tokenizer = struct {
682 },676 },
683677
684 State.CharLiteralHexEscape => switch (c) {678 State.CharLiteralHexEscape => switch (c) {
685 '0'...'9', 'a'...'z', 'A'...'F' => {679 '0'...'9', 'a'...'f', 'A'...'F' => {
686 seen_escape_digits += 1;680 seen_escape_digits += 1;
687 if (seen_escape_digits == expected_escape_digits) {681 if (seen_escape_digits == 2) {
688 state = State.CharLiteralEnd;682 state = State.CharLiteralEnd;
689 }683 }
690 },684 },
...@@ -694,6 +688,43 @@ pub const Tokenizer = struct {...@@ -694,6 +688,43 @@ pub const Tokenizer = struct {
694 },688 },
695 },689 },
696690
691 State.CharLiteralUnicodeEscapeSawU => switch (c) {
692 '{' => {
693 state = State.CharLiteralUnicodeEscape;
694 seen_escape_digits = 0;
695 },
696 else => {
697 result.id = Token.Id.Invalid;
698 state = State.CharLiteralUnicodeInvalid;
699 },
700 },
701
702 State.CharLiteralUnicodeEscape => switch (c) {
703 '0'...'9', 'a'...'f', 'A'...'F' => {
704 seen_escape_digits += 1;
705 },
706 '}' => {
707 if (seen_escape_digits == 0) {
708 result.id = Token.Id.Invalid;
709 state = State.CharLiteralUnicodeInvalid;
710 } else {
711 state = State.CharLiteralEnd;
712 }
713 },
714 else => {
715 result.id = Token.Id.Invalid;
716 state = State.CharLiteralUnicodeInvalid;
717 },
718 },
719
720 State.CharLiteralUnicodeInvalid => switch (c) {
721 // Keep consuming characters until an obvious stopping point.
722 // This consolidates e.g. `u{0ab1Q}` into a single invalid token
723 // instead of creating the tokens `u{0ab1`, `Q`, `}`
724 '0'...'9', 'a'...'z', 'A'...'Z', '}' => {},
725 else => break,
726 },
727
697 State.CharLiteralEnd => switch (c) {728 State.CharLiteralEnd => switch (c) {
698 '\'' => {729 '\'' => {
699 result.id = Token.Id.CharLiteral;730 result.id = Token.Id.CharLiteral;
...@@ -1055,6 +1086,9 @@ pub const Tokenizer = struct {...@@ -1055,6 +1086,9 @@ pub const Tokenizer = struct {
1055 State.CharLiteral,1086 State.CharLiteral,
1056 State.CharLiteralBackslash,1087 State.CharLiteralBackslash,
1057 State.CharLiteralHexEscape,1088 State.CharLiteralHexEscape,
1089 State.CharLiteralUnicodeEscapeSawU,
1090 State.CharLiteralUnicodeEscape,
1091 State.CharLiteralUnicodeInvalid,
1058 State.CharLiteralEnd,1092 State.CharLiteralEnd,
1059 State.StringLiteralBackslash,1093 State.StringLiteralBackslash,
1060 State.LBracketStar,1094 State.LBracketStar,
...@@ -1208,7 +1242,60 @@ test "tokenizer - unknown length pointer and then c pointer" {...@@ -1208,7 +1242,60 @@ test "tokenizer - unknown length pointer and then c pointer" {
1208test "tokenizer - char literal with hex escape" {1242test "tokenizer - char literal with hex escape" {
1209 testTokenize(1243 testTokenize(
1210 \\'\x1b'1244 \\'\x1b'
1211 , [_]Token.Id{Token.Id.CharLiteral});1245 , [_]Token.Id{.CharLiteral});
1246 testTokenize(
1247 \\'\x1'
1248 , [_]Token.Id{ .Invalid, .Invalid });
1249}
1250
1251test "tokenizer - char literal with unicode escapes" {
1252 // Valid unicode escapes
1253 testTokenize(
1254 \\'\u{3}'
1255 , [_]Token.Id{.CharLiteral});
1256 testTokenize(
1257 \\'\u{01}'
1258 , [_]Token.Id{.CharLiteral});
1259 testTokenize(
1260 \\'\u{2a}'
1261 , [_]Token.Id{.CharLiteral});
1262 testTokenize(
1263 \\'\u{3f9}'
1264 , [_]Token.Id{.CharLiteral});
1265 testTokenize(
1266 \\'\u{6E09aBc1523}'
1267 , [_]Token.Id{.CharLiteral});
1268 testTokenize(
1269 \\"\u{440}"
1270 , [_]Token.Id{.StringLiteral});
1271
1272 // Invalid unicode escapes
1273 testTokenize(
1274 \\'\u'
1275 , [_]Token.Id{.Invalid});
1276 testTokenize(
1277 \\'\u{{'
1278 , [_]Token.Id{ .Invalid, .Invalid });
1279 testTokenize(
1280 \\'\u{}'
1281 , [_]Token.Id{ .Invalid, .Invalid });
1282 testTokenize(
1283 \\'\u{s}'
1284 , [_]Token.Id{ .Invalid, .Invalid });
1285 testTokenize(
1286 \\'\u{2z}'
1287 , [_]Token.Id{ .Invalid, .Invalid });
1288 testTokenize(
1289 \\'\u{4a'
1290 , [_]Token.Id{.Invalid});
1291
1292 // Test old-style unicode literals
1293 testTokenize(
1294 \\'\u0333'
1295 , [_]Token.Id{ .Invalid, .Invalid });
1296 testTokenize(
1297 \\'\U0333'
1298 , [_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });
1212}1299}
12131300
1214test "tokenizer - float literal e exponent" {1301test "tokenizer - float literal e exponent" {