authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-07-06 13:14:43-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-07-06 13:14:43-04:00
log21c60922e3514ca15fa505efb380fb37fde8d62a
tree5aea3c7c4f434d2d5aff81e94cf11ac9d1654a78
parent7f618184ade7f5fbe906973e3ca79924fd196641
parente35d49c4d03ea7a84a2d588be8f97108ea616dd7
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #2823 from hryx/unicode-escape

Unicode escapes: support u{N...}

6 files changed, 186 insertions(+), 79 deletions(-)

doc/langref.html.in+4-9
...@@ -566,7 +566,7 @@ test "string literals" {...@@ -566,7 +566,7 @@ test "string literals" {
566 assert(normal_bytes.len == 5);566 assert(normal_bytes.len == 5);
567 assert(normal_bytes[1] == 'e');567 assert(normal_bytes[1] == 'e');
568 assert('e' == '\x65');568 assert('e' == '\x65');
569 assert('\U01f4a9' == 128169);569 assert('\u{1f4a9}' == 128169);
570 assert(mem.eql(u8, "hello", "h\x65llo"));570 assert(mem.eql(u8, "hello", "h\x65llo"));
571571
572 // A C string literal is a null terminated pointer.572 // A C string literal is a null terminated pointer.
...@@ -616,12 +616,8 @@ test "string literals" {...@@ -616,12 +616,8 @@ test "string literals" {
616 <td>hexadecimal 8-bit character code (2 digits)</td>616 <td>hexadecimal 8-bit character code (2 digits)</td>
617 </tr>617 </tr>
618 <tr>618 <tr>
619 <td><code>\uNNNN</code></td>619 <td><code>\u{NNNNNN}</code></td>
620 <td>hexadecimal 16-bit Unicode character code UTF-8 encoded (4 digits)</td>620 <td>hexadecimal Unicode character code UTF-8 encoded (1 or more digits)</td>
621 </tr>
622 <tr>
623 <td><code>\UNNNNNN</code></td>
624 <td>hexadecimal 24-bit Unicode character code UTF-8 encoded (6 digits)</td>
625 </tr>621 </tr>
626 </table>622 </table>
627 </div>623 </div>
...@@ -10011,8 +10007,7 @@ eof &lt;- !....@@ -10011,8 +10007,7 @@ eof &lt;- !.
10011hex &lt;- [0-9a-fA-F]10007hex &lt;- [0-9a-fA-F]
10012char_escape10008char_escape
10013 &lt;- "\\x" hex hex10009 &lt;- "\\x" hex hex
10014 / "\\u" hex hex hex hex10010 / "\\u{" hex+ "}"
10015 / "\\U" hex hex hex hex hex hex
10016 / "\\" [nr\\t'"]10011 / "\\" [nr\\t'"]
10017char_char10012char_char
10018 &lt;- char_escape10013 &lt;- char_escape
src/tokenizer.cpp+61-54
...@@ -190,6 +190,7 @@ enum TokenizeState {...@@ -190,6 +190,7 @@ enum TokenizeState {
190 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"190 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
191 TokenizeStateString,191 TokenizeStateString,
192 TokenizeStateStringEscape,192 TokenizeStateStringEscape,
193 TokenizeStateStringEscapeUnicodeStart,
193 TokenizeStateCharLiteral,194 TokenizeStateCharLiteral,
194 TokenizeStateCharLiteralEnd,195 TokenizeStateCharLiteralEnd,
195 TokenizeStateSawStar,196 TokenizeStateSawStar,
...@@ -241,7 +242,6 @@ struct Tokenize {...@@ -241,7 +242,6 @@ struct Tokenize {
241 int32_t exp_add_amt;242 int32_t exp_add_amt;
242 bool is_exp_negative;243 bool is_exp_negative;
243 size_t char_code_index;244 size_t char_code_index;
244 size_t char_code_end;
245 bool unicode;245 bool unicode;
246 uint32_t char_code;246 uint32_t char_code;
247 int exponent_in_bin_or_dec;247 int exponent_in_bin_or_dec;
...@@ -1071,24 +1071,10 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1071,24 +1071,10 @@ void tokenize(Buf *buf, Tokenization *out) {
1071 t.radix = 16;1071 t.radix = 16;
1072 t.char_code = 0;1072 t.char_code = 0;
1073 t.char_code_index = 0;1073 t.char_code_index = 0;
1074 t.char_code_end = 2;
1075 t.unicode = false;1074 t.unicode = false;
1076 break;1075 break;
1077 case 'u':1076 case 'u':
1078 t.state = TokenizeStateCharCode;1077 t.state = TokenizeStateStringEscapeUnicodeStart;
1079 t.radix = 16;
1080 t.char_code = 0;
1081 t.char_code_index = 0;
1082 t.char_code_end = 4;
1083 t.unicode = true;
1084 break;
1085 case 'U':
1086 t.state = TokenizeStateCharCode;
1087 t.radix = 16;
1088 t.char_code = 0;
1089 t.char_code_index = 0;
1090 t.char_code_end = 6;
1091 t.unicode = true;
1092 break;1078 break;
1093 case 'n':1079 case 'n':
1094 handle_string_escape(&t, '\n');1080 handle_string_escape(&t, '\n');
...@@ -1112,8 +1098,63 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1112,8 +1098,63 @@ void tokenize(Buf *buf, Tokenization *out) {
1112 invalid_char_error(&t, c);1098 invalid_char_error(&t, c);
1113 }1099 }
1114 break;1100 break;
1101 case TokenizeStateStringEscapeUnicodeStart:
1102 switch (c) {
1103 case '{':
1104 t.state = TokenizeStateCharCode;
1105 t.radix = 16;
1106 t.char_code = 0;
1107 t.char_code_index = 0;
1108 t.unicode = true;
1109 break;
1110 default:
1111 invalid_char_error(&t, c);
1112 }
1113 break;
1115 case TokenizeStateCharCode:1114 case TokenizeStateCharCode:
1116 {1115 {
1116 if (t.unicode && c == '}') {
1117 if (t.char_code_index == 0) {
1118 tokenize_error(&t, "empty unicode escape sequence");
1119 break;
1120 }
1121 if (t.char_code > 0x10ffff) {
1122 tokenize_error(&t, "unicode value out of range: %x", t.char_code);
1123 break;
1124 }
1125 if (t.cur_tok->id == TokenIdCharLiteral) {
1126 t.cur_tok->data.char_lit.c = t.char_code;
1127 t.state = TokenizeStateCharLiteralEnd;
1128 } else if (t.char_code <= 0x7f) {
1129 // 00000000 00000000 00000000 0xxxxxxx
1130 handle_string_escape(&t, (uint8_t)t.char_code);
1131 } else if (t.char_code <= 0x7ff) {
1132 // 00000000 00000000 00000xxx xx000000
1133 handle_string_escape(&t, (uint8_t)(0xc0 | (t.char_code >> 6)));
1134 // 00000000 00000000 00000000 00xxxxxx
1135 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1136 } else if (t.char_code <= 0xffff) {
1137 // 00000000 00000000 xxxx0000 00000000
1138 handle_string_escape(&t, (uint8_t)(0xe0 | (t.char_code >> 12)));
1139 // 00000000 00000000 0000xxxx xx000000
1140 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f)));
1141 // 00000000 00000000 00000000 00xxxxxx
1142 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1143 } else if (t.char_code <= 0x10ffff) {
1144 // 00000000 000xxx00 00000000 00000000
1145 handle_string_escape(&t, (uint8_t)(0xf0 | (t.char_code >> 18)));
1146 // 00000000 000000xx xxxx0000 00000000
1147 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 12) & 0x3f)));
1148 // 00000000 00000000 0000xxxx xx000000
1149 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f)));
1150 // 00000000 00000000 00000000 00xxxxxx
1151 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1152 } else {
1153 zig_unreachable();
1154 }
1155 break;
1156 }
1157
1117 uint32_t digit_value = get_digit_value(c);1158 uint32_t digit_value = get_digit_value(c);
1118 if (digit_value >= t.radix) {1159 if (digit_value >= t.radix) {
1119 tokenize_error(&t, "invalid digit: '%c'", c);1160 tokenize_error(&t, "invalid digit: '%c'", c);
...@@ -1123,44 +1164,9 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1123,44 +1164,9 @@ void tokenize(Buf *buf, Tokenization *out) {
1123 t.char_code += digit_value;1164 t.char_code += digit_value;
1124 t.char_code_index += 1;1165 t.char_code_index += 1;
11251166
1126 if (t.char_code_index >= t.char_code_end) {1167 if (!t.unicode && t.char_code_index >= 2) {
1127 if (t.unicode) {1168 assert(t.char_code <= 255);
1128 if (t.char_code > 0x10ffff) {1169 handle_string_escape(&t, (uint8_t)t.char_code);
1129 tokenize_error(&t, "unicode value out of range: %x", t.char_code);
1130 break;
1131 }
1132 if (t.cur_tok->id == TokenIdCharLiteral) {
1133 t.cur_tok->data.char_lit.c = t.char_code;
1134 t.state = TokenizeStateCharLiteralEnd;
1135 } else if (t.char_code <= 0x7f) {
1136 // 00000000 00000000 00000000 0xxxxxxx
1137 handle_string_escape(&t, (uint8_t)t.char_code);
1138 } else if (t.char_code <= 0x7ff) {
1139 // 00000000 00000000 00000xxx xx000000
1140 handle_string_escape(&t, (uint8_t)(0xc0 | (t.char_code >> 6)));
1141 // 00000000 00000000 00000000 00xxxxxx
1142 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1143 } else if (t.char_code <= 0xffff) {
1144 // 00000000 00000000 xxxx0000 00000000
1145 handle_string_escape(&t, (uint8_t)(0xe0 | (t.char_code >> 12)));
1146 // 00000000 00000000 0000xxxx xx000000
1147 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f)));
1148 // 00000000 00000000 00000000 00xxxxxx
1149 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1150 } else if (t.char_code <= 0x10ffff) {
1151 // 00000000 000xxx00 00000000 00000000
1152 handle_string_escape(&t, (uint8_t)(0xf0 | (t.char_code >> 18)));
1153 // 00000000 000000xx xxxx0000 00000000
1154 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 12) & 0x3f)));
1155 // 00000000 00000000 0000xxxx xx000000
1156 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f)));
1157 // 00000000 00000000 00000000 00xxxxxx
1158 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1159 }
1160 } else {
1161 assert(t.char_code <= 255);
1162 handle_string_escape(&t, (uint8_t)t.char_code);
1163 }
1164 }1170 }
1165 }1171 }
1166 break;1172 break;
...@@ -1409,6 +1415,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1409,6 +1415,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1409 tokenize_error(&t, "unterminated string");1415 tokenize_error(&t, "unterminated string");
1410 break;1416 break;
1411 case TokenizeStateStringEscape:1417 case TokenizeStateStringEscape:
1418 case TokenizeStateStringEscapeUnicodeStart:
1412 case TokenizeStateCharCode:1419 case TokenizeStateCharCode:
1413 if (t.cur_tok->id == TokenIdStringLiteral) {1420 if (t.cur_tok->id == TokenIdStringLiteral) {
1414 tokenize_error(&t, "unterminated string");1421 tokenize_error(&t, "unterminated string");
std/zig/parser_test.zig+1-1
...@@ -80,7 +80,7 @@ test "zig fmt: enum literal inside array literal" {...@@ -80,7 +80,7 @@ test "zig fmt: enum literal inside array literal" {
8080
81test "zig fmt: character literal larger than u8" {81test "zig fmt: character literal larger than u8" {
82 try testCanonical(82 try testCanonical(
83 \\const x = '\U01f4a9';83 \\const x = '\u{01f4a9}';
84 \\84 \\
85 );85 );
86}86}
std/zig/tokenizer.zig+100-13
...@@ -240,6 +240,9 @@ pub const Tokenizer = struct {...@@ -240,6 +240,9 @@ pub const Tokenizer = struct {
240 CharLiteral,240 CharLiteral,
241 CharLiteralBackslash,241 CharLiteralBackslash,
242 CharLiteralHexEscape,242 CharLiteralHexEscape,
243 CharLiteralUnicodeEscapeSawU,
244 CharLiteralUnicodeEscape,
245 CharLiteralUnicodeInvalid,
243 CharLiteralEnd,246 CharLiteralEnd,
244 Backslash,247 Backslash,
245 Equal,248 Equal,
...@@ -296,7 +299,6 @@ pub const Tokenizer = struct {...@@ -296,7 +299,6 @@ pub const Tokenizer = struct {
296 .end = undefined,299 .end = undefined,
297 };300 };
298 var seen_escape_digits: usize = undefined;301 var seen_escape_digits: usize = undefined;
299 var expected_escape_digits: usize = undefined;
300 while (self.index < self.buffer.len) : (self.index += 1) {302 while (self.index < self.buffer.len) : (self.index += 1) {
301 const c = self.buffer[self.index];303 const c = self.buffer[self.index];
302 switch (state) {304 switch (state) {
...@@ -661,17 +663,9 @@ pub const Tokenizer = struct {...@@ -661,17 +663,9 @@ pub const Tokenizer = struct {
661 'x' => {663 'x' => {
662 state = State.CharLiteralHexEscape;664 state = State.CharLiteralHexEscape;
663 seen_escape_digits = 0;665 seen_escape_digits = 0;
664 expected_escape_digits = 2;
665 },666 },
666 'u' => {667 'u' => {
667 state = State.CharLiteralHexEscape;668 state = State.CharLiteralUnicodeEscapeSawU;
668 seen_escape_digits = 0;
669 expected_escape_digits = 4;
670 },
671 'U' => {
672 state = State.CharLiteralHexEscape;
673 seen_escape_digits = 0;
674 expected_escape_digits = 6;
675 },669 },
676 else => {670 else => {
677 state = State.CharLiteralEnd;671 state = State.CharLiteralEnd;
...@@ -679,9 +673,9 @@ pub const Tokenizer = struct {...@@ -679,9 +673,9 @@ pub const Tokenizer = struct {
679 },673 },
680674
681 State.CharLiteralHexEscape => switch (c) {675 State.CharLiteralHexEscape => switch (c) {
682 '0'...'9', 'a'...'z', 'A'...'F' => {676 '0'...'9', 'a'...'f', 'A'...'F' => {
683 seen_escape_digits += 1;677 seen_escape_digits += 1;
684 if (seen_escape_digits == expected_escape_digits) {678 if (seen_escape_digits == 2) {
685 state = State.CharLiteralEnd;679 state = State.CharLiteralEnd;
686 }680 }
687 },681 },
...@@ -691,6 +685,43 @@ pub const Tokenizer = struct {...@@ -691,6 +685,43 @@ pub const Tokenizer = struct {
691 },685 },
692 },686 },
693687
688 State.CharLiteralUnicodeEscapeSawU => switch (c) {
689 '{' => {
690 state = State.CharLiteralUnicodeEscape;
691 seen_escape_digits = 0;
692 },
693 else => {
694 result.id = Token.Id.Invalid;
695 state = State.CharLiteralUnicodeInvalid;
696 },
697 },
698
699 State.CharLiteralUnicodeEscape => switch (c) {
700 '0'...'9', 'a'...'f', 'A'...'F' => {
701 seen_escape_digits += 1;
702 },
703 '}' => {
704 if (seen_escape_digits == 0) {
705 result.id = Token.Id.Invalid;
706 state = State.CharLiteralUnicodeInvalid;
707 } else {
708 state = State.CharLiteralEnd;
709 }
710 },
711 else => {
712 result.id = Token.Id.Invalid;
713 state = State.CharLiteralUnicodeInvalid;
714 },
715 },
716
717 State.CharLiteralUnicodeInvalid => switch (c) {
718 // Keep consuming characters until an obvious stopping point.
719 // This consolidates e.g. `u{0ab1Q}` into a single invalid token
720 // instead of creating the tokens `u{0ab1`, `Q`, `}`
721 '0'...'9', 'a'...'z', 'A'...'Z', '}' => {},
722 else => break,
723 },
724
694 State.CharLiteralEnd => switch (c) {725 State.CharLiteralEnd => switch (c) {
695 '\'' => {726 '\'' => {
696 result.id = Token.Id.CharLiteral;727 result.id = Token.Id.CharLiteral;
...@@ -1052,6 +1083,9 @@ pub const Tokenizer = struct {...@@ -1052,6 +1083,9 @@ pub const Tokenizer = struct {
1052 State.CharLiteral,1083 State.CharLiteral,
1053 State.CharLiteralBackslash,1084 State.CharLiteralBackslash,
1054 State.CharLiteralHexEscape,1085 State.CharLiteralHexEscape,
1086 State.CharLiteralUnicodeEscapeSawU,
1087 State.CharLiteralUnicodeEscape,
1088 State.CharLiteralUnicodeInvalid,
1055 State.CharLiteralEnd,1089 State.CharLiteralEnd,
1056 State.StringLiteralBackslash,1090 State.StringLiteralBackslash,
1057 State.LBracketStar,1091 State.LBracketStar,
...@@ -1205,7 +1239,60 @@ test "tokenizer - unknown length pointer and then c pointer" {...@@ -1205,7 +1239,60 @@ test "tokenizer - unknown length pointer and then c pointer" {
1205test "tokenizer - char literal with hex escape" {1239test "tokenizer - char literal with hex escape" {
1206 testTokenize(1240 testTokenize(
1207 \\'\x1b'1241 \\'\x1b'
1208 , [_]Token.Id{Token.Id.CharLiteral});1242 , [_]Token.Id{.CharLiteral});
1243 testTokenize(
1244 \\'\x1'
1245 , [_]Token.Id{ .Invalid, .Invalid });
1246}
1247
1248test "tokenizer - char literal with unicode escapes" {
1249 // Valid unicode escapes
1250 testTokenize(
1251 \\'\u{3}'
1252 , [_]Token.Id{.CharLiteral});
1253 testTokenize(
1254 \\'\u{01}'
1255 , [_]Token.Id{.CharLiteral});
1256 testTokenize(
1257 \\'\u{2a}'
1258 , [_]Token.Id{.CharLiteral});
1259 testTokenize(
1260 \\'\u{3f9}'
1261 , [_]Token.Id{.CharLiteral});
1262 testTokenize(
1263 \\'\u{6E09aBc1523}'
1264 , [_]Token.Id{.CharLiteral});
1265 testTokenize(
1266 \\"\u{440}"
1267 , [_]Token.Id{.StringLiteral});
1268
1269 // Invalid unicode escapes
1270 testTokenize(
1271 \\'\u'
1272 , [_]Token.Id{.Invalid});
1273 testTokenize(
1274 \\'\u{{'
1275 , [_]Token.Id{ .Invalid, .Invalid });
1276 testTokenize(
1277 \\'\u{}'
1278 , [_]Token.Id{ .Invalid, .Invalid });
1279 testTokenize(
1280 \\'\u{s}'
1281 , [_]Token.Id{ .Invalid, .Invalid });
1282 testTokenize(
1283 \\'\u{2z}'
1284 , [_]Token.Id{ .Invalid, .Invalid });
1285 testTokenize(
1286 \\'\u{4a'
1287 , [_]Token.Id{.Invalid});
1288
1289 // Test old-style unicode literals
1290 testTokenize(
1291 \\'\u0333'
1292 , [_]Token.Id{ .Invalid, .Invalid });
1293 testTokenize(
1294 \\'\U0333'
1295 , [_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });
1209}1296}
12101297
1211test "tokenizer - float literal e exponent" {1298test "tokenizer - float literal e exponent" {
test/compile_errors.zig+18
...@@ -5414,6 +5414,24 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {...@@ -5414,6 +5414,24 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
5414 "tmp.zig:1:17: error: invalid carriage return, only '\\n' line endings are supported",5414 "tmp.zig:1:17: error: invalid carriage return, only '\\n' line endings are supported",
5415 );5415 );
54165416
5417 cases.add(
5418 "invalid legacy unicode escape",
5419 \\export fn entry() void {
5420 \\ const a = '\U1234';
5421 \\}
5422 ,
5423 "tmp.zig:2:17: error: invalid character: 'U'",
5424 );
5425
5426 cases.add(
5427 "invalid empty unicode escape",
5428 \\export fn entry() void {
5429 \\ const a = '\u{}';
5430 \\}
5431 ,
5432 "tmp.zig:2:19: error: empty unicode escape sequence",
5433 );
5434
5417 cases.add(5435 cases.add(
5418 "non-printable invalid character",5436 "non-printable invalid character",
5419 "\xff\xfe" ++5437 "\xff\xfe" ++
test/stage1/behavior/misc.zig+2-2
...@@ -189,7 +189,7 @@ test "string escapes" {...@@ -189,7 +189,7 @@ test "string escapes" {
189 expect(mem.eql(u8, "\r", "\x0d"));189 expect(mem.eql(u8, "\r", "\x0d"));
190 expect(mem.eql(u8, "\t", "\x09"));190 expect(mem.eql(u8, "\t", "\x09"));
191 expect(mem.eql(u8, "\\", "\x5c"));191 expect(mem.eql(u8, "\\", "\x5c"));
192 expect(mem.eql(u8, "\u1234\u0069", "\xe1\x88\xb4\x69"));192 expect(mem.eql(u8, "\u{1234}\u{069}\u{1}", "\xe1\x88\xb4\x69\x01"));
193}193}
194194
195test "multiline string" {195test "multiline string" {
...@@ -695,7 +695,7 @@ test "thread local variable" {...@@ -695,7 +695,7 @@ test "thread local variable" {
695}695}
696696
697test "unicode escape in character literal" {697test "unicode escape in character literal" {
698 var a: u24 = '\U01f4a9';698 var a: u24 = '\u{01f4a9}';
699 expect(a == 128169);699 expect(a == 128169);
700}700}
701701