authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-10-09 13:25:41-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-10-09 13:25:41-04:00
log406b70aa56b9a95e768c321dd3caf164add1b49a
treedc89654bce360751073bd712fe51b8c023900636
parentf929a58d5f69c26c25ced89f31f60d0a92ffc46a
parentae7392e504e7765b05d98636cc249cbf92233f5c
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3390 from nrdmn/unicode_character_literals

unicode character literals

4 files changed, 81 insertions(+), 19 deletions(-)

doc/langref.html.in+2-2
...@@ -552,8 +552,7 @@ pub fn main() void {...@@ -552,8 +552,7 @@ pub fn main() void {
552 <p>552 <p>
553 Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as553 Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as
554 {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals554 {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals
555 and character literals. Once https://github.com/ziglang/zig/issues/2097 is implemented,555 and character literals.
556 character literals will be allowed to have a single UTF-8 encoded codepoint.
557 </p>556 </p>
558 {#code_begin|test#}557 {#code_begin|test#}
559const assert = @import("std").debug.assert;558const assert = @import("std").debug.assert;
...@@ -567,6 +566,7 @@ test "string literals" {...@@ -567,6 +566,7 @@ test "string literals" {
567 assert(normal_bytes[1] == 'e');566 assert(normal_bytes[1] == 'e');
568 assert('e' == '\x65');567 assert('e' == '\x65');
569 assert('\u{1f4a9}' == 128169);568 assert('\u{1f4a9}' == 128169);
569 assert('💯' == 128175);
570 assert(mem.eql(u8, "hello", "h\x65llo"));570 assert(mem.eql(u8, "hello", "h\x65llo"));
571571
572 // A C string literal is a null terminated pointer.572 // A C string literal is a null terminated pointer.
lib/std/zig/tokenizer.zig+35-6
...@@ -371,6 +371,7 @@ pub const Tokenizer = struct {...@@ -371,6 +371,7 @@ pub const Tokenizer = struct {
371 CharLiteralUnicodeEscapeSawU,371 CharLiteralUnicodeEscapeSawU,
372 CharLiteralUnicodeEscape,372 CharLiteralUnicodeEscape,
373 CharLiteralUnicodeInvalid,373 CharLiteralUnicodeInvalid,
374 CharLiteralUnicode,
374 CharLiteralEnd,375 CharLiteralEnd,
375 Backslash,376 Backslash,
376 Equal,377 Equal,
...@@ -427,6 +428,7 @@ pub const Tokenizer = struct {...@@ -427,6 +428,7 @@ pub const Tokenizer = struct {
427 .end = undefined,428 .end = undefined,
428 };429 };
429 var seen_escape_digits: usize = undefined;430 var seen_escape_digits: usize = undefined;
431 var remaining_code_units: usize = undefined;
430 while (self.index < self.buffer.len) : (self.index += 1) {432 while (self.index < self.buffer.len) : (self.index += 1) {
431 const c = self.buffer[self.index];433 const c = self.buffer[self.index];
432 switch (state) {434 switch (state) {
...@@ -774,16 +776,23 @@ pub const Tokenizer = struct {...@@ -774,16 +776,23 @@ pub const Tokenizer = struct {
774 '\\' => {776 '\\' => {
775 state = State.CharLiteralBackslash;777 state = State.CharLiteralBackslash;
776 },778 },
777 '\'' => {779 '\'', 0x80...0xbf, 0xf8...0xff => {
778 result.id = Token.Id.Invalid;780 result.id = Token.Id.Invalid;
779 break;781 break;
780 },782 },
783 0xc0...0xdf => { // 110xxxxx
784 remaining_code_units = 1;
785 state = State.CharLiteralUnicode;
786 },
787 0xe0...0xef => { // 1110xxxx
788 remaining_code_units = 2;
789 state = State.CharLiteralUnicode;
790 },
791 0xf0...0xf7 => { // 11110xxx
792 remaining_code_units = 3;
793 state = State.CharLiteralUnicode;
794 },
781 else => {795 else => {
782 if (c < 0x20 or c == 0x7f) {
783 result.id = Token.Id.Invalid;
784 break;
785 }
786
787 state = State.CharLiteralEnd;796 state = State.CharLiteralEnd;
788 },797 },
789 },798 },
...@@ -867,6 +876,19 @@ pub const Tokenizer = struct {...@@ -867,6 +876,19 @@ pub const Tokenizer = struct {
867 },876 },
868 },877 },
869878
879 State.CharLiteralUnicode => switch (c) {
880 0x80...0xbf => {
881 remaining_code_units -= 1;
882 if (remaining_code_units == 0) {
883 state = State.CharLiteralEnd;
884 }
885 },
886 else => {
887 result.id = Token.Id.Invalid;
888 break;
889 },
890 },
891
870 State.MultilineStringLiteralLine => switch (c) {892 State.MultilineStringLiteralLine => switch (c) {
871 '\n' => {893 '\n' => {
872 self.index += 1;894 self.index += 1;
...@@ -1220,6 +1242,7 @@ pub const Tokenizer = struct {...@@ -1220,6 +1242,7 @@ pub const Tokenizer = struct {
1220 State.CharLiteralUnicodeEscape,1242 State.CharLiteralUnicodeEscape,
1221 State.CharLiteralUnicodeInvalid,1243 State.CharLiteralUnicodeInvalid,
1222 State.CharLiteralEnd,1244 State.CharLiteralEnd,
1245 State.CharLiteralUnicode,
1223 State.StringLiteralBackslash,1246 State.StringLiteralBackslash,
1224 State.LBracketStar,1247 State.LBracketStar,
1225 State.LBracketStarC,1248 State.LBracketStarC,
...@@ -1428,6 +1451,12 @@ test "tokenizer - char literal with unicode escapes" {...@@ -1428,6 +1451,12 @@ test "tokenizer - char literal with unicode escapes" {
1428 , [_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });1451 , [_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });
1429}1452}
14301453
1454test "tokenizer - char literal with unicode code point" {
1455 testTokenize(
1456 \\'💩'
1457 , [_]Token.Id{.CharLiteral});
1458}
1459
1431test "tokenizer - float literal e exponent" {1460test "tokenizer - float literal e exponent" {
1432 testTokenize("a = 4.94065645841246544177e-324;\n", [_]Token.Id{1461 testTokenize("a = 4.94065645841246544177e-324;\n", [_]Token.Id{
1433 Token.Id.Identifier,1462 Token.Id.Identifier,
src/tokenizer.cpp+40-11
...@@ -193,6 +193,7 @@ enum TokenizeState {...@@ -193,6 +193,7 @@ enum TokenizeState {
193 TokenizeStateStringEscapeUnicodeStart,193 TokenizeStateStringEscapeUnicodeStart,
194 TokenizeStateCharLiteral,194 TokenizeStateCharLiteral,
195 TokenizeStateCharLiteralEnd,195 TokenizeStateCharLiteralEnd,
196 TokenizeStateCharLiteralUnicode,
196 TokenizeStateSawStar,197 TokenizeStateSawStar,
197 TokenizeStateSawStarPercent,198 TokenizeStateSawStarPercent,
198 TokenizeStateSawSlash,199 TokenizeStateSawSlash,
...@@ -250,6 +251,7 @@ struct Tokenize {...@@ -250,6 +251,7 @@ struct Tokenize {
250 int exponent_in_bin_or_dec;251 int exponent_in_bin_or_dec;
251 BigInt specified_exponent;252 BigInt specified_exponent;
252 BigInt significand;253 BigInt significand;
254 size_t remaining_code_units;
253};255};
254256
255ATTRIBUTE_PRINTF(2, 3)257ATTRIBUTE_PRINTF(2, 3)
...@@ -1221,17 +1223,32 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1221,17 +1223,32 @@ void tokenize(Buf *buf, Tokenization *out) {
1221 }1223 }
1222 break;1224 break;
1223 case TokenizeStateCharLiteral:1225 case TokenizeStateCharLiteral:
1224 switch (c) {1226 if (c == '\'') {
1225 case '\'':1227 tokenize_error(&t, "expected character");
1226 tokenize_error(&t, "expected character");1228 } else if (c == '\\') {
1227 break;1229 t.state = TokenizeStateStringEscape;
1228 case '\\':1230 } else if ((c >= 0x80 && c <= 0xbf) || c >= 0xf8) {
1229 t.state = TokenizeStateStringEscape;1231 // 10xxxxxx
1230 break;1232 // 11111xxx
1231 default:1233 invalid_char_error(&t, c);
1232 t.cur_tok->data.char_lit.c = c;1234 } else if (c >= 0xc0 && c <= 0xdf) {
1233 t.state = TokenizeStateCharLiteralEnd;1235 // 110xxxxx
1234 break;1236 t.cur_tok->data.char_lit.c = c & 0x1f;
1237 t.remaining_code_units = 1;
1238 t.state = TokenizeStateCharLiteralUnicode;
1239 } else if (c >= 0xe0 && c <= 0xef) {
1240 // 1110xxxx
1241 t.cur_tok->data.char_lit.c = c & 0x0f;
1242 t.remaining_code_units = 2;
1243 t.state = TokenizeStateCharLiteralUnicode;
1244 } else if (c >= 0xf0 && c <= 0xf7) {
1245 // 11110xxx
1246 t.cur_tok->data.char_lit.c = c & 0x07;
1247 t.remaining_code_units = 3;
1248 t.state = TokenizeStateCharLiteralUnicode;
1249 } else {
1250 t.cur_tok->data.char_lit.c = c;
1251 t.state = TokenizeStateCharLiteralEnd;
1235 }1252 }
1236 break;1253 break;
1237 case TokenizeStateCharLiteralEnd:1254 case TokenizeStateCharLiteralEnd:
...@@ -1244,6 +1261,17 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1244,6 +1261,17 @@ void tokenize(Buf *buf, Tokenization *out) {
1244 invalid_char_error(&t, c);1261 invalid_char_error(&t, c);
1245 }1262 }
1246 break;1263 break;
1264 case TokenizeStateCharLiteralUnicode:
1265 if (c <= 0x7f || c >= 0xc0) {
1266 invalid_char_error(&t, c);
1267 }
1268 t.cur_tok->data.char_lit.c <<= 6;
1269 t.cur_tok->data.char_lit.c += c & 0x3f;
1270 t.remaining_code_units--;
1271 if (t.remaining_code_units == 0) {
1272 t.state = TokenizeStateCharLiteralEnd;
1273 }
1274 break;
1247 case TokenizeStateZero:1275 case TokenizeStateZero:
1248 switch (c) {1276 switch (c) {
1249 case 'b':1277 case 'b':
...@@ -1479,6 +1507,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1479,6 +1507,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1479 break;1507 break;
1480 case TokenizeStateCharLiteral:1508 case TokenizeStateCharLiteral:
1481 case TokenizeStateCharLiteralEnd:1509 case TokenizeStateCharLiteralEnd:
1510 case TokenizeStateCharLiteralUnicode:
1482 tokenize_error(&t, "unterminated character literal");1511 tokenize_error(&t, "unterminated character literal");
1483 break;1512 break;
1484 case TokenizeStateSymbol:1513 case TokenizeStateSymbol:
test/stage1/behavior/misc.zig+4
...@@ -699,6 +699,10 @@ test "unicode escape in character literal" {...@@ -699,6 +699,10 @@ test "unicode escape in character literal" {
699 expect(a == 128169);699 expect(a == 128169);
700}700}
701701
702test "unicode character in character literal" {
703 expect('💩' == 128169);
704}
705
702test "result location zero sized array inside struct field implicit cast to slice" {706test "result location zero sized array inside struct field implicit cast to slice" {
703 const E = struct {707 const E = struct {
704 entries: []u32,708 entries: []u32,