authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-10-09 13:25:41-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-10-09 13:25:41-04:00
log406b70aa56b9a95e768c321dd3caf164add1b49a
treedc89654bce360751073bd712fe51b8c023900636
parentf929a58d5f69c26c25ced89f31f60d0a92ffc46a
parentae7392e504e7765b05d98636cc249cbf92233f5c
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3390 from nrdmn/unicode_character_literals

unicode character literals

4 files changed, 81 insertions(+), 19 deletions(-)

doc/langref.html.in+2-2
......@@ -552,8 +552,7 @@ pub fn main() void {
552552 <p>
553553 Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as
554554 {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals
555 and character literals. Once https://github.com/ziglang/zig/issues/2097 is implemented,
556 character literals will be allowed to have a single UTF-8 encoded codepoint.
555 and character literals.
557556 </p>
558557 {#code_begin|test#}
559558const assert = @import("std").debug.assert;
......@@ -567,6 +566,7 @@ test "string literals" {
567566 assert(normal_bytes[1] == 'e');
568567 assert('e' == '\x65');
569568 assert('\u{1f4a9}' == 128169);
569 assert('💯' == 128175);
570570 assert(mem.eql(u8, "hello", "h\x65llo"));
571571
572572 // A C string literal is a null terminated pointer.
lib/std/zig/tokenizer.zig+35-6
......@@ -371,6 +371,7 @@ pub const Tokenizer = struct {
371371 CharLiteralUnicodeEscapeSawU,
372372 CharLiteralUnicodeEscape,
373373 CharLiteralUnicodeInvalid,
374 CharLiteralUnicode,
374375 CharLiteralEnd,
375376 Backslash,
376377 Equal,
......@@ -427,6 +428,7 @@ pub const Tokenizer = struct {
427428 .end = undefined,
428429 };
429430 var seen_escape_digits: usize = undefined;
431 var remaining_code_units: usize = undefined;
430432 while (self.index < self.buffer.len) : (self.index += 1) {
431433 const c = self.buffer[self.index];
432434 switch (state) {
......@@ -774,16 +776,23 @@ pub const Tokenizer = struct {
774776 '\\' => {
775777 state = State.CharLiteralBackslash;
776778 },
777 '\'' => {
779 '\'', 0x80...0xbf, 0xf8...0xff => {
778780 result.id = Token.Id.Invalid;
779781 break;
780782 },
783 0xc0...0xdf => { // 110xxxxx
784 remaining_code_units = 1;
785 state = State.CharLiteralUnicode;
786 },
787 0xe0...0xef => { // 1110xxxx
788 remaining_code_units = 2;
789 state = State.CharLiteralUnicode;
790 },
791 0xf0...0xf7 => { // 11110xxx
792 remaining_code_units = 3;
793 state = State.CharLiteralUnicode;
794 },
781795 else => {
782 if (c < 0x20 or c == 0x7f) {
783 result.id = Token.Id.Invalid;
784 break;
785 }
786
787796 state = State.CharLiteralEnd;
788797 },
789798 },
......@@ -867,6 +876,19 @@ pub const Tokenizer = struct {
867876 },
868877 },
869878
879 State.CharLiteralUnicode => switch (c) {
880 0x80...0xbf => {
881 remaining_code_units -= 1;
882 if (remaining_code_units == 0) {
883 state = State.CharLiteralEnd;
884 }
885 },
886 else => {
887 result.id = Token.Id.Invalid;
888 break;
889 },
890 },
891
870892 State.MultilineStringLiteralLine => switch (c) {
871893 '\n' => {
872894 self.index += 1;
......@@ -1220,6 +1242,7 @@ pub const Tokenizer = struct {
12201242 State.CharLiteralUnicodeEscape,
12211243 State.CharLiteralUnicodeInvalid,
12221244 State.CharLiteralEnd,
1245 State.CharLiteralUnicode,
12231246 State.StringLiteralBackslash,
12241247 State.LBracketStar,
12251248 State.LBracketStarC,
......@@ -1428,6 +1451,12 @@ test "tokenizer - char literal with unicode escapes" {
14281451 , [_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });
14291452}
14301453
1454test "tokenizer - char literal with unicode code point" {
1455 testTokenize(
1456 \\'💩'
1457 , [_]Token.Id{.CharLiteral});
1458}
1459
14311460test "tokenizer - float literal e exponent" {
14321461 testTokenize("a = 4.94065645841246544177e-324;\n", [_]Token.Id{
14331462 Token.Id.Identifier,
src/tokenizer.cpp+40-11
......@@ -193,6 +193,7 @@ enum TokenizeState {
193193 TokenizeStateStringEscapeUnicodeStart,
194194 TokenizeStateCharLiteral,
195195 TokenizeStateCharLiteralEnd,
196 TokenizeStateCharLiteralUnicode,
196197 TokenizeStateSawStar,
197198 TokenizeStateSawStarPercent,
198199 TokenizeStateSawSlash,
......@@ -250,6 +251,7 @@ struct Tokenize {
250251 int exponent_in_bin_or_dec;
251252 BigInt specified_exponent;
252253 BigInt significand;
254 size_t remaining_code_units;
253255};
254256
255257ATTRIBUTE_PRINTF(2, 3)
......@@ -1221,17 +1223,32 @@ void tokenize(Buf *buf, Tokenization *out) {
12211223 }
12221224 break;
12231225 case TokenizeStateCharLiteral:
1224 switch (c) {
1225 case '\'':
1226 tokenize_error(&t, "expected character");
1227 break;
1228 case '\\':
1229 t.state = TokenizeStateStringEscape;
1230 break;
1231 default:
1232 t.cur_tok->data.char_lit.c = c;
1233 t.state = TokenizeStateCharLiteralEnd;
1234 break;
1226 if (c == '\'') {
1227 tokenize_error(&t, "expected character");
1228 } else if (c == '\\') {
1229 t.state = TokenizeStateStringEscape;
1230 } else if ((c >= 0x80 && c <= 0xbf) || c >= 0xf8) {
1231 // 10xxxxxx
1232 // 11111xxx
1233 invalid_char_error(&t, c);
1234 } else if (c >= 0xc0 && c <= 0xdf) {
1235 // 110xxxxx
1236 t.cur_tok->data.char_lit.c = c & 0x1f;
1237 t.remaining_code_units = 1;
1238 t.state = TokenizeStateCharLiteralUnicode;
1239 } else if (c >= 0xe0 && c <= 0xef) {
1240 // 1110xxxx
1241 t.cur_tok->data.char_lit.c = c & 0x0f;
1242 t.remaining_code_units = 2;
1243 t.state = TokenizeStateCharLiteralUnicode;
1244 } else if (c >= 0xf0 && c <= 0xf7) {
1245 // 11110xxx
1246 t.cur_tok->data.char_lit.c = c & 0x07;
1247 t.remaining_code_units = 3;
1248 t.state = TokenizeStateCharLiteralUnicode;
1249 } else {
1250 t.cur_tok->data.char_lit.c = c;
1251 t.state = TokenizeStateCharLiteralEnd;
12351252 }
12361253 break;
12371254 case TokenizeStateCharLiteralEnd:
......@@ -1244,6 +1261,17 @@ void tokenize(Buf *buf, Tokenization *out) {
12441261 invalid_char_error(&t, c);
12451262 }
12461263 break;
1264 case TokenizeStateCharLiteralUnicode:
1265 if (c <= 0x7f || c >= 0xc0) {
1266 invalid_char_error(&t, c);
1267 }
1268 t.cur_tok->data.char_lit.c <<= 6;
1269 t.cur_tok->data.char_lit.c += c & 0x3f;
1270 t.remaining_code_units--;
1271 if (t.remaining_code_units == 0) {
1272 t.state = TokenizeStateCharLiteralEnd;
1273 }
1274 break;
12471275 case TokenizeStateZero:
12481276 switch (c) {
12491277 case 'b':
......@@ -1479,6 +1507,7 @@ void tokenize(Buf *buf, Tokenization *out) {
14791507 break;
14801508 case TokenizeStateCharLiteral:
14811509 case TokenizeStateCharLiteralEnd:
1510 case TokenizeStateCharLiteralUnicode:
14821511 tokenize_error(&t, "unterminated character literal");
14831512 break;
14841513 case TokenizeStateSymbol:
test/stage1/behavior/misc.zig+4
......@@ -699,6 +699,10 @@ test "unicode escape in character literal" {
699699 expect(a == 128169);
700700}
701701
702test "unicode character in character literal" {
703 expect('💩' == 128169);
704}
705
702706test "result location zero sized array inside struct field implicit cast to slice" {
703707 const E = struct {
704708 entries: []u32,