authorgravatar for n@nirf.deNick Erdmann <n@nirf.de> 2019-10-06 19:52:35+02:00
committergravatar for n@nirf.deNick Erdmann <n@nirf.de> 2019-10-07 08:18:16+02:00
logae7392e504e7765b05d98636cc249cbf92233f5c
tree42fa9e83943c41a4e84e0152b1deb427dfbdc292
parent571123465b2e030b7b9cf42732ed30f77192fbcd
signaturelock-open Commit is signed but in an unrecognized format.

unicode character literals

4 files changed, 81 insertions(+), 19 deletions(-)

doc/langref.html.in+2-2
...@@ -552,8 +552,7 @@ pub fn main() void {...@@ -552,8 +552,7 @@ pub fn main() void {
552 <p>552 <p>
553 Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as553 Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as
554 {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals554 {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals
555 and character literals. Once https://github.com/ziglang/zig/issues/2097 is implemented,555 and character literals.
556 character literals will be allowed to have a single UTF-8 encoded codepoint.
557 </p>556 </p>
558 {#code_begin|test#}557 {#code_begin|test#}
559const assert = @import("std").debug.assert;558const assert = @import("std").debug.assert;
...@@ -567,6 +566,7 @@ test "string literals" {...@@ -567,6 +566,7 @@ test "string literals" {
567 assert(normal_bytes[1] == 'e');566 assert(normal_bytes[1] == 'e');
568 assert('e' == '\x65');567 assert('e' == '\x65');
569 assert('\u{1f4a9}' == 128169);568 assert('\u{1f4a9}' == 128169);
569 assert('💯' == 128175);
570 assert(mem.eql(u8, "hello", "h\x65llo"));570 assert(mem.eql(u8, "hello", "h\x65llo"));
571571
572 // A C string literal is a null terminated pointer.572 // A C string literal is a null terminated pointer.
lib/std/zig/tokenizer.zig+35-6
...@@ -371,6 +371,7 @@ pub const Tokenizer = struct {...@@ -371,6 +371,7 @@ pub const Tokenizer = struct {
371 CharLiteralUnicodeEscapeSawU,371 CharLiteralUnicodeEscapeSawU,
372 CharLiteralUnicodeEscape,372 CharLiteralUnicodeEscape,
373 CharLiteralUnicodeInvalid,373 CharLiteralUnicodeInvalid,
374 CharLiteralUnicode,
374 CharLiteralEnd,375 CharLiteralEnd,
375 Backslash,376 Backslash,
376 Equal,377 Equal,
...@@ -427,6 +428,7 @@ pub const Tokenizer = struct {...@@ -427,6 +428,7 @@ pub const Tokenizer = struct {
427 .end = undefined,428 .end = undefined,
428 };429 };
429 var seen_escape_digits: usize = undefined;430 var seen_escape_digits: usize = undefined;
431 var remaining_code_units: usize = undefined;
430 while (self.index < self.buffer.len) : (self.index += 1) {432 while (self.index < self.buffer.len) : (self.index += 1) {
431 const c = self.buffer[self.index];433 const c = self.buffer[self.index];
432 switch (state) {434 switch (state) {
...@@ -774,16 +776,23 @@ pub const Tokenizer = struct {...@@ -774,16 +776,23 @@ pub const Tokenizer = struct {
774 '\\' => {776 '\\' => {
775 state = State.CharLiteralBackslash;777 state = State.CharLiteralBackslash;
776 },778 },
777 '\'' => {779 '\'', 0x80...0xbf, 0xf8...0xff => {
778 result.id = Token.Id.Invalid;780 result.id = Token.Id.Invalid;
779 break;781 break;
780 },782 },
783 0xc0...0xdf => { // 110xxxxx
784 remaining_code_units = 1;
785 state = State.CharLiteralUnicode;
786 },
787 0xe0...0xef => { // 1110xxxx
788 remaining_code_units = 2;
789 state = State.CharLiteralUnicode;
790 },
791 0xf0...0xf7 => { // 11110xxx
792 remaining_code_units = 3;
793 state = State.CharLiteralUnicode;
794 },
781 else => {795 else => {
782 if (c < 0x20 or c == 0x7f) {
783 result.id = Token.Id.Invalid;
784 break;
785 }
786
787 state = State.CharLiteralEnd;796 state = State.CharLiteralEnd;
788 },797 },
789 },798 },
...@@ -867,6 +876,19 @@ pub const Tokenizer = struct {...@@ -867,6 +876,19 @@ pub const Tokenizer = struct {
867 },876 },
868 },877 },
869878
879 State.CharLiteralUnicode => switch (c) {
880 0x80...0xbf => {
881 remaining_code_units -= 1;
882 if (remaining_code_units == 0) {
883 state = State.CharLiteralEnd;
884 }
885 },
886 else => {
887 result.id = Token.Id.Invalid;
888 break;
889 },
890 },
891
870 State.MultilineStringLiteralLine => switch (c) {892 State.MultilineStringLiteralLine => switch (c) {
871 '\n' => {893 '\n' => {
872 self.index += 1;894 self.index += 1;
...@@ -1220,6 +1242,7 @@ pub const Tokenizer = struct {...@@ -1220,6 +1242,7 @@ pub const Tokenizer = struct {
1220 State.CharLiteralUnicodeEscape,1242 State.CharLiteralUnicodeEscape,
1221 State.CharLiteralUnicodeInvalid,1243 State.CharLiteralUnicodeInvalid,
1222 State.CharLiteralEnd,1244 State.CharLiteralEnd,
1245 State.CharLiteralUnicode,
1223 State.StringLiteralBackslash,1246 State.StringLiteralBackslash,
1224 State.LBracketStar,1247 State.LBracketStar,
1225 State.LBracketStarC,1248 State.LBracketStarC,
...@@ -1428,6 +1451,12 @@ test "tokenizer - char literal with unicode escapes" {...@@ -1428,6 +1451,12 @@ test "tokenizer - char literal with unicode escapes" {
1428 , [_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });1451 , [_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });
1429}1452}
14301453
1454test "tokenizer - char literal with unicode code point" {
1455 testTokenize(
1456 \\'💩'
1457 , [_]Token.Id{.CharLiteral});
1458}
1459
1431test "tokenizer - float literal e exponent" {1460test "tokenizer - float literal e exponent" {
1432 testTokenize("a = 4.94065645841246544177e-324;\n", [_]Token.Id{1461 testTokenize("a = 4.94065645841246544177e-324;\n", [_]Token.Id{
1433 Token.Id.Identifier,1462 Token.Id.Identifier,
src/tokenizer.cpp+40-11
...@@ -193,6 +193,7 @@ enum TokenizeState {...@@ -193,6 +193,7 @@ enum TokenizeState {
193 TokenizeStateStringEscapeUnicodeStart,193 TokenizeStateStringEscapeUnicodeStart,
194 TokenizeStateCharLiteral,194 TokenizeStateCharLiteral,
195 TokenizeStateCharLiteralEnd,195 TokenizeStateCharLiteralEnd,
196 TokenizeStateCharLiteralUnicode,
196 TokenizeStateSawStar,197 TokenizeStateSawStar,
197 TokenizeStateSawStarPercent,198 TokenizeStateSawStarPercent,
198 TokenizeStateSawSlash,199 TokenizeStateSawSlash,
...@@ -247,6 +248,7 @@ struct Tokenize {...@@ -247,6 +248,7 @@ struct Tokenize {
247 int exponent_in_bin_or_dec;248 int exponent_in_bin_or_dec;
248 BigInt specified_exponent;249 BigInt specified_exponent;
249 BigInt significand;250 BigInt significand;
251 size_t remaining_code_units;
250};252};
251253
252ATTRIBUTE_PRINTF(2, 3)254ATTRIBUTE_PRINTF(2, 3)
...@@ -1176,17 +1178,32 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1176,17 +1178,32 @@ void tokenize(Buf *buf, Tokenization *out) {
1176 }1178 }
1177 break;1179 break;
1178 case TokenizeStateCharLiteral:1180 case TokenizeStateCharLiteral:
1179 switch (c) {1181 if (c == '\'') {
1180 case '\'':1182 tokenize_error(&t, "expected character");
1181 tokenize_error(&t, "expected character");1183 } else if (c == '\\') {
1182 break;1184 t.state = TokenizeStateStringEscape;
1183 case '\\':1185 } else if ((c >= 0x80 && c <= 0xbf) || c >= 0xf8) {
1184 t.state = TokenizeStateStringEscape;1186 // 10xxxxxx
1185 break;1187 // 11111xxx
1186 default:1188 invalid_char_error(&t, c);
1187 t.cur_tok->data.char_lit.c = c;1189 } else if (c >= 0xc0 && c <= 0xdf) {
1188 t.state = TokenizeStateCharLiteralEnd;1190 // 110xxxxx
1189 break;1191 t.cur_tok->data.char_lit.c = c & 0x1f;
1192 t.remaining_code_units = 1;
1193 t.state = TokenizeStateCharLiteralUnicode;
1194 } else if (c >= 0xe0 && c <= 0xef) {
1195 // 1110xxxx
1196 t.cur_tok->data.char_lit.c = c & 0x0f;
1197 t.remaining_code_units = 2;
1198 t.state = TokenizeStateCharLiteralUnicode;
1199 } else if (c >= 0xf0 && c <= 0xf7) {
1200 // 11110xxx
1201 t.cur_tok->data.char_lit.c = c & 0x07;
1202 t.remaining_code_units = 3;
1203 t.state = TokenizeStateCharLiteralUnicode;
1204 } else {
1205 t.cur_tok->data.char_lit.c = c;
1206 t.state = TokenizeStateCharLiteralEnd;
1190 }1207 }
1191 break;1208 break;
1192 case TokenizeStateCharLiteralEnd:1209 case TokenizeStateCharLiteralEnd:
...@@ -1199,6 +1216,17 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1199,6 +1216,17 @@ void tokenize(Buf *buf, Tokenization *out) {
1199 invalid_char_error(&t, c);1216 invalid_char_error(&t, c);
1200 }1217 }
1201 break;1218 break;
1219 case TokenizeStateCharLiteralUnicode:
1220 if (c <= 0x7f || c >= 0xc0) {
1221 invalid_char_error(&t, c);
1222 }
1223 t.cur_tok->data.char_lit.c <<= 6;
1224 t.cur_tok->data.char_lit.c += c & 0x3f;
1225 t.remaining_code_units--;
1226 if (t.remaining_code_units == 0) {
1227 t.state = TokenizeStateCharLiteralEnd;
1228 }
1229 break;
1202 case TokenizeStateZero:1230 case TokenizeStateZero:
1203 switch (c) {1231 switch (c) {
1204 case 'b':1232 case 'b':
...@@ -1434,6 +1462,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1434,6 +1462,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1434 break;1462 break;
1435 case TokenizeStateCharLiteral:1463 case TokenizeStateCharLiteral:
1436 case TokenizeStateCharLiteralEnd:1464 case TokenizeStateCharLiteralEnd:
1465 case TokenizeStateCharLiteralUnicode:
1437 tokenize_error(&t, "unterminated character literal");1466 tokenize_error(&t, "unterminated character literal");
1438 break;1467 break;
1439 case TokenizeStateSymbol:1468 case TokenizeStateSymbol:
test/stage1/behavior/misc.zig+4
...@@ -699,6 +699,10 @@ test "unicode escape in character literal" {...@@ -699,6 +699,10 @@ test "unicode escape in character literal" {
699 expect(a == 128169);699 expect(a == 128169);
700}700}
701701
702test "unicode character in character literal" {
703 expect('💩' == 128169);
704}
705
702test "result location zero sized array inside struct field implicit cast to slice" {706test "result location zero sized array inside struct field implicit cast to slice" {
703 const E = struct {707 const E = struct {
704 entries: []u32,708 entries: []u32,