authorgravatar for n@nirf.deNick Erdmann <n@nirf.de> 2019-10-06 19:52:35+02:00
committergravatar for n@nirf.deNick Erdmann <n@nirf.de> 2019-10-07 08:18:16+02:00
logae7392e504e7765b05d98636cc249cbf92233f5c
tree42fa9e83943c41a4e84e0152b1deb427dfbdc292
parent571123465b2e030b7b9cf42732ed30f77192fbcd
signaturelock-open Commit is signed but in an unrecognized format.

unicode character literals

4 files changed, 81 insertions(+), 19 deletions(-)

doc/langref.html.in+2-2
......@@ -552,8 +552,7 @@ pub fn main() void {
552552 <p>
553553 Character literals have type {#syntax#}comptime_int{#endsyntax#}, the same as
554554 {#link|Integer Literals#}. All {#link|Escape Sequences#} are valid in both string literals
555 and character literals. Once https://github.com/ziglang/zig/issues/2097 is implemented,
556 character literals will be allowed to have a single UTF-8 encoded codepoint.
555 and character literals.
557556 </p>
558557 {#code_begin|test#}
559558const assert = @import("std").debug.assert;
......@@ -567,6 +566,7 @@ test "string literals" {
567566 assert(normal_bytes[1] == 'e');
568567 assert('e' == '\x65');
569568 assert('\u{1f4a9}' == 128169);
569 assert('💯' == 128175);
570570 assert(mem.eql(u8, "hello", "h\x65llo"));
571571
572572 // A C string literal is a null terminated pointer.
lib/std/zig/tokenizer.zig+35-6
......@@ -371,6 +371,7 @@ pub const Tokenizer = struct {
371371 CharLiteralUnicodeEscapeSawU,
372372 CharLiteralUnicodeEscape,
373373 CharLiteralUnicodeInvalid,
374 CharLiteralUnicode,
374375 CharLiteralEnd,
375376 Backslash,
376377 Equal,
......@@ -427,6 +428,7 @@ pub const Tokenizer = struct {
427428 .end = undefined,
428429 };
429430 var seen_escape_digits: usize = undefined;
431 var remaining_code_units: usize = undefined;
430432 while (self.index < self.buffer.len) : (self.index += 1) {
431433 const c = self.buffer[self.index];
432434 switch (state) {
......@@ -774,16 +776,23 @@ pub const Tokenizer = struct {
774776 '\\' => {
775777 state = State.CharLiteralBackslash;
776778 },
777 '\'' => {
779 '\'', 0x80...0xbf, 0xf8...0xff => {
778780 result.id = Token.Id.Invalid;
779781 break;
780782 },
783 0xc0...0xdf => { // 110xxxxx
784 remaining_code_units = 1;
785 state = State.CharLiteralUnicode;
786 },
787 0xe0...0xef => { // 1110xxxx
788 remaining_code_units = 2;
789 state = State.CharLiteralUnicode;
790 },
791 0xf0...0xf7 => { // 11110xxx
792 remaining_code_units = 3;
793 state = State.CharLiteralUnicode;
794 },
781795 else => {
782 if (c < 0x20 or c == 0x7f) {
783 result.id = Token.Id.Invalid;
784 break;
785 }
786
787796 state = State.CharLiteralEnd;
788797 },
789798 },
......@@ -867,6 +876,19 @@ pub const Tokenizer = struct {
867876 },
868877 },
869878
879 State.CharLiteralUnicode => switch (c) {
880 0x80...0xbf => {
881 remaining_code_units -= 1;
882 if (remaining_code_units == 0) {
883 state = State.CharLiteralEnd;
884 }
885 },
886 else => {
887 result.id = Token.Id.Invalid;
888 break;
889 },
890 },
891
870892 State.MultilineStringLiteralLine => switch (c) {
871893 '\n' => {
872894 self.index += 1;
......@@ -1220,6 +1242,7 @@ pub const Tokenizer = struct {
12201242 State.CharLiteralUnicodeEscape,
12211243 State.CharLiteralUnicodeInvalid,
12221244 State.CharLiteralEnd,
1245 State.CharLiteralUnicode,
12231246 State.StringLiteralBackslash,
12241247 State.LBracketStar,
12251248 State.LBracketStarC,
......@@ -1428,6 +1451,12 @@ test "tokenizer - char literal with unicode escapes" {
14281451 , [_]Token.Id{ .Invalid, .IntegerLiteral, .Invalid });
14291452}
14301453
1454test "tokenizer - char literal with unicode code point" {
1455 testTokenize(
1456 \\'💩'
1457 , [_]Token.Id{.CharLiteral});
1458}
1459
14311460test "tokenizer - float literal e exponent" {
14321461 testTokenize("a = 4.94065645841246544177e-324;\n", [_]Token.Id{
14331462 Token.Id.Identifier,
src/tokenizer.cpp+40-11
......@@ -193,6 +193,7 @@ enum TokenizeState {
193193 TokenizeStateStringEscapeUnicodeStart,
194194 TokenizeStateCharLiteral,
195195 TokenizeStateCharLiteralEnd,
196 TokenizeStateCharLiteralUnicode,
196197 TokenizeStateSawStar,
197198 TokenizeStateSawStarPercent,
198199 TokenizeStateSawSlash,
......@@ -247,6 +248,7 @@ struct Tokenize {
247248 int exponent_in_bin_or_dec;
248249 BigInt specified_exponent;
249250 BigInt significand;
251 size_t remaining_code_units;
250252};
251253
252254ATTRIBUTE_PRINTF(2, 3)
......@@ -1176,17 +1178,32 @@ void tokenize(Buf *buf, Tokenization *out) {
11761178 }
11771179 break;
11781180 case TokenizeStateCharLiteral:
1179 switch (c) {
1180 case '\'':
1181 tokenize_error(&t, "expected character");
1182 break;
1183 case '\\':
1184 t.state = TokenizeStateStringEscape;
1185 break;
1186 default:
1187 t.cur_tok->data.char_lit.c = c;
1188 t.state = TokenizeStateCharLiteralEnd;
1189 break;
1181 if (c == '\'') {
1182 tokenize_error(&t, "expected character");
1183 } else if (c == '\\') {
1184 t.state = TokenizeStateStringEscape;
1185 } else if ((c >= 0x80 && c <= 0xbf) || c >= 0xf8) {
1186 // 10xxxxxx
1187 // 11111xxx
1188 invalid_char_error(&t, c);
1189 } else if (c >= 0xc0 && c <= 0xdf) {
1190 // 110xxxxx
1191 t.cur_tok->data.char_lit.c = c & 0x1f;
1192 t.remaining_code_units = 1;
1193 t.state = TokenizeStateCharLiteralUnicode;
1194 } else if (c >= 0xe0 && c <= 0xef) {
1195 // 1110xxxx
1196 t.cur_tok->data.char_lit.c = c & 0x0f;
1197 t.remaining_code_units = 2;
1198 t.state = TokenizeStateCharLiteralUnicode;
1199 } else if (c >= 0xf0 && c <= 0xf7) {
1200 // 11110xxx
1201 t.cur_tok->data.char_lit.c = c & 0x07;
1202 t.remaining_code_units = 3;
1203 t.state = TokenizeStateCharLiteralUnicode;
1204 } else {
1205 t.cur_tok->data.char_lit.c = c;
1206 t.state = TokenizeStateCharLiteralEnd;
11901207 }
11911208 break;
11921209 case TokenizeStateCharLiteralEnd:
......@@ -1199,6 +1216,17 @@ void tokenize(Buf *buf, Tokenization *out) {
11991216 invalid_char_error(&t, c);
12001217 }
12011218 break;
1219 case TokenizeStateCharLiteralUnicode:
1220 if (c <= 0x7f || c >= 0xc0) {
1221 invalid_char_error(&t, c);
1222 }
1223 t.cur_tok->data.char_lit.c <<= 6;
1224 t.cur_tok->data.char_lit.c += c & 0x3f;
1225 t.remaining_code_units--;
1226 if (t.remaining_code_units == 0) {
1227 t.state = TokenizeStateCharLiteralEnd;
1228 }
1229 break;
12021230 case TokenizeStateZero:
12031231 switch (c) {
12041232 case 'b':
......@@ -1434,6 +1462,7 @@ void tokenize(Buf *buf, Tokenization *out) {
14341462 break;
14351463 case TokenizeStateCharLiteral:
14361464 case TokenizeStateCharLiteralEnd:
1465 case TokenizeStateCharLiteralUnicode:
14371466 tokenize_error(&t, "unterminated character literal");
14381467 break;
14391468 case TokenizeStateSymbol:
test/stage1/behavior/misc.zig+4
......@@ -699,6 +699,10 @@ test "unicode escape in character literal" {
699699 expect(a == 128169);
700700}
701701
702test "unicode character in character literal" {
703 expect('💩' == 128169);
704}
705
702706test "result location zero sized array inside struct field implicit cast to slice" {
703707 const E = struct {
704708 entries: []u32,