authorgravatar for codroid@gmail.comhryx <codroid@gmail.com> 2019-07-04 22:40:19-07:00
committergravatar for codroid@gmail.comhryx <codroid@gmail.com> 2019-07-04 22:40:19-07:00
log6bfa8546bbdf6dd644a65876135893339b767bba
treeeffea625eaa95eb85392df60304ff1bc1e144edf
parent8365a7aab49938ff77228b72388301f562287415
signaturelock-open Commit is signed but in an unrecognized format.

Unicode escapes: stage1 tokenizer and behavior tests

3 files changed, 81 insertions(+), 56 deletions(-)

src/tokenizer.cpp+61-54
......@@ -190,6 +190,7 @@ enum TokenizeState {
190190 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
191191 TokenizeStateString,
192192 TokenizeStateStringEscape,
193 TokenizeStateStringEscapeUnicodeStart,
193194 TokenizeStateCharLiteral,
194195 TokenizeStateCharLiteralEnd,
195196 TokenizeStateSawStar,
......@@ -241,7 +242,6 @@ struct Tokenize {
241242 int32_t exp_add_amt;
242243 bool is_exp_negative;
243244 size_t char_code_index;
244 size_t char_code_end;
245245 bool unicode;
246246 uint32_t char_code;
247247 int exponent_in_bin_or_dec;
......@@ -1071,24 +1071,10 @@ void tokenize(Buf *buf, Tokenization *out) {
10711071 t.radix = 16;
10721072 t.char_code = 0;
10731073 t.char_code_index = 0;
1074 t.char_code_end = 2;
10751074 t.unicode = false;
10761075 break;
10771076 case 'u':
1078 t.state = TokenizeStateCharCode;
1079 t.radix = 16;
1080 t.char_code = 0;
1081 t.char_code_index = 0;
1082 t.char_code_end = 4;
1083 t.unicode = true;
1084 break;
1085 case 'U':
1086 t.state = TokenizeStateCharCode;
1087 t.radix = 16;
1088 t.char_code = 0;
1089 t.char_code_index = 0;
1090 t.char_code_end = 6;
1091 t.unicode = true;
1077 t.state = TokenizeStateStringEscapeUnicodeStart;
10921078 break;
10931079 case 'n':
10941080 handle_string_escape(&t, '\n');
......@@ -1112,8 +1098,63 @@ void tokenize(Buf *buf, Tokenization *out) {
11121098 invalid_char_error(&t, c);
11131099 }
11141100 break;
1101 case TokenizeStateStringEscapeUnicodeStart:
1102 switch (c) {
1103 case '{':
1104 t.state = TokenizeStateCharCode;
1105 t.radix = 16;
1106 t.char_code = 0;
1107 t.char_code_index = 0;
1108 t.unicode = true;
1109 break;
1110 default:
1111 invalid_char_error(&t, c);
1112 }
1113 break;
11151114 case TokenizeStateCharCode:
11161115 {
1116 if (t.unicode && c == '}') {
1117 if (t.char_code_index == 0) {
1118 tokenize_error(&t, "empty unicode escape sequence");
1119 break;
1120 }
1121 if (t.char_code > 0x10ffff) {
1122 tokenize_error(&t, "unicode value out of range: %x", t.char_code);
1123 break;
1124 }
1125 if (t.cur_tok->id == TokenIdCharLiteral) {
1126 t.cur_tok->data.char_lit.c = t.char_code;
1127 t.state = TokenizeStateCharLiteralEnd;
1128 } else if (t.char_code <= 0x7f) {
1129 // 00000000 00000000 00000000 0xxxxxxx
1130 handle_string_escape(&t, (uint8_t)t.char_code);
1131 } else if (t.char_code <= 0x7ff) {
1132 // 00000000 00000000 00000xxx xx000000
1133 handle_string_escape(&t, (uint8_t)(0xc0 | (t.char_code >> 6)));
1134 // 00000000 00000000 00000000 00xxxxxx
1135 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1136 } else if (t.char_code <= 0xffff) {
1137 // 00000000 00000000 xxxx0000 00000000
1138 handle_string_escape(&t, (uint8_t)(0xe0 | (t.char_code >> 12)));
1139 // 00000000 00000000 0000xxxx xx000000
1140 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f)));
1141 // 00000000 00000000 00000000 00xxxxxx
1142 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1143 } else if (t.char_code <= 0x10ffff) {
1144 // 00000000 000xxx00 00000000 00000000
1145 handle_string_escape(&t, (uint8_t)(0xf0 | (t.char_code >> 18)));
1146 // 00000000 000000xx xxxx0000 00000000
1147 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 12) & 0x3f)));
1148 // 00000000 00000000 0000xxxx xx000000
1149 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f)));
1150 // 00000000 00000000 00000000 00xxxxxx
1151 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1152 } else {
1153 zig_unreachable();
1154 }
1155 break;
1156 }
1157
11171158 uint32_t digit_value = get_digit_value(c);
11181159 if (digit_value >= t.radix) {
11191160 tokenize_error(&t, "invalid digit: '%c'", c);
......@@ -1123,44 +1164,9 @@ void tokenize(Buf *buf, Tokenization *out) {
11231164 t.char_code += digit_value;
11241165 t.char_code_index += 1;
11251166
1126 if (t.char_code_index >= t.char_code_end) {
1127 if (t.unicode) {
1128 if (t.char_code > 0x10ffff) {
1129 tokenize_error(&t, "unicode value out of range: %x", t.char_code);
1130 break;
1131 }
1132 if (t.cur_tok->id == TokenIdCharLiteral) {
1133 t.cur_tok->data.char_lit.c = t.char_code;
1134 t.state = TokenizeStateCharLiteralEnd;
1135 } else if (t.char_code <= 0x7f) {
1136 // 00000000 00000000 00000000 0xxxxxxx
1137 handle_string_escape(&t, (uint8_t)t.char_code);
1138 } else if (t.char_code <= 0x7ff) {
1139 // 00000000 00000000 00000xxx xx000000
1140 handle_string_escape(&t, (uint8_t)(0xc0 | (t.char_code >> 6)));
1141 // 00000000 00000000 00000000 00xxxxxx
1142 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1143 } else if (t.char_code <= 0xffff) {
1144 // 00000000 00000000 xxxx0000 00000000
1145 handle_string_escape(&t, (uint8_t)(0xe0 | (t.char_code >> 12)));
1146 // 00000000 00000000 0000xxxx xx000000
1147 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f)));
1148 // 00000000 00000000 00000000 00xxxxxx
1149 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1150 } else if (t.char_code <= 0x10ffff) {
1151 // 00000000 000xxx00 00000000 00000000
1152 handle_string_escape(&t, (uint8_t)(0xf0 | (t.char_code >> 18)));
1153 // 00000000 000000xx xxxx0000 00000000
1154 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 12) & 0x3f)));
1155 // 00000000 00000000 0000xxxx xx000000
1156 handle_string_escape(&t, (uint8_t)(0x80 | ((t.char_code >> 6) & 0x3f)));
1157 // 00000000 00000000 00000000 00xxxxxx
1158 handle_string_escape(&t, (uint8_t)(0x80 | (t.char_code & 0x3f)));
1159 }
1160 } else {
1161 assert(t.char_code <= 255);
1162 handle_string_escape(&t, (uint8_t)t.char_code);
1163 }
1167 if (!t.unicode && t.char_code_index >= 2) {
1168 assert(t.char_code <= 255);
1169 handle_string_escape(&t, (uint8_t)t.char_code);
11641170 }
11651171 }
11661172 break;
......@@ -1409,6 +1415,7 @@ void tokenize(Buf *buf, Tokenization *out) {
14091415 tokenize_error(&t, "unterminated string");
14101416 break;
14111417 case TokenizeStateStringEscape:
1418 case TokenizeStateStringEscapeUnicodeStart:
14121419 case TokenizeStateCharCode:
14131420 if (t.cur_tok->id == TokenIdStringLiteral) {
14141421 tokenize_error(&t, "unterminated string");
test/compile_errors.zig+18
......@@ -5414,6 +5414,24 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
54145414 "tmp.zig:1:17: error: invalid carriage return, only '\\n' line endings are supported",
54155415 );
54165416
5417 cases.add(
5418 "invalid legacy unicode escape",
5419 \\export fn entry() void {
5420 \\ const a = '\U1234';
5421 \\}
5422 ,
5423 "tmp.zig:2:17: error: invalid character: 'U'",
5424 );
5425
5426 cases.add(
5427 "invalid empty unicode escape",
5428 \\export fn entry() void {
5429 \\ const a = '\u{}';
5430 \\}
5431 ,
5432 "tmp.zig:2:19: error: empty unicode escape sequence",
5433 );
5434
54175435 cases.add(
54185436 "non-printable invalid character",
54195437 "\xff\xfe" ++
test/stage1/behavior/misc.zig+2-2
......@@ -189,7 +189,7 @@ test "string escapes" {
189189 expect(mem.eql(u8, "\r", "\x0d"));
190190 expect(mem.eql(u8, "\t", "\x09"));
191191 expect(mem.eql(u8, "\\", "\x5c"));
192 expect(mem.eql(u8, "\u1234\u0069", "\xe1\x88\xb4\x69"));
192 expect(mem.eql(u8, "\u{1234}\u{069}\u{1}", "\xe1\x88\xb4\x69\x01"));
193193}
194194
195195test "multiline string" {
......@@ -695,7 +695,7 @@ test "thread local variable" {
695695}
696696
697697test "unicode escape in character literal" {
698 var a: u24 = '\U01f4a9';
698 var a: u24 = '\u{01f4a9}';
699699 expect(a == 128169);
700700}
701701