authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-05-01 14:53:48-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-05-01 14:53:48-07:00
log9ccd0ba9611d7828f42bffca919c7ad3177cbbe1
tree4600e2c82685c1c1fb674d1f200a154c5cf4b1ab
parent037283c3b3e425193eebb766c17336ee55aa0384

implement string escapes

4 files changed, 180 insertions(+), 34 deletions(-)

doc/langref.md+18-21
......@@ -272,10 +272,26 @@ Literal Example Characters Escapes Null Term Type
272272Byte 'H' All ASCII Byte No u8
273273UTF-8 Bytes "hello" All Unicode Byte & Unicode No [5]u8
274274UTF-8 C string c"hello" All Unicode Byte & Unicode Yes &const u8
275UTF-8 Raw String r"A(hello)A" All Unicode None No [5]u8
276UTF-8 Raw C String rc"A(hello)A" All Unicode None Yes &const u8
275UTF-8 Raw String r"X(hello)X" All Unicode None No [5]u8
276UTF-8 Raw C String rc"X(hello)X" All Unicode None Yes &const u8
277277```
278278
279### Escapes
280
281 Escape | Name
282----------|-------------------------------------------------------------------
283 \n | Newline
284 \r | Carriage Return
285 \t | Tab
286 \\ | Backslash
287 \' | Single Quote
288 \" | Double Quote
289 \xNN | hexadecimal 8-bit character code (2 digits)
290 \uNNNN | hexadecimal 16-bit Unicode character code UTF-8 encoded (4 digits)
291 \UNNNNNN | hexadecimal 24-bit Unicode character code UTF-8 encoded (6 digits)
292
293Note that the maximum valid Unicode point is 0x10ffff.
294
279295##### Raw Strings
280296
281297Raw string literals have no escapes and can span across multiple lines. To
......@@ -283,25 +299,6 @@ start a raw string, use 'r"' or 'rc"' followed by unique bytes followed by '('.
283299To end a raw string, use ')' followed by the same unique bytes, followed by '"'.
284300
285301
286```
287Escape Name
288
289\xNN hexadecimal 8-bit character code (exactly 2 digits)
290\n Newline
291\r Carriage return
292\t Tab
293\\ Backslash
294\0 Null
295\' Single quote
296\" Double quote
297```
298
299### Unicode Escapes
300
301 Escape | Name
302------------|-----------------------------------------------
303 \u{NNNNNN} | hexadecimal 24-bit Unicode character code (up to 6 digits)
304
305302#### Numeric Literals
306303
307304```
src/parser.cpp+69-7
......@@ -219,7 +219,7 @@ static uint8_t parse_char_literal(ParseContext *pc, Token *token) {
219219 return return_value;
220220}
221221
222static int get_hex_digit(uint8_t c) {
222static uint32_t get_hex_digit(uint8_t c) {
223223 switch (c) {
224224 case '0': return 0;
225225 case '1': return 1;
......@@ -251,7 +251,7 @@ static int get_hex_digit(uint8_t c) {
251251 case 'F':
252252 return 15;
253253 default:
254 return -1;
254 return UINT32_MAX;
255255 }
256256}
257257
......@@ -279,13 +279,17 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
279279 StateEscape,
280280 StateHex1,
281281 StateHex2,
282 StateUnicode,
282283 };
283284
284285 buf_resize(buf, 0);
285286
287 int unicode_index;
288 int unicode_end;
289
286290 State state = StatePre;
287291 SrcPos pos = {token->start_line, token->start_column};
288 int hex_value = 0;
292 uint32_t hex_value = 0;
289293 for (int i = token->start_pos; i < token->end_pos - 1; i += 1) {
290294 uint8_t c = *((uint8_t*)buf_ptr(pc->buf) + i);
291295
......@@ -348,17 +352,34 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
348352 if (offset_map) offset_map->append(pos);
349353 state = StateStart;
350354 break;
355 case '\'':
356 buf_append_char(buf, '\'');
357 if (offset_map) offset_map->append(pos);
358 state = StateStart;
359 break;
351360 case 'x':
352361 state = StateHex1;
353362 break;
363 case 'u':
364 state = StateUnicode;
365 unicode_index = 0;
366 unicode_end = 4;
367 hex_value = 0;
368 break;
369 case 'U':
370 state = StateUnicode;
371 unicode_index = 0;
372 unicode_end = 6;
373 hex_value = 0;
374 break;
354375 default:
355376 ast_error(pc, token, "invalid escape character");
356377 }
357378 break;
358379 case StateHex1:
359380 {
360 int hex_digit = get_hex_digit(c);
361 if (hex_digit == -1) {
381 uint32_t hex_digit = get_hex_digit(c);
382 if (hex_digit == UINT32_MAX) {
362383 ast_error(pc, token, "invalid hex digit: '%c'", c);
363384 }
364385 hex_value = hex_digit * 16;
......@@ -367,8 +388,8 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
367388 }
368389 case StateHex2:
369390 {
370 int hex_digit = get_hex_digit(c);
371 if (hex_digit == -1) {
391 uint32_t hex_digit = get_hex_digit(c);
392 if (hex_digit == UINT32_MAX) {
372393 ast_error(pc, token, "invalid hex digit: '%c'", c);
373394 }
374395 hex_value += hex_digit;
......@@ -377,6 +398,47 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
377398 state = StateStart;
378399 break;
379400 }
401 case StateUnicode:
402 {
403 uint32_t hex_digit = get_hex_digit(c);
404 if (hex_digit == UINT32_MAX) {
405 ast_error(pc, token, "invalid hex digit: '%c'", c);
406 }
407 hex_value *= 16;
408 hex_value += hex_digit;
409 unicode_index += 1;
410 if (unicode_index >= unicode_end) {
411 if (hex_value <= 0x7f) {
412 // 00000000 00000000 00000000 0xxxxxxx
413 buf_append_char(buf, hex_value);
414 } else if (hex_value <= 0x7ff) {
415 // 00000000 00000000 00000xxx xx000000
416 buf_append_char(buf, (unsigned char)(0xc0 | (hex_value >> 6)));
417 // 00000000 00000000 00000000 00xxxxxx
418 buf_append_char(buf, (unsigned char)(0x80 | (hex_value & 0x3f)));
419 } else if (hex_value <= 0xffff) {
420 // 00000000 00000000 xxxx0000 00000000
421 buf_append_char(buf, (unsigned char)(0xe0 | (hex_value >> 12)));
422 // 00000000 00000000 0000xxxx xx000000
423 buf_append_char(buf, (unsigned char)(0x80 | ((hex_value >> 6) & 0x3f)));
424 // 00000000 00000000 00000000 00xxxxxx
425 buf_append_char(buf, (unsigned char)(0x80 | (hex_value & 0x3f)));
426 } else if (hex_value <= 0x10ffff) {
427 // 00000000 000xxx00 00000000 00000000
428 buf_append_char(buf, (unsigned char)(0xf0 | (hex_value >> 18)));
429 // 00000000 000000xx xxxx0000 00000000
430 buf_append_char(buf, (unsigned char)(0x80 | ((hex_value >> 12) & 0x3f)));
431 // 00000000 00000000 0000xxxx xx000000
432 buf_append_char(buf, (unsigned char)(0x80 | ((hex_value >> 6) & 0x3f)));
433 // 00000000 00000000 00000000 00xxxxxx
434 buf_append_char(buf, (unsigned char)(0x80 | (hex_value & 0x3f)));
435 } else {
436 ast_error(pc, token, "unicode value out of range: %x", hex_value);
437 }
438 state = StateStart;
439 }
440 break;
441 }
380442 }
381443 if (c == '\n') {
382444 pos.line += 1;
src/tokenizer.cpp+82-6
......@@ -103,6 +103,21 @@
103103 ALPHA: \
104104 case '_'
105105
106#define HEX_DIGIT \
107 'a': \
108 case 'b': \
109 case 'c': \
110 case 'd': \
111 case 'e': \
112 case 'f': \
113 case 'A': \
114 case 'B': \
115 case 'C': \
116 case 'D': \
117 case 'E': \
118 case 'F': \
119 case DIGIT
120
106121const char * zig_keywords[] = {
107122 "true", "false", "null", "fn", "return", "var", "const", "extern",
108123 "pub", "export", "use", "if", "else", "goto", "asm",
......@@ -132,11 +147,11 @@ enum TokenizeState {
132147 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
133148 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
134149 TokenizeStateString,
150 TokenizeStateStringEscape,
135151 TokenizeStateRawString,
136152 TokenizeStateRawStringContents,
137153 TokenizeStateRawStringMaybeEnd,
138154 TokenizeStateCharLiteral,
139 TokenizeStateCharLiteralEscape,
140155 TokenizeStateCharLiteralEnd,
141156 TokenizeStateSawStar,
142157 TokenizeStateSawSlash,
......@@ -162,6 +177,7 @@ enum TokenizeState {
162177 TokenizeStateSawDotDot,
163178 TokenizeStateSawQuestionMark,
164179 TokenizeStateSawAtSign,
180 TokenizeStateHex,
165181 TokenizeStateError,
166182};
167183
......@@ -179,6 +195,7 @@ struct Tokenize {
179195 int raw_string_id_start;
180196 int raw_string_id_end;
181197 int raw_string_id_cmp_pos;
198 int hex_chars_left;
182199};
183200
184201__attribute__ ((format (printf, 2, 3)))
......@@ -921,10 +938,63 @@ void tokenize(Buf *buf, Tokenization *out) {
921938 case '\n':
922939 tokenize_error(&t, "use raw string for multiline string literal");
923940 break;
941 case '\\':
942 t.state = TokenizeStateStringEscape;
943 break;
924944 default:
925945 break;
926946 }
927947 break;
948 case TokenizeStateStringEscape:
949 switch (c) {
950 case 'x':
951 t.state = TokenizeStateHex;
952 t.hex_chars_left = 2;
953 break;
954 case 'u':
955 t.state = TokenizeStateHex;
956 t.hex_chars_left = 4;
957 break;
958 case 'U':
959 t.state = TokenizeStateHex;
960 t.hex_chars_left = 6;
961 break;
962 case 'n':
963 case 'r':
964 case '\\':
965 case 't':
966 case '\'':
967 case '"':
968 if (t.cur_tok->id == TokenIdCharLiteral) {
969 t.state = TokenizeStateCharLiteralEnd;
970 } else if (t.cur_tok->id == TokenIdStringLiteral) {
971 t.state = TokenizeStateString;
972 } else {
973 zig_unreachable();
974 }
975 break;
976 default:
977 tokenize_error(&t, "invalid character: '%c'", c);
978 }
979 break;
980 case TokenizeStateHex:
981 switch (c) {
982 case HEX_DIGIT:
983 t.hex_chars_left -= 1;
984 if (t.hex_chars_left == 0) {
985 if (t.cur_tok->id == TokenIdCharLiteral) {
986 t.state = TokenizeStateCharLiteralEnd;
987 } else if (t.cur_tok->id == TokenIdStringLiteral) {
988 t.state = TokenizeStateString;
989 } else {
990 zig_unreachable();
991 }
992 }
993 break;
994 default:
995 tokenize_error(&t, "invalid character: '%c'", c);
996 }
997 break;
928998 case TokenizeStateRawString:
929999 if (c == '(') {
9301000 t.raw_string_id_end = t.pos;
......@@ -963,16 +1033,13 @@ void tokenize(Buf *buf, Tokenization *out) {
9631033 t.state = TokenizeStateStart;
9641034 break;
9651035 case '\\':
966 t.state = TokenizeStateCharLiteralEscape;
1036 t.state = TokenizeStateStringEscape;
9671037 break;
9681038 default:
9691039 t.state = TokenizeStateCharLiteralEnd;
9701040 break;
9711041 }
9721042 break;
973 case TokenizeStateCharLiteralEscape:
974 t.state = TokenizeStateCharLiteralEnd;
975 break;
9761043 case TokenizeStateCharLiteralEnd:
9771044 switch (c) {
9781045 case '\'':
......@@ -1136,13 +1203,22 @@ void tokenize(Buf *buf, Tokenization *out) {
11361203 case TokenizeStateString:
11371204 tokenize_error(&t, "unterminated string");
11381205 break;
1206 case TokenizeStateStringEscape:
1207 case TokenizeStateHex:
1208 if (t.cur_tok->id == TokenIdStringLiteral) {
1209 tokenize_error(&t, "unterminated string");
1210 } else if (t.cur_tok->id == TokenIdCharLiteral) {
1211 tokenize_error(&t, "unterminated character literal");
1212 } else {
1213 zig_unreachable();
1214 }
1215 break;
11391216 case TokenizeStateRawString:
11401217 case TokenizeStateRawStringContents:
11411218 case TokenizeStateRawStringMaybeEnd:
11421219 tokenize_error(&t, "unterminated raw string");
11431220 break;
11441221 case TokenizeStateCharLiteral:
1145 case TokenizeStateCharLiteralEscape:
11461222 case TokenizeStateCharLiteralEnd:
11471223 tokenize_error(&t, "unterminated character literal");
11481224 break;
test/self_hosted.zig+11
......@@ -1398,3 +1398,14 @@ fn test_take_address_of_parameter_noeval(f: f32) {
13981398fn array_mult_operator() {
13991399 assert(str.eql("ab" ** 5, "ababababab"));
14001400}
1401
1402#attribute("test")
1403fn string_escapes() {
1404 assert(str.eql("\"", "\x22"));
1405 assert(str.eql("\'", "\x27"));
1406 assert(str.eql("\n", "\x0a"));
1407 assert(str.eql("\r", "\x0d"));
1408 assert(str.eql("\t", "\x09"));
1409 assert(str.eql("\\", "\x5c"));
1410 assert(str.eql("\u1234\u0069", "\xe1\x88\xb4\x69"));
1411}