authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-05-01 14:53:48-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-05-01 14:53:48-07:00
log9ccd0ba9611d7828f42bffca919c7ad3177cbbe1
tree4600e2c82685c1c1fb674d1f200a154c5cf4b1ab
parent037283c3b3e425193eebb766c17336ee55aa0384

implement string escapes

4 files changed, 180 insertions(+), 34 deletions(-)

doc/langref.md+18-21
...@@ -272,10 +272,26 @@ Literal Example Characters Escapes Null Term Type...@@ -272,10 +272,26 @@ Literal Example Characters Escapes Null Term Type
272Byte 'H' All ASCII Byte No u8272Byte 'H' All ASCII Byte No u8
273UTF-8 Bytes "hello" All Unicode Byte & Unicode No [5]u8273UTF-8 Bytes "hello" All Unicode Byte & Unicode No [5]u8
274UTF-8 C string c"hello" All Unicode Byte & Unicode Yes &const u8274UTF-8 C string c"hello" All Unicode Byte & Unicode Yes &const u8
275UTF-8 Raw String r"A(hello)A" All Unicode None No [5]u8275UTF-8 Raw String r"X(hello)X" All Unicode None No [5]u8
276UTF-8 Raw C String rc"A(hello)A" All Unicode None Yes &const u8276UTF-8 Raw C String rc"X(hello)X" All Unicode None Yes &const u8
277```277```
278278
279### Escapes
280
281 Escape | Name
282----------|-------------------------------------------------------------------
283 \n | Newline
284 \r | Carriage Return
285 \t | Tab
286 \\ | Backslash
287 \' | Single Quote
288 \" | Double Quote
289 \xNN | hexadecimal 8-bit character code (2 digits)
290 \uNNNN | hexadecimal 16-bit Unicode character code UTF-8 encoded (4 digits)
291 \UNNNNNN | hexadecimal 24-bit Unicode character code UTF-8 encoded (6 digits)
292
293Note that the maximum valid Unicode point is 0x10ffff.
294
279##### Raw Strings295##### Raw Strings
280296
281Raw string literals have no escapes and can span across multiple lines. To297Raw string literals have no escapes and can span across multiple lines. To
...@@ -283,25 +299,6 @@ start a raw string, use 'r"' or 'rc"' followed by unique bytes followed by '('....@@ -283,25 +299,6 @@ start a raw string, use 'r"' or 'rc"' followed by unique bytes followed by '('.
283To end a raw string, use ')' followed by the same unique bytes, followed by '"'.299To end a raw string, use ')' followed by the same unique bytes, followed by '"'.
284300
285301
286```
287Escape Name
288
289\xNN hexadecimal 8-bit character code (exactly 2 digits)
290\n Newline
291\r Carriage return
292\t Tab
293\\ Backslash
294\0 Null
295\' Single quote
296\" Double quote
297```
298
299### Unicode Escapes
300
301 Escape | Name
302------------|-----------------------------------------------
303 \u{NNNNNN} | hexadecimal 24-bit Unicode character code (up to 6 digits)
304
305#### Numeric Literals302#### Numeric Literals
306303
307```304```
src/parser.cpp+69-7
...@@ -219,7 +219,7 @@ static uint8_t parse_char_literal(ParseContext *pc, Token *token) {...@@ -219,7 +219,7 @@ static uint8_t parse_char_literal(ParseContext *pc, Token *token) {
219 return return_value;219 return return_value;
220}220}
221221
222static int get_hex_digit(uint8_t c) {222static uint32_t get_hex_digit(uint8_t c) {
223 switch (c) {223 switch (c) {
224 case '0': return 0;224 case '0': return 0;
225 case '1': return 1;225 case '1': return 1;
...@@ -251,7 +251,7 @@ static int get_hex_digit(uint8_t c) {...@@ -251,7 +251,7 @@ static int get_hex_digit(uint8_t c) {
251 case 'F':251 case 'F':
252 return 15;252 return 15;
253 default:253 default:
254 return -1;254 return UINT32_MAX;
255 }255 }
256}256}
257257
...@@ -279,13 +279,17 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool...@@ -279,13 +279,17 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
279 StateEscape,279 StateEscape,
280 StateHex1,280 StateHex1,
281 StateHex2,281 StateHex2,
282 StateUnicode,
282 };283 };
283284
284 buf_resize(buf, 0);285 buf_resize(buf, 0);
285286
287 int unicode_index;
288 int unicode_end;
289
286 State state = StatePre;290 State state = StatePre;
287 SrcPos pos = {token->start_line, token->start_column};291 SrcPos pos = {token->start_line, token->start_column};
288 int hex_value = 0;292 uint32_t hex_value = 0;
289 for (int i = token->start_pos; i < token->end_pos - 1; i += 1) {293 for (int i = token->start_pos; i < token->end_pos - 1; i += 1) {
290 uint8_t c = *((uint8_t*)buf_ptr(pc->buf) + i);294 uint8_t c = *((uint8_t*)buf_ptr(pc->buf) + i);
291295
...@@ -348,17 +352,34 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool...@@ -348,17 +352,34 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
348 if (offset_map) offset_map->append(pos);352 if (offset_map) offset_map->append(pos);
349 state = StateStart;353 state = StateStart;
350 break;354 break;
355 case '\'':
356 buf_append_char(buf, '\'');
357 if (offset_map) offset_map->append(pos);
358 state = StateStart;
359 break;
351 case 'x':360 case 'x':
352 state = StateHex1;361 state = StateHex1;
353 break;362 break;
363 case 'u':
364 state = StateUnicode;
365 unicode_index = 0;
366 unicode_end = 4;
367 hex_value = 0;
368 break;
369 case 'U':
370 state = StateUnicode;
371 unicode_index = 0;
372 unicode_end = 6;
373 hex_value = 0;
374 break;
354 default:375 default:
355 ast_error(pc, token, "invalid escape character");376 ast_error(pc, token, "invalid escape character");
356 }377 }
357 break;378 break;
358 case StateHex1:379 case StateHex1:
359 {380 {
360 int hex_digit = get_hex_digit(c);381 uint32_t hex_digit = get_hex_digit(c);
361 if (hex_digit == -1) {382 if (hex_digit == UINT32_MAX) {
362 ast_error(pc, token, "invalid hex digit: '%c'", c);383 ast_error(pc, token, "invalid hex digit: '%c'", c);
363 }384 }
364 hex_value = hex_digit * 16;385 hex_value = hex_digit * 16;
...@@ -367,8 +388,8 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool...@@ -367,8 +388,8 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
367 }388 }
368 case StateHex2:389 case StateHex2:
369 {390 {
370 int hex_digit = get_hex_digit(c);391 uint32_t hex_digit = get_hex_digit(c);
371 if (hex_digit == -1) {392 if (hex_digit == UINT32_MAX) {
372 ast_error(pc, token, "invalid hex digit: '%c'", c);393 ast_error(pc, token, "invalid hex digit: '%c'", c);
373 }394 }
374 hex_value += hex_digit;395 hex_value += hex_digit;
...@@ -377,6 +398,47 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool...@@ -377,6 +398,47 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
377 state = StateStart;398 state = StateStart;
378 break;399 break;
379 }400 }
401 case StateUnicode:
402 {
403 uint32_t hex_digit = get_hex_digit(c);
404 if (hex_digit == UINT32_MAX) {
405 ast_error(pc, token, "invalid hex digit: '%c'", c);
406 }
407 hex_value *= 16;
408 hex_value += hex_digit;
409 unicode_index += 1;
410 if (unicode_index >= unicode_end) {
411 if (hex_value <= 0x7f) {
412 // 00000000 00000000 00000000 0xxxxxxx
413 buf_append_char(buf, hex_value);
414 } else if (hex_value <= 0x7ff) {
415 // 00000000 00000000 00000xxx xx000000
416 buf_append_char(buf, (unsigned char)(0xc0 | (hex_value >> 6)));
417 // 00000000 00000000 00000000 00xxxxxx
418 buf_append_char(buf, (unsigned char)(0x80 | (hex_value & 0x3f)));
419 } else if (hex_value <= 0xffff) {
420 // 00000000 00000000 xxxx0000 00000000
421 buf_append_char(buf, (unsigned char)(0xe0 | (hex_value >> 12)));
422 // 00000000 00000000 0000xxxx xx000000
423 buf_append_char(buf, (unsigned char)(0x80 | ((hex_value >> 6) & 0x3f)));
424 // 00000000 00000000 00000000 00xxxxxx
425 buf_append_char(buf, (unsigned char)(0x80 | (hex_value & 0x3f)));
426 } else if (hex_value <= 0x10ffff) {
427 // 00000000 000xxx00 00000000 00000000
428 buf_append_char(buf, (unsigned char)(0xf0 | (hex_value >> 18)));
429 // 00000000 000000xx xxxx0000 00000000
430 buf_append_char(buf, (unsigned char)(0x80 | ((hex_value >> 12) & 0x3f)));
431 // 00000000 00000000 0000xxxx xx000000
432 buf_append_char(buf, (unsigned char)(0x80 | ((hex_value >> 6) & 0x3f)));
433 // 00000000 00000000 00000000 00xxxxxx
434 buf_append_char(buf, (unsigned char)(0x80 | (hex_value & 0x3f)));
435 } else {
436 ast_error(pc, token, "unicode value out of range: %x", hex_value);
437 }
438 state = StateStart;
439 }
440 break;
441 }
380 }442 }
381 if (c == '\n') {443 if (c == '\n') {
382 pos.line += 1;444 pos.line += 1;
src/tokenizer.cpp+82-6
...@@ -103,6 +103,21 @@...@@ -103,6 +103,21 @@
103 ALPHA: \103 ALPHA: \
104 case '_'104 case '_'
105105
106#define HEX_DIGIT \
107 'a': \
108 case 'b': \
109 case 'c': \
110 case 'd': \
111 case 'e': \
112 case 'f': \
113 case 'A': \
114 case 'B': \
115 case 'C': \
116 case 'D': \
117 case 'E': \
118 case 'F': \
119 case DIGIT
120
106const char * zig_keywords[] = {121const char * zig_keywords[] = {
107 "true", "false", "null", "fn", "return", "var", "const", "extern",122 "true", "false", "null", "fn", "return", "var", "const", "extern",
108 "pub", "export", "use", "if", "else", "goto", "asm",123 "pub", "export", "use", "if", "else", "goto", "asm",
...@@ -132,11 +147,11 @@ enum TokenizeState {...@@ -132,11 +147,11 @@ enum TokenizeState {
132 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"147 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
133 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"148 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
134 TokenizeStateString,149 TokenizeStateString,
150 TokenizeStateStringEscape,
135 TokenizeStateRawString,151 TokenizeStateRawString,
136 TokenizeStateRawStringContents,152 TokenizeStateRawStringContents,
137 TokenizeStateRawStringMaybeEnd,153 TokenizeStateRawStringMaybeEnd,
138 TokenizeStateCharLiteral,154 TokenizeStateCharLiteral,
139 TokenizeStateCharLiteralEscape,
140 TokenizeStateCharLiteralEnd,155 TokenizeStateCharLiteralEnd,
141 TokenizeStateSawStar,156 TokenizeStateSawStar,
142 TokenizeStateSawSlash,157 TokenizeStateSawSlash,
...@@ -162,6 +177,7 @@ enum TokenizeState {...@@ -162,6 +177,7 @@ enum TokenizeState {
162 TokenizeStateSawDotDot,177 TokenizeStateSawDotDot,
163 TokenizeStateSawQuestionMark,178 TokenizeStateSawQuestionMark,
164 TokenizeStateSawAtSign,179 TokenizeStateSawAtSign,
180 TokenizeStateHex,
165 TokenizeStateError,181 TokenizeStateError,
166};182};
167183
...@@ -179,6 +195,7 @@ struct Tokenize {...@@ -179,6 +195,7 @@ struct Tokenize {
179 int raw_string_id_start;195 int raw_string_id_start;
180 int raw_string_id_end;196 int raw_string_id_end;
181 int raw_string_id_cmp_pos;197 int raw_string_id_cmp_pos;
198 int hex_chars_left;
182};199};
183200
184__attribute__ ((format (printf, 2, 3)))201__attribute__ ((format (printf, 2, 3)))
...@@ -921,10 +938,63 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -921,10 +938,63 @@ void tokenize(Buf *buf, Tokenization *out) {
921 case '\n':938 case '\n':
922 tokenize_error(&t, "use raw string for multiline string literal");939 tokenize_error(&t, "use raw string for multiline string literal");
923 break;940 break;
941 case '\\':
942 t.state = TokenizeStateStringEscape;
943 break;
924 default:944 default:
925 break;945 break;
926 }946 }
927 break;947 break;
948 case TokenizeStateStringEscape:
949 switch (c) {
950 case 'x':
951 t.state = TokenizeStateHex;
952 t.hex_chars_left = 2;
953 break;
954 case 'u':
955 t.state = TokenizeStateHex;
956 t.hex_chars_left = 4;
957 break;
958 case 'U':
959 t.state = TokenizeStateHex;
960 t.hex_chars_left = 6;
961 break;
962 case 'n':
963 case 'r':
964 case '\\':
965 case 't':
966 case '\'':
967 case '"':
968 if (t.cur_tok->id == TokenIdCharLiteral) {
969 t.state = TokenizeStateCharLiteralEnd;
970 } else if (t.cur_tok->id == TokenIdStringLiteral) {
971 t.state = TokenizeStateString;
972 } else {
973 zig_unreachable();
974 }
975 break;
976 default:
977 tokenize_error(&t, "invalid character: '%c'", c);
978 }
979 break;
980 case TokenizeStateHex:
981 switch (c) {
982 case HEX_DIGIT:
983 t.hex_chars_left -= 1;
984 if (t.hex_chars_left == 0) {
985 if (t.cur_tok->id == TokenIdCharLiteral) {
986 t.state = TokenizeStateCharLiteralEnd;
987 } else if (t.cur_tok->id == TokenIdStringLiteral) {
988 t.state = TokenizeStateString;
989 } else {
990 zig_unreachable();
991 }
992 }
993 break;
994 default:
995 tokenize_error(&t, "invalid character: '%c'", c);
996 }
997 break;
928 case TokenizeStateRawString:998 case TokenizeStateRawString:
929 if (c == '(') {999 if (c == '(') {
930 t.raw_string_id_end = t.pos;1000 t.raw_string_id_end = t.pos;
...@@ -963,16 +1033,13 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -963,16 +1033,13 @@ void tokenize(Buf *buf, Tokenization *out) {
963 t.state = TokenizeStateStart;1033 t.state = TokenizeStateStart;
964 break;1034 break;
965 case '\\':1035 case '\\':
966 t.state = TokenizeStateCharLiteralEscape;1036 t.state = TokenizeStateStringEscape;
967 break;1037 break;
968 default:1038 default:
969 t.state = TokenizeStateCharLiteralEnd;1039 t.state = TokenizeStateCharLiteralEnd;
970 break;1040 break;
971 }1041 }
972 break;1042 break;
973 case TokenizeStateCharLiteralEscape:
974 t.state = TokenizeStateCharLiteralEnd;
975 break;
976 case TokenizeStateCharLiteralEnd:1043 case TokenizeStateCharLiteralEnd:
977 switch (c) {1044 switch (c) {
978 case '\'':1045 case '\'':
...@@ -1136,13 +1203,22 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1136,13 +1203,22 @@ void tokenize(Buf *buf, Tokenization *out) {
1136 case TokenizeStateString:1203 case TokenizeStateString:
1137 tokenize_error(&t, "unterminated string");1204 tokenize_error(&t, "unterminated string");
1138 break;1205 break;
1206 case TokenizeStateStringEscape:
1207 case TokenizeStateHex:
1208 if (t.cur_tok->id == TokenIdStringLiteral) {
1209 tokenize_error(&t, "unterminated string");
1210 } else if (t.cur_tok->id == TokenIdCharLiteral) {
1211 tokenize_error(&t, "unterminated character literal");
1212 } else {
1213 zig_unreachable();
1214 }
1215 break;
1139 case TokenizeStateRawString:1216 case TokenizeStateRawString:
1140 case TokenizeStateRawStringContents:1217 case TokenizeStateRawStringContents:
1141 case TokenizeStateRawStringMaybeEnd:1218 case TokenizeStateRawStringMaybeEnd:
1142 tokenize_error(&t, "unterminated raw string");1219 tokenize_error(&t, "unterminated raw string");
1143 break;1220 break;
1144 case TokenizeStateCharLiteral:1221 case TokenizeStateCharLiteral:
1145 case TokenizeStateCharLiteralEscape:
1146 case TokenizeStateCharLiteralEnd:1222 case TokenizeStateCharLiteralEnd:
1147 tokenize_error(&t, "unterminated character literal");1223 tokenize_error(&t, "unterminated character literal");
1148 break;1224 break;
test/self_hosted.zig+11
...@@ -1398,3 +1398,14 @@ fn test_take_address_of_parameter_noeval(f: f32) {...@@ -1398,3 +1398,14 @@ fn test_take_address_of_parameter_noeval(f: f32) {
1398fn array_mult_operator() {1398fn array_mult_operator() {
1399 assert(str.eql("ab" ** 5, "ababababab"));1399 assert(str.eql("ab" ** 5, "ababababab"));
1400}1400}
1401
1402#attribute("test")
1403fn string_escapes() {
1404 assert(str.eql("\"", "\x22"));
1405 assert(str.eql("\'", "\x27"));
1406 assert(str.eql("\n", "\x0a"));
1407 assert(str.eql("\r", "\x0d"));
1408 assert(str.eql("\t", "\x09"));
1409 assert(str.eql("\\", "\x5c"));
1410 assert(str.eql("\u1234\u0069", "\xe1\x88\xb4\x69"));
1411}