authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-04-03 18:44:17-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-04-03 18:59:43-07:00
loge144ddab249af3737f04267c7f1f0f0e093ed314
tree13a209862d84c916b5789caf2caf52d3281b235c
parent5bae9ba0869409598d272a03477641d234ace8e6

add multiline string literal

and make multiple lines in normal string literals an error

7 files changed, 162 insertions(+), 11 deletions(-)

doc/langref.md+13-4
......@@ -267,13 +267,22 @@ from codegen.
267267
268268#### Character and String Literals
269269```
270Literal Example Characters Escapes Null Term Type
270Literal Example Characters Escapes Null Term Type
271271
272Byte 'H' All ASCII Byte No u8
273UTF-8 Bytes "hello" All Unicode Byte & Unicode No [5]u8
274UTF-8 C string c"hello" All Unicode Byte & Unicode Yes &const u8
272Byte 'H' All ASCII Byte No u8
273UTF-8 Bytes "hello" All Unicode Byte & Unicode No [5]u8
274UTF-8 C string c"hello" All Unicode Byte & Unicode Yes &const u8
275UTF-8 Raw String r"A(hello)A" All Unicode None No [5]u8
276UTF-8 Raw C String rc"A(hello)A" All Unicode None Yes &const u8
275277```
276278
279##### Raw Strings
280
281Raw string literals have no escapes and can span across multiple lines. To
282start a raw string, use 'r"' or 'rc"' followed by unique bytes followed by '('.
283To end a raw string, use ')' followed by the same unique bytes, followed by '"'.
284
285
277286```
278287Escape Name
279288
doc/vim/syntax/zig.vim+1-1
......@@ -51,7 +51,7 @@ syn match zigEscape display contained /\\\([nrt0\\'"]\|x\x\{2}\)/
5151syn match zigEscapeUnicode display contained /\\\(u\x\{4}\|U\x\{8}\)/
5252syn match zigEscapeUnicode display contained /\\u{\x\{1,6}}/
5353syn match zigStringContinuation display contained /\\\n\s*/
54syn region zigString start=+c\?"+ skip=+\\\\\|\\"+ end=+"+ contains=zigEscape,zigEscapeUnicode,zigEscapeError,zigStringContinuation,@Spell
54syn region zigString start=+r\?c\?"+ skip=+\\\\\|\\"+ end=+"+ contains=zigEscape,zigEscapeUnicode,zigEscapeError,zigStringContinuation,@Spell
5555syn region zigString start='b\?r\z(#*\)"' end='"\z1' contains=@Spell
5656
5757let b:current_syntax = "zig"
src/parser.cpp+10
......@@ -226,6 +226,16 @@ static uint8_t parse_char_literal(ParseContext *pc, Token *token) {
226226static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool *out_c_str,
227227 ZigList<SrcPos> *offset_map)
228228{
229 if (token->raw_string_start > 0) {
230 uint8_t c1 = *((uint8_t*)buf_ptr(pc->buf) + token->start_pos);
231 uint8_t c2 = *((uint8_t*)buf_ptr(pc->buf) + token->start_pos + 1);
232 assert(c1 == 'r');
233 *out_c_str = (c2 == 'c');
234 const char *str = buf_ptr(pc->buf) + token->raw_string_start;
235 buf_init_from_mem(buf, str, token->raw_string_end - token->raw_string_start);
236 return;
237 }
238
229239 // skip the double quotes at beginning and end
230240 // convert escape sequences
231241 // detect c string literal
src/tokenizer.cpp+104-6
......@@ -30,7 +30,7 @@
3030 '0': \
3131 case DIGIT_NON_ZERO
3232
33#define ALPHA_EXCEPT_C \
33#define ALPHA_EXCEPT_CR \
3434 'a': \
3535 case 'b': \
3636 /*case 'c':*/ \
......@@ -48,7 +48,7 @@
4848 case 'o': \
4949 case 'p': \
5050 case 'q': \
51 case 'r': \
51 /*case 'r':*/ \
5252 case 's': \
5353 case 't': \
5454 case 'u': \
......@@ -85,11 +85,17 @@
8585 case 'Z'
8686
8787#define ALPHA \
88 ALPHA_EXCEPT_C: \
89 case 'c'
88 ALPHA_EXCEPT_CR: \
89 case 'c': \
90 case 'r'
9091
9192#define SYMBOL_CHAR \
92 ALPHA: \
93 SYMBOL_CHAR_EXCEPT_C: \
94 case 'c'
95
96#define SYMBOL_CHAR_EXCEPT_C \
97 ALPHA_EXCEPT_CR: \
98 case 'r': \
9399 case DIGIT: \
94100 case '_'
95101
......@@ -118,12 +124,17 @@ enum TokenizeState {
118124 TokenizeStateStart,
119125 TokenizeStateSymbol,
120126 TokenizeStateSymbolFirst,
127 TokenizeStateSymbolFirstRaw,
128 TokenizeStateFirstR,
121129 TokenizeStateZero, // "0", which might lead to "0x"
122130 TokenizeStateNumber, // "123", "0x123"
123131 TokenizeStateFloatFraction, // "123.456", "0x123.456"
124132 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
125133 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
126134 TokenizeStateString,
135 TokenizeStateRawString,
136 TokenizeStateRawStringContents,
137 TokenizeStateRawStringMaybeEnd,
127138 TokenizeStateCharLiteral,
128139 TokenizeStateSawStar,
129140 TokenizeStateSawSlash,
......@@ -162,6 +173,9 @@ struct Tokenize {
162173 Token *cur_tok;
163174 int multi_line_comment_count;
164175 Tokenization *out;
176 int raw_string_id_start;
177 int raw_string_id_end;
178 int raw_string_id_cmp_pos;
165179};
166180
167181__attribute__ ((format (printf, 2, 3)))
......@@ -193,6 +207,8 @@ static void begin_token(Tokenize *t, TokenId id) {
193207 token->radix = 0;
194208 token->decimal_point_pos = 0;
195209 token->exponent_marker_pos = 0;
210 token->raw_string_start = 0;
211 token->raw_string_end = 0;
196212 t->cur_tok = token;
197213}
198214
......@@ -324,7 +340,11 @@ void tokenize(Buf *buf, Tokenization *out) {
324340 t.state = TokenizeStateSymbolFirst;
325341 begin_token(&t, TokenIdSymbol);
326342 break;
327 case ALPHA_EXCEPT_C:
343 case 'r':
344 t.state = TokenizeStateFirstR;
345 begin_token(&t, TokenIdSymbol);
346 break;
347 case ALPHA_EXCEPT_CR:
328348 case '_':
329349 t.state = TokenizeStateSymbol;
330350 begin_token(&t, TokenIdSymbol);
......@@ -821,6 +841,43 @@ void tokenize(Buf *buf, Tokenization *out) {
821841 continue;
822842 }
823843 break;
844 case TokenizeStateSymbolFirstRaw:
845 switch (c) {
846 case '"':
847 t.cur_tok->id = TokenIdStringLiteral;
848 t.state = TokenizeStateRawString;
849 t.raw_string_id_start = t.pos + 1;
850 break;
851 case SYMBOL_CHAR:
852 t.state = TokenizeStateSymbol;
853 break;
854 default:
855 t.pos -= 1;
856 end_token(&t);
857 t.state = TokenizeStateStart;
858 continue;
859 }
860 break;
861 case TokenizeStateFirstR:
862 switch (c) {
863 case '"':
864 t.cur_tok->id = TokenIdStringLiteral;
865 t.state = TokenizeStateRawString;
866 t.raw_string_id_start = t.pos + 1;
867 break;
868 case 'c':
869 t.state = TokenizeStateSymbolFirstRaw;
870 break;
871 case SYMBOL_CHAR_EXCEPT_C:
872 t.state = TokenizeStateSymbol;
873 break;
874 default:
875 t.pos -= 1;
876 end_token(&t);
877 t.state = TokenizeStateStart;
878 continue;
879 }
880 break;
824881 case TokenizeStateSymbol:
825882 switch (c) {
826883 case SYMBOL_CHAR:
......@@ -838,10 +895,44 @@ void tokenize(Buf *buf, Tokenization *out) {
838895 end_token(&t);
839896 t.state = TokenizeStateStart;
840897 break;
898 case '\n':
899 tokenize_error(&t, "use raw string for multiline string literal");
900 break;
841901 default:
842902 break;
843903 }
844904 break;
905 case TokenizeStateRawString:
906 if (c == '(') {
907 t.raw_string_id_end = t.pos;
908 t.cur_tok->raw_string_start = t.pos + 1;
909 t.state = TokenizeStateRawStringContents;
910 }
911 break;
912 case TokenizeStateRawStringContents:
913 if (c == ')') {
914 t.state = TokenizeStateRawStringMaybeEnd;
915 t.raw_string_id_cmp_pos = t.raw_string_id_start;
916 t.cur_tok->raw_string_end = t.pos;
917 }
918 break;
919 case TokenizeStateRawStringMaybeEnd:
920 if (t.raw_string_id_cmp_pos >= t.raw_string_id_end &&
921 c == '"')
922 {
923 end_token(&t);
924 t.state = TokenizeStateStart;
925 } else if (c != buf_ptr(t.buf)[t.raw_string_id_cmp_pos]) {
926 if (c == ')') {
927 t.raw_string_id_cmp_pos = t.raw_string_id_start;
928 t.cur_tok->raw_string_end = t.pos;
929 } else {
930 t.state = TokenizeStateRawStringContents;
931 }
932 } else {
933 t.raw_string_id_cmp_pos += 1;
934 }
935 break;
845936 case TokenizeStateCharLiteral:
846937 switch (c) {
847938 case '\'':
......@@ -1002,11 +1093,18 @@ void tokenize(Buf *buf, Tokenization *out) {
10021093 case TokenizeStateString:
10031094 tokenize_error(&t, "unterminated string");
10041095 break;
1096 case TokenizeStateRawString:
1097 case TokenizeStateRawStringContents:
1098 case TokenizeStateRawStringMaybeEnd:
1099 tokenize_error(&t, "unterminated raw string");
1100 break;
10051101 case TokenizeStateCharLiteral:
10061102 tokenize_error(&t, "unterminated character literal");
10071103 break;
10081104 case TokenizeStateSymbol:
10091105 case TokenizeStateSymbolFirst:
1106 case TokenizeStateSymbolFirstRaw:
1107 case TokenizeStateFirstR:
10101108 case TokenizeStateZero:
10111109 case TokenizeStateNumber:
10121110 case TokenizeStateFloatFraction:
src/tokenizer.hpp+4
......@@ -112,6 +112,10 @@ struct Token {
112112 int radix; // if != 10, then skip the first 2 characters
113113 int decimal_point_pos; // either exponent_marker_pos or the position of the '.'
114114 int exponent_marker_pos; // either end_pos or the position of the 'e'/'p'
115
116 // for id == TokenIdStringLiteral
117 int raw_string_start;
118 int raw_string_end;
115119};
116120
117121struct Tokenization {
test/run_tests.cpp+6
......@@ -1770,6 +1770,12 @@ fn f() {
17701770 const std = @import("std");
17711771}
17721772 )SOURCE", 1, ".tmp_source.zig:3:17: error: @import invalid inside function bodies");
1773
1774
1775 add_compile_fail_case("normal string with newline", R"SOURCE(
1776const foo = "a
1777b";
1778 )SOURCE", 1, ".tmp_source.zig:2:13: error: use raw string for multiline string literal");
17731779}
17741780
17751781//////////////////////////////////////////////////////////////////////////////
test/self_hosted.zig+24
......@@ -495,7 +495,31 @@ fn count_trailing_zeroes() {
495495}
496496
497497
498#attribute("test")
499fn multiline_string() {
500 const s1 = r"AOEU(
501one
502two)
503three)AOEU";
504 const s2 = "\none\ntwo)\nthree";
505 const s3 = r"(
506one
507two)
508three)";
509 assert(str_eql(s1, s2));
510 assert(str_eql(s3, s2));
511}
512
513
498514
499515fn assert(b: bool) {
500516 if (!b) unreachable{}
501517}
518
519fn str_eql(s1: []u8, s2: []u8) -> bool {
520 if (s1.len != s2.len) return false;
521 for (s1) |c, i| {
522 if (s2[i] != c) return false;
523 }
524 return true;
525}