authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-04-03 18:44:17-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-04-03 18:59:43-07:00
loge144ddab249af3737f04267c7f1f0f0e093ed314
tree13a209862d84c916b5789caf2caf52d3281b235c
parent5bae9ba0869409598d272a03477641d234ace8e6

add multiline string literal

and make multiple lines in normal string literals an error

7 files changed, 162 insertions(+), 11 deletions(-)

doc/langref.md+13-4
...@@ -267,13 +267,22 @@ from codegen....@@ -267,13 +267,22 @@ from codegen.
267267
268#### Character and String Literals268#### Character and String Literals
269```269```
270Literal Example Characters Escapes Null Term Type270Literal Example Characters Escapes Null Term Type
271271
272Byte 'H' All ASCII Byte No u8272Byte 'H' All ASCII Byte No u8
273UTF-8 Bytes "hello" All Unicode Byte & Unicode No [5]u8273UTF-8 Bytes "hello" All Unicode Byte & Unicode No [5]u8
274UTF-8 C string c"hello" All Unicode Byte & Unicode Yes &const u8274UTF-8 C string c"hello" All Unicode Byte & Unicode Yes &const u8
275UTF-8 Raw String r"A(hello)A" All Unicode None No [5]u8
276UTF-8 Raw C String rc"A(hello)A" All Unicode None Yes &const u8
275```277```
276278
279##### Raw Strings
280
281Raw string literals have no escapes and can span across multiple lines. To
282start a raw string, use 'r"' or 'rc"' followed by unique bytes followed by '('.
283To end a raw string, use ')' followed by the same unique bytes, followed by '"'.
284
285
277```286```
278Escape Name287Escape Name
279288
doc/vim/syntax/zig.vim+1-1
...@@ -51,7 +51,7 @@ syn match zigEscape display contained /\\\([nrt0\\'"]\|x\x\{2}\)/...@@ -51,7 +51,7 @@ syn match zigEscape display contained /\\\([nrt0\\'"]\|x\x\{2}\)/
51syn match zigEscapeUnicode display contained /\\\(u\x\{4}\|U\x\{8}\)/51syn match zigEscapeUnicode display contained /\\\(u\x\{4}\|U\x\{8}\)/
52syn match zigEscapeUnicode display contained /\\u{\x\{1,6}}/52syn match zigEscapeUnicode display contained /\\u{\x\{1,6}}/
53syn match zigStringContinuation display contained /\\\n\s*/53syn match zigStringContinuation display contained /\\\n\s*/
54syn region zigString start=+c\?"+ skip=+\\\\\|\\"+ end=+"+ contains=zigEscape,zigEscapeUnicode,zigEscapeError,zigStringContinuation,@Spell54syn region zigString start=+r\?c\?"+ skip=+\\\\\|\\"+ end=+"+ contains=zigEscape,zigEscapeUnicode,zigEscapeError,zigStringContinuation,@Spell
55syn region zigString start='b\?r\z(#*\)"' end='"\z1' contains=@Spell55syn region zigString start='b\?r\z(#*\)"' end='"\z1' contains=@Spell
5656
57let b:current_syntax = "zig"57let b:current_syntax = "zig"
src/parser.cpp+10
...@@ -226,6 +226,16 @@ static uint8_t parse_char_literal(ParseContext *pc, Token *token) {...@@ -226,6 +226,16 @@ static uint8_t parse_char_literal(ParseContext *pc, Token *token) {
226static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool *out_c_str,226static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool *out_c_str,
227 ZigList<SrcPos> *offset_map)227 ZigList<SrcPos> *offset_map)
228{228{
229 if (token->raw_string_start > 0) {
230 uint8_t c1 = *((uint8_t*)buf_ptr(pc->buf) + token->start_pos);
231 uint8_t c2 = *((uint8_t*)buf_ptr(pc->buf) + token->start_pos + 1);
232 assert(c1 == 'r');
233 *out_c_str = (c2 == 'c');
234 const char *str = buf_ptr(pc->buf) + token->raw_string_start;
235 buf_init_from_mem(buf, str, token->raw_string_end - token->raw_string_start);
236 return;
237 }
238
229 // skip the double quotes at beginning and end239 // skip the double quotes at beginning and end
230 // convert escape sequences240 // convert escape sequences
231 // detect c string literal241 // detect c string literal
src/tokenizer.cpp+104-6
...@@ -30,7 +30,7 @@...@@ -30,7 +30,7 @@
30 '0': \30 '0': \
31 case DIGIT_NON_ZERO31 case DIGIT_NON_ZERO
3232
33#define ALPHA_EXCEPT_C \33#define ALPHA_EXCEPT_CR \
34 'a': \34 'a': \
35 case 'b': \35 case 'b': \
36 /*case 'c':*/ \36 /*case 'c':*/ \
...@@ -48,7 +48,7 @@...@@ -48,7 +48,7 @@
48 case 'o': \48 case 'o': \
49 case 'p': \49 case 'p': \
50 case 'q': \50 case 'q': \
51 case 'r': \51 /*case 'r':*/ \
52 case 's': \52 case 's': \
53 case 't': \53 case 't': \
54 case 'u': \54 case 'u': \
...@@ -85,11 +85,17 @@...@@ -85,11 +85,17 @@
85 case 'Z'85 case 'Z'
8686
87#define ALPHA \87#define ALPHA \
88 ALPHA_EXCEPT_C: \88 ALPHA_EXCEPT_CR: \
89 case 'c'89 case 'c': \
90 case 'r'
9091
91#define SYMBOL_CHAR \92#define SYMBOL_CHAR \
92 ALPHA: \93 SYMBOL_CHAR_EXCEPT_C: \
94 case 'c'
95
96#define SYMBOL_CHAR_EXCEPT_C \
97 ALPHA_EXCEPT_CR: \
98 case 'r': \
93 case DIGIT: \99 case DIGIT: \
94 case '_'100 case '_'
95101
...@@ -118,12 +124,17 @@ enum TokenizeState {...@@ -118,12 +124,17 @@ enum TokenizeState {
118 TokenizeStateStart,124 TokenizeStateStart,
119 TokenizeStateSymbol,125 TokenizeStateSymbol,
120 TokenizeStateSymbolFirst,126 TokenizeStateSymbolFirst,
127 TokenizeStateSymbolFirstRaw,
128 TokenizeStateFirstR,
121 TokenizeStateZero, // "0", which might lead to "0x"129 TokenizeStateZero, // "0", which might lead to "0x"
122 TokenizeStateNumber, // "123", "0x123"130 TokenizeStateNumber, // "123", "0x123"
123 TokenizeStateFloatFraction, // "123.456", "0x123.456"131 TokenizeStateFloatFraction, // "123.456", "0x123.456"
124 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"132 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
125 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"133 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
126 TokenizeStateString,134 TokenizeStateString,
135 TokenizeStateRawString,
136 TokenizeStateRawStringContents,
137 TokenizeStateRawStringMaybeEnd,
127 TokenizeStateCharLiteral,138 TokenizeStateCharLiteral,
128 TokenizeStateSawStar,139 TokenizeStateSawStar,
129 TokenizeStateSawSlash,140 TokenizeStateSawSlash,
...@@ -162,6 +173,9 @@ struct Tokenize {...@@ -162,6 +173,9 @@ struct Tokenize {
162 Token *cur_tok;173 Token *cur_tok;
163 int multi_line_comment_count;174 int multi_line_comment_count;
164 Tokenization *out;175 Tokenization *out;
176 int raw_string_id_start;
177 int raw_string_id_end;
178 int raw_string_id_cmp_pos;
165};179};
166180
167__attribute__ ((format (printf, 2, 3)))181__attribute__ ((format (printf, 2, 3)))
...@@ -193,6 +207,8 @@ static void begin_token(Tokenize *t, TokenId id) {...@@ -193,6 +207,8 @@ static void begin_token(Tokenize *t, TokenId id) {
193 token->radix = 0;207 token->radix = 0;
194 token->decimal_point_pos = 0;208 token->decimal_point_pos = 0;
195 token->exponent_marker_pos = 0;209 token->exponent_marker_pos = 0;
210 token->raw_string_start = 0;
211 token->raw_string_end = 0;
196 t->cur_tok = token;212 t->cur_tok = token;
197}213}
198214
...@@ -324,7 +340,11 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -324,7 +340,11 @@ void tokenize(Buf *buf, Tokenization *out) {
324 t.state = TokenizeStateSymbolFirst;340 t.state = TokenizeStateSymbolFirst;
325 begin_token(&t, TokenIdSymbol);341 begin_token(&t, TokenIdSymbol);
326 break;342 break;
327 case ALPHA_EXCEPT_C:343 case 'r':
344 t.state = TokenizeStateFirstR;
345 begin_token(&t, TokenIdSymbol);
346 break;
347 case ALPHA_EXCEPT_CR:
328 case '_':348 case '_':
329 t.state = TokenizeStateSymbol;349 t.state = TokenizeStateSymbol;
330 begin_token(&t, TokenIdSymbol);350 begin_token(&t, TokenIdSymbol);
...@@ -821,6 +841,43 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -821,6 +841,43 @@ void tokenize(Buf *buf, Tokenization *out) {
821 continue;841 continue;
822 }842 }
823 break;843 break;
844 case TokenizeStateSymbolFirstRaw:
845 switch (c) {
846 case '"':
847 t.cur_tok->id = TokenIdStringLiteral;
848 t.state = TokenizeStateRawString;
849 t.raw_string_id_start = t.pos + 1;
850 break;
851 case SYMBOL_CHAR:
852 t.state = TokenizeStateSymbol;
853 break;
854 default:
855 t.pos -= 1;
856 end_token(&t);
857 t.state = TokenizeStateStart;
858 continue;
859 }
860 break;
861 case TokenizeStateFirstR:
862 switch (c) {
863 case '"':
864 t.cur_tok->id = TokenIdStringLiteral;
865 t.state = TokenizeStateRawString;
866 t.raw_string_id_start = t.pos + 1;
867 break;
868 case 'c':
869 t.state = TokenizeStateSymbolFirstRaw;
870 break;
871 case SYMBOL_CHAR_EXCEPT_C:
872 t.state = TokenizeStateSymbol;
873 break;
874 default:
875 t.pos -= 1;
876 end_token(&t);
877 t.state = TokenizeStateStart;
878 continue;
879 }
880 break;
824 case TokenizeStateSymbol:881 case TokenizeStateSymbol:
825 switch (c) {882 switch (c) {
826 case SYMBOL_CHAR:883 case SYMBOL_CHAR:
...@@ -838,10 +895,44 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -838,10 +895,44 @@ void tokenize(Buf *buf, Tokenization *out) {
838 end_token(&t);895 end_token(&t);
839 t.state = TokenizeStateStart;896 t.state = TokenizeStateStart;
840 break;897 break;
898 case '\n':
899 tokenize_error(&t, "use raw string for multiline string literal");
900 break;
841 default:901 default:
842 break;902 break;
843 }903 }
844 break;904 break;
905 case TokenizeStateRawString:
906 if (c == '(') {
907 t.raw_string_id_end = t.pos;
908 t.cur_tok->raw_string_start = t.pos + 1;
909 t.state = TokenizeStateRawStringContents;
910 }
911 break;
912 case TokenizeStateRawStringContents:
913 if (c == ')') {
914 t.state = TokenizeStateRawStringMaybeEnd;
915 t.raw_string_id_cmp_pos = t.raw_string_id_start;
916 t.cur_tok->raw_string_end = t.pos;
917 }
918 break;
919 case TokenizeStateRawStringMaybeEnd:
920 if (t.raw_string_id_cmp_pos >= t.raw_string_id_end &&
921 c == '"')
922 {
923 end_token(&t);
924 t.state = TokenizeStateStart;
925 } else if (c != buf_ptr(t.buf)[t.raw_string_id_cmp_pos]) {
926 if (c == ')') {
927 t.raw_string_id_cmp_pos = t.raw_string_id_start;
928 t.cur_tok->raw_string_end = t.pos;
929 } else {
930 t.state = TokenizeStateRawStringContents;
931 }
932 } else {
933 t.raw_string_id_cmp_pos += 1;
934 }
935 break;
845 case TokenizeStateCharLiteral:936 case TokenizeStateCharLiteral:
846 switch (c) {937 switch (c) {
847 case '\'':938 case '\'':
...@@ -1002,11 +1093,18 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1002,11 +1093,18 @@ void tokenize(Buf *buf, Tokenization *out) {
1002 case TokenizeStateString:1093 case TokenizeStateString:
1003 tokenize_error(&t, "unterminated string");1094 tokenize_error(&t, "unterminated string");
1004 break;1095 break;
1096 case TokenizeStateRawString:
1097 case TokenizeStateRawStringContents:
1098 case TokenizeStateRawStringMaybeEnd:
1099 tokenize_error(&t, "unterminated raw string");
1100 break;
1005 case TokenizeStateCharLiteral:1101 case TokenizeStateCharLiteral:
1006 tokenize_error(&t, "unterminated character literal");1102 tokenize_error(&t, "unterminated character literal");
1007 break;1103 break;
1008 case TokenizeStateSymbol:1104 case TokenizeStateSymbol:
1009 case TokenizeStateSymbolFirst:1105 case TokenizeStateSymbolFirst:
1106 case TokenizeStateSymbolFirstRaw:
1107 case TokenizeStateFirstR:
1010 case TokenizeStateZero:1108 case TokenizeStateZero:
1011 case TokenizeStateNumber:1109 case TokenizeStateNumber:
1012 case TokenizeStateFloatFraction:1110 case TokenizeStateFloatFraction:
src/tokenizer.hpp+4
...@@ -112,6 +112,10 @@ struct Token {...@@ -112,6 +112,10 @@ struct Token {
112 int radix; // if != 10, then skip the first 2 characters112 int radix; // if != 10, then skip the first 2 characters
113 int decimal_point_pos; // either exponent_marker_pos or the position of the '.'113 int decimal_point_pos; // either exponent_marker_pos or the position of the '.'
114 int exponent_marker_pos; // either end_pos or the position of the 'e'/'p'114 int exponent_marker_pos; // either end_pos or the position of the 'e'/'p'
115
116 // for id == TokenIdStringLiteral
117 int raw_string_start;
118 int raw_string_end;
115};119};
116120
117struct Tokenization {121struct Tokenization {
test/run_tests.cpp+6
...@@ -1770,6 +1770,12 @@ fn f() {...@@ -1770,6 +1770,12 @@ fn f() {
1770 const std = @import("std");1770 const std = @import("std");
1771}1771}
1772 )SOURCE", 1, ".tmp_source.zig:3:17: error: @import invalid inside function bodies");1772 )SOURCE", 1, ".tmp_source.zig:3:17: error: @import invalid inside function bodies");
1773
1774
1775 add_compile_fail_case("normal string with newline", R"SOURCE(
1776const foo = "a
1777b";
1778 )SOURCE", 1, ".tmp_source.zig:2:13: error: use raw string for multiline string literal");
1773}1779}
17741780
1775//////////////////////////////////////////////////////////////////////////////1781//////////////////////////////////////////////////////////////////////////////
test/self_hosted.zig+24
...@@ -495,7 +495,31 @@ fn count_trailing_zeroes() {...@@ -495,7 +495,31 @@ fn count_trailing_zeroes() {
495}495}
496496
497497
498#attribute("test")
499fn multiline_string() {
500 const s1 = r"AOEU(
501one
502two)
503three)AOEU";
504 const s2 = "\none\ntwo)\nthree";
505 const s3 = r"(
506one
507two)
508three)";
509 assert(str_eql(s1, s2));
510 assert(str_eql(s3, s2));
511}
512
513
498514
499fn assert(b: bool) {515fn assert(b: bool) {
500 if (!b) unreachable{}516 if (!b) unreachable{}
501}517}
518
519fn str_eql(s1: []u8, s2: []u8) -> bool {
520 if (s1.len != s2.len) return false;
521 for (s1) |c, i| {
522 if (s2[i] != c) return false;
523 }
524 return true;
525}