authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-14 21:24:39+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-01-15 14:20:48-05:00
log0ea96c11ef5eca9cfd19233b6d51b00ba33da716
treedb54183a1c733a3e69da728cb4bc1501297bdf93
parent0a410519559788598fcf698013c1b6389d80c67c

disallow multiline strings in test and library names

6 files changed, 38 insertions(+), 59 deletions(-)

doc/langref.html.in+5-5
...@@ -10212,13 +10212,13 @@ ContainerMembers...@@ -10212,13 +10212,13 @@ ContainerMembers
10212 / ContainerField10212 / ContainerField
10213 /10213 /
1021410214
10215TestDecl &lt;- KEYWORD_test STRINGLITERAL Block10215TestDecl &lt;- KEYWORD_test STRINGLITERALSINGLE Block
1021610216
10217TopLevelComptime &lt;- KEYWORD_comptime BlockExpr10217TopLevelComptime &lt;- KEYWORD_comptime BlockExpr
1021810218
10219TopLevelDecl10219TopLevelDecl
10220 &lt;- (KEYWORD_export / KEYWORD_extern STRINGLITERAL? / KEYWORD_inline)? FnProto (SEMICOLON / Block)10220 &lt;- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / KEYWORD_inline)? FnProto (SEMICOLON / Block)
10221 / (KEYWORD_export / KEYWORD_extern STRINGLITERAL?)? KEYWORD_threadlocal? VarDecl10221 / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl
10222 / KEYWORD_usingnamespace Expr SEMICOLON10222 / KEYWORD_usingnamespace Expr SEMICOLON
1022310223
10224FnProto &lt;- FnCC? KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)10224FnProto &lt;- FnCC? KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)
...@@ -10561,10 +10561,10 @@ INTEGER...@@ -10561,10 +10561,10 @@ INTEGER
10561 / "0o" [0-7]+ skip10561 / "0o" [0-7]+ skip
10562 / "0x" hex+ skip10562 / "0x" hex+ skip
10563 / [0-9]+ skip10563 / [0-9]+ skip
10564STRINGLITERALSINGLE &lt;- "\"" string_char* "\"" skip
10564STRINGLITERAL10565STRINGLITERAL
10565 &lt;- "\"" string_char* "\"" skip10566 &lt;- STRINGLITERALSINGLE
10566 / line_string skip10567 / line_string skip
10567 / line_cstring skip
10568IDENTIFIER10568IDENTIFIER
10569 &lt;- !keyword [A-Za-z_] [A-Za-z0-9_]* skip10569 &lt;- !keyword [A-Za-z_] [A-Za-z0-9_]* skip
10570 / "@\"" string_char* "\"" skip10570 / "@\"" string_char* "\"" skip
lib/std/zig/parse.zig+13-13
...@@ -183,10 +183,10 @@ fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree)...@@ -183,10 +183,10 @@ fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree)
183 return &node.base;183 return &node.base;
184}184}
185185
186/// TestDecl <- KEYWORD_test STRINGLITERAL Block186/// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block
187fn parseTestDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {187fn parseTestDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
188 const test_token = eatToken(it, .Keyword_test) orelse return null;188 const test_token = eatToken(it, .Keyword_test) orelse return null;
189 const name_node = try expectNode(arena, it, tree, parseStringLiteral, AstError{189 const name_node = try expectNode(arena, it, tree, parseStringLiteralSingle, AstError{
190 .ExpectedStringLiteral = AstError.ExpectedStringLiteral{ .token = it.index },190 .ExpectedStringLiteral = AstError.ExpectedStringLiteral{ .token = it.index },
191 });191 });
192 const block_node = try expectNode(arena, it, tree, parseBlock, AstError{192 const block_node = try expectNode(arena, it, tree, parseBlock, AstError{
...@@ -225,15 +225,15 @@ fn parseTopLevelComptime(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*...@@ -225,15 +225,15 @@ fn parseTopLevelComptime(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*
225}225}
226226
227/// TopLevelDecl227/// TopLevelDecl
228/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERAL? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)228/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
229/// / (KEYWORD_export / KEYWORD_extern STRINGLITERAL?)? KEYWORD_threadlocal? VarDecl229/// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl
230/// / KEYWORD_usingnamespace Expr SEMICOLON230/// / KEYWORD_usingnamespace Expr SEMICOLON
231fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {231fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
232 var lib_name: ?*Node = null;232 var lib_name: ?*Node = null;
233 const extern_export_inline_token = blk: {233 const extern_export_inline_token = blk: {
234 if (eatToken(it, .Keyword_export)) |token| break :blk token;234 if (eatToken(it, .Keyword_export)) |token| break :blk token;
235 if (eatToken(it, .Keyword_extern)) |token| {235 if (eatToken(it, .Keyword_extern)) |token| {
236 lib_name = try parseStringLiteral(arena, it, tree);236 lib_name = try parseStringLiteralSingle(arena, it, tree);
237 break :blk token;237 break :blk token;
238 }238 }
239 if (eatToken(it, .Keyword_inline)) |token| break :blk token;239 if (eatToken(it, .Keyword_inline)) |token| break :blk token;
...@@ -285,12 +285,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -285,12 +285,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
285285
286 if (extern_export_inline_token) |token| {286 if (extern_export_inline_token) |token| {
287 if (lib_name) |string_literal_node|287 if (lib_name) |string_literal_node|
288 if (string_literal_node.cast(Node.StringLiteral)) |single| {288 putBackToken(it, string_literal_node.cast(Node.StringLiteral).?.token);
289 putBackToken(it, single.token);
290 } else if (string_literal_node.cast(Node.MultilineStringLiteral)) |multi| {
291 while (multi.lines.pop()) |line|
292 putBackToken(it, line);
293 } else unreachable;
294 putBackToken(it, token);289 putBackToken(it, token);
295 return null;290 return null;
296 }291 }
...@@ -2776,8 +2771,7 @@ fn createLiteral(arena: *Allocator, comptime T: type, token: TokenIndex) !*Node...@@ -2776,8 +2771,7 @@ fn createLiteral(arena: *Allocator, comptime T: type, token: TokenIndex) !*Node
2776 return &result.base;2771 return &result.base;
2777}2772}
27782773
2779// string literal or multiline string literal2774fn parseStringLiteralSingle(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2780fn parseStringLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2781 if (eatToken(it, .StringLiteral)) |token| {2775 if (eatToken(it, .StringLiteral)) |token| {
2782 const node = try arena.create(Node.StringLiteral);2776 const node = try arena.create(Node.StringLiteral);
2783 node.* = Node.StringLiteral{2777 node.* = Node.StringLiteral{
...@@ -2785,6 +2779,12 @@ fn parseStringLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Nod...@@ -2785,6 +2779,12 @@ fn parseStringLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Nod
2785 };2779 };
2786 return &node.base;2780 return &node.base;
2787 }2781 }
2782 return null;
2783}
2784
2785// string literal or multiline string literal
2786fn parseStringLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2787 if (try parseStringLiteralSingle(arena, it, tree)) |node| return node;
27882788
2789 if (eatToken(it, .MultilineStringLiteralLine)) |first_line| {2789 if (eatToken(it, .MultilineStringLiteralLine)) |first_line| {
2790 const node = try arena.create(Node.MultilineStringLiteral);2790 const node = try arena.create(Node.MultilineStringLiteral);
lib/std/zig/parser_test.zig-10
...@@ -2721,16 +2721,6 @@ test "zig fmt: extern without container keyword returns error" {...@@ -2721,16 +2721,6 @@ test "zig fmt: extern without container keyword returns error" {
2721 );2721 );
2722}2722}
27232723
2724test "zig fmt: extern multiline lib name" {
2725 try testError(
2726 \\extern \\super
2727 \\ \\long
2728 \\ \\library
2729 \\ \\name
2730 \\
2731 );
2732}
2733
2734const std = @import("std");2724const std = @import("std");
2735const mem = std.mem;2725const mem = std.mem;
2736const warn = std.debug.warn;2726const warn = std.debug.warn;
src/parser.cpp+15-14
...@@ -141,16 +141,9 @@ static void ast_error(ParseContext *pc, Token *token, const char *format, ...) {...@@ -141,16 +141,9 @@ static void ast_error(ParseContext *pc, Token *token, const char *format, ...) {
141 exit(EXIT_FAILURE);141 exit(EXIT_FAILURE);
142}142}
143143
144static Buf ast_token_str(Buf *input, Token *token) {
145 Buf str = BUF_INIT;
146 buf_init_from_mem(&str, buf_ptr(input) + token->start_pos, token->end_pos - token->start_pos);
147 return str;
148}
149
150ATTRIBUTE_NORETURN144ATTRIBUTE_NORETURN
151static void ast_invalid_token_error(ParseContext *pc, Token *token) {145static void ast_invalid_token_error(ParseContext *pc, Token *token) {
152 Buf token_value = ast_token_str(pc->buf, token);146 ast_error(pc, token, "invalid token: '%s'", token_name(token->id));
153 ast_error(pc, token, "invalid token: '%s'", buf_ptr(&token_value));
154}147}
155148
156static AstNode *ast_create_node_no_line_info(ParseContext *pc, NodeType type) {149static AstNode *ast_create_node_no_line_info(ParseContext *pc, NodeType type) {
...@@ -213,7 +206,7 @@ static void put_back_token(ParseContext *pc) {...@@ -213,7 +206,7 @@ static void put_back_token(ParseContext *pc) {
213static Buf *token_buf(Token *token) {206static Buf *token_buf(Token *token) {
214 if (token == nullptr)207 if (token == nullptr)
215 return nullptr;208 return nullptr;
216 assert(token->id == TokenIdStringLiteral || token->id == TokenIdSymbol);209 assert(token->id == TokenIdStringLiteral || token->id == TokenIdMultilineStringLiteral || token->id == TokenIdSymbol);
217 return &token->data.str_lit.str;210 return &token->data.str_lit.str;
218}211}
219212
...@@ -596,7 +589,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {...@@ -596,7 +589,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
596 return res;589 return res;
597}590}
598591
599// TestDecl <- KEYWORD_test STRINGLITERAL Block592// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block
600static AstNode *ast_parse_test_decl(ParseContext *pc) {593static AstNode *ast_parse_test_decl(ParseContext *pc) {
601 Token *test = eat_token_if(pc, TokenIdKeywordTest);594 Token *test = eat_token_if(pc, TokenIdKeywordTest);
602 if (test == nullptr)595 if (test == nullptr)
...@@ -630,8 +623,8 @@ static AstNode *ast_parse_top_level_comptime(ParseContext *pc) {...@@ -630,8 +623,8 @@ static AstNode *ast_parse_top_level_comptime(ParseContext *pc) {
630}623}
631624
632// TopLevelDecl625// TopLevelDecl
633// <- (KEYWORD_export / KEYWORD_extern STRINGLITERAL? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)626// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
634// / (KEYWORD_export / KEYWORD_extern STRINGLITERAL?)? KEYWORD_threadlocal? VarDecl627// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl
635// / KEYWORD_use Expr SEMICOLON628// / KEYWORD_use Expr SEMICOLON
636static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod, Buf *doc_comments) {629static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod, Buf *doc_comments) {
637 Token *first = eat_token_if(pc, TokenIdKeywordExport);630 Token *first = eat_token_if(pc, TokenIdKeywordExport);
...@@ -1729,6 +1722,8 @@ static AstNode *ast_parse_primary_type_expr(ParseContext *pc) {...@@ -1729,6 +1722,8 @@ static AstNode *ast_parse_primary_type_expr(ParseContext *pc) {
1729 return ast_create_node(pc, NodeTypeUnreachable, unreachable);1722 return ast_create_node(pc, NodeTypeUnreachable, unreachable);
17301723
1731 Token *string_lit = eat_token_if(pc, TokenIdStringLiteral);1724 Token *string_lit = eat_token_if(pc, TokenIdStringLiteral);
1725 if (string_lit == nullptr)
1726 string_lit = eat_token_if(pc, TokenIdMultilineStringLiteral);
1732 if (string_lit != nullptr) {1727 if (string_lit != nullptr) {
1733 AstNode *res = ast_create_node(pc, NodeTypeStringLiteral, string_lit);1728 AstNode *res = ast_create_node(pc, NodeTypeStringLiteral, string_lit);
1734 res->data.string_literal.buf = token_buf(string_lit);1729 res->data.string_literal.buf = token_buf(string_lit);
...@@ -1957,7 +1952,9 @@ static AsmOutput *ast_parse_asm_output_item(ParseContext *pc) {...@@ -1957,7 +1952,9 @@ static AsmOutput *ast_parse_asm_output_item(ParseContext *pc) {
1957 Token *sym_name = expect_token(pc, TokenIdSymbol);1952 Token *sym_name = expect_token(pc, TokenIdSymbol);
1958 expect_token(pc, TokenIdRBracket);1953 expect_token(pc, TokenIdRBracket);
19591954
1960 Token *str = expect_token(pc, TokenIdStringLiteral);1955 Token *str = eat_token_if(pc, TokenIdMultilineStringLiteral);
1956 if (str == nullptr)
1957 str = expect_token(pc, TokenIdStringLiteral);
1961 expect_token(pc, TokenIdLParen);1958 expect_token(pc, TokenIdLParen);
19621959
1963 Token *var_name = eat_token_if(pc, TokenIdSymbol);1960 Token *var_name = eat_token_if(pc, TokenIdSymbol);
...@@ -1999,7 +1996,9 @@ static AsmInput *ast_parse_asm_input_item(ParseContext *pc) {...@@ -1999,7 +1996,9 @@ static AsmInput *ast_parse_asm_input_item(ParseContext *pc) {
1999 Token *sym_name = expect_token(pc, TokenIdSymbol);1996 Token *sym_name = expect_token(pc, TokenIdSymbol);
2000 expect_token(pc, TokenIdRBracket);1997 expect_token(pc, TokenIdRBracket);
20011998
2002 Token *constraint = expect_token(pc, TokenIdStringLiteral);1999 Token *constraint = eat_token_if(pc, TokenIdMultilineStringLiteral);
2000 if (constraint == nullptr)
2001 constraint = expect_token(pc, TokenIdStringLiteral);
2003 expect_token(pc, TokenIdLParen);2002 expect_token(pc, TokenIdLParen);
2004 AstNode *expr = ast_expect(pc, ast_parse_expr);2003 AstNode *expr = ast_expect(pc, ast_parse_expr);
2005 expect_token(pc, TokenIdRParen);2004 expect_token(pc, TokenIdRParen);
...@@ -2018,6 +2017,8 @@ static AstNode *ast_parse_asm_clobbers(ParseContext *pc) {...@@ -2018,6 +2017,8 @@ static AstNode *ast_parse_asm_clobbers(ParseContext *pc) {
20182017
2019 ZigList<Buf *> clobber_list = ast_parse_list<Buf>(pc, TokenIdComma, [](ParseContext *context) {2018 ZigList<Buf *> clobber_list = ast_parse_list<Buf>(pc, TokenIdComma, [](ParseContext *context) {
2020 Token *str = eat_token_if(context, TokenIdStringLiteral);2019 Token *str = eat_token_if(context, TokenIdStringLiteral);
2020 if (str == nullptr)
2021 str = eat_token_if(context, TokenIdMultilineStringLiteral);
2021 if (str != nullptr)2022 if (str != nullptr)
2022 return token_buf(str);2023 return token_buf(str);
2023 return (Buf*)nullptr;2024 return (Buf*)nullptr;
src/tokenizer.cpp+3-16
...@@ -209,7 +209,6 @@ enum TokenizeState {...@@ -209,7 +209,6 @@ enum TokenizeState {
209 TokenizeStateLineString,209 TokenizeStateLineString,
210 TokenizeStateLineStringEnd,210 TokenizeStateLineStringEnd,
211 TokenizeStateLineStringContinue,211 TokenizeStateLineStringContinue,
212 TokenizeStateLineStringContinueC,
213 TokenizeStateSawEq,212 TokenizeStateSawEq,
214 TokenizeStateSawBang,213 TokenizeStateSawBang,
215 TokenizeStateSawLessThan,214 TokenizeStateSawLessThan,
...@@ -266,7 +265,7 @@ static void set_token_id(Tokenize *t, Token *token, TokenId id) {...@@ -266,7 +265,7 @@ static void set_token_id(Tokenize *t, Token *token, TokenId id) {
266 } else if (id == TokenIdFloatLiteral) {265 } else if (id == TokenIdFloatLiteral) {
267 bigfloat_init_32(&token->data.float_lit.bigfloat, 0.0f);266 bigfloat_init_32(&token->data.float_lit.bigfloat, 0.0f);
268 token->data.float_lit.overflow = false;267 token->data.float_lit.overflow = false;
269 } else if (id == TokenIdStringLiteral || id == TokenIdSymbol) {268 } else if (id == TokenIdStringLiteral || id == TokenIdMultilineStringLiteral || id == TokenIdSymbol) {
270 memset(&token->data.str_lit.str, 0, sizeof(Buf));269 memset(&token->data.str_lit.str, 0, sizeof(Buf));
271 buf_resize(&token->data.str_lit.str, 0);270 buf_resize(&token->data.str_lit.str, 0);
272 }271 }
...@@ -503,7 +502,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -503,7 +502,7 @@ void tokenize(Buf *buf, Tokenization *out) {
503 t.state = TokenizeStateSawSlash;502 t.state = TokenizeStateSawSlash;
504 break;503 break;
505 case '\\':504 case '\\':
506 begin_token(&t, TokenIdStringLiteral);505 begin_token(&t, TokenIdMultilineStringLiteral);
507 t.state = TokenizeStateSawBackslash;506 t.state = TokenizeStateSawBackslash;
508 break;507 break;
509 case '%':508 case '%':
...@@ -945,18 +944,6 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -945,18 +944,6 @@ void tokenize(Buf *buf, Tokenization *out) {
945 continue;944 continue;
946 }945 }
947 break;946 break;
948 case TokenizeStateLineStringContinueC:
949 switch (c) {
950 case '\\':
951 t.state = TokenizeStateLineStringContinue;
952 break;
953 default:
954 t.pos -= 1;
955 end_token(&t);
956 t.state = TokenizeStateStart;
957 continue;
958 }
959 break;
960 case TokenizeStateLineStringContinue:947 case TokenizeStateLineStringContinue:
961 switch (c) {948 switch (c) {
962 case '\\':949 case '\\':
...@@ -1471,7 +1458,6 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1471,7 +1458,6 @@ void tokenize(Buf *buf, Tokenization *out) {
1471 case TokenizeStateSawDotDot:1458 case TokenizeStateSawDotDot:
1472 case TokenizeStateSawBackslash:1459 case TokenizeStateSawBackslash:
1473 case TokenizeStateLineStringContinue:1460 case TokenizeStateLineStringContinue:
1474 case TokenizeStateLineStringContinueC:
1475 tokenize_error(&t, "unexpected EOF");1461 tokenize_error(&t, "unexpected EOF");
1476 break;1462 break;
1477 case TokenizeStateLineComment:1463 case TokenizeStateLineComment:
...@@ -1607,6 +1593,7 @@ const char * token_name(TokenId id) {...@@ -1607,6 +1593,7 @@ const char * token_name(TokenId id) {
1607 case TokenIdStar: return "*";1593 case TokenIdStar: return "*";
1608 case TokenIdStarStar: return "**";1594 case TokenIdStarStar: return "**";
1609 case TokenIdStringLiteral: return "StringLiteral";1595 case TokenIdStringLiteral: return "StringLiteral";
1596 case TokenIdMultilineStringLiteral: return "MultilineStringLiteral";
1610 case TokenIdSymbol: return "Symbol";1597 case TokenIdSymbol: return "Symbol";
1611 case TokenIdTilde: return "~";1598 case TokenIdTilde: return "~";
1612 case TokenIdTimesEq: return "*=";1599 case TokenIdTimesEq: return "*=";
src/tokenizer.hpp+2-1
...@@ -124,6 +124,7 @@ enum TokenId {...@@ -124,6 +124,7 @@ enum TokenId {
124 TokenIdStar,124 TokenIdStar,
125 TokenIdStarStar,125 TokenIdStarStar,
126 TokenIdStringLiteral,126 TokenIdStringLiteral,
127 TokenIdMultilineStringLiteral,
127 TokenIdSymbol,128 TokenIdSymbol,
128 TokenIdTilde,129 TokenIdTilde,
129 TokenIdTimesEq,130 TokenIdTimesEq,
...@@ -165,7 +166,7 @@ struct Token {...@@ -165,7 +166,7 @@ struct Token {
165 // TokenIdFloatLiteral166 // TokenIdFloatLiteral
166 TokenFloatLit float_lit;167 TokenFloatLit float_lit;
167168
168 // TokenIdStringLiteral or TokenIdSymbol169 // TokenIdStringLiteral, TokenIdMultilineStringLiteral or TokenIdSymbol
169 TokenStrLit str_lit;170 TokenStrLit str_lit;
170171
171 // TokenIdCharLiteral172 // TokenIdCharLiteral