authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-14 21:24:39+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-01-15 14:20:48-05:00
log0ea96c11ef5eca9cfd19233b6d51b00ba33da716
treedb54183a1c733a3e69da728cb4bc1501297bdf93
parent0a410519559788598fcf698013c1b6389d80c67c

disallow multiline strings in test and library names

6 files changed, 38 insertions(+), 59 deletions(-)

doc/langref.html.in+5-5
......@@ -10212,13 +10212,13 @@ ContainerMembers
1021210212 / ContainerField
1021310213 /
1021410214
10215TestDecl &lt;- KEYWORD_test STRINGLITERAL Block
10215TestDecl &lt;- KEYWORD_test STRINGLITERALSINGLE Block
1021610216
1021710217TopLevelComptime &lt;- KEYWORD_comptime BlockExpr
1021810218
1021910219TopLevelDecl
10220 &lt;- (KEYWORD_export / KEYWORD_extern STRINGLITERAL? / KEYWORD_inline)? FnProto (SEMICOLON / Block)
10221 / (KEYWORD_export / KEYWORD_extern STRINGLITERAL?)? KEYWORD_threadlocal? VarDecl
10220 &lt;- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / KEYWORD_inline)? FnProto (SEMICOLON / Block)
10221 / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl
1022210222 / KEYWORD_usingnamespace Expr SEMICOLON
1022310223
1022410224FnProto &lt;- FnCC? KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)
......@@ -10561,10 +10561,10 @@ INTEGER
1056110561 / "0o" [0-7]+ skip
1056210562 / "0x" hex+ skip
1056310563 / [0-9]+ skip
10564STRINGLITERALSINGLE &lt;- "\"" string_char* "\"" skip
1056410565STRINGLITERAL
10565 &lt;- "\"" string_char* "\"" skip
10566 &lt;- STRINGLITERALSINGLE
1056610567 / line_string skip
10567 / line_cstring skip
1056810568IDENTIFIER
1056910569 &lt;- !keyword [A-Za-z_] [A-Za-z0-9_]* skip
1057010570 / "@\"" string_char* "\"" skip
lib/std/zig/parse.zig+13-13
......@@ -183,10 +183,10 @@ fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree)
183183 return &node.base;
184184}
185185
186/// TestDecl <- KEYWORD_test STRINGLITERAL Block
186/// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block
187187fn parseTestDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
188188 const test_token = eatToken(it, .Keyword_test) orelse return null;
189 const name_node = try expectNode(arena, it, tree, parseStringLiteral, AstError{
189 const name_node = try expectNode(arena, it, tree, parseStringLiteralSingle, AstError{
190190 .ExpectedStringLiteral = AstError.ExpectedStringLiteral{ .token = it.index },
191191 });
192192 const block_node = try expectNode(arena, it, tree, parseBlock, AstError{
......@@ -225,15 +225,15 @@ fn parseTopLevelComptime(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*
225225}
226226
227227/// TopLevelDecl
228/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERAL? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
229/// / (KEYWORD_export / KEYWORD_extern STRINGLITERAL?)? KEYWORD_threadlocal? VarDecl
228/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
229/// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl
230230/// / KEYWORD_usingnamespace Expr SEMICOLON
231231fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
232232 var lib_name: ?*Node = null;
233233 const extern_export_inline_token = blk: {
234234 if (eatToken(it, .Keyword_export)) |token| break :blk token;
235235 if (eatToken(it, .Keyword_extern)) |token| {
236 lib_name = try parseStringLiteral(arena, it, tree);
236 lib_name = try parseStringLiteralSingle(arena, it, tree);
237237 break :blk token;
238238 }
239239 if (eatToken(it, .Keyword_inline)) |token| break :blk token;
......@@ -285,12 +285,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
285285
286286 if (extern_export_inline_token) |token| {
287287 if (lib_name) |string_literal_node|
288 if (string_literal_node.cast(Node.StringLiteral)) |single| {
289 putBackToken(it, single.token);
290 } else if (string_literal_node.cast(Node.MultilineStringLiteral)) |multi| {
291 while (multi.lines.pop()) |line|
292 putBackToken(it, line);
293 } else unreachable;
288 putBackToken(it, string_literal_node.cast(Node.StringLiteral).?.token);
294289 putBackToken(it, token);
295290 return null;
296291 }
......@@ -2776,8 +2771,7 @@ fn createLiteral(arena: *Allocator, comptime T: type, token: TokenIndex) !*Node
27762771 return &result.base;
27772772}
27782773
2779// string literal or multiline string literal
2780fn parseStringLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2774fn parseStringLiteralSingle(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
27812775 if (eatToken(it, .StringLiteral)) |token| {
27822776 const node = try arena.create(Node.StringLiteral);
27832777 node.* = Node.StringLiteral{
......@@ -2785,6 +2779,12 @@ fn parseStringLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Nod
27852779 };
27862780 return &node.base;
27872781 }
2782 return null;
2783}
2784
2785// string literal or multiline string literal
2786fn parseStringLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2787 if (try parseStringLiteralSingle(arena, it, tree)) |node| return node;
27882788
27892789 if (eatToken(it, .MultilineStringLiteralLine)) |first_line| {
27902790 const node = try arena.create(Node.MultilineStringLiteral);
lib/std/zig/parser_test.zig-10
......@@ -2721,16 +2721,6 @@ test "zig fmt: extern without container keyword returns error" {
27212721 );
27222722}
27232723
2724test "zig fmt: extern multiline lib name" {
2725 try testError(
2726 \\extern \\super
2727 \\ \\long
2728 \\ \\library
2729 \\ \\name
2730 \\
2731 );
2732}
2733
27342724const std = @import("std");
27352725const mem = std.mem;
27362726const warn = std.debug.warn;
src/parser.cpp+15-14
......@@ -141,16 +141,9 @@ static void ast_error(ParseContext *pc, Token *token, const char *format, ...) {
141141 exit(EXIT_FAILURE);
142142}
143143
144static Buf ast_token_str(Buf *input, Token *token) {
145 Buf str = BUF_INIT;
146 buf_init_from_mem(&str, buf_ptr(input) + token->start_pos, token->end_pos - token->start_pos);
147 return str;
148}
149
150144ATTRIBUTE_NORETURN
151145static void ast_invalid_token_error(ParseContext *pc, Token *token) {
152 Buf token_value = ast_token_str(pc->buf, token);
153 ast_error(pc, token, "invalid token: '%s'", buf_ptr(&token_value));
146 ast_error(pc, token, "invalid token: '%s'", token_name(token->id));
154147}
155148
156149static AstNode *ast_create_node_no_line_info(ParseContext *pc, NodeType type) {
......@@ -213,7 +206,7 @@ static void put_back_token(ParseContext *pc) {
213206static Buf *token_buf(Token *token) {
214207 if (token == nullptr)
215208 return nullptr;
216 assert(token->id == TokenIdStringLiteral || token->id == TokenIdSymbol);
209 assert(token->id == TokenIdStringLiteral || token->id == TokenIdMultilineStringLiteral || token->id == TokenIdSymbol);
217210 return &token->data.str_lit.str;
218211}
219212
......@@ -596,7 +589,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
596589 return res;
597590}
598591
599// TestDecl <- KEYWORD_test STRINGLITERAL Block
592// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block
600593static AstNode *ast_parse_test_decl(ParseContext *pc) {
601594 Token *test = eat_token_if(pc, TokenIdKeywordTest);
602595 if (test == nullptr)
......@@ -630,8 +623,8 @@ static AstNode *ast_parse_top_level_comptime(ParseContext *pc) {
630623}
631624
632625// TopLevelDecl
633// <- (KEYWORD_export / KEYWORD_extern STRINGLITERAL? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
634// / (KEYWORD_export / KEYWORD_extern STRINGLITERAL?)? KEYWORD_threadlocal? VarDecl
626// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block)
627// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl
635628// / KEYWORD_use Expr SEMICOLON
636629static AstNode *ast_parse_top_level_decl(ParseContext *pc, VisibMod visib_mod, Buf *doc_comments) {
637630 Token *first = eat_token_if(pc, TokenIdKeywordExport);
......@@ -1729,6 +1722,8 @@ static AstNode *ast_parse_primary_type_expr(ParseContext *pc) {
17291722 return ast_create_node(pc, NodeTypeUnreachable, unreachable);
17301723
17311724 Token *string_lit = eat_token_if(pc, TokenIdStringLiteral);
1725 if (string_lit == nullptr)
1726 string_lit = eat_token_if(pc, TokenIdMultilineStringLiteral);
17321727 if (string_lit != nullptr) {
17331728 AstNode *res = ast_create_node(pc, NodeTypeStringLiteral, string_lit);
17341729 res->data.string_literal.buf = token_buf(string_lit);
......@@ -1957,7 +1952,9 @@ static AsmOutput *ast_parse_asm_output_item(ParseContext *pc) {
19571952 Token *sym_name = expect_token(pc, TokenIdSymbol);
19581953 expect_token(pc, TokenIdRBracket);
19591954
1960 Token *str = expect_token(pc, TokenIdStringLiteral);
1955 Token *str = eat_token_if(pc, TokenIdMultilineStringLiteral);
1956 if (str == nullptr)
1957 str = expect_token(pc, TokenIdStringLiteral);
19611958 expect_token(pc, TokenIdLParen);
19621959
19631960 Token *var_name = eat_token_if(pc, TokenIdSymbol);
......@@ -1999,7 +1996,9 @@ static AsmInput *ast_parse_asm_input_item(ParseContext *pc) {
19991996 Token *sym_name = expect_token(pc, TokenIdSymbol);
20001997 expect_token(pc, TokenIdRBracket);
20011998
2002 Token *constraint = expect_token(pc, TokenIdStringLiteral);
1999 Token *constraint = eat_token_if(pc, TokenIdMultilineStringLiteral);
2000 if (constraint == nullptr)
2001 constraint = expect_token(pc, TokenIdStringLiteral);
20032002 expect_token(pc, TokenIdLParen);
20042003 AstNode *expr = ast_expect(pc, ast_parse_expr);
20052004 expect_token(pc, TokenIdRParen);
......@@ -2018,6 +2017,8 @@ static AstNode *ast_parse_asm_clobbers(ParseContext *pc) {
20182017
20192018 ZigList<Buf *> clobber_list = ast_parse_list<Buf>(pc, TokenIdComma, [](ParseContext *context) {
20202019 Token *str = eat_token_if(context, TokenIdStringLiteral);
2020 if (str == nullptr)
2021 str = eat_token_if(context, TokenIdMultilineStringLiteral);
20212022 if (str != nullptr)
20222023 return token_buf(str);
20232024 return (Buf*)nullptr;
src/tokenizer.cpp+3-16
......@@ -209,7 +209,6 @@ enum TokenizeState {
209209 TokenizeStateLineString,
210210 TokenizeStateLineStringEnd,
211211 TokenizeStateLineStringContinue,
212 TokenizeStateLineStringContinueC,
213212 TokenizeStateSawEq,
214213 TokenizeStateSawBang,
215214 TokenizeStateSawLessThan,
......@@ -266,7 +265,7 @@ static void set_token_id(Tokenize *t, Token *token, TokenId id) {
266265 } else if (id == TokenIdFloatLiteral) {
267266 bigfloat_init_32(&token->data.float_lit.bigfloat, 0.0f);
268267 token->data.float_lit.overflow = false;
269 } else if (id == TokenIdStringLiteral || id == TokenIdSymbol) {
268 } else if (id == TokenIdStringLiteral || id == TokenIdMultilineStringLiteral || id == TokenIdSymbol) {
270269 memset(&token->data.str_lit.str, 0, sizeof(Buf));
271270 buf_resize(&token->data.str_lit.str, 0);
272271 }
......@@ -503,7 +502,7 @@ void tokenize(Buf *buf, Tokenization *out) {
503502 t.state = TokenizeStateSawSlash;
504503 break;
505504 case '\\':
506 begin_token(&t, TokenIdStringLiteral);
505 begin_token(&t, TokenIdMultilineStringLiteral);
507506 t.state = TokenizeStateSawBackslash;
508507 break;
509508 case '%':
......@@ -945,18 +944,6 @@ void tokenize(Buf *buf, Tokenization *out) {
945944 continue;
946945 }
947946 break;
948 case TokenizeStateLineStringContinueC:
949 switch (c) {
950 case '\\':
951 t.state = TokenizeStateLineStringContinue;
952 break;
953 default:
954 t.pos -= 1;
955 end_token(&t);
956 t.state = TokenizeStateStart;
957 continue;
958 }
959 break;
960947 case TokenizeStateLineStringContinue:
961948 switch (c) {
962949 case '\\':
......@@ -1471,7 +1458,6 @@ void tokenize(Buf *buf, Tokenization *out) {
14711458 case TokenizeStateSawDotDot:
14721459 case TokenizeStateSawBackslash:
14731460 case TokenizeStateLineStringContinue:
1474 case TokenizeStateLineStringContinueC:
14751461 tokenize_error(&t, "unexpected EOF");
14761462 break;
14771463 case TokenizeStateLineComment:
......@@ -1607,6 +1593,7 @@ const char * token_name(TokenId id) {
16071593 case TokenIdStar: return "*";
16081594 case TokenIdStarStar: return "**";
16091595 case TokenIdStringLiteral: return "StringLiteral";
1596 case TokenIdMultilineStringLiteral: return "MultilineStringLiteral";
16101597 case TokenIdSymbol: return "Symbol";
16111598 case TokenIdTilde: return "~";
16121599 case TokenIdTimesEq: return "*=";
src/tokenizer.hpp+2-1
......@@ -124,6 +124,7 @@ enum TokenId {
124124 TokenIdStar,
125125 TokenIdStarStar,
126126 TokenIdStringLiteral,
127 TokenIdMultilineStringLiteral,
127128 TokenIdSymbol,
128129 TokenIdTilde,
129130 TokenIdTimesEq,
......@@ -165,7 +166,7 @@ struct Token {
165166 // TokenIdFloatLiteral
166167 TokenFloatLit float_lit;
167168
168 // TokenIdStringLiteral or TokenIdSymbol
169 // TokenIdStringLiteral, TokenIdMultilineStringLiteral or TokenIdSymbol
169170 TokenStrLit str_lit;
170171
171172 // TokenIdCharLiteral