authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-08-16 12:24:06+03:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-08-16 06:17:28-07:00
log2151f84d59f8af6e28570cb01a3a134c7b068fa2
tree8ad09e707b33b7f53ed7964aa24bd4fd426ba6a7
parent1e3b6816a8929bb141fb2157197077d74656e7ca

implement new async syntax in self-hosted compiler

8 files changed, 41 insertions(+), 145 deletions(-)

doc/langref.html.in+2-4
......@@ -9987,7 +9987,7 @@ TypeExpr &lt;- PrefixTypeOp* ErrorUnionExpr
99879987ErrorUnionExpr &lt;- SuffixExpr (EXCLAMATIONMARK TypeExpr)?
99889988
99899989SuffixExpr
9990 &lt;- AsyncPrefix PrimaryTypeExpr SuffixOp* FnCallArguments
9990 &lt;- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments
99919991 / PrimaryTypeExpr (SuffixOp / FnCallArguments)*
99929992
99939993PrimaryTypeExpr
......@@ -10063,7 +10063,7 @@ FnCC
1006310063 &lt;- KEYWORD_nakedcc
1006410064 / KEYWORD_stdcallcc
1006510065 / KEYWORD_extern
10066 / KEYWORD_async (LARROW TypeExpr RARROW)?
10066 / KEYWORD_async
1006710067
1006810068ParamDecl &lt;- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType
1006910069
......@@ -10168,8 +10168,6 @@ SuffixOp
1016810168 / DOTASTERISK
1016910169 / DOTQUESTIONMARK
1017010170
10171AsyncPrefix &lt;- KEYWORD_async (LARROW PrefixExpr RARROW)?
10172
1017310171FnCallArguments &lt;- LPAREN ExprList RPAREN
1017410172
1017510173# Ptr specific
src-self-hosted/ir.zig-1
......@@ -1181,7 +1181,6 @@ pub const Builder = struct {
11811181 ast.Node.Id.ErrorTag => return error.Unimplemented,
11821182 ast.Node.Id.AsmInput => return error.Unimplemented,
11831183 ast.Node.Id.AsmOutput => return error.Unimplemented,
1184 ast.Node.Id.AsyncAttribute => return error.Unimplemented,
11851184 ast.Node.Id.ParamDecl => return error.Unimplemented,
11861185 ast.Node.Id.FieldInitializer => return error.Unimplemented,
11871186 ast.Node.Id.EnumLiteral => return error.Unimplemented,
src-self-hosted/translate_c.zig+1-2
......@@ -1037,7 +1037,7 @@ fn transCreateNodeFnCall(c: *Context, fn_expr: *ast.Node) !*ast.Node.SuffixOp {
10371037 .op = ast.Node.SuffixOp.Op{
10381038 .Call = ast.Node.SuffixOp.Op.Call{
10391039 .params = ast.Node.SuffixOp.Op.Call.ParamList.init(c.a()),
1040 .async_attr = null,
1040 .async_token = null,
10411041 },
10421042 },
10431043 .rtoken = undefined, // set after appending args
......@@ -1355,7 +1355,6 @@ fn finishTransFnProto(
13551355 .var_args_token = null, // TODO this field is broken in the AST data model
13561356 .extern_export_inline_token = extern_export_inline_tok,
13571357 .cc_token = cc_tok,
1358 .async_attr = null,
13591358 .body_node = null,
13601359 .lib_name = null,
13611360 .align_expr = null,
src/parser.cpp+15-29
......@@ -113,7 +113,6 @@ static AstNode *ast_parse_multiply_op(ParseContext *pc);
113113static AstNode *ast_parse_prefix_op(ParseContext *pc);
114114static AstNode *ast_parse_prefix_type_op(ParseContext *pc);
115115static AstNode *ast_parse_suffix_op(ParseContext *pc);
116static AstNode *ast_parse_async_prefix(ParseContext *pc);
117116static AstNode *ast_parse_fn_call_argumnets(ParseContext *pc);
118117static AstNode *ast_parse_array_type_start(ParseContext *pc);
119118static AstNode *ast_parse_ptr_type_start(ParseContext *pc);
......@@ -1389,22 +1388,18 @@ static AstNode *ast_parse_error_union_expr(ParseContext *pc) {
13891388}
13901389
13911390// SuffixExpr
1392// <- AsyncPrefix PrimaryTypeExpr SuffixOp* FnCallArguments
1391// <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments
13931392// / PrimaryTypeExpr (SuffixOp / FnCallArguments)*
13941393static AstNode *ast_parse_suffix_expr(ParseContext *pc) {
1395 AstNode *async_call = ast_parse_async_prefix(pc);
1396 if (async_call != nullptr) {
1394 Token *async_token = eat_token_if(pc, TokenIdKeywordAsync);
1395 if (async_token != nullptr) {
13971396 if (eat_token_if(pc, TokenIdKeywordFn) != nullptr) {
13981397 // HACK: If we see the keyword `fn`, then we assume that
13991398 // we are parsing an async fn proto, and not a call.
14001399 // We therefore put back all tokens consumed by the async
14011400 // prefix...
1402 // HACK: This loop is not actually enough to put back all the
1403 // tokens. Let's hope this is fine for most code right now
1404 // and wait till we get the async rework for a syntax update.
1405 do {
1406 put_back_token(pc);
1407 } while (peek_token(pc)->id != TokenIdKeywordAsync);
1401 put_back_token(pc);
1402 put_back_token(pc);
14081403
14091404 return ast_parse_primary_type_expr(pc);
14101405 }
......@@ -1446,10 +1441,14 @@ static AstNode *ast_parse_suffix_expr(ParseContext *pc) {
14461441 ast_invalid_token_error(pc, peek_token(pc));
14471442
14481443 assert(args->type == NodeTypeFnCallExpr);
1449 async_call->data.fn_call_expr.fn_ref_expr = child;
1450 async_call->data.fn_call_expr.params = args->data.fn_call_expr.params;
1451 async_call->data.fn_call_expr.is_builtin = false;
1452 return async_call;
1444
1445 AstNode *res = ast_create_node(pc, NodeTypeFnCallExpr, async_token);
1446 res->data.fn_call_expr.is_async = true;
1447 res->data.fn_call_expr.seen = false;
1448 res->data.fn_call_expr.fn_ref_expr = child;
1449 res->data.fn_call_expr.params = args->data.fn_call_expr.params;
1450 res->data.fn_call_expr.is_builtin = false;
1451 return res;
14531452 }
14541453
14551454 AstNode *res = ast_parse_primary_type_expr(pc);
......@@ -1501,7 +1500,7 @@ static AstNode *ast_parse_suffix_expr(ParseContext *pc) {
15011500// <- BUILTINIDENTIFIER FnCallArguments
15021501// / CHAR_LITERAL
15031502// / ContainerDecl
1504// / DOT IDENTIFIER
1503// / DOT IDENTIFIER
15051504// / ErrorSetDecl
15061505// / FLOAT
15071506// / FnProto
......@@ -2016,7 +2015,7 @@ static AstNode *ast_parse_link_section(ParseContext *pc) {
20162015// <- KEYWORD_nakedcc
20172016// / KEYWORD_stdcallcc
20182017// / KEYWORD_extern
2019// / KEYWORD_async (LARROW TypeExpr RARROW)?
2018// / KEYWORD_async
20202019static Optional<AstNodeFnProto> ast_parse_fn_cc(ParseContext *pc) {
20212020 AstNodeFnProto res = {};
20222021 if (eat_token_if(pc, TokenIdKeywordNakedCC) != nullptr) {
......@@ -2657,19 +2656,6 @@ static AstNode *ast_parse_suffix_op(ParseContext *pc) {
26572656 return nullptr;
26582657}
26592658
2660// AsyncPrefix <- KEYWORD_async (LARROW PrefixExpr RARROW)?
2661static AstNode *ast_parse_async_prefix(ParseContext *pc) {
2662 Token *async = eat_token_if(pc, TokenIdKeywordAsync);
2663 if (async == nullptr)
2664 return nullptr;
2665
2666 AstNode *res = ast_create_node(pc, NodeTypeFnCallExpr, async);
2667 res->data.fn_call_expr.is_async = true;
2668 res->data.fn_call_expr.seen = false;
2669
2670 return res;
2671}
2672
26732659// FnCallArguments <- LPAREN ExprList RPAREN
26742660static AstNode *ast_parse_fn_call_argumnets(ParseContext *pc) {
26752661 Token *paren = eat_token_if(pc, TokenIdLParen);
std/zig/ast.zig+2-35
......@@ -434,7 +434,6 @@ pub const Node = struct {
434434 ErrorTag,
435435 AsmInput,
436436 AsmOutput,
437 AsyncAttribute,
438437 ParamDecl,
439438 FieldInitializer,
440439 };
......@@ -838,36 +837,6 @@ pub const Node = struct {
838837 }
839838 };
840839
841 pub const AsyncAttribute = struct {
842 base: Node,
843 async_token: TokenIndex,
844 allocator_type: ?*Node,
845 rangle_bracket: ?TokenIndex,
846
847 pub fn iterate(self: *AsyncAttribute, index: usize) ?*Node {
848 var i = index;
849
850 if (self.allocator_type) |allocator_type| {
851 if (i < 1) return allocator_type;
852 i -= 1;
853 }
854
855 return null;
856 }
857
858 pub fn firstToken(self: *const AsyncAttribute) TokenIndex {
859 return self.async_token;
860 }
861
862 pub fn lastToken(self: *const AsyncAttribute) TokenIndex {
863 if (self.rangle_bracket) |rangle_bracket| {
864 return rangle_bracket;
865 }
866
867 return self.async_token;
868 }
869 };
870
871840 pub const FnProto = struct {
872841 base: Node,
873842 doc_comments: ?*DocComment,
......@@ -879,7 +848,6 @@ pub const Node = struct {
879848 var_args_token: ?TokenIndex,
880849 extern_export_inline_token: ?TokenIndex,
881850 cc_token: ?TokenIndex,
882 async_attr: ?*AsyncAttribute,
883851 body_node: ?*Node,
884852 lib_name: ?*Node, // populated if this is an extern declaration
885853 align_expr: ?*Node, // populated if align(A) is present
......@@ -935,7 +903,6 @@ pub const Node = struct {
935903
936904 pub fn firstToken(self: *const FnProto) TokenIndex {
937905 if (self.visib_token) |visib_token| return visib_token;
938 if (self.async_attr) |async_attr| return async_attr.firstToken();
939906 if (self.extern_export_inline_token) |extern_export_inline_token| return extern_export_inline_token;
940907 assert(self.lib_name == null);
941908 if (self.cc_token) |cc_token| return cc_token;
......@@ -1699,7 +1666,7 @@ pub const Node = struct {
16991666
17001667 pub const Call = struct {
17011668 params: ParamList,
1702 async_attr: ?*AsyncAttribute,
1669 async_token: ?TokenIndex,
17031670
17041671 pub const ParamList = SegmentedList(*Node, 2);
17051672 };
......@@ -1752,7 +1719,7 @@ pub const Node = struct {
17521719
17531720 pub fn firstToken(self: *const SuffixOp) TokenIndex {
17541721 switch (self.op) {
1755 .Call => |*call_info| if (call_info.async_attr) |async_attr| return async_attr.firstToken(),
1722 .Call => |*call_info| if (call_info.async_token) |async_token| return async_token,
17561723 else => {},
17571724 }
17581725 return self.lhs.firstToken();
std/zig/parse.zig+17-52
......@@ -277,7 +277,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
277277
278278/// FnProto <- FnCC? KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)
279279fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
280 const cc = try parseFnCC(arena, it, tree);
280 const cc = parseFnCC(arena, it, tree);
281281 const fn_token = eatToken(it, .Keyword_fn) orelse {
282282 if (cc == null) return null else return error.ParseError;
283283 };
......@@ -320,7 +320,6 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
320320 .var_args_token = var_args_token,
321321 .extern_export_inline_token = null,
322322 .cc_token = null,
323 .async_attr = null,
324323 .body_node = null,
325324 .lib_name = null,
326325 .align_expr = align_expr,
......@@ -331,7 +330,6 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
331330 switch (kind) {
332331 .CC => |token| fn_proto_node.cc_token = token,
333332 .Extern => |token| fn_proto_node.extern_export_inline_token = token,
334 .Async => |node| fn_proto_node.async_attr = node,
335333 }
336334 }
337335
......@@ -1092,10 +1090,19 @@ fn parseErrorUnionExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No
10921090}
10931091
10941092/// SuffixExpr
1095/// <- AsyncPrefix PrimaryTypeExpr SuffixOp* FnCallArguments
1093/// <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments
10961094/// / PrimaryTypeExpr (SuffixOp / FnCallArguments)*
10971095fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
1098 if (try parseAsyncPrefix(arena, it, tree)) |async_node| {
1096 if (eatToken(it, .Keyword_async)) |async_token| {
1097 if (eatToken(it, .Keyword_fn)) |token_fn| {
1098 // HACK: If we see the keyword `fn`, then we assume that
1099 // we are parsing an async fn proto, and not a call.
1100 // We therefore put back all tokens consumed by the async
1101 // prefix...
1102 putBackToken(it, token_fn);
1103 putBackToken(it, async_token);
1104 return parsePrimaryTypeExpr(arena, it, tree);
1105 }
10991106 // TODO: Implement hack for parsing `async fn ...` in ast_parse_suffix_expr
11001107 var res = try expectNode(arena, it, tree, parsePrimaryTypeExpr, AstError{
11011108 .ExpectedPrimaryTypeExpr = AstError.ExpectedPrimaryTypeExpr{ .token = it.index },
......@@ -1116,7 +1123,6 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
11161123 });
11171124 return null;
11181125 };
1119
11201126 const node = try arena.create(Node.SuffixOp);
11211127 node.* = Node.SuffixOp{
11221128 .base = Node{ .id = .SuffixOp },
......@@ -1124,14 +1130,13 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
11241130 .op = Node.SuffixOp.Op{
11251131 .Call = Node.SuffixOp.Op.Call{
11261132 .params = params.list,
1127 .async_attr = async_node.cast(Node.AsyncAttribute).?,
1133 .async_token = async_token,
11281134 },
11291135 },
11301136 .rtoken = params.rparen,
11311137 };
11321138 return &node.base;
11331139 }
1134
11351140 if (try parsePrimaryTypeExpr(arena, it, tree)) |expr| {
11361141 var res = expr;
11371142
......@@ -1153,7 +1158,7 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
11531158 .op = Node.SuffixOp.Op{
11541159 .Call = Node.SuffixOp.Op.Call{
11551160 .params = params.list,
1156 .async_attr = null,
1161 .async_token = null,
11571162 },
11581163 },
11591164 .rtoken = params.rparen,
......@@ -1653,36 +1658,18 @@ fn parseLinkSection(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
16531658/// <- KEYWORD_nakedcc
16541659/// / KEYWORD_stdcallcc
16551660/// / KEYWORD_extern
1656/// / KEYWORD_async (LARROW TypeExpr RARROW)?
1657fn parseFnCC(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?FnCC {
1661/// / KEYWORD_async
1662fn parseFnCC(arena: *Allocator, it: *TokenIterator, tree: *Tree) ?FnCC {
16581663 if (eatToken(it, .Keyword_nakedcc)) |token| return FnCC{ .CC = token };
16591664 if (eatToken(it, .Keyword_stdcallcc)) |token| return FnCC{ .CC = token };
16601665 if (eatToken(it, .Keyword_extern)) |token| return FnCC{ .Extern = token };
1661 if (eatToken(it, .Keyword_async)) |token| {
1662 const node = try arena.create(Node.AsyncAttribute);
1663 node.* = Node.AsyncAttribute{
1664 .base = Node{ .id = .AsyncAttribute },
1665 .async_token = token,
1666 .allocator_type = null,
1667 .rangle_bracket = null,
1668 };
1669 if (eatToken(it, .AngleBracketLeft)) |_| {
1670 const type_expr = try expectNode(arena, it, tree, parseTypeExpr, AstError{
1671 .ExpectedTypeExpr = AstError.ExpectedTypeExpr{ .token = it.index },
1672 });
1673 const rarrow = try expectToken(it, tree, .AngleBracketRight);
1674 node.allocator_type = type_expr;
1675 node.rangle_bracket = rarrow;
1676 }
1677 return FnCC{ .Async = node };
1678 }
1666 if (eatToken(it, .Keyword_async)) |token| return FnCC{ .CC = token };
16791667 return null;
16801668}
16811669
16821670const FnCC = union(enum) {
16831671 CC: TokenIndex,
16841672 Extern: TokenIndex,
1685 Async: *Node.AsyncAttribute,
16861673};
16871674
16881675/// ParamDecl <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType
......@@ -2409,28 +2396,6 @@ fn parseSuffixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
24092396 return &node.base;
24102397}
24112398
2412/// AsyncPrefix <- KEYWORD_async (LARROW PrefixExpr RARROW)?
2413fn parseAsyncPrefix(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2414 const async_token = eatToken(it, .Keyword_async) orelse return null;
2415 var rangle_bracket: ?TokenIndex = null;
2416 const expr_node = if (eatToken(it, .AngleBracketLeft)) |_| blk: {
2417 const prefix_expr = try expectNode(arena, it, tree, parsePrefixExpr, AstError{
2418 .ExpectedPrefixExpr = AstError.ExpectedPrefixExpr{ .token = it.index },
2419 });
2420 rangle_bracket = try expectToken(it, tree, .AngleBracketRight);
2421 break :blk prefix_expr;
2422 } else null;
2423
2424 const node = try arena.create(Node.AsyncAttribute);
2425 node.* = Node.AsyncAttribute{
2426 .base = Node{ .id = .AsyncAttribute },
2427 .async_token = async_token,
2428 .allocator_type = expr_node,
2429 .rangle_bracket = rangle_bracket,
2430 };
2431 return &node.base;
2432}
2433
24342399/// FnCallArguments <- LPAREN ExprList RPAREN
24352400/// ExprList <- (Expr COMMA)* Expr?
24362401fn parseFnCallArguments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?AnnotatedParamList {
std/zig/parser_test.zig+2-2
......@@ -210,7 +210,7 @@ test "zig fmt: spaces around slice operator" {
210210test "zig fmt: async call in if condition" {
211211 try testCanonical(
212212 \\comptime {
213 \\ if (async<a> b()) {
213 \\ if (async b()) {
214214 \\ a();
215215 \\ }
216216 \\}
......@@ -1118,7 +1118,7 @@ test "zig fmt: first line comment in struct initializer" {
11181118 \\pub async fn acquire(self: *Self) HeldLock {
11191119 \\ return HeldLock{
11201120 \\ // guaranteed allocation elision
1121 \\ .held = await (async self.lock.acquire() catch unreachable),
1121 \\ .held = self.lock.acquire(),
11221122 \\ .value = &self.private_data,
11231123 \\ };
11241124 \\}
std/zig/render.zig+2-20
......@@ -284,20 +284,6 @@ fn renderExpression(
284284 return renderExpression(allocator, stream, tree, indent, start_col, comptime_node.expr, space);
285285 },
286286
287 ast.Node.Id.AsyncAttribute => {
288 const async_attr = @fieldParentPtr(ast.Node.AsyncAttribute, "base", base);
289
290 if (async_attr.allocator_type) |allocator_type| {
291 try renderToken(tree, stream, async_attr.async_token, indent, start_col, Space.None); // async
292
293 try renderToken(tree, stream, tree.nextToken(async_attr.async_token), indent, start_col, Space.None); // <
294 try renderExpression(allocator, stream, tree, indent, start_col, allocator_type, Space.None); // allocator
295 return renderToken(tree, stream, tree.nextToken(allocator_type.lastToken()), indent, start_col, space); // >
296 } else {
297 return renderToken(tree, stream, async_attr.async_token, indent, start_col, space); // async
298 }
299 },
300
301287 ast.Node.Id.Suspend => {
302288 const suspend_node = @fieldParentPtr(ast.Node.Suspend, "base", base);
303289
......@@ -459,8 +445,8 @@ fn renderExpression(
459445
460446 switch (suffix_op.op) {
461447 @TagType(ast.Node.SuffixOp.Op).Call => |*call_info| {
462 if (call_info.async_attr) |async_attr| {
463 try renderExpression(allocator, stream, tree, indent, start_col, &async_attr.base, Space.Space);
448 if (call_info.async_token) |async_token| {
449 try renderToken(tree, stream, async_token, indent, start_col, Space.Space);
464450 }
465451
466452 try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None);
......@@ -1121,10 +1107,6 @@ fn renderExpression(
11211107 try renderToken(tree, stream, cc_token, indent, start_col, Space.Space); // stdcallcc
11221108 }
11231109
1124 if (fn_proto.async_attr) |async_attr| {
1125 try renderExpression(allocator, stream, tree, indent, start_col, &async_attr.base, Space.Space);
1126 }
1127
11281110 const lparen = if (fn_proto.name_token) |name_token| blk: {
11291111 try renderToken(tree, stream, fn_proto.fn_token, indent, start_col, Space.Space); // fn
11301112 try renderToken(tree, stream, name_token, indent, start_col, Space.None); // name