authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-11 15:17:51+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-11 15:17:51+02:00
logdf4c575525a8ab3d190b46af724a718c77e607e3
tree1693a852b2febe3ab10abd3f01087e5f6c8675c9
parentfe7146277d5a7f7c74a8c2f36719c946e6061c50

std.zig.parser now parses inline fn proto

Related #909 Allows parsing of `std/os/zen.zig`.

2 files changed, 87 insertions(+), 57 deletions(-)

std/zig/ast.zig+4-6
...@@ -214,7 +214,7 @@ pub const NodeVarDecl = struct {...@@ -214,7 +214,7 @@ pub const NodeVarDecl = struct {
214 eq_token: Token,214 eq_token: Token,
215 mut_token: Token,215 mut_token: Token,
216 comptime_token: ?Token,216 comptime_token: ?Token,
217 extern_token: ?Token,217 extern_export_token: ?Token,
218 lib_name: ?&Node,218 lib_name: ?&Node,
219 type_node: ?&Node,219 type_node: ?&Node,
220 align_node: ?&Node,220 align_node: ?&Node,
...@@ -245,7 +245,7 @@ pub const NodeVarDecl = struct {...@@ -245,7 +245,7 @@ pub const NodeVarDecl = struct {
245 pub fn firstToken(self: &NodeVarDecl) Token {245 pub fn firstToken(self: &NodeVarDecl) Token {
246 if (self.visib_token) |visib_token| return visib_token;246 if (self.visib_token) |visib_token| return visib_token;
247 if (self.comptime_token) |comptime_token| return comptime_token;247 if (self.comptime_token) |comptime_token| return comptime_token;
248 if (self.extern_token) |extern_token| return extern_token;248 if (self.extern_export_token) |extern_export_token| return extern_export_token;
249 assert(self.lib_name == null);249 assert(self.lib_name == null);
250 return self.mut_token;250 return self.mut_token;
251 }251 }
...@@ -496,8 +496,7 @@ pub const NodeFnProto = struct {...@@ -496,8 +496,7 @@ pub const NodeFnProto = struct {
496 params: ArrayList(&Node),496 params: ArrayList(&Node),
497 return_type: ReturnType,497 return_type: ReturnType,
498 var_args_token: ?Token,498 var_args_token: ?Token,
499 extern_token: ?Token,499 extern_export_inline_token: ?Token,
500 inline_token: ?Token,
501 cc_token: ?Token,500 cc_token: ?Token,
502 async_attr: ?&NodeAsyncAttribute,501 async_attr: ?&NodeAsyncAttribute,
503 body_node: ?&Node,502 body_node: ?&Node,
...@@ -547,9 +546,8 @@ pub const NodeFnProto = struct {...@@ -547,9 +546,8 @@ pub const NodeFnProto = struct {
547546
548 pub fn firstToken(self: &NodeFnProto) Token {547 pub fn firstToken(self: &NodeFnProto) Token {
549 if (self.visib_token) |visib_token| return visib_token;548 if (self.visib_token) |visib_token| return visib_token;
550 if (self.extern_token) |extern_token| return extern_token;549 if (self.extern_export_inline_token) |extern_export_inline_token| return extern_export_inline_token;
551 assert(self.lib_name == null);550 assert(self.lib_name == null);
552 if (self.inline_token) |inline_token| return inline_token;
553 if (self.cc_token) |cc_token| return cc_token;551 if (self.cc_token) |cc_token| return cc_token;
554 return self.fn_token;552 return self.fn_token;
555 }553 }
std/zig/parser.zig+83-51
...@@ -55,7 +55,7 @@ pub const Parser = struct {...@@ -55,7 +55,7 @@ pub const Parser = struct {
55 const TopLevelDeclCtx = struct {55 const TopLevelDeclCtx = struct {
56 decls: &ArrayList(&ast.Node),56 decls: &ArrayList(&ast.Node),
57 visib_token: ?Token,57 visib_token: ?Token,
58 extern_token: ?Token,58 extern_export_inline_token: ?Token,
59 lib_name: ?&ast.Node,59 lib_name: ?&ast.Node,
60 };60 };
6161
...@@ -142,6 +142,7 @@ pub const Parser = struct {...@@ -142,6 +142,7 @@ pub const Parser = struct {
142 const State = union(enum) {142 const State = union(enum) {
143 TopLevel,143 TopLevel,
144 TopLevelExtern: TopLevelDeclCtx,144 TopLevelExtern: TopLevelDeclCtx,
145 TopLevelLibname: TopLevelDeclCtx,
145 TopLevelDecl: TopLevelDeclCtx,146 TopLevelDecl: TopLevelDeclCtx,
146 ContainerExtern: ContainerExternCtx,147 ContainerExtern: ContainerExternCtx,
147 ContainerDecl: &ast.NodeContainerDecl,148 ContainerDecl: &ast.NodeContainerDecl,
...@@ -332,13 +333,13 @@ pub const Parser = struct {...@@ -332,13 +333,13 @@ pub const Parser = struct {
332 root_node.eof_token = token;333 root_node.eof_token = token;
333 return Tree {.root_node = root_node, .arena_allocator = arena_allocator};334 return Tree {.root_node = root_node, .arena_allocator = arena_allocator};
334 },335 },
335 Token.Id.Keyword_pub, Token.Id.Keyword_export => {336 Token.Id.Keyword_pub => {
336 stack.append(State.TopLevel) catch unreachable;337 stack.append(State.TopLevel) catch unreachable;
337 try stack.append(State {338 try stack.append(State {
338 .TopLevelExtern = TopLevelDeclCtx {339 .TopLevelExtern = TopLevelDeclCtx {
339 .decls = &root_node.decls,340 .decls = &root_node.decls,
340 .visib_token = token,341 .visib_token = token,
341 .extern_token = null,342 .extern_export_inline_token = null,
342 .lib_name = null,343 .lib_name = null,
343 }344 }
344 });345 });
...@@ -363,7 +364,7 @@ pub const Parser = struct {...@@ -363,7 +364,7 @@ pub const Parser = struct {
363 .TopLevelExtern = TopLevelDeclCtx {364 .TopLevelExtern = TopLevelDeclCtx {
364 .decls = &root_node.decls,365 .decls = &root_node.decls,
365 .visib_token = null,366 .visib_token = null,
366 .extern_token = null,367 .extern_export_inline_token = null,
367 .lib_name = null,368 .lib_name = null,
368 }369 }
369 });370 });
...@@ -374,39 +375,24 @@ pub const Parser = struct {...@@ -374,39 +375,24 @@ pub const Parser = struct {
374 State.TopLevelExtern => |ctx| {375 State.TopLevelExtern => |ctx| {
375 const token = self.getNextToken();376 const token = self.getNextToken();
376 switch (token.id) {377 switch (token.id) {
377 Token.Id.Keyword_use => {378 Token.Id.Keyword_export, Token.Id.Keyword_inline => {
378 const node = try self.createAttachNode(arena, ctx.decls, ast.NodeUse,
379 ast.NodeUse {
380 .base = undefined,
381 .visib_token = ctx.visib_token,
382 .expr = undefined,
383 .semicolon_token = undefined,
384 }
385 );
386 stack.append(State {379 stack.append(State {
387 .ExpectTokenSave = ExpectTokenSave {380 .TopLevelDecl = TopLevelDeclCtx {
388 .id = Token.Id.Semicolon,381 .decls = ctx.decls,
389 .ptr = &node.semicolon_token,382 .visib_token = ctx.visib_token,
390 }383 .extern_export_inline_token = token,
384 .lib_name = null,
385 },
391 }) catch unreachable;386 }) catch unreachable;
392 try stack.append(State { .Expression = DestPtr { .Field = &node.expr } });
393 continue;387 continue;
394 },388 },
395 Token.Id.Keyword_extern => {389 Token.Id.Keyword_extern => {
396 const lib_name = blk: {
397 const lib_name_token = self.getNextToken();
398 break :blk (try self.parseStringLiteral(arena, lib_name_token)) ?? {
399 self.putBackToken(lib_name_token);
400 break :blk null;
401 };
402 };
403
404 stack.append(State {390 stack.append(State {
405 .TopLevelDecl = TopLevelDeclCtx {391 .TopLevelLibname = TopLevelDeclCtx {
406 .decls = ctx.decls,392 .decls = ctx.decls,
407 .visib_token = ctx.visib_token,393 .visib_token = ctx.visib_token,
408 .extern_token = token,394 .extern_export_inline_token = token,
409 .lib_name = lib_name,395 .lib_name = null,
410 },396 },
411 }) catch unreachable;397 }) catch unreachable;
412 continue;398 continue;
...@@ -418,17 +404,67 @@ pub const Parser = struct {...@@ -418,17 +404,67 @@ pub const Parser = struct {
418 }404 }
419 }405 }
420 },406 },
407
408 State.TopLevelLibname => |ctx| {
409 const lib_name = blk: {
410 const lib_name_token = self.getNextToken();
411 break :blk (try self.parseStringLiteral(arena, lib_name_token)) ?? {
412 self.putBackToken(lib_name_token);
413 break :blk null;
414 };
415 };
416
417 stack.append(State {
418 .TopLevelDecl = TopLevelDeclCtx {
419 .decls = ctx.decls,
420 .visib_token = ctx.visib_token,
421 .extern_export_inline_token = ctx.extern_export_inline_token,
422 .lib_name = lib_name,
423 },
424 }) catch unreachable;
425 },
426
421 State.TopLevelDecl => |ctx| {427 State.TopLevelDecl => |ctx| {
422 const token = self.getNextToken();428 const token = self.getNextToken();
423 switch (token.id) {429 switch (token.id) {
430 Token.Id.Keyword_use => {
431 if (ctx.extern_export_inline_token != null) {
432 try self.parseError(&stack, token, "Invalid token {}", @tagName((??ctx.extern_export_inline_token).id));
433 continue;
434 }
435
436 const node = try self.createAttachNode(arena, ctx.decls, ast.NodeUse,
437 ast.NodeUse {
438 .base = undefined,
439 .visib_token = ctx.visib_token,
440 .expr = undefined,
441 .semicolon_token = undefined,
442 }
443 );
444 stack.append(State {
445 .ExpectTokenSave = ExpectTokenSave {
446 .id = Token.Id.Semicolon,
447 .ptr = &node.semicolon_token,
448 }
449 }) catch unreachable;
450 try stack.append(State { .Expression = DestPtr { .Field = &node.expr } });
451 continue;
452 },
424 Token.Id.Keyword_var, Token.Id.Keyword_const => {453 Token.Id.Keyword_var, Token.Id.Keyword_const => {
454 if (ctx.extern_export_inline_token) |extern_export_inline_token| {
455 if (extern_export_inline_token.id == Token.Id.Keyword_inline) {
456 try self.parseError(&stack, token, "Invalid token {}", @tagName(extern_export_inline_token.id));
457 continue;
458 }
459 }
460
425 const var_decl_node = try self.createAttachNode(arena, ctx.decls, ast.NodeVarDecl,461 const var_decl_node = try self.createAttachNode(arena, ctx.decls, ast.NodeVarDecl,
426 ast.NodeVarDecl {462 ast.NodeVarDecl {
427 .base = undefined,463 .base = undefined,
428 .visib_token = ctx.visib_token,464 .visib_token = ctx.visib_token,
429 .mut_token = token,465 .mut_token = token,
430 .comptime_token = null,466 .comptime_token = null,
431 .extern_token = ctx.extern_token,467 .extern_export_token = ctx.extern_export_inline_token,
432 .type_node = null,468 .type_node = null,
433 .align_node = null,469 .align_node = null,
434 .init_node = null,470 .init_node = null,
...@@ -452,8 +488,7 @@ pub const Parser = struct {...@@ -452,8 +488,7 @@ pub const Parser = struct {
452 .params = ArrayList(&ast.Node).init(arena),488 .params = ArrayList(&ast.Node).init(arena),
453 .return_type = undefined,489 .return_type = undefined,
454 .var_args_token = null,490 .var_args_token = null,
455 .extern_token = ctx.extern_token,491 .extern_export_inline_token = ctx.extern_export_inline_token,
456 .inline_token = null,
457 .cc_token = null,492 .cc_token = null,
458 .async_attr = null,493 .async_attr = null,
459 .body_node = null,494 .body_node = null,
...@@ -475,8 +510,7 @@ pub const Parser = struct {...@@ -475,8 +510,7 @@ pub const Parser = struct {
475 .params = ArrayList(&ast.Node).init(arena),510 .params = ArrayList(&ast.Node).init(arena),
476 .return_type = undefined,511 .return_type = undefined,
477 .var_args_token = null,512 .var_args_token = null,
478 .extern_token = ctx.extern_token,513 .extern_export_inline_token = ctx.extern_export_inline_token,
479 .inline_token = null,
480 .cc_token = token,514 .cc_token = token,
481 .async_attr = null,515 .async_attr = null,
482 .body_node = null,516 .body_node = null,
...@@ -513,8 +547,7 @@ pub const Parser = struct {...@@ -513,8 +547,7 @@ pub const Parser = struct {
513 .params = ArrayList(&ast.Node).init(arena),547 .params = ArrayList(&ast.Node).init(arena),
514 .return_type = undefined,548 .return_type = undefined,
515 .var_args_token = null,549 .var_args_token = null,
516 .extern_token = ctx.extern_token,550 .extern_export_inline_token = ctx.extern_export_inline_token,
517 .inline_token = null,
518 .cc_token = null,551 .cc_token = null,
519 .async_attr = async_node,552 .async_attr = async_node,
520 .body_node = null,553 .body_node = null,
...@@ -752,7 +785,7 @@ pub const Parser = struct {...@@ -752,7 +785,7 @@ pub const Parser = struct {
752 .TopLevelExtern = TopLevelDeclCtx {785 .TopLevelExtern = TopLevelDeclCtx {
753 .decls = &container_decl.fields_and_decls,786 .decls = &container_decl.fields_and_decls,
754 .visib_token = token,787 .visib_token = token,
755 .extern_token = null,788 .extern_export_inline_token = null,
756 .lib_name = null,789 .lib_name = null,
757 }790 }
758 });791 });
...@@ -764,7 +797,7 @@ pub const Parser = struct {...@@ -764,7 +797,7 @@ pub const Parser = struct {
764 .TopLevelExtern = TopLevelDeclCtx {797 .TopLevelExtern = TopLevelDeclCtx {
765 .decls = &container_decl.fields_and_decls,798 .decls = &container_decl.fields_and_decls,
766 .visib_token = token,799 .visib_token = token,
767 .extern_token = null,800 .extern_export_inline_token = null,
768 .lib_name = null,801 .lib_name = null,
769 }802 }
770 });803 });
...@@ -781,7 +814,7 @@ pub const Parser = struct {...@@ -781,7 +814,7 @@ pub const Parser = struct {
781 .TopLevelExtern = TopLevelDeclCtx {814 .TopLevelExtern = TopLevelDeclCtx {
782 .decls = &container_decl.fields_and_decls,815 .decls = &container_decl.fields_and_decls,
783 .visib_token = null,816 .visib_token = null,
784 .extern_token = null,817 .extern_export_inline_token = null,
785 .lib_name = null,818 .lib_name = null,
786 }819 }
787 });820 });
...@@ -1659,8 +1692,7 @@ pub const Parser = struct {...@@ -1659,8 +1692,7 @@ pub const Parser = struct {
1659 .params = ArrayList(&ast.Node).init(arena),1692 .params = ArrayList(&ast.Node).init(arena),
1660 .return_type = undefined,1693 .return_type = undefined,
1661 .var_args_token = null,1694 .var_args_token = null,
1662 .extern_token = token,1695 .extern_export_inline_token = token,
1663 .inline_token = null,
1664 .cc_token = null,1696 .cc_token = null,
1665 .async_attr = null,1697 .async_attr = null,
1666 .body_node = null,1698 .body_node = null,
...@@ -1717,8 +1749,7 @@ pub const Parser = struct {...@@ -1717,8 +1749,7 @@ pub const Parser = struct {
1717 .params = ArrayList(&ast.Node).init(arena),1749 .params = ArrayList(&ast.Node).init(arena),
1718 .return_type = undefined,1750 .return_type = undefined,
1719 .var_args_token = null,1751 .var_args_token = null,
1720 .extern_token = null,1752 .extern_export_inline_token = null,
1721 .inline_token = null,
1722 .cc_token = null,1753 .cc_token = null,
1723 .async_attr = null,1754 .async_attr = null,
1724 .body_node = null,1755 .body_node = null,
...@@ -1740,8 +1771,7 @@ pub const Parser = struct {...@@ -1740,8 +1771,7 @@ pub const Parser = struct {
1740 .params = ArrayList(&ast.Node).init(arena),1771 .params = ArrayList(&ast.Node).init(arena),
1741 .return_type = undefined,1772 .return_type = undefined,
1742 .var_args_token = null,1773 .var_args_token = null,
1743 .extern_token = null,1774 .extern_export_inline_token = null,
1744 .inline_token = null,
1745 .cc_token = token,1775 .cc_token = token,
1746 .async_attr = null,1776 .async_attr = null,
1747 .body_node = null,1777 .body_node = null,
...@@ -2573,7 +2603,7 @@ pub const Parser = struct {...@@ -2573,7 +2603,7 @@ pub const Parser = struct {
2573 .visib_token = null,2603 .visib_token = null,
2574 .mut_token = mut_token,2604 .mut_token = mut_token,
2575 .comptime_token = next,2605 .comptime_token = next,
2576 .extern_token = null,2606 .extern_export_token = null,
2577 .type_node = null,2607 .type_node = null,
2578 .align_node = null,2608 .align_node = null,
2579 .init_node = null,2609 .init_node = null,
...@@ -2601,7 +2631,7 @@ pub const Parser = struct {...@@ -2601,7 +2631,7 @@ pub const Parser = struct {
2601 .visib_token = null,2631 .visib_token = null,
2602 .mut_token = next,2632 .mut_token = next,
2603 .comptime_token = null,2633 .comptime_token = null,
2604 .extern_token = null,2634 .extern_export_token = null,
2605 .type_node = null,2635 .type_node = null,
2606 .align_node = null,2636 .align_node = null,
2607 .init_node = null,2637 .init_node = null,
...@@ -3281,13 +3311,13 @@ pub const Parser = struct {...@@ -3281,13 +3311,13 @@ pub const Parser = struct {
3281 try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(comptime_token) });3311 try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(comptime_token) });
3282 }3312 }
32833313
3284 if (var_decl.extern_token) |extern_token| {3314 if (var_decl.extern_export_token) |extern_export_token| {
3285 if (var_decl.lib_name != null) {3315 if (var_decl.lib_name != null) {
3286 try stack.append(RenderState { .Text = " " });3316 try stack.append(RenderState { .Text = " " });
3287 try stack.append(RenderState { .Expression = ??var_decl.lib_name });3317 try stack.append(RenderState { .Expression = ??var_decl.lib_name });
3288 }3318 }
3289 try stack.append(RenderState { .Text = " " });3319 try stack.append(RenderState { .Text = " " });
3290 try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) });3320 try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_export_token) });
3291 }3321 }
32923322
3293 if (var_decl.visib_token) |visib_token| {3323 if (var_decl.visib_token) |visib_token| {
...@@ -3865,9 +3895,9 @@ pub const Parser = struct {...@@ -3865,9 +3895,9 @@ pub const Parser = struct {
3865 try stack.append(RenderState { .Text = " " });3895 try stack.append(RenderState { .Text = " " });
3866 try stack.append(RenderState { .Expression = lib_name });3896 try stack.append(RenderState { .Expression = lib_name });
3867 }3897 }
3868 if (fn_proto.extern_token) |extern_token| {3898 if (fn_proto.extern_export_inline_token) |extern_export_inline_token| {
3869 try stack.append(RenderState { .Text = " " });3899 try stack.append(RenderState { .Text = " " });
3870 try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_token) });3900 try stack.append(RenderState { .Text = self.tokenizer.getTokenSlice(extern_export_inline_token) });
3871 }3901 }
38723902
3873 if (fn_proto.visib_token) |visib_token| {3903 if (fn_proto.visib_token) |visib_token| {
...@@ -4608,6 +4638,8 @@ test "zig fmt: extern function" {...@@ -4608,6 +4638,8 @@ test "zig fmt: extern function" {
4608 try testCanonical(4638 try testCanonical(
4609 \\extern fn puts(s: &const u8) c_int;4639 \\extern fn puts(s: &const u8) c_int;
4610 \\extern "c" fn puts(s: &const u8) c_int;4640 \\extern "c" fn puts(s: &const u8) c_int;
4641 \\export fn puts(s: &const u8) c_int;
4642 \\inline fn puts(s: &const u8) c_int;
4611 \\4643 \\
4612 );4644 );
4613}4645}