authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-11 14:43:53+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-11 14:43:53+02:00
logfe7146277d5a7f7c74a8c2f36719c946e6061c50
treeaf2fcea408dccc792180726923b6a7301949afd0
parent28ea364e5e60464f68501fdfa69ba41b9cf9c47e

std.zig.parser now accept both string and multiline string for strings

Related #909 Allows it to parse `std/special/compiler_rt/aullrem.zig`, `std/special/compiler_rt/aulldiv.zig` and `std/math/x86_64/sqrt.zig`

2 files changed, 81 insertions(+), 54 deletions(-)

std/zig/ast.zig+7-7
......@@ -1608,7 +1608,7 @@ pub const NodeThisLiteral = struct {
16081608pub const NodeAsmOutput = struct {
16091609 base: Node,
16101610 symbolic_name: &NodeIdentifier,
1611 constraint: &NodeStringLiteral,
1611 constraint: &Node,
16121612 kind: Kind,
16131613
16141614 const Kind = union(enum) {
......@@ -1622,7 +1622,7 @@ pub const NodeAsmOutput = struct {
16221622 if (i < 1) return &self.symbolic_name.base;
16231623 i -= 1;
16241624
1625 if (i < 1) return &self.constraint.base;
1625 if (i < 1) return self.constraint;
16261626 i -= 1;
16271627
16281628 switch (self.kind) {
......@@ -1654,7 +1654,7 @@ pub const NodeAsmOutput = struct {
16541654pub const NodeAsmInput = struct {
16551655 base: Node,
16561656 symbolic_name: &NodeIdentifier,
1657 constraint: &NodeStringLiteral,
1657 constraint: &Node,
16581658 expr: &Node,
16591659
16601660 pub fn iterate(self: &NodeAsmInput, index: usize) ?&Node {
......@@ -1663,7 +1663,7 @@ pub const NodeAsmInput = struct {
16631663 if (i < 1) return &self.symbolic_name.base;
16641664 i -= 1;
16651665
1666 if (i < 1) return &self.constraint.base;
1666 if (i < 1) return self.constraint;
16671667 i -= 1;
16681668
16691669 if (i < 1) return self.expr;
......@@ -1685,11 +1685,11 @@ pub const NodeAsm = struct {
16851685 base: Node,
16861686 asm_token: Token,
16871687 is_volatile: bool,
1688 template: Token,
1688 template: &Node,
16891689 //tokens: ArrayList(AsmToken),
16901690 outputs: ArrayList(&NodeAsmOutput),
16911691 inputs: ArrayList(&NodeAsmInput),
1692 cloppers: ArrayList(&NodeStringLiteral),
1692 cloppers: ArrayList(&Node),
16931693 rparen: Token,
16941694
16951695 pub fn iterate(self: &NodeAsm, index: usize) ?&Node {
......@@ -1701,7 +1701,7 @@ pub const NodeAsm = struct {
17011701 if (i < self.inputs.len) return &self.inputs.at(index).base;
17021702 i -= self.inputs.len;
17031703
1704 if (i < self.cloppers.len) return &self.cloppers.at(index).base;
1704 if (i < self.cloppers.len) return self.cloppers.at(index);
17051705 i -= self.cloppers.len;
17061706
17071707 return null;
std/zig/parser.zig+74-47
......@@ -171,7 +171,7 @@ pub const Parser = struct {
171171 Semicolon: &const &const ast.Node,
172172 AsmOutputItems: &ArrayList(&ast.NodeAsmOutput),
173173 AsmInputItems: &ArrayList(&ast.NodeAsmInput),
174 AsmClopperItems: &ArrayList(&ast.NodeStringLiteral),
174 AsmClopperItems: &ArrayList(&ast.Node),
175175 ExprListItemOrEnd: ExprListCtx,
176176 ExprListCommaOrEnd: ExprListCtx,
177177 FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer),
......@@ -301,7 +301,11 @@ pub const Parser = struct {
301301 Token.Id.Keyword_test => {
302302 stack.append(State.TopLevel) catch unreachable;
303303
304 const name_token = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;
304 const name_token = self.getNextToken();
305 const name = (try self.parseStringLiteral(arena, name_token)) ?? {
306 try self.parseError(&stack, name_token, "expected string literal, found {}", @tagName(name_token.id));
307 continue;
308 };
305309 const lbrace = (try self.expectToken(&stack, Token.Id.LBrace)) ?? continue;
306310
307311 const block = try self.createNode(arena, ast.NodeBlock,
......@@ -317,7 +321,7 @@ pub const Parser = struct {
317321 ast.NodeTestDecl {
318322 .base = undefined,
319323 .test_token = token,
320 .name = &(try self.createLiteral(arena, ast.NodeStringLiteral, name_token)).base,
324 .name = name,
321325 .body_node = &block.base,
322326 }
323327 );
......@@ -389,15 +393,12 @@ pub const Parser = struct {
389393 continue;
390394 },
391395 Token.Id.Keyword_extern => {
392 const lib_name_token = self.getNextToken();
393396 const lib_name = blk: {
394 if (lib_name_token.id == Token.Id.StringLiteral) {
395 const res = try self.createLiteral(arena, ast.NodeStringLiteral, lib_name_token);
396 break :blk &res.base;
397 } else {
397 const lib_name_token = self.getNextToken();
398 break :blk (try self.parseStringLiteral(arena, lib_name_token)) ?? {
398399 self.putBackToken(lib_name_token);
399400 break :blk null;
400 }
401 };
401402 };
402403
403404 stack.append(State {
......@@ -1504,10 +1505,6 @@ pub const Parser = struct {
15041505 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeFloatLiteral, token)).base);
15051506 continue;
15061507 },
1507 Token.Id.StringLiteral => {
1508 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeStringLiteral, token)).base);
1509 continue;
1510 },
15111508 Token.Id.CharLiteral => {
15121509 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeCharLiteral, token)).base);
15131510 continue;
......@@ -1536,24 +1533,8 @@ pub const Parser = struct {
15361533 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeUnreachable, token)).base);
15371534 continue;
15381535 },
1539 Token.Id.MultilineStringLiteralLine => {
1540 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeMultilineStringLiteral,
1541 ast.NodeMultilineStringLiteral {
1542 .base = undefined,
1543 .tokens = ArrayList(Token).init(arena),
1544 }
1545 );
1546 try node.tokens.append(token);
1547 while (true) {
1548 const multiline_str = self.getNextToken();
1549 if (multiline_str.id != Token.Id.MultilineStringLiteralLine) {
1550 self.putBackToken(multiline_str);
1551 break;
1552 }
1553
1554 try node.tokens.append(multiline_str);
1555 }
1556 continue;
1536 Token.Id.StringLiteral, Token.Id.MultilineStringLiteralLine => {
1537 dest_ptr.store((try self.parseStringLiteral(arena, token)) ?? unreachable);
15571538 },
15581539 Token.Id.LParen => {
15591540 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeGroupedExpression,
......@@ -1781,7 +1762,12 @@ pub const Parser = struct {
17811762 break :blk true;
17821763 };
17831764 _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;
1784 const template = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;
1765
1766 const template_token = self.getNextToken();
1767 const template = (try self.parseStringLiteral(arena, template_token)) ?? {
1768 try self.parseError(&stack, template_token, "expected string literal, found {}", @tagName(template_token.id));
1769 continue;
1770 };
17851771 // TODO parse template
17861772
17871773 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeAsm,
......@@ -1793,7 +1779,7 @@ pub const Parser = struct {
17931779 //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena),
17941780 .outputs = ArrayList(&ast.NodeAsmOutput).init(arena),
17951781 .inputs = ArrayList(&ast.NodeAsmInput).init(arena),
1796 .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena),
1782 .cloppers = ArrayList(&ast.Node).init(arena),
17971783 .rparen = undefined,
17981784 }
17991785 );
......@@ -1881,7 +1867,12 @@ pub const Parser = struct {
18811867
18821868 const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue;
18831869 _ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue;
1884 const constraint = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;
1870
1871 const constraint_token = self.getNextToken();
1872 const constraint = (try self.parseStringLiteral(arena, constraint_token)) ?? {
1873 try self.parseError(&stack, constraint_token, "expected string literal, found {}", @tagName(constraint_token.id));
1874 continue;
1875 };
18851876
18861877 _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;
18871878 try stack.append(State { .ExpectToken = Token.Id.RParen });
......@@ -1890,7 +1881,7 @@ pub const Parser = struct {
18901881 ast.NodeAsmOutput {
18911882 .base = undefined,
18921883 .symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name),
1893 .constraint = try self.createLiteral(arena, ast.NodeStringLiteral, constraint),
1884 .constraint = constraint,
18941885 .kind = undefined,
18951886 }
18961887 );
......@@ -1926,7 +1917,12 @@ pub const Parser = struct {
19261917
19271918 const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue;
19281919 _ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue;
1929 const constraint = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;
1920
1921 const constraint_token = self.getNextToken();
1922 const constraint = (try self.parseStringLiteral(arena, constraint_token)) ?? {
1923 try self.parseError(&stack, constraint_token, "expected string literal, found {}", @tagName(constraint_token.id));
1924 continue;
1925 };
19301926
19311927 _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;
19321928 try stack.append(State { .ExpectToken = Token.Id.RParen });
......@@ -1935,7 +1931,7 @@ pub const Parser = struct {
19351931 ast.NodeAsmInput {
19361932 .base = undefined,
19371933 .symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name),
1938 .constraint = try self.createLiteral(arena, ast.NodeStringLiteral, constraint),
1934 .constraint = constraint,
19391935 .expr = undefined,
19401936 }
19411937 );
......@@ -1944,13 +1940,13 @@ pub const Parser = struct {
19441940 },
19451941
19461942 State.AsmClopperItems => |items| {
1947 const string = self.getNextToken();
1948 if (string.id != Token.Id.StringLiteral) {
1949 self.putBackToken(string);
1943 const string_token = self.getNextToken();
1944 const string = (try self.parseStringLiteral(arena, string_token)) ?? {
1945 self.putBackToken(string_token);
19501946 continue;
1951 }
1947 };
1948 try items.append(string);
19521949
1953 try items.append(try self.createLiteral(arena, ast.NodeStringLiteral, string));
19541950 stack.append(State { .AsmClopperItems = items }) catch unreachable;
19551951 try stack.append(State { .IfToken = Token.Id.Comma });
19561952 },
......@@ -2742,6 +2738,37 @@ pub const Parser = struct {
27422738 }
27432739 }
27442740
2741 fn parseStringLiteral(self: &Parser, arena: &mem.Allocator, token: &const Token) !?&ast.Node {
2742 switch (token.id) {
2743 Token.Id.StringLiteral => {
2744 return &(try self.createLiteral(arena, ast.NodeStringLiteral, token)).base;
2745 },
2746 Token.Id.MultilineStringLiteralLine => {
2747 const node = try self.createNode(arena, ast.NodeMultilineStringLiteral,
2748 ast.NodeMultilineStringLiteral {
2749 .base = undefined,
2750 .tokens = ArrayList(Token).init(arena),
2751 }
2752 );
2753 try node.tokens.append(token);
2754 while (true) {
2755 const multiline_str = self.getNextToken();
2756 if (multiline_str.id != Token.Id.MultilineStringLiteralLine) {
2757 self.putBackToken(multiline_str);
2758 break;
2759 }
2760
2761 try node.tokens.append(multiline_str);
2762 }
2763
2764 return &node.base;
2765 },
2766 // TODO: We shouldn't need a cast, but:
2767 // zig: /home/jc/Documents/zig/src/ir.cpp:7962: TypeTableEntry* ir_resolve_peer_types(IrAnalyze*, AstNode*, IrInstruction**, size_t): Assertion `err_set_type != nullptr' failed.
2768 else => return (?&ast.Node)(null),
2769 }
2770 }
2771
27452772 fn parseBlockExpr(self: &Parser, stack: &ArrayList(State), arena: &mem.Allocator, dest_ptr: &const DestPtr, token: &const Token) !bool {
27462773 switch (token.id) {
27472774 Token.Id.Keyword_suspend => {
......@@ -4085,8 +4112,6 @@ pub const Parser = struct {
40854112 try stream.write("volatile ");
40864113 }
40874114
4088 try stream.print("({}", self.tokenizer.getTokenSlice(asm_node.template));
4089
40904115 try stack.append(RenderState { .Indent = indent });
40914116 try stack.append(RenderState { .Text = ")" });
40924117 {
......@@ -4094,7 +4119,7 @@ pub const Parser = struct {
40944119 var i = cloppers.len;
40954120 while (i != 0) {
40964121 i -= 1;
4097 try stack.append(RenderState { .Expression = &cloppers[i].base });
4122 try stack.append(RenderState { .Expression = cloppers[i] });
40984123
40994124 if (i != 0) {
41004125 try stack.append(RenderState { .Text = ", " });
......@@ -4163,6 +4188,8 @@ pub const Parser = struct {
41634188 try stack.append(RenderState.PrintIndent);
41644189 try stack.append(RenderState { .Indent = indent + indent_delta});
41654190 try stack.append(RenderState { .Text = "\n" });
4191 try stack.append(RenderState { .Expression = asm_node.template });
4192 try stack.append(RenderState { .Text = "(" });
41664193 },
41674194 ast.Node.Id.AsmInput => {
41684195 const asm_input = @fieldParentPtr(ast.NodeAsmInput, "base", base);
......@@ -4170,7 +4197,7 @@ pub const Parser = struct {
41704197 try stack.append(RenderState { .Text = ")"});
41714198 try stack.append(RenderState { .Expression = asm_input.expr});
41724199 try stack.append(RenderState { .Text = " ("});
4173 try stack.append(RenderState { .Expression = &asm_input.constraint.base});
4200 try stack.append(RenderState { .Expression = asm_input.constraint });
41744201 try stack.append(RenderState { .Text = "] "});
41754202 try stack.append(RenderState { .Expression = &asm_input.symbolic_name.base});
41764203 try stack.append(RenderState { .Text = "["});
......@@ -4189,7 +4216,7 @@ pub const Parser = struct {
41894216 },
41904217 }
41914218 try stack.append(RenderState { .Text = " ("});
4192 try stack.append(RenderState { .Expression = &asm_output.constraint.base});
4219 try stack.append(RenderState { .Expression = asm_output.constraint });
41934220 try stack.append(RenderState { .Text = "] "});
41944221 try stack.append(RenderState { .Expression = &asm_output.symbolic_name.base});
41954222 try stack.append(RenderState { .Text = "["});