authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-31 21:04:54+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-31 21:04:54+02:00
log4d8f9e2295bb672f70550b3bb5a4cb667f68bb70
tree434ac90b56af673e28fb1b42effb8351c595a7f2
parent4793c3397e17169317d2feded9ca6901ff0e99e8

std.zig.parser now parses multi line strings

3 files changed, 108 insertions(+), 5 deletions(-)

std/zig/ast.zig+21
......@@ -20,6 +20,7 @@ pub const Node = struct {
2020 IntegerLiteral,
2121 FloatLiteral,
2222 StringLiteral,
23 MultilineStringLiteral,
2324 UndefinedLiteral,
2425 BuiltinCall,
2526 Call,
......@@ -40,6 +41,7 @@ pub const Node = struct {
4041 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
4142 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
4243 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
44 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index),
4345 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index),
4446 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
4547 Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index),
......@@ -61,6 +63,7 @@ pub const Node = struct {
6163 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),
6264 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
6365 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
66 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(),
6467 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(),
6568 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
6669 Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(),
......@@ -82,6 +85,7 @@ pub const Node = struct {
8285 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),
8386 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
8487 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
88 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(),
8589 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(),
8690 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
8791 Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(),
......@@ -587,6 +591,23 @@ pub const NodeStringLiteral = struct {
587591 }
588592};
589593
594pub const NodeMultilineStringLiteral = struct {
595 base: Node,
596 tokens: ArrayList(Token),
597
598 pub fn iterate(self: &NodeMultilineStringLiteral, index: usize) ?&Node {
599 return null;
600 }
601
602 pub fn firstToken(self: &NodeMultilineStringLiteral) Token {
603 return self.tokens.at(0);
604 }
605
606 pub fn lastToken(self: &NodeMultilineStringLiteral) Token {
607 return self.tokens.at(self.tokens.len - 1);
608 }
609};
610
590611pub const NodeUndefinedLiteral = struct {
591612 base: Node,
592613 token: Token,
std/zig/parser.zig+48-4
......@@ -456,6 +456,30 @@ pub const Parser = struct {
456456 try stack.append(State.AfterOperand);
457457 continue;
458458 },
459 Token.Id.MultilineStringLiteralLine => {
460 const node = try arena.create(ast.NodeMultilineStringLiteral);
461 *node = ast.NodeMultilineStringLiteral {
462 .base = self.initNode(ast.Node.Id.MultilineStringLiteral),
463 .tokens = ArrayList(Token).init(arena),
464 };
465 try node.tokens.append(token);
466
467 while (true) {
468 const multiline_str = self.getNextToken();
469 if (multiline_str.id != Token.Id.MultilineStringLiteralLine) {
470 self.putBackToken(multiline_str);
471 break;
472 }
473
474 try node.tokens.append(multiline_str);
475 }
476
477 try stack.append(State {
478 .Operand = &node.base
479 });
480 try stack.append(State.AfterOperand);
481 continue;
482 },
459483
460484 else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
461485 }
......@@ -1427,6 +1451,20 @@ pub const Parser = struct {
14271451 const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);
14281452 try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));
14291453 },
1454 ast.Node.Id.MultilineStringLiteral => {
1455 const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base);
1456 try stream.print("\n");
1457
1458 var i : usize = 0;
1459 indent += 4;
1460 while (i < multiline_str_literal.tokens.len) : (i += 1) {
1461 const t = multiline_str_literal.tokens.at(i);
1462 try stream.writeByteNTimes(' ', indent);
1463 try stream.print("{}", self.tokenizer.getTokenSlice(t));
1464 }
1465 try stream.writeByteNTimes(' ', indent);
1466 indent -= 4;
1467 },
14301468 ast.Node.Id.UndefinedLiteral => {
14311469 const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base);
14321470 try stream.print("{}", self.tokenizer.getTokenSlice(undefined_literal.token));
......@@ -1806,6 +1844,16 @@ test "zig fmt: extern function" {
18061844 );
18071845}
18081846
1847test "zig fmt: multiline string" {
1848 try testCanonical(
1849 \\const s =
1850 \\ \\ something
1851 \\ \\ something else
1852 \\ ;
1853 \\
1854 );
1855}
1856
18091857test "zig fmt: values" {
18101858 try testCanonical(
18111859 \\test "values" {
......@@ -1813,10 +1861,6 @@ test "zig fmt: values" {
18131861 \\ 1.0;
18141862 \\ "string";
18151863 \\ c"cstring";
1816 \\ \\ Multi
1817 \\ \\ line
1818 \\ \\ string
1819 \\ ;
18201864 \\ 'c';
18211865 \\ true;
18221866 \\ false;
std/zig/tokenizer.zig+39-1
......@@ -72,6 +72,7 @@ pub const Token = struct {
7272 Invalid,
7373 Identifier,
7474 StringLiteral: StrLitKind,
75 MultilineStringLiteralLine: StrLitKind,
7576 StringIdentifier,
7677 Eof,
7778 Builtin,
......@@ -225,6 +226,9 @@ pub const Tokenizer = struct {
225226 C,
226227 StringLiteral,
227228 StringLiteralBackslash,
229 MultilineStringLiteralLine,
230 MultilineStringLiteralLineBackslash,
231 Backslash,
228232 Equal,
229233 Bang,
230234 Pipe,
......@@ -352,6 +356,10 @@ pub const Tokenizer = struct {
352356 '^' => {
353357 state = State.Caret;
354358 },
359 '\\' => {
360 state = State.Backslash;
361 result.id = Token.Id { .MultilineStringLiteralLine = Token.StrLitKind.Normal };
362 },
355363 '{' => {
356364 result.id = Token.Id.LBrace;
357365 self.index += 1;
......@@ -532,8 +540,17 @@ pub const Tokenizer = struct {
532540 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
533541 else => break,
534542 },
543 State.Backslash => switch (c) {
544 '\\' => {
545 state = State.MultilineStringLiteralLine;
546 },
547 else => break,
548 },
535549 State.C => switch (c) {
536 '\\' => @panic("TODO"),
550 '\\' => {
551 state = State.Backslash;
552 result.id = Token.Id { .MultilineStringLiteralLine = Token.StrLitKind.C };
553 },
537554 '"' => {
538555 state = State.StringLiteral;
539556 result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
......@@ -562,6 +579,24 @@ pub const Tokenizer = struct {
562579 },
563580 },
564581
582 State.MultilineStringLiteralLine => switch (c) {
583 '\\' => {
584 state = State.MultilineStringLiteralLineBackslash;
585 },
586 '\n' => {
587 self.index += 1;
588 break;
589 },
590 else => self.checkLiteralCharacter(),
591 },
592
593 State.MultilineStringLiteralLineBackslash => switch (c) {
594 '\n' => break, // Look for this error later.
595 else => {
596 state = State.MultilineStringLiteralLine;
597 },
598 },
599
565600 State.Bang => switch (c) {
566601 '=' => {
567602 result.id = Token.Id.BangEqual;
......@@ -811,6 +846,7 @@ pub const Tokenizer = struct {
811846 State.FloatFraction,
812847 State.FloatExponentNumber,
813848 State.StringLiteral, // find this error later
849 State.MultilineStringLiteralLine,
814850 State.Builtin => {},
815851
816852 State.Identifier => {
......@@ -825,6 +861,8 @@ pub const Tokenizer = struct {
825861 State.NumberDot,
826862 State.FloatExponentUnsigned,
827863 State.SawAtSign,
864 State.Backslash,
865 State.MultilineStringLiteralLineBackslash,
828866 State.StringLiteralBackslash => {
829867 result.id = Token.Id.Invalid;
830868 },