authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-31 21:28:40+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-31 21:28:40+02:00
log975dc5a390490794df367b6a9869609a8b14d8f8
tree1efe2878e9b1bd001b231c9aa357d1929051ebfa
parent4d8f9e2295bb672f70550b3bb5a4cb667f68bb70

std.zig.parser now parses char literals

3 files changed, 83 insertions(+), 0 deletions(-)

std/zig/ast.zig+21
...@@ -21,6 +21,7 @@ pub const Node = struct {...@@ -21,6 +21,7 @@ pub const Node = struct {
21 FloatLiteral,21 FloatLiteral,
22 StringLiteral,22 StringLiteral,
23 MultilineStringLiteral,23 MultilineStringLiteral,
24 CharLiteral,
24 UndefinedLiteral,25 UndefinedLiteral,
25 BuiltinCall,26 BuiltinCall,
26 Call,27 Call,
...@@ -42,6 +43,7 @@ pub const Node = struct {...@@ -42,6 +43,7 @@ pub const Node = struct {
42 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),43 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
43 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),44 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
44 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index),45 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index),
46 Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index),
45 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index),47 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index),
46 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),48 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
47 Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index),49 Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index),
...@@ -64,6 +66,7 @@ pub const Node = struct {...@@ -64,6 +66,7 @@ pub const Node = struct {
64 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),66 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
65 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),67 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
66 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(),68 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(),
69 Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(),
67 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(),70 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(),
68 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),71 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
69 Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(),72 Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(),
...@@ -86,6 +89,7 @@ pub const Node = struct {...@@ -86,6 +89,7 @@ pub const Node = struct {
86 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),89 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
87 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),90 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
88 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(),91 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(),
92 Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(),
89 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(),93 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(),
90 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),94 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
91 Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(),95 Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(),
...@@ -608,6 +612,23 @@ pub const NodeMultilineStringLiteral = struct {...@@ -608,6 +612,23 @@ pub const NodeMultilineStringLiteral = struct {
608 }612 }
609};613};
610614
615pub const NodeCharLiteral = struct {
616 base: Node,
617 token: Token,
618
619 pub fn iterate(self: &NodeCharLiteral, index: usize) ?&Node {
620 return null;
621 }
622
623 pub fn firstToken(self: &NodeCharLiteral) Token {
624 return self.token;
625 }
626
627 pub fn lastToken(self: &NodeCharLiteral) Token {
628 return self.token;
629 }
630};
631
611pub const NodeUndefinedLiteral = struct {632pub const NodeUndefinedLiteral = struct {
612 base: Node,633 base: Node,
613 token: Token,634 token: Token,
std/zig/parser.zig+16
...@@ -456,6 +456,18 @@ pub const Parser = struct {...@@ -456,6 +456,18 @@ pub const Parser = struct {
456 try stack.append(State.AfterOperand);456 try stack.append(State.AfterOperand);
457 continue;457 continue;
458 },458 },
459 Token.Id.CharLiteral => {
460 const node = try arena.create(ast.NodeCharLiteral);
461 *node = ast.NodeCharLiteral {
462 .base = self.initNode(ast.Node.Id.CharLiteral),
463 .token = token,
464 };
465 try stack.append(State {
466 .Operand = &node.base
467 });
468 try stack.append(State.AfterOperand);
469 continue;
470 },
459 Token.Id.MultilineStringLiteralLine => {471 Token.Id.MultilineStringLiteralLine => {
460 const node = try arena.create(ast.NodeMultilineStringLiteral);472 const node = try arena.create(ast.NodeMultilineStringLiteral);
461 *node = ast.NodeMultilineStringLiteral {473 *node = ast.NodeMultilineStringLiteral {
...@@ -1451,6 +1463,10 @@ pub const Parser = struct {...@@ -1451,6 +1463,10 @@ pub const Parser = struct {
1451 const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);1463 const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);
1452 try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));1464 try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));
1453 },1465 },
1466 ast.Node.Id.CharLiteral => {
1467 const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base);
1468 try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token));
1469 },
1454 ast.Node.Id.MultilineStringLiteral => {1470 ast.Node.Id.MultilineStringLiteral => {
1455 const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base);1471 const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base);
1456 try stream.print("\n");1472 try stream.print("\n");
std/zig/tokenizer.zig+46
...@@ -73,6 +73,7 @@ pub const Token = struct {...@@ -73,6 +73,7 @@ pub const Token = struct {
73 Identifier,73 Identifier,
74 StringLiteral: StrLitKind,74 StringLiteral: StrLitKind,
75 MultilineStringLiteralLine: StrLitKind,75 MultilineStringLiteralLine: StrLitKind,
76 CharLiteral,
76 StringIdentifier,77 StringIdentifier,
77 Eof,78 Eof,
78 Builtin,79 Builtin,
...@@ -228,6 +229,9 @@ pub const Tokenizer = struct {...@@ -228,6 +229,9 @@ pub const Tokenizer = struct {
228 StringLiteralBackslash,229 StringLiteralBackslash,
229 MultilineStringLiteralLine,230 MultilineStringLiteralLine,
230 MultilineStringLiteralLineBackslash,231 MultilineStringLiteralLineBackslash,
232 CharLiteral,
233 CharLiteralBackslash,
234 CharLiteralEnd,
231 Backslash,235 Backslash,
232 Equal,236 Equal,
233 Bang,237 Bang,
...@@ -294,6 +298,9 @@ pub const Tokenizer = struct {...@@ -294,6 +298,9 @@ pub const Tokenizer = struct {
294 state = State.StringLiteral;298 state = State.StringLiteral;
295 result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };299 result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
296 },300 },
301 '\'' => {
302 state = State.CharLiteral;
303 },
297 'a'...'b', 'd'...'z', 'A'...'Z', '_' => {304 'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
298 state = State.Identifier;305 state = State.Identifier;
299 result.id = Token.Id.Identifier;306 result.id = Token.Id.Identifier;
...@@ -579,6 +586,35 @@ pub const Tokenizer = struct {...@@ -579,6 +586,35 @@ pub const Tokenizer = struct {
579 },586 },
580 },587 },
581588
589 State.CharLiteral => switch (c) {
590 '\\' => {
591 state = State.CharLiteralBackslash;
592 },
593 '\'' => break, // Look for this error later.
594 else => {
595 if (c < 0x20 or c == 0x7f)
596 break; // Look for this error later.
597
598 state = State.CharLiteralEnd;
599 }
600 },
601
602 State.CharLiteralBackslash => switch (c) {
603 '\n' => break, // Look for this error later.
604 else => {
605 state = State.CharLiteralEnd;
606 },
607 },
608
609 State.CharLiteralEnd => switch (c) {
610 '\'' => {
611 result.id = Token.Id.CharLiteral;
612 self.index += 1;
613 break;
614 },
615 else => break, // Look for this error later.
616 },
617
582 State.MultilineStringLiteralLine => switch (c) {618 State.MultilineStringLiteralLine => switch (c) {
583 '\\' => {619 '\\' => {
584 state = State.MultilineStringLiteralLineBackslash;620 state = State.MultilineStringLiteralLineBackslash;
...@@ -847,6 +883,7 @@ pub const Tokenizer = struct {...@@ -847,6 +883,7 @@ pub const Tokenizer = struct {
847 State.FloatExponentNumber,883 State.FloatExponentNumber,
848 State.StringLiteral, // find this error later884 State.StringLiteral, // find this error later
849 State.MultilineStringLiteralLine,885 State.MultilineStringLiteralLine,
886 State.CharLiteralEnd,
850 State.Builtin => {},887 State.Builtin => {},
851888
852 State.Identifier => {889 State.Identifier => {
...@@ -863,6 +900,8 @@ pub const Tokenizer = struct {...@@ -863,6 +900,8 @@ pub const Tokenizer = struct {
863 State.SawAtSign,900 State.SawAtSign,
864 State.Backslash,901 State.Backslash,
865 State.MultilineStringLiteralLineBackslash,902 State.MultilineStringLiteralLineBackslash,
903 State.CharLiteral,
904 State.CharLiteralBackslash,
866 State.StringLiteralBackslash => {905 State.StringLiteralBackslash => {
867 result.id = Token.Id.Invalid;906 result.id = Token.Id.Invalid;
868 },907 },
...@@ -1006,9 +1045,16 @@ test "tokenizer" {...@@ -1006,9 +1045,16 @@ test "tokenizer" {
1006 });1045 });
1007}1046}
10081047
1048test "tokenizer - chars" {
1049 testTokenize("'c'", []Token.Id {Token.Id.CharLiteral});
1050}
1051
1009test "tokenizer - invalid token characters" {1052test "tokenizer - invalid token characters" {
1010 testTokenize("#", []Token.Id{Token.Id.Invalid});1053 testTokenize("#", []Token.Id{Token.Id.Invalid});
1011 testTokenize("`", []Token.Id{Token.Id.Invalid});1054 testTokenize("`", []Token.Id{Token.Id.Invalid});
1055 testTokenize("'c", []Token.Id {Token.Id.Invalid});
1056 testTokenize("'", []Token.Id {Token.Id.Invalid});
1057 testTokenize("''", []Token.Id {Token.Id.Invalid});
1012}1058}
10131059
1014test "tokenizer - invalid literal/comment characters" {1060test "tokenizer - invalid literal/comment characters" {