authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-31 21:28:40+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-31 21:28:40+02:00
log975dc5a390490794df367b6a9869609a8b14d8f8
tree1efe2878e9b1bd001b231c9aa357d1929051ebfa
parent4d8f9e2295bb672f70550b3bb5a4cb667f68bb70

std.zig.parser now parses char literals

3 files changed, 83 insertions(+), 0 deletions(-)

std/zig/ast.zig+21
......@@ -21,6 +21,7 @@ pub const Node = struct {
2121 FloatLiteral,
2222 StringLiteral,
2323 MultilineStringLiteral,
24 CharLiteral,
2425 UndefinedLiteral,
2526 BuiltinCall,
2627 Call,
......@@ -42,6 +43,7 @@ pub const Node = struct {
4243 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
4344 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
4445 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index),
46 Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).iterate(index),
4547 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index),
4648 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
4749 Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index),
......@@ -64,6 +66,7 @@ pub const Node = struct {
6466 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
6567 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
6668 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(),
69 Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).firstToken(),
6770 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(),
6871 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
6972 Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(),
......@@ -86,6 +89,7 @@ pub const Node = struct {
8689 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
8790 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
8891 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(),
92 Id.CharLiteral => @fieldParentPtr(NodeCharLiteral, "base", base).lastToken(),
8993 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(),
9094 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
9195 Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(),
......@@ -608,6 +612,23 @@ pub const NodeMultilineStringLiteral = struct {
608612 }
609613};
610614
615pub const NodeCharLiteral = struct {
616 base: Node,
617 token: Token,
618
619 pub fn iterate(self: &NodeCharLiteral, index: usize) ?&Node {
620 return null;
621 }
622
623 pub fn firstToken(self: &NodeCharLiteral) Token {
624 return self.token;
625 }
626
627 pub fn lastToken(self: &NodeCharLiteral) Token {
628 return self.token;
629 }
630};
631
611632pub const NodeUndefinedLiteral = struct {
612633 base: Node,
613634 token: Token,
std/zig/parser.zig+16
......@@ -456,6 +456,18 @@ pub const Parser = struct {
456456 try stack.append(State.AfterOperand);
457457 continue;
458458 },
459 Token.Id.CharLiteral => {
460 const node = try arena.create(ast.NodeCharLiteral);
461 *node = ast.NodeCharLiteral {
462 .base = self.initNode(ast.Node.Id.CharLiteral),
463 .token = token,
464 };
465 try stack.append(State {
466 .Operand = &node.base
467 });
468 try stack.append(State.AfterOperand);
469 continue;
470 },
459471 Token.Id.MultilineStringLiteralLine => {
460472 const node = try arena.create(ast.NodeMultilineStringLiteral);
461473 *node = ast.NodeMultilineStringLiteral {
......@@ -1451,6 +1463,10 @@ pub const Parser = struct {
14511463 const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);
14521464 try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));
14531465 },
1466 ast.Node.Id.CharLiteral => {
1467 const char_literal = @fieldParentPtr(ast.NodeCharLiteral, "base", base);
1468 try stream.print("{}", self.tokenizer.getTokenSlice(char_literal.token));
1469 },
14541470 ast.Node.Id.MultilineStringLiteral => {
14551471 const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base);
14561472 try stream.print("\n");
std/zig/tokenizer.zig+46
......@@ -73,6 +73,7 @@ pub const Token = struct {
7373 Identifier,
7474 StringLiteral: StrLitKind,
7575 MultilineStringLiteralLine: StrLitKind,
76 CharLiteral,
7677 StringIdentifier,
7778 Eof,
7879 Builtin,
......@@ -228,6 +229,9 @@ pub const Tokenizer = struct {
228229 StringLiteralBackslash,
229230 MultilineStringLiteralLine,
230231 MultilineStringLiteralLineBackslash,
232 CharLiteral,
233 CharLiteralBackslash,
234 CharLiteralEnd,
231235 Backslash,
232236 Equal,
233237 Bang,
......@@ -294,6 +298,9 @@ pub const Tokenizer = struct {
294298 state = State.StringLiteral;
295299 result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
296300 },
301 '\'' => {
302 state = State.CharLiteral;
303 },
297304 'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
298305 state = State.Identifier;
299306 result.id = Token.Id.Identifier;
......@@ -579,6 +586,35 @@ pub const Tokenizer = struct {
579586 },
580587 },
581588
589 State.CharLiteral => switch (c) {
590 '\\' => {
591 state = State.CharLiteralBackslash;
592 },
593 '\'' => break, // Look for this error later.
594 else => {
595 if (c < 0x20 or c == 0x7f)
596 break; // Look for this error later.
597
598 state = State.CharLiteralEnd;
599 }
600 },
601
602 State.CharLiteralBackslash => switch (c) {
603 '\n' => break, // Look for this error later.
604 else => {
605 state = State.CharLiteralEnd;
606 },
607 },
608
609 State.CharLiteralEnd => switch (c) {
610 '\'' => {
611 result.id = Token.Id.CharLiteral;
612 self.index += 1;
613 break;
614 },
615 else => break, // Look for this error later.
616 },
617
582618 State.MultilineStringLiteralLine => switch (c) {
583619 '\\' => {
584620 state = State.MultilineStringLiteralLineBackslash;
......@@ -847,6 +883,7 @@ pub const Tokenizer = struct {
847883 State.FloatExponentNumber,
848884 State.StringLiteral, // find this error later
849885 State.MultilineStringLiteralLine,
886 State.CharLiteralEnd,
850887 State.Builtin => {},
851888
852889 State.Identifier => {
......@@ -863,6 +900,8 @@ pub const Tokenizer = struct {
863900 State.SawAtSign,
864901 State.Backslash,
865902 State.MultilineStringLiteralLineBackslash,
903 State.CharLiteral,
904 State.CharLiteralBackslash,
866905 State.StringLiteralBackslash => {
867906 result.id = Token.Id.Invalid;
868907 },
......@@ -1006,9 +1045,16 @@ test "tokenizer" {
10061045 });
10071046}
10081047
1048test "tokenizer - chars" {
1049 testTokenize("'c'", []Token.Id {Token.Id.CharLiteral});
1050}
1051
10091052test "tokenizer - invalid token characters" {
10101053 testTokenize("#", []Token.Id{Token.Id.Invalid});
10111054 testTokenize("`", []Token.Id{Token.Id.Invalid});
1055 testTokenize("'c", []Token.Id {Token.Id.Invalid});
1056 testTokenize("'", []Token.Id {Token.Id.Invalid});
1057 testTokenize("''", []Token.Id {Token.Id.Invalid});
10121058}
10131059
10141060test "tokenizer - invalid literal/comment characters" {