authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-10 11:35:41+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-10 11:35:41+02:00
log34af38e09b33957a9f677e42d57e9cd96f859b76
tree351345ae647f4c138451df5897dac65c9d0b1ba8
parentf85b9f2bf3ac0c5af7804921d2003c2992558cb7

std.zig.tokinizer: fixed failing tests

1 files changed, 19 insertions(+), 7 deletions(-)

std/zig/tokenizer.zig+19-7
......@@ -613,17 +613,25 @@ pub const Tokenizer = struct {
613613 '\\' => {
614614 state = State.CharLiteralBackslash;
615615 },
616 '\'' => break, // Look for this error later.
616 '\'' => {
617 result.id = Token.Id.Invalid;
618 break;
619 },
617620 else => {
618 if (c < 0x20 or c == 0x7f)
619 break; // Look for this error later.
621 if (c < 0x20 or c == 0x7f) {
622 result.id = Token.Id.Invalid;
623 break;
624 }
620625
621626 state = State.CharLiteralEnd;
622627 }
623628 },
624629
625630 State.CharLiteralBackslash => switch (c) {
626 '\n' => break, // Look for this error later.
631 '\n' => {
632 result.id = Token.Id.Invalid;
633 break;
634 },
627635 else => {
628636 state = State.CharLiteralEnd;
629637 },
......@@ -635,7 +643,10 @@ pub const Tokenizer = struct {
635643 self.index += 1;
636644 break;
637645 },
638 else => break, // Look for this error later.
646 else => {
647 result.id = Token.Id.Invalid;
648 break;
649 },
639650 },
640651
641652 State.MultilineStringLiteralLine => switch (c) {
......@@ -903,7 +914,6 @@ pub const Tokenizer = struct {
903914 State.FloatExponentNumber,
904915 State.StringLiteral, // find this error later
905916 State.MultilineStringLiteralLine,
906 State.CharLiteralEnd,
907917 State.Builtin => {},
908918
909919 State.Identifier => {
......@@ -922,6 +932,7 @@ pub const Tokenizer = struct {
922932 State.MultilineStringLiteralLineBackslash,
923933 State.CharLiteral,
924934 State.CharLiteralBackslash,
935 State.CharLiteralEnd,
925936 State.StringLiteralBackslash => {
926937 result.id = Token.Id.Invalid;
927938 },
......@@ -1073,7 +1084,7 @@ test "tokenizer - invalid token characters" {
10731084 testTokenize("`", []Token.Id{Token.Id.Invalid});
10741085 testTokenize("'c", []Token.Id {Token.Id.Invalid});
10751086 testTokenize("'", []Token.Id {Token.Id.Invalid});
1076 testTokenize("''", []Token.Id {Token.Id.Invalid});
1087 testTokenize("''", []Token.Id {Token.Id.Invalid, Token.Id.Invalid});
10771088}
10781089
10791090test "tokenizer - invalid literal/comment characters" {
......@@ -1147,6 +1158,7 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
11471158 var tokenizer = Tokenizer.init(source);
11481159 for (expected_tokens) |expected_token_id| {
11491160 const token = tokenizer.next();
1161 std.debug.warn("{} {}\n", @tagName(expected_token_id), @tagName(token.id));
11501162 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
11511163 switch (expected_token_id) {
11521164 Token.Id.StringLiteral => |expected_kind| {