authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2019-11-06 23:15:19+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-07 02:46:10-05:00
logcf6fb89ced7d0f03b511dba19f02af4a87a70b62
tree8095d46783869770edf8dc57d3c729381bacb99c
parent528908a06e9663a5e6595fc5ff80e017c663dc76
signaturelock-open Commit is signed but in an unrecognized format.

Add small tokenizer test case for #3468

1 files changed, 10 insertions(+), 0 deletions(-)

lib/std/zig/tokenizer.zig+10
......@@ -1625,6 +1625,16 @@ test "tokenizer - UTF-8 BOM is recognized and skipped" {
16251625 });
16261626}
16271627
1628test "correctly parse pointer assignment" {
1629 testTokenize("b.*=3;\n", [_]Token.Id{
1630 Token.Id.Identifier,
1631 Token.Id.PeriodAsterisk,
1632 Token.Id.Equal,
1633 Token.Id.IntegerLiteral,
1634 Token.Id.Semicolon,
1635 });
1636}
1637
16281638fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
16291639 var tokenizer = Tokenizer.init(source);
16301640 for (expected_tokens) |expected_token_id| {