authorgravatar for matthew.h.borkowski@gmail.comMatthew Borkowski <matthew.h.borkowski@gmail.com> 2021-05-25 23:44:55-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-26 04:13:04-04:00
logf750618846f598836e5170c31ac4988df2d6f00d
treed21b11e6fe1916a1c3fc157f310c4b9902797a46
parent44de8849803fd41b5e298926dcb835fcf3d31071

stop tokenizer from recognizing lone `@` or `@` followed by a digit as a builtin

1 files changed, 10 insertions(+), 3 deletions(-)

lib/std/zig/tokenizer.zig+10-3
......@@ -568,12 +568,14 @@ pub const Tokenizer = struct {
568568 result.tag = .identifier;
569569 state = .string_literal;
570570 },
571 else => {
572 // reinterpret as a builtin
573 self.index -= 1;
571 'a'...'z', 'A'...'Z', '_' => {
574572 state = .builtin;
575573 result.tag = .builtin;
576574 },
575 else => {
576 result.tag = .invalid;
577 break;
578 },
577579 },
578580
579581 .ampersand => switch (c) {
......@@ -2053,6 +2055,11 @@ test "tokenizer - multi line string literal with only 1 backslash" {
20532055 try testTokenize("x \\\n;", &.{ .identifier, .invalid, .semicolon });
20542056}
20552057
2058test "tokenizer - invalid builtin identifiers" {
2059 try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren });
2060 try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
2061}
2062
20562063fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) !void {
20572064 var tokenizer = Tokenizer.init(source);
20582065 for (expected_tokens) |expected_token_id| {