authorgravatar for matthew.h.borkowski@gmail.comMatthew Borkowski <matthew.h.borkowski@gmail.com> 2021-05-25 23:44:55-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-26 04:13:04-04:00
logf750618846f598836e5170c31ac4988df2d6f00d
treed21b11e6fe1916a1c3fc157f310c4b9902797a46
parent44de8849803fd41b5e298926dcb835fcf3d31071

stop tokenizer from recognizing lone `@` or `@` followed by a digit as a builtin

1 files changed, 10 insertions(+), 3 deletions(-)

lib/std/zig/tokenizer.zig+10-3
...@@ -568,12 +568,14 @@ pub const Tokenizer = struct {...@@ -568,12 +568,14 @@ pub const Tokenizer = struct {
568 result.tag = .identifier;568 result.tag = .identifier;
569 state = .string_literal;569 state = .string_literal;
570 },570 },
571 else => {571 'a'...'z', 'A'...'Z', '_' => {
572 // reinterpret as a builtin
573 self.index -= 1;
574 state = .builtin;572 state = .builtin;
575 result.tag = .builtin;573 result.tag = .builtin;
576 },574 },
575 else => {
576 result.tag = .invalid;
577 break;
578 },
577 },579 },
578580
579 .ampersand => switch (c) {581 .ampersand => switch (c) {
...@@ -2053,6 +2055,11 @@ test "tokenizer - multi line string literal with only 1 backslash" {...@@ -2053,6 +2055,11 @@ test "tokenizer - multi line string literal with only 1 backslash" {
2053 try testTokenize("x \\\n;", &.{ .identifier, .invalid, .semicolon });2055 try testTokenize("x \\\n;", &.{ .identifier, .invalid, .semicolon });
2054}2056}
20552057
2058test "tokenizer - invalid builtin identifiers" {
2059 try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren });
2060 try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
2061}
2062
2056fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) !void {2063fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) !void {
2057 var tokenizer = Tokenizer.init(source);2064 var tokenizer = Tokenizer.init(source);
2058 for (expected_tokens) |expected_token_id| {2065 for (expected_tokens) |expected_token_id| {