authorgravatar for tssund93@gmail.comTravis <tssund93@gmail.com> 2020-10-28 21:45:58-05:00
committergravatar for tssund93@gmail.comTravis <tssund93@gmail.com> 2020-10-29 12:03:45-05:00
log960b5b518fa7ff7c6760d3e5f3c1256e553e6ba8
tree6f749eb05bb723d43de068c7142ca21d63f08355
parent3c7a49c4947b748c04e314dc89f2ce56562da29c

updated zig tokenizer to handle .*** and added tests

2 files changed, 49 insertions(+), 3 deletions(-)

lib/std/zig/tokenizer.zig+41-3
...@@ -403,6 +403,7 @@ pub const Tokenizer = struct {...@@ -403,6 +403,7 @@ pub const Tokenizer = struct {
403 angle_bracket_angle_bracket_right,403 angle_bracket_angle_bracket_right,
404 period,404 period,
405 period_2,405 period_2,
406 period_asterisk,
406 saw_at_sign,407 saw_at_sign,
407 };408 };
408409
...@@ -979,9 +980,7 @@ pub const Tokenizer = struct {...@@ -979,9 +980,7 @@ pub const Tokenizer = struct {
979 state = .period_2;980 state = .period_2;
980 },981 },
981 '*' => {982 '*' => {
982 result.id = .PeriodAsterisk;983 state = .period_asterisk;
983 self.index += 1;
984 break;
985 },984 },
986 else => {985 else => {
987 result.id = .Period;986 result.id = .Period;
...@@ -1001,6 +1000,17 @@ pub const Tokenizer = struct {...@@ -1001,6 +1000,17 @@ pub const Tokenizer = struct {
1001 },1000 },
1002 },1001 },
10031002
1003 .period_asterisk => switch (c) {
1004 '*' => {
1005 result.id = .Invalid;
1006 break;
1007 },
1008 else => {
1009 result.id = .PeriodAsterisk;
1010 break;
1011 }
1012 },
1013
1004 .slash => switch (c) {1014 .slash => switch (c) {
1005 '/' => {1015 '/' => {
1006 state = .line_comment_start;1016 state = .line_comment_start;
...@@ -1376,6 +1386,9 @@ pub const Tokenizer = struct {...@@ -1376,6 +1386,9 @@ pub const Tokenizer = struct {
1376 .period_2 => {1386 .period_2 => {
1377 result.id = .Ellipsis2;1387 result.id = .Ellipsis2;
1378 },1388 },
1389 .period_asterisk => {
1390 result.id = .PeriodAsterisk;
1391 },
1379 .pipe => {1392 .pipe => {
1380 result.id = .Pipe;1393 result.id = .Pipe;
1381 },1394 },
...@@ -1762,6 +1775,31 @@ test "correctly parse pointer assignment" {...@@ -1762,6 +1775,31 @@ test "correctly parse pointer assignment" {
1762 });1775 });
1763}1776}
17641777
1778test "correctly parse pointer dereference followed by asterisk" {
1779 testTokenize("\"b\".* ** 10", &[_]Token.Id{
1780 .StringLiteral,
1781 .PeriodAsterisk,
1782 .AsteriskAsterisk,
1783 .IntegerLiteral,
1784 });
1785
1786 testTokenize("(\"b\".*)** 10", &[_]Token.Id{
1787 .LParen,
1788 .StringLiteral,
1789 .PeriodAsterisk,
1790 .RParen,
1791 .AsteriskAsterisk,
1792 .IntegerLiteral,
1793 });
1794
1795 testTokenize("\"b\".*** 10", &[_]Token.Id{
1796 .StringLiteral,
1797 .Invalid,
1798 .AsteriskAsterisk,
1799 .IntegerLiteral,
1800 });
1801}
1802
1765test "tokenizer - range literals" {1803test "tokenizer - range literals" {
1766 testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });1804 testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
1767 testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });1805 testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });
test/compile_errors.zig+8
...@@ -8195,4 +8195,12 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {...@@ -8195,4 +8195,12 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
8195 , &[_][]const u8{8195 , &[_][]const u8{
8196 "tmp.zig:4:9: error: expected type '*c_void', found '?*c_void'",8196 "tmp.zig:4:9: error: expected type '*c_void', found '?*c_void'",
8197 });8197 });
8198
8199 cases.add("Issue #6823: don't allow .* to be followed by **",
8200 \\fn foo() void {
8201 \\ var sequence = "repeat".*** 10;
8202 \\}
8203 , &[_][]const u8{
8204 "tmp.zig:2:30: error: `.*` can't be followed by `*`. Are you missing a space?",
8205 });
8198}8206}