authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-01-31 21:57:48-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-01-31 21:57:48-07:00
logbf8fafc37d4182196d108c773ae36c33a109d703
tree12b498ae45d9f102f67aa8e0ea36025345c82590
parent4dca99d3f6b732c415d270f0c97def144ed6d3b7

stage2: tokenizer does not emit line comments anymore

only std.zig.render cares about these, and it can find them in the original source easily enough.

2 files changed, 19 insertions(+), 39 deletions(-)

lib/std/zig/parse.zig-1
...@@ -28,7 +28,6 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!Tree {...@@ -28,7 +28,6 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!Tree {
28 var tokenizer = std.zig.Tokenizer.init(source);28 var tokenizer = std.zig.Tokenizer.init(source);
29 while (true) {29 while (true) {
30 const token = tokenizer.next();30 const token = tokenizer.next();
31 if (token.tag == .LineComment) continue;
32 try tokens.append(gpa, .{31 try tokens.append(gpa, .{
33 .tag = token.tag,32 .tag = token.tag,
34 .start = @intCast(u32, token.loc.start),33 .start = @intCast(u32, token.loc.start),
lib/std/zig/tokenizer.zig+19-38
...@@ -43,7 +43,6 @@ pub const Token = struct {...@@ -43,7 +43,6 @@ pub const Token = struct {
43 .{ "if", .Keyword_if },43 .{ "if", .Keyword_if },
44 .{ "inline", .Keyword_inline },44 .{ "inline", .Keyword_inline },
45 .{ "noalias", .Keyword_noalias },45 .{ "noalias", .Keyword_noalias },
46 .{ "noasync", .Keyword_nosuspend }, // TODO: remove this
47 .{ "noinline", .Keyword_noinline },46 .{ "noinline", .Keyword_noinline },
48 .{ "nosuspend", .Keyword_nosuspend },47 .{ "nosuspend", .Keyword_nosuspend },
49 .{ "null", .Keyword_null },48 .{ "null", .Keyword_null },
...@@ -141,10 +140,8 @@ pub const Token = struct {...@@ -141,10 +140,8 @@ pub const Token = struct {
141 Tilde,140 Tilde,
142 IntegerLiteral,141 IntegerLiteral,
143 FloatLiteral,142 FloatLiteral,
144 LineComment,
145 DocComment,143 DocComment,
146 ContainerDocComment,144 ContainerDocComment,
147 ShebangLine,
148 Keyword_align,145 Keyword_align,
149 Keyword_allowzero,146 Keyword_allowzero,
150 Keyword_and,147 Keyword_and,
...@@ -211,10 +208,8 @@ pub const Token = struct {...@@ -211,10 +208,8 @@ pub const Token = struct {
211 .Builtin => "Builtin",208 .Builtin => "Builtin",
212 .IntegerLiteral => "IntegerLiteral",209 .IntegerLiteral => "IntegerLiteral",
213 .FloatLiteral => "FloatLiteral",210 .FloatLiteral => "FloatLiteral",
214 .LineComment => "LineComment",
215 .DocComment => "DocComment",211 .DocComment => "DocComment",
216 .ContainerDocComment => "ContainerDocComment",212 .ContainerDocComment => "ContainerDocComment",
217 .ShebangLine => "ShebangLine",
218213
219 .Bang => "!",214 .Bang => "!",
220 .Pipe => "|",215 .Pipe => "|",
...@@ -1016,7 +1011,6 @@ pub const Tokenizer = struct {...@@ -1016,7 +1011,6 @@ pub const Tokenizer = struct {
1016 .slash => switch (c) {1011 .slash => switch (c) {
1017 '/' => {1012 '/' => {
1018 state = .line_comment_start;1013 state = .line_comment_start;
1019 result.tag = .LineComment;
1020 },1014 },
1021 '=' => {1015 '=' => {
1022 result.tag = .SlashEqual;1016 result.tag = .SlashEqual;
...@@ -1036,7 +1030,7 @@ pub const Tokenizer = struct {...@@ -1036,7 +1030,7 @@ pub const Tokenizer = struct {
1036 result.tag = .ContainerDocComment;1030 result.tag = .ContainerDocComment;
1037 state = .container_doc_comment;1031 state = .container_doc_comment;
1038 },1032 },
1039 '\n' => break,1033 '\n' => state = .start,
1040 '\t', '\r' => state = .line_comment,1034 '\t', '\r' => state = .line_comment,
1041 else => {1035 else => {
1042 state = .line_comment;1036 state = .line_comment;
...@@ -1061,7 +1055,12 @@ pub const Tokenizer = struct {...@@ -1061,7 +1055,12 @@ pub const Tokenizer = struct {
1061 self.checkLiteralCharacter();1055 self.checkLiteralCharacter();
1062 },1056 },
1063 },1057 },
1064 .line_comment, .doc_comment, .container_doc_comment => switch (c) {1058 .line_comment => switch (c) {
1059 '\n' => state = .start,
1060 '\t', '\r' => {},
1061 else => self.checkLiteralCharacter(),
1062 },
1063 .doc_comment, .container_doc_comment => switch (c) {
1065 '\n' => break,1064 '\n' => break,
1066 '\t', '\r' => {},1065 '\t', '\r' => {},
1067 else => self.checkLiteralCharacter(),1066 else => self.checkLiteralCharacter(),
...@@ -1324,6 +1323,8 @@ pub const Tokenizer = struct {...@@ -1324,6 +1323,8 @@ pub const Tokenizer = struct {
1324 .string_literal, // find this error later1323 .string_literal, // find this error later
1325 .multiline_string_literal_line,1324 .multiline_string_literal_line,
1326 .builtin,1325 .builtin,
1326 .line_comment,
1327 .line_comment_start,
1327 => {},1328 => {},
13281329
1329 .identifier => {1330 .identifier => {
...@@ -1331,9 +1332,6 @@ pub const Tokenizer = struct {...@@ -1331,9 +1332,6 @@ pub const Tokenizer = struct {
1331 result.tag = tag;1332 result.tag = tag;
1332 }1333 }
1333 },1334 },
1334 .line_comment, .line_comment_start => {
1335 result.tag = .LineComment;
1336 },
1337 .doc_comment, .doc_comment_start => {1335 .doc_comment, .doc_comment_start => {
1338 result.tag = .DocComment;1336 result.tag = .DocComment;
1339 },1337 },
...@@ -1614,77 +1612,63 @@ test "tokenizer - invalid literal/comment characters" {...@@ -1614,77 +1612,63 @@ test "tokenizer - invalid literal/comment characters" {
1614 .Invalid,1612 .Invalid,
1615 });1613 });
1616 testTokenize("//\x00", &[_]Token.Tag{1614 testTokenize("//\x00", &[_]Token.Tag{
1617 .LineComment,
1618 .Invalid,1615 .Invalid,
1619 });1616 });
1620 testTokenize("//\x1f", &[_]Token.Tag{1617 testTokenize("//\x1f", &[_]Token.Tag{
1621 .LineComment,
1622 .Invalid,1618 .Invalid,
1623 });1619 });
1624 testTokenize("//\x7f", &[_]Token.Tag{1620 testTokenize("//\x7f", &[_]Token.Tag{
1625 .LineComment,
1626 .Invalid,1621 .Invalid,
1627 });1622 });
1628}1623}
16291624
1630test "tokenizer - utf8" {1625test "tokenizer - utf8" {
1631 testTokenize("//\xc2\x80", &[_]Token.Tag{.LineComment});1626 testTokenize("//\xc2\x80", &[_]Token.Tag{});
1632 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Tag{.LineComment});1627 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Tag{});
1633}1628}
16341629
1635test "tokenizer - invalid utf8" {1630test "tokenizer - invalid utf8" {
1636 testTokenize("//\x80", &[_]Token.Tag{1631 testTokenize("//\x80", &[_]Token.Tag{
1637 .LineComment,
1638 .Invalid,1632 .Invalid,
1639 });1633 });
1640 testTokenize("//\xbf", &[_]Token.Tag{1634 testTokenize("//\xbf", &[_]Token.Tag{
1641 .LineComment,
1642 .Invalid,1635 .Invalid,
1643 });1636 });
1644 testTokenize("//\xf8", &[_]Token.Tag{1637 testTokenize("//\xf8", &[_]Token.Tag{
1645 .LineComment,
1646 .Invalid,1638 .Invalid,
1647 });1639 });
1648 testTokenize("//\xff", &[_]Token.Tag{1640 testTokenize("//\xff", &[_]Token.Tag{
1649 .LineComment,
1650 .Invalid,1641 .Invalid,
1651 });1642 });
1652 testTokenize("//\xc2\xc0", &[_]Token.Tag{1643 testTokenize("//\xc2\xc0", &[_]Token.Tag{
1653 .LineComment,
1654 .Invalid,1644 .Invalid,
1655 });1645 });
1656 testTokenize("//\xe0", &[_]Token.Tag{1646 testTokenize("//\xe0", &[_]Token.Tag{
1657 .LineComment,
1658 .Invalid,1647 .Invalid,
1659 });1648 });
1660 testTokenize("//\xf0", &[_]Token.Tag{1649 testTokenize("//\xf0", &[_]Token.Tag{
1661 .LineComment,
1662 .Invalid,1650 .Invalid,
1663 });1651 });
1664 testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Tag{1652 testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Tag{
1665 .LineComment,
1666 .Invalid,1653 .Invalid,
1667 });1654 });
1668}1655}
16691656
1670test "tokenizer - illegal unicode codepoints" {1657test "tokenizer - illegal unicode codepoints" {
1671 // unicode newline characters.U+0085, U+2028, U+20291658 // unicode newline characters.U+0085, U+2028, U+2029
1672 testTokenize("//\xc2\x84", &[_]Token.Tag{.LineComment});1659 testTokenize("//\xc2\x84", &[_]Token.Tag{});
1673 testTokenize("//\xc2\x85", &[_]Token.Tag{1660 testTokenize("//\xc2\x85", &[_]Token.Tag{
1674 .LineComment,
1675 .Invalid,1661 .Invalid,
1676 });1662 });
1677 testTokenize("//\xc2\x86", &[_]Token.Tag{.LineComment});1663 testTokenize("//\xc2\x86", &[_]Token.Tag{});
1678 testTokenize("//\xe2\x80\xa7", &[_]Token.Tag{.LineComment});1664 testTokenize("//\xe2\x80\xa7", &[_]Token.Tag{});
1679 testTokenize("//\xe2\x80\xa8", &[_]Token.Tag{1665 testTokenize("//\xe2\x80\xa8", &[_]Token.Tag{
1680 .LineComment,
1681 .Invalid,1666 .Invalid,
1682 });1667 });
1683 testTokenize("//\xe2\x80\xa9", &[_]Token.Tag{1668 testTokenize("//\xe2\x80\xa9", &[_]Token.Tag{
1684 .LineComment,
1685 .Invalid,1669 .Invalid,
1686 });1670 });
1687 testTokenize("//\xe2\x80\xaa", &[_]Token.Tag{.LineComment});1671 testTokenize("//\xe2\x80\xaa", &[_]Token.Tag{});
1688}1672}
16891673
1690test "tokenizer - string identifier and builtin fns" {1674test "tokenizer - string identifier and builtin fns" {
...@@ -1719,10 +1703,8 @@ test "tokenizer - comments with literal tab" {...@@ -1719,10 +1703,8 @@ test "tokenizer - comments with literal tab" {
1719 \\/// foo1703 \\/// foo
1720 \\/// /foo1704 \\/// /foo
1721 , &[_]Token.Tag{1705 , &[_]Token.Tag{
1722 .LineComment,
1723 .ContainerDocComment,1706 .ContainerDocComment,
1724 .DocComment,1707 .DocComment,
1725 .LineComment,
1726 .DocComment,1708 .DocComment,
1727 .DocComment,1709 .DocComment,
1728 });1710 });
...@@ -1736,12 +1718,12 @@ test "tokenizer - pipe and then invalid" {...@@ -1736,12 +1718,12 @@ test "tokenizer - pipe and then invalid" {
1736}1718}
17371719
1738test "tokenizer - line comment and doc comment" {1720test "tokenizer - line comment and doc comment" {
1739 testTokenize("//", &[_]Token.Tag{.LineComment});1721 testTokenize("//", &[_]Token.Tag{});
1740 testTokenize("// a / b", &[_]Token.Tag{.LineComment});1722 testTokenize("// a / b", &[_]Token.Tag{});
1741 testTokenize("// /", &[_]Token.Tag{.LineComment});1723 testTokenize("// /", &[_]Token.Tag{});
1742 testTokenize("/// a", &[_]Token.Tag{.DocComment});1724 testTokenize("/// a", &[_]Token.Tag{.DocComment});
1743 testTokenize("///", &[_]Token.Tag{.DocComment});1725 testTokenize("///", &[_]Token.Tag{.DocComment});
1744 testTokenize("////", &[_]Token.Tag{.LineComment});1726 testTokenize("////", &[_]Token.Tag{});
1745 testTokenize("//!", &[_]Token.Tag{.ContainerDocComment});1727 testTokenize("//!", &[_]Token.Tag{.ContainerDocComment});
1746 testTokenize("//!!", &[_]Token.Tag{.ContainerDocComment});1728 testTokenize("//!!", &[_]Token.Tag{.ContainerDocComment});
1747}1729}
...@@ -1754,7 +1736,6 @@ test "tokenizer - line comment followed by identifier" {...@@ -1754,7 +1736,6 @@ test "tokenizer - line comment followed by identifier" {
1754 , &[_]Token.Tag{1736 , &[_]Token.Tag{
1755 .Identifier,1737 .Identifier,
1756 .Comma,1738 .Comma,
1757 .LineComment,
1758 .Identifier,1739 .Identifier,
1759 .Comma,1740 .Comma,
1760 });1741 });