authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-01-31 21:57:48-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-01-31 21:57:48-07:00
logbf8fafc37d4182196d108c773ae36c33a109d703
tree12b498ae45d9f102f67aa8e0ea36025345c82590
parent4dca99d3f6b732c415d270f0c97def144ed6d3b7

stage2: tokenizer does not emit line comments anymore

only std.zig.render cares about these, and it can find them in the original source easily enough.

2 files changed, 19 insertions(+), 39 deletions(-)

lib/std/zig/parse.zig-1
......@@ -28,7 +28,6 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!Tree {
2828 var tokenizer = std.zig.Tokenizer.init(source);
2929 while (true) {
3030 const token = tokenizer.next();
31 if (token.tag == .LineComment) continue;
3231 try tokens.append(gpa, .{
3332 .tag = token.tag,
3433 .start = @intCast(u32, token.loc.start),
lib/std/zig/tokenizer.zig+19-38
......@@ -43,7 +43,6 @@ pub const Token = struct {
4343 .{ "if", .Keyword_if },
4444 .{ "inline", .Keyword_inline },
4545 .{ "noalias", .Keyword_noalias },
46 .{ "noasync", .Keyword_nosuspend }, // TODO: remove this
4746 .{ "noinline", .Keyword_noinline },
4847 .{ "nosuspend", .Keyword_nosuspend },
4948 .{ "null", .Keyword_null },
......@@ -141,10 +140,8 @@ pub const Token = struct {
141140 Tilde,
142141 IntegerLiteral,
143142 FloatLiteral,
144 LineComment,
145143 DocComment,
146144 ContainerDocComment,
147 ShebangLine,
148145 Keyword_align,
149146 Keyword_allowzero,
150147 Keyword_and,
......@@ -211,10 +208,8 @@ pub const Token = struct {
211208 .Builtin => "Builtin",
212209 .IntegerLiteral => "IntegerLiteral",
213210 .FloatLiteral => "FloatLiteral",
214 .LineComment => "LineComment",
215211 .DocComment => "DocComment",
216212 .ContainerDocComment => "ContainerDocComment",
217 .ShebangLine => "ShebangLine",
218213
219214 .Bang => "!",
220215 .Pipe => "|",
......@@ -1016,7 +1011,6 @@ pub const Tokenizer = struct {
10161011 .slash => switch (c) {
10171012 '/' => {
10181013 state = .line_comment_start;
1019 result.tag = .LineComment;
10201014 },
10211015 '=' => {
10221016 result.tag = .SlashEqual;
......@@ -1036,7 +1030,7 @@ pub const Tokenizer = struct {
10361030 result.tag = .ContainerDocComment;
10371031 state = .container_doc_comment;
10381032 },
1039 '\n' => break,
1033 '\n' => state = .start,
10401034 '\t', '\r' => state = .line_comment,
10411035 else => {
10421036 state = .line_comment;
......@@ -1061,7 +1055,12 @@ pub const Tokenizer = struct {
10611055 self.checkLiteralCharacter();
10621056 },
10631057 },
1064 .line_comment, .doc_comment, .container_doc_comment => switch (c) {
1058 .line_comment => switch (c) {
1059 '\n' => state = .start,
1060 '\t', '\r' => {},
1061 else => self.checkLiteralCharacter(),
1062 },
1063 .doc_comment, .container_doc_comment => switch (c) {
10651064 '\n' => break,
10661065 '\t', '\r' => {},
10671066 else => self.checkLiteralCharacter(),
......@@ -1324,6 +1323,8 @@ pub const Tokenizer = struct {
13241323 .string_literal, // find this error later
13251324 .multiline_string_literal_line,
13261325 .builtin,
1326 .line_comment,
1327 .line_comment_start,
13271328 => {},
13281329
13291330 .identifier => {
......@@ -1331,9 +1332,6 @@ pub const Tokenizer = struct {
13311332 result.tag = tag;
13321333 }
13331334 },
1334 .line_comment, .line_comment_start => {
1335 result.tag = .LineComment;
1336 },
13371335 .doc_comment, .doc_comment_start => {
13381336 result.tag = .DocComment;
13391337 },
......@@ -1614,77 +1612,63 @@ test "tokenizer - invalid literal/comment characters" {
16141612 .Invalid,
16151613 });
16161614 testTokenize("//\x00", &[_]Token.Tag{
1617 .LineComment,
16181615 .Invalid,
16191616 });
16201617 testTokenize("//\x1f", &[_]Token.Tag{
1621 .LineComment,
16221618 .Invalid,
16231619 });
16241620 testTokenize("//\x7f", &[_]Token.Tag{
1625 .LineComment,
16261621 .Invalid,
16271622 });
16281623}
16291624
16301625test "tokenizer - utf8" {
1631 testTokenize("//\xc2\x80", &[_]Token.Tag{.LineComment});
1632 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Tag{.LineComment});
1626 testTokenize("//\xc2\x80", &[_]Token.Tag{});
1627 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Tag{});
16331628}
16341629
16351630test "tokenizer - invalid utf8" {
16361631 testTokenize("//\x80", &[_]Token.Tag{
1637 .LineComment,
16381632 .Invalid,
16391633 });
16401634 testTokenize("//\xbf", &[_]Token.Tag{
1641 .LineComment,
16421635 .Invalid,
16431636 });
16441637 testTokenize("//\xf8", &[_]Token.Tag{
1645 .LineComment,
16461638 .Invalid,
16471639 });
16481640 testTokenize("//\xff", &[_]Token.Tag{
1649 .LineComment,
16501641 .Invalid,
16511642 });
16521643 testTokenize("//\xc2\xc0", &[_]Token.Tag{
1653 .LineComment,
16541644 .Invalid,
16551645 });
16561646 testTokenize("//\xe0", &[_]Token.Tag{
1657 .LineComment,
16581647 .Invalid,
16591648 });
16601649 testTokenize("//\xf0", &[_]Token.Tag{
1661 .LineComment,
16621650 .Invalid,
16631651 });
16641652 testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Tag{
1665 .LineComment,
16661653 .Invalid,
16671654 });
16681655}
16691656
16701657test "tokenizer - illegal unicode codepoints" {
16711658 // unicode newline characters.U+0085, U+2028, U+2029
1672 testTokenize("//\xc2\x84", &[_]Token.Tag{.LineComment});
1659 testTokenize("//\xc2\x84", &[_]Token.Tag{});
16731660 testTokenize("//\xc2\x85", &[_]Token.Tag{
1674 .LineComment,
16751661 .Invalid,
16761662 });
1677 testTokenize("//\xc2\x86", &[_]Token.Tag{.LineComment});
1678 testTokenize("//\xe2\x80\xa7", &[_]Token.Tag{.LineComment});
1663 testTokenize("//\xc2\x86", &[_]Token.Tag{});
1664 testTokenize("//\xe2\x80\xa7", &[_]Token.Tag{});
16791665 testTokenize("//\xe2\x80\xa8", &[_]Token.Tag{
1680 .LineComment,
16811666 .Invalid,
16821667 });
16831668 testTokenize("//\xe2\x80\xa9", &[_]Token.Tag{
1684 .LineComment,
16851669 .Invalid,
16861670 });
1687 testTokenize("//\xe2\x80\xaa", &[_]Token.Tag{.LineComment});
1671 testTokenize("//\xe2\x80\xaa", &[_]Token.Tag{});
16881672}
16891673
16901674test "tokenizer - string identifier and builtin fns" {
......@@ -1719,10 +1703,8 @@ test "tokenizer - comments with literal tab" {
17191703 \\/// foo
17201704 \\/// /foo
17211705 , &[_]Token.Tag{
1722 .LineComment,
17231706 .ContainerDocComment,
17241707 .DocComment,
1725 .LineComment,
17261708 .DocComment,
17271709 .DocComment,
17281710 });
......@@ -1736,12 +1718,12 @@ test "tokenizer - pipe and then invalid" {
17361718}
17371719
17381720test "tokenizer - line comment and doc comment" {
1739 testTokenize("//", &[_]Token.Tag{.LineComment});
1740 testTokenize("// a / b", &[_]Token.Tag{.LineComment});
1741 testTokenize("// /", &[_]Token.Tag{.LineComment});
1721 testTokenize("//", &[_]Token.Tag{});
1722 testTokenize("// a / b", &[_]Token.Tag{});
1723 testTokenize("// /", &[_]Token.Tag{});
17421724 testTokenize("/// a", &[_]Token.Tag{.DocComment});
17431725 testTokenize("///", &[_]Token.Tag{.DocComment});
1744 testTokenize("////", &[_]Token.Tag{.LineComment});
1726 testTokenize("////", &[_]Token.Tag{});
17451727 testTokenize("//!", &[_]Token.Tag{.ContainerDocComment});
17461728 testTokenize("//!!", &[_]Token.Tag{.ContainerDocComment});
17471729}
......@@ -1754,7 +1736,6 @@ test "tokenizer - line comment followed by identifier" {
17541736 , &[_]Token.Tag{
17551737 .Identifier,
17561738 .Comma,
1757 .LineComment,
17581739 .Identifier,
17591740 .Comma,
17601741 });