authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:56:59-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:56:59-04:00
log54987c3d8f358e4ded9113a9ee5fa04dc1372a56
tree455cfafce4790a7b209e9703337b5c95b1b5150f
parent0bf7ebcfea7934cb972aef84b25494c92f0dcf6f

std.zig.tokenizer: 3 slashes is doc comment, 4 is line comment

1 files changed, 18 insertions(+), 3 deletions(-)

std/zig/tokenizer.zig+18-3
......@@ -260,6 +260,7 @@ pub const Tokenizer = struct {
260260 Slash,
261261 LineCommentStart,
262262 LineComment,
263 DocCommentStart,
263264 DocComment,
264265 Zero,
265266 IntegerLiteral,
......@@ -840,8 +841,7 @@ pub const Tokenizer = struct {
840841 },
841842 State.LineCommentStart => switch (c) {
842843 '/' => {
843 result.id = Token.Id.DocComment;
844 state = State.DocComment;
844 state = State.DocCommentStart;
845845 },
846846 '\n' => break,
847847 else => {
......@@ -849,6 +849,20 @@ pub const Tokenizer = struct {
849849 self.checkLiteralCharacter();
850850 },
851851 },
852 State.DocCommentStart => switch (c) {
853 '/' => {
854 state = State.LineComment;
855 },
856 '\n' => {
857 result.id = Token.Id.DocComment;
858 break;
859 },
860 else => {
861 state = State.DocComment;
862 result.id = Token.Id.DocComment;
863 self.checkLiteralCharacter();
864 },
865 },
852866 State.LineComment, State.DocComment => switch (c) {
853867 '\n' => break,
854868 else => self.checkLiteralCharacter(),
......@@ -938,7 +952,7 @@ pub const Tokenizer = struct {
938952 State.LineComment => {
939953 result.id = Token.Id.LineComment;
940954 },
941 State.DocComment => {
955 State.DocComment, State.DocCommentStart => {
942956 result.id = Token.Id.DocComment;
943957 },
944958
......@@ -1213,6 +1227,7 @@ test "tokenizer - line comment and doc comment" {
12131227 testTokenize("// /", []Token.Id{Token.Id.LineComment});
12141228 testTokenize("/// a", []Token.Id{Token.Id.DocComment});
12151229 testTokenize("///", []Token.Id{Token.Id.DocComment});
1230 testTokenize("////", []Token.Id{Token.Id.LineComment});
12161231}
12171232
12181233test "tokenizer - line comment followed by identifier" {