authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:56:59-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:56:59-04:00
log54987c3d8f358e4ded9113a9ee5fa04dc1372a56
tree455cfafce4790a7b209e9703337b5c95b1b5150f
parent0bf7ebcfea7934cb972aef84b25494c92f0dcf6f

std.zig.tokenizer: 3 slashes is doc comment, 4 is line comment

1 files changed, 18 insertions(+), 3 deletions(-)

std/zig/tokenizer.zig+18-3
...@@ -260,6 +260,7 @@ pub const Tokenizer = struct {...@@ -260,6 +260,7 @@ pub const Tokenizer = struct {
260 Slash,260 Slash,
261 LineCommentStart,261 LineCommentStart,
262 LineComment,262 LineComment,
263 DocCommentStart,
263 DocComment,264 DocComment,
264 Zero,265 Zero,
265 IntegerLiteral,266 IntegerLiteral,
...@@ -840,8 +841,7 @@ pub const Tokenizer = struct {...@@ -840,8 +841,7 @@ pub const Tokenizer = struct {
840 },841 },
841 State.LineCommentStart => switch (c) {842 State.LineCommentStart => switch (c) {
842 '/' => {843 '/' => {
843 result.id = Token.Id.DocComment;844 state = State.DocCommentStart;
844 state = State.DocComment;
845 },845 },
846 '\n' => break,846 '\n' => break,
847 else => {847 else => {
...@@ -849,6 +849,20 @@ pub const Tokenizer = struct {...@@ -849,6 +849,20 @@ pub const Tokenizer = struct {
849 self.checkLiteralCharacter();849 self.checkLiteralCharacter();
850 },850 },
851 },851 },
852 State.DocCommentStart => switch (c) {
853 '/' => {
854 state = State.LineComment;
855 },
856 '\n' => {
857 result.id = Token.Id.DocComment;
858 break;
859 },
860 else => {
861 state = State.DocComment;
862 result.id = Token.Id.DocComment;
863 self.checkLiteralCharacter();
864 },
865 },
852 State.LineComment, State.DocComment => switch (c) {866 State.LineComment, State.DocComment => switch (c) {
853 '\n' => break,867 '\n' => break,
854 else => self.checkLiteralCharacter(),868 else => self.checkLiteralCharacter(),
...@@ -938,7 +952,7 @@ pub const Tokenizer = struct {...@@ -938,7 +952,7 @@ pub const Tokenizer = struct {
938 State.LineComment => {952 State.LineComment => {
939 result.id = Token.Id.LineComment;953 result.id = Token.Id.LineComment;
940 },954 },
941 State.DocComment => {955 State.DocComment, State.DocCommentStart => {
942 result.id = Token.Id.DocComment;956 result.id = Token.Id.DocComment;
943 },957 },
944958
...@@ -1213,6 +1227,7 @@ test "tokenizer - line comment and doc comment" {...@@ -1213,6 +1227,7 @@ test "tokenizer - line comment and doc comment" {
1213 testTokenize("// /", []Token.Id{Token.Id.LineComment});1227 testTokenize("// /", []Token.Id{Token.Id.LineComment});
1214 testTokenize("/// a", []Token.Id{Token.Id.DocComment});1228 testTokenize("/// a", []Token.Id{Token.Id.DocComment});
1215 testTokenize("///", []Token.Id{Token.Id.DocComment});1229 testTokenize("///", []Token.Id{Token.Id.DocComment});
1230 testTokenize("////", []Token.Id{Token.Id.LineComment});
1216}1231}
12171232
1218test "tokenizer - line comment followed by identifier" {1233test "tokenizer - line comment followed by identifier" {