diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 22d63568bf7777db6a001403441241cf0752748a..b4e9d910d92b2dd19e0bbc54a4ce9225d647817f 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -1494,9 +1494,17 @@ void tokenize(Buf *buf, Tokenization *out) { tokenize_error(&t, "unexpected EOF"); break; case TokenizeStateLineComment: + break; case TokenizeStateSawSlash2: + cancel_token(&t); + break; case TokenizeStateSawSlash3: + set_token_id(&t, t.cur_tok, TokenIdDocComment); + end_token(&t); + break; case TokenizeStateSawSlashBang: + set_token_id(&t, t.cur_tok, TokenIdContainerDocComment); + end_token(&t); break; } if (t.state != TokenizeStateError) {