authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:52:09-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:52:09-04:00
log0bf7ebcfea7934cb972aef84b25494c92f0dcf6f
tree9a600a84b8b35e998ef67681ece89ccd76db7ad6
parentfd2cd38bdb831ef78a0d4ab0973020dfbd348c1f

std.zig.tokenizer: fix handling of line comment / doc comment

2 files changed, 91 insertions(+), 25 deletions(-)

std/zig/parser_test.zig+1
...@@ -124,6 +124,7 @@ test "zig fmt: comments before statements" {...@@ -124,6 +124,7 @@ test "zig fmt: comments before statements" {
124 \\ _ = @import("foo/bar.zig");124 \\ _ = @import("foo/bar.zig");
125 \\125 \\
126 \\ // middle126 \\ // middle
127 \\ // middle2
127 \\128 \\
128 \\ // end129 \\ // end
129 \\}130 \\}
std/zig/tokenizer.zig+90-25
...@@ -260,6 +260,7 @@ pub const Tokenizer = struct {...@@ -260,6 +260,7 @@ pub const Tokenizer = struct {
260 Slash,260 Slash,
261 LineCommentStart,261 LineCommentStart,
262 LineComment,262 LineComment,
263 DocComment,
263 Zero,264 Zero,
264 IntegerLiteral,265 IntegerLiteral,
265 IntegerLiteralWithRadix,266 IntegerLiteralWithRadix,
...@@ -825,6 +826,7 @@ pub const Tokenizer = struct {...@@ -825,6 +826,7 @@ pub const Tokenizer = struct {
825 State.Slash => switch (c) {826 State.Slash => switch (c) {
826 '/' => {827 '/' => {
827 state = State.LineCommentStart;828 state = State.LineCommentStart;
829 result.id = Token.Id.LineComment;
828 },830 },
829 '=' => {831 '=' => {
830 result.id = Token.Id.SlashEqual;832 result.id = Token.Id.SlashEqual;
...@@ -839,15 +841,15 @@ pub const Tokenizer = struct {...@@ -839,15 +841,15 @@ pub const Tokenizer = struct {
839 State.LineCommentStart => switch (c) {841 State.LineCommentStart => switch (c) {
840 '/' => {842 '/' => {
841 result.id = Token.Id.DocComment;843 result.id = Token.Id.DocComment;
842 state = State.LineComment;844 state = State.DocComment;
843 },845 },
844 '\n' => {846 '\n' => break,
845 result.id = Token.Id.LineComment;847 else => {
846 break;848 state = State.LineComment;
849 self.checkLiteralCharacter();
847 },850 },
848 else => self.checkLiteralCharacter(),
849 },851 },
850 State.LineComment => switch (c) {852 State.LineComment, State.DocComment => switch (c) {
851 '\n' => break,853 '\n' => break,
852 else => self.checkLiteralCharacter(),854 else => self.checkLiteralCharacter(),
853 },855 },
...@@ -934,7 +936,10 @@ pub const Tokenizer = struct {...@@ -934,7 +936,10 @@ pub const Tokenizer = struct {
934 },936 },
935 State.LineCommentStart,937 State.LineCommentStart,
936 State.LineComment => {938 State.LineComment => {
937 result.id = Token.Id.Eof;939 result.id = Token.Id.LineComment;
940 },
941 State.DocComment => {
942 result.id = Token.Id.DocComment;
938 },943 },
939944
940 State.NumberDot,945 State.NumberDot,
...@@ -1105,41 +1110,77 @@ test "tokenizer - invalid literal/comment characters" {...@@ -1105,41 +1110,77 @@ test "tokenizer - invalid literal/comment characters" {
1105 Token.Id.Invalid,1110 Token.Id.Invalid,
1106 });1111 });
1107 testTokenize("//\x00", []Token.Id {1112 testTokenize("//\x00", []Token.Id {
1113 Token.Id.LineComment,
1108 Token.Id.Invalid,1114 Token.Id.Invalid,
1109 });1115 });
1110 testTokenize("//\x1f", []Token.Id {1116 testTokenize("//\x1f", []Token.Id {
1117 Token.Id.LineComment,
1111 Token.Id.Invalid,1118 Token.Id.Invalid,
1112 });1119 });
1113 testTokenize("//\x7f", []Token.Id {1120 testTokenize("//\x7f", []Token.Id {
1121 Token.Id.LineComment,
1114 Token.Id.Invalid,1122 Token.Id.Invalid,
1115 });1123 });
1116}1124}
11171125
1118test "tokenizer - utf8" {1126test "tokenizer - utf8" {
1119 testTokenize("//\xc2\x80", []Token.Id{});1127 testTokenize("//\xc2\x80", []Token.Id{Token.Id.LineComment});
1120 testTokenize("//\xf4\x8f\xbf\xbf", []Token.Id{});1128 testTokenize("//\xf4\x8f\xbf\xbf", []Token.Id{Token.Id.LineComment});
1121}1129}
11221130
1123test "tokenizer - invalid utf8" {1131test "tokenizer - invalid utf8" {
1124 testTokenize("//\x80", []Token.Id{Token.Id.Invalid});1132 testTokenize("//\x80", []Token.Id{
1125 testTokenize("//\xbf", []Token.Id{Token.Id.Invalid});1133 Token.Id.LineComment,
1126 testTokenize("//\xf8", []Token.Id{Token.Id.Invalid});1134 Token.Id.Invalid,
1127 testTokenize("//\xff", []Token.Id{Token.Id.Invalid});1135 });
1128 testTokenize("//\xc2\xc0", []Token.Id{Token.Id.Invalid});1136 testTokenize("//\xbf", []Token.Id{
1129 testTokenize("//\xe0", []Token.Id{Token.Id.Invalid});1137 Token.Id.LineComment,
1130 testTokenize("//\xf0", []Token.Id{Token.Id.Invalid});1138 Token.Id.Invalid,
1131 testTokenize("//\xf0\x90\x80\xc0", []Token.Id{Token.Id.Invalid});1139 });
1140 testTokenize("//\xf8", []Token.Id{
1141 Token.Id.LineComment,
1142 Token.Id.Invalid,
1143 });
1144 testTokenize("//\xff", []Token.Id{
1145 Token.Id.LineComment,
1146 Token.Id.Invalid,
1147 });
1148 testTokenize("//\xc2\xc0", []Token.Id{
1149 Token.Id.LineComment,
1150 Token.Id.Invalid,
1151 });
1152 testTokenize("//\xe0", []Token.Id{
1153 Token.Id.LineComment,
1154 Token.Id.Invalid,
1155 });
1156 testTokenize("//\xf0", []Token.Id{
1157 Token.Id.LineComment,
1158 Token.Id.Invalid,
1159 });
1160 testTokenize("//\xf0\x90\x80\xc0", []Token.Id{
1161 Token.Id.LineComment,
1162 Token.Id.Invalid,
1163 });
1132}1164}
11331165
1134test "tokenizer - illegal unicode codepoints" {1166test "tokenizer - illegal unicode codepoints" {
1135 // unicode newline characters.U+0085, U+2028, U+20291167 // unicode newline characters.U+0085, U+2028, U+2029
1136 testTokenize("//\xc2\x84", []Token.Id{});1168 testTokenize("//\xc2\x84", []Token.Id{Token.Id.LineComment});
1137 testTokenize("//\xc2\x85", []Token.Id{Token.Id.Invalid});1169 testTokenize("//\xc2\x85", []Token.Id{
1138 testTokenize("//\xc2\x86", []Token.Id{});1170 Token.Id.LineComment,
1139 testTokenize("//\xe2\x80\xa7", []Token.Id{});1171 Token.Id.Invalid,
1140 testTokenize("//\xe2\x80\xa8", []Token.Id{Token.Id.Invalid});1172 });
1141 testTokenize("//\xe2\x80\xa9", []Token.Id{Token.Id.Invalid});1173 testTokenize("//\xc2\x86", []Token.Id{Token.Id.LineComment});
1142 testTokenize("//\xe2\x80\xaa", []Token.Id{});1174 testTokenize("//\xe2\x80\xa7", []Token.Id{Token.Id.LineComment});
1175 testTokenize("//\xe2\x80\xa8", []Token.Id{
1176 Token.Id.LineComment,
1177 Token.Id.Invalid,
1178 });
1179 testTokenize("//\xe2\x80\xa9", []Token.Id{
1180 Token.Id.LineComment,
1181 Token.Id.Invalid,
1182 });
1183 testTokenize("//\xe2\x80\xaa", []Token.Id{Token.Id.LineComment});
1143}1184}
11441185
1145test "tokenizer - string identifier and builtin fns" {1186test "tokenizer - string identifier and builtin fns" {
...@@ -1166,11 +1207,35 @@ test "tokenizer - pipe and then invalid" {...@@ -1166,11 +1207,35 @@ test "tokenizer - pipe and then invalid" {
1166 });1207 });
1167}1208}
11681209
1210test "tokenizer - line comment and doc comment" {
1211 testTokenize("//", []Token.Id{Token.Id.LineComment});
1212 testTokenize("// a / b", []Token.Id{Token.Id.LineComment});
1213 testTokenize("// /", []Token.Id{Token.Id.LineComment});
1214 testTokenize("/// a", []Token.Id{Token.Id.DocComment});
1215 testTokenize("///", []Token.Id{Token.Id.DocComment});
1216}
1217
1218test "tokenizer - line comment followed by identifier" {
1219 testTokenize(
1220 \\ Unexpected,
1221 \\ // another
1222 \\ Another,
1223 , []Token.Id{
1224 Token.Id.Identifier,
1225 Token.Id.Comma,
1226 Token.Id.LineComment,
1227 Token.Id.Identifier,
1228 Token.Id.Comma,
1229 });
1230}
1231
1169fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {1232fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
1170 var tokenizer = Tokenizer.init(source);1233 var tokenizer = Tokenizer.init(source);
1171 for (expected_tokens) |expected_token_id| {1234 for (expected_tokens) |expected_token_id| {
1172 const token = tokenizer.next();1235 const token = tokenizer.next();
1173 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));1236 if (@TagType(Token.Id)(token.id) != @TagType(Token.Id)(expected_token_id)) {
1237 std.debug.panic("expected {}, found {}\n", @tagName(@TagType(Token.Id)(expected_token_id)), @tagName(@TagType(Token.Id)(token.id)));
1238 }
1174 switch (expected_token_id) {1239 switch (expected_token_id) {
1175 Token.Id.StringLiteral => |expected_kind| {1240 Token.Id.StringLiteral => |expected_kind| {
1176 std.debug.assert(expected_kind == switch (token.id) { Token.Id.StringLiteral => |kind| kind, else => unreachable });1241 std.debug.assert(expected_kind == switch (token.id) { Token.Id.StringLiteral => |kind| kind, else => unreachable });