authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:52:09-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-04-30 00:52:09-04:00
log0bf7ebcfea7934cb972aef84b25494c92f0dcf6f
tree9a600a84b8b35e998ef67681ece89ccd76db7ad6
parentfd2cd38bdb831ef78a0d4ab0973020dfbd348c1f

std.zig.tokenizer: fix handling of line comment / doc comment

2 files changed, 91 insertions(+), 25 deletions(-)

std/zig/parser_test.zig+1
......@@ -124,6 +124,7 @@ test "zig fmt: comments before statements" {
124124 \\ _ = @import("foo/bar.zig");
125125 \\
126126 \\ // middle
127 \\ // middle2
127128 \\
128129 \\ // end
129130 \\}
std/zig/tokenizer.zig+90-25
......@@ -260,6 +260,7 @@ pub const Tokenizer = struct {
260260 Slash,
261261 LineCommentStart,
262262 LineComment,
263 DocComment,
263264 Zero,
264265 IntegerLiteral,
265266 IntegerLiteralWithRadix,
......@@ -825,6 +826,7 @@ pub const Tokenizer = struct {
825826 State.Slash => switch (c) {
826827 '/' => {
827828 state = State.LineCommentStart;
829 result.id = Token.Id.LineComment;
828830 },
829831 '=' => {
830832 result.id = Token.Id.SlashEqual;
......@@ -839,15 +841,15 @@ pub const Tokenizer = struct {
839841 State.LineCommentStart => switch (c) {
840842 '/' => {
841843 result.id = Token.Id.DocComment;
842 state = State.LineComment;
844 state = State.DocComment;
843845 },
844 '\n' => {
845 result.id = Token.Id.LineComment;
846 break;
846 '\n' => break,
847 else => {
848 state = State.LineComment;
849 self.checkLiteralCharacter();
847850 },
848 else => self.checkLiteralCharacter(),
849851 },
850 State.LineComment => switch (c) {
852 State.LineComment, State.DocComment => switch (c) {
851853 '\n' => break,
852854 else => self.checkLiteralCharacter(),
853855 },
......@@ -934,7 +936,10 @@ pub const Tokenizer = struct {
934936 },
935937 State.LineCommentStart,
936938 State.LineComment => {
937 result.id = Token.Id.Eof;
939 result.id = Token.Id.LineComment;
940 },
941 State.DocComment => {
942 result.id = Token.Id.DocComment;
938943 },
939944
940945 State.NumberDot,
......@@ -1105,41 +1110,77 @@ test "tokenizer - invalid literal/comment characters" {
11051110 Token.Id.Invalid,
11061111 });
11071112 testTokenize("//\x00", []Token.Id {
1113 Token.Id.LineComment,
11081114 Token.Id.Invalid,
11091115 });
11101116 testTokenize("//\x1f", []Token.Id {
1117 Token.Id.LineComment,
11111118 Token.Id.Invalid,
11121119 });
11131120 testTokenize("//\x7f", []Token.Id {
1121 Token.Id.LineComment,
11141122 Token.Id.Invalid,
11151123 });
11161124}
11171125
11181126test "tokenizer - utf8" {
1119 testTokenize("//\xc2\x80", []Token.Id{});
1120 testTokenize("//\xf4\x8f\xbf\xbf", []Token.Id{});
1127 testTokenize("//\xc2\x80", []Token.Id{Token.Id.LineComment});
1128 testTokenize("//\xf4\x8f\xbf\xbf", []Token.Id{Token.Id.LineComment});
11211129}
11221130
11231131test "tokenizer - invalid utf8" {
1124 testTokenize("//\x80", []Token.Id{Token.Id.Invalid});
1125 testTokenize("//\xbf", []Token.Id{Token.Id.Invalid});
1126 testTokenize("//\xf8", []Token.Id{Token.Id.Invalid});
1127 testTokenize("//\xff", []Token.Id{Token.Id.Invalid});
1128 testTokenize("//\xc2\xc0", []Token.Id{Token.Id.Invalid});
1129 testTokenize("//\xe0", []Token.Id{Token.Id.Invalid});
1130 testTokenize("//\xf0", []Token.Id{Token.Id.Invalid});
1131 testTokenize("//\xf0\x90\x80\xc0", []Token.Id{Token.Id.Invalid});
1132 testTokenize("//\x80", []Token.Id{
1133 Token.Id.LineComment,
1134 Token.Id.Invalid,
1135 });
1136 testTokenize("//\xbf", []Token.Id{
1137 Token.Id.LineComment,
1138 Token.Id.Invalid,
1139 });
1140 testTokenize("//\xf8", []Token.Id{
1141 Token.Id.LineComment,
1142 Token.Id.Invalid,
1143 });
1144 testTokenize("//\xff", []Token.Id{
1145 Token.Id.LineComment,
1146 Token.Id.Invalid,
1147 });
1148 testTokenize("//\xc2\xc0", []Token.Id{
1149 Token.Id.LineComment,
1150 Token.Id.Invalid,
1151 });
1152 testTokenize("//\xe0", []Token.Id{
1153 Token.Id.LineComment,
1154 Token.Id.Invalid,
1155 });
1156 testTokenize("//\xf0", []Token.Id{
1157 Token.Id.LineComment,
1158 Token.Id.Invalid,
1159 });
1160 testTokenize("//\xf0\x90\x80\xc0", []Token.Id{
1161 Token.Id.LineComment,
1162 Token.Id.Invalid,
1163 });
11321164}
11331165
11341166test "tokenizer - illegal unicode codepoints" {
11351167 // unicode newline characters.U+0085, U+2028, U+2029
1136 testTokenize("//\xc2\x84", []Token.Id{});
1137 testTokenize("//\xc2\x85", []Token.Id{Token.Id.Invalid});
1138 testTokenize("//\xc2\x86", []Token.Id{});
1139 testTokenize("//\xe2\x80\xa7", []Token.Id{});
1140 testTokenize("//\xe2\x80\xa8", []Token.Id{Token.Id.Invalid});
1141 testTokenize("//\xe2\x80\xa9", []Token.Id{Token.Id.Invalid});
1142 testTokenize("//\xe2\x80\xaa", []Token.Id{});
1168 testTokenize("//\xc2\x84", []Token.Id{Token.Id.LineComment});
1169 testTokenize("//\xc2\x85", []Token.Id{
1170 Token.Id.LineComment,
1171 Token.Id.Invalid,
1172 });
1173 testTokenize("//\xc2\x86", []Token.Id{Token.Id.LineComment});
1174 testTokenize("//\xe2\x80\xa7", []Token.Id{Token.Id.LineComment});
1175 testTokenize("//\xe2\x80\xa8", []Token.Id{
1176 Token.Id.LineComment,
1177 Token.Id.Invalid,
1178 });
1179 testTokenize("//\xe2\x80\xa9", []Token.Id{
1180 Token.Id.LineComment,
1181 Token.Id.Invalid,
1182 });
1183 testTokenize("//\xe2\x80\xaa", []Token.Id{Token.Id.LineComment});
11431184}
11441185
11451186test "tokenizer - string identifier and builtin fns" {
......@@ -1166,11 +1207,35 @@ test "tokenizer - pipe and then invalid" {
11661207 });
11671208}
11681209
1210test "tokenizer - line comment and doc comment" {
1211 testTokenize("//", []Token.Id{Token.Id.LineComment});
1212 testTokenize("// a / b", []Token.Id{Token.Id.LineComment});
1213 testTokenize("// /", []Token.Id{Token.Id.LineComment});
1214 testTokenize("/// a", []Token.Id{Token.Id.DocComment});
1215 testTokenize("///", []Token.Id{Token.Id.DocComment});
1216}
1217
1218test "tokenizer - line comment followed by identifier" {
1219 testTokenize(
1220 \\ Unexpected,
1221 \\ // another
1222 \\ Another,
1223 , []Token.Id{
1224 Token.Id.Identifier,
1225 Token.Id.Comma,
1226 Token.Id.LineComment,
1227 Token.Id.Identifier,
1228 Token.Id.Comma,
1229 });
1230}
1231
11691232fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
11701233 var tokenizer = Tokenizer.init(source);
11711234 for (expected_tokens) |expected_token_id| {
11721235 const token = tokenizer.next();
1173 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
1236 if (@TagType(Token.Id)(token.id) != @TagType(Token.Id)(expected_token_id)) {
1237 std.debug.panic("expected {}, found {}\n", @tagName(@TagType(Token.Id)(expected_token_id)), @tagName(@TagType(Token.Id)(token.id)));
1238 }
11741239 switch (expected_token_id) {
11751240 Token.Id.StringLiteral => |expected_kind| {
11761241 std.debug.assert(expected_kind == switch (token.id) { Token.Id.StringLiteral => |kind| kind, else => unreachable });