authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 03:04:02+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:50+02:00
log472ca947c94f703866eec75fc364810e655b4894
tree10639fb387bf4098e34633a42518bb47c784ef0a
parentd75697a6a3e2c8d96819c365dcb5690d4d8028e9
signaturelock-open Commit is signed but in an unrecognized format.

std-c tokenizer add tests

1 files changed, 196 insertions(+), 11 deletions(-)

lib/std/c/tokenizer.zig+196-11
......@@ -31,7 +31,6 @@ pub const Token = struct {
3131 PipeEqual,
3232 Equal,
3333 EqualEqual,
34 EqualAngleBracketRight,
3534 LParen,
3635 RParen,
3736 LBrace,
......@@ -39,7 +38,6 @@ pub const Token = struct {
3938 LBracket,
4039 RBracket,
4140 Period,
42 PeriodAsterisk,
4341 Ellipsis,
4442 Caret,
4543 CaretEqual,
......@@ -253,7 +251,7 @@ pub const Token = struct {
253251pub const Tokenizer = struct {
254252 source: *Source,
255253 index: usize = 0,
256 prev_tok_id: @TagType(Token.Id),
254 prev_tok_id: @TagType(Token.Id) = .Invalid,
257255
258256 pub fn next(self: *Tokenizer) Token {
259257 const start_index = self.index;
......@@ -296,6 +294,7 @@ pub const Tokenizer = struct {
296294 Minus,
297295 Slash,
298296 Ampersand,
297 Hash,
299298 LineComment,
300299 MultiLineComment,
301300 MultiLineCommentAsterisk,
......@@ -329,9 +328,6 @@ pub const Tokenizer = struct {
329328 '\r' => {
330329 state = .Cr;
331330 },
332 ' ', '\t' => {
333 result.start = self.index + 1;
334 },
335331 '"' => {
336332 result.id = .{ .StringLiteral = .None };
337333 state = .StringLiteral;
......@@ -449,6 +445,9 @@ pub const Tokenizer = struct {
449445 '&' => {
450446 state = .Ampersand;
451447 },
448 '#' => {
449 state = .Hash;
450 },
452451 '0' => {
453452 state = .Zero;
454453 },
......@@ -456,9 +455,7 @@ pub const Tokenizer = struct {
456455 state = .IntegerLiteral;
457456 },
458457 else => {
459 result.id = .Invalid;
460 self.index += 1;
461 break;
458 result.start = self.index + 1;
462459 },
463460 },
464461 .Cr => switch (c) {
......@@ -833,6 +830,17 @@ pub const Tokenizer = struct {
833830 break;
834831 },
835832 },
833 .Hash => switch (c) {
834 '#' => {
835 result.id = .HashHash;
836 self.index += 1;
837 break;
838 },
839 else => {
840 result.id = .Hash;
841 break;
842 },
843 },
836844 .LineComment => switch (c) {
837845 '\n' => {
838846 result.id = .LineComment;
......@@ -1069,6 +1077,7 @@ pub const Tokenizer = struct {
10691077 .Minus => result.id = .Minus,
10701078 .Slash => result.id = .Slash,
10711079 .Ampersand => result.id = .Ampersand,
1080 .Hash => result.id = .Hash,
10721081 .Period => result.id = .Period,
10731082 .Pipe => result.id = .Pipe,
10741083 .AngleBracketAngleBracketRight => result.id = .AngleBracketAngleBracketRight,
......@@ -1089,16 +1098,192 @@ pub const Tokenizer = struct {
10891098 }
10901099};
10911100
1101test "operators" {
1102 expectTokens(
1103 \\ ! != | || |= = ==
1104 \\ ( ) { } [ ] . .. ...
1105 \\ ^ ^= + ++ += - -- -=
1106 \\ * *= % %= -> : ; / /=
1107 \\ , & && &= ? < <= <<
1108 \\ <<= > >= >> >>= ~ # ##
1109 \\
1110 ,
1111 &[_]Token.Id{
1112 .Bang,
1113 .BangEqual,
1114 .Pipe,
1115 .PipePipe,
1116 .PipeEqual,
1117 .Equal,
1118 .EqualEqual,
1119 .Nl,
1120
1121 .LParen,
1122 .RParen,
1123 .LBrace,
1124 .RBrace,
1125 .LBracket,
1126 .RBracket,
1127 .Period,
1128 .Period,
1129 .Period,
1130 .Ellipsis,
1131 .Nl,
1132
1133 .Caret,
1134 .CaretEqual,
1135 .Plus,
1136 .PlusPlus,
1137 .PlusEqual,
1138 .Minus,
1139 .MinusMinus,
1140 .MinusEqual,
1141 .Nl,
1142
1143 .Asterisk,
1144 .AsteriskEqual,
1145 .Percent,
1146 .PercentEqual,
1147 .Arrow,
1148 .Colon,
1149 .Semicolon,
1150 .Slash,
1151 .SlashEqual,
1152 .Nl,
1153
1154 .Comma,
1155 .Ampersand,
1156 .AmpersandAmpersand,
1157 .AmpersandEqual,
1158 .QuestionMark,
1159 .AngleBracketLeft,
1160 .AngleBracketLeftEqual,
1161 .AngleBracketAngleBracketLeft,
1162 .Nl,
1163
1164 .AngleBracketAngleBracketLeftEqual,
1165 .AngleBracketRight,
1166 .AngleBracketRightEqual,
1167 .AngleBracketAngleBracketRight,
1168 .AngleBracketAngleBracketRightEqual,
1169 .Tilde,
1170 .Hash,
1171 .HashHash,
1172 .Nl,
1173 },
1174 );
1175}
1176
1177test "keywords" {
1178 expectTokens(
1179 \\auto break case char const continue default do
1180 \\double else enum extern float for goto if int
1181 \\long register return short signed sizeof static
1182 \\struct switch typedef union unsigned void volatile
1183 \\while _Bool _Complex _Imaginary inline restrict _Alignas
1184 \\_Alignof _Atomic _Generic _Noreturn _Static_assert _Thread_local
1185 \\
1186 , &[_]Token.Id{
1187 .Keyword_auto,
1188 .Keyword_break,
1189 .Keyword_case,
1190 .Keyword_char,
1191 .Keyword_const,
1192 .Keyword_continue,
1193 .Keyword_default,
1194 .Keyword_do,
1195 .Nl,
1196
1197 .Keyword_double,
1198 .Keyword_else,
1199 .Keyword_enum,
1200 .Keyword_extern,
1201 .Keyword_float,
1202 .Keyword_for,
1203 .Keyword_goto,
1204 .Keyword_if,
1205 .Keyword_int,
1206 .Nl,
1207
1208 .Keyword_long,
1209 .Keyword_register,
1210 .Keyword_return,
1211 .Keyword_short,
1212 .Keyword_signed,
1213 .Keyword_sizeof,
1214 .Keyword_static,
1215 .Nl,
1216
1217 .Keyword_struct,
1218 .Keyword_switch,
1219 .Keyword_typedef,
1220 .Keyword_union,
1221 .Keyword_unsigned,
1222 .Keyword_void,
1223 .Keyword_volatile,
1224 .Nl,
1225
1226 .Keyword_while,
1227 .Keyword_bool,
1228 .Keyword_complex,
1229 .Keyword_imaginary,
1230 .Keyword_inline,
1231 .Keyword_restrict,
1232 .Keyword_alignas,
1233 .Nl,
1234
1235 .Keyword_alignof,
1236 .Keyword_atomic,
1237 .Keyword_generic,
1238 .Keyword_noreturn,
1239 .Keyword_static_assert,
1240 .Keyword_thread_local,
1241 .Nl,
1242 });
1243}
1244
1245test "preprocessor keywords" {
1246 expectTokens(
1247 \\#include <test>
1248 \\#define
1249 \\#ifdef
1250 \\#ifndef
1251 \\#error
1252 \\#pragma
1253 \\
1254 , &[_]Token.Id{
1255 .Hash,
1256 .Keyword_include,
1257 .MacroString,
1258 .Nl,
1259 .Hash,
1260 .Keyword_define,
1261 .Nl,
1262 .Hash,
1263 .Keyword_ifdef,
1264 .Nl,
1265 .Hash,
1266 .Keyword_ifndef,
1267 .Nl,
1268 .Hash,
1269 .Keyword_error,
1270 .Nl,
1271 .Hash,
1272 .Keyword_pragma,
1273 .Nl,
1274 });
1275}
1276
10921277fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {
10931278 var tokenizer = Tokenizer{
1094 .source = .{
1279 .source = &Source{
10951280 .buffer = source,
10961281 .file_name = undefined,
10971282 },
10981283 };
10991284 for (expected_tokens) |expected_token_id| {
11001285 const token = tokenizer.next();
1101 if (token.id != expected_token_id) {
1286 if (!std.meta.eql(token.id, expected_token_id)) {
11021287 std.debug.panic("expected {}, found {}\n", .{ @tagName(expected_token_id), @tagName(token.id) });
11031288 }
11041289 }