authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 03:04:02+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:50+02:00
log472ca947c94f703866eec75fc364810e655b4894
tree10639fb387bf4098e34633a42518bb47c784ef0a
parentd75697a6a3e2c8d96819c365dcb5690d4d8028e9
signaturelock-open Commit is signed but in an unrecognized format.

std-c tokenizer add tests

1 files changed, 196 insertions(+), 11 deletions(-)

lib/std/c/tokenizer.zig+196-11
...@@ -31,7 +31,6 @@ pub const Token = struct {...@@ -31,7 +31,6 @@ pub const Token = struct {
31 PipeEqual,31 PipeEqual,
32 Equal,32 Equal,
33 EqualEqual,33 EqualEqual,
34 EqualAngleBracketRight,
35 LParen,34 LParen,
36 RParen,35 RParen,
37 LBrace,36 LBrace,
...@@ -39,7 +38,6 @@ pub const Token = struct {...@@ -39,7 +38,6 @@ pub const Token = struct {
39 LBracket,38 LBracket,
40 RBracket,39 RBracket,
41 Period,40 Period,
42 PeriodAsterisk,
43 Ellipsis,41 Ellipsis,
44 Caret,42 Caret,
45 CaretEqual,43 CaretEqual,
...@@ -253,7 +251,7 @@ pub const Token = struct {...@@ -253,7 +251,7 @@ pub const Token = struct {
253pub const Tokenizer = struct {251pub const Tokenizer = struct {
254 source: *Source,252 source: *Source,
255 index: usize = 0,253 index: usize = 0,
256 prev_tok_id: @TagType(Token.Id),254 prev_tok_id: @TagType(Token.Id) = .Invalid,
257255
258 pub fn next(self: *Tokenizer) Token {256 pub fn next(self: *Tokenizer) Token {
259 const start_index = self.index;257 const start_index = self.index;
...@@ -296,6 +294,7 @@ pub const Tokenizer = struct {...@@ -296,6 +294,7 @@ pub const Tokenizer = struct {
296 Minus,294 Minus,
297 Slash,295 Slash,
298 Ampersand,296 Ampersand,
297 Hash,
299 LineComment,298 LineComment,
300 MultiLineComment,299 MultiLineComment,
301 MultiLineCommentAsterisk,300 MultiLineCommentAsterisk,
...@@ -329,9 +328,6 @@ pub const Tokenizer = struct {...@@ -329,9 +328,6 @@ pub const Tokenizer = struct {
329 '\r' => {328 '\r' => {
330 state = .Cr;329 state = .Cr;
331 },330 },
332 ' ', '\t' => {
333 result.start = self.index + 1;
334 },
335 '"' => {331 '"' => {
336 result.id = .{ .StringLiteral = .None };332 result.id = .{ .StringLiteral = .None };
337 state = .StringLiteral;333 state = .StringLiteral;
...@@ -449,6 +445,9 @@ pub const Tokenizer = struct {...@@ -449,6 +445,9 @@ pub const Tokenizer = struct {
449 '&' => {445 '&' => {
450 state = .Ampersand;446 state = .Ampersand;
451 },447 },
448 '#' => {
449 state = .Hash;
450 },
452 '0' => {451 '0' => {
453 state = .Zero;452 state = .Zero;
454 },453 },
...@@ -456,9 +455,7 @@ pub const Tokenizer = struct {...@@ -456,9 +455,7 @@ pub const Tokenizer = struct {
456 state = .IntegerLiteral;455 state = .IntegerLiteral;
457 },456 },
458 else => {457 else => {
459 result.id = .Invalid;458 result.start = self.index + 1;
460 self.index += 1;
461 break;
462 },459 },
463 },460 },
464 .Cr => switch (c) {461 .Cr => switch (c) {
...@@ -833,6 +830,17 @@ pub const Tokenizer = struct {...@@ -833,6 +830,17 @@ pub const Tokenizer = struct {
833 break;830 break;
834 },831 },
835 },832 },
833 .Hash => switch (c) {
834 '#' => {
835 result.id = .HashHash;
836 self.index += 1;
837 break;
838 },
839 else => {
840 result.id = .Hash;
841 break;
842 },
843 },
836 .LineComment => switch (c) {844 .LineComment => switch (c) {
837 '\n' => {845 '\n' => {
838 result.id = .LineComment;846 result.id = .LineComment;
...@@ -1069,6 +1077,7 @@ pub const Tokenizer = struct {...@@ -1069,6 +1077,7 @@ pub const Tokenizer = struct {
1069 .Minus => result.id = .Minus,1077 .Minus => result.id = .Minus,
1070 .Slash => result.id = .Slash,1078 .Slash => result.id = .Slash,
1071 .Ampersand => result.id = .Ampersand,1079 .Ampersand => result.id = .Ampersand,
1080 .Hash => result.id = .Hash,
1072 .Period => result.id = .Period,1081 .Period => result.id = .Period,
1073 .Pipe => result.id = .Pipe,1082 .Pipe => result.id = .Pipe,
1074 .AngleBracketAngleBracketRight => result.id = .AngleBracketAngleBracketRight,1083 .AngleBracketAngleBracketRight => result.id = .AngleBracketAngleBracketRight,
...@@ -1089,16 +1098,192 @@ pub const Tokenizer = struct {...@@ -1089,16 +1098,192 @@ pub const Tokenizer = struct {
1089 }1098 }
1090};1099};
10911100
1101test "operators" {
1102 expectTokens(
1103 \\ ! != | || |= = ==
1104 \\ ( ) { } [ ] . .. ...
1105 \\ ^ ^= + ++ += - -- -=
1106 \\ * *= % %= -> : ; / /=
1107 \\ , & && &= ? < <= <<
1108 \\ <<= > >= >> >>= ~ # ##
1109 \\
1110 ,
1111 &[_]Token.Id{
1112 .Bang,
1113 .BangEqual,
1114 .Pipe,
1115 .PipePipe,
1116 .PipeEqual,
1117 .Equal,
1118 .EqualEqual,
1119 .Nl,
1120
1121 .LParen,
1122 .RParen,
1123 .LBrace,
1124 .RBrace,
1125 .LBracket,
1126 .RBracket,
1127 .Period,
1128 .Period,
1129 .Period,
1130 .Ellipsis,
1131 .Nl,
1132
1133 .Caret,
1134 .CaretEqual,
1135 .Plus,
1136 .PlusPlus,
1137 .PlusEqual,
1138 .Minus,
1139 .MinusMinus,
1140 .MinusEqual,
1141 .Nl,
1142
1143 .Asterisk,
1144 .AsteriskEqual,
1145 .Percent,
1146 .PercentEqual,
1147 .Arrow,
1148 .Colon,
1149 .Semicolon,
1150 .Slash,
1151 .SlashEqual,
1152 .Nl,
1153
1154 .Comma,
1155 .Ampersand,
1156 .AmpersandAmpersand,
1157 .AmpersandEqual,
1158 .QuestionMark,
1159 .AngleBracketLeft,
1160 .AngleBracketLeftEqual,
1161 .AngleBracketAngleBracketLeft,
1162 .Nl,
1163
1164 .AngleBracketAngleBracketLeftEqual,
1165 .AngleBracketRight,
1166 .AngleBracketRightEqual,
1167 .AngleBracketAngleBracketRight,
1168 .AngleBracketAngleBracketRightEqual,
1169 .Tilde,
1170 .Hash,
1171 .HashHash,
1172 .Nl,
1173 },
1174 );
1175}
1176
1177test "keywords" {
1178 expectTokens(
1179 \\auto break case char const continue default do
1180 \\double else enum extern float for goto if int
1181 \\long register return short signed sizeof static
1182 \\struct switch typedef union unsigned void volatile
1183 \\while _Bool _Complex _Imaginary inline restrict _Alignas
1184 \\_Alignof _Atomic _Generic _Noreturn _Static_assert _Thread_local
1185 \\
1186 , &[_]Token.Id{
1187 .Keyword_auto,
1188 .Keyword_break,
1189 .Keyword_case,
1190 .Keyword_char,
1191 .Keyword_const,
1192 .Keyword_continue,
1193 .Keyword_default,
1194 .Keyword_do,
1195 .Nl,
1196
1197 .Keyword_double,
1198 .Keyword_else,
1199 .Keyword_enum,
1200 .Keyword_extern,
1201 .Keyword_float,
1202 .Keyword_for,
1203 .Keyword_goto,
1204 .Keyword_if,
1205 .Keyword_int,
1206 .Nl,
1207
1208 .Keyword_long,
1209 .Keyword_register,
1210 .Keyword_return,
1211 .Keyword_short,
1212 .Keyword_signed,
1213 .Keyword_sizeof,
1214 .Keyword_static,
1215 .Nl,
1216
1217 .Keyword_struct,
1218 .Keyword_switch,
1219 .Keyword_typedef,
1220 .Keyword_union,
1221 .Keyword_unsigned,
1222 .Keyword_void,
1223 .Keyword_volatile,
1224 .Nl,
1225
1226 .Keyword_while,
1227 .Keyword_bool,
1228 .Keyword_complex,
1229 .Keyword_imaginary,
1230 .Keyword_inline,
1231 .Keyword_restrict,
1232 .Keyword_alignas,
1233 .Nl,
1234
1235 .Keyword_alignof,
1236 .Keyword_atomic,
1237 .Keyword_generic,
1238 .Keyword_noreturn,
1239 .Keyword_static_assert,
1240 .Keyword_thread_local,
1241 .Nl,
1242 });
1243}
1244
1245test "preprocessor keywords" {
1246 expectTokens(
1247 \\#include <test>
1248 \\#define
1249 \\#ifdef
1250 \\#ifndef
1251 \\#error
1252 \\#pragma
1253 \\
1254 , &[_]Token.Id{
1255 .Hash,
1256 .Keyword_include,
1257 .MacroString,
1258 .Nl,
1259 .Hash,
1260 .Keyword_define,
1261 .Nl,
1262 .Hash,
1263 .Keyword_ifdef,
1264 .Nl,
1265 .Hash,
1266 .Keyword_ifndef,
1267 .Nl,
1268 .Hash,
1269 .Keyword_error,
1270 .Nl,
1271 .Hash,
1272 .Keyword_pragma,
1273 .Nl,
1274 });
1275}
1276
1092fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {1277fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {
1093 var tokenizer = Tokenizer{1278 var tokenizer = Tokenizer{
1094 .source = .{1279 .source = &Source{
1095 .buffer = source,1280 .buffer = source,
1096 .file_name = undefined,1281 .file_name = undefined,
1097 },1282 },
1098 };1283 };
1099 for (expected_tokens) |expected_token_id| {1284 for (expected_tokens) |expected_token_id| {
1100 const token = tokenizer.next();1285 const token = tokenizer.next();
1101 if (token.id != expected_token_id) {1286 if (!std.meta.eql(token.id, expected_token_id)) {
1102 std.debug.panic("expected {}, found {}\n", .{ @tagName(expected_token_id), @tagName(token.id) });1287 std.debug.panic("expected {}, found {}\n", .{ @tagName(expected_token_id), @tagName(token.id) });
1103 }1288 }
1104 }1289 }