authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 00:19:43+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:49+02:00
log05acc0b0c14c19c9776633cd0d1ebbbbc30c3c47
tree6ee582af66dd869f782135c3a78e758d7024767e
parent04b7cec42e85a74e5933f13a6d3d10c8e0207066
signaturelock-open Commit is signed but in an unrecognized format.

std-c tokenizer more stuff

1 files changed, 556 insertions(+), 2 deletions(-)

lib/std/c/tokenizer.zig+556-2
......@@ -101,16 +101,570 @@ pub const Tokenizer = struct {
101101 };
102102 var state: enum {
103103 Start,
104 Cr,
105 StringLiteral,
106 CharLiteral,
107 Identifier,
108 Equal,
109 Bang,
110 Pipe,
111 Percent,
112 Asterisk,
113 Plus,
114 AngleBracketLeft,
115 AngleBracketAngleBracketLeft,
116 AngleBracketRight,
117 AngleBracketAngleBracketRight,
118 Caret,
119 Period,
120 Minus,
121 Slash,
122 Ampersand,
123 Zero,
124 IntegerLiteralOct,
125 IntegerLiteralBinary,
126 IntegerLiteralHex,
127 IntegerLiteral,
128 IntegerSuffix,
129 IntegerSuffixU,
130 IntegerSuffixL,
131 IntegerSuffixLL,
132 IntegerSuffixUL,
104133 } = .Start;
105134 while (self.index < self.source.buffer.len) : (self.index += 1) {
106135 const c = self.source.buffer[self.index];
107136 switch (state) {
108137 .Start => switch (c) {
109 else => @panic("TODO"),
138 '\n' => {
139 result.id = .Nl;
140 self.index += 1;
141 break;
142 },
143 '\r' => {
144 state = .Cr;
145 },
146 ' ', '\t' => {
147 result.start = self.index + 1;
148 },
149 '"' => {
150 state = .StringLiteral;
151 result.id = .StringLiteral;
152 },
153 '\'' => {
154 state = .CharLiteral;
155 },
156 'a'...'z', 'A'...'Z', '_' => {
157 state = .Identifier;
158 result.id = .Identifier;
159 },
160 '=' => {
161 state = .Equal;
162 },
163 '!' => {
164 state = .Bang;
165 },
166 '|' => {
167 state = .Pipe;
168 },
169 '(' => {
170 result.id = .LParen;
171 self.index += 1;
172 break;
173 },
174 ')' => {
175 result.id = .RParen;
176 self.index += 1;
177 break;
178 },
179 '[' => {
180 result.id = .LBracket;
181 self.index += 1;
182 break;
183 },
184 ']' => {
185 result.id = .RBracket;
186 self.index += 1;
187 break;
188 },
189 ';' => {
190 result.id = .Semicolon;
191 self.index += 1;
192 break;
193 },
194 ',' => {
195 result.id = .Comma;
196 self.index += 1;
197 break;
198 },
199 '?' => {
200 result.id = .QuestionMark;
201 self.index += 1;
202 break;
203 },
204 ':' => {
205 result.id = .Colon;
206 self.index += 1;
207 break;
208 },
209 '%' => {
210 state = .Percent;
211 },
212 '*' => {
213 state = .Asterisk;
214 },
215 '+' => {
216 state = .Plus;
217 },
218 '<' => {
219 state = .AngleBracketLeft;
220 },
221 '>' => {
222 state = .AngleBracketRight;
223 },
224 '^' => {
225 state = .Caret;
226 },
227 '{' => {
228 result.id = .LBrace;
229 self.index += 1;
230 break;
231 },
232 '}' => {
233 result.id = .RBrace;
234 self.index += 1;
235 break;
236 },
237 '~' => {
238 result.id = .Tilde;
239 self.index += 1;
240 break;
241 },
242 '.' => {
243 state = .Period;
244 },
245 '-' => {
246 state = .Minus;
247 },
248 '/' => {
249 state = .Slash;
250 },
251 '&' => {
252 state = .Ampersand;
253 },
254 '0' => {
255 state = .Zero;
256 result.id = .IntegerLiteral;
257 },
258 '1'...'9' => {
259 state = .IntegerLiteral;
260 result.id = .IntegerLiteral;
261 },
262 else => {
263 result.id = .Invalid;
264 self.index += 1;
265 break;
266 },
267 },
268 .Cr => switch (c) {
269 '\n' => {
270 result.id = .Nl;
271 self.index += 1;
272 break;
273 },
274 else => {
275 result.id = .Invalid;
276 break;
277 },
278 },
279 .Identifier => switch (c) {
280 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
281 else => {
282 result.id = .Identifier;
283 break;
284 },
285 },
286 .Equal => switch (c) {
287 '=' => {
288 result.id = .EqualEqual;
289 self.index += 1;
290 break;
291 },
292 else => {
293 result.id = .Equal;
294 break;
295 },
296 },
297 .Bang => switch (c) {
298 '=' => {
299 result.id = .BangEqual;
300 self.index += 1;
301 break;
302 },
303 else => {
304 result.id = .Bang;
305 break;
306 },
307 },
308 .Pipe => switch (c) {
309 '=' => {
310 result.id = .PipeEqual;
311 self.index += 1;
312 break;
313 },
314 '|' => {
315 result.id = .PipePipe;
316 self.index += 1;
317 break;
318 },
319 else => {
320 result.id = .Pipe;
321 break;
322 },
323 },
324 .Percent => switch (c) {
325 '=' => {
326 result.id = .PercentEqual;
327 self.index += 1;
328 break;
329 },
330 else => {
331 result.id = .Id.Percent;
332 break;
333 },
334 },
335 .Asterisk => switch (c) {
336 '=' => {
337 result.id = .AsteriskEqual;
338 self.index += 1;
339 break;
340 },
341 else => {
342 result.id = .Asterisk;
343 break;
344 },
345 },
346 .Plus => switch (c) {
347 '=' => {
348 result.id = .PlusEqual;
349 self.index += 1;
350 break;
351 },
352 '+' => {
353 result.id = .PlusPlus;
354 self.index += 1;
355 break;
356 },
357 else => {
358 result.id = .Plus;
359 break;
360 },
361 },
362 .AngleBracketLeft => switch (c) {
363 '<' => {
364 state = .AngleBracketAngleBracketLeft;
365 },
366 '=' => {
367 result.id = .AngleBracketLeftEqual;
368 self.index += 1;
369 break;
370 },
371 else => {
372 result.id = .AngleBracketLeft;
373 break;
374 },
375 },
376 .AngleBracketAngleBracketLeft => switch (c) {
377 '=' => {
378 result.id = .AngleBracketAngleBracketLeftEqual;
379 self.index += 1;
380 break;
381 },
382 else => {
383 result.id = .AngleBracketAngleBracketLeft;
384 break;
385 },
386 },
387 .AngleBracketRight => switch (c) {
388 '>' => {
389 state = .AngleBracketAngleBracketRight;
390 },
391 '=' => {
392 result.id = .AngleBracketRightEqual;
393 self.index += 1;
394 break;
395 },
396 else => {
397 result.id = .AngleBracketRight;
398 break;
399 },
400 },
401 .AngleBracketAngleBracketRight => switch (c) {
402 '=' => {
403 result.id = .AngleBracketAngleBracketRightEqual;
404 self.index += 1;
405 break;
406 },
407 else => {
408 result.id = .AngleBracketAngleBracketRight;
409 break;
410 },
411 },
412 .Caret => switch (c) {
413 '=' => {
414 result.id = .CaretEqual;
415 self.index += 1;
416 break;
417 },
418 else => {
419 result.id = .Caret;
420 break;
421 },
422 },
423 .Period => switch (c) {
424 '.' => {
425 state = .Period2;
426 },
427 '0'...'9' => {
428 state = .FloatFraction;
429 },
430 else => {
431 result.id = .Period;
432 break;
433 },
434 },
435 .Period2 => switch (c) {
436 '.' => {
437 result.id = .Ellipsis;
438 self.index += 1;
439 break;
440 },
441 else => {
442 result.id = .Period;
443 self.index -= 1;
444 break;
445 },
446 },
447 .Minus => switch (c) {
448 '>' => {
449 result.id = .Arrow;
450 self.index += 1;
451 break;
452 },
453 '=' => {
454 result.id = .MinusEqual;
455 self.index += 1;
456 break;
457 },
458 '-' => {
459 result.id = .MinusMinus;
460 self.index += 1;
461 break;
462 },
463 else => {
464 result.id = .Minus;
465 break;
466 },
467 },
468 .Slash => switch (c) {
469 '/' => {
470 state = .LineComment;
471 result.id = .LineComment;
472 },
473 '=' => {
474 result.id = .SlashEqual;
475 self.index += 1;
476 break;
477 },
478 else => {
479 result.id = .Slash;
480 break;
481 },
482 },
483 .Ampersand => switch (c) {
484 '&' => {
485 result.id = .AmpersandAmpersand;
486 self.index += 1;
487 break;
488 },
489 '=' => {
490 result.id = .AmpersandEqual;
491 self.index += 1;
492 break;
493 },
494 else => {
495 result.id = .Ampersand;
496 break;
497 },
498 },
499 .Zero => switch (c) {
500 '0'...'9' => {
501 state = .IntegerLiteralOct;
502 },
503 'b', 'B' => {
504 state = .IntegerLiteralBinary;
505 },
506 'x', 'X' => {
507 state = .IntegerLiteralHex;
508 },
509 else => {
510 state = .IntegerSuffix;
511 self.index -= 1;
512 },
513 },
514 .IntegerLiteralOct => switch (c) {
515 '0'...'7' => {},
516 else => {
517 state = .IntegerSuffix;
518 self.index -= 1;
519 },
520 },
521 .IntegerLiteralBinary => switch (c) {
522 '0', '1' => {},
523 else => {
524 state = .IntegerSuffix;
525 self.index -= 1;
526 },
527 },
528 .IntegerLiteralHex => switch (c) {
529 '0'...'9', 'a'...'f', 'A'...'F' => {},
530 '.' => {
531 state = .FloatFractionHex;
532 },
533 'p', 'P' => {
534 state = .FloatExponentUnsignedHex;
535 },
536 else => {
537 state = .IntegerSuffix;
538 self.index -= 1;
539 },
540 },
541 .IntegerLiteral => switch (c) {
542 '0'...'9' => {},
543 '.' => {
544 state = .FloatFraction;
545 },
546 'e', 'E' => {
547 state = .FloatExponentUnsigned;
548 },
549 else => {
550 state = .IntegerSuffix;
551 self.index -= 1;
552 },
553 },
554 .IntegerSuffix => switch (c) {
555 'u', 'U' => {
556 state = .IntegerSuffixU;
557 },
558 'l', 'L' => {
559 state = .IntegerSuffixL;
560 },
561 else => {
562 result.id = .IntegerLiteral;
563 break;
564 },
565 },
566 .IntegerSuffixU => switch (c) {
567 'l', 'L' => {
568 state = .IntegerSuffixUL;
569 },
570 else => {
571 result.id = .IntegerLiteral;
572 result.num_suffix = .U;
573 break;
574 },
575 },
576 .IntegerSuffixL => switch (c) {
577 'l', 'L' => {
578 state = .IntegerSuffixLL;
579 },
580 'u', 'U' => {
581 result.id = .IntegerLiteral;
582 result.num_suffix = .LU;
583 self.index += 1;
584 break;
585 },
586 else => {
587 result.id = .IntegerLiteral;
588 result.num_suffix = .L;
589 break;
590 },
591 },
592 .IntegerSuffixLL => switch (c) {
593 'u', 'U' => {
594 result.id = .IntegerLiteral;
595 result.num_suffix = .LLU;
596 self.index += 1;
597 break;
598 },
599 else => {
600 result.id = .IntegerLiteral;
601 result.num_suffix = .LL;
602 break;
603 },
604 },
605 .IntegerSuffixUL => switch (c) {
606 'l', 'L' => {
607 result.id = .IntegerLiteral;
608 result.num_suffix = .LLU;
609 self.index += 1;
610 break;
611 },
612 else => {
613 result.id = .IntegerLiteral;
614 result.num_suffix = .LU;
615 break;
616 },
110617 },
111 else => @panic("TODO"),
618 }
619 } else if (self.index == self.source.buffer.len) {
620 switch (state) {
621 .Identifier => {
622 result.id = .Identifier;
623 },
624 .IntegerLiteralOct,
625 .IntegerLiteralBinary,
626 .IntegerLiteralHex,
627 .IntegerLiteral,
628 .IntegerSuffix,
629 .Zero => result.id = .IntegerLiteral,
630 .IntegerSuffixU => {
631 result.id = .IntegerLiteral;
632 result.num_suffix = .U;
633 },
634 .IntegerSuffixL => {
635 result.id = .IntegerLiteral;
636 result.num_suffix = .L;
637 },
638 .IntegerSuffixLL => {
639 result.id = .IntegerLiteral;
640 result.num_suffix = .LL;
641 },
642 .IntegerSuffixUL => {
643 result.id = .IntegerLiteral;
644 result.num_suffix = .Ul;
645 },
646
647 .Equal => result.id = .Equal,
648 .Bang => result.id = .Bang,
649 .Minus => result.id = .Minus,
650 .Slash => result.id = .Slash,
651 .Ampersand => result.id = .Ampersand,
652 .Period => result.id = .Period,
653 .Period2 => result.id = .Invalid,
654 .Pipe => result.id = .Pipe,
655 .AngleBracketAngleBracketRight => result.id = .AngleBracketAngleBracketRight,
656 .AngleBracketRight => result.id = .AngleBracketRight,
657 .AngleBracketAngleBracketLeft => result.id = .AngleBracketAngleBracketLeft,
658 .AngleBracketLeft => result.id = .AngleBracketLeft,
659 .Plus => result.id = .Plus,
660 .Percent => result.id = .Percent,
661 .Caret => result.id = .Caret,
662 .Asterisk => result.id = .Asterisk,
112663 }
113664 }
665
666 result.end = self.index;
667 return result;
114668 }
115669};
116670