authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-13 23:08:42+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-13 23:08:42+03:00
log2296906e2ad54c387b4b19784148c47a26969cdc
tree963439b81d9d7bdd0f57fa3100ebfcf59146b8f8
parent23c5ff94e9dda07694762fcc829dbac006bb00a1
signaturelock-open Commit is signed but in an unrecognized format.

modernize std.zig.tokenizer

1 files changed, 477 insertions(+), 477 deletions(-)

lib/std/zig/tokenizer.zig+477-477
...@@ -353,64 +353,64 @@ pub const Tokenizer = struct {...@@ -353,64 +353,64 @@ pub const Tokenizer = struct {
353 }353 }
354354
355 const State = enum {355 const State = enum {
356 Start,356 start,
357 Identifier,357 identifier,
358 Builtin,358 builtin,
359 StringLiteral,359 string_literal,
360 StringLiteralBackslash,360 string_literal_backslash,
361 MultilineStringLiteralLine,361 multiline_string_literal_line,
362 CharLiteral,362 char_literal,
363 CharLiteralBackslash,363 char_literal_backslash,
364 CharLiteralHexEscape,364 char_literal_hex_escape,
365 CharLiteralUnicodeEscapeSawU,365 char_literal_unicode_escape_saw_u,
366 CharLiteralUnicodeEscape,366 char_literal_unicode_escape,
367 CharLiteralUnicodeInvalid,367 char_literal_unicode_invalid,
368 CharLiteralUnicode,368 char_literal_unicode,
369 CharLiteralEnd,369 char_literal_end,
370 Backslash,370 backslash,
371 Equal,371 equal,
372 Bang,372 bang,
373 Pipe,373 pipe,
374 Minus,374 minus,
375 MinusPercent,375 minus_percent,
376 Asterisk,376 asterisk,
377 AsteriskPercent,377 asterisk_percent,
378 Slash,378 slash,
379 LineCommentStart,379 line_comment_start,
380 LineComment,380 line_comment,
381 DocCommentStart,381 doc_comment_start,
382 DocComment,382 doc_comment,
383 ContainerDocComment,383 container_doc_comment,
384 Zero,384 zero,
385 IntegerLiteralDec,385 int_literal_dec,
386 IntegerLiteralDecNoUnderscore,386 int_literal_dec_no_underscore,
387 IntegerLiteralBin,387 int_literal_bin,
388 IntegerLiteralBinNoUnderscore,388 int_literal_bin_no_underscore,
389 IntegerLiteralOct,389 int_literal_oct,
390 IntegerLiteralOctNoUnderscore,390 int_literal_oct_no_underscore,
391 IntegerLiteralHex,391 int_literal_hex,
392 IntegerLiteralHexNoUnderscore,392 int_literal_hex_no_underscore,
393 NumberDotDec,393 num_dot_dec,
394 NumberDotHex,394 num_dot_hex,
395 FloatFractionDec,395 float_fraction_dec,
396 FloatFractionDecNoUnderscore,396 float_fraction_dec_no_underscore,
397 FloatFractionHex,397 float_fraction_hex,
398 FloatFractionHexNoUnderscore,398 float_fraction_hex_no_underscore,
399 FloatExponentUnsigned,399 float_exponent_unsigned,
400 FloatExponentNumber,400 float_exponent_num,
401 FloatExponentNumberNoUnderscore,401 float_exponent_num_no_underscore,
402 Ampersand,402 ampersand,
403 Caret,403 caret,
404 Percent,404 percent,
405 Plus,405 plus,
406 PlusPercent,406 plus_percent,
407 AngleBracketLeft,407 angle_bracket_left,
408 AngleBracketAngleBracketLeft,408 angle_bracket_angle_bracket_left,
409 AngleBracketRight,409 angle_bracket_right,
410 AngleBracketAngleBracketRight,410 angle_bracket_angle_bracket_right,
411 Period,411 period,
412 Period2,412 period_2,
413 SawAtSign,413 saw_at_sign,
414 };414 };
415415
416 fn isIdentifierChar(char: u8) bool {416 fn isIdentifierChar(char: u8) bool {
...@@ -423,9 +423,9 @@ pub const Tokenizer = struct {...@@ -423,9 +423,9 @@ pub const Tokenizer = struct {
423 return token;423 return token;
424 }424 }
425 const start_index = self.index;425 const start_index = self.index;
426 var state = State.Start;426 var state: State = .start;
427 var result = Token{427 var result = Token{
428 .id = Token.Id.Eof,428 .id = .Eof,
429 .start = self.index,429 .start = self.index,
430 .end = undefined,430 .end = undefined,
431 };431 };
...@@ -434,40 +434,40 @@ pub const Tokenizer = struct {...@@ -434,40 +434,40 @@ pub const Tokenizer = struct {
434 while (self.index < self.buffer.len) : (self.index += 1) {434 while (self.index < self.buffer.len) : (self.index += 1) {
435 const c = self.buffer[self.index];435 const c = self.buffer[self.index];
436 switch (state) {436 switch (state) {
437 State.Start => switch (c) {437 .start => switch (c) {
438 ' ', '\n', '\t', '\r' => {438 ' ', '\n', '\t', '\r' => {
439 result.start = self.index + 1;439 result.start = self.index + 1;
440 },440 },
441 '"' => {441 '"' => {
442 state = State.StringLiteral;442 state = .string_literal;
443 result.id = Token.Id.StringLiteral;443 result.id = .StringLiteral;
444 },444 },
445 '\'' => {445 '\'' => {
446 state = State.CharLiteral;446 state = .char_literal;
447 },447 },
448 'a'...'z', 'A'...'Z', '_' => {448 'a'...'z', 'A'...'Z', '_' => {
449 state = State.Identifier;449 state = .identifier;
450 result.id = Token.Id.Identifier;450 result.id = .Identifier;
451 },451 },
452 '@' => {452 '@' => {
453 state = State.SawAtSign;453 state = .saw_at_sign;
454 },454 },
455 '=' => {455 '=' => {
456 state = State.Equal;456 state = .equal;
457 },457 },
458 '!' => {458 '!' => {
459 state = State.Bang;459 state = .bang;
460 },460 },
461 '|' => {461 '|' => {
462 state = State.Pipe;462 state = .pipe;
463 },463 },
464 '(' => {464 '(' => {
465 result.id = Token.Id.LParen;465 result.id = .LParen;
466 self.index += 1;466 self.index += 1;
467 break;467 break;
468 },468 },
469 ')' => {469 ')' => {
470 result.id = Token.Id.RParen;470 result.id = .RParen;
471 self.index += 1;471 self.index += 1;
472 break;472 break;
473 },473 },
...@@ -477,213 +477,213 @@ pub const Tokenizer = struct {...@@ -477,213 +477,213 @@ pub const Tokenizer = struct {
477 break;477 break;
478 },478 },
479 ']' => {479 ']' => {
480 result.id = Token.Id.RBracket;480 result.id = .RBracket;
481 self.index += 1;481 self.index += 1;
482 break;482 break;
483 },483 },
484 ';' => {484 ';' => {
485 result.id = Token.Id.Semicolon;485 result.id = .Semicolon;
486 self.index += 1;486 self.index += 1;
487 break;487 break;
488 },488 },
489 ',' => {489 ',' => {
490 result.id = Token.Id.Comma;490 result.id = .Comma;
491 self.index += 1;491 self.index += 1;
492 break;492 break;
493 },493 },
494 '?' => {494 '?' => {
495 result.id = Token.Id.QuestionMark;495 result.id = .QuestionMark;
496 self.index += 1;496 self.index += 1;
497 break;497 break;
498 },498 },
499 ':' => {499 ':' => {
500 result.id = Token.Id.Colon;500 result.id = .Colon;
501 self.index += 1;501 self.index += 1;
502 break;502 break;
503 },503 },
504 '%' => {504 '%' => {
505 state = State.Percent;505 state = .percent;
506 },506 },
507 '*' => {507 '*' => {
508 state = State.Asterisk;508 state = .asterisk;
509 },509 },
510 '+' => {510 '+' => {
511 state = State.Plus;511 state = .plus;
512 },512 },
513 '<' => {513 '<' => {
514 state = State.AngleBracketLeft;514 state = .angle_bracket_left;
515 },515 },
516 '>' => {516 '>' => {
517 state = State.AngleBracketRight;517 state = .angle_bracket_right;
518 },518 },
519 '^' => {519 '^' => {
520 state = State.Caret;520 state = .caret;
521 },521 },
522 '\\' => {522 '\\' => {
523 state = State.Backslash;523 state = .backslash;
524 result.id = Token.Id.MultilineStringLiteralLine;524 result.id = .MultilineStringLiteralLine;
525 },525 },
526 '{' => {526 '{' => {
527 result.id = Token.Id.LBrace;527 result.id = .LBrace;
528 self.index += 1;528 self.index += 1;
529 break;529 break;
530 },530 },
531 '}' => {531 '}' => {
532 result.id = Token.Id.RBrace;532 result.id = .RBrace;
533 self.index += 1;533 self.index += 1;
534 break;534 break;
535 },535 },
536 '~' => {536 '~' => {
537 result.id = Token.Id.Tilde;537 result.id = .Tilde;
538 self.index += 1;538 self.index += 1;
539 break;539 break;
540 },540 },
541 '.' => {541 '.' => {
542 state = State.Period;542 state = .period;
543 },543 },
544 '-' => {544 '-' => {
545 state = State.Minus;545 state = .minus;
546 },546 },
547 '/' => {547 '/' => {
548 state = State.Slash;548 state = .slash;
549 },549 },
550 '&' => {550 '&' => {
551 state = State.Ampersand;551 state = .ampersand;
552 },552 },
553 '0' => {553 '0' => {
554 state = State.Zero;554 state = .zero;
555 result.id = Token.Id.IntegerLiteral;555 result.id = .IntegerLiteral;
556 },556 },
557 '1'...'9' => {557 '1'...'9' => {
558 state = State.IntegerLiteralDec;558 state = .int_literal_dec;
559 result.id = Token.Id.IntegerLiteral;559 result.id = .IntegerLiteral;
560 },560 },
561 else => {561 else => {
562 result.id = Token.Id.Invalid;562 result.id = .Invalid;
563 self.index += 1;563 self.index += 1;
564 break;564 break;
565 },565 },
566 },566 },
567567
568 State.SawAtSign => switch (c) {568 .saw_at_sign => switch (c) {
569 '"' => {569 '"' => {
570 result.id = Token.Id.Identifier;570 result.id = .Identifier;
571 state = State.StringLiteral;571 state = .string_literal;
572 },572 },
573 else => {573 else => {
574 // reinterpret as a builtin574 // reinterpret as a builtin
575 self.index -= 1;575 self.index -= 1;
576 state = State.Builtin;576 state = .builtin;
577 result.id = Token.Id.Builtin;577 result.id = .Builtin;
578 },578 },
579 },579 },
580580
581 State.Ampersand => switch (c) {581 .ampersand => switch (c) {
582 '&' => {582 '&' => {
583 result.id = Token.Id.Invalid_ampersands;583 result.id = .Invalid_ampersands;
584 self.index += 1;584 self.index += 1;
585 break;585 break;
586 },586 },
587 '=' => {587 '=' => {
588 result.id = Token.Id.AmpersandEqual;588 result.id = .AmpersandEqual;
589 self.index += 1;589 self.index += 1;
590 break;590 break;
591 },591 },
592 else => {592 else => {
593 result.id = Token.Id.Ampersand;593 result.id = .Ampersand;
594 break;594 break;
595 },595 },
596 },596 },
597597
598 State.Asterisk => switch (c) {598 .asterisk => switch (c) {
599 '=' => {599 '=' => {
600 result.id = Token.Id.AsteriskEqual;600 result.id = .AsteriskEqual;
601 self.index += 1;601 self.index += 1;
602 break;602 break;
603 },603 },
604 '*' => {604 '*' => {
605 result.id = Token.Id.AsteriskAsterisk;605 result.id = .AsteriskAsterisk;
606 self.index += 1;606 self.index += 1;
607 break;607 break;
608 },608 },
609 '%' => {609 '%' => {
610 state = State.AsteriskPercent;610 state = .asterisk_percent;
611 },611 },
612 else => {612 else => {
613 result.id = Token.Id.Asterisk;613 result.id = .Asterisk;
614 break;614 break;
615 },615 },
616 },616 },
617617
618 State.AsteriskPercent => switch (c) {618 .asterisk_percent => switch (c) {
619 '=' => {619 '=' => {
620 result.id = Token.Id.AsteriskPercentEqual;620 result.id = .AsteriskPercentEqual;
621 self.index += 1;621 self.index += 1;
622 break;622 break;
623 },623 },
624 else => {624 else => {
625 result.id = Token.Id.AsteriskPercent;625 result.id = .AsteriskPercent;
626 break;626 break;
627 },627 },
628 },628 },
629629
630 State.Percent => switch (c) {630 .percent => switch (c) {
631 '=' => {631 '=' => {
632 result.id = Token.Id.PercentEqual;632 result.id = .PercentEqual;
633 self.index += 1;633 self.index += 1;
634 break;634 break;
635 },635 },
636 else => {636 else => {
637 result.id = Token.Id.Percent;637 result.id = .Percent;
638 break;638 break;
639 },639 },
640 },640 },
641641
642 State.Plus => switch (c) {642 .plus => switch (c) {
643 '=' => {643 '=' => {
644 result.id = Token.Id.PlusEqual;644 result.id = .PlusEqual;
645 self.index += 1;645 self.index += 1;
646 break;646 break;
647 },647 },
648 '+' => {648 '+' => {
649 result.id = Token.Id.PlusPlus;649 result.id = .PlusPlus;
650 self.index += 1;650 self.index += 1;
651 break;651 break;
652 },652 },
653 '%' => {653 '%' => {
654 state = State.PlusPercent;654 state = .plus_percent;
655 },655 },
656 else => {656 else => {
657 result.id = Token.Id.Plus;657 result.id = .Plus;
658 break;658 break;
659 },659 },
660 },660 },
661661
662 State.PlusPercent => switch (c) {662 .plus_percent => switch (c) {
663 '=' => {663 '=' => {
664 result.id = Token.Id.PlusPercentEqual;664 result.id = .PlusPercentEqual;
665 self.index += 1;665 self.index += 1;
666 break;666 break;
667 },667 },
668 else => {668 else => {
669 result.id = Token.Id.PlusPercent;669 result.id = .PlusPercent;
670 break;670 break;
671 },671 },
672 },672 },
673673
674 State.Caret => switch (c) {674 .caret => switch (c) {
675 '=' => {675 '=' => {
676 result.id = Token.Id.CaretEqual;676 result.id = .CaretEqual;
677 self.index += 1;677 self.index += 1;
678 break;678 break;
679 },679 },
680 else => {680 else => {
681 result.id = Token.Id.Caret;681 result.id = .Caret;
682 break;682 break;
683 },683 },
684 },684 },
685685
686 State.Identifier => switch (c) {686 .identifier => switch (c) {
687 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},687 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
688 else => {688 else => {
689 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {689 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
...@@ -692,19 +692,19 @@ pub const Tokenizer = struct {...@@ -692,19 +692,19 @@ pub const Tokenizer = struct {
692 break;692 break;
693 },693 },
694 },694 },
695 State.Builtin => switch (c) {695 .builtin => switch (c) {
696 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},696 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
697 else => break,697 else => break,
698 },698 },
699 State.Backslash => switch (c) {699 .backslash => switch (c) {
700 '\\' => {700 '\\' => {
701 state = State.MultilineStringLiteralLine;701 state = .multiline_string_literal_line;
702 },702 },
703 else => break,703 else => break,
704 },704 },
705 State.StringLiteral => switch (c) {705 .string_literal => switch (c) {
706 '\\' => {706 '\\' => {
707 state = State.StringLiteralBackslash;707 state = .string_literal_backslash;
708 },708 },
709 '"' => {709 '"' => {
710 self.index += 1;710 self.index += 1;
...@@ -714,98 +714,98 @@ pub const Tokenizer = struct {...@@ -714,98 +714,98 @@ pub const Tokenizer = struct {
714 else => self.checkLiteralCharacter(),714 else => self.checkLiteralCharacter(),
715 },715 },
716716
717 State.StringLiteralBackslash => switch (c) {717 .string_literal_backslash => switch (c) {
718 '\n', '\r' => break, // Look for this error later.718 '\n', '\r' => break, // Look for this error later.
719 else => {719 else => {
720 state = State.StringLiteral;720 state = .string_literal;
721 },721 },
722 },722 },
723723
724 State.CharLiteral => switch (c) {724 .char_literal => switch (c) {
725 '\\' => {725 '\\' => {
726 state = State.CharLiteralBackslash;726 state = .char_literal_backslash;
727 },727 },
728 '\'', 0x80...0xbf, 0xf8...0xff => {728 '\'', 0x80...0xbf, 0xf8...0xff => {
729 result.id = Token.Id.Invalid;729 result.id = .Invalid;
730 break;730 break;
731 },731 },
732 0xc0...0xdf => { // 110xxxxx732 0xc0...0xdf => { // 110xxxxx
733 remaining_code_units = 1;733 remaining_code_units = 1;
734 state = State.CharLiteralUnicode;734 state = .char_literal_unicode;
735 },735 },
736 0xe0...0xef => { // 1110xxxx736 0xe0...0xef => { // 1110xxxx
737 remaining_code_units = 2;737 remaining_code_units = 2;
738 state = State.CharLiteralUnicode;738 state = .char_literal_unicode;
739 },739 },
740 0xf0...0xf7 => { // 11110xxx740 0xf0...0xf7 => { // 11110xxx
741 remaining_code_units = 3;741 remaining_code_units = 3;
742 state = State.CharLiteralUnicode;742 state = .char_literal_unicode;
743 },743 },
744 else => {744 else => {
745 state = State.CharLiteralEnd;745 state = .char_literal_end;
746 },746 },
747 },747 },
748748
749 State.CharLiteralBackslash => switch (c) {749 .char_literal_backslash => switch (c) {
750 '\n' => {750 '\n' => {
751 result.id = Token.Id.Invalid;751 result.id = .Invalid;
752 break;752 break;
753 },753 },
754 'x' => {754 'x' => {
755 state = State.CharLiteralHexEscape;755 state = .char_literal_hex_escape;
756 seen_escape_digits = 0;756 seen_escape_digits = 0;
757 },757 },
758 'u' => {758 'u' => {
759 state = State.CharLiteralUnicodeEscapeSawU;759 state = .char_literal_unicode_escape_saw_u;
760 },760 },
761 else => {761 else => {
762 state = State.CharLiteralEnd;762 state = .char_literal_end;
763 },763 },
764 },764 },
765765
766 State.CharLiteralHexEscape => switch (c) {766 .char_literal_hex_escape => switch (c) {
767 '0'...'9', 'a'...'f', 'A'...'F' => {767 '0'...'9', 'a'...'f', 'A'...'F' => {
768 seen_escape_digits += 1;768 seen_escape_digits += 1;
769 if (seen_escape_digits == 2) {769 if (seen_escape_digits == 2) {
770 state = State.CharLiteralEnd;770 state = .char_literal_end;
771 }771 }
772 },772 },
773 else => {773 else => {
774 result.id = Token.Id.Invalid;774 result.id = .Invalid;
775 break;775 break;
776 },776 },
777 },777 },
778778
779 State.CharLiteralUnicodeEscapeSawU => switch (c) {779 .char_literal_unicode_escape_saw_u => switch (c) {
780 '{' => {780 '{' => {
781 state = State.CharLiteralUnicodeEscape;781 state = .char_literal_unicode_escape;
782 seen_escape_digits = 0;782 seen_escape_digits = 0;
783 },783 },
784 else => {784 else => {
785 result.id = Token.Id.Invalid;785 result.id = .Invalid;
786 state = State.CharLiteralUnicodeInvalid;786 state = .char_literal_unicode_invalid;
787 },787 },
788 },788 },
789789
790 State.CharLiteralUnicodeEscape => switch (c) {790 .char_literal_unicode_escape => switch (c) {
791 '0'...'9', 'a'...'f', 'A'...'F' => {791 '0'...'9', 'a'...'f', 'A'...'F' => {
792 seen_escape_digits += 1;792 seen_escape_digits += 1;
793 },793 },
794 '}' => {794 '}' => {
795 if (seen_escape_digits == 0) {795 if (seen_escape_digits == 0) {
796 result.id = Token.Id.Invalid;796 result.id = .Invalid;
797 state = State.CharLiteralUnicodeInvalid;797 state = .char_literal_unicode_invalid;
798 } else {798 } else {
799 state = State.CharLiteralEnd;799 state = .char_literal_end;
800 }800 }
801 },801 },
802 else => {802 else => {
803 result.id = Token.Id.Invalid;803 result.id = .Invalid;
804 state = State.CharLiteralUnicodeInvalid;804 state = .char_literal_unicode_invalid;
805 },805 },
806 },806 },
807807
808 State.CharLiteralUnicodeInvalid => switch (c) {808 .char_literal_unicode_invalid => switch (c) {
809 // Keep consuming characters until an obvious stopping point.809 // Keep consuming characters until an obvious stopping point.
810 // This consolidates e.g. `u{0ab1Q}` into a single invalid token810 // This consolidates e.g. `u{0ab1Q}` into a single invalid token
811 // instead of creating the tokens `u{0ab1`, `Q`, `}`811 // instead of creating the tokens `u{0ab1`, `Q`, `}`
...@@ -813,32 +813,32 @@ pub const Tokenizer = struct {...@@ -813,32 +813,32 @@ pub const Tokenizer = struct {
813 else => break,813 else => break,
814 },814 },
815815
816 State.CharLiteralEnd => switch (c) {816 .char_literal_end => switch (c) {
817 '\'' => {817 '\'' => {
818 result.id = Token.Id.CharLiteral;818 result.id = .CharLiteral;
819 self.index += 1;819 self.index += 1;
820 break;820 break;
821 },821 },
822 else => {822 else => {
823 result.id = Token.Id.Invalid;823 result.id = .Invalid;
824 break;824 break;
825 },825 },
826 },826 },
827827
828 State.CharLiteralUnicode => switch (c) {828 .char_literal_unicode => switch (c) {
829 0x80...0xbf => {829 0x80...0xbf => {
830 remaining_code_units -= 1;830 remaining_code_units -= 1;
831 if (remaining_code_units == 0) {831 if (remaining_code_units == 0) {
832 state = State.CharLiteralEnd;832 state = .char_literal_end;
833 }833 }
834 },834 },
835 else => {835 else => {
836 result.id = Token.Id.Invalid;836 result.id = .Invalid;
837 break;837 break;
838 },838 },
839 },839 },
840840
841 State.MultilineStringLiteralLine => switch (c) {841 .multiline_string_literal_line => switch (c) {
842 '\n' => {842 '\n' => {
843 self.index += 1;843 self.index += 1;
844 break;844 break;
...@@ -847,449 +847,449 @@ pub const Tokenizer = struct {...@@ -847,449 +847,449 @@ pub const Tokenizer = struct {
847 else => self.checkLiteralCharacter(),847 else => self.checkLiteralCharacter(),
848 },848 },
849849
850 State.Bang => switch (c) {850 .bang => switch (c) {
851 '=' => {851 '=' => {
852 result.id = Token.Id.BangEqual;852 result.id = .BangEqual;
853 self.index += 1;853 self.index += 1;
854 break;854 break;
855 },855 },
856 else => {856 else => {
857 result.id = Token.Id.Bang;857 result.id = .Bang;
858 break;858 break;
859 },859 },
860 },860 },
861861
862 State.Pipe => switch (c) {862 .pipe => switch (c) {
863 '=' => {863 '=' => {
864 result.id = Token.Id.PipeEqual;864 result.id = .PipeEqual;
865 self.index += 1;865 self.index += 1;
866 break;866 break;
867 },867 },
868 '|' => {868 '|' => {
869 result.id = Token.Id.PipePipe;869 result.id = .PipePipe;
870 self.index += 1;870 self.index += 1;
871 break;871 break;
872 },872 },
873 else => {873 else => {
874 result.id = Token.Id.Pipe;874 result.id = .Pipe;
875 break;875 break;
876 },876 },
877 },877 },
878878
879 State.Equal => switch (c) {879 .equal => switch (c) {
880 '=' => {880 '=' => {
881 result.id = Token.Id.EqualEqual;881 result.id = .EqualEqual;
882 self.index += 1;882 self.index += 1;
883 break;883 break;
884 },884 },
885 '>' => {885 '>' => {
886 result.id = Token.Id.EqualAngleBracketRight;886 result.id = .EqualAngleBracketRight;
887 self.index += 1;887 self.index += 1;
888 break;888 break;
889 },889 },
890 else => {890 else => {
891 result.id = Token.Id.Equal;891 result.id = .Equal;
892 break;892 break;
893 },893 },
894 },894 },
895895
896 State.Minus => switch (c) {896 .minus => switch (c) {
897 '>' => {897 '>' => {
898 result.id = Token.Id.Arrow;898 result.id = .Arrow;
899 self.index += 1;899 self.index += 1;
900 break;900 break;
901 },901 },
902 '=' => {902 '=' => {
903 result.id = Token.Id.MinusEqual;903 result.id = .MinusEqual;
904 self.index += 1;904 self.index += 1;
905 break;905 break;
906 },906 },
907 '%' => {907 '%' => {
908 state = State.MinusPercent;908 state = .minus_percent;
909 },909 },
910 else => {910 else => {
911 result.id = Token.Id.Minus;911 result.id = .Minus;
912 break;912 break;
913 },913 },
914 },914 },
915915
916 State.MinusPercent => switch (c) {916 .minus_percent => switch (c) {
917 '=' => {917 '=' => {
918 result.id = Token.Id.MinusPercentEqual;918 result.id = .MinusPercentEqual;
919 self.index += 1;919 self.index += 1;
920 break;920 break;
921 },921 },
922 else => {922 else => {
923 result.id = Token.Id.MinusPercent;923 result.id = .MinusPercent;
924 break;924 break;
925 },925 },
926 },926 },
927927
928 State.AngleBracketLeft => switch (c) {928 .angle_bracket_left => switch (c) {
929 '<' => {929 '<' => {
930 state = State.AngleBracketAngleBracketLeft;930 state = .angle_bracket_angle_bracket_left;
931 },931 },
932 '=' => {932 '=' => {
933 result.id = Token.Id.AngleBracketLeftEqual;933 result.id = .AngleBracketLeftEqual;
934 self.index += 1;934 self.index += 1;
935 break;935 break;
936 },936 },
937 else => {937 else => {
938 result.id = Token.Id.AngleBracketLeft;938 result.id = .AngleBracketLeft;
939 break;939 break;
940 },940 },
941 },941 },
942942
943 State.AngleBracketAngleBracketLeft => switch (c) {943 .angle_bracket_angle_bracket_left => switch (c) {
944 '=' => {944 '=' => {
945 result.id = Token.Id.AngleBracketAngleBracketLeftEqual;945 result.id = .AngleBracketAngleBracketLeftEqual;
946 self.index += 1;946 self.index += 1;
947 break;947 break;
948 },948 },
949 else => {949 else => {
950 result.id = Token.Id.AngleBracketAngleBracketLeft;950 result.id = .AngleBracketAngleBracketLeft;
951 break;951 break;
952 },952 },
953 },953 },
954954
955 State.AngleBracketRight => switch (c) {955 .angle_bracket_right => switch (c) {
956 '>' => {956 '>' => {
957 state = State.AngleBracketAngleBracketRight;957 state = .angle_bracket_angle_bracket_right;
958 },958 },
959 '=' => {959 '=' => {
960 result.id = Token.Id.AngleBracketRightEqual;960 result.id = .AngleBracketRightEqual;
961 self.index += 1;961 self.index += 1;
962 break;962 break;
963 },963 },
964 else => {964 else => {
965 result.id = Token.Id.AngleBracketRight;965 result.id = .AngleBracketRight;
966 break;966 break;
967 },967 },
968 },968 },
969969
970 State.AngleBracketAngleBracketRight => switch (c) {970 .angle_bracket_angle_bracket_right => switch (c) {
971 '=' => {971 '=' => {
972 result.id = Token.Id.AngleBracketAngleBracketRightEqual;972 result.id = .AngleBracketAngleBracketRightEqual;
973 self.index += 1;973 self.index += 1;
974 break;974 break;
975 },975 },
976 else => {976 else => {
977 result.id = Token.Id.AngleBracketAngleBracketRight;977 result.id = .AngleBracketAngleBracketRight;
978 break;978 break;
979 },979 },
980 },980 },
981981
982 State.Period => switch (c) {982 .period => switch (c) {
983 '.' => {983 '.' => {
984 state = State.Period2;984 state = .period_2;
985 },985 },
986 '*' => {986 '*' => {
987 result.id = Token.Id.PeriodAsterisk;987 result.id = .PeriodAsterisk;
988 self.index += 1;988 self.index += 1;
989 break;989 break;
990 },990 },
991 else => {991 else => {
992 result.id = Token.Id.Period;992 result.id = .Period;
993 break;993 break;
994 },994 },
995 },995 },
996996
997 State.Period2 => switch (c) {997 .period_2 => switch (c) {
998 '.' => {998 '.' => {
999 result.id = Token.Id.Ellipsis3;999 result.id = .Ellipsis3;
1000 self.index += 1;1000 self.index += 1;
1001 break;1001 break;
1002 },1002 },
1003 else => {1003 else => {
1004 result.id = Token.Id.Ellipsis2;1004 result.id = .Ellipsis2;
1005 break;1005 break;
1006 },1006 },
1007 },1007 },
10081008
1009 State.Slash => switch (c) {1009 .slash => switch (c) {
1010 '/' => {1010 '/' => {
1011 state = State.LineCommentStart;1011 state = .line_comment_start;
1012 result.id = Token.Id.LineComment;1012 result.id = .LineComment;
1013 },1013 },
1014 '=' => {1014 '=' => {
1015 result.id = Token.Id.SlashEqual;1015 result.id = .SlashEqual;
1016 self.index += 1;1016 self.index += 1;
1017 break;1017 break;
1018 },1018 },
1019 else => {1019 else => {
1020 result.id = Token.Id.Slash;1020 result.id = .Slash;
1021 break;1021 break;
1022 },1022 },
1023 },1023 },
1024 State.LineCommentStart => switch (c) {1024 .line_comment_start => switch (c) {
1025 '/' => {1025 '/' => {
1026 state = State.DocCommentStart;1026 state = .doc_comment_start;
1027 },1027 },
1028 '!' => {1028 '!' => {
1029 result.id = Token.Id.ContainerDocComment;1029 result.id = .ContainerDocComment;
1030 state = State.ContainerDocComment;1030 state = .container_doc_comment;
1031 },1031 },
1032 '\n' => break,1032 '\n' => break,
1033 else => {1033 else => {
1034 state = State.LineComment;1034 state = .line_comment;
1035 self.checkLiteralCharacter();1035 self.checkLiteralCharacter();
1036 },1036 },
1037 },1037 },
1038 State.DocCommentStart => switch (c) {1038 .doc_comment_start => switch (c) {
1039 '/' => {1039 '/' => {
1040 state = State.LineComment;1040 state = .line_comment;
1041 },1041 },
1042 '\n' => {1042 '\n' => {
1043 result.id = Token.Id.DocComment;1043 result.id = .DocComment;
1044 break;1044 break;
1045 },1045 },
1046 else => {1046 else => {
1047 state = State.DocComment;1047 state = .doc_comment;
1048 result.id = Token.Id.DocComment;1048 result.id = .DocComment;
1049 self.checkLiteralCharacter();1049 self.checkLiteralCharacter();
1050 },1050 },
1051 },1051 },
1052 State.LineComment, State.DocComment, State.ContainerDocComment => switch (c) {1052 .line_comment, .doc_comment, .container_doc_comment => switch (c) {
1053 '\n' => break,1053 '\n' => break,
1054 else => self.checkLiteralCharacter(),1054 else => self.checkLiteralCharacter(),
1055 },1055 },
1056 State.Zero => switch (c) {1056 .zero => switch (c) {
1057 'b' => {1057 'b' => {
1058 state = State.IntegerLiteralBinNoUnderscore;1058 state = .int_literal_bin_no_underscore;
1059 },1059 },
1060 'o' => {1060 'o' => {
1061 state = State.IntegerLiteralOctNoUnderscore;1061 state = .int_literal_oct_no_underscore;
1062 },1062 },
1063 'x' => {1063 'x' => {
1064 state = State.IntegerLiteralHexNoUnderscore;1064 state = .int_literal_hex_no_underscore;
1065 },1065 },
1066 '0'...'9', '_', '.', 'e', 'E' => {1066 '0'...'9', '_', '.', 'e', 'E' => {
1067 // reinterpret as a decimal number1067 // reinterpret as a decimal number
1068 self.index -= 1;1068 self.index -= 1;
1069 state = State.IntegerLiteralDec;1069 state = .int_literal_dec;
1070 },1070 },
1071 else => {1071 else => {
1072 if (isIdentifierChar(c)) {1072 if (isIdentifierChar(c)) {
1073 result.id = Token.Id.Invalid;1073 result.id = .Invalid;
1074 }1074 }
1075 break;1075 break;
1076 },1076 },
1077 },1077 },
1078 State.IntegerLiteralBinNoUnderscore => switch (c) {1078 .int_literal_bin_no_underscore => switch (c) {
1079 '0'...'1' => {1079 '0'...'1' => {
1080 state = State.IntegerLiteralBin;1080 state = .int_literal_bin;
1081 },1081 },
1082 else => {1082 else => {
1083 result.id = Token.Id.Invalid;1083 result.id = .Invalid;
1084 break;1084 break;
1085 },1085 },
1086 },1086 },
1087 State.IntegerLiteralBin => switch (c) {1087 .int_literal_bin => switch (c) {
1088 '_' => {1088 '_' => {
1089 state = State.IntegerLiteralBinNoUnderscore;1089 state = .int_literal_bin_no_underscore;
1090 },1090 },
1091 '0'...'1' => {},1091 '0'...'1' => {},
1092 else => {1092 else => {
1093 if (isIdentifierChar(c)) {1093 if (isIdentifierChar(c)) {
1094 result.id = Token.Id.Invalid;1094 result.id = .Invalid;
1095 }1095 }
1096 break;1096 break;
1097 },1097 },
1098 },1098 },
1099 State.IntegerLiteralOctNoUnderscore => switch (c) {1099 .int_literal_oct_no_underscore => switch (c) {
1100 '0'...'7' => {1100 '0'...'7' => {
1101 state = State.IntegerLiteralOct;1101 state = .int_literal_oct;
1102 },1102 },
1103 else => {1103 else => {
1104 result.id = Token.Id.Invalid;1104 result.id = .Invalid;
1105 break;1105 break;
1106 },1106 },
1107 },1107 },
1108 State.IntegerLiteralOct => switch (c) {1108 .int_literal_oct => switch (c) {
1109 '_' => {1109 '_' => {
1110 state = State.IntegerLiteralOctNoUnderscore;1110 state = .int_literal_oct_no_underscore;
1111 },1111 },
1112 '0'...'7' => {},1112 '0'...'7' => {},
1113 else => {1113 else => {
1114 if (isIdentifierChar(c)) {1114 if (isIdentifierChar(c)) {
1115 result.id = Token.Id.Invalid;1115 result.id = .Invalid;
1116 }1116 }
1117 break;1117 break;
1118 },1118 },
1119 },1119 },
1120 State.IntegerLiteralDecNoUnderscore => switch (c) {1120 .int_literal_dec_no_underscore => switch (c) {
1121 '0'...'9' => {1121 '0'...'9' => {
1122 state = State.IntegerLiteralDec;1122 state = .int_literal_dec;
1123 },1123 },
1124 else => {1124 else => {
1125 result.id = Token.Id.Invalid;1125 result.id = .Invalid;
1126 break;1126 break;
1127 },1127 },
1128 },1128 },
1129 State.IntegerLiteralDec => switch (c) {1129 .int_literal_dec => switch (c) {
1130 '_' => {1130 '_' => {
1131 state = State.IntegerLiteralDecNoUnderscore;1131 state = .int_literal_dec_no_underscore;
1132 },1132 },
1133 '.' => {1133 '.' => {
1134 state = State.NumberDotDec;1134 state = .num_dot_dec;
1135 result.id = Token.Id.FloatLiteral;1135 result.id = .FloatLiteral;
1136 },1136 },
1137 'e', 'E' => {1137 'e', 'E' => {
1138 state = State.FloatExponentUnsigned;1138 state = .float_exponent_unsigned;
1139 result.id = Token.Id.FloatLiteral;1139 result.id = .FloatLiteral;
1140 },1140 },
1141 '0'...'9' => {},1141 '0'...'9' => {},
1142 else => {1142 else => {
1143 if (isIdentifierChar(c)) {1143 if (isIdentifierChar(c)) {
1144 result.id = Token.Id.Invalid;1144 result.id = .Invalid;
1145 }1145 }
1146 break;1146 break;
1147 },1147 },
1148 },1148 },
1149 State.IntegerLiteralHexNoUnderscore => switch (c) {1149 .int_literal_hex_no_underscore => switch (c) {
1150 '0'...'9', 'a'...'f', 'A'...'F' => {1150 '0'...'9', 'a'...'f', 'A'...'F' => {
1151 state = State.IntegerLiteralHex;1151 state = .int_literal_hex;
1152 },1152 },
1153 else => {1153 else => {
1154 result.id = Token.Id.Invalid;1154 result.id = .Invalid;
1155 break;1155 break;
1156 },1156 },
1157 },1157 },
1158 State.IntegerLiteralHex => switch (c) {1158 .int_literal_hex => switch (c) {
1159 '_' => {1159 '_' => {
1160 state = State.IntegerLiteralHexNoUnderscore;1160 state = .int_literal_hex_no_underscore;
1161 },1161 },
1162 '.' => {1162 '.' => {
1163 state = State.NumberDotHex;1163 state = .num_dot_hex;
1164 result.id = Token.Id.FloatLiteral;1164 result.id = .FloatLiteral;
1165 },1165 },
1166 'p', 'P' => {1166 'p', 'P' => {
1167 state = State.FloatExponentUnsigned;1167 state = .float_exponent_unsigned;
1168 result.id = Token.Id.FloatLiteral;1168 result.id = .FloatLiteral;
1169 },1169 },
1170 '0'...'9', 'a'...'f', 'A'...'F' => {},1170 '0'...'9', 'a'...'f', 'A'...'F' => {},
1171 else => {1171 else => {
1172 if (isIdentifierChar(c)) {1172 if (isIdentifierChar(c)) {
1173 result.id = Token.Id.Invalid;1173 result.id = .Invalid;
1174 }1174 }
1175 break;1175 break;
1176 },1176 },
1177 },1177 },
1178 State.NumberDotDec => switch (c) {1178 .num_dot_dec => switch (c) {
1179 '.' => {1179 '.' => {
1180 self.index -= 1;1180 self.index -= 1;
1181 state = State.Start;1181 state = .start;
1182 break;1182 break;
1183 },1183 },
1184 'e', 'E' => {1184 'e', 'E' => {
1185 state = State.FloatExponentUnsigned;1185 state = .float_exponent_unsigned;
1186 },1186 },
1187 '0'...'9' => {1187 '0'...'9' => {
1188 result.id = Token.Id.FloatLiteral;1188 result.id = .FloatLiteral;
1189 state = State.FloatFractionDec;1189 state = .float_fraction_dec;
1190 },1190 },
1191 else => {1191 else => {
1192 if (isIdentifierChar(c)) {1192 if (isIdentifierChar(c)) {
1193 result.id = Token.Id.Invalid;1193 result.id = .Invalid;
1194 }1194 }
1195 break;1195 break;
1196 },1196 },
1197 },1197 },
1198 State.NumberDotHex => switch (c) {1198 .num_dot_hex => switch (c) {
1199 '.' => {1199 '.' => {
1200 self.index -= 1;1200 self.index -= 1;
1201 state = State.Start;1201 state = .start;
1202 break;1202 break;
1203 },1203 },
1204 'p', 'P' => {1204 'p', 'P' => {
1205 state = State.FloatExponentUnsigned;1205 state = .float_exponent_unsigned;
1206 },1206 },
1207 '0'...'9', 'a'...'f', 'A'...'F' => {1207 '0'...'9', 'a'...'f', 'A'...'F' => {
1208 result.id = Token.Id.FloatLiteral;1208 result.id = .FloatLiteral;
1209 state = State.FloatFractionHex;1209 state = .float_fraction_hex;
1210 },1210 },
1211 else => {1211 else => {
1212 if (isIdentifierChar(c)) {1212 if (isIdentifierChar(c)) {
1213 result.id = Token.Id.Invalid;1213 result.id = .Invalid;
1214 }1214 }
1215 break;1215 break;
1216 },1216 },
1217 },1217 },
1218 State.FloatFractionDecNoUnderscore => switch (c) {1218 .float_fraction_dec_no_underscore => switch (c) {
1219 '0'...'9' => {1219 '0'...'9' => {
1220 state = State.FloatFractionDec;1220 state = .float_fraction_dec;
1221 },1221 },
1222 else => {1222 else => {
1223 result.id = Token.Id.Invalid;1223 result.id = .Invalid;
1224 break;1224 break;
1225 },1225 },
1226 },1226 },
1227 State.FloatFractionDec => switch (c) {1227 .float_fraction_dec => switch (c) {
1228 '_' => {1228 '_' => {
1229 state = State.FloatFractionDecNoUnderscore;1229 state = .float_fraction_dec_no_underscore;
1230 },1230 },
1231 'e', 'E' => {1231 'e', 'E' => {
1232 state = State.FloatExponentUnsigned;1232 state = .float_exponent_unsigned;
1233 },1233 },
1234 '0'...'9' => {},1234 '0'...'9' => {},
1235 else => {1235 else => {
1236 if (isIdentifierChar(c)) {1236 if (isIdentifierChar(c)) {
1237 result.id = Token.Id.Invalid;1237 result.id = .Invalid;
1238 }1238 }
1239 break;1239 break;
1240 },1240 },
1241 },1241 },
1242 State.FloatFractionHexNoUnderscore => switch (c) {1242 .float_fraction_hex_no_underscore => switch (c) {
1243 '0'...'9', 'a'...'f', 'A'...'F' => {1243 '0'...'9', 'a'...'f', 'A'...'F' => {
1244 state = State.FloatFractionHex;1244 state = .float_fraction_hex;
1245 },1245 },
1246 else => {1246 else => {
1247 result.id = Token.Id.Invalid;1247 result.id = .Invalid;
1248 break;1248 break;
1249 },1249 },
1250 },1250 },
1251 State.FloatFractionHex => switch (c) {1251 .float_fraction_hex => switch (c) {
1252 '_' => {1252 '_' => {
1253 state = State.FloatFractionHexNoUnderscore;1253 state = .float_fraction_hex_no_underscore;
1254 },1254 },
1255 'p', 'P' => {1255 'p', 'P' => {
1256 state = State.FloatExponentUnsigned;1256 state = .float_exponent_unsigned;
1257 },1257 },
1258 '0'...'9', 'a'...'f', 'A'...'F' => {},1258 '0'...'9', 'a'...'f', 'A'...'F' => {},
1259 else => {1259 else => {
1260 if (isIdentifierChar(c)) {1260 if (isIdentifierChar(c)) {
1261 result.id = Token.Id.Invalid;1261 result.id = .Invalid;
1262 }1262 }
1263 break;1263 break;
1264 },1264 },
1265 },1265 },
1266 State.FloatExponentUnsigned => switch (c) {1266 .float_exponent_unsigned => switch (c) {
1267 '+', '-' => {1267 '+', '-' => {
1268 state = State.FloatExponentNumberNoUnderscore;1268 state = .float_exponent_num_no_underscore;
1269 },1269 },
1270 else => {1270 else => {
1271 // reinterpret as a normal exponent number1271 // reinterpret as a normal exponent number
1272 self.index -= 1;1272 self.index -= 1;
1273 state = State.FloatExponentNumberNoUnderscore;1273 state = .float_exponent_num_no_underscore;
1274 },1274 },
1275 },1275 },
1276 State.FloatExponentNumberNoUnderscore => switch (c) {1276 .float_exponent_num_no_underscore => switch (c) {
1277 '0'...'9' => {1277 '0'...'9' => {
1278 state = State.FloatExponentNumber;1278 state = .float_exponent_num;
1279 },1279 },
1280 else => {1280 else => {
1281 result.id = Token.Id.Invalid;1281 result.id = .Invalid;
1282 break;1282 break;
1283 },1283 },
1284 },1284 },
1285 State.FloatExponentNumber => switch (c) {1285 .float_exponent_num => switch (c) {
1286 '_' => {1286 '_' => {
1287 state = State.FloatExponentNumberNoUnderscore;1287 state = .float_exponent_num_no_underscore;
1288 },1288 },
1289 '0'...'9' => {},1289 '0'...'9' => {},
1290 else => {1290 else => {
1291 if (isIdentifierChar(c)) {1291 if (isIdentifierChar(c)) {
1292 result.id = Token.Id.Invalid;1292 result.id = .Invalid;
1293 }1293 }
1294 break;1294 break;
1295 },1295 },
...@@ -1297,123 +1297,123 @@ pub const Tokenizer = struct {...@@ -1297,123 +1297,123 @@ pub const Tokenizer = struct {
1297 }1297 }
1298 } else if (self.index == self.buffer.len) {1298 } else if (self.index == self.buffer.len) {
1299 switch (state) {1299 switch (state) {
1300 State.Start,1300 .start,
1301 State.IntegerLiteralDec,1301 .int_literal_dec,
1302 State.IntegerLiteralBin,1302 .int_literal_bin,
1303 State.IntegerLiteralOct,1303 .int_literal_oct,
1304 State.IntegerLiteralHex,1304 .int_literal_hex,
1305 State.NumberDotDec,1305 .num_dot_dec,
1306 State.NumberDotHex,1306 .num_dot_hex,
1307 State.FloatFractionDec,1307 .float_fraction_dec,
1308 State.FloatFractionHex,1308 .float_fraction_hex,
1309 State.FloatExponentNumber,1309 .float_exponent_num,
1310 State.StringLiteral, // find this error later1310 .string_literal, // find this error later
1311 State.MultilineStringLiteralLine,1311 .multiline_string_literal_line,
1312 State.Builtin,1312 .builtin,
1313 => {},1313 => {},
13141314
1315 State.Identifier => {1315 .identifier => {
1316 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {1316 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
1317 result.id = id;1317 result.id = id;
1318 }1318 }
1319 },1319 },
1320 State.LineCommentStart, State.LineComment => {1320 .line_comment, .line_comment_start => {
1321 result.id = Token.Id.LineComment;1321 result.id = .LineComment;
1322 },1322 },
1323 State.DocComment, State.DocCommentStart => {1323 .doc_comment, .doc_comment_start => {
1324 result.id = Token.Id.DocComment;1324 result.id = .DocComment;
1325 },1325 },
1326 State.ContainerDocComment => {1326 .container_doc_comment => {
1327 result.id = Token.Id.ContainerDocComment;1327 result.id = .ContainerDocComment;
1328 },1328 },
13291329
1330 State.IntegerLiteralDecNoUnderscore,1330 .int_literal_dec_no_underscore,
1331 State.IntegerLiteralBinNoUnderscore,1331 .int_literal_bin_no_underscore,
1332 State.IntegerLiteralOctNoUnderscore,1332 .int_literal_oct_no_underscore,
1333 State.IntegerLiteralHexNoUnderscore,1333 .int_literal_hex_no_underscore,
1334 State.FloatFractionDecNoUnderscore,1334 .float_fraction_dec_no_underscore,
1335 State.FloatFractionHexNoUnderscore,1335 .float_fraction_hex_no_underscore,
1336 State.FloatExponentNumberNoUnderscore,1336 .float_exponent_num_no_underscore,
1337 State.FloatExponentUnsigned,1337 .float_exponent_unsigned,
1338 State.SawAtSign,1338 .saw_at_sign,
1339 State.Backslash,1339 .backslash,
1340 State.CharLiteral,1340 .char_literal,
1341 State.CharLiteralBackslash,1341 .char_literal_backslash,
1342 State.CharLiteralHexEscape,1342 .char_literal_hex_escape,
1343 State.CharLiteralUnicodeEscapeSawU,1343 .char_literal_unicode_escape_saw_u,
1344 State.CharLiteralUnicodeEscape,1344 .char_literal_unicode_escape,
1345 State.CharLiteralUnicodeInvalid,1345 .char_literal_unicode_invalid,
1346 State.CharLiteralEnd,1346 .char_literal_end,
1347 State.CharLiteralUnicode,1347 .char_literal_unicode,
1348 State.StringLiteralBackslash,1348 .string_literal_backslash,
1349 => {1349 => {
1350 result.id = Token.Id.Invalid;1350 result.id = .Invalid;
1351 },1351 },
13521352
1353 State.Equal => {1353 .equal => {
1354 result.id = Token.Id.Equal;1354 result.id = .Equal;
1355 },1355 },
1356 State.Bang => {1356 .bang => {
1357 result.id = Token.Id.Bang;1357 result.id = .Bang;
1358 },1358 },
1359 State.Minus => {1359 .minus => {
1360 result.id = Token.Id.Minus;1360 result.id = .Minus;
1361 },1361 },
1362 State.Slash => {1362 .slash => {
1363 result.id = Token.Id.Slash;1363 result.id = .Slash;
1364 },1364 },
1365 State.Zero => {1365 .zero => {
1366 result.id = Token.Id.IntegerLiteral;1366 result.id = .IntegerLiteral;
1367 },1367 },
1368 State.Ampersand => {1368 .ampersand => {
1369 result.id = Token.Id.Ampersand;1369 result.id = .Ampersand;
1370 },1370 },
1371 State.Period => {1371 .period => {
1372 result.id = Token.Id.Period;1372 result.id = .Period;
1373 },1373 },
1374 State.Period2 => {1374 .period_2 => {
1375 result.id = Token.Id.Ellipsis2;1375 result.id = .Ellipsis2;
1376 },1376 },
1377 State.Pipe => {1377 .pipe => {
1378 result.id = Token.Id.Pipe;1378 result.id = .Pipe;
1379 },1379 },
1380 State.AngleBracketAngleBracketRight => {1380 .angle_bracket_angle_bracket_right => {
1381 result.id = Token.Id.AngleBracketAngleBracketRight;1381 result.id = .AngleBracketAngleBracketRight;
1382 },1382 },
1383 State.AngleBracketRight => {1383 .angle_bracket_right => {
1384 result.id = Token.Id.AngleBracketRight;1384 result.id = .AngleBracketRight;
1385 },1385 },
1386 State.AngleBracketAngleBracketLeft => {1386 .angle_bracket_angle_bracket_left => {
1387 result.id = Token.Id.AngleBracketAngleBracketLeft;1387 result.id = .AngleBracketAngleBracketLeft;
1388 },1388 },
1389 State.AngleBracketLeft => {1389 .angle_bracket_left => {
1390 result.id = Token.Id.AngleBracketLeft;1390 result.id = .AngleBracketLeft;
1391 },1391 },
1392 State.PlusPercent => {1392 .plus_percent => {
1393 result.id = Token.Id.PlusPercent;1393 result.id = .PlusPercent;
1394 },1394 },
1395 State.Plus => {1395 .plus => {
1396 result.id = Token.Id.Plus;1396 result.id = .Plus;
1397 },1397 },
1398 State.Percent => {1398 .percent => {
1399 result.id = Token.Id.Percent;1399 result.id = .Percent;
1400 },1400 },
1401 State.Caret => {1401 .caret => {
1402 result.id = Token.Id.Caret;1402 result.id = .Caret;
1403 },1403 },
1404 State.AsteriskPercent => {1404 .asterisk_percent => {
1405 result.id = Token.Id.AsteriskPercent;1405 result.id = .AsteriskPercent;
1406 },1406 },
1407 State.Asterisk => {1407 .asterisk => {
1408 result.id = Token.Id.Asterisk;1408 result.id = .Asterisk;
1409 },1409 },
1410 State.MinusPercent => {1410 .minus_percent => {
1411 result.id = Token.Id.MinusPercent;1411 result.id = .MinusPercent;
1412 },1412 },
1413 }1413 }
1414 }1414 }
14151415
1416 if (result.id == Token.Id.Eof) {1416 if (result.id == .Eof) {
1417 if (self.pending_invalid_token) |token| {1417 if (self.pending_invalid_token) |token| {
1418 self.pending_invalid_token = null;1418 self.pending_invalid_token = null;
1419 return token;1419 return token;
...@@ -1428,8 +1428,8 @@ pub const Tokenizer = struct {...@@ -1428,8 +1428,8 @@ pub const Tokenizer = struct {
1428 if (self.pending_invalid_token != null) return;1428 if (self.pending_invalid_token != null) return;
1429 const invalid_length = self.getInvalidCharacterLength();1429 const invalid_length = self.getInvalidCharacterLength();
1430 if (invalid_length == 0) return;1430 if (invalid_length == 0) return;
1431 self.pending_invalid_token = Token{1431 self.pending_invalid_token = .{
1432 .id = Token.Id.Invalid,1432 .id = .Invalid,
1433 .start = self.index,1433 .start = self.index,
1434 .end = self.index + invalid_length,1434 .end = self.index + invalid_length,
1435 };1435 };
...@@ -1474,7 +1474,7 @@ pub const Tokenizer = struct {...@@ -1474,7 +1474,7 @@ pub const Tokenizer = struct {
1474};1474};
14751475
1476test "tokenizer" {1476test "tokenizer" {
1477 testTokenize("test", &[_]Token.Id{Token.Id.Keyword_test});1477 testTokenize("test", &[_]Token.Id{.Keyword_test});
1478}1478}
14791479
1480test "tokenizer - unknown length pointer and then c pointer" {1480test "tokenizer - unknown length pointer and then c pointer" {
...@@ -1482,15 +1482,15 @@ test "tokenizer - unknown length pointer and then c pointer" {...@@ -1482,15 +1482,15 @@ test "tokenizer - unknown length pointer and then c pointer" {
1482 \\[*]u81482 \\[*]u8
1483 \\[*c]u81483 \\[*c]u8
1484 , &[_]Token.Id{1484 , &[_]Token.Id{
1485 Token.Id.LBracket,1485 .LBracket,
1486 Token.Id.Asterisk,1486 .Asterisk,
1487 Token.Id.RBracket,1487 .RBracket,
1488 Token.Id.Identifier,1488 .Identifier,
1489 Token.Id.LBracket,1489 .LBracket,
1490 Token.Id.Asterisk,1490 .Asterisk,
1491 Token.Id.Identifier,1491 .Identifier,
1492 Token.Id.RBracket,1492 .RBracket,
1493 Token.Id.Identifier,1493 .Identifier,
1494 });1494 });
1495}1495}
14961496
...@@ -1561,125 +1561,125 @@ test "tokenizer - char literal with unicode code point" {...@@ -1561,125 +1561,125 @@ test "tokenizer - char literal with unicode code point" {
15611561
1562test "tokenizer - float literal e exponent" {1562test "tokenizer - float literal e exponent" {
1563 testTokenize("a = 4.94065645841246544177e-324;\n", &[_]Token.Id{1563 testTokenize("a = 4.94065645841246544177e-324;\n", &[_]Token.Id{
1564 Token.Id.Identifier,1564 .Identifier,
1565 Token.Id.Equal,1565 .Equal,
1566 Token.Id.FloatLiteral,1566 .FloatLiteral,
1567 Token.Id.Semicolon,1567 .Semicolon,
1568 });1568 });
1569}1569}
15701570
1571test "tokenizer - float literal p exponent" {1571test "tokenizer - float literal p exponent" {
1572 testTokenize("a = 0x1.a827999fcef32p+1022;\n", &[_]Token.Id{1572 testTokenize("a = 0x1.a827999fcef32p+1022;\n", &[_]Token.Id{
1573 Token.Id.Identifier,1573 .Identifier,
1574 Token.Id.Equal,1574 .Equal,
1575 Token.Id.FloatLiteral,1575 .FloatLiteral,
1576 Token.Id.Semicolon,1576 .Semicolon,
1577 });1577 });
1578}1578}
15791579
1580test "tokenizer - chars" {1580test "tokenizer - chars" {
1581 testTokenize("'c'", &[_]Token.Id{Token.Id.CharLiteral});1581 testTokenize("'c'", &[_]Token.Id{.CharLiteral});
1582}1582}
15831583
1584test "tokenizer - invalid token characters" {1584test "tokenizer - invalid token characters" {
1585 testTokenize("#", &[_]Token.Id{Token.Id.Invalid});1585 testTokenize("#", &[_]Token.Id{.Invalid});
1586 testTokenize("`", &[_]Token.Id{Token.Id.Invalid});1586 testTokenize("`", &[_]Token.Id{.Invalid});
1587 testTokenize("'c", &[_]Token.Id{Token.Id.Invalid});1587 testTokenize("'c", &[_]Token.Id{.Invalid});
1588 testTokenize("'", &[_]Token.Id{Token.Id.Invalid});1588 testTokenize("'", &[_]Token.Id{.Invalid});
1589 testTokenize("''", &[_]Token.Id{ Token.Id.Invalid, Token.Id.Invalid });1589 testTokenize("''", &[_]Token.Id{ .Invalid, .Invalid });
1590}1590}
15911591
1592test "tokenizer - invalid literal/comment characters" {1592test "tokenizer - invalid literal/comment characters" {
1593 testTokenize("\"\x00\"", &[_]Token.Id{1593 testTokenize("\"\x00\"", &[_]Token.Id{
1594 Token.Id.StringLiteral,1594 .StringLiteral,
1595 Token.Id.Invalid,1595 .Invalid,
1596 });1596 });
1597 testTokenize("//\x00", &[_]Token.Id{1597 testTokenize("//\x00", &[_]Token.Id{
1598 Token.Id.LineComment,1598 .LineComment,
1599 Token.Id.Invalid,1599 .Invalid,
1600 });1600 });
1601 testTokenize("//\x1f", &[_]Token.Id{1601 testTokenize("//\x1f", &[_]Token.Id{
1602 Token.Id.LineComment,1602 .LineComment,
1603 Token.Id.Invalid,1603 .Invalid,
1604 });1604 });
1605 testTokenize("//\x7f", &[_]Token.Id{1605 testTokenize("//\x7f", &[_]Token.Id{
1606 Token.Id.LineComment,1606 .LineComment,
1607 Token.Id.Invalid,1607 .Invalid,
1608 });1608 });
1609}1609}
16101610
1611test "tokenizer - utf8" {1611test "tokenizer - utf8" {
1612 testTokenize("//\xc2\x80", &[_]Token.Id{Token.Id.LineComment});1612 testTokenize("//\xc2\x80", &[_]Token.Id{.LineComment});
1613 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{Token.Id.LineComment});1613 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{.LineComment});
1614}1614}
16151615
1616test "tokenizer - invalid utf8" {1616test "tokenizer - invalid utf8" {
1617 testTokenize("//\x80", &[_]Token.Id{1617 testTokenize("//\x80", &[_]Token.Id{
1618 Token.Id.LineComment,1618 .LineComment,
1619 Token.Id.Invalid,1619 .Invalid,
1620 });1620 });
1621 testTokenize("//\xbf", &[_]Token.Id{1621 testTokenize("//\xbf", &[_]Token.Id{
1622 Token.Id.LineComment,1622 .LineComment,
1623 Token.Id.Invalid,1623 .Invalid,
1624 });1624 });
1625 testTokenize("//\xf8", &[_]Token.Id{1625 testTokenize("//\xf8", &[_]Token.Id{
1626 Token.Id.LineComment,1626 .LineComment,
1627 Token.Id.Invalid,1627 .Invalid,
1628 });1628 });
1629 testTokenize("//\xff", &[_]Token.Id{1629 testTokenize("//\xff", &[_]Token.Id{
1630 Token.Id.LineComment,1630 .LineComment,
1631 Token.Id.Invalid,1631 .Invalid,
1632 });1632 });
1633 testTokenize("//\xc2\xc0", &[_]Token.Id{1633 testTokenize("//\xc2\xc0", &[_]Token.Id{
1634 Token.Id.LineComment,1634 .LineComment,
1635 Token.Id.Invalid,1635 .Invalid,
1636 });1636 });
1637 testTokenize("//\xe0", &[_]Token.Id{1637 testTokenize("//\xe0", &[_]Token.Id{
1638 Token.Id.LineComment,1638 .LineComment,
1639 Token.Id.Invalid,1639 .Invalid,
1640 });1640 });
1641 testTokenize("//\xf0", &[_]Token.Id{1641 testTokenize("//\xf0", &[_]Token.Id{
1642 Token.Id.LineComment,1642 .LineComment,
1643 Token.Id.Invalid,1643 .Invalid,
1644 });1644 });
1645 testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Id{1645 testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Id{
1646 Token.Id.LineComment,1646 .LineComment,
1647 Token.Id.Invalid,1647 .Invalid,
1648 });1648 });
1649}1649}
16501650
1651test "tokenizer - illegal unicode codepoints" {1651test "tokenizer - illegal unicode codepoints" {
1652 // unicode newline characters.U+0085, U+2028, U+20291652 // unicode newline characters.U+0085, U+2028, U+2029
1653 testTokenize("//\xc2\x84", &[_]Token.Id{Token.Id.LineComment});1653 testTokenize("//\xc2\x84", &[_]Token.Id{.LineComment});
1654 testTokenize("//\xc2\x85", &[_]Token.Id{1654 testTokenize("//\xc2\x85", &[_]Token.Id{
1655 Token.Id.LineComment,1655 .LineComment,
1656 Token.Id.Invalid,1656 .Invalid,
1657 });1657 });
1658 testTokenize("//\xc2\x86", &[_]Token.Id{Token.Id.LineComment});1658 testTokenize("//\xc2\x86", &[_]Token.Id{.LineComment});
1659 testTokenize("//\xe2\x80\xa7", &[_]Token.Id{Token.Id.LineComment});1659 testTokenize("//\xe2\x80\xa7", &[_]Token.Id{.LineComment});
1660 testTokenize("//\xe2\x80\xa8", &[_]Token.Id{1660 testTokenize("//\xe2\x80\xa8", &[_]Token.Id{
1661 Token.Id.LineComment,1661 .LineComment,
1662 Token.Id.Invalid,1662 .Invalid,
1663 });1663 });
1664 testTokenize("//\xe2\x80\xa9", &[_]Token.Id{1664 testTokenize("//\xe2\x80\xa9", &[_]Token.Id{
1665 Token.Id.LineComment,1665 .LineComment,
1666 Token.Id.Invalid,1666 .Invalid,
1667 });1667 });
1668 testTokenize("//\xe2\x80\xaa", &[_]Token.Id{Token.Id.LineComment});1668 testTokenize("//\xe2\x80\xaa", &[_]Token.Id{.LineComment});
1669}1669}
16701670
1671test "tokenizer - string identifier and builtin fns" {1671test "tokenizer - string identifier and builtin fns" {
1672 testTokenize(1672 testTokenize(
1673 \\const @"if" = @import("std");1673 \\const @"if" = @import("std");
1674 , &[_]Token.Id{1674 , &[_]Token.Id{
1675 Token.Id.Keyword_const,1675 .Keyword_const,
1676 Token.Id.Identifier,1676 .Identifier,
1677 Token.Id.Equal,1677 .Equal,
1678 Token.Id.Builtin,1678 .Builtin,
1679 Token.Id.LParen,1679 .LParen,
1680 Token.Id.StringLiteral,1680 .StringLiteral,
1681 Token.Id.RParen,1681 .RParen,
1682 Token.Id.Semicolon,1682 .Semicolon,
1683 });1683 });
1684}1684}
16851685
...@@ -1687,26 +1687,26 @@ test "tokenizer - multiline string literal with literal tab" {...@@ -1687,26 +1687,26 @@ test "tokenizer - multiline string literal with literal tab" {
1687 testTokenize(1687 testTokenize(
1688 \\\\foo bar1688 \\\\foo bar
1689 , &[_]Token.Id{1689 , &[_]Token.Id{
1690 Token.Id.MultilineStringLiteralLine,1690 .MultilineStringLiteralLine,
1691 });1691 });
1692}1692}
16931693
1694test "tokenizer - pipe and then invalid" {1694test "tokenizer - pipe and then invalid" {
1695 testTokenize("||=", &[_]Token.Id{1695 testTokenize("||=", &[_]Token.Id{
1696 Token.Id.PipePipe,1696 .PipePipe,
1697 Token.Id.Equal,1697 .Equal,
1698 });1698 });
1699}1699}
17001700
1701test "tokenizer - line comment and doc comment" {1701test "tokenizer - line comment and doc comment" {
1702 testTokenize("//", &[_]Token.Id{Token.Id.LineComment});1702 testTokenize("//", &[_]Token.Id{.LineComment});
1703 testTokenize("// a / b", &[_]Token.Id{Token.Id.LineComment});1703 testTokenize("// a / b", &[_]Token.Id{.LineComment});
1704 testTokenize("// /", &[_]Token.Id{Token.Id.LineComment});1704 testTokenize("// /", &[_]Token.Id{.LineComment});
1705 testTokenize("/// a", &[_]Token.Id{Token.Id.DocComment});1705 testTokenize("/// a", &[_]Token.Id{.DocComment});
1706 testTokenize("///", &[_]Token.Id{Token.Id.DocComment});1706 testTokenize("///", &[_]Token.Id{.DocComment});
1707 testTokenize("////", &[_]Token.Id{Token.Id.LineComment});1707 testTokenize("////", &[_]Token.Id{.LineComment});
1708 testTokenize("//!", &[_]Token.Id{Token.Id.ContainerDocComment});1708 testTokenize("//!", &[_]Token.Id{.ContainerDocComment});
1709 testTokenize("//!!", &[_]Token.Id{Token.Id.ContainerDocComment});1709 testTokenize("//!!", &[_]Token.Id{.ContainerDocComment});
1710}1710}
17111711
1712test "tokenizer - line comment followed by identifier" {1712test "tokenizer - line comment followed by identifier" {
...@@ -1715,28 +1715,28 @@ test "tokenizer - line comment followed by identifier" {...@@ -1715,28 +1715,28 @@ test "tokenizer - line comment followed by identifier" {
1715 \\ // another1715 \\ // another
1716 \\ Another,1716 \\ Another,
1717 , &[_]Token.Id{1717 , &[_]Token.Id{
1718 Token.Id.Identifier,1718 .Identifier,
1719 Token.Id.Comma,1719 .Comma,
1720 Token.Id.LineComment,1720 .LineComment,
1721 Token.Id.Identifier,1721 .Identifier,
1722 Token.Id.Comma,1722 .Comma,
1723 });1723 });
1724}1724}
17251725
1726test "tokenizer - UTF-8 BOM is recognized and skipped" {1726test "tokenizer - UTF-8 BOM is recognized and skipped" {
1727 testTokenize("\xEF\xBB\xBFa;\n", &[_]Token.Id{1727 testTokenize("\xEF\xBB\xBFa;\n", &[_]Token.Id{
1728 Token.Id.Identifier,1728 .Identifier,
1729 Token.Id.Semicolon,1729 .Semicolon,
1730 });1730 });
1731}1731}
17321732
1733test "correctly parse pointer assignment" {1733test "correctly parse pointer assignment" {
1734 testTokenize("b.*=3;\n", &[_]Token.Id{1734 testTokenize("b.*=3;\n", &[_]Token.Id{
1735 Token.Id.Identifier,1735 .Identifier,
1736 Token.Id.PeriodAsterisk,1736 .PeriodAsterisk,
1737 Token.Id.Equal,1737 .Equal,
1738 Token.Id.IntegerLiteral,1738 .IntegerLiteral,
1739 Token.Id.Semicolon,1739 .Semicolon,
1740 });1740 });
1741}1741}
17421742
...@@ -1979,5 +1979,5 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {...@@ -1979,5 +1979,5 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
1979 }1979 }
1980 }1980 }
1981 const last_token = tokenizer.next();1981 const last_token = tokenizer.next();
1982 std.testing.expect(last_token.id == Token.Id.Eof);1982 std.testing.expect(last_token.id == .Eof);
1983}1983}