authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-13 23:08:42+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-13 23:08:42+03:00
log2296906e2ad54c387b4b19784148c47a26969cdc
tree963439b81d9d7bdd0f57fa3100ebfcf59146b8f8
parent23c5ff94e9dda07694762fcc829dbac006bb00a1
signaturelock-open Commit is signed but in an unrecognized format.

modernize std.zig.tokenizer

1 files changed, 477 insertions(+), 477 deletions(-)

lib/std/zig/tokenizer.zig+477-477
......@@ -353,64 +353,64 @@ pub const Tokenizer = struct {
353353 }
354354
355355 const State = enum {
356 Start,
357 Identifier,
358 Builtin,
359 StringLiteral,
360 StringLiteralBackslash,
361 MultilineStringLiteralLine,
362 CharLiteral,
363 CharLiteralBackslash,
364 CharLiteralHexEscape,
365 CharLiteralUnicodeEscapeSawU,
366 CharLiteralUnicodeEscape,
367 CharLiteralUnicodeInvalid,
368 CharLiteralUnicode,
369 CharLiteralEnd,
370 Backslash,
371 Equal,
372 Bang,
373 Pipe,
374 Minus,
375 MinusPercent,
376 Asterisk,
377 AsteriskPercent,
378 Slash,
379 LineCommentStart,
380 LineComment,
381 DocCommentStart,
382 DocComment,
383 ContainerDocComment,
384 Zero,
385 IntegerLiteralDec,
386 IntegerLiteralDecNoUnderscore,
387 IntegerLiteralBin,
388 IntegerLiteralBinNoUnderscore,
389 IntegerLiteralOct,
390 IntegerLiteralOctNoUnderscore,
391 IntegerLiteralHex,
392 IntegerLiteralHexNoUnderscore,
393 NumberDotDec,
394 NumberDotHex,
395 FloatFractionDec,
396 FloatFractionDecNoUnderscore,
397 FloatFractionHex,
398 FloatFractionHexNoUnderscore,
399 FloatExponentUnsigned,
400 FloatExponentNumber,
401 FloatExponentNumberNoUnderscore,
402 Ampersand,
403 Caret,
404 Percent,
405 Plus,
406 PlusPercent,
407 AngleBracketLeft,
408 AngleBracketAngleBracketLeft,
409 AngleBracketRight,
410 AngleBracketAngleBracketRight,
411 Period,
412 Period2,
413 SawAtSign,
356 start,
357 identifier,
358 builtin,
359 string_literal,
360 string_literal_backslash,
361 multiline_string_literal_line,
362 char_literal,
363 char_literal_backslash,
364 char_literal_hex_escape,
365 char_literal_unicode_escape_saw_u,
366 char_literal_unicode_escape,
367 char_literal_unicode_invalid,
368 char_literal_unicode,
369 char_literal_end,
370 backslash,
371 equal,
372 bang,
373 pipe,
374 minus,
375 minus_percent,
376 asterisk,
377 asterisk_percent,
378 slash,
379 line_comment_start,
380 line_comment,
381 doc_comment_start,
382 doc_comment,
383 container_doc_comment,
384 zero,
385 int_literal_dec,
386 int_literal_dec_no_underscore,
387 int_literal_bin,
388 int_literal_bin_no_underscore,
389 int_literal_oct,
390 int_literal_oct_no_underscore,
391 int_literal_hex,
392 int_literal_hex_no_underscore,
393 num_dot_dec,
394 num_dot_hex,
395 float_fraction_dec,
396 float_fraction_dec_no_underscore,
397 float_fraction_hex,
398 float_fraction_hex_no_underscore,
399 float_exponent_unsigned,
400 float_exponent_num,
401 float_exponent_num_no_underscore,
402 ampersand,
403 caret,
404 percent,
405 plus,
406 plus_percent,
407 angle_bracket_left,
408 angle_bracket_angle_bracket_left,
409 angle_bracket_right,
410 angle_bracket_angle_bracket_right,
411 period,
412 period_2,
413 saw_at_sign,
414414 };
415415
416416 fn isIdentifierChar(char: u8) bool {
......@@ -423,9 +423,9 @@ pub const Tokenizer = struct {
423423 return token;
424424 }
425425 const start_index = self.index;
426 var state = State.Start;
426 var state: State = .start;
427427 var result = Token{
428 .id = Token.Id.Eof,
428 .id = .Eof,
429429 .start = self.index,
430430 .end = undefined,
431431 };
......@@ -434,40 +434,40 @@ pub const Tokenizer = struct {
434434 while (self.index < self.buffer.len) : (self.index += 1) {
435435 const c = self.buffer[self.index];
436436 switch (state) {
437 State.Start => switch (c) {
437 .start => switch (c) {
438438 ' ', '\n', '\t', '\r' => {
439439 result.start = self.index + 1;
440440 },
441441 '"' => {
442 state = State.StringLiteral;
443 result.id = Token.Id.StringLiteral;
442 state = .string_literal;
443 result.id = .StringLiteral;
444444 },
445445 '\'' => {
446 state = State.CharLiteral;
446 state = .char_literal;
447447 },
448448 'a'...'z', 'A'...'Z', '_' => {
449 state = State.Identifier;
450 result.id = Token.Id.Identifier;
449 state = .identifier;
450 result.id = .Identifier;
451451 },
452452 '@' => {
453 state = State.SawAtSign;
453 state = .saw_at_sign;
454454 },
455455 '=' => {
456 state = State.Equal;
456 state = .equal;
457457 },
458458 '!' => {
459 state = State.Bang;
459 state = .bang;
460460 },
461461 '|' => {
462 state = State.Pipe;
462 state = .pipe;
463463 },
464464 '(' => {
465 result.id = Token.Id.LParen;
465 result.id = .LParen;
466466 self.index += 1;
467467 break;
468468 },
469469 ')' => {
470 result.id = Token.Id.RParen;
470 result.id = .RParen;
471471 self.index += 1;
472472 break;
473473 },
......@@ -477,213 +477,213 @@ pub const Tokenizer = struct {
477477 break;
478478 },
479479 ']' => {
480 result.id = Token.Id.RBracket;
480 result.id = .RBracket;
481481 self.index += 1;
482482 break;
483483 },
484484 ';' => {
485 result.id = Token.Id.Semicolon;
485 result.id = .Semicolon;
486486 self.index += 1;
487487 break;
488488 },
489489 ',' => {
490 result.id = Token.Id.Comma;
490 result.id = .Comma;
491491 self.index += 1;
492492 break;
493493 },
494494 '?' => {
495 result.id = Token.Id.QuestionMark;
495 result.id = .QuestionMark;
496496 self.index += 1;
497497 break;
498498 },
499499 ':' => {
500 result.id = Token.Id.Colon;
500 result.id = .Colon;
501501 self.index += 1;
502502 break;
503503 },
504504 '%' => {
505 state = State.Percent;
505 state = .percent;
506506 },
507507 '*' => {
508 state = State.Asterisk;
508 state = .asterisk;
509509 },
510510 '+' => {
511 state = State.Plus;
511 state = .plus;
512512 },
513513 '<' => {
514 state = State.AngleBracketLeft;
514 state = .angle_bracket_left;
515515 },
516516 '>' => {
517 state = State.AngleBracketRight;
517 state = .angle_bracket_right;
518518 },
519519 '^' => {
520 state = State.Caret;
520 state = .caret;
521521 },
522522 '\\' => {
523 state = State.Backslash;
524 result.id = Token.Id.MultilineStringLiteralLine;
523 state = .backslash;
524 result.id = .MultilineStringLiteralLine;
525525 },
526526 '{' => {
527 result.id = Token.Id.LBrace;
527 result.id = .LBrace;
528528 self.index += 1;
529529 break;
530530 },
531531 '}' => {
532 result.id = Token.Id.RBrace;
532 result.id = .RBrace;
533533 self.index += 1;
534534 break;
535535 },
536536 '~' => {
537 result.id = Token.Id.Tilde;
537 result.id = .Tilde;
538538 self.index += 1;
539539 break;
540540 },
541541 '.' => {
542 state = State.Period;
542 state = .period;
543543 },
544544 '-' => {
545 state = State.Minus;
545 state = .minus;
546546 },
547547 '/' => {
548 state = State.Slash;
548 state = .slash;
549549 },
550550 '&' => {
551 state = State.Ampersand;
551 state = .ampersand;
552552 },
553553 '0' => {
554 state = State.Zero;
555 result.id = Token.Id.IntegerLiteral;
554 state = .zero;
555 result.id = .IntegerLiteral;
556556 },
557557 '1'...'9' => {
558 state = State.IntegerLiteralDec;
559 result.id = Token.Id.IntegerLiteral;
558 state = .int_literal_dec;
559 result.id = .IntegerLiteral;
560560 },
561561 else => {
562 result.id = Token.Id.Invalid;
562 result.id = .Invalid;
563563 self.index += 1;
564564 break;
565565 },
566566 },
567567
568 State.SawAtSign => switch (c) {
568 .saw_at_sign => switch (c) {
569569 '"' => {
570 result.id = Token.Id.Identifier;
571 state = State.StringLiteral;
570 result.id = .Identifier;
571 state = .string_literal;
572572 },
573573 else => {
574574 // reinterpret as a builtin
575575 self.index -= 1;
576 state = State.Builtin;
577 result.id = Token.Id.Builtin;
576 state = .builtin;
577 result.id = .Builtin;
578578 },
579579 },
580580
581 State.Ampersand => switch (c) {
581 .ampersand => switch (c) {
582582 '&' => {
583 result.id = Token.Id.Invalid_ampersands;
583 result.id = .Invalid_ampersands;
584584 self.index += 1;
585585 break;
586586 },
587587 '=' => {
588 result.id = Token.Id.AmpersandEqual;
588 result.id = .AmpersandEqual;
589589 self.index += 1;
590590 break;
591591 },
592592 else => {
593 result.id = Token.Id.Ampersand;
593 result.id = .Ampersand;
594594 break;
595595 },
596596 },
597597
598 State.Asterisk => switch (c) {
598 .asterisk => switch (c) {
599599 '=' => {
600 result.id = Token.Id.AsteriskEqual;
600 result.id = .AsteriskEqual;
601601 self.index += 1;
602602 break;
603603 },
604604 '*' => {
605 result.id = Token.Id.AsteriskAsterisk;
605 result.id = .AsteriskAsterisk;
606606 self.index += 1;
607607 break;
608608 },
609609 '%' => {
610 state = State.AsteriskPercent;
610 state = .asterisk_percent;
611611 },
612612 else => {
613 result.id = Token.Id.Asterisk;
613 result.id = .Asterisk;
614614 break;
615615 },
616616 },
617617
618 State.AsteriskPercent => switch (c) {
618 .asterisk_percent => switch (c) {
619619 '=' => {
620 result.id = Token.Id.AsteriskPercentEqual;
620 result.id = .AsteriskPercentEqual;
621621 self.index += 1;
622622 break;
623623 },
624624 else => {
625 result.id = Token.Id.AsteriskPercent;
625 result.id = .AsteriskPercent;
626626 break;
627627 },
628628 },
629629
630 State.Percent => switch (c) {
630 .percent => switch (c) {
631631 '=' => {
632 result.id = Token.Id.PercentEqual;
632 result.id = .PercentEqual;
633633 self.index += 1;
634634 break;
635635 },
636636 else => {
637 result.id = Token.Id.Percent;
637 result.id = .Percent;
638638 break;
639639 },
640640 },
641641
642 State.Plus => switch (c) {
642 .plus => switch (c) {
643643 '=' => {
644 result.id = Token.Id.PlusEqual;
644 result.id = .PlusEqual;
645645 self.index += 1;
646646 break;
647647 },
648648 '+' => {
649 result.id = Token.Id.PlusPlus;
649 result.id = .PlusPlus;
650650 self.index += 1;
651651 break;
652652 },
653653 '%' => {
654 state = State.PlusPercent;
654 state = .plus_percent;
655655 },
656656 else => {
657 result.id = Token.Id.Plus;
657 result.id = .Plus;
658658 break;
659659 },
660660 },
661661
662 State.PlusPercent => switch (c) {
662 .plus_percent => switch (c) {
663663 '=' => {
664 result.id = Token.Id.PlusPercentEqual;
664 result.id = .PlusPercentEqual;
665665 self.index += 1;
666666 break;
667667 },
668668 else => {
669 result.id = Token.Id.PlusPercent;
669 result.id = .PlusPercent;
670670 break;
671671 },
672672 },
673673
674 State.Caret => switch (c) {
674 .caret => switch (c) {
675675 '=' => {
676 result.id = Token.Id.CaretEqual;
676 result.id = .CaretEqual;
677677 self.index += 1;
678678 break;
679679 },
680680 else => {
681 result.id = Token.Id.Caret;
681 result.id = .Caret;
682682 break;
683683 },
684684 },
685685
686 State.Identifier => switch (c) {
686 .identifier => switch (c) {
687687 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
688688 else => {
689689 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
......@@ -692,19 +692,19 @@ pub const Tokenizer = struct {
692692 break;
693693 },
694694 },
695 State.Builtin => switch (c) {
695 .builtin => switch (c) {
696696 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
697697 else => break,
698698 },
699 State.Backslash => switch (c) {
699 .backslash => switch (c) {
700700 '\\' => {
701 state = State.MultilineStringLiteralLine;
701 state = .multiline_string_literal_line;
702702 },
703703 else => break,
704704 },
705 State.StringLiteral => switch (c) {
705 .string_literal => switch (c) {
706706 '\\' => {
707 state = State.StringLiteralBackslash;
707 state = .string_literal_backslash;
708708 },
709709 '"' => {
710710 self.index += 1;
......@@ -714,98 +714,98 @@ pub const Tokenizer = struct {
714714 else => self.checkLiteralCharacter(),
715715 },
716716
717 State.StringLiteralBackslash => switch (c) {
717 .string_literal_backslash => switch (c) {
718718 '\n', '\r' => break, // Look for this error later.
719719 else => {
720 state = State.StringLiteral;
720 state = .string_literal;
721721 },
722722 },
723723
724 State.CharLiteral => switch (c) {
724 .char_literal => switch (c) {
725725 '\\' => {
726 state = State.CharLiteralBackslash;
726 state = .char_literal_backslash;
727727 },
728728 '\'', 0x80...0xbf, 0xf8...0xff => {
729 result.id = Token.Id.Invalid;
729 result.id = .Invalid;
730730 break;
731731 },
732732 0xc0...0xdf => { // 110xxxxx
733733 remaining_code_units = 1;
734 state = State.CharLiteralUnicode;
734 state = .char_literal_unicode;
735735 },
736736 0xe0...0xef => { // 1110xxxx
737737 remaining_code_units = 2;
738 state = State.CharLiteralUnicode;
738 state = .char_literal_unicode;
739739 },
740740 0xf0...0xf7 => { // 11110xxx
741741 remaining_code_units = 3;
742 state = State.CharLiteralUnicode;
742 state = .char_literal_unicode;
743743 },
744744 else => {
745 state = State.CharLiteralEnd;
745 state = .char_literal_end;
746746 },
747747 },
748748
749 State.CharLiteralBackslash => switch (c) {
749 .char_literal_backslash => switch (c) {
750750 '\n' => {
751 result.id = Token.Id.Invalid;
751 result.id = .Invalid;
752752 break;
753753 },
754754 'x' => {
755 state = State.CharLiteralHexEscape;
755 state = .char_literal_hex_escape;
756756 seen_escape_digits = 0;
757757 },
758758 'u' => {
759 state = State.CharLiteralUnicodeEscapeSawU;
759 state = .char_literal_unicode_escape_saw_u;
760760 },
761761 else => {
762 state = State.CharLiteralEnd;
762 state = .char_literal_end;
763763 },
764764 },
765765
766 State.CharLiteralHexEscape => switch (c) {
766 .char_literal_hex_escape => switch (c) {
767767 '0'...'9', 'a'...'f', 'A'...'F' => {
768768 seen_escape_digits += 1;
769769 if (seen_escape_digits == 2) {
770 state = State.CharLiteralEnd;
770 state = .char_literal_end;
771771 }
772772 },
773773 else => {
774 result.id = Token.Id.Invalid;
774 result.id = .Invalid;
775775 break;
776776 },
777777 },
778778
779 State.CharLiteralUnicodeEscapeSawU => switch (c) {
779 .char_literal_unicode_escape_saw_u => switch (c) {
780780 '{' => {
781 state = State.CharLiteralUnicodeEscape;
781 state = .char_literal_unicode_escape;
782782 seen_escape_digits = 0;
783783 },
784784 else => {
785 result.id = Token.Id.Invalid;
786 state = State.CharLiteralUnicodeInvalid;
785 result.id = .Invalid;
786 state = .char_literal_unicode_invalid;
787787 },
788788 },
789789
790 State.CharLiteralUnicodeEscape => switch (c) {
790 .char_literal_unicode_escape => switch (c) {
791791 '0'...'9', 'a'...'f', 'A'...'F' => {
792792 seen_escape_digits += 1;
793793 },
794794 '}' => {
795795 if (seen_escape_digits == 0) {
796 result.id = Token.Id.Invalid;
797 state = State.CharLiteralUnicodeInvalid;
796 result.id = .Invalid;
797 state = .char_literal_unicode_invalid;
798798 } else {
799 state = State.CharLiteralEnd;
799 state = .char_literal_end;
800800 }
801801 },
802802 else => {
803 result.id = Token.Id.Invalid;
804 state = State.CharLiteralUnicodeInvalid;
803 result.id = .Invalid;
804 state = .char_literal_unicode_invalid;
805805 },
806806 },
807807
808 State.CharLiteralUnicodeInvalid => switch (c) {
808 .char_literal_unicode_invalid => switch (c) {
809809 // Keep consuming characters until an obvious stopping point.
810810 // This consolidates e.g. `u{0ab1Q}` into a single invalid token
811811 // instead of creating the tokens `u{0ab1`, `Q`, `}`
......@@ -813,32 +813,32 @@ pub const Tokenizer = struct {
813813 else => break,
814814 },
815815
816 State.CharLiteralEnd => switch (c) {
816 .char_literal_end => switch (c) {
817817 '\'' => {
818 result.id = Token.Id.CharLiteral;
818 result.id = .CharLiteral;
819819 self.index += 1;
820820 break;
821821 },
822822 else => {
823 result.id = Token.Id.Invalid;
823 result.id = .Invalid;
824824 break;
825825 },
826826 },
827827
828 State.CharLiteralUnicode => switch (c) {
828 .char_literal_unicode => switch (c) {
829829 0x80...0xbf => {
830830 remaining_code_units -= 1;
831831 if (remaining_code_units == 0) {
832 state = State.CharLiteralEnd;
832 state = .char_literal_end;
833833 }
834834 },
835835 else => {
836 result.id = Token.Id.Invalid;
836 result.id = .Invalid;
837837 break;
838838 },
839839 },
840840
841 State.MultilineStringLiteralLine => switch (c) {
841 .multiline_string_literal_line => switch (c) {
842842 '\n' => {
843843 self.index += 1;
844844 break;
......@@ -847,449 +847,449 @@ pub const Tokenizer = struct {
847847 else => self.checkLiteralCharacter(),
848848 },
849849
850 State.Bang => switch (c) {
850 .bang => switch (c) {
851851 '=' => {
852 result.id = Token.Id.BangEqual;
852 result.id = .BangEqual;
853853 self.index += 1;
854854 break;
855855 },
856856 else => {
857 result.id = Token.Id.Bang;
857 result.id = .Bang;
858858 break;
859859 },
860860 },
861861
862 State.Pipe => switch (c) {
862 .pipe => switch (c) {
863863 '=' => {
864 result.id = Token.Id.PipeEqual;
864 result.id = .PipeEqual;
865865 self.index += 1;
866866 break;
867867 },
868868 '|' => {
869 result.id = Token.Id.PipePipe;
869 result.id = .PipePipe;
870870 self.index += 1;
871871 break;
872872 },
873873 else => {
874 result.id = Token.Id.Pipe;
874 result.id = .Pipe;
875875 break;
876876 },
877877 },
878878
879 State.Equal => switch (c) {
879 .equal => switch (c) {
880880 '=' => {
881 result.id = Token.Id.EqualEqual;
881 result.id = .EqualEqual;
882882 self.index += 1;
883883 break;
884884 },
885885 '>' => {
886 result.id = Token.Id.EqualAngleBracketRight;
886 result.id = .EqualAngleBracketRight;
887887 self.index += 1;
888888 break;
889889 },
890890 else => {
891 result.id = Token.Id.Equal;
891 result.id = .Equal;
892892 break;
893893 },
894894 },
895895
896 State.Minus => switch (c) {
896 .minus => switch (c) {
897897 '>' => {
898 result.id = Token.Id.Arrow;
898 result.id = .Arrow;
899899 self.index += 1;
900900 break;
901901 },
902902 '=' => {
903 result.id = Token.Id.MinusEqual;
903 result.id = .MinusEqual;
904904 self.index += 1;
905905 break;
906906 },
907907 '%' => {
908 state = State.MinusPercent;
908 state = .minus_percent;
909909 },
910910 else => {
911 result.id = Token.Id.Minus;
911 result.id = .Minus;
912912 break;
913913 },
914914 },
915915
916 State.MinusPercent => switch (c) {
916 .minus_percent => switch (c) {
917917 '=' => {
918 result.id = Token.Id.MinusPercentEqual;
918 result.id = .MinusPercentEqual;
919919 self.index += 1;
920920 break;
921921 },
922922 else => {
923 result.id = Token.Id.MinusPercent;
923 result.id = .MinusPercent;
924924 break;
925925 },
926926 },
927927
928 State.AngleBracketLeft => switch (c) {
928 .angle_bracket_left => switch (c) {
929929 '<' => {
930 state = State.AngleBracketAngleBracketLeft;
930 state = .angle_bracket_angle_bracket_left;
931931 },
932932 '=' => {
933 result.id = Token.Id.AngleBracketLeftEqual;
933 result.id = .AngleBracketLeftEqual;
934934 self.index += 1;
935935 break;
936936 },
937937 else => {
938 result.id = Token.Id.AngleBracketLeft;
938 result.id = .AngleBracketLeft;
939939 break;
940940 },
941941 },
942942
943 State.AngleBracketAngleBracketLeft => switch (c) {
943 .angle_bracket_angle_bracket_left => switch (c) {
944944 '=' => {
945 result.id = Token.Id.AngleBracketAngleBracketLeftEqual;
945 result.id = .AngleBracketAngleBracketLeftEqual;
946946 self.index += 1;
947947 break;
948948 },
949949 else => {
950 result.id = Token.Id.AngleBracketAngleBracketLeft;
950 result.id = .AngleBracketAngleBracketLeft;
951951 break;
952952 },
953953 },
954954
955 State.AngleBracketRight => switch (c) {
955 .angle_bracket_right => switch (c) {
956956 '>' => {
957 state = State.AngleBracketAngleBracketRight;
957 state = .angle_bracket_angle_bracket_right;
958958 },
959959 '=' => {
960 result.id = Token.Id.AngleBracketRightEqual;
960 result.id = .AngleBracketRightEqual;
961961 self.index += 1;
962962 break;
963963 },
964964 else => {
965 result.id = Token.Id.AngleBracketRight;
965 result.id = .AngleBracketRight;
966966 break;
967967 },
968968 },
969969
970 State.AngleBracketAngleBracketRight => switch (c) {
970 .angle_bracket_angle_bracket_right => switch (c) {
971971 '=' => {
972 result.id = Token.Id.AngleBracketAngleBracketRightEqual;
972 result.id = .AngleBracketAngleBracketRightEqual;
973973 self.index += 1;
974974 break;
975975 },
976976 else => {
977 result.id = Token.Id.AngleBracketAngleBracketRight;
977 result.id = .AngleBracketAngleBracketRight;
978978 break;
979979 },
980980 },
981981
982 State.Period => switch (c) {
982 .period => switch (c) {
983983 '.' => {
984 state = State.Period2;
984 state = .period_2;
985985 },
986986 '*' => {
987 result.id = Token.Id.PeriodAsterisk;
987 result.id = .PeriodAsterisk;
988988 self.index += 1;
989989 break;
990990 },
991991 else => {
992 result.id = Token.Id.Period;
992 result.id = .Period;
993993 break;
994994 },
995995 },
996996
997 State.Period2 => switch (c) {
997 .period_2 => switch (c) {
998998 '.' => {
999 result.id = Token.Id.Ellipsis3;
999 result.id = .Ellipsis3;
10001000 self.index += 1;
10011001 break;
10021002 },
10031003 else => {
1004 result.id = Token.Id.Ellipsis2;
1004 result.id = .Ellipsis2;
10051005 break;
10061006 },
10071007 },
10081008
1009 State.Slash => switch (c) {
1009 .slash => switch (c) {
10101010 '/' => {
1011 state = State.LineCommentStart;
1012 result.id = Token.Id.LineComment;
1011 state = .line_comment_start;
1012 result.id = .LineComment;
10131013 },
10141014 '=' => {
1015 result.id = Token.Id.SlashEqual;
1015 result.id = .SlashEqual;
10161016 self.index += 1;
10171017 break;
10181018 },
10191019 else => {
1020 result.id = Token.Id.Slash;
1020 result.id = .Slash;
10211021 break;
10221022 },
10231023 },
1024 State.LineCommentStart => switch (c) {
1024 .line_comment_start => switch (c) {
10251025 '/' => {
1026 state = State.DocCommentStart;
1026 state = .doc_comment_start;
10271027 },
10281028 '!' => {
1029 result.id = Token.Id.ContainerDocComment;
1030 state = State.ContainerDocComment;
1029 result.id = .ContainerDocComment;
1030 state = .container_doc_comment;
10311031 },
10321032 '\n' => break,
10331033 else => {
1034 state = State.LineComment;
1034 state = .line_comment;
10351035 self.checkLiteralCharacter();
10361036 },
10371037 },
1038 State.DocCommentStart => switch (c) {
1038 .doc_comment_start => switch (c) {
10391039 '/' => {
1040 state = State.LineComment;
1040 state = .line_comment;
10411041 },
10421042 '\n' => {
1043 result.id = Token.Id.DocComment;
1043 result.id = .DocComment;
10441044 break;
10451045 },
10461046 else => {
1047 state = State.DocComment;
1048 result.id = Token.Id.DocComment;
1047 state = .doc_comment;
1048 result.id = .DocComment;
10491049 self.checkLiteralCharacter();
10501050 },
10511051 },
1052 State.LineComment, State.DocComment, State.ContainerDocComment => switch (c) {
1052 .line_comment, .doc_comment, .container_doc_comment => switch (c) {
10531053 '\n' => break,
10541054 else => self.checkLiteralCharacter(),
10551055 },
1056 State.Zero => switch (c) {
1056 .zero => switch (c) {
10571057 'b' => {
1058 state = State.IntegerLiteralBinNoUnderscore;
1058 state = .int_literal_bin_no_underscore;
10591059 },
10601060 'o' => {
1061 state = State.IntegerLiteralOctNoUnderscore;
1061 state = .int_literal_oct_no_underscore;
10621062 },
10631063 'x' => {
1064 state = State.IntegerLiteralHexNoUnderscore;
1064 state = .int_literal_hex_no_underscore;
10651065 },
10661066 '0'...'9', '_', '.', 'e', 'E' => {
10671067 // reinterpret as a decimal number
10681068 self.index -= 1;
1069 state = State.IntegerLiteralDec;
1069 state = .int_literal_dec;
10701070 },
10711071 else => {
10721072 if (isIdentifierChar(c)) {
1073 result.id = Token.Id.Invalid;
1073 result.id = .Invalid;
10741074 }
10751075 break;
10761076 },
10771077 },
1078 State.IntegerLiteralBinNoUnderscore => switch (c) {
1078 .int_literal_bin_no_underscore => switch (c) {
10791079 '0'...'1' => {
1080 state = State.IntegerLiteralBin;
1080 state = .int_literal_bin;
10811081 },
10821082 else => {
1083 result.id = Token.Id.Invalid;
1083 result.id = .Invalid;
10841084 break;
10851085 },
10861086 },
1087 State.IntegerLiteralBin => switch (c) {
1087 .int_literal_bin => switch (c) {
10881088 '_' => {
1089 state = State.IntegerLiteralBinNoUnderscore;
1089 state = .int_literal_bin_no_underscore;
10901090 },
10911091 '0'...'1' => {},
10921092 else => {
10931093 if (isIdentifierChar(c)) {
1094 result.id = Token.Id.Invalid;
1094 result.id = .Invalid;
10951095 }
10961096 break;
10971097 },
10981098 },
1099 State.IntegerLiteralOctNoUnderscore => switch (c) {
1099 .int_literal_oct_no_underscore => switch (c) {
11001100 '0'...'7' => {
1101 state = State.IntegerLiteralOct;
1101 state = .int_literal_oct;
11021102 },
11031103 else => {
1104 result.id = Token.Id.Invalid;
1104 result.id = .Invalid;
11051105 break;
11061106 },
11071107 },
1108 State.IntegerLiteralOct => switch (c) {
1108 .int_literal_oct => switch (c) {
11091109 '_' => {
1110 state = State.IntegerLiteralOctNoUnderscore;
1110 state = .int_literal_oct_no_underscore;
11111111 },
11121112 '0'...'7' => {},
11131113 else => {
11141114 if (isIdentifierChar(c)) {
1115 result.id = Token.Id.Invalid;
1115 result.id = .Invalid;
11161116 }
11171117 break;
11181118 },
11191119 },
1120 State.IntegerLiteralDecNoUnderscore => switch (c) {
1120 .int_literal_dec_no_underscore => switch (c) {
11211121 '0'...'9' => {
1122 state = State.IntegerLiteralDec;
1122 state = .int_literal_dec;
11231123 },
11241124 else => {
1125 result.id = Token.Id.Invalid;
1125 result.id = .Invalid;
11261126 break;
11271127 },
11281128 },
1129 State.IntegerLiteralDec => switch (c) {
1129 .int_literal_dec => switch (c) {
11301130 '_' => {
1131 state = State.IntegerLiteralDecNoUnderscore;
1131 state = .int_literal_dec_no_underscore;
11321132 },
11331133 '.' => {
1134 state = State.NumberDotDec;
1135 result.id = Token.Id.FloatLiteral;
1134 state = .num_dot_dec;
1135 result.id = .FloatLiteral;
11361136 },
11371137 'e', 'E' => {
1138 state = State.FloatExponentUnsigned;
1139 result.id = Token.Id.FloatLiteral;
1138 state = .float_exponent_unsigned;
1139 result.id = .FloatLiteral;
11401140 },
11411141 '0'...'9' => {},
11421142 else => {
11431143 if (isIdentifierChar(c)) {
1144 result.id = Token.Id.Invalid;
1144 result.id = .Invalid;
11451145 }
11461146 break;
11471147 },
11481148 },
1149 State.IntegerLiteralHexNoUnderscore => switch (c) {
1149 .int_literal_hex_no_underscore => switch (c) {
11501150 '0'...'9', 'a'...'f', 'A'...'F' => {
1151 state = State.IntegerLiteralHex;
1151 state = .int_literal_hex;
11521152 },
11531153 else => {
1154 result.id = Token.Id.Invalid;
1154 result.id = .Invalid;
11551155 break;
11561156 },
11571157 },
1158 State.IntegerLiteralHex => switch (c) {
1158 .int_literal_hex => switch (c) {
11591159 '_' => {
1160 state = State.IntegerLiteralHexNoUnderscore;
1160 state = .int_literal_hex_no_underscore;
11611161 },
11621162 '.' => {
1163 state = State.NumberDotHex;
1164 result.id = Token.Id.FloatLiteral;
1163 state = .num_dot_hex;
1164 result.id = .FloatLiteral;
11651165 },
11661166 'p', 'P' => {
1167 state = State.FloatExponentUnsigned;
1168 result.id = Token.Id.FloatLiteral;
1167 state = .float_exponent_unsigned;
1168 result.id = .FloatLiteral;
11691169 },
11701170 '0'...'9', 'a'...'f', 'A'...'F' => {},
11711171 else => {
11721172 if (isIdentifierChar(c)) {
1173 result.id = Token.Id.Invalid;
1173 result.id = .Invalid;
11741174 }
11751175 break;
11761176 },
11771177 },
1178 State.NumberDotDec => switch (c) {
1178 .num_dot_dec => switch (c) {
11791179 '.' => {
11801180 self.index -= 1;
1181 state = State.Start;
1181 state = .start;
11821182 break;
11831183 },
11841184 'e', 'E' => {
1185 state = State.FloatExponentUnsigned;
1185 state = .float_exponent_unsigned;
11861186 },
11871187 '0'...'9' => {
1188 result.id = Token.Id.FloatLiteral;
1189 state = State.FloatFractionDec;
1188 result.id = .FloatLiteral;
1189 state = .float_fraction_dec;
11901190 },
11911191 else => {
11921192 if (isIdentifierChar(c)) {
1193 result.id = Token.Id.Invalid;
1193 result.id = .Invalid;
11941194 }
11951195 break;
11961196 },
11971197 },
1198 State.NumberDotHex => switch (c) {
1198 .num_dot_hex => switch (c) {
11991199 '.' => {
12001200 self.index -= 1;
1201 state = State.Start;
1201 state = .start;
12021202 break;
12031203 },
12041204 'p', 'P' => {
1205 state = State.FloatExponentUnsigned;
1205 state = .float_exponent_unsigned;
12061206 },
12071207 '0'...'9', 'a'...'f', 'A'...'F' => {
1208 result.id = Token.Id.FloatLiteral;
1209 state = State.FloatFractionHex;
1208 result.id = .FloatLiteral;
1209 state = .float_fraction_hex;
12101210 },
12111211 else => {
12121212 if (isIdentifierChar(c)) {
1213 result.id = Token.Id.Invalid;
1213 result.id = .Invalid;
12141214 }
12151215 break;
12161216 },
12171217 },
1218 State.FloatFractionDecNoUnderscore => switch (c) {
1218 .float_fraction_dec_no_underscore => switch (c) {
12191219 '0'...'9' => {
1220 state = State.FloatFractionDec;
1220 state = .float_fraction_dec;
12211221 },
12221222 else => {
1223 result.id = Token.Id.Invalid;
1223 result.id = .Invalid;
12241224 break;
12251225 },
12261226 },
1227 State.FloatFractionDec => switch (c) {
1227 .float_fraction_dec => switch (c) {
12281228 '_' => {
1229 state = State.FloatFractionDecNoUnderscore;
1229 state = .float_fraction_dec_no_underscore;
12301230 },
12311231 'e', 'E' => {
1232 state = State.FloatExponentUnsigned;
1232 state = .float_exponent_unsigned;
12331233 },
12341234 '0'...'9' => {},
12351235 else => {
12361236 if (isIdentifierChar(c)) {
1237 result.id = Token.Id.Invalid;
1237 result.id = .Invalid;
12381238 }
12391239 break;
12401240 },
12411241 },
1242 State.FloatFractionHexNoUnderscore => switch (c) {
1242 .float_fraction_hex_no_underscore => switch (c) {
12431243 '0'...'9', 'a'...'f', 'A'...'F' => {
1244 state = State.FloatFractionHex;
1244 state = .float_fraction_hex;
12451245 },
12461246 else => {
1247 result.id = Token.Id.Invalid;
1247 result.id = .Invalid;
12481248 break;
12491249 },
12501250 },
1251 State.FloatFractionHex => switch (c) {
1251 .float_fraction_hex => switch (c) {
12521252 '_' => {
1253 state = State.FloatFractionHexNoUnderscore;
1253 state = .float_fraction_hex_no_underscore;
12541254 },
12551255 'p', 'P' => {
1256 state = State.FloatExponentUnsigned;
1256 state = .float_exponent_unsigned;
12571257 },
12581258 '0'...'9', 'a'...'f', 'A'...'F' => {},
12591259 else => {
12601260 if (isIdentifierChar(c)) {
1261 result.id = Token.Id.Invalid;
1261 result.id = .Invalid;
12621262 }
12631263 break;
12641264 },
12651265 },
1266 State.FloatExponentUnsigned => switch (c) {
1266 .float_exponent_unsigned => switch (c) {
12671267 '+', '-' => {
1268 state = State.FloatExponentNumberNoUnderscore;
1268 state = .float_exponent_num_no_underscore;
12691269 },
12701270 else => {
12711271 // reinterpret as a normal exponent number
12721272 self.index -= 1;
1273 state = State.FloatExponentNumberNoUnderscore;
1273 state = .float_exponent_num_no_underscore;
12741274 },
12751275 },
1276 State.FloatExponentNumberNoUnderscore => switch (c) {
1276 .float_exponent_num_no_underscore => switch (c) {
12771277 '0'...'9' => {
1278 state = State.FloatExponentNumber;
1278 state = .float_exponent_num;
12791279 },
12801280 else => {
1281 result.id = Token.Id.Invalid;
1281 result.id = .Invalid;
12821282 break;
12831283 },
12841284 },
1285 State.FloatExponentNumber => switch (c) {
1285 .float_exponent_num => switch (c) {
12861286 '_' => {
1287 state = State.FloatExponentNumberNoUnderscore;
1287 state = .float_exponent_num_no_underscore;
12881288 },
12891289 '0'...'9' => {},
12901290 else => {
12911291 if (isIdentifierChar(c)) {
1292 result.id = Token.Id.Invalid;
1292 result.id = .Invalid;
12931293 }
12941294 break;
12951295 },
......@@ -1297,123 +1297,123 @@ pub const Tokenizer = struct {
12971297 }
12981298 } else if (self.index == self.buffer.len) {
12991299 switch (state) {
1300 State.Start,
1301 State.IntegerLiteralDec,
1302 State.IntegerLiteralBin,
1303 State.IntegerLiteralOct,
1304 State.IntegerLiteralHex,
1305 State.NumberDotDec,
1306 State.NumberDotHex,
1307 State.FloatFractionDec,
1308 State.FloatFractionHex,
1309 State.FloatExponentNumber,
1310 State.StringLiteral, // find this error later
1311 State.MultilineStringLiteralLine,
1312 State.Builtin,
1300 .start,
1301 .int_literal_dec,
1302 .int_literal_bin,
1303 .int_literal_oct,
1304 .int_literal_hex,
1305 .num_dot_dec,
1306 .num_dot_hex,
1307 .float_fraction_dec,
1308 .float_fraction_hex,
1309 .float_exponent_num,
1310 .string_literal, // find this error later
1311 .multiline_string_literal_line,
1312 .builtin,
13131313 => {},
13141314
1315 State.Identifier => {
1315 .identifier => {
13161316 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
13171317 result.id = id;
13181318 }
13191319 },
1320 State.LineCommentStart, State.LineComment => {
1321 result.id = Token.Id.LineComment;
1322 },
1323 State.DocComment, State.DocCommentStart => {
1324 result.id = Token.Id.DocComment;
1325 },
1326 State.ContainerDocComment => {
1327 result.id = Token.Id.ContainerDocComment;
1328 },
1329
1330 State.IntegerLiteralDecNoUnderscore,
1331 State.IntegerLiteralBinNoUnderscore,
1332 State.IntegerLiteralOctNoUnderscore,
1333 State.IntegerLiteralHexNoUnderscore,
1334 State.FloatFractionDecNoUnderscore,
1335 State.FloatFractionHexNoUnderscore,
1336 State.FloatExponentNumberNoUnderscore,
1337 State.FloatExponentUnsigned,
1338 State.SawAtSign,
1339 State.Backslash,
1340 State.CharLiteral,
1341 State.CharLiteralBackslash,
1342 State.CharLiteralHexEscape,
1343 State.CharLiteralUnicodeEscapeSawU,
1344 State.CharLiteralUnicodeEscape,
1345 State.CharLiteralUnicodeInvalid,
1346 State.CharLiteralEnd,
1347 State.CharLiteralUnicode,
1348 State.StringLiteralBackslash,
1320 .line_comment, .line_comment_start => {
1321 result.id = .LineComment;
1322 },
1323 .doc_comment, .doc_comment_start => {
1324 result.id = .DocComment;
1325 },
1326 .container_doc_comment => {
1327 result.id = .ContainerDocComment;
1328 },
1329
1330 .int_literal_dec_no_underscore,
1331 .int_literal_bin_no_underscore,
1332 .int_literal_oct_no_underscore,
1333 .int_literal_hex_no_underscore,
1334 .float_fraction_dec_no_underscore,
1335 .float_fraction_hex_no_underscore,
1336 .float_exponent_num_no_underscore,
1337 .float_exponent_unsigned,
1338 .saw_at_sign,
1339 .backslash,
1340 .char_literal,
1341 .char_literal_backslash,
1342 .char_literal_hex_escape,
1343 .char_literal_unicode_escape_saw_u,
1344 .char_literal_unicode_escape,
1345 .char_literal_unicode_invalid,
1346 .char_literal_end,
1347 .char_literal_unicode,
1348 .string_literal_backslash,
13491349 => {
1350 result.id = Token.Id.Invalid;
1350 result.id = .Invalid;
13511351 },
13521352
1353 State.Equal => {
1354 result.id = Token.Id.Equal;
1353 .equal => {
1354 result.id = .Equal;
13551355 },
1356 State.Bang => {
1357 result.id = Token.Id.Bang;
1356 .bang => {
1357 result.id = .Bang;
13581358 },
1359 State.Minus => {
1360 result.id = Token.Id.Minus;
1359 .minus => {
1360 result.id = .Minus;
13611361 },
1362 State.Slash => {
1363 result.id = Token.Id.Slash;
1362 .slash => {
1363 result.id = .Slash;
13641364 },
1365 State.Zero => {
1366 result.id = Token.Id.IntegerLiteral;
1365 .zero => {
1366 result.id = .IntegerLiteral;
13671367 },
1368 State.Ampersand => {
1369 result.id = Token.Id.Ampersand;
1368 .ampersand => {
1369 result.id = .Ampersand;
13701370 },
1371 State.Period => {
1372 result.id = Token.Id.Period;
1371 .period => {
1372 result.id = .Period;
13731373 },
1374 State.Period2 => {
1375 result.id = Token.Id.Ellipsis2;
1374 .period_2 => {
1375 result.id = .Ellipsis2;
13761376 },
1377 State.Pipe => {
1378 result.id = Token.Id.Pipe;
1377 .pipe => {
1378 result.id = .Pipe;
13791379 },
1380 State.AngleBracketAngleBracketRight => {
1381 result.id = Token.Id.AngleBracketAngleBracketRight;
1380 .angle_bracket_angle_bracket_right => {
1381 result.id = .AngleBracketAngleBracketRight;
13821382 },
1383 State.AngleBracketRight => {
1384 result.id = Token.Id.AngleBracketRight;
1383 .angle_bracket_right => {
1384 result.id = .AngleBracketRight;
13851385 },
1386 State.AngleBracketAngleBracketLeft => {
1387 result.id = Token.Id.AngleBracketAngleBracketLeft;
1386 .angle_bracket_angle_bracket_left => {
1387 result.id = .AngleBracketAngleBracketLeft;
13881388 },
1389 State.AngleBracketLeft => {
1390 result.id = Token.Id.AngleBracketLeft;
1389 .angle_bracket_left => {
1390 result.id = .AngleBracketLeft;
13911391 },
1392 State.PlusPercent => {
1393 result.id = Token.Id.PlusPercent;
1392 .plus_percent => {
1393 result.id = .PlusPercent;
13941394 },
1395 State.Plus => {
1396 result.id = Token.Id.Plus;
1395 .plus => {
1396 result.id = .Plus;
13971397 },
1398 State.Percent => {
1399 result.id = Token.Id.Percent;
1398 .percent => {
1399 result.id = .Percent;
14001400 },
1401 State.Caret => {
1402 result.id = Token.Id.Caret;
1401 .caret => {
1402 result.id = .Caret;
14031403 },
1404 State.AsteriskPercent => {
1405 result.id = Token.Id.AsteriskPercent;
1404 .asterisk_percent => {
1405 result.id = .AsteriskPercent;
14061406 },
1407 State.Asterisk => {
1408 result.id = Token.Id.Asterisk;
1407 .asterisk => {
1408 result.id = .Asterisk;
14091409 },
1410 State.MinusPercent => {
1411 result.id = Token.Id.MinusPercent;
1410 .minus_percent => {
1411 result.id = .MinusPercent;
14121412 },
14131413 }
14141414 }
14151415
1416 if (result.id == Token.Id.Eof) {
1416 if (result.id == .Eof) {
14171417 if (self.pending_invalid_token) |token| {
14181418 self.pending_invalid_token = null;
14191419 return token;
......@@ -1428,8 +1428,8 @@ pub const Tokenizer = struct {
14281428 if (self.pending_invalid_token != null) return;
14291429 const invalid_length = self.getInvalidCharacterLength();
14301430 if (invalid_length == 0) return;
1431 self.pending_invalid_token = Token{
1432 .id = Token.Id.Invalid,
1431 self.pending_invalid_token = .{
1432 .id = .Invalid,
14331433 .start = self.index,
14341434 .end = self.index + invalid_length,
14351435 };
......@@ -1474,7 +1474,7 @@ pub const Tokenizer = struct {
14741474};
14751475
14761476test "tokenizer" {
1477 testTokenize("test", &[_]Token.Id{Token.Id.Keyword_test});
1477 testTokenize("test", &[_]Token.Id{.Keyword_test});
14781478}
14791479
14801480test "tokenizer - unknown length pointer and then c pointer" {
......@@ -1482,15 +1482,15 @@ test "tokenizer - unknown length pointer and then c pointer" {
14821482 \\[*]u8
14831483 \\[*c]u8
14841484 , &[_]Token.Id{
1485 Token.Id.LBracket,
1486 Token.Id.Asterisk,
1487 Token.Id.RBracket,
1488 Token.Id.Identifier,
1489 Token.Id.LBracket,
1490 Token.Id.Asterisk,
1491 Token.Id.Identifier,
1492 Token.Id.RBracket,
1493 Token.Id.Identifier,
1485 .LBracket,
1486 .Asterisk,
1487 .RBracket,
1488 .Identifier,
1489 .LBracket,
1490 .Asterisk,
1491 .Identifier,
1492 .RBracket,
1493 .Identifier,
14941494 });
14951495}
14961496
......@@ -1561,125 +1561,125 @@ test "tokenizer - char literal with unicode code point" {
15611561
15621562test "tokenizer - float literal e exponent" {
15631563 testTokenize("a = 4.94065645841246544177e-324;\n", &[_]Token.Id{
1564 Token.Id.Identifier,
1565 Token.Id.Equal,
1566 Token.Id.FloatLiteral,
1567 Token.Id.Semicolon,
1564 .Identifier,
1565 .Equal,
1566 .FloatLiteral,
1567 .Semicolon,
15681568 });
15691569}
15701570
15711571test "tokenizer - float literal p exponent" {
15721572 testTokenize("a = 0x1.a827999fcef32p+1022;\n", &[_]Token.Id{
1573 Token.Id.Identifier,
1574 Token.Id.Equal,
1575 Token.Id.FloatLiteral,
1576 Token.Id.Semicolon,
1573 .Identifier,
1574 .Equal,
1575 .FloatLiteral,
1576 .Semicolon,
15771577 });
15781578}
15791579
15801580test "tokenizer - chars" {
1581 testTokenize("'c'", &[_]Token.Id{Token.Id.CharLiteral});
1581 testTokenize("'c'", &[_]Token.Id{.CharLiteral});
15821582}
15831583
15841584test "tokenizer - invalid token characters" {
1585 testTokenize("#", &[_]Token.Id{Token.Id.Invalid});
1586 testTokenize("`", &[_]Token.Id{Token.Id.Invalid});
1587 testTokenize("'c", &[_]Token.Id{Token.Id.Invalid});
1588 testTokenize("'", &[_]Token.Id{Token.Id.Invalid});
1589 testTokenize("''", &[_]Token.Id{ Token.Id.Invalid, Token.Id.Invalid });
1585 testTokenize("#", &[_]Token.Id{.Invalid});
1586 testTokenize("`", &[_]Token.Id{.Invalid});
1587 testTokenize("'c", &[_]Token.Id{.Invalid});
1588 testTokenize("'", &[_]Token.Id{.Invalid});
1589 testTokenize("''", &[_]Token.Id{ .Invalid, .Invalid });
15901590}
15911591
15921592test "tokenizer - invalid literal/comment characters" {
15931593 testTokenize("\"\x00\"", &[_]Token.Id{
1594 Token.Id.StringLiteral,
1595 Token.Id.Invalid,
1594 .StringLiteral,
1595 .Invalid,
15961596 });
15971597 testTokenize("//\x00", &[_]Token.Id{
1598 Token.Id.LineComment,
1599 Token.Id.Invalid,
1598 .LineComment,
1599 .Invalid,
16001600 });
16011601 testTokenize("//\x1f", &[_]Token.Id{
1602 Token.Id.LineComment,
1603 Token.Id.Invalid,
1602 .LineComment,
1603 .Invalid,
16041604 });
16051605 testTokenize("//\x7f", &[_]Token.Id{
1606 Token.Id.LineComment,
1607 Token.Id.Invalid,
1606 .LineComment,
1607 .Invalid,
16081608 });
16091609}
16101610
16111611test "tokenizer - utf8" {
1612 testTokenize("//\xc2\x80", &[_]Token.Id{Token.Id.LineComment});
1613 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{Token.Id.LineComment});
1612 testTokenize("//\xc2\x80", &[_]Token.Id{.LineComment});
1613 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{.LineComment});
16141614}
16151615
16161616test "tokenizer - invalid utf8" {
16171617 testTokenize("//\x80", &[_]Token.Id{
1618 Token.Id.LineComment,
1619 Token.Id.Invalid,
1618 .LineComment,
1619 .Invalid,
16201620 });
16211621 testTokenize("//\xbf", &[_]Token.Id{
1622 Token.Id.LineComment,
1623 Token.Id.Invalid,
1622 .LineComment,
1623 .Invalid,
16241624 });
16251625 testTokenize("//\xf8", &[_]Token.Id{
1626 Token.Id.LineComment,
1627 Token.Id.Invalid,
1626 .LineComment,
1627 .Invalid,
16281628 });
16291629 testTokenize("//\xff", &[_]Token.Id{
1630 Token.Id.LineComment,
1631 Token.Id.Invalid,
1630 .LineComment,
1631 .Invalid,
16321632 });
16331633 testTokenize("//\xc2\xc0", &[_]Token.Id{
1634 Token.Id.LineComment,
1635 Token.Id.Invalid,
1634 .LineComment,
1635 .Invalid,
16361636 });
16371637 testTokenize("//\xe0", &[_]Token.Id{
1638 Token.Id.LineComment,
1639 Token.Id.Invalid,
1638 .LineComment,
1639 .Invalid,
16401640 });
16411641 testTokenize("//\xf0", &[_]Token.Id{
1642 Token.Id.LineComment,
1643 Token.Id.Invalid,
1642 .LineComment,
1643 .Invalid,
16441644 });
16451645 testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Id{
1646 Token.Id.LineComment,
1647 Token.Id.Invalid,
1646 .LineComment,
1647 .Invalid,
16481648 });
16491649}
16501650
16511651test "tokenizer - illegal unicode codepoints" {
16521652 // unicode newline characters.U+0085, U+2028, U+2029
1653 testTokenize("//\xc2\x84", &[_]Token.Id{Token.Id.LineComment});
1653 testTokenize("//\xc2\x84", &[_]Token.Id{.LineComment});
16541654 testTokenize("//\xc2\x85", &[_]Token.Id{
1655 Token.Id.LineComment,
1656 Token.Id.Invalid,
1655 .LineComment,
1656 .Invalid,
16571657 });
1658 testTokenize("//\xc2\x86", &[_]Token.Id{Token.Id.LineComment});
1659 testTokenize("//\xe2\x80\xa7", &[_]Token.Id{Token.Id.LineComment});
1658 testTokenize("//\xc2\x86", &[_]Token.Id{.LineComment});
1659 testTokenize("//\xe2\x80\xa7", &[_]Token.Id{.LineComment});
16601660 testTokenize("//\xe2\x80\xa8", &[_]Token.Id{
1661 Token.Id.LineComment,
1662 Token.Id.Invalid,
1661 .LineComment,
1662 .Invalid,
16631663 });
16641664 testTokenize("//\xe2\x80\xa9", &[_]Token.Id{
1665 Token.Id.LineComment,
1666 Token.Id.Invalid,
1665 .LineComment,
1666 .Invalid,
16671667 });
1668 testTokenize("//\xe2\x80\xaa", &[_]Token.Id{Token.Id.LineComment});
1668 testTokenize("//\xe2\x80\xaa", &[_]Token.Id{.LineComment});
16691669}
16701670
16711671test "tokenizer - string identifier and builtin fns" {
16721672 testTokenize(
16731673 \\const @"if" = @import("std");
16741674 , &[_]Token.Id{
1675 Token.Id.Keyword_const,
1676 Token.Id.Identifier,
1677 Token.Id.Equal,
1678 Token.Id.Builtin,
1679 Token.Id.LParen,
1680 Token.Id.StringLiteral,
1681 Token.Id.RParen,
1682 Token.Id.Semicolon,
1675 .Keyword_const,
1676 .Identifier,
1677 .Equal,
1678 .Builtin,
1679 .LParen,
1680 .StringLiteral,
1681 .RParen,
1682 .Semicolon,
16831683 });
16841684}
16851685
......@@ -1687,26 +1687,26 @@ test "tokenizer - multiline string literal with literal tab" {
16871687 testTokenize(
16881688 \\\\foo bar
16891689 , &[_]Token.Id{
1690 Token.Id.MultilineStringLiteralLine,
1690 .MultilineStringLiteralLine,
16911691 });
16921692}
16931693
16941694test "tokenizer - pipe and then invalid" {
16951695 testTokenize("||=", &[_]Token.Id{
1696 Token.Id.PipePipe,
1697 Token.Id.Equal,
1696 .PipePipe,
1697 .Equal,
16981698 });
16991699}
17001700
17011701test "tokenizer - line comment and doc comment" {
1702 testTokenize("//", &[_]Token.Id{Token.Id.LineComment});
1703 testTokenize("// a / b", &[_]Token.Id{Token.Id.LineComment});
1704 testTokenize("// /", &[_]Token.Id{Token.Id.LineComment});
1705 testTokenize("/// a", &[_]Token.Id{Token.Id.DocComment});
1706 testTokenize("///", &[_]Token.Id{Token.Id.DocComment});
1707 testTokenize("////", &[_]Token.Id{Token.Id.LineComment});
1708 testTokenize("//!", &[_]Token.Id{Token.Id.ContainerDocComment});
1709 testTokenize("//!!", &[_]Token.Id{Token.Id.ContainerDocComment});
1702 testTokenize("//", &[_]Token.Id{.LineComment});
1703 testTokenize("// a / b", &[_]Token.Id{.LineComment});
1704 testTokenize("// /", &[_]Token.Id{.LineComment});
1705 testTokenize("/// a", &[_]Token.Id{.DocComment});
1706 testTokenize("///", &[_]Token.Id{.DocComment});
1707 testTokenize("////", &[_]Token.Id{.LineComment});
1708 testTokenize("//!", &[_]Token.Id{.ContainerDocComment});
1709 testTokenize("//!!", &[_]Token.Id{.ContainerDocComment});
17101710}
17111711
17121712test "tokenizer - line comment followed by identifier" {
......@@ -1715,28 +1715,28 @@ test "tokenizer - line comment followed by identifier" {
17151715 \\ // another
17161716 \\ Another,
17171717 , &[_]Token.Id{
1718 Token.Id.Identifier,
1719 Token.Id.Comma,
1720 Token.Id.LineComment,
1721 Token.Id.Identifier,
1722 Token.Id.Comma,
1718 .Identifier,
1719 .Comma,
1720 .LineComment,
1721 .Identifier,
1722 .Comma,
17231723 });
17241724}
17251725
17261726test "tokenizer - UTF-8 BOM is recognized and skipped" {
17271727 testTokenize("\xEF\xBB\xBFa;\n", &[_]Token.Id{
1728 Token.Id.Identifier,
1729 Token.Id.Semicolon,
1728 .Identifier,
1729 .Semicolon,
17301730 });
17311731}
17321732
17331733test "correctly parse pointer assignment" {
17341734 testTokenize("b.*=3;\n", &[_]Token.Id{
1735 Token.Id.Identifier,
1736 Token.Id.PeriodAsterisk,
1737 Token.Id.Equal,
1738 Token.Id.IntegerLiteral,
1739 Token.Id.Semicolon,
1735 .Identifier,
1736 .PeriodAsterisk,
1737 .Equal,
1738 .IntegerLiteral,
1739 .Semicolon,
17401740 });
17411741}
17421742
......@@ -1979,5 +1979,5 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
19791979 }
19801980 }
19811981 const last_token = tokenizer.next();
1982 std.testing.expect(last_token.id == Token.Id.Eof);
1982 std.testing.expect(last_token.id == .Eof);
19831983}