authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-09-10 19:59:43-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-09-10 19:59:43-07:00
logb81782366bcdaf18ad0066e029cfdb2ed028bddd
tree4cf606081ccc8b029f66ad9b8ecf56ecdcc604cd
parent421fbd939c450b34f310f380466cdc0d3d15133e
parent0e84fedbe040bca2993115c5ac198e8695a506a1
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #21367 from RetroDev256/faster-tokenizer

Update the zig tokenizer to use labeled switch statements

3 files changed, 456 insertions(+), 564 deletions(-)

lib/std/zig/tokenizer.zig+450-564
...@@ -402,7 +402,6 @@ pub const Tokenizer = struct {...@@ -402,7 +402,6 @@ pub const Tokenizer = struct {
402 /// After this returns invalid, it will reset on the next newline, returning tokens starting from there.402 /// After this returns invalid, it will reset on the next newline, returning tokens starting from there.
403 /// An eof token will always be returned at the end.403 /// An eof token will always be returned at the end.
404 pub fn next(self: *Tokenizer) Token {404 pub fn next(self: *Tokenizer) Token {
405 var state: State = .start;
406 var result: Token = .{405 var result: Token = .{
407 .tag = undefined,406 .tag = undefined,
408 .loc = .{407 .loc = .{
...@@ -410,676 +409,557 @@ pub const Tokenizer = struct {...@@ -410,676 +409,557 @@ pub const Tokenizer = struct {
410 .end = undefined,409 .end = undefined,
411 },410 },
412 };411 };
413 while (true) : (self.index += 1) {412 state: switch (State.start) {
414 const c = self.buffer[self.index];413 .start => switch (self.buffer[self.index]) {
415 switch (state) {414 0 => {
416 .start => switch (c) {415 if (self.index == self.buffer.len) {
417 0 => {416 return .{
418 if (self.index == self.buffer.len) return .{
419 .tag = .eof,417 .tag = .eof,
420 .loc = .{418 .loc = .{
421 .start = self.index,419 .start = self.index,
422 .end = self.index,420 .end = self.index,
423 },421 },
424 };422 };
425 state = .invalid;423 } else {
426 },424 continue :state .invalid;
427 ' ', '\n', '\t', '\r' => {425 }
428 result.loc.start = self.index + 1;426 },
429 },427 ' ', '\n', '\t', '\r' => {
430 '"' => {428 self.index += 1;
431 state = .string_literal;429 result.loc.start = self.index;
432 result.tag = .string_literal;430 continue :state .start;
433 },431 },
434 '\'' => {432 '"' => {
435 state = .char_literal;433 result.tag = .string_literal;
436 result.tag = .char_literal;434 continue :state .string_literal;
437 },435 },
438 'a'...'z', 'A'...'Z', '_' => {436 '\'' => {
439 state = .identifier;437 result.tag = .char_literal;
440 result.tag = .identifier;438 continue :state .char_literal;
441 },
442 '@' => {
443 state = .saw_at_sign;
444 },
445 '=' => {
446 state = .equal;
447 },
448 '!' => {
449 state = .bang;
450 },
451 '|' => {
452 state = .pipe;
453 },
454 '(' => {
455 result.tag = .l_paren;
456 self.index += 1;
457 break;
458 },
459 ')' => {
460 result.tag = .r_paren;
461 self.index += 1;
462 break;
463 },
464 '[' => {
465 result.tag = .l_bracket;
466 self.index += 1;
467 break;
468 },
469 ']' => {
470 result.tag = .r_bracket;
471 self.index += 1;
472 break;
473 },
474 ';' => {
475 result.tag = .semicolon;
476 self.index += 1;
477 break;
478 },
479 ',' => {
480 result.tag = .comma;
481 self.index += 1;
482 break;
483 },
484 '?' => {
485 result.tag = .question_mark;
486 self.index += 1;
487 break;
488 },
489 ':' => {
490 result.tag = .colon;
491 self.index += 1;
492 break;
493 },
494 '%' => {
495 state = .percent;
496 },
497 '*' => {
498 state = .asterisk;
499 },
500 '+' => {
501 state = .plus;
502 },
503 '<' => {
504 state = .angle_bracket_left;
505 },
506 '>' => {
507 state = .angle_bracket_right;
508 },
509 '^' => {
510 state = .caret;
511 },
512 '\\' => {
513 state = .backslash;
514 result.tag = .multiline_string_literal_line;
515 },
516 '{' => {
517 result.tag = .l_brace;
518 self.index += 1;
519 break;
520 },
521 '}' => {
522 result.tag = .r_brace;
523 self.index += 1;
524 break;
525 },
526 '~' => {
527 result.tag = .tilde;
528 self.index += 1;
529 break;
530 },
531 '.' => {
532 state = .period;
533 },
534 '-' => {
535 state = .minus;
536 },
537 '/' => {
538 state = .slash;
539 },
540 '&' => {
541 state = .ampersand;
542 },
543 '0'...'9' => {
544 state = .int;
545 result.tag = .number_literal;
546 },
547 else => {
548 state = .invalid;
549 },
550 },439 },
440 'a'...'z', 'A'...'Z', '_' => {
441 result.tag = .identifier;
442 continue :state .identifier;
443 },
444 '@' => continue :state .saw_at_sign,
445 '=' => continue :state .equal,
446 '!' => continue :state .bang,
447 '|' => continue :state .pipe,
448 '(' => {
449 result.tag = .l_paren;
450 self.index += 1;
451 },
452 ')' => {
453 result.tag = .r_paren;
454 self.index += 1;
455 },
456 '[' => {
457 result.tag = .l_bracket;
458 self.index += 1;
459 },
460 ']' => {
461 result.tag = .r_bracket;
462 self.index += 1;
463 },
464 ';' => {
465 result.tag = .semicolon;
466 self.index += 1;
467 },
468 ',' => {
469 result.tag = .comma;
470 self.index += 1;
471 },
472 '?' => {
473 result.tag = .question_mark;
474 self.index += 1;
475 },
476 ':' => {
477 result.tag = .colon;
478 self.index += 1;
479 },
480 '%' => continue :state .percent,
481 '*' => continue :state .asterisk,
482 '+' => continue :state .plus,
483 '<' => continue :state .angle_bracket_left,
484 '>' => continue :state .angle_bracket_right,
485 '^' => continue :state .caret,
486 '\\' => {
487 result.tag = .multiline_string_literal_line;
488 continue :state .backslash;
489 },
490 '{' => {
491 result.tag = .l_brace;
492 self.index += 1;
493 },
494 '}' => {
495 result.tag = .r_brace;
496 self.index += 1;
497 },
498 '~' => {
499 result.tag = .tilde;
500 self.index += 1;
501 },
502 '.' => continue :state .period,
503 '-' => continue :state .minus,
504 '/' => continue :state .slash,
505 '&' => continue :state .ampersand,
506 '0'...'9' => {
507 result.tag = .number_literal;
508 self.index += 1;
509 continue :state .int;
510 },
511 else => continue :state .invalid,
512 },
551513
552 .expect_newline => switch (c) {514 .expect_newline => {
515 self.index += 1;
516 switch (self.buffer[self.index]) {
553 0 => {517 0 => {
554 if (self.index == self.buffer.len) {518 if (self.index == self.buffer.len) {
555 result.tag = .invalid;519 result.tag = .invalid;
556 break;520 } else {
521 continue :state .invalid;
557 }522 }
558 state = .invalid;
559 },523 },
560 '\n' => {524 '\n' => {
561 result.loc.start = self.index + 1;525 self.index += 1;
562 state = .start;526 result.loc.start = self.index;
563 },527 continue :state .start;
564 else => {
565 state = .invalid;
566 },528 },
567 },529 else => continue :state .invalid,
530 }
531 },
568532
569 .invalid => switch (c) {533 .invalid => {
534 self.index += 1;
535 switch (self.buffer[self.index]) {
570 0 => if (self.index == self.buffer.len) {536 0 => if (self.index == self.buffer.len) {
571 result.tag = .invalid;537 result.tag = .invalid;
572 break;
573 },
574 '\n' => {
575 result.tag = .invalid;
576 break;
577 },538 },
578 else => continue,539 '\n' => result.tag = .invalid,
579 },540 else => continue :state .invalid,
541 }
542 },
580543
581 .saw_at_sign => switch (c) {544 .saw_at_sign => {
582 0, '\n' => {545 self.index += 1;
583 result.tag = .invalid;546 switch (self.buffer[self.index]) {
584 break;547 0, '\n' => result.tag = .invalid,
585 },
586 '"' => {548 '"' => {
587 result.tag = .identifier;549 result.tag = .identifier;
588 state = .string_literal;550 continue :state .string_literal;
589 },551 },
590 'a'...'z', 'A'...'Z', '_' => {552 'a'...'z', 'A'...'Z', '_' => {
591 state = .builtin;
592 result.tag = .builtin;553 result.tag = .builtin;
554 continue :state .builtin;
593 },555 },
594 else => {556 else => continue :state .invalid,
595 state = .invalid;557 }
596 },558 },
597 },
598559
599 .ampersand => switch (c) {560 .ampersand => {
561 self.index += 1;
562 switch (self.buffer[self.index]) {
600 '=' => {563 '=' => {
601 result.tag = .ampersand_equal;564 result.tag = .ampersand_equal;
602 self.index += 1;565 self.index += 1;
603 break;
604 },
605 else => {
606 result.tag = .ampersand;
607 break;
608 },566 },
609 },567 else => result.tag = .ampersand,
568 }
569 },
610570
611 .asterisk => switch (c) {571 .asterisk => {
572 self.index += 1;
573 switch (self.buffer[self.index]) {
612 '=' => {574 '=' => {
613 result.tag = .asterisk_equal;575 result.tag = .asterisk_equal;
614 self.index += 1;576 self.index += 1;
615 break;
616 },577 },
617 '*' => {578 '*' => {
618 result.tag = .asterisk_asterisk;579 result.tag = .asterisk_asterisk;
619 self.index += 1;580 self.index += 1;
620 break;
621 },
622 '%' => {
623 state = .asterisk_percent;
624 },
625 '|' => {
626 state = .asterisk_pipe;
627 },581 },
628 else => {582 '%' => continue :state .asterisk_percent,
629 result.tag = .asterisk;583 '|' => continue :state .asterisk_pipe,
630 break;584 else => result.tag = .asterisk,
631 },585 }
632 },586 },
633587
634 .asterisk_percent => switch (c) {588 .asterisk_percent => {
589 self.index += 1;
590 switch (self.buffer[self.index]) {
635 '=' => {591 '=' => {
636 result.tag = .asterisk_percent_equal;592 result.tag = .asterisk_percent_equal;
637 self.index += 1;593 self.index += 1;
638 break;
639 },
640 else => {
641 result.tag = .asterisk_percent;
642 break;
643 },594 },
644 },595 else => result.tag = .asterisk_percent,
596 }
597 },
645598
646 .asterisk_pipe => switch (c) {599 .asterisk_pipe => {
600 self.index += 1;
601 switch (self.buffer[self.index]) {
647 '=' => {602 '=' => {
648 result.tag = .asterisk_pipe_equal;603 result.tag = .asterisk_pipe_equal;
649 self.index += 1;604 self.index += 1;
650 break;
651 },
652 else => {
653 result.tag = .asterisk_pipe;
654 break;
655 },605 },
656 },606 else => result.tag = .asterisk_pipe,
607 }
608 },
657609
658 .percent => switch (c) {610 .percent => {
611 self.index += 1;
612 switch (self.buffer[self.index]) {
659 '=' => {613 '=' => {
660 result.tag = .percent_equal;614 result.tag = .percent_equal;
661 self.index += 1;615 self.index += 1;
662 break;
663 },616 },
664 else => {617 else => result.tag = .percent,
665 result.tag = .percent;618 }
666 break;619 },
667 },
668 },
669620
670 .plus => switch (c) {621 .plus => {
622 self.index += 1;
623 switch (self.buffer[self.index]) {
671 '=' => {624 '=' => {
672 result.tag = .plus_equal;625 result.tag = .plus_equal;
673 self.index += 1;626 self.index += 1;
674 break;
675 },627 },
676 '+' => {628 '+' => {
677 result.tag = .plus_plus;629 result.tag = .plus_plus;
678 self.index += 1;630 self.index += 1;
679 break;
680 },
681 '%' => {
682 state = .plus_percent;
683 },
684 '|' => {
685 state = .plus_pipe;
686 },
687 else => {
688 result.tag = .plus;
689 break;
690 },631 },
691 },632 '%' => continue :state .plus_percent,
633 '|' => continue :state .plus_pipe,
634 else => result.tag = .plus,
635 }
636 },
692637
693 .plus_percent => switch (c) {638 .plus_percent => {
639 self.index += 1;
640 switch (self.buffer[self.index]) {
694 '=' => {641 '=' => {
695 result.tag = .plus_percent_equal;642 result.tag = .plus_percent_equal;
696 self.index += 1;643 self.index += 1;
697 break;
698 },
699 else => {
700 result.tag = .plus_percent;
701 break;
702 },644 },
703 },645 else => result.tag = .plus_percent,
646 }
647 },
704648
705 .plus_pipe => switch (c) {649 .plus_pipe => {
650 self.index += 1;
651 switch (self.buffer[self.index]) {
706 '=' => {652 '=' => {
707 result.tag = .plus_pipe_equal;653 result.tag = .plus_pipe_equal;
708 self.index += 1;654 self.index += 1;
709 break;
710 },655 },
711 else => {656 else => result.tag = .plus_pipe,
712 result.tag = .plus_pipe;657 }
713 break;658 },
714 },
715 },
716659
717 .caret => switch (c) {660 .caret => {
661 self.index += 1;
662 switch (self.buffer[self.index]) {
718 '=' => {663 '=' => {
719 result.tag = .caret_equal;664 result.tag = .caret_equal;
720 self.index += 1;665 self.index += 1;
721 break;
722 },
723 else => {
724 result.tag = .caret;
725 break;
726 },666 },
727 },667 else => result.tag = .caret,
668 }
669 },
728670
729 .identifier => switch (c) {671 .identifier => {
730 'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,672 self.index += 1;
673 switch (self.buffer[self.index]) {
674 'a'...'z', 'A'...'Z', '_', '0'...'9' => continue :state .identifier,
731 else => {675 else => {
732 if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |tag| {676 const ident = self.buffer[result.loc.start..self.index];
677 if (Token.getKeyword(ident)) |tag| {
733 result.tag = tag;678 result.tag = tag;
734 }679 }
735 break;
736 },
737 },
738 .builtin => switch (c) {
739 'a'...'z', 'A'...'Z', '_', '0'...'9' => continue,
740 else => break,
741 },
742 .backslash => switch (c) {
743 0 => {
744 result.tag = .invalid;
745 break;
746 },680 },
747 '\\' => {681 }
748 state = .multiline_string_literal_line;682 },
749 },683 .builtin => {
750 '\n' => {684 self.index += 1;
751 result.tag = .invalid;685 switch (self.buffer[self.index]) {
752 break;686 'a'...'z', 'A'...'Z', '_', '0'...'9' => continue :state .builtin,
753 },687 else => {},
754 else => {688 }
755 state = .invalid;689 },
756 },690 .backslash => {
757 },691 self.index += 1;
758 .string_literal => switch (c) {692 switch (self.buffer[self.index]) {
693 0 => result.tag = .invalid,
694 '\\' => continue :state .multiline_string_literal_line,
695 '\n' => result.tag = .invalid,
696 else => continue :state .invalid,
697 }
698 },
699 .string_literal => {
700 self.index += 1;
701 switch (self.buffer[self.index]) {
759 0 => {702 0 => {
760 if (self.index != self.buffer.len) {703 if (self.index != self.buffer.len) {
761 state = .invalid;704 continue :state .invalid;
762 continue;705 } else {
706 result.tag = .invalid;
763 }707 }
764 result.tag = .invalid;
765 break;
766 },
767 '\n' => {
768 result.tag = .invalid;
769 break;
770 },
771 '\\' => {
772 state = .string_literal_backslash;
773 },
774 '"' => {
775 self.index += 1;
776 break;
777 },708 },
709 '\n' => result.tag = .invalid,
710 '\\' => continue :state .string_literal_backslash,
711 '"' => self.index += 1,
778 0x01...0x09, 0x0b...0x1f, 0x7f => {712 0x01...0x09, 0x0b...0x1f, 0x7f => {
779 state = .invalid;713 continue :state .invalid;
780 },714 },
781 else => continue,715 else => continue :state .string_literal,
782 },716 }
717 },
783718
784 .string_literal_backslash => switch (c) {719 .string_literal_backslash => {
785 0, '\n' => {720 self.index += 1;
786 result.tag = .invalid;721 switch (self.buffer[self.index]) {
787 break;722 0, '\n' => result.tag = .invalid,
788 },723 else => continue :state .string_literal,
789 else => {724 }
790 state = .string_literal;725 },
791 },
792 },
793726
794 .char_literal => switch (c) {727 .char_literal => {
728 self.index += 1;
729 switch (self.buffer[self.index]) {
795 0 => {730 0 => {
796 if (self.index != self.buffer.len) {731 if (self.index != self.buffer.len) {
797 state = .invalid;732 continue :state .invalid;
798 continue;733 } else {
734 result.tag = .invalid;
799 }735 }
800 result.tag = .invalid;
801 break;
802 },
803 '\n' => {
804 result.tag = .invalid;
805 break;
806 },
807 '\\' => {
808 state = .char_literal_backslash;
809 },
810 '\'' => {
811 self.index += 1;
812 break;
813 },736 },
737 '\n' => result.tag = .invalid,
738 '\\' => continue :state .char_literal_backslash,
739 '\'' => self.index += 1,
814 0x01...0x09, 0x0b...0x1f, 0x7f => {740 0x01...0x09, 0x0b...0x1f, 0x7f => {
815 state = .invalid;741 continue :state .invalid;
816 },742 },
817 else => continue,743 else => continue :state .char_literal,
818 },744 }
745 },
819746
820 .char_literal_backslash => switch (c) {747 .char_literal_backslash => {
748 self.index += 1;
749 switch (self.buffer[self.index]) {
821 0 => {750 0 => {
822 if (self.index != self.buffer.len) {751 if (self.index != self.buffer.len) {
823 state = .invalid;752 continue :state .invalid;
824 continue;753 } else {
754 result.tag = .invalid;
825 }755 }
826 result.tag = .invalid;
827 break;
828 },
829 '\n' => {
830 result.tag = .invalid;
831 break;
832 },756 },
757 '\n' => result.tag = .invalid,
833 0x01...0x09, 0x0b...0x1f, 0x7f => {758 0x01...0x09, 0x0b...0x1f, 0x7f => {
834 state = .invalid;759 continue :state .invalid;
835 },760 },
836 else => {761 else => continue :state .char_literal,
837 state = .char_literal;762 }
838 },763 },
839 },
840764
841 .multiline_string_literal_line => switch (c) {765 .multiline_string_literal_line => {
842 0 => {766 self.index += 1;
843 if (self.index != self.buffer.len) {767 switch (self.buffer[self.index]) {
844 state = .invalid;768 0 => if (self.index != self.buffer.len) {
845 continue;769 continue :state .invalid;
846 }
847 break;
848 },
849 '\n' => {
850 break;
851 },
852 '\r' => {
853 if (self.buffer[self.index + 1] == '\n') {
854 break;
855 } else {
856 state = .invalid;
857 }
858 },770 },
859 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {771 '\n' => {},
860 state = .invalid;772 '\r' => if (self.buffer[self.index + 1] != '\n') {
773 continue :state .invalid;
861 },774 },
862 else => continue,775 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => continue :state .invalid,
863 },776 else => continue :state .multiline_string_literal_line,
777 }
778 },
864779
865 .bang => switch (c) {780 .bang => {
781 self.index += 1;
782 switch (self.buffer[self.index]) {
866 '=' => {783 '=' => {
867 result.tag = .bang_equal;784 result.tag = .bang_equal;
868 self.index += 1;785 self.index += 1;
869 break;
870 },786 },
871 else => {787 else => result.tag = .bang,
872 result.tag = .bang;788 }
873 break;789 },
874 },
875 },
876790
877 .pipe => switch (c) {791 .pipe => {
792 self.index += 1;
793 switch (self.buffer[self.index]) {
878 '=' => {794 '=' => {
879 result.tag = .pipe_equal;795 result.tag = .pipe_equal;
880 self.index += 1;796 self.index += 1;
881 break;
882 },797 },
883 '|' => {798 '|' => {
884 result.tag = .pipe_pipe;799 result.tag = .pipe_pipe;
885 self.index += 1;800 self.index += 1;
886 break;
887 },
888 else => {
889 result.tag = .pipe;
890 break;
891 },801 },
892 },802 else => result.tag = .pipe,
803 }
804 },
893805
894 .equal => switch (c) {806 .equal => {
807 self.index += 1;
808 switch (self.buffer[self.index]) {
895 '=' => {809 '=' => {
896 result.tag = .equal_equal;810 result.tag = .equal_equal;
897 self.index += 1;811 self.index += 1;
898 break;
899 },812 },
900 '>' => {813 '>' => {
901 result.tag = .equal_angle_bracket_right;814 result.tag = .equal_angle_bracket_right;
902 self.index += 1;815 self.index += 1;
903 break;
904 },816 },
905 else => {817 else => result.tag = .equal,
906 result.tag = .equal;818 }
907 break;819 },
908 },
909 },
910820
911 .minus => switch (c) {821 .minus => {
822 self.index += 1;
823 switch (self.buffer[self.index]) {
912 '>' => {824 '>' => {
913 result.tag = .arrow;825 result.tag = .arrow;
914 self.index += 1;826 self.index += 1;
915 break;
916 },827 },
917 '=' => {828 '=' => {
918 result.tag = .minus_equal;829 result.tag = .minus_equal;
919 self.index += 1;830 self.index += 1;
920 break;
921 },
922 '%' => {
923 state = .minus_percent;
924 },831 },
925 '|' => {832 '%' => continue :state .minus_percent,
926 state = .minus_pipe;833 '|' => continue :state .minus_pipe,
927 },834 else => result.tag = .minus,
928 else => {835 }
929 result.tag = .minus;836 },
930 break;
931 },
932 },
933837
934 .minus_percent => switch (c) {838 .minus_percent => {
839 self.index += 1;
840 switch (self.buffer[self.index]) {
935 '=' => {841 '=' => {
936 result.tag = .minus_percent_equal;842 result.tag = .minus_percent_equal;
937 self.index += 1;843 self.index += 1;
938 break;
939 },
940 else => {
941 result.tag = .minus_percent;
942 break;
943 },844 },
944 },845 else => result.tag = .minus_percent,
945 .minus_pipe => switch (c) {846 }
847 },
848 .minus_pipe => {
849 self.index += 1;
850 switch (self.buffer[self.index]) {
946 '=' => {851 '=' => {
947 result.tag = .minus_pipe_equal;852 result.tag = .minus_pipe_equal;
948 self.index += 1;853 self.index += 1;
949 break;
950 },854 },
951 else => {855 else => result.tag = .minus_pipe,
952 result.tag = .minus_pipe;856 }
953 break;857 },
954 },
955 },
956858
957 .angle_bracket_left => switch (c) {859 .angle_bracket_left => {
958 '<' => {860 self.index += 1;
959 state = .angle_bracket_angle_bracket_left;861 switch (self.buffer[self.index]) {
960 },862 '<' => continue :state .angle_bracket_angle_bracket_left,
961 '=' => {863 '=' => {
962 result.tag = .angle_bracket_left_equal;864 result.tag = .angle_bracket_left_equal;
963 self.index += 1;865 self.index += 1;
964 break;
965 },866 },
966 else => {867 else => result.tag = .angle_bracket_left,
967 result.tag = .angle_bracket_left;868 }
968 break;869 },
969 },
970 },
971870
972 .angle_bracket_angle_bracket_left => switch (c) {871 .angle_bracket_angle_bracket_left => {
872 self.index += 1;
873 switch (self.buffer[self.index]) {
973 '=' => {874 '=' => {
974 result.tag = .angle_bracket_angle_bracket_left_equal;875 result.tag = .angle_bracket_angle_bracket_left_equal;
975 self.index += 1;876 self.index += 1;
976 break;
977 },
978 '|' => {
979 state = .angle_bracket_angle_bracket_left_pipe;
980 },
981 else => {
982 result.tag = .angle_bracket_angle_bracket_left;
983 break;
984 },877 },
985 },878 '|' => continue :state .angle_bracket_angle_bracket_left_pipe,
879 else => result.tag = .angle_bracket_angle_bracket_left,
880 }
881 },
986882
987 .angle_bracket_angle_bracket_left_pipe => switch (c) {883 .angle_bracket_angle_bracket_left_pipe => {
884 self.index += 1;
885 switch (self.buffer[self.index]) {
988 '=' => {886 '=' => {
989 result.tag = .angle_bracket_angle_bracket_left_pipe_equal;887 result.tag = .angle_bracket_angle_bracket_left_pipe_equal;
990 self.index += 1;888 self.index += 1;
991 break;
992 },889 },
993 else => {890 else => result.tag = .angle_bracket_angle_bracket_left_pipe,
994 result.tag = .angle_bracket_angle_bracket_left_pipe;891 }
995 break;892 },
996 },
997 },
998893
999 .angle_bracket_right => switch (c) {894 .angle_bracket_right => {
1000 '>' => {895 self.index += 1;
1001 state = .angle_bracket_angle_bracket_right;896 switch (self.buffer[self.index]) {
1002 },897 '>' => continue :state .angle_bracket_angle_bracket_right,
1003 '=' => {898 '=' => {
1004 result.tag = .angle_bracket_right_equal;899 result.tag = .angle_bracket_right_equal;
1005 self.index += 1;900 self.index += 1;
1006 break;
1007 },
1008 else => {
1009 result.tag = .angle_bracket_right;
1010 break;
1011 },901 },
1012 },902 else => result.tag = .angle_bracket_right,
903 }
904 },
1013905
1014 .angle_bracket_angle_bracket_right => switch (c) {906 .angle_bracket_angle_bracket_right => {
907 self.index += 1;
908 switch (self.buffer[self.index]) {
1015 '=' => {909 '=' => {
1016 result.tag = .angle_bracket_angle_bracket_right_equal;910 result.tag = .angle_bracket_angle_bracket_right_equal;
1017 self.index += 1;911 self.index += 1;
1018 break;
1019 },912 },
1020 else => {913 else => result.tag = .angle_bracket_angle_bracket_right,
1021 result.tag = .angle_bracket_angle_bracket_right;914 }
1022 break;915 },
1023 },
1024 },
1025916
1026 .period => switch (c) {917 .period => {
1027 '.' => {918 self.index += 1;
1028 state = .period_2;919 switch (self.buffer[self.index]) {
1029 },920 '.' => continue :state .period_2,
1030 '*' => {921 '*' => continue :state .period_asterisk,
1031 state = .period_asterisk;922 else => result.tag = .period,
1032 },923 }
1033 else => {924 },
1034 result.tag = .period;
1035 break;
1036 },
1037 },
1038925
1039 .period_2 => switch (c) {926 .period_2 => {
927 self.index += 1;
928 switch (self.buffer[self.index]) {
1040 '.' => {929 '.' => {
1041 result.tag = .ellipsis3;930 result.tag = .ellipsis3;
1042 self.index += 1;931 self.index += 1;
1043 break;
1044 },932 },
1045 else => {933 else => result.tag = .ellipsis2,
1046 result.tag = .ellipsis2;934 }
1047 break;935 },
1048 },
1049 },
1050936
1051 .period_asterisk => switch (c) {937 .period_asterisk => {
1052 '*' => {938 self.index += 1;
1053 result.tag = .invalid_periodasterisks;939 switch (self.buffer[self.index]) {
1054 break;940 '*' => result.tag = .invalid_periodasterisks,
1055 },941 else => result.tag = .period_asterisk,
1056 else => {942 }
1057 result.tag = .period_asterisk;943 },
1058 break;
1059 },
1060 },
1061944
1062 .slash => switch (c) {945 .slash => {
1063 '/' => {946 self.index += 1;
1064 state = .line_comment_start;947 switch (self.buffer[self.index]) {
1065 },948 '/' => continue :state .line_comment_start,
1066 '=' => {949 '=' => {
1067 result.tag = .slash_equal;950 result.tag = .slash_equal;
1068 self.index += 1;951 self.index += 1;
1069 break;
1070 },
1071 else => {
1072 result.tag = .slash;
1073 break;
1074 },952 },
1075 },953 else => result.tag = .slash,
1076 .line_comment_start => switch (c) {954 }
955 },
956 .line_comment_start => {
957 self.index += 1;
958 switch (self.buffer[self.index]) {
1077 0 => {959 0 => {
1078 if (self.index != self.buffer.len) {960 if (self.index != self.buffer.len) {
1079 state = .invalid;961 continue :state .invalid;
1080 continue;962 } else return .{
1081 }
1082 return .{
1083 .tag = .eof,963 .tag = .eof,
1084 .loc = .{964 .loc = .{
1085 .start = self.index,965 .start = self.index,
...@@ -1087,58 +967,51 @@ pub const Tokenizer = struct {...@@ -1087,58 +967,51 @@ pub const Tokenizer = struct {
1087 },967 },
1088 };968 };
1089 },969 },
1090 '/' => {
1091 state = .doc_comment_start;
1092 },
1093 '!' => {970 '!' => {
1094 result.tag = .container_doc_comment;971 result.tag = .container_doc_comment;
1095 state = .doc_comment;972 continue :state .doc_comment;
1096 },
1097 '\r' => {
1098 state = .expect_newline;
1099 },973 },
1100 '\n' => {974 '\n' => {
1101 state = .start;975 self.index += 1;
1102 result.loc.start = self.index + 1;976 result.loc.start = self.index;
977 continue :state .start;
1103 },978 },
979 '/' => continue :state .doc_comment_start,
980 '\r' => continue :state .expect_newline,
1104 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {981 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1105 state = .invalid;982 continue :state .invalid;
1106 },
1107 else => {
1108 state = .line_comment;
1109 },
1110 },
1111 .doc_comment_start => switch (c) {
1112 0, '\n' => {
1113 result.tag = .doc_comment;
1114 break;
1115 },983 },
984 else => continue :state .line_comment,
985 }
986 },
987 .doc_comment_start => {
988 self.index += 1;
989 switch (self.buffer[self.index]) {
990 0, '\n' => result.tag = .doc_comment,
1116 '\r' => {991 '\r' => {
1117 if (self.buffer[self.index + 1] == '\n') {992 if (self.buffer[self.index + 1] == '\n') {
1118 result.tag = .doc_comment;993 result.tag = .doc_comment;
1119 break;
1120 } else {994 } else {
1121 state = .invalid;995 continue :state .invalid;
1122 }996 }
1123 },997 },
1124 '/' => {998 '/' => continue :state .line_comment,
1125 state = .line_comment;
1126 },
1127 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {999 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1128 state = .invalid;1000 continue :state .invalid;
1129 },1001 },
1130 else => {1002 else => {
1131 state = .doc_comment;
1132 result.tag = .doc_comment;1003 result.tag = .doc_comment;
1004 continue :state .doc_comment;
1133 },1005 },
1134 },1006 }
1135 .line_comment => switch (c) {1007 },
1008 .line_comment => {
1009 self.index += 1;
1010 switch (self.buffer[self.index]) {
1136 0 => {1011 0 => {
1137 if (self.index != self.buffer.len) {1012 if (self.index != self.buffer.len) {
1138 state = .invalid;1013 continue :state .invalid;
1139 continue;1014 } else return .{
1140 }
1141 return .{
1142 .tag = .eof,1015 .tag = .eof,
1143 .loc = .{1016 .loc = .{
1144 .start = self.index,1017 .start = self.index,
...@@ -1146,72 +1019,85 @@ pub const Tokenizer = struct {...@@ -1146,72 +1019,85 @@ pub const Tokenizer = struct {
1146 },1019 },
1147 };1020 };
1148 },1021 },
1149 '\r' => {
1150 state = .expect_newline;
1151 },
1152 '\n' => {1022 '\n' => {
1153 state = .start;1023 self.index += 1;
1154 result.loc.start = self.index + 1;1024 result.loc.start = self.index;
1025 continue :state .start;
1155 },1026 },
1027 '\r' => continue :state .expect_newline,
1156 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {1028 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1157 state = .invalid;1029 continue :state .invalid;
1158 },1030 },
1159 else => continue,1031 else => continue :state .line_comment,
1160 },1032 }
1161 .doc_comment => switch (c) {1033 },
1162 0, '\n' => {1034 .doc_comment => {
1163 break;1035 self.index += 1;
1164 },1036 switch (self.buffer[self.index]) {
1165 '\r' => {1037 0, '\n' => {},
1166 if (self.buffer[self.index + 1] == '\n') {1038 '\r' => if (self.buffer[self.index + 1] != '\n') {
1167 break;1039 continue :state .invalid;
1168 } else {
1169 state = .invalid;
1170 }
1171 },1040 },
1172 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {1041 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1173 state = .invalid;1042 continue :state .invalid;
1174 },1043 },
1175 else => continue,1044 else => continue :state .doc_comment,
1045 }
1046 },
1047 .int => switch (self.buffer[self.index]) {
1048 '.' => continue :state .int_period,
1049 '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {
1050 self.index += 1;
1051 continue :state .int;
1176 },1052 },
1177 .int => switch (c) {1053 'e', 'E', 'p', 'P' => {
1178 '.' => state = .int_period,1054 continue :state .int_exponent;
1179 '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,
1180 'e', 'E', 'p', 'P' => state = .int_exponent,
1181 else => break,
1182 },1055 },
1183 .int_exponent => switch (c) {1056 else => {},
1057 },
1058 .int_exponent => {
1059 self.index += 1;
1060 switch (self.buffer[self.index]) {
1184 '-', '+' => {1061 '-', '+' => {
1185 state = .float;1062 self.index += 1;
1186 },1063 continue :state .float;
1187 else => {
1188 self.index -= 1;
1189 state = .int;
1190 },1064 },
1191 },1065 else => continue :state .int,
1192 .int_period => switch (c) {1066 }
1067 },
1068 .int_period => {
1069 self.index += 1;
1070 switch (self.buffer[self.index]) {
1193 '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {1071 '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {
1194 state = .float;1072 self.index += 1;
1073 continue :state .float;
1195 },1074 },
1196 'e', 'E', 'p', 'P' => state = .float_exponent,1075 'e', 'E', 'p', 'P' => {
1197 else => {1076 continue :state .float_exponent;
1198 self.index -= 1;
1199 break;
1200 },1077 },
1078 else => self.index -= 1,
1079 }
1080 },
1081 .float => switch (self.buffer[self.index]) {
1082 '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => {
1083 self.index += 1;
1084 continue :state .float;
1201 },1085 },
1202 .float => switch (c) {1086 'e', 'E', 'p', 'P' => {
1203 '_', 'a'...'d', 'f'...'o', 'q'...'z', 'A'...'D', 'F'...'O', 'Q'...'Z', '0'...'9' => continue,1087 continue :state .float_exponent;
1204 'e', 'E', 'p', 'P' => state = .float_exponent,
1205 else => break,
1206 },1088 },
1207 .float_exponent => switch (c) {1089 else => {},
1208 '-', '+' => state = .float,1090 },
1209 else => {1091 .float_exponent => {
1210 self.index -= 1;1092 self.index += 1;
1211 state = .float;1093 switch (self.buffer[self.index]) {
1094 '-', '+' => {
1095 self.index += 1;
1096 continue :state .float;
1212 },1097 },
1213 },1098 else => continue :state .float,
1214 }1099 }
1100 },
1215 }1101 }
12161102
1217 result.loc.end = self.index;1103 result.loc.end = self.index;
src/codegen/llvm.zig+6
...@@ -6544,6 +6544,12 @@ pub const FuncGen = struct {...@@ -6544,6 +6544,12 @@ pub const FuncGen = struct {
65446544
6545 const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: {6545 const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: {
6546 if (!is_dispatch_loop) break :jmp_table null;6546 if (!is_dispatch_loop) break :jmp_table null;
6547
6548 // Workaround for:
6549 // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/lib/MC/WasmObjectWriter.cpp#L560
6550 // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll
6551 if (zcu.comp.getTarget().isWasm()) break :jmp_table null;
6552
6547 // On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just6553 // On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just
6548 // about acceptable - it won't fill L1d cache on most CPUs.6554 // about acceptable - it won't fill L1d cache on most CPUs.
6549 const max_table_len = 1024;6555 const max_table_len = 1024;
stage1/zig1.wasm
Binary files a/stage1/zig1.wasm and b/stage1/zig1.wasm differ