authorgravatar for MyvarHD@gmail.comEmile Badenhorst <MyvarHD@gmail.com> 2023-07-03 17:47:08+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-07-03 17:47:08+02:00
log116a99d3cef91046bf132b6fcecbd2708f912896
treee86f4008a64b87bc0868a277bd962b74c177ac9d
parentde227ace14e09c7c17a60f716b88d2327fee201d
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Autodoc: Javascript port of zig lexer. (#16306)

* Most of the js lexer ported * Fixed lexical Ranges * Mini test framework * More testing and bug fixing in zig js lexer * Removed bad import * Implemented good html generation Closes #15863

1 files changed, 2109 insertions(+), 0 deletions(-)

lib/docs/ziglexer.js created+2109
...@@ -0,0 +1,2109 @@
1const Tag = {
2 invalid: "invalid",
3 identifier: "identifier",
4 string_literal: "string_literal",
5 multiline_string_literal_line: "multiline_string_literal_line",
6 char_literal: "char_literal",
7 eof: "eof",
8 builtin: "builtin",
9 number_literal: "number_literal",
10 doc_comment: "doc_comment",
11 container_doc_comment: "container_doc_comment",
12 invalid_periodasterisks: "invalid_periodasterisks",
13 bang: "bang",
14 pipe: "pipe",
15 pipe_pipe: "pipe_pipe",
16 pipe_equal: "pipe_equal",
17 equal: "equal",
18 equal_equal: "equal_equal",
19 equal_angle_bracket_right: "equal_angle_bracket_right",
20 bang_equal: "bang_equal",
21 l_paren: "l_paren",
22 r_paren: "r_paren",
23 semicolon: "semicolon",
24 percent: "percent",
25 percent_equal: "percent_equal",
26 l_brace: "l_brace",
27 r_brace: "r_brace",
28 l_bracket: "l_bracket",
29 r_bracket: "r_bracket",
30 period: "period",
31 period_asterisk: "period_asterisk",
32 ellipsis2: "ellipsis2",
33 ellipsis3: "ellipsis3",
34 caret: "caret",
35 caret_equal: "caret_equal",
36 plus: "plus",
37 plus_plus: "plus_plus",
38 plus_equal: "plus_equal",
39 plus_percent: "plus_percent",
40 plus_percent_equal: "plus_percent_equal",
41 plus_pipe: "plus_pipe",
42 plus_pipe_equal: "plus_pipe_equal",
43 minus: "minus",
44 minus_equal: "minus_equal",
45 minus_percent: "minus_percent",
46 minus_percent_equal: "minus_percent_equal",
47 minus_pipe: "minus_pipe",
48 minus_pipe_equal: "minus_pipe_equal",
49 asterisk: "asterisk",
50 asterisk_equal: "asterisk_equal",
51 asterisk_asterisk: "asterisk_asterisk",
52 asterisk_percent: "asterisk_percent",
53 asterisk_percent_equal: "asterisk_percent_equal",
54 asterisk_pipe: "asterisk_pipe",
55 asterisk_pipe_equal: "asterisk_pipe_equal",
56 arrow: "arrow",
57 colon: "colon",
58 slash: "slash",
59 slash_equal: "slash_equal",
60 comma: "comma",
61 ampersand: "ampersand",
62 ampersand_equal: "ampersand_equal",
63 question_mark: "question_mark",
64 angle_bracket_left: "angle_bracket_left",
65 angle_bracket_left_equal: "angle_bracket_left_equal",
66 angle_bracket_angle_bracket_left: "angle_bracket_angle_bracket_left",
67 angle_bracket_angle_bracket_left_equal: "angle_bracket_angle_bracket_left_equal",
68 angle_bracket_angle_bracket_left_pipe: "angle_bracket_angle_bracket_left_pipe",
69 angle_bracket_angle_bracket_left_pipe_equal: "angle_bracket_angle_bracket_left_pipe_equal",
70 angle_bracket_right: "angle_bracket_right",
71 angle_bracket_right_equal: "angle_bracket_right_equal",
72 angle_bracket_angle_bracket_right: "angle_bracket_angle_bracket_right",
73 angle_bracket_angle_bracket_right_equal: "angle_bracket_angle_bracket_right_equal",
74 tilde: "tilde",
75 keyword_addrspace: "keyword_addrspace",
76 keyword_align: "keyword_align",
77 keyword_allowzero: "keyword_allowzero",
78 keyword_and: "keyword_and",
79 keyword_anyframe: "keyword_anyframe",
80 keyword_anytype: "keyword_anytype",
81 keyword_asm: "keyword_asm",
82 keyword_async: "keyword_async",
83 keyword_await: "keyword_await",
84 keyword_break: "keyword_break",
85 keyword_callconv: "keyword_callconv",
86 keyword_catch: "keyword_catch",
87 keyword_comptime: "keyword_comptime",
88 keyword_const: "keyword_const",
89 keyword_continue: "keyword_continue",
90 keyword_defer: "keyword_defer",
91 keyword_else: "keyword_else",
92 keyword_enum: "keyword_enum",
93 keyword_errdefer: "keyword_errdefer",
94 keyword_error: "keyword_error",
95 keyword_export: "keyword_export",
96 keyword_extern: "keyword_extern",
97 keyword_fn: "keyword_fn",
98 keyword_for: "keyword_for",
99 keyword_if: "keyword_if",
100 keyword_inline: "keyword_inline",
101 keyword_noalias: "keyword_noalias",
102 keyword_noinline: "keyword_noinline",
103 keyword_nosuspend: "keyword_nosuspend",
104 keyword_opaque: "keyword_opaque",
105 keyword_or: "keyword_or",
106 keyword_orelse: "keyword_orelse",
107 keyword_packed: "keyword_packed",
108 keyword_pub: "keyword_pub",
109 keyword_resume: "keyword_resume",
110 keyword_return: "keyword_return",
111 keyword_linksection: "keyword_linksection",
112 keyword_struct: "keyword_struct",
113 keyword_suspend: "keyword_suspend",
114 keyword_switch: "keyword_switch",
115 keyword_test: "keyword_test",
116 keyword_threadlocal: "keyword_threadlocal",
117 keyword_try: "keyword_try",
118 keyword_union: "keyword_union",
119 keyword_unreachable: "keyword_unreachable",
120 keyword_usingnamespace: "keyword_usingnamespace",
121 keyword_var: "keyword_var",
122 keyword_volatile: "keyword_volatile",
123 keyword_while: "keyword_while"
124}
125
126const State = {
127 start: 0,
128 identifier: 1,
129 builtin: 2,
130 string_literal: 3,
131 string_literal_backslash: 4,
132 multiline_string_literal_line: 5,
133 char_literal: 6,
134 char_literal_backslash: 7,
135 char_literal_hex_escape: 8,
136 char_literal_unicode_escape_saw_u: 9,
137 char_literal_unicode_escape: 10,
138 char_literal_unicode_invalid: 11,
139 char_literal_unicode: 12,
140 char_literal_end: 13,
141 backslash: 14,
142 equal: 15,
143 bang: 16,
144 pipe: 17,
145 minus: 18,
146 minus_percent: 19,
147 minus_pipe: 20,
148 asterisk: 21,
149 asterisk_percent: 22,
150 asterisk_pipe: 23,
151 slash: 24,
152 line_comment_start: 25,
153 line_comment: 26,
154 doc_comment_start: 27,
155 doc_comment: 28,
156 int: 29,
157 int_exponent: 30,
158 int_period: 31,
159 float: 32,
160 float_exponent: 33,
161 ampersand: 34,
162 caret: 35,
163 percent: 36,
164 plus: 37,
165 plus_percent: 38,
166 plus_pipe: 39,
167 angle_bracket_left: 40,
168 angle_bracket_angle_bracket_left: 41,
169 angle_bracket_angle_bracket_left_pipe: 42,
170 angle_bracket_right: 43,
171 angle_bracket_angle_bracket_right: 44,
172 period: 45,
173 period_2: 46,
174 period_asterisk: 47,
175 saw_at_sign: 48,
176}
177
178const keywords = {
179 "addrspace": Tag.keyword_addrspace,
180 "align": Tag.keyword_align,
181 "allowzero": Tag.keyword_allowzero,
182 "and": Tag.keyword_and,
183 "anyframe": Tag.keyword_anyframe,
184 "anytype": Tag.keyword_anytype,
185 "asm": Tag.keyword_asm,
186 "async": Tag.keyword_async,
187 "await": Tag.keyword_await,
188 "break": Tag.keyword_break,
189 "callconv": Tag.keyword_callconv,
190 "catch": Tag.keyword_catch,
191 "comptime": Tag.keyword_comptime,
192 "const": Tag.keyword_const,
193 "continue": Tag.keyword_continue,
194 "defer": Tag.keyword_defer,
195 "else": Tag.keyword_else,
196 "enum": Tag.keyword_enum,
197 "errdefer": Tag.keyword_errdefer,
198 "error": Tag.keyword_error,
199 "export": Tag.keyword_export,
200 "extern": Tag.keyword_extern,
201 "fn": Tag.keyword_fn,
202 "for": Tag.keyword_for,
203 "if": Tag.keyword_if,
204 "inline": Tag.keyword_inline,
205 "noalias": Tag.keyword_noalias,
206 "noinline": Tag.keyword_noinline,
207 "nosuspend": Tag.keyword_nosuspend,
208 "opaque": Tag.keyword_opaque,
209 "or": Tag.keyword_or,
210 "orelse": Tag.keyword_orelse,
211 "packed": Tag.keyword_packed,
212 "pub": Tag.keyword_pub,
213 "resume": Tag.keyword_resume,
214 "return": Tag.keyword_return,
215 "linksection": Tag.keyword_linksection,
216 "struct": Tag.keyword_struct,
217 "suspend": Tag.keyword_suspend,
218 "switch": Tag.keyword_switch,
219 "test": Tag.keyword_test,
220 "threadlocal": Tag.keyword_threadlocal,
221 "try": Tag.keyword_try,
222 "union": Tag.keyword_union,
223 "unreachable": Tag.keyword_unreachable,
224 "usingnamespace": Tag.keyword_usingnamespace,
225 "var": Tag.keyword_var,
226 "volatile": Tag.keyword_volatile,
227 "while": Tag.keyword_while,
228};
229
230function make_token(tag, start, end) {
231 return {
232 tag: tag,
233 loc: {
234 start: start,
235 end: end
236 }
237 }
238
239}
240
241function dump_tokens(tokens, raw_source) {
242
243 //TODO: this is not very fast
244 function find_tag_key(tag) {
245 for (const [key, value] of Object.entries(Tag)) {
246 if (value == tag) return key;
247 }
248 }
249
250 for (let i = 0; i < tokens.length; i++) {
251 const tok = tokens[i];
252 const z = raw_source.substring(tok.loc.start, tok.loc.end).toLowerCase();
253 console.log(`${find_tag_key(tok.tag)} "${tok.tag}" '${z}'`)
254 }
255}
256
257
258
259function tokenize_zig_source(raw_source) {
260
261 var index = 0;
262 var flag = false;
263
264 let seen_escape_digits = undefined;
265 let remaining_code_units = undefined;
266
267 const next = () => {
268 let state = State.start;
269
270 var result = {
271 tag: -1,
272 loc: {
273 start: index,
274 end: undefined,
275 },
276 };
277
278 //having a while (true) loop seems like a bad idea the loop should never
279 //take more iterations than twice the length of the source code
280 const MAX_ITERATIONS = raw_source.length * 2;
281 let iterations = 0;
282
283 while (iterations <= MAX_ITERATIONS) {
284
285 if (flag) {
286 return make_token(Tag.eof, index - 2, index - 2);
287 }
288 iterations += 1; // avoid death loops
289
290 var c = raw_source[index];
291
292 if (c === undefined) {
293 c = ' '; // push the last token
294 flag = true;
295 }
296
297 switch (state) {
298 case State.start:
299 switch (c) {
300 case 0: {
301 if (index != raw_source.length) {
302 result.tag = Tag.invalid;
303 result.loc.start = index;
304 index += 1;
305 result.loc.end = index;
306 return result;
307 }
308 result.loc.end = index;
309 return result;
310 }
311 case ' ':
312 case '\n':
313 case '\t':
314 case '\r': {
315 result.loc.start = index + 1;
316 break;
317 }
318 case '"': {
319 state = State.string_literal;
320 result.tag = Tag.string_literal;
321 break;
322 }
323 case '\'': {
324 state = State.char_literal;
325 break;
326 }
327 case 'a':
328 case 'b':
329 case 'c':
330 case 'd':
331 case 'e':
332 case 'f':
333 case 'g':
334 case 'h':
335 case 'i':
336 case 'j':
337 case 'k':
338 case 'l':
339 case 'm':
340 case 'n':
341 case 'o':
342 case 'p':
343 case 'q':
344 case 'r':
345 case 's':
346 case 't':
347 case 'u':
348 case 'v':
349 case 'w':
350 case 'x':
351 case 'y':
352 case 'z':
353 case 'A':
354 case 'B':
355 case 'C':
356 case 'D':
357 case 'E':
358 case 'F':
359 case 'G':
360 case 'H':
361 case 'I':
362 case 'J':
363 case 'K':
364 case 'L':
365 case 'M':
366 case 'N':
367 case 'O':
368 case 'P':
369 case 'Q':
370 case 'R':
371 case 'S':
372 case 'T':
373 case 'U':
374 case 'V':
375 case 'W':
376 case 'X':
377 case 'Y':
378 case 'Z':
379 case '_': {
380 state = State.identifier;
381 result.tag = Tag.identifier;
382 break;
383 }
384 case '@': {
385 state = State.saw_at_sign;
386 break;
387 }
388 case '=': {
389 state = State.equal;
390 break;
391 }
392 case '!': {
393 state = State.bang;
394 break;
395 }
396 case '|': {
397 state = State.pipe;
398 break;
399 }
400 case '(': {
401 result.tag = Tag.l_paren;
402 index += 1;
403 result.loc.end = index;
404
405 return result;
406
407 }
408 case ')': {
409 result.tag = Tag.r_paren;
410 index += 1; result.loc.end = index;
411 return result;
412
413 }
414 case '[': {
415 result.tag = Tag.l_bracket;
416 index += 1; result.loc.end = index;
417 return result;
418
419 }
420 case ']': {
421 result.tag = Tag.r_bracket;
422 index += 1; result.loc.end = index;
423 return result;
424
425 }
426 case ';': {
427 result.tag = Tag.semicolon;
428 index += 1; result.loc.end = index;
429 return result;
430
431 }
432 case ',': {
433 result.tag = Tag.comma;
434 index += 1; result.loc.end = index;
435 return result;
436
437 }
438 case '?': {
439 result.tag = Tag.question_mark;
440 index += 1; result.loc.end = index;
441 return result;
442
443 }
444 case ':': {
445 result.tag = Tag.colon;
446 index += 1; result.loc.end = index;
447 return result;
448
449 }
450 case '%': {
451 state = State.percent; break;
452 }
453 case '*': {
454 state = State.asterisk; break;
455 }
456 case '+': {
457 state = State.plus; break;
458 }
459 case '<': {
460 state = State.angle_bracket_left; break;
461 }
462 case '>': {
463 state = State.angle_bracket_right; break;
464 }
465 case '^': {
466 state = State.caret; break;
467 }
468 case '\\': {
469 state = State.backslash;
470 result.tag = Tag.multiline_string_literal_line; break;
471 }
472 case '{': {
473 result.tag = Tag.l_brace;
474 index += 1; result.loc.end = index;
475 return result;
476
477 }
478 case '}': {
479 result.tag = Tag.r_brace;
480 index += 1; result.loc.end = index;
481 return result;
482
483 }
484 case '~': {
485 result.tag = Tag.tilde;
486 index += 1; result.loc.end = index;
487 return result;
488
489 }
490 case '.': {
491 state = State.period; break;
492 }
493 case '-': {
494 state = State.minus; break;
495 }
496 case '/': {
497 state = State.slash; break;
498 }
499 case '&': {
500 state = State.ampersand; break;
501 }
502 case '0':
503 case '1':
504 case '2':
505 case '3':
506 case '4':
507 case '5':
508 case '6':
509 case '7':
510 case '8':
511 case '9':
512 {
513 state = State.int;
514 result.tag = Tag.number_literal; break;
515 }
516 default: {
517 result.tag = Tag.invalid;
518 result.loc.end = index;
519 index += 1;
520 return result;
521 }
522 }
523 break;
524 case State.saw_at_sign:
525 switch (c) {
526 case '"': {
527 result.tag = Tag.identifier;
528 state = State.string_literal; break;
529 }
530 case 'a':
531 case 'b':
532 case 'c':
533 case 'd':
534 case 'e':
535 case 'f':
536 case 'g':
537 case 'h':
538 case 'i':
539 case 'j':
540 case 'k':
541 case 'l':
542 case 'm':
543 case 'n':
544 case 'o':
545 case 'p':
546 case 'q':
547 case 'r':
548 case 's':
549 case 't':
550 case 'u':
551 case 'v':
552 case 'w':
553 case 'x':
554 case 'y':
555 case 'z':
556 case 'A':
557 case 'B':
558 case 'C':
559 case 'D':
560 case 'E':
561 case 'F':
562 case 'G':
563 case 'H':
564 case 'I':
565 case 'J':
566 case 'K':
567 case 'L':
568 case 'M':
569 case 'N':
570 case 'O':
571 case 'P':
572 case 'Q':
573 case 'R':
574 case 'S':
575 case 'T':
576 case 'U':
577 case 'V':
578 case 'W':
579 case 'X':
580 case 'Y':
581 case 'Z':
582 case '_': {
583 state = State.builtin;
584 result.tag = Tag.builtin;
585 break;
586 }
587 default: {
588 result.tag = Tag.invalid;
589 result.loc.end = index;
590 return result;
591 }
592 }
593 break;
594 case State.ampersand:
595 switch (c) {
596 case '=': {
597 result.tag = Tag.ampersand_equal;
598 index += 1; result.loc.end = index;
599 return result;
600 }
601 default: {
602 result.tag = Tag.ampersand; result.loc.end = index;
603 return result;
604 }
605 }
606 break;
607 case State.asterisk: switch (c) {
608 case '=': {
609 result.tag = Tag.asterisk_equal;
610 index += 1; result.loc.end = index;
611 return result;
612 }
613 case '*': {
614 result.tag = Tag.asterisk_asterisk;
615 index += 1; result.loc.end = index;
616 return result;
617 }
618 case '%': {
619 state = State.asterisk_percent; break;
620 }
621 case '|': {
622 state = State.asterisk_pipe; break;
623 }
624 default: {
625 result.tag = Tag.asterisk;
626 result.loc.end = index;
627 return result;
628 }
629 }
630 break;
631 case State.asterisk_percent:
632 switch (c) {
633 case '=': {
634 result.tag = Tag.asterisk_percent_equal;
635 index += 1; result.loc.end = index;
636 return result;
637 }
638 default: {
639 result.tag = Tag.asterisk_percent;
640 result.loc.end = index;
641 return result;
642 }
643 }
644 break;
645 case State.asterisk_pipe:
646 switch (c) {
647 case '=': {
648 result.tag = Tag.asterisk_pipe_equal;
649 index += 1; result.loc.end = index;
650 return result;
651 }
652 default: {
653 result.tag = Tag.asterisk_pipe; result.loc.end = index;
654 return result;
655 }
656 }
657 break;
658 case State.percent:
659 switch (c) {
660 case '=': {
661 result.tag = Tag.percent_equal;
662 index += 1; result.loc.end = index;
663 return result;
664 }
665 default: {
666 result.tag = Tag.percent; result.loc.end = index;
667 return result;
668 }
669 }
670 break;
671 case State.plus:
672 switch (c) {
673 case '=': {
674 result.tag = Tag.plus_equal;
675 index += 1; result.loc.end = index;
676 return result;
677 }
678 case '+': {
679 result.tag = Tag.plus_plus;
680 index += 1; result.loc.end = index;
681 return result;
682 }
683 case '%': {
684 state = State.plus_percent; break;
685 }
686 case '|': {
687 state = State.plus_pipe; break;
688 }
689 default: {
690 result.tag = Tag.plus; result.loc.end = index;
691 return result;
692 }
693 }
694 break;
695 case State.plus_percent:
696 switch (c) {
697 case '=': {
698 result.tag = Tag.plus_percent_equal;
699 index += 1; result.loc.end = index;
700 return result;
701 }
702 default: {
703 result.tag = Tag.plus_percent; result.loc.end = index;
704 return result;
705 }
706 }
707 break;
708 case State.plus_pipe:
709 switch (c) {
710 case '=': {
711 result.tag = Tag.plus_pipe_equal;
712 index += 1; result.loc.end = index;
713 return result;
714 }
715 default: {
716 result.tag = Tag.plus_pipe; result.loc.end = index;
717 return result;
718 }
719 }
720 break;
721 case State.caret:
722 switch (c) {
723 case '=': {
724 result.tag = Tag.caret_equal;
725 index += 1; result.loc.end = index;
726 return result;
727 }
728 default: {
729 result.tag = Tag.caret; result.loc.end = index;
730 return result;
731 }
732 }
733 break;
734 case State.identifier:
735 switch (c) {
736 case 'a':
737 case 'b':
738 case 'c':
739 case 'd':
740 case 'e':
741 case 'f':
742 case 'g':
743 case 'h':
744 case 'i':
745 case 'j':
746 case 'k':
747 case 'l':
748 case 'm':
749 case 'n':
750 case 'o':
751 case 'p':
752 case 'q':
753 case 'r':
754 case 's':
755 case 't':
756 case 'u':
757 case 'v':
758 case 'w':
759 case 'x':
760 case 'y':
761 case 'z':
762 case 'A':
763 case 'B':
764 case 'C':
765 case 'D':
766 case 'E':
767 case 'F':
768 case 'G':
769 case 'H':
770 case 'I':
771 case 'J':
772 case 'K':
773 case 'L':
774 case 'M':
775 case 'N':
776 case 'O':
777 case 'P':
778 case 'Q':
779 case 'R':
780 case 'S':
781 case 'T':
782 case 'U':
783 case 'V':
784 case 'W':
785 case 'X':
786 case 'Y':
787 case 'Z':
788 case '_':
789 case '0':
790 case '1':
791 case '2':
792 case '3':
793 case '4':
794 case '5':
795 case '6':
796 case '7':
797 case '8':
798 case '9': break;
799 default: {
800 // if (Token.getKeyword(buffer[result.loc.start..index])) | tag | {
801 const z = raw_source.substring(result.loc.start, index);
802 if (z in keywords) {
803 result.tag = keywords[z];
804 }
805 result.loc.end = index;
806 return result;
807 }
808
809
810 }
811 break;
812 case State.builtin: switch (c) {
813 case 'a':
814 case 'b':
815 case 'c':
816 case 'd':
817 case 'e':
818 case 'f':
819 case 'g':
820 case 'h':
821 case 'i':
822 case 'j':
823 case 'k':
824 case 'l':
825 case 'm':
826 case 'n':
827 case 'o':
828 case 'p':
829 case 'q':
830 case 'r':
831 case 's':
832 case 't':
833 case 'u':
834 case 'v':
835 case 'w':
836 case 'x':
837 case 'y':
838 case 'z':
839 case 'A':
840 case 'B':
841 case 'C':
842 case 'D':
843 case 'E':
844 case 'F':
845 case 'G':
846 case 'H':
847 case 'I':
848 case 'J':
849 case 'K':
850 case 'L':
851 case 'M':
852 case 'N':
853 case 'O':
854 case 'P':
855 case 'Q':
856 case 'R':
857 case 'S':
858 case 'T':
859 case 'U':
860 case 'V':
861 case 'W':
862 case 'X':
863 case 'Y':
864 case 'Z':
865 case '_':
866 case '0':
867 case '1':
868 case '2':
869 case '3':
870 case '4':
871 case '5':
872 case '6':
873 case '7':
874 case '8':
875 case '9': break;
876 default: result.loc.end = index;
877 return result;
878 }
879 break;
880 case State.backslash:
881 switch (c) {
882 case '\\': {
883 state = State.multiline_string_literal_line;
884 break;
885 }
886 default: {
887 result.tag = Tag.invalid;
888 result.loc.end = index;
889 return result;
890 }
891 }
892 break;
893 case State.string_literal:
894 switch (c) {
895 case '\\': {
896 state = State.string_literal_backslash; break;
897 }
898 case '"': {
899 index += 1;
900 result.loc.end = index;
901
902 return result;
903 }
904 case 0: {
905 //TODO: PORT
906 // if (index == buffer.len) {
907 // result.tag = .invalid;
908 // break;
909 // } else {
910 // checkLiteralCharacter();
911 // }
912 result.loc.end = index;
913 return result;
914 }
915 case '\n': {
916 result.tag = Tag.invalid;
917 result.loc.end = index;
918 return result;
919 }
920 //TODO: PORT
921 //default: checkLiteralCharacter(),
922 }
923 break;
924 case State.string_literal_backslash:
925 switch (c) {
926 case 0:
927 case '\n': {
928 result.tag = Tag.invalid;
929 result.loc.end = index;
930 return result;
931 }
932 default: {
933 state = State.string_literal; break;
934 }
935 }
936 break;
937 case State.char_literal: switch (c) {
938 case 0: {
939 result.tag = Tag.invalid;
940 result.loc.end = index;
941 return result;
942 }
943 case '\\': {
944 state = State.char_literal_backslash;
945 break;
946 }
947 //TODO: PORT
948 // '\'', 0x80...0xbf, 0xf8...0xff => {
949 // result.tag = .invalid;
950 // break;
951 // },
952 // 0xc0...0xdf => { // 110xxxxx
953 // remaining_code_units = 1;
954 // state = .char_literal_unicode;
955 // },
956 // 0xe0...0xef => { // 1110xxxx
957 // remaining_code_units = 2;
958 // state = .char_literal_unicode;
959 // },
960 // 0xf0...0xf7 => { // 11110xxx
961 // remaining_code_units = 3;
962 // state = .char_literal_unicode;
963 // },
964
965 // case 0x80:
966 // case 0x81:
967 // case 0x82:
968 // case 0x83:
969 // case 0x84:
970 // case 0x85:
971 // case 0x86:
972 // case 0x87:
973 // case 0x88:
974 // case 0x89:
975 // case 0x8a:
976 // case 0x8b:
977 // case 0x8c:
978 // case 0x8d:
979 // case 0x8e:
980 // case 0x8f:
981 // case 0x90:
982 // case 0x91:
983 // case 0x92:
984 // case 0x93:
985 // case 0x94:
986 // case 0x95:
987 // case 0x96:
988 // case 0x97:
989 // case 0x98:
990 // case 0x99:
991 // case 0x9a:
992 // case 0x9b:
993 // case 0x9c:
994 // case 0x9d:
995 // case 0x9e:
996 // case 0x9f:
997 // case 0xa0:
998 // case 0xa1:
999 // case 0xa2:
1000 // case 0xa3:
1001 // case 0xa4:
1002 // case 0xa5:
1003 // case 0xa6:
1004 // case 0xa7:
1005 // case 0xa8:
1006 // case 0xa9:
1007 // case 0xaa:
1008 // case 0xab:
1009 // case 0xac:
1010 // case 0xad:
1011 // case 0xae:
1012 // case 0xaf:
1013 // case 0xb0:
1014 // case 0xb1:
1015 // case 0xb2:
1016 // case 0xb3:
1017 // case 0xb4:
1018 // case 0xb5:
1019 // case 0xb6:
1020 // case 0xb7:
1021 // case 0xb8:
1022 // case 0xb9:
1023 // case 0xba:
1024 // case 0xbb:
1025 // case 0xbc:
1026 // case 0xbd:
1027 // case 0xbe:
1028 // case 0xbf:
1029 // case 0xf8:
1030 // case 0xf9:
1031 // case 0xfa:
1032 // case 0xfb:
1033 // case 0xfc:
1034 // case 0xfd:
1035 // case 0xfe:
1036 // case 0xff:
1037 // result.tag = .invalid;
1038 // break;
1039 // case 0xc0:
1040 // case 0xc1:
1041 // case 0xc2:
1042 // case 0xc3:
1043 // case 0xc4:
1044 // case 0xc5:
1045 // case 0xc6:
1046 // case 0xc7:
1047 // case 0xc8:
1048 // case 0xc9:
1049 // case 0xca:
1050 // case 0xcb:
1051 // case 0xcc:
1052 // case 0xcd:
1053 // case 0xce:
1054 // case 0xcf:
1055 // case 0xd0:
1056 // case 0xd1:
1057 // case 0xd2:
1058 // case 0xd3:
1059 // case 0xd4:
1060 // case 0xd5:
1061 // case 0xd6:
1062 // case 0xd7:
1063 // case 0xd8:
1064 // case 0xd9:
1065 // case 0xda:
1066 // case 0xdb:
1067 // case 0xdc:
1068 // case 0xdd:
1069 // case 0xde:
1070 // case 0xdf:
1071 // remaining_code_units = 1;
1072 // state = .char_literal_unicode;
1073 // case 0xe0:
1074 // case 0xe1:
1075 // case 0xe2:
1076 // case 0xe3:
1077 // case 0xe4:
1078 // case 0xe5:
1079 // case 0xe6:
1080 // case 0xe7:
1081 // case 0xe8:
1082 // case 0xe9:
1083 // case 0xea:
1084 // case 0xeb:
1085 // case 0xec:
1086 // case 0xed:
1087 // case 0xee:
1088 // case 0xef:
1089 // remaining_code_units = 2;
1090 // state = .char_literal_unicode;
1091 // case 0xf0:
1092 // case 0xf1:
1093 // case 0xf2:
1094 // case 0xf3:
1095 // case 0xf4:
1096 // case 0xf5:
1097 // case 0xf6:
1098 // case 0xf7:
1099 // remaining_code_units = 3;
1100 // state = .char_literal_unicode;
1101
1102 case '\n': {
1103 result.tag = Tag.invalid;
1104 result.loc.end = index;
1105 return result;
1106 }
1107 default: {
1108 state = State.char_literal_end; break;
1109 }
1110 }
1111 break;
1112 case State.char_literal_backslash:
1113 switch (c) {
1114 case 0:
1115 case '\n': {
1116 result.tag = Tag.invalid;
1117 result.loc.end = index;
1118 return result;
1119 }
1120 case 'x': {
1121 state = State.char_literal_hex_escape;
1122 seen_escape_digits = 0; break;
1123 }
1124 case 'u': {
1125 state = State.char_literal_unicode_escape_saw_u; break;
1126 }
1127 default: {
1128 state = State.char_literal_end; break;
1129 }
1130 }
1131 break;
1132 case State.char_literal_hex_escape:
1133 switch (c) {
1134 case '0':
1135 case '1':
1136 case '2':
1137 case '3':
1138 case '4':
1139 case '5':
1140 case '6':
1141 case '7':
1142 case '8':
1143 case '9':
1144 case 'a':
1145 case 'b':
1146 case 'c':
1147 case 'd':
1148 case 'e':
1149 case 'f':
1150 case 'A':
1151 case 'B':
1152 case 'C':
1153 case 'D':
1154 case 'E':
1155 case 'F': {
1156 seen_escape_digits += 1;
1157 if (seen_escape_digits == 2) {
1158 state = State.char_literal_end;
1159 } break;
1160 }
1161 default: {
1162 result.tag = Tag.invalid;
1163 esult.loc.end = index;
1164 return result;
1165 }
1166 }
1167 break;
1168 case State.char_literal_unicode_escape_saw_u:
1169 switch (c) {
1170 case 0: {
1171 result.tag = Tag.invalid;
1172 result.loc.end = index;
1173 return result;
1174 }
1175 case '{': {
1176 state = State.char_literal_unicode_escape; break;
1177 }
1178 default: {
1179 result.tag = Tag.invalid;
1180 state = State.char_literal_unicode_invalid; break;
1181 }
1182 }
1183 break;
1184 case State.char_literal_unicode_escape:
1185 switch (c) {
1186 case 0: {
1187 result.tag = Tag.invalid;
1188 result.loc.end = index;
1189 return result;
1190 }
1191 case '0':
1192 case '1':
1193 case '2':
1194 case '3':
1195 case '4':
1196 case '5':
1197 case '6':
1198 case '7':
1199 case '8':
1200 case '9':
1201 case 'a':
1202 case 'b':
1203 case 'c':
1204 case 'd':
1205 case 'e':
1206 case 'f':
1207 case 'A':
1208 case 'B':
1209 case 'C':
1210 case 'D':
1211 case 'E':
1212 case 'F': break;
1213 case '}': {
1214 state = State.char_literal_end; // too many/few digits handled later
1215 break;
1216 }
1217 default: {
1218 result.tag = Tag.invalid;
1219 state = State.char_literal_unicode_invalid; break;
1220 }
1221 }
1222 break;
1223 case State.char_literal_unicode_invalid:
1224 switch (c) {
1225 // Keep consuming characters until an obvious stopping point.
1226 // This consolidates e.g. `u{0ab1Q}` into a single invalid token
1227 // instead of creating the tokens `u{0ab1`, `Q`, `}`
1228 case 'a':
1229 case 'b':
1230 case 'c':
1231 case 'd':
1232 case 'e':
1233 case 'f':
1234 case 'g':
1235 case 'h':
1236 case 'i':
1237 case 'j':
1238 case 'k':
1239 case 'l':
1240 case 'm':
1241 case 'n':
1242 case 'o':
1243 case 'p':
1244 case 'q':
1245 case 'r':
1246 case 's':
1247 case 't':
1248 case 'u':
1249 case 'v':
1250 case 'w':
1251 case 'x':
1252 case 'y':
1253 case 'z':
1254 case 'A':
1255 case 'B':
1256 case 'C':
1257 case 'D':
1258 case 'E':
1259 case 'F':
1260 case 'G':
1261 case 'H':
1262 case 'I':
1263 case 'J':
1264 case 'K':
1265 case 'L':
1266 case 'M':
1267 case 'N':
1268 case 'O':
1269 case 'P':
1270 case 'Q':
1271 case 'R':
1272 case 'S':
1273 case 'T':
1274 case 'U':
1275 case 'V':
1276 case 'W':
1277 case 'X':
1278 case 'Y':
1279 case 'Z':
1280 case '}':
1281 case '0':
1282 case '1':
1283 case '2':
1284 case '3':
1285 case '4':
1286 case '5':
1287 case '6':
1288 case '7':
1289 case '8':
1290 case '9': break;
1291 default: break;
1292 }
1293 break;
1294 case State.char_literal_end:
1295 switch (c) {
1296 case '\'': {
1297 result.tag = Tag.char_literal;
1298 index += 1;
1299 result.loc.end = index;
1300 return result;
1301 }
1302 default: {
1303 result.tag = Tag.invalid;
1304 result.loc.end = index;
1305 return result;
1306 }
1307 }
1308 break;
1309 case State.char_literal_unicode:
1310 switch (c) {
1311 // 0x80...0xbf => {
1312 // remaining_code_units -= 1;
1313 // if (remaining_code_units == 0) {
1314 // state = .char_literal_end;
1315 // }
1316 // },
1317 default: {
1318 result.tag = Tag.invalid;
1319 result.loc.end = index;
1320 return result;
1321 }
1322 }
1323 break;
1324 case State.multiline_string_literal_line:
1325 switch (c) {
1326 case 0:
1327 result.loc.end = index;
1328 return result;
1329 case '\n': {
1330
1331 index += 1;
1332 result.loc.end = index;
1333 return result;
1334 }
1335 case '\t': break;
1336 //TODO: PORT
1337 //default: checkLiteralCharacter(),
1338
1339 }
1340 break;
1341 case State.bang:
1342 switch (c) {
1343 case '=': {
1344 result.tag = Tag.bang_equal;
1345 index += 1;
1346 result.loc.end = index;
1347 return result;
1348 }
1349 default: {
1350 result.tag = Tag.bang;
1351 result.loc.end = index;
1352 return result;
1353 }
1354 }
1355 break;
1356 case State.pipe:
1357 switch (c) {
1358 case '=': {
1359 result.tag = Tag.pipe_equal;
1360 index += 1;
1361 result.loc.end = index;
1362 return result;
1363 }
1364 case '|': {
1365 result.tag = Tag.pipe_pipe;
1366 index += 1;
1367 result.loc.end = index;
1368 return result;
1369 }
1370 default: {
1371 result.tag = Tag.pipe;
1372 result.loc.end = index;
1373 return result;
1374 }
1375 }
1376 break;
1377 case State.equal: switch (c) {
1378 case '=': {
1379 result.tag = Tag.equal_equal;
1380 index += 1;
1381 result.loc.end = index;
1382 return result;
1383 }
1384 case '>': {
1385 result.tag = Tag.equal_angle_bracket_right;
1386 index += 1;
1387 result.loc.end = index;
1388 return result;
1389 }
1390 default: {
1391 result.tag = Tag.equal;
1392 result.loc.end = index;
1393 return result;
1394 }
1395 }
1396 break;
1397 case State.minus: switch (c) {
1398 case '>': {
1399 result.tag = Tag.arrow;
1400 index += 1;
1401 result.loc.end = index;
1402 return result;
1403 }
1404 case '=': {
1405 result.tag = Tag.minus_equal;
1406 index += 1;
1407 result.loc.end = index;
1408 return result;
1409 }
1410 case '%': {
1411 state = State.minus_percent; break;
1412 }
1413 case '|': {
1414 state = State.minus_pipe; break;
1415 }
1416 default: {
1417 result.tag = Tag.minus;
1418 result.loc.end = index;
1419 return result;
1420 }
1421 }
1422 break;
1423 case State.minus_percent:
1424 switch (c) {
1425 case '=': {
1426 result.tag = Tag.minus_percent_equal;
1427 index += 1;
1428 result.loc.end = index;
1429 return result;
1430 }
1431 default: {
1432 result.tag = Tag.minus_percent;
1433 result.loc.end = index;
1434 return result;
1435 }
1436 }
1437 break;
1438 case State.minus_pipe:
1439 switch (c) {
1440 case '=': {
1441 result.tag = Tag.minus_pipe_equal;
1442 index += 1;
1443 result.loc.end = index;
1444 return result;
1445 }
1446 default: {
1447 result.tag = Tag.minus_pipe;
1448 result.loc.end = index;
1449 return result;
1450 }
1451 }
1452 break;
1453 case State.angle_bracket_left:
1454 switch (c) {
1455 case '<': {
1456 state = State.angle_bracket_angle_bracket_left; break;
1457 }
1458 case '=': {
1459 result.tag = Tag.angle_bracket_left_equal;
1460 index += 1;
1461 result.loc.end = index;
1462 return result;
1463 }
1464 default: {
1465 result.tag = Tag.angle_bracket_left;
1466 result.loc.end = index;
1467 return result;
1468 }
1469 }
1470 break;
1471 case State.angle_bracket_angle_bracket_left:
1472 switch (c) {
1473 case '=': {
1474 result.tag = Tag.angle_bracket_angle_bracket_left_equal;
1475 index += 1;
1476 result.loc.end = index;
1477 return result;
1478 }
1479 case '|': {
1480 state = State.angle_bracket_angle_bracket_left_pipe;
1481 }
1482 default: {
1483 result.tag = Tag.angle_bracket_angle_bracket_left;
1484 result.loc.end = index;
1485 return result;
1486 }
1487 }
1488 break;
1489 case State.angle_bracket_angle_bracket_left_pipe:
1490 switch (c) {
1491 case '=': {
1492 result.tag = Tag.angle_bracket_angle_bracket_left_pipe_equal;
1493 index += 1;
1494 result.loc.end = index;
1495 return result;
1496 }
1497 default: {
1498 result.tag = Tag.angle_bracket_angle_bracket_left_pipe;
1499 result.loc.end = index;
1500 return result;
1501 }
1502 }
1503 break;
1504 case State.angle_bracket_right:
1505 switch (c) {
1506 case '>': {
1507 state = State.angle_bracket_angle_bracket_right; break;
1508 }
1509 case '=': {
1510 result.tag = Tag.angle_bracket_right_equal;
1511 index += 1;
1512 result.loc.end = index;
1513 return result;
1514 }
1515 default: {
1516 result.tag = Tag.angle_bracket_right;
1517 result.loc.end = index;
1518 return result;
1519 }
1520 }
1521 break;
1522 case State.angle_bracket_angle_bracket_right:
1523 switch (c) {
1524 case '=': {
1525 result.tag = Tag.angle_bracket_angle_bracket_right_equal;
1526 index += 1;
1527 result.loc.end = index;
1528 return result;
1529 }
1530 default: {
1531 result.tag = Tag.angle_bracket_angle_bracket_right;
1532 result.loc.end = index;
1533 return result;
1534 }
1535 }
1536 break;
1537 case State.period:
1538 switch (c) {
1539 case '.': {
1540 state = State.period_2; break;
1541 }
1542 case '*': {
1543 state = State.period_asterisk; break;
1544 }
1545 default: {
1546 result.tag = Tag.period;
1547 result.loc.end = index;
1548 return result;
1549 }
1550 }
1551 break;
1552 case State.period_2:
1553 switch (c) {
1554 case '.': {
1555 result.tag = Tag.ellipsis3;
1556 index += 1;
1557 result.loc.end = index;
1558 return result;
1559 }
1560 default: {
1561 result.tag = Tag.ellipsis2;
1562 result.loc.end = index;
1563 return result;
1564 }
1565 }
1566 break;
1567 case State.period_asterisk:
1568 switch (c) {
1569 case '*': {
1570 result.tag = Tag.invalid_periodasterisks;
1571 result.loc.end = index;
1572 return result;
1573 }
1574 default: {
1575 result.tag = Tag.period_asterisk;
1576 result.loc.end = index;
1577 return result;
1578 }
1579 }
1580 break;
1581 case State.slash:
1582 switch (c) {
1583 case '/': {
1584 state = State.line_comment_start;
1585 break;
1586 }
1587 case '=': {
1588 result.tag = Tag.slash_equal;
1589 index += 1;
1590 result.loc.end = index;
1591 return result;
1592 }
1593 default: {
1594 result.tag = Tag.slash;
1595 result.loc.end = index;
1596 return result;
1597 }
1598 } break;
1599 case State.line_comment_start:
1600 switch (c) {
1601 case 0: {
1602 if (index != raw_source.length) {
1603 result.tag = Tag.invalid;
1604 index += 1;
1605 }
1606 result.loc.end = index;
1607 return result;
1608 }
1609 case '/': {
1610 state = State.doc_comment_start; break;
1611 }
1612 case '!': {
1613 result.tag = Tag.container_doc_comment;
1614 state = State.doc_comment; break;
1615 }
1616 case '\n': {
1617 state = State.start;
1618 result.loc.start = index + 1; break;
1619 }
1620 case '\t':
1621 state = State.line_comment; break;
1622 default: {
1623 state = State.line_comment;
1624 //TODO: PORT
1625 //checkLiteralCharacter();
1626 break;
1627 }
1628 } break;
1629 case State.doc_comment_start:
1630 switch (c) {
1631 case '/': {
1632 state = State.line_comment; break;
1633 }
1634 case 0:
1635 case '\n':
1636 {
1637 result.tag = Tag.doc_comment;
1638 result.loc.end = index;
1639 return result;
1640 }
1641 case '\t': {
1642 state = State.doc_comment;
1643 result.tag = Tag.doc_comment; break;
1644 }
1645 default: {
1646 state = State.doc_comment;
1647 result.tag = Tag.doc_comment;
1648 //TODO: PORT
1649 //checkLiteralCharacter();
1650 break;
1651 }
1652 } break;
1653 case State.line_comment:
1654 switch (c) {
1655 case 0: {
1656 if (index != raw_source.length) {
1657 result.tag = Tag.invalid;
1658 index += 1;
1659 }
1660 result.loc.end = index;
1661 return result;
1662 }
1663 case '\n': {
1664 state = State.start;
1665 result.loc.start = index + 1;
1666 break;
1667 }
1668 case '\t': break;
1669 //TODO: PORT
1670 //default: checkLiteralCharacter(),
1671 } break;
1672 case State.doc_comment:
1673 switch (c) {
1674 case 0://
1675 case '\n':
1676 result.loc.end = index;
1677 return result;
1678 case '\t': break;
1679 //TODOL PORT
1680 // default: checkLiteralCharacter(),
1681 default:
1682 break;
1683 } break;
1684 case State.int:
1685 switch (c) {
1686 case '.':
1687 state = State.int_period;
1688 break;
1689 case '_':
1690 case 'a':
1691 case 'b':
1692 case 'c':
1693 case 'd':
1694 case 'f':
1695 case 'g':
1696 case 'h':
1697 case 'i':
1698 case 'j':
1699 case 'k':
1700 case 'l':
1701 case 'm':
1702 case 'n':
1703 case 'o':
1704 case 'q':
1705 case 'r':
1706 case 's':
1707 case 't':
1708 case 'u':
1709 case 'v':
1710 case 'w':
1711 case 'x':
1712 case 'y':
1713 case 'z':
1714 case 'A':
1715 case 'B':
1716 case 'C':
1717 case 'D':
1718 case 'F':
1719 case 'G':
1720 case 'H':
1721 case 'I':
1722 case 'J':
1723 case 'K':
1724 case 'L':
1725 case 'M':
1726 case 'N':
1727 case 'O':
1728 case 'Q':
1729 case 'R':
1730 case 'S':
1731 case 'T':
1732 case 'U':
1733 case 'V':
1734 case 'W':
1735 case 'X':
1736 case 'Y':
1737 case 'Z':
1738 case '0':
1739 case '1':
1740 case '2':
1741 case '3':
1742 case '4':
1743 case '5':
1744 case '6':
1745 case '7':
1746 case '8':
1747 case '9':
1748 break;
1749 case 'e':
1750 case 'E':
1751 case 'p':
1752 case 'P':
1753 state = State.int_exponent;
1754 break;
1755 default: result.loc.end = index;
1756 return result;
1757 } break;
1758 case State.int_exponent:
1759 switch (c) {
1760 case '-':
1761 case '+':
1762 {
1763 ``
1764 state = State.float; break;
1765 }
1766 default: {
1767 index -= 1;
1768 state = State.int; break;
1769 }
1770 } break;
1771 case State.int_period: switch (c) {
1772 case '_':
1773 case 'a':
1774 case 'b':
1775 case 'c':
1776 case 'd':
1777 case 'f':
1778 case 'g':
1779 case 'h':
1780 case 'i':
1781 case 'j':
1782 case 'k':
1783 case 'l':
1784 case 'm':
1785 case 'n':
1786 case 'o':
1787 case 'q':
1788 case 'r':
1789 case 's':
1790 case 't':
1791 case 'u':
1792 case 'v':
1793 case 'w':
1794 case 'x':
1795 case 'y':
1796 case 'z':
1797 case 'A':
1798 case 'B':
1799 case 'C':
1800 case 'D':
1801 case 'F':
1802 case 'G':
1803 case 'H':
1804 case 'I':
1805 case 'J':
1806 case 'K':
1807 case 'L':
1808 case 'M':
1809 case 'N':
1810 case 'O':
1811 case 'Q':
1812 case 'R':
1813 case 'S':
1814 case 'T':
1815 case 'U':
1816 case 'V':
1817 case 'W':
1818 case 'X':
1819 case 'Y':
1820 case 'Z':
1821 case '0':
1822 case '1':
1823 case '2':
1824 case '3':
1825 case '4':
1826 case '5':
1827 case '6':
1828 case '7':
1829 case '8':
1830 case '9': {
1831 state = State.float; break;
1832 }
1833 case 'e':
1834 case 'E':
1835 case 'p':
1836 case 'P':
1837 state = State.float_exponent; break;
1838 default: {
1839 index -= 1;
1840 result.loc.end = index;
1841 return result;
1842 }
1843 } break;
1844 case State.float:
1845 switch (c) {
1846 case '_':
1847 case 'a':
1848 case 'b':
1849 case 'c':
1850 case 'd':
1851 case 'f':
1852 case 'g':
1853 case 'h':
1854 case 'i':
1855 case 'j':
1856 case 'k':
1857 case 'l':
1858 case 'm':
1859 case 'n':
1860 case 'o':
1861 case 'q':
1862 case 'r':
1863 case 's':
1864 case 't':
1865 case 'u':
1866 case 'v':
1867 case 'w':
1868 case 'x':
1869 case 'y':
1870 case 'z':
1871 case 'A':
1872 case 'B':
1873 case 'C':
1874 case 'D':
1875 case 'F':
1876 case 'G':
1877 case 'H':
1878 case 'I':
1879 case 'J':
1880 case 'K':
1881 case 'L':
1882 case 'M':
1883 case 'N':
1884 case 'O':
1885 case 'Q':
1886 case 'R':
1887 case 'S':
1888 case 'T':
1889 case 'U':
1890 case 'V':
1891 case 'W':
1892 case 'X':
1893 case 'Y':
1894 case 'Z':
1895 case '0':
1896 case '1':
1897 case '2':
1898 case '3':
1899 case '4':
1900 case '5':
1901 case '6':
1902 case '7':
1903 case '8':
1904 case '9':
1905 break;
1906
1907 case 'e':
1908 case 'E':
1909 case 'p':
1910 case 'P':
1911 state = State.float_exponent; break;
1912 default: result.loc.end = index;
1913 return result;
1914 } break;
1915 case State.float_exponent:
1916 switch (c) {
1917 case '-':
1918 case '+':
1919 state = State.float; break;
1920 default: {
1921 index -= 1;
1922 state = State.float; break;
1923 }
1924 }
1925 break;
1926 }
1927 index += 1;
1928 }
1929
1930 //TODO: PORT
1931 // if (result.tag == Tag.eof) {
1932 // if (pending_invalid_token) | token | {
1933 // pending_invalid_token = null;
1934 // return token;
1935 // }
1936 // result.loc.start = sindex;
1937 // }
1938
1939 result.loc.end = index;
1940 return result;
1941
1942 }
1943
1944 toks = []
1945
1946 for (let i = 0; i < raw_source.length * 2; i++) {
1947 const tok = next();
1948 toks.push(tok);
1949
1950 if (tok.tag == Tag.eof) {
1951 break;
1952 }
1953 }
1954
1955 return toks;
1956}
1957
1958
1959function generate_html_for_src(src) {
1960 var toks = tokenize_zig_source(src);
1961 var html = [];
1962
1963 html.push("<pre>");
1964 let offset = 0;
1965 for (let z = 0; z < toks.length; z++) {
1966 const t = toks[z];
1967
1968 if(t.tag == Tag.eof)
1969 break;
1970
1971 const spanStart = `<span class="zig_${t.tag}">`
1972 const spanEnd = `</span>`
1973
1974 src = `${src.slice(0, t.loc.start + offset)}` + spanStart + `${src.slice(t.loc.start + offset)}`;
1975 offset += spanStart.length;
1976
1977 src = `${src.slice(0, t.loc.end + offset)}` + spanEnd + `${src.slice(t.loc.end + offset)}`;
1978 offset += spanEnd.length;
1979 }
1980
1981
1982 html.push(src);
1983 html.push("</pre>");
1984
1985 return html.join("");
1986
1987}
1988
1989//const fs = require('fs');
1990//const src = fs.readFileSync("../std/c.zig", 'utf8');
1991//console.log(generate_html_for_src(src));
1992
1993
1994// gist for zig_lexer_test code: https://gist.github.com/Myvar/2684ba4fb86b975274629d6f21eddc7b
1995// // Just for testing not to commit in pr
1996// var isNode = new Function("try {return this===global;}catch(e){return false;}");
1997// if (isNode()) {
1998
1999
2000// //const s = "const std = @import(\"std\");";
2001// //const toksa = tokenize_zig_source(s);
2002// //dump_tokens(toksa, s);
2003// //console.log(JSON.stringify(toksa));
2004
2005// const fs = require('fs');
2006
2007// function testFile(fileName) {
2008// //console.log(fileName);
2009// var exec = require('child_process').execFileSync;
2010// var passed = true;
2011// const zig_data = exec('./zig_lexer_test', [fileName]);
2012// const data = fs.readFileSync(fileName, 'utf8');
2013
2014// const toks = tokenize_zig_source(data);
2015// const a_json = toks;
2016
2017// // dump_tokens(a_json, data);
2018// // return;
2019
2020// const b_json = JSON.parse(zig_data.toString());
2021
2022// if (a_json.length !== b_json.length) {
2023// console.log("FAILED a and be is not the same length");
2024// passed = false;
2025// //return;
2026// }
2027
2028// let len = a_json.length;
2029// if (len >= b_json.length) len = b_json.length;
2030
2031// for (let i = 0; i < len; i++) {
2032// const a = a_json[i];
2033// const b = b_json[i];
2034
2035// // console.log(a.tag + " == " + b.tag);
2036
2037// if (a.tag !== b.tag) {
2038
2039// // console.log("Around here:");
2040// // console.log(
2041// // data.substring(b_json[i - 2].loc.start, b_json[i - 2].loc.end),
2042// // data.substring(b_json[i - 1].loc.start, b_json[i - 1].loc.end),
2043// // data.substring(b_json[i].loc.start, b_json[i].loc.end),
2044// // data.substring(b_json[i + 1].loc.start, b_json[i + 1].loc.end),
2045// // data.substring(b_json[i + 2].loc.start, b_json[i + 2].loc.end),
2046// // );
2047
2048// console.log("TAG: a != b");
2049// console.log("js", a.tag);
2050// console.log("zig", b.tag);
2051// passed = false;
2052// return;
2053// }
2054
2055// if (a.tag !== Tag.eof && a.loc.start !== b.loc.start) {
2056// console.log("START: a != b");
2057
2058// console.log("js", "\"" + data.substring(a_json[i ].loc.start, a_json[i].loc.end) + "\"");
2059// console.log("zig", "\"" + data.substring(b_json[i ].loc.start, b_json[i].loc.end) + "\"");
2060
2061
2062// passed = false;
2063// return;
2064// }
2065
2066// // if (a.tag !== Tag.eof && a.loc.end !== b.loc.end) {
2067// // console.log("END: a != b");
2068// // // console.log("Around here:");
2069// // // console.log(
2070// // // // data.substring(b_json[i - 2].loc.start, b_json[i - 2].loc.end),
2071// // // // data.substring(b_json[i - 1].loc.start, b_json[i - 1].loc.end),
2072// // // data.substring(b_json[i ].loc.start, b_json[i].loc.end),
2073// // // // data.substring(b_json[i + 1].loc.start, b_json[i + 1].loc.end),
2074// // // // data.substring(b_json[i + 2].loc.start, b_json[i + 2].loc.end),
2075// // // );
2076// // console.log("js", "\"" + data.substring(a_json[i ].loc.start, a_json[i].loc.end) + "\"");
2077// // console.log("zig", "\"" + data.substring(b_json[i ].loc.start, b_json[i].loc.end) + "\"");
2078// // passed = false;
2079// // return;
2080// // }
2081// }
2082// return passed;
2083// }
2084// var path = require('path');
2085// function fromDir(startPath, filter) {
2086// if (!fs.existsSync(startPath)) {
2087// console.log("no dir ", startPath);
2088// return;
2089// }
2090// var files = fs.readdirSync(startPath);
2091// for (var i = 0; i < files.length; i++) {
2092// var filename = path.join(startPath, files[i]);
2093// var stat = fs.lstatSync(filename);
2094// if (stat.isDirectory()) {
2095// fromDir(filename, filter); //recurse
2096// } else if (filename.endsWith(filter)) {
2097// try {
2098// console.log('-- TESTING: ', filename);
2099// console.log("\t\t", testFile(filename));
2100// }
2101// catch {
2102// }
2103// };
2104// };
2105// };
2106// fromDir('../std', '.zig');
2107// //console.log(testFile("/home/myvar/code/zig/lib/std/fmt/errol.zig"));
2108// //console.log(testFile("test.zig"));
2109// }
\ No newline at end of file