authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2015-12-14 23:13:53-07:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2015-12-15 04:05:53-07:00
logcf88fcb2ad31b64296d906ef525e76bbe6c3f65e
tree46ce6c05ed94593ae29dd26f577095662905f55a
parentde7c5ea1026db1c32fac522c71e9d5c57de6ed14

tokenizer parses all number literal types

2 files changed, 151 insertions(+), 7 deletions(-)

src/tokenizer.cpp+146-7
...@@ -13,12 +13,11 @@...@@ -13,12 +13,11 @@
13#include <stdio.h>13#include <stdio.h>
1414
15#define WHITESPACE \15#define WHITESPACE \
16 ' ': \16 ' ': \
17 case '\n'17 case '\n'
1818
19#define DIGIT \19#define DIGIT_NON_ZERO \
20 '0': \20 '1': \
21 case '1': \
22 case '2': \21 case '2': \
23 case '3': \22 case '3': \
24 case '4': \23 case '4': \
...@@ -27,10 +26,14 @@...@@ -27,10 +26,14 @@
27 case '7': \26 case '7': \
28 case '8': \27 case '8': \
29 case '9'28 case '9'
29#define DIGIT \
30 '0': \
31 case DIGIT_NON_ZERO
3032
31#define ALPHA_EXCEPT_C \33#define ALPHA_EXCEPT_C \
32 'a': \34 'a': \
33 case 'b': \35 case 'b': \
36 /*case 'c':*/ \
34 case 'd': \37 case 'd': \
35 case 'e': \38 case 'e': \
36 case 'f': \39 case 'f': \
...@@ -94,7 +97,11 @@ enum TokenizeState {...@@ -94,7 +97,11 @@ enum TokenizeState {
94 TokenizeStateStart,97 TokenizeStateStart,
95 TokenizeStateSymbol,98 TokenizeStateSymbol,
96 TokenizeStateSymbolFirst,99 TokenizeStateSymbolFirst,
97 TokenizeStateNumber,100 TokenizeStateZero, // "0", which might lead to "0x"
101 TokenizeStateNumber, // "123", "0x123"
102 TokenizeStateFloatFraction, // "123.456", "0x123.456"
103 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
104 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
98 TokenizeStateString,105 TokenizeStateString,
99 TokenizeStateSawStar,106 TokenizeStateSawStar,
100 TokenizeStateSawSlash,107 TokenizeStateSawSlash,
...@@ -172,6 +179,16 @@ static void end_token(Tokenize *t) {...@@ -172,6 +179,16 @@ static void end_token(Tokenize *t) {
172 assert(t->cur_tok);179 assert(t->cur_tok);
173 t->cur_tok->end_pos = t->pos + 1;180 t->cur_tok->end_pos = t->pos + 1;
174181
182 // normalize number literal parsing stuff
183 if (t->cur_tok->id == TokenIdNumberLiteral) {
184 if (t->cur_tok->exponent_marker_pos == 0) {
185 t->cur_tok->exponent_marker_pos = t->cur_tok->end_pos;
186 }
187 if (t->cur_tok->decimal_point_pos == 0) {
188 t->cur_tok->decimal_point_pos = t->cur_tok->exponent_marker_pos;
189 }
190 }
191
175 char *token_mem = buf_ptr(t->buf) + t->cur_tok->start_pos;192 char *token_mem = buf_ptr(t->buf) + t->cur_tok->start_pos;
176 int token_len = t->cur_tok->end_pos - t->cur_tok->start_pos;193 int token_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
177194
...@@ -218,6 +235,26 @@ static void end_token(Tokenize *t) {...@@ -218,6 +235,26 @@ static void end_token(Tokenize *t) {
218 t->cur_tok = nullptr;235 t->cur_tok = nullptr;
219}236}
220237
238static bool is_exponent_signifier(uint8_t c, int radix) {
239 if (radix == 16) {
240 return c == 'p' || c == 'P';
241 } else {
242 return c == 'e' || c == 'E';
243 }
244}
245static int get_digit_value(uint8_t c) {
246 if ('0' <= c && c <= '9') {
247 return c - '0';
248 }
249 if ('A' <= c && c <= 'Z') {
250 return c - 'A' + 10;
251 }
252 if ('a' <= c && c <= 'z') {
253 return c - 'a' + 10;
254 }
255 return -1;
256}
257
221void tokenize(Buf *buf, Tokenization *out) {258void tokenize(Buf *buf, Tokenization *out) {
222 Tokenize t = {0};259 Tokenize t = {0};
223 t.out = out;260 t.out = out;
...@@ -245,9 +282,15 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -245,9 +282,15 @@ void tokenize(Buf *buf, Tokenization *out) {
245 t.state = TokenizeStateSymbol;282 t.state = TokenizeStateSymbol;
246 begin_token(&t, TokenIdSymbol);283 begin_token(&t, TokenIdSymbol);
247 break;284 break;
248 case DIGIT:285 case '0':
286 t.state = TokenizeStateZero;
287 begin_token(&t, TokenIdNumberLiteral);
288 t.cur_tok->radix = 10;
289 break;
290 case DIGIT_NON_ZERO:
249 t.state = TokenizeStateNumber;291 t.state = TokenizeStateNumber;
250 begin_token(&t, TokenIdNumberLiteral);292 begin_token(&t, TokenIdNumberLiteral);
293 t.cur_tok->radix = 10;
251 break;294 break;
252 case '"':295 case '"':
253 begin_token(&t, TokenIdStringLiteral);296 begin_token(&t, TokenIdStringLiteral);
...@@ -701,10 +744,102 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -701,10 +744,102 @@ void tokenize(Buf *buf, Tokenization *out) {
701 break;744 break;
702 }745 }
703 break;746 break;
747 case TokenizeStateZero:
748 switch (c) {
749 case 'b':
750 t.cur_tok->radix = 2;
751 break;
752 case 'o':
753 t.cur_tok->radix = 8;
754 break;
755 case 'x':
756 t.cur_tok->radix = 16;
757 break;
758 default:
759 // reinterpret as normal number
760 t.pos -= 1;
761 t.state = TokenizeStateNumber;
762 continue;
763 }
764 break;
704 case TokenizeStateNumber:765 case TokenizeStateNumber:
766 {
767 if (c == '.') {
768 t.cur_tok->decimal_point_pos = t.pos;
769 t.state = TokenizeStateFloatFraction;
770 break;
771 }
772 if (is_exponent_signifier(c, t.cur_tok->radix)) {
773 t.cur_tok->exponent_marker_pos = t.pos;
774 t.state = TokenizeStateFloatExponentUnsigned;
775 break;
776 }
777 if (c == '_') {
778 tokenize_error(&t, "invalid character: '%c'", c);
779 break;
780 }
781 int digit_value = get_digit_value(c);
782 if (digit_value >= 0) {
783 if (digit_value >= t.cur_tok->radix) {
784 tokenize_error(&t, "invalid character: '%c'", c);
785 }
786 // normal digit
787 } else {
788 // not my char
789 t.pos -= 1;
790 end_token(&t);
791 t.state = TokenizeStateStart;
792 continue;
793 }
794 break;
795 }
796 case TokenizeStateFloatFraction:
797 {
798 if (is_exponent_signifier(c, t.cur_tok->radix)) {
799 t.cur_tok->exponent_marker_pos = t.pos;
800 t.state = TokenizeStateFloatExponentUnsigned;
801 break;
802 }
803 if (c == '_') {
804 tokenize_error(&t, "invalid character: '%c'", c);
805 break;
806 }
807 int digit_value = get_digit_value(c);
808 if (digit_value >= 0) {
809 if (digit_value >= t.cur_tok->radix) {
810 tokenize_error(&t, "invalid character: '%c'", c);
811 }
812 // normal digit
813 } else {
814 // not my char
815 t.pos -= 1;
816 end_token(&t);
817 t.state = TokenizeStateStart;
818 continue;
819 }
820 break;
821 }
822 case TokenizeStateFloatExponentUnsigned:
823 switch (c) {
824 case '+':
825 case '-':
826 t.state = TokenizeStateFloatExponentNumber;
827 break;
828 default:
829 // reinterpret as normal exponent number
830 t.pos -= 1;
831 t.state = TokenizeStateFloatExponentNumber;
832 continue;
833 }
834 break;
835 case TokenizeStateFloatExponentNumber:
705 switch (c) {836 switch (c) {
706 case DIGIT:837 case DIGIT:
707 break;838 break;
839 case ALPHA:
840 case '_':
841 tokenize_error(&t, "invalid character: '%c'", c);
842 break;
708 default:843 default:
709 t.pos -= 1;844 t.pos -= 1;
710 end_token(&t);845 end_token(&t);
...@@ -750,7 +885,11 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -750,7 +885,11 @@ void tokenize(Buf *buf, Tokenization *out) {
750 break;885 break;
751 case TokenizeStateSymbol:886 case TokenizeStateSymbol:
752 case TokenizeStateSymbolFirst:887 case TokenizeStateSymbolFirst:
888 case TokenizeStateZero:
753 case TokenizeStateNumber:889 case TokenizeStateNumber:
890 case TokenizeStateFloatFraction:
891 case TokenizeStateFloatExponentUnsigned:
892 case TokenizeStateFloatExponentNumber:
754 case TokenizeStateSawStar:893 case TokenizeStateSawStar:
755 case TokenizeStateSawSlash:894 case TokenizeStateSawSlash:
756 case TokenizeStateSawPercent:895 case TokenizeStateSawPercent:
src/tokenizer.hpp+5
...@@ -88,6 +88,11 @@ struct Token {...@@ -88,6 +88,11 @@ struct Token {
88 int end_pos;88 int end_pos;
89 int start_line;89 int start_line;
90 int start_column;90 int start_column;
91
92 // for id == TokenIdNumberLiteral
93 int radix; // if != 10, then skip the first 2 characters
94 int decimal_point_pos; // either exponent_marker_pos or the position of the '.'
95 int exponent_marker_pos; // either end_pos or the position of the 'e'/'p'
91};96};
9297
93struct Tokenization {98struct Tokenization {