authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2015-12-14 23:13:53-07:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2015-12-15 04:05:53-07:00
logcf88fcb2ad31b64296d906ef525e76bbe6c3f65e
tree46ce6c05ed94593ae29dd26f577095662905f55a
parentde7c5ea1026db1c32fac522c71e9d5c57de6ed14

tokenizer parses all number literal types

2 files changed, 151 insertions(+), 7 deletions(-)

src/tokenizer.cpp+146-7
......@@ -13,12 +13,11 @@
1313#include <stdio.h>
1414
1515#define WHITESPACE \
16 ' ': \
16 ' ': \
1717 case '\n'
1818
19#define DIGIT \
20 '0': \
21 case '1': \
19#define DIGIT_NON_ZERO \
20 '1': \
2221 case '2': \
2322 case '3': \
2423 case '4': \
......@@ -27,10 +26,14 @@
2726 case '7': \
2827 case '8': \
2928 case '9'
29#define DIGIT \
30 '0': \
31 case DIGIT_NON_ZERO
3032
3133#define ALPHA_EXCEPT_C \
32 'a': \
34 'a': \
3335 case 'b': \
36 /*case 'c':*/ \
3437 case 'd': \
3538 case 'e': \
3639 case 'f': \
......@@ -94,7 +97,11 @@ enum TokenizeState {
9497 TokenizeStateStart,
9598 TokenizeStateSymbol,
9699 TokenizeStateSymbolFirst,
97 TokenizeStateNumber,
100 TokenizeStateZero, // "0", which might lead to "0x"
101 TokenizeStateNumber, // "123", "0x123"
102 TokenizeStateFloatFraction, // "123.456", "0x123.456"
103 TokenizeStateFloatExponentUnsigned, // "123.456e", "123e", "0x123p"
104 TokenizeStateFloatExponentNumber, // "123.456e-", "123.456e5", "123.456e5e-5"
98105 TokenizeStateString,
99106 TokenizeStateSawStar,
100107 TokenizeStateSawSlash,
......@@ -172,6 +179,16 @@ static void end_token(Tokenize *t) {
172179 assert(t->cur_tok);
173180 t->cur_tok->end_pos = t->pos + 1;
174181
182 // normalize number literal parsing stuff
183 if (t->cur_tok->id == TokenIdNumberLiteral) {
184 if (t->cur_tok->exponent_marker_pos == 0) {
185 t->cur_tok->exponent_marker_pos = t->cur_tok->end_pos;
186 }
187 if (t->cur_tok->decimal_point_pos == 0) {
188 t->cur_tok->decimal_point_pos = t->cur_tok->exponent_marker_pos;
189 }
190 }
191
175192 char *token_mem = buf_ptr(t->buf) + t->cur_tok->start_pos;
176193 int token_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
177194
......@@ -218,6 +235,26 @@ static void end_token(Tokenize *t) {
218235 t->cur_tok = nullptr;
219236}
220237
238static bool is_exponent_signifier(uint8_t c, int radix) {
239 if (radix == 16) {
240 return c == 'p' || c == 'P';
241 } else {
242 return c == 'e' || c == 'E';
243 }
244}
245static int get_digit_value(uint8_t c) {
246 if ('0' <= c && c <= '9') {
247 return c - '0';
248 }
249 if ('A' <= c && c <= 'Z') {
250 return c - 'A' + 10;
251 }
252 if ('a' <= c && c <= 'z') {
253 return c - 'a' + 10;
254 }
255 return -1;
256}
257
221258void tokenize(Buf *buf, Tokenization *out) {
222259 Tokenize t = {0};
223260 t.out = out;
......@@ -245,9 +282,15 @@ void tokenize(Buf *buf, Tokenization *out) {
245282 t.state = TokenizeStateSymbol;
246283 begin_token(&t, TokenIdSymbol);
247284 break;
248 case DIGIT:
285 case '0':
286 t.state = TokenizeStateZero;
287 begin_token(&t, TokenIdNumberLiteral);
288 t.cur_tok->radix = 10;
289 break;
290 case DIGIT_NON_ZERO:
249291 t.state = TokenizeStateNumber;
250292 begin_token(&t, TokenIdNumberLiteral);
293 t.cur_tok->radix = 10;
251294 break;
252295 case '"':
253296 begin_token(&t, TokenIdStringLiteral);
......@@ -701,10 +744,102 @@ void tokenize(Buf *buf, Tokenization *out) {
701744 break;
702745 }
703746 break;
747 case TokenizeStateZero:
748 switch (c) {
749 case 'b':
750 t.cur_tok->radix = 2;
751 break;
752 case 'o':
753 t.cur_tok->radix = 8;
754 break;
755 case 'x':
756 t.cur_tok->radix = 16;
757 break;
758 default:
759 // reinterpret as normal number
760 t.pos -= 1;
761 t.state = TokenizeStateNumber;
762 continue;
763 }
764 break;
704765 case TokenizeStateNumber:
766 {
767 if (c == '.') {
768 t.cur_tok->decimal_point_pos = t.pos;
769 t.state = TokenizeStateFloatFraction;
770 break;
771 }
772 if (is_exponent_signifier(c, t.cur_tok->radix)) {
773 t.cur_tok->exponent_marker_pos = t.pos;
774 t.state = TokenizeStateFloatExponentUnsigned;
775 break;
776 }
777 if (c == '_') {
778 tokenize_error(&t, "invalid character: '%c'", c);
779 break;
780 }
781 int digit_value = get_digit_value(c);
782 if (digit_value >= 0) {
783 if (digit_value >= t.cur_tok->radix) {
784 tokenize_error(&t, "invalid character: '%c'", c);
785 }
786 // normal digit
787 } else {
788 // not my char
789 t.pos -= 1;
790 end_token(&t);
791 t.state = TokenizeStateStart;
792 continue;
793 }
794 break;
795 }
796 case TokenizeStateFloatFraction:
797 {
798 if (is_exponent_signifier(c, t.cur_tok->radix)) {
799 t.cur_tok->exponent_marker_pos = t.pos;
800 t.state = TokenizeStateFloatExponentUnsigned;
801 break;
802 }
803 if (c == '_') {
804 tokenize_error(&t, "invalid character: '%c'", c);
805 break;
806 }
807 int digit_value = get_digit_value(c);
808 if (digit_value >= 0) {
809 if (digit_value >= t.cur_tok->radix) {
810 tokenize_error(&t, "invalid character: '%c'", c);
811 }
812 // normal digit
813 } else {
814 // not my char
815 t.pos -= 1;
816 end_token(&t);
817 t.state = TokenizeStateStart;
818 continue;
819 }
820 break;
821 }
822 case TokenizeStateFloatExponentUnsigned:
823 switch (c) {
824 case '+':
825 case '-':
826 t.state = TokenizeStateFloatExponentNumber;
827 break;
828 default:
829 // reinterpret as normal exponent number
830 t.pos -= 1;
831 t.state = TokenizeStateFloatExponentNumber;
832 continue;
833 }
834 break;
835 case TokenizeStateFloatExponentNumber:
705836 switch (c) {
706837 case DIGIT:
707838 break;
839 case ALPHA:
840 case '_':
841 tokenize_error(&t, "invalid character: '%c'", c);
842 break;
708843 default:
709844 t.pos -= 1;
710845 end_token(&t);
......@@ -750,7 +885,11 @@ void tokenize(Buf *buf, Tokenization *out) {
750885 break;
751886 case TokenizeStateSymbol:
752887 case TokenizeStateSymbolFirst:
888 case TokenizeStateZero:
753889 case TokenizeStateNumber:
890 case TokenizeStateFloatFraction:
891 case TokenizeStateFloatExponentUnsigned:
892 case TokenizeStateFloatExponentNumber:
754893 case TokenizeStateSawStar:
755894 case TokenizeStateSawSlash:
756895 case TokenizeStateSawPercent:
src/tokenizer.hpp+5
......@@ -88,6 +88,11 @@ struct Token {
8888 int end_pos;
8989 int start_line;
9090 int start_column;
91
92 // for id == TokenIdNumberLiteral
93 int radix; // if != 10, then skip the first 2 characters
94 int decimal_point_pos; // either exponent_marker_pos or the position of the '.'
95 int exponent_marker_pos; // either end_pos or the position of the 'e'/'p'
9196};
9297
9398struct Tokenization {