authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-28 13:26:16-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-28 13:26:16-07:00
log9de2c1b76e2b38e35bb17cd0a3943a61c730da76
tree1e23f3926d2757e546107bf75f913a2fe75f3c3d
parent01e0a4b2f06d60a58b3d2df410776b164d05c1ec

stage1: tokenizer: backport recent stage2 changes

This backports ec10595b65490126e1b7d231f655be6d7d198acf stage2 tokenizer logic to stage1.

1 files changed, 2 insertions(+), 8 deletions(-)

src/stage1/tokenizer.cpp+2-8
...@@ -1290,12 +1290,9 @@ void tokenize(const char *source, Tokenization *out) {...@@ -1290,12 +1290,9 @@ void tokenize(const char *source, Tokenization *out) {
1290 case DIGIT:1290 case DIGIT:
1291 t.state = TokenizeState_float_fraction_dec;1291 t.state = TokenizeState_float_fraction_dec;
1292 break;1292 break;
1293 case ALPHA_EXCEPT_E:1293 default:
1294 invalid_char_error(&t, c);1294 invalid_char_error(&t, c);
1295 break;1295 break;
1296 default:
1297 t.state = TokenizeState_start;
1298 continue;
1299 }1296 }
1300 break;1297 break;
1301 case TokenizeState_num_dot_hex:1298 case TokenizeState_num_dot_hex:
...@@ -1316,12 +1313,9 @@ void tokenize(const char *source, Tokenization *out) {...@@ -1316,12 +1313,9 @@ void tokenize(const char *source, Tokenization *out) {
1316 t.out->ids.last() = TokenIdFloatLiteral;1313 t.out->ids.last() = TokenIdFloatLiteral;
1317 t.state = TokenizeState_float_fraction_hex;1314 t.state = TokenizeState_float_fraction_hex;
1318 break;1315 break;
1319 case ALPHA_EXCEPT_HEX_AND_P:1316 default:
1320 invalid_char_error(&t, c);1317 invalid_char_error(&t, c);
1321 break;1318 break;
1322 default:
1323 t.state = TokenizeState_start;
1324 continue;
1325 }1319 }
1326 break;1320 break;
1327 case TokenizeState_float_fraction_dec_no_underscore:1321 case TokenizeState_float_fraction_dec_no_underscore: