authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-28 13:26:16-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-28 13:26:16-07:00
log9de2c1b76e2b38e35bb17cd0a3943a61c730da76
tree1e23f3926d2757e546107bf75f913a2fe75f3c3d
parent01e0a4b2f06d60a58b3d2df410776b164d05c1ec

stage1: tokenizer: backport recent stage2 changes

This backports ec10595b65490126e1b7d231f655be6d7d198acf stage2 tokenizer logic to stage1.

1 files changed, 2 insertions(+), 8 deletions(-)

src/stage1/tokenizer.cpp+2-8
......@@ -1290,12 +1290,9 @@ void tokenize(const char *source, Tokenization *out) {
12901290 case DIGIT:
12911291 t.state = TokenizeState_float_fraction_dec;
12921292 break;
1293 case ALPHA_EXCEPT_E:
1293 default:
12941294 invalid_char_error(&t, c);
12951295 break;
1296 default:
1297 t.state = TokenizeState_start;
1298 continue;
12991296 }
13001297 break;
13011298 case TokenizeState_num_dot_hex:
......@@ -1316,12 +1313,9 @@ void tokenize(const char *source, Tokenization *out) {
13161313 t.out->ids.last() = TokenIdFloatLiteral;
13171314 t.state = TokenizeState_float_fraction_hex;
13181315 break;
1319 case ALPHA_EXCEPT_HEX_AND_P:
1316 default:
13201317 invalid_char_error(&t, c);
13211318 break;
1322 default:
1323 t.state = TokenizeState_start;
1324 continue;
13251319 }
13261320 break;
13271321 case TokenizeState_float_fraction_dec_no_underscore: