From 7aac21c6f59b70deea6ced617f7b6a550e92bab4 Mon Sep 17 00:00:00 2001 From: momumi <57862114+momumi@users.noreply.github.com> Date: Sun, 15 Mar 2020 11:37:36 +1000 Subject: [PATCH 1/7] allow `_` separators in number literals (stage 1) * Underscores `_` may be placed between two digits in a int/float literal * Consecutive underscores are not allowed * Fixed parsing bug in exponents of hexadecimal float literals. Exponents should always be base 10, but hex characters would be parsed inside the exponent and everything after them would be ignored. eg: `0x1.0p1ab1` would be parsed as `0x1.0p1`. --- doc/langref.html.in | 11 ++ lib/std/special/compiler_rt/floatundisf.zig | 38 +++--- src/parse_f128.c | 79 ++++++++--- src/tokenizer.cpp | 137 ++++++++++++-------- test/compile_errors.zig | 96 ++++++++++++++ test/stage1/behavior/math.zig | 28 ++++ 6 files changed, 297 insertions(+), 92 deletions(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index 3a7892fd455e1cf87a50090e8d39adf2decc3869..616edd44ebbc497434a17a57a8336c087abbb24a 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -885,6 +885,12 @@ const hex_int = 0xff; const another_hex_int = 0xFF; const octal_int = 0o755; const binary_int = 0b11110000; + +// underscores may be placed between two digits as a visual separator +const one_billion = 1_000_000_000; +const binary_mask = 0b1_1111_1111; +const permissions = 0o7_5_5; +const big_address = 0xFF80_0000_0000_0000; {#code_end#} {#header_close#} {#header_open|Runtime Integer Values#} @@ -947,6 +953,11 @@ const yet_another = 123.0e+77; const hex_floating_point = 0x103.70p-5; const another_hex_float = 0x103.70; const yet_another_hex_float = 0x103.70P-5; + +// underscores may be placed between two digits as a visual separator +const lightspeed = 299_792_458.000_000; +const nanosecond = 0.000_000_001; +const more_hex = 0x1234_5678.9ABC_CDEFp-10; {#code_end#}

There is no syntax for NaN, infinity, or negative infinity. For these special values, diff --git a/lib/std/special/compiler_rt/floatundisf.zig b/lib/std/special/compiler_rt/floatundisf.zig index 41ff02daee686ec0f0b2c0c838e13590881287fa..ff242721d6e09f8e600077b02fa4ec4c196ccf26 100644 --- a/lib/std/special/compiler_rt/floatundisf.zig +++ b/lib/std/special/compiler_rt/floatundisf.zig @@ -69,23 +69,23 @@ test "floatundisf" { test__floatundisf(0, 0.0); test__floatundisf(1, 1.0); test__floatundisf(2, 2.0); - test__floatundisf(0x7FFFFF8000000000, 0x1.FFFFFEp+62F); - test__floatundisf(0x7FFFFF0000000000, 0x1.FFFFFCp+62F); - test__floatundisf(0x8000008000000000, 0x1p+63F); - test__floatundisf(0x8000010000000000, 0x1.000002p+63F); - test__floatundisf(0x8000000000000000, 0x1p+63F); - test__floatundisf(0x8000000000000001, 0x1p+63F); - test__floatundisf(0xFFFFFFFFFFFFFFFE, 0x1p+64F); - test__floatundisf(0xFFFFFFFFFFFFFFFF, 0x1p+64F); - test__floatundisf(0x0007FB72E8000000, 0x1.FEDCBAp+50F); - test__floatundisf(0x0007FB72EA000000, 0x1.FEDCBAp+50F); - test__floatundisf(0x0007FB72EB000000, 0x1.FEDCBAp+50F); - test__floatundisf(0x0007FB72EBFFFFFF, 0x1.FEDCBAp+50F); - test__floatundisf(0x0007FB72EC000000, 0x1.FEDCBCp+50F); - test__floatundisf(0x0007FB72E8000001, 0x1.FEDCBAp+50F); - test__floatundisf(0x0007FB72E6000000, 0x1.FEDCBAp+50F); - test__floatundisf(0x0007FB72E7000000, 0x1.FEDCBAp+50F); - test__floatundisf(0x0007FB72E7FFFFFF, 0x1.FEDCBAp+50F); - test__floatundisf(0x0007FB72E4000001, 0x1.FEDCBAp+50F); - test__floatundisf(0x0007FB72E4000000, 0x1.FEDCB8p+50F); + test__floatundisf(0x7FFFFF8000000000, 0x1.FFFFFEp+62); + test__floatundisf(0x7FFFFF0000000000, 0x1.FFFFFCp+62); + test__floatundisf(0x8000008000000000, 0x1p+63); + test__floatundisf(0x8000010000000000, 0x1.000002p+63); + test__floatundisf(0x8000000000000000, 0x1p+63); + test__floatundisf(0x8000000000000001, 0x1p+63); + test__floatundisf(0xFFFFFFFFFFFFFFFE, 0x1p+64); + test__floatundisf(0xFFFFFFFFFFFFFFFF, 0x1p+64); + test__floatundisf(0x0007FB72E8000000, 0x1.FEDCBAp+50); + test__floatundisf(0x0007FB72EA000000, 0x1.FEDCBAp+50); + test__floatundisf(0x0007FB72EB000000, 0x1.FEDCBAp+50); + test__floatundisf(0x0007FB72EBFFFFFF, 0x1.FEDCBAp+50); + test__floatundisf(0x0007FB72EC000000, 0x1.FEDCBCp+50); + test__floatundisf(0x0007FB72E8000001, 0x1.FEDCBAp+50); + test__floatundisf(0x0007FB72E6000000, 0x1.FEDCBAp+50); + test__floatundisf(0x0007FB72E7000000, 0x1.FEDCBAp+50); + test__floatundisf(0x0007FB72E7FFFFFF, 0x1.FEDCBAp+50); + test__floatundisf(0x0007FB72E4000001, 0x1.FEDCBAp+50); + test__floatundisf(0x0007FB72E4000000, 0x1.FEDCB8p+50); } diff --git a/src/parse_f128.c b/src/parse_f128.c index cffb3796b482d10089ee81b00939a67d9e483e65..9b5c287a3c015388a6e749fcc0a07379b3d2e58f 100644 --- a/src/parse_f128.c +++ b/src/parse_f128.c @@ -165,22 +165,36 @@ static long long scanexp(struct MuslFILE *f, int pok) int x; long long y; int neg = 0; - + c = shgetc(f); if (c=='+' || c=='-') { neg = (c=='-'); c = shgetc(f); if (c-'0'>=10U && pok) shunget(f); } - if (c-'0'>=10U) { + if (c-'0'>=10U && c!='_') { shunget(f); return LLONG_MIN; } - for (x=0; c-'0'<10U && xdata.int_lit.bigint, 0); - bigint_init_unsigned(&t.specified_exponent, 0); break; case DIGIT_NON_ZERO: t.state = TokenizeStateNumber; begin_token(&t, TokenIdIntLiteral); + t.is_trailing_underscore = false; t.radix = 10; - t.exp_add_amt = 1; - t.exponent_in_bin_or_dec = 0; bigint_init_unsigned(&t.cur_tok->data.int_lit.bigint, get_digit_value(c)); - bigint_init_unsigned(&t.specified_exponent, 0); break; case '"': begin_token(&t, TokenIdStringLiteral); @@ -1189,17 +1184,15 @@ void tokenize(Buf *buf, Tokenization *out) { switch (c) { case 'b': t.radix = 2; - t.state = TokenizeStateNumber; + t.state = TokenizeStateNumberNoUnderscore; break; case 'o': t.radix = 8; - t.exp_add_amt = 3; - t.state = TokenizeStateNumber; + t.state = TokenizeStateNumberNoUnderscore; break; case 'x': t.radix = 16; - t.exp_add_amt = 4; - t.state = TokenizeStateNumber; + t.state = TokenizeStateNumberNoUnderscore; break; default: // reinterpret as normal number @@ -1208,9 +1201,27 @@ void tokenize(Buf *buf, Tokenization *out) { continue; } break; + case TokenizeStateNumberNoUnderscore: + if (c == '_') { + invalid_char_error(&t, c); + break; + } else if (get_digit_value(c) < t.radix) { + t.is_trailing_underscore = false; + t.state = TokenizeStateNumber; + } + // fall through case TokenizeStateNumber: { + if (c == '_') { + t.is_trailing_underscore = true; + t.state = TokenizeStateNumberNoUnderscore; + break; + } if (c == '.') { + if (t.is_trailing_underscore) { + invalid_char_error(&t, c); + break; + } if (t.radix != 16 && t.radix != 10) { invalid_char_error(&t, c); } @@ -1222,13 +1233,18 @@ void tokenize(Buf *buf, Tokenization *out) { invalid_char_error(&t, c); } t.state = TokenizeStateFloatExponentUnsigned; + t.radix = 10; // exponent is always base 10 assert(t.cur_tok->id == TokenIdIntLiteral); - bigint_init_bigint(&t.significand, &t.cur_tok->data.int_lit.bigint); set_token_id(&t, t.cur_tok, TokenIdFloatLiteral); break; } uint32_t digit_value = get_digit_value(c); if (digit_value >= t.radix) { + if (t.is_trailing_underscore) { + invalid_char_error(&t, c); + break; + } + if (is_symbol_char(c)) { invalid_char_error(&t, c); } @@ -1259,20 +1275,37 @@ void tokenize(Buf *buf, Tokenization *out) { continue; } t.pos -= 1; - t.state = TokenizeStateFloatFraction; + t.state = TokenizeStateFloatFractionNoUnderscore; assert(t.cur_tok->id == TokenIdIntLiteral); - bigint_init_bigint(&t.significand, &t.cur_tok->data.int_lit.bigint); set_token_id(&t, t.cur_tok, TokenIdFloatLiteral); continue; } + case TokenizeStateFloatFractionNoUnderscore: + if (c == '_') { + invalid_char_error(&t, c); + } else if (get_digit_value(c) < t.radix) { + t.is_trailing_underscore = false; + t.state = TokenizeStateFloatFraction; + } + // fall through case TokenizeStateFloatFraction: { + if (c == '_') { + t.is_trailing_underscore = true; + t.state = TokenizeStateFloatFractionNoUnderscore; + break; + } if (is_exponent_signifier(c, t.radix)) { t.state = TokenizeStateFloatExponentUnsigned; + t.radix = 10; // exponent is always base 10 break; } uint32_t digit_value = get_digit_value(c); if (digit_value >= t.radix) { + if (t.is_trailing_underscore) { + invalid_char_error(&t, c); + break; + } if (is_symbol_char(c)) { invalid_char_error(&t, c); } @@ -1282,46 +1315,47 @@ void tokenize(Buf *buf, Tokenization *out) { t.state = TokenizeStateStart; continue; } - t.exponent_in_bin_or_dec -= t.exp_add_amt; - if (t.radix == 10) { - // For now we use strtod to parse decimal floats, so we just have to get to the - // end of the token. - break; - } - BigInt digit_value_bi; - bigint_init_unsigned(&digit_value_bi, digit_value); - BigInt radix_bi; - bigint_init_unsigned(&radix_bi, t.radix); - - BigInt multiplied; - bigint_mul(&multiplied, &t.significand, &radix_bi); - - bigint_add(&t.significand, &multiplied, &digit_value_bi); - break; + // we use parse_f128 to generate the float literal, so just + // need to get to the end of the token } + break; case TokenizeStateFloatExponentUnsigned: switch (c) { case '+': - t.is_exp_negative = false; - t.state = TokenizeStateFloatExponentNumber; + t.state = TokenizeStateFloatExponentNumberNoUnderscore; break; case '-': - t.is_exp_negative = true; - t.state = TokenizeStateFloatExponentNumber; + t.state = TokenizeStateFloatExponentNumberNoUnderscore; break; default: // reinterpret as normal exponent number t.pos -= 1; - t.is_exp_negative = false; - t.state = TokenizeStateFloatExponentNumber; + t.state = TokenizeStateFloatExponentNumberNoUnderscore; continue; } break; + case TokenizeStateFloatExponentNumberNoUnderscore: + if (c == '_') { + invalid_char_error(&t, c); + } else if (get_digit_value(c) < t.radix) { + t.is_trailing_underscore = false; + t.state = TokenizeStateFloatExponentNumber; + } + // fall through case TokenizeStateFloatExponentNumber: { + if (c == '_') { + t.is_trailing_underscore = true; + t.state = TokenizeStateFloatExponentNumberNoUnderscore; + break; + } uint32_t digit_value = get_digit_value(c); if (digit_value >= t.radix) { + if (t.is_trailing_underscore) { + invalid_char_error(&t, c); + break; + } if (is_symbol_char(c)) { invalid_char_error(&t, c); } @@ -1331,21 +1365,9 @@ void tokenize(Buf *buf, Tokenization *out) { t.state = TokenizeStateStart; continue; } - if (t.radix == 10) { - // For now we use strtod to parse decimal floats, so we just have to get to the - // end of the token. - break; - } - BigInt digit_value_bi; - bigint_init_unsigned(&digit_value_bi, digit_value); - BigInt radix_bi; - bigint_init_unsigned(&radix_bi, 10); - - BigInt multiplied; - bigint_mul(&multiplied, &t.specified_exponent, &radix_bi); - - bigint_add(&t.specified_exponent, &multiplied, &digit_value_bi); + // we use parse_f128 to generate the float literal, so just + // need to get to the end of the token } break; case TokenizeStateSawDash: @@ -1399,6 +1421,9 @@ void tokenize(Buf *buf, Tokenization *out) { case TokenizeStateStart: case TokenizeStateError: break; + case TokenizeStateNumberNoUnderscore: + case TokenizeStateFloatFractionNoUnderscore: + case TokenizeStateFloatExponentNumberNoUnderscore: case TokenizeStateNumberDot: tokenize_error(&t, "unterminated number literal"); break; diff --git a/test/compile_errors.zig b/test/compile_errors.zig index f894a152a74b4514d2e1b8b7f598efc4c9486102..83fe1def6290c587797db155f2cf18ac8bc8b609 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -389,6 +389,102 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { "tmp.zig:5:29: error: invalid token: '.'", }); + cases.add("invalid underscore placement in float literal - 1", + \\fn main() void { + \\ var bad: f128 = 0._0; + \\}) + , &[_][]const u8{ + "tmp.zig:2:23: error: invalid character: '_'", + }); + + cases.add("invalid underscore placement in float literal - 2", + \\fn main() void { + \\ var bad: f128 = 0_.0; + \\}) + , &[_][]const u8{ + "tmp.zig:2:23: error: invalid character: '.'", + }); + + cases.add("invalid underscore placement in float literal - 3", + \\fn main() void { + \\ var bad: f128 = 0.0_; + \\}) + , &[_][]const u8{ + "tmp.zig:2:25: error: invalid character: ';'", + }); + + cases.add("invalid underscore placement in float literal - 4", + \\fn main() void { + \\ var bad: f128 = 1.0e_1; + \\}) + , &[_][]const u8{ + "tmp.zig:2:25: error: invalid character: '_'", + }); + + cases.add("invalid underscore placement in float literal - 5", + \\fn main() void { + \\ var bad: f128 = 1.0e+_1; + \\}) + , &[_][]const u8{ + "tmp.zig:2:26: error: invalid character: '_'", + }); + + cases.add("invalid underscore placement in float literal - 6", + \\fn main() void { + \\ var bad: f128 = 1.0e-_1; + \\}) + , &[_][]const u8{ + "tmp.zig:2:26: error: invalid character: '_'", + }); + + cases.add("invalid underscore placement in float literal - 7", + \\fn main() void { + \\ var bad: f128 = 1.0e-1_; + \\}) + , &[_][]const u8{ + "tmp.zig:2:28: error: invalid character: ';'", + }); + + cases.add("invalid underscore placement in float literal - 9", + \\fn main() void { + \\ var bad: f128 = 1__0.0e-1; + \\}) + , &[_][]const u8{ + "tmp.zig:2:23: error: invalid character: '_'", + }); + + cases.add("invalid underscore placement in float literal - 10", + \\fn main() void { + \\ var bad: f128 = 1.0__0e-1; + \\}) + , &[_][]const u8{ + "tmp.zig:2:25: error: invalid character: '_'", + }); + + cases.add("invalid underscore placement in float literal - 11", + \\fn main() void { + \\ var bad: f128 = 1.0e-1__0; + \\}) + , &[_][]const u8{ + "tmp.zig:2:28: error: invalid character: '_'", + }); + + cases.add("invalid underscore placement in float literal - 12", + \\fn main() void { + \\ var bad: f128 = 0_x0.0; + \\}) + , &[_][]const u8{ + "tmp.zig:2:23: error: invalid character: 'x'", + }); + + cases.add("invalid underscore placement in float literal - 13", + \\fn main() void { + \\ var bad: f128 = 0x_0.0; + \\}) + , &[_][]const u8{ + "tmp.zig:2:23: error: invalid character: '_'", + }); + cases.add("var args without c calling conv", \\fn foo(args: ...) void {} \\comptime { diff --git a/test/stage1/behavior/math.zig b/test/stage1/behavior/math.zig index fb70fb7e4420e6c1eae4ca786ea323cc7d73b312..b342597acffa549cf80f6d27f35fa5f9555990a7 100644 --- a/test/stage1/behavior/math.zig +++ b/test/stage1/behavior/math.zig @@ -411,6 +411,34 @@ test "quad hex float literal parsing accurate" { comptime S.doTheTest(); } +test "underscore separator parsing" { + expect(0_0_0_0 == 0); + expect(1_234_567 == 1234567); + expect(001_234_567 == 1234567); + expect(0_0_1_2_3_4_5_6_7 == 1234567); + + expect(0b0_0_0_0 == 0); + expect(0b1010_1010 == 0b10101010); + expect(0b0000_1010_1010 == 0b10101010); + expect(0b1_0_1_0_1_0_1_0 == 0b10101010); + + expect(0o0_0_0_0 == 0); + expect(0o1010_1010 == 0o10101010); + expect(0o0000_1010_1010 == 0o10101010); + expect(0o1_0_1_0_1_0_1_0 == 0o10101010); + + expect(0x0_0_0_0 == 0); + expect(0x1010_1010 == 0x10101010); + expect(0x0000_1010_1010 == 0x10101010); + expect(0x1_0_1_0_1_0_1_0 == 0x10101010); + + expect(123_456.789_000e1_0 == 123456.789000e10); + expect(0_1_2_3_4_5_6.7_8_9_0_0_0e0_0_1_0 == 123456.789000e10); + + expect(0x1234_5678.9ABC_DEF0p-1_0 == 0x12345678.9ABCDEF0p-10); + expect(0x1_2_3_4_5_6_7_8.9_A_B_C_D_E_F_0p-0_0_0_1_0 == 0x12345678.9ABCDEF0p-10); +} + test "hex float literal within range" { const a = 0x1.0p16383; const b = 0x0.1p16387; -- 2.54.0 From 925f71085269a8072ea9c49025a534fa47fbec44 Mon Sep 17 00:00:00 2001 From: momumi <57862114+momumi@users.noreply.github.com> Date: Sun, 15 Mar 2020 13:05:24 +1000 Subject: [PATCH 2/7] make parsing `0.0_e1` an error --- src/tokenizer.cpp | 8 ++++++++ test/compile_errors.zig | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 73efae203731ca60a466717a12035845c0a4a6c6..22d63568bf7777db6a001403441241cf0752748a 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -1229,6 +1229,10 @@ void tokenize(Buf *buf, Tokenization *out) { break; } if (is_exponent_signifier(c, t.radix)) { + if (t.is_trailing_underscore) { + invalid_char_error(&t, c); + break; + } if (t.radix != 16 && t.radix != 10) { invalid_char_error(&t, c); } @@ -1296,6 +1300,10 @@ void tokenize(Buf *buf, Tokenization *out) { break; } if (is_exponent_signifier(c, t.radix)) { + if (t.is_trailing_underscore) { + invalid_char_error(&t, c); + break; + } t.state = TokenizeStateFloatExponentUnsigned; t.radix = 10; // exponent is always base 10 break; diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 83fe1def6290c587797db155f2cf18ac8bc8b609..2b40fec106cefd9610739e32376268f775c840de 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -485,6 +485,14 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { "tmp.zig:2:23: error: invalid character: '_'", }); + cases.add("invalid underscore placement in float literal - 14", + \\fn main() void { + \\ var bad: f128 = 0x0.0_p1; + \\}) + , &[_][]const u8{ + "tmp.zig:2:27: error: invalid character: 'p'", + }); + cases.add("var args without c calling conv", \\fn foo(args: ...) void {} \\comptime { -- 2.54.0 From 47f7e6658077fb4f55c36e9c62ed5012ef8aace2 Mon Sep 17 00:00:00 2001 From: momumi <57862114+momumi@users.noreply.github.com> Date: Sun, 15 Mar 2020 23:42:29 +1000 Subject: [PATCH 3/7] add more test cases for invalid number literals --- test/compile_errors.zig | 76 +++++++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 2b40fec106cefd9610739e32376268f775c840de..73bf60216e479a979e2f8724dc1317ed7d8dd44e 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -384,15 +384,31 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { \\ var bad_float :f32 = 0.0; \\ bad_float = bad_float + .20; \\ std.debug.assert(bad_float < 1.0); - \\}) + \\} , &[_][]const u8{ "tmp.zig:5:29: error: invalid token: '.'", }); + cases.add("invalid exponent in float literal - 1", + \\fn main() void { + \\ var bad: f128 = 0x1.0p1ab1; + \\} + , &[_][]const u8{ + "tmp.zig:2:28: error: invalid character: 'a'", + }); + + cases.add("invalid exponent in float literal - 2", + \\fn main() void { + \\ var bad: f128 = 0x1.0p50F; + \\} + , &[_][]const u8{ + "tmp.zig:2:29: error: invalid character: 'F'", + }); + cases.add("invalid underscore placement in float literal - 1", \\fn main() void { \\ var bad: f128 = 0._0; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:23: error: invalid character: '_'", }); @@ -400,7 +416,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 2", \\fn main() void { \\ var bad: f128 = 0_.0; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:23: error: invalid character: '.'", }); @@ -408,7 +424,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 3", \\fn main() void { \\ var bad: f128 = 0.0_; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:25: error: invalid character: ';'", }); @@ -416,7 +432,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 4", \\fn main() void { \\ var bad: f128 = 1.0e_1; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:25: error: invalid character: '_'", }); @@ -424,7 +440,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 5", \\fn main() void { \\ var bad: f128 = 1.0e+_1; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:26: error: invalid character: '_'", }); @@ -432,7 +448,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 6", \\fn main() void { \\ var bad: f128 = 1.0e-_1; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:26: error: invalid character: '_'", }); @@ -440,7 +456,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 7", \\fn main() void { \\ var bad: f128 = 1.0e-1_; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:28: error: invalid character: ';'", }); @@ -448,7 +464,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 9", \\fn main() void { \\ var bad: f128 = 1__0.0e-1; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:23: error: invalid character: '_'", }); @@ -456,7 +472,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 10", \\fn main() void { \\ var bad: f128 = 1.0__0e-1; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:25: error: invalid character: '_'", }); @@ -464,7 +480,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 11", \\fn main() void { \\ var bad: f128 = 1.0e-1__0; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:28: error: invalid character: '_'", }); @@ -472,7 +488,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 12", \\fn main() void { \\ var bad: f128 = 0_x0.0; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:23: error: invalid character: 'x'", }); @@ -480,7 +496,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 13", \\fn main() void { \\ var bad: f128 = 0x_0.0; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:23: error: invalid character: '_'", }); @@ -488,11 +504,43 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { cases.add("invalid underscore placement in float literal - 14", \\fn main() void { \\ var bad: f128 = 0x0.0_p1; - \\}) + \\} , &[_][]const u8{ "tmp.zig:2:27: error: invalid character: 'p'", }); + cases.add("invalid underscore placement in int literal - 1", + \\fn main() void { + \\ var bad: u128 = 0010_; + \\} + , &[_][]const u8{ + "tmp.zig:2:26: error: invalid character: ';'", + }); + + cases.add("invalid underscore placement in int literal - 2", + \\fn main() void { + \\ var bad: u128 = 0b0010_; + \\} + , &[_][]const u8{ + "tmp.zig:2:28: error: invalid character: ';'", + }); + + cases.add("invalid underscore placement in int literal - 3", + \\fn main() void { + \\ var bad: u128 = 0o0010_; + \\} + , &[_][]const u8{ + "tmp.zig:2:28: error: invalid character: ';'", + }); + + cases.add("invalid underscore placement in int literal - 4", + \\fn main() void { + \\ var bad: u128 = 0x0010_; + \\} + , &[_][]const u8{ + "tmp.zig:2:28: error: invalid character: ';'", + }); + cases.add("var args without c calling conv", \\fn foo(args: ...) void {} \\comptime { -- 2.54.0 From 138dab45248c718b8d38fa0a8eefe56f40fe617c Mon Sep 17 00:00:00 2001 From: momumi <57862114+momumi@users.noreply.github.com> Date: Sun, 22 Mar 2020 10:35:19 +1000 Subject: [PATCH 4/7] add number `_` separators for stage 2 tokenizer --- lib/std/zig/tokenizer.zig | 423 +++++++++++++++++++++++++++++++++----- 1 file changed, 372 insertions(+), 51 deletions(-) diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index f6c71479e7be2166ae1466dba766fdd50d5d4273..8c9b39d34f056463d38321b1b338f050ee9101aa 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -387,17 +387,23 @@ pub const Tokenizer = struct { DocComment, ContainerDocComment, Zero, - IntegerLiteral, - IntegerLiteralWithRadix, - IntegerLiteralWithRadixHex, - NumberDot, + IntegerLiteralDec, + IntegerLiteralDecNoUnderscore, + IntegerLiteralBin, + IntegerLiteralBinNoUnderscore, + IntegerLiteralOct, + IntegerLiteralOctNoUnderscore, + IntegerLiteralHex, + IntegerLiteralHexNoUnderscore, + NumberDotDec, NumberDotHex, - FloatFraction, + FloatFractionDec, + FloatFractionDecNoUnderscore, FloatFractionHex, + FloatFractionHexNoUnderscore, FloatExponentUnsigned, - FloatExponentUnsignedHex, FloatExponentNumber, - FloatExponentNumberHex, + FloatExponentNumberNoUnderscore, Ampersand, Caret, Percent, @@ -550,7 +556,7 @@ pub const Tokenizer = struct { result.id = Token.Id.IntegerLiteral; }, '1'...'9' => { - state = State.IntegerLiteral; + state = State.IntegerLiteralDec; result.id = Token.Id.IntegerLiteral; }, else => { @@ -1048,55 +1054,122 @@ pub const Tokenizer = struct { else => self.checkLiteralCharacter(), }, State.Zero => switch (c) { - 'b', 'o' => { - state = State.IntegerLiteralWithRadix; + 'b' => { + state = State.IntegerLiteralBinNoUnderscore; + }, + 'o' => { + state = State.IntegerLiteralOctNoUnderscore; }, 'x' => { - state = State.IntegerLiteralWithRadixHex; + state = State.IntegerLiteralHexNoUnderscore; }, else => { // reinterpret as a normal number self.index -= 1; - state = State.IntegerLiteral; + state = State.IntegerLiteralDec; }, }, - State.IntegerLiteral => switch (c) { + State.IntegerLiteralBinNoUnderscore => switch (c) { + '0'...'1' => { + state = State.IntegerLiteralBin; + }, + else => { + result.id = Token.Id.Invalid; + }, + }, + State.IntegerLiteralBin => switch (c) { + '_' => { + state = State.IntegerLiteralBinNoUnderscore; + }, + '0'...'1' => {}, + '2'...'9', 'a'...'z', 'A'...'Z' => { + result.id = Token.Id.Invalid; + }, + else => break, + }, + State.IntegerLiteralOctNoUnderscore => switch (c) { + '0'...'7' => { + state = State.IntegerLiteralOct; + }, + else => { + result.id = Token.Id.Invalid; + }, + }, + State.IntegerLiteralOct => switch (c) { + '_' => { + state = State.IntegerLiteralOctNoUnderscore; + }, + '0'...'7' => {}, + '8'...'9', 'a'...'z', 'A'...'Z' => { + result.id = Token.Id.Invalid; + }, + else => break, + }, + State.IntegerLiteralDecNoUnderscore => switch (c) { + '0'...'9' => { + state = State.IntegerLiteralDec; + }, + else => { + result.id = Token.Id.Invalid; + }, + }, + State.IntegerLiteralDec => switch (c) { + '_' => { + state = State.IntegerLiteralDecNoUnderscore; + }, '.' => { - state = State.NumberDot; + state = State.NumberDotDec; + result.id = Token.Id.FloatLiteral; }, - 'p', 'P', 'e', 'E' => { + 'e', 'E' => { state = State.FloatExponentUnsigned; + result.id = Token.Id.FloatLiteral; }, '0'...'9' => {}, + 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => { + result.id = Token.Id.Invalid; + }, else => break, }, - State.IntegerLiteralWithRadix => switch (c) { - '.' => { - state = State.NumberDot; + State.IntegerLiteralHexNoUnderscore => switch (c) { + '0'...'9', 'a'...'f', 'A'...'F' => { + state = State.IntegerLiteralHex; + }, + else => { + result.id = Token.Id.Invalid; }, - '0'...'9' => {}, - else => break, }, - State.IntegerLiteralWithRadixHex => switch (c) { + State.IntegerLiteralHex => switch (c) { + '_' => { + state = State.IntegerLiteralHexNoUnderscore; + }, '.' => { state = State.NumberDotHex; + result.id = Token.Id.FloatLiteral; }, 'p', 'P' => { - state = State.FloatExponentUnsignedHex; + state = State.FloatExponentUnsigned; + result.id = Token.Id.FloatLiteral; }, '0'...'9', 'a'...'f', 'A'...'F' => {}, + 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => { + result.id = Token.Id.Invalid; + }, else => break, }, - State.NumberDot => switch (c) { + State.NumberDotDec => switch (c) { '.' => { self.index -= 1; state = State.Start; break; }, + 'e', 'E' => { + state = State.FloatExponentUnsigned; + }, else => { self.index -= 1; result.id = Token.Id.FloatLiteral; - state = State.FloatFraction; + state = State.FloatFractionDecNoUnderscore; }, }, State.NumberDotHex => switch (c) { @@ -1105,65 +1178,98 @@ pub const Tokenizer = struct { state = State.Start; break; }, + 'p', 'P' => { + state = State.FloatExponentUnsigned; + }, else => { self.index -= 1; result.id = Token.Id.FloatLiteral; + state = State.FloatFractionHexNoUnderscore; + }, + }, + State.FloatFractionDecNoUnderscore => switch (c) { + '0'...'9' => { + state = State.FloatFractionDec; + }, + else => { + result.id = Token.Id.Invalid; + }, + }, + State.FloatFractionDec => switch (c) { + '_' => { + state = State.FloatFractionDecNoUnderscore; + }, + 'e', 'E' => { + state = State.FloatExponentUnsigned; + }, + '0'...'9' => {}, + 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => { + result.id = Token.Id.Invalid; + }, + else => break, + }, + State.FloatFractionHexNoUnderscore => switch (c) { + '0'...'9', 'a'...'f', 'A'...'F' => { state = State.FloatFractionHex; }, - }, - State.FloatFraction => switch (c) { - 'e', 'E' => { - state = State.FloatExponentUnsigned; + else => { + result.id = Token.Id.Invalid; }, - '0'...'9' => {}, - else => break, }, State.FloatFractionHex => switch (c) { + '_' => { + state = State.FloatFractionHexNoUnderscore; + }, 'p', 'P' => { - state = State.FloatExponentUnsignedHex; + state = State.FloatExponentUnsigned; }, '0'...'9', 'a'...'f', 'A'...'F' => {}, + 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => { + result.id = Token.Id.Invalid; + }, else => break, }, State.FloatExponentUnsigned => switch (c) { '+', '-' => { - state = State.FloatExponentNumber; + state = State.FloatExponentNumberNoUnderscore; }, else => { // reinterpret as a normal exponent number self.index -= 1; - state = State.FloatExponentNumber; + state = State.FloatExponentNumberNoUnderscore; }, }, - State.FloatExponentUnsignedHex => switch (c) { - '+', '-' => { - state = State.FloatExponentNumberHex; + State.FloatExponentNumberNoUnderscore => switch (c) { + '0'...'9' => { + state = State.FloatExponentNumber; }, else => { - // reinterpret as a normal exponent number - self.index -= 1; - state = State.FloatExponentNumberHex; + result.id = Token.Id.Invalid; }, }, State.FloatExponentNumber => switch (c) { + '_' => { + state = State.FloatExponentNumberNoUnderscore; + }, '0'...'9' => {}, - else => break, - }, - State.FloatExponentNumberHex => switch (c) { - '0'...'9', 'a'...'f', 'A'...'F' => {}, + 'a'...'z', 'A'...'Z' => { + result.id = Token.Id.Invalid; + }, else => break, }, } } else if (self.index == self.buffer.len) { switch (state) { State.Start, - State.IntegerLiteral, - State.IntegerLiteralWithRadix, - State.IntegerLiteralWithRadixHex, - State.FloatFraction, + State.IntegerLiteralDec, + State.IntegerLiteralBin, + State.IntegerLiteralOct, + State.IntegerLiteralHex, + State.NumberDotDec, + State.NumberDotHex, + State.FloatFractionDec, State.FloatFractionHex, State.FloatExponentNumber, - State.FloatExponentNumberHex, State.StringLiteral, // find this error later State.MultilineStringLiteralLine, State.Builtin, @@ -1184,10 +1290,14 @@ pub const Tokenizer = struct { result.id = Token.Id.ContainerDocComment; }, - State.NumberDot, - State.NumberDotHex, + State.IntegerLiteralDecNoUnderscore, + State.IntegerLiteralBinNoUnderscore, + State.IntegerLiteralOctNoUnderscore, + State.IntegerLiteralHexNoUnderscore, + State.FloatFractionDecNoUnderscore, + State.FloatFractionHexNoUnderscore, + State.FloatExponentNumberNoUnderscore, State.FloatExponentUnsigned, - State.FloatExponentUnsignedHex, State.SawAtSign, State.Backslash, State.CharLiteral, @@ -1585,6 +1695,217 @@ test "correctly parse pointer assignment" { }); } +test "tokenizer - number literals decimal" { + testTokenize("1", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("2", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("3", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("4", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("5", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("6", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("7", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("8", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("9", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0a", &[_]Token.Id{ .Invalid }); + testTokenize("9b", &[_]Token.Id{ .Invalid }); + testTokenize("1z", &[_]Token.Id{ .Invalid }); + testTokenize("1z_1", &[_]Token.Id{ .Invalid }); + testTokenize("9z3", &[_]Token.Id{ .Invalid }); + + testTokenize("0_0", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0001", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("01234567890", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("012_345_6789_0", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0_1_2_3_4_5_6_7_8_9_0", &[_]Token.Id{ .IntegerLiteral }); + + testTokenize("00_", &[_]Token.Id{ .Invalid }); + testTokenize("0_0_", &[_]Token.Id{ .Invalid }); + testTokenize("0__0", &[_]Token.Id{ .Invalid }); + testTokenize("0_0f", &[_]Token.Id{ .Invalid }); + testTokenize("0_0_f", &[_]Token.Id{ .Invalid }); + testTokenize("1_,", &[_]Token.Id{ .Invalid }); + + testTokenize("1.", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0.0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("1.0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("10.0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("1e0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("1e100", &[_]Token.Id{ .FloatLiteral }); + testTokenize("1.e100", &[_]Token.Id{ .FloatLiteral }); + testTokenize("1.0e100", &[_]Token.Id{ .FloatLiteral }); + testTokenize("1.0e+100", &[_]Token.Id{ .FloatLiteral }); + testTokenize("1.0e-100", &[_]Token.Id{ .FloatLiteral }); + testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{ .FloatLiteral }); + + testTokenize("1e", &[_]Token.Id{ .Invalid }); + testTokenize("1.0e1f0", &[_]Token.Id{ .Invalid }); + testTokenize("1.0p100", &[_]Token.Id{ .Invalid }); + testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Minus, .IntegerLiteral }); + testTokenize("1.0p1f0", &[_]Token.Id{ .Invalid }); + testTokenize("1.0_,", &[_]Token.Id{ .Invalid }); + testTokenize("1.0e,", &[_]Token.Id{ .Invalid }); +} + + +test "tokenizer - number literals binary" { + testTokenize("0b0", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0b1", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0b2", &[_]Token.Id{ .Invalid }); + testTokenize("0b3", &[_]Token.Id{ .Invalid }); + testTokenize("0b4", &[_]Token.Id{ .Invalid }); + testTokenize("0b5", &[_]Token.Id{ .Invalid }); + testTokenize("0b6", &[_]Token.Id{ .Invalid }); + testTokenize("0b7", &[_]Token.Id{ .Invalid }); + testTokenize("0b8", &[_]Token.Id{ .Invalid }); + testTokenize("0b9", &[_]Token.Id{ .Invalid }); + testTokenize("0ba", &[_]Token.Id{ .Invalid }); + testTokenize("0bb", &[_]Token.Id{ .Invalid }); + testTokenize("0bc", &[_]Token.Id{ .Invalid }); + testTokenize("0bd", &[_]Token.Id{ .Invalid }); + testTokenize("0be", &[_]Token.Id{ .Invalid }); + testTokenize("0bf", &[_]Token.Id{ .Invalid }); + testTokenize("0bz", &[_]Token.Id{ .Invalid }); + + testTokenize("0b0000_0000", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0b1111_1111", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0b10_10_10_10", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0b0_1_0_1_0_1_0_1", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period }); + testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral }); + + testTokenize("0B0", &[_]Token.Id{ .Invalid }); + testTokenize("0b_", &[_]Token.Id{ .Invalid }); + testTokenize("0b_0", &[_]Token.Id{ .Invalid }); + testTokenize("0b1_", &[_]Token.Id{ .Invalid }); + testTokenize("0b0__1", &[_]Token.Id{ .Invalid }); + testTokenize("0b0_1_", &[_]Token.Id{ .Invalid }); + testTokenize("0b1e", &[_]Token.Id{ .Invalid }); + testTokenize("0b1p", &[_]Token.Id{ .Invalid }); + testTokenize("0b1e0", &[_]Token.Id{ .Invalid }); + testTokenize("0b1p0", &[_]Token.Id{ .Invalid }); + testTokenize("0b1_,", &[_]Token.Id{ .Invalid }); +} + +test "tokenizer - number literals octal" { + testTokenize("0o0", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o1", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o2", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o3", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o4", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o5", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o6", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o7", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o8", &[_]Token.Id{ .Invalid }); + testTokenize("0o9", &[_]Token.Id{ .Invalid }); + testTokenize("0oa", &[_]Token.Id{ .Invalid }); + testTokenize("0ob", &[_]Token.Id{ .Invalid }); + testTokenize("0oc", &[_]Token.Id{ .Invalid }); + testTokenize("0od", &[_]Token.Id{ .Invalid }); + testTokenize("0oe", &[_]Token.Id{ .Invalid }); + testTokenize("0of", &[_]Token.Id{ .Invalid }); + testTokenize("0oz", &[_]Token.Id{ .Invalid }); + + testTokenize("0o01234567", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o0123_4567", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o01_23_45_67", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o0_1_2_3_4_5_6_7", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period }); + testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral }); + + testTokenize("0O0", &[_]Token.Id{ .Invalid }); + testTokenize("0o_", &[_]Token.Id{ .Invalid }); + testTokenize("0o_0", &[_]Token.Id{ .Invalid }); + testTokenize("0o1_", &[_]Token.Id{ .Invalid }); + testTokenize("0o0__1", &[_]Token.Id{ .Invalid }); + testTokenize("0o0_1_", &[_]Token.Id{ .Invalid }); + testTokenize("0o1e", &[_]Token.Id{ .Invalid }); + testTokenize("0o1p", &[_]Token.Id{ .Invalid }); + testTokenize("0o1e0", &[_]Token.Id{ .Invalid }); + testTokenize("0o1p0", &[_]Token.Id{ .Invalid }); + testTokenize("0o_,", &[_]Token.Id{ .Invalid }); +} + +test "tokenizer - number literals hexadeciaml" { + testTokenize("0x0", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x1", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x2", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x3", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x4", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x5", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x6", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x7", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x8", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x9", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xa", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xb", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xc", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xd", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xe", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xf", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xA", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xB", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xC", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xD", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xE", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0xF", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x0z", &[_]Token.Id{ .Invalid }); + testTokenize("0xz", &[_]Token.Id{ .Invalid }); + + testTokenize("0x0123456789ABCDEF", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{ .IntegerLiteral }); + + testTokenize("0X0", &[_]Token.Id{ .Invalid }); + testTokenize("0x_", &[_]Token.Id{ .Invalid }); + testTokenize("0x_1", &[_]Token.Id{ .Invalid }); + testTokenize("0x1_", &[_]Token.Id{ .Invalid }); + testTokenize("0x0__1", &[_]Token.Id{ .Invalid }); + testTokenize("0x0_1_", &[_]Token.Id{ .Invalid }); + testTokenize("0x_,", &[_]Token.Id{ .Invalid }); + + testTokenize("0x1.", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0x1.0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xF.", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xF.0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xF.F", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xF.Fp0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xF.FP0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0x1p0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xfp0", &[_]Token.Id{ .FloatLiteral }); + + testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0x0.0p0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xff.ffp10", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xff.ffP10", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xff.p10", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xffp10", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xff_ff.ff_ffp1_0_0_0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &[_]Token.Id{ .FloatLiteral }); + + testTokenize("0x1e", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x1e0", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x1p", &[_]Token.Id{ .Invalid }); + testTokenize("0xfp0z1", &[_]Token.Id{ .Invalid }); + testTokenize("0xff.ffpff", &[_]Token.Id{ .Invalid }); + testTokenize("0x0_.0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0._0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0.0_", &[_]Token.Id{ .Invalid }); + testTokenize("0x0_p0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0_.p0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0._p0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0._0p0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0.0p_0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0.0p+_0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0.0p-_0", &[_]Token.Id{ .Invalid }); + testTokenize("0x0.0p0_", &[_]Token.Id{ .Invalid }); +} + + fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| { -- 2.54.0 From 29324e6f393011ee010b80f2c88d946f082bef22 Mon Sep 17 00:00:00 2001 From: momumi <57862114+momumi@users.noreply.github.com> Date: Sun, 22 Mar 2020 12:41:11 +1000 Subject: [PATCH 5/7] fix formatting in tokenizer tests --- lib/std/zig/tokenizer.zig | 361 +++++++++++++++++++------------------- 1 file changed, 180 insertions(+), 181 deletions(-) diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 8c9b39d34f056463d38321b1b338f050ee9101aa..6beaba0fe54d33d79a89b41a81a770f501c56896 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -1696,216 +1696,215 @@ test "correctly parse pointer assignment" { } test "tokenizer - number literals decimal" { - testTokenize("1", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("2", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("3", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("4", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("5", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("6", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("7", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("8", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("9", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0a", &[_]Token.Id{ .Invalid }); - testTokenize("9b", &[_]Token.Id{ .Invalid }); - testTokenize("1z", &[_]Token.Id{ .Invalid }); - testTokenize("1z_1", &[_]Token.Id{ .Invalid }); - testTokenize("9z3", &[_]Token.Id{ .Invalid }); + testTokenize("0", &[_]Token.Id{.IntegerLiteral}); + testTokenize("1", &[_]Token.Id{.IntegerLiteral}); + testTokenize("2", &[_]Token.Id{.IntegerLiteral}); + testTokenize("3", &[_]Token.Id{.IntegerLiteral}); + testTokenize("4", &[_]Token.Id{.IntegerLiteral}); + testTokenize("5", &[_]Token.Id{.IntegerLiteral}); + testTokenize("6", &[_]Token.Id{.IntegerLiteral}); + testTokenize("7", &[_]Token.Id{.IntegerLiteral}); + testTokenize("8", &[_]Token.Id{.IntegerLiteral}); + testTokenize("9", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0a", &[_]Token.Id{.Invalid}); + testTokenize("9b", &[_]Token.Id{.Invalid}); + testTokenize("1z", &[_]Token.Id{.Invalid}); + testTokenize("1z_1", &[_]Token.Id{.Invalid}); + testTokenize("9z3", &[_]Token.Id{.Invalid}); - testTokenize("0_0", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0001", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("01234567890", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("012_345_6789_0", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0_1_2_3_4_5_6_7_8_9_0", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0_0", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0001", &[_]Token.Id{.IntegerLiteral}); + testTokenize("01234567890", &[_]Token.Id{.IntegerLiteral}); + testTokenize("012_345_6789_0", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0_1_2_3_4_5_6_7_8_9_0", &[_]Token.Id{.IntegerLiteral}); - testTokenize("00_", &[_]Token.Id{ .Invalid }); - testTokenize("0_0_", &[_]Token.Id{ .Invalid }); - testTokenize("0__0", &[_]Token.Id{ .Invalid }); - testTokenize("0_0f", &[_]Token.Id{ .Invalid }); - testTokenize("0_0_f", &[_]Token.Id{ .Invalid }); - testTokenize("1_,", &[_]Token.Id{ .Invalid }); + testTokenize("00_", &[_]Token.Id{.Invalid}); + testTokenize("0_0_", &[_]Token.Id{.Invalid}); + testTokenize("0__0", &[_]Token.Id{.Invalid}); + testTokenize("0_0f", &[_]Token.Id{.Invalid}); + testTokenize("0_0_f", &[_]Token.Id{.Invalid}); + testTokenize("1_,", &[_]Token.Id{.Invalid}); - testTokenize("1.", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0.0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("1.0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("10.0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("1e0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("1e100", &[_]Token.Id{ .FloatLiteral }); - testTokenize("1.e100", &[_]Token.Id{ .FloatLiteral }); - testTokenize("1.0e100", &[_]Token.Id{ .FloatLiteral }); - testTokenize("1.0e+100", &[_]Token.Id{ .FloatLiteral }); - testTokenize("1.0e-100", &[_]Token.Id{ .FloatLiteral }); - testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("1.", &[_]Token.Id{.FloatLiteral}); + testTokenize("0.0", &[_]Token.Id{.FloatLiteral}); + testTokenize("1.0", &[_]Token.Id{.FloatLiteral}); + testTokenize("10.0", &[_]Token.Id{.FloatLiteral}); + testTokenize("1e0", &[_]Token.Id{.FloatLiteral}); + testTokenize("1e100", &[_]Token.Id{.FloatLiteral}); + testTokenize("1.e100", &[_]Token.Id{.FloatLiteral}); + testTokenize("1.0e100", &[_]Token.Id{.FloatLiteral}); + testTokenize("1.0e+100", &[_]Token.Id{.FloatLiteral}); + testTokenize("1.0e-100", &[_]Token.Id{.FloatLiteral}); + testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{.FloatLiteral}); - testTokenize("1e", &[_]Token.Id{ .Invalid }); - testTokenize("1.0e1f0", &[_]Token.Id{ .Invalid }); - testTokenize("1.0p100", &[_]Token.Id{ .Invalid }); + testTokenize("1e", &[_]Token.Id{.Invalid}); + testTokenize("1.0e1f0", &[_]Token.Id{.Invalid}); + testTokenize("1.0p100", &[_]Token.Id{.Invalid}); testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Minus, .IntegerLiteral }); - testTokenize("1.0p1f0", &[_]Token.Id{ .Invalid }); - testTokenize("1.0_,", &[_]Token.Id{ .Invalid }); - testTokenize("1.0e,", &[_]Token.Id{ .Invalid }); + testTokenize("1.0p1f0", &[_]Token.Id{.Invalid}); + testTokenize("1.0_,", &[_]Token.Id{.Invalid}); + testTokenize("1.0e,", &[_]Token.Id{.Invalid}); } - test "tokenizer - number literals binary" { - testTokenize("0b0", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0b1", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0b2", &[_]Token.Id{ .Invalid }); - testTokenize("0b3", &[_]Token.Id{ .Invalid }); - testTokenize("0b4", &[_]Token.Id{ .Invalid }); - testTokenize("0b5", &[_]Token.Id{ .Invalid }); - testTokenize("0b6", &[_]Token.Id{ .Invalid }); - testTokenize("0b7", &[_]Token.Id{ .Invalid }); - testTokenize("0b8", &[_]Token.Id{ .Invalid }); - testTokenize("0b9", &[_]Token.Id{ .Invalid }); - testTokenize("0ba", &[_]Token.Id{ .Invalid }); - testTokenize("0bb", &[_]Token.Id{ .Invalid }); - testTokenize("0bc", &[_]Token.Id{ .Invalid }); - testTokenize("0bd", &[_]Token.Id{ .Invalid }); - testTokenize("0be", &[_]Token.Id{ .Invalid }); - testTokenize("0bf", &[_]Token.Id{ .Invalid }); - testTokenize("0bz", &[_]Token.Id{ .Invalid }); + testTokenize("0b0", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0b1", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0b2", &[_]Token.Id{.Invalid}); + testTokenize("0b3", &[_]Token.Id{.Invalid}); + testTokenize("0b4", &[_]Token.Id{.Invalid}); + testTokenize("0b5", &[_]Token.Id{.Invalid}); + testTokenize("0b6", &[_]Token.Id{.Invalid}); + testTokenize("0b7", &[_]Token.Id{.Invalid}); + testTokenize("0b8", &[_]Token.Id{.Invalid}); + testTokenize("0b9", &[_]Token.Id{.Invalid}); + testTokenize("0ba", &[_]Token.Id{.Invalid}); + testTokenize("0bb", &[_]Token.Id{.Invalid}); + testTokenize("0bc", &[_]Token.Id{.Invalid}); + testTokenize("0bd", &[_]Token.Id{.Invalid}); + testTokenize("0be", &[_]Token.Id{.Invalid}); + testTokenize("0bf", &[_]Token.Id{.Invalid}); + testTokenize("0bz", &[_]Token.Id{.Invalid}); - testTokenize("0b0000_0000", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0b1111_1111", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0b10_10_10_10", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0b0_1_0_1_0_1_0_1", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0b0000_0000", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0b1111_1111", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0b10_10_10_10", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0b0_1_0_1_0_1_0_1", &[_]Token.Id{.IntegerLiteral}); testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period }); testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral }); - testTokenize("0B0", &[_]Token.Id{ .Invalid }); - testTokenize("0b_", &[_]Token.Id{ .Invalid }); - testTokenize("0b_0", &[_]Token.Id{ .Invalid }); - testTokenize("0b1_", &[_]Token.Id{ .Invalid }); - testTokenize("0b0__1", &[_]Token.Id{ .Invalid }); - testTokenize("0b0_1_", &[_]Token.Id{ .Invalid }); - testTokenize("0b1e", &[_]Token.Id{ .Invalid }); - testTokenize("0b1p", &[_]Token.Id{ .Invalid }); - testTokenize("0b1e0", &[_]Token.Id{ .Invalid }); - testTokenize("0b1p0", &[_]Token.Id{ .Invalid }); - testTokenize("0b1_,", &[_]Token.Id{ .Invalid }); + testTokenize("0B0", &[_]Token.Id{.Invalid}); + testTokenize("0b_", &[_]Token.Id{.Invalid}); + testTokenize("0b_0", &[_]Token.Id{.Invalid}); + testTokenize("0b1_", &[_]Token.Id{.Invalid}); + testTokenize("0b0__1", &[_]Token.Id{.Invalid}); + testTokenize("0b0_1_", &[_]Token.Id{.Invalid}); + testTokenize("0b1e", &[_]Token.Id{.Invalid}); + testTokenize("0b1p", &[_]Token.Id{.Invalid}); + testTokenize("0b1e0", &[_]Token.Id{.Invalid}); + testTokenize("0b1p0", &[_]Token.Id{.Invalid}); + testTokenize("0b1_,", &[_]Token.Id{.Invalid}); } test "tokenizer - number literals octal" { - testTokenize("0o0", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o1", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o2", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o3", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o4", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o5", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o6", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o7", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o8", &[_]Token.Id{ .Invalid }); - testTokenize("0o9", &[_]Token.Id{ .Invalid }); - testTokenize("0oa", &[_]Token.Id{ .Invalid }); - testTokenize("0ob", &[_]Token.Id{ .Invalid }); - testTokenize("0oc", &[_]Token.Id{ .Invalid }); - testTokenize("0od", &[_]Token.Id{ .Invalid }); - testTokenize("0oe", &[_]Token.Id{ .Invalid }); - testTokenize("0of", &[_]Token.Id{ .Invalid }); - testTokenize("0oz", &[_]Token.Id{ .Invalid }); + testTokenize("0o0", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o1", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o2", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o3", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o4", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o5", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o6", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o7", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o8", &[_]Token.Id{.Invalid}); + testTokenize("0o9", &[_]Token.Id{.Invalid}); + testTokenize("0oa", &[_]Token.Id{.Invalid}); + testTokenize("0ob", &[_]Token.Id{.Invalid}); + testTokenize("0oc", &[_]Token.Id{.Invalid}); + testTokenize("0od", &[_]Token.Id{.Invalid}); + testTokenize("0oe", &[_]Token.Id{.Invalid}); + testTokenize("0of", &[_]Token.Id{.Invalid}); + testTokenize("0oz", &[_]Token.Id{.Invalid}); - testTokenize("0o01234567", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o0123_4567", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o01_23_45_67", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0o0_1_2_3_4_5_6_7", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0o01234567", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o0123_4567", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o01_23_45_67", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0o0_1_2_3_4_5_6_7", &[_]Token.Id{.IntegerLiteral}); testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period }); testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral }); - testTokenize("0O0", &[_]Token.Id{ .Invalid }); - testTokenize("0o_", &[_]Token.Id{ .Invalid }); - testTokenize("0o_0", &[_]Token.Id{ .Invalid }); - testTokenize("0o1_", &[_]Token.Id{ .Invalid }); - testTokenize("0o0__1", &[_]Token.Id{ .Invalid }); - testTokenize("0o0_1_", &[_]Token.Id{ .Invalid }); - testTokenize("0o1e", &[_]Token.Id{ .Invalid }); - testTokenize("0o1p", &[_]Token.Id{ .Invalid }); - testTokenize("0o1e0", &[_]Token.Id{ .Invalid }); - testTokenize("0o1p0", &[_]Token.Id{ .Invalid }); - testTokenize("0o_,", &[_]Token.Id{ .Invalid }); + testTokenize("0O0", &[_]Token.Id{.Invalid}); + testTokenize("0o_", &[_]Token.Id{.Invalid}); + testTokenize("0o_0", &[_]Token.Id{.Invalid}); + testTokenize("0o1_", &[_]Token.Id{.Invalid}); + testTokenize("0o0__1", &[_]Token.Id{.Invalid}); + testTokenize("0o0_1_", &[_]Token.Id{.Invalid}); + testTokenize("0o1e", &[_]Token.Id{.Invalid}); + testTokenize("0o1p", &[_]Token.Id{.Invalid}); + testTokenize("0o1e0", &[_]Token.Id{.Invalid}); + testTokenize("0o1p0", &[_]Token.Id{.Invalid}); + testTokenize("0o_,", &[_]Token.Id{.Invalid}); } test "tokenizer - number literals hexadeciaml" { - testTokenize("0x0", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x1", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x2", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x3", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x4", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x5", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x6", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x7", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x8", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x9", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xa", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xb", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xc", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xd", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xe", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xf", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xA", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xB", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xC", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xD", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xE", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0xF", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x0z", &[_]Token.Id{ .Invalid }); - testTokenize("0xz", &[_]Token.Id{ .Invalid }); + testTokenize("0x0", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x1", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x2", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x3", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x4", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x5", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x6", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x7", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x8", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x9", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xa", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xb", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xc", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xd", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xe", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xf", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xA", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xB", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xC", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xD", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xE", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0xF", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x0z", &[_]Token.Id{.Invalid}); + testTokenize("0xz", &[_]Token.Id{.Invalid}); - testTokenize("0x0123456789ABCDEF", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{ .IntegerLiteral }); + testTokenize("0x0123456789ABCDEF", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{.IntegerLiteral}); - testTokenize("0X0", &[_]Token.Id{ .Invalid }); - testTokenize("0x_", &[_]Token.Id{ .Invalid }); - testTokenize("0x_1", &[_]Token.Id{ .Invalid }); - testTokenize("0x1_", &[_]Token.Id{ .Invalid }); - testTokenize("0x0__1", &[_]Token.Id{ .Invalid }); - testTokenize("0x0_1_", &[_]Token.Id{ .Invalid }); - testTokenize("0x_,", &[_]Token.Id{ .Invalid }); + testTokenize("0X0", &[_]Token.Id{.Invalid}); + testTokenize("0x_", &[_]Token.Id{.Invalid}); + testTokenize("0x_1", &[_]Token.Id{.Invalid}); + testTokenize("0x1_", &[_]Token.Id{.Invalid}); + testTokenize("0x0__1", &[_]Token.Id{.Invalid}); + testTokenize("0x0_1_", &[_]Token.Id{.Invalid}); + testTokenize("0x_,", &[_]Token.Id{.Invalid}); - testTokenize("0x1.", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0x1.0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xF.", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xF.0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xF.F", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xF.Fp0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xF.FP0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0x1p0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xfp0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0x1.", &[_]Token.Id{.FloatLiteral}); + testTokenize("0x1.0", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xF.", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xF.0", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xF.F", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xF.Fp0", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xF.FP0", &[_]Token.Id{.FloatLiteral}); + testTokenize("0x1p0", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xfp0", &[_]Token.Id{.FloatLiteral}); - testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0x0.0p0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xff.ffp10", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xff.ffP10", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xff.p10", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xffp10", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xff_ff.ff_ffp1_0_0_0", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &[_]Token.Id{ .FloatLiteral }); - testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &[_]Token.Id{ .FloatLiteral }); + testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{.FloatLiteral}); + testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{.FloatLiteral}); + testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{.FloatLiteral}); + testTokenize("0x0.0p0", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xff.ffp10", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xff.ffP10", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xff.p10", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xffp10", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xff_ff.ff_ffp1_0_0_0", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xf_f_f_f.f_f_f_fp+1_000", &[_]Token.Id{.FloatLiteral}); + testTokenize("0xf_f_f_f.f_f_f_fp-1_00_0", &[_]Token.Id{.FloatLiteral}); - testTokenize("0x1e", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x1e0", &[_]Token.Id{ .IntegerLiteral }); - testTokenize("0x1p", &[_]Token.Id{ .Invalid }); - testTokenize("0xfp0z1", &[_]Token.Id{ .Invalid }); - testTokenize("0xff.ffpff", &[_]Token.Id{ .Invalid }); - testTokenize("0x0_.0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0._0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0.0_", &[_]Token.Id{ .Invalid }); - testTokenize("0x0_p0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0_.p0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0._p0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0._0p0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0.0p_0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0.0p+_0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0.0p-_0", &[_]Token.Id{ .Invalid }); - testTokenize("0x0.0p0_", &[_]Token.Id{ .Invalid }); + testTokenize("0x1e", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x1e0", &[_]Token.Id{.IntegerLiteral}); + testTokenize("0x1p", &[_]Token.Id{.Invalid}); + testTokenize("0xfp0z1", &[_]Token.Id{.Invalid}); + testTokenize("0xff.ffpff", &[_]Token.Id{.Invalid}); + testTokenize("0x0_.0", &[_]Token.Id{.Invalid}); + testTokenize("0x0._0", &[_]Token.Id{.Invalid}); + testTokenize("0x0.0_", &[_]Token.Id{.Invalid}); + testTokenize("0x0_p0", &[_]Token.Id{.Invalid}); + testTokenize("0x0_.p0", &[_]Token.Id{.Invalid}); + testTokenize("0x0._p0", &[_]Token.Id{.Invalid}); + testTokenize("0x0.0_p0", &[_]Token.Id{.Invalid}); + testTokenize("0x0._0p0", &[_]Token.Id{.Invalid}); + testTokenize("0x0.0_p0", &[_]Token.Id{.Invalid}); + testTokenize("0x0.0p_0", &[_]Token.Id{.Invalid}); + testTokenize("0x0.0p+_0", &[_]Token.Id{.Invalid}); + testTokenize("0x0.0p-_0", &[_]Token.Id{.Invalid}); + testTokenize("0x0.0p0_", &[_]Token.Id{.Invalid}); } - fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| { -- 2.54.0 From 8de45e51438d7f593825ae2134ab7dfc46f3bab2 Mon Sep 17 00:00:00 2001 From: momumi <57862114+momumi@users.noreply.github.com> Date: Sun, 22 Mar 2020 13:45:31 +1000 Subject: [PATCH 6/7] update parsing of int literals in self-hosted * update std.math.big.Int.setString() to ignore underscores and make it case insensitive * fix issue in ir.zig with leading zeroes in integer literals --- lib/std/math/big/int.zig | 25 +++++++++++++++++++++++-- src-self-hosted/ir.zig | 7 +++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/lib/std/math/big/int.zig b/lib/std/math/big/int.zig index 95d0764f6866e429225e8872dc205c045fe2d9cf..3557a407989465d2e03365b344dd4593b61f2e5b 100644 --- a/lib/std/math/big/int.zig +++ b/lib/std/math/big/int.zig @@ -373,6 +373,7 @@ pub const Int = struct { const d = switch (ch) { '0'...'9' => ch - '0', 'a'...'f' => (ch - 'a') + 0xa, + 'A'...'F' => (ch - 'A') + 0xa, else => return error.InvalidCharForDigit, }; @@ -393,8 +394,9 @@ pub const Int = struct { /// Set self from the string representation `value`. /// - /// value must contain only digits <= `base`. Base prefixes are not allowed (e.g. 0x43 should - /// simply be 43). + /// `value` must contain only digits <= `base` and is case insensitive. Base prefixes are + /// not allowed (e.g. 0x43 should simply be 43). Underscores in the input string are + /// ignored and can be used as digit separators. /// /// Returns an error if memory could not be allocated or `value` has invalid digits for the /// requested base. @@ -415,6 +417,9 @@ pub const Int = struct { try self.set(0); for (value[i..]) |ch| { + if (ch == '_') { + continue; + } const d = try charToDigit(ch, base); const ap_d = Int.initFixed(([_]Limb{d})[0..]); @@ -1582,6 +1587,22 @@ test "big.int string negative" { testing.expect((try a.to(i32)) == -1023); } +test "big.int string set number with underscores" { + var a = try Int.init(testing.allocator); + defer a.deinit(); + + try a.setString(10, "__1_2_0_3_1_7_2_4_1_2_0_____9_1__2__4_7_8_1_2_4_1_2_9_0_8_4_7_1_2_4___"); + testing.expect((try a.to(u128)) == 120317241209124781241290847124); +} + +test "big.int string set case insensitive number" { + var a = try Int.init(testing.allocator); + defer a.deinit(); + + try a.setString(16, "aB_cD_eF"); + testing.expect((try a.to(u32)) == 0xabcdef); +} + test "big.int string set bad char error" { var a = try Int.init(testing.allocator); defer a.deinit(); diff --git a/src-self-hosted/ir.zig b/src-self-hosted/ir.zig index 2e65962d415a5857df0d80c87257746a525e8268..e3d13b43db98dfb436e15cfde950b67ed279be0f 100644 --- a/src-self-hosted/ir.zig +++ b/src-self-hosted/ir.zig @@ -1311,13 +1311,16 @@ pub const Builder = struct { var base: u8 = undefined; var rest: []const u8 = undefined; if (int_token.len >= 3 and int_token[0] == '0') { + rest = int_token[2..]; base = switch (int_token[1]) { 'b' => 2, 'o' => 8, 'x' => 16, - else => unreachable, + else => { + base = 10; + rest = int_token; + }, }; - rest = int_token[2..]; } else { base = 10; rest = int_token; -- 2.54.0 From 2d18178c27060ff9b9b4a5b56941617dc47868d0 Mon Sep 17 00:00:00 2001 From: momumi <57862114+momumi@users.noreply.github.com> Date: Mon, 23 Mar 2020 09:21:34 +1000 Subject: [PATCH 7/7] minor fixes and more tests for _ separators * Make the tokenizer spit out an Invalid token on the first invalid character found in the number literal. * More parsing and tokenizer tests for number literals * fix invalid switch statement in ir.zig --- lib/std/zig/parser_test.zig | 69 +++++++++ lib/std/zig/tokenizer.zig | 275 ++++++++++++++++++++++-------------- src-self-hosted/ir.zig | 10 +- 3 files changed, 242 insertions(+), 112 deletions(-) diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index d00568e49f92a9139d923eeb2c2b377606252566..00f6b33673a16cfa6f5eb95ee16776552aa495db 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -2800,6 +2800,75 @@ test "zig fmt: extern without container keyword returns error" { ); } +test "zig fmt: integer literals with underscore separators" { + try testTransform( + \\const + \\ x = + \\ 1_234_567 + \\ +(0b0_1-0o7_0+0xff_FF ) + 0_0; + , + \\const x = 1_234_567 + (0b0_1 - 0o7_0 + 0xff_FF) + 0_0; + \\ + ); +} + +test "zig fmt: hex literals with underscore separators" { + try testTransform( + \\pub fn orMask(a: [ 1_000 ]u64, b: [ 1_000] u64) [1_000]u64 { + \\ var c: [1_000]u64 = [1]u64{ 0xFFFF_FFFF_FFFF_FFFF}**1_000; + \\ for (c [ 0_0 .. ]) |_, i| { + \\ c[i] = (a[i] | b[i]) & 0xCCAA_CCAA_CCAA_CCAA; + \\ } + \\ return c; + \\} + \\ + \\ + , + \\pub fn orMask(a: [1_000]u64, b: [1_000]u64) [1_000]u64 { + \\ var c: [1_000]u64 = [1]u64{0xFFFF_FFFF_FFFF_FFFF} ** 1_000; + \\ for (c[0_0..]) |_, i| { + \\ c[i] = (a[i] | b[i]) & 0xCCAA_CCAA_CCAA_CCAA; + \\ } + \\ return c; + \\} + \\ + ); +} + +test "zig fmt: decimal float literals with underscore separators" { + try testTransform( + \\pub fn main() void { + \\ const a:f64=(10.0e-0+(10.e+0))+10_00.00_00e-2+00_00.00_10e+4; + \\ const b:f64=010.0--0_10.+0_1_0.0_0+1e2; + \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b }); + \\} + , + \\pub fn main() void { + \\ const a: f64 = (10.0e-0 + (10.e+0)) + 10_00.00_00e-2 + 00_00.00_10e+4; + \\ const b: f64 = 010.0 - -0_10. + 0_1_0.0_0 + 1e2; + \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b }); + \\} + \\ + ); +} + +test "zig fmt: hexadeciaml float literals with underscore separators" { + try testTransform( + \\pub fn main() void { + \\ const a: f64 = (0x10.0p-0+(0x10.p+0))+0x10_00.00_00p-8+0x00_00.00_10p+16; + \\ const b: f64 = 0x0010.0--0x00_10.+0x10.00+0x1p4; + \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b }); + \\} + , + \\pub fn main() void { + \\ const a: f64 = (0x10.0p-0 + (0x10.p+0)) + 0x10_00.00_00p-8 + 0x00_00.00_10p+16; + \\ const b: f64 = 0x0010.0 - -0x00_10. + 0x10.00 + 0x1p4; + \\ std.debug.warn("a: {}, b: {} -> a+b: {}\n", .{ a, b, a + b }); + \\} + \\ + ); +} + const std = @import("std"); const mem = std.mem; const warn = std.debug.warn; diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 6beaba0fe54d33d79a89b41a81a770f501c56896..6cb66595a7e62db72691b5239798fcdefc2dbad1 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -418,6 +418,10 @@ pub const Tokenizer = struct { SawAtSign, }; + fn isIdentifierChar(char: u8) bool { + return std.ascii.isAlNum(char) or char == '_'; + } + pub fn next(self: *Tokenizer) Token { if (self.pending_invalid_token) |token| { self.pending_invalid_token = null; @@ -1063,11 +1067,17 @@ pub const Tokenizer = struct { 'x' => { state = State.IntegerLiteralHexNoUnderscore; }, - else => { - // reinterpret as a normal number + '0'...'9', '_', '.', 'e', 'E' => { + // reinterpret as a decimal number self.index -= 1; state = State.IntegerLiteralDec; }, + else => { + if (isIdentifierChar(c)) { + result.id = Token.Id.Invalid; + } + break; + }, }, State.IntegerLiteralBinNoUnderscore => switch (c) { '0'...'1' => { @@ -1075,6 +1085,7 @@ pub const Tokenizer = struct { }, else => { result.id = Token.Id.Invalid; + break; }, }, State.IntegerLiteralBin => switch (c) { @@ -1082,10 +1093,12 @@ pub const Tokenizer = struct { state = State.IntegerLiteralBinNoUnderscore; }, '0'...'1' => {}, - '2'...'9', 'a'...'z', 'A'...'Z' => { - result.id = Token.Id.Invalid; + else => { + if (isIdentifierChar(c)) { + result.id = Token.Id.Invalid; + } + break; }, - else => break, }, State.IntegerLiteralOctNoUnderscore => switch (c) { '0'...'7' => { @@ -1093,6 +1106,7 @@ pub const Tokenizer = struct { }, else => { result.id = Token.Id.Invalid; + break; }, }, State.IntegerLiteralOct => switch (c) { @@ -1100,10 +1114,12 @@ pub const Tokenizer = struct { state = State.IntegerLiteralOctNoUnderscore; }, '0'...'7' => {}, - '8'...'9', 'a'...'z', 'A'...'Z' => { - result.id = Token.Id.Invalid; + else => { + if (isIdentifierChar(c)) { + result.id = Token.Id.Invalid; + } + break; }, - else => break, }, State.IntegerLiteralDecNoUnderscore => switch (c) { '0'...'9' => { @@ -1111,6 +1127,7 @@ pub const Tokenizer = struct { }, else => { result.id = Token.Id.Invalid; + break; }, }, State.IntegerLiteralDec => switch (c) { @@ -1126,10 +1143,12 @@ pub const Tokenizer = struct { result.id = Token.Id.FloatLiteral; }, '0'...'9' => {}, - 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => { - result.id = Token.Id.Invalid; + else => { + if (isIdentifierChar(c)) { + result.id = Token.Id.Invalid; + } + break; }, - else => break, }, State.IntegerLiteralHexNoUnderscore => switch (c) { '0'...'9', 'a'...'f', 'A'...'F' => { @@ -1137,6 +1156,7 @@ pub const Tokenizer = struct { }, else => { result.id = Token.Id.Invalid; + break; }, }, State.IntegerLiteralHex => switch (c) { @@ -1152,10 +1172,12 @@ pub const Tokenizer = struct { result.id = Token.Id.FloatLiteral; }, '0'...'9', 'a'...'f', 'A'...'F' => {}, - 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => { - result.id = Token.Id.Invalid; + else => { + if (isIdentifierChar(c)) { + result.id = Token.Id.Invalid; + } + break; }, - else => break, }, State.NumberDotDec => switch (c) { '.' => { @@ -1166,10 +1188,15 @@ pub const Tokenizer = struct { 'e', 'E' => { state = State.FloatExponentUnsigned; }, - else => { - self.index -= 1; + '0'...'9' => { result.id = Token.Id.FloatLiteral; - state = State.FloatFractionDecNoUnderscore; + state = State.FloatFractionDec; + }, + else => { + if (isIdentifierChar(c)) { + result.id = Token.Id.Invalid; + } + break; }, }, State.NumberDotHex => switch (c) { @@ -1181,10 +1208,15 @@ pub const Tokenizer = struct { 'p', 'P' => { state = State.FloatExponentUnsigned; }, - else => { - self.index -= 1; + '0'...'9', 'a'...'f', 'A'...'F' => { result.id = Token.Id.FloatLiteral; - state = State.FloatFractionHexNoUnderscore; + state = State.FloatFractionHex; + }, + else => { + if (isIdentifierChar(c)) { + result.id = Token.Id.Invalid; + } + break; }, }, State.FloatFractionDecNoUnderscore => switch (c) { @@ -1193,6 +1225,7 @@ pub const Tokenizer = struct { }, else => { result.id = Token.Id.Invalid; + break; }, }, State.FloatFractionDec => switch (c) { @@ -1203,10 +1236,12 @@ pub const Tokenizer = struct { state = State.FloatExponentUnsigned; }, '0'...'9' => {}, - 'a'...'d', 'f'...'z', 'A'...'D', 'F'...'Z' => { - result.id = Token.Id.Invalid; + else => { + if (isIdentifierChar(c)) { + result.id = Token.Id.Invalid; + } + break; }, - else => break, }, State.FloatFractionHexNoUnderscore => switch (c) { '0'...'9', 'a'...'f', 'A'...'F' => { @@ -1214,6 +1249,7 @@ pub const Tokenizer = struct { }, else => { result.id = Token.Id.Invalid; + break; }, }, State.FloatFractionHex => switch (c) { @@ -1224,10 +1260,12 @@ pub const Tokenizer = struct { state = State.FloatExponentUnsigned; }, '0'...'9', 'a'...'f', 'A'...'F' => {}, - 'g'...'o', 'q'...'z', 'G'...'O', 'Q'...'Z' => { - result.id = Token.Id.Invalid; + else => { + if (isIdentifierChar(c)) { + result.id = Token.Id.Invalid; + } + break; }, - else => break, }, State.FloatExponentUnsigned => switch (c) { '+', '-' => { @@ -1245,6 +1283,7 @@ pub const Tokenizer = struct { }, else => { result.id = Token.Id.Invalid; + break; }, }, State.FloatExponentNumber => switch (c) { @@ -1252,10 +1291,12 @@ pub const Tokenizer = struct { state = State.FloatExponentNumberNoUnderscore; }, '0'...'9' => {}, - 'a'...'z', 'A'...'Z' => { - result.id = Token.Id.Invalid; + else => { + if (isIdentifierChar(c)) { + result.id = Token.Id.Invalid; + } + break; }, - else => break, }, } } else if (self.index == self.buffer.len) { @@ -1706,11 +1747,11 @@ test "tokenizer - number literals decimal" { testTokenize("7", &[_]Token.Id{.IntegerLiteral}); testTokenize("8", &[_]Token.Id{.IntegerLiteral}); testTokenize("9", &[_]Token.Id{.IntegerLiteral}); - testTokenize("0a", &[_]Token.Id{.Invalid}); - testTokenize("9b", &[_]Token.Id{.Invalid}); - testTokenize("1z", &[_]Token.Id{.Invalid}); - testTokenize("1z_1", &[_]Token.Id{.Invalid}); - testTokenize("9z3", &[_]Token.Id{.Invalid}); + testTokenize("0a", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("9b", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1z", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1z_1", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("9z3", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0_0", &[_]Token.Id{.IntegerLiteral}); testTokenize("0001", &[_]Token.Id{.IntegerLiteral}); @@ -1720,15 +1761,17 @@ test "tokenizer - number literals decimal" { testTokenize("00_", &[_]Token.Id{.Invalid}); testTokenize("0_0_", &[_]Token.Id{.Invalid}); - testTokenize("0__0", &[_]Token.Id{.Invalid}); - testTokenize("0_0f", &[_]Token.Id{.Invalid}); - testTokenize("0_0_f", &[_]Token.Id{.Invalid}); - testTokenize("1_,", &[_]Token.Id{.Invalid}); + testTokenize("0__0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0_0f", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0_0_f", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0_0_f_00", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1_,", &[_]Token.Id{ .Invalid, .Comma }); testTokenize("1.", &[_]Token.Id{.FloatLiteral}); testTokenize("0.0", &[_]Token.Id{.FloatLiteral}); testTokenize("1.0", &[_]Token.Id{.FloatLiteral}); testTokenize("10.0", &[_]Token.Id{.FloatLiteral}); + testTokenize("0e0", &[_]Token.Id{.FloatLiteral}); testTokenize("1e0", &[_]Token.Id{.FloatLiteral}); testTokenize("1e100", &[_]Token.Id{.FloatLiteral}); testTokenize("1.e100", &[_]Token.Id{.FloatLiteral}); @@ -1736,34 +1779,47 @@ test "tokenizer - number literals decimal" { testTokenize("1.0e+100", &[_]Token.Id{.FloatLiteral}); testTokenize("1.0e-100", &[_]Token.Id{.FloatLiteral}); testTokenize("1_0_0_0.0_0_0_0_0_1e1_0_0_0", &[_]Token.Id{.FloatLiteral}); + testTokenize("1.+", &[_]Token.Id{ .FloatLiteral, .Plus }); testTokenize("1e", &[_]Token.Id{.Invalid}); - testTokenize("1.0e1f0", &[_]Token.Id{.Invalid}); - testTokenize("1.0p100", &[_]Token.Id{.Invalid}); - testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Minus, .IntegerLiteral }); - testTokenize("1.0p1f0", &[_]Token.Id{.Invalid}); - testTokenize("1.0_,", &[_]Token.Id{.Invalid}); - testTokenize("1.0e,", &[_]Token.Id{.Invalid}); + testTokenize("1.0e1f0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1.0p100", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1.0p-100", &[_]Token.Id{ .Invalid, .Identifier, .Minus, .IntegerLiteral }); + testTokenize("1.0p1f0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1.0_,", &[_]Token.Id{ .Invalid, .Comma }); + testTokenize("1_.0", &[_]Token.Id{ .Invalid, .Period, .IntegerLiteral }); + testTokenize("1._", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1.a", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1.z", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1._0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1._+", &[_]Token.Id{ .Invalid, .Identifier, .Plus }); + testTokenize("1._e", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1.0e", &[_]Token.Id{.Invalid}); + testTokenize("1.0e,", &[_]Token.Id{ .Invalid, .Comma }); + testTokenize("1.0e_", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1.0e+_", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1.0e-_", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("1.0e0_+", &[_]Token.Id{ .Invalid, .Plus }); } test "tokenizer - number literals binary" { testTokenize("0b0", &[_]Token.Id{.IntegerLiteral}); testTokenize("0b1", &[_]Token.Id{.IntegerLiteral}); - testTokenize("0b2", &[_]Token.Id{.Invalid}); - testTokenize("0b3", &[_]Token.Id{.Invalid}); - testTokenize("0b4", &[_]Token.Id{.Invalid}); - testTokenize("0b5", &[_]Token.Id{.Invalid}); - testTokenize("0b6", &[_]Token.Id{.Invalid}); - testTokenize("0b7", &[_]Token.Id{.Invalid}); - testTokenize("0b8", &[_]Token.Id{.Invalid}); - testTokenize("0b9", &[_]Token.Id{.Invalid}); - testTokenize("0ba", &[_]Token.Id{.Invalid}); - testTokenize("0bb", &[_]Token.Id{.Invalid}); - testTokenize("0bc", &[_]Token.Id{.Invalid}); - testTokenize("0bd", &[_]Token.Id{.Invalid}); - testTokenize("0be", &[_]Token.Id{.Invalid}); - testTokenize("0bf", &[_]Token.Id{.Invalid}); - testTokenize("0bz", &[_]Token.Id{.Invalid}); + testTokenize("0b2", &[_]Token.Id{ .Invalid, .IntegerLiteral }); + testTokenize("0b3", &[_]Token.Id{ .Invalid, .IntegerLiteral }); + testTokenize("0b4", &[_]Token.Id{ .Invalid, .IntegerLiteral }); + testTokenize("0b5", &[_]Token.Id{ .Invalid, .IntegerLiteral }); + testTokenize("0b6", &[_]Token.Id{ .Invalid, .IntegerLiteral }); + testTokenize("0b7", &[_]Token.Id{ .Invalid, .IntegerLiteral }); + testTokenize("0b8", &[_]Token.Id{ .Invalid, .IntegerLiteral }); + testTokenize("0b9", &[_]Token.Id{ .Invalid, .IntegerLiteral }); + testTokenize("0ba", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0bb", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0bc", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0bd", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0be", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0bf", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0bz", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0b0000_0000", &[_]Token.Id{.IntegerLiteral}); testTokenize("0b1111_1111", &[_]Token.Id{.IntegerLiteral}); @@ -1772,17 +1828,17 @@ test "tokenizer - number literals binary" { testTokenize("0b1.", &[_]Token.Id{ .IntegerLiteral, .Period }); testTokenize("0b1.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral }); - testTokenize("0B0", &[_]Token.Id{.Invalid}); - testTokenize("0b_", &[_]Token.Id{.Invalid}); - testTokenize("0b_0", &[_]Token.Id{.Invalid}); + testTokenize("0B0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0b_", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0b_0", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0b1_", &[_]Token.Id{.Invalid}); - testTokenize("0b0__1", &[_]Token.Id{.Invalid}); + testTokenize("0b0__1", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0b0_1_", &[_]Token.Id{.Invalid}); - testTokenize("0b1e", &[_]Token.Id{.Invalid}); - testTokenize("0b1p", &[_]Token.Id{.Invalid}); - testTokenize("0b1e0", &[_]Token.Id{.Invalid}); - testTokenize("0b1p0", &[_]Token.Id{.Invalid}); - testTokenize("0b1_,", &[_]Token.Id{.Invalid}); + testTokenize("0b1e", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0b1p", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0b1e0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0b1p0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0b1_,", &[_]Token.Id{ .Invalid, .Comma }); } test "tokenizer - number literals octal" { @@ -1794,15 +1850,15 @@ test "tokenizer - number literals octal" { testTokenize("0o5", &[_]Token.Id{.IntegerLiteral}); testTokenize("0o6", &[_]Token.Id{.IntegerLiteral}); testTokenize("0o7", &[_]Token.Id{.IntegerLiteral}); - testTokenize("0o8", &[_]Token.Id{.Invalid}); - testTokenize("0o9", &[_]Token.Id{.Invalid}); - testTokenize("0oa", &[_]Token.Id{.Invalid}); - testTokenize("0ob", &[_]Token.Id{.Invalid}); - testTokenize("0oc", &[_]Token.Id{.Invalid}); - testTokenize("0od", &[_]Token.Id{.Invalid}); - testTokenize("0oe", &[_]Token.Id{.Invalid}); - testTokenize("0of", &[_]Token.Id{.Invalid}); - testTokenize("0oz", &[_]Token.Id{.Invalid}); + testTokenize("0o8", &[_]Token.Id{ .Invalid, .IntegerLiteral }); + testTokenize("0o9", &[_]Token.Id{ .Invalid, .IntegerLiteral }); + testTokenize("0oa", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0ob", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0oc", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0od", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0oe", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0of", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0oz", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0o01234567", &[_]Token.Id{.IntegerLiteral}); testTokenize("0o0123_4567", &[_]Token.Id{.IntegerLiteral}); @@ -1811,17 +1867,17 @@ test "tokenizer - number literals octal" { testTokenize("0o7.", &[_]Token.Id{ .IntegerLiteral, .Period }); testTokenize("0o7.0", &[_]Token.Id{ .IntegerLiteral, .Period, .IntegerLiteral }); - testTokenize("0O0", &[_]Token.Id{.Invalid}); - testTokenize("0o_", &[_]Token.Id{.Invalid}); - testTokenize("0o_0", &[_]Token.Id{.Invalid}); + testTokenize("0O0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0o_", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0o_0", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0o1_", &[_]Token.Id{.Invalid}); - testTokenize("0o0__1", &[_]Token.Id{.Invalid}); + testTokenize("0o0__1", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0o0_1_", &[_]Token.Id{.Invalid}); - testTokenize("0o1e", &[_]Token.Id{.Invalid}); - testTokenize("0o1p", &[_]Token.Id{.Invalid}); - testTokenize("0o1e0", &[_]Token.Id{.Invalid}); - testTokenize("0o1p0", &[_]Token.Id{.Invalid}); - testTokenize("0o_,", &[_]Token.Id{.Invalid}); + testTokenize("0o1e", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0o1p", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0o1e0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0o1p0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0o_,", &[_]Token.Id{ .Invalid, .Identifier, .Comma }); } test "tokenizer - number literals hexadeciaml" { @@ -1847,21 +1903,21 @@ test "tokenizer - number literals hexadeciaml" { testTokenize("0xD", &[_]Token.Id{.IntegerLiteral}); testTokenize("0xE", &[_]Token.Id{.IntegerLiteral}); testTokenize("0xF", &[_]Token.Id{.IntegerLiteral}); - testTokenize("0x0z", &[_]Token.Id{.Invalid}); - testTokenize("0xz", &[_]Token.Id{.Invalid}); + testTokenize("0x0z", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0xz", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0x0123456789ABCDEF", &[_]Token.Id{.IntegerLiteral}); testTokenize("0x0123_4567_89AB_CDEF", &[_]Token.Id{.IntegerLiteral}); testTokenize("0x01_23_45_67_89AB_CDE_F", &[_]Token.Id{.IntegerLiteral}); testTokenize("0x0_1_2_3_4_5_6_7_8_9_A_B_C_D_E_F", &[_]Token.Id{.IntegerLiteral}); - testTokenize("0X0", &[_]Token.Id{.Invalid}); - testTokenize("0x_", &[_]Token.Id{.Invalid}); - testTokenize("0x_1", &[_]Token.Id{.Invalid}); + testTokenize("0X0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x_", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x_1", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0x1_", &[_]Token.Id{.Invalid}); - testTokenize("0x0__1", &[_]Token.Id{.Invalid}); + testTokenize("0x0__1", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0x0_1_", &[_]Token.Id{.Invalid}); - testTokenize("0x_,", &[_]Token.Id{.Invalid}); + testTokenize("0x_,", &[_]Token.Id{ .Invalid, .Identifier, .Comma }); testTokenize("0x1.", &[_]Token.Id{.FloatLiteral}); testTokenize("0x1.0", &[_]Token.Id{.FloatLiteral}); @@ -1872,10 +1928,12 @@ test "tokenizer - number literals hexadeciaml" { testTokenize("0xF.FP0", &[_]Token.Id{.FloatLiteral}); testTokenize("0x1p0", &[_]Token.Id{.FloatLiteral}); testTokenize("0xfp0", &[_]Token.Id{.FloatLiteral}); + testTokenize("0x1.+0xF.", &[_]Token.Id{ .FloatLiteral, .Plus, .FloatLiteral }); testTokenize("0x0123456.789ABCDEF", &[_]Token.Id{.FloatLiteral}); testTokenize("0x0_123_456.789_ABC_DEF", &[_]Token.Id{.FloatLiteral}); testTokenize("0x0_1_2_3_4_5_6.7_8_9_A_B_C_D_E_F", &[_]Token.Id{.FloatLiteral}); + testTokenize("0x0p0", &[_]Token.Id{.FloatLiteral}); testTokenize("0x0.0p0", &[_]Token.Id{.FloatLiteral}); testTokenize("0xff.ffp10", &[_]Token.Id{.FloatLiteral}); testTokenize("0xff.ffP10", &[_]Token.Id{.FloatLiteral}); @@ -1888,21 +1946,24 @@ test "tokenizer - number literals hexadeciaml" { testTokenize("0x1e", &[_]Token.Id{.IntegerLiteral}); testTokenize("0x1e0", &[_]Token.Id{.IntegerLiteral}); testTokenize("0x1p", &[_]Token.Id{.Invalid}); - testTokenize("0xfp0z1", &[_]Token.Id{.Invalid}); - testTokenize("0xff.ffpff", &[_]Token.Id{.Invalid}); - testTokenize("0x0_.0", &[_]Token.Id{.Invalid}); - testTokenize("0x0._0", &[_]Token.Id{.Invalid}); + testTokenize("0xfp0z1", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0xff.ffpff", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x0.p", &[_]Token.Id{.Invalid}); + testTokenize("0x0.z", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x0._", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x0_.0", &[_]Token.Id{ .Invalid, .Period, .IntegerLiteral }); + testTokenize("0x0_.0.0", &[_]Token.Id{ .Invalid, .Period, .FloatLiteral }); + testTokenize("0x0._0", &[_]Token.Id{ .Invalid, .Identifier }); testTokenize("0x0.0_", &[_]Token.Id{.Invalid}); - testTokenize("0x0_p0", &[_]Token.Id{.Invalid}); - testTokenize("0x0_.p0", &[_]Token.Id{.Invalid}); - testTokenize("0x0._p0", &[_]Token.Id{.Invalid}); - testTokenize("0x0.0_p0", &[_]Token.Id{.Invalid}); - testTokenize("0x0._0p0", &[_]Token.Id{.Invalid}); - testTokenize("0x0.0_p0", &[_]Token.Id{.Invalid}); - testTokenize("0x0.0p_0", &[_]Token.Id{.Invalid}); - testTokenize("0x0.0p+_0", &[_]Token.Id{.Invalid}); - testTokenize("0x0.0p-_0", &[_]Token.Id{.Invalid}); - testTokenize("0x0.0p0_", &[_]Token.Id{.Invalid}); + testTokenize("0x0_p0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x0_.p0", &[_]Token.Id{ .Invalid, .Period, .Identifier }); + testTokenize("0x0._p0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x0.0_p0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x0._0p0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x0.0p_0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x0.0p+_0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x0.0p-_0", &[_]Token.Id{ .Invalid, .Identifier }); + testTokenize("0x0.0p0_", &[_]Token.Id{ .Invalid, .Eof }); } fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void { diff --git a/src-self-hosted/ir.zig b/src-self-hosted/ir.zig index e3d13b43db98dfb436e15cfde950b67ed279be0f..63dc67a6fb4ae4173eb4785e966ae784a979894d 100644 --- a/src-self-hosted/ir.zig +++ b/src-self-hosted/ir.zig @@ -1312,15 +1312,15 @@ pub const Builder = struct { var rest: []const u8 = undefined; if (int_token.len >= 3 and int_token[0] == '0') { rest = int_token[2..]; - base = switch (int_token[1]) { - 'b' => 2, - 'o' => 8, - 'x' => 16, + switch (int_token[1]) { + 'b' => base = 2, + 'o' => base = 8, + 'x' => base = 16, else => { base = 10; rest = int_token; }, - }; + } } else { base = 10; rest = int_token; -- 2.54.0