Commit 3560f61c1630c36dddaf46e88efc0ccc7b91cf44 | zig

author	Andrew Kelley <andrew@ziglang.org> 2019-03-22 10:16:46-04:00
committer	GitHub <noreply@github.com> 2019-03-22 10:16:46-04:00
log	3560f61c1630c36dddaf46e88efc0ccc7b91cf44
tree	829f8ed0ee6b5c4e582eb18ab147f38c8577e42a
parent	23af502d04d9d6bcdce40777a1fe16cbdec9cef0
parent	e3b70fe4ba5c9b5139055e2ffdde526414c86538
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #2087 from ziglang/float-parsing

fix some hex literal parsing bugs

2 files changed, 117 insertions(+), 37 deletions(-)

src
- tokenizer.cpp
test
- stage1
  - behavior
    math.zig

src/tokenizer.cpp+35-37

 @@ -326,51 +326,49 @@ static void end_float_token(Tokenize *t) {
         return;
     }
-    // A SoftFloat-3d float128 is represented internally as a standard
+    // A SoftFloat-3e float128 is represented internally as a standard
-    // quad-precision float with 15bit exponent and 113bit fractional.
+    // quad-precision float with 15bit exponent and 112bit fractional.
     union { uint64_t repr[2]; float128_t actual; } f_bits;
     if (bigint_cmp_zero(&t->significand) == CmpEQ) {
         f_bits.repr[0] = 0;
         f_bits.repr[1] = 0;
     } else {
-        // normalize the significand
+        int significand_magnitude_in_bin = 127 - bigint_clz(&t->significand, 128);
-        if (t->radix == 10) {
+        t->exponent_in_bin_or_dec += significand_magnitude_in_bin;
-            zig_panic("TODO: decimal floats");
+        if (!(-16382 <= t->exponent_in_bin_or_dec && t->exponent_in_bin_or_dec <= 16383)) {
-        } else {
+            t->cur_tok->data.float_lit.overflow = true;
-            int significand_magnitude_in_bin = 127 - bigint_clz(&t->significand, 128);
+            return;
-            t->exponent_in_bin_or_dec += significand_magnitude_in_bin;
+        }
-            if (!(-16382 <= t->exponent_in_bin_or_dec && t->exponent_in_bin_or_dec <= 16383)) {
-                t->cur_tok->data.float_lit.overflow = true;
-                return;
-            }
-            uint64_t sig_bits[2] = {0, 0};
-            bigint_write_twos_complement(&t->significand, (uint8_t*) sig_bits, 128, false);
-            const uint64_t shift = 112 - significand_magnitude_in_bin;
-            const uint64_t exp_shift = 48;
-            // Mask the sign bit to 0 since always non-negative lex
-            const uint64_t exp_mask = 0xffffull << exp_shift;
-            // must be special-cased to avoid undefined behavior on shift == 64
-            if (shift == 128) {
-                f_bits.repr[0] = 0;
-                f_bits.repr[1] = sig_bits[0];
-            } else if (shift == 0) {
-                f_bits.repr[0] = sig_bits[0];
-                f_bits.repr[1] = sig_bits[1];
-            } else if (shift >= 64) {
-                f_bits.repr[0] = 0;
-                f_bits.repr[1] = sig_bits[0] << (shift - 64);
-            } else {
-                f_bits.repr[0] = sig_bits[0] << shift;
-                f_bits.repr[1] = (sig_bits[1] << shift) | (sig_bits[0] >> (64 - shift));
-            }
-            f_bits.repr[1] &= ~exp_mask;
+        // Shift bits of significand so they are left-justified at the 112-bit
-            f_bits.repr[1] |= (uint64_t)(t->exponent_in_bin_or_dec + 16383) << exp_shift;
+        // mark. We truncate excess bits and lose precision. No rounding.
+        //
+        // -16 <= shift <= 112
+        //
+        // NOTE: The loss of precision could be considered a limitation of using
+        // 128-bit floats. In stage2 we should use an arbitrary precision
+        // float/rational type to represent these and avoid this.
+        const int shift = 112 - significand_magnitude_in_bin;
+        bigint_write_twos_complement(&t->significand, (uint8_t*) f_bits.repr, 128, false);
+        if (shift >= 64) {
+            f_bits.repr[1] = f_bits.repr[0] << (shift - 64);
+            f_bits.repr[0] = 0;
+        } else if (shift > 0) {
+            f_bits.repr[1] = (f_bits.repr[1] << shift) | (f_bits.repr[0] >> (64 - shift));
+            f_bits.repr[0] = f_bits.repr[0] << shift;
+        } else if (shift < 0) {
+            int positive_shift = -shift;
+            assert(positive_shift <= 16);
+            f_bits.repr[0] = (f_bits.repr[0] >> positive_shift) | (f_bits.repr[1] << (64 - positive_shift));
+            f_bits.repr[1] = f_bits.repr[1] >> positive_shift;
         }
+        // Lexer separates negative sign from value so this is always non-negative.
+        const uint64_t exp_mask = 0xffffull << 48;
+        f_bits.repr[1] &= ~exp_mask;
+        f_bits.repr[1] |= (uint64_t)(t->exponent_in_bin_or_dec + 16383) << 48;
     }
     bigfloat_init_128(&t->cur_tok->data.float_lit.bigfloat, f_bits.actual);

test/stage1/behavior/math.zig+82

...	@@ -307,6 +307,88 @@ test "quad hex float literal parsing accurate" {	...	@@ -307,6 +307,88 @@ test "quad hex float literal parsing accurate" {
307	// implied 1 is dropped, with an exponent of 0 (0x3fff) after biasing.	307	// implied 1 is dropped, with an exponent of 0 (0x3fff) after biasing.
308	const expected: u128 = 0x3fff1111222233334444555566667777;	308	const expected: u128 = 0x3fff1111222233334444555566667777;
309	expect(@bitCast(u128, a) == expected);	309	expect(@bitCast(u128, a) == expected);
		310
		311	// non-normalized
		312	const b: f128 = 0x11.111222233334444555566667777p-4;
		313	expect(@bitCast(u128, b) == expected);
		314
		315	const S = struct {
		316	fn doTheTest() void {
		317	{
		318	var f: f128 = 0x1.2eab345678439abcdefea56782346p+5;
		319	expect(@bitCast(u128, f) == 0x40042eab345678439abcdefea5678234);
		320	}
		321	{
		322	var f: f128 = 0x1.edcb34a235253948765432134674fp-1;
		323	expect(@bitCast(u128, f) == 0x3ffeedcb34a235253948765432134674);
		324	}
		325	{
		326	var f: f128 = 0x1.353e45674d89abacc3a2ebf3ff4ffp-50;
		327	expect(@bitCast(u128, f) == 0x3fcd353e45674d89abacc3a2ebf3ff4f);
		328	}
		329	{
		330	var f: f128 = 0x1.ed8764648369535adf4be3214567fp-9;
		331	expect(@bitCast(u128, f) == 0x3ff6ed8764648369535adf4be3214567);
		332	}
		333	const exp2ft = []f64{
		334	0x1.6a09e667f3bcdp-1,
		335	0x1.7a11473eb0187p-1,
		336	0x1.8ace5422aa0dbp-1,
		337	0x1.9c49182a3f090p-1,
		338	0x1.ae89f995ad3adp-1,
		339	0x1.c199bdd85529cp-1,
		340	0x1.d5818dcfba487p-1,
		341	0x1.ea4afa2a490dap-1,
		342	0x1.0000000000000p+0,
		343	0x1.0b5586cf9890fp+0,
		344	0x1.172b83c7d517bp+0,
		345	0x1.2387a6e756238p+0,
		346	0x1.306fe0a31b715p+0,
		347	0x1.3dea64c123422p+0,
		348	0x1.4bfdad5362a27p+0,
		349	0x1.5ab07dd485429p+0,
		350	0x1.8p23,
		351	0x1.62e430p-1,
		352	0x1.ebfbe0p-3,
		353	0x1.c6b348p-5,
		354	0x1.3b2c9cp-7,
		355	0x1.0p127,
		356	-0x1.0p-149,
		357	};
		358
		359	const answers = []u64{
		360	0x3fe6a09e667f3bcd,
		361	0x3fe7a11473eb0187,
		362	0x3fe8ace5422aa0db,
		363	0x3fe9c49182a3f090,
		364	0x3feae89f995ad3ad,
		365	0x3fec199bdd85529c,
		366	0x3fed5818dcfba487,
		367	0x3feea4afa2a490da,
		368	0x3ff0000000000000,
		369	0x3ff0b5586cf9890f,
		370	0x3ff172b83c7d517b,
		371	0x3ff2387a6e756238,
		372	0x3ff306fe0a31b715,
		373	0x3ff3dea64c123422,
		374	0x3ff4bfdad5362a27,
		375	0x3ff5ab07dd485429,
		376	0x4168000000000000,
		377	0x3fe62e4300000000,
		378	0x3fcebfbe00000000,
		379	0x3fac6b3480000000,
		380	0x3f83b2c9c0000000,
		381	0x47e0000000000000,
		382	0xb6a0000000000000,
		383	};
		384
		385	for (exp2ft) \|x, i\| {
		386	expect(@bitCast(u64, x) == answers[i]);
		387	}
		388	}
		389	};
		390	S.doTheTest();
		391	comptime S.doTheTest();
310	}	392	}
311		393
312	test "hex float literal within range" {	394	test "hex float literal within range" {