authorgravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-04-18 20:22:43-07:00
committergravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-04-18 20:46:03-07:00
logd760cae2b1f77c239cc4b47a54124ff761aa8a7a
treec7e04bed99d8533608fcf6eebeea4d529cb0d332
parent5195b87639a1dc56b90751d0aecc4fcf4f2a1bb0

compiler_rt: implement __mulxf3 for f80

3 files changed, 157 insertions(+), 50 deletions(-)

lib/std/special/compiler_rt.zig+7-4
......@@ -226,23 +226,26 @@ comptime {
226226 @export(__addsf3, .{ .name = "__addsf3", .linkage = linkage });
227227 const __adddf3 = @import("compiler_rt/addXf3.zig").__adddf3;
228228 @export(__adddf3, .{ .name = "__adddf3", .linkage = linkage });
229 const __addtf3 = @import("compiler_rt/addXf3.zig").__addtf3;
230 @export(__addtf3, .{ .name = "__addtf3", .linkage = linkage });
231229 const __addxf3 = @import("compiler_rt/addXf3.zig").__addxf3;
232230 @export(__addxf3, .{ .name = "__addxf3", .linkage = linkage });
231 const __addtf3 = @import("compiler_rt/addXf3.zig").__addtf3;
232 @export(__addtf3, .{ .name = "__addtf3", .linkage = linkage });
233
233234 const __subsf3 = @import("compiler_rt/addXf3.zig").__subsf3;
234235 @export(__subsf3, .{ .name = "__subsf3", .linkage = linkage });
235236 const __subdf3 = @import("compiler_rt/addXf3.zig").__subdf3;
236237 @export(__subdf3, .{ .name = "__subdf3", .linkage = linkage });
237 const __subtf3 = @import("compiler_rt/addXf3.zig").__subtf3;
238 @export(__subtf3, .{ .name = "__subtf3", .linkage = linkage });
239238 const __subxf3 = @import("compiler_rt/addXf3.zig").__subxf3;
240239 @export(__subxf3, .{ .name = "__subxf3", .linkage = linkage });
240 const __subtf3 = @import("compiler_rt/addXf3.zig").__subtf3;
241 @export(__subtf3, .{ .name = "__subtf3", .linkage = linkage });
241242
242243 const __mulsf3 = @import("compiler_rt/mulXf3.zig").__mulsf3;
243244 @export(__mulsf3, .{ .name = "__mulsf3", .linkage = linkage });
244245 const __muldf3 = @import("compiler_rt/mulXf3.zig").__muldf3;
245246 @export(__muldf3, .{ .name = "__muldf3", .linkage = linkage });
247 const __mulxf3 = @import("compiler_rt/mulXf3.zig").__mulxf3;
248 @export(__mulxf3, .{ .name = "__mulxf3", .linkage = linkage });
246249 const __multf3 = @import("compiler_rt/mulXf3.zig").__multf3;
247250 @export(__multf3, .{ .name = "__multf3", .linkage = linkage });
248251
lib/std/special/compiler_rt/mulXf3.zig+83-46
......@@ -3,12 +3,16 @@
33// https://github.com/llvm/llvm-project/blob/2ffb1b0413efa9a24eb3c49e710e36f92e2cb50b/compiler-rt/lib/builtins/fp_mul_impl.inc
44
55const std = @import("std");
6const math = std.math;
67const builtin = @import("builtin");
78const compiler_rt = @import("../compiler_rt.zig");
89
910pub fn __multf3(a: f128, b: f128) callconv(.C) f128 {
1011 return mulXf3(f128, a, b);
1112}
13pub fn __mulxf3(a: f80, b: f80) callconv(.C) f80 {
14 return mulXf3(f80, a, b);
15}
1216pub fn __muldf3(a: f64, b: f64) callconv(.C) f64 {
1317 return mulXf3(f64, a, b);
1418}
......@@ -29,30 +33,36 @@ pub fn __aeabi_dmul(a: f64, b: f64) callconv(.C) f64 {
2933fn mulXf3(comptime T: type, a: T, b: T) T {
3034 @setRuntimeSafety(builtin.is_test);
3135 const typeWidth = @typeInfo(T).Float.bits;
36 const significandBits = math.floatMantissaBits(T);
37 const fractionalBits = math.floatFractionalBits(T);
38 const exponentBits = math.floatExponentBits(T);
39
3240 const Z = std.meta.Int(.unsigned, typeWidth);
3341
34 const significandBits = std.math.floatMantissaBits(T);
35 const exponentBits = std.math.floatExponentBits(T);
42 // ZSignificand is large enough to contain the significand, including an explicit integer bit
43 const ZSignificand = PowerOfTwoSignificandZ(T);
44 const ZSignificandBits = @typeInfo(ZSignificand).Int.bits;
3645
46 const roundBit = (1 << (ZSignificandBits - 1));
3747 const signBit = (@as(Z, 1) << (significandBits + exponentBits));
3848 const maxExponent = ((1 << exponentBits) - 1);
3949 const exponentBias = (maxExponent >> 1);
4050
41 const implicitBit = (@as(Z, 1) << significandBits);
42 const quietBit = implicitBit >> 1;
43 const significandMask = implicitBit - 1;
51 const integerBit = (@as(ZSignificand, 1) << fractionalBits);
52 const quietBit = integerBit >> 1;
53 const significandMask = (@as(Z, 1) << significandBits) - 1;
4454
4555 const absMask = signBit - 1;
46 const exponentMask = absMask ^ significandMask;
47 const qnanRep = exponentMask | quietBit;
48 const infRep = @bitCast(Z, std.math.inf(T));
56 const qnanRep = @bitCast(Z, math.nan(T)) | quietBit;
57 const infRep = @bitCast(Z, math.inf(T));
58 const minNormalRep = @bitCast(Z, math.floatMin(T));
4959
5060 const aExponent = @truncate(u32, (@bitCast(Z, a) >> significandBits) & maxExponent);
5161 const bExponent = @truncate(u32, (@bitCast(Z, b) >> significandBits) & maxExponent);
5262 const productSign: Z = (@bitCast(Z, a) ^ @bitCast(Z, b)) & signBit;
5363
54 var aSignificand: Z = @bitCast(Z, a) & significandMask;
55 var bSignificand: Z = @bitCast(Z, b) & significandMask;
64 var aSignificand: ZSignificand = @intCast(ZSignificand, @bitCast(Z, a) & significandMask);
65 var bSignificand: ZSignificand = @intCast(ZSignificand, @bitCast(Z, b) & significandMask);
5666 var scale: i32 = 0;
5767
5868 // Detect if a or b is zero, denormal, infinity, or NaN.
......@@ -93,38 +103,40 @@ fn mulXf3(comptime T: type, a: T, b: T) T {
93103 // one or both of a or b is denormal, the other (if applicable) is a
94104 // normal number. Renormalize one or both of a and b, and set scale to
95105 // include the necessary exponent adjustment.
96 if (aAbs < implicitBit) scale += normalize(T, &aSignificand);
97 if (bAbs < implicitBit) scale += normalize(T, &bSignificand);
106 if (aAbs < minNormalRep) scale += normalize(T, &aSignificand);
107 if (bAbs < minNormalRep) scale += normalize(T, &bSignificand);
98108 }
99109
100110 // Or in the implicit significand bit. (If we fell through from the
101111 // denormal path it was already set by normalize( ), but setting it twice
102112 // won't hurt anything.)
103 aSignificand |= implicitBit;
104 bSignificand |= implicitBit;
113 aSignificand |= integerBit;
114 bSignificand |= integerBit;
105115
106116 // Get the significand of a*b. Before multiplying the significands, shift
107117 // one of them left to left-align it in the field. Thus, the product will
108118 // have (exponentBits + 2) integral digits, all but two of which must be
109119 // zero. Normalizing this result is just a conditional left-shift by one
110120 // and bumping the exponent accordingly.
111 var productHi: Z = undefined;
112 var productLo: Z = undefined;
113 wideMultiply(Z, aSignificand, bSignificand << exponentBits, &productHi, &productLo);
121 var productHi: ZSignificand = undefined;
122 var productLo: ZSignificand = undefined;
123 const left_align_shift = ZSignificandBits - fractionalBits - 1;
124 wideMultiply(ZSignificand, aSignificand, bSignificand << left_align_shift, &productHi, &productLo);
114125
115 var productExponent: i32 = @bitCast(i32, aExponent +% bExponent) -% exponentBias +% scale;
126 var productExponent: i32 = @intCast(i32, aExponent + bExponent) - exponentBias + scale;
116127
117128 // Normalize the significand, adjust exponent if needed.
118 if ((productHi & implicitBit) != 0) {
129 if ((productHi & integerBit) != 0) {
119130 productExponent +%= 1;
120131 } else {
121 productHi = (productHi << 1) | (productLo >> (typeWidth - 1));
132 productHi = (productHi << 1) | (productLo >> (ZSignificandBits - 1));
122133 productLo = productLo << 1;
123134 }
124135
125136 // If we have overflowed the type, return +/- infinity.
126137 if (productExponent >= maxExponent) return @bitCast(T, infRep | productSign);
127138
139 var result: Z = undefined;
128140 if (productExponent <= 0) {
129141 // Result is denormal before rounding
130142 //
......@@ -133,35 +145,49 @@ fn mulXf3(comptime T: type, a: T, b: T) T {
133145 // handle this case separately, but we make it a special case to
134146 // simplify the shift logic.
135147 const shift: u32 = @truncate(u32, @as(Z, 1) -% @bitCast(u32, productExponent));
136 if (shift >= typeWidth) return @bitCast(T, productSign);
148 if (shift >= ZSignificandBits) return @bitCast(T, productSign);
137149
138150 // Otherwise, shift the significand of the result so that the round
139151 // bit is the high bit of productLo.
140 wideRightShiftWithSticky(Z, &productHi, &productLo, shift);
152 const sticky = wideShrWithTruncation(ZSignificand, &productHi, &productLo, shift);
153 productLo |= @boolToInt(sticky);
154 result = productHi;
141155 } else {
142156 // Result is normal before rounding; insert the exponent.
143 productHi &= significandMask;
144 productHi |= @as(Z, @bitCast(u32, productExponent)) << significandBits;
157 result = productHi & significandMask;
158 result |= @intCast(Z, productExponent) << significandBits;
145159 }
146160
147 // Insert the sign of the result:
148 productHi |= productSign;
149
150161 // Final rounding. The final result may overflow to infinity, or underflow
151162 // to zero, but those are the correct results in those cases. We use the
152163 // default IEEE-754 round-to-nearest, ties-to-even rounding mode.
153 if (productLo > signBit) productHi +%= 1;
154 if (productLo == signBit) productHi +%= productHi & 1;
155 return @bitCast(T, productHi);
164 if (productLo > roundBit) result +%= 1;
165 if (productLo == roundBit) result +%= result & 1;
166
167 // Restore any explicit integer bit, if it was rounded off
168 if (significandBits != fractionalBits) {
169 if ((result >> significandBits) != 0) result |= integerBit;
170 }
171
172 // Insert the sign of the result:
173 result |= productSign;
174
175 return @bitCast(T, result);
156176}
157177
158178fn wideMultiply(comptime Z: type, a: Z, b: Z, hi: *Z, lo: *Z) void {
159179 @setRuntimeSafety(builtin.is_test);
160180 switch (Z) {
181 u16 => {
182 // 16x16 --> 32 bit multiply
183 const product = @as(u32, a) * @as(u32, b);
184 hi.* = @intCast(u16, product >> 16);
185 lo.* = @truncate(u16, product);
186 },
161187 u32 => {
162188 // 32x32 --> 64 bit multiply
163189 const product = @as(u64, a) * @as(u64, b);
164 hi.* = @truncate(u32, product >> 32);
190 hi.* = @intCast(u32, product >> 32);
165191 lo.* = @truncate(u32, product);
166192 },
167193 u64 => {
......@@ -170,7 +196,7 @@ fn wideMultiply(comptime Z: type, a: Z, b: Z, hi: *Z, lo: *Z) void {
170196 return @truncate(u32, x);
171197 }
172198 fn hiWord(x: u64) u64 {
173 return @truncate(u32, x >> 32);
199 return @intCast(u32, x >> 32);
174200 }
175201 };
176202 // 64x64 -> 128 wide multiply for platforms that don't have such an operation;
......@@ -264,34 +290,45 @@ fn wideMultiply(comptime Z: type, a: Z, b: Z, hi: *Z, lo: *Z) void {
264290 }
265291}
266292
267fn normalize(comptime T: type, significand: *std.meta.Int(.unsigned, @typeInfo(T).Float.bits)) i32 {
293/// Returns a power-of-two integer type that is large enough to contain
294/// the significand of T, including an explicit integer bit
295fn PowerOfTwoSignificandZ(comptime T: type) type {
296 const bits = math.ceilPowerOfTwoAssert(u16, math.floatFractionalBits(T) + 1);
297 return std.meta.Int(.unsigned, bits);
298}
299
300fn normalize(comptime T: type, significand: *PowerOfTwoSignificandZ(T)) i32 {
268301 @setRuntimeSafety(builtin.is_test);
269 const Z = std.meta.Int(.unsigned, @typeInfo(T).Float.bits);
270 const significandBits = std.math.floatMantissaBits(T);
271 const implicitBit = @as(Z, 1) << significandBits;
302 const Z = PowerOfTwoSignificandZ(T);
303 const integerBit = @as(Z, 1) << math.floatFractionalBits(T);
272304
273 const shift = @clz(Z, significand.*) - @clz(Z, implicitBit);
274 significand.* <<= @intCast(std.math.Log2Int(Z), shift);
305 const shift = @clz(Z, significand.*) - @clz(Z, integerBit);
306 significand.* <<= @intCast(math.Log2Int(Z), shift);
275307 return @as(i32, 1) - shift;
276308}
277309
278fn wideRightShiftWithSticky(comptime Z: type, hi: *Z, lo: *Z, count: u32) void {
310// Returns `true` if the right shift is inexact (i.e. any bit shifted out is non-zero)
311//
312// This is analogous to an shr version of `@shlWithOverflow`
313fn wideShrWithTruncation(comptime Z: type, hi: *Z, lo: *Z, count: u32) bool {
279314 @setRuntimeSafety(builtin.is_test);
280315 const typeWidth = @typeInfo(Z).Int.bits;
281 const S = std.math.Log2Int(Z);
316 const S = math.Log2Int(Z);
317 var inexact = false;
282318 if (count < typeWidth) {
283 const sticky = @boolToInt((lo.* << @intCast(S, typeWidth -% count)) != 0);
284 lo.* = (hi.* << @intCast(S, typeWidth -% count)) | (lo.* >> @intCast(S, count)) | sticky;
319 inexact = (lo.* << @intCast(S, typeWidth -% count)) != 0;
320 lo.* = (hi.* << @intCast(S, typeWidth -% count)) | (lo.* >> @intCast(S, count));
285321 hi.* = hi.* >> @intCast(S, count);
286322 } else if (count < 2 * typeWidth) {
287 const sticky = @boolToInt((hi.* << @intCast(S, 2 * typeWidth -% count) | lo.*) != 0);
288 lo.* = hi.* >> @intCast(S, count -% typeWidth) | sticky;
323 inexact = (hi.* << @intCast(S, 2 * typeWidth -% count) | lo.*) != 0;
324 lo.* = hi.* >> @intCast(S, count -% typeWidth);
289325 hi.* = 0;
290326 } else {
291 const sticky = @boolToInt((hi.* | lo.*) != 0);
292 lo.* = sticky;
327 inexact = (hi.* | lo.*) != 0;
328 lo.* = 0;
293329 hi.* = 0;
294330 }
331 return inexact;
295332}
296333
297334test {
lib/std/special/compiler_rt/mulXf3_test.zig+67
......@@ -2,10 +2,15 @@
22//
33// https://github.com/llvm/llvm-project/blob/2ffb1b0413efa9a24eb3c49e710e36f92e2cb50b/compiler-rt/test/builtins/Unit/multf3_test.c
44
5const std = @import("std");
6const math = std.math;
57const qnan128 = @bitCast(f128, @as(u128, 0x7fff800000000000) << 64);
68const inf128 = @bitCast(f128, @as(u128, 0x7fff000000000000) << 64);
79
810const __multf3 = @import("mulXf3.zig").__multf3;
11const __mulxf3 = @import("mulXf3.zig").__mulxf3;
12const __muldf3 = @import("mulXf3.zig").__muldf3;
13const __mulsf3 = @import("mulXf3.zig").__mulsf3;
914
1015// return true if equal
1116// use two 64-bit integers intead of one 128-bit integer
......@@ -97,4 +102,66 @@ test "multf3" {
97102 0x3f90000000000000,
98103 0x0,
99104 );
105
106 try test__multf3(0x1.0000_0000_0000_0000_0000_0000_0001p+0, 0x1.8p+5, 0x4004_8000_0000_0000, 0x0000_0000_0000_0002);
107 try test__multf3(0x1.0000_0000_0000_0000_0000_0000_0002p+0, 0x1.8p+5, 0x4004_8000_0000_0000, 0x0000_0000_0000_0003);
108}
109
110const qnan80 = @bitCast(f80, @bitCast(u80, math.nan(f80)) | (1 << (math.floatFractionalBits(f80) - 1)));
111
112fn test__mulxf3(a: f80, b: f80, expected: u80) !void {
113 const x = __mulxf3(a, b);
114 const rep = @bitCast(u80, x);
115
116 if (rep == expected)
117 return;
118
119 if (math.isNan(@bitCast(f80, expected)) and math.isNan(x))
120 return; // We don't currently test NaN payload propagation
121
122 return error.TestFailed;
123}
124
125test "mulxf3" {
126 // NaN * any = NaN
127 try test__mulxf3(qnan80, 0x1.23456789abcdefp+5, @bitCast(u80, qnan80));
128 try test__mulxf3(@bitCast(f80, @as(u80, 0x7fff_8000_8000_3000_0000)), 0x1.23456789abcdefp+5, @bitCast(u80, qnan80));
129
130 // any * NaN = NaN
131 try test__mulxf3(0x1.23456789abcdefp+5, qnan80, @bitCast(u80, qnan80));
132 try test__mulxf3(0x1.23456789abcdefp+5, @bitCast(f80, @as(u80, 0x7fff_8000_8000_3000_0000)), @bitCast(u80, qnan80));
133
134 // NaN * inf = NaN
135 try test__mulxf3(qnan80, math.inf(f80), @bitCast(u80, qnan80));
136
137 // inf * NaN = NaN
138 try test__mulxf3(math.inf(f80), qnan80, @bitCast(u80, qnan80));
139
140 // inf * inf = inf
141 try test__mulxf3(math.inf(f80), math.inf(f80), @bitCast(u80, math.inf(f80)));
142
143 // inf * -inf = -inf
144 try test__mulxf3(math.inf(f80), -math.inf(f80), @bitCast(u80, -math.inf(f80)));
145
146 // -inf + inf = -inf
147 try test__mulxf3(-math.inf(f80), math.inf(f80), @bitCast(u80, -math.inf(f80)));
148
149 // inf * any = inf
150 try test__mulxf3(math.inf(f80), 0x1.2335653452436234723489432abcdefp+5, @bitCast(u80, math.inf(f80)));
151
152 // any * inf = inf
153 try test__mulxf3(0x1.2335653452436234723489432abcdefp+5, math.inf(f80), @bitCast(u80, math.inf(f80)));
154
155 // any * any
156 try test__mulxf3(0x1.0p+0, 0x1.dcba987654321p+5, 0x4004_ee5d_4c3b_2a19_0800);
157 try test__mulxf3(0x1.0000_0000_0000_0004p+0, 0x1.8p+5, 0x4004_C000_0000_0000_0003); // exact
158
159 try test__mulxf3(0x1.0000_0000_0000_0002p+0, 0x1.0p+5, 0x4004_8000_0000_0000_0001); // exact
160 try test__mulxf3(0x1.0000_0000_0000_0002p+0, 0x1.7ffep+5, 0x4004_BFFF_0000_0000_0001); // round down
161 try test__mulxf3(0x1.0000_0000_0000_0002p+0, 0x1.8p+5, 0x4004_C000_0000_0000_0002); // round up to even
162 try test__mulxf3(0x1.0000_0000_0000_0002p+0, 0x1.8002p+5, 0x4004_C001_0000_0000_0002); // round up
163 try test__mulxf3(0x1.0000_0000_0000_0002p+0, 0x1.0p+6, 0x4005_8000_0000_0000_0001); // exact
164
165 try test__mulxf3(0x1.0000_0001p+0, 0x1.0000_0001p+0, 0x3FFF_8000_0001_0000_0000); // round down to even
166 try test__mulxf3(0x1.0000_0001p+0, 0x1.0000_0001_0002p+0, 0x3FFF_8000_0001_0001_0001); // round up
100167}