authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-03-06 15:23:21-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-03-06 16:11:39-07:00
log71b8760d3b145c92dc6e331aefff7dac5cabebeb
tree66b92748616634b689eb5c984f143042132d5e6c
parent6637335981f7179b449fced78cfd4052b1618051

stage2: rework `@mulAdd`

* mul_add AIR instruction: use `pl_op` instead of `ty_pl`. The type is always the same as the operand; no need to waste bytes redundantly storing the type. * AstGen: use coerced_ty for all the operands except for one which we use to communicate the type. * Sema: use the correct source location for requireRuntimeBlock in handling of `@mulAdd`. * native backends: handle liveness even for the functions that are TODO. * C backend: implement `@mulAdd`. It lowers to libc calls. * LLVM backend: make `@mulAdd` handle all float types. - improved fptrunc and fpext to handle f80 with compiler-rt calls. * Value.mulAdd: handle all float types and use the `@mulAdd` builtin. * behavior tests: revert the changes to testing `@mulAdd`. These changes broke the test coverage, making it only tested at compile-time. Improved f80 support: * std.math.fma handles f80 * move fma functions from freestanding libc to compiler-rt - add __fmax and fmal - make __fmax and fmaq only exported when they don't alias fmal. - make their linkage weak just like the rest of compiler-rt symbols. * removed `longDoubleIsF128` and replaced it with `longDoubleIs` which takes a type as a parameter. The implementation is now more accurate and handles more targets. Similarly, in stage2 the function CTypes.sizeInBits is more accurate for long double for more targets.

19 files changed, 405 insertions(+), 219 deletions(-)

lib/std/math/fma.zig+2
......@@ -19,6 +19,8 @@ pub fn fma(comptime T: type, x: T, y: T, z: T) T {
1919 // TODO this is not correct for some targets
2020 c_longdouble => @floatCast(c_longdouble, fma128(x, y, z)),
2121
22 f80 => @floatCast(f80, fma128(x, y, z)),
23
2224 else => @compileError("fma not implemented for " ++ @typeName(T)),
2325 };
2426}
lib/std/special/c.zig+1-19
......@@ -12,7 +12,7 @@ const maxInt = std.math.maxInt;
1212const native_os = builtin.os.tag;
1313const native_arch = builtin.cpu.arch;
1414const native_abi = builtin.abi;
15const long_double_is_f128 = builtin.target.longDoubleIsF128();
15const long_double_is_f128 = builtin.target.longDoubleIs(f128);
1616
1717const is_wasm = switch (native_arch) {
1818 .wasm32, .wasm64 => true,
......@@ -90,10 +90,6 @@ comptime {
9090 @export(fmod, .{ .name = "fmod", .linkage = .Strong });
9191 @export(fmodf, .{ .name = "fmodf", .linkage = .Strong });
9292
93 @export(fma, .{ .name = "fma", .linkage = .Strong });
94 @export(fmaf, .{ .name = "fmaf", .linkage = .Strong });
95 @export(fmal, .{ .name = "fmal", .linkage = .Strong });
96
9793 @export(sincos, .{ .name = "sincos", .linkage = .Strong });
9894 @export(sincosf, .{ .name = "sincosf", .linkage = .Strong });
9995
......@@ -561,20 +557,6 @@ test "fmod, fmodf" {
561557 }
562558}
563559
564fn fmaf(a: f32, b: f32, c: f32) callconv(.C) f32 {
565 return math.fma(f32, a, b, c);
566}
567
568fn fma(a: f64, b: f64, c: f64) callconv(.C) f64 {
569 return math.fma(f64, a, b, c);
570}
571fn fmal(a: c_longdouble, b: c_longdouble, c: c_longdouble) callconv(.C) c_longdouble {
572 if (!long_double_is_f128) {
573 @panic("TODO implement this");
574 }
575 return math.fma(c_longdouble, a, b, c);
576}
577
578560fn sincos(a: f64, r_sin: *f64, r_cos: *f64) callconv(.C) void {
579561 r_sin.* = math.sin(a);
580562 r_cos.* = math.cos(a);
lib/std/special/compiler_rt.zig+24-2
......@@ -19,7 +19,8 @@ const strong_linkage = if (is_test)
1919else
2020 std.builtin.GlobalLinkage.Strong;
2121
22const long_double_is_f128 = builtin.target.longDoubleIsF128();
22const long_double_is_f80 = builtin.target.longDoubleIs(f80);
23const long_double_is_f128 = builtin.target.longDoubleIs(f128);
2324
2425comptime {
2526 // These files do their own comptime exporting logic.
......@@ -758,14 +759,35 @@ comptime {
758759 @export(floorf, .{ .name = "floorf", .linkage = linkage });
759760 @export(floor, .{ .name = "floor", .linkage = linkage });
760761 @export(floorl, .{ .name = "floorl", .linkage = linkage });
761 @export(fmaq, .{ .name = "fmaq", .linkage = linkage });
762
763 @export(fma, .{ .name = "fma", .linkage = linkage });
764 @export(fmaf, .{ .name = "fmaf", .linkage = linkage });
765 @export(fmal, .{ .name = "fmal", .linkage = linkage });
766 if (!long_double_is_f80) {
767 @export(__fmax, .{ .name = "__fmax", .linkage = linkage });
768 }
769 if (!long_double_is_f128) {
770 @export(fmaq, .{ .name = "fmaq", .linkage = linkage });
771 }
762772}
763773
764774const math = std.math;
765775
776fn fmaf(a: f32, b: f32, c: f32) callconv(.C) f32 {
777 return math.fma(f32, a, b, c);
778}
779fn fma(a: f64, b: f64, c: f64) callconv(.C) f64 {
780 return math.fma(f64, a, b, c);
781}
782fn __fmax(a: f80, b: f80, c: f80) callconv(.C) f80 {
783 return math.fma(f80, a, b, c);
784}
766785fn fmaq(a: f128, b: f128, c: f128) callconv(.C) f128 {
767786 return math.fma(f128, a, b, c);
768787}
788fn fmal(a: c_longdouble, b: c_longdouble, c: c_longdouble) callconv(.C) c_longdouble {
789 return math.fma(c_longdouble, a, b, c);
790}
769791
770792// TODO add intrinsics for these (and probably the double version too)
771793// and have the math stuff use the intrinsic. same as @mod and @rem
lib/std/target.zig+49-3
......@@ -1714,9 +1714,55 @@ pub const Target = struct {
17141714 };
17151715 }
17161716
1717 pub inline fn longDoubleIsF128(target: Target) bool {
1718 return switch (target.cpu.arch) {
1719 .riscv64, .aarch64, .aarch64_be, .aarch64_32, .s390x, .mips64, .mips64el => true,
1717 pub inline fn longDoubleIs(target: Target, comptime F: type) bool {
1718 if (target.abi == .msvc) {
1719 return F == f64;
1720 }
1721 return switch (F) {
1722 f128 => switch (target.cpu.arch) {
1723 .riscv64,
1724 .aarch64,
1725 .aarch64_be,
1726 .aarch64_32,
1727 .s390x,
1728 .mips64,
1729 .mips64el,
1730 .sparc,
1731 .sparcv9,
1732 .sparcel,
1733 .powerpc,
1734 .powerpcle,
1735 .powerpc64,
1736 .powerpc64le,
1737 => true,
1738
1739 else => false,
1740 },
1741 f80 => switch (target.cpu.arch) {
1742 .x86_64, .i386 => true,
1743 else => false,
1744 },
1745 f64 => switch (target.cpu.arch) {
1746 .x86_64,
1747 .i386,
1748 .riscv64,
1749 .aarch64,
1750 .aarch64_be,
1751 .aarch64_32,
1752 .s390x,
1753 .mips64,
1754 .mips64el,
1755 .sparc,
1756 .sparcv9,
1757 .sparcel,
1758 .powerpc,
1759 .powerpcle,
1760 .powerpc64,
1761 .powerpc64le,
1762 => false,
1763
1764 else => true,
1765 },
17201766 else => false,
17211767 };
17221768 }
src/Air.zig+4-8
......@@ -580,7 +580,8 @@ pub const Inst = struct {
580580 prefetch,
581581
582582 /// Computes `(a * b) + c`, but only rounds once.
583 /// Uses the `ty_pl` field.
583 /// Uses the `pl_op` field with payload `Bin`.
584 /// The operand is the addend. The mulends are lhs and rhs.
584585 mul_add,
585586
586587 /// Implements @fieldParentPtr builtin.
......@@ -728,12 +729,6 @@ pub const Bin = struct {
728729 rhs: Inst.Ref,
729730};
730731
731pub const MulAdd = struct {
732 mulend1: Inst.Ref,
733 mulend2: Inst.Ref,
734 addend: Inst.Ref,
735};
736
737732pub const FieldParentPtr = struct {
738733 field_ptr: Inst.Ref,
739734 field_index: u32,
......@@ -899,7 +894,6 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
899894 .aggregate_init,
900895 .union_init,
901896 .field_parent_ptr,
902 .mul_add,
903897 => return air.getRefType(datas[inst].ty_pl.ty),
904898
905899 .not,
......@@ -997,6 +991,8 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
997991 return ptr_ty.elemType();
998992 },
999993
994 .mul_add => return air.typeOf(datas[inst].pl_op.operand),
995
1000996 .add_with_overflow,
1001997 .sub_with_overflow,
1002998 .mul_with_overflow,
src/AstGen.zig+2-2
......@@ -7309,8 +7309,8 @@ fn builtinCall(
73097309 },
73107310 .mul_add => {
73117311 const float_type = try typeExpr(gz, scope, params[0]);
7312 const mulend1 = try expr(gz, scope, .{ .ty = float_type }, params[1]);
7313 const mulend2 = try expr(gz, scope, .{ .ty = float_type }, params[2]);
7312 const mulend1 = try expr(gz, scope, .{ .coerced_ty = float_type }, params[1]);
7313 const mulend2 = try expr(gz, scope, .{ .coerced_ty = float_type }, params[2]);
73147314 const addend = try expr(gz, scope, .{ .ty = float_type }, params[3]);
73157315 const result = try gz.addPlNode(.mul_add, node, Zir.Inst.MulAdd{
73167316 .mulend1 = mulend1,
src/Liveness.zig+3-2
......@@ -465,8 +465,9 @@ fn analyzeInst(
465465 return trackOperands(a, new_set, inst, main_tomb, .{ extra.ptr, extra.expected_value, extra.new_value });
466466 },
467467 .mul_add => {
468 const extra = a.air.extraData(Air.MulAdd, inst_datas[inst].ty_pl.payload).data;
469 return trackOperands(a, new_set, inst, main_tomb, .{ extra.mulend1, extra.mulend2, extra.addend });
468 const pl_op = inst_datas[inst].pl_op;
469 const extra = a.air.extraData(Air.Bin, pl_op.payload).data;
470 return trackOperands(a, new_set, inst, main_tomb, .{ extra.lhs, extra.rhs, pl_op.operand });
470471 },
471472 .atomic_load => {
472473 const ptr = inst_datas[inst].atomic_load.ptr;
src/Sema.zig+40-41
......@@ -13525,48 +13525,26 @@ fn zirMulAdd(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.
1352513525 const mulend2_src: LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };
1352613526 const addend_src: LazySrcLoc = .{ .node_offset_builtin_call_arg3 = inst_data.src_node };
1352713527
13528 const mulend1 = sema.resolveInst(extra.mulend1);
13529 const mulend2 = sema.resolveInst(extra.mulend2);
1353013528 const addend = sema.resolveInst(extra.addend);
13531 // All args have the same type
13532 const ty = sema.typeOf(mulend1);
13533 switch (ty.zigTypeTag()) {
13534 .ComptimeFloat, .Float => {},
13535 .Vector => {
13536 const scalar_ty = ty.scalarType();
13537 switch (scalar_ty.zigTypeTag()) {
13538 .ComptimeFloat, .Float => {},
13539 else => return sema.fail(block, src, "expected vector of floats or float type, found '{}'", .{scalar_ty}),
13540 }
13541 },
13542 else => return sema.fail(block, src, "expected vector of floats or float type, found '{}'", .{ty}),
13543 }
13529 const ty = sema.typeOf(addend);
13530 const mulend1 = try sema.coerce(block, ty, sema.resolveInst(extra.mulend1), mulend1_src);
13531 const mulend2 = try sema.coerce(block, ty, sema.resolveInst(extra.mulend2), mulend2_src);
1354413532
1354513533 const target = sema.mod.getTarget();
13534
1354613535 switch (ty.zigTypeTag()) {
1354713536 .ComptimeFloat, .Float => {
1354813537 const maybe_mulend1 = try sema.resolveMaybeUndefVal(block, mulend1_src, mulend1);
1354913538 const maybe_mulend2 = try sema.resolveMaybeUndefVal(block, mulend2_src, mulend2);
1355013539 const maybe_addend = try sema.resolveMaybeUndefVal(block, addend_src, addend);
1355113540
13552 if (maybe_mulend1) |mulend1_val| {
13553 if (mulend1_val.isUndef())
13554 return sema.addConstUndef(ty);
13555 }
13556
13557 if (maybe_mulend2) |mulend2_val| {
13558 if (mulend2_val.isUndef())
13559 return sema.addConstUndef(ty);
13560 }
13561
13562 if (maybe_addend) |addend_val| {
13563 if (addend_val.isUndef())
13564 return sema.addConstUndef(ty);
13565 }
13566
13567 if (maybe_mulend1) |mulend1_val| {
13541 const runtime_src = if (maybe_mulend1) |mulend1_val| rs: {
1356813542 if (maybe_mulend2) |mulend2_val| {
13543 if (mulend2_val.isUndef()) return sema.addConstUndef(ty);
13544
1356913545 if (maybe_addend) |addend_val| {
13546 if (addend_val.isUndef()) return sema.addConstUndef(ty);
13547
1357013548 const result_val = try Value.mulAdd(
1357113549 ty,
1357213550 mulend1_val,
......@@ -13576,25 +13554,46 @@ fn zirMulAdd(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.
1357613554 target,
1357713555 );
1357813556 return sema.addConstant(ty, result_val);
13557 } else {
13558 break :rs addend_src;
1357913559 }
13560 } else {
13561 if (maybe_addend) |addend_val| {
13562 if (addend_val.isUndef()) return sema.addConstUndef(ty);
13563 }
13564 break :rs mulend2_src;
1358013565 }
13581 }
13566 } else rs: {
13567 if (maybe_mulend2) |mulend2_val| {
13568 if (mulend2_val.isUndef()) return sema.addConstUndef(ty);
13569 }
13570 if (maybe_addend) |addend_val| {
13571 if (addend_val.isUndef()) return sema.addConstUndef(ty);
13572 }
13573 break :rs mulend1_src;
13574 };
1358213575
13583 try sema.requireRuntimeBlock(block, src);
13576 try sema.requireRuntimeBlock(block, runtime_src);
1358413577 return block.addInst(.{
1358513578 .tag = .mul_add,
13586 .data = .{ .ty_pl = .{
13587 .ty = try sema.addType(ty),
13588 .payload = try sema.addExtra(Air.MulAdd{
13589 .mulend1 = mulend1,
13590 .mulend2 = mulend2,
13591 .addend = addend,
13579 .data = .{ .pl_op = .{
13580 .operand = addend,
13581 .payload = try sema.addExtra(Air.Bin{
13582 .lhs = mulend1,
13583 .rhs = mulend2,
1359213584 }),
1359313585 } },
1359413586 });
1359513587 },
13596 .Vector => return sema.fail(block, src, "TODO: implement @mulAdd for vectors", .{}),
13597 else => unreachable,
13588 .Vector => {
13589 const scalar_ty = ty.scalarType();
13590 switch (scalar_ty.zigTypeTag()) {
13591 .ComptimeFloat, .Float => {},
13592 else => return sema.fail(block, src, "expected vector of floats or float type, found '{}'", .{scalar_ty}),
13593 }
13594 return sema.fail(block, src, "TODO: implement @mulAdd for vectors", .{});
13595 },
13596 else => return sema.fail(block, src, "expected vector of floats or float type, found '{}'", .{ty}),
1359813597 }
1359913598}
1360013599
src/Zir.zig+2
......@@ -891,6 +891,8 @@ pub const Inst = struct {
891891 atomic_store,
892892 /// Implements the `@mulAdd` builtin.
893893 /// Uses the `pl_node` union field with payload `MulAdd`.
894 /// The addend communicates the type of the builtin.
895 /// The mulends need to be coerced to the same type.
894896 mul_add,
895897 /// Implements the `@call` builtin.
896898 /// Uses the `pl_node` union field with payload `BuiltinCall`.
src/arch/aarch64/CodeGen.zig+6-2
......@@ -3654,8 +3654,12 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
36543654}
36553655
36563656fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
3657 _ = inst;
3658 return self.fail("TODO implement airMulAdd for aarch64", .{});
3657 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
3658 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
3659 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
3660 return self.fail("TODO implement airMulAdd for aarch64", .{});
3661 };
3662 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
36593663}
36603664
36613665fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
src/arch/arm/CodeGen.zig+6-2
......@@ -4088,8 +4088,12 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
40884088}
40894089
40904090fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
4091 _ = inst;
4092 return self.fail("TODO implement airMulAdd for arm", .{});
4091 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
4092 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
4093 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
4094 return self.fail("TODO implement airMulAdd for arm", .{});
4095 };
4096 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
40934097}
40944098
40954099fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
src/arch/riscv64/CodeGen.zig+6-2
......@@ -2205,8 +2205,12 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
22052205}
22062206
22072207fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
2208 _ = inst;
2209 return self.fail("TODO implement airMulAdd for riscv64", .{});
2208 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
2209 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
2210 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
2211 return self.fail("TODO implement airMulAdd for riscv64", .{});
2212 };
2213 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
22102214}
22112215
22122216fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
src/arch/x86_64/CodeGen.zig+6-2
......@@ -5561,8 +5561,12 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
55615561}
55625562
55635563fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
5564 _ = inst;
5565 return self.fail("TODO implement airMulAdd for x86_64", .{});
5564 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
5565 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
5566 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else {
5567 return self.fail("TODO implement airMulAdd for x86_64", .{});
5568 };
5569 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, pl_op.operand });
55665570}
55675571
55685572fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
src/codegen/c.zig+31-1
......@@ -16,6 +16,7 @@ const trace = @import("../tracy.zig").trace;
1616const LazySrcLoc = Module.LazySrcLoc;
1717const Air = @import("../Air.zig");
1818const Liveness = @import("../Liveness.zig");
19const CType = @import("../type.zig").CType;
1920
2021const Mutability = enum { Const, Mut };
2122const BigIntConst = std.math.big.int.Const;
......@@ -1635,7 +1636,7 @@ fn genBody(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, OutO
16351636 .trunc_float,
16361637 => |tag| return f.fail("TODO: C backend: implement unary op for tag '{s}'", .{@tagName(tag)}),
16371638
1638 .mul_add => return f.fail("TODO: C backend: implement @mulAdd", .{}),
1639 .mul_add => try airMulAdd(f, inst),
16391640
16401641 .add_with_overflow => try airAddWithOverflow(f, inst),
16411642 .sub_with_overflow => try airSubWithOverflow(f, inst),
......@@ -3623,6 +3624,35 @@ fn airWasmMemoryGrow(f: *Function, inst: Air.Inst.Index) !CValue {
36233624 return local;
36243625}
36253626
3627fn airMulAdd(f: *Function, inst: Air.Inst.Index) !CValue {
3628 if (f.liveness.isUnused(inst)) return CValue.none;
3629 const pl_op = f.air.instructions.items(.data)[inst].pl_op;
3630 const extra = f.air.extraData(Air.Bin, pl_op.payload).data;
3631 const inst_ty = f.air.typeOfIndex(inst);
3632 const mulend1 = try f.resolveInst(extra.lhs);
3633 const mulend2 = try f.resolveInst(extra.rhs);
3634 const addend = try f.resolveInst(pl_op.operand);
3635 const writer = f.object.writer();
3636 const target = f.object.dg.module.getTarget();
3637 const fn_name = switch (inst_ty.floatBits(target)) {
3638 16, 32 => "fmaf",
3639 64 => "fma",
3640 80 => if (CType.longdouble.sizeInBits(target) == 80) "fmal" else "__fmax",
3641 128 => if (CType.longdouble.sizeInBits(target) == 128) "fmal" else "fmaq",
3642 else => unreachable,
3643 };
3644 const local = try f.allocLocal(inst_ty, .Const);
3645 try writer.writeAll(" = ");
3646 try writer.print("{s}(", .{fn_name});
3647 try f.writeCValue(writer, mulend1);
3648 try writer.writeAll(", ");
3649 try f.writeCValue(writer, mulend2);
3650 try writer.writeAll(", ");
3651 try f.writeCValue(writer, addend);
3652 try writer.writeAll(");\n");
3653 return local;
3654}
3655
36263656fn toMemoryOrder(order: std.builtin.AtomicOrder) [:0]const u8 {
36273657 return switch (order) {
36283658 .Unordered => "memory_order_relaxed",
src/codegen/llvm.zig+127-31
......@@ -18,6 +18,7 @@ const target_util = @import("../target.zig");
1818const Value = @import("../value.zig").Value;
1919const Type = @import("../type.zig").Type;
2020const LazySrcLoc = Module.LazySrcLoc;
21const CType = @import("../type.zig").CType;
2122
2223const Error = error{ OutOfMemory, CodegenFail };
2324
......@@ -2189,12 +2190,12 @@ pub const FuncGen = struct {
21892190 .min => try self.airMin(inst),
21902191 .max => try self.airMax(inst),
21912192 .slice => try self.airSlice(inst),
2193 .mul_add => try self.airMulAdd(inst),
21922194
21932195 .add_with_overflow => try self.airOverflow(inst, "llvm.sadd.with.overflow", "llvm.uadd.with.overflow"),
21942196 .sub_with_overflow => try self.airOverflow(inst, "llvm.ssub.with.overflow", "llvm.usub.with.overflow"),
21952197 .mul_with_overflow => try self.airOverflow(inst, "llvm.smul.with.overflow", "llvm.umul.with.overflow"),
21962198 .shl_with_overflow => try self.airShlWithOverflow(inst),
2197 .mul_add => try self.airMulAdd(inst),
21982199
21992200 .bit_and, .bool_and => try self.airAnd(inst),
22002201 .bit_or, .bool_or => try self.airOr(inst),
......@@ -3844,43 +3845,43 @@ pub const FuncGen = struct {
38443845 }
38453846
38463847 fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
3847 if (self.liveness.isUnused(inst))
3848 return null;
3848 if (self.liveness.isUnused(inst)) return null;
38493849
3850 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
3851 const extra = self.air.extraData(Air.MulAdd, ty_pl.payload).data;
3850 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
3851 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
38523852
3853 const mulend1 = try self.resolveInst(extra.mulend1);
3854 const mulend2 = try self.resolveInst(extra.mulend2);
3855 const addend = try self.resolveInst(extra.addend);
3853 const mulend1 = try self.resolveInst(extra.lhs);
3854 const mulend2 = try self.resolveInst(extra.rhs);
3855 const addend = try self.resolveInst(pl_op.operand);
38563856
38573857 const ty = self.air.typeOfIndex(inst);
38583858 const llvm_ty = try self.dg.llvmType(ty);
38593859 const target = self.dg.module.getTarget();
38603860
3861 const fn_val = switch (ty.floatBits(target)) {
3862 16, 32, 64 => blk: {
3863 break :blk self.getIntrinsic("llvm.fma", &.{llvm_ty});
3864 },
3865 // TODO: using `llvm.fma` for f80 does not seem to work for all targets, needs further investigation.
3866 80 => return self.dg.todo("Implement mulAdd for f80", .{}),
3867 128 => blk: {
3868 // LLVM incorrectly lowers the fma builtin for f128 to fmal, which is for
3869 // `long double`. On some targets this will be correct; on others it will be incorrect.
3870 if (target.longDoubleIsF128()) {
3871 break :blk self.getIntrinsic("llvm.fma", &.{llvm_ty});
3872 } else {
3873 break :blk self.dg.object.llvm_module.getNamedFunction("fmaq") orelse fn_blk: {
3874 const param_types = [_]*const llvm.Type{ llvm_ty, llvm_ty, llvm_ty };
3875 const fn_type = llvm.functionType(llvm_ty, &param_types, param_types.len, .False);
3876 break :fn_blk self.dg.object.llvm_module.addFunction("fmaq", fn_type);
3877 };
3878 }
3879 },
3861 const Strat = union(enum) {
3862 intrinsic,
3863 libc: [*:0]const u8,
3864 };
3865 const strat: Strat = switch (ty.floatBits(target)) {
3866 16, 32, 64 => Strat.intrinsic,
3867 80 => if (CType.longdouble.sizeInBits(target) == 80) Strat{ .intrinsic = {} } else Strat{ .libc = "__fmax" },
3868 // LLVM always lowers the fma builtin for f128 to fmal, which is for `long double`.
3869 // On some targets this will be correct; on others it will be incorrect.
3870 128 => if (CType.longdouble.sizeInBits(target) == 128) Strat{ .intrinsic = {} } else Strat{ .libc = "fmaq" },
38803871 else => unreachable,
38813872 };
3873
3874 const llvm_fn = switch (strat) {
3875 .intrinsic => self.getIntrinsic("llvm.fma", &.{llvm_ty}),
3876 .libc => |fn_name| self.dg.object.llvm_module.getNamedFunction(fn_name) orelse b: {
3877 const param_types = [_]*const llvm.Type{ llvm_ty, llvm_ty, llvm_ty };
3878 const fn_type = llvm.functionType(llvm_ty, &param_types, param_types.len, .False);
3879 break :b self.dg.object.llvm_module.addFunction(fn_name, fn_type);
3880 },
3881 };
3882
38823883 const params = [_]*const llvm.Value{ mulend1, mulend2, addend };
3883 return self.builder.buildCall(fn_val, &params, params.len, .C, .Auto, "");
3884 return self.builder.buildCall(llvm_fn, &params, params.len, .C, .Auto, "");
38843885 }
38853886
38863887 fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
......@@ -4061,8 +4062,15 @@ pub const FuncGen = struct {
40614062
40624063 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
40634064 const operand = try self.resolveInst(ty_op.operand);
4064 const dest_llvm_ty = try self.dg.llvmType(self.air.typeOfIndex(inst));
4065
4065 const operand_ty = self.air.typeOf(ty_op.operand);
4066 const dest_ty = self.air.typeOfIndex(inst);
4067 const target = self.dg.module.getTarget();
4068 const dest_bits = dest_ty.floatBits(target);
4069 const src_bits = operand_ty.floatBits(target);
4070 if (!backendSupportsF80(target) and (src_bits == 80 or dest_bits == 80)) {
4071 return softF80TruncOrExt(self, operand, src_bits, dest_bits);
4072 }
4073 const dest_llvm_ty = try self.dg.llvmType(dest_ty);
40664074 return self.builder.buildFPTrunc(operand, dest_llvm_ty, "");
40674075 }
40684076
......@@ -4072,8 +4080,15 @@ pub const FuncGen = struct {
40724080
40734081 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
40744082 const operand = try self.resolveInst(ty_op.operand);
4083 const operand_ty = self.air.typeOf(ty_op.operand);
4084 const dest_ty = self.air.typeOfIndex(inst);
4085 const target = self.dg.module.getTarget();
4086 const dest_bits = dest_ty.floatBits(target);
4087 const src_bits = operand_ty.floatBits(target);
4088 if (!backendSupportsF80(target) and (src_bits == 80 or dest_bits == 80)) {
4089 return softF80TruncOrExt(self, operand, src_bits, dest_bits);
4090 }
40754091 const dest_llvm_ty = try self.dg.llvmType(self.air.typeOfIndex(inst));
4076
40774092 return self.builder.buildFPExt(operand, dest_llvm_ty, "");
40784093 }
40794094
......@@ -5105,6 +5120,87 @@ pub const FuncGen = struct {
51055120 return null;
51065121 }
51075122
5123 fn softF80TruncOrExt(
5124 self: *FuncGen,
5125 operand: *const llvm.Value,
5126 src_bits: u16,
5127 dest_bits: u16,
5128 ) !?*const llvm.Value {
5129 const target = self.dg.module.getTarget();
5130
5131 var param_llvm_ty: *const llvm.Type = self.context.intType(80);
5132 var ret_llvm_ty: *const llvm.Type = param_llvm_ty;
5133 var fn_name: [*:0]const u8 = undefined;
5134 var arg = operand;
5135 var final_cast: ?*const llvm.Type = null;
5136
5137 assert(src_bits == 80 or dest_bits == 80);
5138
5139 if (src_bits == 80) switch (dest_bits) {
5140 16 => {
5141 // See corresponding condition at definition of
5142 // __truncxfhf2 in compiler-rt.
5143 if (target.cpu.arch.isAARCH64()) {
5144 ret_llvm_ty = self.context.halfType();
5145 } else {
5146 ret_llvm_ty = self.context.intType(16);
5147 final_cast = self.context.halfType();
5148 }
5149 fn_name = "__truncxfhf2";
5150 },
5151 32 => {
5152 ret_llvm_ty = self.context.floatType();
5153 fn_name = "__truncxfsf2";
5154 },
5155 64 => {
5156 ret_llvm_ty = self.context.doubleType();
5157 fn_name = "__truncxfdf2";
5158 },
5159 80 => return operand,
5160 128 => {
5161 ret_llvm_ty = self.context.fp128Type();
5162 fn_name = "__extendxftf2";
5163 },
5164 else => unreachable,
5165 } else switch (src_bits) {
5166 16 => {
5167 // See corresponding condition at definition of
5168 // __extendhfxf2 in compiler-rt.
5169 param_llvm_ty = if (target.cpu.arch.isAARCH64())
5170 self.context.halfType()
5171 else
5172 self.context.intType(16);
5173 arg = self.builder.buildBitCast(arg, param_llvm_ty, "");
5174 fn_name = "__extendhfxf2";
5175 },
5176 32 => {
5177 param_llvm_ty = self.context.floatType();
5178 fn_name = "__extendsfxf2";
5179 },
5180 64 => {
5181 param_llvm_ty = self.context.doubleType();
5182 fn_name = "__extenddfxf2";
5183 },
5184 80 => return operand,
5185 128 => {
5186 param_llvm_ty = self.context.fp128Type();
5187 fn_name = "__trunctfxf2";
5188 },
5189 else => unreachable,
5190 }
5191
5192 const llvm_fn = self.dg.object.llvm_module.getNamedFunction(fn_name) orelse f: {
5193 const param_types = [_]*const llvm.Type{param_llvm_ty};
5194 const fn_type = llvm.functionType(ret_llvm_ty, &param_types, param_types.len, .False);
5195 break :f self.dg.object.llvm_module.addFunction(fn_name, fn_type);
5196 };
5197
5198 var args: [1]*const llvm.Value = .{arg};
5199 const result = self.builder.buildCall(llvm_fn, &args, args.len, .C, .Auto, "");
5200 const final_cast_llvm_ty = final_cast orelse return result;
5201 return self.builder.buildBitCast(result, final_cast_llvm_ty, "");
5202 }
5203
51085204 fn getErrorNameTable(self: *FuncGen) !*const llvm.Value {
51095205 if (self.dg.object.error_name_table) |table| {
51105206 return table;
src/print_air.zig+5-5
......@@ -360,14 +360,14 @@ const Writer = struct {
360360 }
361361
362362 fn writeMulAdd(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
363 const ty_pl = w.air.instructions.items(.data)[inst].ty_pl;
364 const extra = w.air.extraData(Air.MulAdd, ty_pl.payload).data;
363 const pl_op = w.air.instructions.items(.data)[inst].pl_op;
364 const extra = w.air.extraData(Air.Bin, pl_op.payload).data;
365365
366 try w.writeOperand(s, inst, 0, extra.mulend1);
366 try w.writeOperand(s, inst, 0, extra.lhs);
367367 try s.writeAll(", ");
368 try w.writeOperand(s, inst, 1, extra.mulend2);
368 try w.writeOperand(s, inst, 1, extra.rhs);
369369 try s.writeAll(", ");
370 try w.writeOperand(s, inst, 2, extra.addend);
370 try w.writeOperand(s, inst, 2, pl_op.operand);
371371 }
372372
373373 fn writeFence(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
src/type.zig+55-67
......@@ -5436,33 +5436,36 @@ pub const CType = enum {
54365436 switch (target.os.tag) {
54375437 .freestanding, .other => switch (target.cpu.arch) {
54385438 .msp430 => switch (self) {
5439 .short,
5440 .ushort,
5441 .int,
5442 .uint,
5443 => return 16,
5444 .long,
5445 .ulong,
5446 => return 32,
5447 .longlong,
5448 .ulonglong,
5449 => return 64,
5450 .longdouble => @panic("TODO figure out what kind of float `long double` is on this target"),
5439 .short, .ushort, .int, .uint => return 16,
5440 .long, .ulong => return 32,
5441 .longlong, .ulonglong, .longdouble => return 64,
54515442 },
54525443 else => switch (self) {
5453 .short,
5454 .ushort,
5455 => return 16,
5456 .int,
5457 .uint,
5458 => return 32,
5459 .long,
5460 .ulong,
5461 => return target.cpu.arch.ptrBitWidth(),
5462 .longlong,
5463 .ulonglong,
5464 => return 64,
5465 .longdouble => @panic("TODO figure out what kind of float `long double` is on this target"),
5444 .short, .ushort => return 16,
5445 .int, .uint => return 32,
5446 .long, .ulong => return target.cpu.arch.ptrBitWidth(),
5447 .longlong, .ulonglong => return 64,
5448 .longdouble => switch (target.cpu.arch) {
5449 .i386, .x86_64 => return 80,
5450
5451 .riscv64,
5452 .aarch64,
5453 .aarch64_be,
5454 .aarch64_32,
5455 .s390x,
5456 .mips64,
5457 .mips64el,
5458 .sparc,
5459 .sparcv9,
5460 .sparcel,
5461 .powerpc,
5462 .powerpcle,
5463 .powerpc64,
5464 .powerpc64le,
5465 => return 128,
5466
5467 else => return 64,
5468 },
54665469 },
54675470 },
54685471
......@@ -5477,19 +5480,13 @@ pub const CType = enum {
54775480 .plan9,
54785481 .solaris,
54795482 => switch (self) {
5480 .short,
5481 .ushort,
5482 => return 16,
5483 .int,
5484 .uint,
5485 => return 32,
5486 .long,
5487 .ulong,
5488 => return target.cpu.arch.ptrBitWidth(),
5489 .longlong,
5490 .ulonglong,
5491 => return 64,
5483 .short, .ushort => return 16,
5484 .int, .uint => return 32,
5485 .long, .ulong => return target.cpu.arch.ptrBitWidth(),
5486 .longlong, .ulonglong => return 64,
54925487 .longdouble => switch (target.cpu.arch) {
5488 .i386, .x86_64 => return 80,
5489
54935490 .riscv64,
54945491 .aarch64,
54955492 .aarch64_be,
......@@ -5497,40 +5494,33 @@ pub const CType = enum {
54975494 .s390x,
54985495 .mips64,
54995496 .mips64el,
5497 .sparc,
5498 .sparcv9,
5499 .sparcel,
5500 .powerpc,
5501 .powerpcle,
5502 .powerpc64,
5503 .powerpc64le,
55005504 => return 128,
55015505
5502 else => return 80,
5506 else => return 64,
55035507 },
55045508 },
55055509
55065510 .windows, .uefi => switch (self) {
5507 .short,
5508 .ushort,
5509 => return 16,
5510 .int,
5511 .uint,
5512 .long,
5513 .ulong,
5514 => return 32,
5515 .longlong,
5516 .ulonglong,
5517 => return 64,
5518 .longdouble => @panic("TODO figure out what kind of float `long double` is on this target"),
5519 },
5520
5521 .ios => switch (self) {
5522 .short,
5523 .ushort,
5524 => return 16,
5525 .int,
5526 .uint,
5527 => return 32,
5528 .long,
5529 .ulong,
5530 .longlong,
5531 .ulonglong,
5532 => return 64,
5533 .longdouble => @panic("TODO figure out what kind of float `long double` is on this target"),
5511 .short, .ushort => return 16,
5512 .int, .uint, .long, .ulong => return 32,
5513 .longlong, .ulonglong, .longdouble => return 64,
5514 },
5515
5516 .ios, .tvos, .watchos => switch (self) {
5517 .short, .ushort => return 16,
5518 .int, .uint => return 32,
5519 .long, .ulong, .longlong, .ulonglong => return 64,
5520 .longdouble => switch (target.cpu.arch) {
5521 .i386, .x86_64 => return 80,
5522 else => return 64,
5523 },
55345524 },
55355525
55365526 .ananas,
......@@ -5549,8 +5539,6 @@ pub const CType = enum {
55495539 .amdhsa,
55505540 .ps4,
55515541 .elfiamcu,
5552 .tvos,
5553 .watchos,
55545542 .mesa3d,
55555543 .contiki,
55565544 .amdpal,
src/value.zig+19-12
......@@ -4020,37 +4020,44 @@ pub const Value = extern union {
40204020 }
40214021 }
40224022
4023 pub fn mulAdd(float_type: Type, mulend1: Value, mulend2: Value, addend: Value, arena: Allocator, target: Target) Allocator.Error!Value {
4023 pub fn mulAdd(
4024 float_type: Type,
4025 mulend1: Value,
4026 mulend2: Value,
4027 addend: Value,
4028 arena: Allocator,
4029 target: Target,
4030 ) Allocator.Error!Value {
40244031 switch (float_type.floatBits(target)) {
40254032 16 => {
4026 if (true) {
4027 // TODO: missing f16 implementation of FMA in `std.math.fma` or compiler-rt
4028 @panic("TODO implement mulAdd for f16");
4029 }
4033 const m1 = mulend1.toFloat(f16);
4034 const m2 = mulend2.toFloat(f16);
4035 const a = addend.toFloat(f16);
4036 return Value.Tag.float_16.create(arena, @mulAdd(f16, m1, m2, a));
40304037 },
40314038 32 => {
40324039 const m1 = mulend1.toFloat(f32);
40334040 const m2 = mulend2.toFloat(f32);
40344041 const a = addend.toFloat(f32);
4035 return Value.Tag.float_32.create(arena, std.math.fma(f32, m1, m2, a));
4042 return Value.Tag.float_32.create(arena, @mulAdd(f32, m1, m2, a));
40364043 },
40374044 64 => {
40384045 const m1 = mulend1.toFloat(f64);
40394046 const m2 = mulend2.toFloat(f64);
40404047 const a = addend.toFloat(f64);
4041 return Value.Tag.float_64.create(arena, std.math.fma(f64, m1, m2, a));
4048 return Value.Tag.float_64.create(arena, @mulAdd(f64, m1, m2, a));
40424049 },
40434050 80 => {
4044 if (true) {
4045 // TODO: missing f80 implementation of FMA in `std.math.fma` or compiler-rt
4046 @panic("TODO implement mulAdd for f80");
4047 }
4051 const m1 = mulend1.toFloat(f80);
4052 const m2 = mulend2.toFloat(f80);
4053 const a = addend.toFloat(f80);
4054 return Value.Tag.float_80.create(arena, @mulAdd(f80, m1, m2, a));
40484055 },
40494056 128 => {
40504057 const m1 = mulend1.toFloat(f128);
40514058 const m2 = mulend2.toFloat(f128);
40524059 const a = addend.toFloat(f128);
4053 return Value.Tag.float_128.create(arena, std.math.fma(f128, m1, m2, a));
4060 return Value.Tag.float_128.create(arena, @mulAdd(f128, m1, m2, a));
40544061 },
40554062 else => unreachable,
40564063 }
test/behavior/muladd.zig+17-18
......@@ -2,8 +2,8 @@ const builtin = @import("builtin");
22const expect = @import("std").testing.expect;
33
44test "@mulAdd" {
5 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
65 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
6 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
77 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
88 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
99 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
......@@ -13,22 +13,22 @@ test "@mulAdd" {
1313}
1414
1515fn testMulAdd() !void {
16 if (builtin.zig_backend == .stage1) {
17 const a: f16 = 5.5;
18 const b: f16 = 2.5;
19 const c: f16 = 6.25;
16 {
17 var a: f16 = 5.5;
18 var b: f16 = 2.5;
19 var c: f16 = 6.25;
2020 try expect(@mulAdd(f16, a, b, c) == 20);
2121 }
2222 {
23 const a: f32 = 5.5;
24 const b: f32 = 2.5;
25 const c: f32 = 6.25;
23 var a: f32 = 5.5;
24 var b: f32 = 2.5;
25 var c: f32 = 6.25;
2626 try expect(@mulAdd(f32, a, b, c) == 20);
2727 }
2828 {
29 const a: f64 = 5.5;
30 const b: f64 = 2.5;
31 const c: f64 = 6.25;
29 var a: f64 = 5.5;
30 var b: f64 = 2.5;
31 var c: f64 = 6.25;
3232 try expect(@mulAdd(f64, a, b, c) == 20);
3333 }
3434}
......@@ -39,9 +39,7 @@ test "@mulAdd f80" {
3939 return error.SkipZigTest;
4040 }
4141
42 // TODO: missing f80 implementation of FMA in `std.math.fma` or compiler-rt
43 // comptime try testMulAdd80();
44
42 comptime try testMulAdd80();
4543 try testMulAdd80();
4644}
4745
......@@ -53,11 +51,12 @@ fn testMulAdd80() !void {
5351}
5452
5553test "@mulAdd f128" {
56 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
5754 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
55 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
5856 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
5957 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
6058 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
59
6160 if (builtin.os.tag == .macos and builtin.cpu.arch == .aarch64) {
6261 // https://github.com/ziglang/zig/issues/9900
6362 return error.SkipZigTest;
......@@ -68,8 +67,8 @@ test "@mulAdd f128" {
6867}
6968
7069fn testMulAdd128() !void {
71 const a: f16 = 5.5;
72 const b: f128 = 2.5;
73 const c: f128 = 6.25;
70 var a: f16 = 5.5;
71 var b: f128 = 2.5;
72 var c: f128 = 6.25;
7473 try expect(@mulAdd(f128, a, b, c) == 20);
7574}