authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-10-05 14:20:12-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-10-05 14:22:38-07:00
log4ee91bb8a83fbddaf266500196525308525e2605
tree5aa0ee6ec57501a7607013cc0a9688341d11f5b8
parent5e153b53828875a0c5b6c893ff40add07ac0aafd

stage1: work around LLVM's buggy fma lowering

* move fmaq from freestanding libc to compiler_rt, unconditionally exported weak_odr. * stage1: add fmaf, fmal, fmaq as symbols that compiler-rt might generate calls to. * stage1: lower `@mulAdd` directly to a call to `fmaq` instead of to the LLVM intrinsic because LLVM will lower it to `fmal` even when the target's `long double` is not equivalent to `f128`. This commit is intended to fix the test suite which is failing on the previous commit.

5 files changed, 44 insertions(+), 15 deletions(-)

lib/std/special/c_stage1.zig-4
......@@ -656,10 +656,6 @@ export fn ceil(x: f64) f64 {
656656 return math.ceil(x);
657657}
658658
659export fn fmal(a: f128, b: f128, c: f128) f128 {
660 return math.fma(f128, a, b, c);
661}
662
663659export fn fma(a: f64, b: f64, c: f64) f64 {
664660 return math.fma(f64, a, b, c);
665661}
lib/std/special/compiler_rt.zig+6
......@@ -616,9 +616,15 @@ comptime {
616616 @export(__mulodi4, .{ .name = "__mulodi4", .linkage = linkage });
617617
618618 _ = @import("compiler_rt/atomics.zig");
619
620 @export(fmaq, .{ .name = "fmaq", .linkage = linkage });
619621 }
620622}
621623
624fn fmaq(a: f128, b: f128, c: f128) callconv(.C) f128 {
625 return std.math.fma(f128, a, b, c);
626}
627
622628// Avoid dragging in the runtime safety mechanisms into this .o file,
623629// unless we're trying to test this file.
624630pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace) noreturn {
src/stage1/codegen.cpp+21-11
......@@ -57,6 +57,9 @@ static const char *symbols_that_llvm_depends_on[] = {
5757 "log10",
5858 "log2",
5959 "fma",
60 "fmaf",
61 "fmal",
62 "fmaq",
6063 "fabs",
6164 "minnum",
6265 "maxnum",
......@@ -832,10 +835,25 @@ static LLVMValueRef get_float_fn(CodeGen *g, ZigType *type_entry, ZigLLVMFnId fn
832835
833836 bool is_vector = (type_entry->id == ZigTypeIdVector);
834837 ZigType *float_type = is_vector ? type_entry->data.vector.elem_type : type_entry;
838 uint32_t float_bits = float_type->data.floating.bit_count;
839
840 // LLVM incorrectly lowers the fma builtin for f128 to fmal, which is for
841 // `long double`. On some targets this will be correct; on others it will be incorrect.
842 if (fn_id == ZigLLVMFnIdFMA && float_bits == 128 &&
843 !target_long_double_is_f128(g->zig_target))
844 {
845 LLVMValueRef existing_llvm_fn = LLVMGetNamedFunction(g->module, "fmaq");
846 if (existing_llvm_fn != nullptr) return existing_llvm_fn;
847
848 LLVMTypeRef float_type_ref = get_llvm_type(g, type_entry);
849 LLVMTypeRef return_elem_types[3] = { float_type_ref, float_type_ref, float_type_ref };
850 LLVMTypeRef fn_type = LLVMFunctionType(float_type_ref, return_elem_types, 3, false);
851 return LLVMAddFunction(g->module, "fmaq", fn_type);
852 }
835853
836854 ZigLLVMFnKey key = {};
837855 key.id = fn_id;
838 key.data.floating.bit_count = (uint32_t)float_type->data.floating.bit_count;
856 key.data.floating.bit_count = float_bits;
839857 key.data.floating.vector_len = is_vector ? (uint32_t)type_entry->data.vector.len : 0;
840858 key.data.floating.op = op;
841859
......@@ -861,11 +879,7 @@ static LLVMValueRef get_float_fn(CodeGen *g, ZigType *type_entry, ZigLLVMFnId fn
861879 else
862880 sprintf(fn_name, "llvm.%s.f%" PRIu32, name, key.data.floating.bit_count);
863881 LLVMTypeRef float_type_ref = get_llvm_type(g, type_entry);
864 LLVMTypeRef return_elem_types[3] = {
865 float_type_ref,
866 float_type_ref,
867 float_type_ref,
868 };
882 LLVMTypeRef return_elem_types[3] = { float_type_ref, float_type_ref, float_type_ref };
869883 LLVMTypeRef fn_type = LLVMFunctionType(float_type_ref, return_elem_types, num_args, false);
870884 LLVMValueRef fn_val = LLVMAddFunction(g->module, fn_name, fn_type);
871885 assert(LLVMGetIntrinsicID(fn_val));
......@@ -6583,11 +6597,7 @@ static LLVMValueRef ir_render_mul_add(CodeGen *g, Stage1Air *executable, Stage1A
65836597 assert(instruction->base.value->type->id == ZigTypeIdFloat ||
65846598 instruction->base.value->type->id == ZigTypeIdVector);
65856599 LLVMValueRef fn_val = get_float_fn(g, instruction->base.value->type, ZigLLVMFnIdFMA, BuiltinFnIdMulAdd);
6586 LLVMValueRef args[3] = {
6587 op1,
6588 op2,
6589 op3,
6590 };
6600 LLVMValueRef args[3] = { op1, op2, op3 };
65916601 return LLVMBuildCall(g->builder, fn_val, args, 3, "");
65926602}
65936603
src/stage1/target.cpp+16
......@@ -999,6 +999,22 @@ bool target_has_debug_info(const ZigTarget *target) {
999999 return !target_is_wasm(target);
10001000}
10011001
1002bool target_long_double_is_f128(const ZigTarget *target) {
1003 switch (target->arch) {
1004 case ZigLLVM_riscv64:
1005 case ZigLLVM_aarch64:
1006 case ZigLLVM_aarch64_be:
1007 case ZigLLVM_aarch64_32:
1008 case ZigLLVM_systemz:
1009 case ZigLLVM_mips64:
1010 case ZigLLVM_mips64el:
1011 return true;
1012
1013 default:
1014 return false;
1015 }
1016}
1017
10021018bool target_is_riscv(const ZigTarget *target) {
10031019 return target->arch == ZigLLVM_riscv32 || target->arch == ZigLLVM_riscv64;
10041020}
src/stage1/target.hpp+1
......@@ -79,6 +79,7 @@ bool target_is_riscv(const ZigTarget *target);
7979bool target_is_sparc(const ZigTarget *target);
8080bool target_is_android(const ZigTarget *target);
8181bool target_has_debug_info(const ZigTarget *target);
82bool target_long_double_is_f128(const ZigTarget *target);
8283
8384uint32_t target_arch_pointer_bit_width(ZigLLVM_ArchType arch);
8485uint32_t target_arch_largest_atomic_bits(ZigLLVM_ArchType arch);