authorgravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-04-26 15:16:37-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-04-27 12:22:09-07:00
log0e8242b905514f838f36757f2f890241e7448554
tree53a5788dd61def8792e76621efb216218266de92
parent41dd2beaacade94c5c98400a4a655aea07b9e2f3

stage1: Manually lower softfloat ops when needed

Updates stage1 to manually lower softfloat operations for all unary floating point operations, extension/truncation, and arithmetic.

5 files changed, 390 insertions(+), 328 deletions(-)

src/stage1/analyze.cpp+3-1
...@@ -10375,7 +10375,7 @@ void ZigValue::dump() {...@@ -10375,7 +10375,7 @@ void ZigValue::dump() {
1037510375
10376// float ops that take a single argument10376// float ops that take a single argument
10377//TODO Powi, Pow, minnum, maxnum, maximum, minimum, copysign, lround, llround, lrint, llrint10377//TODO Powi, Pow, minnum, maxnum, maximum, minimum, copysign, lround, llround, lrint, llrint
10378const char *float_op_to_name(BuiltinFnId op) {10378const char *float_un_op_to_name(BuiltinFnId op) {
10379 switch (op) {10379 switch (op) {
10380 case BuiltinFnIdSqrt:10380 case BuiltinFnIdSqrt:
10381 return "sqrt";10381 return "sqrt";
...@@ -10405,6 +10405,8 @@ const char *float_op_to_name(BuiltinFnId op) {...@@ -10405,6 +10405,8 @@ const char *float_op_to_name(BuiltinFnId op) {
10405 return "nearbyint";10405 return "nearbyint";
10406 case BuiltinFnIdRound:10406 case BuiltinFnIdRound:
10407 return "round";10407 return "round";
10408 case BuiltinFnIdMulAdd:
10409 return "fma";
10408 default:10410 default:
10409 zig_unreachable();10411 zig_unreachable();
10410 }10412 }
src/stage1/analyze.hpp+1-1
...@@ -307,7 +307,7 @@ void copy_const_val(CodeGen *g, ZigValue *dest, ZigValue *src);...@@ -307,7 +307,7 @@ void copy_const_val(CodeGen *g, ZigValue *dest, ZigValue *src);
307bool type_has_optional_repr(ZigType *ty);307bool type_has_optional_repr(ZigType *ty);
308bool is_opt_err_set(ZigType *ty);308bool is_opt_err_set(ZigType *ty);
309bool type_is_numeric(ZigType *ty);309bool type_is_numeric(ZigType *ty);
310const char *float_op_to_name(BuiltinFnId op);310const char *float_un_op_to_name(BuiltinFnId op);
311311
312#define src_assert(OK, SOURCE_NODE) src_assert_impl((OK), (SOURCE_NODE), __FILE__, __LINE__)312#define src_assert(OK, SOURCE_NODE) src_assert_impl((OK), (SOURCE_NODE), __FILE__, __LINE__)
313313
src/stage1/codegen.cpp+382-322
...@@ -869,7 +869,7 @@ static LLVMValueRef get_float_fn(CodeGen *g, ZigType *type_entry, ZigLLVMFnId fn...@@ -869,7 +869,7 @@ static LLVMValueRef get_float_fn(CodeGen *g, ZigType *type_entry, ZigLLVMFnId fn
869 name = "fma";869 name = "fma";
870 num_args = 3;870 num_args = 3;
871 } else if (fn_id == ZigLLVMFnIdFloatOp) {871 } else if (fn_id == ZigLLVMFnIdFloatOp) {
872 name = float_op_to_name(op);872 name = float_un_op_to_name(op);
873 num_args = 1;873 num_args = 1;
874 } else {874 } else {
875 zig_unreachable();875 zig_unreachable();
...@@ -1604,8 +1604,49 @@ static LLVMValueRef gen_assert_zero(CodeGen *g, LLVMValueRef expr_val, ZigType *...@@ -1604,8 +1604,49 @@ static LLVMValueRef gen_assert_zero(CodeGen *g, LLVMValueRef expr_val, ZigType *
1604 return nullptr;1604 return nullptr;
1605}1605}
16061606
1607static const char *get_compiler_rt_type_abbrev(ZigType *type) {
1608 uint16_t bits;
1609 if (type->id == ZigTypeIdFloat) {
1610 bits = type->data.floating.bit_count;
1611 } else if (type->id == ZigTypeIdInt) {
1612 bits = type->data.integral.bit_count;
1613 } else {
1614 zig_unreachable();
1615 }
1616 switch (bits) {
1617 case 16:
1618 return "h";
1619 case 32:
1620 return "s";
1621 case 64:
1622 return "d";
1623 case 80:
1624 return "x";
1625 case 128:
1626 return "t";
1627 default:
1628 zig_unreachable();
1629 }
1630}
1631
1632static const char *get_math_h_type_abbrev(CodeGen *g, ZigType *float_type) {
1633 if (float_type == g->builtin_types.entry_f16)
1634 return "h"; // Non-standard
1635 else if (float_type == g->builtin_types.entry_f32)
1636 return "s";
1637 else if (float_type == g->builtin_types.entry_f64)
1638 return "";
1639 else if (float_type == g->builtin_types.entry_f80)
1640 return "x"; // Non-standard
1641 else if (float_type == g->builtin_types.entry_c_longdouble)
1642 return "l";
1643 else if (float_type == g->builtin_types.entry_f128)
1644 return "q"; // Non-standard
1645 else
1646 zig_unreachable();
1647}
16071648
1608static LLVMValueRef gen_soft_f80_widen_or_shorten(CodeGen *g, ZigType *actual_type,1649static LLVMValueRef gen_soft_float_widen_or_shorten(CodeGen *g, ZigType *actual_type,
1609 ZigType *wanted_type, LLVMValueRef expr_val)1650 ZigType *wanted_type, LLVMValueRef expr_val)
1610{1651{
1611 ZigType *scalar_actual_type = (actual_type->id == ZigTypeIdVector) ?1652 ZigType *scalar_actual_type = (actual_type->id == ZigTypeIdVector) ?
...@@ -1615,87 +1656,47 @@ static LLVMValueRef gen_soft_f80_widen_or_shorten(CodeGen *g, ZigType *actual_ty...@@ -1615,87 +1656,47 @@ static LLVMValueRef gen_soft_f80_widen_or_shorten(CodeGen *g, ZigType *actual_ty
1615 uint64_t actual_bits = scalar_actual_type->data.floating.bit_count;1656 uint64_t actual_bits = scalar_actual_type->data.floating.bit_count;
1616 uint64_t wanted_bits = scalar_wanted_type->data.floating.bit_count;1657 uint64_t wanted_bits = scalar_wanted_type->data.floating.bit_count;
16171658
16181659 if (actual_bits == wanted_bits)
1619 LLVMTypeRef param_type;1660 return expr_val;
1620 LLVMTypeRef return_type;
1621 const char *func_name;
16221661
1623 LLVMValueRef result;1662 LLVMValueRef result;
1624 bool castTruncatedToF16 = false;1663 bool castTruncatedToF16 = false;
16251664
1626 if (actual_bits == wanted_bits) {1665 char fn_name[64];
1627 return expr_val;1666 if (wanted_bits < actual_bits) {
1628 } else if (actual_bits == 80) {1667 sprintf(fn_name, "__trunc%sf%sf2",
1629 param_type = g->builtin_types.entry_f80->llvm_type;1668 get_compiler_rt_type_abbrev(scalar_actual_type),
1630 switch (wanted_bits) {1669 get_compiler_rt_type_abbrev(scalar_wanted_type));
1631 case 16:1670 } else {
1632 // Only Arm has a native f16 type, other platforms soft-implement it1671 sprintf(fn_name, "__extend%sf%sf2",
1633 // using u16 instead.1672 get_compiler_rt_type_abbrev(scalar_actual_type),
1634 if (target_is_arm(g->zig_target)) {1673 get_compiler_rt_type_abbrev(scalar_wanted_type));
1635 return_type = g->builtin_types.entry_f16->llvm_type;1674 }
1636 } else {1675
1637 return_type = g->builtin_types.entry_u16->llvm_type;1676 LLVMTypeRef return_type = scalar_wanted_type->llvm_type;
1638 castTruncatedToF16 = true;1677 LLVMTypeRef param_type = scalar_actual_type->llvm_type;
1639 }1678
1640 func_name = "__truncxfhf2";1679 if (!target_is_arm(g->zig_target)) {
1641 break;1680 // Only Arm has a native f16 type, other platforms soft-implement it using u16 instead.
1642 case 32:1681 if (scalar_wanted_type == g->builtin_types.entry_f16) {
1643 return_type = g->builtin_types.entry_f32->llvm_type;1682 return_type = g->builtin_types.entry_u16->llvm_type;
1644 func_name = "__truncxfsf2";1683 castTruncatedToF16 = true;
1645 break;
1646 case 64:
1647 return_type = g->builtin_types.entry_f64->llvm_type;
1648 func_name = "__truncxfdf2";
1649 break;
1650 case 128:
1651 return_type = g->builtin_types.entry_f128->llvm_type;
1652 func_name = "__extendxftf2";
1653 break;
1654 default:
1655 zig_unreachable();
1656 }1684 }
1657 } else if (wanted_bits == 80) {1685 if (scalar_actual_type == g->builtin_types.entry_f16) {
1658 return_type = g->builtin_types.entry_f80->llvm_type;1686 param_type = g->builtin_types.entry_u16->llvm_type;
1659 switch (actual_bits) {1687 expr_val = LLVMBuildBitCast(g->builder, expr_val, param_type, "");
1660 case 16:
1661 // Only Arm has a native f16 type, other platforms soft-implement it
1662 // using u16 instead.
1663 if (target_is_arm(g->zig_target)) {
1664 param_type = g->builtin_types.entry_f16->llvm_type;
1665 } else {
1666 param_type = g->builtin_types.entry_u16->llvm_type;
1667 expr_val = LLVMBuildBitCast(g->builder, expr_val, param_type, "");
1668 }
1669 func_name = "__extendhfxf2";
1670 break;
1671 case 32:
1672 param_type = g->builtin_types.entry_f32->llvm_type;
1673 func_name = "__extendsfxf2";
1674 break;
1675 case 64:
1676 param_type = g->builtin_types.entry_f64->llvm_type;
1677 func_name = "__extenddfxf2";
1678 break;
1679 case 128:
1680 param_type = g->builtin_types.entry_f128->llvm_type;
1681 func_name = "__trunctfxf2";
1682 break;
1683 default:
1684 zig_unreachable();
1685 }1688 }
1686 } else {
1687 zig_unreachable();
1688 }1689 }
16891690
1690 LLVMValueRef func_ref = LLVMGetNamedFunction(g->module, func_name);1691 LLVMValueRef func_ref = LLVMGetNamedFunction(g->module, fn_name);
1691 if (func_ref == nullptr) {1692 if (func_ref == nullptr) {
1692 LLVMTypeRef fn_type = LLVMFunctionType(return_type, &param_type, 1, false);1693 LLVMTypeRef fn_type = LLVMFunctionType(return_type, &param_type, 1, false);
1693 func_ref = LLVMAddFunction(g->module, func_name, fn_type);1694 func_ref = LLVMAddFunction(g->module, fn_name, fn_type);
1694 }1695 }
16951696
1696 result = LLVMBuildCall(g->builder, func_ref, &expr_val, 1, "");1697 result = LLVMBuildCall(g->builder, func_ref, &expr_val, 1, "");
16971698
1698 // On non-Arm platforms we need to bitcast __truncxfhf2 result back to f161699 // On non-Arm platforms we need to bitcast __trunc<>fhf2 result back to f16
1699 if (castTruncatedToF16) {1700 if (castTruncatedToF16) {
1700 result = LLVMBuildBitCast(g->builder, result, g->builtin_types.entry_f16->llvm_type, "");1701 result = LLVMBuildBitCast(g->builder, result, g->builtin_types.entry_f16->llvm_type, "");
1701 }1702 }
...@@ -1721,7 +1722,7 @@ static LLVMValueRef gen_widen_or_shorten(CodeGen *g, bool want_runtime_safety, Z...@@ -1721,7 +1722,7 @@ static LLVMValueRef gen_widen_or_shorten(CodeGen *g, bool want_runtime_safety, Z
1721 || scalar_wanted_type == g->builtin_types.entry_f80)1722 || scalar_wanted_type == g->builtin_types.entry_f80)
1722 && !target_has_f80(g->zig_target))1723 && !target_has_f80(g->zig_target))
1723 {1724 {
1724 return gen_soft_f80_widen_or_shorten(g, actual_type, wanted_type, expr_val);1725 return gen_soft_float_widen_or_shorten(g, actual_type, wanted_type, expr_val);
1725 }1726 }
1726 actual_bits = scalar_actual_type->data.floating.bit_count;1727 actual_bits = scalar_actual_type->data.floating.bit_count;
1727 wanted_bits = scalar_wanted_type->data.floating.bit_count;1728 wanted_bits = scalar_wanted_type->data.floating.bit_count;
...@@ -2978,10 +2979,50 @@ static LLVMValueRef gen_overflow_shr_op(CodeGen *g, ZigType *operand_type,...@@ -2978,10 +2979,50 @@ static LLVMValueRef gen_overflow_shr_op(CodeGen *g, ZigType *operand_type,
2978 return result;2979 return result;
2979}2980}
29802981
2981static LLVMValueRef gen_float_op(CodeGen *g, LLVMValueRef val, ZigType *type_entry, BuiltinFnId op) {2982static LLVMValueRef get_soft_float_fn(CodeGen *g, const char *name, int param_count, LLVMTypeRef param_type, LLVMTypeRef return_type) {
2982 assert(type_entry->id == ZigTypeIdFloat || type_entry->id == ZigTypeIdVector);2983 LLVMValueRef existing_llvm_fn = LLVMGetNamedFunction(g->module, name);
2983 LLVMValueRef floor_fn = get_float_fn(g, type_entry, ZigLLVMFnIdFloatOp, op);2984 if (existing_llvm_fn != nullptr) return existing_llvm_fn;
2984 return LLVMBuildCall(g->builder, floor_fn, &val, 1, "");2985 LLVMValueRef existing_llvm_alias = LLVMGetNamedGlobalAlias(g->module, name, strlen(name));
2986 if (existing_llvm_alias != nullptr) return LLVMAliasGetAliasee(existing_llvm_alias);
2987
2988 LLVMTypeRef param_types[3] = { param_type, param_type, param_type };
2989 LLVMTypeRef fn_type = LLVMFunctionType(return_type, param_types, param_count, false);
2990 return LLVMAddFunction(g->module, name, fn_type);
2991}
2992
2993static LLVMValueRef gen_soft_float_un_op(CodeGen *g, LLVMValueRef op, ZigType *operand_type, BuiltinFnId op_id) {
2994 uint32_t vector_len = operand_type->id == ZigTypeIdVector ? operand_type->data.vector.len : 0;
2995
2996 char fn_name[64];
2997 sprintf(fn_name, "%s%s", float_un_op_to_name(op_id), get_math_h_type_abbrev(g, operand_type));
2998 LLVMValueRef func_ref = get_soft_float_fn(g, fn_name, 1, operand_type->llvm_type, operand_type->llvm_type);
2999
3000 LLVMValueRef result;
3001 if (vector_len == 0) {
3002 return LLVMBuildCall(g->builder, func_ref, &op, 1, "");
3003 } else {
3004 result = build_alloca(g, operand_type, "", 0);
3005 LLVMTypeRef usize_ref = g->builtin_types.entry_usize->llvm_type;
3006 for (uint32_t i = 0; i < vector_len; i++) {
3007 LLVMValueRef index_value = LLVMConstInt(usize_ref, i, false);
3008 LLVMValueRef param = LLVMBuildExtractElement(g->builder, op, index_value, "");
3009 LLVMValueRef call_result = LLVMBuildCall(g->builder, func_ref, &param, 1, "");
3010 LLVMBuildInsertElement(g->builder, LLVMBuildLoad(g->builder, result, ""),
3011 call_result, index_value, "");
3012 }
3013 return LLVMBuildLoad(g->builder, result, "");
3014 }
3015}
3016
3017static LLVMValueRef gen_float_un_op(CodeGen *g, LLVMValueRef operand, ZigType *operand_type, BuiltinFnId op) {
3018 assert(operand_type->id == ZigTypeIdFloat || operand_type->id == ZigTypeIdVector);
3019 ZigType *elem_type = operand_type->id == ZigTypeIdVector ? operand_type->data.vector.elem_type : operand_type;
3020 if ((elem_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target)) ||
3021 (elem_type == g->builtin_types.entry_f128 && !target_long_double_is_f128(g->zig_target))) {
3022 return gen_soft_float_un_op(g, operand, operand_type, op);
3023 }
3024 LLVMValueRef float_op_fn = get_float_fn(g, operand_type, ZigLLVMFnIdFloatOp, op);
3025 return LLVMBuildCall(g->builder, float_op_fn, &operand, 1, "");
2985}3026}
29863027
2987enum DivKind {3028enum DivKind {
...@@ -3088,7 +3129,7 @@ static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast...@@ -3088,7 +3129,7 @@ static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast
3088 case DivKindExact:3129 case DivKindExact:
3089 if (want_runtime_safety) {3130 if (want_runtime_safety) {
3090 // Safety check: a / b == floor(a / b)3131 // Safety check: a / b == floor(a / b)
3091 LLVMValueRef floored = gen_float_op(g, result, operand_type, BuiltinFnIdFloor);3132 LLVMValueRef floored = gen_float_un_op(g, result, operand_type, BuiltinFnIdFloor);
30923133
3093 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactOk");3134 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactOk");
3094 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactFail");3135 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactFail");
...@@ -3105,9 +3146,9 @@ static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast...@@ -3105,9 +3146,9 @@ static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast
3105 }3146 }
3106 return result;3147 return result;
3107 case DivKindTrunc:3148 case DivKindTrunc:
3108 return gen_float_op(g, result, operand_type, BuiltinFnIdTrunc);3149 return gen_float_un_op(g, result, operand_type, BuiltinFnIdTrunc);
3109 case DivKindFloor:3150 case DivKindFloor:
3110 return gen_float_op(g, result, operand_type, BuiltinFnIdFloor);3151 return gen_float_un_op(g, result, operand_type, BuiltinFnIdFloor);
3111 }3152 }
3112 zig_unreachable();3153 zig_unreachable();
3113 }3154 }
...@@ -3269,17 +3310,7 @@ static void gen_shift_rhs_check(CodeGen *g, ZigType *lhs_type, ZigType *rhs_type...@@ -3269,17 +3310,7 @@ static void gen_shift_rhs_check(CodeGen *g, ZigType *lhs_type, ZigType *rhs_type
3269 }3310 }
3270}3311}
32713312
3272static LLVMValueRef get_soft_f80_bin_op_func(CodeGen *g, const char *name, int param_count, LLVMTypeRef return_type) {3313enum Icmp {
3273 LLVMValueRef existing_llvm_fn = LLVMGetNamedFunction(g->module, name);
3274 if (existing_llvm_fn != nullptr) return existing_llvm_fn;
3275
3276 LLVMTypeRef float_type_ref = g->builtin_types.entry_f80->llvm_type;
3277 LLVMTypeRef param_types[2] = { float_type_ref, float_type_ref };
3278 LLVMTypeRef fn_type = LLVMFunctionType(return_type, param_types, param_count, false);
3279 return LLVMAddFunction(g->module, name, fn_type);
3280}
3281
3282enum SoftF80Icmp {
3283 NONE,3314 NONE,
3284 EQ_ZERO,3315 EQ_ZERO,
3285 NE_ZERO,3316 NE_ZERO,
...@@ -3289,7 +3320,7 @@ enum SoftF80Icmp {...@@ -3289,7 +3320,7 @@ enum SoftF80Icmp {
3289 EQ_ONE,3320 EQ_ONE,
3290};3321};
32913322
3292static LLVMValueRef add_f80_icmp(CodeGen *g, LLVMValueRef val, SoftF80Icmp kind) {3323static LLVMValueRef add_icmp(CodeGen *g, LLVMValueRef val, Icmp kind) {
3293 switch (kind) {3324 switch (kind) {
3294 case NONE:3325 case NONE:
3295 return val;3326 return val;
...@@ -3322,22 +3353,123 @@ static LLVMValueRef add_f80_icmp(CodeGen *g, LLVMValueRef val, SoftF80Icmp kind)...@@ -3322,22 +3353,123 @@ static LLVMValueRef add_f80_icmp(CodeGen *g, LLVMValueRef val, SoftF80Icmp kind)
3322 }3353 }
3323}3354}
33243355
3325static LLVMValueRef ir_render_soft_f80_bin_op(CodeGen *g, Stage1Air *executable,3356static LLVMValueRef gen_soft_int_to_float_op(CodeGen *g, LLVMValueRef value_ref, ZigType *operand_type, ZigType *result_type) {
3326 Stage1AirInstBinOp *bin_op_instruction)3357 uint32_t vector_len = operand_type->id == ZigTypeIdVector ? operand_type->data.vector.len : 0;
3327{
3328 IrBinOp op_id = bin_op_instruction->op_id;
3329 Stage1AirInst *op1 = bin_op_instruction->op1;
3330 Stage1AirInst *op2 = bin_op_instruction->op2;
3331 uint32_t vector_len = op1->value->type->id == ZigTypeIdVector ? op1->value->type->data.vector.len : 0;
33323358
3333 LLVMValueRef op1_value = ir_llvm_value(g, op1);3359 // Handle integers of non-pot bitsize by widening them.
3334 LLVMValueRef op2_value = ir_llvm_value(g, op2);3360 const size_t bitsize = operand_type->data.integral.bit_count;
3361 const bool is_signed = operand_type->data.integral.is_signed;
3362 if (bitsize < 32 || !is_power_of_2(bitsize)) {
3363 const size_t wider_bitsize = bitsize < 32 ? 32 : round_to_next_power_of_2(bitsize);
3364 ZigType *const wider_type = get_int_type(g, is_signed, wider_bitsize);
3365 value_ref = gen_widen_or_shorten(g, false, operand_type, wider_type, value_ref);
3366 operand_type = wider_type;
3367 }
3368 assert(bitsize <= 128);
3369
3370 const char *int_compiler_rt_type_abbrev = get_compiler_rt_type_abbrev(operand_type);
3371 const char *float_compiler_rt_type_abbrev = get_compiler_rt_type_abbrev(result_type);
3372
3373 char fn_name[64];
3374 if (is_signed) {
3375 sprintf(fn_name, "__float%si%sf", int_compiler_rt_type_abbrev, float_compiler_rt_type_abbrev);
3376 } else {
3377 sprintf(fn_name, "__floatun%si%sf", int_compiler_rt_type_abbrev, float_compiler_rt_type_abbrev);
3378 }
3379
3380 int param_count = 1;
3381 LLVMValueRef func_ref = get_soft_float_fn(g, fn_name, param_count, operand_type->llvm_type, result_type->llvm_type);
3382
3383 LLVMValueRef result;
3384 if (vector_len == 0) {
3385 LLVMValueRef params[1] = {value_ref};
3386 result = LLVMBuildCall(g->builder, func_ref, params, param_count, "");
3387 } else {
3388 ZigType *alloca_ty = operand_type;
3389 result = build_alloca(g, alloca_ty, "", 0);
3390
3391 LLVMTypeRef usize_ref = g->builtin_types.entry_usize->llvm_type;
3392 for (uint32_t i = 0; i < vector_len; i++) {
3393 LLVMValueRef index_value = LLVMConstInt(usize_ref, i, false);
3394 LLVMValueRef params[1] = {
3395 LLVMBuildExtractElement(g->builder, value_ref, index_value, ""),
3396 };
3397 LLVMValueRef call_result = LLVMBuildCall(g->builder, func_ref, params, param_count, "");
3398 LLVMBuildInsertElement(g->builder, LLVMBuildLoad(g->builder, result, ""),
3399 call_result, index_value, "");
3400 }
3401
3402 result = LLVMBuildLoad(g->builder, result, "");
3403 }
3404 return result;
3405}
33353406
3336 bool div_exact_safety_check = false;3407static LLVMValueRef gen_soft_float_to_int_op(CodeGen *g, LLVMValueRef value_ref, ZigType *operand_type, ZigType *result_type) {
3337 LLVMTypeRef return_type = g->builtin_types.entry_f80->llvm_type;3408 uint32_t vector_len = operand_type->id == ZigTypeIdVector ? operand_type->data.vector.len : 0;
3409
3410 // Handle integers of non-pot bitsize by truncating a sufficiently wide pot integer
3411 const size_t bitsize = result_type->data.integral.bit_count;
3412 const bool is_signed = result_type->data.integral.is_signed;
3413 ZigType * wider_type = result_type;
3414 if (bitsize < 32 || !is_power_of_2(bitsize)) {
3415 const size_t wider_bitsize = bitsize < 32 ? 32 : round_to_next_power_of_2(bitsize);
3416 wider_type = get_int_type(g, is_signed, wider_bitsize);
3417 }
3418 assert(bitsize <= 128);
3419
3420 const char *float_compiler_rt_type_abbrev = get_compiler_rt_type_abbrev(operand_type);
3421 const char *int_compiler_rt_type_abbrev = get_compiler_rt_type_abbrev(wider_type);
3422
3423 char fn_name[64];
3424 if (is_signed) {
3425 sprintf(fn_name, "__fix%sf%si", float_compiler_rt_type_abbrev, int_compiler_rt_type_abbrev);
3426 } else {
3427 sprintf(fn_name, "__fixuns%sf%si", float_compiler_rt_type_abbrev, int_compiler_rt_type_abbrev);
3428 }
3429
3430 int param_count = 1;
3431 LLVMValueRef func_ref = get_soft_float_fn(g, fn_name, param_count, operand_type->llvm_type, wider_type->llvm_type);
3432
3433 LLVMValueRef result;
3434 if (vector_len == 0) {
3435 LLVMValueRef params[1] = {value_ref};
3436 result = LLVMBuildCall(g->builder, func_ref, params, param_count, "");
3437 } else {
3438 ZigType *alloca_ty = operand_type;
3439 result = build_alloca(g, alloca_ty, "", 0);
3440
3441 LLVMTypeRef usize_ref = g->builtin_types.entry_usize->llvm_type;
3442 for (uint32_t i = 0; i < vector_len; i++) {
3443 LLVMValueRef index_value = LLVMConstInt(usize_ref, i, false);
3444 LLVMValueRef params[1] = {
3445 LLVMBuildExtractElement(g->builder, value_ref, index_value, ""),
3446 };
3447 LLVMValueRef call_result = LLVMBuildCall(g->builder, func_ref, params, param_count, "");
3448 LLVMBuildInsertElement(g->builder, LLVMBuildLoad(g->builder, result, ""),
3449 call_result, index_value, "");
3450 }
3451
3452 result = LLVMBuildLoad(g->builder, result, "");
3453 }
3454
3455 // Handle integers of non-pot bitsize by shortening them on the output
3456 if (result_type != wider_type) {
3457 return gen_widen_or_shorten(g, false, wider_type, result_type, result);
3458 }
3459 return result;
3460}
3461
3462static LLVMValueRef gen_soft_float_bin_op(CodeGen *g, LLVMValueRef op1_value, LLVMValueRef op2_value, ZigType *operand_type, IrBinOp op_id) {
3463 uint32_t vector_len = operand_type->id == ZigTypeIdVector ? operand_type->data.vector.len : 0;
3464
3465 LLVMTypeRef return_type = operand_type->llvm_type;
3338 int param_count = 2;3466 int param_count = 2;
3339 const char *func_name;3467
3340 SoftF80Icmp res_icmp = NONE;3468 const char *compiler_rt_type_abbrev = get_compiler_rt_type_abbrev(operand_type);
3469 const char *math_h_type_abbrev = get_math_h_type_abbrev(g, operand_type);
3470
3471 char fn_name[64];
3472 Icmp res_icmp = NONE;
3341 switch (op_id) {3473 switch (op_id) {
3342 case IrBinOpInvalid:3474 case IrBinOpInvalid:
3343 case IrBinOpArrayCat:3475 case IrBinOpArrayCat:
...@@ -3362,152 +3494,129 @@ static LLVMValueRef ir_render_soft_f80_bin_op(CodeGen *g, Stage1Air *executable,...@@ -3362,152 +3494,129 @@ static LLVMValueRef ir_render_soft_f80_bin_op(CodeGen *g, Stage1Air *executable,
3362 zig_unreachable();3494 zig_unreachable();
3363 case IrBinOpCmpEq:3495 case IrBinOpCmpEq:
3364 return_type = g->builtin_types.entry_i32->llvm_type;3496 return_type = g->builtin_types.entry_i32->llvm_type;
3365 func_name = "__eqxf2";3497 sprintf(fn_name, "__eq%sf2", compiler_rt_type_abbrev);
3366 res_icmp = EQ_ZERO;3498 res_icmp = EQ_ZERO;
3367 break;3499 break;
3368 case IrBinOpCmpNotEq:3500 case IrBinOpCmpNotEq:
3369 return_type = g->builtin_types.entry_i32->llvm_type;3501 return_type = g->builtin_types.entry_i32->llvm_type;
3370 func_name = "__nexf2";3502 sprintf(fn_name, "__ne%sf2", compiler_rt_type_abbrev);
3371 res_icmp = NE_ZERO;3503 res_icmp = NE_ZERO;
3372 break;3504 break;
3373 case IrBinOpCmpLessOrEq:3505 case IrBinOpCmpLessOrEq:
3374 return_type = g->builtin_types.entry_i32->llvm_type;3506 return_type = g->builtin_types.entry_i32->llvm_type;
3375 func_name = "__lexf2";3507 sprintf(fn_name, "__le%sf2", compiler_rt_type_abbrev);
3376 res_icmp = LE_ZERO;3508 res_icmp = LE_ZERO;
3377 break;3509 break;
3378 case IrBinOpCmpLessThan:3510 case IrBinOpCmpLessThan:
3379 return_type = g->builtin_types.entry_i32->llvm_type;3511 return_type = g->builtin_types.entry_i32->llvm_type;
3380 func_name = "__lexf2";3512 sprintf(fn_name, "__le%sf2", compiler_rt_type_abbrev);
3381 res_icmp = EQ_NEG;3513 res_icmp = EQ_NEG;
3382 break;3514 break;
3383 case IrBinOpCmpGreaterOrEq:3515 case IrBinOpCmpGreaterOrEq:
3384 return_type = g->builtin_types.entry_i32->llvm_type;3516 return_type = g->builtin_types.entry_i32->llvm_type;
3385 func_name = "__gexf2";3517 sprintf(fn_name, "__ge%sf2", compiler_rt_type_abbrev);
3386 res_icmp = GE_ZERO;3518 res_icmp = GE_ZERO;
3387 break;3519 break;
3388 case IrBinOpCmpGreaterThan:3520 case IrBinOpCmpGreaterThan:
3389 return_type = g->builtin_types.entry_i32->llvm_type;3521 return_type = g->builtin_types.entry_i32->llvm_type;
3390 func_name = "__gexf2";3522 sprintf(fn_name, "__ge%sf2", compiler_rt_type_abbrev);
3391 res_icmp = EQ_ONE;3523 res_icmp = EQ_ONE;
3392 break;3524 break;
3393 case IrBinOpMaximum:3525 case IrBinOpMaximum:
3394 func_name = "__fmaxx";3526 sprintf(fn_name, "fmax%s", math_h_type_abbrev);
3395 break;3527 break;
3396 case IrBinOpMinimum:3528 case IrBinOpMinimum:
3397 func_name = "__fminx";3529 sprintf(fn_name, "fmin%s", math_h_type_abbrev);
3398 break;3530 break;
3399 case IrBinOpMult:3531 case IrBinOpMult:
3400 func_name = "__mulxf3";3532 sprintf(fn_name, "__mul%sf3", compiler_rt_type_abbrev);
3401 break;3533 break;
3402 case IrBinOpAdd:3534 case IrBinOpAdd:
3403 func_name = "__addxf3";3535 sprintf(fn_name, "__add%sf3", compiler_rt_type_abbrev);
3404 break;3536 break;
3405 case IrBinOpSub:3537 case IrBinOpSub:
3406 func_name = "__subxf3";3538 sprintf(fn_name, "__sub%sf3", compiler_rt_type_abbrev);
3407 break;3539 break;
3408 case IrBinOpDivUnspecified:3540 case IrBinOpDivUnspecified:
3409 func_name = "__divxf3";
3410 break;
3411 case IrBinOpDivExact:3541 case IrBinOpDivExact:
3412 func_name = "__divxf3";
3413 div_exact_safety_check = bin_op_instruction->safety_check_on &&
3414 ir_want_runtime_safety(g, &bin_op_instruction->base);
3415 break;
3416 case IrBinOpDivTrunc:3542 case IrBinOpDivTrunc:
3417 param_count = 1;
3418 func_name = "__truncx";
3419 break;
3420 case IrBinOpDivFloor:3543 case IrBinOpDivFloor:
3421 param_count = 1;3544 sprintf(fn_name, "__div%sf3", compiler_rt_type_abbrev);
3422 func_name = "__floorx";
3423 break;3545 break;
3424 case IrBinOpRemRem:3546 case IrBinOpRemRem:
3425 param_count = 1;
3426 func_name = "__remx";
3427 break;
3428 case IrBinOpRemMod:3547 case IrBinOpRemMod:
3429 param_count = 1;3548 sprintf(fn_name, "fmod%s", math_h_type_abbrev);
3430 func_name = "__modx";
3431 break;3549 break;
3432 default:3550 default:
3433 zig_unreachable();3551 zig_unreachable();
3434 }3552 }
34353553
3436 LLVMValueRef func_ref = get_soft_f80_bin_op_func(g, func_name, param_count, return_type);3554 LLVMValueRef func_ref = get_soft_float_fn(g, fn_name, param_count, operand_type->llvm_type, return_type);
34373555
3438 LLVMValueRef result;3556 LLVMValueRef result;
3439 if (vector_len == 0) {3557 if (vector_len == 0) {
3440 LLVMValueRef params[2] = {op1_value, op2_value};3558 LLVMValueRef params[2] = {op1_value, op2_value};
3441 result = LLVMBuildCall(g->builder, func_ref, params, param_count, "");3559 result = LLVMBuildCall(g->builder, func_ref, params, param_count, "");
3442 result = add_f80_icmp(g, result, res_icmp);3560 result = add_icmp(g, result, res_icmp);
3443 } else {3561 } else {
3444 ZigType *alloca_ty = op1->value->type;3562 ZigType *alloca_ty = operand_type;
3445 if (res_icmp != NONE) alloca_ty = get_vector_type(g, vector_len, g->builtin_types.entry_bool);3563 if (res_icmp != NONE) alloca_ty = get_vector_type(g, vector_len, g->builtin_types.entry_bool);
3446 result = build_alloca(g, alloca_ty, "", 0);3564 result = build_alloca(g, alloca_ty, "", 0);
3447 }
3448
3449 LLVMTypeRef usize_ref = g->builtin_types.entry_usize->llvm_type;
3450 for (uint32_t i = 0; i < vector_len; i++) {
3451 LLVMValueRef index_value = LLVMConstInt(usize_ref, i, false);
3452 LLVMValueRef params[2] = {
3453 LLVMBuildExtractElement(g->builder, op1_value, index_value, ""),
3454 LLVMBuildExtractElement(g->builder, op2_value, index_value, ""),
3455 };
3456 LLVMValueRef call_result = LLVMBuildCall(g->builder, func_ref, params, param_count, "");
3457 call_result = add_f80_icmp(g, call_result, res_icmp);
3458 LLVMBuildInsertElement(g->builder, LLVMBuildLoad(g->builder, result, ""),
3459 call_result, index_value, "");
3460 }
3461
3462 if (div_exact_safety_check) {
3463 // Safety check: a / b == floor(a / b)
3464 LLVMValueRef floor_func = get_soft_f80_bin_op_func(g, "__floorx", 1, return_type);
3465 LLVMValueRef eq_func = get_soft_f80_bin_op_func(g, "__eqxf2", 2, g->builtin_types.entry_i32->llvm_type);
3466
3467 LLVMValueRef ok_bit;
3468 if (vector_len == 0) {
3469 LLVMValueRef floored = LLVMBuildCall(g->builder, floor_func, &result, 1, "");
3470
3471 LLVMValueRef params[2] = {result, floored};
3472 ok_bit = LLVMBuildCall(g->builder, eq_func, params, 2, "");
3473 } else {
3474 ZigType *bool_vec_ty = get_vector_type(g, vector_len, g->builtin_types.entry_bool);
3475 ok_bit = build_alloca(g, bool_vec_ty, "", 0);
3476 }
34773565
3566 LLVMTypeRef usize_ref = g->builtin_types.entry_usize->llvm_type;
3478 for (uint32_t i = 0; i < vector_len; i++) {3567 for (uint32_t i = 0; i < vector_len; i++) {
3479 LLVMValueRef index_value = LLVMConstInt(usize_ref, i, false);3568 LLVMValueRef index_value = LLVMConstInt(usize_ref, i, false);
3480 LLVMValueRef div_res = LLVMBuildExtractElement(g->builder,
3481 LLVMBuildLoad(g->builder, result, ""), index_value, "");
3482
3483 LLVMValueRef params[2] = {3569 LLVMValueRef params[2] = {
3484 div_res,3570 LLVMBuildExtractElement(g->builder, op1_value, index_value, ""),
3485 LLVMBuildCall(g->builder, floor_func, &div_res, 1, ""),3571 LLVMBuildExtractElement(g->builder, op2_value, index_value, ""),
3486 };3572 };
3487 LLVMValueRef cmp_res = LLVMBuildCall(g->builder, eq_func, params, 2, "");3573 LLVMValueRef call_result = LLVMBuildCall(g->builder, func_ref, params, param_count, "");
3488 cmp_res = LLVMBuildTrunc(g->builder, cmp_res, g->builtin_types.entry_bool->llvm_type, "");3574 call_result = add_icmp(g, call_result, res_icmp);
3489 LLVMBuildInsertElement(g->builder, LLVMBuildLoad(g->builder, ok_bit, ""),3575 LLVMBuildInsertElement(g->builder, LLVMBuildLoad(g->builder, result, ""),
3490 cmp_res, index_value, "");3576 call_result, index_value, "");
3491 }3577 }
34923578
3493 if (vector_len != 0) {3579 result = LLVMBuildLoad(g->builder, result, "");
3494 ok_bit = ZigLLVMBuildAndReduce(g->builder, LLVMBuildLoad(g->builder, ok_bit, ""));3580 }
3495 }
3496 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactOk");
3497 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactFail");
34983581
3499 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);3582 // Some operations are implemented as compound ops and require us to perform some
3583 // more operations before we obtain the final result
3584 switch (op_id) {
3585 case IrBinOpDivTrunc:
3586 return gen_float_un_op(g, result, operand_type, BuiltinFnIdTrunc);
3587 case IrBinOpDivFloor:
3588 return gen_float_un_op(g, result, operand_type, BuiltinFnIdFloor);
3589 case IrBinOpRemMod:
3590 {
3591 LLVMValueRef b = gen_soft_float_bin_op(g, result, op2_value, operand_type, IrBinOpAdd);
3592 LLVMValueRef wrapped_result = gen_soft_float_bin_op(g, b, op2_value, operand_type, IrBinOpRemRem);
3593 LLVMValueRef zero = LLVMConstNull(operand_type->llvm_type);
3594 LLVMValueRef ltz = gen_soft_float_bin_op(g, op1_value, zero, operand_type, IrBinOpCmpLessThan);
35003595
3501 LLVMPositionBuilderAtEnd(g->builder, fail_block);3596 return LLVMBuildSelect(g->builder, ltz, wrapped_result, result, "");
3502 gen_safety_crash(g, PanicMsgIdExactDivisionRemainder);3597 }
3598 case IrBinOpDivExact:
3599 {
3600 LLVMValueRef floored = gen_float_un_op(g, result, operand_type, BuiltinFnIdFloor);
3601 LLVMValueRef ok_bit = gen_soft_float_bin_op(g, result, floored, operand_type, IrBinOpCmpEq);
3602 if (vector_len != 0) {
3603 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
3604 }
35033605
3504 LLVMPositionBuilderAtEnd(g->builder, ok_block);3606 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactOk");
3505 }3607 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactFail");
3608 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
35063609
3507 if (vector_len != 0) {3610 LLVMPositionBuilderAtEnd(g->builder, fail_block);
3508 result = LLVMBuildLoad(g->builder, result, "");3611 gen_safety_crash(g, PanicMsgIdExactDivisionRemainder);
3612
3613 LLVMPositionBuilderAtEnd(g->builder, ok_block);
3614 }
3615 return result;
3616 default:
3617 return result;
3509 }3618 }
3510 return result;3619 zig_unreachable();
3511}3620}
35123621
3513static LLVMValueRef ir_render_bin_op(CodeGen *g, Stage1Air *executable,3622static LLVMValueRef ir_render_bin_op(CodeGen *g, Stage1Air *executable,
...@@ -3519,8 +3628,13 @@ static LLVMValueRef ir_render_bin_op(CodeGen *g, Stage1Air *executable,...@@ -3519,8 +3628,13 @@ static LLVMValueRef ir_render_bin_op(CodeGen *g, Stage1Air *executable,
35193628
3520 ZigType *operand_type = op1->value->type;3629 ZigType *operand_type = op1->value->type;
3521 ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ? operand_type->data.vector.elem_type : operand_type;3630 ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ? operand_type->data.vector.elem_type : operand_type;
3522 if (scalar_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target)) {3631 if ((scalar_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target)) ||
3523 return ir_render_soft_f80_bin_op(g, executable, bin_op_instruction);3632 (scalar_type == g->builtin_types.entry_f128 && !target_long_double_is_f128(g->zig_target))) {
3633 // LLVM incorrectly lowers the soft float calls for f128 as if they operated on `long double`.
3634 // On some targets this will be incorrect, so we manually lower the call ourselves.
3635 LLVMValueRef op1_value = ir_llvm_value(g, op1);
3636 LLVMValueRef op2_value = ir_llvm_value(g, op2);
3637 return gen_soft_float_bin_op(g, op1_value, op2_value, operand_type, op_id);
3524 }3638 }
35253639
35263640
...@@ -3828,10 +3942,17 @@ static LLVMValueRef ir_render_cast(CodeGen *g, Stage1Air *executable,...@@ -3828,10 +3942,17 @@ static LLVMValueRef ir_render_cast(CodeGen *g, Stage1Air *executable,
3828 }3942 }
3829 case CastOpIntToFloat:3943 case CastOpIntToFloat:
3830 assert(actual_type->id == ZigTypeIdInt);3944 assert(actual_type->id == ZigTypeIdInt);
3831 if (actual_type->data.integral.is_signed) {3945 {
3832 return LLVMBuildSIToFP(g->builder, expr_val, get_llvm_type(g, wanted_type), "");3946 if ((wanted_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target)) ||
3833 } else {3947 (wanted_type == g->builtin_types.entry_f128 && !target_long_double_is_f128(g->zig_target))) {
3834 return LLVMBuildUIToFP(g->builder, expr_val, get_llvm_type(g, wanted_type), "");3948 return gen_soft_int_to_float_op(g, expr_val, actual_type, wanted_type);
3949 } else {
3950 if (actual_type->data.integral.is_signed) {
3951 return LLVMBuildSIToFP(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3952 } else {
3953 return LLVMBuildUIToFP(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3954 }
3955 }
3835 }3956 }
3836 case CastOpFloatToInt: {3957 case CastOpFloatToInt: {
3837 assert(wanted_type->id == ZigTypeIdInt);3958 assert(wanted_type->id == ZigTypeIdInt);
...@@ -3840,18 +3961,28 @@ static LLVMValueRef ir_render_cast(CodeGen *g, Stage1Air *executable,...@@ -3840,18 +3961,28 @@ static LLVMValueRef ir_render_cast(CodeGen *g, Stage1Air *executable,
3840 bool want_safety = ir_want_runtime_safety(g, &cast_instruction->base);3961 bool want_safety = ir_want_runtime_safety(g, &cast_instruction->base);
38413962
3842 LLVMValueRef result;3963 LLVMValueRef result;
3843 if (wanted_type->data.integral.is_signed) {3964 if ((actual_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target)) ||
3844 result = LLVMBuildFPToSI(g->builder, expr_val, get_llvm_type(g, wanted_type), "");3965 (actual_type == g->builtin_types.entry_f128 && !target_long_double_is_f128(g->zig_target))) {
3966 result = gen_soft_float_to_int_op(g, expr_val, actual_type, wanted_type);
3845 } else {3967 } else {
3846 result = LLVMBuildFPToUI(g->builder, expr_val, get_llvm_type(g, wanted_type), "");3968 if (wanted_type->data.integral.is_signed) {
3969 result = LLVMBuildFPToSI(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3970 } else {
3971 result = LLVMBuildFPToUI(g->builder, expr_val, get_llvm_type(g, wanted_type), "");
3972 }
3847 }3973 }
38483974
3849 if (want_safety) {3975 if (want_safety) {
3850 LLVMValueRef back_to_float;3976 LLVMValueRef back_to_float;
3851 if (wanted_type->data.integral.is_signed) {3977 if ((actual_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target)) ||
3852 back_to_float = LLVMBuildSIToFP(g->builder, result, LLVMTypeOf(expr_val), "");3978 (actual_type == g->builtin_types.entry_f128 && !target_long_double_is_f128(g->zig_target))) {
3979 back_to_float = gen_soft_int_to_float_op(g, result, wanted_type, actual_type);
3853 } else {3980 } else {
3854 back_to_float = LLVMBuildUIToFP(g->builder, result, LLVMTypeOf(expr_val), "");3981 if (wanted_type->data.integral.is_signed) {
3982 back_to_float = LLVMBuildSIToFP(g->builder, result, LLVMTypeOf(expr_val), "");
3983 } else {
3984 back_to_float = LLVMBuildUIToFP(g->builder, result, LLVMTypeOf(expr_val), "");
3985 }
3855 }3986 }
3856 LLVMValueRef difference = LLVMBuildFSub(g->builder, expr_val, back_to_float, "");3987 LLVMValueRef difference = LLVMBuildFSub(g->builder, expr_val, back_to_float, "");
3857 LLVMValueRef one_pos = LLVMConstReal(LLVMTypeOf(expr_val), 1.0f);3988 LLVMValueRef one_pos = LLVMConstReal(LLVMTypeOf(expr_val), 1.0f);
...@@ -4151,42 +4282,46 @@ static LLVMValueRef ir_render_binary_not(CodeGen *g, Stage1Air *executable,...@@ -4151,42 +4282,46 @@ static LLVMValueRef ir_render_binary_not(CodeGen *g, Stage1Air *executable,
4151 return LLVMBuildNot(g->builder, operand, "");4282 return LLVMBuildNot(g->builder, operand, "");
4152}4283}
41534284
4154static LLVMValueRef ir_gen_soft_f80_neg(CodeGen *g, ZigType *op_type, LLVMValueRef operand) {4285static LLVMValueRef gen_soft_float_neg(CodeGen *g, ZigType *operand_type, LLVMValueRef operand) {
4155 uint32_t vector_len = op_type->id == ZigTypeIdVector ? op_type->data.vector.len : 0;4286 uint32_t vector_len = operand_type->id == ZigTypeIdVector ? operand_type->data.vector.len : 0;
4287 uint16_t num_bits = operand_type->data.floating.bit_count;
41564288
4157 LLVMTypeRef llvm_i80 = LLVMIntType(80);4289 ZigType *iX_type = get_int_type(g, true, num_bits);
4158 LLVMValueRef sign_mask = LLVMConstInt(llvm_i80, 1, false);4290 LLVMValueRef sign_mask = LLVMConstInt(iX_type->llvm_type, 1, false);
4159 sign_mask = LLVMConstShl(sign_mask, LLVMConstInt(llvm_i80, 79, false));4291 sign_mask = LLVMConstShl(sign_mask, LLVMConstInt(iX_type->llvm_type, num_bits - 1, false));
41604292
4161 LLVMValueRef result;
4162 if (vector_len == 0) {4293 if (vector_len == 0) {
4163 result = LLVMBuildXor(g->builder, operand, sign_mask, "");4294 LLVMValueRef bitcasted_operand = LLVMBuildBitCast(g->builder, operand, iX_type->llvm_type, "");
4295 LLVMValueRef result = LLVMBuildXor(g->builder, bitcasted_operand, sign_mask, "");
4296
4297 return LLVMBuildBitCast(g->builder, result, operand_type->llvm_type, "");
4164 } else {4298 } else {
4165 result = build_alloca(g, op_type, "", 0);4299 LLVMTypeRef usize_ref = g->builtin_types.entry_usize->llvm_type;
4166 }4300 ZigType *iX_vector_type = get_vector_type(g, vector_len, iX_type);
41674301
4168 LLVMTypeRef usize_ref = g->builtin_types.entry_usize->llvm_type;4302 LLVMValueRef result = build_alloca(g, iX_vector_type, "", 0);
4169 for (uint32_t i = 0; i < vector_len; i++) {4303 LLVMValueRef bitcasted_operand = LLVMBuildBitCast(g->builder, operand, iX_vector_type->llvm_type, "");
4170 LLVMValueRef index_value = LLVMConstInt(usize_ref, i, false);4304 for (uint32_t i = 0; i < vector_len; i++) {
4171 LLVMValueRef xor_operand = LLVMBuildExtractElement(g->builder, operand, index_value, "");4305 LLVMValueRef index_value = LLVMConstInt(usize_ref, i, false);
4172 LLVMValueRef xor_result = LLVMBuildXor(g->builder, xor_operand, sign_mask, "");4306 LLVMValueRef elem = LLVMBuildExtractElement(g->builder, bitcasted_operand, index_value, "");
4173 LLVMBuildInsertElement(g->builder, LLVMBuildLoad(g->builder, result, ""),4307 LLVMValueRef result_elem = LLVMBuildXor(g->builder, elem, sign_mask, "");
4174 xor_result, index_value, "");4308 LLVMBuildInsertElement(g->builder, LLVMBuildLoad(g->builder, result, ""),
4175 }4309 result_elem, index_value, "");
4176 if (vector_len != 0) {4310 }
4177 result = LLVMBuildLoad(g->builder, result, "");4311 return LLVMBuildBitCast(g->builder, LLVMBuildLoad(g->builder, result, ""), operand_type->llvm_type, "");
4178 }4312 }
4179 return result;
4180}4313}
41814314
4182static LLVMValueRef ir_gen_negation(CodeGen *g, Stage1AirInst *inst, Stage1AirInst *operand, bool wrapping) {4315static LLVMValueRef gen_negation(CodeGen *g, Stage1AirInst *inst, Stage1AirInst *operand, bool wrapping) {
4183 LLVMValueRef llvm_operand = ir_llvm_value(g, operand);4316 LLVMValueRef llvm_operand = ir_llvm_value(g, operand);
4184 ZigType *operand_type = operand->value->type;4317 ZigType *operand_type = operand->value->type;
4185 ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ?4318 ZigType *scalar_type = (operand_type->id == ZigTypeIdVector) ?
4186 operand_type->data.vector.elem_type : operand_type;4319 operand_type->data.vector.elem_type : operand_type;
41874320
4188 if (scalar_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target))4321 if ((scalar_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target)) ||
4189 return ir_gen_soft_f80_neg(g, operand_type, llvm_operand);4322 (scalar_type == g->builtin_types.entry_f128 && !target_long_double_is_f128(g->zig_target))) {
4323 return gen_soft_float_neg(g, operand_type, llvm_operand);
4324 }
41904325
4191 if (scalar_type->id == ZigTypeIdFloat) {4326 if (scalar_type->id == ZigTypeIdFloat) {
4192 ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, inst));4327 ZigLLVMSetFastMath(g->builder, ir_want_fast_math(g, inst));
...@@ -4210,7 +4345,7 @@ static LLVMValueRef ir_gen_negation(CodeGen *g, Stage1AirInst *inst, Stage1AirIn...@@ -4210,7 +4345,7 @@ static LLVMValueRef ir_gen_negation(CodeGen *g, Stage1AirInst *inst, Stage1AirIn
4210static LLVMValueRef ir_render_negation(CodeGen *g, Stage1Air *executable,4345static LLVMValueRef ir_render_negation(CodeGen *g, Stage1Air *executable,
4211 Stage1AirInstNegation *inst)4346 Stage1AirInstNegation *inst)
4212{4347{
4213 return ir_gen_negation(g, &inst->base, inst->operand, inst->wrapping);4348 return gen_negation(g, &inst->base, inst->operand, inst->wrapping);
4214}4349}
42154350
4216static LLVMValueRef ir_render_bool_not(CodeGen *g, Stage1Air *executable, Stage1AirInstBoolNot *instruction) {4351static LLVMValueRef ir_render_bool_not(CodeGen *g, Stage1Air *executable, Stage1AirInstBoolNot *instruction) {
...@@ -7024,110 +7159,34 @@ static LLVMValueRef ir_render_atomic_store(CodeGen *g, Stage1Air *executable,...@@ -7024,110 +7159,34 @@ static LLVMValueRef ir_render_atomic_store(CodeGen *g, Stage1Air *executable,
7024 return nullptr;7159 return nullptr;
7025}7160}
70267161
7027static LLVMValueRef ir_render_soft_f80_float_op(CodeGen *g, Stage1Air *executable, Stage1AirInstFloatOp *instruction) {
7028 ZigType *op_type = instruction->operand->value->type;
7029 uint32_t vector_len = op_type->id == ZigTypeIdVector ? op_type->data.vector.len : 0;
7030
7031 const char *func_name;
7032 switch (instruction->fn_id) {
7033 case BuiltinFnIdSqrt:
7034 func_name = "__sqrtx";
7035 break;
7036 case BuiltinFnIdSin:
7037 func_name = "__sinx";
7038 break;
7039 case BuiltinFnIdCos:
7040 func_name = "__cosx";
7041 break;
7042 case BuiltinFnIdExp:
7043 func_name = "__expx";
7044 break;
7045 case BuiltinFnIdExp2:
7046 func_name = "__exp2x";
7047 break;
7048 case BuiltinFnIdLog:
7049 func_name = "__logx";
7050 break;
7051 case BuiltinFnIdLog2:
7052 func_name = "__log2x";
7053 break;
7054 case BuiltinFnIdLog10:
7055 func_name = "__log10x";
7056 break;
7057 case BuiltinFnIdFabs:
7058 func_name = "__fabsx";
7059 break;
7060 case BuiltinFnIdFloor:
7061 func_name = "__floorx";
7062 break;
7063 case BuiltinFnIdCeil:
7064 func_name = "__ceilx";
7065 break;
7066 case BuiltinFnIdTrunc:
7067 func_name = "__truncx";
7068 break;
7069 case BuiltinFnIdNearbyInt:
7070 func_name = "__nearbyintx";
7071 break;
7072 case BuiltinFnIdRound:
7073 func_name = "__roundx";
7074 break;
7075 default:
7076 zig_unreachable();
7077 }
7078
7079
7080 LLVMValueRef func_ref = LLVMGetNamedFunction(g->module, func_name);
7081 if (func_ref == nullptr) {
7082 LLVMTypeRef f80_ref = g->builtin_types.entry_f80->llvm_type;
7083 LLVMTypeRef fn_type = LLVMFunctionType(f80_ref, &f80_ref, 1, false);
7084 func_ref = LLVMAddFunction(g->module, func_name, fn_type);
7085 }
7086
7087 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
7088 LLVMValueRef result;
7089 if (vector_len == 0) {
7090 result = LLVMBuildCall(g->builder, func_ref, &operand, 1, "");
7091 } else {
7092 result = build_alloca(g, instruction->operand->value->type, "", 0);
7093 }
7094
7095 LLVMTypeRef usize_ref = g->builtin_types.entry_usize->llvm_type;
7096 for (uint32_t i = 0; i < vector_len; i++) {
7097 LLVMValueRef index_value = LLVMConstInt(usize_ref, i, false);
7098 LLVMValueRef param = LLVMBuildExtractElement(g->builder, operand, index_value, "");
7099 LLVMValueRef call_result = LLVMBuildCall(g->builder, func_ref, &param, 1, "");
7100 LLVMBuildInsertElement(g->builder, LLVMBuildLoad(g->builder, result, ""),
7101 call_result, index_value, "");
7102 }
7103 if (vector_len != 0) {
7104 result = LLVMBuildLoad(g->builder, result, "");
7105 }
7106 return result;
7107}
7108
7109static LLVMValueRef ir_render_float_op(CodeGen *g, Stage1Air *executable, Stage1AirInstFloatOp *instruction) {7162static LLVMValueRef ir_render_float_op(CodeGen *g, Stage1Air *executable, Stage1AirInstFloatOp *instruction) {
7110 ZigType *op_type = instruction->operand->value->type;
7111 op_type = op_type->id == ZigTypeIdVector ? op_type->data.vector.elem_type : op_type;
7112 if (op_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target)) {
7113 return ir_render_soft_f80_float_op(g, executable, instruction);
7114 }
7115 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);7163 LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
7116 LLVMValueRef fn_val = get_float_fn(g, instruction->base.value->type, ZigLLVMFnIdFloatOp, instruction->fn_id);7164 ZigType *operand_type = instruction->operand->value->type;
7117 return LLVMBuildCall(g->builder, fn_val, &operand, 1, "");7165 return gen_float_un_op(g, operand, operand_type, instruction->fn_id);
7118}7166}
71197167
7120static LLVMValueRef ir_render_soft_f80_mul_add(CodeGen *g, Stage1Air *executable, Stage1AirInstMulAdd *instruction) {7168static LLVMValueRef ir_render_soft_mul_add(CodeGen *g, Stage1Air *executable, Stage1AirInstMulAdd *instruction, ZigType *float_type) {
7121 ZigType *op_type = instruction->op1->value->type;7169 ZigType *operand_type = instruction->op1->value->type;
7122 uint32_t vector_len = op_type->id == ZigTypeIdVector ? op_type->data.vector.len : 0;7170 uint32_t vector_len = operand_type->id == ZigTypeIdVector ? operand_type->data.vector.len : 0;
7171
7172 const char *fn_name;
7173 if (float_type == g->builtin_types.entry_f32)
7174 fn_name = "fmaf";
7175 else if (float_type == g->builtin_types.entry_f64)
7176 fn_name = "fma";
7177 else if (float_type == g->builtin_types.entry_f80)
7178 fn_name = "__fmax";
7179 else if (float_type == g->builtin_types.entry_f128)
7180 fn_name = "fmaq";
7181 else
7182 zig_unreachable();
71237183
7124 const char *func_name = "__fmax";7184 LLVMValueRef func_ref = LLVMGetNamedFunction(g->module, fn_name);
7125 LLVMValueRef func_ref = LLVMGetNamedFunction(g->module, func_name);
7126 if (func_ref == nullptr) {7185 if (func_ref == nullptr) {
7127 LLVMTypeRef f80_ref = g->builtin_types.entry_f80->llvm_type;7186 LLVMTypeRef float_type_ref = float_type->llvm_type;
7128 LLVMTypeRef params[3] = { f80_ref, f80_ref, f80_ref };7187 LLVMTypeRef params[3] = { float_type_ref, float_type_ref, float_type_ref };
7129 LLVMTypeRef fn_type = LLVMFunctionType(f80_ref, params, 3, false);7188 LLVMTypeRef fn_type = LLVMFunctionType(float_type_ref, params, 3, false);
7130 func_ref = LLVMAddFunction(g->module, func_name, fn_type);7189 func_ref = LLVMAddFunction(g->module, fn_name, fn_type);
7131 }7190 }
71327191
7133 LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);7192 LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);
...@@ -7161,10 +7220,11 @@ static LLVMValueRef ir_render_soft_f80_mul_add(CodeGen *g, Stage1Air *executable...@@ -7161,10 +7220,11 @@ static LLVMValueRef ir_render_soft_f80_mul_add(CodeGen *g, Stage1Air *executable
7161}7220}
71627221
7163static LLVMValueRef ir_render_mul_add(CodeGen *g, Stage1Air *executable, Stage1AirInstMulAdd *instruction) {7222static LLVMValueRef ir_render_mul_add(CodeGen *g, Stage1Air *executable, Stage1AirInstMulAdd *instruction) {
7164 ZigType *op_type = instruction->op1->value->type;7223 ZigType *operand_type = instruction->op1->value->type;
7165 op_type = op_type->id == ZigTypeIdVector ? op_type->data.vector.elem_type : op_type;7224 operand_type = operand_type->id == ZigTypeIdVector ? operand_type->data.vector.elem_type : operand_type;
7166 if (op_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target)) {7225 if ((operand_type == g->builtin_types.entry_f80 && !target_has_f80(g->zig_target)) ||
7167 return ir_render_soft_f80_mul_add(g, executable, instruction);7226 (operand_type == g->builtin_types.entry_f128 && !target_long_double_is_f128(g->zig_target))) {
7227 return ir_render_soft_mul_add(g, executable, instruction, operand_type);
7168 }7228 }
7169 LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);7229 LLVMValueRef op1 = ir_llvm_value(g, instruction->op1);
7170 LLVMValueRef op2 = ir_llvm_value(g, instruction->op2);7230 LLVMValueRef op2 = ir_llvm_value(g, instruction->op2);
src/stage1/ir.cpp+2-2
...@@ -24300,7 +24300,7 @@ static ErrorMsg *ir_eval_float_op(IrAnalyze *ira, Scope *scope, AstNode *source_...@@ -24300,7 +24300,7 @@ static ErrorMsg *ir_eval_float_op(IrAnalyze *ira, Scope *scope, AstNode *source_
24300 case BuiltinFnIdLog2:24300 case BuiltinFnIdLog2:
24301 return ir_add_error_node(ira, source_node,24301 return ir_add_error_node(ira, source_node,
24302 buf_sprintf("compiler bug: TODO: implement '%s' for type '%s'. See https://github.com/ziglang/zig/issues/4026",24302 buf_sprintf("compiler bug: TODO: implement '%s' for type '%s'. See https://github.com/ziglang/zig/issues/4026",
24303 float_op_to_name(fop), buf_ptr(&float_type->name)));24303 float_un_op_to_name(fop), buf_ptr(&float_type->name)));
24304 default:24304 default:
24305 zig_unreachable();24305 zig_unreachable();
24306 }24306 }
...@@ -24344,7 +24344,7 @@ static ErrorMsg *ir_eval_float_op(IrAnalyze *ira, Scope *scope, AstNode *source_...@@ -24344,7 +24344,7 @@ static ErrorMsg *ir_eval_float_op(IrAnalyze *ira, Scope *scope, AstNode *source_
24344 case BuiltinFnIdLog2:24344 case BuiltinFnIdLog2:
24345 return ir_add_error_node(ira, source_node,24345 return ir_add_error_node(ira, source_node,
24346 buf_sprintf("compiler bug: TODO: implement '%s' for type '%s'. See https://github.com/ziglang/zig/issues/4026",24346 buf_sprintf("compiler bug: TODO: implement '%s' for type '%s'. See https://github.com/ziglang/zig/issues/4026",
24347 float_op_to_name(fop), buf_ptr(&float_type->name)));24347 float_un_op_to_name(fop), buf_ptr(&float_type->name)));
24348 default:24348 default:
24349 zig_unreachable();24349 zig_unreachable();
24350 }24350 }
src/stage1/ir_print.cpp+2-2
...@@ -2558,13 +2558,13 @@ static void ir_print_add_implicit_return_type(IrPrintSrc *irp, Stage1ZirInstAddI...@@ -2558,13 +2558,13 @@ static void ir_print_add_implicit_return_type(IrPrintSrc *irp, Stage1ZirInstAddI
2558}2558}
25592559
2560static void ir_print_float_op(IrPrintSrc *irp, Stage1ZirInstFloatOp *instruction) {2560static void ir_print_float_op(IrPrintSrc *irp, Stage1ZirInstFloatOp *instruction) {
2561 fprintf(irp->f, "@%s(", float_op_to_name(instruction->fn_id));2561 fprintf(irp->f, "@%s(", float_un_op_to_name(instruction->fn_id));
2562 ir_print_other_inst_src(irp, instruction->operand);2562 ir_print_other_inst_src(irp, instruction->operand);
2563 fprintf(irp->f, ")");2563 fprintf(irp->f, ")");
2564}2564}
25652565
2566static void ir_print_float_op(IrPrintGen *irp, Stage1AirInstFloatOp *instruction) {2566static void ir_print_float_op(IrPrintGen *irp, Stage1AirInstFloatOp *instruction) {
2567 fprintf(irp->f, "@%s(", float_op_to_name(instruction->fn_id));2567 fprintf(irp->f, "@%s(", float_un_op_to_name(instruction->fn_id));
2568 ir_print_other_inst_gen(irp, instruction->operand);2568 ir_print_other_inst_gen(irp, instruction->operand);
2569 fprintf(irp->f, ")");2569 fprintf(irp->f, ")");
2570}2570}