authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-10-04 18:23:52+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-10-05 04:51:45-04:00
log22b5e47839cf34c1e4a7c5e6dc256e041b4bf8fc
treea940511e3d881231d60276f824a6a9055877dc83
parent7c5a24e08cd0bffd2a5cce6d1fd592a7d2bee678

stage1: Implement @reduce builtin for vector types

The builtin folds a Vector(N,T) into a scalar T using a specified operator. Closes #2698

8 files changed, 430 insertions(+), 39 deletions(-)

lib/std/builtin.zig+10
......@@ -98,6 +98,16 @@ pub const AtomicOrder = enum {
9898 SeqCst,
9999};
100100
101/// This data structure is used by the Zig language code generation and
102/// therefore must be kept in sync with the compiler implementation.
103pub const ReduceOp = enum {
104 And,
105 Or,
106 Xor,
107 Min,
108 Max,
109};
110
101111/// This data structure is used by the Zig language code generation and
102112/// therefore must be kept in sync with the compiler implementation.
103113pub const AtomicRmwOp = enum {
src/stage1/all_types.hpp+26
......@@ -1821,6 +1821,7 @@ enum BuiltinFnId {
18211821 BuiltinFnIdWasmMemorySize,
18221822 BuiltinFnIdWasmMemoryGrow,
18231823 BuiltinFnIdSrc,
1824 BuiltinFnIdReduce,
18241825};
18251826
18261827struct BuiltinFnEntry {
......@@ -2436,6 +2437,15 @@ enum AtomicOrder {
24362437 AtomicOrderSeqCst,
24372438};
24382439
2440// synchronized with code in define_builtin_compile_vars
2441enum ReduceOp {
2442 ReduceOp_and,
2443 ReduceOp_or,
2444 ReduceOp_xor,
2445 ReduceOp_min,
2446 ReduceOp_max,
2447};
2448
24392449// synchronized with the code in define_builtin_compile_vars
24402450enum AtomicRmwOp {
24412451 AtomicRmwOp_xchg,
......@@ -2545,6 +2555,7 @@ enum IrInstSrcId {
25452555 IrInstSrcIdEmbedFile,
25462556 IrInstSrcIdCmpxchg,
25472557 IrInstSrcIdFence,
2558 IrInstSrcIdReduce,
25482559 IrInstSrcIdTruncate,
25492560 IrInstSrcIdIntCast,
25502561 IrInstSrcIdFloatCast,
......@@ -2667,6 +2678,7 @@ enum IrInstGenId {
26672678 IrInstGenIdErrName,
26682679 IrInstGenIdCmpxchg,
26692680 IrInstGenIdFence,
2681 IrInstGenIdReduce,
26702682 IrInstGenIdTruncate,
26712683 IrInstGenIdShuffleVector,
26722684 IrInstGenIdSplat,
......@@ -3516,6 +3528,20 @@ struct IrInstGenFence {
35163528 AtomicOrder order;
35173529};
35183530
3531struct IrInstSrcReduce {
3532 IrInstSrc base;
3533
3534 IrInstSrc *op;
3535 IrInstSrc *value;
3536};
3537
3538struct IrInstGenReduce {
3539 IrInstGen base;
3540
3541 ReduceOp op;
3542 IrInstGen *value;
3543};
3544
35193545struct IrInstSrcTruncate {
35203546 IrInstSrc base;
35213547
src/stage1/codegen.cpp+56-39
......@@ -2583,36 +2583,6 @@ static LLVMValueRef ir_render_return(CodeGen *g, IrExecutableGen *executable, Ir
25832583 return nullptr;
25842584}
25852585
2586enum class ScalarizePredicate {
2587 // Returns true iff all the elements in the vector are 1.
2588 // Equivalent to folding all the bits with `and`.
2589 All,
2590 // Returns true iff there's at least one element in the vector that is 1.
2591 // Equivalent to folding all the bits with `or`.
2592 Any,
2593};
2594
2595// Collapses a <N x i1> vector into a single i1 according to the given predicate
2596static LLVMValueRef scalarize_cmp_result(CodeGen *g, LLVMValueRef val, ScalarizePredicate predicate) {
2597 assert(LLVMGetTypeKind(LLVMTypeOf(val)) == LLVMVectorTypeKind);
2598 LLVMTypeRef scalar_type = LLVMIntType(LLVMGetVectorSize(LLVMTypeOf(val)));
2599 LLVMValueRef casted = LLVMBuildBitCast(g->builder, val, scalar_type, "");
2600
2601 switch (predicate) {
2602 case ScalarizePredicate::Any: {
2603 LLVMValueRef all_zeros = LLVMConstNull(scalar_type);
2604 return LLVMBuildICmp(g->builder, LLVMIntNE, casted, all_zeros, "");
2605 }
2606 case ScalarizePredicate::All: {
2607 LLVMValueRef all_ones = LLVMConstAllOnes(scalar_type);
2608 return LLVMBuildICmp(g->builder, LLVMIntEQ, casted, all_ones, "");
2609 }
2610 }
2611
2612 zig_unreachable();
2613}
2614
2615
26162586static LLVMValueRef gen_overflow_shl_op(CodeGen *g, ZigType *operand_type,
26172587 LLVMValueRef val1, LLVMValueRef val2)
26182588{
......@@ -2637,7 +2607,7 @@ static LLVMValueRef gen_overflow_shl_op(CodeGen *g, ZigType *operand_type,
26372607 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowOk");
26382608 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowFail");
26392609 if (operand_type->id == ZigTypeIdVector) {
2640 ok_bit = scalarize_cmp_result(g, ok_bit, ScalarizePredicate::All);
2610 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
26412611 }
26422612 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
26432613
......@@ -2668,7 +2638,7 @@ static LLVMValueRef gen_overflow_shr_op(CodeGen *g, ZigType *operand_type,
26682638 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowOk");
26692639 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "OverflowFail");
26702640 if (operand_type->id == ZigTypeIdVector) {
2671 ok_bit = scalarize_cmp_result(g, ok_bit, ScalarizePredicate::All);
2641 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
26722642 }
26732643 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
26742644
......@@ -2745,7 +2715,7 @@ static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast
27452715 }
27462716
27472717 if (operand_type->id == ZigTypeIdVector) {
2748 is_zero_bit = scalarize_cmp_result(g, is_zero_bit, ScalarizePredicate::Any);
2718 is_zero_bit = ZigLLVMBuildOrReduce(g->builder, is_zero_bit);
27492719 }
27502720
27512721 LLVMBasicBlockRef div_zero_fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivZeroFail");
......@@ -2770,7 +2740,7 @@ static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast
27702740 LLVMValueRef den_is_neg_1 = LLVMBuildICmp(g->builder, LLVMIntEQ, val2, neg_1_value, "");
27712741 LLVMValueRef overflow_fail_bit = LLVMBuildAnd(g->builder, num_is_int_min, den_is_neg_1, "");
27722742 if (operand_type->id == ZigTypeIdVector) {
2773 overflow_fail_bit = scalarize_cmp_result(g, overflow_fail_bit, ScalarizePredicate::Any);
2743 overflow_fail_bit = ZigLLVMBuildOrReduce(g->builder, overflow_fail_bit);
27742744 }
27752745 LLVMBuildCondBr(g->builder, overflow_fail_bit, overflow_fail_block, overflow_ok_block);
27762746
......@@ -2795,7 +2765,7 @@ static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast
27952765 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactFail");
27962766 LLVMValueRef ok_bit = LLVMBuildFCmp(g->builder, LLVMRealOEQ, floored, result, "");
27972767 if (operand_type->id == ZigTypeIdVector) {
2798 ok_bit = scalarize_cmp_result(g, ok_bit, ScalarizePredicate::All);
2768 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
27992769 }
28002770 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
28012771
......@@ -2812,7 +2782,7 @@ static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast
28122782 LLVMBasicBlockRef end_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivTruncEnd");
28132783 LLVMValueRef ltz = LLVMBuildFCmp(g->builder, LLVMRealOLT, val1, zero, "");
28142784 if (operand_type->id == ZigTypeIdVector) {
2815 ltz = scalarize_cmp_result(g, ltz, ScalarizePredicate::Any);
2785 ltz = ZigLLVMBuildOrReduce(g->builder, ltz);
28162786 }
28172787 LLVMBuildCondBr(g->builder, ltz, ltz_block, gez_block);
28182788
......@@ -2864,7 +2834,7 @@ static LLVMValueRef gen_div(CodeGen *g, bool want_runtime_safety, bool want_fast
28642834 LLVMBasicBlockRef fail_block = LLVMAppendBasicBlock(g->cur_fn_val, "DivExactFail");
28652835 LLVMValueRef ok_bit = LLVMBuildICmp(g->builder, LLVMIntEQ, remainder_val, zero, "");
28662836 if (operand_type->id == ZigTypeIdVector) {
2867 ok_bit = scalarize_cmp_result(g, ok_bit, ScalarizePredicate::All);
2837 ok_bit = ZigLLVMBuildAndReduce(g->builder, ok_bit);
28682838 }
28692839 LLVMBuildCondBr(g->builder, ok_bit, ok_block, fail_block);
28702840
......@@ -2928,7 +2898,7 @@ static LLVMValueRef gen_rem(CodeGen *g, bool want_runtime_safety, bool want_fast
29282898 }
29292899
29302900 if (operand_type->id == ZigTypeIdVector) {
2931 is_zero_bit = scalarize_cmp_result(g, is_zero_bit, ScalarizePredicate::Any);
2901 is_zero_bit = ZigLLVMBuildOrReduce(g->builder, is_zero_bit);
29322902 }
29332903
29342904 LLVMBasicBlockRef rem_zero_ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "RemZeroOk");
......@@ -2985,7 +2955,7 @@ static void gen_shift_rhs_check(CodeGen *g, ZigType *lhs_type, ZigType *rhs_type
29852955 LLVMBasicBlockRef ok_block = LLVMAppendBasicBlock(g->cur_fn_val, "CheckOk");
29862956 LLVMValueRef less_than_bit = LLVMBuildICmp(g->builder, LLVMIntULT, value, bit_count_value, "");
29872957 if (rhs_type->id == ZigTypeIdVector) {
2988 less_than_bit = scalarize_cmp_result(g, less_than_bit, ScalarizePredicate::Any);
2958 less_than_bit = ZigLLVMBuildOrReduce(g->builder, less_than_bit);
29892959 }
29902960 LLVMBuildCondBr(g->builder, less_than_bit, ok_block, fail_block);
29912961
......@@ -5470,6 +5440,50 @@ static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutableGen *executable, I
54705440 return result_loc;
54715441}
54725442
5443static LLVMValueRef ir_render_reduce(CodeGen *g, IrExecutableGen *executable, IrInstGenReduce *instruction) {
5444 LLVMValueRef value = ir_llvm_value(g, instruction->value);
5445
5446 ZigType *value_type = instruction->value->value->type;
5447 assert(value_type->id == ZigTypeIdVector);
5448 ZigType *scalar_type = value_type->data.vector.elem_type;
5449
5450 LLVMValueRef result_val;
5451 switch (instruction->op) {
5452 case ReduceOp_and:
5453 assert(scalar_type->id == ZigTypeIdInt || scalar_type->id == ZigTypeIdBool);
5454 result_val = ZigLLVMBuildAndReduce(g->builder, value);
5455 break;
5456 case ReduceOp_or:
5457 assert(scalar_type->id == ZigTypeIdInt || scalar_type->id == ZigTypeIdBool);
5458 result_val = ZigLLVMBuildOrReduce(g->builder, value);
5459 break;
5460 case ReduceOp_xor:
5461 assert(scalar_type->id == ZigTypeIdInt || scalar_type->id == ZigTypeIdBool);
5462 result_val = ZigLLVMBuildXorReduce(g->builder, value);
5463 break;
5464 case ReduceOp_min: {
5465 if (scalar_type->id == ZigTypeIdInt) {
5466 const bool is_signed = scalar_type->data.integral.is_signed;
5467 result_val = ZigLLVMBuildIntMinReduce(g->builder, value, is_signed);
5468 } else if (scalar_type->id == ZigTypeIdFloat) {
5469 result_val = ZigLLVMBuildFPMinReduce(g->builder, value);
5470 } else zig_unreachable();
5471 } break;
5472 case ReduceOp_max: {
5473 if (scalar_type->id == ZigTypeIdInt) {
5474 const bool is_signed = scalar_type->data.integral.is_signed;
5475 result_val = ZigLLVMBuildIntMaxReduce(g->builder, value, is_signed);
5476 } else if (scalar_type->id == ZigTypeIdFloat) {
5477 result_val = ZigLLVMBuildFPMaxReduce(g->builder, value);
5478 } else zig_unreachable();
5479 } break;
5480 default:
5481 zig_unreachable();
5482 }
5483
5484 return result_val;
5485}
5486
54735487static LLVMValueRef ir_render_fence(CodeGen *g, IrExecutableGen *executable, IrInstGenFence *instruction) {
54745488 LLVMAtomicOrdering atomic_order = to_LLVMAtomicOrdering(instruction->order);
54755489 LLVMBuildFence(g->builder, atomic_order, false, "");
......@@ -6674,6 +6688,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, IrExecutableGen *executabl
66746688 return ir_render_cmpxchg(g, executable, (IrInstGenCmpxchg *)instruction);
66756689 case IrInstGenIdFence:
66766690 return ir_render_fence(g, executable, (IrInstGenFence *)instruction);
6691 case IrInstGenIdReduce:
6692 return ir_render_reduce(g, executable, (IrInstGenReduce *)instruction);
66776693 case IrInstGenIdTruncate:
66786694 return ir_render_truncate(g, executable, (IrInstGenTruncate *)instruction);
66796695 case IrInstGenIdBoolNot:
......@@ -8630,6 +8646,7 @@ static void define_builtin_fns(CodeGen *g) {
86308646 create_builtin_fn(g, BuiltinFnIdWasmMemorySize, "wasmMemorySize", 1);
86318647 create_builtin_fn(g, BuiltinFnIdWasmMemoryGrow, "wasmMemoryGrow", 2);
86328648 create_builtin_fn(g, BuiltinFnIdSrc, "src", 0);
8649 create_builtin_fn(g, BuiltinFnIdReduce, "reduce", 2);
86338650}
86348651
86358652static const char *bool_to_str(bool b) {
src/stage1/ir.cpp+227
......@@ -402,6 +402,8 @@ static void destroy_instruction_src(IrInstSrc *inst) {
402402 return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcCmpxchg *>(inst));
403403 case IrInstSrcIdFence:
404404 return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcFence *>(inst));
405 case IrInstSrcIdReduce:
406 return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcReduce *>(inst));
405407 case IrInstSrcIdTruncate:
406408 return heap::c_allocator.destroy(reinterpret_cast<IrInstSrcTruncate *>(inst));
407409 case IrInstSrcIdIntCast:
......@@ -636,6 +638,8 @@ void destroy_instruction_gen(IrInstGen *inst) {
636638 return heap::c_allocator.destroy(reinterpret_cast<IrInstGenCmpxchg *>(inst));
637639 case IrInstGenIdFence:
638640 return heap::c_allocator.destroy(reinterpret_cast<IrInstGenFence *>(inst));
641 case IrInstGenIdReduce:
642 return heap::c_allocator.destroy(reinterpret_cast<IrInstGenReduce *>(inst));
639643 case IrInstGenIdTruncate:
640644 return heap::c_allocator.destroy(reinterpret_cast<IrInstGenTruncate *>(inst));
641645 case IrInstGenIdShuffleVector:
......@@ -1311,6 +1315,10 @@ static constexpr IrInstSrcId ir_inst_id(IrInstSrcFence *) {
13111315 return IrInstSrcIdFence;
13121316}
13131317
1318static constexpr IrInstSrcId ir_inst_id(IrInstSrcReduce *) {
1319 return IrInstSrcIdReduce;
1320}
1321
13141322static constexpr IrInstSrcId ir_inst_id(IrInstSrcTruncate *) {
13151323 return IrInstSrcIdTruncate;
13161324}
......@@ -1775,6 +1783,10 @@ static constexpr IrInstGenId ir_inst_id(IrInstGenFence *) {
17751783 return IrInstGenIdFence;
17761784}
17771785
1786static constexpr IrInstGenId ir_inst_id(IrInstGenReduce *) {
1787 return IrInstGenIdReduce;
1788}
1789
17781790static constexpr IrInstGenId ir_inst_id(IrInstGenTruncate *) {
17791791 return IrInstGenIdTruncate;
17801792}
......@@ -3502,6 +3514,29 @@ static IrInstGen *ir_build_fence_gen(IrAnalyze *ira, IrInst *source_instr, Atomi
35023514 return &instruction->base;
35033515}
35043516
3517static IrInstSrc *ir_build_reduce(IrBuilderSrc *irb, Scope *scope, AstNode *source_node, IrInstSrc *op, IrInstSrc *value) {
3518 IrInstSrcReduce *instruction = ir_build_instruction<IrInstSrcReduce>(irb, scope, source_node);
3519 instruction->op = op;
3520 instruction->value = value;
3521
3522 ir_ref_instruction(op, irb->current_basic_block);
3523 ir_ref_instruction(value, irb->current_basic_block);
3524
3525 return &instruction->base;
3526}
3527
3528static IrInstGen *ir_build_reduce_gen(IrAnalyze *ira, IrInst *source_instruction, ReduceOp op, IrInstGen *value, ZigType *result_type) {
3529 IrInstGenReduce *instruction = ir_build_inst_gen<IrInstGenReduce>(&ira->new_irb,
3530 source_instruction->scope, source_instruction->source_node);
3531 instruction->base.value->type = result_type;
3532 instruction->op = op;
3533 instruction->value = value;
3534
3535 ir_ref_inst_gen(value);
3536
3537 return &instruction->base;
3538}
3539
35053540static IrInstSrc *ir_build_truncate(IrBuilderSrc *irb, Scope *scope, AstNode *source_node,
35063541 IrInstSrc *dest_type, IrInstSrc *target)
35073542{
......@@ -6580,6 +6615,21 @@ static IrInstSrc *ir_gen_builtin_fn_call(IrBuilderSrc *irb, Scope *scope, AstNod
65806615 IrInstSrc *fence = ir_build_fence(irb, scope, node, arg0_value);
65816616 return ir_lval_wrap(irb, scope, fence, lval, result_loc);
65826617 }
6618 case BuiltinFnIdReduce:
6619 {
6620 AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
6621 IrInstSrc *arg0_value = ir_gen_node(irb, arg0_node, scope);
6622 if (arg0_value == irb->codegen->invalid_inst_src)
6623 return arg0_value;
6624
6625 AstNode *arg1_node = node->data.fn_call_expr.params.at(1);
6626 IrInstSrc *arg1_value = ir_gen_node(irb, arg1_node, scope);
6627 if (arg1_value == irb->codegen->invalid_inst_src)
6628 return arg1_value;
6629
6630 IrInstSrc *reduce = ir_build_reduce(irb, scope, node, arg0_value, arg1_value);
6631 return ir_lval_wrap(irb, scope, reduce, lval, result_loc);
6632 }
65836633 case BuiltinFnIdDivExact:
65846634 {
65856635 AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
......@@ -15932,6 +15982,24 @@ static bool ir_resolve_comptime(IrAnalyze *ira, IrInstGen *value, bool *out) {
1593215982 return ir_resolve_bool(ira, value, out);
1593315983}
1593415984
15985static bool ir_resolve_reduce_op(IrAnalyze *ira, IrInstGen *value, ReduceOp *out) {
15986 if (type_is_invalid(value->value->type))
15987 return false;
15988
15989 ZigType *reduce_op_type = get_builtin_type(ira->codegen, "ReduceOp");
15990
15991 IrInstGen *casted_value = ir_implicit_cast(ira, value, reduce_op_type);
15992 if (type_is_invalid(casted_value->value->type))
15993 return false;
15994
15995 ZigValue *const_val = ir_resolve_const(ira, casted_value, UndefBad);
15996 if (!const_val)
15997 return false;
15998
15999 *out = (ReduceOp)bigint_as_u32(&const_val->data.x_enum_tag);
16000 return true;
16001}
16002
1593516003static bool ir_resolve_atomic_order(IrAnalyze *ira, IrInstGen *value, AtomicOrder *out) {
1593616004 if (type_is_invalid(value->value->type))
1593716005 return false;
......@@ -26802,6 +26870,161 @@ static IrInstGen *ir_analyze_instruction_cmpxchg(IrAnalyze *ira, IrInstSrcCmpxch
2680226870 success_order, failure_order, instruction->is_weak, result_loc);
2680326871}
2680426872
26873static ErrorMsg *ir_eval_reduce(IrAnalyze *ira, IrInst *source_instr, ReduceOp op, ZigValue *value, ZigValue *out_value) {
26874 assert(value->type->id == ZigTypeIdVector);
26875 ZigType *scalar_type = value->type->data.vector.elem_type;
26876 const size_t len = value->type->data.vector.len;
26877 assert(len > 0);
26878
26879 out_value->type = scalar_type;
26880 out_value->special = ConstValSpecialStatic;
26881
26882 if (scalar_type->id == ZigTypeIdBool) {
26883 ZigValue *first_elem_val = &value->data.x_array.data.s_none.elements[0];
26884
26885 bool result = first_elem_val->data.x_bool;
26886 for (size_t i = 1; i < len; i++) {
26887 ZigValue *elem_val = &value->data.x_array.data.s_none.elements[i];
26888
26889 switch (op) {
26890 case ReduceOp_and:
26891 result = result && elem_val->data.x_bool;
26892 if (!result) break; // Short circuit
26893 break;
26894 case ReduceOp_or:
26895 result = result || elem_val->data.x_bool;
26896 if (result) break; // Short circuit
26897 break;
26898 case ReduceOp_xor:
26899 result = result != elem_val->data.x_bool;
26900 break;
26901 default:
26902 zig_unreachable();
26903 }
26904 }
26905
26906 out_value->data.x_bool = result;
26907 return nullptr;
26908 }
26909
26910 if (op != ReduceOp_min && op != ReduceOp_max) {
26911 ZigValue *first_elem_val = &value->data.x_array.data.s_none.elements[0];
26912
26913 copy_const_val(ira->codegen, out_value, first_elem_val);
26914
26915 for (size_t i = 1; i < len; i++) {
26916 ZigValue *elem_val = &value->data.x_array.data.s_none.elements[i];
26917
26918 IrBinOp bin_op;
26919 switch (op) {
26920 case ReduceOp_and: bin_op = IrBinOpBinAnd; break;
26921 case ReduceOp_or: bin_op = IrBinOpBinOr; break;
26922 case ReduceOp_xor: bin_op = IrBinOpBinXor; break;
26923 default: zig_unreachable();
26924 }
26925
26926 ErrorMsg *msg = ir_eval_math_op_scalar(ira, source_instr, scalar_type,
26927 out_value, bin_op, elem_val, out_value);
26928 if (msg != nullptr)
26929 return msg;
26930 }
26931
26932 return nullptr;
26933 }
26934
26935 ZigValue *candidate_elem_val = &value->data.x_array.data.s_none.elements[0];
26936
26937 ZigValue *dummy_cmp_value = ira->codegen->pass1_arena->create<ZigValue>();
26938 for (size_t i = 1; i < len; i++) {
26939 ZigValue *elem_val = &value->data.x_array.data.s_none.elements[i];
26940
26941 IrBinOp bin_op;
26942 switch (op) {
26943 case ReduceOp_min: bin_op = IrBinOpCmpLessThan; break;
26944 case ReduceOp_max: bin_op = IrBinOpCmpGreaterThan; break;
26945 default: zig_unreachable();
26946 }
26947
26948 ErrorMsg *msg = ir_eval_bin_op_cmp_scalar(ira, source_instr,
26949 elem_val, bin_op, candidate_elem_val, dummy_cmp_value);
26950 if (msg != nullptr)
26951 return msg;
26952
26953 if (dummy_cmp_value->data.x_bool)
26954 candidate_elem_val = elem_val;
26955 }
26956
26957 ira->codegen->pass1_arena->destroy(dummy_cmp_value);
26958 copy_const_val(ira->codegen, out_value, candidate_elem_val);
26959
26960 return nullptr;
26961}
26962
26963static IrInstGen *ir_analyze_instruction_reduce(IrAnalyze *ira, IrInstSrcReduce *instruction) {
26964 IrInstGen *op_inst = instruction->op->child;
26965 if (type_is_invalid(op_inst->value->type))
26966 return ira->codegen->invalid_inst_gen;
26967
26968 IrInstGen *value_inst = instruction->value->child;
26969 if (type_is_invalid(value_inst->value->type))
26970 return ira->codegen->invalid_inst_gen;
26971
26972 ZigType *value_type = value_inst->value->type;
26973 if (value_type->id != ZigTypeIdVector) {
26974 ir_add_error(ira, &value_inst->base,
26975 buf_sprintf("expected vector type, found '%s'",
26976 buf_ptr(&value_type->name)));
26977 return ira->codegen->invalid_inst_gen;
26978 }
26979
26980 ReduceOp op;
26981 if (!ir_resolve_reduce_op(ira, op_inst, &op))
26982 return ira->codegen->invalid_inst_gen;
26983
26984 ZigType *elem_type = value_type->data.vector.elem_type;
26985 switch (elem_type->id) {
26986 case ZigTypeIdInt:
26987 break;
26988 case ZigTypeIdBool:
26989 if (op > ReduceOp_xor) {
26990 ir_add_error(ira, &op_inst->base,
26991 buf_sprintf("invalid operation for '%s' type",
26992 buf_ptr(&elem_type->name)));
26993 return ira->codegen->invalid_inst_gen;
26994 } break;
26995 case ZigTypeIdFloat:
26996 if (op < ReduceOp_min) {
26997 ir_add_error(ira, &op_inst->base,
26998 buf_sprintf("invalid operation for '%s' type",
26999 buf_ptr(&elem_type->name)));
27000 return ira->codegen->invalid_inst_gen;
27001 } break;
27002 default:
27003 // Vectors cannot have child types other than those listed above
27004 zig_unreachable();
27005 }
27006
27007 // special case zero bit types
27008 switch (type_has_one_possible_value(ira->codegen, elem_type)) {
27009 case OnePossibleValueInvalid:
27010 return ira->codegen->invalid_inst_gen;
27011 case OnePossibleValueYes:
27012 return ir_const_move(ira, &instruction->base.base,
27013 get_the_one_possible_value(ira->codegen, elem_type));
27014 case OnePossibleValueNo:
27015 break;
27016 }
27017
27018 if (instr_is_comptime(value_inst)) {
27019 IrInstGen *result = ir_const(ira, &instruction->base.base, elem_type);
27020 if (ir_eval_reduce(ira, &instruction->base.base, op, value_inst->value, result->value))
27021 return ira->codegen->invalid_inst_gen;
27022 return result;
27023 }
27024
27025 return ir_build_reduce_gen(ira, &instruction->base.base, op, value_inst, elem_type);
27026}
27027
2680527028static IrInstGen *ir_analyze_instruction_fence(IrAnalyze *ira, IrInstSrcFence *instruction) {
2680627029 IrInstGen *order_inst = instruction->order->child;
2680727030 if (type_is_invalid(order_inst->value->type))
......@@ -31550,6 +31773,8 @@ static IrInstGen *ir_analyze_instruction_base(IrAnalyze *ira, IrInstSrc *instruc
3155031773 return ir_analyze_instruction_cmpxchg(ira, (IrInstSrcCmpxchg *)instruction);
3155131774 case IrInstSrcIdFence:
3155231775 return ir_analyze_instruction_fence(ira, (IrInstSrcFence *)instruction);
31776 case IrInstSrcIdReduce:
31777 return ir_analyze_instruction_reduce(ira, (IrInstSrcReduce *)instruction);
3155331778 case IrInstSrcIdTruncate:
3155431779 return ir_analyze_instruction_truncate(ira, (IrInstSrcTruncate *)instruction);
3155531780 case IrInstSrcIdIntCast:
......@@ -31937,6 +32162,7 @@ bool ir_inst_gen_has_side_effects(IrInstGen *instruction) {
3193732162 case IrInstGenIdNegation:
3193832163 case IrInstGenIdNegationWrapping:
3193932164 case IrInstGenIdWasmMemorySize:
32165 case IrInstGenIdReduce:
3194032166 return false;
3194132167
3194232168 case IrInstGenIdAsm:
......@@ -32106,6 +32332,7 @@ bool ir_inst_src_has_side_effects(IrInstSrc *instruction) {
3210632332 case IrInstSrcIdSpillEnd:
3210732333 case IrInstSrcIdWasmMemorySize:
3210832334 case IrInstSrcIdSrc:
32335 case IrInstSrcIdReduce:
3210932336 return false;
3211032337
3211132338 case IrInstSrcIdAsm:
src/stage1/ir_print.cpp+35
......@@ -200,6 +200,8 @@ const char* ir_inst_src_type_str(IrInstSrcId id) {
200200 return "SrcCmpxchg";
201201 case IrInstSrcIdFence:
202202 return "SrcFence";
203 case IrInstSrcIdReduce:
204 return "SrcReduce";
203205 case IrInstSrcIdTruncate:
204206 return "SrcTruncate";
205207 case IrInstSrcIdIntCast:
......@@ -436,6 +438,8 @@ const char* ir_inst_gen_type_str(IrInstGenId id) {
436438 return "GenCmpxchg";
437439 case IrInstGenIdFence:
438440 return "GenFence";
441 case IrInstGenIdReduce:
442 return "GenReduce";
439443 case IrInstGenIdTruncate:
440444 return "GenTruncate";
441445 case IrInstGenIdBoolNot:
......@@ -1584,6 +1588,14 @@ static void ir_print_fence(IrPrintSrc *irp, IrInstSrcFence *instruction) {
15841588 fprintf(irp->f, ")");
15851589}
15861590
1591static void ir_print_reduce(IrPrintSrc *irp, IrInstSrcReduce *instruction) {
1592 fprintf(irp->f, "@reduce(");
1593 ir_print_other_inst_src(irp, instruction->op);
1594 fprintf(irp->f, ", ");
1595 ir_print_other_inst_src(irp, instruction->value);
1596 fprintf(irp->f, ")");
1597}
1598
15871599static const char *atomic_order_str(AtomicOrder order) {
15881600 switch (order) {
15891601 case AtomicOrderUnordered: return "Unordered";
......@@ -1600,6 +1612,23 @@ static void ir_print_fence(IrPrintGen *irp, IrInstGenFence *instruction) {
16001612 fprintf(irp->f, "fence %s", atomic_order_str(instruction->order));
16011613}
16021614
1615static const char *reduce_op_str(ReduceOp op) {
1616 switch (op) {
1617 case ReduceOp_and: return "And";
1618 case ReduceOp_or: return "Or";
1619 case ReduceOp_xor: return "Xor";
1620 case ReduceOp_min: return "Min";
1621 case ReduceOp_max: return "Max";
1622 }
1623 zig_unreachable();
1624}
1625
1626static void ir_print_reduce(IrPrintGen *irp, IrInstGenReduce *instruction) {
1627 fprintf(irp->f, "@reduce(.%s, ", reduce_op_str(instruction->op));
1628 ir_print_other_inst_gen(irp, instruction->value);
1629 fprintf(irp->f, ")");
1630}
1631
16031632static void ir_print_truncate(IrPrintSrc *irp, IrInstSrcTruncate *instruction) {
16041633 fprintf(irp->f, "@truncate(");
16051634 ir_print_other_inst_src(irp, instruction->dest_type);
......@@ -2749,6 +2778,9 @@ static void ir_print_inst_src(IrPrintSrc *irp, IrInstSrc *instruction, bool trai
27492778 case IrInstSrcIdFence:
27502779 ir_print_fence(irp, (IrInstSrcFence *)instruction);
27512780 break;
2781 case IrInstSrcIdReduce:
2782 ir_print_reduce(irp, (IrInstSrcReduce *)instruction);
2783 break;
27522784 case IrInstSrcIdTruncate:
27532785 ir_print_truncate(irp, (IrInstSrcTruncate *)instruction);
27542786 break;
......@@ -3097,6 +3129,9 @@ static void ir_print_inst_gen(IrPrintGen *irp, IrInstGen *instruction, bool trai
30973129 case IrInstGenIdFence:
30983130 ir_print_fence(irp, (IrInstGenFence *)instruction);
30993131 break;
3132 case IrInstGenIdReduce:
3133 ir_print_reduce(irp, (IrInstGenReduce *)instruction);
3134 break;
31003135 case IrInstGenIdTruncate:
31013136 ir_print_truncate(irp, (IrInstGenTruncate *)instruction);
31023137 break;
src/zig_llvm.cpp+28
......@@ -1123,6 +1123,34 @@ LLVMValueRef ZigLLVMBuildAtomicRMW(LLVMBuilderRef B, enum ZigLLVM_AtomicRMWBinOp
11231123 singleThread ? SyncScope::SingleThread : SyncScope::System));
11241124}
11251125
1126LLVMValueRef ZigLLVMBuildAndReduce(LLVMBuilderRef B, LLVMValueRef Val) {
1127 return wrap(unwrap(B)->CreateAndReduce(unwrap(Val)));
1128}
1129
1130LLVMValueRef ZigLLVMBuildOrReduce(LLVMBuilderRef B, LLVMValueRef Val) {
1131 return wrap(unwrap(B)->CreateOrReduce(unwrap(Val)));
1132}
1133
1134LLVMValueRef ZigLLVMBuildXorReduce(LLVMBuilderRef B, LLVMValueRef Val) {
1135 return wrap(unwrap(B)->CreateXorReduce(unwrap(Val)));
1136}
1137
1138LLVMValueRef ZigLLVMBuildIntMaxReduce(LLVMBuilderRef B, LLVMValueRef Val, bool is_signed) {
1139 return wrap(unwrap(B)->CreateIntMaxReduce(unwrap(Val), is_signed));
1140}
1141
1142LLVMValueRef ZigLLVMBuildIntMinReduce(LLVMBuilderRef B, LLVMValueRef Val, bool is_signed) {
1143 return wrap(unwrap(B)->CreateIntMinReduce(unwrap(Val), is_signed));
1144}
1145
1146LLVMValueRef ZigLLVMBuildFPMaxReduce(LLVMBuilderRef B, LLVMValueRef Val) {
1147 return wrap(unwrap(B)->CreateFPMaxReduce(unwrap(Val)));
1148}
1149
1150LLVMValueRef ZigLLVMBuildFPMinReduce(LLVMBuilderRef B, LLVMValueRef Val) {
1151 return wrap(unwrap(B)->CreateFPMinReduce(unwrap(Val)));
1152}
1153
11261154static_assert((Triple::ArchType)ZigLLVM_UnknownArch == Triple::UnknownArch, "");
11271155static_assert((Triple::ArchType)ZigLLVM_arm == Triple::arm, "");
11281156static_assert((Triple::ArchType)ZigLLVM_armeb == Triple::armeb, "");
src/zig_llvm.h+8
......@@ -455,6 +455,14 @@ LLVMValueRef ZigLLVMBuildAtomicRMW(LLVMBuilderRef B, enum ZigLLVM_AtomicRMWBinOp
455455 LLVMValueRef PTR, LLVMValueRef Val,
456456 LLVMAtomicOrdering ordering, LLVMBool singleThread);
457457
458LLVMValueRef ZigLLVMBuildAndReduce(LLVMBuilderRef B, LLVMValueRef Val);
459LLVMValueRef ZigLLVMBuildOrReduce(LLVMBuilderRef B, LLVMValueRef Val);
460LLVMValueRef ZigLLVMBuildXorReduce(LLVMBuilderRef B, LLVMValueRef Val);
461LLVMValueRef ZigLLVMBuildIntMaxReduce(LLVMBuilderRef B, LLVMValueRef Val, bool is_signed);
462LLVMValueRef ZigLLVMBuildIntMinReduce(LLVMBuilderRef B, LLVMValueRef Val, bool is_signed);
463LLVMValueRef ZigLLVMBuildFPMaxReduce(LLVMBuilderRef B, LLVMValueRef Val);
464LLVMValueRef ZigLLVMBuildFPMinReduce(LLVMBuilderRef B, LLVMValueRef Val);
465
458466#define ZigLLVM_DIFlags_Zero 0U
459467#define ZigLLVM_DIFlags_Private 1U
460468#define ZigLLVM_DIFlags_Protected 2U
test/stage1/behavior/vector.zig+40
......@@ -484,3 +484,43 @@ test "vector shift operators" {
484484 S.doTheTest();
485485 comptime S.doTheTest();
486486}
487
488test "vector reduce operation" {
489 const S = struct {
490 fn doTheTestReduce(comptime op: builtin.ReduceOp, x: anytype, expected: anytype) void {
491 const N = @typeInfo(@TypeOf(x)).Array.len;
492 const TX = @typeInfo(@TypeOf(x)).Array.child;
493
494 var r = @reduce(op, @as(Vector(N, TX), x));
495 expectEqual(expected, r);
496 }
497 fn doTheTest() void {
498 doTheTestReduce(.And, [4]bool{ true, false, true, true }, @as(bool, false));
499 doTheTestReduce(.Or, [4]bool{ false, true, false, false }, @as(bool, true));
500 doTheTestReduce(.Xor, [4]bool{ true, true, true, false }, @as(bool, true));
501
502 doTheTestReduce(.And, [4]u1{ 1, 0, 1, 1 }, @as(u1, 0));
503 doTheTestReduce(.Or, [4]u1{ 0, 1, 0, 0 }, @as(u1, 1));
504 doTheTestReduce(.Xor, [4]u1{ 1, 1, 1, 0 }, @as(u1, 1));
505
506 doTheTestReduce(.And, [4]u32{ 0xffffffff, 0xffff5555, 0xaaaaffff, 0x10101010 }, @as(u32, 0x1010));
507 doTheTestReduce(.Or, [4]u32{ 0xffff0000, 0xff00, 0xf0, 0xf }, ~@as(u32, 0));
508 doTheTestReduce(.Xor, [4]u32{ 0x00000000, 0x33333333, 0x88888888, 0x44444444 }, ~@as(u32, 0));
509
510 doTheTestReduce(.Min, [4]i32{ 1234567, -386, 0, 3 }, @as(i32, -386));
511 doTheTestReduce(.Max, [4]i32{ 1234567, -386, 0, 3 }, @as(i32, 1234567));
512
513 doTheTestReduce(.Min, [4]u32{ 99, 9999, 9, 99999 }, @as(u32, 9));
514 doTheTestReduce(.Max, [4]u32{ 99, 9999, 9, 99999 }, @as(u32, 99999));
515
516 doTheTestReduce(.Min, [4]f32{ -10.3, 10.0e9, 13.0, -100.0 }, @as(f32, -100.0));
517 doTheTestReduce(.Max, [4]f32{ -10.3, 10.0e9, 13.0, -100.0 }, @as(f32, 10.0e9));
518
519 doTheTestReduce(.Min, [4]f64{ -10.3, 10.0e9, 13.0, -100.0 }, @as(f64, -100.0));
520 doTheTestReduce(.Max, [4]f64{ -10.3, 10.0e9, 13.0, -100.0 }, @as(f64, 10.0e9));
521 }
522 };
523
524 S.doTheTest();
525 comptime S.doTheTest();
526}