authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-11-09 15:16:49+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-11-12 16:00:16+00:00
log69f39868b4125e79e4070a88bbdfcd3643dbc90d
tree7e90f08d2b5d1cb234957dbe0b6a48034afe4e30
parent99a7884308d288bd39df9192c9094439b179ff60
signaturelock-open Commit is signed but in an unrecognized format.

Air.Legalize: revert to loops for scalarizations

I had tried unrolling the loops to avoid requiring the `vector_store_elem` instruction, but it's arguably a problem to generate O(N) code for an operation on `@Vector(N, T)`. In addition, that lowering emitted a lot of `.aggregate_init` instructions, which is itself a quite difficult operation to codegen. This requires reintroducing runtime vector indexing internally. However, I've put it in a couple of instructions which are intended only for use by `Air.Legalize`, named `legalize_vec_elem_val` (like `array_elem_val`, but for indexing a vector with a runtime-known index) and `legalize_vec_store_elem` (like the old `vector_store_elem` instruction). These are explicitly documented as *not* being emitted by Sema, so need only be implemented by backends if they actually use an `Air.Legalize.Feature` which emits them (otherwise they can be marked as `unreachable`).

14 files changed, 1548 insertions(+), 301 deletions(-)

src/Air.zig+26-3
......@@ -660,8 +660,8 @@ pub const Inst = struct {
660660 /// Given a pointer to a slice, return a pointer to the pointer of the slice.
661661 /// Uses the `ty_op` field.
662662 ptr_slice_ptr_ptr,
663 /// Given an (array value or vector value) and element index,
664 /// return the element value at that index.
663 /// Given an (array value or vector value) and element index, return the element value at
664 /// that index. If the lhs is a vector value, the index is guaranteed to be comptime-known.
665665 /// Result type is the element type of the array operand.
666666 /// Uses the `bin_op` field.
667667 array_elem_val,
......@@ -915,6 +915,26 @@ pub const Inst = struct {
915915 /// Operand is unused and set to Ref.none
916916 work_group_id,
917917
918 // The remaining instructions are not emitted by Sema. They are only emitted by `Legalize`,
919 // depending on the enabled features. As such, backends can consider them `unreachable` if
920 // they do not enable the relevant legalizations.
921
922 /// Given a pointer to a vector, a runtime-known index, and a scalar value, store the value
923 /// into the vector at the given index. Zig does not support this operation, but `Legalize`
924 /// may emit it when scalarizing vector operations.
925 ///
926 /// Uses the `pl_op` field with payload `Bin`. `operand` is the vector pointer. `lhs` is the
927 /// element index of type `usize`. `rhs` is the element value. Result is always void.
928 legalize_vec_store_elem,
929 /// Given a vector value and a runtime-known index, return the element value at that index.
930 /// This instruction is similar to `array_elem_val`; the only difference is that the index
931 /// here is runtime-known, which is usually not allowed for vectors. `Legalize` may emit
932 /// this instruction when scalarizing vector operations.
933 ///
934 /// Uses the `bin_op` field. `lhs` is the vector pointer. `rhs` is the element index. Result
935 /// type is the vector element type.
936 legalize_vec_elem_val,
937
918938 pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {
919939 switch (op) {
920940 .lt => return if (optimized) .cmp_lt_optimized else .cmp_lt,
......@@ -1681,6 +1701,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
16811701 .prefetch,
16821702 .set_err_return_trace,
16831703 .c_va_end,
1704 .legalize_vec_store_elem,
16841705 => return .void,
16851706
16861707 .slice_len,
......@@ -1699,7 +1720,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
16991720 return .fromInterned(ip.funcTypeReturnType(callee_ty.toIntern()));
17001721 },
17011722
1702 .slice_elem_val, .ptr_elem_val, .array_elem_val => {
1723 .slice_elem_val, .ptr_elem_val, .array_elem_val, .legalize_vec_elem_val => {
17031724 const ptr_ty = air.typeOf(datas[@intFromEnum(inst)].bin_op.lhs, ip);
17041725 return ptr_ty.childTypeIp(ip);
17051726 },
......@@ -1857,6 +1878,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
18571878 .intcast_safe,
18581879 .int_from_float_safe,
18591880 .int_from_float_optimized_safe,
1881 .legalize_vec_store_elem,
18601882 => true,
18611883
18621884 .add,
......@@ -2002,6 +2024,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
20022024 .work_item_id,
20032025 .work_group_size,
20042026 .work_group_id,
2027 .legalize_vec_elem_val,
20052028 => false,
20062029
20072030 .is_non_null_ptr, .is_null_ptr, .is_non_err_ptr, .is_err_ptr => air.typeOf(data.un_op, ip).isVolatilePtrIp(ip),
src/Air/Legalize.zig+818-288
......@@ -320,28 +320,36 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
320320 .xor,
321321 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
322322 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
323 if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
323 if (l.typeOf(bin_op.lhs).isVector(zcu)) {
324 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
325 }
324326 },
325327 .add_safe => if (l.features.has(.expand_add_safe)) {
326328 assert(!l.features.has(.scalarize_add_safe)); // it doesn't make sense to do both
327329 continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow));
328330 } else if (l.features.has(.scalarize_add_safe)) {
329331 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
330 if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
332 if (l.typeOf(bin_op.lhs).isVector(zcu)) {
333 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
334 }
331335 },
332336 .sub_safe => if (l.features.has(.expand_sub_safe)) {
333337 assert(!l.features.has(.scalarize_sub_safe)); // it doesn't make sense to do both
334338 continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow));
335339 } else if (l.features.has(.scalarize_sub_safe)) {
336340 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
337 if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
341 if (l.typeOf(bin_op.lhs).isVector(zcu)) {
342 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
343 }
338344 },
339345 .mul_safe => if (l.features.has(.expand_mul_safe)) {
340346 assert(!l.features.has(.scalarize_mul_safe)); // it doesn't make sense to do both
341347 continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow));
342348 } else if (l.features.has(.scalarize_mul_safe)) {
343349 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
344 if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
350 if (l.typeOf(bin_op.lhs).isVector(zcu)) {
351 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
352 }
345353 },
346354 .ptr_add, .ptr_sub => {},
347355 inline .add_with_overflow,
......@@ -350,7 +358,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
350358 .shl_with_overflow,
351359 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
352360 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
353 if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst));
361 if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) {
362 continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst));
363 }
354364 },
355365 .alloc => {},
356366 .inferred_alloc, .inferred_alloc_comptime => unreachable,
......@@ -387,7 +397,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
387397 }
388398 }
389399 }
390 if (l.features.has(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op);
400 if (l.features.has(comptime .scalarize(air_tag))) {
401 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
402 }
391403 }
392404 },
393405 inline .not,
......@@ -406,7 +418,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
406418 .float_from_int,
407419 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
408420 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
409 if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
421 if (ty_op.ty.toType().isVector(zcu)) {
422 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
423 }
410424 },
411425 .bitcast => if (l.features.has(.scalarize_bitcast)) {
412426 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {
......@@ -418,21 +432,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
418432 continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
419433 } else if (l.features.has(.scalarize_intcast_safe)) {
420434 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
421 if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
435 if (ty_op.ty.toType().isVector(zcu)) {
436 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
437 }
422438 },
423439 .int_from_float_safe => if (l.features.has(.expand_int_from_float_safe)) {
424440 assert(!l.features.has(.scalarize_int_from_float_safe));
425441 continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, false));
426442 } else if (l.features.has(.scalarize_int_from_float_safe)) {
427443 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
428 if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
444 if (ty_op.ty.toType().isVector(zcu)) {
445 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
446 }
429447 },
430448 .int_from_float_optimized_safe => if (l.features.has(.expand_int_from_float_optimized_safe)) {
431449 assert(!l.features.has(.scalarize_int_from_float_optimized_safe));
432450 continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, true));
433451 } else if (l.features.has(.scalarize_int_from_float_optimized_safe)) {
434452 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
435 if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
453 if (ty_op.ty.toType().isVector(zcu)) {
454 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
455 }
436456 },
437457 .block, .loop => {
438458 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
......@@ -467,7 +487,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
467487 .neg_optimized,
468488 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
469489 const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;
470 if (l.typeOf(un_op).isVector(zcu)) continue :inst try l.scalarize(inst, .un_op);
490 if (l.typeOf(un_op).isVector(zcu)) {
491 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .un_op));
492 }
471493 },
472494 .cmp_lt,
473495 .cmp_lt_optimized,
......@@ -484,7 +506,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
484506 => {},
485507 inline .cmp_vector, .cmp_vector_optimized => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
486508 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
487 if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .cmp_vector);
509 if (ty_pl.ty.toType().isVector(zcu)) {
510 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .cmp_vector));
511 }
488512 },
489513 .cond_br => {
490514 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
......@@ -614,9 +638,15 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
614638 else => {},
615639 }
616640 },
617 .shuffle_one => if (l.features.has(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one),
618 .shuffle_two => if (l.features.has(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two),
619 .select => if (l.features.has(.scalarize_select)) continue :inst try l.scalarize(inst, .select),
641 .shuffle_one => if (l.features.has(.scalarize_shuffle_one)) {
642 continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleOneBlockPayload(inst));
643 },
644 .shuffle_two => if (l.features.has(.scalarize_shuffle_two)) {
645 continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleTwoBlockPayload(inst));
646 },
647 .select => if (l.features.has(.scalarize_select)) {
648 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .select));
649 },
620650 .memset,
621651 .memset_safe,
622652 .memcpy,
......@@ -657,7 +687,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
657687 .union_init, .prefetch => {},
658688 .mul_add => if (l.features.has(.scalarize_mul_add)) {
659689 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
660 if (l.typeOf(pl_op.operand).isVector(zcu)) continue :inst try l.scalarize(inst, .pl_op_bin);
690 if (l.typeOf(pl_op.operand).isVector(zcu)) {
691 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .pl_op_bin));
692 }
661693 },
662694 .field_parent_ptr,
663695 .wasm_memory_size,
......@@ -675,96 +707,123 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
675707 .work_item_id,
676708 .work_group_size,
677709 .work_group_id,
710 .legalize_vec_elem_val,
711 .legalize_vec_store_elem,
678712 => {},
679713 }
680714 }
681715}
682716
683const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, cmp_vector, shuffle_one, shuffle_two, select };
684/// inline to propagate comptime-known `replaceInst` result.
685inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag {
686 return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form));
687}
688fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Data {
717const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, cmp_vector, select };
718fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: ScalarizeForm) Error!Air.Inst.Data {
689719 const pt = l.pt;
690720 const zcu = pt.zcu;
691 const gpa = zcu.gpa;
692721
693722 const orig = l.air_instructions.get(@intFromEnum(orig_inst));
694723 const res_ty = l.typeOfIndex(orig_inst);
695 const res_len = res_ty.vectorLen(zcu);
724 const result_is_array = switch (res_ty.zigTypeTag(zcu)) {
725 .vector => false,
726 .array => true,
727 else => unreachable,
728 };
729 const res_len = res_ty.arrayLen(zcu);
730 const res_elem_ty = res_ty.childType(zcu);
696731
697 const inst_per_elem = switch (form) {
732 if (result_is_array) {
733 // This is only allowed when legalizing an elementwise bitcast.
734 assert(orig.tag == .bitcast);
735 assert(form == .ty_op);
736 }
737
738 // Our output will be a loop doing elementwise stores:
739 //
740 // %1 = block(@Vector(N, Scalar), {
741 // %2 = alloc(*usize)
742 // %3 = alloc(*@Vector(N, Scalar))
743 // %4 = store(%2, @zero_usize)
744 // %5 = loop({
745 // %6 = load(%2)
746 // %7 = <scalar result of operation at index %5>
747 // %8 = legalize_vec_store_elem(%3, %5, %6)
748 // %9 = cmp_eq(%6, <usize, N-1>)
749 // %10 = cond_br(%9, {
750 // %11 = load(%3)
751 // %12 = br(%1, %11)
752 // }, {
753 // %13 = add(%6, @one_usize)
754 // %14 = store(%2, %13)
755 // %15 = repeat(%5)
756 // })
757 // })
758 // })
759 //
760 // If scalarizing an elementwise bitcast, the result might be an array, in which case
761 // `legalize_vec_store_elem` becomes two instructions (`ptr_elem_ptr` and `store`).
762 // Therefore, there are 13 or 14 instructions in the block, plus however many are
763 // needed to compute each result element for `form`.
764 const inst_per_form: usize = switch (form) {
698765 .un_op, .ty_op => 2,
699766 .bin_op, .cmp_vector => 3,
700767 .pl_op_bin => 4,
701 .shuffle_one, .shuffle_two => 1,
702768 .select => 7,
703769 };
770 const max_inst_per_form = 7; // maximum value in the above switch
771 var inst_buf: [14 + max_inst_per_form]Air.Inst.Index = undefined;
704772
705 var sfba_state = std.heap.stackFallback(@sizeOf([inst_per_elem * 32 + 2]Air.Inst.Index) + @sizeOf([32]Air.Inst.Ref), gpa);
706 const sfba = sfba_state.get();
773 var main_block: Block = .init(&inst_buf);
774 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
707775
708 // Plus 2 extra instructions for `aggregate_init` and `br`.
709 const inst_buf = try sfba.alloc(Air.Inst.Index, inst_per_elem * res_len + 2);
710 defer sfba.free(inst_buf);
776 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
777 const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(res_ty)).toRef();
711778
712 var main_block: Block = .init(inst_buf);
713 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
779 _ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
714780
715 const elem_buf = try sfba.alloc(Air.Inst.Ref, res_len);
716 defer sfba.free(elem_buf);
781 var loop: Loop = .init(l, &main_block);
782 loop.block = .init(main_block.stealRemainingCapacity());
717783
718 switch (form) {
719 .un_op => {
784 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
785 const elem_val: Air.Inst.Ref = switch (form) {
786 .un_op => elem: {
720787 const orig_operand = orig.data.un_op;
721 const un_op_tag = orig.tag;
722 for (elem_buf, 0..) |*elem, elem_idx| {
723 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
724 const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();
725 elem.* = main_block.addUnOp(l, un_op_tag, operand).toRef();
726 }
788 const operand = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operand, index_val).toRef();
789 break :elem loop.block.addUnOp(l, orig.tag, operand).toRef();
727790 },
728 .ty_op => {
791 .ty_op => elem: {
729792 const orig_operand = orig.data.ty_op.operand;
730 const orig_ty: Type = .fromInterned(orig.data.ty_op.ty.toInterned().?);
731 const scalar_ty = orig_ty.childType(zcu);
732 const ty_op_tag = orig.tag;
733 for (elem_buf, 0..) |*elem, elem_idx| {
734 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
735 const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();
736 elem.* = main_block.addTyOp(l, ty_op_tag, scalar_ty, operand).toRef();
737 }
793 const operand_is_array = switch (l.typeOf(orig_operand).zigTypeTag(zcu)) {
794 .vector => false,
795 .array => true,
796 else => unreachable,
797 };
798 const operand = loop.block.addBinOp(
799 l,
800 if (operand_is_array) .array_elem_val else .legalize_vec_elem_val,
801 orig_operand,
802 index_val,
803 ).toRef();
804 break :elem loop.block.addTyOp(l, orig.tag, res_elem_ty, operand).toRef();
738805 },
739 .bin_op => {
740 const orig_operands = orig.data.bin_op;
741 const bin_op_tag = orig.tag;
742 for (elem_buf, 0..) |*elem, elem_idx| {
743 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
744 const lhs = main_block.addBinOp(l, .array_elem_val, orig_operands.lhs, elem_idx_ref).toRef();
745 const rhs = main_block.addBinOp(l, .array_elem_val, orig_operands.rhs, elem_idx_ref).toRef();
746 elem.* = main_block.addBinOp(l, bin_op_tag, lhs, rhs).toRef();
747 }
806 .bin_op => elem: {
807 const orig_bin = orig.data.bin_op;
808 const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
809 const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
810 break :elem loop.block.addBinOp(l, orig.tag, lhs, rhs).toRef();
748811 },
749 .pl_op_bin => {
812 .pl_op_bin => elem: {
750813 const orig_operand = orig.data.pl_op.operand;
751 const orig_payload = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
752 const pl_op_tag = orig.tag;
753 for (elem_buf, 0..) |*elem, elem_idx| {
754 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
755 const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();
756 const lhs = main_block.addBinOp(l, .array_elem_val, orig_payload.lhs, elem_idx_ref).toRef();
757 const rhs = main_block.addBinOp(l, .array_elem_val, orig_payload.rhs, elem_idx_ref).toRef();
758 elem.* = main_block.add(l, .{
759 .tag = pl_op_tag,
760 .data = .{ .pl_op = .{
761 .payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
762 .operand = operand,
763 } },
764 }).toRef();
765 }
814 const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
815 const operand = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operand, index_val).toRef();
816 const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
817 const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
818 break :elem loop.block.add(l, .{
819 .tag = orig.tag,
820 .data = .{ .pl_op = .{
821 .operand = operand,
822 .payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
823 } },
824 }).toRef();
766825 },
767 .cmp_vector => {
826 .cmp_vector => elem: {
768827 const orig_payload = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
769828 const cmp_op = orig_payload.compareOperator();
770829 const optimized = switch (orig.tag) {
......@@ -772,116 +831,393 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form:
772831 .cmp_vector_optimized => true,
773832 else => unreachable,
774833 };
775 for (elem_buf, 0..) |*elem, elem_idx| {
776 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
777 const lhs = main_block.addBinOp(l, .array_elem_val, orig_payload.lhs, elem_idx_ref).toRef();
778 const rhs = main_block.addBinOp(l, .array_elem_val, orig_payload.rhs, elem_idx_ref).toRef();
779 elem.* = main_block.addCmpScalar(l, cmp_op, lhs, rhs, optimized).toRef();
780 }
781 },
782 .shuffle_one => {
783 const shuffle = l.getTmpAir().unwrapShuffleOne(zcu, orig_inst);
784 for (elem_buf, shuffle.mask) |*elem, mask| elem.* = switch (mask.unwrap()) {
785 .value => |val| .fromIntern(val),
786 .elem => |src_idx| elem: {
787 const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
788 break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand, src_idx_ref).toRef();
789 },
790 };
834 const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_payload.lhs, index_val).toRef();
835 const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_payload.rhs, index_val).toRef();
836 break :elem loop.block.addCmpScalar(l, cmp_op, lhs, rhs, optimized).toRef();
791837 },
792 .shuffle_two => {
793 const shuffle = l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst);
794 const scalar_ty = res_ty.childType(zcu);
795 for (elem_buf, shuffle.mask) |*elem, mask| elem.* = switch (mask.unwrap()) {
796 .undef => .fromValue(try pt.undefValue(scalar_ty)),
797 .a_elem => |src_idx| elem: {
798 const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
799 break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand_a, src_idx_ref).toRef();
800 },
801 .b_elem => |src_idx| elem: {
802 const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
803 break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand_b, src_idx_ref).toRef();
804 },
805 };
806 },
807 .select => {
838 .select => elem: {
808839 const orig_cond = orig.data.pl_op.operand;
809840 const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
810 const res_scalar_ty = res_ty.childType(zcu);
811 for (elem_buf, 0..) |*elem, elem_idx| {
812 // Payload to be populated later; we need the index early for `br`s.
813 const elem_block_inst = main_block.add(l, .{
814 .tag = .block,
815 .data = .{ .ty_pl = .{
816 .ty = .fromType(res_scalar_ty),
817 .payload = undefined,
818 } },
819 });
820 var elem_block: Block = .init(main_block.stealCapacity(2));
821841
822 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
823 const cond = elem_block.addBinOp(l, .array_elem_val, orig_cond, elem_idx_ref).toRef();
824 var condbr: CondBr = .init(l, cond, &elem_block, .{});
842 const elem_block_inst = loop.block.add(l, .{
843 .tag = .block,
844 .data = .{ .ty_pl = .{
845 .ty = .fromType(res_elem_ty),
846 .payload = undefined,
847 } },
848 });
849 var elem_block: Block = .init(loop.block.stealCapacity(2));
850 const cond = elem_block.addBinOp(l, .legalize_vec_elem_val, orig_cond, index_val).toRef();
851
852 var condbr: CondBr = .init(l, cond, &elem_block, .{});
825853
826 condbr.then_block = .init(main_block.stealCapacity(2));
827 const lhs = condbr.then_block.addBinOp(l, .array_elem_val, orig_bin.lhs, elem_idx_ref).toRef();
828 condbr.then_block.addBr(l, elem_block_inst, lhs);
854 condbr.then_block = .init(loop.block.stealCapacity(2));
855 const lhs = condbr.then_block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
856 condbr.then_block.addBr(l, elem_block_inst, lhs);
829857
830 condbr.else_block = .init(main_block.stealCapacity(2));
831 const rhs = condbr.else_block.addBinOp(l, .array_elem_val, orig_bin.rhs, elem_idx_ref).toRef();
832 condbr.else_block.addBr(l, elem_block_inst, rhs);
858 condbr.else_block = .init(loop.block.stealCapacity(2));
859 const rhs = condbr.else_block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
860 condbr.else_block.addBr(l, elem_block_inst, rhs);
833861
834 try condbr.finish(l);
862 try condbr.finish(l);
835863
836 const inst_data = l.air_instructions.items(.data);
837 inst_data[@intFromEnum(elem_block_inst)].ty_pl.payload = try l.addBlockBody(elem_block.body());
864 const inst_data = l.air_instructions.items(.data);
865 inst_data[@intFromEnum(elem_block_inst)].ty_pl.payload = try l.addBlockBody(elem_block.body());
838866
839 elem.* = elem_block_inst.toRef();
840 }
867 break :elem elem_block_inst.toRef();
841868 },
869 };
870 _ = loop.block.stealCapacity(max_inst_per_form - inst_per_form);
871 if (result_is_array) {
872 const elem_ptr = loop.block.add(l, .{
873 .tag = .ptr_elem_ptr,
874 .data = .{ .ty_pl = .{
875 .ty = .fromType(try pt.singleMutPtrType(res_elem_ty)),
876 .payload = try l.addExtra(Air.Bin, .{
877 .lhs = result_ptr,
878 .rhs = index_val,
879 }),
880 } },
881 }).toRef();
882 _ = loop.block.addBinOp(l, .store, elem_ptr, elem_val);
883 } else {
884 _ = loop.block.add(l, .{
885 .tag = .legalize_vec_store_elem,
886 .data = .{ .pl_op = .{
887 .operand = result_ptr,
888 .payload = try l.addExtra(Air.Bin, .{
889 .lhs = index_val,
890 .rhs = elem_val,
891 }),
892 } },
893 });
894 _ = loop.block.stealCapacity(1);
842895 }
896 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, res_len - 1))).toRef();
843897
844 const result = main_block.add(l, .{
845 .tag = .aggregate_init,
846 .data = .{ .ty_pl = .{
847 .ty = .fromType(res_ty),
848 .payload = payload: {
849 const idx = l.air_extra.items.len;
850 try l.air_extra.appendSlice(gpa, @ptrCast(elem_buf));
851 break :payload @intCast(idx);
852 },
853 } },
854 }).toRef();
898 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
899 condbr.then_block = .init(loop.block.stealRemainingCapacity());
900 const result_val = condbr.then_block.addTyOp(l, .load, res_ty, result_ptr).toRef();
901 condbr.then_block.addBr(l, orig_inst, result_val);
855902
856 main_block.addBr(l, orig_inst, result);
903 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
904 const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
905 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
906 _ = condbr.else_block.add(l, .{
907 .tag = .repeat,
908 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
909 });
857910
858 // Some `form` values may intentionally not use the full instruction buffer.
859 switch (form) {
860 .un_op,
861 .ty_op,
862 .bin_op,
863 .pl_op_bin,
864 .cmp_vector,
865 .select,
866 => {},
867 .shuffle_one,
868 .shuffle_two,
869 => _ = main_block.stealRemainingCapacity(),
870 }
911 try condbr.finish(l);
912
913 try loop.finish(l);
871914
872915 return .{ .ty_pl = .{
873916 .ty = .fromType(res_ty),
874917 .payload = try l.addBlockBody(main_block.body()),
875918 } };
876919}
877fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {
920fn scalarizeShuffleOneBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
878921 const pt = l.pt;
879922 const zcu = pt.zcu;
880923 const gpa = zcu.gpa;
881924
925 const shuffle = l.getTmpAir().unwrapShuffleOne(zcu, orig_inst);
926
927 // We're going to emit something like this:
928 //
929 // var x: @Vector(N, T) = all_comptime_known_elems;
930 // for (out_idxs, in_idxs) |i, j| x[i] = operand[j];
931 //
932 // So we must first compute `out_idxs` and `in_idxs`.
933
882934 var sfba_state = std.heap.stackFallback(512, gpa);
883935 const sfba = sfba_state.get();
884936
937 const out_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
938 defer sfba.free(out_idxs_buf);
939
940 const in_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
941 defer sfba.free(in_idxs_buf);
942
943 var n: usize = 0;
944 for (shuffle.mask, 0..) |mask, out_idx| switch (mask.unwrap()) {
945 .value => {},
946 .elem => |in_idx| {
947 out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
948 in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
949 n += 1;
950 },
951 };
952
953 const init_val: Value = init: {
954 const undef_val = try pt.undefValue(shuffle.result_ty.childType(zcu));
955 const elems = try sfba.alloc(InternPool.Index, shuffle.mask.len);
956 defer sfba.free(elems);
957 for (shuffle.mask, elems) |mask, *elem| elem.* = switch (mask.unwrap()) {
958 .value => |ip_index| ip_index,
959 .elem => undef_val.toIntern(),
960 };
961 break :init try pt.aggregateValue(shuffle.result_ty, elems);
962 };
963
964 // %1 = block(@Vector(N, T), {
965 // %2 = alloc(*@Vector(N, T))
966 // %3 = alloc(*usize)
967 // %4 = store(%2, <init_val>)
968 // %5 = [addScalarizedShuffle]
969 // %6 = load(%2)
970 // %7 = br(%1, %6)
971 // })
972
973 var inst_buf: [6]Air.Inst.Index = undefined;
974 var main_block: Block = .init(&inst_buf);
975 try l.air_instructions.ensureUnusedCapacity(gpa, 19);
976
977 const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(shuffle.result_ty)).toRef();
978 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
979
980 _ = main_block.addBinOp(l, .store, result_ptr, .fromValue(init_val));
981
982 try l.addScalarizedShuffle(
983 &main_block,
984 shuffle.operand,
985 result_ptr,
986 index_ptr,
987 out_idxs_buf[0..n],
988 in_idxs_buf[0..n],
989 );
990
991 const result_val = main_block.addTyOp(l, .load, shuffle.result_ty, result_ptr).toRef();
992 main_block.addBr(l, orig_inst, result_val);
993
994 return .{ .ty_pl = .{
995 .ty = .fromType(shuffle.result_ty),
996 .payload = try l.addBlockBody(main_block.body()),
997 } };
998}
999fn scalarizeShuffleTwoBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1000 const pt = l.pt;
1001 const zcu = pt.zcu;
1002 const gpa = zcu.gpa;
1003
1004 const shuffle = l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst);
1005
1006 // We're going to emit something like this:
1007 //
1008 // var x: @Vector(N, T) = undefined;
1009 // for (out_idxs_a, in_idxs_a) |i, j| x[i] = operand_a[j];
1010 // for (out_idxs_b, in_idxs_b) |i, j| x[i] = operand_b[j];
1011 //
1012 // The AIR will look like this:
1013 //
1014 // %1 = block(@Vector(N, T), {
1015 // %2 = alloc(*@Vector(N, T))
1016 // %3 = alloc(*usize)
1017 // %4 = store(%2, <@Vector(N, T), undefined>)
1018 // %5 = [addScalarizedShuffle]
1019 // %6 = [addScalarizedShuffle]
1020 // %7 = load(%2)
1021 // %8 = br(%1, %7)
1022 // })
1023
1024 var sfba_state = std.heap.stackFallback(512, gpa);
1025 const sfba = sfba_state.get();
1026
1027 const out_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
1028 defer sfba.free(out_idxs_buf);
1029
1030 const in_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
1031 defer sfba.free(in_idxs_buf);
1032
1033 // Iterate `shuffle.mask` before doing anything, because modifying AIR invalidates it.
1034 const out_idxs_a, const in_idxs_a, const out_idxs_b, const in_idxs_b = idxs: {
1035 var n: usize = 0;
1036 for (shuffle.mask, 0..) |mask, out_idx| switch (mask.unwrap()) {
1037 .undef, .b_elem => {},
1038 .a_elem => |in_idx| {
1039 out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
1040 in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
1041 n += 1;
1042 },
1043 };
1044 const a_len = n;
1045 for (shuffle.mask, 0..) |mask, out_idx| switch (mask.unwrap()) {
1046 .undef, .a_elem => {},
1047 .b_elem => |in_idx| {
1048 out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
1049 in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
1050 n += 1;
1051 },
1052 };
1053 break :idxs .{
1054 out_idxs_buf[0..a_len],
1055 in_idxs_buf[0..a_len],
1056 out_idxs_buf[a_len..n],
1057 in_idxs_buf[a_len..n],
1058 };
1059 };
1060
1061 var inst_buf: [7]Air.Inst.Index = undefined;
1062 var main_block: Block = .init(&inst_buf);
1063 try l.air_instructions.ensureUnusedCapacity(gpa, 33);
1064
1065 const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(shuffle.result_ty)).toRef();
1066 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1067
1068 _ = main_block.addBinOp(l, .store, result_ptr, .fromValue(try pt.undefValue(shuffle.result_ty)));
1069
1070 if (out_idxs_a.len == 0) {
1071 _ = main_block.stealCapacity(1);
1072 } else {
1073 try l.addScalarizedShuffle(
1074 &main_block,
1075 shuffle.operand_a,
1076 result_ptr,
1077 index_ptr,
1078 out_idxs_a,
1079 in_idxs_a,
1080 );
1081 }
1082
1083 if (out_idxs_b.len == 0) {
1084 _ = main_block.stealCapacity(1);
1085 } else {
1086 try l.addScalarizedShuffle(
1087 &main_block,
1088 shuffle.operand_b,
1089 result_ptr,
1090 index_ptr,
1091 out_idxs_b,
1092 in_idxs_b,
1093 );
1094 }
1095
1096 const result_val = main_block.addTyOp(l, .load, shuffle.result_ty, result_ptr).toRef();
1097 main_block.addBr(l, orig_inst, result_val);
1098
1099 return .{ .ty_pl = .{
1100 .ty = .fromType(shuffle.result_ty),
1101 .payload = try l.addBlockBody(main_block.body()),
1102 } };
1103}
1104/// Adds code to `parent_block` which behaves like this loop:
1105///
1106/// for (out_idxs, in_idxs) |i, j| result_vec_ptr[i] = operand_vec[j];
1107///
1108/// The actual AIR adds exactly one instruction to `parent_block` itself and 14 instructions
1109/// overall, and is as follows:
1110///
1111/// %1 = block(void, {
1112/// %2 = store(index_ptr, @zero_usize)
1113/// %3 = loop({
1114/// %4 = load(index_ptr)
1115/// %5 = ptr_elem_val(out_idxs_ptr, %4)
1116/// %6 = ptr_elem_val(in_idxs_ptr, %4)
1117/// %7 = legalize_vec_elem_val(operand_vec, %6)
1118/// %8 = legalize_vec_store_elem(result_vec_ptr, %4, %7)
1119/// %9 = cmp_eq(%4, <usize, out_idxs.len-1>)
1120/// %10 = cond_br(%9, {
1121/// %11 = br(%1, @void_value)
1122/// }, {
1123/// %12 = add(%4, @one_usize)
1124/// %13 = store(index_ptr, %12)
1125/// %14 = repeat(%3)
1126/// })
1127/// })
1128/// })
1129///
1130/// The caller is responsible for reserving space in `l.air_instructions`.
1131fn addScalarizedShuffle(
1132 l: *Legalize,
1133 parent_block: *Block,
1134 operand_vec: Air.Inst.Ref,
1135 result_vec_ptr: Air.Inst.Ref,
1136 index_ptr: Air.Inst.Ref,
1137 out_idxs: []const InternPool.Index,
1138 in_idxs: []const InternPool.Index,
1139) Error!void {
1140 const pt = l.pt;
1141
1142 assert(out_idxs.len == in_idxs.len);
1143 const n = out_idxs.len;
1144
1145 const idxs_ty = try pt.arrayType(.{ .len = n, .child = .usize_type });
1146 const idxs_ptr_ty = try pt.singleConstPtrType(idxs_ty);
1147 const manyptr_usize_ty = try pt.manyConstPtrType(.usize);
1148
1149 const out_idxs_ptr = try pt.intern(.{ .ptr = .{
1150 .ty = manyptr_usize_ty.toIntern(),
1151 .base_addr = .{ .uav = .{
1152 .val = (try pt.aggregateValue(idxs_ty, out_idxs)).toIntern(),
1153 .orig_ty = idxs_ptr_ty.toIntern(),
1154 } },
1155 .byte_offset = 0,
1156 } });
1157 const in_idxs_ptr = try pt.intern(.{ .ptr = .{
1158 .ty = manyptr_usize_ty.toIntern(),
1159 .base_addr = .{ .uav = .{
1160 .val = (try pt.aggregateValue(idxs_ty, in_idxs)).toIntern(),
1161 .orig_ty = idxs_ptr_ty.toIntern(),
1162 } },
1163 .byte_offset = 0,
1164 } });
1165
1166 const main_block_inst = parent_block.add(l, .{
1167 .tag = .block,
1168 .data = .{ .ty_pl = .{
1169 .ty = .void_type,
1170 .payload = undefined,
1171 } },
1172 });
1173
1174 var inst_buf: [13]Air.Inst.Index = undefined;
1175 var main_block: Block = .init(&inst_buf);
1176
1177 _ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1178
1179 var loop: Loop = .init(l, &main_block);
1180 loop.block = .init(main_block.stealRemainingCapacity());
1181
1182 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1183 const in_idx_val = loop.block.addBinOp(l, .ptr_elem_val, .fromIntern(in_idxs_ptr), index_val).toRef();
1184 const out_idx_val = loop.block.addBinOp(l, .ptr_elem_val, .fromIntern(out_idxs_ptr), index_val).toRef();
1185
1186 const elem_val = loop.block.addBinOp(l, .legalize_vec_elem_val, operand_vec, in_idx_val).toRef();
1187 _ = loop.block.add(l, .{
1188 .tag = .legalize_vec_store_elem,
1189 .data = .{ .pl_op = .{
1190 .operand = result_vec_ptr,
1191 .payload = try l.addExtra(Air.Bin, .{
1192 .lhs = out_idx_val,
1193 .rhs = elem_val,
1194 }),
1195 } },
1196 });
1197
1198 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, n - 1))).toRef();
1199 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1200 condbr.then_block = .init(loop.block.stealRemainingCapacity());
1201 condbr.then_block.addBr(l, main_block_inst, .void_value);
1202
1203 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1204 const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1205 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1206 _ = condbr.else_block.add(l, .{
1207 .tag = .repeat,
1208 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1209 });
1210
1211 try condbr.finish(l);
1212 try loop.finish(l);
1213
1214 const inst_data = l.air_instructions.items(.data);
1215 inst_data[@intFromEnum(main_block_inst)].ty_pl.payload = try l.addBlockBody(main_block.body());
1216}
1217fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {
1218 const pt = l.pt;
1219 const zcu = pt.zcu;
1220
8851221 const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
8861222
8871223 const dest_ty = ty_op.ty.toType();
......@@ -920,72 +1256,204 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
9201256 const uint_ty = try pt.intType(.unsigned, num_bits);
9211257 const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
9221258
923 const inst_buf = try sfba.alloc(Air.Inst.Index, len: {
924 const operand_to_uint_len: u64 = if (operand_legal) 1 else (operand_ty.arrayLen(zcu) * 5);
925 const uint_to_dest_len: u64 = if (dest_legal) 1 else (dest_ty.arrayLen(zcu) * 3 + 1);
926 break :len @intCast(operand_to_uint_len + uint_to_dest_len + 1);
927 });
928 defer sfba.free(inst_buf);
929 var main_block: Block = .init(inst_buf);
930 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
1259 var inst_buf: [39]Air.Inst.Index = undefined;
1260 var main_block: Block = .init(&inst_buf);
1261 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
9311262
9321263 // First, convert `operand_ty` to `uint_ty` (`uN`).
9331264
9341265 const uint_val: Air.Inst.Ref = uint_val: {
935 if (operand_legal) break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
936
937 const bits_per_elem: u16 = @intCast(operand_ty.childType(zcu).bitSize(zcu));
938 const bits_per_elem_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, bits_per_elem));
939 const elem_uint_ty = try pt.intType(.unsigned, bits_per_elem);
940
941 var cur_uint: Air.Inst.Ref = .fromValue(try pt.intValue(uint_ty, 0));
942 var elem_idx = operand_ty.arrayLen(zcu);
943 while (elem_idx > 0) {
944 elem_idx -= 1;
945 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
946 const orig_elem = main_block.addBinOp(l, .array_elem_val, ty_op.operand, elem_idx_ref).toRef();
947 const elem_as_uint = main_block.addBitCast(l, elem_uint_ty, orig_elem);
948 const elem_extended = main_block.addTyOp(l, .intcast, uint_ty, elem_as_uint).toRef();
949 cur_uint = main_block.addBinOp(l, .shl_exact, cur_uint, bits_per_elem_ref).toRef();
950 cur_uint = main_block.addBinOp(l, .bit_or, cur_uint, elem_extended).toRef();
1266 if (operand_legal) {
1267 _ = main_block.stealCapacity(19);
1268 break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
9511269 }
952 break :uint_val cur_uint;
1270
1271 // %1 = block({
1272 // %2 = alloc(*usize)
1273 // %3 = alloc(*uN)
1274 // %4 = store(%2, <usize, operand_len>)
1275 // %5 = store(%3, <uN, 0>)
1276 // %6 = loop({
1277 // %7 = load(%2)
1278 // %8 = array_elem_val(orig_operand, %7)
1279 // %9 = bitcast(uE, %8)
1280 // %10 = intcast(uN, %9)
1281 // %11 = load(%3)
1282 // %12 = shl_exact(%11, <uS, E>)
1283 // %13 = bit_or(%12, %10)
1284 // %14 = cmp_eq(%4, @zero_usize)
1285 // %15 = cond_br(%14, {
1286 // %16 = br(%1, %13)
1287 // }, {
1288 // %17 = store(%3, %13)
1289 // %18 = sub(%7, @one_usize)
1290 // %19 = store(%2, %18)
1291 // %20 = repeat(%6)
1292 // })
1293 // })
1294 // })
1295
1296 const elem_bits = operand_ty.childType(zcu).bitSize(zcu);
1297 const elem_bits_val = try pt.intValue(shift_ty, elem_bits);
1298 const elem_uint_ty = try pt.intType(.unsigned, @intCast(elem_bits));
1299
1300 const uint_block_inst = main_block.add(l, .{
1301 .tag = .block,
1302 .data = .{ .ty_pl = .{
1303 .ty = .fromType(uint_ty),
1304 .payload = undefined,
1305 } },
1306 });
1307 var uint_block: Block = .init(main_block.stealCapacity(19));
1308
1309 const index_ptr = uint_block.addTy(l, .alloc, .ptr_usize).toRef();
1310 const result_ptr = uint_block.addTy(l, .alloc, try pt.singleMutPtrType(uint_ty)).toRef();
1311 _ = uint_block.addBinOp(
1312 l,
1313 .store,
1314 index_ptr,
1315 .fromValue(try pt.intValue(.usize, operand_ty.arrayLen(zcu))),
1316 );
1317 _ = uint_block.addBinOp(l, .store, result_ptr, .fromValue(try pt.intValue(uint_ty, 0)));
1318
1319 var loop: Loop = .init(l, &uint_block);
1320 loop.block = .init(uint_block.stealRemainingCapacity());
1321
1322 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1323 const raw_elem = loop.block.addBinOp(
1324 l,
1325 if (operand_ty.zigTypeTag(zcu) == .vector) .legalize_vec_elem_val else .array_elem_val,
1326 ty_op.operand,
1327 index_val,
1328 ).toRef();
1329 const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);
1330 const elem_extended = loop.block.addTyOp(l, .intcast, uint_ty, elem_uint).toRef();
1331 const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();
1332 const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();
1333 const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();
1334
1335 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .zero_usize).toRef();
1336 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1337
1338 condbr.then_block = .init(loop.block.stealRemainingCapacity());
1339 condbr.then_block.addBr(l, uint_block_inst, new_result);
1340
1341 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1342 _ = condbr.else_block.addBinOp(l, .store, result_ptr, new_result);
1343 const new_index_val = condbr.else_block.addBinOp(l, .sub, index_val, .one_usize).toRef();
1344 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1345 _ = condbr.else_block.add(l, .{
1346 .tag = .repeat,
1347 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1348 });
1349
1350 try condbr.finish(l);
1351 try loop.finish(l);
1352
1353 const inst_data = l.air_instructions.items(.data);
1354 inst_data[@intFromEnum(uint_block_inst)].ty_pl.payload = try l.addBlockBody(uint_block.body());
1355
1356 break :uint_val uint_block_inst.toRef();
9531357 };
9541358
9551359 // Now convert `uint_ty` (`uN`) to `dest_ty`.
9561360
957 const result: Air.Inst.Ref = result: {
958 if (dest_legal) break :result main_block.addBitCast(l, dest_ty, uint_val);
1361 if (dest_legal) {
1362 _ = main_block.stealCapacity(17);
1363 const result = main_block.addBitCast(l, dest_ty, uint_val);
1364 main_block.addBr(l, orig_inst, result);
1365 } else {
1366 // %1 = alloc(*usize)
1367 // %2 = alloc(*@Vector(N, Result))
1368 // %3 = store(%1, @zero_usize)
1369 // %4 = loop({
1370 // %5 = load(%1)
1371 // %6 = mul(%5, <usize, E>)
1372 // %7 = intcast(uS, %6)
1373 // %8 = shr(uint_val, %7)
1374 // %9 = trunc(uE, %8)
1375 // %10 = bitcast(Result, %9)
1376 // %11 = legalize_vec_store_elem(%2, %5, %10)
1377 // %12 = cmp_eq(%5, <usize, vec_len>)
1378 // %13 = cond_br(%12, {
1379 // %14 = load(%2)
1380 // %15 = br(%0, %14)
1381 // }, {
1382 // %16 = add(%5, @one_usize)
1383 // %17 = store(%1, %16)
1384 // %18 = repeat(%4)
1385 // })
1386 // })
1387 //
1388 // The result might be an array, in which case `legalize_vec_store_elem`
1389 // becomes `ptr_elem_ptr` followed by `store`.
9591390
9601391 const elem_ty = dest_ty.childType(zcu);
961 const bits_per_elem: u16 = @intCast(elem_ty.bitSize(zcu));
962 const bits_per_elem_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, bits_per_elem));
963 const elem_uint_ty = try pt.intType(.unsigned, bits_per_elem);
964
965 const elem_buf = try sfba.alloc(Air.Inst.Ref, dest_ty.arrayLen(zcu));
966 defer sfba.free(elem_buf);
967
968 var cur_uint = uint_val;
969 for (elem_buf) |*elem| {
970 const elem_as_uint = main_block.addTyOp(l, .trunc, elem_uint_ty, cur_uint).toRef();
971 elem.* = main_block.addBitCast(l, elem_ty, elem_as_uint);
972 cur_uint = main_block.addBinOp(l, .shr, cur_uint, bits_per_elem_ref).toRef();
1392 const elem_bits = elem_ty.bitSize(zcu);
1393 const elem_uint_ty = try pt.intType(.unsigned, @intCast(elem_bits));
1394
1395 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1396 const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(dest_ty)).toRef();
1397 _ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1398
1399 var loop: Loop = .init(l, &main_block);
1400 loop.block = .init(main_block.stealRemainingCapacity());
1401
1402 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1403 const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();
1404 const casted_bit_offset = loop.block.addTyOp(l, .intcast, shift_ty, bit_offset).toRef();
1405 const shifted_uint = loop.block.addBinOp(l, .shr, index_val, casted_bit_offset).toRef();
1406 const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();
1407 const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);
1408 switch (dest_ty.zigTypeTag(zcu)) {
1409 .array => {
1410 const elem_ptr = loop.block.add(l, .{
1411 .tag = .ptr_elem_ptr,
1412 .data = .{ .ty_pl = .{
1413 .ty = .fromType(try pt.singleMutPtrType(elem_ty)),
1414 .payload = try l.addExtra(Air.Bin, .{
1415 .lhs = result_ptr,
1416 .rhs = index_val,
1417 }),
1418 } },
1419 }).toRef();
1420 _ = loop.block.addBinOp(l, .store, elem_ptr, elem_val);
1421 },
1422 .vector => {
1423 _ = loop.block.add(l, .{
1424 .tag = .legalize_vec_store_elem,
1425 .data = .{ .pl_op = .{
1426 .operand = result_ptr,
1427 .payload = try l.addExtra(Air.Bin, .{
1428 .lhs = index_val,
1429 .rhs = elem_val,
1430 }),
1431 } },
1432 });
1433 _ = loop.block.stealCapacity(1);
1434 },
1435 else => unreachable,
9731436 }
9741437
975 break :result main_block.add(l, .{
976 .tag = .aggregate_init,
977 .data = .{ .ty_pl = .{
978 .ty = .fromType(dest_ty),
979 .payload = payload: {
980 const idx = l.air_extra.items.len;
981 try l.air_extra.appendSlice(gpa, @ptrCast(elem_buf));
982 break :payload @intCast(idx);
983 },
984 } },
985 }).toRef();
986 };
1438 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, dest_ty.arrayLen(zcu) - 1))).toRef();
9871439
988 main_block.addBr(l, orig_inst, result);
1440 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1441
1442 condbr.then_block = .init(loop.block.stealRemainingCapacity());
1443 const result_val = condbr.then_block.addTyOp(l, .load, dest_ty, result_ptr).toRef();
1444 condbr.then_block.addBr(l, orig_inst, result_val);
1445
1446 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1447 const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1448 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1449 _ = condbr.else_block.add(l, .{
1450 .tag = .repeat,
1451 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1452 });
1453
1454 try condbr.finish(l);
1455 try loop.finish(l);
1456 }
9891457
9901458 return .{ .ty_pl = .{
9911459 .ty = .fromType(dest_ty),
......@@ -995,10 +1463,6 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
9951463fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
9961464 const pt = l.pt;
9971465 const zcu = pt.zcu;
998 const gpa = zcu.gpa;
999
1000 var sfba_state = std.heap.stackFallback(512, gpa);
1001 const sfba = sfba_state.get();
10021466
10031467 const orig = l.air_instructions.get(@intFromEnum(orig_inst));
10041468 const orig_operands = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;
......@@ -1015,89 +1479,127 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!
10151479 const scalar_int_ty = vec_int_ty.childType(zcu);
10161480 const scalar_tuple_ty = try pt.overflowArithmeticTupleType(scalar_int_ty);
10171481
1018 const elems_len = vec_int_ty.vectorLen(zcu);
1019
1020 const inst_buf = try sfba.alloc(Air.Inst.Index, 5 * elems_len + 4);
1021 defer sfba.free(inst_buf);
1482 // %1 = block(struct { @Vector(N, Int), @Vector(N, u1) }, {
1483 // %2 = alloc(*usize)
1484 // %3 = alloc(*struct { @Vector(N, Int), @Vector(N, u1) })
1485 // %4 = struct_field_ptr_index_0(*@Vector(N, Int), %3)
1486 // %5 = struct_field_ptr_index_1(*@Vector(N, u1), %3)
1487 // %6 = store(%2, @zero_usize)
1488 // %7 = loop({
1489 // %8 = load(%2)
1490 // %9 = legalize_vec_elem_val(orig_lhs, %8)
1491 // %10 = legalize_vec_elem_val(orig_rhs, %8)
1492 // %11 = ???_with_overflow(struct { Int, u1 }, %9, %10)
1493 // %12 = struct_field_val(%11, 0)
1494 // %13 = struct_field_val(%11, 1)
1495 // %14 = legalize_vec_store_elem(%4, %8, %12)
1496 // %15 = legalize_vec_store_elem(%4, %8, %13)
1497 // %16 = cmp_eq(%8, <usize, N-1>)
1498 // %17 = cond_br(%16, {
1499 // %18 = load(%3)
1500 // %19 = br(%1, %18)
1501 // }, {
1502 // %20 = add(%8, @one_usize)
1503 // %21 = store(%2, %20)
1504 // %22 = repeat(%7)
1505 // })
1506 // })
1507 // })
10221508
1023 var main_block: Block = .init(inst_buf);
1024 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
1509 const elems_len = vec_int_ty.vectorLen(zcu);
10251510
1026 const int_elem_buf = try sfba.alloc(Air.Inst.Ref, elems_len);
1027 defer sfba.free(int_elem_buf);
1028 const overflow_elem_buf = try sfba.alloc(Air.Inst.Ref, elems_len);
1029 defer sfba.free(overflow_elem_buf);
1030
1031 for (int_elem_buf, overflow_elem_buf, 0..) |*int_elem, *overflow_elem, elem_idx| {
1032 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
1033 const lhs = main_block.addBinOp(l, .array_elem_val, orig_operands.lhs, elem_idx_ref).toRef();
1034 const rhs = main_block.addBinOp(l, .array_elem_val, orig_operands.rhs, elem_idx_ref).toRef();
1035 const elem_result = main_block.add(l, .{
1036 .tag = orig.tag,
1037 .data = .{ .ty_pl = .{
1038 .ty = .fromType(scalar_tuple_ty),
1039 .payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
1040 } },
1041 }).toRef();
1042 int_elem.* = main_block.add(l, .{
1043 .tag = .struct_field_val,
1044 .data = .{ .ty_pl = .{
1045 .ty = .fromType(scalar_int_ty),
1046 .payload = try l.addExtra(Air.StructField, .{
1047 .struct_operand = elem_result,
1048 .field_index = 0,
1049 }),
1050 } },
1051 }).toRef();
1052 overflow_elem.* = main_block.add(l, .{
1053 .tag = .struct_field_val,
1054 .data = .{ .ty_pl = .{
1055 .ty = .bool_type,
1056 .payload = try l.addExtra(Air.StructField, .{
1057 .struct_operand = elem_result,
1058 .field_index = 1,
1059 }),
1060 } },
1061 }).toRef();
1062 }
1511 var inst_buf: [21]Air.Inst.Index = undefined;
1512 var main_block: Block = .init(&inst_buf);
1513 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
10631514
1064 const int_vec = main_block.add(l, .{
1065 .tag = .aggregate_init,
1515 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1516 const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(vec_tuple_ty)).toRef();
1517 const result_int_ptr = main_block.addTyOp(
1518 l,
1519 .struct_field_ptr_index_0,
1520 try pt.singleMutPtrType(vec_int_ty),
1521 result_ptr,
1522 ).toRef();
1523 const result_overflow_ptr = main_block.addTyOp(
1524 l,
1525 .struct_field_ptr_index_1,
1526 try pt.singleMutPtrType(vec_overflow_ty),
1527 result_ptr,
1528 ).toRef();
1529
1530 _ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1531
1532 var loop: Loop = .init(l, &main_block);
1533 loop.block = .init(main_block.stealRemainingCapacity());
1534
1535 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1536 const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operands.lhs, index_val).toRef();
1537 const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operands.rhs, index_val).toRef();
1538 const elem_result = loop.block.add(l, .{
1539 .tag = orig.tag,
10661540 .data = .{ .ty_pl = .{
1067 .ty = .fromType(vec_int_ty),
1068 .payload = payload: {
1069 const idx = l.air_extra.items.len;
1070 try l.air_extra.appendSlice(gpa, @ptrCast(int_elem_buf));
1071 break :payload @intCast(idx);
1072 },
1541 .ty = .fromType(scalar_tuple_ty),
1542 .payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
10731543 } },
10741544 }).toRef();
1075 const overflow_vec = main_block.add(l, .{
1076 .tag = .aggregate_init,
1545 const int_elem = loop.block.add(l, .{
1546 .tag = .struct_field_val,
10771547 .data = .{ .ty_pl = .{
1078 .ty = .fromType(vec_overflow_ty),
1079 .payload = payload: {
1080 const idx = l.air_extra.items.len;
1081 try l.air_extra.appendSlice(gpa, @ptrCast(overflow_elem_buf));
1082 break :payload @intCast(idx);
1083 },
1548 .ty = .fromType(scalar_int_ty),
1549 .payload = try l.addExtra(Air.StructField, .{
1550 .struct_operand = elem_result,
1551 .field_index = 0,
1552 }),
10841553 } },
10851554 }).toRef();
1086
1087 const tuple_elems: [2]Air.Inst.Ref = .{ int_vec, overflow_vec };
1088 const result = main_block.add(l, .{
1089 .tag = .aggregate_init,
1555 const overflow_elem = loop.block.add(l, .{
1556 .tag = .struct_field_val,
10901557 .data = .{ .ty_pl = .{
1091 .ty = .fromType(vec_tuple_ty),
1092 .payload = payload: {
1093 const idx = l.air_extra.items.len;
1094 try l.air_extra.appendSlice(gpa, @ptrCast(&tuple_elems));
1095 break :payload @intCast(idx);
1096 },
1558 .ty = .u1_type,
1559 .payload = try l.addExtra(Air.StructField, .{
1560 .struct_operand = elem_result,
1561 .field_index = 1,
1562 }),
10971563 } },
10981564 }).toRef();
1565 _ = loop.block.add(l, .{
1566 .tag = .legalize_vec_store_elem,
1567 .data = .{ .pl_op = .{
1568 .operand = result_int_ptr,
1569 .payload = try l.addExtra(Air.Bin, .{
1570 .lhs = index_val,
1571 .rhs = int_elem,
1572 }),
1573 } },
1574 });
1575 _ = loop.block.add(l, .{
1576 .tag = .legalize_vec_store_elem,
1577 .data = .{ .pl_op = .{
1578 .operand = result_overflow_ptr,
1579 .payload = try l.addExtra(Air.Bin, .{
1580 .lhs = index_val,
1581 .rhs = overflow_elem,
1582 }),
1583 } },
1584 });
10991585
1100 main_block.addBr(l, orig_inst, result);
1586 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, elems_len - 1))).toRef();
1587 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1588
1589 condbr.then_block = .init(loop.block.stealRemainingCapacity());
1590 const result_val = condbr.then_block.addTyOp(l, .load, vec_tuple_ty, result_ptr).toRef();
1591 condbr.then_block.addBr(l, orig_inst, result_val);
1592
1593 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1594 const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1595 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1596 _ = condbr.else_block.add(l, .{
1597 .tag = .repeat,
1598 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1599 });
1600
1601 try condbr.finish(l);
1602 try loop.finish(l);
11011603
11021604 return .{ .ty_pl = .{
11031605 .ty = .fromType(vec_tuple_ty),
......@@ -1288,7 +1790,7 @@ fn safeIntFromFloatBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, optimiz
12881790
12891791 // We emit 9 instructions in the worst case.
12901792 var inst_buf: [9]Air.Inst.Index = undefined;
1291 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1793 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
12921794 var main_block: Block = .init(&inst_buf);
12931795
12941796 // This check is a bit annoying because of floating-point rounding and the fact that this
......@@ -1771,6 +2273,9 @@ const Block = struct {
17712273 .data = .{ .br = .{ .block_inst = target, .operand = operand } },
17722274 });
17732275 }
2276 fn addTy(b: *Block, l: *Legalize, tag: Air.Inst.Tag, ty: Type) Air.Inst.Index {
2277 return b.add(l, .{ .tag = tag, .data = .{ .ty = ty } });
2278 }
17742279 fn addBinOp(b: *Block, l: *Legalize, tag: Air.Inst.Tag, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) Air.Inst.Index {
17752280 return b.add(l, .{
17762281 .tag = tag,
......@@ -1921,6 +2426,31 @@ const Block = struct {
19212426 }
19222427};
19232428
2429const Loop = struct {
2430 inst: Air.Inst.Index,
2431 block: Block,
2432
2433 /// The return value has `block` initialized to `undefined`; it is the caller's reponsibility
2434 /// to initialize it.
2435 fn init(l: *Legalize, parent_block: *Block) Loop {
2436 return .{
2437 .inst = parent_block.add(l, .{
2438 .tag = .loop,
2439 .data = .{ .ty_pl = .{
2440 .ty = .noreturn_type,
2441 .payload = undefined,
2442 } },
2443 }),
2444 .block = undefined,
2445 };
2446 }
2447
2448 fn finish(loop: Loop, l: *Legalize) Error!void {
2449 const data = &l.air_instructions.items(.data)[@intFromEnum(loop.inst)];
2450 data.ty_pl.payload = try l.addBlockBody(loop.block.body());
2451 }
2452};
2453
19242454const CondBr = struct {
19252455 inst: Air.Inst.Index,
19262456 hints: Air.CondBr.BranchHints,
src/Air/Liveness.zig+7
......@@ -458,6 +458,7 @@ fn analyzeInst(
458458 .memset_safe,
459459 .memcpy,
460460 .memmove,
461 .legalize_vec_elem_val,
461462 => {
462463 const o = inst_datas[@intFromEnum(inst)].bin_op;
463464 return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none });
......@@ -769,6 +770,12 @@ fn analyzeInst(
769770 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
770771 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none });
771772 },
773
774 .legalize_vec_store_elem => {
775 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
776 const bin = a.air.extraData(Air.Bin, pl_op.payload).data;
777 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, bin.lhs, bin.rhs });
778 },
772779 }
773780}
774781
src/Air/Liveness/Verify.zig+6
......@@ -272,6 +272,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
272272 .memset_safe,
273273 .memcpy,
274274 .memmove,
275 .legalize_vec_elem_val,
275276 => {
276277 const bin_op = data[@intFromEnum(inst)].bin_op;
277278 try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none });
......@@ -577,6 +578,11 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
577578
578579 try self.verifyInst(inst);
579580 },
581 .legalize_vec_store_elem => {
582 const pl_op = data[@intFromEnum(inst)].pl_op;
583 const bin = self.air.extraData(Air.Bin, pl_op.payload).data;
584 try self.verifyInstOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });
585 },
580586 }
581587 }
582588}
src/Air/print.zig+14
......@@ -171,6 +171,7 @@ const Writer = struct {
171171 .memmove,
172172 .memset,
173173 .memset_safe,
174 .legalize_vec_elem_val,
174175 => try w.writeBinOp(s, inst),
175176
176177 .is_null,
......@@ -331,6 +332,7 @@ const Writer = struct {
331332 .reduce, .reduce_optimized => try w.writeReduce(s, inst),
332333 .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),
333334 .runtime_nav_ptr => try w.writeRuntimeNavPtr(s, inst),
335 .legalize_vec_store_elem => try w.writeLegalizeVecStoreElem(s, inst),
334336
335337 .work_item_id,
336338 .work_group_size,
......@@ -508,6 +510,18 @@ const Writer = struct {
508510 try w.writeOperand(s, inst, 2, pl_op.operand);
509511 }
510512
513 fn writeLegalizeVecStoreElem(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {
514 const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
515 const bin = w.air.extraData(Air.Bin, pl_op.payload).data;
516
517 try w.writeOperand(s, inst, 0, pl_op.operand);
518 try s.writeAll(", ");
519 try w.writeOperand(s, inst, 1, bin.lhs);
520 try s.writeAll(", ");
521 try w.writeOperand(s, inst, 2, bin.rhs);
522 try s.writeAll(", ");
523 }
524
511525 fn writeShuffleOne(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {
512526 const unwrapped = w.air.unwrapShuffleOne(w.pt.zcu, inst);
513527 try w.writeType(s, unwrapped.result_ty);
src/Air/types_resolved.zig+2
......@@ -88,6 +88,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
8888 .atomic_store_monotonic,
8989 .atomic_store_release,
9090 .atomic_store_seq_cst,
91 .legalize_vec_elem_val,
9192 => {
9293 if (!checkRef(data.bin_op.lhs, zcu)) return false;
9394 if (!checkRef(data.bin_op.rhs, zcu)) return false;
......@@ -322,6 +323,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
322323
323324 .select,
324325 .mul_add,
326 .legalize_vec_store_elem,
325327 => {
326328 const bin = air.extraData(Air.Bin, data.pl_op.payload).data;
327329 if (!checkRef(data.pl_op.operand, zcu)) return false;
src/Sema.zig+14-9
......@@ -15930,16 +15930,21 @@ fn zirOverflowArithmetic(
1593015930 }
1593115931 }
1593215932 // If either of the arguments is one, the result is the other and no overflow occured.
15933 const scalar_one = try pt.intValue(dest_ty.scalarType(zcu), 1);
15934 const vec_one = try sema.splat(dest_ty, scalar_one);
15935 if (maybe_lhs_val) |lhs_val| {
15936 if (!lhs_val.isUndef(zcu) and try sema.compareAll(lhs_val, .eq, vec_one, dest_ty)) {
15937 break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };
15933 const dest_scalar_ty = dest_ty.scalarType(zcu);
15934 const dest_scalar_int = dest_scalar_ty.intInfo(zcu);
15935 // We could still be working with i1, where '1' is not a legal value!
15936 if (!(dest_scalar_int.bits == 1 and dest_scalar_int.signedness == .signed)) {
15937 const scalar_one = try pt.intValue(dest_scalar_ty, 1);
15938 const vec_one = try sema.splat(dest_ty, scalar_one);
15939 if (maybe_lhs_val) |lhs_val| {
15940 if (!lhs_val.isUndef(zcu) and try sema.compareAll(lhs_val, .eq, vec_one, dest_ty)) {
15941 break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };
15942 }
1593815943 }
15939 }
15940 if (maybe_rhs_val) |rhs_val| {
15941 if (!rhs_val.isUndef(zcu) and try sema.compareAll(rhs_val, .eq, vec_one, dest_ty)) {
15942 break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };
15944 if (maybe_rhs_val) |rhs_val| {
15945 if (!rhs_val.isUndef(zcu) and try sema.compareAll(rhs_val, .eq, vec_one, dest_ty)) {
15946 break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };
15947 }
1594315948 }
1594415949 }
1594515950
src/codegen/aarch64/Select.zig+9
......@@ -134,6 +134,10 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
134134 var air_inst_index = air_body[air_body_index];
135135 const initial_def_order_len = isel.def_order.count();
136136 air_tag: switch (air_tags[@intFromEnum(air_inst_index)]) {
137 // No "scalarize" legalizations are enabled, so these instructions never appear.
138 .legalize_vec_elem_val => unreachable,
139 .legalize_vec_store_elem => unreachable,
140
137141 .arg,
138142 .ret_addr,
139143 .frame_addr,
......@@ -950,6 +954,11 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
950954 };
951955 air_tag: switch (air.next().?) {
952956 else => |air_tag| return isel.fail("unimplemented {t}", .{air_tag}),
957
958 // No "scalarize" legalizations are enabled, so these instructions never appear.
959 .legalize_vec_elem_val => unreachable,
960 .legalize_vec_store_elem => unreachable,
961
953962 .arg => {
954963 const arg_vi = isel.live_values.fetchRemove(air.inst_index).?.value;
955964 defer arg_vi.deref(isel);
src/codegen/c.zig+4
......@@ -3325,6 +3325,10 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
33253325 // zig fmt: off
33263326 .inferred_alloc, .inferred_alloc_comptime => unreachable,
33273327
3328 // No "scalarize" legalizations are enabled, so these instructions never appear.
3329 .legalize_vec_elem_val => unreachable,
3330 .legalize_vec_store_elem => unreachable,
3331
33283332 .arg => try airArg(f, inst),
33293333
33303334 .breakpoint => try airBreakpoint(f),
src/codegen/llvm.zig+5
......@@ -4886,6 +4886,11 @@ pub const FuncGen = struct {
48864886
48874887 const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
48884888 // zig fmt: off
4889
4890 // No "scalarize" legalizations are enabled, so these instructions never appear.
4891 .legalize_vec_elem_val => unreachable,
4892 .legalize_vec_store_elem => unreachable,
4893
48894894 .add => try self.airAdd(inst, .normal),
48904895 .add_optimized => try self.airAdd(inst, .fast),
48914896 .add_wrap => try self.airAddWrap(inst),
src/codegen/riscv64/CodeGen.zig+5
......@@ -1391,6 +1391,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
13911391 const tag = air_tags[@intFromEnum(inst)];
13921392 switch (tag) {
13931393 // zig fmt: off
1394
1395 // No "scalarize" legalizations are enabled, so these instructions never appear.
1396 .legalize_vec_elem_val => unreachable,
1397 .legalize_vec_store_elem => unreachable,
1398
13941399 .add,
13951400 .add_wrap,
13961401 .sub,
src/codegen/sparc64/CodeGen.zig+5
......@@ -479,6 +479,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
479479 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
480480 switch (air_tags[@intFromEnum(inst)]) {
481481 // zig fmt: off
482
483 // No "scalarize" legalizations are enabled, so these instructions never appear.
484 .legalize_vec_elem_val => unreachable,
485 .legalize_vec_store_elem => unreachable,
486
482487 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
483488 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
484489
src/codegen/wasm/CodeGen.zig+4
......@@ -1786,6 +1786,10 @@ fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum
17861786fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
17871787 const air_tags = cg.air.instructions.items(.tag);
17881788 return switch (air_tags[@intFromEnum(inst)]) {
1789 // No "scalarize" legalizations are enabled, so these instructions never appear.
1790 .legalize_vec_elem_val => unreachable,
1791 .legalize_vec_store_elem => unreachable,
1792
17891793 .inferred_alloc, .inferred_alloc_comptime => unreachable,
17901794
17911795 .add => cg.airBinOp(inst, .add),
src/codegen/x86_64/CodeGen.zig+629-1
......@@ -103926,7 +103926,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103926103926 try ops[0].toOffset(0, cg);
103927103927 try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg);
103928103928 },
103929 .array_elem_val => {
103929 .array_elem_val, .legalize_vec_elem_val => {
103930103930 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
103931103931 const array_ty = cg.typeOf(bin_op.lhs);
103932103932 const res_ty = array_ty.elemType2(zcu);
......@@ -173061,6 +173061,634 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
173061173061 .c_va_copy => try cg.airVaCopy(inst),
173062173062 .c_va_end => try cg.airVaEnd(inst),
173063173063 .c_va_start => try cg.airVaStart(inst),
173064 .legalize_vec_store_elem => {
173065 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
173066 const bin = cg.air.extraData(Air.Bin, pl_op.payload).data;
173067 // vector_ptr, index, elem_val
173068 var ops = try cg.tempsFromOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });
173069 cg.select(&.{}, &.{}, &ops, comptime &.{ .{
173070 .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },
173071 .patterns = &.{
173072 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } },
173073 },
173074 .extra_temps = .{
173075 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
173076 .unused,
173077 .unused,
173078 .unused,
173079 .unused,
173080 .unused,
173081 .unused,
173082 .unused,
173083 .unused,
173084 .unused,
173085 .unused,
173086 },
173087 .clobbers = .{ .eflags = true },
173088 .each = .{ .once = &.{
173089 .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ },
173090 .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ },
173091 .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ },
173092 } },
173093 }, .{
173094 .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },
173095 .patterns = &.{
173096 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } },
173097 },
173098 .extra_temps = .{
173099 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
173100 .unused,
173101 .unused,
173102 .unused,
173103 .unused,
173104 .unused,
173105 .unused,
173106 .unused,
173107 .unused,
173108 .unused,
173109 .unused,
173110 },
173111 .clobbers = .{ .eflags = true },
173112 .each = .{ .once = &.{
173113 .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ },
173114 .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ },
173115 .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ },
173116 } },
173117 }, .{
173118 .required_features = .{ .cmov, null, null, null },
173119 .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },
173120 .patterns = &.{
173121 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173122 },
173123 .extra_temps = .{
173124 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
173125 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
173126 .unused,
173127 .unused,
173128 .unused,
173129 .unused,
173130 .unused,
173131 .unused,
173132 .unused,
173133 .unused,
173134 .unused,
173135 },
173136 .clobbers = .{ .eflags = true },
173137 .each = .{ .once = &.{
173138 .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ },
173139 .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ },
173140 .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ },
173141 .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ },
173142 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173143 .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ },
173144 .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ },
173145 } },
173146 }, .{
173147 .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },
173148 .patterns = &.{
173149 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173150 },
173151 .extra_temps = .{
173152 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
173153 .unused,
173154 .unused,
173155 .unused,
173156 .unused,
173157 .unused,
173158 .unused,
173159 .unused,
173160 .unused,
173161 .unused,
173162 .unused,
173163 },
173164 .clobbers = .{ .eflags = true },
173165 .each = .{ .once = &.{
173166 .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ },
173167 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173168 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
173169 .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ },
173170 .{ ._, ._mp, .j, .@"1f", ._, ._, ._ },
173171 .{ .@"0:", ._s, .bt, .tmp0d, .src1d, ._, ._ },
173172 .{ .@"1:", ._, .mov, .lea(.src0b), .tmp0b, ._, ._ },
173173 } },
173174 }, .{
173175 .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool },
173176 .patterns = &.{
173177 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } },
173178 },
173179 .clobbers = .{ .eflags = true },
173180 .each = .{ .once = &.{
173181 .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ },
173182 } },
173183 }, .{
173184 .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool },
173185 .patterns = &.{
173186 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } },
173187 },
173188 .clobbers = .{ .eflags = true },
173189 .each = .{ .once = &.{
173190 .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ },
173191 } },
173192 }, .{
173193 .required_features = .{ .cmov, null, null, null },
173194 .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool },
173195 .patterns = &.{
173196 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173197 },
173198 .extra_temps = .{
173199 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
173200 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
173201 .unused,
173202 .unused,
173203 .unused,
173204 .unused,
173205 .unused,
173206 .unused,
173207 .unused,
173208 .unused,
173209 .unused,
173210 },
173211 .clobbers = .{ .eflags = true },
173212 .each = .{ .once = &.{
173213 .{ ._, ._, .movzx, .tmp0d, .lea(.src0w), ._, ._ },
173214 .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ },
173215 .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ },
173216 .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ },
173217 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173218 .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ },
173219 .{ ._, ._, .mov, .lea(.src0w), .tmp0w, ._, ._ },
173220 } },
173221 }, .{
173222 .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool },
173223 .patterns = &.{
173224 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173225 },
173226 .clobbers = .{ .eflags = true },
173227 .each = .{ .once = &.{
173228 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173229 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
173230 .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ },
173231 .{ ._, ._mp, .j, .@"0f", ._, ._, ._ },
173232 .{ .@"1:", ._s, .bt, .lea(.src0w), .src1w, ._, ._ },
173233 } },
173234 }, .{
173235 .src_constraints = .{ .ptr_any_bool_vec, .any, .bool },
173236 .patterns = &.{
173237 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } },
173238 },
173239 .clobbers = .{ .eflags = true },
173240 .each = .{ .once = &.{
173241 .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ },
173242 } },
173243 }, .{
173244 .src_constraints = .{ .ptr_any_bool_vec, .any, .bool },
173245 .patterns = &.{
173246 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } },
173247 },
173248 .clobbers = .{ .eflags = true },
173249 .each = .{ .once = &.{
173250 .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ },
173251 } },
173252 }, .{
173253 .required_features = .{ .cmov, null, null, null },
173254 .src_constraints = .{ .{ .ptr_bool_vec = .dword }, .any, .bool },
173255 .patterns = &.{
173256 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173257 },
173258 .extra_temps = .{
173259 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
173260 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
173261 .unused,
173262 .unused,
173263 .unused,
173264 .unused,
173265 .unused,
173266 .unused,
173267 .unused,
173268 .unused,
173269 .unused,
173270 },
173271 .clobbers = .{ .eflags = true },
173272 .each = .{ .once = &.{
173273 .{ ._, ._, .mov, .tmp0d, .lea(.src0d), ._, ._ },
173274 .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ },
173275 .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ },
173276 .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ },
173277 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173278 .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ },
173279 .{ ._, ._, .mov, .lea(.src0d), .tmp0d, ._, ._ },
173280 } },
173281 }, .{
173282 .required_features = .{ .@"64bit", .cmov, null, null },
173283 .src_constraints = .{ .{ .ptr_bool_vec = .qword }, .any, .bool },
173284 .patterns = &.{
173285 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173286 },
173287 .extra_temps = .{
173288 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
173289 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
173290 .unused,
173291 .unused,
173292 .unused,
173293 .unused,
173294 .unused,
173295 .unused,
173296 .unused,
173297 .unused,
173298 .unused,
173299 },
173300 .clobbers = .{ .eflags = true },
173301 .each = .{ .once = &.{
173302 .{ ._, ._, .mov, .tmp0q, .lea(.src0q), ._, ._ },
173303 .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ },
173304 .{ ._, ._r, .bt, .tmp1q, .src1q, ._, ._ },
173305 .{ ._, ._s, .bt, .tmp0q, .src1q, ._, ._ },
173306 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173307 .{ ._, ._z, .cmov, .tmp0q, .tmp1q, ._, ._ },
173308 .{ ._, ._, .mov, .lea(.src0q), .tmp0q, ._, ._ },
173309 } },
173310 }, .{
173311 .required_features = .{ .cmov, null, null, null },
173312 .src_constraints = .{ .ptr_any_bool_vec, .any, .bool },
173313 .patterns = &.{
173314 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173315 },
173316 .extra_temps = .{
173317 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
173318 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
173319 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
173320 .unused,
173321 .unused,
173322 .unused,
173323 .unused,
173324 .unused,
173325 .unused,
173326 .unused,
173327 .unused,
173328 },
173329 .clobbers = .{ .eflags = true },
173330 .each = .{ .once = &.{
173331 .{ ._, ._, .mov, .tmp0d, .src1d, ._, ._ },
173332 .{ ._, ._r, .sh, .tmp0d, .ui(5), ._, ._ },
173333 .{ ._, ._, .mov, .tmp1d, .leasi(.src0d, .@"4", .tmp0), ._, ._ },
173334 .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ },
173335 .{ ._, ._r, .bt, .tmp2d, .src1d, ._, ._ },
173336 .{ ._, ._s, .bt, .tmp1d, .src1d, ._, ._ },
173337 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173338 .{ ._, ._z, .cmov, .tmp1d, .tmp2d, ._, ._ },
173339 .{ ._, ._, .mov, .leasi(.src0d, .@"4", .tmp0), .tmp1d, ._, ._ },
173340 } },
173341 }, .{
173342 .src_constraints = .{ .ptr_any_bool_vec, .any, .bool },
173343 .patterns = &.{
173344 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173345 },
173346 .clobbers = .{ .eflags = true },
173347 .each = .{ .once = &.{
173348 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173349 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
173350 .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ },
173351 .{ ._, ._mp, .j, .@"0f", ._, ._, ._ },
173352 .{ .@"1:", ._s, .bt, .lea(.src0d), .src1d, ._, ._ },
173353 } },
173354 }, .{
173355 .src_constraints = .{ .any, .any, .{ .int = .byte } },
173356 .patterns = &.{
173357 .{ .src = .{ .to_gpr, .simm32, .imm8 } },
173358 .{ .src = .{ .to_gpr, .simm32, .to_gpr } },
173359 },
173360 .each = .{ .once = &.{
173361 .{ ._, ._, .mov, .leaa(.src0b, .add_src0_elem_size_mul_src1), .src2b, ._, ._ },
173362 } },
173363 }, .{
173364 .src_constraints = .{ .any, .any, .{ .int = .byte } },
173365 .patterns = &.{
173366 .{ .src = .{ .to_gpr, .to_gpr, .imm8 } },
173367 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173368 },
173369 .each = .{ .once = &.{
173370 .{ ._, ._, .mov, .leai(.src0b, .src1), .src2b, ._, ._ },
173371 } },
173372 }, .{
173373 .src_constraints = .{ .any, .any, .{ .int = .word } },
173374 .patterns = &.{
173375 .{ .src = .{ .to_gpr, .simm32, .imm16 } },
173376 .{ .src = .{ .to_gpr, .simm32, .to_gpr } },
173377 },
173378 .each = .{ .once = &.{
173379 .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2w, ._, ._ },
173380 } },
173381 }, .{
173382 .src_constraints = .{ .any, .any, .{ .int = .word } },
173383 .patterns = &.{
173384 .{ .src = .{ .to_gpr, .to_gpr, .imm16 } },
173385 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173386 },
173387 .each = .{ .once = &.{
173388 .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .src2w, ._, ._ },
173389 } },
173390 }, .{
173391 .required_features = .{ .avx, null, null, null },
173392 .src_constraints = .{ .any, .any, .{ .float = .word } },
173393 .patterns = &.{
173394 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173395 },
173396 .each = .{ .once = &.{
173397 .{ ._, .vp_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ },
173398 } },
173399 }, .{
173400 .required_features = .{ .sse4_1, null, null, null },
173401 .src_constraints = .{ .any, .any, .{ .float = .word } },
173402 .patterns = &.{
173403 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173404 },
173405 .each = .{ .once = &.{
173406 .{ ._, .p_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ },
173407 } },
173408 }, .{
173409 .required_features = .{ .sse2, null, null, null },
173410 .src_constraints = .{ .any, .any, .{ .float = .word } },
173411 .patterns = &.{
173412 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173413 },
173414 .extra_temps = .{
173415 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
173416 .unused,
173417 .unused,
173418 .unused,
173419 .unused,
173420 .unused,
173421 .unused,
173422 .unused,
173423 .unused,
173424 .unused,
173425 .unused,
173426 },
173427 .each = .{ .once = &.{
173428 .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ },
173429 .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp0w, ._, ._ },
173430 } },
173431 }, .{
173432 .required_features = .{ .sse, null, null, null },
173433 .src_constraints = .{ .any, .any, .{ .float = .word } },
173434 .patterns = &.{
173435 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173436 },
173437 .extra_temps = .{
173438 .{ .type = .f32, .kind = .mem },
173439 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
173440 .unused,
173441 .unused,
173442 .unused,
173443 .unused,
173444 .unused,
173445 .unused,
173446 .unused,
173447 .unused,
173448 .unused,
173449 },
173450 .each = .{ .once = &.{
173451 .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ },
173452 .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ },
173453 .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp1w, ._, ._ },
173454 } },
173455 }, .{
173456 .required_features = .{ .avx, null, null, null },
173457 .src_constraints = .{ .any, .any, .{ .float = .word } },
173458 .patterns = &.{
173459 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173460 },
173461 .each = .{ .once = &.{
173462 .{ ._, .vp_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ },
173463 } },
173464 }, .{
173465 .required_features = .{ .sse4_1, null, null, null },
173466 .src_constraints = .{ .any, .any, .{ .float = .word } },
173467 .patterns = &.{
173468 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173469 },
173470 .each = .{ .once = &.{
173471 .{ ._, .p_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ },
173472 } },
173473 }, .{
173474 .required_features = .{ .sse2, null, null, null },
173475 .src_constraints = .{ .any, .any, .{ .float = .word } },
173476 .patterns = &.{
173477 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173478 },
173479 .extra_temps = .{
173480 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
173481 .unused,
173482 .unused,
173483 .unused,
173484 .unused,
173485 .unused,
173486 .unused,
173487 .unused,
173488 .unused,
173489 .unused,
173490 .unused,
173491 },
173492 .each = .{ .once = &.{
173493 .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ },
173494 .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp0w, ._, ._ },
173495 } },
173496 }, .{
173497 .required_features = .{ .sse, null, null, null },
173498 .src_constraints = .{ .any, .any, .{ .float = .word } },
173499 .patterns = &.{
173500 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173501 },
173502 .extra_temps = .{
173503 .{ .type = .f32, .kind = .mem },
173504 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
173505 .unused,
173506 .unused,
173507 .unused,
173508 .unused,
173509 .unused,
173510 .unused,
173511 .unused,
173512 .unused,
173513 .unused,
173514 },
173515 .each = .{ .once = &.{
173516 .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ },
173517 .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ },
173518 .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp1w, ._, ._ },
173519 } },
173520 }, .{
173521 .src_constraints = .{ .any, .any, .{ .int = .dword } },
173522 .patterns = &.{
173523 .{ .src = .{ .to_gpr, .simm32, .imm32 } },
173524 .{ .src = .{ .to_gpr, .simm32, .to_gpr } },
173525 },
173526 .each = .{ .once = &.{
173527 .{ ._, ._, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2d, ._, ._ },
173528 } },
173529 }, .{
173530 .src_constraints = .{ .any, .any, .{ .int = .dword } },
173531 .patterns = &.{
173532 .{ .src = .{ .to_gpr, .to_gpr, .imm32 } },
173533 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173534 },
173535 .each = .{ .once = &.{
173536 .{ ._, ._, .mov, .leasi(.src0d, .@"4", .src1), .src2d, ._, ._ },
173537 } },
173538 }, .{
173539 .required_features = .{ .avx, null, null, null },
173540 .src_constraints = .{ .any, .any, .{ .float = .dword } },
173541 .patterns = &.{
173542 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173543 },
173544 .each = .{ .once = &.{
173545 .{ ._, .v_ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
173546 } },
173547 }, .{
173548 .required_features = .{ .sse, null, null, null },
173549 .src_constraints = .{ .any, .any, .{ .float = .dword } },
173550 .patterns = &.{
173551 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173552 },
173553 .each = .{ .once = &.{
173554 .{ ._, ._ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
173555 } },
173556 }, .{
173557 .required_features = .{ .avx, null, null, null },
173558 .src_constraints = .{ .any, .any, .{ .float = .dword } },
173559 .patterns = &.{
173560 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173561 },
173562 .each = .{ .once = &.{
173563 .{ ._, .v_ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ },
173564 } },
173565 }, .{
173566 .required_features = .{ .sse, null, null, null },
173567 .src_constraints = .{ .any, .any, .{ .float = .dword } },
173568 .patterns = &.{
173569 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173570 },
173571 .each = .{ .once = &.{
173572 .{ ._, ._ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ },
173573 } },
173574 }, .{
173575 .required_features = .{ .@"64bit", null, null, null },
173576 .src_constraints = .{ .any, .any, .{ .int = .qword } },
173577 .patterns = &.{
173578 .{ .src = .{ .to_gpr, .simm32, .simm32 } },
173579 .{ .src = .{ .to_gpr, .simm32, .to_gpr } },
173580 },
173581 .each = .{ .once = &.{
173582 .{ ._, ._, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2q, ._, ._ },
173583 } },
173584 }, .{
173585 .required_features = .{ .@"64bit", null, null, null },
173586 .src_constraints = .{ .any, .any, .{ .int = .qword } },
173587 .patterns = &.{
173588 .{ .src = .{ .to_gpr, .to_gpr, .simm32 } },
173589 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173590 },
173591 .each = .{ .once = &.{
173592 .{ ._, ._, .mov, .leasi(.src0q, .@"8", .src1), .src2q, ._, ._ },
173593 } },
173594 }, .{
173595 .required_features = .{ .avx, null, null, null },
173596 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173597 .patterns = &.{
173598 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173599 },
173600 .each = .{ .once = &.{
173601 .{ ._, .v_sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
173602 } },
173603 }, .{
173604 .required_features = .{ .sse2, null, null, null },
173605 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173606 .patterns = &.{
173607 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173608 },
173609 .each = .{ .once = &.{
173610 .{ ._, ._sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
173611 } },
173612 }, .{
173613 .required_features = .{ .sse, null, null, null },
173614 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173615 .patterns = &.{
173616 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173617 },
173618 .each = .{ .once = &.{
173619 .{ ._, ._ps, .movl, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
173620 } },
173621 }, .{
173622 .required_features = .{ .avx, null, null, null },
173623 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173624 .patterns = &.{
173625 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173626 },
173627 .each = .{ .once = &.{
173628 .{ ._, .v_sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ },
173629 } },
173630 }, .{
173631 .required_features = .{ .sse2, null, null, null },
173632 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173633 .patterns = &.{
173634 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173635 },
173636 .each = .{ .once = &.{
173637 .{ ._, ._sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ },
173638 } },
173639 }, .{
173640 .required_features = .{ .sse, null, null, null },
173641 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173642 .patterns = &.{
173643 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173644 },
173645 .each = .{ .once = &.{
173646 .{ ._, ._ps, .movl, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ },
173647 } },
173648 } }) catch |err| switch (err) {
173649 error.SelectFailed => {
173650 const elem_size = cg.typeOf(bin.rhs).abiSize(zcu);
173651 while (try ops[0].toRegClass(true, .general_purpose, cg) or
173652 try ops[1].toRegClass(true, .general_purpose, cg))
173653 {}
173654 const base_reg = ops[0].tracking(cg).short.register.to64();
173655 const rhs_reg = ops[1].tracking(cg).short.register.to64();
173656 if (!std.math.isPowerOfTwo(elem_size)) {
173657 try cg.spillEflagsIfOccupied();
173658 try cg.asmRegisterRegisterImmediate(
173659 .{ .i_, .mul },
173660 rhs_reg,
173661 rhs_reg,
173662 .u(elem_size),
173663 );
173664 try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{
173665 .base = .{ .reg = base_reg },
173666 .mod = .{ .rm = .{ .index = rhs_reg } },
173667 });
173668 } else if (elem_size > 8) {
173669 try cg.spillEflagsIfOccupied();
173670 try cg.asmRegisterImmediate(
173671 .{ ._l, .sh },
173672 rhs_reg,
173673 .u(std.math.log2_int(u64, elem_size)),
173674 );
173675 try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{
173676 .base = .{ .reg = base_reg },
173677 .mod = .{ .rm = .{ .index = rhs_reg } },
173678 });
173679 } else try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{
173680 .base = .{ .reg = base_reg },
173681 .mod = .{ .rm = .{
173682 .index = rhs_reg,
173683 .scale = .fromFactor(@intCast(elem_size)),
173684 } },
173685 });
173686 try ops[0].store(&ops[2], .{}, cg);
173687 },
173688 else => |e| return e,
173689 };
173690 for (ops) |op| try op.die(cg);
173691 },
173064173692 .work_item_id, .work_group_size, .work_group_id => unreachable,
173065173693 }
173066173694 try cg.resetTemps(@enumFromInt(0));