authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-11-09 15:16:49+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-11-12 16:00:16+00:00
log69f39868b4125e79e4070a88bbdfcd3643dbc90d
tree7e90f08d2b5d1cb234957dbe0b6a48034afe4e30
parent99a7884308d288bd39df9192c9094439b179ff60
signaturelock-open Commit is signed but in an unrecognized format.

Air.Legalize: revert to loops for scalarizations

I had tried unrolling the loops to avoid requiring the `vector_store_elem` instruction, but it's arguably a problem to generate O(N) code for an operation on `@Vector(N, T)`. In addition, that lowering emitted a lot of `.aggregate_init` instructions, which is itself a quite difficult operation to codegen. This requires reintroducing runtime vector indexing internally. However, I've put it in a couple of instructions which are intended only for use by `Air.Legalize`, named `legalize_vec_elem_val` (like `array_elem_val`, but for indexing a vector with a runtime-known index) and `legalize_vec_store_elem` (like the old `vector_store_elem` instruction). These are explicitly documented as *not* being emitted by Sema, so need only be implemented by backends if they actually use an `Air.Legalize.Feature` which emits them (otherwise they can be marked as `unreachable`).

14 files changed, 1548 insertions(+), 301 deletions(-)

src/Air.zig+26-3
...@@ -660,8 +660,8 @@ pub const Inst = struct {...@@ -660,8 +660,8 @@ pub const Inst = struct {
660 /// Given a pointer to a slice, return a pointer to the pointer of the slice.660 /// Given a pointer to a slice, return a pointer to the pointer of the slice.
661 /// Uses the `ty_op` field.661 /// Uses the `ty_op` field.
662 ptr_slice_ptr_ptr,662 ptr_slice_ptr_ptr,
663 /// Given an (array value or vector value) and element index,663 /// Given an (array value or vector value) and element index, return the element value at
664 /// return the element value at that index.664 /// that index. If the lhs is a vector value, the index is guaranteed to be comptime-known.
665 /// Result type is the element type of the array operand.665 /// Result type is the element type of the array operand.
666 /// Uses the `bin_op` field.666 /// Uses the `bin_op` field.
667 array_elem_val,667 array_elem_val,
...@@ -915,6 +915,26 @@ pub const Inst = struct {...@@ -915,6 +915,26 @@ pub const Inst = struct {
915 /// Operand is unused and set to Ref.none915 /// Operand is unused and set to Ref.none
916 work_group_id,916 work_group_id,
917917
918 // The remaining instructions are not emitted by Sema. They are only emitted by `Legalize`,
919 // depending on the enabled features. As such, backends can consider them `unreachable` if
920 // they do not enable the relevant legalizations.
921
922 /// Given a pointer to a vector, a runtime-known index, and a scalar value, store the value
923 /// into the vector at the given index. Zig does not support this operation, but `Legalize`
924 /// may emit it when scalarizing vector operations.
925 ///
926 /// Uses the `pl_op` field with payload `Bin`. `operand` is the vector pointer. `lhs` is the
927 /// element index of type `usize`. `rhs` is the element value. Result is always void.
928 legalize_vec_store_elem,
929 /// Given a vector value and a runtime-known index, return the element value at that index.
930 /// This instruction is similar to `array_elem_val`; the only difference is that the index
931 /// here is runtime-known, which is usually not allowed for vectors. `Legalize` may emit
932 /// this instruction when scalarizing vector operations.
933 ///
934 /// Uses the `bin_op` field. `lhs` is the vector pointer. `rhs` is the element index. Result
935 /// type is the vector element type.
936 legalize_vec_elem_val,
937
918 pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {938 pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {
919 switch (op) {939 switch (op) {
920 .lt => return if (optimized) .cmp_lt_optimized else .cmp_lt,940 .lt => return if (optimized) .cmp_lt_optimized else .cmp_lt,
...@@ -1681,6 +1701,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)...@@ -1681,6 +1701,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
1681 .prefetch,1701 .prefetch,
1682 .set_err_return_trace,1702 .set_err_return_trace,
1683 .c_va_end,1703 .c_va_end,
1704 .legalize_vec_store_elem,
1684 => return .void,1705 => return .void,
16851706
1686 .slice_len,1707 .slice_len,
...@@ -1699,7 +1720,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)...@@ -1699,7 +1720,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
1699 return .fromInterned(ip.funcTypeReturnType(callee_ty.toIntern()));1720 return .fromInterned(ip.funcTypeReturnType(callee_ty.toIntern()));
1700 },1721 },
17011722
1702 .slice_elem_val, .ptr_elem_val, .array_elem_val => {1723 .slice_elem_val, .ptr_elem_val, .array_elem_val, .legalize_vec_elem_val => {
1703 const ptr_ty = air.typeOf(datas[@intFromEnum(inst)].bin_op.lhs, ip);1724 const ptr_ty = air.typeOf(datas[@intFromEnum(inst)].bin_op.lhs, ip);
1704 return ptr_ty.childTypeIp(ip);1725 return ptr_ty.childTypeIp(ip);
1705 },1726 },
...@@ -1857,6 +1878,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {...@@ -1857,6 +1878,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1857 .intcast_safe,1878 .intcast_safe,
1858 .int_from_float_safe,1879 .int_from_float_safe,
1859 .int_from_float_optimized_safe,1880 .int_from_float_optimized_safe,
1881 .legalize_vec_store_elem,
1860 => true,1882 => true,
18611883
1862 .add,1884 .add,
...@@ -2002,6 +2024,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {...@@ -2002,6 +2024,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
2002 .work_item_id,2024 .work_item_id,
2003 .work_group_size,2025 .work_group_size,
2004 .work_group_id,2026 .work_group_id,
2027 .legalize_vec_elem_val,
2005 => false,2028 => false,
20062029
2007 .is_non_null_ptr, .is_null_ptr, .is_non_err_ptr, .is_err_ptr => air.typeOf(data.un_op, ip).isVolatilePtrIp(ip),2030 .is_non_null_ptr, .is_null_ptr, .is_non_err_ptr, .is_err_ptr => air.typeOf(data.un_op, ip).isVolatilePtrIp(ip),
src/Air/Legalize.zig+818-288
...@@ -320,28 +320,36 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -320,28 +320,36 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
320 .xor,320 .xor,
321 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {321 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
322 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;322 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
323 if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);323 if (l.typeOf(bin_op.lhs).isVector(zcu)) {
324 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
325 }
324 },326 },
325 .add_safe => if (l.features.has(.expand_add_safe)) {327 .add_safe => if (l.features.has(.expand_add_safe)) {
326 assert(!l.features.has(.scalarize_add_safe)); // it doesn't make sense to do both328 assert(!l.features.has(.scalarize_add_safe)); // it doesn't make sense to do both
327 continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow));329 continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow));
328 } else if (l.features.has(.scalarize_add_safe)) {330 } else if (l.features.has(.scalarize_add_safe)) {
329 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;331 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
330 if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);332 if (l.typeOf(bin_op.lhs).isVector(zcu)) {
333 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
334 }
331 },335 },
332 .sub_safe => if (l.features.has(.expand_sub_safe)) {336 .sub_safe => if (l.features.has(.expand_sub_safe)) {
333 assert(!l.features.has(.scalarize_sub_safe)); // it doesn't make sense to do both337 assert(!l.features.has(.scalarize_sub_safe)); // it doesn't make sense to do both
334 continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow));338 continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow));
335 } else if (l.features.has(.scalarize_sub_safe)) {339 } else if (l.features.has(.scalarize_sub_safe)) {
336 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;340 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
337 if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);341 if (l.typeOf(bin_op.lhs).isVector(zcu)) {
342 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
343 }
338 },344 },
339 .mul_safe => if (l.features.has(.expand_mul_safe)) {345 .mul_safe => if (l.features.has(.expand_mul_safe)) {
340 assert(!l.features.has(.scalarize_mul_safe)); // it doesn't make sense to do both346 assert(!l.features.has(.scalarize_mul_safe)); // it doesn't make sense to do both
341 continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow));347 continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow));
342 } else if (l.features.has(.scalarize_mul_safe)) {348 } else if (l.features.has(.scalarize_mul_safe)) {
343 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;349 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
344 if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);350 if (l.typeOf(bin_op.lhs).isVector(zcu)) {
351 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
352 }
345 },353 },
346 .ptr_add, .ptr_sub => {},354 .ptr_add, .ptr_sub => {},
347 inline .add_with_overflow,355 inline .add_with_overflow,
...@@ -350,7 +358,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -350,7 +358,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
350 .shl_with_overflow,358 .shl_with_overflow,
351 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {359 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
352 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;360 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
353 if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst));361 if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) {
362 continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst));
363 }
354 },364 },
355 .alloc => {},365 .alloc => {},
356 .inferred_alloc, .inferred_alloc_comptime => unreachable,366 .inferred_alloc, .inferred_alloc_comptime => unreachable,
...@@ -387,7 +397,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -387,7 +397,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
387 }397 }
388 }398 }
389 }399 }
390 if (l.features.has(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op);400 if (l.features.has(comptime .scalarize(air_tag))) {
401 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
402 }
391 }403 }
392 },404 },
393 inline .not,405 inline .not,
...@@ -406,7 +418,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -406,7 +418,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
406 .float_from_int,418 .float_from_int,
407 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {419 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
408 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;420 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
409 if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);421 if (ty_op.ty.toType().isVector(zcu)) {
422 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
423 }
410 },424 },
411 .bitcast => if (l.features.has(.scalarize_bitcast)) {425 .bitcast => if (l.features.has(.scalarize_bitcast)) {
412 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {426 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {
...@@ -418,21 +432,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -418,21 +432,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
418 continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));432 continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
419 } else if (l.features.has(.scalarize_intcast_safe)) {433 } else if (l.features.has(.scalarize_intcast_safe)) {
420 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;434 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
421 if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);435 if (ty_op.ty.toType().isVector(zcu)) {
436 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
437 }
422 },438 },
423 .int_from_float_safe => if (l.features.has(.expand_int_from_float_safe)) {439 .int_from_float_safe => if (l.features.has(.expand_int_from_float_safe)) {
424 assert(!l.features.has(.scalarize_int_from_float_safe));440 assert(!l.features.has(.scalarize_int_from_float_safe));
425 continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, false));441 continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, false));
426 } else if (l.features.has(.scalarize_int_from_float_safe)) {442 } else if (l.features.has(.scalarize_int_from_float_safe)) {
427 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;443 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
428 if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);444 if (ty_op.ty.toType().isVector(zcu)) {
445 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
446 }
429 },447 },
430 .int_from_float_optimized_safe => if (l.features.has(.expand_int_from_float_optimized_safe)) {448 .int_from_float_optimized_safe => if (l.features.has(.expand_int_from_float_optimized_safe)) {
431 assert(!l.features.has(.scalarize_int_from_float_optimized_safe));449 assert(!l.features.has(.scalarize_int_from_float_optimized_safe));
432 continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, true));450 continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, true));
433 } else if (l.features.has(.scalarize_int_from_float_optimized_safe)) {451 } else if (l.features.has(.scalarize_int_from_float_optimized_safe)) {
434 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;452 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
435 if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);453 if (ty_op.ty.toType().isVector(zcu)) {
454 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
455 }
436 },456 },
437 .block, .loop => {457 .block, .loop => {
438 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;458 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...@@ -467,7 +487,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -467,7 +487,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
467 .neg_optimized,487 .neg_optimized,
468 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {488 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
469 const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;489 const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;
470 if (l.typeOf(un_op).isVector(zcu)) continue :inst try l.scalarize(inst, .un_op);490 if (l.typeOf(un_op).isVector(zcu)) {
491 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .un_op));
492 }
471 },493 },
472 .cmp_lt,494 .cmp_lt,
473 .cmp_lt_optimized,495 .cmp_lt_optimized,
...@@ -484,7 +506,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -484,7 +506,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
484 => {},506 => {},
485 inline .cmp_vector, .cmp_vector_optimized => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {507 inline .cmp_vector, .cmp_vector_optimized => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
486 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;508 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
487 if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .cmp_vector);509 if (ty_pl.ty.toType().isVector(zcu)) {
510 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .cmp_vector));
511 }
488 },512 },
489 .cond_br => {513 .cond_br => {
490 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;514 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
...@@ -614,9 +638,15 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -614,9 +638,15 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
614 else => {},638 else => {},
615 }639 }
616 },640 },
617 .shuffle_one => if (l.features.has(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one),641 .shuffle_one => if (l.features.has(.scalarize_shuffle_one)) {
618 .shuffle_two => if (l.features.has(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two),642 continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleOneBlockPayload(inst));
619 .select => if (l.features.has(.scalarize_select)) continue :inst try l.scalarize(inst, .select),643 },
644 .shuffle_two => if (l.features.has(.scalarize_shuffle_two)) {
645 continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleTwoBlockPayload(inst));
646 },
647 .select => if (l.features.has(.scalarize_select)) {
648 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .select));
649 },
620 .memset,650 .memset,
621 .memset_safe,651 .memset_safe,
622 .memcpy,652 .memcpy,
...@@ -657,7 +687,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -657,7 +687,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
657 .union_init, .prefetch => {},687 .union_init, .prefetch => {},
658 .mul_add => if (l.features.has(.scalarize_mul_add)) {688 .mul_add => if (l.features.has(.scalarize_mul_add)) {
659 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;689 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
660 if (l.typeOf(pl_op.operand).isVector(zcu)) continue :inst try l.scalarize(inst, .pl_op_bin);690 if (l.typeOf(pl_op.operand).isVector(zcu)) {
691 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .pl_op_bin));
692 }
661 },693 },
662 .field_parent_ptr,694 .field_parent_ptr,
663 .wasm_memory_size,695 .wasm_memory_size,
...@@ -675,96 +707,123 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -675,96 +707,123 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
675 .work_item_id,707 .work_item_id,
676 .work_group_size,708 .work_group_size,
677 .work_group_id,709 .work_group_id,
710 .legalize_vec_elem_val,
711 .legalize_vec_store_elem,
678 => {},712 => {},
679 }713 }
680 }714 }
681}715}
682716
683const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, cmp_vector, shuffle_one, shuffle_two, select };717const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, cmp_vector, select };
684/// inline to propagate comptime-known `replaceInst` result.718fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: ScalarizeForm) Error!Air.Inst.Data {
685inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag {
686 return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form));
687}
688fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Data {
689 const pt = l.pt;719 const pt = l.pt;
690 const zcu = pt.zcu;720 const zcu = pt.zcu;
691 const gpa = zcu.gpa;
692721
693 const orig = l.air_instructions.get(@intFromEnum(orig_inst));722 const orig = l.air_instructions.get(@intFromEnum(orig_inst));
694 const res_ty = l.typeOfIndex(orig_inst);723 const res_ty = l.typeOfIndex(orig_inst);
695 const res_len = res_ty.vectorLen(zcu);724 const result_is_array = switch (res_ty.zigTypeTag(zcu)) {
725 .vector => false,
726 .array => true,
727 else => unreachable,
728 };
729 const res_len = res_ty.arrayLen(zcu);
730 const res_elem_ty = res_ty.childType(zcu);
696731
697 const inst_per_elem = switch (form) {732 if (result_is_array) {
733 // This is only allowed when legalizing an elementwise bitcast.
734 assert(orig.tag == .bitcast);
735 assert(form == .ty_op);
736 }
737
738 // Our output will be a loop doing elementwise stores:
739 //
740 // %1 = block(@Vector(N, Scalar), {
741 // %2 = alloc(*usize)
742 // %3 = alloc(*@Vector(N, Scalar))
743 // %4 = store(%2, @zero_usize)
744 // %5 = loop({
745 // %6 = load(%2)
746 // %7 = <scalar result of operation at index %5>
747 // %8 = legalize_vec_store_elem(%3, %5, %6)
748 // %9 = cmp_eq(%6, <usize, N-1>)
749 // %10 = cond_br(%9, {
750 // %11 = load(%3)
751 // %12 = br(%1, %11)
752 // }, {
753 // %13 = add(%6, @one_usize)
754 // %14 = store(%2, %13)
755 // %15 = repeat(%5)
756 // })
757 // })
758 // })
759 //
760 // If scalarizing an elementwise bitcast, the result might be an array, in which case
761 // `legalize_vec_store_elem` becomes two instructions (`ptr_elem_ptr` and `store`).
762 // Therefore, there are 13 or 14 instructions in the block, plus however many are
763 // needed to compute each result element for `form`.
764 const inst_per_form: usize = switch (form) {
698 .un_op, .ty_op => 2,765 .un_op, .ty_op => 2,
699 .bin_op, .cmp_vector => 3,766 .bin_op, .cmp_vector => 3,
700 .pl_op_bin => 4,767 .pl_op_bin => 4,
701 .shuffle_one, .shuffle_two => 1,
702 .select => 7,768 .select => 7,
703 };769 };
770 const max_inst_per_form = 7; // maximum value in the above switch
771 var inst_buf: [14 + max_inst_per_form]Air.Inst.Index = undefined;
704772
705 var sfba_state = std.heap.stackFallback(@sizeOf([inst_per_elem * 32 + 2]Air.Inst.Index) + @sizeOf([32]Air.Inst.Ref), gpa);773 var main_block: Block = .init(&inst_buf);
706 const sfba = sfba_state.get();774 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
707775
708 // Plus 2 extra instructions for `aggregate_init` and `br`.776 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
709 const inst_buf = try sfba.alloc(Air.Inst.Index, inst_per_elem * res_len + 2);777 const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(res_ty)).toRef();
710 defer sfba.free(inst_buf);
711778
712 var main_block: Block = .init(inst_buf);779 _ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
713 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
714780
715 const elem_buf = try sfba.alloc(Air.Inst.Ref, res_len);781 var loop: Loop = .init(l, &main_block);
716 defer sfba.free(elem_buf);782 loop.block = .init(main_block.stealRemainingCapacity());
717783
718 switch (form) {784 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
719 .un_op => {785 const elem_val: Air.Inst.Ref = switch (form) {
786 .un_op => elem: {
720 const orig_operand = orig.data.un_op;787 const orig_operand = orig.data.un_op;
721 const un_op_tag = orig.tag;788 const operand = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operand, index_val).toRef();
722 for (elem_buf, 0..) |*elem, elem_idx| {789 break :elem loop.block.addUnOp(l, orig.tag, operand).toRef();
723 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
724 const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();
725 elem.* = main_block.addUnOp(l, un_op_tag, operand).toRef();
726 }
727 },790 },
728 .ty_op => {791 .ty_op => elem: {
729 const orig_operand = orig.data.ty_op.operand;792 const orig_operand = orig.data.ty_op.operand;
730 const orig_ty: Type = .fromInterned(orig.data.ty_op.ty.toInterned().?);793 const operand_is_array = switch (l.typeOf(orig_operand).zigTypeTag(zcu)) {
731 const scalar_ty = orig_ty.childType(zcu);794 .vector => false,
732 const ty_op_tag = orig.tag;795 .array => true,
733 for (elem_buf, 0..) |*elem, elem_idx| {796 else => unreachable,
734 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));797 };
735 const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();798 const operand = loop.block.addBinOp(
736 elem.* = main_block.addTyOp(l, ty_op_tag, scalar_ty, operand).toRef();799 l,
737 }800 if (operand_is_array) .array_elem_val else .legalize_vec_elem_val,
801 orig_operand,
802 index_val,
803 ).toRef();
804 break :elem loop.block.addTyOp(l, orig.tag, res_elem_ty, operand).toRef();
738 },805 },
739 .bin_op => {806 .bin_op => elem: {
740 const orig_operands = orig.data.bin_op;807 const orig_bin = orig.data.bin_op;
741 const bin_op_tag = orig.tag;808 const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
742 for (elem_buf, 0..) |*elem, elem_idx| {809 const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
743 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));810 break :elem loop.block.addBinOp(l, orig.tag, lhs, rhs).toRef();
744 const lhs = main_block.addBinOp(l, .array_elem_val, orig_operands.lhs, elem_idx_ref).toRef();
745 const rhs = main_block.addBinOp(l, .array_elem_val, orig_operands.rhs, elem_idx_ref).toRef();
746 elem.* = main_block.addBinOp(l, bin_op_tag, lhs, rhs).toRef();
747 }
748 },811 },
749 .pl_op_bin => {812 .pl_op_bin => elem: {
750 const orig_operand = orig.data.pl_op.operand;813 const orig_operand = orig.data.pl_op.operand;
751 const orig_payload = l.extraData(Air.Bin, orig.data.pl_op.payload).data;814 const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
752 const pl_op_tag = orig.tag;815 const operand = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operand, index_val).toRef();
753 for (elem_buf, 0..) |*elem, elem_idx| {816 const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
754 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));817 const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
755 const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();818 break :elem loop.block.add(l, .{
756 const lhs = main_block.addBinOp(l, .array_elem_val, orig_payload.lhs, elem_idx_ref).toRef();819 .tag = orig.tag,
757 const rhs = main_block.addBinOp(l, .array_elem_val, orig_payload.rhs, elem_idx_ref).toRef();820 .data = .{ .pl_op = .{
758 elem.* = main_block.add(l, .{821 .operand = operand,
759 .tag = pl_op_tag,822 .payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
760 .data = .{ .pl_op = .{823 } },
761 .payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),824 }).toRef();
762 .operand = operand,
763 } },
764 }).toRef();
765 }
766 },825 },
767 .cmp_vector => {826 .cmp_vector => elem: {
768 const orig_payload = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;827 const orig_payload = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
769 const cmp_op = orig_payload.compareOperator();828 const cmp_op = orig_payload.compareOperator();
770 const optimized = switch (orig.tag) {829 const optimized = switch (orig.tag) {
...@@ -772,116 +831,393 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form:...@@ -772,116 +831,393 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form:
772 .cmp_vector_optimized => true,831 .cmp_vector_optimized => true,
773 else => unreachable,832 else => unreachable,
774 };833 };
775 for (elem_buf, 0..) |*elem, elem_idx| {834 const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_payload.lhs, index_val).toRef();
776 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));835 const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_payload.rhs, index_val).toRef();
777 const lhs = main_block.addBinOp(l, .array_elem_val, orig_payload.lhs, elem_idx_ref).toRef();836 break :elem loop.block.addCmpScalar(l, cmp_op, lhs, rhs, optimized).toRef();
778 const rhs = main_block.addBinOp(l, .array_elem_val, orig_payload.rhs, elem_idx_ref).toRef();
779 elem.* = main_block.addCmpScalar(l, cmp_op, lhs, rhs, optimized).toRef();
780 }
781 },
782 .shuffle_one => {
783 const shuffle = l.getTmpAir().unwrapShuffleOne(zcu, orig_inst);
784 for (elem_buf, shuffle.mask) |*elem, mask| elem.* = switch (mask.unwrap()) {
785 .value => |val| .fromIntern(val),
786 .elem => |src_idx| elem: {
787 const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
788 break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand, src_idx_ref).toRef();
789 },
790 };
791 },837 },
792 .shuffle_two => {838 .select => elem: {
793 const shuffle = l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst);
794 const scalar_ty = res_ty.childType(zcu);
795 for (elem_buf, shuffle.mask) |*elem, mask| elem.* = switch (mask.unwrap()) {
796 .undef => .fromValue(try pt.undefValue(scalar_ty)),
797 .a_elem => |src_idx| elem: {
798 const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
799 break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand_a, src_idx_ref).toRef();
800 },
801 .b_elem => |src_idx| elem: {
802 const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
803 break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand_b, src_idx_ref).toRef();
804 },
805 };
806 },
807 .select => {
808 const orig_cond = orig.data.pl_op.operand;839 const orig_cond = orig.data.pl_op.operand;
809 const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;840 const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
810 const res_scalar_ty = res_ty.childType(zcu);
811 for (elem_buf, 0..) |*elem, elem_idx| {
812 // Payload to be populated later; we need the index early for `br`s.
813 const elem_block_inst = main_block.add(l, .{
814 .tag = .block,
815 .data = .{ .ty_pl = .{
816 .ty = .fromType(res_scalar_ty),
817 .payload = undefined,
818 } },
819 });
820 var elem_block: Block = .init(main_block.stealCapacity(2));
821841
822 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));842 const elem_block_inst = loop.block.add(l, .{
823 const cond = elem_block.addBinOp(l, .array_elem_val, orig_cond, elem_idx_ref).toRef();843 .tag = .block,
824 var condbr: CondBr = .init(l, cond, &elem_block, .{});844 .data = .{ .ty_pl = .{
845 .ty = .fromType(res_elem_ty),
846 .payload = undefined,
847 } },
848 });
849 var elem_block: Block = .init(loop.block.stealCapacity(2));
850 const cond = elem_block.addBinOp(l, .legalize_vec_elem_val, orig_cond, index_val).toRef();
851
852 var condbr: CondBr = .init(l, cond, &elem_block, .{});
825853
826 condbr.then_block = .init(main_block.stealCapacity(2));854 condbr.then_block = .init(loop.block.stealCapacity(2));
827 const lhs = condbr.then_block.addBinOp(l, .array_elem_val, orig_bin.lhs, elem_idx_ref).toRef();855 const lhs = condbr.then_block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
828 condbr.then_block.addBr(l, elem_block_inst, lhs);856 condbr.then_block.addBr(l, elem_block_inst, lhs);
829857
830 condbr.else_block = .init(main_block.stealCapacity(2));858 condbr.else_block = .init(loop.block.stealCapacity(2));
831 const rhs = condbr.else_block.addBinOp(l, .array_elem_val, orig_bin.rhs, elem_idx_ref).toRef();859 const rhs = condbr.else_block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
832 condbr.else_block.addBr(l, elem_block_inst, rhs);860 condbr.else_block.addBr(l, elem_block_inst, rhs);
833861
834 try condbr.finish(l);862 try condbr.finish(l);
835863
836 const inst_data = l.air_instructions.items(.data);864 const inst_data = l.air_instructions.items(.data);
837 inst_data[@intFromEnum(elem_block_inst)].ty_pl.payload = try l.addBlockBody(elem_block.body());865 inst_data[@intFromEnum(elem_block_inst)].ty_pl.payload = try l.addBlockBody(elem_block.body());
838866
839 elem.* = elem_block_inst.toRef();867 break :elem elem_block_inst.toRef();
840 }
841 },868 },
869 };
870 _ = loop.block.stealCapacity(max_inst_per_form - inst_per_form);
871 if (result_is_array) {
872 const elem_ptr = loop.block.add(l, .{
873 .tag = .ptr_elem_ptr,
874 .data = .{ .ty_pl = .{
875 .ty = .fromType(try pt.singleMutPtrType(res_elem_ty)),
876 .payload = try l.addExtra(Air.Bin, .{
877 .lhs = result_ptr,
878 .rhs = index_val,
879 }),
880 } },
881 }).toRef();
882 _ = loop.block.addBinOp(l, .store, elem_ptr, elem_val);
883 } else {
884 _ = loop.block.add(l, .{
885 .tag = .legalize_vec_store_elem,
886 .data = .{ .pl_op = .{
887 .operand = result_ptr,
888 .payload = try l.addExtra(Air.Bin, .{
889 .lhs = index_val,
890 .rhs = elem_val,
891 }),
892 } },
893 });
894 _ = loop.block.stealCapacity(1);
842 }895 }
896 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, res_len - 1))).toRef();
843897
844 const result = main_block.add(l, .{898 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
845 .tag = .aggregate_init,899 condbr.then_block = .init(loop.block.stealRemainingCapacity());
846 .data = .{ .ty_pl = .{900 const result_val = condbr.then_block.addTyOp(l, .load, res_ty, result_ptr).toRef();
847 .ty = .fromType(res_ty),901 condbr.then_block.addBr(l, orig_inst, result_val);
848 .payload = payload: {
849 const idx = l.air_extra.items.len;
850 try l.air_extra.appendSlice(gpa, @ptrCast(elem_buf));
851 break :payload @intCast(idx);
852 },
853 } },
854 }).toRef();
855902
856 main_block.addBr(l, orig_inst, result);903 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
904 const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
905 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
906 _ = condbr.else_block.add(l, .{
907 .tag = .repeat,
908 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
909 });
857910
858 // Some `form` values may intentionally not use the full instruction buffer.911 try condbr.finish(l);
859 switch (form) {912
860 .un_op,913 try loop.finish(l);
861 .ty_op,
862 .bin_op,
863 .pl_op_bin,
864 .cmp_vector,
865 .select,
866 => {},
867 .shuffle_one,
868 .shuffle_two,
869 => _ = main_block.stealRemainingCapacity(),
870 }
871914
872 return .{ .ty_pl = .{915 return .{ .ty_pl = .{
873 .ty = .fromType(res_ty),916 .ty = .fromType(res_ty),
874 .payload = try l.addBlockBody(main_block.body()),917 .payload = try l.addBlockBody(main_block.body()),
875 } };918 } };
876}919}
877fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {920fn scalarizeShuffleOneBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
878 const pt = l.pt;921 const pt = l.pt;
879 const zcu = pt.zcu;922 const zcu = pt.zcu;
880 const gpa = zcu.gpa;923 const gpa = zcu.gpa;
881924
925 const shuffle = l.getTmpAir().unwrapShuffleOne(zcu, orig_inst);
926
927 // We're going to emit something like this:
928 //
929 // var x: @Vector(N, T) = all_comptime_known_elems;
930 // for (out_idxs, in_idxs) |i, j| x[i] = operand[j];
931 //
932 // So we must first compute `out_idxs` and `in_idxs`.
933
882 var sfba_state = std.heap.stackFallback(512, gpa);934 var sfba_state = std.heap.stackFallback(512, gpa);
883 const sfba = sfba_state.get();935 const sfba = sfba_state.get();
884936
937 const out_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
938 defer sfba.free(out_idxs_buf);
939
940 const in_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
941 defer sfba.free(in_idxs_buf);
942
943 var n: usize = 0;
944 for (shuffle.mask, 0..) |mask, out_idx| switch (mask.unwrap()) {
945 .value => {},
946 .elem => |in_idx| {
947 out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
948 in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
949 n += 1;
950 },
951 };
952
953 const init_val: Value = init: {
954 const undef_val = try pt.undefValue(shuffle.result_ty.childType(zcu));
955 const elems = try sfba.alloc(InternPool.Index, shuffle.mask.len);
956 defer sfba.free(elems);
957 for (shuffle.mask, elems) |mask, *elem| elem.* = switch (mask.unwrap()) {
958 .value => |ip_index| ip_index,
959 .elem => undef_val.toIntern(),
960 };
961 break :init try pt.aggregateValue(shuffle.result_ty, elems);
962 };
963
964 // %1 = block(@Vector(N, T), {
965 // %2 = alloc(*@Vector(N, T))
966 // %3 = alloc(*usize)
967 // %4 = store(%2, <init_val>)
968 // %5 = [addScalarizedShuffle]
969 // %6 = load(%2)
970 // %7 = br(%1, %6)
971 // })
972
973 var inst_buf: [6]Air.Inst.Index = undefined;
974 var main_block: Block = .init(&inst_buf);
975 try l.air_instructions.ensureUnusedCapacity(gpa, 19);
976
977 const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(shuffle.result_ty)).toRef();
978 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
979
980 _ = main_block.addBinOp(l, .store, result_ptr, .fromValue(init_val));
981
982 try l.addScalarizedShuffle(
983 &main_block,
984 shuffle.operand,
985 result_ptr,
986 index_ptr,
987 out_idxs_buf[0..n],
988 in_idxs_buf[0..n],
989 );
990
991 const result_val = main_block.addTyOp(l, .load, shuffle.result_ty, result_ptr).toRef();
992 main_block.addBr(l, orig_inst, result_val);
993
994 return .{ .ty_pl = .{
995 .ty = .fromType(shuffle.result_ty),
996 .payload = try l.addBlockBody(main_block.body()),
997 } };
998}
999fn scalarizeShuffleTwoBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1000 const pt = l.pt;
1001 const zcu = pt.zcu;
1002 const gpa = zcu.gpa;
1003
1004 const shuffle = l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst);
1005
1006 // We're going to emit something like this:
1007 //
1008 // var x: @Vector(N, T) = undefined;
1009 // for (out_idxs_a, in_idxs_a) |i, j| x[i] = operand_a[j];
1010 // for (out_idxs_b, in_idxs_b) |i, j| x[i] = operand_b[j];
1011 //
1012 // The AIR will look like this:
1013 //
1014 // %1 = block(@Vector(N, T), {
1015 // %2 = alloc(*@Vector(N, T))
1016 // %3 = alloc(*usize)
1017 // %4 = store(%2, <@Vector(N, T), undefined>)
1018 // %5 = [addScalarizedShuffle]
1019 // %6 = [addScalarizedShuffle]
1020 // %7 = load(%2)
1021 // %8 = br(%1, %7)
1022 // })
1023
1024 var sfba_state = std.heap.stackFallback(512, gpa);
1025 const sfba = sfba_state.get();
1026
1027 const out_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
1028 defer sfba.free(out_idxs_buf);
1029
1030 const in_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
1031 defer sfba.free(in_idxs_buf);
1032
1033 // Iterate `shuffle.mask` before doing anything, because modifying AIR invalidates it.
1034 const out_idxs_a, const in_idxs_a, const out_idxs_b, const in_idxs_b = idxs: {
1035 var n: usize = 0;
1036 for (shuffle.mask, 0..) |mask, out_idx| switch (mask.unwrap()) {
1037 .undef, .b_elem => {},
1038 .a_elem => |in_idx| {
1039 out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
1040 in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
1041 n += 1;
1042 },
1043 };
1044 const a_len = n;
1045 for (shuffle.mask, 0..) |mask, out_idx| switch (mask.unwrap()) {
1046 .undef, .a_elem => {},
1047 .b_elem => |in_idx| {
1048 out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
1049 in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
1050 n += 1;
1051 },
1052 };
1053 break :idxs .{
1054 out_idxs_buf[0..a_len],
1055 in_idxs_buf[0..a_len],
1056 out_idxs_buf[a_len..n],
1057 in_idxs_buf[a_len..n],
1058 };
1059 };
1060
1061 var inst_buf: [7]Air.Inst.Index = undefined;
1062 var main_block: Block = .init(&inst_buf);
1063 try l.air_instructions.ensureUnusedCapacity(gpa, 33);
1064
1065 const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(shuffle.result_ty)).toRef();
1066 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1067
1068 _ = main_block.addBinOp(l, .store, result_ptr, .fromValue(try pt.undefValue(shuffle.result_ty)));
1069
1070 if (out_idxs_a.len == 0) {
1071 _ = main_block.stealCapacity(1);
1072 } else {
1073 try l.addScalarizedShuffle(
1074 &main_block,
1075 shuffle.operand_a,
1076 result_ptr,
1077 index_ptr,
1078 out_idxs_a,
1079 in_idxs_a,
1080 );
1081 }
1082
1083 if (out_idxs_b.len == 0) {
1084 _ = main_block.stealCapacity(1);
1085 } else {
1086 try l.addScalarizedShuffle(
1087 &main_block,
1088 shuffle.operand_b,
1089 result_ptr,
1090 index_ptr,
1091 out_idxs_b,
1092 in_idxs_b,
1093 );
1094 }
1095
1096 const result_val = main_block.addTyOp(l, .load, shuffle.result_ty, result_ptr).toRef();
1097 main_block.addBr(l, orig_inst, result_val);
1098
1099 return .{ .ty_pl = .{
1100 .ty = .fromType(shuffle.result_ty),
1101 .payload = try l.addBlockBody(main_block.body()),
1102 } };
1103}
1104/// Adds code to `parent_block` which behaves like this loop:
1105///
1106/// for (out_idxs, in_idxs) |i, j| result_vec_ptr[i] = operand_vec[j];
1107///
1108/// The actual AIR adds exactly one instruction to `parent_block` itself and 14 instructions
1109/// overall, and is as follows:
1110///
1111/// %1 = block(void, {
1112/// %2 = store(index_ptr, @zero_usize)
1113/// %3 = loop({
1114/// %4 = load(index_ptr)
1115/// %5 = ptr_elem_val(out_idxs_ptr, %4)
1116/// %6 = ptr_elem_val(in_idxs_ptr, %4)
1117/// %7 = legalize_vec_elem_val(operand_vec, %6)
1118/// %8 = legalize_vec_store_elem(result_vec_ptr, %4, %7)
1119/// %9 = cmp_eq(%4, <usize, out_idxs.len-1>)
1120/// %10 = cond_br(%9, {
1121/// %11 = br(%1, @void_value)
1122/// }, {
1123/// %12 = add(%4, @one_usize)
1124/// %13 = store(index_ptr, %12)
1125/// %14 = repeat(%3)
1126/// })
1127/// })
1128/// })
1129///
1130/// The caller is responsible for reserving space in `l.air_instructions`.
1131fn addScalarizedShuffle(
1132 l: *Legalize,
1133 parent_block: *Block,
1134 operand_vec: Air.Inst.Ref,
1135 result_vec_ptr: Air.Inst.Ref,
1136 index_ptr: Air.Inst.Ref,
1137 out_idxs: []const InternPool.Index,
1138 in_idxs: []const InternPool.Index,
1139) Error!void {
1140 const pt = l.pt;
1141
1142 assert(out_idxs.len == in_idxs.len);
1143 const n = out_idxs.len;
1144
1145 const idxs_ty = try pt.arrayType(.{ .len = n, .child = .usize_type });
1146 const idxs_ptr_ty = try pt.singleConstPtrType(idxs_ty);
1147 const manyptr_usize_ty = try pt.manyConstPtrType(.usize);
1148
1149 const out_idxs_ptr = try pt.intern(.{ .ptr = .{
1150 .ty = manyptr_usize_ty.toIntern(),
1151 .base_addr = .{ .uav = .{
1152 .val = (try pt.aggregateValue(idxs_ty, out_idxs)).toIntern(),
1153 .orig_ty = idxs_ptr_ty.toIntern(),
1154 } },
1155 .byte_offset = 0,
1156 } });
1157 const in_idxs_ptr = try pt.intern(.{ .ptr = .{
1158 .ty = manyptr_usize_ty.toIntern(),
1159 .base_addr = .{ .uav = .{
1160 .val = (try pt.aggregateValue(idxs_ty, in_idxs)).toIntern(),
1161 .orig_ty = idxs_ptr_ty.toIntern(),
1162 } },
1163 .byte_offset = 0,
1164 } });
1165
1166 const main_block_inst = parent_block.add(l, .{
1167 .tag = .block,
1168 .data = .{ .ty_pl = .{
1169 .ty = .void_type,
1170 .payload = undefined,
1171 } },
1172 });
1173
1174 var inst_buf: [13]Air.Inst.Index = undefined;
1175 var main_block: Block = .init(&inst_buf);
1176
1177 _ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1178
1179 var loop: Loop = .init(l, &main_block);
1180 loop.block = .init(main_block.stealRemainingCapacity());
1181
1182 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1183 const in_idx_val = loop.block.addBinOp(l, .ptr_elem_val, .fromIntern(in_idxs_ptr), index_val).toRef();
1184 const out_idx_val = loop.block.addBinOp(l, .ptr_elem_val, .fromIntern(out_idxs_ptr), index_val).toRef();
1185
1186 const elem_val = loop.block.addBinOp(l, .legalize_vec_elem_val, operand_vec, in_idx_val).toRef();
1187 _ = loop.block.add(l, .{
1188 .tag = .legalize_vec_store_elem,
1189 .data = .{ .pl_op = .{
1190 .operand = result_vec_ptr,
1191 .payload = try l.addExtra(Air.Bin, .{
1192 .lhs = out_idx_val,
1193 .rhs = elem_val,
1194 }),
1195 } },
1196 });
1197
1198 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, n - 1))).toRef();
1199 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1200 condbr.then_block = .init(loop.block.stealRemainingCapacity());
1201 condbr.then_block.addBr(l, main_block_inst, .void_value);
1202
1203 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1204 const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1205 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1206 _ = condbr.else_block.add(l, .{
1207 .tag = .repeat,
1208 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1209 });
1210
1211 try condbr.finish(l);
1212 try loop.finish(l);
1213
1214 const inst_data = l.air_instructions.items(.data);
1215 inst_data[@intFromEnum(main_block_inst)].ty_pl.payload = try l.addBlockBody(main_block.body());
1216}
1217fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {
1218 const pt = l.pt;
1219 const zcu = pt.zcu;
1220
885 const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;1221 const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
8861222
887 const dest_ty = ty_op.ty.toType();1223 const dest_ty = ty_op.ty.toType();
...@@ -920,72 +1256,204 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -920,72 +1256,204 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
920 const uint_ty = try pt.intType(.unsigned, num_bits);1256 const uint_ty = try pt.intType(.unsigned, num_bits);
921 const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));1257 const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
9221258
923 const inst_buf = try sfba.alloc(Air.Inst.Index, len: {1259 var inst_buf: [39]Air.Inst.Index = undefined;
924 const operand_to_uint_len: u64 = if (operand_legal) 1 else (operand_ty.arrayLen(zcu) * 5);1260 var main_block: Block = .init(&inst_buf);
925 const uint_to_dest_len: u64 = if (dest_legal) 1 else (dest_ty.arrayLen(zcu) * 3 + 1);1261 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
926 break :len @intCast(operand_to_uint_len + uint_to_dest_len + 1);
927 });
928 defer sfba.free(inst_buf);
929 var main_block: Block = .init(inst_buf);
930 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
9311262
932 // First, convert `operand_ty` to `uint_ty` (`uN`).1263 // First, convert `operand_ty` to `uint_ty` (`uN`).
9331264
934 const uint_val: Air.Inst.Ref = uint_val: {1265 const uint_val: Air.Inst.Ref = uint_val: {
935 if (operand_legal) break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);1266 if (operand_legal) {
9361267 _ = main_block.stealCapacity(19);
937 const bits_per_elem: u16 = @intCast(operand_ty.childType(zcu).bitSize(zcu));1268 break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
938 const bits_per_elem_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, bits_per_elem));
939 const elem_uint_ty = try pt.intType(.unsigned, bits_per_elem);
940
941 var cur_uint: Air.Inst.Ref = .fromValue(try pt.intValue(uint_ty, 0));
942 var elem_idx = operand_ty.arrayLen(zcu);
943 while (elem_idx > 0) {
944 elem_idx -= 1;
945 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
946 const orig_elem = main_block.addBinOp(l, .array_elem_val, ty_op.operand, elem_idx_ref).toRef();
947 const elem_as_uint = main_block.addBitCast(l, elem_uint_ty, orig_elem);
948 const elem_extended = main_block.addTyOp(l, .intcast, uint_ty, elem_as_uint).toRef();
949 cur_uint = main_block.addBinOp(l, .shl_exact, cur_uint, bits_per_elem_ref).toRef();
950 cur_uint = main_block.addBinOp(l, .bit_or, cur_uint, elem_extended).toRef();
951 }1269 }
952 break :uint_val cur_uint;1270
1271 // %1 = block({
1272 // %2 = alloc(*usize)
1273 // %3 = alloc(*uN)
1274 // %4 = store(%2, <usize, operand_len>)
1275 // %5 = store(%3, <uN, 0>)
1276 // %6 = loop({
1277 // %7 = load(%2)
1278 // %8 = array_elem_val(orig_operand, %7)
1279 // %9 = bitcast(uE, %8)
1280 // %10 = intcast(uN, %9)
1281 // %11 = load(%3)
1282 // %12 = shl_exact(%11, <uS, E>)
1283 // %13 = bit_or(%12, %10)
1284 // %14 = cmp_eq(%4, @zero_usize)
1285 // %15 = cond_br(%14, {
1286 // %16 = br(%1, %13)
1287 // }, {
1288 // %17 = store(%3, %13)
1289 // %18 = sub(%7, @one_usize)
1290 // %19 = store(%2, %18)
1291 // %20 = repeat(%6)
1292 // })
1293 // })
1294 // })
1295
1296 const elem_bits = operand_ty.childType(zcu).bitSize(zcu);
1297 const elem_bits_val = try pt.intValue(shift_ty, elem_bits);
1298 const elem_uint_ty = try pt.intType(.unsigned, @intCast(elem_bits));
1299
1300 const uint_block_inst = main_block.add(l, .{
1301 .tag = .block,
1302 .data = .{ .ty_pl = .{
1303 .ty = .fromType(uint_ty),
1304 .payload = undefined,
1305 } },
1306 });
1307 var uint_block: Block = .init(main_block.stealCapacity(19));
1308
1309 const index_ptr = uint_block.addTy(l, .alloc, .ptr_usize).toRef();
1310 const result_ptr = uint_block.addTy(l, .alloc, try pt.singleMutPtrType(uint_ty)).toRef();
1311 _ = uint_block.addBinOp(
1312 l,
1313 .store,
1314 index_ptr,
1315 .fromValue(try pt.intValue(.usize, operand_ty.arrayLen(zcu))),
1316 );
1317 _ = uint_block.addBinOp(l, .store, result_ptr, .fromValue(try pt.intValue(uint_ty, 0)));
1318
1319 var loop: Loop = .init(l, &uint_block);
1320 loop.block = .init(uint_block.stealRemainingCapacity());
1321
1322 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1323 const raw_elem = loop.block.addBinOp(
1324 l,
1325 if (operand_ty.zigTypeTag(zcu) == .vector) .legalize_vec_elem_val else .array_elem_val,
1326 ty_op.operand,
1327 index_val,
1328 ).toRef();
1329 const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);
1330 const elem_extended = loop.block.addTyOp(l, .intcast, uint_ty, elem_uint).toRef();
1331 const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();
1332 const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();
1333 const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();
1334
1335 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .zero_usize).toRef();
1336 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1337
1338 condbr.then_block = .init(loop.block.stealRemainingCapacity());
1339 condbr.then_block.addBr(l, uint_block_inst, new_result);
1340
1341 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1342 _ = condbr.else_block.addBinOp(l, .store, result_ptr, new_result);
1343 const new_index_val = condbr.else_block.addBinOp(l, .sub, index_val, .one_usize).toRef();
1344 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1345 _ = condbr.else_block.add(l, .{
1346 .tag = .repeat,
1347 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1348 });
1349
1350 try condbr.finish(l);
1351 try loop.finish(l);
1352
1353 const inst_data = l.air_instructions.items(.data);
1354 inst_data[@intFromEnum(uint_block_inst)].ty_pl.payload = try l.addBlockBody(uint_block.body());
1355
1356 break :uint_val uint_block_inst.toRef();
953 };1357 };
9541358
955 // Now convert `uint_ty` (`uN`) to `dest_ty`.1359 // Now convert `uint_ty` (`uN`) to `dest_ty`.
9561360
957 const result: Air.Inst.Ref = result: {1361 if (dest_legal) {
958 if (dest_legal) break :result main_block.addBitCast(l, dest_ty, uint_val);1362 _ = main_block.stealCapacity(17);
1363 const result = main_block.addBitCast(l, dest_ty, uint_val);
1364 main_block.addBr(l, orig_inst, result);
1365 } else {
1366 // %1 = alloc(*usize)
1367 // %2 = alloc(*@Vector(N, Result))
1368 // %3 = store(%1, @zero_usize)
1369 // %4 = loop({
1370 // %5 = load(%1)
1371 // %6 = mul(%5, <usize, E>)
1372 // %7 = intcast(uS, %6)
1373 // %8 = shr(uint_val, %7)
1374 // %9 = trunc(uE, %8)
1375 // %10 = bitcast(Result, %9)
1376 // %11 = legalize_vec_store_elem(%2, %5, %10)
1377 // %12 = cmp_eq(%5, <usize, vec_len>)
1378 // %13 = cond_br(%12, {
1379 // %14 = load(%2)
1380 // %15 = br(%0, %14)
1381 // }, {
1382 // %16 = add(%5, @one_usize)
1383 // %17 = store(%1, %16)
1384 // %18 = repeat(%4)
1385 // })
1386 // })
1387 //
1388 // The result might be an array, in which case `legalize_vec_store_elem`
1389 // becomes `ptr_elem_ptr` followed by `store`.
9591390
960 const elem_ty = dest_ty.childType(zcu);1391 const elem_ty = dest_ty.childType(zcu);
961 const bits_per_elem: u16 = @intCast(elem_ty.bitSize(zcu));1392 const elem_bits = elem_ty.bitSize(zcu);
962 const bits_per_elem_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, bits_per_elem));1393 const elem_uint_ty = try pt.intType(.unsigned, @intCast(elem_bits));
963 const elem_uint_ty = try pt.intType(.unsigned, bits_per_elem);1394
9641395 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
965 const elem_buf = try sfba.alloc(Air.Inst.Ref, dest_ty.arrayLen(zcu));1396 const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(dest_ty)).toRef();
966 defer sfba.free(elem_buf);1397 _ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
9671398
968 var cur_uint = uint_val;1399 var loop: Loop = .init(l, &main_block);
969 for (elem_buf) |*elem| {1400 loop.block = .init(main_block.stealRemainingCapacity());
970 const elem_as_uint = main_block.addTyOp(l, .trunc, elem_uint_ty, cur_uint).toRef();1401
971 elem.* = main_block.addBitCast(l, elem_ty, elem_as_uint);1402 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
972 cur_uint = main_block.addBinOp(l, .shr, cur_uint, bits_per_elem_ref).toRef();1403 const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();
1404 const casted_bit_offset = loop.block.addTyOp(l, .intcast, shift_ty, bit_offset).toRef();
1405 const shifted_uint = loop.block.addBinOp(l, .shr, index_val, casted_bit_offset).toRef();
1406 const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();
1407 const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);
1408 switch (dest_ty.zigTypeTag(zcu)) {
1409 .array => {
1410 const elem_ptr = loop.block.add(l, .{
1411 .tag = .ptr_elem_ptr,
1412 .data = .{ .ty_pl = .{
1413 .ty = .fromType(try pt.singleMutPtrType(elem_ty)),
1414 .payload = try l.addExtra(Air.Bin, .{
1415 .lhs = result_ptr,
1416 .rhs = index_val,
1417 }),
1418 } },
1419 }).toRef();
1420 _ = loop.block.addBinOp(l, .store, elem_ptr, elem_val);
1421 },
1422 .vector => {
1423 _ = loop.block.add(l, .{
1424 .tag = .legalize_vec_store_elem,
1425 .data = .{ .pl_op = .{
1426 .operand = result_ptr,
1427 .payload = try l.addExtra(Air.Bin, .{
1428 .lhs = index_val,
1429 .rhs = elem_val,
1430 }),
1431 } },
1432 });
1433 _ = loop.block.stealCapacity(1);
1434 },
1435 else => unreachable,
973 }1436 }
9741437
975 break :result main_block.add(l, .{1438 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, dest_ty.arrayLen(zcu) - 1))).toRef();
976 .tag = .aggregate_init,
977 .data = .{ .ty_pl = .{
978 .ty = .fromType(dest_ty),
979 .payload = payload: {
980 const idx = l.air_extra.items.len;
981 try l.air_extra.appendSlice(gpa, @ptrCast(elem_buf));
982 break :payload @intCast(idx);
983 },
984 } },
985 }).toRef();
986 };
9871439
988 main_block.addBr(l, orig_inst, result);1440 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1441
1442 condbr.then_block = .init(loop.block.stealRemainingCapacity());
1443 const result_val = condbr.then_block.addTyOp(l, .load, dest_ty, result_ptr).toRef();
1444 condbr.then_block.addBr(l, orig_inst, result_val);
1445
1446 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1447 const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1448 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1449 _ = condbr.else_block.add(l, .{
1450 .tag = .repeat,
1451 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1452 });
1453
1454 try condbr.finish(l);
1455 try loop.finish(l);
1456 }
9891457
990 return .{ .ty_pl = .{1458 return .{ .ty_pl = .{
991 .ty = .fromType(dest_ty),1459 .ty = .fromType(dest_ty),
...@@ -995,10 +1463,6 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -995,10 +1463,6 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
995fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {1463fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
996 const pt = l.pt;1464 const pt = l.pt;
997 const zcu = pt.zcu;1465 const zcu = pt.zcu;
998 const gpa = zcu.gpa;
999
1000 var sfba_state = std.heap.stackFallback(512, gpa);
1001 const sfba = sfba_state.get();
10021466
1003 const orig = l.air_instructions.get(@intFromEnum(orig_inst));1467 const orig = l.air_instructions.get(@intFromEnum(orig_inst));
1004 const orig_operands = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;1468 const orig_operands = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;
...@@ -1015,89 +1479,127 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!...@@ -1015,89 +1479,127 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!
1015 const scalar_int_ty = vec_int_ty.childType(zcu);1479 const scalar_int_ty = vec_int_ty.childType(zcu);
1016 const scalar_tuple_ty = try pt.overflowArithmeticTupleType(scalar_int_ty);1480 const scalar_tuple_ty = try pt.overflowArithmeticTupleType(scalar_int_ty);
10171481
1018 const elems_len = vec_int_ty.vectorLen(zcu);1482 // %1 = block(struct { @Vector(N, Int), @Vector(N, u1) }, {
10191483 // %2 = alloc(*usize)
1020 const inst_buf = try sfba.alloc(Air.Inst.Index, 5 * elems_len + 4);1484 // %3 = alloc(*struct { @Vector(N, Int), @Vector(N, u1) })
1021 defer sfba.free(inst_buf);1485 // %4 = struct_field_ptr_index_0(*@Vector(N, Int), %3)
1486 // %5 = struct_field_ptr_index_1(*@Vector(N, u1), %3)
1487 // %6 = store(%2, @zero_usize)
1488 // %7 = loop({
1489 // %8 = load(%2)
1490 // %9 = legalize_vec_elem_val(orig_lhs, %8)
1491 // %10 = legalize_vec_elem_val(orig_rhs, %8)
1492 // %11 = ???_with_overflow(struct { Int, u1 }, %9, %10)
1493 // %12 = struct_field_val(%11, 0)
1494 // %13 = struct_field_val(%11, 1)
1495 // %14 = legalize_vec_store_elem(%4, %8, %12)
1496 // %15 = legalize_vec_store_elem(%4, %8, %13)
1497 // %16 = cmp_eq(%8, <usize, N-1>)
1498 // %17 = cond_br(%16, {
1499 // %18 = load(%3)
1500 // %19 = br(%1, %18)
1501 // }, {
1502 // %20 = add(%8, @one_usize)
1503 // %21 = store(%2, %20)
1504 // %22 = repeat(%7)
1505 // })
1506 // })
1507 // })
10221508
1023 var main_block: Block = .init(inst_buf);1509 const elems_len = vec_int_ty.vectorLen(zcu);
1024 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
10251510
1026 const int_elem_buf = try sfba.alloc(Air.Inst.Ref, elems_len);1511 var inst_buf: [21]Air.Inst.Index = undefined;
1027 defer sfba.free(int_elem_buf);1512 var main_block: Block = .init(&inst_buf);
1028 const overflow_elem_buf = try sfba.alloc(Air.Inst.Ref, elems_len);1513 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1029 defer sfba.free(overflow_elem_buf);
1030
1031 for (int_elem_buf, overflow_elem_buf, 0..) |*int_elem, *overflow_elem, elem_idx| {
1032 const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
1033 const lhs = main_block.addBinOp(l, .array_elem_val, orig_operands.lhs, elem_idx_ref).toRef();
1034 const rhs = main_block.addBinOp(l, .array_elem_val, orig_operands.rhs, elem_idx_ref).toRef();
1035 const elem_result = main_block.add(l, .{
1036 .tag = orig.tag,
1037 .data = .{ .ty_pl = .{
1038 .ty = .fromType(scalar_tuple_ty),
1039 .payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
1040 } },
1041 }).toRef();
1042 int_elem.* = main_block.add(l, .{
1043 .tag = .struct_field_val,
1044 .data = .{ .ty_pl = .{
1045 .ty = .fromType(scalar_int_ty),
1046 .payload = try l.addExtra(Air.StructField, .{
1047 .struct_operand = elem_result,
1048 .field_index = 0,
1049 }),
1050 } },
1051 }).toRef();
1052 overflow_elem.* = main_block.add(l, .{
1053 .tag = .struct_field_val,
1054 .data = .{ .ty_pl = .{
1055 .ty = .bool_type,
1056 .payload = try l.addExtra(Air.StructField, .{
1057 .struct_operand = elem_result,
1058 .field_index = 1,
1059 }),
1060 } },
1061 }).toRef();
1062 }
10631514
1064 const int_vec = main_block.add(l, .{1515 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1065 .tag = .aggregate_init,1516 const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(vec_tuple_ty)).toRef();
1517 const result_int_ptr = main_block.addTyOp(
1518 l,
1519 .struct_field_ptr_index_0,
1520 try pt.singleMutPtrType(vec_int_ty),
1521 result_ptr,
1522 ).toRef();
1523 const result_overflow_ptr = main_block.addTyOp(
1524 l,
1525 .struct_field_ptr_index_1,
1526 try pt.singleMutPtrType(vec_overflow_ty),
1527 result_ptr,
1528 ).toRef();
1529
1530 _ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1531
1532 var loop: Loop = .init(l, &main_block);
1533 loop.block = .init(main_block.stealRemainingCapacity());
1534
1535 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1536 const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operands.lhs, index_val).toRef();
1537 const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operands.rhs, index_val).toRef();
1538 const elem_result = loop.block.add(l, .{
1539 .tag = orig.tag,
1066 .data = .{ .ty_pl = .{1540 .data = .{ .ty_pl = .{
1067 .ty = .fromType(vec_int_ty),1541 .ty = .fromType(scalar_tuple_ty),
1068 .payload = payload: {1542 .payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
1069 const idx = l.air_extra.items.len;
1070 try l.air_extra.appendSlice(gpa, @ptrCast(int_elem_buf));
1071 break :payload @intCast(idx);
1072 },
1073 } },1543 } },
1074 }).toRef();1544 }).toRef();
1075 const overflow_vec = main_block.add(l, .{1545 const int_elem = loop.block.add(l, .{
1076 .tag = .aggregate_init,1546 .tag = .struct_field_val,
1077 .data = .{ .ty_pl = .{1547 .data = .{ .ty_pl = .{
1078 .ty = .fromType(vec_overflow_ty),1548 .ty = .fromType(scalar_int_ty),
1079 .payload = payload: {1549 .payload = try l.addExtra(Air.StructField, .{
1080 const idx = l.air_extra.items.len;1550 .struct_operand = elem_result,
1081 try l.air_extra.appendSlice(gpa, @ptrCast(overflow_elem_buf));1551 .field_index = 0,
1082 break :payload @intCast(idx);1552 }),
1083 },
1084 } },1553 } },
1085 }).toRef();1554 }).toRef();
10861555 const overflow_elem = loop.block.add(l, .{
1087 const tuple_elems: [2]Air.Inst.Ref = .{ int_vec, overflow_vec };1556 .tag = .struct_field_val,
1088 const result = main_block.add(l, .{
1089 .tag = .aggregate_init,
1090 .data = .{ .ty_pl = .{1557 .data = .{ .ty_pl = .{
1091 .ty = .fromType(vec_tuple_ty),1558 .ty = .u1_type,
1092 .payload = payload: {1559 .payload = try l.addExtra(Air.StructField, .{
1093 const idx = l.air_extra.items.len;1560 .struct_operand = elem_result,
1094 try l.air_extra.appendSlice(gpa, @ptrCast(&tuple_elems));1561 .field_index = 1,
1095 break :payload @intCast(idx);1562 }),
1096 },
1097 } },1563 } },
1098 }).toRef();1564 }).toRef();
1565 _ = loop.block.add(l, .{
1566 .tag = .legalize_vec_store_elem,
1567 .data = .{ .pl_op = .{
1568 .operand = result_int_ptr,
1569 .payload = try l.addExtra(Air.Bin, .{
1570 .lhs = index_val,
1571 .rhs = int_elem,
1572 }),
1573 } },
1574 });
1575 _ = loop.block.add(l, .{
1576 .tag = .legalize_vec_store_elem,
1577 .data = .{ .pl_op = .{
1578 .operand = result_overflow_ptr,
1579 .payload = try l.addExtra(Air.Bin, .{
1580 .lhs = index_val,
1581 .rhs = overflow_elem,
1582 }),
1583 } },
1584 });
10991585
1100 main_block.addBr(l, orig_inst, result);1586 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, elems_len - 1))).toRef();
1587 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1588
1589 condbr.then_block = .init(loop.block.stealRemainingCapacity());
1590 const result_val = condbr.then_block.addTyOp(l, .load, vec_tuple_ty, result_ptr).toRef();
1591 condbr.then_block.addBr(l, orig_inst, result_val);
1592
1593 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1594 const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1595 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1596 _ = condbr.else_block.add(l, .{
1597 .tag = .repeat,
1598 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1599 });
1600
1601 try condbr.finish(l);
1602 try loop.finish(l);
11011603
1102 return .{ .ty_pl = .{1604 return .{ .ty_pl = .{
1103 .ty = .fromType(vec_tuple_ty),1605 .ty = .fromType(vec_tuple_ty),
...@@ -1288,7 +1790,7 @@ fn safeIntFromFloatBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, optimiz...@@ -1288,7 +1790,7 @@ fn safeIntFromFloatBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, optimiz
12881790
1289 // We emit 9 instructions in the worst case.1791 // We emit 9 instructions in the worst case.
1290 var inst_buf: [9]Air.Inst.Index = undefined;1792 var inst_buf: [9]Air.Inst.Index = undefined;
1291 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);1793 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
1292 var main_block: Block = .init(&inst_buf);1794 var main_block: Block = .init(&inst_buf);
12931795
1294 // This check is a bit annoying because of floating-point rounding and the fact that this1796 // This check is a bit annoying because of floating-point rounding and the fact that this
...@@ -1771,6 +2273,9 @@ const Block = struct {...@@ -1771,6 +2273,9 @@ const Block = struct {
1771 .data = .{ .br = .{ .block_inst = target, .operand = operand } },2273 .data = .{ .br = .{ .block_inst = target, .operand = operand } },
1772 });2274 });
1773 }2275 }
2276 fn addTy(b: *Block, l: *Legalize, tag: Air.Inst.Tag, ty: Type) Air.Inst.Index {
2277 return b.add(l, .{ .tag = tag, .data = .{ .ty = ty } });
2278 }
1774 fn addBinOp(b: *Block, l: *Legalize, tag: Air.Inst.Tag, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) Air.Inst.Index {2279 fn addBinOp(b: *Block, l: *Legalize, tag: Air.Inst.Tag, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) Air.Inst.Index {
1775 return b.add(l, .{2280 return b.add(l, .{
1776 .tag = tag,2281 .tag = tag,
...@@ -1921,6 +2426,31 @@ const Block = struct {...@@ -1921,6 +2426,31 @@ const Block = struct {
1921 }2426 }
1922};2427};
19232428
2429const Loop = struct {
2430 inst: Air.Inst.Index,
2431 block: Block,
2432
2433 /// The return value has `block` initialized to `undefined`; it is the caller's reponsibility
2434 /// to initialize it.
2435 fn init(l: *Legalize, parent_block: *Block) Loop {
2436 return .{
2437 .inst = parent_block.add(l, .{
2438 .tag = .loop,
2439 .data = .{ .ty_pl = .{
2440 .ty = .noreturn_type,
2441 .payload = undefined,
2442 } },
2443 }),
2444 .block = undefined,
2445 };
2446 }
2447
2448 fn finish(loop: Loop, l: *Legalize) Error!void {
2449 const data = &l.air_instructions.items(.data)[@intFromEnum(loop.inst)];
2450 data.ty_pl.payload = try l.addBlockBody(loop.block.body());
2451 }
2452};
2453
1924const CondBr = struct {2454const CondBr = struct {
1925 inst: Air.Inst.Index,2455 inst: Air.Inst.Index,
1926 hints: Air.CondBr.BranchHints,2456 hints: Air.CondBr.BranchHints,
src/Air/Liveness.zig+7
...@@ -458,6 +458,7 @@ fn analyzeInst(...@@ -458,6 +458,7 @@ fn analyzeInst(
458 .memset_safe,458 .memset_safe,
459 .memcpy,459 .memcpy,
460 .memmove,460 .memmove,
461 .legalize_vec_elem_val,
461 => {462 => {
462 const o = inst_datas[@intFromEnum(inst)].bin_op;463 const o = inst_datas[@intFromEnum(inst)].bin_op;
463 return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none });464 return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none });
...@@ -769,6 +770,12 @@ fn analyzeInst(...@@ -769,6 +770,12 @@ fn analyzeInst(
769 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;770 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
770 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none });771 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none });
771 },772 },
773
774 .legalize_vec_store_elem => {
775 const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
776 const bin = a.air.extraData(Air.Bin, pl_op.payload).data;
777 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, bin.lhs, bin.rhs });
778 },
772 }779 }
773}780}
774781
src/Air/Liveness/Verify.zig+6
...@@ -272,6 +272,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {...@@ -272,6 +272,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
272 .memset_safe,272 .memset_safe,
273 .memcpy,273 .memcpy,
274 .memmove,274 .memmove,
275 .legalize_vec_elem_val,
275 => {276 => {
276 const bin_op = data[@intFromEnum(inst)].bin_op;277 const bin_op = data[@intFromEnum(inst)].bin_op;
277 try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none });278 try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none });
...@@ -577,6 +578,11 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {...@@ -577,6 +578,11 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
577578
578 try self.verifyInst(inst);579 try self.verifyInst(inst);
579 },580 },
581 .legalize_vec_store_elem => {
582 const pl_op = data[@intFromEnum(inst)].pl_op;
583 const bin = self.air.extraData(Air.Bin, pl_op.payload).data;
584 try self.verifyInstOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });
585 },
580 }586 }
581 }587 }
582}588}
src/Air/print.zig+14
...@@ -171,6 +171,7 @@ const Writer = struct {...@@ -171,6 +171,7 @@ const Writer = struct {
171 .memmove,171 .memmove,
172 .memset,172 .memset,
173 .memset_safe,173 .memset_safe,
174 .legalize_vec_elem_val,
174 => try w.writeBinOp(s, inst),175 => try w.writeBinOp(s, inst),
175176
176 .is_null,177 .is_null,
...@@ -331,6 +332,7 @@ const Writer = struct {...@@ -331,6 +332,7 @@ const Writer = struct {
331 .reduce, .reduce_optimized => try w.writeReduce(s, inst),332 .reduce, .reduce_optimized => try w.writeReduce(s, inst),
332 .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),333 .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),
333 .runtime_nav_ptr => try w.writeRuntimeNavPtr(s, inst),334 .runtime_nav_ptr => try w.writeRuntimeNavPtr(s, inst),
335 .legalize_vec_store_elem => try w.writeLegalizeVecStoreElem(s, inst),
334336
335 .work_item_id,337 .work_item_id,
336 .work_group_size,338 .work_group_size,
...@@ -508,6 +510,18 @@ const Writer = struct {...@@ -508,6 +510,18 @@ const Writer = struct {
508 try w.writeOperand(s, inst, 2, pl_op.operand);510 try w.writeOperand(s, inst, 2, pl_op.operand);
509 }511 }
510512
513 fn writeLegalizeVecStoreElem(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {
514 const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
515 const bin = w.air.extraData(Air.Bin, pl_op.payload).data;
516
517 try w.writeOperand(s, inst, 0, pl_op.operand);
518 try s.writeAll(", ");
519 try w.writeOperand(s, inst, 1, bin.lhs);
520 try s.writeAll(", ");
521 try w.writeOperand(s, inst, 2, bin.rhs);
522 try s.writeAll(", ");
523 }
524
511 fn writeShuffleOne(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {525 fn writeShuffleOne(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {
512 const unwrapped = w.air.unwrapShuffleOne(w.pt.zcu, inst);526 const unwrapped = w.air.unwrapShuffleOne(w.pt.zcu, inst);
513 try w.writeType(s, unwrapped.result_ty);527 try w.writeType(s, unwrapped.result_ty);
src/Air/types_resolved.zig+2
...@@ -88,6 +88,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {...@@ -88,6 +88,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
88 .atomic_store_monotonic,88 .atomic_store_monotonic,
89 .atomic_store_release,89 .atomic_store_release,
90 .atomic_store_seq_cst,90 .atomic_store_seq_cst,
91 .legalize_vec_elem_val,
91 => {92 => {
92 if (!checkRef(data.bin_op.lhs, zcu)) return false;93 if (!checkRef(data.bin_op.lhs, zcu)) return false;
93 if (!checkRef(data.bin_op.rhs, zcu)) return false;94 if (!checkRef(data.bin_op.rhs, zcu)) return false;
...@@ -322,6 +323,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {...@@ -322,6 +323,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
322323
323 .select,324 .select,
324 .mul_add,325 .mul_add,
326 .legalize_vec_store_elem,
325 => {327 => {
326 const bin = air.extraData(Air.Bin, data.pl_op.payload).data;328 const bin = air.extraData(Air.Bin, data.pl_op.payload).data;
327 if (!checkRef(data.pl_op.operand, zcu)) return false;329 if (!checkRef(data.pl_op.operand, zcu)) return false;
src/Sema.zig+14-9
...@@ -15930,16 +15930,21 @@ fn zirOverflowArithmetic(...@@ -15930,16 +15930,21 @@ fn zirOverflowArithmetic(
15930 }15930 }
15931 }15931 }
15932 // If either of the arguments is one, the result is the other and no overflow occured.15932 // If either of the arguments is one, the result is the other and no overflow occured.
15933 const scalar_one = try pt.intValue(dest_ty.scalarType(zcu), 1);15933 const dest_scalar_ty = dest_ty.scalarType(zcu);
15934 const vec_one = try sema.splat(dest_ty, scalar_one);15934 const dest_scalar_int = dest_scalar_ty.intInfo(zcu);
15935 if (maybe_lhs_val) |lhs_val| {15935 // We could still be working with i1, where '1' is not a legal value!
15936 if (!lhs_val.isUndef(zcu) and try sema.compareAll(lhs_val, .eq, vec_one, dest_ty)) {15936 if (!(dest_scalar_int.bits == 1 and dest_scalar_int.signedness == .signed)) {
15937 break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };15937 const scalar_one = try pt.intValue(dest_scalar_ty, 1);
15938 const vec_one = try sema.splat(dest_ty, scalar_one);
15939 if (maybe_lhs_val) |lhs_val| {
15940 if (!lhs_val.isUndef(zcu) and try sema.compareAll(lhs_val, .eq, vec_one, dest_ty)) {
15941 break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };
15942 }
15938 }15943 }
15939 }15944 if (maybe_rhs_val) |rhs_val| {
15940 if (maybe_rhs_val) |rhs_val| {15945 if (!rhs_val.isUndef(zcu) and try sema.compareAll(rhs_val, .eq, vec_one, dest_ty)) {
15941 if (!rhs_val.isUndef(zcu) and try sema.compareAll(rhs_val, .eq, vec_one, dest_ty)) {15946 break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };
15942 break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };15947 }
15943 }15948 }
15944 }15949 }
1594515950
src/codegen/aarch64/Select.zig+9
...@@ -134,6 +134,10 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {...@@ -134,6 +134,10 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
134 var air_inst_index = air_body[air_body_index];134 var air_inst_index = air_body[air_body_index];
135 const initial_def_order_len = isel.def_order.count();135 const initial_def_order_len = isel.def_order.count();
136 air_tag: switch (air_tags[@intFromEnum(air_inst_index)]) {136 air_tag: switch (air_tags[@intFromEnum(air_inst_index)]) {
137 // No "scalarize" legalizations are enabled, so these instructions never appear.
138 .legalize_vec_elem_val => unreachable,
139 .legalize_vec_store_elem => unreachable,
140
137 .arg,141 .arg,
138 .ret_addr,142 .ret_addr,
139 .frame_addr,143 .frame_addr,
...@@ -950,6 +954,11 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,...@@ -950,6 +954,11 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
950 };954 };
951 air_tag: switch (air.next().?) {955 air_tag: switch (air.next().?) {
952 else => |air_tag| return isel.fail("unimplemented {t}", .{air_tag}),956 else => |air_tag| return isel.fail("unimplemented {t}", .{air_tag}),
957
958 // No "scalarize" legalizations are enabled, so these instructions never appear.
959 .legalize_vec_elem_val => unreachable,
960 .legalize_vec_store_elem => unreachable,
961
953 .arg => {962 .arg => {
954 const arg_vi = isel.live_values.fetchRemove(air.inst_index).?.value;963 const arg_vi = isel.live_values.fetchRemove(air.inst_index).?.value;
955 defer arg_vi.deref(isel);964 defer arg_vi.deref(isel);
src/codegen/c.zig+4
...@@ -3325,6 +3325,10 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {...@@ -3325,6 +3325,10 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
3325 // zig fmt: off3325 // zig fmt: off
3326 .inferred_alloc, .inferred_alloc_comptime => unreachable,3326 .inferred_alloc, .inferred_alloc_comptime => unreachable,
33273327
3328 // No "scalarize" legalizations are enabled, so these instructions never appear.
3329 .legalize_vec_elem_val => unreachable,
3330 .legalize_vec_store_elem => unreachable,
3331
3328 .arg => try airArg(f, inst),3332 .arg => try airArg(f, inst),
33293333
3330 .breakpoint => try airBreakpoint(f),3334 .breakpoint => try airBreakpoint(f),
src/codegen/llvm.zig+5
...@@ -4886,6 +4886,11 @@ pub const FuncGen = struct {...@@ -4886,6 +4886,11 @@ pub const FuncGen = struct {
48864886
4887 const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {4887 const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
4888 // zig fmt: off4888 // zig fmt: off
4889
4890 // No "scalarize" legalizations are enabled, so these instructions never appear.
4891 .legalize_vec_elem_val => unreachable,
4892 .legalize_vec_store_elem => unreachable,
4893
4889 .add => try self.airAdd(inst, .normal),4894 .add => try self.airAdd(inst, .normal),
4890 .add_optimized => try self.airAdd(inst, .fast),4895 .add_optimized => try self.airAdd(inst, .fast),
4891 .add_wrap => try self.airAddWrap(inst),4896 .add_wrap => try self.airAddWrap(inst),
src/codegen/riscv64/CodeGen.zig+5
...@@ -1391,6 +1391,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {...@@ -1391,6 +1391,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1391 const tag = air_tags[@intFromEnum(inst)];1391 const tag = air_tags[@intFromEnum(inst)];
1392 switch (tag) {1392 switch (tag) {
1393 // zig fmt: off1393 // zig fmt: off
1394
1395 // No "scalarize" legalizations are enabled, so these instructions never appear.
1396 .legalize_vec_elem_val => unreachable,
1397 .legalize_vec_store_elem => unreachable,
1398
1394 .add,1399 .add,
1395 .add_wrap,1400 .add_wrap,
1396 .sub,1401 .sub,
src/codegen/sparc64/CodeGen.zig+5
...@@ -479,6 +479,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -479,6 +479,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
479 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();479 self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
480 switch (air_tags[@intFromEnum(inst)]) {480 switch (air_tags[@intFromEnum(inst)]) {
481 // zig fmt: off481 // zig fmt: off
482
483 // No "scalarize" legalizations are enabled, so these instructions never appear.
484 .legalize_vec_elem_val => unreachable,
485 .legalize_vec_store_elem => unreachable,
486
482 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),487 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
483 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),488 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
484489
src/codegen/wasm/CodeGen.zig+4
...@@ -1786,6 +1786,10 @@ fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum...@@ -1786,6 +1786,10 @@ fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum
1786fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {1786fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1787 const air_tags = cg.air.instructions.items(.tag);1787 const air_tags = cg.air.instructions.items(.tag);
1788 return switch (air_tags[@intFromEnum(inst)]) {1788 return switch (air_tags[@intFromEnum(inst)]) {
1789 // No "scalarize" legalizations are enabled, so these instructions never appear.
1790 .legalize_vec_elem_val => unreachable,
1791 .legalize_vec_store_elem => unreachable,
1792
1789 .inferred_alloc, .inferred_alloc_comptime => unreachable,1793 .inferred_alloc, .inferred_alloc_comptime => unreachable,
17901794
1791 .add => cg.airBinOp(inst, .add),1795 .add => cg.airBinOp(inst, .add),
src/codegen/x86_64/CodeGen.zig+629-1
...@@ -103926,7 +103926,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -103926,7 +103926,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103926 try ops[0].toOffset(0, cg);103926 try ops[0].toOffset(0, cg);
103927 try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg);103927 try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg);
103928 },103928 },
103929 .array_elem_val => {103929 .array_elem_val, .legalize_vec_elem_val => {
103930 const bin_op = air_datas[@intFromEnum(inst)].bin_op;103930 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
103931 const array_ty = cg.typeOf(bin_op.lhs);103931 const array_ty = cg.typeOf(bin_op.lhs);
103932 const res_ty = array_ty.elemType2(zcu);103932 const res_ty = array_ty.elemType2(zcu);
...@@ -173061,6 +173061,634 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -173061,6 +173061,634 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
173061 .c_va_copy => try cg.airVaCopy(inst),173061 .c_va_copy => try cg.airVaCopy(inst),
173062 .c_va_end => try cg.airVaEnd(inst),173062 .c_va_end => try cg.airVaEnd(inst),
173063 .c_va_start => try cg.airVaStart(inst),173063 .c_va_start => try cg.airVaStart(inst),
173064 .legalize_vec_store_elem => {
173065 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
173066 const bin = cg.air.extraData(Air.Bin, pl_op.payload).data;
173067 // vector_ptr, index, elem_val
173068 var ops = try cg.tempsFromOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });
173069 cg.select(&.{}, &.{}, &ops, comptime &.{ .{
173070 .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },
173071 .patterns = &.{
173072 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } },
173073 },
173074 .extra_temps = .{
173075 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
173076 .unused,
173077 .unused,
173078 .unused,
173079 .unused,
173080 .unused,
173081 .unused,
173082 .unused,
173083 .unused,
173084 .unused,
173085 .unused,
173086 },
173087 .clobbers = .{ .eflags = true },
173088 .each = .{ .once = &.{
173089 .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ },
173090 .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ },
173091 .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ },
173092 } },
173093 }, .{
173094 .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },
173095 .patterns = &.{
173096 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } },
173097 },
173098 .extra_temps = .{
173099 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
173100 .unused,
173101 .unused,
173102 .unused,
173103 .unused,
173104 .unused,
173105 .unused,
173106 .unused,
173107 .unused,
173108 .unused,
173109 .unused,
173110 },
173111 .clobbers = .{ .eflags = true },
173112 .each = .{ .once = &.{
173113 .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ },
173114 .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ },
173115 .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ },
173116 } },
173117 }, .{
173118 .required_features = .{ .cmov, null, null, null },
173119 .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },
173120 .patterns = &.{
173121 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173122 },
173123 .extra_temps = .{
173124 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
173125 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
173126 .unused,
173127 .unused,
173128 .unused,
173129 .unused,
173130 .unused,
173131 .unused,
173132 .unused,
173133 .unused,
173134 .unused,
173135 },
173136 .clobbers = .{ .eflags = true },
173137 .each = .{ .once = &.{
173138 .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ },
173139 .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ },
173140 .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ },
173141 .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ },
173142 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173143 .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ },
173144 .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ },
173145 } },
173146 }, .{
173147 .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },
173148 .patterns = &.{
173149 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173150 },
173151 .extra_temps = .{
173152 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
173153 .unused,
173154 .unused,
173155 .unused,
173156 .unused,
173157 .unused,
173158 .unused,
173159 .unused,
173160 .unused,
173161 .unused,
173162 .unused,
173163 },
173164 .clobbers = .{ .eflags = true },
173165 .each = .{ .once = &.{
173166 .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ },
173167 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173168 .{ ._, ._nz, .j, .@"0f", ._, ._, ._ },
173169 .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ },
173170 .{ ._, ._mp, .j, .@"1f", ._, ._, ._ },
173171 .{ .@"0:", ._s, .bt, .tmp0d, .src1d, ._, ._ },
173172 .{ .@"1:", ._, .mov, .lea(.src0b), .tmp0b, ._, ._ },
173173 } },
173174 }, .{
173175 .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool },
173176 .patterns = &.{
173177 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } },
173178 },
173179 .clobbers = .{ .eflags = true },
173180 .each = .{ .once = &.{
173181 .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ },
173182 } },
173183 }, .{
173184 .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool },
173185 .patterns = &.{
173186 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } },
173187 },
173188 .clobbers = .{ .eflags = true },
173189 .each = .{ .once = &.{
173190 .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ },
173191 } },
173192 }, .{
173193 .required_features = .{ .cmov, null, null, null },
173194 .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool },
173195 .patterns = &.{
173196 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173197 },
173198 .extra_temps = .{
173199 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
173200 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
173201 .unused,
173202 .unused,
173203 .unused,
173204 .unused,
173205 .unused,
173206 .unused,
173207 .unused,
173208 .unused,
173209 .unused,
173210 },
173211 .clobbers = .{ .eflags = true },
173212 .each = .{ .once = &.{
173213 .{ ._, ._, .movzx, .tmp0d, .lea(.src0w), ._, ._ },
173214 .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ },
173215 .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ },
173216 .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ },
173217 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173218 .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ },
173219 .{ ._, ._, .mov, .lea(.src0w), .tmp0w, ._, ._ },
173220 } },
173221 }, .{
173222 .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool },
173223 .patterns = &.{
173224 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173225 },
173226 .clobbers = .{ .eflags = true },
173227 .each = .{ .once = &.{
173228 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173229 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
173230 .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ },
173231 .{ ._, ._mp, .j, .@"0f", ._, ._, ._ },
173232 .{ .@"1:", ._s, .bt, .lea(.src0w), .src1w, ._, ._ },
173233 } },
173234 }, .{
173235 .src_constraints = .{ .ptr_any_bool_vec, .any, .bool },
173236 .patterns = &.{
173237 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } },
173238 },
173239 .clobbers = .{ .eflags = true },
173240 .each = .{ .once = &.{
173241 .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ },
173242 } },
173243 }, .{
173244 .src_constraints = .{ .ptr_any_bool_vec, .any, .bool },
173245 .patterns = &.{
173246 .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } },
173247 },
173248 .clobbers = .{ .eflags = true },
173249 .each = .{ .once = &.{
173250 .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ },
173251 } },
173252 }, .{
173253 .required_features = .{ .cmov, null, null, null },
173254 .src_constraints = .{ .{ .ptr_bool_vec = .dword }, .any, .bool },
173255 .patterns = &.{
173256 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173257 },
173258 .extra_temps = .{
173259 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
173260 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
173261 .unused,
173262 .unused,
173263 .unused,
173264 .unused,
173265 .unused,
173266 .unused,
173267 .unused,
173268 .unused,
173269 .unused,
173270 },
173271 .clobbers = .{ .eflags = true },
173272 .each = .{ .once = &.{
173273 .{ ._, ._, .mov, .tmp0d, .lea(.src0d), ._, ._ },
173274 .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ },
173275 .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ },
173276 .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ },
173277 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173278 .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ },
173279 .{ ._, ._, .mov, .lea(.src0d), .tmp0d, ._, ._ },
173280 } },
173281 }, .{
173282 .required_features = .{ .@"64bit", .cmov, null, null },
173283 .src_constraints = .{ .{ .ptr_bool_vec = .qword }, .any, .bool },
173284 .patterns = &.{
173285 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173286 },
173287 .extra_temps = .{
173288 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
173289 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
173290 .unused,
173291 .unused,
173292 .unused,
173293 .unused,
173294 .unused,
173295 .unused,
173296 .unused,
173297 .unused,
173298 .unused,
173299 },
173300 .clobbers = .{ .eflags = true },
173301 .each = .{ .once = &.{
173302 .{ ._, ._, .mov, .tmp0q, .lea(.src0q), ._, ._ },
173303 .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ },
173304 .{ ._, ._r, .bt, .tmp1q, .src1q, ._, ._ },
173305 .{ ._, ._s, .bt, .tmp0q, .src1q, ._, ._ },
173306 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173307 .{ ._, ._z, .cmov, .tmp0q, .tmp1q, ._, ._ },
173308 .{ ._, ._, .mov, .lea(.src0q), .tmp0q, ._, ._ },
173309 } },
173310 }, .{
173311 .required_features = .{ .cmov, null, null, null },
173312 .src_constraints = .{ .ptr_any_bool_vec, .any, .bool },
173313 .patterns = &.{
173314 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173315 },
173316 .extra_temps = .{
173317 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
173318 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
173319 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
173320 .unused,
173321 .unused,
173322 .unused,
173323 .unused,
173324 .unused,
173325 .unused,
173326 .unused,
173327 .unused,
173328 },
173329 .clobbers = .{ .eflags = true },
173330 .each = .{ .once = &.{
173331 .{ ._, ._, .mov, .tmp0d, .src1d, ._, ._ },
173332 .{ ._, ._r, .sh, .tmp0d, .ui(5), ._, ._ },
173333 .{ ._, ._, .mov, .tmp1d, .leasi(.src0d, .@"4", .tmp0), ._, ._ },
173334 .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ },
173335 .{ ._, ._r, .bt, .tmp2d, .src1d, ._, ._ },
173336 .{ ._, ._s, .bt, .tmp1d, .src1d, ._, ._ },
173337 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173338 .{ ._, ._z, .cmov, .tmp1d, .tmp2d, ._, ._ },
173339 .{ ._, ._, .mov, .leasi(.src0d, .@"4", .tmp0), .tmp1d, ._, ._ },
173340 } },
173341 }, .{
173342 .src_constraints = .{ .ptr_any_bool_vec, .any, .bool },
173343 .patterns = &.{
173344 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173345 },
173346 .clobbers = .{ .eflags = true },
173347 .each = .{ .once = &.{
173348 .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ },
173349 .{ ._, ._nz, .j, .@"1f", ._, ._, ._ },
173350 .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ },
173351 .{ ._, ._mp, .j, .@"0f", ._, ._, ._ },
173352 .{ .@"1:", ._s, .bt, .lea(.src0d), .src1d, ._, ._ },
173353 } },
173354 }, .{
173355 .src_constraints = .{ .any, .any, .{ .int = .byte } },
173356 .patterns = &.{
173357 .{ .src = .{ .to_gpr, .simm32, .imm8 } },
173358 .{ .src = .{ .to_gpr, .simm32, .to_gpr } },
173359 },
173360 .each = .{ .once = &.{
173361 .{ ._, ._, .mov, .leaa(.src0b, .add_src0_elem_size_mul_src1), .src2b, ._, ._ },
173362 } },
173363 }, .{
173364 .src_constraints = .{ .any, .any, .{ .int = .byte } },
173365 .patterns = &.{
173366 .{ .src = .{ .to_gpr, .to_gpr, .imm8 } },
173367 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173368 },
173369 .each = .{ .once = &.{
173370 .{ ._, ._, .mov, .leai(.src0b, .src1), .src2b, ._, ._ },
173371 } },
173372 }, .{
173373 .src_constraints = .{ .any, .any, .{ .int = .word } },
173374 .patterns = &.{
173375 .{ .src = .{ .to_gpr, .simm32, .imm16 } },
173376 .{ .src = .{ .to_gpr, .simm32, .to_gpr } },
173377 },
173378 .each = .{ .once = &.{
173379 .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2w, ._, ._ },
173380 } },
173381 }, .{
173382 .src_constraints = .{ .any, .any, .{ .int = .word } },
173383 .patterns = &.{
173384 .{ .src = .{ .to_gpr, .to_gpr, .imm16 } },
173385 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173386 },
173387 .each = .{ .once = &.{
173388 .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .src2w, ._, ._ },
173389 } },
173390 }, .{
173391 .required_features = .{ .avx, null, null, null },
173392 .src_constraints = .{ .any, .any, .{ .float = .word } },
173393 .patterns = &.{
173394 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173395 },
173396 .each = .{ .once = &.{
173397 .{ ._, .vp_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ },
173398 } },
173399 }, .{
173400 .required_features = .{ .sse4_1, null, null, null },
173401 .src_constraints = .{ .any, .any, .{ .float = .word } },
173402 .patterns = &.{
173403 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173404 },
173405 .each = .{ .once = &.{
173406 .{ ._, .p_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ },
173407 } },
173408 }, .{
173409 .required_features = .{ .sse2, null, null, null },
173410 .src_constraints = .{ .any, .any, .{ .float = .word } },
173411 .patterns = &.{
173412 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173413 },
173414 .extra_temps = .{
173415 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
173416 .unused,
173417 .unused,
173418 .unused,
173419 .unused,
173420 .unused,
173421 .unused,
173422 .unused,
173423 .unused,
173424 .unused,
173425 .unused,
173426 },
173427 .each = .{ .once = &.{
173428 .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ },
173429 .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp0w, ._, ._ },
173430 } },
173431 }, .{
173432 .required_features = .{ .sse, null, null, null },
173433 .src_constraints = .{ .any, .any, .{ .float = .word } },
173434 .patterns = &.{
173435 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173436 },
173437 .extra_temps = .{
173438 .{ .type = .f32, .kind = .mem },
173439 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
173440 .unused,
173441 .unused,
173442 .unused,
173443 .unused,
173444 .unused,
173445 .unused,
173446 .unused,
173447 .unused,
173448 .unused,
173449 },
173450 .each = .{ .once = &.{
173451 .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ },
173452 .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ },
173453 .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp1w, ._, ._ },
173454 } },
173455 }, .{
173456 .required_features = .{ .avx, null, null, null },
173457 .src_constraints = .{ .any, .any, .{ .float = .word } },
173458 .patterns = &.{
173459 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173460 },
173461 .each = .{ .once = &.{
173462 .{ ._, .vp_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ },
173463 } },
173464 }, .{
173465 .required_features = .{ .sse4_1, null, null, null },
173466 .src_constraints = .{ .any, .any, .{ .float = .word } },
173467 .patterns = &.{
173468 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173469 },
173470 .each = .{ .once = &.{
173471 .{ ._, .p_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ },
173472 } },
173473 }, .{
173474 .required_features = .{ .sse2, null, null, null },
173475 .src_constraints = .{ .any, .any, .{ .float = .word } },
173476 .patterns = &.{
173477 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173478 },
173479 .extra_temps = .{
173480 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
173481 .unused,
173482 .unused,
173483 .unused,
173484 .unused,
173485 .unused,
173486 .unused,
173487 .unused,
173488 .unused,
173489 .unused,
173490 .unused,
173491 },
173492 .each = .{ .once = &.{
173493 .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ },
173494 .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp0w, ._, ._ },
173495 } },
173496 }, .{
173497 .required_features = .{ .sse, null, null, null },
173498 .src_constraints = .{ .any, .any, .{ .float = .word } },
173499 .patterns = &.{
173500 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173501 },
173502 .extra_temps = .{
173503 .{ .type = .f32, .kind = .mem },
173504 .{ .type = .f16, .kind = .{ .rc = .general_purpose } },
173505 .unused,
173506 .unused,
173507 .unused,
173508 .unused,
173509 .unused,
173510 .unused,
173511 .unused,
173512 .unused,
173513 .unused,
173514 },
173515 .each = .{ .once = &.{
173516 .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ },
173517 .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ },
173518 .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp1w, ._, ._ },
173519 } },
173520 }, .{
173521 .src_constraints = .{ .any, .any, .{ .int = .dword } },
173522 .patterns = &.{
173523 .{ .src = .{ .to_gpr, .simm32, .imm32 } },
173524 .{ .src = .{ .to_gpr, .simm32, .to_gpr } },
173525 },
173526 .each = .{ .once = &.{
173527 .{ ._, ._, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2d, ._, ._ },
173528 } },
173529 }, .{
173530 .src_constraints = .{ .any, .any, .{ .int = .dword } },
173531 .patterns = &.{
173532 .{ .src = .{ .to_gpr, .to_gpr, .imm32 } },
173533 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173534 },
173535 .each = .{ .once = &.{
173536 .{ ._, ._, .mov, .leasi(.src0d, .@"4", .src1), .src2d, ._, ._ },
173537 } },
173538 }, .{
173539 .required_features = .{ .avx, null, null, null },
173540 .src_constraints = .{ .any, .any, .{ .float = .dword } },
173541 .patterns = &.{
173542 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173543 },
173544 .each = .{ .once = &.{
173545 .{ ._, .v_ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
173546 } },
173547 }, .{
173548 .required_features = .{ .sse, null, null, null },
173549 .src_constraints = .{ .any, .any, .{ .float = .dword } },
173550 .patterns = &.{
173551 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173552 },
173553 .each = .{ .once = &.{
173554 .{ ._, ._ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
173555 } },
173556 }, .{
173557 .required_features = .{ .avx, null, null, null },
173558 .src_constraints = .{ .any, .any, .{ .float = .dword } },
173559 .patterns = &.{
173560 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173561 },
173562 .each = .{ .once = &.{
173563 .{ ._, .v_ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ },
173564 } },
173565 }, .{
173566 .required_features = .{ .sse, null, null, null },
173567 .src_constraints = .{ .any, .any, .{ .float = .dword } },
173568 .patterns = &.{
173569 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173570 },
173571 .each = .{ .once = &.{
173572 .{ ._, ._ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ },
173573 } },
173574 }, .{
173575 .required_features = .{ .@"64bit", null, null, null },
173576 .src_constraints = .{ .any, .any, .{ .int = .qword } },
173577 .patterns = &.{
173578 .{ .src = .{ .to_gpr, .simm32, .simm32 } },
173579 .{ .src = .{ .to_gpr, .simm32, .to_gpr } },
173580 },
173581 .each = .{ .once = &.{
173582 .{ ._, ._, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2q, ._, ._ },
173583 } },
173584 }, .{
173585 .required_features = .{ .@"64bit", null, null, null },
173586 .src_constraints = .{ .any, .any, .{ .int = .qword } },
173587 .patterns = &.{
173588 .{ .src = .{ .to_gpr, .to_gpr, .simm32 } },
173589 .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } },
173590 },
173591 .each = .{ .once = &.{
173592 .{ ._, ._, .mov, .leasi(.src0q, .@"8", .src1), .src2q, ._, ._ },
173593 } },
173594 }, .{
173595 .required_features = .{ .avx, null, null, null },
173596 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173597 .patterns = &.{
173598 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173599 },
173600 .each = .{ .once = &.{
173601 .{ ._, .v_sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
173602 } },
173603 }, .{
173604 .required_features = .{ .sse2, null, null, null },
173605 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173606 .patterns = &.{
173607 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173608 },
173609 .each = .{ .once = &.{
173610 .{ ._, ._sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
173611 } },
173612 }, .{
173613 .required_features = .{ .sse, null, null, null },
173614 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173615 .patterns = &.{
173616 .{ .src = .{ .to_gpr, .simm32, .to_sse } },
173617 },
173618 .each = .{ .once = &.{
173619 .{ ._, ._ps, .movl, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ },
173620 } },
173621 }, .{
173622 .required_features = .{ .avx, null, null, null },
173623 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173624 .patterns = &.{
173625 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173626 },
173627 .each = .{ .once = &.{
173628 .{ ._, .v_sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ },
173629 } },
173630 }, .{
173631 .required_features = .{ .sse2, null, null, null },
173632 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173633 .patterns = &.{
173634 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173635 },
173636 .each = .{ .once = &.{
173637 .{ ._, ._sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ },
173638 } },
173639 }, .{
173640 .required_features = .{ .sse, null, null, null },
173641 .src_constraints = .{ .any, .any, .{ .float = .qword } },
173642 .patterns = &.{
173643 .{ .src = .{ .to_gpr, .to_gpr, .to_sse } },
173644 },
173645 .each = .{ .once = &.{
173646 .{ ._, ._ps, .movl, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ },
173647 } },
173648 } }) catch |err| switch (err) {
173649 error.SelectFailed => {
173650 const elem_size = cg.typeOf(bin.rhs).abiSize(zcu);
173651 while (try ops[0].toRegClass(true, .general_purpose, cg) or
173652 try ops[1].toRegClass(true, .general_purpose, cg))
173653 {}
173654 const base_reg = ops[0].tracking(cg).short.register.to64();
173655 const rhs_reg = ops[1].tracking(cg).short.register.to64();
173656 if (!std.math.isPowerOfTwo(elem_size)) {
173657 try cg.spillEflagsIfOccupied();
173658 try cg.asmRegisterRegisterImmediate(
173659 .{ .i_, .mul },
173660 rhs_reg,
173661 rhs_reg,
173662 .u(elem_size),
173663 );
173664 try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{
173665 .base = .{ .reg = base_reg },
173666 .mod = .{ .rm = .{ .index = rhs_reg } },
173667 });
173668 } else if (elem_size > 8) {
173669 try cg.spillEflagsIfOccupied();
173670 try cg.asmRegisterImmediate(
173671 .{ ._l, .sh },
173672 rhs_reg,
173673 .u(std.math.log2_int(u64, elem_size)),
173674 );
173675 try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{
173676 .base = .{ .reg = base_reg },
173677 .mod = .{ .rm = .{ .index = rhs_reg } },
173678 });
173679 } else try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{
173680 .base = .{ .reg = base_reg },
173681 .mod = .{ .rm = .{
173682 .index = rhs_reg,
173683 .scale = .fromFactor(@intCast(elem_size)),
173684 } },
173685 });
173686 try ops[0].store(&ops[2], .{}, cg);
173687 },
173688 else => |e| return e,
173689 };
173690 for (ops) |op| try op.die(cg);
173691 },
173064 .work_item_id, .work_group_size, .work_group_id => unreachable,173692 .work_item_id, .work_group_size, .work_group_id => unreachable,
173065 }173693 }
173066 try cg.resetTemps(@enumFromInt(0));173694 try cg.resetTemps(@enumFromInt(0));