Commit 9a7d28fe58c0dbd34b0e41fe0e4fed6d57c7ad45 | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2025-10-30 09:11:36+00:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2025-11-12 16:00:15+00:00
log	9a7d28fe58c0dbd34b0e41fe0e4fed6d57c7ad45
tree	e991c9a41bd17eecc626138f9bc8c720f08ff276
parent	ee0a0f119c25cb5e4436aa705f672316e80a74d1
signature	Commit is signed but in an unrecognized format.

Legalize: rewrite several legalizations

The main goal of this change was to avoid emitting the
`vector_store_elem` AIR tag, because this represents an operation which
Zig no longer supports (and hence Sema no longer emits) as of 010d9a6
(because runtime vector indices are now forbidden). Backends should not
need to lower this operation, so I rewrote the legalizations which
emitted it (scalarizations of vector operations) to instead unroll the
loop and hence emit comptime-known vector indices.

In doing this, I actually reworked those legalizations to use a
different strategy; instead of using an `alloc` and storing to
individual vector elements, the vector is constructed by-val, for
instance by performing the scalar operation on all elements and passing
them to an `aggregate_init`. This is vastly simpler to implement in
Legalize, conceptually simpler, and doesn't severely pessimise memory
usage, because a non-optimizing backend will store the full vector on
the stack either way.

Given the above rationale, I also ended up reworking several other
legalizations to use simpler lowerings. The legalizations in question
were bitcast scalarization, `struct_field_val` of `packed struct`s
(where we just bitcast to an integer and perform the appropriate
shift/trunc sequence), and `aggregate_init` of a `packed struct` (also
implemented in terms of integer bitwise operations with bitcasts to and
from the actual types). This hugely simplified some parts of `Legalize`.

So, `Legalize` is now much simpler, and the `vector_store_elem`
instruction is no longer emitted by any part of the compiler so can be
removed in a future commit.

1 files changed, 480 insertions(+), 1383 deletions(-)

src
- Air
  - Legalize.zig

src/Air/Legalize.zig+480-1383

...	...	@@ -14,7 +14,7 @@ features: if (switch (dev.env) {
14	14	return comptime bootstrap_features.contains(feature);
15	15	}
16	16	/// `inline` to propagate comptime-known result.
17		fn hasAny(_: @This(), comptime features: []const Feature) bool {
	17	inline fn hasAny(_: @This(), comptime features: []const Feature) bool {
18	18	return comptime !bootstrap_features.intersectWith(.initMany(features)).eql(.initEmpty());
19	19	}
20	20	} else struct {
...	...	@@ -154,9 +154,9 @@ pub const Feature = enum {
154	154	/// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
155	155	/// first byte of memory until bit pointers know their backing type.
156	156	expand_packed_store,
157		/// Replace `struct_field_val` of a packed field with a `store` and packed `load`.
	157	/// Replace `struct_field_val` of a packed field with a `bitcast` to integer, `shr`, `trunc`, and `bitcast` to field type.
158	158	expand_packed_struct_field_val,
159		/// Replace `aggregate_init` of a packed aggregate with a series a packed `store`s followed by a `load`.
	159	/// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.
160	160	expand_packed_aggregate_init,
161	161
162	162	fn scalarize(tag: Air.Inst.Tag) Feature {
...	...	@@ -409,40 +409,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
409	409	if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
410	410	},
411	411	.bitcast => if (l.features.has(.scalarize_bitcast)) {
412		const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
413
414		const to_ty = ty_op.ty.toType();
415		const to_ty_tag = to_ty.zigTypeTag(zcu);
416		const to_ty_legal = legal: switch (to_ty_tag) {
417		else => true,
418		.array, .vector => {
419		if (to_ty.arrayLen(zcu) == 1) break :legal true;
420		const to_elem_ty = to_ty.childType(zcu);
421		break :legal to_elem_ty.bitSize(zcu) == 8 * to_elem_ty.abiSize(zcu);
422		},
423		};
424
425		const from_ty = l.typeOf(ty_op.operand);
426		const from_ty_legal = legal: switch (from_ty.zigTypeTag(zcu)) {
427		else => true,
428		.array, .vector => {
429		if (from_ty.arrayLen(zcu) == 1) break :legal true;
430		const from_elem_ty = from_ty.childType(zcu);
431		break :legal from_elem_ty.bitSize(zcu) == 8 * from_elem_ty.abiSize(zcu);
432		},
433		};
434
435		if (!to_ty_legal and !from_ty_legal and to_ty.arrayLen(zcu) == from_ty.arrayLen(zcu)) switch (to_ty_tag) {
436		else => unreachable,
437		.array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastToArrayBlockPayload(inst)),
438		.vector => continue :inst try l.scalarize(inst, .bitcast),
439		};
440		if (!to_ty_legal) switch (to_ty_tag) {
441		else => unreachable,
442		.array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultArrayBlockPayload(inst)),
443		.vector => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultVectorBlockPayload(inst)),
444		};
445		if (!from_ty_legal) continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastOperandBlockPayload(inst));
	412	if (try l.scalarizeBitcastBlockPayload(inst)) \|payload\| {
	413	continue :inst l.replaceInst(inst, .block, payload);
	414	}
446	415	},
447	416	.intcast_safe => if (l.features.has(.expand_intcast_safe)) {
448	417	assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both
...	...	@@ -570,13 +539,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
570	539	.load => if (l.features.has(.expand_packed_load)) {
571	540	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
572	541	const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu);
573		if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst));
	542	if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
	543	continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst));
	544	}
574	545	},
575	546	.ret, .ret_safe, .ret_load => {},
576	547	.store, .store_safe => if (l.features.has(.expand_packed_store)) {
577	548	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
578	549	const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu);
579		if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst));
	550	if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
	551	continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst));
	552	}
580	553	},
581	554	.unreach,
582	555	.optional_payload,
...	...	@@ -624,7 +597,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
624	597	switch (vector_ty.vectorLen(zcu)) {
625	598	0 => unreachable,
626	599	1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
627		.ty = Air.internedToRef(vector_ty.childType(zcu).toIntern()),
	600	.ty = .fromType(vector_ty.childType(zcu)),
628	601	.operand = reduce.operand,
629	602	} }),
630	603	else => {},
...	...	@@ -666,9 +639,18 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
666	639	const agg_ty = ty_pl.ty.toType();
667	640	switch (agg_ty.zigTypeTag(zcu)) {
668	641	else => {},
669		.@"struct", .@"union" => switch (agg_ty.containerLayout(zcu)) {
	642	.@"union" => unreachable,
	643	.@"struct" => switch (agg_ty.containerLayout(zcu)) {
670	644	.auto, .@"extern" => {},
671		.@"packed" => continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst)),
	645	.@"packed" => switch (agg_ty.structFieldCount(zcu)) {
	646	0 => unreachable,
	647	// An `aggregate_init` of a packed struct with 1 field is just a fancy bitcast.
	648	1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
	649	.ty = .fromType(agg_ty),
	650	.operand = @enumFromInt(l.air_extra.items[ty_pl.payload]),
	651	} }),
	652	else => continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst)),
	653	},
672	654	},
673	655	}
674	656	},
...	...	@@ -685,7 +667,6 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
685	667	.set_err_return_trace,
686	668	.addrspace_cast,
687	669	.save_err_return_trace_index,
688		.vector_store_elem,
689	670	.runtime_nav_ptr,
690	671	.c_va_arg,
691	672	.c_va_copy,
...	...	@@ -699,7 +680,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
699	680	}
700	681	}
701	682
702		const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, bitcast, cmp_vector, shuffle_one, shuffle_two, select };
	683	const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, cmp_vector, shuffle_one, shuffle_two, select };
703	684	/// inline to propagate comptime-known `replaceInst` result.
704	685	inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag {
705	686	return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form));
...	...	@@ -707,1160 +688,420 @@ inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: Scal
707	688	fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Data {
708	689	const pt = l.pt;
709	690	const zcu = pt.zcu;
	691	const gpa = zcu.gpa;
710	692
711	693	const orig = l.air_instructions.get(@intFromEnum(orig_inst));
712	694	const res_ty = l.typeOfIndex(orig_inst);
713	695	const res_len = res_ty.vectorLen(zcu);
714	696
715		const extra_insts = switch (form) {
716		.un_op, .ty_op, .bitcast => 1,
717		.bin_op, .cmp_vector => 2,
718		.pl_op_bin => 3,
719		.shuffle_one, .shuffle_two => 13,
720		.select => 6,
	697	const inst_per_elem = switch (form) {
	698	.un_op, .ty_op => 2,
	699	.bin_op, .cmp_vector => 3,
	700	.pl_op_bin => 4,
	701	.shuffle_one, .shuffle_two => 1,
	702	.select => 7,
721	703	};
722		var inst_buf: [5 + extra_insts + 9]Air.Inst.Index = undefined;
723		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
724	704
725		var res_block: Block = .init(&inst_buf);
726		{
727		const res_alloc_inst = res_block.add(l, .{
728		.tag = .alloc,
729		.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
730		});
731		const index_alloc_inst = res_block.add(l, .{
732		.tag = .alloc,
733		.data = .{ .ty = .ptr_usize },
734		});
735		_ = res_block.add(l, .{
736		.tag = .store,
737		.data = .{ .bin_op = .{
738		.lhs = index_alloc_inst.toRef(),
739		.rhs = .zero_usize,
740		} },
741		});
	705	var sfba_state = std.heap.stackFallback(@sizeOf([inst_per_elem * 32 + 2]Air.Inst.Index) + @sizeOf([32]Air.Inst.Ref), gpa);
	706	const sfba = sfba_state.get();
742	707
743		var loop: Loop = .init(l, &res_block);
744		loop.block = .init(res_block.stealRemainingCapacity());
745		{
746		const cur_index_inst = loop.block.add(l, .{
747		.tag = .load,
748		.data = .{ .ty_op = .{
749		.ty = .usize_type,
750		.operand = index_alloc_inst.toRef(),
751		} },
752		});
753		_ = loop.block.add(l, .{
754		.tag = .vector_store_elem,
755		.data = .{ .vector_store_elem = .{
756		.vector_ptr = res_alloc_inst.toRef(),
757		.payload = try l.addExtra(Air.Bin, .{
758		.lhs = cur_index_inst.toRef(),
759		.rhs = res_elem: switch (form) {
760		.un_op => loop.block.add(l, .{
761		.tag = orig.tag,
762		.data = .{ .un_op = loop.block.add(l, .{
763		.tag = .array_elem_val,
764		.data = .{ .bin_op = .{
765		.lhs = orig.data.un_op,
766		.rhs = cur_index_inst.toRef(),
767		} },
768		}).toRef() },
769		}).toRef(),
770		.ty_op => loop.block.add(l, .{
771		.tag = orig.tag,
772		.data = .{ .ty_op = .{
773		.ty = Air.internedToRef(res_ty.childType(zcu).toIntern()),
774		.operand = loop.block.add(l, .{
775		.tag = .array_elem_val,
776		.data = .{ .bin_op = .{
777		.lhs = orig.data.ty_op.operand,
778		.rhs = cur_index_inst.toRef(),
779		} },
780		}).toRef(),
781		} },
782		}).toRef(),
783		.bin_op => loop.block.add(l, .{
784		.tag = orig.tag,
785		.data = .{ .bin_op = .{
786		.lhs = loop.block.add(l, .{
787		.tag = .array_elem_val,
788		.data = .{ .bin_op = .{
789		.lhs = orig.data.bin_op.lhs,
790		.rhs = cur_index_inst.toRef(),
791		} },
792		}).toRef(),
793		.rhs = loop.block.add(l, .{
794		.tag = .array_elem_val,
795		.data = .{ .bin_op = .{
796		.lhs = orig.data.bin_op.rhs,
797		.rhs = cur_index_inst.toRef(),
798		} },
799		}).toRef(),
800		} },
801		}).toRef(),
802		.pl_op_bin => {
803		const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
804		break :res_elem loop.block.add(l, .{
805		.tag = orig.tag,
806		.data = .{ .pl_op = .{
807		.payload = try l.addExtra(Air.Bin, .{
808		.lhs = loop.block.add(l, .{
809		.tag = .array_elem_val,
810		.data = .{ .bin_op = .{
811		.lhs = extra.lhs,
812		.rhs = cur_index_inst.toRef(),
813		} },
814		}).toRef(),
815		.rhs = loop.block.add(l, .{
816		.tag = .array_elem_val,
817		.data = .{ .bin_op = .{
818		.lhs = extra.rhs,
819		.rhs = cur_index_inst.toRef(),
820		} },
821		}).toRef(),
822		}),
823		.operand = loop.block.add(l, .{
824		.tag = .array_elem_val,
825		.data = .{ .bin_op = .{
826		.lhs = orig.data.pl_op.operand,
827		.rhs = cur_index_inst.toRef(),
828		} },
829		}).toRef(),
830		} },
831		}).toRef();
832		},
833		.bitcast => loop.block.addBitCast(l, res_ty.childType(zcu), loop.block.add(l, .{
834		.tag = .array_elem_val,
835		.data = .{ .bin_op = .{
836		.lhs = orig.data.ty_op.operand,
837		.rhs = cur_index_inst.toRef(),
838		} },
839		}).toRef()),
840		.cmp_vector => {
841		const extra = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
842		break :res_elem (try loop.block.addCmp(
843		l,
844		extra.compareOperator(),
845		loop.block.add(l, .{
846		.tag = .array_elem_val,
847		.data = .{ .bin_op = .{
848		.lhs = extra.lhs,
849		.rhs = cur_index_inst.toRef(),
850		} },
851		}).toRef(),
852		loop.block.add(l, .{
853		.tag = .array_elem_val,
854		.data = .{ .bin_op = .{
855		.lhs = extra.rhs,
856		.rhs = cur_index_inst.toRef(),
857		} },
858		}).toRef(),
859		.{ .optimized = switch (orig.tag) {
860		else => unreachable,
861		.cmp_vector => false,
862		.cmp_vector_optimized => true,
863		} },
864		)).toRef();
865		},
866		.shuffle_one, .shuffle_two => {
867		const ip = &zcu.intern_pool;
868		const unwrapped = switch (form) {
869		else => comptime unreachable,
870		.shuffle_one => l.getTmpAir().unwrapShuffleOne(zcu, orig_inst),
871		.shuffle_two => l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst),
872		};
873		const operand_a = switch (form) {
874		else => comptime unreachable,
875		.shuffle_one => unwrapped.operand,
876		.shuffle_two => unwrapped.operand_a,
877		};
878		const operand_a_len = l.typeOf(operand_a).vectorLen(zcu);
879		const elem_ty = res_ty.childType(zcu);
880		var res_elem: Result = .init(l, elem_ty, &loop.block);
881		res_elem.block = .init(loop.block.stealCapacity(extra_insts));
882		{
883		const ExpectedContents = extern struct {
884		mask_elems: [128]InternPool.Index,
885		ct_elems: switch (form) {
886		else => unreachable,
887		.shuffle_one => extern struct {
888		keys: [152]InternPool.Index,
889		header: u8 align(@alignOf(u32)),
890		index: [256][2]u8,
891		},
892		.shuffle_two => void,
893		},
894		};
895		var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
896		std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa);
897		const gpa = stack.get();
	708	// Plus 2 extra instructions for `aggregate_init` and `br`.
	709	const inst_buf = try sfba.alloc(Air.Inst.Index, inst_per_elem * res_len + 2);
	710	defer sfba.free(inst_buf);
898	711
899		const mask_elems = try gpa.alloc(InternPool.Index, res_len);
900		defer gpa.free(mask_elems);
901
902		var ct_elems: switch (form) {
903		else => unreachable,
904		.shuffle_one => std.AutoArrayHashMapUnmanaged(InternPool.Index, void),
905		.shuffle_two => struct {
906		const empty: @This() = .{};
907		inline fn deinit(_: @This(), _: std.mem.Allocator) void {}
908		inline fn ensureTotalCapacity(_: @This(), _: std.mem.Allocator, _: usize) error{}!void {}
909		},
910		} = .empty;
911		defer ct_elems.deinit(gpa);
912		try ct_elems.ensureTotalCapacity(gpa, res_len);
913
914		const mask_elem_ty = try pt.intType(.signed, 1 + Type.smallestUnsignedBits(@max(operand_a_len, switch (form) {
915		else => comptime unreachable,
916		.shuffle_one => res_len,
917		.shuffle_two => l.typeOf(unwrapped.operand_b).vectorLen(zcu),
918		})));
919		for (mask_elems, unwrapped.mask) \|mask_elem_val, mask_elem\| mask_elem_val. = (try pt.intValue(mask_elem_ty, switch (form) {
920		else => comptime unreachable,
921		.shuffle_one => switch (mask_elem.unwrap()) {
922		.elem => \|index\| index,
923		.value => \|elem_val\| if (ip.isUndef(elem_val))
924		operand_a_len
925		else
926		~@as(i33, @intCast((ct_elems.getOrPutAssumeCapacity(elem_val)).index)),
927		},
928		.shuffle_two => switch (mask_elem.unwrap()) {
929		.a_elem => \|a_index\| a_index,
930		.b_elem => \|b_index\| ~@as(i33, b_index),
931		.undef => operand_a_len,
932		},
933		})).toIntern();
934		const mask_ty = try pt.arrayType(.{
935		.len = res_len,
936		.child = mask_elem_ty.toIntern(),
937		});
938		const mask_elem_inst = res_elem.block.add(l, .{
939		.tag = .ptr_elem_val,
940		.data = .{ .bin_op = .{
941		.lhs = Air.internedToRef(try pt.intern(.{ .ptr = .{
942		.ty = (try pt.manyConstPtrType(mask_elem_ty)).toIntern(),
943		.base_addr = .{ .uav = .{
944		.val = (try pt.aggregateValue(mask_ty, mask_elems)).toIntern(),
945		.orig_ty = (try pt.singleConstPtrType(mask_ty)).toIntern(),
946		} },
947		.byte_offset = 0,
948		} })),
949		.rhs = cur_index_inst.toRef(),
950		} },
951		});
952		var def_cond_br: CondBr = .init(l, (try res_elem.block.addCmp(
953		l,
954		.lt,
955		mask_elem_inst.toRef(),
956		try pt.intRef(mask_elem_ty, operand_a_len),
957		.{},
958		)).toRef(), &res_elem.block, .{});
959		def_cond_br.then_block = .init(res_elem.block.stealRemainingCapacity());
960		{
961		const operand_b_used = switch (form) {
962		else => comptime unreachable,
963		.shuffle_one => ct_elems.count() > 0,
964		.shuffle_two => true,
965		};
966		var operand_cond_br: CondBr = undefined;
967		operand_cond_br.then_block = if (operand_b_used) then_block: {
968		operand_cond_br = .init(l, (try def_cond_br.then_block.addCmp(
969		l,
970		.gte,
971		mask_elem_inst.toRef(),
972		try pt.intRef(mask_elem_ty, 0),
973		.{},
974		)).toRef(), &def_cond_br.then_block, .{});
975		break :then_block .init(def_cond_br.then_block.stealRemainingCapacity());
976		} else def_cond_br.then_block;
977		_ = operand_cond_br.then_block.add(l, .{
978		.tag = .br,
979		.data = .{ .br = .{
980		.block_inst = res_elem.inst,
981		.operand = operand_cond_br.then_block.add(l, .{
982		.tag = .array_elem_val,
983		.data = .{ .bin_op = .{
984		.lhs = operand_a,
985		.rhs = operand_cond_br.then_block.add(l, .{
986		.tag = .intcast,
987		.data = .{ .ty_op = .{
988		.ty = .usize_type,
989		.operand = mask_elem_inst.toRef(),
990		} },
991		}).toRef(),
992		} },
993		}).toRef(),
994		} },
995		});
996		if (operand_b_used) {
997		operand_cond_br.else_block = .init(operand_cond_br.then_block.stealRemainingCapacity());
998		_ = operand_cond_br.else_block.add(l, .{
999		.tag = .br,
1000		.data = .{ .br = .{
1001		.block_inst = res_elem.inst,
1002		.operand = if (switch (form) {
1003		else => comptime unreachable,
1004		.shuffle_one => ct_elems.count() > 1,
1005		.shuffle_two => true,
1006		}) operand_cond_br.else_block.add(l, .{
1007		.tag = switch (form) {
1008		else => comptime unreachable,
1009		.shuffle_one => .ptr_elem_val,
1010		.shuffle_two => .array_elem_val,
1011		},
1012		.data = .{ .bin_op = .{
1013		.lhs = operand_b: switch (form) {
1014		else => comptime unreachable,
1015		.shuffle_one => {
1016		const ct_elems_ty = try pt.arrayType(.{
1017		.len = ct_elems.count(),
1018		.child = elem_ty.toIntern(),
1019		});
1020		break :operand_b Air.internedToRef(try pt.intern(.{ .ptr = .{
1021		.ty = (try pt.manyConstPtrType(elem_ty)).toIntern(),
1022		.base_addr = .{ .uav = .{
1023		.val = (try pt.aggregateValue(ct_elems_ty, ct_elems.keys())).toIntern(),
1024		.orig_ty = (try pt.singleConstPtrType(ct_elems_ty)).toIntern(),
1025		} },
1026		.byte_offset = 0,
1027		} }));
1028		},
1029		.shuffle_two => unwrapped.operand_b,
1030		},
1031		.rhs = operand_cond_br.else_block.add(l, .{
1032		.tag = .intcast,
1033		.data = .{ .ty_op = .{
1034		.ty = .usize_type,
1035		.operand = operand_cond_br.else_block.add(l, .{
1036		.tag = .not,
1037		.data = .{ .ty_op = .{
1038		.ty = Air.internedToRef(mask_elem_ty.toIntern()),
1039		.operand = mask_elem_inst.toRef(),
1040		} },
1041		}).toRef(),
1042		} },
1043		}).toRef(),
1044		} },
1045		}).toRef() else res_elem_br: {
1046		_ = operand_cond_br.else_block.stealCapacity(3);
1047		break :res_elem_br Air.internedToRef(ct_elems.keys()[0]);
1048		},
1049		} },
1050		});
1051		def_cond_br.else_block = .init(operand_cond_br.else_block.stealRemainingCapacity());
1052		try operand_cond_br.finish(l);
1053		} else {
1054		def_cond_br.then_block = operand_cond_br.then_block;
1055		_ = def_cond_br.then_block.stealCapacity(6);
1056		def_cond_br.else_block = .init(def_cond_br.then_block.stealRemainingCapacity());
1057		}
1058		}
1059		_ = def_cond_br.else_block.add(l, .{
1060		.tag = .br,
1061		.data = .{ .br = .{
1062		.block_inst = res_elem.inst,
1063		.operand = try pt.undefRef(elem_ty),
1064		} },
1065		});
1066		try def_cond_br.finish(l);
1067		}
1068		try res_elem.finish(l);
1069		break :res_elem res_elem.inst.toRef();
1070		},
1071		.select => {
1072		const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
1073		var res_elem: Result = .init(l, l.typeOf(extra.lhs).childType(zcu), &loop.block);
1074		res_elem.block = .init(loop.block.stealCapacity(extra_insts));
1075		{
1076		var select_cond_br: CondBr = .init(l, res_elem.block.add(l, .{
1077		.tag = .array_elem_val,
1078		.data = .{ .bin_op = .{
1079		.lhs = orig.data.pl_op.operand,
1080		.rhs = cur_index_inst.toRef(),
1081		} },
1082		}).toRef(), &res_elem.block, .{});
1083		select_cond_br.then_block = .init(res_elem.block.stealRemainingCapacity());
1084		_ = select_cond_br.then_block.add(l, .{
1085		.tag = .br,
1086		.data = .{ .br = .{
1087		.block_inst = res_elem.inst,
1088		.operand = select_cond_br.then_block.add(l, .{
1089		.tag = .array_elem_val,
1090		.data = .{ .bin_op = .{
1091		.lhs = extra.lhs,
1092		.rhs = cur_index_inst.toRef(),
1093		} },
1094		}).toRef(),
1095		} },
1096		});
1097		select_cond_br.else_block = .init(select_cond_br.then_block.stealRemainingCapacity());
1098		_ = select_cond_br.else_block.add(l, .{
1099		.tag = .br,
1100		.data = .{ .br = .{
1101		.block_inst = res_elem.inst,
1102		.operand = select_cond_br.else_block.add(l, .{
1103		.tag = .array_elem_val,
1104		.data = .{ .bin_op = .{
1105		.lhs = extra.rhs,
1106		.rhs = cur_index_inst.toRef(),
1107		} },
1108		}).toRef(),
1109		} },
1110		});
1111		try select_cond_br.finish(l);
1112		}
1113		try res_elem.finish(l);
1114		break :res_elem res_elem.inst.toRef();
1115		},
1116		},
1117		}),
1118		} },
1119		});
	712	var main_block: Block = .init(inst_buf);
	713	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
1120	714
1121		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1122		l,
1123		.lt,
1124		cur_index_inst.toRef(),
1125		try pt.intRef(.usize, res_len - 1),
1126		.{},
1127		)).toRef(), &loop.block, .{});
1128		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1129		{
1130		_ = loop_cond_br.then_block.add(l, .{
1131		.tag = .store,
1132		.data = .{ .bin_op = .{
1133		.lhs = index_alloc_inst.toRef(),
1134		.rhs = loop_cond_br.then_block.add(l, .{
1135		.tag = .add,
1136		.data = .{ .bin_op = .{
1137		.lhs = cur_index_inst.toRef(),
1138		.rhs = .one_usize,
1139		} },
1140		}).toRef(),
	715	const elem_buf = try sfba.alloc(Air.Inst.Ref, res_len);
	716	defer sfba.free(elem_buf);
	717
	718	switch (form) {
	719	.un_op => {
	720	const orig_operand = orig.data.un_op;
	721	const un_op_tag = orig.tag;
	722	for (elem_buf, 0..) \|*elem, elem_idx\| {
	723	const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
	724	const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();
	725	elem.* = main_block.addUnOp(l, un_op_tag, operand).toRef();
	726	}
	727	},
	728	.ty_op => {
	729	const orig_operand = orig.data.ty_op.operand;
	730	const orig_ty: Type = .fromInterned(orig.data.ty_op.ty.toInterned().?);
	731	const scalar_ty = orig_ty.childType(zcu);
	732	const ty_op_tag = orig.tag;
	733	for (elem_buf, 0..) \|*elem, elem_idx\| {
	734	const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
	735	const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();
	736	elem.* = main_block.addTyOp(l, ty_op_tag, scalar_ty, operand).toRef();
	737	}
	738	},
	739	.bin_op => {
	740	const orig_operands = orig.data.bin_op;
	741	const bin_op_tag = orig.tag;
	742	for (elem_buf, 0..) \|*elem, elem_idx\| {
	743	const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
	744	const lhs = main_block.addBinOp(l, .array_elem_val, orig_operands.lhs, elem_idx_ref).toRef();
	745	const rhs = main_block.addBinOp(l, .array_elem_val, orig_operands.rhs, elem_idx_ref).toRef();
	746	elem.* = main_block.addBinOp(l, bin_op_tag, lhs, rhs).toRef();
	747	}
	748	},
	749	.pl_op_bin => {
	750	const orig_operand = orig.data.pl_op.operand;
	751	const orig_payload = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
	752	const pl_op_tag = orig.tag;
	753	for (elem_buf, 0..) \|*elem, elem_idx\| {
	754	const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
	755	const operand = main_block.addBinOp(l, .array_elem_val, orig_operand, elem_idx_ref).toRef();
	756	const lhs = main_block.addBinOp(l, .array_elem_val, orig_payload.lhs, elem_idx_ref).toRef();
	757	const rhs = main_block.addBinOp(l, .array_elem_val, orig_payload.rhs, elem_idx_ref).toRef();
	758	elem.* = main_block.add(l, .{
	759	.tag = pl_op_tag,
	760	.data = .{ .pl_op = .{
	761	.payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
	762	.operand = operand,
1141	763	} },
1142		});
1143		_ = loop_cond_br.then_block.add(l, .{
1144		.tag = .repeat,
1145		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1146		});
	764	}).toRef();
1147	765	}
1148		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1149		_ = loop_cond_br.else_block.add(l, .{
1150		.tag = .br,
1151		.data = .{ .br = .{
1152		.block_inst = orig_inst,
1153		.operand = loop_cond_br.else_block.add(l, .{
1154		.tag = .load,
1155		.data = .{ .ty_op = .{
1156		.ty = Air.internedToRef(res_ty.toIntern()),
1157		.operand = res_alloc_inst.toRef(),
1158		} },
1159		}).toRef(),
1160		} },
1161		});
1162		try loop_cond_br.finish(l);
1163		}
1164		try loop.finish(l);
1165		}
1166		return .{ .ty_pl = .{
1167		.ty = Air.internedToRef(res_ty.toIntern()),
1168		.payload = try l.addBlockBody(res_block.body()),
1169		} };
1170		}
1171		fn scalarizeBitcastToArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1172		const pt = l.pt;
1173		const zcu = pt.zcu;
	766	},
	767	.cmp_vector => {
	768	const orig_payload = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
	769	const cmp_op = orig_payload.compareOperator();
	770	const optimized = switch (orig.tag) {
	771	.cmp_vector => false,
	772	.cmp_vector_optimized => true,
	773	else => unreachable,
	774	};
	775	for (elem_buf, 0..) \|*elem, elem_idx\| {
	776	const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
	777	const lhs = main_block.addBinOp(l, .array_elem_val, orig_payload.lhs, elem_idx_ref).toRef();
	778	const rhs = main_block.addBinOp(l, .array_elem_val, orig_payload.rhs, elem_idx_ref).toRef();
	779	elem.* = main_block.addCmpScalar(l, cmp_op, lhs, rhs, optimized).toRef();
	780	}
	781	},
	782	.shuffle_one => {
	783	const shuffle = l.getTmpAir().unwrapShuffleOne(zcu, orig_inst);
	784	for (elem_buf, shuffle.mask) \|elem, mask\| elem. = switch (mask.unwrap()) {
	785	.value => \|val\| .fromIntern(val),
	786	.elem => \|src_idx\| elem: {
	787	const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
	788	break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand, src_idx_ref).toRef();
	789	},
	790	};
	791	},
	792	.shuffle_two => {
	793	const shuffle = l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst);
	794	const scalar_ty = res_ty.childType(zcu);
	795	for (elem_buf, shuffle.mask) \|elem, mask\| elem. = switch (mask.unwrap()) {
	796	.undef => .fromValue(try pt.undefValue(scalar_ty)),
	797	.a_elem => \|src_idx\| elem: {
	798	const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
	799	break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand_a, src_idx_ref).toRef();
	800	},
	801	.b_elem => \|src_idx\| elem: {
	802	const src_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, src_idx));
	803	break :elem main_block.addBinOp(l, .array_elem_val, shuffle.operand_b, src_idx_ref).toRef();
	804	},
	805	};
	806	},
	807	.select => {
	808	const orig_cond = orig.data.pl_op.operand;
	809	const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
	810	const res_scalar_ty = res_ty.childType(zcu);
	811	for (elem_buf, 0..) \|*elem, elem_idx\| {
	812	// Payload to be populated later; we need the index early for `br`s.
	813	const elem_block_inst = main_block.add(l, .{
	814	.tag = .block,
	815	.data = .{ .ty_pl = .{
	816	.ty = .fromType(res_scalar_ty),
	817	.payload = undefined,
	818	} },
	819	});
	820	var elem_block: Block = .init(main_block.stealCapacity(2));
1174	821
1175		const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1176		const res_ty = orig_ty_op.ty.toType();
1177		const res_elem_ty = res_ty.childType(zcu);
1178		const res_len = res_ty.arrayLen(zcu);
	822	const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
	823	const cond = elem_block.addBinOp(l, .array_elem_val, orig_cond, elem_idx_ref).toRef();
	824	var condbr: CondBr = .init(l, cond, &elem_block, .{});
1179	825
1180		var inst_buf: [16]Air.Inst.Index = undefined;
1181		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	826	condbr.then_block = .init(main_block.stealCapacity(2));
	827	const lhs = condbr.then_block.addBinOp(l, .array_elem_val, orig_bin.lhs, elem_idx_ref).toRef();
	828	condbr.then_block.addBr(l, elem_block_inst, lhs);
1182	829
1183		var res_block: Block = .init(&inst_buf);
1184		{
1185		const res_alloc_inst = res_block.add(l, .{
1186		.tag = .alloc,
1187		.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1188		});
1189		const index_alloc_inst = res_block.add(l, .{
1190		.tag = .alloc,
1191		.data = .{ .ty = .ptr_usize },
1192		});
1193		_ = res_block.add(l, .{
1194		.tag = .store,
1195		.data = .{ .bin_op = .{
1196		.lhs = index_alloc_inst.toRef(),
1197		.rhs = .zero_usize,
1198		} },
1199		});
	830	condbr.else_block = .init(main_block.stealCapacity(2));
	831	const rhs = condbr.else_block.addBinOp(l, .array_elem_val, orig_bin.rhs, elem_idx_ref).toRef();
	832	condbr.else_block.addBr(l, elem_block_inst, rhs);
1200	833
1201		var loop: Loop = .init(l, &res_block);
1202		loop.block = .init(res_block.stealRemainingCapacity());
1203		{
1204		const cur_index_inst = loop.block.add(l, .{
1205		.tag = .load,
1206		.data = .{ .ty_op = .{
1207		.ty = .usize_type,
1208		.operand = index_alloc_inst.toRef(),
1209		} },
1210		});
1211		_ = loop.block.add(l, .{
1212		.tag = .store,
1213		.data = .{ .bin_op = .{
1214		.lhs = loop.block.add(l, .{
1215		.tag = .ptr_elem_ptr,
1216		.data = .{ .ty_pl = .{
1217		.ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()),
1218		.payload = try l.addExtra(Air.Bin, .{
1219		.lhs = res_alloc_inst.toRef(),
1220		.rhs = cur_index_inst.toRef(),
1221		}),
1222		} },
1223		}).toRef(),
1224		.rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1225		.tag = .array_elem_val,
1226		.data = .{ .bin_op = .{
1227		.lhs = orig_ty_op.operand,
1228		.rhs = cur_index_inst.toRef(),
1229		} },
1230		}).toRef()),
1231		} },
1232		});
	834	try condbr.finish(l);
1233	835
1234		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1235		l,
1236		.lt,
1237		cur_index_inst.toRef(),
1238		try pt.intRef(.usize, res_len - 1),
1239		.{},
1240		)).toRef(), &loop.block, .{});
1241		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1242		{
1243		_ = loop_cond_br.then_block.add(l, .{
1244		.tag = .store,
1245		.data = .{ .bin_op = .{
1246		.lhs = index_alloc_inst.toRef(),
1247		.rhs = loop_cond_br.then_block.add(l, .{
1248		.tag = .add,
1249		.data = .{ .bin_op = .{
1250		.lhs = cur_index_inst.toRef(),
1251		.rhs = .one_usize,
1252		} },
1253		}).toRef(),
1254		} },
1255		});
1256		_ = loop_cond_br.then_block.add(l, .{
1257		.tag = .repeat,
1258		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1259		});
	836	const inst_data = l.air_instructions.items(.data);
	837	inst_data[@intFromEnum(elem_block_inst)].ty_pl.payload = try l.addBlockBody(elem_block.body());
	838
	839	elem.* = elem_block_inst.toRef();
1260	840	}
1261		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1262		_ = loop_cond_br.else_block.add(l, .{
1263		.tag = .br,
1264		.data = .{ .br = .{
1265		.block_inst = orig_inst,
1266		.operand = loop_cond_br.else_block.add(l, .{
1267		.tag = .load,
1268		.data = .{ .ty_op = .{
1269		.ty = Air.internedToRef(res_ty.toIntern()),
1270		.operand = res_alloc_inst.toRef(),
1271		} },
1272		}).toRef(),
1273		} },
1274		});
1275		try loop_cond_br.finish(l);
1276		}
1277		try loop.finish(l);
	841	},
1278	842	}
	843
	844	const result = main_block.add(l, .{
	845	.tag = .aggregate_init,
	846	.data = .{ .ty_pl = .{
	847	.ty = .fromType(res_ty),
	848	.payload = payload: {
	849	const idx = l.air_extra.items.len;
	850	try l.air_extra.appendSlice(gpa, @ptrCast(elem_buf));
	851	break :payload @intCast(idx);
	852	},
	853	} },
	854	}).toRef();
	855
	856	main_block.addBr(l, orig_inst, result);
	857
	858	// Some `form` values may intentionally not use the full instruction buffer.
	859	switch (form) {
	860	.un_op,
	861	.ty_op,
	862	.bin_op,
	863	.pl_op_bin,
	864	.cmp_vector,
	865	.select,
	866	=> {},
	867	.shuffle_one,
	868	.shuffle_two,
	869	=> _ = main_block.stealRemainingCapacity(),
	870	}
	871
1279	872	return .{ .ty_pl = .{
1280		.ty = Air.internedToRef(res_ty.toIntern()),
1281		.payload = try l.addBlockBody(res_block.body()),
	873	.ty = .fromType(res_ty),
	874	.payload = try l.addBlockBody(main_block.body()),
1282	875	} };
1283	876	}
1284		fn scalarizeBitcastOperandBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
	877	fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {
1285	878	const pt = l.pt;
1286	879	const zcu = pt.zcu;
	880	const gpa = zcu.gpa;
1287	881
1288		const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1289		const res_ty = orig_ty_op.ty.toType();
1290		const operand_ty = l.typeOf(orig_ty_op.operand);
1291		const int_bits: u16 = @intCast(operand_ty.bitSize(zcu));
1292		const int_ty = try pt.intType(.unsigned, int_bits);
1293		const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1294		const elem_bits: u16 = @intCast(operand_ty.childType(zcu).bitSize(zcu));
1295		const elem_int_ty = try pt.intType(.unsigned, elem_bits);
1296
1297		var inst_buf: [22]Air.Inst.Index = undefined;
1298		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	882	var sfba_state = std.heap.stackFallback(512, gpa);
	883	const sfba = sfba_state.get();
1299	884
1300		var res_block: Block = .init(&inst_buf);
1301		{
1302		const int_alloc_inst = res_block.add(l, .{
1303		.tag = .alloc,
1304		.data = .{ .ty = try pt.singleMutPtrType(int_ty) },
1305		});
1306		_ = res_block.add(l, .{
1307		.tag = .store,
1308		.data = .{ .bin_op = .{
1309		.lhs = int_alloc_inst.toRef(),
1310		.rhs = try pt.intRef(int_ty, 0),
1311		} },
1312		});
1313		const index_alloc_inst = res_block.add(l, .{
1314		.tag = .alloc,
1315		.data = .{ .ty = .ptr_usize },
1316		});
1317		_ = res_block.add(l, .{
1318		.tag = .store,
1319		.data = .{ .bin_op = .{
1320		.lhs = index_alloc_inst.toRef(),
1321		.rhs = .zero_usize,
1322		} },
1323		});
	885	const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1324	886
1325		var loop: Loop = .init(l, &res_block);
1326		loop.block = .init(res_block.stealRemainingCapacity());
1327		{
1328		const cur_index_inst = loop.block.add(l, .{
1329		.tag = .load,
1330		.data = .{ .ty_op = .{
1331		.ty = .usize_type,
1332		.operand = index_alloc_inst.toRef(),
1333		} },
1334		});
1335		const cur_int_inst = loop.block.add(l, .{
1336		.tag = .bit_or,
1337		.data = .{ .bin_op = .{
1338		.lhs = loop.block.add(l, .{
1339		.tag = .shl_exact,
1340		.data = .{ .bin_op = .{
1341		.lhs = loop.block.add(l, .{
1342		.tag = .intcast,
1343		.data = .{ .ty_op = .{
1344		.ty = Air.internedToRef(int_ty.toIntern()),
1345		.operand = loop.block.addBitCast(l, elem_int_ty, loop.block.add(l, .{
1346		.tag = .array_elem_val,
1347		.data = .{ .bin_op = .{
1348		.lhs = orig_ty_op.operand,
1349		.rhs = cur_index_inst.toRef(),
1350		} },
1351		}).toRef()),
1352		} },
1353		}).toRef(),
1354		.rhs = loop.block.add(l, .{
1355		.tag = .mul,
1356		.data = .{ .bin_op = .{
1357		.lhs = loop.block.add(l, .{
1358		.tag = .intcast,
1359		.data = .{ .ty_op = .{
1360		.ty = Air.internedToRef(shift_ty.toIntern()),
1361		.operand = cur_index_inst.toRef(),
1362		} },
1363		}).toRef(),
1364		.rhs = try pt.intRef(shift_ty, elem_bits),
1365		} },
1366		}).toRef(),
1367		} },
1368		}).toRef(),
1369		.rhs = loop.block.add(l, .{
1370		.tag = .load,
1371		.data = .{ .ty_op = .{
1372		.ty = Air.internedToRef(int_ty.toIntern()),
1373		.operand = int_alloc_inst.toRef(),
1374		} },
1375		}).toRef(),
1376		} },
1377		});
	887	const dest_ty = ty_op.ty.toType();
	888	const dest_legal = switch (dest_ty.zigTypeTag(zcu)) {
	889	else => true,
	890	.array, .vector => legal: {
	891	if (dest_ty.arrayLen(zcu) == 1) break :legal true;
	892	const dest_elem_ty = dest_ty.childType(zcu);
	893	break :legal dest_elem_ty.bitSize(zcu) == 8 * dest_elem_ty.abiSize(zcu);
	894	},
	895	};
1378	896
1379		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1380		l,
1381		.lt,
1382		cur_index_inst.toRef(),
1383		try pt.intRef(.usize, operand_ty.arrayLen(zcu) - 1),
1384		.{},
1385		)).toRef(), &loop.block, .{});
1386		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1387		{
1388		_ = loop_cond_br.then_block.add(l, .{
1389		.tag = .store,
1390		.data = .{ .bin_op = .{
1391		.lhs = int_alloc_inst.toRef(),
1392		.rhs = cur_int_inst.toRef(),
1393		} },
1394		});
1395		_ = loop_cond_br.then_block.add(l, .{
1396		.tag = .store,
1397		.data = .{ .bin_op = .{
1398		.lhs = index_alloc_inst.toRef(),
1399		.rhs = loop_cond_br.then_block.add(l, .{
1400		.tag = .add,
1401		.data = .{ .bin_op = .{
1402		.lhs = cur_index_inst.toRef(),
1403		.rhs = .one_usize,
1404		} },
1405		}).toRef(),
1406		} },
1407		});
1408		_ = loop_cond_br.then_block.add(l, .{
1409		.tag = .repeat,
1410		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1411		});
1412		}
1413		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1414		_ = loop_cond_br.else_block.add(l, .{
1415		.tag = .br,
1416		.data = .{ .br = .{
1417		.block_inst = orig_inst,
1418		.operand = loop_cond_br.else_block.addBitCast(l, res_ty, cur_int_inst.toRef()),
1419		} },
1420		});
1421		try loop_cond_br.finish(l);
1422		}
1423		try loop.finish(l);
1424		}
1425		return .{ .ty_pl = .{
1426		.ty = Air.internedToRef(res_ty.toIntern()),
1427		.payload = try l.addBlockBody(res_block.body()),
1428		} };
1429		}
1430		fn scalarizeBitcastResultArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1431		const pt = l.pt;
1432		const zcu = pt.zcu;
	897	const operand_ty = l.typeOf(ty_op.operand);
	898	const operand_legal = switch (operand_ty.zigTypeTag(zcu)) {
	899	else => true,
	900	.array, .vector => legal: {
	901	if (operand_ty.arrayLen(zcu) == 1) break :legal true;
	902	const operand_elem_ty = operand_ty.childType(zcu);
	903	break :legal operand_elem_ty.bitSize(zcu) == 8 * operand_elem_ty.abiSize(zcu);
	904	},
	905	};
1433	906
1434		const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1435		const res_ty = orig_ty_op.ty.toType();
1436		const int_bits: u16 = @intCast(res_ty.bitSize(zcu));
1437		const int_ty = try pt.intType(.unsigned, int_bits);
1438		const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1439		const res_elem_ty = res_ty.childType(zcu);
1440		const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu));
1441		const elem_int_ty = try pt.intType(.unsigned, elem_bits);
1442
1443		var inst_buf: [20]Air.Inst.Index = undefined;
1444		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	907	if (dest_legal and operand_legal) return null;
1445	908
1446		var res_block: Block = .init(&inst_buf);
1447		{
1448		const res_alloc_inst = res_block.add(l, .{
1449		.tag = .alloc,
1450		.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1451		});
1452		const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand);
1453		const index_alloc_inst = res_block.add(l, .{
1454		.tag = .alloc,
1455		.data = .{ .ty = .ptr_usize },
1456		});
1457		_ = res_block.add(l, .{
1458		.tag = .store,
1459		.data = .{ .bin_op = .{
1460		.lhs = index_alloc_inst.toRef(),
1461		.rhs = .zero_usize,
1462		} },
1463		});
	909	if (!operand_legal and !dest_legal and operand_ty.arrayLen(zcu) == dest_ty.arrayLen(zcu)) {
	910	// from_ty and to_ty are both arrays or vectors of types with the same bit size,
	911	// so we can do an elementwise bitcast.
	912	return try l.scalarizeBlockPayload(orig_inst, .ty_op);
	913	}
1464	914
1465		var loop: Loop = .init(l, &res_block);
1466		loop.block = .init(res_block.stealRemainingCapacity());
1467		{
1468		const cur_index_inst = loop.block.add(l, .{
1469		.tag = .load,
1470		.data = .{ .ty_op = .{
1471		.ty = .usize_type,
1472		.operand = index_alloc_inst.toRef(),
1473		} },
1474		});
1475		_ = loop.block.add(l, .{
1476		.tag = .store,
1477		.data = .{ .bin_op = .{
1478		.lhs = loop.block.add(l, .{
1479		.tag = .ptr_elem_ptr,
1480		.data = .{ .ty_pl = .{
1481		.ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()),
1482		.payload = try l.addExtra(Air.Bin, .{
1483		.lhs = res_alloc_inst.toRef(),
1484		.rhs = cur_index_inst.toRef(),
1485		}),
1486		} },
1487		}).toRef(),
1488		.rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1489		.tag = .trunc,
1490		.data = .{ .ty_op = .{
1491		.ty = Air.internedToRef(elem_int_ty.toIntern()),
1492		.operand = loop.block.add(l, .{
1493		.tag = .shr,
1494		.data = .{ .bin_op = .{
1495		.lhs = int_ref,
1496		.rhs = loop.block.add(l, .{
1497		.tag = .mul,
1498		.data = .{ .bin_op = .{
1499		.lhs = loop.block.add(l, .{
1500		.tag = .intcast,
1501		.data = .{ .ty_op = .{
1502		.ty = Air.internedToRef(shift_ty.toIntern()),
1503		.operand = cur_index_inst.toRef(),
1504		} },
1505		}).toRef(),
1506		.rhs = try pt.intRef(shift_ty, elem_bits),
1507		} },
1508		}).toRef(),
1509		} },
1510		}).toRef(),
1511		} },
1512		}).toRef()),
1513		} },
1514		});
	915	// Fallback path. Our strategy is to use an unsigned integer type as an intermediate
	916	// "bag of bits" representation which can be manipulated by bitwise operations.
1515	917
1516		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1517		l,
1518		.lt,
1519		cur_index_inst.toRef(),
1520		try pt.intRef(.usize, res_ty.arrayLen(zcu) - 1),
1521		.{},
1522		)).toRef(), &loop.block, .{});
1523		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1524		{
1525		_ = loop_cond_br.then_block.add(l, .{
1526		.tag = .store,
1527		.data = .{ .bin_op = .{
1528		.lhs = index_alloc_inst.toRef(),
1529		.rhs = loop_cond_br.then_block.add(l, .{
1530		.tag = .add,
1531		.data = .{ .bin_op = .{
1532		.lhs = cur_index_inst.toRef(),
1533		.rhs = .one_usize,
1534		} },
1535		}).toRef(),
1536		} },
1537		});
1538		_ = loop_cond_br.then_block.add(l, .{
1539		.tag = .repeat,
1540		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1541		});
1542		}
1543		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1544		_ = loop_cond_br.else_block.add(l, .{
1545		.tag = .br,
1546		.data = .{ .br = .{
1547		.block_inst = orig_inst,
1548		.operand = loop_cond_br.else_block.add(l, .{
1549		.tag = .load,
1550		.data = .{ .ty_op = .{
1551		.ty = Air.internedToRef(res_ty.toIntern()),
1552		.operand = res_alloc_inst.toRef(),
1553		} },
1554		}).toRef(),
1555		} },
1556		});
1557		try loop_cond_br.finish(l);
	918	const num_bits: u16 = @intCast(dest_ty.bitSize(zcu));
	919	assert(operand_ty.bitSize(zcu) == num_bits);
	920	const uint_ty = try pt.intType(.unsigned, num_bits);
	921	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
	922
	923	const inst_buf = try sfba.alloc(Air.Inst.Index, len: {
	924	const operand_to_uint_len: u64 = if (operand_legal) 1 else (operand_ty.arrayLen(zcu) * 5);
	925	const uint_to_dest_len: u64 = if (dest_legal) 1 else (dest_ty.arrayLen(zcu) * 3 + 1);
	926	break :len @intCast(operand_to_uint_len + uint_to_dest_len + 1);
	927	});
	928	defer sfba.free(inst_buf);
	929	var main_block: Block = .init(inst_buf);
	930	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
	931
	932	// First, convert `operand_ty` to `uint_ty` (`uN`).
	933
	934	const uint_val: Air.Inst.Ref = uint_val: {
	935	if (operand_legal) break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
	936
	937	const bits_per_elem: u16 = @intCast(operand_ty.childType(zcu).bitSize(zcu));
	938	const bits_per_elem_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, bits_per_elem));
	939	const elem_uint_ty = try pt.intType(.unsigned, bits_per_elem);
	940
	941	var cur_uint: Air.Inst.Ref = .fromValue(try pt.intValue(uint_ty, 0));
	942	var elem_idx = operand_ty.arrayLen(zcu);
	943	while (elem_idx > 0) {
	944	elem_idx -= 1;
	945	const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
	946	const orig_elem = main_block.addBinOp(l, .array_elem_val, ty_op.operand, elem_idx_ref).toRef();
	947	const elem_as_uint = main_block.addBitCast(l, elem_uint_ty, orig_elem);
	948	const elem_extended = main_block.addTyOp(l, .intcast, uint_ty, elem_as_uint).toRef();
	949	cur_uint = main_block.addBinOp(l, .shl_exact, cur_uint, bits_per_elem_ref).toRef();
	950	cur_uint = main_block.addBinOp(l, .bit_or, cur_uint, elem_extended).toRef();
1558	951	}
1559		try loop.finish(l);
1560		}
1561		return .{ .ty_pl = .{
1562		.ty = Air.internedToRef(res_ty.toIntern()),
1563		.payload = try l.addBlockBody(res_block.body()),
1564		} };
1565		}
1566		fn scalarizeBitcastResultVectorBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1567		const pt = l.pt;
1568		const zcu = pt.zcu;
	952	break :uint_val cur_uint;
	953	};
1569	954
1570		const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1571		const res_ty = orig_ty_op.ty.toType();
1572		const int_bits: u16 = @intCast(res_ty.bitSize(zcu));
1573		const int_ty = try pt.intType(.unsigned, int_bits);
1574		const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1575		const res_elem_ty = res_ty.childType(zcu);
1576		const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu));
1577		const elem_int_ty = try pt.intType(.unsigned, elem_bits);
1578
1579		var inst_buf: [19]Air.Inst.Index = undefined;
1580		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	955	// Now convert `uint_ty` (`uN`) to `dest_ty`.
1581	956
1582		var res_block: Block = .init(&inst_buf);
1583		{
1584		const res_alloc_inst = res_block.add(l, .{
1585		.tag = .alloc,
1586		.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1587		});
1588		const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand);
1589		const index_alloc_inst = res_block.add(l, .{
1590		.tag = .alloc,
1591		.data = .{ .ty = .ptr_usize },
1592		});
1593		_ = res_block.add(l, .{
1594		.tag = .store,
1595		.data = .{ .bin_op = .{
1596		.lhs = index_alloc_inst.toRef(),
1597		.rhs = .zero_usize,
1598		} },
1599		});
	957	const result: Air.Inst.Ref = result: {
	958	if (dest_legal) break :result main_block.addBitCast(l, dest_ty, uint_val);
1600	959
1601		var loop: Loop = .init(l, &res_block);
1602		loop.block = .init(res_block.stealRemainingCapacity());
1603		{
1604		const cur_index_inst = loop.block.add(l, .{
1605		.tag = .load,
1606		.data = .{ .ty_op = .{
1607		.ty = .usize_type,
1608		.operand = index_alloc_inst.toRef(),
1609		} },
1610		});
1611		_ = loop.block.add(l, .{
1612		.tag = .vector_store_elem,
1613		.data = .{ .vector_store_elem = .{
1614		.vector_ptr = res_alloc_inst.toRef(),
1615		.payload = try l.addExtra(Air.Bin, .{
1616		.lhs = cur_index_inst.toRef(),
1617		.rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1618		.tag = .trunc,
1619		.data = .{ .ty_op = .{
1620		.ty = Air.internedToRef(elem_int_ty.toIntern()),
1621		.operand = loop.block.add(l, .{
1622		.tag = .shr,
1623		.data = .{ .bin_op = .{
1624		.lhs = int_ref,
1625		.rhs = loop.block.add(l, .{
1626		.tag = .mul,
1627		.data = .{ .bin_op = .{
1628		.lhs = loop.block.add(l, .{
1629		.tag = .intcast,
1630		.data = .{ .ty_op = .{
1631		.ty = Air.internedToRef(shift_ty.toIntern()),
1632		.operand = cur_index_inst.toRef(),
1633		} },
1634		}).toRef(),
1635		.rhs = try pt.intRef(shift_ty, elem_bits),
1636		} },
1637		}).toRef(),
1638		} },
1639		}).toRef(),
1640		} },
1641		}).toRef()),
1642		}),
1643		} },
1644		});
	960	const elem_ty = dest_ty.childType(zcu);
	961	const bits_per_elem: u16 = @intCast(elem_ty.bitSize(zcu));
	962	const bits_per_elem_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, bits_per_elem));
	963	const elem_uint_ty = try pt.intType(.unsigned, bits_per_elem);
1645	964
1646		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1647		l,
1648		.lt,
1649		cur_index_inst.toRef(),
1650		try pt.intRef(.usize, res_ty.vectorLen(zcu) - 1),
1651		.{},
1652		)).toRef(), &loop.block, .{});
1653		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1654		{
1655		_ = loop_cond_br.then_block.add(l, .{
1656		.tag = .store,
1657		.data = .{ .bin_op = .{
1658		.lhs = index_alloc_inst.toRef(),
1659		.rhs = loop_cond_br.then_block.add(l, .{
1660		.tag = .add,
1661		.data = .{ .bin_op = .{
1662		.lhs = cur_index_inst.toRef(),
1663		.rhs = .one_usize,
1664		} },
1665		}).toRef(),
1666		} },
1667		});
1668		_ = loop_cond_br.then_block.add(l, .{
1669		.tag = .repeat,
1670		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1671		});
1672		}
1673		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1674		_ = loop_cond_br.else_block.add(l, .{
1675		.tag = .br,
1676		.data = .{ .br = .{
1677		.block_inst = orig_inst,
1678		.operand = loop_cond_br.else_block.add(l, .{
1679		.tag = .load,
1680		.data = .{ .ty_op = .{
1681		.ty = Air.internedToRef(res_ty.toIntern()),
1682		.operand = res_alloc_inst.toRef(),
1683		} },
1684		}).toRef(),
1685		} },
1686		});
1687		try loop_cond_br.finish(l);
	965	const elem_buf = try sfba.alloc(Air.Inst.Ref, dest_ty.arrayLen(zcu));
	966	defer sfba.free(elem_buf);
	967
	968	var cur_uint = uint_val;
	969	for (elem_buf) \|*elem\| {
	970	const elem_as_uint = main_block.addTyOp(l, .trunc, elem_uint_ty, cur_uint).toRef();
	971	elem.* = main_block.addBitCast(l, elem_ty, elem_as_uint);
	972	cur_uint = main_block.addBinOp(l, .shr, cur_uint, bits_per_elem_ref).toRef();
1688	973	}
1689		try loop.finish(l);
1690		}
	974
	975	break :result main_block.add(l, .{
	976	.tag = .aggregate_init,
	977	.data = .{ .ty_pl = .{
	978	.ty = .fromType(dest_ty),
	979	.payload = payload: {
	980	const idx = l.air_extra.items.len;
	981	try l.air_extra.appendSlice(gpa, @ptrCast(elem_buf));
	982	break :payload @intCast(idx);
	983	},
	984	} },
	985	}).toRef();
	986	};
	987
	988	main_block.addBr(l, orig_inst, result);
	989
1691	990	return .{ .ty_pl = .{
1692		.ty = Air.internedToRef(res_ty.toIntern()),
1693		.payload = try l.addBlockBody(res_block.body()),
	991	.ty = .fromType(dest_ty),
	992	.payload = try l.addBlockBody(main_block.body()),
1694	993	} };
1695	994	}
1696	995	fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1697	996	const pt = l.pt;
1698	997	const zcu = pt.zcu;
	998	const gpa = zcu.gpa;
	999
	1000	var sfba_state = std.heap.stackFallback(512, gpa);
	1001	const sfba = sfba_state.get();
1699	1002
1700	1003	const orig = l.air_instructions.get(@intFromEnum(orig_inst));
1701		const res_ty = l.typeOfIndex(orig_inst);
1702		const wrapped_res_ty = res_ty.fieldType(0, zcu);
1703		const wrapped_res_scalar_ty = wrapped_res_ty.childType(zcu);
1704		const res_len = wrapped_res_ty.vectorLen(zcu);
	1004	const orig_operands = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;
1705	1005
1706		var inst_buf: [21]Air.Inst.Index = undefined;
1707		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	1006	const vec_tuple_ty = l.typeOfIndex(orig_inst);
	1007	const vec_int_ty = vec_tuple_ty.fieldType(0, zcu);
	1008	const vec_overflow_ty = vec_tuple_ty.fieldType(1, zcu);
1708	1009
1709		var res_block: Block = .init(&inst_buf);
1710		{
1711		const res_alloc_inst = res_block.add(l, .{
1712		.tag = .alloc,
1713		.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1714		});
1715		const ptr_wrapped_res_inst = res_block.add(l, .{
1716		.tag = .struct_field_ptr_index_0,
1717		.data = .{ .ty_op = .{
1718		.ty = Air.internedToRef((try pt.singleMutPtrType(wrapped_res_ty)).toIntern()),
1719		.operand = res_alloc_inst.toRef(),
	1010	assert(l.typeOf(orig_operands.lhs).toIntern() == vec_int_ty.toIntern());
	1011	if (orig.tag != .shl_with_overflow) {
	1012	assert(l.typeOf(orig_operands.rhs).toIntern() == vec_int_ty.toIntern());
	1013	}
	1014
	1015	const scalar_int_ty = vec_int_ty.childType(zcu);
	1016	const scalar_tuple_ty = try pt.overflowArithmeticTupleType(scalar_int_ty);
	1017
	1018	const elems_len = vec_int_ty.vectorLen(zcu);
	1019
	1020	const inst_buf = try sfba.alloc(Air.Inst.Index, 5 * elems_len + 4);
	1021	defer sfba.free(inst_buf);
	1022
	1023	var main_block: Block = .init(inst_buf);
	1024	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
	1025
	1026	const int_elem_buf = try sfba.alloc(Air.Inst.Ref, elems_len);
	1027	defer sfba.free(int_elem_buf);
	1028	const overflow_elem_buf = try sfba.alloc(Air.Inst.Ref, elems_len);
	1029	defer sfba.free(overflow_elem_buf);
	1030
	1031	for (int_elem_buf, overflow_elem_buf, 0..) \|int_elem, overflow_elem, elem_idx\| {
	1032	const elem_idx_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_idx));
	1033	const lhs = main_block.addBinOp(l, .array_elem_val, orig_operands.lhs, elem_idx_ref).toRef();
	1034	const rhs = main_block.addBinOp(l, .array_elem_val, orig_operands.rhs, elem_idx_ref).toRef();
	1035	const elem_result = main_block.add(l, .{
	1036	.tag = orig.tag,
	1037	.data = .{ .ty_pl = .{
	1038	.ty = .fromType(scalar_tuple_ty),
	1039	.payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
1720	1040	} },
1721		});
1722		const ptr_overflow_res_inst = res_block.add(l, .{
1723		.tag = .struct_field_ptr_index_1,
1724		.data = .{ .ty_op = .{
1725		.ty = Air.internedToRef((try pt.singleMutPtrType(res_ty.fieldType(1, zcu))).toIntern()),
1726		.operand = res_alloc_inst.toRef(),
	1041	}).toRef();
	1042	int_elem.* = main_block.add(l, .{
	1043	.tag = .struct_field_val,
	1044	.data = .{ .ty_pl = .{
	1045	.ty = .fromType(scalar_int_ty),
	1046	.payload = try l.addExtra(Air.StructField, .{
	1047	.struct_operand = elem_result,
	1048	.field_index = 0,
	1049	}),
1727	1050	} },
1728		});
1729		const index_alloc_inst = res_block.add(l, .{
1730		.tag = .alloc,
1731		.data = .{ .ty = .ptr_usize },
1732		});
1733		_ = res_block.add(l, .{
1734		.tag = .store,
1735		.data = .{ .bin_op = .{
1736		.lhs = index_alloc_inst.toRef(),
1737		.rhs = .zero_usize,
	1051	}).toRef();
	1052	overflow_elem.* = main_block.add(l, .{
	1053	.tag = .struct_field_val,
	1054	.data = .{ .ty_pl = .{
	1055	.ty = .bool_type,
	1056	.payload = try l.addExtra(Air.StructField, .{
	1057	.struct_operand = elem_result,
	1058	.field_index = 1,
	1059	}),
1738	1060	} },
1739		});
	1061	}).toRef();
	1062	}
1740	1063
1741		var loop: Loop = .init(l, &res_block);
1742		loop.block = .init(res_block.stealRemainingCapacity());
1743		{
1744		const cur_index_inst = loop.block.add(l, .{
1745		.tag = .load,
1746		.data = .{ .ty_op = .{
1747		.ty = .usize_type,
1748		.operand = index_alloc_inst.toRef(),
1749		} },
1750		});
1751		const extra = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;
1752		const res_elem = loop.block.add(l, .{
1753		.tag = orig.tag,
1754		.data = .{ .ty_pl = .{
1755		.ty = Air.internedToRef(try zcu.intern_pool.getTupleType(zcu.gpa, pt.tid, .{
1756		.types = &.{ wrapped_res_scalar_ty.toIntern(), .u1_type },
1757		.values = &(.{.none} ** 2),
1758		})),
1759		.payload = try l.addExtra(Air.Bin, .{
1760		.lhs = loop.block.add(l, .{
1761		.tag = .array_elem_val,
1762		.data = .{ .bin_op = .{
1763		.lhs = extra.lhs,
1764		.rhs = cur_index_inst.toRef(),
1765		} },
1766		}).toRef(),
1767		.rhs = loop.block.add(l, .{
1768		.tag = .array_elem_val,
1769		.data = .{ .bin_op = .{
1770		.lhs = extra.rhs,
1771		.rhs = cur_index_inst.toRef(),
1772		} },
1773		}).toRef(),
1774		}),
1775		} },
1776		});
1777		_ = loop.block.add(l, .{
1778		.tag = .vector_store_elem,
1779		.data = .{ .vector_store_elem = .{
1780		.vector_ptr = ptr_overflow_res_inst.toRef(),
1781		.payload = try l.addExtra(Air.Bin, .{
1782		.lhs = cur_index_inst.toRef(),
1783		.rhs = loop.block.add(l, .{
1784		.tag = .struct_field_val,
1785		.data = .{ .ty_pl = .{
1786		.ty = .u1_type,
1787		.payload = try l.addExtra(Air.StructField, .{
1788		.struct_operand = res_elem.toRef(),
1789		.field_index = 1,
1790		}),
1791		} },
1792		}).toRef(),
1793		}),
1794		} },
1795		});
1796		_ = loop.block.add(l, .{
1797		.tag = .vector_store_elem,
1798		.data = .{ .vector_store_elem = .{
1799		.vector_ptr = ptr_wrapped_res_inst.toRef(),
1800		.payload = try l.addExtra(Air.Bin, .{
1801		.lhs = cur_index_inst.toRef(),
1802		.rhs = loop.block.add(l, .{
1803		.tag = .struct_field_val,
1804		.data = .{ .ty_pl = .{
1805		.ty = Air.internedToRef(wrapped_res_scalar_ty.toIntern()),
1806		.payload = try l.addExtra(Air.StructField, .{
1807		.struct_operand = res_elem.toRef(),
1808		.field_index = 0,
1809		}),
1810		} },
1811		}).toRef(),
1812		}),
1813		} },
1814		});
	1064	const int_vec = main_block.add(l, .{
	1065	.tag = .aggregate_init,
	1066	.data = .{ .ty_pl = .{
	1067	.ty = .fromType(vec_int_ty),
	1068	.payload = payload: {
	1069	const idx = l.air_extra.items.len;
	1070	try l.air_extra.appendSlice(gpa, @ptrCast(int_elem_buf));
	1071	break :payload @intCast(idx);
	1072	},
	1073	} },
	1074	}).toRef();
	1075	const overflow_vec = main_block.add(l, .{
	1076	.tag = .aggregate_init,
	1077	.data = .{ .ty_pl = .{
	1078	.ty = .fromType(vec_overflow_ty),
	1079	.payload = payload: {
	1080	const idx = l.air_extra.items.len;
	1081	try l.air_extra.appendSlice(gpa, @ptrCast(overflow_elem_buf));
	1082	break :payload @intCast(idx);
	1083	},
	1084	} },
	1085	}).toRef();
	1086
	1087	const tuple_elems: [2]Air.Inst.Ref = .{ int_vec, overflow_vec };
	1088	const result = main_block.add(l, .{
	1089	.tag = .aggregate_init,
	1090	.data = .{ .ty_pl = .{
	1091	.ty = .fromType(vec_tuple_ty),
	1092	.payload = payload: {
	1093	const idx = l.air_extra.items.len;
	1094	try l.air_extra.appendSlice(gpa, @ptrCast(&tuple_elems));
	1095	break :payload @intCast(idx);
	1096	},
	1097	} },
	1098	}).toRef();
	1099
	1100	main_block.addBr(l, orig_inst, result);
1815	1101
1816		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1817		l,
1818		.lt,
1819		cur_index_inst.toRef(),
1820		try pt.intRef(.usize, res_len - 1),
1821		.{},
1822		)).toRef(), &loop.block, .{});
1823		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1824		{
1825		_ = loop_cond_br.then_block.add(l, .{
1826		.tag = .store,
1827		.data = .{ .bin_op = .{
1828		.lhs = index_alloc_inst.toRef(),
1829		.rhs = loop_cond_br.then_block.add(l, .{
1830		.tag = .add,
1831		.data = .{ .bin_op = .{
1832		.lhs = cur_index_inst.toRef(),
1833		.rhs = .one_usize,
1834		} },
1835		}).toRef(),
1836		} },
1837		});
1838		_ = loop_cond_br.then_block.add(l, .{
1839		.tag = .repeat,
1840		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1841		});
1842		}
1843		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1844		_ = loop_cond_br.else_block.add(l, .{
1845		.tag = .br,
1846		.data = .{ .br = .{
1847		.block_inst = orig_inst,
1848		.operand = loop_cond_br.else_block.add(l, .{
1849		.tag = .load,
1850		.data = .{ .ty_op = .{
1851		.ty = Air.internedToRef(res_ty.toIntern()),
1852		.operand = res_alloc_inst.toRef(),
1853		} },
1854		}).toRef(),
1855		} },
1856		});
1857		try loop_cond_br.finish(l);
1858		}
1859		try loop.finish(l);
1860		}
1861	1102	return .{ .ty_pl = .{
1862		.ty = Air.internedToRef(res_ty.toIntern()),
1863		.payload = try l.addBlockBody(res_block.body()),
	1103	.ty = .fromType(vec_tuple_ty),
	1104	.payload = try l.addBlockBody(main_block.body()),
1864	1105	} };
1865	1106	}
1866	1107
...	...	@@ -2231,37 +1472,6 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
2231	1472	} };
2232	1473	}
2233	1474
2234		fn expandBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2235		const pt = l.pt;
2236		const zcu = pt.zcu;
2237		const ip = &zcu.intern_pool;
2238
2239		const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
2240		const res_ty = orig_ty_op.ty.toType();
2241		const res_ty_key = ip.indexToKey(res_ty.toIntern());
2242		const operand_ty = l.typeOf(orig_ty_op.operand);
2243		const operand_ty_key = ip.indexToKey(operand_ty.toIntern());
2244		_ = res_ty_key;
2245		_ = operand_ty_key;
2246
2247		var inst_buf: [1]Air.Inst.Index = undefined;
2248		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2249
2250		var res_block: Block = .init(&inst_buf);
2251		{
2252		_ = res_block.add(l, .{
2253		.tag = .br,
2254		.data = .{ .br = .{
2255		.block_inst = orig_inst,
2256		.operand = try pt.undefRef(res_ty),
2257		} },
2258		});
2259		}
2260		return .{ .ty_pl = .{
2261		.ty = Air.internedToRef(res_ty.toIntern()),
2262		.payload = try l.addBlockBody(res_block.body()),
2263		} };
2264		}
2265	1475	fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2266	1476	const pt = l.pt;
2267	1477	const zcu = pt.zcu;
...	...	@@ -2431,89 +1641,73 @@ fn packedStructFieldValBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Err
2431	1641	const field_ty = orig_ty_pl.ty.toType();
2432	1642	const agg_ty = l.typeOf(orig_extra.struct_operand);
2433	1643
	1644	const agg_bits: u16 = @intCast(agg_ty.bitSize(zcu));
	1645	const bit_offset = zcu.structPackedFieldBitOffset(zcu.typeToStruct(agg_ty).?, orig_extra.field_index);
	1646
	1647	const agg_int_ty = try pt.intType(.unsigned, agg_bits);
	1648	const field_int_ty = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
	1649
	1650	const agg_shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, agg_bits));
	1651	const bit_offset_ref: Air.Inst.Ref = .fromValue(try pt.intValue(agg_shift_ty, bit_offset));
	1652
2434	1653	var inst_buf: [5]Air.Inst.Index = undefined;
	1654	var main_block: Block = .init(&inst_buf);
2435	1655	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2436	1656
2437		var res_block: Block = .init(&inst_buf);
2438		{
2439		const agg_alloc_inst = res_block.add(l, .{
2440		.tag = .alloc,
2441		.data = .{ .ty = try pt.singleMutPtrType(agg_ty) },
2442		});
2443		_ = res_block.add(l, .{
2444		.tag = .store,
2445		.data = .{ .bin_op = .{
2446		.lhs = agg_alloc_inst.toRef(),
2447		.rhs = orig_extra.struct_operand,
2448		} },
2449		});
2450		_ = res_block.add(l, .{
2451		.tag = .br,
2452		.data = .{ .br = .{
2453		.block_inst = orig_inst,
2454		.operand = res_block.add(l, .{
2455		.tag = .load,
2456		.data = .{ .ty_op = .{
2457		.ty = Air.internedToRef(field_ty.toIntern()),
2458		.operand = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), orig_extra.field_index)).toRef(),
2459		} },
2460		}).toRef(),
2461		} },
2462		});
2463		}
	1657	const agg_int = main_block.addBitCast(l, agg_int_ty, orig_extra.struct_operand);
	1658	const shifted_agg_int = main_block.addBinOp(l, .shr, agg_int, bit_offset_ref).toRef();
	1659	const field_int = main_block.addTyOp(l, .trunc, field_int_ty, shifted_agg_int).toRef();
	1660	const field_val = main_block.addBitCast(l, field_ty, field_int);
	1661	main_block.addBr(l, orig_inst, field_val);
	1662
2464	1663	return .{ .ty_pl = .{
2465		.ty = Air.internedToRef(field_ty.toIntern()),
2466		.payload = try l.addBlockBody(res_block.body()),
	1664	.ty = .fromType(field_ty),
	1665	.payload = try l.addBlockBody(main_block.body()),
2467	1666	} };
2468	1667	}
2469	1668	fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2470	1669	const pt = l.pt;
2471	1670	const zcu = pt.zcu;
	1671	const gpa = zcu.gpa;
2472	1672
2473	1673	const orig_ty_pl = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_pl;
2474		const field_ty = orig_ty_pl.ty.toType();
2475	1674	const agg_ty = orig_ty_pl.ty.toType();
2476	1675	const agg_field_count = agg_ty.structFieldCount(zcu);
2477	1676
2478		const ExpectedContents = [1 + 2 * 32 + 2]Air.Inst.Index;
2479		var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
2480		std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa);
2481		const gpa = stack.get();
	1677	var sfba_state = std.heap.stackFallback(@sizeOf([4 * 32 + 2]Air.Inst.Index), gpa);
	1678	const sfba = sfba_state.get();
2482	1679
2483		const inst_buf = try gpa.alloc(Air.Inst.Index, 1 + 2 * agg_field_count + 2);
2484		defer gpa.free(inst_buf);
2485		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	1680	const inst_buf = try sfba.alloc(Air.Inst.Index, 4 * agg_field_count + 2);
	1681	defer sfba.free(inst_buf);
2486	1682
2487		var res_block: Block = .init(inst_buf);
2488		{
2489		const agg_alloc_inst = res_block.add(l, .{
2490		.tag = .alloc,
2491		.data = .{ .ty = try pt.singleMutPtrType(agg_ty) },
2492		});
2493		for (0..agg_field_count, orig_ty_pl.payload..) \|field_index, extra_index\| _ = res_block.add(l, .{
2494		.tag = .store,
2495		.data = .{ .bin_op = .{
2496		.lhs = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), field_index)).toRef(),
2497		.rhs = @enumFromInt(l.air_extra.items[extra_index]),
2498		} },
2499		});
2500		_ = res_block.add(l, .{
2501		.tag = .br,
2502		.data = .{ .br = .{
2503		.block_inst = orig_inst,
2504		.operand = res_block.add(l, .{
2505		.tag = .load,
2506		.data = .{ .ty_op = .{
2507		.ty = Air.internedToRef(field_ty.toIntern()),
2508		.operand = agg_alloc_inst.toRef(),
2509		} },
2510		}).toRef(),
2511		} },
2512		});
	1683	var main_block: Block = .init(inst_buf);
	1684	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
	1685
	1686	const num_bits: u16 = @intCast(agg_ty.bitSize(zcu));
	1687	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
	1688	const uint_ty = try pt.intType(.unsigned, num_bits);
	1689	var cur_uint: Air.Inst.Ref = .fromValue(try pt.intValue(uint_ty, 0));
	1690
	1691	var field_idx = agg_field_count;
	1692	while (field_idx > 0) {
	1693	field_idx -= 1;
	1694	const field_ty = agg_ty.fieldType(field_idx, zcu);
	1695	const field_uint_ty = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
	1696	const field_bit_size_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, field_ty.bitSize(zcu)));
	1697	const field_val: Air.Inst.Ref = @enumFromInt(l.air_extra.items[orig_ty_pl.payload + field_idx]);
	1698
	1699	const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();
	1700	const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);
	1701	const field_extended = main_block.addTyOp(l, .intcast, uint_ty, field_as_uint).toRef();
	1702	cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();
2513	1703	}
	1704
	1705	const result = main_block.addBitCast(l, agg_ty, cur_uint);
	1706	main_block.addBr(l, orig_inst, result);
	1707
2514	1708	return .{ .ty_pl = .{
2515		.ty = Air.internedToRef(field_ty.toIntern()),
2516		.payload = try l.addBlockBody(res_block.body()),
	1709	.ty = .fromType(agg_ty),
	1710	.payload = try l.addBlockBody(main_block.body()),
2517	1711	} };
2518	1712	}
2519	1713
...	...	@@ -2571,6 +1765,33 @@ const Block = struct {
2571	1765	b.len += 1;
2572	1766	return inst;
2573	1767	}
	1768	fn addBr(b: Block, l: Legalize, target: Air.Inst.Index, operand: Air.Inst.Ref) void {
	1769	_ = b.add(l, .{
	1770	.tag = .br,
	1771	.data = .{ .br = .{ .block_inst = target, .operand = operand } },
	1772	});
	1773	}
	1774	fn addBinOp(b: Block, l: Legalize, tag: Air.Inst.Tag, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) Air.Inst.Index {
	1775	return b.add(l, .{
	1776	.tag = tag,
	1777	.data = .{ .bin_op = .{ .lhs = lhs, .rhs = rhs } },
	1778	});
	1779	}
	1780	fn addUnOp(b: Block, l: Legalize, tag: Air.Inst.Tag, operand: Air.Inst.Ref) Air.Inst.Index {
	1781	return b.add(l, .{
	1782	.tag = tag,
	1783	.data = .{ .un_op = operand },
	1784	});
	1785	}
	1786	fn addTyOp(b: Block, l: Legalize, tag: Air.Inst.Tag, ty: Type, operand: Air.Inst.Ref) Air.Inst.Index {
	1787	return b.add(l, .{
	1788	.tag = tag,
	1789	.data = .{ .ty_op = .{
	1790	.ty = .fromType(ty),
	1791	.operand = operand,
	1792	} },
	1793	});
	1794	}
2574	1795
2575	1796	/// Adds the code to call the panic handler `panic_id`. This is usually `.call` then `.unreach`,
2576	1797	/// but if `Zcu.Feature.panic_fn` is unsupported, we lower to `.trap` instead.
...	...	@@ -2625,14 +1846,27 @@ const Block = struct {
2625	1846	} },
2626	1847	});
2627	1848	}
	1849	return addCmpScalar(b, l, op, lhs, rhs, opts.optimized);
	1850	}
	1851
	1852	/// Similar to `addCmp`, but for scalars only. Unlike `addCmp`, this function is
	1853	/// infallible, because it doesn't need to add entries to `extra`.
	1854	fn addCmpScalar(
	1855	b: *Block,
	1856	l: *Legalize,
	1857	op: std.math.CompareOperator,
	1858	lhs: Air.Inst.Ref,
	1859	rhs: Air.Inst.Ref,
	1860	optimized: bool,
	1861	) Air.Inst.Index {
2628	1862	return b.add(l, .{
2629	1863	.tag = switch (op) {
2630		.lt => if (opts.optimized) .cmp_lt_optimized else .cmp_lt,
2631		.lte => if (opts.optimized) .cmp_lte_optimized else .cmp_lte,
2632		.eq => if (opts.optimized) .cmp_eq_optimized else .cmp_eq,
2633		.gte => if (opts.optimized) .cmp_gte_optimized else .cmp_gte,
2634		.gt => if (opts.optimized) .cmp_gt_optimized else .cmp_gt,
2635		.neq => if (opts.optimized) .cmp_neq_optimized else .cmp_neq,
	1864	.lt => if (optimized) .cmp_lt_optimized else .cmp_lt,
	1865	.lte => if (optimized) .cmp_lte_optimized else .cmp_lte,
	1866	.eq => if (optimized) .cmp_eq_optimized else .cmp_eq,
	1867	.gte => if (optimized) .cmp_gte_optimized else .cmp_gte,
	1868	.gt => if (optimized) .cmp_gt_optimized else .cmp_gt,
	1869	.neq => if (optimized) .cmp_neq_optimized else .cmp_neq,
2636	1870	},
2637	1871	.data = .{ .bin_op = .{
2638	1872	.lhs = lhs,
...	...	@@ -2641,93 +1875,6 @@ const Block = struct {
2641	1875	});
2642	1876	}
2643	1877
2644		/// Adds a `struct_field_ptr*` instruction to `b`. This is a fairly thin wrapper around `add`
2645		/// that selects the optimized instruction encoding to use, although it does compute the
2646		/// proper field pointer type.
2647		fn addStructFieldPtr(
2648		b: *Block,
2649		l: *Legalize,
2650		struct_operand: Air.Inst.Ref,
2651		field_index: usize,
2652		) Error!Air.Inst.Index {
2653		const pt = l.pt;
2654		const zcu = pt.zcu;
2655
2656		const agg_ptr_ty = l.typeOf(struct_operand);
2657		const agg_ptr_info = agg_ptr_ty.ptrInfo(zcu);
2658		const agg_ty: Type = .fromInterned(agg_ptr_info.child);
2659		const agg_ptr_align = switch (agg_ptr_info.flags.alignment) {
2660		.none => agg_ty.abiAlignment(zcu),
2661		else => \|agg_ptr_align\| agg_ptr_align,
2662		};
2663		const agg_layout = agg_ty.containerLayout(zcu);
2664		const field_ty = agg_ty.fieldType(field_index, zcu);
2665		var field_ptr_info: InternPool.Key.PtrType = .{
2666		.child = field_ty.toIntern(),
2667		.flags = .{
2668		.is_const = agg_ptr_info.flags.is_const,
2669		.is_volatile = agg_ptr_info.flags.is_volatile,
2670		.address_space = agg_ptr_info.flags.address_space,
2671		},
2672		};
2673		field_ptr_info.flags.alignment = field_ptr_align: switch (agg_layout) {
2674		.auto => agg_ty.fieldAlignment(field_index, zcu).min(agg_ptr_align),
2675		.@"extern" => switch (agg_ty.zigTypeTag(zcu)) {
2676		else => unreachable,
2677		.@"struct" => .fromLog2Units(@min(
2678		agg_ptr_align.toLog2Units(),
2679		@ctz(agg_ty.structFieldOffset(field_index, zcu)),
2680		)),
2681		.@"union" => agg_ptr_align,
2682		},
2683		.@"packed" => switch (agg_ty.zigTypeTag(zcu)) {
2684		else => unreachable,
2685		.@"struct" => {
2686		const packed_offset = agg_ty.packedStructFieldPtrInfo(agg_ptr_ty, @intCast(field_index), pt);
2687		field_ptr_info.packed_offset = packed_offset;
2688		break :field_ptr_align agg_ptr_align;
2689		},
2690		.@"union" => {
2691		field_ptr_info.packed_offset = .{
2692		.host_size = switch (agg_ptr_info.packed_offset.host_size) {
2693		0 => @intCast(agg_ty.abiSize(zcu)),
2694		else => \|host_size\| host_size,
2695		},
2696		.bit_offset = agg_ptr_info.packed_offset.bit_offset,
2697		};
2698		break :field_ptr_align agg_ptr_align;
2699		},
2700		},
2701		};
2702		const field_ptr_ty = try pt.ptrType(field_ptr_info);
2703		const field_ptr_ty_ref = Air.internedToRef(field_ptr_ty.toIntern());
2704		return switch (field_index) {
2705		inline 0...3 => \|ct_field_index\| b.add(l, .{
2706		.tag = switch (ct_field_index) {
2707		0 => .struct_field_ptr_index_0,
2708		1 => .struct_field_ptr_index_1,
2709		2 => .struct_field_ptr_index_2,
2710		3 => .struct_field_ptr_index_3,
2711		else => comptime unreachable,
2712		},
2713		.data = .{ .ty_op = .{
2714		.ty = field_ptr_ty_ref,
2715		.operand = struct_operand,
2716		} },
2717		}),
2718		else => b.add(l, .{
2719		.tag = .struct_field_ptr,
2720		.data = .{ .ty_pl = .{
2721		.ty = field_ptr_ty_ref,
2722		.payload = try l.addExtra(Air.StructField, .{
2723		.struct_operand = struct_operand,
2724		.field_index = @intCast(field_index),
2725		}),
2726		} },
2727		}),
2728		};
2729		}
2730
2731	1878	/// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for
2732	1879	/// no-op casts.
2733	1880	fn addBitCast(
...	...	@@ -2774,56 +1921,6 @@ const Block = struct {
2774	1921	}
2775	1922	};
2776	1923
2777		const Result = struct {
2778		inst: Air.Inst.Index,
2779		block: Block,
2780
2781		/// The return value has `block` initialized to `undefined`; it is the caller's reponsibility
2782		/// to initialize it.
2783		fn init(l: Legalize, ty: Type, parent_block: Block) Result {
2784		return .{
2785		.inst = parent_block.add(l, .{
2786		.tag = .block,
2787		.data = .{ .ty_pl = .{
2788		.ty = Air.internedToRef(ty.toIntern()),
2789		.payload = undefined,
2790		} },
2791		}),
2792		.block = undefined,
2793		};
2794		}
2795
2796		fn finish(res: Result, l: *Legalize) Error!void {
2797		const data = &l.air_instructions.items(.data)[@intFromEnum(res.inst)];
2798		data.ty_pl.payload = try l.addBlockBody(res.block.body());
2799		}
2800		};
2801
2802		const Loop = struct {
2803		inst: Air.Inst.Index,
2804		block: Block,
2805
2806		/// The return value has `block` initialized to `undefined`; it is the caller's reponsibility
2807		/// to initialize it.
2808		fn init(l: Legalize, parent_block: Block) Loop {
2809		return .{
2810		.inst = parent_block.add(l, .{
2811		.tag = .loop,
2812		.data = .{ .ty_pl = .{
2813		.ty = .noreturn_type,
2814		.payload = undefined,
2815		} },
2816		}),
2817		.block = undefined,
2818		};
2819		}
2820
2821		fn finish(loop: Loop, l: *Legalize) Error!void {
2822		const data = &l.air_instructions.items(.data)[@intFromEnum(loop.inst)];
2823		data.ty_pl.payload = try l.addBlockBody(loop.block.body());
2824		}
2825		};
2826
2827	1924	const CondBr = struct {
2828	1925	inst: Air.Inst.Index,
2829	1926	hints: Air.CondBr.BranchHints,