Meg's Mirrors

zig

1	const FuncGen = @This();
2
3	object: *Object,
4	nav_index: InternPool.Nav.Index,
5	pt: Zcu.PerThread,
6	gpa: Allocator,
7	air: Air,
8	liveness: Air.Liveness,
9	wip: Builder.WipFunction,
10	is_naked: bool,
11	fuzz: ?Fuzz,
12
13	file: Builder.Metadata,
14	scope: Builder.Metadata,
15
16	inlined_at: Builder.Metadata.Optional,
17
18	base_line: u32,
19	prev_dbg_line: u32,
20	prev_dbg_column: u32,
21
22	/// This stores the LLVM values used in a function, such that they can be referred to
23	/// in other instructions. This table is cleared before every function is generated.
24	func_inst_table: std.AutoHashMapUnmanaged(Air.Inst.Ref, Builder.Value),
25
26	/// If the return type is sret, this is the result pointer. Otherwise null.
27	/// Note that this can disagree with isByRef for the return type in the case
28	/// of C ABI functions.
29	ret_ptr: Builder.Value,
30	/// Any function that needs to perform Valgrind client requests needs an array alloca
31	/// instruction, however a maximum of one per function is needed.
32	valgrind_client_request_array: Builder.Value = .none,
33	/// These fields are used to refer to the LLVM value of the function parameters
34	/// in an Arg instruction.
35	/// This list may be shorter than the list according to the zig type system;
36	/// it omits 0-bit types. If the function uses sret as the first parameter,
37	/// this slice does not include it.
38	args: []const Builder.Value,
39	arg_index: u32,
40	arg_inline_index: u32,
41
42	err_ret_trace: Builder.Value,
43
44	/// This data structure is used to implement breaking to blocks.
45	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
46	parent_bb: Builder.Function.Block.Index,
47	breaks: *BreakList,
48	}),
49
50	/// Maps `loop` instructions to the bb to branch to to repeat the loop.
51	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, Builder.Function.Block.Index),
52
53	/// Maps `loop_switch_br` instructions to the information required to lower
54	/// dispatches (`switch_dispatch` instructions).
55	switch_dispatch_info: std.AutoHashMapUnmanaged(Air.Inst.Index, SwitchDispatchInfo),
56
57	sync_scope: Builder.SyncScope,
58
59	disable_intrinsics: bool,
60
61	/// Have we seen loads or stores involving `allowzero` pointers?
62	allowzero_access: bool,
63
64	/// In general, codegen should never emit errors; we cannot report useful source locations for them
65	/// and they don't really play nicely with incremental compilation. The LLVM backend mostly obeys
66	/// this rule. Where it does not, it calls `todo` to emit an error, and results in this error set
67	/// being used for the function
68	///
69	/// Please avoid using this error set in new code. Ideally, every fallible function in this file
70	/// should have the error set `Allocator.Error`.
71	const TodoError = Zcu.CodegenFailError;
72
73	/// Avoid introducing new calls to this function---see documentation comment on `TodoError`.
74	fn todo(fg: *FuncGen, comptime format: []const u8, args: anytype) TodoError {
75	@branchHint(.cold);
76	return fg.object.zcu.codegenFail(
77	fg.nav_index,
78	"TODO (LLVM): " ++ format,
79	args,
80	);
81	}
82
83	fn ownerModule(fg: *const FuncGen) *Module {
84	return fg.object.zcu.navFileScope(fg.nav_index).mod.?;
85	}
86
87	fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void {
88	// LLVM already considers null pointers to be valid in non-generic address spaces, so avoid
89	// pessimizing optimization for functions with accesses to such pointers.
90	if (info.flags.address_space == .generic and info.flags.is_allowzero) self.allowzero_access = true;
91	}
92
93	pub const Fuzz = struct {
94	counters_variable: Builder.Variable.Index,
95	pcs: std.ArrayList(Builder.Constant),
96
97	fn deinit(f: *Fuzz, gpa: Allocator) void {
98	f.pcs.deinit(gpa);
99	f.* = undefined;
100	}
101	};
102
103	const SwitchDispatchInfo = struct {
104	/// These are the blocks corresponding to each switch case.
105	/// The final element corresponds to the `else` case.
106	/// Slices allocated into `gpa`.
107	case_blocks: []Builder.Function.Block.Index,
108	/// This is `.none` if `jmp_table` is set, since we won't use a `switch` instruction to dispatch.
109	switch_weights: Builder.Function.Instruction.BrCond.Weights,
110	/// If not `null`, we have manually constructed a jump table to reach the desired block.
111	/// `table` can be used if the value is between `min` and `max` inclusive.
112	/// We perform this lowering manually to avoid some questionable behavior from LLVM.
113	/// See `airSwitchBr` for details.
114	jmp_table: ?JmpTable,
115
116	const JmpTable = struct {
117	min: Builder.Constant,
118	max: Builder.Constant,
119	in_bounds_hint: enum { none, unpredictable, likely, unlikely },
120	/// Pointer to the jump table itself, to be used with `indirectbr`.
121	/// The index into the jump table is the dispatch condition minus `min`.
122	/// The table values are `blockaddress` constants corresponding to blocks in `case_blocks`.
123	table: Builder.Constant,
124	/// `true` if `table` conatins a reference to the `else` block.
125	/// In this case, the `indirectbr` must include the `else` block in its target list.
126	table_includes_else: bool,
127	};
128	};
129
130	const BreakList = union {
131	list: std.MultiArrayList(struct {
132	bb: Builder.Function.Block.Index,
133	val: Builder.Value,
134	}),
135	len: usize,
136	};
137
138	pub fn deinit(self: *FuncGen) void {
139	const gpa = self.gpa;
140	if (self.fuzz) |*f| f.deinit(self.gpa);
141	self.wip.deinit();
142	self.func_inst_table.deinit(gpa);
143	self.blocks.deinit(gpa);
144	self.loops.deinit(gpa);
145	var it = self.switch_dispatch_info.valueIterator();
146	while (it.next()) |info| {
147	self.gpa.free(info.case_blocks);
148	}
149	self.switch_dispatch_info.deinit(gpa);
150	}
151
152	fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) Allocator.Error!Builder.Value {
153	const gpa = self.gpa;
154	const gop = try self.func_inst_table.getOrPut(gpa, inst);
155	if (gop.found_existing) return gop.value_ptr.*;
156
157	const llvm_val = try self.resolveValue(.fromInterned(inst.toInterned().?));
158	gop.value_ptr.* = llvm_val.toValue();
159	return llvm_val.toValue();
160	}
161
162	fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
163	const o = self.object;
164	const zcu = o.zcu;
165	const ty = val.typeOf(zcu);
166	if (!isByRef(ty, zcu)) {
167	return o.lowerValue(val.toIntern(), .as_value);
168	} else {
169	// We need a pointer to a global constant, i.e. a UAV.
170	return o.lowerUavRef(
171	val.toIntern(),
172	ty.abiAlignment(zcu).toLlvm(),
173	target_util.defaultAddressSpace(zcu.getTarget(), .global_constant),
174	);
175	}
176	}
177
178	/// Populates `fg.ret_ptr`, `fg.err_ret_trace`, and `fg.args` based on the parameters of the
179	/// function type, then generates the entire function body.
180	///
181	/// The caller may initialize `fg.ret_ptr`, `fg.err_ret_trace`, and `fg.args` to undefined.
182	pub fn genMainBody(fg: *FuncGen) TodoError!void {
183	const o = fg.object;
184	const zcu = o.zcu;
185	const ip = &zcu.intern_pool;
186	const comp = zcu.comp;
187	const gpa = comp.gpa;
188
189	const fn_ty: Type = .fromInterned(ip.getNav(fg.nav_index).resolved.?.type);
190	const fn_info = zcu.typeToFunc(fn_ty).?;
191	const param_types = fn_info.param_types.get(ip);
192
193	var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types.get(ip));
194
195	// Populate `fg.ret_ptr`...
196	fg.ret_ptr = switch (try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type))) {
197	.sret => rp: {
198	defer it.llvm_index += 1;
199	break :rp fg.wip.arg(it.llvm_index);
200	},
201	else => .none,
202	};
203	// ...and `fg.err_ret_trace`...
204	if (fn_info.cc == .auto and comp.config.any_error_tracing) {
205	fg.err_ret_trace = fg.wip.arg(it.llvm_index);
206	it.llvm_index += 1;
207	} else {
208	fg.err_ret_trace = .none;
209	}
210	// ...and as for `fg.args`, we'll put all of the arguments into this ArrayList, and once that's
211	// done we'll use its buffer as `fg.args`.
212	var args: std.ArrayList(Builder.Value) = .empty;
213	defer args.deinit(gpa);
214
215	while (try it.next()) |lowering| {
216	try args.ensureUnusedCapacity(gpa, 1);
217
218	switch (lowering) {
219	.no_bits => continue,
220	.byval => {
221	assert(it.byval_attr == null);
222	const param_index = it.zig_index - 1;
223	const param_ty: Type = .fromInterned(param_types[param_index]);
224	const param = fg.wip.arg(it.llvm_index - 1);
225
226	if (isByRef(param_ty, zcu)) {
227	const alignment = param_ty.abiAlignment(zcu).toLlvm();
228	const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
229	// We don't need to handle non-ABI-sized integer types in memory here since they
230	// are never by-ref.
231	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
232	args.appendAssumeCapacity(arg_ptr);
233	} else {
234	args.appendAssumeCapacity(param);
235	}
236	},
237	.byref, .byref_mut => {
238	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
239	const param = fg.wip.arg(it.llvm_index - 1);
240	const alignment = if (it.byval_attr) |byval_attr| byval_attr.alignment else .none;
241
242	if (alignment == .none and isByRef(param_ty, zcu)) {
243	args.appendAssumeCapacity(param);
244	} else {
245	args.appendAssumeCapacity(try fg.load(param, alignment, param_ty, .normal));
246	}
247	},
248	.abi_sized_int => {
249	assert(it.byval_attr == null);
250	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
251	const param = fg.wip.arg(it.llvm_index - 1);
252
253	const alignment = param_ty.abiAlignment(zcu).toLlvm();
254	const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
255	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
256
257	if (isByRef(param_ty, zcu)) {
258	args.appendAssumeCapacity(arg_ptr);
259	} else {
260	args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
261	}
262	},
263	.slice => {
264	assert(it.byval_attr == null);
265	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
266	assert(!isByRef(param_ty, zcu));
267	const slice_val = try fg.wip.buildAggregate(
268	try o.lowerType(param_ty, .as_value),
269	&.{ fg.wip.arg(it.llvm_index - 2), fg.wip.arg(it.llvm_index - 1) },
270	"",
271	);
272	args.appendAssumeCapacity(slice_val);
273	},
274	.multiple_llvm_types => {
275	assert(it.byval_attr == null);
276	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
277	const param_alignment = param_ty.abiAlignment(zcu);
278	const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
279	const arg_ptr = try fg.buildAlloca(llvm_ty, param_alignment.toLlvm());
280	const llvm_args_start = it.llvm_index - it.types_len;
281	for (llvm_args_start.., it.offsets_buffer[0..it.types_len]) |llvm_arg_index, offset| {
282	const param = fg.wip.arg(@intCast(llvm_arg_index));
283	const part_ptr = try fg.ptraddConst(arg_ptr, offset);
284	_ = try fg.wip.store(.normal, param, part_ptr, param_alignment.offset(offset).toLlvm());
285	}
286
287	if (isByRef(param_ty, zcu)) {
288	args.appendAssumeCapacity(arg_ptr);
289	} else {
290	args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
291	}
292	},
293	.float_array => {
294	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
295	const param = fg.wip.arg(it.llvm_index - 1);
296
297	const alignment = param_ty.abiAlignment(zcu).toLlvm();
298	const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
299	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
300
301	if (isByRef(param_ty, zcu)) {
302	args.appendAssumeCapacity(arg_ptr);
303	} else {
304	args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
305	}
306	},
307	.i32_array, .i64_array => {
308	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
309	const param = fg.wip.arg(it.llvm_index - 1);
310
311	const alignment = param_ty.abiAlignment(zcu).toLlvm();
312	const arg_ptr = try fg.buildAlloca(param.typeOfWip(&fg.wip), alignment);
313	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
314
315	if (isByRef(param_ty, zcu)) {
316	args.appendAssumeCapacity(arg_ptr);
317	} else {
318	args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
319	}
320	},
321	}
322	}
323
324	fg.args = args.items;
325
326	try fg.genBody(fg.air.getMainBody(), .poi);
327	}
328
329	fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void {
330	const o = self.object;
331	const zcu = self.object.zcu;
332	const ip = &zcu.intern_pool;
333	const air_tags = self.air.instructions.items(.tag);
334	switch (coverage_point) {
335	.none => {},
336	.poi => if (self.fuzz) |*fuzz| {
337	const poi_index = fuzz.pcs.items.len;
338	const base_ptr = fuzz.counters_variable.toValue(&o.builder);
339	const ptr = try self.ptraddConst(base_ptr, poi_index);
340	const one = try o.builder.intValue(.i8, 1);
341	_ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, "");
342
343	// LLVM does not allow blockaddress on the entry block.
344	const pc = if (self.wip.cursor.block == .entry)
345	self.wip.function.toConst(&o.builder)
346	else
347	try o.builder.blockAddrConst(self.wip.function, self.wip.cursor.block);
348	const gpa = self.gpa;
349	try fuzz.pcs.append(gpa, pc);
350	},
351	}
352	for (body) |inst| {
353	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
354
355	const val: Builder.Value = switch (air_tags[@backingInt(inst)]) {
356	// zig fmt: off
357
358	// Required due to `.scalarize_bit_cast_vector_non_elementwise` being enabled.
359	.legalize_vec_elem_val => try self.airLegalizeVecElemVal(inst),
360	.legalize_vec_store_elem => try self.airLegalizeVecStoreElem(inst),
361
362	// No soft float legalizations are enabled.
363	.legalize_compiler_rt_call => unreachable,
364
365	.add => try self.airAdd(inst, .normal),
366	.add_optimized => try self.airAdd(inst, .fast),
367	.add_wrap => try self.airAddWrap(inst),
368	.add_sat => try self.airAddSat(inst),
369
370	.sub => try self.airSub(inst, .normal),
371	.sub_optimized => try self.airSub(inst, .fast),
372	.sub_wrap => try self.airSubWrap(inst),
373	.sub_sat => try self.airSubSat(inst),
374
375	.mul => try self.airMul(inst, .normal),
376	.mul_optimized => try self.airMul(inst, .fast),
377	.mul_wrap => try self.airMulWrap(inst),
378	.mul_sat => try self.airMulSat(inst),
379
380	.add_safe => try self.airSafeArithmetic(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
381	.sub_safe => try self.airSafeArithmetic(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
382	.mul_safe => try self.airSafeArithmetic(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
383
384	.div_float => try self.airDivFloat(inst, .normal),
385	.div_trunc => try self.airDivTrunc(inst, .normal),
386	.div_floor => try self.airDivFloor(inst, .normal),
387	.div_ceil => try self.airDivCeil(inst, .normal),
388	.div_exact => try self.airDivExact(inst, .normal),
389	.rem => try self.airRem(inst, .normal),
390	.mod => try self.airMod(inst, .normal),
391	.abs => try self.airAbs(inst),
392	.ptr_add => try self.airPtrAdd(inst),
393	.ptr_sub => try self.airPtrSub(inst),
394	.shl => try self.airShl(inst),
395	.shl_sat => try self.airShlSat(inst),
396	.shl_exact => try self.airShlExact(inst),
397	.min => try self.airMin(inst),
398	.max => try self.airMax(inst),
399	.slice => try self.airSlice(inst),
400	.mul_add => try self.airMulAdd(inst),
401
402	.div_float_optimized => try self.airDivFloat(inst, .fast),
403	.div_trunc_optimized => try self.airDivTrunc(inst, .fast),
404	.div_floor_optimized => try self.airDivFloor(inst, .fast),
405	.div_ceil_optimized => try self.airDivCeil(inst, .fast),
406	.div_exact_optimized => try self.airDivExact(inst, .fast),
407	.rem_optimized => try self.airRem(inst, .fast),
408	.mod_optimized => try self.airMod(inst, .fast),
409
410	.add_with_overflow => try self.airOverflow(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
411	.sub_with_overflow => try self.airOverflow(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
412	.mul_with_overflow => try self.airOverflow(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
413	.shl_with_overflow => try self.airShlWithOverflow(inst),
414
415	.bit_and => try self.airAnd(inst),
416	.bit_or => try self.airOr(inst),
417	.xor => try self.airXor(inst),
418	.shr => try self.airShr(inst, false),
419	.shr_exact => try self.airShr(inst, true),
420
421	.sqrt => try self.airUnaryOp(inst, .sqrt),
422	.sin => try self.airUnaryOp(inst, .sin),
423	.cos => try self.airUnaryOp(inst, .cos),
424	.tan => try self.airUnaryOp(inst, .tan),
425	.exp => try self.airUnaryOp(inst, .exp),
426	.exp2 => try self.airUnaryOp(inst, .exp2),
427	.log => try self.airUnaryOp(inst, .log),
428	.log2 => try self.airUnaryOp(inst, .log2),
429	.log10 => try self.airUnaryOp(inst, .log10),
430	.floor => try self.airUnaryOp(inst, .floor),
431	.ceil => try self.airUnaryOp(inst, .ceil),
432	.round => try self.airUnaryOp(inst, .round),
433	.trunc_float => try self.airUnaryOp(inst, .trunc),
434
435	.neg => try self.airNeg(inst, .normal),
436	.neg_optimized => try self.airNeg(inst, .fast),
437
438	.cmp_eq => try self.airCmp(inst, .eq, .normal),
439	.cmp_gt => try self.airCmp(inst, .gt, .normal),
440	.cmp_gte => try self.airCmp(inst, .gte, .normal),
441	.cmp_lt => try self.airCmp(inst, .lt, .normal),
442	.cmp_lte => try self.airCmp(inst, .lte, .normal),
443	.cmp_neq => try self.airCmp(inst, .neq, .normal),
444
445	.cmp_eq_optimized => try self.airCmp(inst, .eq, .fast),
446	.cmp_gt_optimized => try self.airCmp(inst, .gt, .fast),
447	.cmp_gte_optimized => try self.airCmp(inst, .gte, .fast),
448	.cmp_lt_optimized => try self.airCmp(inst, .lt, .fast),
449	.cmp_lte_optimized => try self.airCmp(inst, .lte, .fast),
450	.cmp_neq_optimized => try self.airCmp(inst, .neq, .fast),
451
452	.cmp_vector => try self.airCmpVector(inst, .normal),
453	.cmp_vector_optimized => try self.airCmpVector(inst, .fast),
454	.cmp_lte_errors_len => try self.airCmpLteErrorsLen(inst),
455
456	.is_non_null => try self.airIsNonNull(inst, false, .ne),
457	.is_non_null_ptr => try self.airIsNonNull(inst, true , .ne),
458	.is_null => try self.airIsNonNull(inst, false, .eq),
459	.is_null_ptr => try self.airIsNonNull(inst, true , .eq),
460
461	.is_non_err => try self.airIsErr(inst, .eq, false),
462	.is_non_err_ptr => try self.airIsErr(inst, .eq, true),
463	.is_err => try self.airIsErr(inst, .ne, false),
464	.is_err_ptr => try self.airIsErr(inst, .ne, true),
465
466	.alloc => try self.airAlloc(inst),
467	.ret_ptr => try self.airRetPtr(inst),
468	.arg => try self.airArg(inst),
469	.bit_cast => try self.airBitCast(inst, false),
470	.bit_cast_safe => try self.airBitCast(inst, true),
471	.ptr_cast => try self.airNopCast(inst),
472	.ptr_from_int => try self.airPtrFromInt(inst),
473	.int_from_ptr => try self.airIntFromPtr(inst),
474	.error_cast => try self.airNopCast(inst),
475	.error_from_int => try self.airNopCast(inst),
476	.int_from_error => try self.airNopCast(inst),
477	.union_from_enum => try self.airUnionFromEnum(inst),
478	.breakpoint => try self.airBreakpoint(inst),
479	.ret_addr => try self.airRetAddr(inst),
480	.frame_addr => try self.airFrameAddress(inst),
481	.@"try" => try self.airTry(inst, false),
482	.try_cold => try self.airTry(inst, true),
483	.try_ptr => try self.airTryPtr(inst, false),
484	.try_ptr_cold => try self.airTryPtr(inst, true),
485	.int_cast => try self.airIntCast(inst, false),
486	.int_cast_safe => try self.airIntCast(inst, true),
487	.trunc => try self.airTrunc(inst),
488	.fptrunc => try self.airFptrunc(inst),
489	.fpext => try self.airFpext(inst),
490	.load => try self.airLoad(inst),
491	.not => try self.airNot(inst),
492	.store => try self.airStore(inst, false),
493	.store_safe => try self.airStore(inst, true),
494	.assembly => try self.airAssembly(inst),
495	.slice_ptr => try self.airSliceField(inst, 0),
496	.slice_len => try self.airSliceField(inst, 1),
497
498	.ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
499	.ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
500
501	.int_from_float => try self.airIntFromFloat(inst, .normal),
502	.int_from_float_optimized => try self.airIntFromFloat(inst, .fast),
503	.int_from_float_safe => unreachable, // handled by `legalizeFeatures`
504	.int_from_float_optimized_safe => unreachable, // handled by `legalizeFeatures`
505
506	.array_to_slice => try self.airArrayToSlice(inst),
507	.array_to_vector => try self.airArrayToVector(inst),
508	.float_from_int => try self.airFloatFromInt(inst),
509	.cmpxchg_weak => try self.airCmpxchg(inst, .weak),
510	.cmpxchg_strong => try self.airCmpxchg(inst, .strong),
511	.atomic_rmw => try self.airAtomicRmw(inst),
512	.atomic_load => try self.airAtomicLoad(inst),
513	.memset => try self.airMemset(inst, false),
514	.memset_safe => try self.airMemset(inst, true),
515	.memcpy => try self.airMemcpy(inst),
516	.memmove => try self.airMemmove(inst),
517	.set_union_tag => try self.airSetUnionTag(inst),
518	.get_union_tag => try self.airGetUnionTag(inst),
519	.clz => try self.airClzCtz(inst, .ctlz),
520	.ctz => try self.airClzCtz(inst, .cttz),
521	.popcount => try self.airBitOp(inst, .ctpop),
522	.byte_swap => try self.airByteSwap(inst),
523	.bit_reverse => try self.airBitOp(inst, .bitreverse),
524	.tag_name => try self.airTagName(inst),
525	.error_name => try self.airErrorName(inst),
526	.splat => try self.airSplat(inst),
527	.select => try self.airSelect(inst),
528	.shuffle_one => try self.airShuffleOne(inst),
529	.shuffle_two => try self.airShuffleTwo(inst),
530	.aggregate_init => try self.airAggregateInit(inst),
531	.union_init => try self.airUnionInit(inst),
532	.prefetch => try self.airPrefetch(inst),
533	.addrspace_cast => try self.airAddrSpaceCast(inst),
534
535	.is_named_enum_value => try self.airIsNamedEnumValue(inst),
536	.error_set_has_value => try self.airErrorSetHasValue(inst),
537
538	.reduce => try self.airReduce(inst, .normal),
539	.reduce_optimized => try self.airReduce(inst, .fast),
540
541	.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
542	.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
543	.atomic_store_release => try self.airAtomicStore(inst, .release),
544	.atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
545
546	.struct_field_ptr => try self.airStructFieldPtr(inst),
547	.agg_field_val => try self.airAggFieldVal(inst),
548
549	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
550	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
551	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
552	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
553
554	.field_parent_ptr => try self.airFieldParentPtr(inst),
555
556	.array_elem_val => try self.airArrayElemVal(inst),
557	.slice_elem_val => try self.airSliceElemVal(inst),
558	.slice_elem_ptr => try self.airSliceElemPtr(inst),
559	.ptr_elem_val => try self.airPtrElemVal(inst),
560	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
561
562	.optional_payload => try self.airOptionalPayload(inst),
563	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
564	.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
565
566	.unwrap_errunion_payload => try self.airErrUnionPayload(inst),
567	.unwrap_errunion_payload_ptr => try self.airErrUnionPayloadPtr(inst),
568	.unwrap_errunion_err => try self.airErrUnionErr(inst, false),
569	.unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
570	.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
571	.err_return_trace => try self.airErrReturnTrace(inst),
572	.set_err_return_trace => try self.airSetErrReturnTrace(inst),
573	.save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),
574
575	.wrap_optional => try self.airWrapOptional(inst),
576	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
577	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
578
579	.wasm_memory_size => try self.airWasmMemorySize(inst),
580	.wasm_memory_grow => try self.airWasmMemoryGrow(inst),
581
582	.runtime_nav_ptr => try self.airRuntimeNavPtr(inst),
583
584	.inferred_alloc, .inferred_alloc_comptime => unreachable,
585
586	.dbg_stmt => try self.airDbgStmt(inst),
587	.dbg_empty_stmt => try self.airDbgEmptyStmt(inst),
588	.dbg_var_ptr => try self.airDbgVarPtr(inst),
589	.dbg_var_val => try self.airDbgVarVal(inst, false),
590	.dbg_arg_inline => try self.airDbgVarVal(inst, true),
591
592	.c_va_arg => try self.airCVaArg(inst),
593	.c_va_copy => try self.airCVaCopy(inst),
594	.c_va_end => try self.airCVaEnd(inst),
595	.c_va_start => try self.airCVaStart(inst),
596
597	.work_item_id => try self.airWorkItemId(inst),
598	.work_group_size => try self.airWorkGroupSize(inst),
599	.work_group_id => try self.airWorkGroupId(inst),
600	.spirv_runtime_array_len => unreachable,
601
602	// Instructions that are known to always be `noreturn` based on their tag.
603	.br => return self.airBr(inst),
604	.repeat => return self.airRepeat(inst),
605	.switch_dispatch => return self.airSwitchDispatch(inst),
606	.cond_br => return self.airCondBr(inst),
607	.switch_br => return self.airSwitchBr(inst, false),
608	.loop_switch_br => return self.airSwitchBr(inst, true),
609	.loop => return self.airLoop(inst),
610	.ret => return self.airRet(inst, false),
611	.ret_safe => return self.airRet(inst, true),
612	.ret_load => return self.airRetLoad(inst),
613	.trap => return self.airTrap(inst),
614	.unreach => return self.airUnreach(inst),
615
616	// Instructions which may be `noreturn`.
617	.block => res: {
618	const block = self.air.unwrapBlock(inst);
619	const res = try self.lowerBlock(inst, null, block.body);
620	if (block.ty.isNoReturn(zcu)) return;
621	break :res res;
622	},
623	.dbg_inline_block => res: {
624	const block = self.air.unwrapDbgBlock(inst);
625	self.arg_inline_index = 0;
626	const res = try self.lowerBlock(inst, block.func, block.body);
627	if (block.ty.isNoReturn(zcu)) return;
628	break :res res;
629	},
630	.call, .call_always_tail, .call_never_tail, .call_never_inline => |tag| res: {
631	const res = try self.airCall(inst, switch (tag) {
632	.call => .auto,
633	.call_always_tail => .always_tail,
634	.call_never_tail => .never_tail,
635	.call_never_inline => .never_inline,
636	else => unreachable,
637	});
638	// TODO: the AIR we emit for calls is a bit weird - the instruction has
639	// type `noreturn`, but there are instructions (and maybe a safety check) following
640	// nonetheless. The `unreachable` or safety check should be emitted by backends instead.
641	//if (self.typeOfIndex(inst).isNoReturn(mod)) return;
642	break :res res;
643	},
644
645	// zig fmt: on
646	};
647	if (val != .none) try self.func_inst_table.putNoClobber(self.gpa, inst.toRef(), val);
648	}
649	unreachable;
650	}
651
652	fn genBodyDebugScope(
653	self: *FuncGen,
654	maybe_inline_func: ?InternPool.Index,
655	body: []const Air.Inst.Index,
656	coverage_point: Air.CoveragePoint,
657	) TodoError!void {
658	const o = self.object;
659
660	if (self.wip.strip) return self.genBody(body, coverage_point);
661
662	const old_debug_location = self.wip.debug_location;
663	const old_file = self.file;
664	const old_inlined_at = self.inlined_at;
665	const old_base_line = self.base_line;
666	defer if (maybe_inline_func) |_| {
667	self.wip.debug_location = old_debug_location;
668	self.file = old_file;
669	self.inlined_at = old_inlined_at;
670	self.base_line = old_base_line;
671	};
672
673	const old_scope = self.scope;
674	defer self.scope = old_scope;
675
676	if (maybe_inline_func) |inline_func| {
677	const zcu = o.zcu;
678	const ip = &zcu.intern_pool;
679
680	const func = zcu.funcInfo(inline_func);
681	const nav = ip.getNav(func.owner_nav);
682	const file_scope = zcu.navFileScopeIndex(func.owner_nav);
683	const mod = zcu.fileByIndex(file_scope).mod.?;
684
685	self.file = try o.getDebugFile(file_scope);
686
687	self.base_line = zcu.navSrcLine(func.owner_nav);
688	const line_number = self.base_line + 1;
689	self.inlined_at = try self.wip.debug_location.toMetadata(&o.builder);
690
691	self.scope = try o.builder.debugSubprogram(
692	self.file,
693	try o.builder.metadataString(nav.name.toSlice(&zcu.intern_pool)),
694	try o.builder.metadataString(nav.fqn.toSlice(&zcu.intern_pool)),
695	line_number,
696	line_number + func.lbrace_line,
697	try o.builder.debugSubroutineType(null),
698	.{
699	.di_flags = .{ .StaticMember = true },
700	.sp_flags = .{
701	.Optimized = mod.optimize_mode != .debug,
702	.Definition = true,
703	.LocalToUnit = true, // inline functions cannot be exported
704	},
705	},
706	o.debug_compile_unit.unwrap().?,
707	);
708	}
709
710	self.scope = try o.builder.debugLexicalBlock(
711	self.scope,
712	self.file,
713	self.prev_dbg_line,
714	self.prev_dbg_column,
715	);
716	self.wip.debug_location = .{ .location = .{
717	.line = self.prev_dbg_line,
718	.column = self.prev_dbg_column,
719	.scope = self.scope.toOptional(),
720	.inlined_at = self.inlined_at,
721	} };
722	defer self.wip.debug_location.location.scope = old_scope.toOptional();
723
724	try self.genBody(body, coverage_point);
725	}
726
727	fn airCall(fg: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) Allocator.Error!Builder.Value {
728	const o = fg.object;
729	const zcu = o.zcu;
730	const air_call = fg.air.unwrapCall(inst);
731	const args = air_call.args;
732	const ip = &zcu.intern_pool;
733	const callee_ty = fg.typeOf(air_call.callee);
734	const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
735	.@"fn" => callee_ty,
736	.pointer => callee_ty.childType(zcu),
737	else => unreachable,
738	};
739	const fn_info = zcu.typeToFunc(zig_fn_ty).?;
740	const llvm_fn = llvm_fn: {
741	// If the callee is a function *body*, we need to use a pointer to the global.
742	if (air_call.callee.toInterned()) |ip_index| switch (ip.indexToKey(ip_index)) {
743	.@"extern" => |e| break :llvm_fn (try o.lowerNavRef(e.owner_nav)).toValue(),
744	.func => |f| break :llvm_fn (try o.lowerNavRef(f.owner_nav)).toValue(),
745	else => {},
746	};
747	// Otherwise, the operand is already a function pointer (possibly runtime-known).
748	break :llvm_fn try fg.resolveInst(air_call.callee);
749	};
750
751	const arg_types = try fg.gpa.alloc(InternPool.Index, args.len);
752	defer fg.gpa.free(arg_types);
753	const arg_values = try fg.gpa.alloc(Builder.Value, args.len);
754	defer fg.gpa.free(arg_values);
755	for (arg_types, arg_values, args) |*arg_type, *arg_value, arg| {
756	const arg_ty = fg.typeOf(arg);
757	arg_type.* = arg_ty.toIntern();
758	arg_value.* = if (arg_ty.hasRuntimeBits(zcu)) try fg.resolveInst(arg) else .none;
759	}
760	return fg.buildCall(.{
761	.is_unused = fg.liveness.isUnused(inst),
762	.modifier = modifier,
763	}, try o.lowerType(zig_fn_ty, .as_value), llvm_fn, .fromIntern(fn_info, ip), arg_types, arg_values);
764	}
765
766	fn buildCall(
767	fg: *FuncGen,
768	opts: struct {
769	is_unused: bool = false,
770	modifier: std.lang.CallModifier = .auto,
771	},
772	llvm_fn_ty: Builder.Type,
773	llvm_fn: Builder.Value,
774	fn_info: Object.FuncInfo,
775	arg_types: []const InternPool.Index,
776	arg_values: []const Builder.Value,
777	) Allocator.Error!Builder.Value {
778	const o = fg.object;
779	const pt = fg.pt;
780	const zcu = o.zcu;
781	const return_type: Type = .fromInterned(fn_info.return_type);
782	const target = zcu.getTarget();
783	const ret_strat = try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type));
784
785	var llvm_args: std.ArrayList(Builder.Value) = .empty;
786	defer llvm_args.deinit(fg.gpa);
787
788	var attributes: Builder.FunctionAttributes.Wip = .{};
789	defer attributes.deinit(&o.builder);
790
791	if (fg.disable_intrinsics) {
792	try attributes.addFnAttr(.nobuiltin, &o.builder);
793	}
794
795	switch (opts.modifier) {
796	.auto, .always_tail => {},
797	.never_tail, .never_inline => try attributes.addFnAttr(.@"noinline", &o.builder),
798	.no_suspend, .always_inline, .compile_time => unreachable,
799	}
800
801	const sret_alloc: ?Builder.Value = switch (ret_strat) {
802	.sret => sret_alloc: {
803	const alignment = return_type.abiAlignment(zcu).toLlvm();
804	try o.addSRetFnAttributes(&attributes, try o.lowerType(return_type, .in_memory), alignment, .callsite);
805
806	const ptr = try fg.buildZigAlloca(return_type, .none);
807	try llvm_args.append(fg.gpa, ptr);
808	break :sret_alloc ptr;
809	},
810	else => sret_alloc: {
811	if (ccAbiPromoteInt(fn_info.cc, zcu, .fromInterned(fn_info.return_type))) |s| switch (s) {
812	.signed => try attributes.addRetAttr(.signext, &o.builder),
813	.unsigned => try attributes.addRetAttr(.zeroext, &o.builder),
814	};
815	break :sret_alloc null;
816	},
817	};
818
819	const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
820	if (err_return_tracing) {
821	assert(fg.err_ret_trace != .none);
822	try llvm_args.append(fg.gpa, fg.err_ret_trace);
823	}
824
825	var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types);
826	while (try it.nextCall(arg_types)) |lowering| {
827	const arg_ty: Type = .fromInterned(arg_types[it.zig_index - 1]);
828	const arg_val = arg_values[it.zig_index - 1];
829	switch (lowering) {
830	.no_bits => continue,
831	.byval => {
832	if (isByRef(arg_ty, zcu)) {
833	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
834	// We don't need to handle non-ABI-sized integer types in memory here since they are
835	// never by-ref.
836	const llvm_arg_ty = try o.lowerType(arg_ty, .memory_access);
837	const loaded = try fg.wip.load(.normal, llvm_arg_ty, arg_val, alignment, "");
838	try llvm_args.append(fg.gpa, loaded);
839	} else {
840	try llvm_args.append(fg.gpa, arg_val);
841	}
842	},
843	.byref => {
844	if (isByRef(arg_ty, zcu)) {
845	try llvm_args.append(fg.gpa, arg_val);
846	} else {
847	const arg_ptr = try fg.buildZigAlloca(arg_ty, .none);
848	try fg.store(arg_ptr, .none, arg_val, arg_ty, .normal);
849	try llvm_args.append(fg.gpa, arg_ptr);
850	}
851	},
852	.byref_mut => {
853	const arg_ptr = try fg.buildZigAlloca(arg_ty, .none);
854	try fg.store(arg_ptr, .none, arg_val, arg_ty, .normal);
855	try llvm_args.append(fg.gpa, arg_ptr);
856	},
857	.abi_sized_int => {
858	const int_llvm_ty = try o.builder.intType(@intCast(arg_ty.abiSize(zcu) * 8));
859
860	if (isByRef(arg_ty, zcu)) {
861	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
862	const loaded = try fg.wip.load(.normal, int_llvm_ty, arg_val, alignment, "");
863	try llvm_args.append(fg.gpa, loaded);
864	} else {
865	// LLVM does not allow bitcasting structs so we must allocate
866	// a local, store as one type, and then load as another type.
867	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
868	const ptr = try fg.buildAlloca(int_llvm_ty, alignment);
869	try fg.store(ptr, .none, arg_val, arg_ty, .normal);
870	const loaded = try fg.wip.load(.normal, int_llvm_ty, ptr, alignment, "");
871	try llvm_args.append(fg.gpa, loaded);
872	}
873	},
874	.slice => {
875	const ptr = try fg.wip.extractValue(arg_val, &.{0}, "");
876	const len = try fg.wip.extractValue(arg_val, &.{1}, "");
877	try llvm_args.appendSlice(fg.gpa, &.{ ptr, len });
878	},
879	.multiple_llvm_types => {
880	const arg_alignment = arg_ty.abiAlignment(zcu);
881	const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
882	const arg_ptr = try fg.buildAlloca(llvm_ty, arg_alignment.toLlvm());
883	try fg.store(arg_ptr, .none, arg_val, arg_ty, .normal);
884
885	try llvm_args.ensureUnusedCapacity(fg.gpa, it.types_len);
886	for (it.types_buffer[0..it.types_len], it.offsets_buffer[0..it.types_len]) |field_ty, offset| {
887	const field_ptr = try fg.ptraddConst(arg_ptr, offset);
888	const loaded = try fg.wip.load(.normal, field_ty, field_ptr, arg_alignment.offset(offset).toLlvm(), "");
889	llvm_args.appendAssumeCapacity(loaded);
890	}
891	},
892	.float_array => |count| {
893	const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
894	const ptr = try fg.buildZigAlloca(arg_ty, .none);
895	try fg.store(ptr, .none, arg_val, arg_ty, .normal);
896	break :ptr ptr;
897	} else arg_val;
898
899	const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?, .memory_access);
900	const array_ty = try o.builder.arrayType(count, float_ty);
901
902	const loaded = try fg.wip.load(.normal, array_ty, arg_ptr, arg_ty.abiAlignment(zcu).toLlvm(), "");
903	try llvm_args.append(fg.gpa, loaded);
904	},
905	.i32_array, .i64_array => |arr_len| {
906	const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
907
908	const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
909	const ptr = try fg.buildZigAlloca(arg_ty, .none);
910	try fg.store(ptr, .none, arg_val, arg_ty, .normal);
911	break :ptr ptr;
912	} else arg_val;
913
914	const array_ty = try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
915	const loaded = try fg.wip.load(.normal, array_ty, arg_ptr, arg_ty.abiAlignment(zcu).toLlvm(), "");
916	try llvm_args.append(fg.gpa, loaded);
917	},
918	}
919	}
920
921	const cc_info = llvm.toLlvmCallConv(fn_info.cc, target).?;
922
923	{
924	// Add argument attributes.
925	it = iterateParamTypes(o, fn_info.cc, fn_info.param_types);
926	it.llvm_index += @intFromBool(ret_strat == .sret);
927	it.llvm_index += @intFromBool(err_return_tracing);
928	var remaining_inreg_int = cc_info.inreg_int_params;
929	var remaining_inreg_float = cc_info.inreg_float_params;
930	while (try it.next()) |lowering| switch (lowering) {
931	.byval => {
932	const param_index = it.zig_index - 1;
933	const param_ty = Type.fromInterned(fn_info.param_types[param_index]);
934	if (!isByRef(param_ty, zcu)) {
935	try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
936	}
937
938	if (remaining_inreg_int > 0 and
939	(param_ty.isPtrAtRuntime(zcu) or
940	(param_ty.isAbiInt(zcu) and param_ty.abiSize(zcu) <= Type.usize.abiSize(zcu))))
941	{
942	try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
943	remaining_inreg_int -= 1;
944	}
945
946	if (remaining_inreg_float > 0 and
947	param_ty.zigTypeTag(zcu) == .float)
948	{
949	try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
950	remaining_inreg_float -= 1;
951	}
952	},
953	.byref => {
954	const param_index = it.zig_index - 1;
955	const param_ty: Type = .fromInterned(fn_info.param_types[param_index]);
956	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, it.byval_attr, param_ty);
957	},
958	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
959	// No attributes needed for these.
960	.no_bits,
961	.abi_sized_int,
962	.multiple_llvm_types,
963	.float_array,
964	.i32_array,
965	.i64_array,
966	=> continue,
967
968	.slice => {
969	assert(it.byval_attr == null);
970	const param_ty = Type.fromInterned(fn_info.param_types[it.zig_index - 1]);
971	const ptr_info = param_ty.ptrInfo(zcu);
972	const llvm_arg_i = it.llvm_index - 2;
973
974	if (math.cast(u5, it.zig_index - 1)) |i| {
975	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
976	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
977	}
978	}
979	if (param_ty.zigTypeTag(zcu) != .optional and
980	!ptr_info.flags.is_allowzero and
981	ptr_info.flags.address_space == .generic)
982	{
983	try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
984	}
985	if (ptr_info.flags.is_const) {
986	try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
987	}
988	const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) {
989	else => |a| .wrap(a.toLlvm()),
990	.none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)),
991	};
992	try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder);
993	},
994	};
995	}
996
997	const call = try fg.wip.call(
998	switch (opts.modifier) {
999	.auto, .never_inline => .normal,
1000	.never_tail => .notail,
1001	.always_tail => .musttail,
1002	.no_suspend, .always_inline, .compile_time => unreachable,
1003	},
1004	cc_info.llvm_cc,
1005	try attributes.finish(&o.builder),
1006	llvm_fn_ty,
1007	llvm_fn,
1008	llvm_args.items,
1009	"",
1010	);
1011
1012	if (opts.is_unused) return .none;
1013	if (fn_info.return_type == .noreturn_type and opts.modifier != .always_tail) return .none;
1014
1015	// We exit this `switch` if we have a pointer to the return value.
1016	const ret_val_ptr: Builder.Value = switch (ret_strat) {
1017	.void => return .none,
1018	.by_val => return call,
1019
1020	.sret => sret_alloc.?,
1021	.mem_cast => |llvm_ret_ty| ret_val_ptr: {
1022	const alignment = return_type.abiAlignment(zcu).toLlvm();
1023	const ptr = try fg.buildAlloca(llvm_ret_ty, alignment);
1024	_ = try fg.wip.store(.normal, call, ptr, alignment);
1025	break :ret_val_ptr ptr;
1026	},
1027	};
1028	if (isByRef(return_type, zcu)) {
1029	return ret_val_ptr;
1030	} else {
1031	return fg.load(ret_val_ptr, .none, return_type, .normal);
1032	}
1033	}
1034
1035	fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!void {
1036	const o = fg.object;
1037	const zcu = o.zcu;
1038	const target = zcu.getTarget();
1039	const panic_func = zcu.funcInfo(zcu.std_lang_decl_values.get(panic_id.toStdLangDecl()));
1040	const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;
1041	const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty), .as_value);
1042
1043	const llvm_panic_fn_ref = try o.lowerNavRef(panic_func.owner_nav);
1044
1045	const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto;
1046	if (has_err_trace) assert(fg.err_ret_trace != .none);
1047	_ = try fg.wip.callIntrinsicAssumeCold();
1048	_ = try fg.wip.call(
1049	.normal,
1050	llvm.toLlvmCallConvTag(fn_info.cc, target).?,
1051	.none,
1052	llvm_panic_fn_ty,
1053	llvm_panic_fn_ref.toValue(),
1054	if (has_err_trace) &.{fg.err_ret_trace} else &.{},
1055	"",
1056	);
1057	_ = try fg.wip.@"unreachable"();
1058	}
1059
1060	fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!void {
1061	const o = self.object;
1062	const zcu = o.zcu;
1063	const ip = &zcu.intern_pool;
1064	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
1065
1066	const ret_ty = self.typeOf(un_op);
1067
1068	const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1069
1070	const ret_strat = try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type));
1071	const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
1072	const ret_ty_align = ret_ty.abiAlignment(zcu);
1073
1074	if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
1075	const rp = switch (self.ret_ptr) {
1076	.none => try self.buildZigAlloca(ret_ty, .none),
1077	else => |rp| rp,
1078	};
1079	const len = try o.builder.intValue(try o.lowerType(.usize, .as_value), ret_ty.abiSize(zcu));
1080	_ = try self.wip.callMemSet(
1081	rp,
1082	ret_ty_align.toLlvm(),
1083	try o.builder.intValue(.i8, 0xaa),
1084	len,
1085	.normal,
1086	self.disable_intrinsics,
1087	);
1088	const owner_mod = self.ownerModule();
1089	if (owner_mod.valgrind) {
1090	try self.valgrindMarkUndef(rp, len);
1091	}
1092	switch (ret_strat) {
1093	.void => unreachable, // value is undef so return type cannot be OPV
1094	.sret => {
1095	// We just stored directly to `self.ret_ptr`.
1096	_ = try self.wip.retVoid();
1097	},
1098	.by_val => {
1099	const loaded = try self.load(rp, .none, ret_ty, .normal);
1100	_ = try self.wip.ret(loaded);
1101	},
1102	.mem_cast => |llvm_abi_ret_ty| {
1103	const loaded = try self.wip.load(.normal, llvm_abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
1104	_ = try self.wip.ret(loaded);
1105	},
1106	}
1107	return;
1108	}
1109
1110	switch (ret_strat) {
1111	.void => _ = try self.wip.retVoid(),
1112	.sret => {
1113	const operand = try self.resolveInst(un_op);
1114	try self.store(self.ret_ptr, .none, operand, ret_ty, .normal);
1115	_ = try self.wip.retVoid();
1116	},
1117	.by_val => {
1118	assert(!isByRef(ret_ty, zcu));
1119	const operand = try self.resolveInst(un_op);
1120	_ = try self.wip.ret(operand);
1121	},
1122	.mem_cast => |llvm_ret_ty| {
1123	const operand = try self.resolveInst(un_op);
1124	const ptr: Builder.Value = if (!isByRef(ret_ty, zcu)) ptr: {
1125	const ptr = try self.buildZigAlloca(ret_ty, .none);
1126	try self.store(ptr, .none, operand, ret_ty, .normal);
1127	break :ptr ptr;
1128	} else operand;
1129	const ret_val = try self.wip.load(.normal, llvm_ret_ty, ptr, ret_ty_align.toLlvm(), "");
1130	_ = try self.wip.ret(ret_val);
1131	},
1132	}
1133	}
1134
1135	fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
1136	const o = self.object;
1137	const zcu = o.zcu;
1138	const ip = &zcu.intern_pool;
1139	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
1140	const ptr_ty = self.typeOf(un_op);
1141	const ret_ty = ptr_ty.childType(zcu);
1142	const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1143	const ptr = try self.resolveInst(un_op);
1144	switch (try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type))) {
1145	.void => _ = try self.wip.retVoid(),
1146	.sret => {
1147	assert(self.ret_ptr != .none);
1148	_ = try self.wip.retVoid();
1149	},
1150	.by_val => {
1151	assert(self.ret_ptr == .none);
1152	const loaded = try self.load(ptr, .none, ret_ty, .normal);
1153	_ = try self.wip.ret(loaded);
1154	},
1155	.mem_cast => |llvm_abi_ret_ty| {
1156	assert(self.ret_ptr == .none);
1157	const ret_ty_align = ret_ty.abiAlignment(zcu);
1158	const loaded = try self.wip.load(.normal, llvm_abi_ret_ty, ptr, ret_ty_align.toLlvm(), "");
1159	_ = try self.wip.ret(loaded);
1160	},
1161	}
1162	}
1163
1164	fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
1165	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1166	const list = try self.resolveInst(ty_op.operand);
1167	const arg_ty = ty_op.ty;
1168	const llvm_arg_ty = try self.object.lowerType(arg_ty, .as_value);
1169
1170	return self.wip.vaArg(list, llvm_arg_ty, "");
1171	}
1172
1173	fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
1174	const o = self.object;
1175	const zcu = o.zcu;
1176	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1177	const src_list = try self.resolveInst(ty_op.operand);
1178	const va_list_ty = ty_op.ty;
1179
1180	const dest_list = try self.buildZigAlloca(va_list_ty, .none);
1181
1182	_ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
1183	return if (isByRef(va_list_ty, zcu))
1184	dest_list
1185	else
1186	try self.load(dest_list, .none, va_list_ty, .normal);
1187	}
1188
1189	fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
1190	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
1191	const src_list = try self.resolveInst(un_op);
1192
1193	_ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, "");
1194	return .none;
1195	}
1196
1197	fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
1198	const o = self.object;
1199	const zcu = o.zcu;
1200	const va_list_ty = self.typeOfIndex(inst);
1201
1202	const dest_list = try self.buildZigAlloca(va_list_ty, .none);
1203
1204	_ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
1205	return if (isByRef(va_list_ty, zcu))
1206	dest_list
1207	else
1208	try self.load(dest_list, .none, va_list_ty, .normal);
1209	}
1210
1211	fn airCmp(
1212	self: *FuncGen,
1213	inst: Air.Inst.Index,
1214	op: math.CompareOperator,
1215	fast: Builder.FastMathKind,
1216	) Allocator.Error!Builder.Value {
1217	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1218	const lhs = try self.resolveInst(bin_op.lhs);
1219	const rhs = try self.resolveInst(bin_op.rhs);
1220	const operand_ty = self.typeOf(bin_op.lhs);
1221
1222	return self.cmp(fast, op, operand_ty, lhs, rhs);
1223	}
1224
1225	fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
1226	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
1227	const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
1228
1229	const lhs = try self.resolveInst(extra.lhs);
1230	const rhs = try self.resolveInst(extra.rhs);
1231	const vec_ty = self.typeOf(extra.lhs);
1232	const cmp_op = extra.compareOperator();
1233
1234	return self.cmp(fast, cmp_op, vec_ty, lhs, rhs);
1235	}
1236
1237	fn airCmpLteErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
1238	const o = self.object;
1239	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
1240	const operand = try self.resolveInst(un_op);
1241	const errors_len_ptr = try o.getErrorsLen();
1242	const errors_len_val = try self.load(errors_len_ptr.toValue(&o.builder), .none, .anyerror, .normal);
1243	return self.wip.icmp(.ule, operand, errors_len_val, "");
1244	}
1245
1246	fn cmp(
1247	self: *FuncGen,
1248	fast: Builder.FastMathKind,
1249	op: math.CompareOperator,
1250	operand_ty: Type,
1251	lhs: Builder.Value,
1252	rhs: Builder.Value,
1253	) Allocator.Error!Builder.Value {
1254	const o = self.object;
1255	const zcu = o.zcu;
1256	const scalar_ty = operand_ty.scalarType(zcu);
1257	const int_ty = switch (scalar_ty.zigTypeTag(zcu)) {
1258	.int, .bool, .pointer, .error_set => scalar_ty,
1259	.optional => blk: {
1260	const payload_ty = operand_ty.optionalChild(zcu);
1261	if (!payload_ty.hasRuntimeBits(zcu) or
1262	operand_ty.optionalReprIsPayload(zcu))
1263	{
1264	break :blk operand_ty;
1265	}
1266	// We need to emit instructions to check for equality/inequality
1267	// of optionals that are not pointers.
1268	const lhs_non_null = try self.optCmpNull(.ne, scalar_ty, lhs, .normal);
1269	const rhs_non_null = try self.optCmpNull(.ne, scalar_ty, rhs, .normal);
1270	const llvm_i2 = try o.builder.intType(2);
1271	const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, "");
1272	const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, "");
1273	const lhs_shifted = try self.wip.bin(.shl, lhs_non_null_i2, try o.builder.intValue(llvm_i2, 1), "");
1274	const lhs_rhs_ored = try self.wip.bin(.@"or", lhs_shifted, rhs_non_null_i2, "");
1275	const both_null_block = try self.wip.block(1, "BothNull");
1276	const mixed_block = try self.wip.block(1, "Mixed");
1277	const both_pl_block = try self.wip.block(1, "BothNonNull");
1278	const end_block = try self.wip.block(3, "End");
1279	var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none);
1280	defer wip_switch.finish(&self.wip);
1281	try wip_switch.addCase(
1282	try o.builder.intConst(llvm_i2, 0b00),
1283	both_null_block,
1284	&self.wip,
1285	);
1286	try wip_switch.addCase(
1287	try o.builder.intConst(llvm_i2, 0b11),
1288	both_pl_block,
1289	&self.wip,
1290	);
1291
1292	self.wip.cursor = .{ .block = both_null_block };
1293	_ = try self.wip.br(end_block);
1294
1295	self.wip.cursor = .{ .block = mixed_block };
1296	_ = try self.wip.br(end_block);
1297
1298	self.wip.cursor = .{ .block = both_pl_block };
1299	const lhs_payload = try self.optPayloadHandle(lhs, scalar_ty, true);
1300	const rhs_payload = try self.optPayloadHandle(rhs, scalar_ty, true);
1301	const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload);
1302	_ = try self.wip.br(end_block);
1303	const both_pl_block_end = self.wip.cursor.block;
1304
1305	self.wip.cursor = .{ .block = end_block };
1306	const llvm_i1_0 = Builder.Value.false;
1307	const llvm_i1_1 = Builder.Value.true;
1308	const incoming_values: [3]Builder.Value = .{
1309	switch (op) {
1310	.eq => llvm_i1_1,
1311	.neq => llvm_i1_0,
1312	else => unreachable,
1313	},
1314	switch (op) {
1315	.eq => llvm_i1_0,
1316	.neq => llvm_i1_1,
1317	else => unreachable,
1318	},
1319	payload_cmp,
1320	};
1321
1322	const phi = try self.wip.phi(.i1, "");
1323	phi.finish(
1324	&incoming_values,
1325	&.{ both_null_block, mixed_block, both_pl_block_end },
1326	&self.wip,
1327	);
1328	return phi.toValue();
1329	},
1330	.float => return self.buildFloatCmp(fast, op, operand_ty, .{ lhs, rhs }),
1331	.@"enum", .@"struct", .@"union" => scalar_ty.backingIntType(zcu),
1332	else => unreachable,
1333	};
1334	const is_signed = int_ty.isSignedInt(zcu);
1335	const cond: Builder.IntegerCondition = switch (op) {
1336	.eq => .eq,
1337	.neq => .ne,
1338	.lt => if (is_signed) .slt else .ult,
1339	.lte => if (is_signed) .sle else .ule,
1340	.gt => if (is_signed) .sgt else .ugt,
1341	.gte => if (is_signed) .sge else .uge,
1342	};
1343	return self.wip.icmp(cond, lhs, rhs, "");
1344	}
1345
1346	fn lowerBlock(
1347	self: *FuncGen,
1348	inst: Air.Inst.Index,
1349	maybe_inline_func: ?InternPool.Index,
1350	body: []const Air.Inst.Index,
1351	) TodoError!Builder.Value {
1352	const o = self.object;
1353	const zcu = o.zcu;
1354	const inst_ty = self.typeOfIndex(inst);
1355
1356	if (inst_ty.isNoReturn(zcu)) {
1357	try self.genBodyDebugScope(maybe_inline_func, body, .none);
1358	return .none;
1359	}
1360
1361	const have_block_result = inst_ty.hasRuntimeBits(zcu);
1362
1363	var breaks: BreakList = if (have_block_result) .{ .list = .{} } else .{ .len = 0 };
1364	defer if (have_block_result) breaks.list.deinit(self.gpa);
1365
1366	const parent_bb = try self.wip.block(0, "Block");
1367	try self.blocks.putNoClobber(self.gpa, inst, .{
1368	.parent_bb = parent_bb,
1369	.breaks = &breaks,
1370	});
1371	defer assert(self.blocks.remove(inst));
1372
1373	try self.genBodyDebugScope(maybe_inline_func, body, .none);
1374
1375	self.wip.cursor = .{ .block = parent_bb };
1376
1377	// Create a phi node only if the block returns a value.
1378	if (have_block_result) {
1379	const llvm_ty: Builder.Type = switch (isByRef(inst_ty, zcu)) {
1380	true => .ptr,
1381	false => try o.lowerType(inst_ty, .as_value),
1382	};
1383	parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);
1384	const phi = try self.wip.phi(llvm_ty, "");
1385	phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);
1386	return phi.toValue();
1387	} else {
1388	parent_bb.ptr(&self.wip).incoming = @intCast(breaks.len);
1389	return .none;
1390	}
1391	}
1392
1393	fn airBr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
1394	const zcu = self.object.zcu;
1395	const branch = self.air.instructions.items(.data)[@backingInt(inst)].br;
1396	const block = self.blocks.get(branch.block_inst).?;
1397
1398	// Add the values to the lists only if the break provides a value.
1399	const operand_ty = self.typeOf(branch.operand);
1400	if (operand_ty.hasRuntimeBits(zcu)) {
1401	const val = try self.resolveInst(branch.operand);
1402
1403	// For the phi node, we need the basic blocks and the values of the
1404	// break instructions.
1405	try block.breaks.list.append(self.gpa, .{ .bb = self.wip.cursor.block, .val = val });
1406	} else block.breaks.len += 1;
1407	_ = try self.wip.br(block.parent_bb);
1408	}
1409
1410	fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
1411	const repeat = self.air.instructions.items(.data)[@backingInt(inst)].repeat;
1412	const loop_bb = self.loops.get(repeat.loop_inst).?;
1413	loop_bb.ptr(&self.wip).incoming += 1;
1414	_ = try self.wip.br(loop_bb);
1415	}
1416
1417	fn lowerSwitchDispatch(
1418	self: *FuncGen,
1419	switch_inst: Air.Inst.Index,
1420	cond_ref: Air.Inst.Ref,
1421	dispatch_info: SwitchDispatchInfo,
1422	) Allocator.Error!void {
1423	const o = self.object;
1424	const zcu = o.zcu;
1425	const cond_ty = self.typeOf(cond_ref);
1426	const switch_br = self.air.unwrapSwitch(switch_inst);
1427
1428	if (cond_ref.toInterned()) |cond_ip_index| {
1429	const cond_val: Value = .fromInterned(cond_ip_index);
1430	// Comptime-known dispatch. Iterate the cases to find the correct
1431	// one, and branch to the corresponding element of `case_blocks`.
1432	var it = switch_br.iterateCases();
1433	const target_case_idx = target: while (it.next()) |case| {
1434	for (case.items) |item| {
1435	const val = Value.fromInterned(item.toInterned().?);
1436	if (cond_val.compareHetero(.eq, val, zcu)) break :target case.idx;
1437	}
1438	for (case.ranges) |range| {
1439	const low = Value.fromInterned(range[0].toInterned().?);
1440	const high = Value.fromInterned(range[1].toInterned().?);
1441	if (cond_val.compareHetero(.gte, low, zcu) and
1442	cond_val.compareHetero(.lte, high, zcu))
1443	{
1444	break :target case.idx;
1445	}
1446	}
1447	} else dispatch_info.case_blocks.len - 1;
1448	const target_block = dispatch_info.case_blocks[target_case_idx];
1449	target_block.ptr(&self.wip).incoming += 1;
1450	_ = try self.wip.br(target_block);
1451	return;
1452	}
1453
1454	// Runtime-known dispatch.
1455	const cond = try self.resolveInst(cond_ref);
1456
1457	if (dispatch_info.jmp_table) |jmp_table| {
1458	// We should use the constructed jump table.
1459	// First, check the bounds to branch to the `else` case if needed.
1460	const inbounds = try self.wip.bin(
1461	.@"and",
1462	try self.cmp(.normal, .gte, cond_ty, cond, jmp_table.min.toValue()),
1463	try self.cmp(.normal, .lte, cond_ty, cond, jmp_table.max.toValue()),
1464	"",
1465	);
1466	const jmp_table_block = try self.wip.block(1, "Then");
1467	const else_block = dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1];
1468	else_block.ptr(&self.wip).incoming += 1;
1469	_ = try self.wip.brCond(inbounds, jmp_table_block, else_block, switch (jmp_table.in_bounds_hint) {
1470	.none => .none,
1471	.unpredictable => .unpredictable,
1472	.likely => .then_likely,
1473	.unlikely => .else_likely,
1474	});
1475
1476	self.wip.cursor = .{ .block = jmp_table_block };
1477
1478	// Figure out the list of blocks we might branch to.
1479	// This includes all case blocks, but it might not include the `else` block if
1480	// the table is dense.
1481	const target_blocks_len = dispatch_info.case_blocks.len - @intFromBool(!jmp_table.table_includes_else);
1482	const target_blocks = dispatch_info.case_blocks[0..target_blocks_len];
1483
1484	// Make sure to cast the index to a usize so it's not treated as negative!
1485	const table_index = try self.wip.conv(
1486	.unsigned,
1487	try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),
1488	try o.lowerType(.usize, .as_value),
1489	"",
1490	);
1491	const target_ptr_ptr = try self.ptraddScaled(
1492	jmp_table.table.toValue(),
1493	table_index,
1494	Type.usize.abiSize(zcu),
1495	);
1496	const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, "");
1497
1498	// Do the branch!
1499	_ = try self.wip.indirectbr(target_ptr, target_blocks);
1500
1501	// Mark all target blocks as having one more incoming branch.
1502	for (target_blocks) |case_block| {
1503	case_block.ptr(&self.wip).incoming += 1;
1504	}
1505
1506	return;
1507	}
1508
1509	// We must lower to an actual LLVM `switch` instruction.
1510	// The switch prongs will correspond to our scalar cases. Ranges will
1511	// be handled by conditional branches in the `else` prong.
1512
1513	const llvm_usize = try o.lowerType(.usize, .as_value);
1514	const cond_int = if (cond_ty.zigTypeTag(zcu) == .pointer)
1515	try self.wip.cast(.ptrtoint, cond, llvm_usize, "")
1516	else
1517	cond;
1518
1519	const llvm_cases_len, const last_range_case = info: {
1520	var llvm_cases_len: u32 = 0;
1521	var last_range_case: ?u32 = null;
1522	var it = switch_br.iterateCases();
1523	while (it.next()) |case| {
1524	if (case.ranges.len > 0) last_range_case = case.idx;
1525	llvm_cases_len += @intCast(case.items.len);
1526	}
1527	break :info .{ llvm_cases_len, last_range_case };
1528	};
1529
1530	// The `else` of the LLVM `switch` is the actual `else` prong only
1531	// if there are no ranges. Otherwise, the `else` will have a
1532	// conditional chain before the "true" `else` prong.
1533	const llvm_else_block = if (last_range_case == null)
1534	dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
1535	else
1536	try self.wip.block(0, "RangeTest");
1537
1538	llvm_else_block.ptr(&self.wip).incoming += 1;
1539
1540	var wip_switch = try self.wip.@"switch"(cond_int, llvm_else_block, llvm_cases_len, dispatch_info.switch_weights);
1541	defer wip_switch.finish(&self.wip);
1542
1543	// Construct the actual cases. Set the cursor to the `else` block so
1544	// we can construct ranges at the same time as scalar cases.
1545	self.wip.cursor = .{ .block = llvm_else_block };
1546
1547	var it = switch_br.iterateCases();
1548	while (it.next()) |case| {
1549	const case_block = dispatch_info.case_blocks[case.idx];
1550
1551	for (case.items) |item| {
1552	const llvm_item = (try self.resolveInst(item)).toConst().?;
1553	const llvm_int_item = if (cond_ty.zigTypeTag(zcu) == .pointer)
1554	try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize)
1555	else
1556	llvm_item;
1557	try wip_switch.addCase(llvm_int_item, case_block, &self.wip);
1558	}
1559	case_block.ptr(&self.wip).incoming += @intCast(case.items.len);
1560
1561	if (case.ranges.len == 0) continue;
1562
1563	// Add a conditional for the ranges, directing to the relevant bb.
1564	// We don't need to consider `cold` branch hints since that information is stored
1565	// in the target bb body, but we do care about likely/unlikely/unpredictable.
1566
1567	const hint = switch_br.getHint(case.idx);
1568
1569	var range_cond: ?Builder.Value = null;
1570	for (case.ranges) |range| {
1571	const llvm_min = try self.resolveInst(range[0]);
1572	const llvm_max = try self.resolveInst(range[1]);
1573	const cond_part = try self.wip.bin(
1574	.@"and",
1575	try self.cmp(.normal, .gte, cond_ty, cond, llvm_min),
1576	try self.cmp(.normal, .lte, cond_ty, cond, llvm_max),
1577	"",
1578	);
1579	if (range_cond) |prev| {
1580	range_cond = try self.wip.bin(.@"or", prev, cond_part, "");
1581	} else range_cond = cond_part;
1582	}
1583
1584	// If the check fails, we either branch to the "true" `else` case,
1585	// or to the next range condition.
1586	const range_else_block = if (case.idx == last_range_case.?)
1587	dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
1588	else
1589	try self.wip.block(0, "RangeTest");
1590
1591	_ = try self.wip.brCond(range_cond.?, case_block, range_else_block, switch (hint) {
1592	.none, .cold => .none,
1593	.unpredictable => .unpredictable,
1594	.likely => .then_likely,
1595	.unlikely => .else_likely,
1596	});
1597	case_block.ptr(&self.wip).incoming += 1;
1598	range_else_block.ptr(&self.wip).incoming += 1;
1599
1600	// Construct the next range conditional (if any) in the false branch.
1601	self.wip.cursor = .{ .block = range_else_block };
1602	}
1603	}
1604
1605	fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
1606	const br = self.air.instructions.items(.data)[@backingInt(inst)].br;
1607	const dispatch_info = self.switch_dispatch_info.get(br.block_inst).?;
1608	return self.lowerSwitchDispatch(br.block_inst, br.operand, dispatch_info);
1609	}
1610
1611	fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) TodoError!void {
1612	const cond_br = self.air.unwrapCondBr(inst);
1613	const cond = try self.resolveInst(cond_br.condition);
1614	const then_body = cond_br.then_body;
1615	const else_body = cond_br.else_body;
1616
1617	const Hint = enum {
1618	none,
1619	unpredictable,
1620	then_likely,
1621	else_likely,
1622	then_cold,
1623	else_cold,
1624	};
1625	const hint: Hint = switch (cond_br.branch_hints.true) {
1626	.none => switch (cond_br.branch_hints.false) {
1627	.none => .none,
1628	.likely => .else_likely,
1629	.unlikely => .then_likely,
1630	.cold => .else_cold,
1631	.unpredictable => .unpredictable,
1632	},
1633	.likely => switch (cond_br.branch_hints.false) {
1634	.none => .then_likely,
1635	.likely => .unpredictable,
1636	.unlikely => .then_likely,
1637	.cold => .else_cold,
1638	.unpredictable => .unpredictable,
1639	},
1640	.unlikely => switch (cond_br.branch_hints.false) {
1641	.none => .else_likely,
1642	.likely => .else_likely,
1643	.unlikely => .unpredictable,
1644	.cold => .else_cold,
1645	.unpredictable => .unpredictable,
1646	},
1647	.cold => .then_cold,
1648	.unpredictable => .unpredictable,
1649	};
1650
1651	const then_block = try self.wip.block(1, "Then");
1652	const else_block = try self.wip.block(1, "Else");
1653	_ = try self.wip.brCond(cond, then_block, else_block, switch (hint) {
1654	.none, .then_cold, .else_cold => .none,
1655	.unpredictable => .unpredictable,
1656	.then_likely => .then_likely,
1657	.else_likely => .else_likely,
1658	});
1659
1660	self.wip.cursor = .{ .block = then_block };
1661	if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold();
1662	try self.genBodyDebugScope(null, then_body, cond_br.branch_hints.then_cov);
1663
1664	self.wip.cursor = .{ .block = else_block };
1665	if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold();
1666	try self.genBodyDebugScope(null, else_body, cond_br.branch_hints.else_cov);
1667
1668	// No need to reset the insert cursor since this instruction is noreturn.
1669	}
1670
1671	fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) TodoError!Builder.Value {
1672	const unwrapped_try = self.air.unwrapTry(inst);
1673	const err_union = try self.resolveInst(unwrapped_try.error_union);
1674	const body = unwrapped_try.else_body;
1675	const err_union_ty = self.typeOf(unwrapped_try.error_union);
1676	const is_unused = self.liveness.isUnused(inst);
1677	return lowerTry(self, err_union, body, err_union_ty, false, .none, is_unused, err_cold);
1678	}
1679
1680	fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) TodoError!Builder.Value {
1681	const zcu = self.object.zcu;
1682	const unwrapped_try = self.air.unwrapTryPtr(inst);
1683	const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr);
1684	const body = unwrapped_try.else_body;
1685	const err_union_ptr_ty = self.typeOf(unwrapped_try.error_union_ptr);
1686	const err_union_ty = err_union_ptr_ty.childType(zcu);
1687	const is_unused = self.liveness.isUnused(inst);
1688
1689	self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu));
1690
1691	return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), is_unused, err_cold);
1692	}
1693
1694	fn lowerTry(
1695	fg: *FuncGen,
1696	err_union: Builder.Value,
1697	body: []const Air.Inst.Index,
1698	err_union_ty: Type,
1699	operand_is_ptr: bool,
1700	operand_ptr_align: InternPool.Alignment,
1701	is_unused: bool,
1702	err_cold: bool,
1703	) TodoError!Builder.Value {
1704	const o = fg.object;
1705	const zcu = o.zcu;
1706	const payload_ty = err_union_ty.errorUnionPayload(zcu);
1707	const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
1708
1709	const operand_align: InternPool.Alignment = if (operand_is_ptr) operand_ptr_align else err_union_ty.abiAlignment(zcu);
1710
1711	if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
1712	const loaded = loaded: {
1713	if (payload_has_bits) {
1714	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1715	} else if (!operand_is_ptr) {
1716	break :loaded err_union;
1717	}
1718
1719	const offset = codegen.errUnionErrorOffset(payload_ty, zcu);
1720	const err_field_ptr = try fg.ptraddConst(err_union, offset);
1721	break :loaded try fg.load(
1722	err_field_ptr,
1723	operand_align.offset(offset),
1724	.anyerror,
1725	if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
1726	);
1727	};
1728	const zero = try o.builder.intValue(try o.errorIntType(.as_value), 0);
1729	const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
1730
1731	const return_block = try fg.wip.block(1, "TryRet");
1732	const continue_block = try fg.wip.block(1, "TryCont");
1733	_ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely);
1734
1735	fg.wip.cursor = .{ .block = return_block };
1736	if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold();
1737	try fg.genBodyDebugScope(null, body, .poi);
1738
1739	fg.wip.cursor = .{ .block = continue_block };
1740	}
1741	if (is_unused) return .none;
1742
1743	if (!operand_is_ptr) {
1744	assert(payload_has_bits); // otherwise the result should be comptime-known
1745	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1746	}
1747
1748	const offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
1749	const payload_ptr = try fg.ptraddConst(err_union, offset);
1750	if (operand_is_ptr) {
1751	return payload_ptr;
1752	} else {
1753	return fg.load(payload_ptr, operand_align.offset(offset), payload_ty, .normal);
1754	}
1755	}
1756
1757	fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) TodoError!void {
1758	const o = self.object;
1759	const zcu = o.zcu;
1760
1761	const switch_br = self.air.unwrapSwitch(inst);
1762
1763	// For `loop_switch_br`, we need these BBs prepared ahead of time to generate dispatches.
1764	// For `switch_br`, they allow us to sometimes generate better IR by sharing a BB between
1765	// scalar and range cases in the same prong.
1766	// +1 for `else` case. This is not the same as the LLVM `else` prong, as that may first contain
1767	// conditionals to handle ranges.
1768	const case_blocks = try self.gpa.alloc(Builder.Function.Block.Index, switch_br.cases_len + 1);
1769	defer self.gpa.free(case_blocks);
1770	// We set incoming as 0 for now, and increment it as we construct dispatches.
1771	for (case_blocks[0 .. case_blocks.len - 1]) |*b| b.* = try self.wip.block(0, "Case");
1772	case_blocks[case_blocks.len - 1] = try self.wip.block(0, "Default");
1773
1774	// There's a special case here to manually generate a jump table in some cases.
1775	//
1776	// Labeled switch in Zig is intended to follow the "direct threading" pattern. We would ideally use a jump
1777	// table, and each `continue` has its own indirect `jmp`, to allow the branch predictor to more accurately
1778	// use data patterns to predict future dispatches. The problem, however, is that LLVM emits fascinatingly
1779	// bad asm for this. Not only does it not share the jump table -- which we really need it to do to prevent
1780	// destroying the cache -- but it also actually generates slightly different jump tables for each case,
1781	// and *a separate conditional branch beforehand* to handle dispatching back to the case we're currently
1782	// within(!!).
1783	//
1784	// This asm is really, really, not what we want. As such, we will construct the jump table manually where
1785	// appropriate (the values are dense and relatively few), and use it when lowering dispatches.
1786
1787	const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: {
1788	if (!is_dispatch_loop) break :jmp_table null;
1789
1790	// Workaround for:
1791	// * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/lib/MC/WasmObjectWriter.cpp#L560
1792	// * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll
1793	if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null;
1794
1795	// On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just
1796	// about acceptable - it won't fill L1d cache on most CPUs.
1797	const max_table_len = 1024;
1798
1799	const cond_ty = self.typeOf(switch_br.operand);
1800	switch (cond_ty.zigTypeTag(zcu)) {
1801	.bool, .pointer => break :jmp_table null,
1802	.@"enum", .int, .error_set, .@"struct", .@"union" => {},
1803	else => unreachable,
1804	}
1805
1806	if (cond_ty.intInfo(zcu).signedness == .signed) break :jmp_table null;
1807
1808	// Don't worry about the size of the type -- it's irrelevant, because the prong values could be fairly dense.
1809	// If they are, then we will construct a jump table.
1810	const min, const max = self.switchCaseItemRange(switch_br) orelse break :jmp_table null;
1811	const min_int = min.getUnsignedInt(zcu) orelse break :jmp_table null;
1812	const max_int = max.getUnsignedInt(zcu) orelse break :jmp_table null;
1813	const table_len = max_int - min_int + 1;
1814	if (table_len > max_table_len) break :jmp_table null;
1815
1816	const table_elems = try self.gpa.alloc(Builder.Constant, @intCast(table_len));
1817	defer self.gpa.free(table_elems);
1818
1819	// Set them all to the `else` branch, then iterate over the AIR switch
1820	// and replace all values which correspond to other prongs.
1821	@memset(table_elems, try o.builder.blockAddrConst(
1822	self.wip.function,
1823	case_blocks[case_blocks.len - 1],
1824	));
1825	var item_count: u32 = 0;
1826	var it = switch_br.iterateCases();
1827	while (it.next()) |case| {
1828	const case_block = case_blocks[case.idx];
1829	const case_block_addr = try o.builder.blockAddrConst(
1830	self.wip.function,
1831	case_block,
1832	);
1833	for (case.items) |item| {
1834	const val = Value.fromInterned(item.toInterned().?);
1835	const table_idx = val.toUnsignedInt(zcu) - min_int;
1836	table_elems[@intCast(table_idx)] = case_block_addr;
1837	item_count += 1;
1838	}
1839	for (case.ranges) |range| {
1840	const low = Value.fromInterned(range[0].toInterned().?);
1841	const high = Value.fromInterned(range[1].toInterned().?);
1842	const low_idx = low.toUnsignedInt(zcu) - min_int;
1843	const high_idx = high.toUnsignedInt(zcu) - min_int;
1844	@memset(table_elems[@intCast(low_idx)..@intCast(high_idx + 1)], case_block_addr);
1845	item_count += @intCast(high_idx + 1 - low_idx);
1846	}
1847	}
1848
1849	const table_llvm_ty = try o.builder.arrayType(table_elems.len, .ptr);
1850	const table_val = try o.builder.arrayConst(table_llvm_ty, table_elems);
1851
1852	const table_variable = try o.builder.addVariable(
1853	try o.builder.strtabStringFmt("__jmptab_{d}", .{@backingInt(inst)}),
1854	table_llvm_ty,
1855	.default,
1856	);
1857	try table_variable.setInitializer(table_val, &o.builder);
1858	const table_global = table_variable.ptrConst(&o.builder).global;
1859	table_global.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
1860	table_global.setUnnamedAddr(.unnamed_addr, &o.builder);
1861
1862	const table_includes_else = item_count != table_len;
1863
1864	break :jmp_table .{
1865	.min = try o.lowerValue(min.toIntern(), .as_value),
1866	.max = try o.lowerValue(max.toIntern(), .as_value),
1867	.in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {
1868	.none, .cold => .none,
1869	.unpredictable => .unpredictable,
1870	.likely => .likely,
1871	.unlikely => .unlikely,
1872	},
1873	.table = table_global.toConst(),
1874	.table_includes_else = table_includes_else,
1875	};
1876	};
1877
1878	const weights: Builder.Function.Instruction.BrCond.Weights = weights: {
1879	if (jmp_table != null) break :weights .none; // not used
1880
1881	// First pass. If any weights are `.unpredictable`, unpredictable.
1882	// If all are `.none` or `.cold`, none.
1883	var any_likely = false;
1884	for (0..switch_br.cases_len) |case_idx| {
1885	switch (switch_br.getHint(@intCast(case_idx))) {
1886	.none, .cold => {},
1887	.likely, .unlikely => any_likely = true,
1888	.unpredictable => break :weights .unpredictable,
1889	}
1890	}
1891	switch (switch_br.getElseHint()) {
1892	.none, .cold => {},
1893	.likely, .unlikely => any_likely = true,
1894	.unpredictable => break :weights .unpredictable,
1895	}
1896	if (!any_likely) break :weights .none;
1897
1898	const llvm_cases_len = llvm_cases_len: {
1899	var len: u32 = 0;
1900	var it = switch_br.iterateCases();
1901	while (it.next()) |case| len += @intCast(case.items.len);
1902	break :llvm_cases_len len;
1903	};
1904
1905	var weights = try self.gpa.alloc(Builder.Metadata, 1 + llvm_cases_len + 1);
1906	defer self.gpa.free(weights);
1907	var weight_idx: usize = 0;
1908
1909	const branch_weights_str = try o.builder.metadataString("branch_weights");
1910	weights[weight_idx] = branch_weights_str.toMetadata();
1911	weight_idx += 1;
1912
1913	const else_weight: u32 = switch (switch_br.getElseHint()) {
1914	.unpredictable => unreachable,
1915	.none, .cold => 1000,
1916	.likely => 2000,
1917	.unlikely => 1,
1918	};
1919	weights[weight_idx] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight));
1920	weight_idx += 1;
1921
1922	var it = switch_br.iterateCases();
1923	while (it.next()) |case| {
1924	const weight_val: u32 = switch (switch_br.getHint(case.idx)) {
1925	.unpredictable => unreachable,
1926	.none, .cold => 1000,
1927	.likely => 2000,
1928	.unlikely => 1,
1929	};
1930	const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val));
1931	@memset(weights[weight_idx..][0..case.items.len], weight_meta);
1932	weight_idx += case.items.len;
1933	}
1934
1935	assert(weight_idx == weights.len);
1936	break :weights .fromMetadata(try o.builder.metadataTuple(weights));
1937	};
1938
1939	const dispatch_info: SwitchDispatchInfo = .{
1940	.case_blocks = case_blocks,
1941	.switch_weights = weights,
1942	.jmp_table = jmp_table,
1943	};
1944
1945	if (is_dispatch_loop) {
1946	try self.switch_dispatch_info.putNoClobber(self.gpa, inst, dispatch_info);
1947	}
1948	defer if (is_dispatch_loop) {
1949	assert(self.switch_dispatch_info.remove(inst));
1950	};
1951
1952	// Generate the initial dispatch.
1953	// If this is a simple `switch_br`, this is the only dispatch.
1954	try self.lowerSwitchDispatch(inst, switch_br.operand, dispatch_info);
1955
1956	// Iterate the cases and generate their bodies.
1957	var it = switch_br.iterateCases();
1958	while (it.next()) |case| {
1959	const case_block = case_blocks[case.idx];
1960	self.wip.cursor = .{ .block = case_block };
1961	if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold();
1962	try self.genBodyDebugScope(null, case.body, .none);
1963	}
1964	self.wip.cursor = .{ .block = case_blocks[case_blocks.len - 1] };
1965	const else_body = it.elseBody();
1966	if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold();
1967	if (else_body.len > 0) {
1968	try self.genBodyDebugScope(null, it.elseBody(), .none);
1969	} else {
1970	_ = try self.wip.@"unreachable"();
1971	}
1972	}
1973
1974	fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) ?[2]Value {
1975	const zcu = self.object.zcu;
1976	var it = switch_br.iterateCases();
1977	var min: ?Value = null;
1978	var max: ?Value = null;
1979	while (it.next()) |case| {
1980	for (case.items) |item| {
1981	const val = Value.fromInterned(item.toInterned().?);
1982	const low = if (min) |m| val.compareHetero(.lt, m, zcu) else true;
1983	const high = if (max) |m| val.compareHetero(.gt, m, zcu) else true;
1984	if (low) min = val;
1985	if (high) max = val;
1986	}
1987	for (case.ranges) |range| {
1988	const vals: [2]Value = .{
1989	Value.fromInterned(range[0].toInterned().?),
1990	Value.fromInterned(range[1].toInterned().?),
1991	};
1992	const low = if (min) |m| vals[0].compareHetero(.lt, m, zcu) else true;
1993	const high = if (max) |m| vals[1].compareHetero(.gt, m, zcu) else true;
1994	if (low) min = vals[0];
1995	if (high) max = vals[1];
1996	}
1997	}
1998	if (min == null) {
1999	assert(max == null);
2000	return null;
2001	}
2002	return .{ min.?, max.? };
2003	}
2004
2005	fn airLoop(self: *FuncGen, inst: Air.Inst.Index) TodoError!void {
2006	const block = self.air.unwrapBlock(inst);
2007	const body = block.body;
2008	const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time
2009	_ = try self.wip.br(loop_block);
2010
2011	try self.loops.putNoClobber(self.gpa, inst, loop_block);
2012	defer assert(self.loops.remove(inst));
2013
2014	self.wip.cursor = .{ .block = loop_block };
2015	try self.genBodyDebugScope(null, body, .none);
2016	}
2017
2018	fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2019	const o = self.object;
2020	const zcu = o.zcu;
2021	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2022	const operand_ty = self.typeOf(ty_op.operand);
2023	const array_ty = operand_ty.childType(zcu);
2024	const llvm_usize = try o.lowerType(.usize, .as_value);
2025	const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
2026	const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst), .as_value);
2027	const operand = try self.resolveInst(ty_op.operand);
2028	return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
2029	}
2030
2031	fn airArrayToVector(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2032	const o = fg.object;
2033	const zcu = o.zcu;
2034	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2035	const array_ty = fg.typeOf(ty_op.operand);
2036	const vector_ty = fg.typeOfIndex(inst);
2037	const elem_ty = vector_ty.childType(zcu);
2038	const operand = try fg.resolveInst(ty_op.operand);
2039
2040	assert(array_ty.arrayLen(zcu) == vector_ty.vectorLen(zcu));
2041	assert(array_ty.childType(zcu).toIntern() == elem_ty.toIntern());
2042	assert(isByRef(array_ty, zcu)); // the operand is runtime-known, so the array has runtime bits
2043
2044	// A by-ref vector is lowered as `[n x T]` with the same element representation as the array,
2045	// so the operand is already the result.
2046	if (isByRef(vector_ty, zcu)) return operand;
2047
2048	// LLVM lays `<n x T>` out as `n` consecutive `T`s, just like `[n]T`, so long as `T` is
2049	// accessed as the same type it is used as; then this is one load.
2050	if ((try o.lowerType(elem_ty, .memory_access)) == (try o.lowerType(elem_ty, .as_value)) and
2051	// f80 has an unusual in-memory representation with padding bytes, so is
2052	// not eligible for this optimization
2053	!(elem_ty.isRuntimeFloat() and elem_ty.floatBits(zcu.getTarget()) == 80))
2054	{
2055	return fg.load(operand, array_ty.abiAlignment(zcu), vector_ty, .normal);
2056	}
2057
2058	const llvm_usize = try o.lowerType(.usize, .as_value);
2059	const elem_size = elem_ty.abiSize(zcu);
2060	var vector = try o.builder.poisonValue(try o.lowerType(vector_ty, .as_value));
2061	for (0..@intCast(vector_ty.vectorLen(zcu))) |elem_index| {
2062	const elem_ptr = try fg.ptraddScaled(
2063	operand,
2064	try o.builder.intValue(llvm_usize, elem_index),
2065	elem_size,
2066	);
2067	const elem = try fg.load(elem_ptr, .none, elem_ty, .normal);
2068	vector = try fg.wip.insertElement(vector, elem, try o.builder.intValue(.i32, elem_index), "");
2069	}
2070	return vector;
2071	}
2072
2073	fn airFloatFromInt(fg: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2074	const o = fg.object;
2075	const zcu = o.zcu;
2076	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2077
2078	const operand = try fg.resolveInst(ty_op.operand);
2079	const operand_ty = fg.typeOf(ty_op.operand);
2080	const operand_scalar_ty = operand_ty.scalarType(zcu);
2081	const operand_scalar_info = operand_scalar_ty.intInfo(zcu);
2082
2083	const dest_ty = fg.typeOfIndex(inst);
2084	const dest_scalar_ty = dest_ty.scalarType(zcu);
2085	const target = zcu.getTarget();
2086
2087	if (intrinsicsAllowed(.compiler_rt, dest_scalar_ty, target))
2088	return fg.wip.conv(.fromStdLang(operand_scalar_info.signedness), operand, try o.lowerType(dest_ty, .as_value), "");
2089
2090	const rt_int_ty = compilerRtPromoteInt(operand_scalar_info) orelse {
2091	return fg.todo("float_from_int on {d} bit integer", .{operand_scalar_info.bits});
2092	};
2093	const vector_len = if (operand_ty.isVector(zcu)) operand_ty.vectorLen(zcu) else null;
2094	const rt_llvm_int_ty = try o.lowerType(rt_int_ty, .as_value);
2095	const extended = try fg.wip.conv(
2096	.fromStdLang(operand_scalar_info.signedness),
2097	operand,
2098	if (vector_len) |len|
2099	try o.builder.vectorType(.normal, len, rt_llvm_int_ty)
2100	else
2101	rt_llvm_int_ty,
2102	"",
2103	);
2104	const fn_name = try o.builder.strtabStringFmt("__float{s}{s}i{s}f", .{
2105	switch (operand_scalar_info.signedness) {
2106	.signed => "",
2107	.unsigned => "un",
2108	},
2109	compilerRtIntAbbrev(rt_int_ty.intInfo(zcu).bits),
2110	compilerRtFloatAbbrev(target, dest_scalar_ty.floatBits(target)),
2111	});
2112	return fg.buildElementwiseCall(fn_name, .{
2113	.cc = target.cCallingConvention().?,
2114	.param_types = &.{rt_int_ty.toIntern()},
2115	.return_type = dest_scalar_ty.toIntern(),
2116	}, &.{extended}, vector_len);
2117	}
2118
2119	fn airIntFromFloat(
2120	fg: *FuncGen,
2121	inst: Air.Inst.Index,
2122	fast: Builder.FastMathKind,
2123	) TodoError!Builder.Value {
2124	_ = fast;
2125
2126	const o = fg.object;
2127	const zcu = o.zcu;
2128	const target = zcu.getTarget();
2129	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2130
2131	const operand = try fg.resolveInst(ty_op.operand);
2132	const operand_ty = fg.typeOf(ty_op.operand);
2133	const operand_scalar_ty = operand_ty.scalarType(zcu);
2134
2135	const dest_ty = fg.typeOfIndex(inst);
2136	const dest_scalar_ty = dest_ty.scalarType(zcu);
2137	const dest_llvm_ty = try o.lowerType(dest_ty, .as_value);
2138	const dest_scalar_info = dest_scalar_ty.intInfo(zcu);
2139
2140	if (intrinsicsAllowed(.compiler_rt, operand_scalar_ty, target)) {
2141	// TODO set fast math flag
2142	return fg.wip.conv(.fromStdLang(dest_scalar_info.signedness), operand, dest_llvm_ty, "");
2143	}
2144
2145	const rt_int_ty = compilerRtPromoteInt(dest_scalar_info) orelse {
2146	return fg.todo("int_from_float to {d} bit integer", .{dest_scalar_info.bits});
2147	};
2148	const fn_name = try o.builder.strtabStringFmt("__fix{s}{s}f{s}i", .{
2149	switch (dest_scalar_info.signedness) {
2150	.signed => "",
2151	.unsigned => "uns",
2152	},
2153	compilerRtFloatAbbrev(target, operand_scalar_ty.floatBits(target)),
2154	compilerRtIntAbbrev(rt_int_ty.intInfo(zcu).bits),
2155	});
2156	const result = try fg.buildElementwiseCall(fn_name, .{
2157	.cc = target.cCallingConvention().?,
2158	.param_types = &.{operand_scalar_ty.toIntern()},
2159	.return_type = rt_int_ty.toIntern(),
2160	}, &.{operand}, if (operand_ty.isVector(zcu)) operand_ty.vectorLen(zcu) else null);
2161	return fg.wip.cast(.trunc, result, try o.lowerType(dest_ty, .as_value), "");
2162	}
2163
2164	fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
2165	const zcu = fg.object.zcu;
2166	return if (ty.isSlice(zcu)) fg.wip.extractValue(ptr, &.{0}, "") else ptr;
2167	}
2168
2169	fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
2170	const o = fg.object;
2171	const zcu = o.zcu;
2172	const llvm_usize = try o.lowerType(.usize, .as_value);
2173	switch (ty.ptrSize(zcu)) {
2174	.slice => {
2175	const len = try fg.wip.extractValue(ptr, &.{1}, "");
2176	const elem_ty = ty.childType(zcu);
2177	const abi_size = elem_ty.abiSize(zcu);
2178	if (abi_size == 1) return len;
2179	const abi_size_llvm_val = try o.builder.intValue(llvm_usize, abi_size);
2180	return fg.wip.bin(.@"mul nuw", len, abi_size_llvm_val, "");
2181	},
2182	.one => {
2183	const array_ty = ty.childType(zcu);
2184	const elem_ty = array_ty.childType(zcu);
2185	const abi_size = elem_ty.abiSize(zcu);
2186	return o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu) * abi_size);
2187	},
2188	.many, .c => unreachable,
2189	}
2190	}
2191
2192	fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) Allocator.Error!Builder.Value {
2193	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2194	const operand = try self.resolveInst(ty_op.operand);
2195	return self.wip.extractValue(operand, &.{index}, "");
2196	}
2197
2198	fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: u1) Allocator.Error!Builder.Value {
2199	const zcu = self.object.zcu;
2200	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2201	const slice_ptr = try self.resolveInst(ty_op.operand);
2202	return self.ptraddConst(slice_ptr, index * Type.usize.abiSize(zcu));
2203	}
2204
2205	fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2206	const zcu = self.object.zcu;
2207	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2208	const slice_ty = self.typeOf(bin_op.lhs);
2209	const slice = try self.resolveInst(bin_op.lhs);
2210	const index = try self.resolveInst(bin_op.rhs);
2211	const slice_info = slice_ty.ptrInfo(zcu);
2212	assert(slice_info.flags.size == .slice);
2213	const elem_ty: Type = .fromInterned(slice_info.child);
2214	const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
2215	const ptr = try self.ptraddScaled(base_ptr, index, elem_ty.abiSize(zcu));
2216	const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
2217	const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
2218	self.maybeMarkAllowZeroAccess(slice_info);
2219	return self.load(ptr, elem_align, elem_ty, access_kind);
2220	}
2221
2222	fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2223	const zcu = self.object.zcu;
2224	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2225	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
2226	const slice_ty = self.typeOf(bin_op.lhs);
2227
2228	const slice = try self.resolveInst(bin_op.lhs);
2229	const index = try self.resolveInst(bin_op.rhs);
2230	const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
2231	return self.ptraddScaled(base_ptr, index, slice_ty.childType(zcu).abiSize(zcu));
2232	}
2233
2234	fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2235	const zcu = self.object.zcu;
2236
2237	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2238	const array_ty = self.typeOf(bin_op.lhs);
2239	const array_llvm_val = try self.resolveInst(bin_op.lhs);
2240	const rhs = try self.resolveInst(bin_op.rhs);
2241	const elem_ty = array_ty.childType(zcu);
2242	if (isByRef(array_ty, zcu)) {
2243	const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu));
2244	return self.load(elem_ptr, .none, elem_ty, .normal);
2245	}
2246
2247	// This branch can be reached for vectors, which are always by-value.
2248	return self.wip.extractElement(array_llvm_val, rhs, "");
2249	}
2250
2251	fn airLegalizeVecElemVal(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2252	const bin_op = fg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2253	const vec = try fg.resolveInst(bin_op.lhs);
2254	const index = try fg.resolveInst(bin_op.rhs);
2255	return fg.wip.extractElement(vec, index, "");
2256	}
2257	fn airLegalizeVecStoreElem(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2258	const zcu = fg.object.zcu;
2259
2260	const pl_op = fg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
2261	const extra = fg.air.extraData(Air.Bin, pl_op.payload).data;
2262
2263	const ptr_ty = fg.typeOf(pl_op.operand);
2264	const vec_ty = ptr_ty.childType(zcu);
2265
2266	const ptr_align = ptr_ty.ptrAlignment(zcu);
2267
2268	const vec_ptr = try fg.resolveInst(pl_op.operand);
2269	const index = try fg.resolveInst(extra.lhs);
2270	const elem = try fg.resolveInst(extra.rhs);
2271
2272	const old_vec = try fg.load(vec_ptr, ptr_align, vec_ty, .normal);
2273	const new_vec = try fg.wip.insertElement(old_vec, elem, index, "");
2274	try fg.store(vec_ptr, ptr_align, new_vec, vec_ty, .normal);
2275
2276	return .none;
2277	}
2278
2279	fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2280	const zcu = self.object.zcu;
2281	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2282	const ptr_ty = self.typeOf(bin_op.lhs);
2283	const elem_ty = ptr_ty.indexableElem(zcu);
2284	const base_ptr = try self.resolveInst(bin_op.lhs);
2285	const rhs = try self.resolveInst(bin_op.rhs);
2286
2287	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
2288
2289	return self.load(
2290	try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)),
2291	ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)),
2292	elem_ty,
2293	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2294	);
2295	}
2296
2297	fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2298	const zcu = self.object.zcu;
2299	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2300	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
2301	const ptr_ty = self.typeOf(bin_op.lhs);
2302	const elem_ty = ptr_ty.indexableElem(zcu);
2303	assert(elem_ty.hasRuntimeBits(zcu));
2304
2305	const base_ptr = try self.resolveInst(bin_op.lhs);
2306	const rhs = try self.resolveInst(bin_op.rhs);
2307
2308	return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));
2309	}
2310
2311	fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2312	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2313	const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
2314	const struct_ptr = try self.resolveInst(struct_field.struct_operand);
2315	const struct_ptr_ty = self.typeOf(struct_field.struct_operand);
2316	return self.fieldPtr(struct_ptr, struct_ptr_ty, struct_field.field_index);
2317	}
2318
2319	fn airStructFieldPtrIndex(
2320	self: *FuncGen,
2321	inst: Air.Inst.Index,
2322	field_index: u32,
2323	) Allocator.Error!Builder.Value {
2324	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2325	const struct_ptr = try self.resolveInst(ty_op.operand);
2326	const struct_ptr_ty = self.typeOf(ty_op.operand);
2327	return self.fieldPtr(struct_ptr, struct_ptr_ty, field_index);
2328	}
2329
2330	fn airAggFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2331	const o = self.object;
2332	const zcu = o.zcu;
2333	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2334	const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
2335	const struct_ty = self.typeOf(struct_field.struct_operand);
2336	const struct_llvm_val = try self.resolveInst(struct_field.struct_operand);
2337	const field_index = struct_field.field_index;
2338	const field_ty = struct_ty.fieldType(field_index, zcu);
2339	assert(field_ty.hasRuntimeBits(zcu));
2340
2341	if (!isByRef(struct_ty, zcu)) {
2342	// All auto/extern struct/union types are by-ref, unless they have no runtime bits, in which
2343	// case we shouldn't be seeing this instruction to begin with. Therefore we must be dealing
2344	// with a `packed struct` or `packed union`.
2345	assert(struct_ty.containerLayout(zcu) == .@"packed");
2346	assert(!isByRef(field_ty, zcu));
2347	const field_int_val: Builder.Value = switch (struct_ty.zigTypeTag(zcu)) {
2348	.@"struct" => field_int_val: {
2349	const llvm_field_int_ty = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
2350	const bit_offset = zcu.structPackedFieldBitOffset(
2351	zcu.intern_pool.loadStructType(struct_ty.toIntern()),
2352	field_index,
2353	);
2354	const shift_bits = try o.builder.intValue(struct_llvm_val.typeOfWip(&self.wip), bit_offset);
2355	const shifted = try self.wip.bin(.lshr, struct_llvm_val, shift_bits, "");
2356	break :field_int_val try self.wip.cast(.trunc, shifted, llvm_field_int_ty, "");
2357	},
2358	.@"union" => struct_llvm_val,
2359	else => unreachable,
2360	};
2361	switch (field_ty.zigTypeTag(zcu)) {
2362	else => unreachable, // not packable
2363	.void => unreachable, // opv bug in sema
2364	.int, .bool, .@"enum", .@"struct", .@"union" => {
2365	// Represented as integers, so already done
2366	return field_int_val;
2367	},
2368	.float => {
2369	// bitcast int->float
2370	return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty, .as_value), "");
2371	},
2372	}
2373	}
2374
2375	const offset: u64 = switch (struct_ty.zigTypeTag(zcu)) {
2376	.@"struct" => struct_ty.structFieldOffset(field_index, zcu),
2377	.@"union" => struct_ty.unionGetLayout(zcu).payloadOffset(),
2378	else => unreachable,
2379	};
2380
2381	const struct_ptr_align = struct_ty.abiAlignment(zcu);
2382	const field_ptr = try self.ptraddConst(struct_llvm_val, offset);
2383	const field_ptr_align = struct_ptr_align.offset(offset);
2384
2385	return self.load(field_ptr, field_ptr_align, field_ty, .normal);
2386	}
2387
2388	fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2389	const o = self.object;
2390	const zcu = o.zcu;
2391	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
2392	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
2393
2394	const field_ptr = try self.resolveInst(extra.field_ptr);
2395
2396	const parent_ty = ty_pl.ty.childType(zcu);
2397	const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
2398	if (field_offset == 0) return field_ptr;
2399
2400	const res_ty = try o.lowerType(ty_pl.ty, .as_value);
2401	const llvm_usize = try o.lowerType(.usize, .as_value);
2402
2403	const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");
2404	const base_ptr_int = try self.wip.bin(
2405	.@"sub nuw",
2406	field_ptr_int,
2407	try o.builder.intValue(llvm_usize, field_offset),
2408	"",
2409	);
2410	return self.wip.cast(.inttoptr, base_ptr_int, res_ty, "");
2411	}
2412
2413	fn airNot(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2414	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2415	const operand = try self.resolveInst(ty_op.operand);
2416
2417	return self.wip.not(operand, "");
2418	}
2419
2420	fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
2421	_ = inst;
2422	_ = try self.wip.@"unreachable"();
2423	}
2424
2425	fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2426	const dbg_stmt = self.air.instructions.items(.data)[@backingInt(inst)].dbg_stmt;
2427	self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1);
2428	self.prev_dbg_column = @intCast(dbg_stmt.column + 1);
2429
2430	self.wip.debug_location = .{ .location = .{
2431	.line = self.prev_dbg_line,
2432	.column = self.prev_dbg_column,
2433	.scope = self.scope.toOptional(),
2434	.inlined_at = self.inlined_at,
2435	} };
2436
2437	return .none;
2438	}
2439
2440	fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2441	_ = self;
2442	_ = inst;
2443	return .none;
2444	}
2445
2446	fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2447	const o = self.object;
2448	const pt = self.pt;
2449	const zcu = o.zcu;
2450	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
2451	const operand = try self.resolveInst(pl_op.operand);
2452	const name: Air.NullTerminatedString = @fromBackingInt(@intCast(pl_op.payload));
2453	const ptr_ty = self.typeOf(pl_op.operand);
2454
2455	const debug_local_var = try o.builder.debugLocalVar(
2456	try o.builder.metadataString(name.toSlice(self.air)),
2457	self.file,
2458	self.scope,
2459	self.prev_dbg_line,
2460	try o.getDebugType(pt, ptr_ty.childType(zcu)),
2461	);
2462
2463	_ = try self.wip.callIntrinsic(
2464	.normal,
2465	.none,
2466	.@"dbg.declare",
2467	&.{},
2468	&.{
2469	(try self.wip.debugValue(operand)).toValue(),
2470	debug_local_var.toValue(),
2471	(try o.builder.debugExpression(&.{})).toValue(),
2472	},
2473	"",
2474	);
2475
2476	return .none;
2477	}
2478
2479	fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Error!Builder.Value {
2480	const o = self.object;
2481	const pt = self.pt;
2482	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
2483	const operand = try self.resolveInst(pl_op.operand);
2484	const operand_ty = self.typeOf(pl_op.operand);
2485	const name: Air.NullTerminatedString = @fromBackingInt(@intCast(pl_op.payload));
2486	const name_slice = name.toSlice(self.air);
2487	const metadata_name = if (name_slice.len > 0) try o.builder.metadataString(name_slice) else null;
2488	const debug_local_var = if (is_arg) try o.builder.debugParameter(
2489	metadata_name,
2490	self.file,
2491	self.scope,
2492	self.prev_dbg_line,
2493	try o.getDebugType(pt, operand_ty),
2494	arg_no: {
2495	self.arg_inline_index += 1;
2496	break :arg_no self.arg_inline_index;
2497	},
2498	) else try o.builder.debugLocalVar(
2499	metadata_name,
2500	self.file,
2501	self.scope,
2502	self.prev_dbg_line,
2503	try o.getDebugType(pt, operand_ty),
2504	);
2505
2506	const zcu = o.zcu;
2507	const owner_mod = self.ownerModule();
2508	if (isByRef(operand_ty, zcu)) {
2509	_ = try self.wip.callIntrinsic(
2510	.normal,
2511	.none,
2512	.@"dbg.declare",
2513	&.{},
2514	&.{
2515	(try self.wip.debugValue(operand)).toValue(),
2516	debug_local_var.toValue(),
2517	(try o.builder.debugExpression(&.{})).toValue(),
2518	},
2519	"",
2520	);
2521	} else if (owner_mod.optimize_mode == .debug and !self.is_naked) {
2522	// We avoid taking this path for naked functions because there's no guarantee that such
2523	// functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
2524
2525	const alloca = try self.buildZigAlloca(operand_ty, .none);
2526	try self.store(alloca, .none, operand, operand_ty, .normal);
2527	_ = try self.wip.callIntrinsic(
2528	.normal,
2529	.none,
2530	.@"dbg.declare",
2531	&.{},
2532	&.{
2533	(try self.wip.debugValue(alloca)).toValue(),
2534	debug_local_var.toValue(),
2535	(try o.builder.debugExpression(&.{})).toValue(),
2536	},
2537	"",
2538	);
2539	} else {
2540	_ = try self.wip.callIntrinsic(
2541	.normal,
2542	.none,
2543	.@"dbg.value",
2544	&.{},
2545	&.{
2546	(try self.wip.debugValue(operand)).toValue(),
2547	debug_local_var.toValue(),
2548	(try o.builder.debugExpression(&.{})).toValue(),
2549	},
2550	"",
2551	);
2552	}
2553	return .none;
2554	}
2555
2556	fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2557	// Eventually, the Zig compiler needs to be reworked to have inline
2558	// assembly go through the same parsing code regardless of backend, and
2559	// have LLVM-flavored inline assembly be *output* from that assembler.
2560	// We don't have such an assembler implemented yet though. For now,
2561	// this implementation feeds the inline assembly code directly to LLVM.
2562
2563	const o = self.object;
2564	const unwrapped_asm = self.air.unwrapAsm(inst);
2565	const is_volatile = unwrapped_asm.is_volatile;
2566	const gpa = self.gpa;
2567
2568	const outputs = unwrapped_asm.outputs;
2569	const inputs = unwrapped_asm.inputs;
2570
2571	var llvm_constraints: std.ArrayList(u8) = .empty;
2572	defer llvm_constraints.deinit(gpa);
2573
2574	var arena_allocator = std.heap.ArenaAllocator.init(gpa);
2575	defer arena_allocator.deinit();
2576	const arena = arena_allocator.allocator();
2577
2578	// The exact number of return / parameter values depends on which output values
2579	// are passed by reference as indirect outputs (determined below).
2580	const max_return_count = outputs.len;
2581	const llvm_ret_types = try arena.alloc(Builder.Type, max_return_count);
2582	const llvm_ret_indirect = try arena.alloc(bool, max_return_count);
2583	const llvm_rw_vals = try arena.alloc(Builder.Value, max_return_count);
2584
2585	const max_param_count = max_return_count + inputs.len + outputs.len;
2586	const llvm_param_types = try arena.alloc(Builder.Type, max_param_count);
2587	const llvm_param_values = try arena.alloc(Builder.Value, max_param_count);
2588	// This stores whether we need to add an elementtype attribute and
2589	// if so, the element type itself.
2590	const llvm_param_attrs = try arena.alloc(Builder.Type, max_param_count);
2591	const zcu = o.zcu;
2592	const ip = &zcu.intern_pool;
2593	const target = zcu.getTarget();
2594
2595	var llvm_ret_i: usize = 0;
2596	var llvm_param_i: usize = 0;
2597	var total_i: usize = 0;
2598
2599	var name_map: std.array_hash_map.String(u16) = .empty;
2600	try name_map.ensureUnusedCapacity(arena, max_param_count);
2601
2602	var it = unwrapped_asm.iterateOutputs();
2603	while (it.next()) |output| {
2604	const constraint = output.constraint;
2605	const name = output.name;
2606
2607	try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 3);
2608	if (total_i != 0) {
2609	llvm_constraints.appendAssumeCapacity(',');
2610	}
2611	llvm_constraints.appendAssumeCapacity('=');
2612
2613	if (output.operand != .none) {
2614	const output_inst = try self.resolveInst(output.operand);
2615	const output_ty = self.typeOf(output.operand);
2616	assert(output_ty.zigTypeTag(zcu) == .pointer);
2617	const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu), .as_value);
2618
2619	switch (constraint[0]) {
2620	'=' => {},
2621	'+' => llvm_rw_vals[output.index] = output_inst,
2622	else => return self.todo("unsupported output constraint on output type '{c}'", .{
2623	constraint[0],
2624	}),
2625	}
2626
2627	self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu));
2628
2629	// Pass any non-return outputs indirectly, if the constraint accepts a memory location
2630	llvm_ret_indirect[output.index] = constraintAllowsMemory(constraint);
2631	if (llvm_ret_indirect[output.index]) {
2632	// Pass the result by reference as an indirect output (e.g. "=*m")
2633	llvm_constraints.appendAssumeCapacity('*');
2634
2635	llvm_param_values[llvm_param_i] = output_inst;
2636	llvm_param_types[llvm_param_i] = output_inst.typeOfWip(&self.wip);
2637	llvm_param_attrs[llvm_param_i] = elem_llvm_ty;
2638	llvm_param_i += 1;
2639	} else {
2640	// Pass the result directly (e.g. "=r")
2641	llvm_ret_types[llvm_ret_i] = elem_llvm_ty;
2642	llvm_ret_i += 1;
2643	}
2644	} else {
2645	switch (constraint[0]) {
2646	'=' => {},
2647	else => return self.todo("unsupported output constraint on result type '{s}'", .{
2648	constraint,
2649	}),
2650	}
2651
2652	llvm_ret_indirect[output.index] = false;
2653
2654	const ret_ty = self.typeOfIndex(inst);
2655	llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty, .as_value);
2656	llvm_ret_i += 1;
2657	}
2658
2659	// LLVM uses commas internally to separate different constraints,
2660	// alternative constraints are achieved with pipes.
2661	// We still allow the user to use commas in a way that is similar
2662	// to GCC's inline assembly.
2663	// http://llvm.org/docs/LangRef.html#constraint-codes
2664	for (constraint[1..]) |byte| {
2665	switch (byte) {
2666	',' => llvm_constraints.appendAssumeCapacity('|'),
2667	'*' => {}, // Indirect outputs are handled above
2668	else => llvm_constraints.appendAssumeCapacity(byte),
2669	}
2670	}
2671
2672	if (!std.mem.eql(u8, name, "_")) {
2673	const gop = name_map.getOrPutAssumeCapacity(name);
2674	if (gop.found_existing) return self.todo("duplicate asm output name '{s}'", .{name});
2675	gop.value_ptr.* = @intCast(total_i);
2676	}
2677	total_i += 1;
2678	}
2679
2680	it = unwrapped_asm.iterateInputs();
2681	while (it.next()) |input| {
2682	const constraint = input.constraint;
2683	const name = input.name;
2684
2685	const arg_llvm_value = try self.resolveInst(input.operand);
2686	const arg_ty = self.typeOf(input.operand);
2687	const is_by_ref = isByRef(arg_ty, zcu);
2688	if (is_by_ref) {
2689	if (constraintAllowsMemory(constraint)) {
2690	llvm_param_values[llvm_param_i] = arg_llvm_value;
2691	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
2692	} else {
2693	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2694	const arg_llvm_ty = try o.lowerType(arg_ty, .as_value);
2695	const load_inst = try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
2696	llvm_param_values[llvm_param_i] = load_inst;
2697	llvm_param_types[llvm_param_i] = arg_llvm_ty;
2698	}
2699	} else {
2700	if (constraintAllowsRegister(constraint)) {
2701	llvm_param_values[llvm_param_i] = arg_llvm_value;
2702	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
2703	} else {
2704	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2705	const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);
2706	try self.store(arg_ptr, .none, arg_llvm_value, arg_ty, .normal);
2707	llvm_param_values[llvm_param_i] = arg_ptr;
2708	llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);
2709	}
2710	}
2711
2712	try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 1);
2713	if (total_i != 0) {
2714	llvm_constraints.appendAssumeCapacity(',');
2715	}
2716	for (constraint) |byte| {
2717	llvm_constraints.appendAssumeCapacity(switch (byte) {
2718	',' => '|',
2719	else => byte,
2720	});
2721	}
2722
2723	if (!std.mem.eql(u8, name, "_")) {
2724	const gop = name_map.getOrPutAssumeCapacity(name);
2725	if (gop.found_existing) return self.todo("duplicate asm input name '{s}'", .{name});
2726	gop.value_ptr.* = @intCast(total_i);
2727	}
2728
2729	// In the case of indirect inputs, LLVM requires the callsite to have
2730	// an elementtype(<ty>) attribute.
2731	llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {
2732	if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));
2733
2734	break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu), .as_value);
2735	} else .none;
2736
2737	llvm_param_i += 1;
2738	total_i += 1;
2739	}
2740
2741	it = unwrapped_asm.iterateOutputs();
2742	while (it.next()) |output| {
2743	const constraint = output.constraint;
2744
2745	if (constraint[0] != '+') continue;
2746
2747	const rw_ty = self.typeOf(output.operand);
2748	const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu), .as_value);
2749	if (llvm_ret_indirect[output.index]) {
2750	llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
2751	llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
2752	} else {
2753	const access_kind: Builder.MemoryAccessKind = if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
2754	const loaded = try self.load(llvm_rw_vals[output.index], .none, rw_ty.childType(zcu), access_kind);
2755	llvm_param_values[llvm_param_i] = loaded;
2756	llvm_param_types[llvm_param_i] = llvm_elem_ty;
2757	}
2758
2759	try llvm_constraints.print(gpa, ",{d}", .{output.index});
2760
2761	// In the case of indirect inputs, LLVM requires the callsite to have
2762	// an elementtype(<ty>) attribute.
2763	llvm_param_attrs[llvm_param_i] = if (llvm_ret_indirect[output.index]) llvm_elem_ty else .none;
2764
2765	llvm_param_i += 1;
2766	total_i += 1;
2767	}
2768
2769	if (total_i != 0) try llvm_constraints.append(gpa, ',');
2770	const clobbers_val: Value = .fromInterned(unwrapped_asm.clobbers);
2771	const clobbers_ty = clobbers_val.typeOf(zcu);
2772	var clobbers_bigint_buf: Value.BigIntSpace = undefined;
2773	const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu);
2774	for (0..clobbers_ty.structFieldCount(zcu)) |field_index| {
2775	assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type);
2776	const limb_bits = @bitSizeOf(std.math.big.Limb);
2777	if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false
2778	switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) {
2779	0 => continue, // field is false
2780	1 => {}, // field is true
2781	}
2782	const name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
2783	total_i += try appendConstraints(gpa, &llvm_constraints, name, target);
2784	}
2785
2786	// We have finished scanning through all inputs/outputs, so the number of
2787	// parameters and return values is known.
2788	const param_count = llvm_param_i;
2789	const return_count = llvm_ret_i;
2790
2791	// For some targets, Clang unconditionally adds some clobbers to all inline assembly.
2792	// While this is probably not strictly necessary, if we don't follow Clang's lead
2793	// here then we may risk tripping LLVM bugs since anything not used by Clang tends
2794	// to be buggy and regress often.
2795	switch (target.cpu.arch) {
2796	.x86_64, .x86 => {
2797	try llvm_constraints.appendSlice(gpa, "~{dirflag},~{fpsr},~{flags},");
2798	total_i += 3;
2799	},
2800	.mips, .mipsel, .mips64, .mips64el => {
2801	try llvm_constraints.appendSlice(gpa, "~{$1},");
2802	total_i += 1;
2803	},
2804	else => {},
2805	}
2806
2807	if (std.mem.endsWith(u8, llvm_constraints.items, ",")) llvm_constraints.items.len -= 1;
2808
2809	const asm_source = unwrapped_asm.source;
2810
2811	// hackety hacks until stage2 has proper inline asm in the frontend.
2812	var rendered_template = std.array_list.Managed(u8).init(gpa);
2813	defer rendered_template.deinit();
2814
2815	const State = enum { start, percent, input, modifier };
2816
2817	var state: State = .start;
2818
2819	var name_start: usize = undefined;
2820	var modifier_start: usize = undefined;
2821	for (asm_source, 0..) |byte, i| {
2822	switch (state) {
2823	.start => switch (byte) {
2824	'%' => state = .percent,
2825	'$' => try rendered_template.appendSlice("$$"),
2826	else => try rendered_template.append(byte),
2827	},
2828	.percent => switch (byte) {
2829	'%' => {
2830	try rendered_template.append('%');
2831	state = .start;
2832	},
2833	'[' => {
2834	try rendered_template.append('$');
2835	try rendered_template.append('{');
2836	name_start = i + 1;
2837	state = .input;
2838	},
2839	'=' => {
2840	try rendered_template.appendSlice("${:uid}");
2841	state = .start;
2842	},
2843	else => {
2844	try rendered_template.append('%');
2845	try rendered_template.append(byte);
2846	state = .start;
2847	},
2848	},
2849	.input => switch (byte) {
2850	']', ':' => {
2851	const name = asm_source[name_start..i];
2852
2853	const index = name_map.get(name) orelse {
2854	// we should validate the assembly in Sema; by now it is too late
2855	return self.todo("unknown input or output name: '{s}'", .{name});
2856	};
2857	try rendered_template.print("{d}", .{index});
2858	if (byte == ':') {
2859	try rendered_template.append(':');
2860	modifier_start = i + 1;
2861	state = .modifier;
2862	} else {
2863	try rendered_template.append('}');
2864	state = .start;
2865	}
2866	},
2867	else => {},
2868	},
2869	.modifier => switch (byte) {
2870	']' => {
2871	try rendered_template.appendSlice(asm_source[modifier_start..i]);
2872	try rendered_template.append('}');
2873	state = .start;
2874	},
2875	else => {},
2876	},
2877	}
2878	}
2879
2880	var attributes: Builder.FunctionAttributes.Wip = .{};
2881	defer attributes.deinit(&o.builder);
2882	for (llvm_param_attrs[0..param_count], 0..) |llvm_elem_ty, i| if (llvm_elem_ty != .none)
2883	try attributes.addParamAttr(i, .{ .elementtype = llvm_elem_ty }, &o.builder);
2884
2885	const ret_llvm_ty = switch (return_count) {
2886	0 => .void,
2887	1 => llvm_ret_types[0],
2888	else => try o.builder.structType(.normal, llvm_ret_types),
2889	};
2890	const llvm_fn_ty = try o.builder.fnType(ret_llvm_ty, llvm_param_types[0..param_count], .normal);
2891	const call = try self.wip.callAsm(
2892	try attributes.finish(&o.builder),
2893	llvm_fn_ty,
2894	.{ .sideeffect = is_volatile },
2895	try o.builder.string(rendered_template.items),
2896	try o.builder.string(llvm_constraints.items),
2897	llvm_param_values[0..param_count],
2898	"",
2899	);
2900
2901	var ret_val = call;
2902	llvm_ret_i = 0;
2903	for (outputs, 0..) |output, i| {
2904	if (llvm_ret_indirect[i]) continue;
2905
2906	const output_value = if (return_count > 1)
2907	try self.wip.extractValue(call, &[_]u32{@intCast(llvm_ret_i)}, "")
2908	else
2909	call;
2910
2911	if (output != .none) {
2912	const output_ptr = try self.resolveInst(output);
2913	const output_ptr_ty = self.typeOf(output);
2914	try self.store(
2915	output_ptr,
2916	output_ptr_ty.ptrAlignment(zcu),
2917	output_value,
2918	output_ptr_ty.childType(zcu),
2919	if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2920	);
2921	} else {
2922	ret_val = output_value;
2923	}
2924	llvm_ret_i += 1;
2925	}
2926
2927	return ret_val;
2928	}
2929
2930	fn airIsNonNull(
2931	self: *FuncGen,
2932	inst: Air.Inst.Index,
2933	operand_is_ptr: bool,
2934	cond: Builder.IntegerCondition,
2935	) Allocator.Error!Builder.Value {
2936	const o = self.object;
2937	const zcu = o.zcu;
2938	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
2939	const operand = try self.resolveInst(un_op);
2940	const operand_ty = self.typeOf(un_op);
2941	const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
2942	const payload_ty = optional_ty.optionalChild(zcu);
2943
2944	const access_kind: Builder.MemoryAccessKind =
2945	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
2946
2947	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
2948
2949	if (optional_ty.optionalReprIsPayload(zcu)) {
2950	const loaded = if (operand_is_ptr)
2951	try self.load(operand, operand_ty.ptrAlignment(zcu), optional_ty, access_kind)
2952	else
2953	operand;
2954	if (payload_ty.isSlice(zcu)) {
2955	const slice_ptr = try self.wip.extractValue(loaded, &.{0}, "");
2956	const ptr_ty = try o.builder.ptrType(llvm.toLlvmAddressSpace(
2957	payload_ty.ptrAddressSpace(zcu),
2958	zcu.getTarget(),
2959	));
2960	return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
2961	}
2962	return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(try o.lowerType(optional_ty, .as_value)), "");
2963	}
2964
2965	comptime assert(optional_layout_version == 3);
2966
2967	if (!payload_ty.hasRuntimeBits(zcu)) {
2968	const loaded = if (operand_is_ptr)
2969	try self.load(operand, operand_ty.ptrAlignment(zcu), optional_ty, access_kind)
2970	else
2971	operand;
2972	return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
2973	}
2974
2975	return self.optCmpNull(cond, optional_ty, operand, access_kind);
2976	}
2977
2978	fn airIsErr(
2979	self: *FuncGen,
2980	inst: Air.Inst.Index,
2981	cond: Builder.IntegerCondition,
2982	operand_is_ptr: bool,
2983	) Allocator.Error!Builder.Value {
2984	const o = self.object;
2985	const zcu = o.zcu;
2986	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
2987	const operand = try self.resolveInst(un_op);
2988	const operand_ty = self.typeOf(un_op);
2989	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
2990	const payload_ty = err_union_ty.errorUnionPayload(zcu);
2991	const zero_err = try o.builder.intValue(try o.errorIntType(.as_value), 0);
2992
2993	const access_kind: Builder.MemoryAccessKind =
2994	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
2995
2996	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
2997	return switch (cond) {
2998	.eq => .true, // 0 == 0
2999	.ne => .false, // 0 != 0
3000	else => unreachable,
3001	};
3002	}
3003
3004	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
3005
3006	if (!payload_ty.hasRuntimeBits(zcu)) {
3007	const loaded = if (operand_is_ptr)
3008	try self.load(operand, operand_ty.ptrAlignment(zcu), err_union_ty, access_kind)
3009	else
3010	operand;
3011	return self.wip.icmp(cond, loaded, zero_err, "");
3012	}
3013	assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref
3014
3015	const err_align = if (operand_is_ptr)
3016	operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu))
3017	else
3018	.none;
3019	const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3020	const loaded = try self.load(err_field_ptr, err_align, .anyerror, access_kind);
3021	return self.wip.icmp(cond, loaded, zero_err, "");
3022	}
3023
3024	fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3025	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3026	const operand = try self.resolveInst(ty_op.operand);
3027	// If `Type.optionalReprIsPayload`, then the address should be the same. Otherwise, optional
3028	// layouts always put the payload at offset 0, so... the address should still be the same.
3029	return operand;
3030	}
3031
3032	fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3033	comptime assert(optional_layout_version == 3);
3034
3035	const o = self.object;
3036	const zcu = o.zcu;
3037	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3038	const operand = try self.resolveInst(ty_op.operand);
3039	const optional_ptr_ty = self.typeOf(ty_op.operand);
3040	const optional_ty = optional_ptr_ty.childType(zcu);
3041	const payload_ty = optional_ty.optionalChild(zcu);
3042
3043	const access_kind: Builder.MemoryAccessKind =
3044	if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
3045
3046	if (!payload_ty.hasRuntimeBits(zcu)) {
3047	self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
3048
3049	// We have a pointer to a i8. We need to set it to 1 and then return the same pointer.
3050	// Default alignment store because align of the non null bit is 1 anyway.
3051	try self.store(operand, .@"1", .true, .bool, access_kind);
3052	return operand;
3053	}
3054	if (optional_ty.optionalReprIsPayload(zcu)) {
3055	// The payload and the optional are the same value.
3056	// Setting to non-null will be done when the payload is set.
3057	return operand;
3058	}
3059
3060	// First set the non-null bit. It's always immediately after the payload (no padding) because it
3061	// has alignment 1.
3062	const non_null_ptr = try self.ptraddConst(operand, payload_ty.abiSize(zcu));
3063
3064	self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
3065
3066	// Default alignment store because align of the non null bit is 1 anyway.
3067	try self.store(non_null_ptr, .@"1", .true, .bool, access_kind);
3068
3069	// Then return the payload pointer (only if it's used).
3070	if (self.liveness.isUnused(inst)) return .none;
3071
3072	return operand; // payload is at offset 0
3073	}
3074
3075	fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3076	const zcu = self.object.zcu;
3077	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3078	const operand = try self.resolveInst(ty_op.operand);
3079	const optional_ty = self.typeOf(ty_op.operand);
3080	const payload_ty = self.typeOfIndex(inst);
3081	if (!payload_ty.hasRuntimeBits(zcu)) return .none;
3082
3083	if (optional_ty.optionalReprIsPayload(zcu)) {
3084	// Payload value is the same as the optional value.
3085	return operand;
3086	}
3087
3088	return self.optPayloadHandle(operand, optional_ty, false);
3089	}
3090
3091	fn airErrUnionPayload(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3092	const o = fg.object;
3093	const zcu = o.zcu;
3094	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3095	const operand = try fg.resolveInst(ty_op.operand);
3096	const err_union_ty = fg.typeOf(ty_op.operand);
3097	const payload_ty = fg.typeOfIndex(inst);
3098
3099	assert(payload_ty.hasRuntimeBits(zcu));
3100	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3101
3102	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
3103	const payload_ptr = try fg.ptraddConst(operand, payload_offset);
3104	return fg.load(payload_ptr, err_union_ty.abiAlignment(zcu).offset(payload_offset), payload_ty, .normal);
3105	}
3106
3107	fn airErrUnionPayloadPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3108	const o = fg.object;
3109	const zcu = o.zcu;
3110	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3111	const operand = try fg.resolveInst(ty_op.operand);
3112	const payload_ty = fg.typeOfIndex(inst).childType(zcu);
3113	return fg.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3114	}
3115
3116	fn airErrUnionErr(
3117	self: *FuncGen,
3118	inst: Air.Inst.Index,
3119	operand_is_ptr: bool,
3120	) Allocator.Error!Builder.Value {
3121	const o = self.object;
3122	const zcu = o.zcu;
3123	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3124	const operand = try self.resolveInst(ty_op.operand);
3125	const operand_ty = self.typeOf(ty_op.operand);
3126	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3127
3128	const access_kind: Builder.MemoryAccessKind =
3129	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
3130
3131	const payload_ty = err_union_ty.errorUnionPayload(zcu);
3132
3133	if (payload_ty.hasRuntimeBits(zcu)) {
3134	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3135	} else if (!operand_is_ptr) {
3136	return operand;
3137	}
3138
3139	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
3140
3141	const ptr_align = if (operand_is_ptr) operand_ty.ptrAlignment(zcu) else err_union_ty.abiAlignment(zcu);
3142
3143	const err_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
3144	const err_align = ptr_align.offset(err_offset);
3145	const err_ptr = try self.ptraddConst(operand, err_offset);
3146
3147	return self.load(err_ptr, err_align, .anyerror, access_kind);
3148	}
3149
3150	fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3151	const o = self.object;
3152	const zcu = o.zcu;
3153	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3154	const operand = try self.resolveInst(ty_op.operand);
3155	const err_union_ptr_ty = self.typeOf(ty_op.operand);
3156	const err_union_ty = err_union_ptr_ty.childType(zcu);
3157	const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);
3158
3159	const payload_ty = err_union_ty.errorUnionPayload(zcu);
3160	const non_error_val = try o.builder.intValue(try o.errorIntType(.as_value), 0);
3161
3162	const access_kind: Builder.MemoryAccessKind =
3163	if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
3164
3165	self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
3166
3167	{
3168	// First set the non-error value.
3169	const error_off = codegen.errUnionErrorOffset(payload_ty, zcu);
3170	const error_ptr = try self.ptraddConst(operand, error_off);
3171	try self.store(error_ptr, err_union_ptr_align.offset(error_off), non_error_val, .anyerror, access_kind);
3172	}
3173
3174	// Then return the payload pointer (only if it is used).
3175	if (self.liveness.isUnused(inst)) return .none;
3176	return self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3177	}
3178
3179	fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) Allocator.Error!Builder.Value {
3180	assert(self.err_ret_trace != .none);
3181	return self.err_ret_trace;
3182	}
3183
3184	fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3185	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
3186	self.err_ret_trace = try self.resolveInst(un_op);
3187	return .none;
3188	}
3189
3190	fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3191	const zcu = self.object.zcu;
3192
3193	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
3194	const struct_ty = ty_pl.ty;
3195	const field_index = ty_pl.payload;
3196
3197	assert(self.err_ret_trace != .none);
3198
3199	const field_ty = struct_ty.fieldType(field_index, zcu);
3200	const field_offset = struct_ty.structFieldOffset(field_index, zcu);
3201	const field_align = struct_ty.abiAlignment(zcu).offset(field_offset);
3202	const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset);
3203	return self.load(field_ptr, field_align, field_ty, .normal);
3204	}
3205
3206	fn airWrapOptional(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3207	const o = self.object;
3208	const zcu = o.zcu;
3209	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3210	const payload_ty = self.typeOf(ty_op.operand);
3211	comptime assert(optional_layout_version == 3);
3212	assert(payload_ty.hasRuntimeBits(zcu));
3213	const operand = try self.resolveInst(ty_op.operand);
3214	const optional_ty = self.typeOfIndex(inst);
3215	if (optional_ty.optionalReprIsPayload(zcu)) return operand;
3216	assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload`
3217	const optional_ptr = try self.buildZigAlloca(optional_ty, .none);
3218
3219	const payload_ptr = optional_ptr; // payload always at offset 0
3220	try self.store(payload_ptr, .none, operand, payload_ty, .normal);
3221
3222	// Non-null bit immediately after payload (no padding because the bit has alignment 1).
3223	const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu));
3224	try self.store(non_null_ptr, .none, .true, .bool, .normal);
3225
3226	return optional_ptr;
3227	}
3228
3229	fn airWrapErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3230	const o = self.object;
3231	const zcu = o.zcu;
3232	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3233	const err_un_ty = self.typeOfIndex(inst);
3234	const operand = try self.resolveInst(ty_op.operand);
3235	const payload_ty = self.typeOf(ty_op.operand);
3236	assert(payload_ty.hasRuntimeBits(zcu));
3237	assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3238	const ok_err_code = try o.builder.intValue(try o.errorIntType(.as_value), 0);
3239
3240	const result_ptr = try self.buildZigAlloca(err_un_ty, .none);
3241
3242	const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3243	try self.store(err_ptr, .none, ok_err_code, .anyerror, .normal);
3244
3245	const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3246	try self.store(payload_ptr, .none, operand, payload_ty, .normal);
3247
3248	return result_ptr;
3249	}
3250
3251	fn airWrapErrUnionErr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3252	const o = self.object;
3253	const zcu = o.zcu;
3254	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
3255	const err_un_ty = self.typeOfIndex(inst);
3256	const payload_ty = err_un_ty.errorUnionPayload(zcu);
3257	const operand = try self.resolveInst(ty_op.operand);
3258	if (!payload_ty.hasRuntimeBits(zcu)) return operand;
3259	assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3260
3261	const result_ptr = try self.buildZigAlloca(err_un_ty, .none);
3262
3263	const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3264	try self.store(err_ptr, .none, operand, .anyerror, .normal);
3265
3266	const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3267	// TODO store undef to payload_ptr
3268	_ = payload_ptr;
3269
3270	return result_ptr;
3271	}
3272
3273	fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3274	const o = self.object;
3275	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
3276	const index = pl_op.payload;
3277	const llvm_usize = try o.lowerType(.usize, .as_value);
3278	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{
3279	try o.builder.intValue(.i32, index),
3280	}, "");
3281	}
3282
3283	fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3284	const o = self.object;
3285	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
3286	const index = pl_op.payload;
3287	const llvm_isize = try o.lowerType(.isize, .as_value);
3288	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{
3289	try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),
3290	}, "");
3291	}
3292
3293	fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3294	const o = fg.object;
3295	const ty_nav = fg.air.instructions.items(.data)[@backingInt(inst)].ty_nav;
3296	const llvm_ptr = try o.lowerNavRef(ty_nav.nav);
3297	return llvm_ptr.toValue();
3298	}
3299
3300	fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3301	const o = self.object;
3302	const zcu = o.zcu;
3303	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3304	const lhs = try self.resolveInst(bin_op.lhs);
3305	const rhs = try self.resolveInst(bin_op.rhs);
3306	const inst_ty = self.typeOfIndex(inst);
3307	const scalar_ty = inst_ty.scalarType(zcu);
3308
3309	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, .normal, inst_ty, 2, .{ lhs, rhs });
3310	return self.wip.callIntrinsic(
3311	.normal,
3312	.none,
3313	if (scalar_ty.isSignedInt(zcu)) .smin else .umin,
3314	&.{try o.lowerType(inst_ty, .as_value)},
3315	&.{ lhs, rhs },
3316	"",
3317	);
3318	}
3319
3320	fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3321	const o = self.object;
3322	const zcu = o.zcu;
3323	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3324	const lhs = try self.resolveInst(bin_op.lhs);
3325	const rhs = try self.resolveInst(bin_op.rhs);
3326	const inst_ty = self.typeOfIndex(inst);
3327	const scalar_ty = inst_ty.scalarType(zcu);
3328
3329	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, .normal, inst_ty, 2, .{ lhs, rhs });
3330	return self.wip.callIntrinsic(
3331	.normal,
3332	.none,
3333	if (scalar_ty.isSignedInt(zcu)) .smax else .umax,
3334	&.{try o.lowerType(inst_ty, .as_value)},
3335	&.{ lhs, rhs },
3336	"",
3337	);
3338	}
3339
3340	fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3341	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
3342	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3343	const ptr = try self.resolveInst(bin_op.lhs);
3344	const len = try self.resolveInst(bin_op.rhs);
3345	const inst_ty = self.typeOfIndex(inst);
3346	return self.wip.buildAggregate(try self.object.lowerType(inst_ty, .as_value), &.{ ptr, len }, "");
3347	}
3348
3349	fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
3350	const zcu = self.object.zcu;
3351	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3352	const lhs = try self.resolveInst(bin_op.lhs);
3353	const rhs = try self.resolveInst(bin_op.rhs);
3354	const inst_ty = self.typeOfIndex(inst);
3355	const scalar_ty = inst_ty.scalarType(zcu);
3356
3357	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, fast, inst_ty, 2, .{ lhs, rhs });
3358	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"add nsw" else .@"add nuw", lhs, rhs, "");
3359	}
3360
3361	fn airSafeArithmetic(
3362	fg: *FuncGen,
3363	inst: Air.Inst.Index,
3364	signed_intrinsic: Builder.Intrinsic,
3365	unsigned_intrinsic: Builder.Intrinsic,
3366	) Allocator.Error!Builder.Value {
3367	const o = fg.object;
3368	const zcu = o.zcu;
3369
3370	const bin_op = fg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3371	const lhs = try fg.resolveInst(bin_op.lhs);
3372	const rhs = try fg.resolveInst(bin_op.rhs);
3373	const inst_ty = fg.typeOfIndex(inst);
3374	const scalar_ty = inst_ty.scalarType(zcu);
3375
3376	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3377	const llvm_inst_ty = try o.lowerType(inst_ty, .as_value);
3378	const results =
3379	try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");
3380
3381	const overflow_bits = try fg.wip.extractValue(results, &.{1}, "");
3382	const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip);
3383	const overflow_bit = switch (inst_ty.zigTypeTag(zcu)) {
3384	.vector => try fg.wip.callIntrinsic(
3385	.normal,
3386	.none,
3387	.@"vector.reduce.or",
3388	&.{overflow_bits_ty},
3389	&.{overflow_bits},
3390	"",
3391	),
3392	else => overflow_bits,
3393	};
3394
3395	const fail_block = try fg.wip.block(1, "OverflowFail");
3396	const ok_block = try fg.wip.block(1, "OverflowOk");
3397	_ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none);
3398
3399	fg.wip.cursor = .{ .block = fail_block };
3400	try fg.buildSimplePanic(.integer_overflow);
3401
3402	fg.wip.cursor = .{ .block = ok_block };
3403	return fg.wip.extractValue(results, &.{0}, "");
3404	}
3405
3406	fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3407	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3408	const lhs = try self.resolveInst(bin_op.lhs);
3409	const rhs = try self.resolveInst(bin_op.rhs);
3410
3411	return self.wip.bin(.add, lhs, rhs, "");
3412	}
3413
3414	fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3415	const o = self.object;
3416	const zcu = o.zcu;
3417	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3418	const lhs = try self.resolveInst(bin_op.lhs);
3419	const rhs = try self.resolveInst(bin_op.rhs);
3420	const inst_ty = self.typeOfIndex(inst);
3421	const scalar_ty = inst_ty.scalarType(zcu);
3422	assert(scalar_ty.zigTypeTag(zcu) == .int);
3423	return self.wip.callIntrinsic(
3424	.normal,
3425	.none,
3426	if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",
3427	&.{try o.lowerType(inst_ty, .as_value)},
3428	&.{ lhs, rhs },
3429	"",
3430	);
3431	}
3432
3433	fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
3434	const zcu = self.object.zcu;
3435	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3436	const lhs = try self.resolveInst(bin_op.lhs);
3437	const rhs = try self.resolveInst(bin_op.rhs);
3438	const inst_ty = self.typeOfIndex(inst);
3439	const scalar_ty = inst_ty.scalarType(zcu);
3440
3441	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, fast, inst_ty, 2, .{ lhs, rhs });
3442	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"sub nsw" else .@"sub nuw", lhs, rhs, "");
3443	}
3444
3445	fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3446	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3447	const lhs = try self.resolveInst(bin_op.lhs);
3448	const rhs = try self.resolveInst(bin_op.rhs);
3449
3450	return self.wip.bin(.sub, lhs, rhs, "");
3451	}
3452
3453	fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3454	const o = self.object;
3455	const zcu = o.zcu;
3456	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3457	const lhs = try self.resolveInst(bin_op.lhs);
3458	const rhs = try self.resolveInst(bin_op.rhs);
3459	const inst_ty = self.typeOfIndex(inst);
3460	const scalar_ty = inst_ty.scalarType(zcu);
3461	assert(scalar_ty.zigTypeTag(zcu) == .int);
3462	return self.wip.callIntrinsic(
3463	.normal,
3464	.none,
3465	if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",
3466	&.{try o.lowerType(inst_ty, .as_value)},
3467	&.{ lhs, rhs },
3468	"",
3469	);
3470	}
3471
3472	fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
3473	const zcu = self.object.zcu;
3474	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3475	const lhs = try self.resolveInst(bin_op.lhs);
3476	const rhs = try self.resolveInst(bin_op.rhs);
3477	const inst_ty = self.typeOfIndex(inst);
3478	const scalar_ty = inst_ty.scalarType(zcu);
3479
3480	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, fast, inst_ty, 2, .{ lhs, rhs });
3481	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"mul nsw" else .@"mul nuw", lhs, rhs, "");
3482	}
3483
3484	fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3485	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3486	const lhs = try self.resolveInst(bin_op.lhs);
3487	const rhs = try self.resolveInst(bin_op.rhs);
3488
3489	return self.wip.bin(.mul, lhs, rhs, "");
3490	}
3491
3492	fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3493	const o = self.object;
3494	const zcu = o.zcu;
3495	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3496	const lhs = try self.resolveInst(bin_op.lhs);
3497	const rhs = try self.resolveInst(bin_op.rhs);
3498	const inst_ty = self.typeOfIndex(inst);
3499	const scalar_ty = inst_ty.scalarType(zcu);
3500	assert(scalar_ty.zigTypeTag(zcu) == .int);
3501	return self.wip.callIntrinsic(
3502	.normal,
3503	.none,
3504	if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",
3505	&.{try o.lowerType(inst_ty, .as_value)},
3506	&.{ lhs, rhs, .@"0" },
3507	"",
3508	);
3509	}
3510
3511	fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
3512	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3513	const lhs = try self.resolveInst(bin_op.lhs);
3514	const rhs = try self.resolveInst(bin_op.rhs);
3515	const inst_ty = self.typeOfIndex(inst);
3516
3517	return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
3518	}
3519
3520	fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
3521	const zcu = self.object.zcu;
3522	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3523	const lhs = try self.resolveInst(bin_op.lhs);
3524	const rhs = try self.resolveInst(bin_op.rhs);
3525	const inst_ty = self.typeOfIndex(inst);
3526	const scalar_ty = inst_ty.scalarType(zcu);
3527
3528	if (scalar_ty.isRuntimeFloat()) {
3529	const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
3530	return self.buildFloatOp(.trunc, fast, inst_ty, 1, .{result});
3531	}
3532	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .sdiv else .udiv, lhs, rhs, "");
3533	}
3534
3535	fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
3536	const o = self.object;
3537	const zcu = o.zcu;
3538	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3539	const lhs = try self.resolveInst(bin_op.lhs);
3540	const rhs = try self.resolveInst(bin_op.rhs);
3541	const inst_ty = self.typeOfIndex(inst);
3542	const scalar_ty = inst_ty.scalarType(zcu);
3543
3544	if (scalar_ty.isRuntimeFloat()) {
3545	const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
3546	return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});
3547	}
3548	if (scalar_ty.isSignedInt(zcu)) {
3549	const scalar_llvm_ty = try o.lowerType(scalar_ty, .as_value);
3550	const inst_llvm_ty = try o.lowerType(inst_ty, .as_value);
3551
3552	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
3553	var bfa_buf: ExpectedContents = undefined;
3554	var bfa: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), self.gpa);
3555	const allocator = bfa.allocator();
3556
3557	const scalar_bits = scalar_ty.intInfo(zcu).bits;
3558	var smin_big_int: std.math.big.int.Mutable = .{
3559	.limbs = try allocator.alloc(
3560	std.math.big.Limb,
3561	std.math.big.int.calcTwosCompLimbCount(scalar_bits),
3562	),
3563	.len = undefined,
3564	.positive = undefined,
3565	};
3566	defer allocator.free(smin_big_int.limbs);
3567	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
3568	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
3569	scalar_llvm_ty,
3570	smin_big_int.toConst(),
3571	));
3572
3573	const div = try self.wip.bin(.sdiv, lhs, rhs, "divFloor.div");
3574	const rem = try self.wip.bin(.srem, lhs, rhs, "divFloor.rem");
3575	const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "divFloor.rhs_sign");
3576	const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "divFloor.rem_xor_rhs_sign");
3577	const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "divFloor.need_correction");
3578	const correction = try self.wip.cast(.sext, need_correction, inst_llvm_ty, "divFloor.correction");
3579	return self.wip.bin(.@"add nsw", div, correction, "divFloor");
3580	}
3581	return self.wip.bin(.udiv, lhs, rhs, "");
3582	}
3583
3584	fn airDivCeil(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
3585	const o = self.object;
3586	const zcu = o.zcu;
3587	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3588	const lhs = try self.resolveInst(bin_op.lhs);
3589	const rhs = try self.resolveInst(bin_op.rhs);
3590	const inst_ty = self.typeOfIndex(inst);
3591	const scalar_ty = inst_ty.scalarType(zcu);
3592
3593	if (scalar_ty.isRuntimeFloat()) {
3594	const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
3595	return self.buildFloatOp(.ceil, fast, inst_ty, 1, .{result});
3596	}
3597	if (scalar_ty.isSignedInt(zcu)) {
3598	const scalar_llvm_ty = try o.lowerType(scalar_ty, .as_value);
3599	const inst_llvm_ty = try o.lowerType(inst_ty, .as_value);
3600
3601	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
3602	var bfa_buf: ExpectedContents = undefined;
3603	var bfa: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), self.gpa);
3604	const allocator = bfa.allocator();
3605
3606	const scalar_bits = scalar_ty.intInfo(zcu).bits;
3607	var smin_big_int: std.math.big.int.Mutable = .{
3608	.limbs = try allocator.alloc(
3609	std.math.big.Limb,
3610	std.math.big.int.calcTwosCompLimbCount(scalar_bits),
3611	),
3612	.len = undefined,
3613	.positive = undefined,
3614	};
3615	defer allocator.free(smin_big_int.limbs);
3616	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
3617	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
3618	scalar_llvm_ty,
3619	smin_big_int.toConst(),
3620	));
3621
3622	const zero = try o.builder.splatValue(
3623	inst_llvm_ty,
3624	try o.builder.intConst(scalar_llvm_ty, 0),
3625	);
3626
3627	const div = try self.wip.bin(.sdiv, lhs, rhs, "divCeil.div");
3628	const rem = try self.wip.bin(.srem, lhs, rhs, "divCeil.rem");
3629
3630	const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "divCeil.rhs_sign");
3631	const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "divCeil.rem_xor_rhs_sign");
3632
3633	const need_correction = try self.wip.icmp(.sgt, rem_xor_rhs_sign, zero, "divCeil.need_correction");
3634
3635	const correction = try self.wip.cast(.zext, need_correction, inst_llvm_ty, "divCeil.correction");
3636	return self.wip.bin(.@"add nsw", div, correction, "divCeil");
3637	} else {
3638	const scalar_llvm_ty = try o.lowerType(scalar_ty, .as_value);
3639	const inst_llvm_ty = try o.lowerType(inst_ty, .as_value);
3640
3641	const zero = try o.builder.splatValue(
3642	inst_llvm_ty,
3643	try o.builder.intConst(scalar_llvm_ty, 0),
3644	);
3645
3646	const div = try self.wip.bin(.udiv, lhs, rhs, "divCeil.div");
3647	const rem = try self.wip.bin(.urem, lhs, rhs, "divCeil.rem");
3648
3649	const rem_nonzero = try self.wip.icmp(.ne, rem, zero, "divCeil.rem_nonzero");
3650	const correction = try self.wip.cast(.zext, rem_nonzero, inst_llvm_ty, "divCeil.correction");
3651
3652	return self.wip.bin(.@"add nuw", div, correction, "divCeil");
3653	}
3654	}
3655
3656	fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
3657	const zcu = self.object.zcu;
3658	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3659	const lhs = try self.resolveInst(bin_op.lhs);
3660	const rhs = try self.resolveInst(bin_op.rhs);
3661	const inst_ty = self.typeOfIndex(inst);
3662	const scalar_ty = inst_ty.scalarType(zcu);
3663
3664	if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
3665	return self.wip.bin(
3666	if (scalar_ty.isSignedInt(zcu)) .@"sdiv exact" else .@"udiv exact",
3667	lhs,
3668	rhs,
3669	"",
3670	);
3671	}
3672
3673	fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
3674	const zcu = self.object.zcu;
3675	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3676	const lhs = try self.resolveInst(bin_op.lhs);
3677	const rhs = try self.resolveInst(bin_op.rhs);
3678	const inst_ty = self.typeOfIndex(inst);
3679	const scalar_ty = inst_ty.scalarType(zcu);
3680
3681	if (scalar_ty.isRuntimeFloat())
3682	return self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
3683	return self.wip.bin(if (scalar_ty.isSignedInt(zcu))
3684	.srem
3685	else
3686	.urem, lhs, rhs, "");
3687	}
3688
3689	fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
3690	const o = self.object;
3691	const zcu = o.zcu;
3692	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
3693	const lhs = try self.resolveInst(bin_op.lhs);
3694	const rhs = try self.resolveInst(bin_op.rhs);
3695	const inst_ty = self.typeOfIndex(inst);
3696	const scalar_ty = inst_ty.scalarType(zcu);
3697
3698	if (scalar_ty.isRuntimeFloat()) {
3699	const a = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
3700	const b = try self.buildFloatOp(.add, fast, inst_ty, 2, .{ a, rhs });
3701	const c = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ b, rhs });
3702	const zero = if (isByRef(inst_ty, zcu)) zero: {
3703	const zero = try o.builder.zeroInitConst(try o.lowerType(inst_ty, .in_memory));
3704	break :zero try o.lowerConstRef(zero, inst_ty.abiAlignment(zcu).toLlvm());
3705	} else try o.builder.zeroInitConst(try o.lowerType(inst_ty, .as_value));
3706	const ltz = try self.buildFloatCmp(fast, .lt, inst_ty, .{ lhs, zero.toValue() });
3707	return self.wip.select(fast, ltz, c, a, "");
3708	}
3709	if (scalar_ty.isSignedInt(zcu)) {
3710	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
3711	var bfa_buf: ExpectedContents = undefined;
3712	var bfa: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), self.gpa);
3713	const allocator = bfa.allocator();
3714
3715	const inst_llvm_ty = try o.lowerType(inst_ty, .as_value);
3716	const scalar_bits = scalar_ty.intInfo(zcu).bits;
3717	var smin_big_int: std.math.big.int.Mutable = .{
3718	.limbs = try allocator.alloc(
3719	std.math.big.Limb,
3720	std.math.big.int.calcTwosCompLimbCount(scalar_bits),
3721	),
3722	.len = undefined,
3723	.positive = undefined,
3724	};
3725	defer allocator.free(smin_big_int.limbs);
3726	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
3727	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
3728	try o.lowerType(scalar_ty, .as_value),
3729	smin_big_int.toConst(),
3730	));
3731
3732	const rem = try self.wip.bin(.srem, lhs, rhs, "mod.rem");
3733	const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "mod.rhs_sign");
3734	const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "mod.rem_xor_rhs_sign");
3735	const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "mod.need_correction");
3736	const zero = try o.builder.zeroInitValue(inst_llvm_ty);
3737	const correction = try self.wip.select(.normal, need_correction, rhs, zero, "mod.correction");
3738	return self.wip.bin(.@"add nsw", correction, rem, "mod");
3739	}
3740	return self.wip.bin(.urem, lhs, rhs, "");
3741	}
3742
3743	fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3744	const zcu = self.object.zcu;
3745	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
3746	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3747	const ptr_or_slice = try self.resolveInst(bin_op.lhs);
3748	const index = try self.resolveInst(bin_op.rhs);
3749	const ptr_ty = self.typeOf(bin_op.lhs);
3750	const elem_ty = ptr_ty.indexableElem(zcu);
3751	const ptr = switch (ptr_ty.ptrSize(zcu)) {
3752	.one, .many, .c => ptr_or_slice,
3753	.slice => try self.wip.extractValue(ptr_or_slice, &.{0}, ""),
3754	};
3755	return self.ptraddScaled(ptr, index, elem_ty.abiSize(zcu));
3756	}
3757
3758	fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3759	const o = self.object;
3760	const zcu = o.zcu;
3761	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
3762	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3763	const ptr_or_slice = try self.resolveInst(bin_op.lhs);
3764	const llvm_usize_ty = try o.lowerType(.usize, .as_value);
3765	const ptr_ty = self.typeOf(bin_op.lhs);
3766	const elem_ty = ptr_ty.indexableElem(zcu);
3767	const ptr = switch (ptr_ty.ptrSize(zcu)) {
3768	.one, .many, .c => ptr_or_slice,
3769	.slice => try self.wip.extractValue(ptr_or_slice, &.{0}, ""),
3770	};
3771	const scale_val = try o.builder.intValue(llvm_usize_ty, -@as(i65, elem_ty.abiSize(zcu)));
3772	const positive_index = try self.resolveInst(bin_op.rhs);
3773	const negative_offset = try self.wip.bin(.@"mul nsw", positive_index, scale_val, "");
3774	return self.wip.gep(.inbounds, .i8, ptr, &.{negative_offset}, "");
3775	}
3776
3777	fn airOverflow(
3778	self: *FuncGen,
3779	inst: Air.Inst.Index,
3780	signed_intrinsic: Builder.Intrinsic,
3781	unsigned_intrinsic: Builder.Intrinsic,
3782	) Allocator.Error!Builder.Value {
3783	const o = self.object;
3784	const zcu = o.zcu;
3785	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
3786	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
3787
3788	const lhs = try self.resolveInst(extra.lhs);
3789	const rhs = try self.resolveInst(extra.rhs);
3790
3791	const lhs_ty = self.typeOf(extra.lhs);
3792	const scalar_ty = lhs_ty.scalarType(zcu);
3793	const inst_ty = self.typeOfIndex(inst);
3794	assert(isByRef(inst_ty, zcu)); // auto structs are by-ref
3795
3796	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3797	const llvm_lhs_ty = try o.lowerType(lhs_ty, .as_value);
3798	const results =
3799	try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");
3800
3801	const result_val = try self.wip.extractValue(results, &.{0}, "");
3802	const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
3803
3804	const result_alignment = inst_ty.abiAlignment(zcu);
3805	const alloca_inst = try self.buildZigAlloca(inst_ty, .none);
3806
3807	{
3808	// Store to 'result: IntType' field
3809	const field_off = inst_ty.structFieldOffset(0, zcu);
3810	const field_ptr = try self.ptraddConst(alloca_inst, field_off);
3811	try self.store(field_ptr, result_alignment.offset(field_off), result_val, lhs_ty, .normal);
3812	}
3813
3814	{
3815	// Store to 'overflow: u1' field
3816	const field_off = inst_ty.structFieldOffset(1, zcu);
3817	const field_ptr = try self.ptraddConst(alloca_inst, field_off);
3818	try self.store(field_ptr, result_alignment.offset(field_off), overflow_bit, inst_ty.fieldType(1, zcu), .normal);
3819	}
3820
3821	return alloca_inst;
3822	}
3823
3824	fn buildElementwiseCall(
3825	fg: *FuncGen,
3826	fn_name: Builder.StrtabString,
3827	fn_info: Object.FuncInfo,
3828	arg_values: []const Builder.Value,
3829	vector_len: ?u32,
3830	) Allocator.Error!Builder.Value {
3831	const o = fg.object;
3832	const zcu = o.zcu;
3833	const llvm_fn = try fg.object.getLibcFunction(fg.pt, fn_name, fn_info);
3834
3835	const iterations = vector_len orelse 1;
3836	const ret_ty: Type = .fromInterned(fn_info.return_type);
3837	const ret_is_by_ref = isByRef(ret_ty, zcu);
3838	if (iterations > 1 and (fn_info.return_type == .void_type or ret_is_by_ref) and
3839	for (fn_info.param_types) |param_type| {
3840	if (!isByRef(.fromInterned(param_type), zcu)) break false;
3841	} else true)
3842	{
3843	const entry_block = fg.wip.cursor.block;
3844	const loop_block = try fg.wip.block(2, "elementwise.loop");
3845	const done_block = try fg.wip.block(1, "elementwise.done");
3846
3847	const result_ptr = if (fn_info.return_type == .void_type) .none else result_ptr: {
3848	const ret_llvm_ty = try o.lowerType(ret_ty, .in_memory);
3849	break :result_ptr try fg.buildAlloca(
3850	if (vector_len) |len| try o.builder.arrayType(len, ret_llvm_ty) else ret_llvm_ty,
3851	ret_ty.abiAlignment(zcu).toLlvm(),
3852	);
3853	};
3854	_ = try fg.wip.br(loop_block);
3855
3856	fg.wip.cursor = .{ .block = loop_block };
3857	const index = try fg.wip.phi(.i32, "elementwise.index");
3858
3859	var arg_elems_buf: [3]Builder.Value = undefined;
3860	const arg_elems = arg_elems_buf[0..arg_values.len];
3861	for (arg_elems, fn_info.param_types, arg_values) |*arg_elem, param_type, arg_value| {
3862	const arg_elem_ptr = try fg.ptraddScaled(arg_value, index.toValue(), Type.fromInterned(param_type).abiSize(zcu));
3863	arg_elem.* = try fg.load(arg_elem_ptr, .none, .fromInterned(param_type), .normal);
3864	}
3865	const result_elem = try fg.buildCall(.{}, llvm_fn.typeOf(&o.builder), llvm_fn.toValue(&o.builder), fn_info, fn_info.param_types, arg_elems);
3866	if (fn_info.return_type == .void_type) {
3867	assert(result_elem == .none);
3868	} else if (result_elem != .none) {
3869	const result_elem_ptr = try fg.ptraddScaled(result_ptr, index.toValue(), ret_ty.abiSize(zcu));
3870	try fg.store(result_elem_ptr, .none, result_elem, ret_ty, .normal);
3871	}
3872
3873	const next_index = try fg.wip.bin(.@"add nuw", index.toValue(), try o.builder.intValue(.i32, 1), "elementwise.next_index");
3874	index.finish(&.{ try o.builder.intValue(.i32, 0), next_index }, &.{ entry_block, loop_block }, &fg.wip);
3875	const is_done = try fg.wip.icmp(.eq, next_index, try o.builder.intValue(.i32, iterations), "elementwise.is_done");
3876	_ = try fg.wip.brCond(is_done, done_block, loop_block, .none);
3877
3878	fg.wip.cursor = .{ .block = done_block };
3879	return result_ptr;
3880	}
3881
3882	var result = if (fn_info.return_type == .void_type) .none else if (ret_is_by_ref) result: {
3883	const ret_llvm_ty = try o.lowerType(ret_ty, .in_memory);
3884	break :result try fg.buildAlloca(
3885	if (vector_len) |len| try o.builder.arrayType(len, ret_llvm_ty) else ret_llvm_ty,
3886	ret_ty.abiAlignment(zcu).toLlvm(),
3887	);
3888	} else if (vector_len) |len| try o.builder.poisonValue(
3889	try o.builder.vectorType(.normal, len, try o.lowerType(ret_ty, .as_value)),
3890	) else .none;
3891	for (0..iterations) |index| {
3892	const index_value = try o.builder.intValue(.i32, index);
3893	var arg_elems_buf: [3]Builder.Value = undefined;
3894	const arg_elems = arg_elems_buf[0..arg_values.len];
3895	for (arg_elems, fn_info.param_types, arg_values) |*arg_elem_value, param_type, arg_value| {
3896	const arg_ty: Type = .fromInterned(param_type);
3897	if (isByRef(arg_ty, zcu)) {
3898	const arg_elem_ptr = try fg.ptraddConst(arg_value, index * arg_ty.abiSize(zcu));
3899	arg_elem_value.* = try fg.load(arg_elem_ptr, .none, .fromInterned(param_type), .normal);
3900	} else if (vector_len) |_| {
3901	arg_elem_value.* = try fg.wip.extractElement(arg_value, index_value, "elementwise.arg_elem");
3902	} else arg_elem_value.* = arg_value;
3903	}
3904	const result_elem = try fg.buildCall(.{}, llvm_fn.typeOf(&o.builder), llvm_fn.toValue(&o.builder), fn_info, fn_info.param_types, arg_elems);
3905	if (fn_info.return_type == .void_type) {
3906	assert(result_elem == .none);
3907	} else if (ret_is_by_ref) {
3908	const result_elem_ptr = try fg.ptraddConst(result, index * ret_ty.abiSize(zcu));
3909	try fg.store(result_elem_ptr, .none, result_elem, ret_ty, .normal);
3910	} else if (vector_len) |_| {
3911	result = try fg.wip.insertElement(result, result_elem, index_value, "elementwise.result");
3912	} else {
3913	assert(result == .none);
3914	result = result_elem;
3915	}
3916	}
3917	return result;
3918	}
3919
3920	/// Creates a floating point comparison by lowering to the appropriate
3921	/// hardware instruction or softfloat routine for the target
3922	fn buildFloatCmp(
3923	fg: *FuncGen,
3924	fast: Builder.FastMathKind,
3925	pred: math.CompareOperator,
3926	ty: Type,
3927	params: [2]Builder.Value,
3928	) Allocator.Error!Builder.Value {
3929	const o = fg.object;
3930	const zcu = o.zcu;
3931	const target = zcu.getTarget();
3932	const scalar_ty = ty.scalarType(zcu);
3933
3934	if (intrinsicsAllowed(.compiler_rt, scalar_ty, target)) {
3935	const cond: Builder.FloatCondition = switch (pred) {
3936	.eq => .oeq,
3937	.neq => .une,
3938	.lt => .olt,
3939	.lte => .ole,
3940	.gt => .ogt,
3941	.gte => .oge,
3942	};
3943	return fg.wip.fcmp(fast, cond, params[0], params[1], "");
3944	}
3945
3946	const fn_name = try o.builder.strtabStringFmt("__{s}{s}f2", .{
3947	switch (pred) {
3948	.neq => "ne",
3949	.eq => "eq",
3950	.lt => "lt",
3951	.lte => "le",
3952	.gt => "gt",
3953	.gte => "ge",
3954	},
3955	compilerRtFloatAbbrev(target, scalar_ty.floatBits(target)),
3956	});
3957	const result = try fg.buildElementwiseCall(fn_name, .{
3958	.cc = target.cCallingConvention().?,
3959	.param_types = &.{ scalar_ty.toIntern(), scalar_ty.toIntern() },
3960	.return_type = .i32_type,
3961	}, &params, if (ty.isVector(zcu)) ty.vectorLen(zcu) else null);
3962	return fg.wip.icmp(switch (pred) {
3963	.eq => .eq,
3964	.neq => .ne,
3965	.lt => .slt,
3966	.lte => .sle,
3967	.gt => .sgt,
3968	.gte => .sge,
3969	}, result, try o.builder.splatValue(result.typeOfWip(&fg.wip), .@"0"), "");
3970	}
3971
3972	const FloatOp = enum {
3973	add,
3974	ceil,
3975	cos,
3976	div,
3977	exp,
3978	exp2,
3979	fabs,
3980	floor,
3981	fma,
3982	fmax,
3983	fmin,
3984	fmod,
3985	log,
3986	log10,
3987	log2,
3988	mul,
3989	neg,
3990	round,
3991	sin,
3992	sqrt,
3993	sub,
3994	tan,
3995	trunc,
3996	};
3997
3998	/// Creates a floating point operation (add, sub, fma, sqrt, exp, etc.)
3999	/// by lowering to the appropriate hardware instruction or softfloat
4000	/// routine for the target
4001	fn buildFloatOp(
4002	fg: *FuncGen,
4003	comptime op: FloatOp,
4004	fast: Builder.FastMathKind,
4005	ty: Type,
4006	comptime params_len: usize,
4007	params: [params_len]Builder.Value,
4008	) Allocator.Error!Builder.Value {
4009	const o = fg.object;
4010	const zcu = o.zcu;
4011	const target = zcu.getTarget();
4012	const scalar_ty = ty.scalarType(zcu);
4013
4014	switch (op) {
4015	// Some operations are dedicated LLVM instructions, not available as intrinsics
4016	.neg => if (intrinsicsAllowed(.compiler_rt, scalar_ty, target)) return fg.wip.un(.fneg, params[0], ""),
4017	.add, .sub, .mul, .div, .fmod => if (intrinsicsAllowed(switch (op) {
4018	else => unreachable,
4019	.add, .sub, .mul, .div => .compiler_rt,
4020	.fmod => .libc,
4021	}, scalar_ty, target)) return fg.wip.bin(switch (fast) {
4022	.normal => switch (op) {
4023	.add => .fadd,
4024	.sub => .fsub,
4025	.mul => .fmul,
4026	.div => .fdiv,
4027	.fmod => .frem,
4028	else => unreachable,
4029	},
4030	.fast => switch (op) {
4031	.add => .@"fadd fast",
4032	.sub => .@"fsub fast",
4033	.mul => .@"fmul fast",
4034	.div => .@"fdiv fast",
4035	.fmod => .@"frem fast",
4036	else => unreachable,
4037	},
4038	}, params[0], params[1], ""),
4039	.fma,
4040	.fmax,
4041	.fmin,
4042	.ceil,
4043	.cos,
4044	.exp,
4045	.exp2,
4046	.fabs,
4047	.floor,
4048	.log,
4049	.log10,
4050	.log2,
4051	.round,
4052	.sin,
4053	.sqrt,
4054	.tan,
4055	.trunc,
4056	=> if (intrinsicsAllowed(.libc, scalar_ty, target)) return fg.wip.callIntrinsic(fast, .none, switch (op) {
4057	.fma => .fma,
4058	.fmax => .maxnum,
4059	.fmin => .minnum,
4060	.ceil => .ceil,
4061	.cos => .cos,
4062	.exp => .exp,
4063	.exp2 => .exp2,
4064	.fabs => .fabs,
4065	.floor => .floor,
4066	.log => .log,
4067	.log10 => .log10,
4068	.log2 => .log2,
4069	.round => .round,
4070	.sin => .sin,
4071	.sqrt => .sqrt,
4072	.tan => .tan,
4073	.trunc => .trunc,
4074	else => unreachable,
4075	}, &.{try o.lowerType(ty, .as_value)}, &params, ""),
4076	}
4077
4078	const float_bits = scalar_ty.floatBits(target);
4079	const fn_name = switch (op) {
4080	// In these cases we can generate a softfloat operation by modifying the sign bit using a bitwise operation.
4081	.neg, .fabs => if (isByRef(scalar_ty, zcu)) {
4082	const is_vector = ty.toIntern() != scalar_ty.toIntern();
4083	const result_ptr = try fg.buildZigAlloca(ty, .none);
4084	const entry_block = fg.wip.cursor.block;
4085	const loop_block, const done_block, const llvm_usize_ty, const offset, const elem, const result_elem = if (is_vector) loop: {
4086	const loop_block = try fg.wip.block(2, "neg_fabs.loop");
4087	const done_block = try fg.wip.block(1, "neg_fabs.done");
4088	_ = try fg.wip.br(loop_block);
4089
4090	fg.wip.cursor = .{ .block = loop_block };
4091	const llvm_usize_ty = try o.lowerType(.usize, .as_value);
4092	const offset = try fg.wip.phi(llvm_usize_ty, "neg_fabs.offset");
4093	break :loop .{
4094	loop_block,
4095	done_block,
4096	llvm_usize_ty,
4097	offset,
4098	try fg.ptraddScaled(params[0], offset.toValue(), 1),
4099	try fg.ptraddScaled(result_ptr, offset.toValue(), 1),
4100	};
4101	} else .{ undefined, undefined, undefined, undefined, params[0], result_ptr };
4102	switch (scalar_ty.floatBits(target)) {
4103	else => unreachable,
4104	80 => {
4105	const f80_layout = o.softF80Layout(.{}) catch unreachable;
4106	const mantissa = try fg.load(
4107	try fg.ptraddConst(elem, f80_layout.mantissa_offset),
4108	f80_layout.alignment.offset(f80_layout.mantissa_offset),
4109	.u64,
4110	.normal,
4111	);
4112	const exponent = try fg.load(
4113	try fg.ptraddConst(elem, f80_layout.exponent_offset),
4114	f80_layout.alignment.offset(f80_layout.exponent_offset),
4115	.u16,
4116	.normal,
4117	);
4118	const exponent_sign_bit: u16 = 1 << (16 - 1);
4119	const updated_exponent = try fg.wip.bin(switch (op) {
4120	else => unreachable,
4121	.neg => .xor,
4122	.fabs => .@"and",
4123	}, exponent, try o.builder.intValue(.i16, switch (op) {
4124	else => unreachable,
4125	.neg => exponent_sign_bit,
4126	.fabs => exponent_sign_bit - 1,
4127	}), "neg_fabs.updated_exponent");
4128	try fg.store(
4129	try fg.ptraddConst(result_elem, f80_layout.mantissa_offset),
4130	f80_layout.alignment.offset(f80_layout.mantissa_offset),
4131	mantissa,
4132	.u64,
4133	.normal,
4134	);
4135	try fg.store(
4136	try fg.ptraddConst(result_elem, f80_layout.exponent_offset),
4137	f80_layout.alignment.offset(f80_layout.exponent_offset),
4138	updated_exponent,
4139	.u16,
4140	.normal,
4141	);
4142	},
4143	128 => {
4144	const f128_layout = o.softF128Layout(.{}) catch unreachable;
4145	const lo = try fg.load(
4146	try fg.ptraddConst(elem, f128_layout.lo_offset),
4147	f128_layout.alignment.offset(f128_layout.lo_offset),
4148	.u64,
4149	.normal,
4150	);
4151	const hi = try fg.load(
4152	try fg.ptraddConst(elem, f128_layout.hi_offset),
4153	f128_layout.alignment.offset(f128_layout.hi_offset),
4154	.u64,
4155	.normal,
4156	);
4157	const hi_sign_bit: u64 = 1 << (64 - 1);
4158	const updated_hi = try fg.wip.bin(switch (op) {
4159	else => unreachable,
4160	.neg => .xor,
4161	.fabs => .@"and",
4162	}, hi, try o.builder.intValue(.i64, switch (op) {
4163	else => unreachable,
4164	.neg => hi_sign_bit,
4165	.fabs => hi_sign_bit - 1,
4166	}), "neg_fabs.updated_hi");
4167	try fg.store(
4168	try fg.ptraddConst(result_elem, f128_layout.lo_offset),
4169	f128_layout.alignment.offset(f128_layout.lo_offset),
4170	lo,
4171	.u64,
4172	.normal,
4173	);
4174	try fg.store(
4175	try fg.ptraddConst(result_elem, f128_layout.hi_offset),
4176	f128_layout.alignment.offset(f128_layout.hi_offset),
4177	updated_hi,
4178	.u64,
4179	.normal,
4180	);
4181	},
4182	}
4183	if (is_vector) {
4184	const next_offset = try fg.wip.bin(.@"add nuw", offset.toValue(), try o.builder.intValue(llvm_usize_ty, scalar_ty.abiSize(zcu)), "neg_fabs.next_offset");
4185	offset.finish(&.{ try o.builder.intValue(llvm_usize_ty, 0), next_offset }, &.{ entry_block, loop_block }, &fg.wip);
4186	const is_done = try fg.wip.icmp(.eq, next_offset, try o.builder.intValue(llvm_usize_ty, ty.abiSize(zcu)), "neg_fabs.is_done");
4187	_ = try fg.wip.brCond(is_done, done_block, loop_block, .none);
4188
4189	fg.wip.cursor = .{ .block = done_block };
4190	}
4191	return result_ptr;
4192	} else {
4193	const int_ty = try o.builder.intType(@intCast(float_bits));
4194	const cast_ty = switch (ty.zigTypeTag(zcu)) {
4195	.vector => try o.builder.vectorType(.normal, ty.vectorLen(zcu), int_ty),
4196	else => int_ty,
4197	};
4198	const sign_bit = @as(u128, 1) << @intCast(float_bits - 1);
4199	const bitwise_rhs = try o.builder.splatValue(cast_ty, try o.builder.intConst(int_ty, switch (op) {
4200	else => unreachable,
4201	.neg => sign_bit,
4202	.fabs => sign_bit - 1,
4203	}));
4204	const bitcasted_operand = try fg.wip.cast(.bitcast, params[0], cast_ty, "");
4205	const result = try fg.wip.bin(switch (op) {
4206	else => unreachable,
4207	.neg => .xor,
4208	.fabs => .@"and",
4209	}, bitcasted_operand, bitwise_rhs, "");
4210	const llvm_ty = try o.lowerType(ty, .as_value);
4211	return fg.wip.cast(.bitcast, result, llvm_ty, "");
4212	},
4213	.add, .sub, .div, .mul => try o.builder.strtabStringFmt("__{s}{s}f3", .{
4214	@tagName(op), compilerRtFloatAbbrev(target, float_bits),
4215	}),
4216	.ceil,
4217	.cos,
4218	.exp,
4219	.exp2,
4220	.floor,
4221	.fma,
4222	.fmax,
4223	.fmin,
4224	.fmod,
4225	.log,
4226	.log10,
4227	.log2,
4228	.round,
4229	.sin,
4230	.sqrt,
4231	.tan,
4232	.trunc,
4233	=> try o.builder.strtabStringFmt("{s}{s}{s}", .{
4234	libcFloatPrefix(float_bits), @tagName(op), libcFloatSuffix(float_bits),
4235	}),
4236	};
4237	return fg.buildElementwiseCall(fn_name, .{
4238	.cc = target.cCallingConvention().?,
4239	.param_types = &@as([params_len]InternPool.Index, @splat(scalar_ty.toIntern())),
4240	.return_type = scalar_ty.toIntern(),
4241	}, &params, if (ty.isVector(zcu)) ty.vectorLen(zcu) else null);
4242	}
4243
4244	/// Creates a floating point cast operation by lowering to the specified softfloat routine.
4245	fn buildFloatCastCall(
4246	fg: *FuncGen,
4247	dest_ty: Type,
4248	fn_name: Builder.StrtabString,
4249	operand_ty: Type,
4250	operand: Builder.Value,
4251	) Allocator.Error!Builder.Value {
4252	const zcu = fg.object.zcu;
4253	return fg.buildElementwiseCall(fn_name, .{
4254	.cc = zcu.getTarget().cCallingConvention().?,
4255	.param_types = &.{operand_ty.scalarType(zcu).toIntern()},
4256	.return_type = dest_ty.scalarType(zcu).toIntern(),
4257	}, &.{operand}, if (operand_ty.isVector(zcu)) operand_ty.vectorLen(zcu) else null);
4258	}
4259
4260	fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4261	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
4262	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
4263
4264	const mulend1 = try self.resolveInst(extra.lhs);
4265	const mulend2 = try self.resolveInst(extra.rhs);
4266	const addend = try self.resolveInst(pl_op.operand);
4267
4268	const ty = self.typeOfIndex(inst);
4269	return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend });
4270	}
4271
4272	fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4273	const o = self.object;
4274	const zcu = o.zcu;
4275	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
4276	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
4277
4278	const lhs = try self.resolveInst(extra.lhs);
4279	const rhs = try self.resolveInst(extra.rhs);
4280
4281	const lhs_ty = self.typeOf(extra.lhs);
4282	if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu)) {
4283	// `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize`
4284	// features which we do not use. Therefore this branch is currently impossible.
4285	unreachable;
4286	}
4287
4288	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
4289
4290	const dest_ty = self.typeOfIndex(inst);
4291	assert(isByRef(dest_ty, zcu)); // auto structs are by-ref
4292
4293	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .as_value), "");
4294
4295	const result = try self.wip.bin(.shl, lhs, casted_rhs, "");
4296	const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
4297	.ashr
4298	else
4299	.lshr, result, casted_rhs, "");
4300
4301	const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
4302
4303	const result_alignment = dest_ty.abiAlignment(zcu);
4304	const alloca_inst = try self.buildZigAlloca(dest_ty, .none);
4305
4306	{
4307	// Store to 'result: IntType' field
4308	const field_off = dest_ty.structFieldOffset(0, zcu);
4309	const field_ptr = try self.ptraddConst(alloca_inst, field_off);
4310	try self.store(field_ptr, result_alignment.offset(field_off), result, lhs_ty, .normal);
4311	}
4312
4313	{
4314	// Store to 'overflow: u1' field
4315	const field_off = dest_ty.structFieldOffset(1, zcu);
4316	const field_ptr = try self.ptraddConst(alloca_inst, field_off);
4317	try self.store(field_ptr, result_alignment.offset(field_off), overflow_bit, dest_ty.fieldType(1, zcu), .normal);
4318	}
4319
4320	return alloca_inst;
4321	}
4322
4323	fn airAnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4324	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4325	const lhs = try self.resolveInst(bin_op.lhs);
4326	const rhs = try self.resolveInst(bin_op.rhs);
4327	return self.wip.bin(.@"and", lhs, rhs, "");
4328	}
4329
4330	fn airOr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4331	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4332	const lhs = try self.resolveInst(bin_op.lhs);
4333	const rhs = try self.resolveInst(bin_op.rhs);
4334	return self.wip.bin(.@"or", lhs, rhs, "");
4335	}
4336
4337	fn airXor(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4338	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4339	const lhs = try self.resolveInst(bin_op.lhs);
4340	const rhs = try self.resolveInst(bin_op.rhs);
4341	return self.wip.bin(.xor, lhs, rhs, "");
4342	}
4343
4344	fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4345	const o = self.object;
4346	const zcu = o.zcu;
4347	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4348
4349	const lhs = try self.resolveInst(bin_op.lhs);
4350	const rhs = try self.resolveInst(bin_op.rhs);
4351
4352	const lhs_ty = self.typeOf(bin_op.lhs);
4353	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) {
4354	// `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize`
4355	// features which we do not use. Therefore this branch is currently impossible.
4356	unreachable;
4357	}
4358	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
4359
4360	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .as_value), "");
4361	return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
4362	.@"shl nsw"
4363	else
4364	.@"shl nuw", lhs, casted_rhs, "");
4365	}
4366
4367	fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4368	const o = self.object;
4369	const zcu = o.zcu;
4370	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4371
4372	const lhs = try self.resolveInst(bin_op.lhs);
4373	const rhs = try self.resolveInst(bin_op.rhs);
4374
4375	const lhs_ty = self.typeOf(bin_op.lhs);
4376	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) {
4377	// `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize`
4378	// features which we do not use. Therefore this branch is currently impossible.
4379	unreachable;
4380	}
4381	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .as_value), "");
4382	return self.wip.bin(.shl, lhs, casted_rhs, "");
4383	}
4384
4385	fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4386	const o = self.object;
4387	const zcu = o.zcu;
4388	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4389
4390	const lhs = try self.resolveInst(bin_op.lhs);
4391	const rhs = try self.resolveInst(bin_op.rhs);
4392
4393	const lhs_ty = self.typeOf(bin_op.lhs);
4394	const lhs_info = lhs_ty.intInfo(zcu);
4395	const llvm_lhs_ty = try o.lowerType(lhs_ty, .as_value);
4396	const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu), .as_value);
4397
4398	const rhs_ty = self.typeOf(bin_op.rhs);
4399	if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) {
4400	// `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize`
4401	// features which we do not use. Therefore this branch is currently impossible.
4402	unreachable;
4403	}
4404	const rhs_info = rhs_ty.intInfo(zcu);
4405	assert(rhs_info.signedness == .unsigned);
4406	const llvm_rhs_ty = try o.lowerType(rhs_ty, .as_value);
4407	const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu), .as_value);
4408
4409	const result = try self.wip.callIntrinsic(
4410	.normal,
4411	.none,
4412	switch (lhs_info.signedness) {
4413	.signed => .@"sshl.sat",
4414	.unsigned => .@"ushl.sat",
4415	},
4416	&.{llvm_lhs_ty},
4417	&.{ lhs, try self.wip.conv(.unsigned, rhs, llvm_lhs_ty, "") },
4418	"",
4419	);
4420
4421	// LLVM langref says "If b is (statically or dynamically) equal to or
4422	// larger than the integer bit width of the arguments, the result is a
4423	// poison value."
4424	// However Zig semantics says that saturating shift left can never produce
4425	// undefined; instead it saturates.
4426	if (rhs_info.bits <= math.log2_int(u16, lhs_info.bits)) return result;
4427	const bits = try o.builder.splatValue(
4428	llvm_rhs_ty,
4429	try o.builder.intConst(llvm_rhs_scalar_ty, lhs_info.bits),
4430	);
4431	const in_range = try self.wip.icmp(.ult, rhs, bits, "");
4432	const lhs_sat = lhs_sat: switch (lhs_info.signedness) {
4433	.signed => {
4434	const zero = try o.builder.splatValue(
4435	llvm_lhs_ty,
4436	try o.builder.intConst(llvm_lhs_scalar_ty, 0),
4437	);
4438	const smin = try o.builder.splatValue(
4439	llvm_lhs_ty,
4440	try minIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
4441	);
4442	const smax = try o.builder.splatValue(
4443	llvm_lhs_ty,
4444	try maxIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
4445	);
4446	const lhs_lt_zero = try self.wip.icmp(.slt, lhs, zero, "");
4447	const slimit = try self.wip.select(.normal, lhs_lt_zero, smin, smax, "");
4448	const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
4449	break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, slimit, "");
4450	},
4451	.unsigned => {
4452	const zero = try o.builder.splatValue(
4453	llvm_lhs_ty,
4454	try o.builder.intConst(llvm_lhs_scalar_ty, 0),
4455	);
4456	const umax = try o.builder.splatValue(
4457	llvm_lhs_ty,
4458	try o.builder.intConst(llvm_lhs_scalar_ty, -1),
4459	);
4460	const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
4461	break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, umax, "");
4462	},
4463	};
4464	return self.wip.select(.normal, in_range, result, lhs_sat, "");
4465	}
4466
4467	fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!Builder.Value {
4468	const o = self.object;
4469	const zcu = o.zcu;
4470	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4471
4472	const lhs = try self.resolveInst(bin_op.lhs);
4473	const rhs = try self.resolveInst(bin_op.rhs);
4474
4475	const lhs_ty = self.typeOf(bin_op.lhs);
4476	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) {
4477	// `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize`
4478	// features which we do not use. Therefore this branch is currently impossible.
4479	unreachable;
4480	}
4481	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
4482
4483	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .as_value), "");
4484	const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);
4485
4486	return self.wip.bin(if (is_exact)
4487	if (is_signed_int) .@"ashr exact" else .@"lshr exact"
4488	else if (is_signed_int) .ashr else .lshr, lhs, casted_rhs, "");
4489	}
4490
4491	fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4492	const o = self.object;
4493	const zcu = o.zcu;
4494	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4495	const operand = try self.resolveInst(ty_op.operand);
4496	const operand_ty = self.typeOf(ty_op.operand);
4497	const scalar_ty = operand_ty.scalarType(zcu);
4498
4499	switch (scalar_ty.zigTypeTag(zcu)) {
4500	.int => return self.wip.callIntrinsic(
4501	.normal,
4502	.none,
4503	.abs,
4504	&.{try o.lowerType(operand_ty, .as_value)},
4505	&.{ operand, .false },
4506	"",
4507	),
4508	.float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),
4509	else => unreachable,
4510	}
4511	}
4512
4513	fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
4514	const o = fg.object;
4515	const zcu = o.zcu;
4516	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4517	const dest_ty = fg.typeOfIndex(inst);
4518	const dest_llvm_ty = try o.lowerType(dest_ty, .as_value);
4519	const operand = try fg.resolveInst(ty_op.operand);
4520	const operand_ty = fg.typeOf(ty_op.operand);
4521	const operand_info = operand_ty.intInfo(zcu);
4522
4523	const dest_is_enum = dest_ty.zigTypeTag(zcu) == .@"enum";
4524
4525	bounds_check: {
4526	const dest_scalar = dest_ty.scalarType(zcu);
4527	const operand_scalar = operand_ty.scalarType(zcu);
4528
4529	const dest_info = dest_ty.intInfo(zcu);
4530
4531	const have_min_check, const have_max_check = c: {
4532	const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed);
4533	const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed);
4534
4535	const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0;
4536	const operand_maybe_neg = operand_info.signedness == .signed and operand_info.bits > 0;
4537
4538	break :c .{
4539	operand_maybe_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits),
4540	dest_pos_bits < operand_pos_bits,
4541	};
4542	};
4543
4544	if (!have_min_check and !have_max_check) break :bounds_check;
4545
4546	const operand_llvm_ty = try o.lowerType(operand_ty, .as_value);
4547	const operand_scalar_llvm_ty = try o.lowerType(operand_scalar, .as_value);
4548
4549	const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
4550	assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));
4551
4552	const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .integer_out_of_bounds;
4553
4554	if (have_min_check) {
4555	const min_const_scalar = try minIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
4556	const min_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, min_const_scalar) else min_const_scalar.toValue();
4557	const ok_maybe_vec = try fg.cmp(.normal, .gte, operand_ty, operand, min_val);
4558	const ok = if (is_vector) ok: {
4559	const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
4560	break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
4561	} else ok_maybe_vec;
4562	if (safety) {
4563	const fail_block = try fg.wip.block(1, "IntMinFail");
4564	const ok_block = try fg.wip.block(1, "IntMinOk");
4565	_ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
4566	fg.wip.cursor = .{ .block = fail_block };
4567	try fg.buildSimplePanic(panic_id);
4568	fg.wip.cursor = .{ .block = ok_block };
4569	} else {
4570	_ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
4571	}
4572	}
4573
4574	if (have_max_check) {
4575	const max_const_scalar = try maxIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
4576	const max_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, max_const_scalar) else max_const_scalar.toValue();
4577	const ok_maybe_vec = try fg.cmp(.normal, .lte, operand_ty, operand, max_val);
4578	const ok = if (is_vector) ok: {
4579	const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
4580	break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
4581	} else ok_maybe_vec;
4582	if (safety) {
4583	const fail_block = try fg.wip.block(1, "IntMaxFail");
4584	const ok_block = try fg.wip.block(1, "IntMaxOk");
4585	_ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
4586	fg.wip.cursor = .{ .block = fail_block };
4587	try fg.buildSimplePanic(panic_id);
4588	fg.wip.cursor = .{ .block = ok_block };
4589	} else {
4590	_ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
4591	}
4592	}
4593	}
4594
4595	const result = try fg.wip.conv(switch (operand_info.signedness) {
4596	.signed => .signed,
4597	.unsigned => .unsigned,
4598	}, operand, dest_llvm_ty, "");
4599
4600	if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) {
4601	const llvm_fn = try o.getIsNamedEnumValueFunction(dest_ty);
4602	const is_valid_enum_val = try fg.wip.call(
4603	.normal,
4604	.fastcc,
4605	.none,
4606	llvm_fn.typeOf(&o.builder),
4607	llvm_fn.toValue(&o.builder),
4608	&.{result},
4609	"",
4610	);
4611	const fail_block = try fg.wip.block(1, "ValidEnumFail");
4612	const ok_block = try fg.wip.block(1, "ValidEnumOk");
4613	_ = try fg.wip.brCond(is_valid_enum_val, ok_block, fail_block, .none);
4614	fg.wip.cursor = .{ .block = fail_block };
4615	try fg.buildSimplePanic(.invalid_enum_value);
4616	fg.wip.cursor = .{ .block = ok_block };
4617	}
4618
4619	return result;
4620	}
4621
4622	fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4623	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4624	const operand = try self.resolveInst(ty_op.operand);
4625	const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst), .as_value);
4626	return self.wip.cast(.trunc, operand, dest_llvm_ty, "");
4627	}
4628
4629	fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4630	const o = self.object;
4631	const zcu = o.zcu;
4632	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4633	const operand = try self.resolveInst(ty_op.operand);
4634	const operand_ty = self.typeOf(ty_op.operand);
4635	const operand_scalar_ty = operand_ty.scalarType(zcu);
4636	const dest_ty = self.typeOfIndex(inst);
4637	const dest_scalar_ty = dest_ty.scalarType(zcu);
4638	const target = zcu.getTarget();
4639
4640	if (intrinsicsAllowed(.compiler_rt, dest_scalar_ty, target) and
4641	intrinsicsAllowed(.compiler_rt, operand_scalar_ty, target))
4642	return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty, .as_value), "");
4643	const dest_bits = dest_scalar_ty.floatBits(target);
4644	const src_bits = operand_scalar_ty.floatBits(target);
4645	const fn_name = try o.builder.strtabStringFmt("__trunc{s}f{s}f2", .{
4646	compilerRtFloatAbbrev(target, src_bits), compilerRtFloatAbbrev(target, dest_bits),
4647	});
4648	return self.buildFloatCastCall(dest_ty, fn_name, operand_ty, operand);
4649	}
4650
4651	fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4652	const o = self.object;
4653	const zcu = o.zcu;
4654	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4655	const operand = try self.resolveInst(ty_op.operand);
4656	const operand_ty = self.typeOf(ty_op.operand);
4657	const operand_scalar_ty = operand_ty.scalarType(zcu);
4658	const dest_ty = self.typeOfIndex(inst);
4659	const dest_scalar_ty = dest_ty.scalarType(zcu);
4660	const target = zcu.getTarget();
4661
4662	if (intrinsicsAllowed(.compiler_rt, dest_scalar_ty, target) and
4663	intrinsicsAllowed(.compiler_rt, operand_scalar_ty, target))
4664	return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty, .as_value), "");
4665	const dest_bits = dest_scalar_ty.floatBits(target);
4666	const src_bits = operand_scalar_ty.floatBits(target);
4667	const fn_name = try o.builder.strtabStringFmt("__extend{s}f{s}f2", .{
4668	compilerRtFloatAbbrev(target, src_bits), compilerRtFloatAbbrev(target, dest_bits),
4669	});
4670	return self.buildFloatCastCall(dest_ty, fn_name, operand_ty, operand);
4671	}
4672
4673	fn airBitCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
4674	const o = fg.object;
4675	const zcu = o.zcu;
4676
4677	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4678	const operand_ty = fg.typeOf(ty_op.operand);
4679	const dest_ty = fg.typeOfIndex(inst);
4680	const operand = try fg.resolveInst(ty_op.operand);
4681
4682	// We have the following `Air.Legalize` features enabled:
4683	//
4684	// * `.scalarize_bit_cast_array`
4685	// * `.scalarize_bit_cast_vector_non_elementwise`
4686	//
4687	// That means the `bit_cast` instructions we might see are limited to the following:
4688	//
4689	// * bool/int/float <-> bool/int/float
4690	// * `@Vector(n, A)` <-> `@Vector(n, B)`
4691	//
4692	// Most of these cases can be handled by LLVM's `bitcast` instruction, except when
4693	// a non-native type like `f80` is used.
4694
4695	if (isByRef(operand_ty, zcu)) {
4696	const operand_scalar_ty = operand_ty.scalarType(zcu);
4697	const target = zcu.getTarget();
4698	const bits = operand_scalar_ty.floatBits(target);
4699	const dest_scalar_ty = dest_ty.scalarType(zcu);
4700	if (isByRef(dest_ty, zcu)) {
4701	assert(dest_scalar_ty.floatBits(target) == bits);
4702	return operand;
4703	}
4704	assert(dest_scalar_ty.intInfo(zcu).bits == bits);
4705
4706	const len = if (operand_ty.toIntern() != operand_scalar_ty.toIntern())
4707	operand_ty.vectorLen(zcu)
4708	else
4709	null;
4710	const operand_scalar_size = operand_scalar_ty.abiSize(zcu);
4711	var result = if (len) |_|
4712	try o.builder.poisonValue(try o.lowerType(dest_ty, .as_value))
4713	else
4714	undefined;
4715	for (0..len orelse 1) |index| {
4716	const result_elem = result_elem: switch (bits) {
4717	else => unreachable,
4718	80 => {
4719	const f80_layout = o.softF80Layout(.{}) catch unreachable;
4720	const mantissa = try fg.load(
4721	try fg.ptraddConst(operand, operand_scalar_size * index + f80_layout.mantissa_offset),
4722	f80_layout.alignment.offset(f80_layout.mantissa_offset),
4723	.u64,
4724	.normal,
4725	);
4726	const exponent = try fg.load(
4727	try fg.ptraddConst(operand, operand_scalar_size * index + f80_layout.exponent_offset),
4728	f80_layout.alignment.offset(f80_layout.exponent_offset),
4729	.u16,
4730	.normal,
4731	);
4732	const casted_mantissa = try fg.wip.cast(.zext, mantissa, .i80, "bitCast.casted_mantissa");
4733	const casted_exponent = try fg.wip.cast(.zext, exponent, .i80, "bitCast.casted_exponent");
4734	const shifted_exponent = try fg.wip.bin(.@"shl nuw", casted_exponent, try o.builder.intValue(.i80, 64), "bitCast.shifted_exponent");
4735	break :result_elem try fg.wip.bin(.@"or", casted_mantissa, shifted_exponent, "bitCast.result_elem");
4736	},
4737	128 => {
4738	const f128_layout = o.softF128Layout(.{}) catch unreachable;
4739	const lo = try fg.load(
4740	try fg.ptraddConst(operand, operand_scalar_size * index + f128_layout.lo_offset),
4741	f128_layout.alignment.offset(f128_layout.lo_offset),
4742	.u64,
4743	.normal,
4744	);
4745	const hi = try fg.load(
4746	try fg.ptraddConst(operand, operand_scalar_size * index + f128_layout.hi_offset),
4747	f128_layout.alignment.offset(f128_layout.hi_offset),
4748	.u64,
4749	.normal,
4750	);
4751	const casted_lo = try fg.wip.cast(.zext, lo, .i128, "bitCast.casted_lo");
4752	const casted_hi = try fg.wip.cast(.zext, hi, .i128, "bitCast.casted_hi");
4753	const shifted_hi = try fg.wip.bin(.@"shl nuw", casted_hi, try o.builder.intValue(.i128, 64), "bitCast.shifted_hi");
4754	break :result_elem try fg.wip.bin(.@"or", casted_lo, shifted_hi, "bitCast.result_elem");
4755	},
4756	};
4757	result = if (len) |_|
4758	try fg.wip.insertElement(result, result_elem, try o.builder.intValue(.i32, index), "elementwise.result")
4759	else
4760	result_elem;
4761	}
4762	return result;
4763	}
4764
4765	if (isByRef(dest_ty, zcu)) {
4766	const dest_scalar_ty = dest_ty.scalarType(zcu);
4767	const bits = dest_scalar_ty.floatBits(zcu.getTarget());
4768	assert(dest_scalar_ty.isRuntimeFloat());
4769	const operand_scalar_ty = operand_ty.scalarType(zcu);
4770	assert(operand_scalar_ty.intInfo(zcu).bits == bits);
4771
4772	const len = if (operand_ty.toIntern() != operand_scalar_ty.toIntern())
4773	operand_ty.vectorLen(zcu)
4774	else
4775	null;
4776	const operand_scalar_size = operand_scalar_ty.abiSize(zcu);
4777	const result_ptr = try fg.buildZigAlloca(dest_ty, .none);
4778	for (0..len orelse 1) |index| {
4779	const operand_elem = if (len) |_|
4780	try fg.wip.extractElement(operand, try o.builder.intValue(.i32, index), "elementwise.operand_elem")
4781	else
4782	operand;
4783	switch (bits) {
4784	else => unreachable,
4785	80 => {
4786	const f80_layout = o.softF80Layout(.{}) catch unreachable;
4787	const mantissa = try fg.wip.cast(.trunc, operand_elem, .i64, "bitCast.mantissa");
4788	const shifted_exponent = try fg.wip.bin(.lshr, operand_elem, try o.builder.intValue(.i80, 64), "bitCast.shifted_exponent");
4789	const exponent = try fg.wip.cast(.@"trunc nuw", shifted_exponent, .i16, "bitCast.exponent");
4790	try fg.store(
4791	try fg.ptraddConst(result_ptr, operand_scalar_size * index + f80_layout.mantissa_offset),
4792	f80_layout.alignment.offset(f80_layout.mantissa_offset),
4793	mantissa,
4794	.u64,
4795	.normal,
4796	);
4797	try fg.store(
4798	try fg.ptraddConst(result_ptr, operand_scalar_size * index + f80_layout.exponent_offset),
4799	f80_layout.alignment.offset(f80_layout.exponent_offset),
4800	exponent,
4801	.u16,
4802	.normal,
4803	);
4804	},
4805	128 => {
4806	const f128_layout = o.softF128Layout(.{}) catch unreachable;
4807	const lo = try fg.wip.cast(.trunc, operand_elem, .i64, "bitCast.lo");
4808	const shifted_hi = try fg.wip.bin(.lshr, operand_elem, try o.builder.intValue(.i128, 64), "bitCast.shifted_hi");
4809	const hi = try fg.wip.cast(.@"trunc nuw", shifted_hi, .i64, "bitCast.hi");
4810	try fg.store(
4811	try fg.ptraddConst(result_ptr, operand_scalar_size * index + f128_layout.lo_offset),
4812	f128_layout.alignment.offset(f128_layout.lo_offset),
4813	lo,
4814	.u64,
4815	.normal,
4816	);
4817	try fg.store(
4818	try fg.ptraddConst(result_ptr, operand_scalar_size * index + f128_layout.hi_offset),
4819	f128_layout.alignment.offset(f128_layout.hi_offset),
4820	hi,
4821	.u64,
4822	.normal,
4823	);
4824	},
4825	}
4826	}
4827	return result_ptr;
4828	}
4829
4830	const llvm_dest_ty = try o.lowerType(dest_ty, .as_value);
4831	const result = try fg.wip.cast(.bitcast, operand, llvm_dest_ty, "");
4832	if (safety and dest_ty.zigTypeTag(zcu) == .@"enum" and !dest_ty.isNonexhaustiveEnum(zcu)) {
4833	const llvm_fn = try o.getIsNamedEnumValueFunction(dest_ty);
4834	const is_valid_enum_val = try fg.wip.call(
4835	.normal,
4836	.fastcc,
4837	.none,
4838	llvm_fn.typeOf(&o.builder),
4839	llvm_fn.toValue(&o.builder),
4840	&.{result},
4841	"",
4842	);
4843	const fail_block = try fg.wip.block(1, "ValidEnumFail");
4844	const ok_block = try fg.wip.block(1, "ValidEnumOk");
4845	_ = try fg.wip.brCond(is_valid_enum_val, ok_block, fail_block, .none);
4846	fg.wip.cursor = .{ .block = fail_block };
4847	try fg.buildSimplePanic(.invalid_enum_value);
4848	fg.wip.cursor = .{ .block = ok_block };
4849	}
4850	return result;
4851	}
4852
4853	fn airNopCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4854	const zcu = fg.object.zcu;
4855	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4856	const operand_ty = fg.typeOf(ty_op.operand);
4857	const dest_ty = fg.typeOfIndex(inst);
4858	assert(isByRef(operand_ty, zcu) == isByRef(dest_ty, zcu));
4859	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
4860	return fg.resolveInst(ty_op.operand);
4861	}
4862
4863	fn airPtrFromInt(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4864	const o = fg.object;
4865	const zcu = o.zcu;
4866	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4867	const operand_ty = fg.typeOf(ty_op.operand);
4868	const dest_ty = fg.typeOfIndex(inst);
4869	assert(operand_ty.scalarType(zcu).toIntern() == .usize_type);
4870	assert(dest_ty.scalarType(zcu).isPtrAtRuntime(zcu));
4871
4872	const operand = try fg.resolveInst(ty_op.operand);
4873	const llvm_dest_ty = try o.lowerType(dest_ty, .as_value);
4874	return fg.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
4875	}
4876
4877	fn airIntFromPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4878	const o = fg.object;
4879	const zcu = o.zcu;
4880	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4881	const operand_ty = fg.typeOf(ty_op.operand);
4882	const dest_ty = fg.typeOfIndex(inst);
4883	assert(operand_ty.scalarType(zcu).isPtrAtRuntime(zcu));
4884	assert(dest_ty.scalarType(zcu).toIntern() == .usize_type);
4885
4886	const operand = try fg.resolveInst(ty_op.operand);
4887	const llvm_dest_ty = try o.lowerType(dest_ty, .as_value);
4888	return fg.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
4889	}
4890
4891	fn airUnionFromEnum(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4892	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4893	const enum_ty = fg.typeOf(ty_op.operand);
4894	const union_ty = fg.typeOfIndex(inst);
4895	const enum_val = try fg.resolveInst(ty_op.operand);
4896	const union_ptr = try fg.buildZigAlloca(union_ty, .none);
4897	try fg.store(union_ptr, .none, enum_val, enum_ty, .normal);
4898	return union_ptr;
4899	}
4900
4901	fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4902	const o = self.object;
4903	const pt = self.pt;
4904	const zcu = o.zcu;
4905	const arg_val = self.args[self.arg_index];
4906	self.arg_index += 1;
4907
4908	// llvm does not support debug info for naked function arguments
4909	if (self.is_naked) return arg_val;
4910
4911	const inst_ty = self.typeOfIndex(inst);
4912
4913	const func = zcu.funcInfo(zcu.navValue(self.nav_index).toIntern());
4914	const func_zir = func.zir_body_inst.resolveFull(&zcu.intern_pool).?;
4915	const file = zcu.fileByIndex(func_zir.file);
4916
4917	const mod = file.mod.?;
4918	if (mod.strip) return arg_val;
4919	const arg = self.air.instructions.items(.data)[@backingInt(inst)].arg;
4920	const zir = &file.zir.?;
4921	const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
4922
4923	const lbrace_line = zcu.navSrcLine(func.owner_nav) + func.lbrace_line + 1;
4924	const lbrace_col = func.lbrace_column + 1;
4925
4926	const debug_parameter = try o.builder.debugParameter(
4927	if (name.len > 0) try o.builder.metadataString(name) else null,
4928	self.file,
4929	self.scope,
4930	lbrace_line,
4931	try o.getDebugType(pt, inst_ty),
4932	self.arg_index,
4933	);
4934
4935	const old_location = self.wip.debug_location;
4936	self.wip.debug_location = .{ .location = .{
4937	.line = lbrace_line,
4938	.column = lbrace_col,
4939	.scope = self.scope.toOptional(),
4940	.inlined_at = .none,
4941	} };
4942
4943	if (isByRef(inst_ty, zcu)) {
4944	_ = try self.wip.callIntrinsic(
4945	.normal,
4946	.none,
4947	.@"dbg.declare",
4948	&.{},
4949	&.{
4950	(try self.wip.debugValue(arg_val)).toValue(),
4951	debug_parameter.toValue(),
4952	(try o.builder.debugExpression(&.{})).toValue(),
4953	},
4954	"",
4955	);
4956	} else if (mod.optimize_mode == .debug) {
4957	const alloca = try self.buildZigAlloca(inst_ty, .none);
4958	try self.store(alloca, .none, arg_val, inst_ty, .normal);
4959	_ = try self.wip.callIntrinsic(
4960	.normal,
4961	.none,
4962	.@"dbg.declare",
4963	&.{},
4964	&.{
4965	(try self.wip.debugValue(alloca)).toValue(),
4966	debug_parameter.toValue(),
4967	(try o.builder.debugExpression(&.{})).toValue(),
4968	},
4969	"",
4970	);
4971	} else {
4972	_ = try self.wip.callIntrinsic(
4973	.normal,
4974	.none,
4975	.@"dbg.value",
4976	&.{},
4977	&.{
4978	(try self.wip.debugValue(arg_val)).toValue(),
4979	debug_parameter.toValue(),
4980	(try o.builder.debugExpression(&.{})).toValue(),
4981	},
4982	"",
4983	);
4984	}
4985
4986	self.wip.debug_location = old_location;
4987	return arg_val;
4988	}
4989
4990	fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4991	const o = self.object;
4992	const zcu = o.zcu;
4993	const ptr_ty = self.typeOfIndex(inst);
4994	const ptr_align = ptr_ty.ptrAlignment(zcu);
4995	const elem_ty = ptr_ty.childType(zcu);
4996	if (!elem_ty.hasRuntimeBits(zcu)) {
4997	return (try o.lowerPtrToVoid(ptr_align.toLlvm(), ptr_ty.ptrAddressSpace(zcu))).toValue();
4998	}
4999	return self.buildZigAlloca(elem_ty, ptr_align);
5000	}
5001
5002	fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5003	if (self.ret_ptr != .none) return self.ret_ptr;
5004	const o = self.object;
5005	const zcu = o.zcu;
5006	const ptr_ty = self.typeOfIndex(inst);
5007	const ptr_align = ptr_ty.ptrAlignment(zcu);
5008	const elem_ty = ptr_ty.childType(zcu);
5009	if (!elem_ty.hasRuntimeBits(zcu)) {
5010	return (try o.lowerPtrToVoid(ptr_align.toLlvm(), ptr_ty.ptrAddressSpace(zcu))).toValue();
5011	}
5012	return self.buildZigAlloca(elem_ty, ptr_align);
5013	}
5014
5015	fn buildZigAlloca(fg: *FuncGen, ty: Type, @"align": InternPool.Alignment) Allocator.Error!Builder.Value {
5016	const o = fg.object;
5017	const resolved_align: InternPool.Alignment = switch (@"align") {
5018	.none => ty.abiAlignment(o.zcu),
5019	else => |a| a,
5020	};
5021	return fg.buildAlloca(try o.lowerType(ty, .in_memory), resolved_align.toLlvm());
5022	}
5023
5024	/// Unlike `WipFunction.alloca`, this puts the alloca instruction at the top of the function.
5025	fn buildAlloca(
5026	fg: *FuncGen,
5027	llvm_ty: Builder.Type,
5028	alignment: Builder.Alignment,
5029	) Allocator.Error!Builder.Value {
5030	const wip = &fg.wip;
5031
5032	const alloca = blk: {
5033	const prev_cursor = wip.cursor;
5034	const prev_debug_location = wip.debug_location;
5035	defer {
5036	wip.cursor = prev_cursor;
5037	if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
5038	wip.debug_location = prev_debug_location;
5039	}
5040
5041	wip.cursor = .{ .block = .entry };
5042	wip.debug_location = .no_location;
5043	const address_space = llvmAllocaAddressSpace(fg.object.zcu.getTarget());
5044	break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
5045	};
5046
5047	// The pointer returned from this function should have the generic address space,
5048	// if this isn't the case then cast it to the generic address space.
5049	return fg.wip.conv(.unneeded, alloca, .ptr, "");
5050	}
5051
5052	fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
5053	const o = fg.object;
5054	const zcu = o.zcu;
5055	const bin_op = fg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5056	const ptr = try fg.resolveInst(bin_op.lhs);
5057	const ptr_ty = fg.typeOf(bin_op.lhs);
5058	const ptr_info = ptr_ty.ptrInfo(zcu);
5059	const ptr_alignment = ptr_ty.ptrAlignment(zcu);
5060
5061	const elem_ty = fg.typeOf(bin_op.rhs);
5062	assert(elem_ty.hasRuntimeBits(zcu));
5063
5064	fg.maybeMarkAllowZeroAccess(ptr_info);
5065
5066	const access_kind: Builder.MemoryAccessKind = switch (ptr_info.flags.is_volatile) {
5067	true => .@"volatile",
5068	false => .normal,
5069	};
5070
5071	const val_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
5072	if (val_is_undef and !fg.needMemsetWorkaround(elem_ty.abiSize(zcu))) {
5073	const owner_mod = fg.ownerModule();
5074
5075	// Even if safety is disabled, we still emit a memset to undefined since it conveys
5076	// extra information to LLVM, and LLVM will optimize it out. Safety makes the difference
5077	// between using 0xaa or actual undefined for the fill byte.
5078	//
5079	// However, for Debug builds specifically, we avoid emitting the memset because LLVM
5080	// will neither use the information nor get rid of the memset, thus leaving an
5081	// unexpected call in the user's code. This is problematic if the code in question is
5082	// not ready to correctly make calls yet, such as in our early PIE startup code, or in
5083	// the early stages of a dynamic linker, etc.
5084	if (!safety and owner_mod.optimize_mode == .debug) {
5085	return .none;
5086	}
5087
5088	const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
5089	if (needs_bitmask) {
5090	// TODO: only some bits are to be undef, we cannot write with a simple memset.
5091	// meanwhile, ignore the write rather than stomping over valid bits.
5092	// https://github.com/ziglang/zig/issues/15337
5093	return .none;
5094	}
5095
5096	const len = try o.builder.intValue(try o.lowerType(.usize, .as_value), elem_ty.abiSize(zcu));
5097	_ = try fg.wip.callMemSet(
5098	ptr,
5099	ptr_alignment.toLlvm(),
5100	if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
5101	len,
5102	access_kind,
5103	fg.disable_intrinsics,
5104	);
5105	if (safety and owner_mod.valgrind) {
5106	try fg.valgrindMarkUndef(ptr, len);
5107	}
5108	return .none;
5109	}
5110
5111	const elem = try fg.resolveInst(bin_op.rhs);
5112
5113	if (ptr_info.flags.vector_index != .none) {
5114	if (isByRef(elem_ty, zcu)) {
5115	const offset = @backingInt(ptr_info.flags.vector_index) * elem_ty.abiSize(zcu);
5116	const elem_ptr = try fg.ptraddConst(ptr, offset);
5117	try fg.store(elem_ptr, ptr_alignment.offset(offset), elem, elem_ty, access_kind);
5118	} else {
5119	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
5120	const vec_ty = try fg.pt.vectorType(.{
5121	.len = ptr_info.packed_offset.host_size,
5122	.child = elem_ty.toIntern(),
5123	});
5124
5125	const loaded_vector = try fg.load(ptr, ptr_alignment, vec_ty, access_kind);
5126	const index_val = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
5127	const modified_vector = try fg.wip.insertElement(loaded_vector, elem, index_val, "");
5128
5129	try fg.store(ptr, ptr_alignment, modified_vector, vec_ty, access_kind);
5130	}
5131
5132	return .none;
5133	}
5134
5135	if (ptr_info.packed_offset.host_size != 0) {
5136	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
5137	const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
5138	const llvm_backing_int_ty = try o.lowerType(backing_int_ty, .as_value);
5139
5140	const backing_int_val = try fg.load(ptr, ptr_alignment, backing_int_ty, access_kind);
5141
5142	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
5143	const shift_amt = try o.builder.intConst(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
5144
5145	// Convert to equally-sized integer type in order to perform the bit
5146	// operations on the value to store
5147	const new_val_bits_type = try o.builder.intType(@intCast(elem_bits));
5148	const new_val_bits = if (elem_ty.isPtrAtRuntime(zcu))
5149	try fg.wip.cast(.ptrtoint, elem, new_val_bits_type, "")
5150	else
5151	try fg.wip.cast(.bitcast, elem, new_val_bits_type, "");
5152
5153	const mask_val = blk: {
5154	const zext = try fg.wip.cast(
5155	.zext,
5156	try o.builder.intValue(new_val_bits_type, -1),
5157	llvm_backing_int_ty,
5158	"",
5159	);
5160	const shl = try fg.wip.bin(.shl, zext, shift_amt.toValue(), "");
5161	break :blk try fg.wip.bin(
5162	.xor,
5163	shl,
5164	try o.builder.intValue(llvm_backing_int_ty, -1),
5165	"",
5166	);
5167	};
5168
5169	const masked_backing_int_val = try fg.wip.bin(.@"and", backing_int_val, mask_val, "");
5170	const extended_new_val = try fg.wip.cast(.zext, new_val_bits, llvm_backing_int_ty, "");
5171	const shifted_new_val = try fg.wip.bin(.shl, extended_new_val, shift_amt.toValue(), "");
5172	const new_backing_int_val = try fg.wip.bin(.@"or", shifted_new_val, masked_backing_int_val, "");
5173
5174	try fg.store(ptr, ptr_alignment, new_backing_int_val, backing_int_ty, access_kind);
5175	return .none;
5176	}
5177
5178	try fg.store(ptr, ptr_alignment, elem, elem_ty, access_kind);
5179	return .none;
5180	}
5181
5182	fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5183	const o = fg.object;
5184	const zcu = o.zcu;
5185	const ty_op = fg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5186	const ptr_ty = fg.typeOf(ty_op.operand);
5187	const ptr_info = ptr_ty.ptrInfo(zcu);
5188	const ptr = try fg.resolveInst(ty_op.operand);
5189	const elem_ty = ptr_ty.childType(zcu);
5190	const ptr_align = ptr_ty.ptrAlignment(zcu);
5191
5192	fg.maybeMarkAllowZeroAccess(ptr_info);
5193
5194	const access_kind: Builder.MemoryAccessKind =
5195	if (ptr_info.flags.is_volatile) .@"volatile" else .normal;
5196
5197	if (ptr_info.flags.vector_index != .none) {
5198	if (isByRef(elem_ty, zcu)) {
5199	const elem_size = elem_ty.abiSize(zcu);
5200	const offset = @backingInt(ptr_info.flags.vector_index) * elem_size;
5201	const elem_ptr = try fg.ptraddConst(ptr, offset);
5202	return fg.load(elem_ptr, ptr_align.offset(offset), elem_ty, access_kind);
5203	} else {
5204	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
5205	const vec_ty = try fg.pt.vectorType(.{
5206	.len = ptr_info.packed_offset.host_size,
5207	.child = elem_ty.toIntern(),
5208	});
5209	const vector_val = try fg.load(ptr, ptr_align, vec_ty, access_kind);
5210	const index_val = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
5211	return fg.wip.extractElement(vector_val, index_val, "");
5212	}
5213	}
5214
5215	if (ptr_info.packed_offset.host_size == 0) {
5216	return fg.load(ptr, ptr_align, elem_ty, access_kind);
5217	}
5218
5219	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
5220	const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
5221	const llvm_backing_int_ty = try o.lowerType(backing_int_ty, .as_value);
5222
5223	const backing_int_val = try fg.load(ptr, ptr_align, backing_int_ty, .normal);
5224
5225	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
5226	const shift_amt = try o.builder.intValue(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
5227	const shifted_value = try fg.wip.bin(.lshr, backing_int_val, shift_amt, "");
5228
5229	if (isByRef(elem_ty, zcu)) {
5230	const result_ptr = try fg.buildZigAlloca(elem_ty, .none);
5231	switch (elem_ty.floatBits(zcu.getTarget())) {
5232	else => unreachable,
5233	80 => {
5234	const f80_layout = o.softF80Layout(.{}) catch unreachable;
5235	const mantissa = try fg.wip.cast(.trunc, shifted_value, .i64, "load.mantissa");
5236	const shifted_exponent = try fg.wip.bin(
5237	.lshr,
5238	backing_int_val,
5239	try o.builder.intValue(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset + 64),
5240	"load.shifted_exponent",
5241	);
5242	const exponent = try fg.wip.cast(.trunc, shifted_exponent, .i16, "load.exponent");
5243
5244	try fg.store(
5245	try fg.ptraddConst(result_ptr, f80_layout.mantissa_offset),
5246	f80_layout.alignment.offset(f80_layout.mantissa_offset),
5247	mantissa,
5248	.u64,
5249	.normal,
5250	);
5251	try fg.store(
5252	try fg.ptraddConst(result_ptr, f80_layout.exponent_offset),
5253	f80_layout.alignment.offset(f80_layout.exponent_offset),
5254	exponent,
5255	.u16,
5256	.normal,
5257	);
5258	},
5259	128 => {
5260	const f128_layout = o.softF128Layout(.{}) catch unreachable;
5261	const lo = try fg.wip.cast(.trunc, shifted_value, .i64, "load.lo");
5262	const shifted_hi = try fg.wip.bin(
5263	.lshr,
5264	backing_int_val,
5265	try o.builder.intValue(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset + 64),
5266	"load.shifted_hi",
5267	);
5268	const hi = try fg.wip.cast(.trunc, shifted_hi, .i64, "load.hi");
5269
5270	try fg.store(
5271	try fg.ptraddConst(result_ptr, f128_layout.lo_offset),
5272	f128_layout.alignment.offset(f128_layout.lo_offset),
5273	lo,
5274	.u64,
5275	.normal,
5276	);
5277	try fg.store(
5278	try fg.ptraddConst(result_ptr, f128_layout.hi_offset),
5279	f128_layout.alignment.offset(f128_layout.hi_offset),
5280	hi,
5281	.u64,
5282	.normal,
5283	);
5284	},
5285	}
5286	return result_ptr;
5287	}
5288
5289	const elem_llvm_ty = try o.lowerType(elem_ty, .as_value);
5290
5291	if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
5292	const same_size_int = try o.builder.intType(@intCast(elem_bits));
5293	const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, "");
5294	return fg.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
5295	}
5296
5297	if (elem_ty.isPtrAtRuntime(zcu)) {
5298	const same_size_int = try o.builder.intType(@intCast(elem_bits));
5299	const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, "");
5300	return fg.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, "");
5301	}
5302
5303	return fg.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
5304	}
5305
5306	fn airTrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
5307	_ = inst;
5308	const target = self.object.zcu.getTarget();
5309	if ((target.cpu.arch == .mips or target.cpu.arch == .mipsel) and
5310	target.cpu.has(.mips, .notraps))
5311	{
5312	// Emit a MIPS `break` instruction followed by an infinite loop (to fulfil the noreturn)
5313	// since this CPU does not support trap instructions.
5314	const o = self.object;
5315	_ = try self.wip.callAsm(
5316	.none,
5317	try o.builder.fnType(.void, &.{}, .normal),
5318	.{ .sideeffect = true },
5319	try o.builder.string("break\n0:\nj 0b\nnop"),
5320	try o.builder.string("~{memory}"),
5321	&.{},
5322	"",
5323	);
5324	} else {
5325	_ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, "");
5326	}
5327	_ = try self.wip.@"unreachable"();
5328	}
5329
5330	fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5331	_ = inst;
5332	_ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, "");
5333	return .none;
5334	}
5335
5336	fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5337	_ = inst;
5338	const o = self.object;
5339	const llvm_usize = try o.lowerType(.usize, .as_value);
5340	if (!target_util.supportsReturnAddress(self.object.zcu.getTarget(), self.ownerModule().optimize_mode)) {
5341	// https://github.com/ziglang/zig/issues/11946
5342	return o.builder.intValue(llvm_usize, 0);
5343	}
5344	const result = try self.wip.callIntrinsic(.normal, .none, .returnaddress, &.{}, &.{.@"0"}, "");
5345	return self.wip.cast(.ptrtoint, result, llvm_usize, "");
5346	}
5347
5348	fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5349	_ = inst;
5350	const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");
5351	return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize, .as_value), "");
5352	}
5353
5354	fn airCmpxchg(
5355	self: *FuncGen,
5356	inst: Air.Inst.Index,
5357	kind: Builder.Function.Instruction.CmpXchg.Kind,
5358	) Allocator.Error!Builder.Value {
5359	const o = self.object;
5360	const zcu = o.zcu;
5361	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5362	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
5363	const ptr = try self.resolveInst(extra.ptr);
5364	const ptr_ty = self.typeOf(extra.ptr);
5365	var expected_value = try self.resolveInst(extra.expected_value);
5366	var new_value = try self.resolveInst(extra.new_value);
5367	const operand_ty = ptr_ty.childType(zcu);
5368	const llvm_operand_ty = try o.lowerType(operand_ty, .as_value);
5369	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);
5370	if (llvm_abi_ty != .none) {
5371	// operand needs widening and truncating
5372	const signedness: Builder.Function.Instruction.Cast.Signedness =
5373	if (operand_ty.isSignedInt(zcu)) .signed else .unsigned;
5374	expected_value = try self.wip.conv(signedness, expected_value, llvm_abi_ty, "");
5375	new_value = try self.wip.conv(signedness, new_value, llvm_abi_ty, "");
5376	}
5377
5378	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
5379
5380	const result = try self.wip.cmpxchg(
5381	kind,
5382	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
5383	ptr,
5384	expected_value,
5385	new_value,
5386	self.sync_scope,
5387	toLlvmAtomicOrdering(extra.successOrder()),
5388	toLlvmAtomicOrdering(extra.failureOrder()),
5389	ptr_ty.ptrAlignment(zcu).toLlvm(),
5390	"",
5391	);
5392
5393	const optional_ty = self.typeOfIndex(inst);
5394
5395	var payload = try self.wip.extractValue(result, &.{0}, "");
5396	if (llvm_abi_ty != .none) payload = try self.wip.cast(.trunc, payload, llvm_operand_ty, "");
5397	const success_bit = try self.wip.extractValue(result, &.{1}, "");
5398
5399	if (optional_ty.optionalReprIsPayload(zcu)) {
5400	const zero = try o.builder.zeroInitValue(payload.typeOfWip(&self.wip));
5401	return self.wip.select(.normal, success_bit, zero, payload, "");
5402	}
5403
5404	assert(!isByRef(operand_ty, zcu)); // can only cmpxchg non-by-ref types
5405	assert(isByRef(optional_ty, zcu)); // all optionals are by-ref
5406
5407	comptime assert(optional_layout_version == 3);
5408
5409	const non_null_bit = try self.wip.not(success_bit, "");
5410
5411	const payload_align = operand_ty.abiAlignment(zcu);
5412	const alloca_inst = try self.buildZigAlloca(optional_ty, .none);
5413
5414	// Payload is always the first field at offset 0, so address is `alloca_inst`
5415	try self.store(alloca_inst, .none, payload, operand_ty, .normal);
5416
5417	// Non-null bit is after payload with no padding because it has alignment 1
5418	const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu));
5419	try self.store(non_null_ptr, payload_align, non_null_bit, .bool, .normal);
5420
5421	return alloca_inst;
5422	}
5423
5424	fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5425	const o = self.object;
5426	const zcu = o.zcu;
5427	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
5428	const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
5429	const ptr = try self.resolveInst(pl_op.operand);
5430	const ptr_ty = self.typeOf(pl_op.operand);
5431	const operand_ty = ptr_ty.childType(zcu);
5432	const operand = try self.resolveInst(extra.operand);
5433	const is_signed_int = operand_ty.isSignedInt(zcu);
5434	const is_float = operand_ty.isRuntimeFloat();
5435	const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
5436	const ordering = toLlvmAtomicOrdering(extra.ordering());
5437	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg);
5438	const llvm_operand_ty = try o.lowerType(operand_ty, .as_value);
5439
5440	const access_kind: Builder.MemoryAccessKind =
5441	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
5442	const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
5443
5444	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
5445
5446	if (llvm_abi_ty != .none) {
5447	// operand needs widening and truncating or bitcasting.
5448	return self.wip.cast(if (is_float) .bitcast else .trunc, try self.wip.atomicrmw(
5449	access_kind,
5450	op,
5451	ptr,
5452	try self.wip.cast(
5453	if (is_float) .bitcast else if (is_signed_int) .sext else .zext,
5454	operand,
5455	llvm_abi_ty,
5456	"",
5457	),
5458	self.sync_scope,
5459	ordering,
5460	ptr_alignment,
5461	"",
5462	), llvm_operand_ty, "");
5463	}
5464
5465	// If we are storing a pointer we need to convert to and from a plain old integer.
5466	const non_ptr_operand = switch (operand_ty.zigTypeTag(zcu)) {
5467	.pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize, .as_value), ""),
5468	else => operand,
5469	};
5470
5471	const raw_result = try self.wip.atomicrmw(
5472	access_kind,
5473	op,
5474	ptr,
5475	non_ptr_operand,
5476	self.sync_scope,
5477	ordering,
5478	ptr_alignment,
5479	"",
5480	);
5481
5482	// ...and then convert the result back.
5483	switch (operand_ty.zigTypeTag(zcu)) {
5484	.pointer => return self.wip.cast(.inttoptr, raw_result, llvm_operand_ty, ""),
5485	else => return raw_result,
5486	}
5487	}
5488
5489	fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5490	const o = self.object;
5491	const zcu = o.zcu;
5492	const atomic_load = self.air.instructions.items(.data)[@backingInt(inst)].atomic_load;
5493	const ptr = try self.resolveInst(atomic_load.ptr);
5494	const ptr_ty = self.typeOf(atomic_load.ptr);
5495	const info = ptr_ty.ptrInfo(zcu);
5496	const elem_ty = Type.fromInterned(info.child);
5497	if (!elem_ty.hasRuntimeBits(zcu)) return .none;
5498	const ordering = toLlvmAtomicOrdering(atomic_load.order);
5499	const llvm_abi_ty = try self.getAtomicAbiType(elem_ty, false);
5500	const ptr_alignment = (if (info.flags.alignment != .none)
5501	@as(InternPool.Alignment, info.flags.alignment)
5502	else
5503	Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();
5504	const access_kind: Builder.MemoryAccessKind =
5505	if (info.flags.is_volatile) .@"volatile" else .normal;
5506	const elem_llvm_ty = try o.lowerType(elem_ty, .as_value);
5507
5508	self.maybeMarkAllowZeroAccess(info);
5509
5510	if (llvm_abi_ty != .none) {
5511	// operand needs widening and truncating
5512	const loaded = try self.wip.loadAtomic(
5513	access_kind,
5514	llvm_abi_ty,
5515	ptr,
5516	self.sync_scope,
5517	ordering,
5518	ptr_alignment,
5519	"",
5520	);
5521	return self.wip.cast(.trunc, loaded, elem_llvm_ty, "");
5522	}
5523	return self.wip.loadAtomic(
5524	access_kind,
5525	elem_llvm_ty,
5526	ptr,
5527	self.sync_scope,
5528	ordering,
5529	ptr_alignment,
5530	"",
5531	);
5532	}
5533
5534	fn airAtomicStore(
5535	self: *FuncGen,
5536	inst: Air.Inst.Index,
5537	ordering: Builder.AtomicOrdering,
5538	) Allocator.Error!Builder.Value {
5539	const zcu = self.object.zcu;
5540	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5541	const ptr_ty = self.typeOf(bin_op.lhs);
5542	const operand_ty = ptr_ty.childType(zcu);
5543	if (!operand_ty.hasRuntimeBits(zcu)) return .none;
5544	const ptr = try self.resolveInst(bin_op.lhs);
5545	var element = try self.resolveInst(bin_op.rhs);
5546	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);
5547
5548	if (llvm_abi_ty != .none) {
5549	// operand needs widening
5550	element = try self.wip.conv(
5551	if (operand_ty.isSignedInt(zcu)) .signed else .unsigned,
5552	element,
5553	llvm_abi_ty,
5554	"",
5555	);
5556	}
5557
5558	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
5559
5560	assert(!isByRef(operand_ty, zcu));
5561
5562	_ = try self.wip.storeAtomic(
5563	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
5564	element,
5565	ptr,
5566	self.sync_scope,
5567	ordering,
5568	ptr_ty.ptrAlignment(zcu).toLlvm(),
5569	);
5570
5571	return .none;
5572	}
5573
5574	fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
5575	const o = self.object;
5576	const zcu = o.zcu;
5577	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5578	const dest_slice = try self.resolveInst(bin_op.lhs);
5579	const ptr_ty = self.typeOf(bin_op.lhs);
5580	const elem_ty = self.typeOf(bin_op.rhs);
5581	const dest_ptr_align = ptr_ty.ptrAlignment(zcu);
5582	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);
5583	const access_kind: Builder.MemoryAccessKind =
5584	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
5585
5586	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
5587
5588	const allow_byte_memset = !self.needMemsetWorkaround(switch (ptr_ty.ptrSize(zcu)) {
5589	.one => ptr_ty.childType(zcu).abiSize(zcu),
5590	.slice => null,
5591	.many, .c => unreachable,
5592	});
5593	const len_bytes = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5594
5595	try self.lowerMemset(
5596	dest_ptr,
5597	dest_ptr_align,
5598	bin_op.rhs,
5599	elem_ty,
5600	len_bytes,
5601	access_kind,
5602	safety,
5603	allow_byte_memset,
5604	);
5605	return .none;
5606	}
5607
5608	fn lowerMemset(
5609	self: *FuncGen,
5610	dest_ptr: Builder.Value,
5611	dest_ptr_align: InternPool.Alignment,
5612	elem_ref: Air.Inst.Ref,
5613	elem_ty: Type,
5614	len_bytes: Builder.Value,
5615	access_kind: Builder.MemoryAccessKind,
5616	safety: bool,
5617	allow_byte_memset: bool,
5618	) Allocator.Error!void {
5619	const o = self.object;
5620	const zcu = o.zcu;
5621
5622	if (allow_byte_memset) if (elem_ref.toInterned()) |elem_ip_index| {
5623	const elem_val: Value = .fromInterned(elem_ip_index);
5624	if (elem_val.isUndef(zcu)) {
5625	// Even if safety is disabled, we still emit a memset to undefined since it conveys
5626	// extra information to LLVM. However, safety makes the difference between using
5627	// 0xaa or actual undefined for the fill byte.
5628	const fill_byte = if (safety)
5629	try o.builder.intValue(.i8, 0xaa)
5630	else
5631	try o.builder.undefValue(.i8);
5632	_ = try self.wip.callMemSet(
5633	dest_ptr,
5634	dest_ptr_align.toLlvm(),
5635	fill_byte,
5636	len_bytes,
5637	access_kind,
5638	self.disable_intrinsics,
5639	);
5640	const owner_mod = self.ownerModule();
5641	if (safety and owner_mod.valgrind) {
5642	try self.valgrindMarkUndef(dest_ptr, len_bytes);
5643	}
5644	return;
5645	}
5646
5647	// Test if the element value is compile-time known to be a
5648	// repeating byte pattern, for example, `@as(u64, 0)` has a
5649	// repeating byte pattern of 0 bytes. In such case, the memset
5650	// intrinsic can be used.
5651	if (try elem_val.hasRepeatedByteRepr(zcu)) |byte_val| {
5652	const fill_byte = try o.builder.intValue(.i8, byte_val);
5653	_ = try self.wip.callMemSet(
5654	dest_ptr,
5655	dest_ptr_align.toLlvm(),
5656	fill_byte,
5657	len_bytes,
5658	access_kind,
5659	self.disable_intrinsics,
5660	);
5661	return;
5662	}
5663	};
5664
5665	const value = try self.resolveInst(elem_ref);
5666	const elem_abi_size = elem_ty.abiSize(zcu);
5667
5668	intrinsic: {
5669	if (!allow_byte_memset) break :intrinsic;
5670	if (elem_abi_size != 1) break :intrinsic;
5671	// To use LLVM's intrinsic, we need to convert the operand to a raw 8-bit integer value.
5672	const fill_byte: Builder.Value = byte: {
5673	if (isByRef(elem_ty, zcu)) {
5674	break :byte try self.load(value, elem_ty.abiAlignment(zcu), .u8, .normal);
5675	}
5676	if (elem_ty.isAbiInt(zcu)) {
5677	const info = elem_ty.intInfo(zcu);
5678	break :byte try self.wip.conv(switch (info.signedness) {
5679	.unsigned => .unsigned,
5680	.signed => .signed,
5681	}, value, .i8, "");
5682	}
5683	if (elem_ty.toIntern() == .bool_type) {
5684	break :byte try self.wip.cast(.zext, value, .i8, "");
5685	}
5686	break :intrinsic;
5687	};
5688	// Great, we can use the intrinsic!
5689	_ = try self.wip.callMemSet(
5690	dest_ptr,
5691	dest_ptr_align.toLlvm(),
5692	fill_byte,
5693	len_bytes,
5694	access_kind,
5695	self.disable_intrinsics,
5696	);
5697	return;
5698	}
5699
5700	// non-byte-sized element. lower with a loop. something like this:
5701
5702	// entry:
5703	// ...
5704	// %end_ptr = getelementptr %ptr, %len
5705	// br %loop
5706	// loop:
5707	// %it_ptr = phi body %next_ptr, entry %ptr
5708	// %end = cmp eq %it_ptr, %end_ptr
5709	// br %end, %body, %end
5710	// body:
5711	// store %it_ptr, %value
5712	// %next_ptr = getelementptr %it_ptr, 1
5713	// br %loop
5714	// end:
5715	// ...
5716	const entry_block = self.wip.cursor.block;
5717	const loop_block = try self.wip.block(2, "InlineMemsetLoop");
5718	const body_block = try self.wip.block(1, "InlineMemsetBody");
5719	const end_block = try self.wip.block(1, "InlineMemsetEnd");
5720
5721	const end_ptr = try self.ptraddScaled(dest_ptr, len_bytes, 1);
5722	_ = try self.wip.br(loop_block);
5723
5724	self.wip.cursor = .{ .block = loop_block };
5725	const it_ptr = try self.wip.phi(.ptr, "");
5726	const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, "");
5727	_ = try self.wip.brCond(end, body_block, end_block, .none);
5728
5729	self.wip.cursor = .{ .block = body_block };
5730	const elem_abi_align = elem_ty.abiAlignment(zcu);
5731	const it_ptr_align: InternPool.Alignment = dest_ptr_align.min(elem_abi_align);
5732	try self.store(it_ptr.toValue(), it_ptr_align, value, elem_ty, access_kind);
5733	const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size);
5734	_ = try self.wip.br(loop_block);
5735
5736	self.wip.cursor = .{ .block = end_block };
5737	it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip);
5738	return;
5739	}
5740
5741	fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5742	const zcu = self.object.zcu;
5743	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5744	const dest_slice = try self.resolveInst(bin_op.lhs);
5745	const dest_ptr_ty = self.typeOf(bin_op.lhs);
5746	const src_slice = try self.resolveInst(bin_op.rhs);
5747	const src_ptr_ty = self.typeOf(bin_op.rhs);
5748	const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
5749	const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
5750	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
5751	const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
5752	dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
5753
5754	self.maybeMarkAllowZeroAccess(dest_ptr_ty.ptrInfo(zcu));
5755	self.maybeMarkAllowZeroAccess(src_ptr_ty.ptrInfo(zcu));
5756
5757	_ = try self.wip.callMemCpy(
5758	dest_ptr,
5759	dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
5760	src_ptr,
5761	src_ptr_ty.ptrAlignment(zcu).toLlvm(),
5762	len,
5763	access_kind,
5764	self.disable_intrinsics,
5765	);
5766	return .none;
5767	}
5768
5769	fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5770	const zcu = self.object.zcu;
5771	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5772	const dest_slice = try self.resolveInst(bin_op.lhs);
5773	const dest_ptr_ty = self.typeOf(bin_op.lhs);
5774	const src_slice = try self.resolveInst(bin_op.rhs);
5775	const src_ptr_ty = self.typeOf(bin_op.rhs);
5776	const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
5777	const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
5778	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
5779	const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
5780	dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
5781
5782	_ = try self.wip.callMemMove(
5783	dest_ptr,
5784	dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
5785	src_ptr,
5786	src_ptr_ty.ptrAlignment(zcu).toLlvm(),
5787	len,
5788	access_kind,
5789	);
5790	return .none;
5791	}
5792
5793	fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5794	const zcu = self.object.zcu;
5795	const bin_op = self.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5796	const un_ptr_ty = self.typeOf(bin_op.lhs);
5797	const un_ty = un_ptr_ty.childType(zcu);
5798	const layout = un_ty.unionGetLayout(zcu);
5799
5800	if (layout.tag_size == 0) return .none; // TODO: stop Sema emitting this
5801
5802	const access_kind: Builder.MemoryAccessKind =
5803	if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
5804
5805	self.maybeMarkAllowZeroAccess(un_ptr_ty.ptrInfo(zcu));
5806
5807	const union_ptr = try self.resolveInst(bin_op.lhs);
5808	const new_tag = try self.resolveInst(bin_op.rhs);
5809	const tag_ty = self.typeOf(bin_op.rhs);
5810	const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);
5811	const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());
5812	const tag_ptr_align = union_ptr_align.offset(layout.tagOffset());
5813	try self.store(tag_field_ptr, tag_ptr_align, new_tag, tag_ty, access_kind);
5814	return .none;
5815	}
5816
5817	fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5818	const o = self.object;
5819	const zcu = o.zcu;
5820	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5821	const un_ty = self.typeOf(ty_op.operand);
5822	const layout = un_ty.unionGetLayout(zcu);
5823	assert(layout.tag_size != 0);
5824	const operand = try self.resolveInst(ty_op.operand);
5825	assert(isByRef(un_ty, zcu));
5826	const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());
5827	return self.load(tag_field_ptr, .none, un_ty.unionTagTypeRuntime(zcu).?, .normal);
5828	}
5829
5830	fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) Allocator.Error!Builder.Value {
5831	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
5832	const operand = try self.resolveInst(un_op);
5833	const operand_ty = self.typeOf(un_op);
5834
5835	return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand});
5836	}
5837
5838	fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
5839	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
5840	const operand = try self.resolveInst(un_op);
5841	const operand_ty = self.typeOf(un_op);
5842
5843	return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand});
5844	}
5845
5846	fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value {
5847	const o = self.object;
5848	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5849	const inst_ty = self.typeOfIndex(inst);
5850	const operand_ty = self.typeOf(ty_op.operand);
5851	const operand = try self.resolveInst(ty_op.operand);
5852
5853	const result = try self.wip.callIntrinsic(
5854	.normal,
5855	.none,
5856	intrinsic,
5857	&.{try o.lowerType(operand_ty, .as_value)},
5858	&.{ operand, .false },
5859	"",
5860	);
5861	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .as_value), "");
5862	}
5863
5864	fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value {
5865	const o = self.object;
5866	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5867	const inst_ty = self.typeOfIndex(inst);
5868	const operand_ty = self.typeOf(ty_op.operand);
5869	const operand = try self.resolveInst(ty_op.operand);
5870
5871	const result = try self.wip.callIntrinsic(
5872	.normal,
5873	.none,
5874	intrinsic,
5875	&.{try o.lowerType(operand_ty, .as_value)},
5876	&.{operand},
5877	"",
5878	);
5879	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .as_value), "");
5880	}
5881
5882	fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5883	const o = self.object;
5884	const zcu = o.zcu;
5885	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5886	const operand_ty = self.typeOf(ty_op.operand);
5887	var bits = operand_ty.intInfo(zcu).bits;
5888	assert(bits % 8 == 0);
5889
5890	const inst_ty = self.typeOfIndex(inst);
5891	var operand = try self.resolveInst(ty_op.operand);
5892	var llvm_operand_ty = try o.lowerType(operand_ty, .as_value);
5893
5894	if (bits % 16 == 8) {
5895	// If not an even byte-multiple, we need zero-extend + shift-left 1 byte
5896	// The truncated result at the end will be the correct bswap
5897	const scalar_ty = try o.builder.intType(@intCast(bits + 8));
5898	if (operand_ty.zigTypeTag(zcu) == .vector) {
5899	const vec_len = operand_ty.vectorLen(zcu);
5900	llvm_operand_ty = try o.builder.vectorType(.normal, vec_len, scalar_ty);
5901	} else llvm_operand_ty = scalar_ty;
5902
5903	const shift_amt =
5904	try o.builder.splatValue(llvm_operand_ty, try o.builder.intConst(scalar_ty, 8));
5905	const extended = try self.wip.cast(.zext, operand, llvm_operand_ty, "");
5906	operand = try self.wip.bin(.shl, extended, shift_amt, "");
5907
5908	bits = bits + 8;
5909	}
5910
5911	const result =
5912	try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");
5913	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .as_value), "");
5914	}
5915
5916	fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5917	const o = self.object;
5918	const zcu = o.zcu;
5919	const ip = &zcu.intern_pool;
5920	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5921	const operand = try self.resolveInst(ty_op.operand);
5922	const error_set_ty = ty_op.ty;
5923
5924	const names = error_set_ty.errorSetNames(zcu);
5925	const valid_block = try self.wip.block(@intCast(names.len), "Valid");
5926	const invalid_block = try self.wip.block(1, "Invalid");
5927	const end_block = try self.wip.block(2, "End");
5928	var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none);
5929	defer wip_switch.finish(&self.wip);
5930
5931	for (0..names.len) |name_index| {
5932	const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
5933	const this_tag_int_value = try o.builder.intConst(try o.errorIntType(.as_value), err_int);
5934	try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);
5935	}
5936	self.wip.cursor = .{ .block = valid_block };
5937	_ = try self.wip.br(end_block);
5938
5939	self.wip.cursor = .{ .block = invalid_block };
5940	_ = try self.wip.br(end_block);
5941
5942	self.wip.cursor = .{ .block = end_block };
5943	const phi = try self.wip.phi(.i1, "");
5944	phi.finish(&.{ .true, .false }, &.{ valid_block, invalid_block }, &self.wip);
5945	return phi.toValue();
5946	}
5947
5948	fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5949	const o = self.object;
5950	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
5951	const operand = try self.resolveInst(un_op);
5952	const enum_ty = self.typeOf(un_op);
5953
5954	const llvm_fn = try o.getIsNamedEnumValueFunction(enum_ty);
5955	return self.wip.call(
5956	.normal,
5957	.fastcc,
5958	.none,
5959	llvm_fn.typeOf(&o.builder),
5960	llvm_fn.toValue(&o.builder),
5961	&.{operand},
5962	"",
5963	);
5964	}
5965
5966	fn airTagName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5967	const o = self.object;
5968	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
5969	const operand = try self.resolveInst(un_op);
5970	const enum_ty = self.typeOf(un_op);
5971
5972	const llvm_fn = try o.getEnumTagNameFunction(enum_ty);
5973	return self.wip.call(
5974	.normal,
5975	.fastcc,
5976	.none,
5977	llvm_fn.typeOf(&o.builder),
5978	llvm_fn.toValue(&o.builder),
5979	&.{operand},
5980	"",
5981	);
5982	}
5983
5984	fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5985	const o = self.object;
5986	const zcu = o.zcu;
5987	const un_op = self.air.instructions.items(.data)[@backingInt(inst)].un_op;
5988	const operand = try self.resolveInst(un_op);
5989	const slice_ty = self.typeOfIndex(inst);
5990
5991	// If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
5992	const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize, .as_value), "");
5993
5994	const error_name_table_ptr = try o.getErrorNameTable();
5995	const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));
5996	return self.load(error_name_ptr, .none, slice_ty, .normal);
5997	}
5998
5999	fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6000	const o = self.object;
6001	const zcu = o.zcu;
6002	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6003	const result_ty = self.typeOfIndex(inst);
6004	switch (result_ty.zigTypeTag(zcu)) {
6005	.vector => {
6006	const scalar = try self.resolveInst(ty_op.operand);
6007	return self.wip.splatVector(try o.lowerType(result_ty, .as_value), scalar, "");
6008	},
6009	.array => {
6010	assert(isByRef(result_ty, zcu));
6011
6012	const result_ptr = try self.buildZigAlloca(result_ty, .none);
6013	const array_info = result_ty.arrayInfo(zcu);
6014	const elem_size = array_info.elem_type.abiSize(zcu);
6015	const len_bytes = array_info.len * elem_size;
6016	const len_bytes_llvm = try o.builder.intValue(try o.lowerType(.usize, .as_value), len_bytes);
6017
6018	try self.lowerMemset(
6019	result_ptr,
6020	result_ty.abiAlignment(zcu),
6021	ty_op.operand,
6022	array_info.elem_type,
6023	len_bytes_llvm,
6024	.normal,
6025	false,
6026	!self.needMemsetWorkaround(len_bytes),
6027	);
6028
6029	if (array_info.sentinel) |sent_val| {
6030	const sent_ptr = try self.ptraddConst(result_ptr, len_bytes);
6031	const sent_elem = try self.resolveValue(sent_val);
6032	try self.store(sent_ptr, .none, sent_elem.toValue(), array_info.elem_type, .normal);
6033	}
6034
6035	return result_ptr;
6036	},
6037	else => unreachable,
6038	}
6039	}
6040
6041	fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6042	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
6043	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
6044	const pred = try self.resolveInst(pl_op.operand);
6045	const a = try self.resolveInst(extra.lhs);
6046	const b = try self.resolveInst(extra.rhs);
6047
6048	return self.wip.select(.normal, pred, a, b, "");
6049	}
6050
6051	fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6052	const o = fg.object;
6053	const zcu = o.zcu;
6054	const gpa = zcu.gpa;
6055
6056	const unwrapped = fg.air.unwrapShuffleOne(zcu, inst);
6057
6058	const operand = try fg.resolveInst(unwrapped.operand);
6059	const mask = unwrapped.mask;
6060	const operand_ty = fg.typeOf(unwrapped.operand);
6061	const llvm_operand_ty = try o.lowerType(operand_ty, .as_value);
6062	const llvm_result_ty = try o.lowerType(unwrapped.result_ty, .as_value);
6063	const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu), .as_value);
6064	const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
6065	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
6066	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
6067
6068	// LLVM requires that the two input vectors have the same length, so lowering isn't trivial.
6069	// And, in the words of jacobly0: "llvm sucks at shuffles so we do have to hold its hand at
6070	// least a bit". So, there are two cases here.
6071	//
6072	// If the operand length equals the mask length, we do just the one `shufflevector`, where
6073	// the second operand is a constant vector with comptime-known elements at the right indices
6074	// and poison values elsewhere (in the indices which won't be selected).
6075	//
6076	// Otherwise, we lower to *two* `shufflevector` instructions. The first shuffles the runtime
6077	// operand with an all-poison vector to extract and correctly position all of the runtime
6078	// elements. We also make a constant vector with all of the comptime elements correctly
6079	// positioned. Then, our second instruction selects elements from those "runtime-or-poison"
6080	// and "comptime-or-poison" vectors to compute the result.
6081
6082	// This buffer is used primarily for the mask constants.
6083	const llvm_elem_buf = try gpa.alloc(Builder.Constant, mask.len);
6084	defer gpa.free(llvm_elem_buf);
6085
6086	// ...but first, we'll collect all of the comptime-known values.
6087	var any_defined_comptime_value = false;
6088	for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| {
6089	llvm_elem.* = switch (mask_elem.unwrap()) {
6090	.elem => llvm_poison_elem,
6091	.value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
6092	any_defined_comptime_value = true;
6093	break :elem try o.lowerValue(val, .as_value);
6094	} else llvm_poison_elem,
6095	};
6096	}
6097	// This vector is like the result, but runtime elements are replaced with poison.
6098	const comptime_and_poison: Builder.Value = if (any_defined_comptime_value) vec: {
6099	break :vec try o.builder.vectorValue(llvm_result_ty, llvm_elem_buf);
6100	} else try o.builder.poisonValue(llvm_result_ty);
6101
6102	if (operand_ty.vectorLen(zcu) == mask.len) {
6103	// input length equals mask/output length, so we lower to one instruction
6104	for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| {
6105	llvm_elem.* = switch (mask_elem.unwrap()) {
6106	.elem => |idx| try o.builder.intConst(.i32, idx),
6107	.value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
6108	break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
6109	} else llvm_poison_mask_elem,
6110	};
6111	}
6112	return fg.wip.shuffleVector(
6113	operand,
6114	comptime_and_poison,
6115	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
6116	"",
6117	);
6118	}
6119
6120	for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| {
6121	llvm_elem.* = switch (mask_elem.unwrap()) {
6122	.elem => |idx| try o.builder.intConst(.i32, idx),
6123	.value => llvm_poison_mask_elem,
6124	};
6125	}
6126	// This vector is like our result, but all comptime-known elements are poison.
6127	const runtime_and_poison = try fg.wip.shuffleVector(
6128	operand,
6129	try o.builder.poisonValue(llvm_operand_ty),
6130	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
6131	"",
6132	);
6133
6134	if (!any_defined_comptime_value) {
6135	// `comptime_and_poison` is just poison; a second shuffle would be a nop.
6136	return runtime_and_poison;
6137	}
6138
6139	// In this second shuffle, the inputs, the mask, and the output all have the same length.
6140	for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| {
6141	llvm_elem.* = switch (mask_elem.unwrap()) {
6142	.elem => try o.builder.intConst(.i32, elem_idx),
6143	.value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
6144	break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
6145	} else llvm_poison_mask_elem,
6146	};
6147	}
6148	// Merge the runtime and comptime elements with the mask we just built.
6149	return fg.wip.shuffleVector(
6150	runtime_and_poison,
6151	comptime_and_poison,
6152	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
6153	"",
6154	);
6155	}
6156
6157	fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6158	const o = fg.object;
6159	const zcu = o.zcu;
6160	const gpa = zcu.gpa;
6161
6162	const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
6163
6164	const mask = unwrapped.mask;
6165	const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu), .as_value);
6166	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
6167	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
6168
6169	// This is kind of simpler than in `airShuffleOne`. We extend the shorter vector to the
6170	// length of the longer one with an initial `shufflevector` if necessary, and then do the
6171	// actual computation with a second `shufflevector`.
6172
6173	const operand_a_len = fg.typeOf(unwrapped.operand_a).vectorLen(zcu);
6174	const operand_b_len = fg.typeOf(unwrapped.operand_b).vectorLen(zcu);
6175	const operand_len: u32 = @max(operand_a_len, operand_b_len);
6176
6177	// If we need to extend an operand, this is the type that mask will have.
6178	const llvm_operand_mask_ty = try o.builder.vectorType(.normal, operand_len, .i32);
6179
6180	const llvm_elem_buf = try gpa.alloc(Builder.Constant, @max(mask.len, operand_len));
6181	defer gpa.free(llvm_elem_buf);
6182
6183	const operand_a: Builder.Value = extend: {
6184	const raw = try fg.resolveInst(unwrapped.operand_a);
6185	if (operand_a_len == operand_len) break :extend raw;
6186	// Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
6187	const mask_elems = llvm_elem_buf[0..operand_len];
6188	for (mask_elems[0..operand_a_len], 0..) |*llvm_elem, elem_idx| {
6189	llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
6190	}
6191	@memset(mask_elems[operand_a_len..], llvm_poison_mask_elem);
6192	const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_a_len, llvm_elem_ty);
6193	break :extend try fg.wip.shuffleVector(
6194	raw,
6195	try o.builder.poisonValue(llvm_this_operand_ty),
6196	try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
6197	"",
6198	);
6199	};
6200	const operand_b: Builder.Value = extend: {
6201	const raw = try fg.resolveInst(unwrapped.operand_b);
6202	if (operand_b_len == operand_len) break :extend raw;
6203	// Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
6204	const mask_elems = llvm_elem_buf[0..operand_len];
6205	for (mask_elems[0..operand_b_len], 0..) |*llvm_elem, elem_idx| {
6206	llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
6207	}
6208	@memset(mask_elems[operand_b_len..], llvm_poison_mask_elem);
6209	const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_b_len, llvm_elem_ty);
6210	break :extend try fg.wip.shuffleVector(
6211	raw,
6212	try o.builder.poisonValue(llvm_this_operand_ty),
6213	try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
6214	"",
6215	);
6216	};
6217
6218	// `operand_a` and `operand_b` now have the same length (we've extended the shorter one with
6219	// an initial shuffle if necessary). Now for the easy bit.
6220
6221	const mask_elems = llvm_elem_buf[0..mask.len];
6222	for (mask, mask_elems) |mask_elem, *llvm_mask_elem| {
6223	llvm_mask_elem.* = switch (mask_elem.unwrap()) {
6224	.a_elem => |idx| try o.builder.intConst(.i32, idx),
6225	.b_elem => |idx| try o.builder.intConst(.i32, operand_len + idx),
6226	.undef => llvm_poison_mask_elem,
6227	};
6228	}
6229	return fg.wip.shuffleVector(
6230	operand_a,
6231	operand_b,
6232	try o.builder.vectorValue(llvm_mask_ty, mask_elems),
6233	"",
6234	);
6235	}
6236
6237	fn airReduce(fg: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
6238	const o = fg.object;
6239	const zcu = o.zcu;
6240	const target = zcu.getTarget();
6241
6242	const reduce = fg.air.instructions.items(.data)[@backingInt(inst)].reduce;
6243	const operand = try fg.resolveInst(reduce.operand);
6244	const operand_ty = fg.typeOf(reduce.operand);
6245	const scalar_ty = fg.typeOfIndex(inst);
6246
6247	switch (reduce.operation) {
6248	.And, .Or, .Xor => return fg.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
6249	.And => .@"vector.reduce.and",
6250	.Or => .@"vector.reduce.or",
6251	.Xor => .@"vector.reduce.xor",
6252	else => unreachable,
6253	}, &.{try o.lowerType(operand_ty, .as_value)}, &.{operand}, ""),
6254	.Min, .Max => switch (scalar_ty.zigTypeTag(zcu)) {
6255	.int => return fg.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
6256	.Min => if (scalar_ty.isSignedInt(zcu))
6257	.@"vector.reduce.smin"
6258	else
6259	.@"vector.reduce.umin",
6260	.Max => if (scalar_ty.isSignedInt(zcu))
6261	.@"vector.reduce.smax"
6262	else
6263	.@"vector.reduce.umax",
6264	else => unreachable,
6265	}, &.{try o.lowerType(operand_ty, .as_value)}, &.{operand}, ""),
6266	.float => if (intrinsicsAllowed(.libc, scalar_ty, target))
6267	return fg.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
6268	.Min => .@"vector.reduce.fmin",
6269	.Max => .@"vector.reduce.fmax",
6270	else => unreachable,
6271	}, &.{try o.lowerType(operand_ty, .as_value)}, &.{operand}, ""),
6272	else => unreachable,
6273	},
6274	.Add, .Mul => switch (scalar_ty.zigTypeTag(zcu)) {
6275	.int => return fg.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
6276	.Add => .@"vector.reduce.add",
6277	.Mul => .@"vector.reduce.mul",
6278	else => unreachable,
6279	}, &.{try o.lowerType(operand_ty, .as_value)}, &.{operand}, ""),
6280	.float => if (intrinsicsAllowed(.compiler_rt, scalar_ty, target))
6281	return fg.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
6282	.Add => .@"vector.reduce.fadd",
6283	.Mul => .@"vector.reduce.fmul",
6284	else => unreachable,
6285	}, &.{try o.lowerType(operand_ty, .as_value)}, &.{ switch (reduce.operation) {
6286	.Add => try o.builder.fpValue(try o.lowerType(scalar_ty, .as_value), -0.0),
6287	.Mul => try o.builder.fpValue(try o.lowerType(scalar_ty, .as_value), 1.0),
6288	else => unreachable,
6289	}, operand }, ""),
6290	else => unreachable,
6291	},
6292	}
6293
6294	// Reduction could not be performed with intrinsics.
6295	// Use a manual loop over a softfloat call instead.
6296	const float_bits = scalar_ty.floatBits(target);
6297	const fn_name = switch (reduce.operation) {
6298	.Min => try o.builder.strtabStringFmt("{s}fmin{s}", .{
6299	libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
6300	}),
6301	.Max => try o.builder.strtabStringFmt("{s}fmax{s}", .{
6302	libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
6303	}),
6304	.Add => try o.builder.strtabStringFmt("__add{s}f3", .{
6305	compilerRtFloatAbbrev(target, float_bits),
6306	}),
6307	.Mul => try o.builder.strtabStringFmt("__mul{s}f3", .{
6308	compilerRtFloatAbbrev(target, float_bits),
6309	}),
6310	else => unreachable,
6311	};
6312	const fn_info: Object.FuncInfo = .{
6313	.cc = target.cCallingConvention().?,
6314	.param_types = &.{ scalar_ty.toIntern(), scalar_ty.toIntern() },
6315	.return_type = scalar_ty.toIntern(),
6316	};
6317	const llvm_fn = try fg.object.getLibcFunction(fg.pt, fn_name, fn_info);
6318	const init = switch (float_bits) {
6319	else => unreachable,
6320	16 => try o.f16Const(switch (reduce.operation) {
6321	else => unreachable,
6322	.Min, .Max => std.math.nan(f16),
6323	.Add => -0.0,
6324	.Mul => 1.0,
6325	}),
6326	32 => try o.f32Const(switch (reduce.operation) {
6327	else => unreachable,
6328	.Min, .Max => std.math.nan(f32),
6329	.Add => -0.0,
6330	.Mul => 1.0,
6331	}),
6332	64 => try o.f64Const(switch (reduce.operation) {
6333	else => unreachable,
6334	.Min, .Max => std.math.nan(f64),
6335	.Add => -0.0,
6336	.Mul => 1.0,
6337	}),
6338	80 => try o.f80Const(switch (reduce.operation) {
6339	else => unreachable,
6340	.Min, .Max => std.math.nan(f80),
6341	.Add => -0.0,
6342	.Mul => 1.0,
6343	}),
6344	128 => try o.f128Const(switch (reduce.operation) {
6345	else => unreachable,
6346	.Min, .Max => std.math.nan(f128),
6347	.Add => -0.0,
6348	.Mul => 1.0,
6349	}),
6350	};
6351	const iterations = operand_ty.vectorLen(zcu);
6352	const is_by_ref = isByRef(operand_ty, zcu);
6353	if (iterations > 1 and is_by_ref) {
6354	const init_ref = try o.lowerConstRef(init, scalar_ty.abiAlignment(zcu).toLlvm());
6355
6356	const entry_block = fg.wip.cursor.block;
6357	const loop_block = try fg.wip.block(2, "reduce.loop");
6358	const done_block = try fg.wip.block(1, "reduce.loop");
6359
6360	_ = try fg.wip.br(loop_block);
6361
6362	fg.wip.cursor = .{ .block = loop_block };
6363	const index = try fg.wip.phi(.i32, "reduce.index");
6364	const result = try fg.wip.phi(.ptr, "reduce.result");
6365
6366	const rhs_elem_ptr = try fg.ptraddScaled(operand, index.toValue(), scalar_ty.abiSize(zcu));
6367	const rhs_elem = try fg.load(rhs_elem_ptr, .none, scalar_ty, .normal);
6368	const next_result = try fg.buildCall(.{}, llvm_fn.typeOf(&o.builder), llvm_fn.toValue(&o.builder), fn_info, fn_info.param_types, &.{ result.toValue(), rhs_elem });
6369
6370	const next_index = try fg.wip.bin(.@"add nuw", index.toValue(), try o.builder.intValue(.i32, 1), "reduce.next_index");
6371	index.finish(&.{ try o.builder.intValue(.i32, 0), next_index }, &.{ entry_block, loop_block }, &fg.wip);
6372	result.finish(&.{ init_ref.toValue(), next_result }, &.{ entry_block, loop_block }, &fg.wip);
6373	const is_done = try fg.wip.icmp(.eq, next_index, try o.builder.intValue(.i32, iterations), "reduce.is_done");
6374	_ = try fg.wip.brCond(is_done, done_block, loop_block, .none);
6375
6376	fg.wip.cursor = .{ .block = done_block };
6377	return next_result;
6378	}
6379	var result = init.toValue();
6380	for (0..iterations) |index| {
6381	const index_value = try o.builder.intValue(.i32, index);
6382	const rhs_elem = if (is_by_ref) rhs_elem: {
6383	const rhs_elem_ptr = try fg.ptraddConst(operand, index * scalar_ty.abiSize(zcu));
6384	break :rhs_elem try fg.load(rhs_elem_ptr, .none, scalar_ty, .normal);
6385	} else try fg.wip.extractElement(operand, index_value, "reduce.rhs_elem");
6386	result = try fg.buildCall(.{}, llvm_fn.typeOf(&o.builder), llvm_fn.toValue(&o.builder), fn_info, fn_info.param_types, &.{ result, rhs_elem });
6387	}
6388	return result;
6389	}
6390
6391	fn airAggregateInit(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6392	const o = fg.object;
6393	const zcu = o.zcu;
6394	const ip = &zcu.intern_pool;
6395	const ty_pl = fg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6396	const result_ty = fg.typeOfIndex(inst);
6397	const len: usize = @intCast(result_ty.arrayLen(zcu));
6398	const elements: []const Air.Inst.Ref = @ptrCast(fg.air.extra.items[ty_pl.payload..][0..len]);
6399
6400	switch (result_ty.zigTypeTag(zcu)) {
6401	.vector => if (isByRef(result_ty, zcu)) {
6402	const elem_ty = result_ty.childType(zcu);
6403	const elem_size = elem_ty.abiSize(zcu);
6404	const result_ptr = try fg.buildZigAlloca(result_ty, .none);
6405	for (elements, 0..) |elem, elem_index| {
6406	const elem_ptr = try fg.ptraddConst(result_ptr, elem_index * elem_size);
6407	const llvm_elem = try fg.resolveInst(elem);
6408	try fg.store(elem_ptr, .none, llvm_elem, elem_ty, .normal);
6409	}
6410	return result_ptr;
6411	} else {
6412	const llvm_result_ty = try o.lowerType(result_ty, .as_value);
6413	var vector = try o.builder.poisonValue(llvm_result_ty);
6414	for (elements, 0..) |elem, elem_index| {
6415	const elem_index_val = try o.builder.intValue(.i32, elem_index);
6416	const llvm_elem = try fg.resolveInst(elem);
6417	vector = try fg.wip.insertElement(vector, llvm_elem, elem_index_val, "");
6418	}
6419	return vector;
6420	},
6421	.@"struct" => switch (result_ty.containerLayout(zcu)) {
6422	.@"packed" => {
6423	const struct_type = ip.loadStructType(result_ty.toIntern());
6424	const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type);
6425	const big_bits = backing_int_ty.bitSize(zcu);
6426	const int_ty = try o.builder.intType(@intCast(big_bits));
6427	comptime assert(Type.packed_struct_layout_version == 2);
6428	var running_int = try o.builder.intValue(int_ty, 0);
6429	var running_bits: u16 = 0;
6430	for (elements, struct_type.field_types.get(ip)) |elem, field_ty| {
6431	if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
6432
6433	const non_int_val = try fg.resolveInst(elem);
6434	const ty_bit_size: u16 = @intCast(Type.fromInterned(field_ty).bitSize(zcu));
6435	const small_int_ty = try o.builder.intType(ty_bit_size);
6436	const small_int_val = if (Type.fromInterned(field_ty).isPtrAtRuntime(zcu))
6437	try fg.wip.cast(.ptrtoint, non_int_val, small_int_ty, "")
6438	else
6439	try fg.wip.cast(.bitcast, non_int_val, small_int_ty, "");
6440	const shift_rhs = try o.builder.intValue(int_ty, running_bits);
6441	const extended_int_val =
6442	try fg.wip.conv(.unsigned, small_int_val, int_ty, "");
6443	const shifted = try fg.wip.bin(.shl, extended_int_val, shift_rhs, "");
6444	running_int = try fg.wip.bin(.@"or", running_int, shifted, "");
6445	running_bits += ty_bit_size;
6446	}
6447	return running_int;
6448	},
6449	.auto, .@"extern" => {
6450	assert(isByRef(result_ty, zcu));
6451	// TODO in debug builds init to undef so that the padding will be 0xaa
6452	// even if we fully populate the fields.
6453	const struct_align = result_ty.abiAlignment(zcu);
6454	const alloca_inst = try fg.buildZigAlloca(result_ty, .none);
6455
6456	for (elements, 0..) |elem, field_index| {
6457	if (result_ty.structFieldIsComptime(field_index, zcu)) continue;
6458	const field_ty = result_ty.fieldType(field_index, zcu);
6459	if (!field_ty.hasRuntimeBits(zcu)) continue;
6460	const offset = result_ty.structFieldOffset(field_index, zcu);
6461	const field_ptr = try fg.ptraddConst(alloca_inst, offset);
6462	const field_ptr_align = struct_align.offset(offset);
6463
6464	const llvm_field_val = try fg.resolveInst(elem);
6465
6466	try fg.store(field_ptr, field_ptr_align, llvm_field_val, field_ty, .normal);
6467	}
6468
6469	return alloca_inst;
6470	},
6471	},
6472	.array => {
6473	assert(isByRef(result_ty, zcu));
6474
6475	const alloca_inst = try fg.buildZigAlloca(result_ty, .none);
6476
6477	const array_info = result_ty.arrayInfo(zcu);
6478
6479	const elem_size = array_info.elem_type.abiSize(zcu);
6480
6481	for (elements, 0..) |elem, i| {
6482	const elem_ptr = try fg.ptraddConst(alloca_inst, elem_size * i);
6483	const llvm_elem = try fg.resolveInst(elem);
6484	try fg.store(elem_ptr, .none, llvm_elem, array_info.elem_type, .normal);
6485	}
6486	if (array_info.sentinel) |sent_val| {
6487	const elem_ptr = try fg.ptraddConst(alloca_inst, elem_size * array_info.len);
6488	const llvm_elem = try fg.resolveValue(sent_val);
6489	try fg.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type, .normal);
6490	}
6491
6492	return alloca_inst;
6493	},
6494	else => unreachable,
6495	}
6496	}
6497
6498	fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6499	const o = self.object;
6500	const zcu = o.zcu;
6501	const ip = &zcu.intern_pool;
6502	const ty_pl = self.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6503	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
6504	const union_ty = self.typeOfIndex(inst);
6505	const union_obj = zcu.typeToUnion(union_ty).?;
6506
6507	assert(union_obj.layout != .@"packed");
6508
6509	const layout = Type.getUnionLayout(union_obj, zcu);
6510
6511	assert(layout.payload_size != 0); // otherwise the value would be comptime-known
6512	assert(isByRef(union_ty, zcu));
6513
6514	const result_ptr = try self.buildZigAlloca(union_ty, layout.abi_align);
6515	const llvm_payload = try self.resolveInst(extra.init);
6516	const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6517	assert(field_ty.hasRuntimeBits(zcu));
6518
6519	{
6520	const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset());
6521	try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty, .normal);
6522	}
6523
6524	if (layout.tag_size != 0) {
6525	const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
6526	const loaded_enum = ip.loadEnumType(tag_ty.toIntern());
6527	const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {
6528	.none => try o.builder.intConst(
6529	try o.lowerType(.fromInterned(union_obj.enum_tag_type), .as_value),
6530	extra.field_index, // auto-numbered
6531	),
6532	else => |tag_val_ip| try o.lowerValue(tag_val_ip, .as_value),
6533	};
6534	const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());
6535	try self.store(tag_ptr, layout.tag_align, llvm_tag_val.toValue(), tag_ty, .normal);
6536	}
6537
6538	return result_ptr;
6539	}
6540
6541	fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6542	const o = self.object;
6543	const prefetch = self.air.instructions.items(.data)[@backingInt(inst)].prefetch;
6544
6545	comptime assert(@backingInt(std.lang.PrefetchOptions.Rw.read) == 0);
6546	comptime assert(@backingInt(std.lang.PrefetchOptions.Rw.write) == 1);
6547
6548	comptime assert(prefetch.locality >= 0);
6549	comptime assert(prefetch.locality <= 3);
6550
6551	comptime assert(@backingInt(std.lang.PrefetchOptions.Cache.instruction) == 0);
6552	comptime assert(@backingInt(std.lang.PrefetchOptions.Cache.data) == 1);
6553
6554	// LLVM fails during codegen of instruction cache prefetchs for these architectures.
6555	// This is an LLVM bug as the prefetch intrinsic should be a noop if not supported
6556	// by the target.
6557	// To work around this, don't emit llvm.prefetch in this case.
6558	// See https://bugs.llvm.org/show_bug.cgi?id=21037
6559	const zcu = self.object.zcu;
6560	const target = zcu.getTarget();
6561	switch (prefetch.cache) {
6562	.instruction => switch (target.cpu.arch) {
6563	.x86_64,
6564	.x86,
6565	.powerpc,
6566	.powerpcle,
6567	.powerpc64,
6568	.powerpc64le,
6569	=> return .none,
6570	.arm, .armeb, .thumb, .thumbeb => {
6571	switch (prefetch.rw) {
6572	.write => return .none,
6573	else => {},
6574	}
6575	},
6576	else => {},
6577	},
6578	.data => {},
6579	}
6580
6581	_ = try self.wip.callIntrinsic(.normal, .none, .prefetch, &.{.ptr}, &.{
6582	try self.sliceOrArrayPtr(try self.resolveInst(prefetch.ptr), self.typeOf(prefetch.ptr)),
6583	try o.builder.intValue(.i32, prefetch.rw),
6584	try o.builder.intValue(.i32, prefetch.locality),
6585	try o.builder.intValue(.i32, prefetch.cache),
6586	}, "");
6587	return .none;
6588	}
6589
6590	fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6591	const ty_op = self.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6592	const inst_ty = self.typeOfIndex(inst);
6593	const operand = try self.resolveInst(ty_op.operand);
6594	return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty, .as_value), "");
6595	}
6596
6597	fn workIntrinsic(
6598	self: *FuncGen,
6599	dimension: u32,
6600	default: u32,
6601	comptime basename: []const u8,
6602	) Allocator.Error!Builder.Value {
6603	return self.wip.callIntrinsic(.normal, .none, switch (dimension) {
6604	0 => @field(Builder.Intrinsic, basename ++ ".x"),
6605	1 => @field(Builder.Intrinsic, basename ++ ".y"),
6606	2 => @field(Builder.Intrinsic, basename ++ ".z"),
6607	else => return self.object.builder.intValue(.i32, default),
6608	}, &.{}, &.{}, "");
6609	}
6610
6611	fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6612	const target = self.object.zcu.getTarget();
6613
6614	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
6615	const dimension = pl_op.payload;
6616
6617	return switch (target.cpu.arch) {
6618	.amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workitem.id"),
6619	.nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.tid"),
6620	else => unreachable,
6621	};
6622	}
6623
6624	fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6625	const target = self.object.zcu.getTarget();
6626
6627	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
6628	const dimension = pl_op.payload;
6629
6630	switch (target.cpu.arch) {
6631	.amdgcn => {
6632	if (dimension >= 3) return .@"1";
6633
6634	// Fetch the dispatch pointer, which points to this structure:
6635	// https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/adae6c61e10d371f7cbc3d0e94ae2c070cab18a4/src/inc/hsa.h#L2913
6636	const dispatch_ptr =
6637	try self.wip.callIntrinsic(.normal, .none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, "");
6638
6639	// Load the work_group_* member from the struct as u16.
6640	// Just treat the dispatch pointer as an array of u16 to keep things simple.
6641	const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2);
6642	return self.load(workgroup_size_ptr, .@"2", .u16, .normal);
6643	},
6644	.nvptx, .nvptx64 => {
6645	return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
6646	},
6647	else => unreachable,
6648	}
6649	}
6650
6651	fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
6652	const target = self.object.zcu.getTarget();
6653
6654	const pl_op = self.air.instructions.items(.data)[@backingInt(inst)].pl_op;
6655	const dimension = pl_op.payload;
6656
6657	return switch (target.cpu.arch) {
6658	.amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workgroup.id"),
6659	.nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.ctaid"),
6660	else => unreachable,
6661	};
6662	}
6663
6664	/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
6665	fn optCmpNull(
6666	self: *FuncGen,
6667	cond: Builder.IntegerCondition,
6668	opt_ty: Type,
6669	opt_ptr: Builder.Value,
6670	access_kind: Builder.MemoryAccessKind,
6671	) Allocator.Error!Builder.Value {
6672	const zcu = self.object.zcu;
6673	assert(isByRef(opt_ty, zcu));
6674	comptime assert(optional_layout_version == 3);
6675	// Non-null bit is always after the payload, with no padding because it has alignment 1.
6676	const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu));
6677	const non_null = try self.load(non_null_ptr, .@"1", .bool, access_kind);
6678	return self.wip.icmp(cond, non_null, .false, "");
6679	}
6680
6681	/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
6682	fn optPayloadHandle(
6683	fg: *FuncGen,
6684	opt_ptr: Builder.Value,
6685	opt_ty: Type,
6686	can_elide_load: bool,
6687	) Allocator.Error!Builder.Value {
6688	const zcu = fg.object.zcu;
6689	assert(isByRef(opt_ty, zcu));
6690	const payload_ty = opt_ty.optionalChild(zcu);
6691
6692	// Payload is first field so always at the same address as the optional itself.
6693	const payload_ptr = opt_ptr;
6694
6695	if (can_elide_load and isByRef(payload_ty, zcu)) return payload_ptr;
6696
6697	return fg.load(payload_ptr, .none, payload_ty, .normal);
6698	}
6699
6700	fn fieldPtr(
6701	self: *FuncGen,
6702	aggregate_ptr: Builder.Value,
6703	aggregate_ptr_ty: Type,
6704	field_index: u32,
6705	) Allocator.Error!Builder.Value {
6706	const zcu = self.object.zcu;
6707	const aggregate_ty = aggregate_ptr_ty.childType(zcu);
6708	if (aggregate_ty.containerLayout(zcu) == .@"packed") {
6709	// A pointer to a bitpack field is equivalent to a pointer to the whole bitpack; the
6710	// bit offset is represented in the pointer *type*.
6711	return aggregate_ptr;
6712	}
6713	const offset: u64 = switch (aggregate_ty.zigTypeTag(zcu)) {
6714	.@"struct" => aggregate_ty.structFieldOffset(field_index, zcu),
6715	.@"union" => aggregate_ty.unionGetLayout(zcu).payloadOffset(),
6716	else => unreachable,
6717	};
6718	return self.ptraddConst(aggregate_ptr, offset);
6719	}
6720
6721	/// Non-atomic, non-bitpacked load of type `load_ty` from pointer `ptr`.
6722	///
6723	/// `ptr` has alignment `ptr_align`, or `load_ty.abiAlignment(zcu)` if `ptr_align` is `.none`.
6724	///
6725	/// If `load_ty` is a by-ref type, then the value is copied to a new alloca with a memcpy, and a
6726	/// pointer to that alloca is returned.
6727	fn load(
6728	fg: *FuncGen,
6729	ptr: Builder.Value,
6730	ptr_align: InternPool.Alignment,
6731	load_ty: Type,
6732	access_kind: Builder.MemoryAccessKind,
6733	) Allocator.Error!Builder.Value {
6734	const o = fg.object;
6735	const zcu = o.zcu;
6736
6737	const abi_align = load_ty.abiAlignment(zcu);
6738	const abi_size = load_ty.abiSize(zcu);
6739
6740	const llvm_ptr_align: Builder.Alignment = switch (ptr_align) {
6741	.none => abi_align.toLlvm(),
6742	else => |a| a.toLlvm(),
6743	};
6744
6745	if (isByRef(load_ty, zcu)) {
6746	const llvm_usize_ty = try o.lowerType(.usize, .as_value);
6747	const result_ptr = try fg.buildZigAlloca(load_ty, .none);
6748	_ = try fg.wip.callMemCpy(
6749	result_ptr,
6750	abi_align.toLlvm(),
6751	ptr,
6752	llvm_ptr_align,
6753	try o.builder.intValue(llvm_usize_ty, abi_size),
6754	access_kind,
6755	fg.disable_intrinsics,
6756	);
6757	return result_ptr;
6758	}
6759
6760	const llvm_access_ty = try o.lowerType(load_ty, .memory_access);
6761	const llvm_value_ty = try o.lowerType(load_ty, .as_value);
6762
6763	if (llvm_access_ty != llvm_value_ty) {
6764	const signedness: std.lang.Signedness = switch (load_ty.toIntern()) {
6765	.bool_type => .unsigned,
6766	else => load_ty.intInfo(zcu).signedness,
6767	};
6768	// `load_ty` is an integer type with padding bits. In theory, we shouldn't need any special
6769	// handling for these, as LLVM's documented semantics are a valid implementation of Zig's
6770	// semantics. However:
6771	//
6772	// * LLVM's lowering for these integer types generally leads to poor codegen, as integers
6773	// are only extended to the next byte, instead of to the next "natural" integer type.
6774	//
6775	// * Clang never emits loads or stores of these types, so LLVM's support for them is rather
6776	// flaky---we have encountered several LLVM bugs caused by incorrect handling of them.
6777	//
6778	// Therefore, we handle these memory accesses specially: in this case we will actually load
6779	// the next-largest "natural" integer type and then truncate to `load_ty`.
6780	const loaded = try fg.wip.load(access_kind, llvm_access_ty, ptr, llvm_ptr_align, "");
6781	// For packed structs, current Zig semantics don't really allow us to make the padding bits
6782	// well-defined. This should be solved once https://github.com/ziglang/zig/issues/24061 is
6783	// implemented, but until then, do a normal trunc for packed types.
6784	return fg.wip.cast(switch (load_ty.zigTypeTag(zcu)) {
6785	.@"struct", .@"union" => .trunc,
6786	else => switch (signedness) {
6787	.unsigned => .@"trunc nuw",
6788	.signed => .@"trunc nsw",
6789	},
6790	}, loaded, llvm_value_ty, "");
6791	}
6792
6793	// `load_ty` is a simple by-val type which requires no special handling.
6794	return fg.wip.load(access_kind, llvm_value_ty, ptr, llvm_ptr_align, "");
6795	}
6796
6797	/// Non-atomic, non-bitpacked store of `elem` to pointer `ptr`.
6798	///
6799	/// `ptr` has alignment `ptr_align`, or `elem_ty.abiAlignment(zcu)` if `ptr_align` is `.none`.
6800	///
6801	/// If `elem_ty` is a by-ref type, then `elem` is itself a pointer, and a memcpy is emitted.
6802	fn store(
6803	fg: *FuncGen,
6804	ptr: Builder.Value,
6805	ptr_align: InternPool.Alignment,
6806	elem: Builder.Value,
6807	elem_ty: Type,
6808	access_kind: Builder.MemoryAccessKind,
6809	) Allocator.Error!void {
6810	const o = fg.object;
6811	const zcu = o.zcu;
6812
6813	const abi_align = elem_ty.abiAlignment(zcu);
6814	const abi_size = elem_ty.abiSize(zcu);
6815
6816	const llvm_ptr_align = switch (ptr_align) {
6817	.none => abi_align.toLlvm(),
6818	else => ptr_align.toLlvm(),
6819	};
6820
6821	if (isByRef(elem_ty, zcu)) {
6822	const llvm_usize_ty = try o.lowerType(.usize, .as_value);
6823	_ = try fg.wip.callMemCpy(
6824	ptr,
6825	llvm_ptr_align,
6826	elem,
6827	abi_align.toLlvm(),
6828	try o.builder.intValue(llvm_usize_ty, abi_size),
6829	access_kind,
6830	fg.disable_intrinsics,
6831	);
6832	return;
6833	}
6834
6835	assert(elem.typeOfWip(&fg.wip) == try o.lowerType(elem_ty, .as_value));
6836
6837	const llvm_access_ty = try o.lowerType(elem_ty, .memory_access);
6838	const llvm_value_ty = try o.lowerType(elem_ty, .as_value);
6839
6840	if (llvm_access_ty != llvm_value_ty) {
6841	const signedness: std.lang.Signedness = switch (elem_ty.toIntern()) {
6842	.bool_type => .unsigned,
6843	else => elem_ty.intInfo(zcu).signedness,
6844	};
6845	// `elem_ty` is an integer type with padding bits, so we need to handle it specially---see
6846	// the corresponding comment in `FuncGen.load` for more details.
6847	const extended = try fg.wip.cast(switch (signedness) {
6848	.unsigned => .zext,
6849	.signed => .sext,
6850	}, elem, llvm_access_ty, "");
6851	_ = try fg.wip.store(access_kind, extended, ptr, llvm_ptr_align);
6852	return;
6853	}
6854
6855	// `elem_ty` is a simple by-val type which requires no special handling.
6856	_ = try fg.wip.store(access_kind, elem, ptr, llvm_ptr_align);
6857	}
6858
6859	fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
6860	const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
6861	const o = fg.object;
6862	const usize_ty = try o.lowerType(.usize, .as_value);
6863	const zero = try o.builder.intValue(usize_ty, 0);
6864	const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);
6865	const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");
6866	_ = try valgrindClientRequest(fg, zero, req, ptr_as_usize, len, zero, zero, zero);
6867	}
6868
6869	fn valgrindClientRequest(
6870	fg: *FuncGen,
6871	default_value: Builder.Value,
6872	request: Builder.Value,
6873	a1: Builder.Value,
6874	a2: Builder.Value,
6875	a3: Builder.Value,
6876	a4: Builder.Value,
6877	a5: Builder.Value,
6878	) Allocator.Error!Builder.Value {
6879	const o = fg.object;
6880	const zcu = o.zcu;
6881	const target = zcu.getTarget();
6882	if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
6883
6884	const llvm_usize = try o.lowerType(.usize, .as_value);
6885	const usize_align = Type.usize.abiAlignment(zcu).toLlvm();
6886
6887	const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
6888	const array_ptr = if (fg.valgrind_client_request_array == .none) a: {
6889	const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_align);
6890	fg.valgrind_client_request_array = array_ptr;
6891	break :a array_ptr;
6892	} else fg.valgrind_client_request_array;
6893	const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
6894	for (array_elements, 0..) |elem, i| {
6895	const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu));
6896	try fg.store(elem_ptr, .none, elem, .usize, .normal);
6897	}
6898
6899	const arch_specific: struct {
6900	template: [:0]const u8,
6901	constraints: [:0]const u8,
6902	} = switch (target.cpu.arch) {
6903	.arm, .armeb, .thumb, .thumbeb => .{
6904	.template =
6905	\\ mov r12, r12, ror #3 ; mov r12, r12, ror #13
6906	\\ mov r12, r12, ror #29 ; mov r12, r12, ror #19
6907	\\ orr r10, r10, r10
6908	,
6909	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
6910	},
6911	.aarch64, .aarch64_be => .{
6912	.template =
6913	\\ ror x12, x12, #3 ; ror x12, x12, #13
6914	\\ ror x12, x12, #51 ; ror x12, x12, #61
6915	\\ orr x10, x10, x10
6916	,
6917	.constraints = "={x3},{x4},{x3},~{cc},~{memory}",
6918	},
6919	.mips, .mipsel => .{
6920	.template =
6921	\\ srl $$0, $$0, 13
6922	\\ srl $$0, $$0, 29
6923	\\ srl $$0, $$0, 3
6924	\\ srl $$0, $$0, 19
6925	\\ or $$13, $$13, $$13
6926	,
6927	.constraints = "={$11},{$12},{$11},~{memory},~{$1}",
6928	},
6929	.mips64, .mips64el => .{
6930	.template =
6931	\\ dsll $$0, $$0, 3 ; dsll $$0, $$0, 13
6932	\\ dsll $$0, $$0, 29 ; dsll $$0, $$0, 19
6933	\\ or $$13, $$13, $$13
6934	,
6935	.constraints = "={$11},{$12},{$11},~{memory},~{$1}",
6936	},
6937	.powerpc, .powerpcle => .{
6938	.template =
6939	\\ rlwinm 0, 0, 3, 0, 31 ; rlwinm 0, 0, 13, 0, 31
6940	\\ rlwinm 0, 0, 29, 0, 31 ; rlwinm 0, 0, 19, 0, 31
6941	\\ or 1, 1, 1
6942	,
6943	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
6944	},
6945	.powerpc64, .powerpc64le => .{
6946	.template =
6947	\\ rotldi 0, 0, 3 ; rotldi 0, 0, 13
6948	\\ rotldi 0, 0, 61 ; rotldi 0, 0, 51
6949	\\ or 1, 1, 1
6950	,
6951	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
6952	},
6953	.riscv64 => .{
6954	.template =
6955	\\ .option push
6956	\\ .option norvc
6957	\\ srli zero, zero, 3
6958	\\ srli zero, zero, 13
6959	\\ srli zero, zero, 51
6960	\\ srli zero, zero, 61
6961	\\ or a0, a0, a0
6962	\\ .option pop
6963	,
6964	.constraints = "={a3},{a4},{a3},~{cc},~{memory}",
6965	},
6966	.s390x => .{
6967	.template =
6968	\\ lr %r15, %r15
6969	\\ lr %r1, %r1
6970	\\ lr %r2, %r2
6971	\\ lr %r3, %r3
6972	\\ lr %r2, %r2
6973	,
6974	.constraints = "={r3},{r2},{r3},~{cc},~{memory}",
6975	},
6976	.x86 => .{
6977	.template =
6978	\\ roll $$3, %edi ; roll $$13, %edi
6979	\\ roll $$61, %edi ; roll $$51, %edi
6980	\\ xchgl %ebx, %ebx
6981	,
6982	.constraints = "={edx},{eax},{edx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
6983	},
6984	.x86_64 => .{
6985	.template =
6986	\\ rolq $$3, %rdi ; rolq $$13, %rdi
6987	\\ rolq $$61, %rdi ; rolq $$51, %rdi
6988	\\ xchgq %rbx, %rbx
6989	,
6990	.constraints = "={rdx},{rax},{rdx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
6991	},
6992	else => unreachable,
6993	};
6994
6995	return fg.wip.callAsm(
6996	.none,
6997	try o.builder.fnType(llvm_usize, &.{ llvm_usize, llvm_usize }, .normal),
6998	.{ .sideeffect = true },
6999	try o.builder.string(arch_specific.template),
7000	try o.builder.string(arch_specific.constraints),
7001	&.{ try fg.wip.cast(.ptrtoint, array_ptr, llvm_usize, ""), default_value },
7002	"",
7003	);
7004	}
7005
7006	fn typeOf(fg: *FuncGen, inst: Air.Inst.Ref) Type {
7007	const zcu = fg.object.zcu;
7008	return fg.air.typeOf(inst, &zcu.intern_pool);
7009	}
7010
7011	fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type {
7012	const zcu = fg.object.zcu;
7013	return fg.air.typeOfIndex(inst, &zcu.intern_pool);
7014	}
7015
7016	const ParamTypeIterator = struct {
7017	object: *Object,
7018	cc: std.lang.CallingConvention,
7019	param_types: []const InternPool.Index,
7020	zig_index: u32,
7021	llvm_index: u32,
7022	types_len: u32,
7023	types_buffer: [8]Builder.Type,
7024	offsets_buffer: [9]u64,
7025	byval_attr: ?Object.Byval,
7026
7027	const Lowering = union(enum) {
7028	no_bits,
7029	byval,
7030	byref,
7031	byref_mut,
7032	abi_sized_int,
7033	multiple_llvm_types,
7034	slice,
7035	float_array: u8,
7036	i32_array: u8,
7037	i64_array: u8,
7038	};
7039
7040	pub fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering {
7041	if (it.zig_index >= it.param_types.len) return null;
7042	const ty = it.param_types[it.zig_index];
7043	it.byval_attr = null;
7044	return nextInner(it, Type.fromInterned(ty));
7045	}
7046
7047	/// `airCall` uses this instead of `next` so that it can take into account variadic functions.
7048	fn nextCall(it: *ParamTypeIterator, arg_types: []const InternPool.Index) Allocator.Error!?Lowering {
7049	if (it.zig_index >= it.param_types.len) {
7050	if (it.zig_index >= arg_types.len) {
7051	return null;
7052	} else {
7053	return nextInner(it, .fromInterned(arg_types[it.zig_index]));
7054	}
7055	} else {
7056	return nextInner(it, .fromInterned(it.param_types[it.zig_index]));
7057	}
7058	}
7059
7060	fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
7061	const zcu = it.object.zcu;
7062	ty.assertHasLayout(zcu);
7063	if (!ty.hasRuntimeBits(zcu)) {
7064	it.zig_index += 1;
7065	return .no_bits;
7066	}
7067	switch (it.cc) {
7068	.@"inline" => unreachable,
7069	.auto => {
7070	it.zig_index += 1;
7071	it.llvm_index += 1;
7072
7073	// Match the c calling convention in some cases to avoid llvm bugs.
7074	const target = zcu.getTarget();
7075	if (target.cpu.arch == .x86_64 and ty.isVector(zcu) and ty.childType(zcu).toIntern() == .bool_type) return switch (ty.vectorLen(zcu)) {
7076	0 => .no_bits,
7077	1...32 => .abi_sized_int,
7078	33...64 => {
7079	it.types_buffer[0..1].* = .{.double};
7080	it.offsets_buffer[0..2].* = .{ 0, 8 };
7081	it.types_len = 1;
7082	return .multiple_llvm_types;
7083	},
7084	else => .byval,
7085	};
7086
7087	if (ty.isSlice(zcu) or
7088	(ty.zigTypeTag(zcu) == .optional and ty.optionalChild(zcu).isSlice(zcu) and !ty.ptrAllowsZero(zcu)))
7089	{
7090	it.llvm_index += 1;
7091	return .slice;
7092	}
7093	if (isByRef(ty, zcu)) return .byref;
7094	return .byval;
7095	},
7096	.async => {
7097	@panic("TODO implement async function lowering in the LLVM backend");
7098	},
7099	.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => {
7100	it.zig_index += 1;
7101	it.llvm_index += 1;
7102	switch (aarch64_c_abi.classifyType(ty, zcu)) {
7103	.memory => return .byref_mut,
7104	.float_array => |len| return .{ .float_array = len },
7105	.byval => return .byval,
7106	.integer => {
7107	it.types_buffer[0..1].* = .{.i64};
7108	it.offsets_buffer[0..2].* = .{ 0, 8 };
7109	it.types_len = 1;
7110	return .multiple_llvm_types;
7111	},
7112	.double_integer => return .{ .i64_array = 2 },
7113	}
7114	},
7115	.arm_aapcs, .arm_aapcs_vfp => {
7116	it.zig_index += 1;
7117	it.llvm_index += 1;
7118	switch (arm_c_abi.classifyType(ty, zcu, .arg)) {
7119	.memory => {
7120	it.byval_attr = .{};
7121	return .byref;
7122	},
7123	.byval => return .byval,
7124	.i32_array => |size| return .{ .i32_array = size },
7125	.i64_array => |size| return .{ .i64_array = size },
7126	}
7127	},
7128	.loongarch32_ilp32, .loongarch64_lp64 => switch (loongarch_c_abi.classifyType(ty, zcu)) {
7129	.ignored => {
7130	it.zig_index += 1;
7131	return .no_bits;
7132	},
7133	.gar, .far => {
7134	it.zig_index += 1;
7135	it.llvm_index += 1;
7136	return .byval;
7137	},
7138	.member => |member_ty| {
7139	it.types_buffer[0..1].* = .{
7140	try it.object.lowerType(member_ty, .as_value),
7141	};
7142	it.offsets_buffer[0..2].* = .{ 0, member_ty.abiSize(zcu) };
7143	it.types_len = 1;
7144	it.zig_index += 1;
7145	it.llvm_index += 1;
7146	return .multiple_llvm_types;
7147	},
7148	.member_pair => |member_tys| {
7149	it.types_buffer[0..2].* = .{
7150	try it.object.lowerType(member_tys[0], .as_value),
7151	try it.object.lowerType(member_tys[1], .as_value),
7152	};
7153	const first_size = member_tys[0].abiSize(zcu);
7154	const second_size = member_tys[0].abiSize(zcu);
7155	it.offsets_buffer[0..3].* = .{ 0, first_size, first_size + second_size };
7156	it.types_len = 2;
7157	it.zig_index += 1;
7158	it.llvm_index += 2;
7159	return .multiple_llvm_types;
7160	},
7161	.memory_gar => {
7162	switch (it.cc) {
7163	else => unreachable,
7164	.loongarch32_ilp32 => {
7165	it.types_buffer[0..1].* = .{.i32};
7166	it.offsets_buffer[0..2].* = .{ 0, 4 };
7167	},
7168	.loongarch64_lp64 => {
7169	it.types_buffer[0..1].* = .{.i64};
7170	it.offsets_buffer[0..2].* = .{ 0, 8 };
7171	},
7172	}
7173	it.types_len = 1;
7174	it.zig_index += 1;
7175	it.llvm_index += 1;
7176	return .multiple_llvm_types;
7177	},
7178	.memory_gar_pair => {
7179	it.zig_index += 1;
7180	it.llvm_index += 1;
7181	return switch (it.cc) {
7182	else => unreachable,
7183	.loongarch32_ilp32 => .{ .i32_array = 2 },
7184	.loongarch64_lp64 => .{ .i64_array = 2 },
7185	};
7186	},
7187	.address => {
7188	it.zig_index += 1;
7189	it.llvm_index += 1;
7190	return .byref;
7191	},
7192	},
7193	.mips_o32 => {
7194	it.zig_index += 1;
7195	it.llvm_index += 1;
7196	switch (mips_c_abi.classifyType(ty, zcu, .arg)) {
7197	.memory => {
7198	it.byval_attr = .{};
7199	return .byref;
7200	},
7201	.byval => return .byval,
7202	.i32_array => |size| return .{ .i32_array = size },
7203	}
7204	},
7205	.powerpc64_elf_v2 => {
7206	it.zig_index += 1;
7207	it.llvm_index += 1;
7208	if (isByRef(ty, zcu)) return switch (ty.abiSize(zcu)) {
7209	1...8 => .abi_sized_int,
7210	9...64 => |abi_size| .{ .i64_array = @intCast(@divCeil(abi_size, 8)) },
7211	else => .byref,
7212	};
7213	return .byval; // TODO
7214	},
7215	.riscv64_lp64, .riscv32_ilp32 => {
7216	it.zig_index += 1;
7217	it.llvm_index += 1;
7218	switch (riscv_c_abi.classifyType(ty, zcu)) {
7219	.memory => return .byref_mut,
7220	.byval => return .byval,
7221	.integer => return .abi_sized_int,
7222	.double_integer => return .{ .i64_array = 2 },
7223	.fields => {
7224	it.types_len = 0;
7225	var field_it: InternPool.LoadedStructType.RuntimeOrderIterator = if (zcu.typeToStruct(ty)) |loaded_struct|
7226	loaded_struct.iterateRuntimeOrder(&zcu.intern_pool)
7227	else
7228	.{ .runtime_order = null, .fields_len = ty.structFieldCount(zcu), .next_index = 0 };
7229	while (field_it.next()) |field_index| {
7230	const field_ty = ty.fieldType(field_index, zcu);
7231	if (!field_ty.hasRuntimeBits(zcu)) continue;
7232	it.types_buffer[it.types_len] = try it.object.lowerType(field_ty, .as_value);
7233	it.offsets_buffer[it.types_len] = ty.structFieldOffset(field_index, zcu);
7234	it.types_len += 1;
7235	}
7236	it.offsets_buffer[it.types_len] = ty.abiSize(zcu);
7237	it.llvm_index += it.types_len - 1;
7238	return .multiple_llvm_types;
7239	},
7240	}
7241	},
7242	.s390x_sysv, .s390x_sysv_vx => {
7243	it.zig_index += 1;
7244	switch (s390x_c_abi.classifyType(ty, .arg, zcu)) {
7245	.none => return .no_bits,
7246	.double_or_float, .vector, .simple => {
7247	it.llvm_index += 1;
7248	return .byval;
7249	},
7250	.simple_aggregate => {
7251	it.llvm_index += 1;
7252	return .abi_sized_int;
7253	},
7254	.pointer => {
7255	it.llvm_index += 1;
7256	return .byref_mut;
7257	},
7258	}
7259	},
7260	.wasm_mvp => switch (wasm_c_abi.classifyTypeForLlvm(ty, zcu)) {
7261	.direct => |scalar_ty| {
7262	if (isScalar(zcu, ty)) {
7263	it.zig_index += 1;
7264	it.llvm_index += 1;
7265	return .byval;
7266	} else {
7267	it.types_buffer[0..1].* = .{try it.object.lowerType(scalar_ty, .as_value)};
7268	it.offsets_buffer[0..2].* = .{ 0, scalar_ty.abiSize(zcu) };
7269	it.types_len = 1;
7270	it.zig_index += 1;
7271	it.llvm_index += 1;
7272	return .multiple_llvm_types;
7273	}
7274	},
7275	.indirect => {
7276	it.zig_index += 1;
7277	it.llvm_index += 1;
7278	it.byval_attr = .{};
7279	return .byref;
7280	},
7281	},
7282	.x86_stdcall => {
7283	it.zig_index += 1;
7284	it.llvm_index += 1;
7285
7286	if (isScalar(zcu, ty)) {
7287	return .byval;
7288	} else {
7289	it.byval_attr = .{};
7290	return .byref;
7291	}
7292	},
7293	.x86_sysv, .x86_win, .x86_mingw => {
7294	if (isByRef(ty, zcu)) {
7295	var items_buf: [1]codegen.FlattenedItem = undefined;
7296	if (codegen.flattenType(&items_buf, ty, zcu, .{
7297	.allow_arrays = false,
7298	})) |items| one_float: {
7299	if (items.len != 1 or items[0].offset != 0) break :one_float;
7300	const item_ty = items[0].type orelse break :one_float;
7301	if (!item_ty.isRuntimeFloat()) break :one_float;
7302	it.types_buffer[0..1].*, it.offsets_buffer[0..2].* =
7303	switch (item_ty.floatBits(zcu.getTarget())) {
7304	else => unreachable,
7305	32 => .{ .{.float}, .{ 0, 4 } },
7306	64 => .{ .{.double}, .{ 0, 8 } },
7307	16, 80, 128 => break :one_float,
7308	};
7309	it.types_len = 1;
7310	it.zig_index += 1;
7311	it.llvm_index += 1;
7312	return .multiple_llvm_types;
7313	}
7314	it.zig_index += 1;
7315	it.llvm_index += 1;
7316	it.byval_attr = .{ .alignment = .@"4" };
7317	return .byref;
7318	}
7319	if (ty.isAbiInt(zcu)) switch (ty.intInfo(zcu).bits) {
7320	else => unreachable,
7321	8, 16, 32, 64 => {
7322	it.zig_index += 1;
7323	it.llvm_index += 1;
7324	return .byval;
7325	},
7326	128 => {
7327	it.types_buffer[0..2].* = .{ .i64, .i64 };
7328	it.offsets_buffer[0..3].* = .{ 0, 8, 16 };
7329	it.types_len = 2;
7330	it.zig_index += 1;
7331	it.llvm_index += 2;
7332	return .multiple_llvm_types;
7333	},
7334	};
7335	it.zig_index += 1;
7336	it.llvm_index += 1;
7337	return .byval;
7338	},
7339	.x86_64_sysv, .x86_64_x32 => return try it.next_x86_64_sysv(ty),
7340	.x86_64_win => return it.next_x86_64_win(ty),
7341	// TODO investigate other callconvs
7342	else => {
7343	it.zig_index += 1;
7344	it.llvm_index += 1;
7345	return .byval;
7346	},
7347	}
7348	}
7349
7350	fn next_x86_64_win(it: *ParamTypeIterator, ty: Type) Lowering {
7351	const zcu = it.object.zcu;
7352	switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) {
7353	.integer => {
7354	if (isScalar(zcu, ty)) {
7355	it.zig_index += 1;
7356	it.llvm_index += 1;
7357	return .byval;
7358	} else {
7359	it.zig_index += 1;
7360	it.llvm_index += 1;
7361	return .abi_sized_int;
7362	}
7363	},
7364	.sse,
7365	.bool_vector_mask,
7366	.integer_per_element,
7367	.sse_per_element,
7368	.sse_sse_x87_per_qword,
7369	.sse_per_xword,
7370	.sse_per_yword,
7371	.sse_per_zword,
7372	=> {
7373	it.zig_index += 1;
7374	it.llvm_index += 1;
7375	return .byval;
7376	},
7377	.sseup, .x87, .x87up, .none, .float, .float_combine => unreachable,
7378	.memory => {
7379	it.zig_index += 1;
7380	it.llvm_index += 1;
7381	return .byref_mut;
7382	},
7383	.win_i128 => {
7384	it.zig_index += 1;
7385	it.llvm_index += 1;
7386	return .byref;
7387	},
7388	}
7389	}
7390
7391	fn next_x86_64_sysv(it: *ParamTypeIterator, ty: Type) Allocator.Error!Lowering {
7392	const o = it.object;
7393	const zcu = o.zcu;
7394	const target = zcu.getTarget();
7395	const classes = x86_64_abi.classifySystemV(ty, zcu, target, .arg);
7396	var types_len: u32 = 0;
7397	const classes_len = for (classes, 0..) |class, class_index| switch (class) {
7398	.integer => {
7399	it.types_buffer[types_len] = try o.builder.intType(@min(8 * ty.abiSize(zcu) - 64 * class_index, 64));
7400	it.offsets_buffer[types_len] = 8 * class_index;
7401	types_len += 1;
7402	},
7403	.sse => {
7404	it.types_buffer[types_len] = .double;
7405	it.offsets_buffer[types_len] = 8 * class_index;
7406	types_len += 1;
7407	},
7408	.sseup => {
7409	if (it.types_buffer[types_len - 1] == .double) {
7410	if (ty.isVector(zcu)) {
7411	it.zig_index += 1;
7412	it.llvm_index += 1;
7413	return .byval;
7414	}
7415	it.types_buffer[types_len - 1] = .fp128;
7416	} else {
7417	it.types_buffer[types_len] = .double;
7418	it.offsets_buffer[types_len] = 8 * class_index;
7419	types_len += 1;
7420	}
7421	},
7422	.float => {
7423	it.types_buffer[types_len] = .float;
7424	it.offsets_buffer[types_len] = 8 * class_index;
7425	types_len += 1;
7426	},
7427	.float_combine => {
7428	it.types_buffer[types_len] = try it.object.builder.vectorType(.normal, 2, .float);
7429	it.offsets_buffer[types_len] = 8 * class_index;
7430	types_len += 1;
7431	},
7432	.x87 => {
7433	it.zig_index += 1;
7434	it.llvm_index += 1;
7435	it.byval_attr = .{};
7436	return .byref;
7437	},
7438	.x87up => unreachable,
7439	.none => break class_index,
7440	.memory => {
7441	it.zig_index += 1;
7442	it.llvm_index += 1;
7443	it.byval_attr = .{};
7444	return .byref;
7445	},
7446	.win_i128 => unreachable, // windows only
7447	.bool_vector_mask,
7448	.integer_per_element,
7449	.sse_per_element,
7450	.sse_sse_x87_per_qword,
7451	.sse_per_xword,
7452	.sse_per_yword,
7453	.sse_per_zword,
7454	=> {
7455	it.zig_index += 1;
7456	it.llvm_index += 1;
7457	return .byval;
7458	},
7459	} else classes.len;
7460	if (types_len > 1) {
7461	if (it.llvm_index + classes_len > 6) {
7462	it.zig_index += 1;
7463	it.llvm_index += 1;
7464	it.byval_attr = .{};
7465	return .byref;
7466	}
7467	} else if (!isByRef(ty, zcu)) {
7468	const llvm_ty = try o.lowerType(ty, .as_value);
7469	if (it.types_buffer[0] == llvm_ty or
7470	(it.types_buffer[0] == .i64 and llvm_ty.isPointer(&o.builder)))
7471	{
7472	it.zig_index += 1;
7473	it.llvm_index += 1;
7474	return .byval;
7475	}
7476	}
7477	it.offsets_buffer[types_len] = 8 * classes_len;
7478	it.types_len = types_len;
7479	it.llvm_index += types_len;
7480	it.zig_index += 1;
7481	return .multiple_llvm_types;
7482	}
7483	};
7484	pub fn iterateParamTypes(
7485	object: *Object,
7486	cc: std.lang.CallingConvention,
7487	param_types: []const InternPool.Index,
7488	) ParamTypeIterator {
7489	return .{
7490	.object = object,
7491	.cc = cc,
7492	.param_types = param_types,
7493	.zig_index = 0,
7494	.llvm_index = 0,
7495	.types_len = undefined,
7496	.types_buffer = undefined,
7497	.offsets_buffer = undefined,
7498	.byval_attr = null,
7499	};
7500	}
7501
7502	pub const FnReturnStrat = union(enum) {
7503	/// The function return type is OPV (zero-bit), so the LLVM function return type is `void`.
7504	void,
7505	/// An sret parameter is used. The LLVM function return type is `void`.
7506	sret,
7507	/// The function's return type directly corresponds to the LLVM function return type.
7508	///
7509	/// The return type is by-val, i.e. `isByRef` returns `false`.
7510	by_val,
7511	/// The LLVM function returns the given `Builder.Type` by reinterpreting memory containing the
7512	/// actual return value. The actual return type may be by-val or by-ref.
7513	mem_cast: Builder.Type,
7514
7515	fn forceByVal(o: *Object, ret_ty: Type) Allocator.Error!FnReturnStrat {
7516	if (!isByRef(ret_ty, o.zcu)) return .by_val;
7517	return .{ .mem_cast = try o.lowerType(ret_ty, .in_memory) };
7518	}
7519	};
7520	/// In order to support the C calling convention, some return types need to be lowered
7521	/// completely differently in the function prototype to honor the C ABI, and then
7522	/// be effectively bitcasted to the actual return type.
7523	pub fn fnReturnStrat(o: *Object, cc: std.lang.CallingConvention, ret_ty: Type) Allocator.Error!FnReturnStrat {
7524	const zcu = o.zcu;
7525	ret_ty.assertHasLayout(zcu);
7526	if (!ret_ty.hasRuntimeBits(zcu)) return .void;
7527	return switch (cc) {
7528	.@"inline" => unreachable,
7529	.auto => {
7530	// Match the c calling convention in some cases to avoid llvm bugs.
7531	const target = zcu.getTarget();
7532	if (target.cpu.arch == .x86_64 and ret_ty.isVector(zcu) and ret_ty.childType(zcu).toIntern() == .bool_type) return switch (ret_ty.vectorLen(zcu)) {
7533	0 => .void,
7534	1...8 => .{ .mem_cast = .i8 },
7535	9...16 => .{ .mem_cast = .i16 },
7536	17...32 => .{ .mem_cast = .i32 },
7537	33...64 => .{ .mem_cast = .double },
7538	else => .by_val,
7539	};
7540	return if (isByRef(ret_ty, zcu)) .sret else .by_val;
7541	},
7542	.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(ret_ty, zcu)) {
7543	.memory => .sret,
7544	.float_array, .byval => .forceByVal(o, ret_ty),
7545	.integer => .{ .mem_cast = .i64 },
7546	.double_integer => .{ .mem_cast = try o.builder.arrayType(2, .i64) },
7547	},
7548	.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(ret_ty, zcu, .ret)) {
7549	.memory, .i64_array => .sret,
7550	.i32_array => |len| if (len == 1) .{ .mem_cast = .i32 } else .sret,
7551	.byval => .forceByVal(o, ret_ty),
7552	},
7553	.loongarch32_ilp32, .loongarch64_lp64 => switch (loongarch_c_abi.classifyType(ret_ty, zcu)) {
7554	.ignored => .void,
7555	.gar, .far => .by_val,
7556	.member => |member_ty| .{ .mem_cast = try o.lowerType(member_ty, .as_value) },
7557	.member_pair => |member_tys| .{ .mem_cast = try o.builder.structType(.normal, &.{
7558	try o.lowerType(member_tys[0], .as_value),
7559	try o.lowerType(member_tys[1], .as_value),
7560	}) },
7561	.memory_gar => .{ .mem_cast = switch (cc) {
7562	else => unreachable,
7563	.loongarch32_ilp32 => .i32,
7564	.loongarch64_lp64 => .i64,
7565	} },
7566	.memory_gar_pair => .{ .mem_cast = try o.builder.arrayType(2, switch (cc) {
7567	else => unreachable,
7568	.loongarch32_ilp32 => .i32,
7569	.loongarch64_lp64 => .i64,
7570	}) },
7571	.address => .sret,
7572	},
7573	.mips_o32 => switch (mips_c_abi.classifyType(ret_ty, zcu, .ret)) {
7574	.memory, .i32_array => .sret,
7575	.byval => .forceByVal(o, ret_ty),
7576	},
7577	.powerpc64_elf_v2 => if (isByRef(ret_ty, zcu)) switch (ret_ty.abiSize(zcu)) {
7578	1...8 => .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) },
7579	9...16 => .{ .mem_cast = try o.builder.structType(.normal, &.{ .i64, .i64 }) },
7580	else => .sret,
7581	} else .by_val, // TODO
7582	.riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(ret_ty, zcu)) {
7583	.memory => .sret,
7584	.integer => .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) },
7585	.double_integer => {
7586	const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {
7587	.riscv64, .riscv64be => .i64,
7588	.riscv32, .riscv32be => .i32,
7589	else => unreachable,
7590	};
7591	return .{ .mem_cast = try o.builder.structType(.normal, &.{ integer, integer }) };
7592	},
7593	.byval => .forceByVal(o, ret_ty),
7594	.fields => {
7595	var types_len: usize = 0;
7596	var types: [8]Builder.Type = undefined;
7597	for (0..ret_ty.structFieldCount(zcu)) |field_index| {
7598	const field_ty = ret_ty.fieldType(field_index, zcu);
7599	if (!field_ty.hasRuntimeBits(zcu)) continue;
7600	types[types_len] = try o.lowerType(field_ty, .as_value);
7601	types_len += 1;
7602	}
7603	return .{ .mem_cast = try o.builder.structType(.normal, types[0..types_len]) };
7604	},
7605	},
7606	.s390x_sysv, .s390x_sysv_vx => switch (s390x_c_abi.classifyType(ret_ty, .ret, zcu)) {
7607	.none => .void,
7608	.double_or_float, .vector, .simple => .by_val,
7609	.simple_aggregate => unreachable,
7610	.pointer => .sret,
7611	},
7612	.wasm_mvp => switch (wasm_c_abi.classifyTypeForLlvm(ret_ty, zcu)) {
7613	.direct => |scalar_ty| if (scalar_ty.toIntern() == ret_ty.toIntern()) {
7614	assert(!isByRef(ret_ty, zcu));
7615	return .by_val;
7616	} else .{ .mem_cast = try o.lowerType(scalar_ty, .as_value) },
7617	.indirect => .sret,
7618	},
7619	.x86_stdcall => if (isScalar(zcu, ret_ty)) {
7620	assert(!isByRef(ret_ty, zcu));
7621	return .by_val;
7622	} else .sret,
7623	.x86_fastcall => fnReturnStrat_x86_fastcall(o, zcu, ret_ty),
7624	.x86_sysv, .x86_win, .x86_mingw => if (isByRef(ret_ty, zcu)) {
7625	switch (cc) {
7626	else => unreachable,
7627	.x86_sysv => return .sret,
7628	.x86_win => {},
7629	.x86_mingw => {
7630	var items_buf: [1]codegen.FlattenedItem = undefined;
7631	if (codegen.flattenType(&items_buf, ret_ty, zcu, .{})) |items| one_float: {
7632	if (items.len != 1 or items[0].offset != 0) break :one_float;
7633	const item_ty = items[0].type orelse break :one_float;
7634	if (!item_ty.isRuntimeFloat()) break :one_float;
7635	return .{ .mem_cast = switch (item_ty.floatBits(zcu.getTarget())) {
7636	else => unreachable,
7637	16 => .half,
7638	32 => .float,
7639	64 => .double,
7640	80, 128 => break :one_float,
7641	} };
7642	}
7643	},
7644	}
7645	return switch (ret_ty.abiSize(zcu)) {
7646	0 => .void,
7647	1 => .{ .mem_cast = .i8 },
7648	2 => .{ .mem_cast = .i16 },
7649	4 => .{ .mem_cast = .i32 },
7650	8 => .{ .mem_cast = .i64 },
7651	else => .sret,
7652	};
7653	} else if (ret_ty.isAbiInt(zcu) and ret_ty.intInfo(zcu).bits > 64)
7654	.sret
7655	else
7656	.by_val,
7657	.x86_64_sysv, .x86_64_x32 => fnReturnStrat_x86_64_sysv(o, ret_ty),
7658	.x86_64_win => fnReturnStrat_x86_64_win(o, ret_ty),
7659	// TODO investigate other callconvs
7660	else => .forceByVal(o, ret_ty),
7661	};
7662	}
7663
7664	fn fnReturnStrat_x86_fastcall(o: *Object, zcu: *Zcu, ty: Type) Allocator.Error!FnReturnStrat {
7665	if (isScalar(zcu, ty)) {
7666	assert(!isByRef(ty, zcu));
7667	return .by_val;
7668	}
7669	const tag = ty.zigTypeTag(zcu);
7670	if (tag == .@"struct" or tag == .@"union") {
7671	const size = ty.abiSize(zcu);
7672	if (size == 1 or size == 2 or size == 4 or size == 8) {
7673	return .{ .mem_cast = try o.builder.intType(@intCast(size * 8)) };
7674	}
7675	}
7676	return .sret;
7677	}
7678
7679	fn fnReturnStrat_x86_64_win(o: *Object, ret_ty: Type) Allocator.Error!FnReturnStrat {
7680	const zcu = o.zcu;
7681	switch (x86_64_abi.classifyWindows(ret_ty, zcu, zcu.getTarget(), .ret)) {
7682	.integer => if (isScalar(zcu, ret_ty)) {
7683	assert(!isByRef(ret_ty, zcu));
7684	return .by_val;
7685	} else {
7686	return .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) };
7687	},
7688	.win_i128 => return .{ .mem_cast = try o.builder.vectorType(.normal, 2, .i64) },
7689	.memory => return .sret,
7690
7691	.sse,
7692	.bool_vector_mask,
7693	.integer_per_element,
7694	.sse_per_element,
7695	.sse_sse_x87_per_qword,
7696	.sse_per_xword,
7697	.sse_per_yword,
7698	.sse_per_zword,
7699	=> {
7700	assert(!isByRef(ret_ty, zcu));
7701	return .by_val;
7702	},
7703	.sseup,
7704	.x87,
7705	.x87up,
7706	.none,
7707	.float,
7708	.float_combine,
7709	=> unreachable,
7710	}
7711	}
7712
7713	fn fnReturnStrat_x86_64_sysv(o: *Object, ret_ty: Type) Allocator.Error!FnReturnStrat {
7714	const zcu = o.zcu;
7715	const classes = x86_64_abi.classifySystemV(ret_ty, zcu, zcu.getTarget(), .ret);
7716	var types_buffer: [8]Builder.Type = undefined;
7717	var types_len: u32 = 0;
7718	for (classes, 0..) |class, class_index| switch (class) {
7719	.integer => {
7720	types_buffer[types_len] = try o.builder.intType(@min(8 * ret_ty.abiSize(zcu) - 64 * class_index, 64));
7721	types_len += 1;
7722	},
7723	.sse => {
7724	types_buffer[types_len] = .double;
7725	types_len += 1;
7726	},
7727	.sseup => {
7728	if (types_buffer[types_len - 1] == .double) {
7729	if (ret_ty.isVector(zcu)) return .by_val;
7730	types_buffer[types_len - 1] = .fp128;
7731	} else {
7732	types_buffer[types_len] = .double;
7733	types_len += 1;
7734	}
7735	},
7736	.float => {
7737	types_buffer[types_len] = .float;
7738	types_len += 1;
7739	},
7740	.float_combine => {
7741	types_buffer[types_len] = try o.builder.vectorType(.normal, 2, .float);
7742	types_len += 1;
7743	},
7744	.x87 => {
7745	if (types_len > 0 or classes[2] != .none) return .sret;
7746	types_buffer[types_len] = .x86_fp80;
7747	types_len += 1;
7748	},
7749	.x87up => continue,
7750	.none => break,
7751	.memory => return if (ret_ty.isVector(zcu)) .by_val else .sret,
7752	.win_i128 => unreachable, // windows only
7753	.bool_vector_mask,
7754	.integer_per_element,
7755	.sse_per_element,
7756	.sse_sse_x87_per_qword,
7757	.sse_per_xword,
7758	.sse_per_yword,
7759	.sse_per_zword,
7760	=> return .by_val,
7761	};
7762	if (types_len > 1) return .{ .mem_cast = try o.builder.structType(.normal, types_buffer[0..types_len]) };
7763	if (!isByRef(ret_ty, zcu)) {
7764	const llvm_ty = try o.lowerType(ret_ty, .as_value);
7765	if (types_buffer[0] == llvm_ty) return .by_val;
7766	if (types_buffer[0] == .i64 and llvm_ty.isPointer(&o.builder)) return .by_val;
7767	if (types_buffer[0] == .double and llvm_ty.isVector(&o.builder) and
7768	llvm_ty.vectorLen(&o.builder) == 1 and
7769	llvm_ty.scalarType(&o.builder) == .double) return .by_val;
7770	}
7771	return .{ .mem_cast = types_buffer[0] };
7772	}
7773
7774	/// This function deliberately does not handle `_BitInt` because it typically
7775	/// has different ABI than regular integer types, and there is currently no way
7776	/// to determine whether a Zig integer type is meant to represent e.g. `int`
7777	/// or `_BitInt(32)`.
7778	pub fn ccAbiPromoteInt(cc: std.lang.CallingConvention, zcu: *Zcu, ty: Type) ?std.lang.Signedness {
7779	switch (cc) {
7780	.auto, .@"inline", .async => return null,
7781	else => {},
7782	}
7783
7784	const target = zcu.getTarget();
7785	const int_info: std.lang.Type.Int = if (ty.toIntern() == .bool_type)
7786	.{ .signedness = .unsigned, .bits = 1 }
7787	else if (ty.isAbiInt(zcu))
7788	ty.intInfo(zcu)
7789	else if (ty.isRuntimeFloat()) switch (ty.floatBits(target)) {
7790	else => unreachable,
7791	16, 32, 64 => |bits| switch (std.zig.target.compilerRtFloatAbi(target, bits)) {
7792	.hard => return null,
7793	.soft => .{ .signedness = .unsigned, .bits = bits },
7794	},
7795	80, 128 => return null,
7796	} else return null;
7797
7798	assert(int_info.bits == 0 or (int_info.bits == 1 and ty.toIntern() == .bool_type) or std.math.isPowerOfTwo(int_info.bits));
7799
7800	return switch (target.cpu.arch) {
7801	.aarch64,
7802	.aarch64_be,
7803	=> switch (target.os.tag) {
7804	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => switch (int_info.bits) {
7805	1, 8, 16 => int_info.signedness,
7806	else => null,
7807	},
7808	else => null,
7809	},
7810
7811	.avr,
7812	=> switch (int_info.bits) {
7813	1, 8 => int_info.signedness,
7814	else => null,
7815	},
7816
7817	.lanai,
7818	=> null,
7819
7820	.loongarch64,
7821	.riscv64,
7822	.riscv64be,
7823	=> switch (int_info.bits) {
7824	1, 8, 16 => int_info.signedness,
7825	32 => .signed,
7826	else => null,
7827	},
7828
7829	.mips,
7830	.mipsel,
7831	.mips64,
7832	.mips64el,
7833	=> switch (int_info.bits) {
7834	1, 8, 16, 64 => int_info.signedness,
7835	32 => .signed,
7836	else => null,
7837	},
7838
7839	.powerpc64,
7840	.powerpc64le,
7841	.s390x,
7842	.sparc64,
7843	.ve,
7844	=> switch (int_info.bits) {
7845	1, 8, 16, 32 => int_info.signedness,
7846	else => null,
7847	},
7848
7849	else => switch (int_info.bits) {
7850	1, 8, 16 => int_info.signedness,
7851	else => null,
7852	},
7853	};
7854	}
7855
7856	fn isScalar(zcu: *Zcu, ty: Type) bool {
7857	return switch (ty.zigTypeTag(zcu)) {
7858	.void,
7859	.bool,
7860	.noreturn,
7861	.int,
7862	.float,
7863	.pointer,
7864	.optional,
7865	.error_set,
7866	.@"enum",
7867	.@"anyframe",
7868	.vector,
7869	=> true,
7870
7871	.@"struct" => ty.containerLayout(zcu) == .@"packed",
7872	.@"union" => ty.containerLayout(zcu) == .@"packed",
7873	else => false,
7874	};
7875	}
7876
7877	/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
7878	/// or as an LLVM value.
7879	pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
7880	return switch (ty.zigTypeTag(zcu)) {
7881	.type,
7882	.comptime_int,
7883	.comptime_float,
7884	.enum_literal,
7885	.undefined,
7886	.null,
7887	.@"opaque",
7888	.spirv,
7889	=> unreachable,
7890
7891	.noreturn,
7892	.void,
7893	.bool,
7894	.int,
7895	.pointer,
7896	.error_set,
7897	.@"fn",
7898	.@"enum",
7899	.@"anyframe",
7900	=> false,
7901
7902	.float, .vector => {
7903	const target = zcu.getTarget();
7904	const scalar_ty = ty.scalarType(zcu);
7905	return if (scalar_ty.isRuntimeFloat()) switch (scalar_ty.floatBits(target)) {
7906	else => unreachable,
7907	16, 32, 64 => false,
7908	80, 128 => |bits| switch (std.zig.target.compilerRtFloatAbi(target, bits)) {
7909	.hard => false,
7910	.soft => true,
7911	},
7912	} else false;
7913	},
7914
7915	.array,
7916	.frame,
7917	=> ty.hasRuntimeBits(zcu),
7918
7919	.error_union => ty.errorUnionPayload(zcu).hasRuntimeBits(zcu),
7920
7921	.optional => !ty.optionalReprIsPayload(zcu) and ty.optionalChild(zcu).hasRuntimeBits(zcu),
7922
7923	.@"struct" => switch (ty.containerLayout(zcu)) {
7924	.@"packed" => false,
7925	.auto, .@"extern" => ty.hasRuntimeBits(zcu),
7926	},
7927	.@"union" => switch (ty.containerLayout(zcu)) {
7928	.@"packed" => false,
7929	else => ty.hasRuntimeBits(zcu),
7930	},
7931	};
7932	}
7933
7934	/// If the operand type of an atomic operation is not byte sized we need to
7935	/// widen it before using it and then truncate the result.
7936	/// RMW exchange of floating-point values is bitcasted to same-sized integer
7937	/// types to work around a LLVM deficiency when targeting ARM/AArch64.
7938	fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type {
7939	const zcu = fg.object.zcu;
7940	switch (ty.zigTypeTag(zcu)) {
7941	.int, .@"enum", .@"struct", .@"union" => {},
7942	.float => {
7943	if (!is_rmw_xchg) return .none;
7944	return fg.object.builder.intType(@intCast(ty.abiSize(zcu) * 8));
7945	},
7946	.bool => return .i8,
7947	else => return .none,
7948	}
7949	const bit_count = ty.bitSize(zcu);
7950	if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) {
7951	return fg.object.builder.intType(@intCast(ty.abiSize(zcu) * 8));
7952	} else {
7953	return .none;
7954	}
7955	}
7956
7957	fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
7958	if (offset == 0) return ptr;
7959	const o = fg.object;
7960	const llvm_usize_ty = try o.lowerType(.usize, .as_value);
7961	const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
7962	return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
7963	}
7964	fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value {
7965	if (scale == 0) return ptr;
7966	// Right now LLVM seems to fare a bit worse with an explicit `mul nuw` instruction than it does
7967	// if we use a bigger type for the GEP, so we'll do that. As I understand it, it has not yet
7968	// been decided whether the planned `ptradd` instruction will accept a scale or not; if it does
7969	// not then presumably upstream will improve their handling of explicit `mul nuw` computing the
7970	// offset.
7971	const llvm_scale_ty = try fg.object.builder.arrayType(scale, .i8);
7972	return fg.wip.gep(.inbounds, llvm_scale_ty, ptr, &.{index}, "");
7973	}
7974
7975	fn compilerRtPromoteInt(int_info: InternPool.Key.IntType) ?Type {
7976	if (int_info.bits <= 32) return switch (int_info.signedness) {
7977	.signed => .i32,
7978	.unsigned => .u32,
7979	};
7980	if (int_info.bits <= 64) return switch (int_info.signedness) {
7981	.signed => .i64,
7982	.unsigned => .u64,
7983	};
7984	if (int_info.bits <= 128) return switch (int_info.signedness) {
7985	.signed => .i128,
7986	.unsigned => .u128,
7987	};
7988	return null;
7989	}
7990
7991	/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location
7992	///
7993	/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
7994	fn constraintAllowsMemory(constraint: []const u8) bool {
7995	// TODO: This implementation is woefully incomplete.
7996	for (constraint) |byte| {
7997	switch (byte) {
7998	'=', '*', ',', '&' => {},
7999	'm', 'o', 'X', 'g' => return true,
8000	else => {},
8001	}
8002	} else return false;
8003	}
8004
8005	/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register
8006	///
8007	/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
8008	fn constraintAllowsRegister(constraint: []const u8) bool {
8009	// TODO: This implementation is woefully incomplete.
8010	for (constraint) |byte| {
8011	switch (byte) {
8012	'=', '*', ',', '&' => {},
8013	'm', 'o' => {},
8014	else => return true,
8015	}
8016	} else return false;
8017	}
8018
8019	/// Appends zero or more LLVM constraints to `llvm_constraints`, returning how many were added.
8020	fn appendConstraints(
8021	gpa: Allocator,
8022	llvm_constraints: *std.ArrayList(u8),
8023	zig_name: []const u8,
8024	target: *const std.Target,
8025	) error{OutOfMemory}!usize {
8026	switch (target.cpu.arch) {
8027	.mips, .mipsel, .mips64, .mips64el => if (mips_clobber_overrides.get(zig_name)) |llvm_tag| {
8028	const llvm_name = @tagName(llvm_tag);
8029	try llvm_constraints.ensureUnusedCapacity(gpa, llvm_name.len + 4);
8030	llvm_constraints.appendSliceAssumeCapacity("~{");
8031	llvm_constraints.appendSliceAssumeCapacity(llvm_name);
8032	llvm_constraints.appendSliceAssumeCapacity("},");
8033	return 1;
8034	},
8035	else => {},
8036	}
8037
8038	try llvm_constraints.ensureUnusedCapacity(gpa, zig_name.len + 4);
8039	llvm_constraints.appendSliceAssumeCapacity("~{");
8040	llvm_constraints.appendSliceAssumeCapacity(zig_name);
8041	llvm_constraints.appendSliceAssumeCapacity("},");
8042	return 1;
8043	}
8044
8045	/// LLVM does not support all relevant intrinsics for all targets, so we
8046	/// may need to manually generate a compiler-rt call using a soft type.
8047	fn intrinsicsAllowed(kind: enum { compiler_rt, libc }, scalar_ty: Type, target: *const std.Target) bool {
8048	if (!scalar_ty.isRuntimeFloat()) return true;
8049	const bits = scalar_ty.floatBits(target);
8050	switch (kind) {
8051	.compiler_rt => {},
8052	// Since upstream musl/msvc do not actually define the *f128 functions, llvm decides
8053	// that it is a much better idea to just emit a call to the entirely wrong function as
8054	// a fallback. We wouldn't want any linker errors when trying to perform an operation
8055	// that isn't actually implemented anywhere, now would we!
8056	.libc => if (bits == 128 and target.cpu.arch.isX86() and !target.abi.isGnu()) return false,
8057	}
8058	return switch (std.zig.target.compilerRtFloatAbi(target, bits)) {
8059	.hard => true,
8060	.soft => false,
8061	};
8062	}
8063
8064	fn toLlvmAtomicOrdering(atomic_order: std.lang.AtomicOrder) Builder.AtomicOrdering {
8065	return switch (atomic_order) {
8066	.unordered => .unordered,
8067	.monotonic => .monotonic,
8068	.acquire => .acquire,
8069	.release => .release,
8070	.acq_rel => .acq_rel,
8071	.seq_cst => .seq_cst,
8072	};
8073	}
8074
8075	fn toLlvmAtomicRmwBinOp(
8076	op: std.lang.AtomicRmwOp,
8077	is_signed: bool,
8078	is_float: bool,
8079	) Builder.Function.Instruction.AtomicRmw.Operation {
8080	return switch (op) {
8081	.Xchg => .xchg,
8082	.Add => if (is_float) .fadd else return .add,
8083	.Sub => if (is_float) .fsub else return .sub,
8084	.And => .@"and",
8085	.Nand => .nand,
8086	.Or => .@"or",
8087	.Xor => .xor,
8088	.Max => if (is_float) .fmax else if (is_signed) .max else return .umax,
8089	.Min => if (is_float) .fmin else if (is_signed) .min else return .umin,
8090	};
8091	}
8092
8093	fn minIntConst(b: *Builder, min_ty: Type, as_ty: Builder.Type, zcu: *const Zcu) Allocator.Error!Builder.Constant {
8094	const info = min_ty.intInfo(zcu);
8095	if (info.signedness == .unsigned) {
8096	return b.intConst(as_ty, 0);
8097	}
8098	if (std.math.cast(u6, info.bits - 1)) |shift| {
8099	const min_val: i64 = @as(i64, std.math.minInt(i64)) >> (63 - shift);
8100	return b.intConst(as_ty, min_val);
8101	}
8102	var res: std.math.big.int.Managed = try .init(zcu.gpa);
8103	defer res.deinit();
8104	try res.setTwosCompIntLimit(.min, info.signedness, info.bits);
8105	return b.bigIntConst(as_ty, res.toConst());
8106	}
8107
8108	fn maxIntConst(b: *Builder, max_ty: Type, as_ty: Builder.Type, zcu: *const Zcu) Allocator.Error!Builder.Constant {
8109	const info = max_ty.intInfo(zcu);
8110	switch (info.bits) {
8111	0 => return b.intConst(as_ty, 0),
8112	1 => switch (info.signedness) {
8113	.signed => return b.intConst(as_ty, 0),
8114	.unsigned => return b.intConst(as_ty, 1),
8115	},
8116	else => {},
8117	}
8118	const unsigned_bits = switch (info.signedness) {
8119	.unsigned => info.bits,
8120	.signed => info.bits - 1,
8121	};
8122	if (std.math.cast(u6, unsigned_bits)) |shift| {
8123	const max_val: u64 = (@as(u64, 1) << shift) - 1;
8124	return b.intConst(as_ty, max_val);
8125	}
8126	var res: std.math.big.int.Managed = try .init(zcu.gpa);
8127	defer res.deinit();
8128	try res.setTwosCompIntLimit(.max, info.signedness, info.bits);
8129	return b.bigIntConst(as_ty, res.toConst());
8130	}
8131
8132	/// On some targets, local values that are in the generic address space must be generated into a
8133	/// different address, space and then cast back to the generic address space.
8134	/// For example, on GPUs local variable declarations must be generated into the local address space.
8135	/// This function returns the address space local values should be generated into.
8136	fn llvmAllocaAddressSpace(target: *const std.Target) Builder.AddrSpace {
8137	return switch (target.cpu.arch) {
8138	// On amdgcn, locals should be generated into the private address space.
8139	// To make Zig not impossible to use, these are then converted to addresses in the
8140	// generic address space and treates as regular pointers. This is the way that HIP also does it.
8141	.amdgcn => Builder.AddrSpace.amdgpu.private,
8142	else => .default,
8143	};
8144	}
8145
8146	/// Due to an LLVM bug, calls to `@llvm.memset.inline.*` with large constant length arguments cause
8147	/// LLVM to crash. As a mitigation, this function returns `true` if we should avoid emitting a
8148	/// memset call of the given length.
8149	///
8150	/// Most of our call sites are just setting memory to `undefined`, so can simply skip the memset
8151	/// call if we return `true`.
8152	///
8153	/// Upstream issue: https://github.com/llvm/llvm-project/issues/189161
8154	/// Zig issue: https://codeberg.org/ziglang/zig/issues/31701
8155	fn needMemsetWorkaround(fg: *const FuncGen, maybe_len: ?u64) bool {
8156	if (!fg.disable_intrinsics) {
8157	// The bug is limited to `@llvm.memset.inline.*`: normal memset calls are fine.
8158	return false;
8159	}
8160	const len = maybe_len orelse {
8161	// We don't think the length is constant, but a trivial optimization on LLVM's side could
8162	// turn it into one and potentially trigger the bug. Therefore, always apply the workaround
8163	// if the length is not a known constant.
8164	return true;
8165	};
8166	// Empirically, the crash first happens at 1048561 bytes, which is 1 MiB less 15 bytes. To be
8167	// safe (just in case the limit is target-specific or something like that), let's just set the
8168	// cap at half of that, i.e. 512 KiB.
8169	return len > 1024 * 512;
8170	}
8171
8172	const mips_clobber_overrides = std.StaticStringMap(enum {
8173	@"$msair",
8174	@"$msacsr",
8175	@"$msaaccess",
8176	@"$msasave",
8177	@"$msamodify",
8178	@"$msarequest",
8179	@"$msamap",
8180	@"$msaunmap",
8181	@"$f0",
8182	@"$f1",
8183	@"$f2",
8184	@"$f3",
8185	@"$f4",
8186	@"$f5",
8187	@"$f6",
8188	@"$f7",
8189	@"$f8",
8190	@"$f9",
8191	@"$f10",
8192	@"$f11",
8193	@"$f12",
8194	@"$f13",
8195	@"$f14",
8196	@"$f15",
8197	@"$f16",
8198	@"$f17",
8199	@"$f18",
8200	@"$f19",
8201	@"$f20",
8202	@"$f21",
8203	@"$f22",
8204	@"$f23",
8205	@"$f24",
8206	@"$f25",
8207	@"$f26",
8208	@"$f27",
8209	@"$f28",
8210	@"$f29",
8211	@"$f30",
8212	@"$f31",
8213	@"$fcc0",
8214	@"$fcc1",
8215	@"$fcc2",
8216	@"$fcc3",
8217	@"$fcc4",
8218	@"$fcc5",
8219	@"$fcc6",
8220	@"$fcc7",
8221	@"$w0",
8222	@"$w1",
8223	@"$w2",
8224	@"$w3",
8225	@"$w4",
8226	@"$w5",
8227	@"$w6",
8228	@"$w7",
8229	@"$w8",
8230	@"$w9",
8231	@"$w10",
8232	@"$w11",
8233	@"$w12",
8234	@"$w13",
8235	@"$w14",
8236	@"$w15",
8237	@"$w16",
8238	@"$w17",
8239	@"$w18",
8240	@"$w19",
8241	@"$w20",
8242	@"$w21",
8243	@"$w22",
8244	@"$w23",
8245	@"$w24",
8246	@"$w25",
8247	@"$w26",
8248	@"$w27",
8249	@"$w28",
8250	@"$w29",
8251	@"$w30",
8252	@"$w31",
8253	@"$0",
8254	@"$1",
8255	@"$2",
8256	@"$3",
8257	@"$4",
8258	@"$5",
8259	@"$6",
8260	@"$7",
8261	@"$8",
8262	@"$9",
8263	@"$10",
8264	@"$11",
8265	@"$12",
8266	@"$13",
8267	@"$14",
8268	@"$15",
8269	@"$16",
8270	@"$17",
8271	@"$18",
8272	@"$19",
8273	@"$20",
8274	@"$21",
8275	@"$22",
8276	@"$23",
8277	@"$24",
8278	@"$25",
8279	@"$26",
8280	@"$27",
8281	@"$28",
8282	@"$29",
8283	@"$30",
8284	@"$31",
8285	}).initComptime(.{
8286	.{ "msa_ir", .@"$msair" },
8287	.{ "msa_csr", .@"$msacsr" },
8288	.{ "msa_access", .@"$msaaccess" },
8289	.{ "msa_save", .@"$msasave" },
8290	.{ "msa_modify", .@"$msamodify" },
8291	.{ "msa_request", .@"$msarequest" },
8292	.{ "msa_map", .@"$msamap" },
8293	.{ "msa_unmap", .@"$msaunmap" },
8294	.{ "f0", .@"$f0" },
8295	.{ "f1", .@"$f1" },
8296	.{ "f2", .@"$f2" },
8297	.{ "f3", .@"$f3" },
8298	.{ "f4", .@"$f4" },
8299	.{ "f5", .@"$f5" },
8300	.{ "f6", .@"$f6" },
8301	.{ "f7", .@"$f7" },
8302	.{ "f8", .@"$f8" },
8303	.{ "f9", .@"$f9" },
8304	.{ "f10", .@"$f10" },
8305	.{ "f11", .@"$f11" },
8306	.{ "f12", .@"$f12" },
8307	.{ "f13", .@"$f13" },
8308	.{ "f14", .@"$f14" },
8309	.{ "f15", .@"$f15" },
8310	.{ "f16", .@"$f16" },
8311	.{ "f17", .@"$f17" },
8312	.{ "f18", .@"$f18" },
8313	.{ "f19", .@"$f19" },
8314	.{ "f20", .@"$f20" },
8315	.{ "f21", .@"$f21" },
8316	.{ "f22", .@"$f22" },
8317	.{ "f23", .@"$f23" },
8318	.{ "f24", .@"$f24" },
8319	.{ "f25", .@"$f25" },
8320	.{ "f26", .@"$f26" },
8321	.{ "f27", .@"$f27" },
8322	.{ "f28", .@"$f28" },
8323	.{ "f29", .@"$f29" },
8324	.{ "f30", .@"$f30" },
8325	.{ "f31", .@"$f31" },
8326	.{ "fcc0", .@"$fcc0" },
8327	.{ "fcc1", .@"$fcc1" },
8328	.{ "fcc2", .@"$fcc2" },
8329	.{ "fcc3", .@"$fcc3" },
8330	.{ "fcc4", .@"$fcc4" },
8331	.{ "fcc5", .@"$fcc5" },
8332	.{ "fcc6", .@"$fcc6" },
8333	.{ "fcc7", .@"$fcc7" },
8334	.{ "w0", .@"$w0" },
8335	.{ "w1", .@"$w1" },
8336	.{ "w2", .@"$w2" },
8337	.{ "w3", .@"$w3" },
8338	.{ "w4", .@"$w4" },
8339	.{ "w5", .@"$w5" },
8340	.{ "w6", .@"$w6" },
8341	.{ "w7", .@"$w7" },
8342	.{ "w8", .@"$w8" },
8343	.{ "w9", .@"$w9" },
8344	.{ "w10", .@"$w10" },
8345	.{ "w11", .@"$w11" },
8346	.{ "w12", .@"$w12" },
8347	.{ "w13", .@"$w13" },
8348	.{ "w14", .@"$w14" },
8349	.{ "w15", .@"$w15" },
8350	.{ "w16", .@"$w16" },
8351	.{ "w17", .@"$w17" },
8352	.{ "w18", .@"$w18" },
8353	.{ "w19", .@"$w19" },
8354	.{ "w20", .@"$w20" },
8355	.{ "w21", .@"$w21" },
8356	.{ "w22", .@"$w22" },
8357	.{ "w23", .@"$w23" },
8358	.{ "w24", .@"$w24" },
8359	.{ "w25", .@"$w25" },
8360	.{ "w26", .@"$w26" },
8361	.{ "w27", .@"$w27" },
8362	.{ "w28", .@"$w28" },
8363	.{ "w29", .@"$w29" },
8364	.{ "w30", .@"$w30" },
8365	.{ "w31", .@"$w31" },
8366	.{ "r0", .@"$0" },
8367	.{ "r1", .@"$1" },
8368	.{ "r2", .@"$2" },
8369	.{ "r3", .@"$3" },
8370	.{ "r4", .@"$4" },
8371	.{ "r5", .@"$5" },
8372	.{ "r6", .@"$6" },
8373	.{ "r7", .@"$7" },
8374	.{ "r8", .@"$8" },
8375	.{ "r9", .@"$9" },
8376	.{ "r10", .@"$10" },
8377	.{ "r11", .@"$11" },
8378	.{ "r12", .@"$12" },
8379	.{ "r13", .@"$13" },
8380	.{ "r14", .@"$14" },
8381	.{ "r15", .@"$15" },
8382	.{ "r16", .@"$16" },
8383	.{ "r17", .@"$17" },
8384	.{ "r18", .@"$18" },
8385	.{ "r19", .@"$19" },
8386	.{ "r20", .@"$20" },
8387	.{ "r21", .@"$21" },
8388	.{ "r22", .@"$22" },
8389	.{ "r23", .@"$23" },
8390	.{ "r24", .@"$24" },
8391	.{ "r25", .@"$25" },
8392	.{ "r26", .@"$26" },
8393	.{ "r27", .@"$27" },
8394	.{ "r28", .@"$28" },
8395	.{ "r29", .@"$29" },
8396	.{ "r30", .@"$30" },
8397	.{ "r31", .@"$31" },
8398	});
8399
8400	const std = @import("std");
8401	const Allocator = std.mem.Allocator;
8402	const Builder = std.zig.llvm.Builder;
8403	const assert = std.debug.assert;
8404	const math = std.math;
8405
8406	const aarch64_c_abi = @import("../aarch64/abi.zig");
8407	const arm_c_abi = @import("../arm/abi.zig");
8408	const loongarch_c_abi = @import("../loongarch/abi.zig");
8409	const mips_c_abi = @import("../mips/abi.zig");
8410	const riscv_c_abi = @import("../riscv64/abi.zig");
8411	const s390x_c_abi = @import("../s390x/abi.zig");
8412	const wasm_c_abi = @import("../wasm/abi.zig");
8413	const x86_64_abi = @import("../x86_64/abi.zig");
8414
8415	const Zcu = @import("../../Zcu.zig");
8416	const Air = @import("../../Air.zig");
8417	const Module = @import("../../Module.zig");
8418	const InternPool = @import("../../InternPool.zig");
8419	const Value = @import("../../Value.zig");
8420	const Type = @import("../../Type.zig");
8421	const codegen = @import("../../codegen.zig");
8422
8423	const target_util = @import("../../target.zig");
8424	const libcFloatPrefix = target_util.libcFloatPrefix;
8425	const libcFloatSuffix = target_util.libcFloatSuffix;
8426	const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
8427	const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev;
8428
8429	const llvm = @import("../llvm.zig");
8430	const Object = llvm.Object;
8431	const optional_layout_version = llvm.optional_layout_version;