Commit e3ee37f983ffe655b5e9516c12a1a0d69a967e12 | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2026-03-21 14:26:21+00:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2026-03-28 16:45:54+00:00
log	e3ee37f983ffe655b5e9516c12a1a0d69a967e12
tree	eb5973e573b39619e2932a974239f1ddc99d2e73
parent	3dc5f13989676ae0bfb9389e2b162e3945b38241
signature	Commit is signed but in an unrecognized format.

llvm: start separating out function codegen

This mostly just moves `FuncGen` into its own file, but there are also a
few more cleanups, such as removing `NavGen` (it no longer served any
purpose) and slightly simplifying the logic for emitting codegen errors.

2 files changed, 8567 insertions(+), 8578 deletions(-)

src
- codegen
  - llvm.zig
  - llvm
    FuncGen.zig

src/codegen/llvm.zig+422-8578

...	@@ -1,16 +1,24 @@	...	@@ -1,16 +1,24 @@
1	const builtin = @import("builtin");	1	const builtin = @import("builtin");
2		2
		3	const FuncGen = @import("llvm/FuncGen.zig");
		4	const buildAllocaInner = FuncGen.buildAllocaInner;
		5	const isByRef = FuncGen.isByRef;
		6	const firstParamSRet = FuncGen.firstParamSRet;
		7	const lowerFnRetTy = FuncGen.lowerFnRetTy;
		8	const iterateParamTypes = FuncGen.iterateParamTypes;
		9	const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt;
		10	const aarch64_c_abi = @import("aarch64/abi.zig");
		11
3	const std = @import("std");	12	const std = @import("std");
4	const Io = std.Io;	13	const Io = std.Io;
5	const assert = std.debug.assert;	14	const assert = std.debug.assert;
6	const Allocator = std.mem.Allocator;	15	const Allocator = std.mem.Allocator;
7	const log = std.log.scoped(.codegen);	16	const log = std.log.scoped(.codegen);
8	const math = std.math;
9	const DW = std.dwarf;	17	const DW = std.dwarf;
10	const Builder = std.zig.llvm.Builder;	18	const Builder = std.zig.llvm.Builder;
11		19
12	const build_options = @import("build_options");	20	const build_options = @import("build_options");
13	const llvm = if (build_options.have_llvm)	21	const bindings = if (build_options.have_llvm)
14	@import("llvm/bindings.zig")	22	@import("llvm/bindings.zig")
15	else	23	else
16	@compileError("LLVM unavailable");	24	@compileError("LLVM unavailable");
...	@@ -24,21 +32,9 @@ const Air = @import("../Air.zig");	...	@@ -24,21 +32,9 @@ const Air = @import("../Air.zig");
24	const Value = @import("../Value.zig");	32	const Value = @import("../Value.zig");
25	const Type = @import("../Type.zig");	33	const Type = @import("../Type.zig");
26	const codegen = @import("../codegen.zig");	34	const codegen = @import("../codegen.zig");
27	const x86_64_abi = @import("x86_64/abi.zig");
28	const wasm_c_abi = @import("wasm/abi.zig");
29	const aarch64_c_abi = @import("aarch64/abi.zig");
30	const arm_c_abi = @import("arm/abi.zig");
31	const riscv_c_abi = @import("riscv64/abi.zig");
32	const mips_c_abi = @import("mips/abi.zig");
33	const dev = @import("../dev.zig");	35	const dev = @import("../dev.zig");
34		36
35	const target_util = @import("../target.zig");	37	const target_util = @import("../target.zig");
36	const libcFloatPrefix = target_util.libcFloatPrefix;
37	const libcFloatSuffix = target_util.libcFloatSuffix;
38	const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev;
39	const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
40
41	const Error = error{ OutOfMemory, CodegenFail };
42		38
43	pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {	39	pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {
44	return comptime &.initMany(&.{	40	return comptime &.initMany(&.{
...	@@ -493,7 +489,7 @@ pub fn dataLayout(target: *const std.Target) []const u8 {	...	@@ -493,7 +489,7 @@ pub fn dataLayout(target: *const std.Target) []const u8 {
493	};	489	};
494	}	490	}
495		491
496	// Avoid depending on `llvm.CodeModel` in the bitcode-only case.	492	// Avoid depending on `bindings.CodeModel` in the bitcode-only case.
497	const CodeModel = enum {	493	const CodeModel = enum {
498	default,	494	default,
499	tiny,	495	tiny,
...	@@ -602,7 +598,7 @@ pub const Object = struct {	...	@@ -602,7 +598,7 @@ pub const Object = struct {
602		598
603	pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn;	599	pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn;
604		600
605	pub const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);	601	const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);
606		602
607	pub fn create(arena: Allocator, comp: *Compilation) !Ptr {	603	pub fn create(arena: Allocator, comp: *Compilation) !Ptr {
608	dev.check(.llvm_backend);	604	dev.check(.llvm_backend);
...	@@ -1026,20 +1022,20 @@ pub const Object = struct {	...	@@ -1026,20 +1022,20 @@ pub const Object = struct {
1026		1022
1027	initializeLLVMTarget(comp.root_mod.resolved_target.result.cpu.arch);	1023	initializeLLVMTarget(comp.root_mod.resolved_target.result.cpu.arch);
1028		1024
1029	const context: *llvm.Context = llvm.Context.create();	1025	const context: *bindings.Context = .create();
1030	errdefer context.dispose();	1026	errdefer context.dispose();
1031		1027
1032	const bitcode_memory_buffer = llvm.MemoryBuffer.createMemoryBufferWithMemoryRange(	1028	const bitcode_memory_buffer = bindings.MemoryBuffer.createMemoryBufferWithMemoryRange(
1033	@ptrCast(bitcode.ptr),	1029	@ptrCast(bitcode.ptr),
1034	bitcode.len * 4,	1030	bitcode.len * 4,
1035	"BitcodeBuffer",	1031	"BitcodeBuffer",
1036	llvm.Bool.False,	1032	bindings.Bool.False,
1037	);	1033	);
1038	defer bitcode_memory_buffer.dispose();	1034	defer bitcode_memory_buffer.dispose();
1039		1035
1040	context.enableBrokenDebugInfoCheck();	1036	context.enableBrokenDebugInfoCheck();
1041		1037
1042	var module: *llvm.Module = undefined;	1038	var module: *bindings.Module = undefined;
1043	if (context.parseBitcodeInContext2(bitcode_memory_buffer, &module).toBool() or context.getBrokenDebugInfo()) {	1039	if (context.parseBitcodeInContext2(bitcode_memory_buffer, &module).toBool() or context.getBrokenDebugInfo()) {
1044	return diags.fail("Failed to parse bitcode", .{});	1040	return diags.fail("Failed to parse bitcode", .{});
1045	}	1041	}
...	@@ -1047,28 +1043,28 @@ pub const Object = struct {	...	@@ -1047,28 +1043,28 @@ pub const Object = struct {
1047	};	1043	};
1048	defer context.dispose();	1044	defer context.dispose();
1049		1045
1050	var target: *llvm.Target = undefined;	1046	var target: *bindings.Target = undefined;
1051	var error_message: [*:0]const u8 = undefined;	1047	var error_message: [*:0]const u8 = undefined;
1052	if (llvm.Target.getFromTriple(target_triple_sentinel, &target, &error_message).toBool()) {	1048	if (bindings.Target.getFromTriple(target_triple_sentinel, &target, &error_message).toBool()) {
1053	defer llvm.disposeMessage(error_message);	1049	defer bindings.disposeMessage(error_message);
1054	return diags.fail("LLVM failed to parse '{s}': {s}", .{ target_triple_sentinel, error_message });	1050	return diags.fail("LLVM failed to parse '{s}': {s}", .{ target_triple_sentinel, error_message });
1055	}	1051	}
1056		1052
1057	const optimize_mode = comp.root_mod.optimize_mode;	1053	const optimize_mode = comp.root_mod.optimize_mode;
1058		1054
1059	const opt_level: llvm.CodeGenOptLevel = if (optimize_mode == .Debug)	1055	const opt_level: bindings.CodeGenOptLevel = if (optimize_mode == .Debug)
1060	.None	1056	.None
1061	else	1057	else
1062	.Aggressive;	1058	.Aggressive;
1063		1059
1064	const reloc_mode: llvm.RelocMode = if (comp.root_mod.pic)	1060	const reloc_mode: bindings.RelocMode = if (comp.root_mod.pic)
1065	.PIC	1061	.PIC
1066	else if (comp.config.link_mode == .dynamic)	1062	else if (comp.config.link_mode == .dynamic)
1067	llvm.RelocMode.DynamicNoPIC	1063	bindings.RelocMode.DynamicNoPIC
1068	else	1064	else
1069	.Static;	1065	.Static;
1070		1066
1071	const code_model: llvm.CodeModel = switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) {	1067	const code_model: bindings.CodeModel = switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) {
1072	.default => .Default,	1068	.default => .Default,
1073	.tiny => .Tiny,	1069	.tiny => .Tiny,
1074	.small => .Small,	1070	.small => .Small,
...	@@ -1077,12 +1073,12 @@ pub const Object = struct {	...	@@ -1077,12 +1073,12 @@ pub const Object = struct {
1077	.large => .Large,	1073	.large => .Large,
1078	};	1074	};
1079		1075
1080	const float_abi: llvm.TargetMachine.FloatABI = if (comp.root_mod.resolved_target.result.abi.float() == .hard)	1076	const float_abi: bindings.TargetMachine.FloatABI = if (comp.root_mod.resolved_target.result.abi.float() == .hard)
1081	.Hard	1077	.Hard
1082	else	1078	else
1083	.Soft;	1079	.Soft;
1084		1080
1085	var target_machine = llvm.TargetMachine.create(	1081	var target_machine = bindings.TargetMachine.create(
1086	target,	1082	target,
1087	target_triple_sentinel,	1083	target_triple_sentinel,
1088	if (comp.root_mod.resolved_target.result.cpu.model.llvm_name) \|s\| s.ptr else null,	1084	if (comp.root_mod.resolved_target.result.cpu.model.llvm_name) \|s\| s.ptr else null,
...	@@ -1105,7 +1101,7 @@ pub const Object = struct {	...	@@ -1105,7 +1101,7 @@ pub const Object = struct {
1105	// Unfortunately, LLVM shits the bed when we ask for both binary and assembly.	1101	// Unfortunately, LLVM shits the bed when we ask for both binary and assembly.
1106	// So we call the entire pipeline multiple times if this is requested.	1102	// So we call the entire pipeline multiple times if this is requested.
1107	// var error_message: [*:0]const u8 = undefined;	1103	// var error_message: [*:0]const u8 = undefined;
1108	var lowered_options: llvm.TargetMachine.EmitOptions = .{	1104	var lowered_options: bindings.TargetMachine.EmitOptions = .{
1109	.is_debug = options.is_debug,	1105	.is_debug = options.is_debug,
1110	.is_small = options.is_small,	1106	.is_small = options.is_small,
1111	.time_report_out = null, // set below to make sure it's only set for a single `emitToFile`	1107	.time_report_out = null, // set below to make sure it's only set for a single `emitToFile`
...	@@ -1154,7 +1150,7 @@ pub const Object = struct {	...	@@ -1154,7 +1150,7 @@ pub const Object = struct {
1154	};	1150	};
1155	if (options.asm_path != null and options.bin_path != null) {	1151	if (options.asm_path != null and options.bin_path != null) {
1156	if (target_machine.emitToFile(module, &error_message, &lowered_options)) {	1152	if (target_machine.emitToFile(module, &error_message, &lowered_options)) {
1157	defer llvm.disposeMessage(error_message);	1153	defer bindings.disposeMessage(error_message);
1158	return diags.fail("LLVM failed to emit bin={s} ir={s}: {s}", .{	1154	return diags.fail("LLVM failed to emit bin={s} ir={s}: {s}", .{
1159	emit_bin_msg, post_llvm_ir_msg, error_message,	1155	emit_bin_msg, post_llvm_ir_msg, error_message,
1160	});	1156	});
...	@@ -1170,7 +1166,7 @@ pub const Object = struct {	...	@@ -1170,7 +1166,7 @@ pub const Object = struct {
1170		1166
1171	lowered_options.asm_filename = if (options.asm_path) \|x\| x.ptr else null;	1167	lowered_options.asm_filename = if (options.asm_path) \|x\| x.ptr else null;
1172	if (target_machine.emitToFile(module, &error_message, &lowered_options)) {	1168	if (target_machine.emitToFile(module, &error_message, &lowered_options)) {
1173	defer llvm.disposeMessage(error_message);	1169	defer bindings.disposeMessage(error_message);
1174	return diags.fail("LLVM failed to emit asm={s} bin={s} ir={s} bc={s}: {s}", .{	1170	return diags.fail("LLVM failed to emit asm={s} bin={s} ir={s} bc={s}: {s}", .{
1175	emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg, error_message,	1171	emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg, error_message,
1176	});	1172	});
...	@@ -1201,13 +1197,6 @@ pub const Object = struct {	...	@@ -1201,13 +1197,6 @@ pub const Object = struct {
1201	const fn_info = zcu.typeToFunc(fn_ty).?;	1197	const fn_info = zcu.typeToFunc(fn_ty).?;
1202	const target = &owner_mod.resolved_target.result;	1198	const target = &owner_mod.resolved_target.result;
1203		1199
1204	var ng: NavGen = .{
1205	.object = o,
1206	.nav_index = func.owner_nav,
1207	.pt = pt,
1208	.err_msg = null,
1209	};
1210
1211	const function_index = try o.resolveLlvmFunction(pt, func.owner_nav);	1200	const function_index = try o.resolveLlvmFunction(pt, func.owner_nav);
1212		1201
1213	var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder);	1202	var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder);
...	@@ -1293,9 +1282,7 @@ pub const Object = struct {	...	@@ -1293,9 +1282,7 @@ pub const Object = struct {
1293		1282
1294	var llvm_arg_i: u32 = 0;	1283	var llvm_arg_i: u32 = 0;
1295		1284
1296	// This gets the LLVM values from the function and stores them in `ng.args`.	1285	const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: {
1297	const sret = firstParamSRet(fn_info, zcu, target);
1298	const ret_ptr: Builder.Value = if (sret) param: {
1299	const param = wip.arg(llvm_arg_i);	1286	const param = wip.arg(llvm_arg_i);
1300	llvm_arg_i += 1;	1287	llvm_arg_i += 1;
1301	break :param param;	1288	break :param param;
...	@@ -1398,7 +1385,7 @@ pub const Object = struct {	...	@@ -1398,7 +1385,7 @@ pub const Object = struct {
1398	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);	1385	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1399	const ptr_info = param_ty.ptrInfo(zcu);	1386	const ptr_info = param_ty.ptrInfo(zcu);
1400		1387
1401	if (math.cast(u5, it.zig_index - 1)) \|i\| {	1388	if (std.math.cast(u5, it.zig_index - 1)) \|i\| {
1402	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {	1389	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
1403	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);	1390	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
1404	}	1391	}
...	@@ -1543,10 +1530,12 @@ pub const Object = struct {	...	@@ -1543,10 +1530,12 @@ pub const Object = struct {
1543	};	1530	};
1544		1531
1545	var fg: FuncGen = .{	1532	var fg: FuncGen = .{
		1533	.object = o,
		1534	.nav_index = func.owner_nav,
		1535	.pt = pt,
1546	.gpa = gpa,	1536	.gpa = gpa,
1547	.air = air.*,	1537	.air = air.*,
1548	.liveness = liveness.*.?,	1538	.liveness = liveness.*.?,
1549	.ng = &ng,
1550	.wip = wip,	1539	.wip = wip,
1551	.is_naked = fn_info.cc == .naked,	1540	.is_naked = fn_info.cc == .naked,
1552	.fuzz = fuzz,	1541	.fuzz = fuzz,
...	@@ -1561,20 +1550,19 @@ pub const Object = struct {	...	@@ -1561,20 +1550,19 @@ pub const Object = struct {
1561	.sync_scope = if (owner_mod.single_threaded) .singlethread else .system,	1550	.sync_scope = if (owner_mod.single_threaded) .singlethread else .system,
1562	.file = file,	1551	.file = file,
1563	.scope = subprogram,	1552	.scope = subprogram,
		1553	.inlined_at = .none,
1564	.base_line = zcu.navSrcLine(func.owner_nav),	1554	.base_line = zcu.navSrcLine(func.owner_nav),
1565	.prev_dbg_line = 0,	1555	.prev_dbg_line = 0,
1566	.prev_dbg_column = 0,	1556	.prev_dbg_column = 0,
1567	.err_ret_trace = err_ret_trace,	1557	.err_ret_trace = err_ret_trace,
1568	.disable_intrinsics = disable_intrinsics,	1558	.disable_intrinsics = disable_intrinsics,
		1559	.allowzero_access = false,
1569	};	1560	};
1570	defer fg.deinit();	1561	defer fg.deinit();
1571	deinit_wip = false;	1562	deinit_wip = false;
1572		1563
1573	fg.genBody(air.getMainBody(), .poi) catch \|err\| switch (err) {	1564	fg.genBody(air.getMainBody(), .poi) catch \|err\| switch (err) {
1574	error.CodegenFail => switch (zcu.codegenFailMsg(func.owner_nav, ng.err_msg.?)) {	1565	error.CodegenFail => return, // MLUGG TODO
1575	error.CodegenFail => return,
1576	error.OutOfMemory => \|e\| return e,
1577	},
1578	else => \|e\| return e,	1566	else => \|e\| return e,
1579	};	1567	};
1580		1568
...	@@ -1618,15 +1606,142 @@ pub const Object = struct {	...	@@ -1618,15 +1606,142 @@ pub const Object = struct {
1618	try o.flushTypePool(pt);	1606	try o.flushTypePool(pt);
1619	}	1607	}
1620		1608
1621	pub fn updateNav(self: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void {	1609	pub fn updateNav(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void {
1622	var ng: NavGen = .{	1610	const zcu = pt.zcu;
1623	.object = self,	1611	const ip = &zcu.intern_pool;
1624	.nav_index = nav_index,	1612
1625	.pt = pt,	1613	const nav = ip.getNav(nav_index);
1626	.err_msg = null,	1614	const resolved = nav.resolved.?;
		1615
		1616	const lib_name, const linkage, const visibility: Builder.Visibility, const is_dll_import, const init_val, const owner_nav = switch (ip.indexToKey(resolved.value)) {
		1617	else => .{ .none, .internal, .default, false, resolved.value, nav_index },
		1618	.@"extern" => \|e\| .{ e.lib_name, e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import, .none, e.owner_nav },
1627	};	1619	};
1628	try ng.genDecl();	1620	const ty: Type = .fromInterned(nav.resolved.?.type);
1629	try self.flushTypePool(pt);	1621
		1622	if (linkage != .internal and ip.isFunctionType(ty.toIntern())) {
		1623	const function_index = try o.resolveLlvmFunction(pt, owner_nav);
		1624	// Add parameter attributes which weren't set by `resolveLlvmFunction`
		1625	const fn_info = zcu.typeToFunc(ty).?;
		1626	var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder);
		1627	defer attributes.deinit(&o.builder);
		1628	var it = iterateParamTypes(o, pt, fn_info);
		1629	if (firstParamSRet(fn_info, zcu, zcu.getTarget())) it.llvm_index += 1;
		1630	if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) it.llvm_index += 1;
		1631	while (try it.next()) \|lowering\| switch (lowering) {
		1632	.byval => {
		1633	const param_index = it.zig_index - 1;
		1634	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
		1635	if (!isByRef(param_ty, zcu)) {
		1636	try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
		1637	}
		1638	},
		1639	.byref => {
		1640	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
		1641	const param_llvm_ty = try o.lowerType(pt, param_ty);
		1642	const alignment = param_ty.abiAlignment(zcu);
		1643	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty);
		1644	},
		1645	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
		1646	// No attributes needed for these.
		1647	.no_bits,
		1648	.abi_sized_int,
		1649	.multiple_llvm_types,
		1650	.float_array,
		1651	.i32_array,
		1652	.i64_array,
		1653	=> continue,
		1654
		1655	.slice => unreachable, // extern functions do not support slice types.
		1656	};
		1657	function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
		1658	} else {
		1659	const variable_index = try o.resolveGlobalNav(pt, nav_index);
		1660	variable_index.setAlignment(zcu.navAlignment(nav_index).toLlvm(), &o.builder);
		1661	if (resolved.@"linksection".toSlice(ip)) \|section\|
		1662	variable_index.setSection(try o.builder.string(section), &o.builder);
		1663	if (resolved.@"const") variable_index.setMutability(.constant, &o.builder);
		1664	try variable_index.setInitializer(switch (init_val) {
		1665	.none => .no_init,
		1666	else => try o.lowerValue(pt, init_val),
		1667	}, &o.builder);
		1668	variable_index.setVisibility(visibility, &o.builder);
		1669
		1670	const file_scope = zcu.navFileScopeIndex(nav_index);
		1671	const mod = zcu.fileByIndex(file_scope).mod.?;
		1672	if (resolved.@"threadlocal" and !mod.single_threaded)
		1673	variable_index.setThreadLocal(.generaldynamic, &o.builder);
		1674
		1675	const line_number = zcu.navSrcLine(nav_index) + 1;
		1676
		1677	if (!mod.strip) {
		1678	const debug_file = try o.getDebugFile(pt, file_scope);
		1679
		1680	const debug_global_var = try o.builder.debugGlobalVar(
		1681	try o.builder.metadataString(nav.name.toSlice(ip)), // Name
		1682	try o.builder.metadataStringFromStrtabString(variable_index.name(&o.builder)), // Linkage name
		1683	debug_file, // File
		1684	debug_file, // Scope
		1685	line_number,
		1686	try o.getDebugType(pt, ty),
		1687	variable_index,
		1688	.{ .local = linkage == .internal },
		1689	);
		1690
		1691	const debug_expression = try o.builder.debugExpression(&.{});
		1692
		1693	const debug_global_var_expression = try o.builder.debugGlobalVarExpression(
		1694	debug_global_var,
		1695	debug_expression,
		1696	);
		1697
		1698	variable_index.setGlobalVariableExpression(debug_global_var_expression, &o.builder);
		1699	try o.debug_globals.append(o.gpa, debug_global_var_expression);
		1700	}
		1701	}
		1702
		1703	switch (linkage) {
		1704	.internal => {},
		1705	.strong, .weak => {
		1706	const global_index = o.nav_map.get(nav_index).?;
		1707
		1708	const decl_name = decl_name: {
		1709	if (zcu.getTarget().cpu.arch.isWasm() and ty.zigTypeTag(zcu) == .@"fn") {
		1710	if (lib_name.toSlice(ip)) \|lib_name_slice\| {
		1711	if (!std.mem.eql(u8, lib_name_slice, "c")) {
		1712	break :decl_name try o.builder.strtabStringFmt("{f}\|{s}", .{ nav.name.fmt(ip), lib_name_slice });
		1713	}
		1714	}
		1715	}
		1716	break :decl_name try o.builder.strtabString(nav.name.toSlice(ip));
		1717	};
		1718
		1719	if (o.builder.getGlobal(decl_name)) \|other_global\| {
		1720	if (other_global != global_index) {
		1721	// Another global already has this name; just use it in place of this global.
		1722	try global_index.replace(other_global, &o.builder);
		1723	return;
		1724	}
		1725	}
		1726
		1727	try global_index.rename(decl_name, &o.builder);
		1728	global_index.setUnnamedAddr(.default, &o.builder);
		1729	if (is_dll_import) {
		1730	global_index.setDllStorageClass(.dllimport, &o.builder);
		1731	} else if (zcu.comp.config.dll_export_fns) {
		1732	global_index.setDllStorageClass(.default, &o.builder);
		1733	}
		1734
		1735	global_index.setLinkage(switch (linkage) {
		1736	.internal => unreachable,
		1737	.strong => .external,
		1738	.weak => .extern_weak,
		1739	.link_once => unreachable,
		1740	}, &o.builder);
		1741	global_index.setVisibility(visibility, &o.builder);
		1742	},
		1743	.link_once => unreachable,
		1744	}
1630	}	1745	}
1631		1746
1632	fn flushTypePool(o: *Object, pt: Zcu.PerThread) Allocator.Error!void {	1747	fn flushTypePool(o: *Object, pt: Zcu.PerThread) Allocator.Error!void {
...	@@ -1904,7 +2019,7 @@ pub const Object = struct {	...	@@ -1904,7 +2019,7 @@ pub const Object = struct {
1904	}	2019	}
1905	}	2020	}
1906		2021
1907	fn getDebugFile(o: *Object, pt: Zcu.PerThread, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata {	2022	pub fn getDebugFile(o: *Object, pt: Zcu.PerThread, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata {
1908	const gpa = o.gpa;	2023	const gpa = o.gpa;
1909	const gop = try o.debug_file_map.getOrPut(gpa, file_index);	2024	const gop = try o.debug_file_map.getOrPut(gpa, file_index);
1910	errdefer assert(o.debug_file_map.remove(file_index));	2025	errdefer assert(o.debug_file_map.remove(file_index));
...	@@ -1920,7 +2035,7 @@ pub const Object = struct {	...	@@ -1920,7 +2035,7 @@ pub const Object = struct {
1920	return gop.value_ptr.*;	2035	return gop.value_ptr.*;
1921	}	2036	}
1922		2037
1923	fn getDebugType(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Metadata {	2038	pub fn getDebugType(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Metadata {
1924	assert(!o.builder.strip);	2039	assert(!o.builder.strip);
1925	const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern());	2040	const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern());
1926	return o.debug_types.items[@intFromEnum(index)];	2041	return o.debug_types.items[@intFromEnum(index)];
...	@@ -2212,13 +2327,14 @@ pub const Object = struct {	...	@@ -2212,13 +2327,14 @@ pub const Object = struct {
2212	defer debug_param_types.deinit(gpa);	2327	defer debug_param_types.deinit(gpa);
2213		2328
2214	// Return type goes first.	2329	// Return type goes first.
2215	const sret = firstParamSRet(fn_info, zcu, target);	2330	if (firstParamSRet(fn_info, zcu, target)) {
2216	const ret_ty: Type = if (sret) .void else .fromInterned(fn_info.return_type);	2331	// Actual return type is void, then first arg is the sret pointer.
2217	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ret_ty));	2332	const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type));
2218		2333	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void));
2219	if (sret) {
2220	const ptr_ty = try pt.singleMutPtrType(Type.fromInterned(fn_info.return_type));
2221	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ptr_ty));	2334	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ptr_ty));
		2335	} else {
		2336	const ret_ty: Type = .fromInterned(fn_info.return_type);
		2337	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ret_ty));
2222	}	2338	}
2223		2339
2224	if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) {	2340	if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) {
...	@@ -2661,7 +2777,7 @@ pub const Object = struct {	...	@@ -2661,7 +2777,7 @@ pub const Object = struct {
2661	/// If the llvm function does not exist, create it.	2777	/// If the llvm function does not exist, create it.
2662	/// Note that this can be called before the function's semantic analysis has	2778	/// Note that this can be called before the function's semantic analysis has
2663	/// completed, so if any attributes rely on that, they must be done in updateFunc, not here.	2779	/// completed, so if any attributes rely on that, they must be done in updateFunc, not here.
2664	fn resolveLlvmFunction(	2780	pub fn resolveLlvmFunction(
2665	o: *Object,	2781	o: *Object,
2666	pt: Zcu.PerThread,	2782	pt: Zcu.PerThread,
2667	nav_index: InternPool.Nav.Index,	2783	nav_index: InternPool.Nav.Index,
...	@@ -2677,7 +2793,6 @@ pub const Object = struct {	...	@@ -2677,7 +2793,6 @@ pub const Object = struct {
2677		2793
2678	const fn_info = zcu.typeToFunc(ty).?;	2794	const fn_info = zcu.typeToFunc(ty).?;
2679	const target = &owner_mod.resolved_target.result;	2795	const target = &owner_mod.resolved_target.result;
2680	const sret = firstParamSRet(fn_info, zcu, target);
2681		2796
2682	const is_extern, const lib_name = if (nav.getExtern(ip)) \|@"extern"\|	2797	const is_extern, const lib_name = if (nav.getExtern(ip)) \|@"extern"\|
2683	.{ true, @"extern".lib_name }	2798	.{ true, @"extern".lib_name }
...	@@ -2712,7 +2827,7 @@ pub const Object = struct {	...	@@ -2712,7 +2827,7 @@ pub const Object = struct {
2712	}	2827	}
2713		2828
2714	var llvm_arg_i: u32 = 0;	2829	var llvm_arg_i: u32 = 0;
2715	if (sret) {	2830	if (firstParamSRet(fn_info, zcu, target)) {
2716	// Sret pointers must not be address 0	2831	// Sret pointers must not be address 0
2717	try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);	2832	try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
2718	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);	2833	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
...	@@ -2988,11 +3103,11 @@ pub const Object = struct {	...	@@ -2988,11 +3103,11 @@ pub const Object = struct {
2988	return variable_index;	3103	return variable_index;
2989	}	3104	}
2990		3105
2991	fn errorIntType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Type {	3106	pub fn errorIntType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Type {
2992	return o.builder.intType(pt.zcu.errorSetBits());	3107	return o.builder.intType(pt.zcu.errorSetBits());
2993	}	3108	}
2994		3109
2995	fn lowerType(o: *Object, pt: Zcu.PerThread, t: Type) Allocator.Error!Builder.Type {	3110	pub fn lowerType(o: *Object, pt: Zcu.PerThread, t: Type) Allocator.Error!Builder.Type {
2996	const zcu = pt.zcu;	3111	const zcu = pt.zcu;
2997	const target = zcu.getTarget();	3112	const target = zcu.getTarget();
2998	const ip = &zcu.intern_pool;	3113	const ip = &zcu.intern_pool;
...	@@ -3408,7 +3523,7 @@ pub const Object = struct {	...	@@ -3408,7 +3523,7 @@ pub const Object = struct {
3408	return gop.value_ptr.*;	3523	return gop.value_ptr.*;
3409	},	3524	},
3410	.enum_type => try o.lowerType(pt, t.intTagType(zcu)),	3525	.enum_type => try o.lowerType(pt, t.intTagType(zcu)),
3411	.func_type => \|func_type\| try o.lowerTypeFn(pt, func_type),	3526	.func_type => \|func_type\| try o.lowerFnType(pt, func_type),
3412	.error_set_type, .inferred_error_set_type => try o.errorIntType(pt),	3527	.error_set_type, .inferred_error_set_type => try o.errorIntType(pt),
3413	// values, not types	3528	// values, not types
3414	.undef,	3529	.undef,
...	@@ -3434,7 +3549,7 @@ pub const Object = struct {	...	@@ -3434,7 +3549,7 @@ pub const Object = struct {
3434	};	3549	};
3435	}	3550	}
3436		3551
3437	fn lowerTypeFn(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {	3552	fn lowerFnType(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
3438	const zcu = pt.zcu;	3553	const zcu = pt.zcu;
3439	const ip = &zcu.intern_pool;	3554	const ip = &zcu.intern_pool;
3440	const target = zcu.getTarget();	3555	const target = zcu.getTarget();
...	@@ -3500,7 +3615,7 @@ pub const Object = struct {	...	@@ -3500,7 +3615,7 @@ pub const Object = struct {
3500	);	3615	);
3501	}	3616	}
3502		3617
3503	fn lowerValue(o: *Object, pt: Zcu.PerThread, arg_val: InternPool.Index) Allocator.Error!Builder.Constant {	3618	pub fn lowerValue(o: *Object, pt: Zcu.PerThread, arg_val: InternPool.Index) Allocator.Error!Builder.Constant {
3504	const zcu = pt.zcu;	3619	const zcu = pt.zcu;
3505	const ip = &zcu.intern_pool;	3620	const ip = &zcu.intern_pool;
3506	const target = zcu.getTarget();	3621	const target = zcu.getTarget();
...	@@ -4076,7 +4191,7 @@ pub const Object = struct {	...	@@ -4076,7 +4191,7 @@ pub const Object = struct {
4076		4191
4077	/// This logic is very similar to `lowerNavRefValue` but for anonymous declarations.	4192	/// This logic is very similar to `lowerNavRefValue` but for anonymous declarations.
4078	/// Maybe the logic could be unified.	4193	/// Maybe the logic could be unified.
4079	fn lowerUavRef(	4194	pub fn lowerUavRef(
4080	o: *Object,	4195	o: *Object,
4081	pt: Zcu.PerThread,	4196	pt: Zcu.PerThread,
4082	uav: InternPool.Key.Ptr.BaseAddr.Uav,	4197	uav: InternPool.Key.Ptr.BaseAddr.Uav,
...	@@ -4113,7 +4228,7 @@ pub const Object = struct {	...	@@ -4113,7 +4228,7 @@ pub const Object = struct {
4113	return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty));	4228	return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty));
4114	}	4229	}
4115		4230
4116	fn lowerNavRefValue(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant {	4231	pub fn lowerNavRefValue(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant {
4117	const zcu = pt.zcu;	4232	const zcu = pt.zcu;
4118	const ip = &zcu.intern_pool;	4233	const ip = &zcu.intern_pool;
4119		4234
...	@@ -4139,7 +4254,7 @@ pub const Object = struct {	...	@@ -4139,7 +4254,7 @@ pub const Object = struct {
4139	return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty));	4254	return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty));
4140	}	4255	}
4141		4256
4142	fn lowerPtrToVoid(o: *Object, pt: Zcu.PerThread, ptr_ty: Type) Allocator.Error!Builder.Constant {	4257	pub fn lowerPtrToVoid(o: *Object, pt: Zcu.PerThread, ptr_ty: Type) Allocator.Error!Builder.Constant {
4143	const zcu = pt.zcu;	4258	const zcu = pt.zcu;
4144	// Even though we are pointing at something which has zero bits (e.g. `void`),	4259	// Even though we are pointing at something which has zero bits (e.g. `void`),
4145	// Pointers are defined to have bits. So we must return something here.	4260	// Pointers are defined to have bits. So we must return something here.
...	@@ -4163,30 +4278,7 @@ pub const Object = struct {	...	@@ -4163,30 +4278,7 @@ pub const Object = struct {
4163	return o.builder.castConst(.inttoptr, try o.builder.intConst(llvm_usize, int), llvm_ptr_ty);	4278	return o.builder.castConst(.inttoptr, try o.builder.intConst(llvm_usize, int), llvm_ptr_ty);
4164	}	4279	}
4165		4280
4166	/// If the operand type of an atomic operation is not byte sized we need to	4281	pub fn addByValParamAttrs(
4167	/// widen it before using it and then truncate the result.
4168	/// RMW exchange of floating-point values is bitcasted to same-sized integer
4169	/// types to work around a LLVM deficiency when targeting ARM/AArch64.
4170	fn getAtomicAbiType(o: *Object, pt: Zcu.PerThread, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type {
4171	const zcu = pt.zcu;
4172	switch (ty.zigTypeTag(zcu)) {
4173	.int, .@"enum", .@"struct", .@"union" => {},
4174	.float => {
4175	if (!is_rmw_xchg) return .none;
4176	return o.builder.intType(@intCast(ty.abiSize(zcu) * 8));
4177	},
4178	.bool => return .i8,
4179	else => return .none,
4180	}
4181	const bit_count = ty.bitSize(zcu);
4182	if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) {
4183	return o.builder.intType(@intCast(ty.abiSize(zcu) * 8));
4184	} else {
4185	return .none;
4186	}
4187	}
4188
4189	fn addByValParamAttrs(
4190	o: *Object,	4282	o: *Object,
4191	pt: Zcu.PerThread,	4283	pt: Zcu.PerThread,
4192	attributes: *Builder.FunctionAttributes.Wip,	4284	attributes: *Builder.FunctionAttributes.Wip,
...	@@ -4198,7 +4290,7 @@ pub const Object = struct {	...	@@ -4198,7 +4290,7 @@ pub const Object = struct {
4198	const zcu = pt.zcu;	4290	const zcu = pt.zcu;
4199	if (param_ty.isPtrAtRuntime(zcu)) {	4291	if (param_ty.isPtrAtRuntime(zcu)) {
4200	const ptr_info = param_ty.ptrInfo(zcu);	4292	const ptr_info = param_ty.ptrInfo(zcu);
4201	if (math.cast(u5, param_index)) \|i\| {	4293	if (std.math.cast(u5, param_index)) \|i\| {
4202	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {	4294	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
4203	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);	4295	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
4204	}	4296	}
...	@@ -4232,7 +4324,7 @@ pub const Object = struct {	...	@@ -4232,7 +4324,7 @@ pub const Object = struct {
4232	};	4324	};
4233	}	4325	}
4234		4326
4235	fn addByRefParamAttrs(	4327	pub fn addByRefParamAttrs(
4236	o: *Object,	4328	o: *Object,
4237	attributes: *Builder.FunctionAttributes.Wip,	4329	attributes: *Builder.FunctionAttributes.Wip,
4238	llvm_arg_i: u32,	4330	llvm_arg_i: u32,
...	@@ -4246,14 +4338,15 @@ pub const Object = struct {	...	@@ -4246,14 +4338,15 @@ pub const Object = struct {
4246	if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);	4338	if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);
4247	}	4339	}
4248		4340
4249	fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint {	4341	pub fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint {
4250	return o.struct_field_map.get(.{	4342	return o.struct_field_map.get(.{
4251	.struct_ty = struct_ty.toIntern(),	4343	.struct_ty = struct_ty.toIntern(),
4252	.field_index = @intCast(field_index),	4344	.field_index = @intCast(field_index),
4253	});	4345	});
4254	}	4346	}
4255		4347
4256	fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index {	4348	/// MLUGG TODO: this is super dumb
		4349	pub fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index {
4257	const name = try o.builder.strtabString(lt_errors_fn_name);	4350	const name = try o.builder.strtabString(lt_errors_fn_name);
4258	if (o.builder.getGlobal(name)) \|llvm_fn\| return llvm_fn.ptrConst(&o.builder).kind.function;	4351	if (o.builder.getGlobal(name)) \|llvm_fn\| return llvm_fn.ptrConst(&o.builder).kind.function;
4259		4352
...	@@ -4275,7 +4368,7 @@ pub const Object = struct {	...	@@ -4275,7 +4368,7 @@ pub const Object = struct {
4275	return function_index;	4368	return function_index;
4276	}	4369	}
4277		4370
4278	fn getEnumTagNameFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index {	4371	pub fn getEnumTagNameFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index {
4279	const zcu = pt.zcu;	4372	const zcu = pt.zcu;
4280	const ip = &zcu.intern_pool;	4373	const ip = &zcu.intern_pool;
4281	const enum_type = ip.loadEnumType(enum_ty.toIntern());	4374	const enum_type = ip.loadEnumType(enum_ty.toIntern());
...	@@ -4353,12 +4446,12 @@ pub const Object = struct {	...	@@ -4353,12 +4446,12 @@ pub const Object = struct {
4353	return function_index;	4446	return function_index;
4354	}	4447	}
4355		4448
4356	fn lazyAbiAlignment(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Alignment.Lazy {	4449	pub fn lazyAbiAlignment(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Alignment.Lazy {
4357	const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern());	4450	const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern());
4358	return o.lazy_abi_aligns.items[@intFromEnum(index)];	4451	return o.lazy_abi_aligns.items[@intFromEnum(index)];
4359	}	4452	}
4360		4453
4361	fn updateIsNamedEnumValueFunction(	4454	pub fn updateIsNamedEnumValueFunction(
4362	o: *Object,	4455	o: *Object,
4363	pt: Zcu.PerThread,	4456	pt: Zcu.PerThread,
4364	enum_ty: Type,	4457	enum_ty: Type,
...	@@ -4411,7248 +4504,104 @@ pub const Object = struct {	...	@@ -4411,7248 +4504,104 @@ pub const Object = struct {
4411	}	4504	}
4412	};	4505	};
4413		4506
4414	pub const NavGen = struct {	4507	const CallingConventionInfo = struct {
4415	object: *Object,	4508	/// The LLVM calling convention to use.
4416	nav_index: InternPool.Nav.Index,	4509	llvm_cc: Builder.CallConv,
4417	pt: Zcu.PerThread,	4510	/// Whether to use an `alignstack` attribute to forcibly re-align the stack pointer in the function's prologue.
4418	err_msg: ?*Zcu.ErrorMsg,	4511	align_stack: bool,
		4512	/// Whether the function needs a `naked` attribute.
		4513	naked: bool,
		4514	/// How many leading parameters to apply the `inreg` attribute to.
		4515	inreg_param_count: u2 = 0,
		4516	};
4419		4517
4420	fn ownerModule(ng: NavGen) *Package.Module {	4518	pub fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: *const std.Target) ?CallingConventionInfo {
4421	return ng.pt.zcu.navFileScope(ng.nav_index).mod.?;	4519	const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null;
		4520	const incoming_stack_alignment: ?u64, const register_params: u2 = switch (cc) {
		4521	inline else => \|pl\| switch (@TypeOf(pl)) {
		4522	void => .{ null, 0 },
		4523	std.builtin.CallingConvention.ArcInterruptOptions,
		4524	std.builtin.CallingConvention.ArmInterruptOptions,
		4525	std.builtin.CallingConvention.RiscvInterruptOptions,
		4526	std.builtin.CallingConvention.ShInterruptOptions,
		4527	std.builtin.CallingConvention.MicroblazeInterruptOptions,
		4528	std.builtin.CallingConvention.MipsInterruptOptions,
		4529	std.builtin.CallingConvention.CommonOptions,
		4530	=> .{ pl.incoming_stack_alignment, 0 },
		4531	std.builtin.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params },
		4532	else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)),
		4533	},
		4534	};
		4535	return .{
		4536	.llvm_cc = llvm_cc,
		4537	.align_stack = if (incoming_stack_alignment) \|a\| need_align: {
		4538	const normal_stack_align = target.stackAlignment();
		4539	break :need_align a < normal_stack_align;
		4540	} else false,
		4541	.naked = cc == .naked,
		4542	.inreg_param_count = register_params,
		4543	};
		4544	}
		4545	pub fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv {
		4546	if (target.cCallingConvention()) \|default_c\| {
		4547	if (cc_tag == default_c) {
		4548	return .ccc;
		4549	}
4422	}	4550	}
		4551	return switch (cc_tag) {
		4552	.@"inline" => unreachable,
		4553	.auto, .async => .fastcc,
		4554	.naked => .ccc,
		4555	.x86_64_sysv => .x86_64_sysvcc,
		4556	.x86_64_win => .win64cc,
		4557	.x86_64_regcall_v3_sysv => if (target.cpu.arch == .x86_64 and target.os.tag != .windows)
		4558	.x86_regcallcc
		4559	else
		4560	null,
		4561	.x86_64_regcall_v4_win => if (target.cpu.arch == .x86_64 and target.os.tag == .windows)
		4562	.x86_regcallcc // we use the "RegCallv4" module flag to make this correct
		4563	else
		4564	null,
		4565	.x86_64_vectorcall => .x86_vectorcallcc,
		4566	.x86_64_interrupt => .x86_intrcc,
		4567	.x86_stdcall => .x86_stdcallcc,
		4568	.x86_fastcall => .x86_fastcallcc,
		4569	.x86_thiscall => .x86_thiscallcc,
		4570	.x86_regcall_v3 => if (target.cpu.arch == .x86 and target.os.tag != .windows)
		4571	.x86_regcallcc
		4572	else
		4573	null,
		4574	.x86_regcall_v4_win => if (target.cpu.arch == .x86 and target.os.tag == .windows)
		4575	.x86_regcallcc // we use the "RegCallv4" module flag to make this correct
		4576	else
		4577	null,
		4578	.x86_vectorcall => .x86_vectorcallcc,
		4579	.x86_interrupt => .x86_intrcc,
		4580	.aarch64_vfabi => .aarch64_vector_pcs,
		4581	.aarch64_vfabi_sve => .aarch64_sve_vector_pcs,
		4582	.arm_aapcs => .arm_aapcscc,
		4583	.arm_aapcs_vfp => .arm_aapcs_vfpcc,
		4584	.riscv64_lp64_v => .riscv_vectorcallcc,
		4585	.riscv32_ilp32_v => .riscv_vectorcallcc,
		4586	.avr_builtin => .avr_builtincc,
		4587	.avr_signal => .avr_signalcc,
		4588	.avr_interrupt => .avr_intrcc,
		4589	.m68k_rtd => .m68k_rtdcc,
		4590	.m68k_interrupt => .m68k_intrcc,
		4591	.msp430_interrupt => .msp430_intrcc,
		4592	.amdgcn_kernel => .amdgpu_kernel,
		4593	.amdgcn_cs => .amdgpu_cs,
		4594	.nvptx_device => .ptx_device,
		4595	.nvptx_kernel => .ptx_kernel,
4423		4596
4424	fn todo(ng: *NavGen, comptime format: []const u8, args: anytype) Error {	4597	// Calling conventions which LLVM uses function attributes for.
4425	@branchHint(.cold);	4598	.riscv64_interrupt,
4426	assert(ng.err_msg == null);	4599	.riscv32_interrupt,
4427	const o = ng.object;	4600	.arm_interrupt,
4428	const gpa = o.gpa;	4601	.mips64_interrupt,
4429	const src_loc = ng.pt.zcu.navSrcLoc(ng.nav_index);	4602	.mips_interrupt,
4430	ng.err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "TODO (LLVM): " ++ format, args);	4603	.csky_interrupt,
4431	return error.CodegenFail;	4604	=> .ccc,
4432	}
4433
4434	fn genDecl(ng: *NavGen) !void {
4435	const o = ng.object;
4436	const pt = ng.pt;
4437	const zcu = pt.zcu;
4438	const ip = &zcu.intern_pool;
4439	const nav_index = ng.nav_index;
4440	const nav = ip.getNav(nav_index);
4441	const resolved = nav.resolved.?;
4442
4443	const lib_name, const linkage, const visibility: Builder.Visibility, const is_dll_import, const init_val, const owner_nav = switch (ip.indexToKey(resolved.value)) {
4444	else => .{ .none, .internal, .default, false, resolved.value, nav_index },
4445	.@"extern" => \|e\| .{ e.lib_name, e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import, .none, e.owner_nav },
4446	};
4447	const ty: Type = .fromInterned(nav.resolved.?.type);
4448
4449	if (linkage != .internal and ip.isFunctionType(ty.toIntern())) {
4450	const function_index = try o.resolveLlvmFunction(pt, owner_nav);
4451	// Add parameter attributes which weren't set by `resolveLlvmFunction`
4452	const fn_info = zcu.typeToFunc(ty).?;
4453	var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder);
4454	defer attributes.deinit(&o.builder);
4455	var it = iterateParamTypes(o, pt, fn_info);
4456	if (firstParamSRet(fn_info, zcu, zcu.getTarget())) it.llvm_index += 1;
4457	if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) it.llvm_index += 1;
4458	while (try it.next()) \|lowering\| switch (lowering) {
4459	.byval => {
4460	const param_index = it.zig_index - 1;
4461	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
4462	if (!isByRef(param_ty, zcu)) {
4463	try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
4464	}
4465	},
4466	.byref => {
4467	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
4468	const param_llvm_ty = try o.lowerType(pt, param_ty);
4469	const alignment = param_ty.abiAlignment(zcu);
4470	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty);
4471	},
4472	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
4473	// No attributes needed for these.
4474	.no_bits,
4475	.abi_sized_int,
4476	.multiple_llvm_types,
4477	.float_array,
4478	.i32_array,
4479	.i64_array,
4480	=> continue,
4481
4482	.slice => unreachable, // extern functions do not support slice types.
4483	};
4484	function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
4485	} else {
4486	const variable_index = try o.resolveGlobalNav(pt, nav_index);
4487	variable_index.setAlignment(zcu.navAlignment(nav_index).toLlvm(), &o.builder);
4488	if (resolved.@"linksection".toSlice(ip)) \|section\|
4489	variable_index.setSection(try o.builder.string(section), &o.builder);
4490	if (resolved.@"const") variable_index.setMutability(.constant, &o.builder);
4491	try variable_index.setInitializer(switch (init_val) {
4492	.none => .no_init,
4493	else => try o.lowerValue(pt, init_val),
4494	}, &o.builder);
4495	variable_index.setVisibility(visibility, &o.builder);
4496
4497	const file_scope = zcu.navFileScopeIndex(nav_index);
4498	const mod = zcu.fileByIndex(file_scope).mod.?;
4499	if (resolved.@"threadlocal" and !mod.single_threaded)
4500	variable_index.setThreadLocal(.generaldynamic, &o.builder);
4501
4502	const line_number = zcu.navSrcLine(nav_index) + 1;
4503
4504	if (!mod.strip) {
4505	const debug_file = try o.getDebugFile(pt, file_scope);
4506
4507	const debug_global_var = try o.builder.debugGlobalVar(
4508	try o.builder.metadataString(nav.name.toSlice(ip)), // Name
4509	try o.builder.metadataStringFromStrtabString(variable_index.name(&o.builder)), // Linkage name
4510	debug_file, // File
4511	debug_file, // Scope
4512	line_number,
4513	try o.getDebugType(pt, ty),
4514	variable_index,
4515	.{ .local = linkage == .internal },
4516	);
4517
4518	const debug_expression = try o.builder.debugExpression(&.{});
4519
4520	const debug_global_var_expression = try o.builder.debugGlobalVarExpression(
4521	debug_global_var,
4522	debug_expression,
4523	);
4524
4525	variable_index.setGlobalVariableExpression(debug_global_var_expression, &o.builder);
4526	try o.debug_globals.append(o.gpa, debug_global_var_expression);
4527	}
4528	}
4529
4530	switch (linkage) {
4531	.internal => {},
4532	.strong, .weak => {
4533	const global_index = o.nav_map.get(nav_index).?;
4534
4535	const decl_name = decl_name: {
4536	if (zcu.getTarget().cpu.arch.isWasm() and ty.zigTypeTag(zcu) == .@"fn") {
4537	if (lib_name.toSlice(ip)) \|lib_name_slice\| {
4538	if (!std.mem.eql(u8, lib_name_slice, "c")) {
4539	break :decl_name try o.builder.strtabStringFmt("{f}\|{s}", .{ nav.name.fmt(ip), lib_name_slice });
4540	}
4541	}
4542	}
4543	break :decl_name try o.builder.strtabString(nav.name.toSlice(ip));
4544	};
4545
4546	if (o.builder.getGlobal(decl_name)) \|other_global\| {
4547	if (other_global != global_index) {
4548	// Another global already has this name; just use it in place of this global.
4549	try global_index.replace(other_global, &o.builder);
4550	return;
4551	}
4552	}
4553
4554	try global_index.rename(decl_name, &o.builder);
4555	global_index.setUnnamedAddr(.default, &o.builder);
4556	if (is_dll_import) {
4557	global_index.setDllStorageClass(.dllimport, &o.builder);
4558	} else if (zcu.comp.config.dll_export_fns) {
4559	global_index.setDllStorageClass(.default, &o.builder);
4560	}
4561
4562	global_index.setLinkage(switch (linkage) {
4563	.internal => unreachable,
4564	.strong => .external,
4565	.weak => .extern_weak,
4566	.link_once => unreachable,
4567	}, &o.builder);
4568	global_index.setVisibility(visibility, &o.builder);
4569	},
4570	.link_once => unreachable,
4571	}
4572	}
4573	};
4574
4575	pub const FuncGen = struct {
4576	gpa: Allocator,
4577	ng: *NavGen,
4578	air: Air,
4579	liveness: Air.Liveness,
4580	wip: Builder.WipFunction,
4581	is_naked: bool,
4582	fuzz: ?Fuzz,
4583
4584	file: Builder.Metadata,
4585	scope: Builder.Metadata,
4586
4587	inlined_at: Builder.Metadata.Optional = .none,
4588
4589	base_line: u32,
4590	prev_dbg_line: c_uint,
4591	prev_dbg_column: c_uint,
4592
4593	/// This stores the LLVM values used in a function, such that they can be referred to
4594	/// in other instructions. This table is cleared before every function is generated.
4595	func_inst_table: std.AutoHashMapUnmanaged(Air.Inst.Ref, Builder.Value),
4596
4597	/// If the return type is sret, this is the result pointer. Otherwise null.
4598	/// Note that this can disagree with isByRef for the return type in the case
4599	/// of C ABI functions.
4600	ret_ptr: Builder.Value,
4601	/// Any function that needs to perform Valgrind client requests needs an array alloca
4602	/// instruction, however a maximum of one per function is needed.
4603	valgrind_client_request_array: Builder.Value = .none,
4604	/// These fields are used to refer to the LLVM value of the function parameters
4605	/// in an Arg instruction.
4606	/// This list may be shorter than the list according to the zig type system;
4607	/// it omits 0-bit types. If the function uses sret as the first parameter,
4608	/// this slice does not include it.
4609	args: []const Builder.Value,
4610	arg_index: u32,
4611	arg_inline_index: u32,
4612
4613	err_ret_trace: Builder.Value = .none,
4614
4615	/// This data structure is used to implement breaking to blocks.
4616	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
4617	parent_bb: Builder.Function.Block.Index,
4618	breaks: *BreakList,
4619	}),
4620
4621	/// Maps `loop` instructions to the bb to branch to to repeat the loop.
4622	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, Builder.Function.Block.Index),
4623
4624	/// Maps `loop_switch_br` instructions to the information required to lower
4625	/// dispatches (`switch_dispatch` instructions).
4626	switch_dispatch_info: std.AutoHashMapUnmanaged(Air.Inst.Index, SwitchDispatchInfo),
4627
4628	sync_scope: Builder.SyncScope,
4629
4630	disable_intrinsics: bool,
4631
4632	/// Have we seen loads or stores involving `allowzero` pointers?
4633	allowzero_access: bool = false,
4634
4635	fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void {
4636	// LLVM already considers null pointers to be valid in non-generic address spaces, so avoid
4637	// pessimizing optimization for functions with accesses to such pointers.
4638	if (info.flags.address_space == .generic and info.flags.is_allowzero) self.allowzero_access = true;
4639	}
4640
4641	const Fuzz = struct {
4642	counters_variable: Builder.Variable.Index,
4643	pcs: std.ArrayList(Builder.Constant),
4644
4645	fn deinit(f: *Fuzz, gpa: Allocator) void {
4646	f.pcs.deinit(gpa);
4647	f.* = undefined;
4648	}
4649	};
4650
4651	const SwitchDispatchInfo = struct {
4652	/// These are the blocks corresponding to each switch case.
4653	/// The final element corresponds to the `else` case.
4654	/// Slices allocated into `gpa`.
4655	case_blocks: []Builder.Function.Block.Index,
4656	/// This is `.none` if `jmp_table` is set, since we won't use a `switch` instruction to dispatch.
4657	switch_weights: Builder.Function.Instruction.BrCond.Weights,
4658	/// If not `null`, we have manually constructed a jump table to reach the desired block.
4659	/// `table` can be used if the value is between `min` and `max` inclusive.
4660	/// We perform this lowering manually to avoid some questionable behavior from LLVM.
4661	/// See `airSwitchBr` for details.
4662	jmp_table: ?JmpTable,
4663
4664	const JmpTable = struct {
4665	min: Builder.Constant,
4666	max: Builder.Constant,
4667	in_bounds_hint: enum { none, unpredictable, likely, unlikely },
4668	/// Pointer to the jump table itself, to be used with `indirectbr`.
4669	/// The index into the jump table is the dispatch condition minus `min`.
4670	/// The table values are `blockaddress` constants corresponding to blocks in `case_blocks`.
4671	table: Builder.Constant,
4672	/// `true` if `table` conatins a reference to the `else` block.
4673	/// In this case, the `indirectbr` must include the `else` block in its target list.
4674	table_includes_else: bool,
4675	};
4676	};
4677
4678	const BreakList = union {
4679	list: std.MultiArrayList(struct {
4680	bb: Builder.Function.Block.Index,
4681	val: Builder.Value,
4682	}),
4683	len: usize,
4684	};
4685
4686	fn deinit(self: *FuncGen) void {
4687	const gpa = self.gpa;
4688	if (self.fuzz) \|*f\| f.deinit(self.gpa);
4689	self.wip.deinit();
4690	self.func_inst_table.deinit(gpa);
4691	self.blocks.deinit(gpa);
4692	self.loops.deinit(gpa);
4693	var it = self.switch_dispatch_info.valueIterator();
4694	while (it.next()) \|info\| {
4695	self.gpa.free(info.case_blocks);
4696	}
4697	self.switch_dispatch_info.deinit(gpa);
4698	}
4699
4700	fn todo(self: *FuncGen, comptime format: []const u8, args: anytype) Error {
4701	@branchHint(.cold);
4702	return self.ng.todo(format, args);
4703	}
4704
4705	fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value {
4706	const gpa = self.gpa;
4707	const gop = try self.func_inst_table.getOrPut(gpa, inst);
4708	if (gop.found_existing) return gop.value_ptr.*;
4709
4710	const llvm_val = try self.resolveValue((try self.air.value(inst, self.ng.pt)).?);
4711	gop.value_ptr.* = llvm_val.toValue();
4712	return llvm_val.toValue();
4713	}
4714
4715	fn resolveValue(self: *FuncGen, val: Value) Error!Builder.Constant {
4716	const o = self.ng.object;
4717	const pt = self.ng.pt;
4718	const zcu = pt.zcu;
4719	const ty = val.typeOf(zcu);
4720	const llvm_val = try o.lowerValue(pt, val.toIntern());
4721	if (!isByRef(ty, zcu)) return llvm_val;
4722
4723	// We have an LLVM value but we need to create a global constant and
4724	// set the value as its initializer, and then return a pointer to the global.
4725	const target = zcu.getTarget();
4726	const variable_index = try o.builder.addVariable(
4727	.empty,
4728	llvm_val.typeOf(&o.builder),
4729	toLlvmGlobalAddressSpace(.generic, target),
4730	);
4731	try variable_index.setInitializer(llvm_val, &o.builder);
4732	variable_index.setLinkage(.private, &o.builder);
4733	variable_index.setMutability(.constant, &o.builder);
4734	variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
4735	variable_index.setAlignment(ty.abiAlignment(zcu).toLlvm(), &o.builder);
4736	return o.builder.convConst(
4737	variable_index.toConst(&o.builder),
4738	try o.builder.ptrType(toLlvmAddressSpace(.generic, target)),
4739	);
4740	}
4741
4742	fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) Error!void {
4743	const o = self.ng.object;
4744	const zcu = self.ng.pt.zcu;
4745	const ip = &zcu.intern_pool;
4746	const air_tags = self.air.instructions.items(.tag);
4747	switch (coverage_point) {
4748	.none => {},
4749	.poi => if (self.fuzz) \|*fuzz\| {
4750	const poi_index = fuzz.pcs.items.len;
4751	const base_ptr = fuzz.counters_variable.toValue(&o.builder);
4752	const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{
4753	try o.builder.intValue(.i32, poi_index),
4754	}, "");
4755	const one = try o.builder.intValue(.i8, 1);
4756	_ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, "");
4757
4758	// LLVM does not allow blockaddress on the entry block.
4759	const pc = if (self.wip.cursor.block == .entry)
4760	self.wip.function.toConst(&o.builder)
4761	else
4762	try o.builder.blockAddrConst(self.wip.function, self.wip.cursor.block);
4763	const gpa = self.gpa;
4764	try fuzz.pcs.append(gpa, pc);
4765	},
4766	}
4767	for (body, 0..) \|inst, i\| {
4768	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
4769
4770	const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
4771	// zig fmt: off
4772
4773	// No "scalarize" legalizations are enabled, so these instructions never appear.
4774	.legalize_vec_elem_val => unreachable,
4775	.legalize_vec_store_elem => unreachable,
4776	// No soft float legalizations are enabled.
4777	.legalize_compiler_rt_call => unreachable,
4778
4779	.add => try self.airAdd(inst, .normal),
4780	.add_optimized => try self.airAdd(inst, .fast),
4781	.add_wrap => try self.airAddWrap(inst),
4782	.add_sat => try self.airAddSat(inst),
4783
4784	.sub => try self.airSub(inst, .normal),
4785	.sub_optimized => try self.airSub(inst, .fast),
4786	.sub_wrap => try self.airSubWrap(inst),
4787	.sub_sat => try self.airSubSat(inst),
4788
4789	.mul => try self.airMul(inst, .normal),
4790	.mul_optimized => try self.airMul(inst, .fast),
4791	.mul_wrap => try self.airMulWrap(inst),
4792	.mul_sat => try self.airMulSat(inst),
4793
4794	.add_safe => try self.airSafeArithmetic(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
4795	.sub_safe => try self.airSafeArithmetic(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
4796	.mul_safe => try self.airSafeArithmetic(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
4797
4798	.div_float => try self.airDivFloat(inst, .normal),
4799	.div_trunc => try self.airDivTrunc(inst, .normal),
4800	.div_floor => try self.airDivFloor(inst, .normal),
4801	.div_exact => try self.airDivExact(inst, .normal),
4802	.rem => try self.airRem(inst, .normal),
4803	.mod => try self.airMod(inst, .normal),
4804	.abs => try self.airAbs(inst),
4805	.ptr_add => try self.airPtrAdd(inst),
4806	.ptr_sub => try self.airPtrSub(inst),
4807	.shl => try self.airShl(inst),
4808	.shl_sat => try self.airShlSat(inst),
4809	.shl_exact => try self.airShlExact(inst),
4810	.min => try self.airMin(inst),
4811	.max => try self.airMax(inst),
4812	.slice => try self.airSlice(inst),
4813	.mul_add => try self.airMulAdd(inst),
4814
4815	.div_float_optimized => try self.airDivFloat(inst, .fast),
4816	.div_trunc_optimized => try self.airDivTrunc(inst, .fast),
4817	.div_floor_optimized => try self.airDivFloor(inst, .fast),
4818	.div_exact_optimized => try self.airDivExact(inst, .fast),
4819	.rem_optimized => try self.airRem(inst, .fast),
4820	.mod_optimized => try self.airMod(inst, .fast),
4821
4822	.add_with_overflow => try self.airOverflow(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
4823	.sub_with_overflow => try self.airOverflow(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
4824	.mul_with_overflow => try self.airOverflow(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
4825	.shl_with_overflow => try self.airShlWithOverflow(inst),
4826
4827	.bit_and, .bool_and => try self.airAnd(inst),
4828	.bit_or, .bool_or => try self.airOr(inst),
4829	.xor => try self.airXor(inst),
4830	.shr => try self.airShr(inst, false),
4831	.shr_exact => try self.airShr(inst, true),
4832
4833	.sqrt => try self.airUnaryOp(inst, .sqrt),
4834	.sin => try self.airUnaryOp(inst, .sin),
4835	.cos => try self.airUnaryOp(inst, .cos),
4836	.tan => try self.airUnaryOp(inst, .tan),
4837	.exp => try self.airUnaryOp(inst, .exp),
4838	.exp2 => try self.airUnaryOp(inst, .exp2),
4839	.log => try self.airUnaryOp(inst, .log),
4840	.log2 => try self.airUnaryOp(inst, .log2),
4841	.log10 => try self.airUnaryOp(inst, .log10),
4842	.floor => try self.airUnaryOp(inst, .floor),
4843	.ceil => try self.airUnaryOp(inst, .ceil),
4844	.round => try self.airUnaryOp(inst, .round),
4845	.trunc_float => try self.airUnaryOp(inst, .trunc),
4846
4847	.neg => try self.airNeg(inst, .normal),
4848	.neg_optimized => try self.airNeg(inst, .fast),
4849
4850	.cmp_eq => try self.airCmp(inst, .eq, .normal),
4851	.cmp_gt => try self.airCmp(inst, .gt, .normal),
4852	.cmp_gte => try self.airCmp(inst, .gte, .normal),
4853	.cmp_lt => try self.airCmp(inst, .lt, .normal),
4854	.cmp_lte => try self.airCmp(inst, .lte, .normal),
4855	.cmp_neq => try self.airCmp(inst, .neq, .normal),
4856
4857	.cmp_eq_optimized => try self.airCmp(inst, .eq, .fast),
4858	.cmp_gt_optimized => try self.airCmp(inst, .gt, .fast),
4859	.cmp_gte_optimized => try self.airCmp(inst, .gte, .fast),
4860	.cmp_lt_optimized => try self.airCmp(inst, .lt, .fast),
4861	.cmp_lte_optimized => try self.airCmp(inst, .lte, .fast),
4862	.cmp_neq_optimized => try self.airCmp(inst, .neq, .fast),
4863
4864	.cmp_vector => try self.airCmpVector(inst, .normal),
4865	.cmp_vector_optimized => try self.airCmpVector(inst, .fast),
4866	.cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),
4867
4868	.is_non_null => try self.airIsNonNull(inst, false, .ne),
4869	.is_non_null_ptr => try self.airIsNonNull(inst, true , .ne),
4870	.is_null => try self.airIsNonNull(inst, false, .eq),
4871	.is_null_ptr => try self.airIsNonNull(inst, true , .eq),
4872
4873	.is_non_err => try self.airIsErr(inst, .eq, false),
4874	.is_non_err_ptr => try self.airIsErr(inst, .eq, true),
4875	.is_err => try self.airIsErr(inst, .ne, false),
4876	.is_err_ptr => try self.airIsErr(inst, .ne, true),
4877
4878	.alloc => try self.airAlloc(inst),
4879	.ret_ptr => try self.airRetPtr(inst),
4880	.arg => try self.airArg(inst),
4881	.bitcast => try self.airBitCast(inst),
4882	.breakpoint => try self.airBreakpoint(inst),
4883	.ret_addr => try self.airRetAddr(inst),
4884	.frame_addr => try self.airFrameAddress(inst),
4885	.@"try" => try self.airTry(inst, false),
4886	.try_cold => try self.airTry(inst, true),
4887	.try_ptr => try self.airTryPtr(inst, false),
4888	.try_ptr_cold => try self.airTryPtr(inst, true),
4889	.intcast => try self.airIntCast(inst, false),
4890	.intcast_safe => try self.airIntCast(inst, true),
4891	.trunc => try self.airTrunc(inst),
4892	.fptrunc => try self.airFptrunc(inst),
4893	.fpext => try self.airFpext(inst),
4894	.load => try self.airLoad(inst),
4895	.not => try self.airNot(inst),
4896	.store => try self.airStore(inst, false),
4897	.store_safe => try self.airStore(inst, true),
4898	.assembly => try self.airAssembly(inst),
4899	.slice_ptr => try self.airSliceField(inst, 0),
4900	.slice_len => try self.airSliceField(inst, 1),
4901
4902	.ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
4903	.ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
4904
4905	.int_from_float => try self.airIntFromFloat(inst, .normal),
4906	.int_from_float_optimized => try self.airIntFromFloat(inst, .fast),
4907	.int_from_float_safe => unreachable, // handled by `legalizeFeatures`
4908	.int_from_float_optimized_safe => unreachable, // handled by `legalizeFeatures`
4909
4910	.array_to_slice => try self.airArrayToSlice(inst),
4911	.float_from_int => try self.airFloatFromInt(inst),
4912	.cmpxchg_weak => try self.airCmpxchg(inst, .weak),
4913	.cmpxchg_strong => try self.airCmpxchg(inst, .strong),
4914	.atomic_rmw => try self.airAtomicRmw(inst),
4915	.atomic_load => try self.airAtomicLoad(inst),
4916	.memset => try self.airMemset(inst, false),
4917	.memset_safe => try self.airMemset(inst, true),
4918	.memcpy => try self.airMemcpy(inst),
4919	.memmove => try self.airMemmove(inst),
4920	.set_union_tag => try self.airSetUnionTag(inst),
4921	.get_union_tag => try self.airGetUnionTag(inst),
4922	.clz => try self.airClzCtz(inst, .ctlz),
4923	.ctz => try self.airClzCtz(inst, .cttz),
4924	.popcount => try self.airBitOp(inst, .ctpop),
4925	.byte_swap => try self.airByteSwap(inst),
4926	.bit_reverse => try self.airBitOp(inst, .bitreverse),
4927	.tag_name => try self.airTagName(inst),
4928	.error_name => try self.airErrorName(inst),
4929	.splat => try self.airSplat(inst),
4930	.select => try self.airSelect(inst),
4931	.shuffle_one => try self.airShuffleOne(inst),
4932	.shuffle_two => try self.airShuffleTwo(inst),
4933	.aggregate_init => try self.airAggregateInit(inst),
4934	.union_init => try self.airUnionInit(inst),
4935	.prefetch => try self.airPrefetch(inst),
4936	.addrspace_cast => try self.airAddrSpaceCast(inst),
4937
4938	.is_named_enum_value => try self.airIsNamedEnumValue(inst),
4939	.error_set_has_value => try self.airErrorSetHasValue(inst),
4940
4941	.reduce => try self.airReduce(inst, .normal),
4942	.reduce_optimized => try self.airReduce(inst, .fast),
4943
4944	.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
4945	.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
4946	.atomic_store_release => try self.airAtomicStore(inst, .release),
4947	.atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
4948
4949	.struct_field_ptr => try self.airStructFieldPtr(inst),
4950	.struct_field_val => try self.airStructFieldVal(inst),
4951
4952	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
4953	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
4954	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
4955	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
4956
4957	.field_parent_ptr => try self.airFieldParentPtr(inst),
4958
4959	.array_elem_val => try self.airArrayElemVal(inst),
4960	.slice_elem_val => try self.airSliceElemVal(inst),
4961	.slice_elem_ptr => try self.airSliceElemPtr(inst),
4962	.ptr_elem_val => try self.airPtrElemVal(inst),
4963	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
4964
4965	.optional_payload => try self.airOptionalPayload(inst),
4966	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
4967	.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
4968
4969	.unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),
4970	.unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),
4971	.unwrap_errunion_err => try self.airErrUnionErr(inst, false),
4972	.unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
4973	.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
4974	.err_return_trace => try self.airErrReturnTrace(inst),
4975	.set_err_return_trace => try self.airSetErrReturnTrace(inst),
4976	.save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),
4977
4978	.wrap_optional => try self.airWrapOptional(body[i..]),
4979	.wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]),
4980	.wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]),
4981
4982	.wasm_memory_size => try self.airWasmMemorySize(inst),
4983	.wasm_memory_grow => try self.airWasmMemoryGrow(inst),
4984
4985	.runtime_nav_ptr => try self.airRuntimeNavPtr(inst),
4986
4987	.inferred_alloc, .inferred_alloc_comptime => unreachable,
4988
4989	.dbg_stmt => try self.airDbgStmt(inst),
4990	.dbg_empty_stmt => try self.airDbgEmptyStmt(inst),
4991	.dbg_var_ptr => try self.airDbgVarPtr(inst),
4992	.dbg_var_val => try self.airDbgVarVal(inst, false),
4993	.dbg_arg_inline => try self.airDbgVarVal(inst, true),
4994
4995	.c_va_arg => try self.airCVaArg(inst),
4996	.c_va_copy => try self.airCVaCopy(inst),
4997	.c_va_end => try self.airCVaEnd(inst),
4998	.c_va_start => try self.airCVaStart(inst),
4999
5000	.work_item_id => try self.airWorkItemId(inst),
5001	.work_group_size => try self.airWorkGroupSize(inst),
5002	.work_group_id => try self.airWorkGroupId(inst),
5003
5004	// Instructions that are known to always be `noreturn` based on their tag.
5005	.br => return self.airBr(inst),
5006	.repeat => return self.airRepeat(inst),
5007	.switch_dispatch => return self.airSwitchDispatch(inst),
5008	.cond_br => return self.airCondBr(inst),
5009	.switch_br => return self.airSwitchBr(inst, false),
5010	.loop_switch_br => return self.airSwitchBr(inst, true),
5011	.loop => return self.airLoop(inst),
5012	.ret => return self.airRet(inst, false),
5013	.ret_safe => return self.airRet(inst, true),
5014	.ret_load => return self.airRetLoad(inst),
5015	.trap => return self.airTrap(inst),
5016	.unreach => return self.airUnreach(inst),
5017
5018	// Instructions which may be `noreturn`.
5019	.block => res: {
5020	const res = try self.airBlock(inst);
5021	if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
5022	break :res res;
5023	},
5024	.dbg_inline_block => res: {
5025	const res = try self.airDbgInlineBlock(inst);
5026	if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
5027	break :res res;
5028	},
5029	.call, .call_always_tail, .call_never_tail, .call_never_inline => \|tag\| res: {
5030	const res = try self.airCall(inst, switch (tag) {
5031	.call => .auto,
5032	.call_always_tail => .always_tail,
5033	.call_never_tail => .never_tail,
5034	.call_never_inline => .never_inline,
5035	else => unreachable,
5036	});
5037	// TODO: the AIR we emit for calls is a bit weird - the instruction has
5038	// type `noreturn`, but there are instructions (and maybe a safety check) following
5039	// nonetheless. The `unreachable` or safety check should be emitted by backends instead.
5040	//if (self.typeOfIndex(inst).isNoReturn(mod)) return;
5041	break :res res;
5042	},
5043
5044	// zig fmt: on
5045	};
5046	if (val != .none) try self.func_inst_table.putNoClobber(self.gpa, inst.toRef(), val);
5047	}
5048	unreachable;
5049	}
5050
5051	fn genBodyDebugScope(
5052	self: *FuncGen,
5053	maybe_inline_func: ?InternPool.Index,
5054	body: []const Air.Inst.Index,
5055	coverage_point: Air.CoveragePoint,
5056	) Error!void {
5057	if (self.wip.strip) return self.genBody(body, coverage_point);
5058
5059	const old_debug_location = self.wip.debug_location;
5060	const old_file = self.file;
5061	const old_inlined_at = self.inlined_at;
5062	const old_base_line = self.base_line;
5063	defer if (maybe_inline_func) \|_\| {
5064	self.wip.debug_location = old_debug_location;
5065	self.file = old_file;
5066	self.inlined_at = old_inlined_at;
5067	self.base_line = old_base_line;
5068	};
5069
5070	const old_scope = self.scope;
5071	defer self.scope = old_scope;
5072
5073	if (maybe_inline_func) \|inline_func\| {
5074	const o = self.ng.object;
5075	const pt = self.ng.pt;
5076	const zcu = pt.zcu;
5077	const ip = &zcu.intern_pool;
5078
5079	const func = zcu.funcInfo(inline_func);
5080	const nav = ip.getNav(func.owner_nav);
5081	const file_scope = zcu.navFileScopeIndex(func.owner_nav);
5082	const mod = zcu.fileByIndex(file_scope).mod.?;
5083
5084	self.file = try o.getDebugFile(pt, file_scope);
5085
5086	self.base_line = zcu.navSrcLine(func.owner_nav);
5087	const line_number = self.base_line + 1;
5088	self.inlined_at = try self.wip.debug_location.toMetadata(&o.builder);
5089
5090	const fn_ty = try pt.funcType(.{
5091	.param_types = &.{},
5092	.return_type = .void_type,
5093	});
5094
5095	self.scope = try o.builder.debugSubprogram(
5096	self.file,
5097	try o.builder.metadataString(nav.name.toSlice(&zcu.intern_pool)),
5098	try o.builder.metadataString(nav.fqn.toSlice(&zcu.intern_pool)),
5099	line_number,
5100	line_number + func.lbrace_line,
5101	try o.getDebugType(pt, fn_ty),
5102	.{
5103	.di_flags = .{ .StaticMember = true },
5104	.sp_flags = .{
5105	.Optimized = mod.optimize_mode != .Debug,
5106	.Definition = true,
5107	.LocalToUnit = true, // inline functions cannot be exported
5108	},
5109	},
5110	o.debug_compile_unit.unwrap().?,
5111	);
5112	}
5113
5114	self.scope = try self.ng.object.builder.debugLexicalBlock(
5115	self.scope,
5116	self.file,
5117	self.prev_dbg_line,
5118	self.prev_dbg_column,
5119	);
5120	self.wip.debug_location = .{ .location = .{
5121	.line = self.prev_dbg_line,
5122	.column = self.prev_dbg_column,
5123	.scope = self.scope.toOptional(),
5124	.inlined_at = self.inlined_at,
5125	} };
5126
5127	try self.genBody(body, coverage_point);
5128	}
5129
5130	pub const CallAttr = enum {
5131	Auto,
5132	NeverTail,
5133	NeverInline,
5134	AlwaysTail,
5135	AlwaysInline,
5136	};
5137
5138	fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value {
5139	const air_call = self.air.unwrapCall(inst);
5140	const args = air_call.args;
5141	const o = self.ng.object;
5142	const pt = self.ng.pt;
5143	const zcu = pt.zcu;
5144	const ip = &zcu.intern_pool;
5145	const callee_ty = self.typeOf(air_call.callee);
5146	const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
5147	.@"fn" => callee_ty,
5148	.pointer => callee_ty.childType(zcu),
5149	else => unreachable,
5150	};
5151	const fn_info = zcu.typeToFunc(zig_fn_ty).?;
5152	const return_type = Type.fromInterned(fn_info.return_type);
5153	const llvm_fn = try self.resolveInst(air_call.callee);
5154	const target = zcu.getTarget();
5155	const sret = firstParamSRet(fn_info, zcu, target);
5156
5157	var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);
5158	defer llvm_args.deinit();
5159
5160	var attributes: Builder.FunctionAttributes.Wip = .{};
5161	defer attributes.deinit(&o.builder);
5162
5163	if (self.disable_intrinsics) {
5164	try attributes.addFnAttr(.nobuiltin, &o.builder);
5165	}
5166
5167	switch (modifier) {
5168	.auto, .always_tail => {},
5169	.never_tail, .never_inline => try attributes.addFnAttr(.@"noinline", &o.builder),
5170	.no_suspend, .always_inline, .compile_time => unreachable,
5171	}
5172
5173	const ret_ptr = if (!sret) null else blk: {
5174	const llvm_ret_ty = try o.lowerType(pt, return_type);
5175	try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);
5176
5177	const alignment = return_type.abiAlignment(zcu).toLlvm();
5178	const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);
5179	try llvm_args.append(ret_ptr);
5180	break :blk ret_ptr;
5181	};
5182
5183	const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
5184	if (err_return_tracing) {
5185	assert(self.err_ret_trace != .none);
5186	try llvm_args.append(self.err_ret_trace);
5187	}
5188
5189	var it = iterateParamTypes(o, pt, fn_info);
5190	while (try it.nextCall(self, args)) \|lowering\| switch (lowering) {
5191	.no_bits => continue,
5192	.byval => {
5193	const arg = args[it.zig_index - 1];
5194	const param_ty = self.typeOf(arg);
5195	const llvm_arg = try self.resolveInst(arg);
5196	const llvm_param_ty = try o.lowerType(pt, param_ty);
5197	if (isByRef(param_ty, zcu)) {
5198	const alignment = param_ty.abiAlignment(zcu).toLlvm();
5199	const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
5200	try llvm_args.append(loaded);
5201	} else {
5202	try llvm_args.append(llvm_arg);
5203	}
5204	},
5205	.byref => {
5206	const arg = args[it.zig_index - 1];
5207	const param_ty = self.typeOf(arg);
5208	const llvm_arg = try self.resolveInst(arg);
5209	if (isByRef(param_ty, zcu)) {
5210	try llvm_args.append(llvm_arg);
5211	} else {
5212	const alignment = param_ty.abiAlignment(zcu).toLlvm();
5213	const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);
5214	const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
5215	_ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
5216	try llvm_args.append(arg_ptr);
5217	}
5218	},
5219	.byref_mut => {
5220	const arg = args[it.zig_index - 1];
5221	const param_ty = self.typeOf(arg);
5222	const llvm_arg = try self.resolveInst(arg);
5223
5224	const alignment = param_ty.abiAlignment(zcu).toLlvm();
5225	const param_llvm_ty = try o.lowerType(pt, param_ty);
5226	const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
5227	if (isByRef(param_ty, zcu)) {
5228	const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
5229	_ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
5230	} else {
5231	_ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
5232	}
5233	try llvm_args.append(arg_ptr);
5234	},
5235	.abi_sized_int => {
5236	const arg = args[it.zig_index - 1];
5237	const param_ty = self.typeOf(arg);
5238	const llvm_arg = try self.resolveInst(arg);
5239	const int_llvm_ty = try o.builder.intType(@intCast(param_ty.abiSize(zcu) * 8));
5240
5241	if (isByRef(param_ty, zcu)) {
5242	const alignment = param_ty.abiAlignment(zcu).toLlvm();
5243	const loaded = try self.wip.load(.normal, int_llvm_ty, llvm_arg, alignment, "");
5244	try llvm_args.append(loaded);
5245	} else {
5246	// LLVM does not allow bitcasting structs so we must allocate
5247	// a local, store as one type, and then load as another type.
5248	const alignment = param_ty.abiAlignment(zcu).toLlvm();
5249	const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);
5250	_ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);
5251	const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");
5252	try llvm_args.append(loaded);
5253	}
5254	},
5255	.slice => {
5256	const arg = args[it.zig_index - 1];
5257	const llvm_arg = try self.resolveInst(arg);
5258	const ptr = try self.wip.extractValue(llvm_arg, &.{0}, "");
5259	const len = try self.wip.extractValue(llvm_arg, &.{1}, "");
5260	try llvm_args.appendSlice(&.{ ptr, len });
5261	},
5262	.multiple_llvm_types => {
5263	const arg = args[it.zig_index - 1];
5264	const param_ty = self.typeOf(arg);
5265	const llvm_types = it.types_buffer[0..it.types_len];
5266	const llvm_arg = try self.resolveInst(arg);
5267	const is_by_ref = isByRef(param_ty, zcu);
5268	const arg_ptr = if (is_by_ref) llvm_arg else ptr: {
5269	const alignment = param_ty.abiAlignment(zcu).toLlvm();
5270	const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
5271	_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
5272	break :ptr ptr;
5273	};
5274
5275	const llvm_ty = try o.builder.structType(.normal, llvm_types);
5276	try llvm_args.ensureUnusedCapacity(it.types_len);
5277	for (llvm_types, 0..) \|field_ty, i\| {
5278	const alignment =
5279	Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8));
5280	const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, "");
5281	const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, "");
5282	llvm_args.appendAssumeCapacity(loaded);
5283	}
5284	},
5285	.float_array => \|count\| {
5286	const arg = args[it.zig_index - 1];
5287	const arg_ty = self.typeOf(arg);
5288	var llvm_arg = try self.resolveInst(arg);
5289	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
5290	if (!isByRef(arg_ty, zcu)) {
5291	const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
5292	_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
5293	llvm_arg = ptr;
5294	}
5295
5296	const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);
5297	const array_ty = try o.builder.arrayType(count, float_ty);
5298
5299	const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
5300	try llvm_args.append(loaded);
5301	},
5302	.i32_array, .i64_array => \|arr_len\| {
5303	const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
5304	const arg = args[it.zig_index - 1];
5305	const arg_ty = self.typeOf(arg);
5306	var llvm_arg = try self.resolveInst(arg);
5307	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
5308	if (!isByRef(arg_ty, zcu)) {
5309	const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
5310	_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
5311	llvm_arg = ptr;
5312	}
5313
5314	const array_ty =
5315	try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
5316	const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
5317	try llvm_args.append(loaded);
5318	},
5319	};
5320
5321	{
5322	// Add argument attributes.
5323	it = iterateParamTypes(o, pt, fn_info);
5324	it.llvm_index += @intFromBool(sret);
5325	it.llvm_index += @intFromBool(err_return_tracing);
5326	while (try it.next()) \|lowering\| switch (lowering) {
5327	.byval => {
5328	const param_index = it.zig_index - 1;
5329	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
5330	if (!isByRef(param_ty, zcu)) {
5331	try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
5332	}
5333	},
5334	.byref => {
5335	const param_index = it.zig_index - 1;
5336	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
5337	const param_llvm_ty = try o.lowerType(pt, param_ty);
5338	const alignment = param_ty.abiAlignment(zcu).toLlvm();
5339	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
5340	},
5341	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
5342	// No attributes needed for these.
5343	.no_bits,
5344	.abi_sized_int,
5345	.multiple_llvm_types,
5346	.float_array,
5347	.i32_array,
5348	.i64_array,
5349	=> continue,
5350
5351	.slice => {
5352	assert(!it.byval_attr);
5353	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
5354	const ptr_info = param_ty.ptrInfo(zcu);
5355	const llvm_arg_i = it.llvm_index - 2;
5356
5357	if (math.cast(u5, it.zig_index - 1)) \|i\| {
5358	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
5359	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
5360	}
5361	}
5362	if (param_ty.zigTypeTag(zcu) != .optional and
5363	!ptr_info.flags.is_allowzero and
5364	ptr_info.flags.address_space == .generic)
5365	{
5366	try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
5367	}
5368	if (ptr_info.flags.is_const) {
5369	try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
5370	}
5371	const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) {
5372	else => \|a\| .wrap(a.toLlvm()),
5373	.none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)),
5374	};
5375	try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder);
5376	},
5377	};
5378	}
5379
5380	const call = try self.wip.call(
5381	switch (modifier) {
5382	.auto, .never_inline => .normal,
5383	.never_tail => .notail,
5384	.always_tail => .musttail,
5385	.no_suspend, .always_inline, .compile_time => unreachable,
5386	},
5387	toLlvmCallConvTag(fn_info.cc, target).?,
5388	try attributes.finish(&o.builder),
5389	try o.lowerType(pt, zig_fn_ty),
5390	llvm_fn,
5391	llvm_args.items,
5392	"",
5393	);
5394
5395	if (fn_info.return_type == .noreturn_type and modifier != .always_tail) {
5396	return .none;
5397	}
5398
5399	if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) {
5400	return .none;
5401	}
5402
5403	const llvm_ret_ty = try o.lowerType(pt, return_type);
5404	if (ret_ptr) \|rp\| {
5405	if (isByRef(return_type, zcu)) {
5406	return rp;
5407	} else {
5408	// our by-ref status disagrees with sret so we must load.
5409	const return_alignment = return_type.abiAlignment(zcu).toLlvm();
5410	return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, "");
5411	}
5412	}
5413
5414	const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
5415
5416	if (abi_ret_ty != llvm_ret_ty) {
5417	// In this case the function return type is honoring the calling convention by having
5418	// a different LLVM type than the usual one. We solve this here at the callsite
5419	// by using our canonical type, then loading it if necessary.
5420	const alignment = return_type.abiAlignment(zcu).toLlvm();
5421	const rp = try self.buildAlloca(abi_ret_ty, alignment);
5422	_ = try self.wip.store(.normal, call, rp, alignment);
5423	return if (isByRef(return_type, zcu))
5424	rp
5425	else
5426	try self.wip.load(.normal, llvm_ret_ty, rp, alignment, "");
5427	}
5428
5429	if (isByRef(return_type, zcu)) {
5430	// our by-ref status disagrees with sret so we must allocate, store,
5431	// and return the allocation pointer.
5432	const alignment = return_type.abiAlignment(zcu).toLlvm();
5433	const rp = try self.buildAlloca(llvm_ret_ty, alignment);
5434	_ = try self.wip.store(.normal, call, rp, alignment);
5435	return rp;
5436	} else {
5437	return call;
5438	}
5439	}
5440
5441	fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) !void {
5442	const o = fg.ng.object;
5443	const pt = fg.ng.pt;
5444	const zcu = pt.zcu;
5445	const target = zcu.getTarget();
5446	const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin()));
5447	const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;
5448	const panic_global = try o.resolveLlvmFunction(pt, panic_func.owner_nav);
5449
5450	const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto;
5451	if (has_err_trace) assert(fg.err_ret_trace != .none);
5452	_ = try fg.wip.callIntrinsicAssumeCold();
5453	_ = try fg.wip.call(
5454	.normal,
5455	toLlvmCallConvTag(fn_info.cc, target).?,
5456	.none,
5457	panic_global.typeOf(&o.builder),
5458	panic_global.toValue(&o.builder),
5459	if (has_err_trace) &.{fg.err_ret_trace} else &.{},
5460	"",
5461	);
5462	_ = try fg.wip.@"unreachable"();
5463	}
5464
5465	fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void {
5466	const o = self.ng.object;
5467	const pt = self.ng.pt;
5468	const zcu = pt.zcu;
5469	const ip = &zcu.intern_pool;
5470	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5471	const ret_ty = self.typeOf(un_op);
5472
5473	if (self.ret_ptr != .none) {
5474	const ptr_ty = try pt.singleMutPtrType(ret_ty);
5475
5476	const operand = try self.resolveInst(un_op);
5477	const val_is_undef = if (try self.air.value(un_op, pt)) \|val\| val.isUndef(zcu) else false;
5478	if (val_is_undef and safety) undef: {
5479	const ptr_info = ptr_ty.ptrInfo(zcu);
5480	const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
5481	if (needs_bitmask) {
5482	// TODO: only some bits are to be undef, we cannot write with a simple memset.
5483	// meanwhile, ignore the write rather than stomping over valid bits.
5484	// https://github.com/ziglang/zig/issues/15337
5485	break :undef;
5486	}
5487	const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu));
5488	_ = try self.wip.callMemSet(
5489	self.ret_ptr,
5490	ptr_ty.ptrAlignment(zcu).toLlvm(),
5491	try o.builder.intValue(.i8, 0xaa),
5492	len,
5493	.normal,
5494	self.disable_intrinsics,
5495	);
5496	const owner_mod = self.ng.ownerModule();
5497	if (owner_mod.valgrind) {
5498	try self.valgrindMarkUndef(self.ret_ptr, len);
5499	}
5500	_ = try self.wip.retVoid();
5501	return;
5502	}
5503
5504	const unwrapped_operand = operand.unwrap();
5505	const unwrapped_ret = self.ret_ptr.unwrap();
5506
5507	// Return value was stored previously
5508	if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) {
5509	_ = try self.wip.retVoid();
5510	return;
5511	}
5512
5513	try self.store(self.ret_ptr, ptr_ty, operand, .none);
5514	_ = try self.wip.retVoid();
5515	return;
5516	}
5517	const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.ng.nav_index).resolved.?.type)).?;
5518	if (!ret_ty.hasRuntimeBits(zcu)) {
5519	if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
5520	// Functions with an empty error set are emitted with an error code
5521	// return type and return zero so they can be function pointers coerced
5522	// to functions that return anyerror.
5523	_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0));
5524	} else {
5525	_ = try self.wip.retVoid();
5526	}
5527	return;
5528	}
5529
5530	const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
5531	const operand = try self.resolveInst(un_op);
5532	const val_is_undef = if (try self.air.value(un_op, pt)) \|val\| val.isUndef(zcu) else false;
5533	const alignment = ret_ty.abiAlignment(zcu).toLlvm();
5534
5535	if (val_is_undef and safety) {
5536	const llvm_ret_ty = operand.typeOfWip(&self.wip);
5537	const rp = try self.buildAlloca(llvm_ret_ty, alignment);
5538	const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu));
5539	_ = try self.wip.callMemSet(
5540	rp,
5541	alignment,
5542	try o.builder.intValue(.i8, 0xaa),
5543	len,
5544	.normal,
5545	self.disable_intrinsics,
5546	);
5547	const owner_mod = self.ng.ownerModule();
5548	if (owner_mod.valgrind) {
5549	try self.valgrindMarkUndef(rp, len);
5550	}
5551	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
5552	return;
5553	}
5554
5555	if (isByRef(ret_ty, zcu)) {
5556	// operand is a pointer however self.ret_ptr is null so that means
5557	// we need to return a value.
5558	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, ""));
5559	return;
5560	}
5561
5562	const llvm_ret_ty = operand.typeOfWip(&self.wip);
5563	if (abi_ret_ty == llvm_ret_ty) {
5564	_ = try self.wip.ret(operand);
5565	return;
5566	}
5567
5568	const rp = try self.buildAlloca(llvm_ret_ty, alignment);
5569	_ = try self.wip.store(.normal, operand, rp, alignment);
5570	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
5571	return;
5572	}
5573
5574	fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void {
5575	const o = self.ng.object;
5576	const pt = self.ng.pt;
5577	const zcu = pt.zcu;
5578	const ip = &zcu.intern_pool;
5579	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5580	const ptr_ty = self.typeOf(un_op);
5581	const ret_ty = ptr_ty.childType(zcu);
5582	const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.ng.nav_index).resolved.?.type)).?;
5583	if (!ret_ty.hasRuntimeBits(zcu)) {
5584	if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
5585	// Functions with an empty error set are emitted with an error code
5586	// return type and return zero so they can be function pointers coerced
5587	// to functions that return anyerror.
5588	_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0));
5589	} else {
5590	_ = try self.wip.retVoid();
5591	}
5592	return;
5593	}
5594	if (self.ret_ptr != .none) {
5595	_ = try self.wip.retVoid();
5596	return;
5597	}
5598	const ptr = try self.resolveInst(un_op);
5599	const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
5600	const alignment = ret_ty.abiAlignment(zcu).toLlvm();
5601	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, ""));
5602	return;
5603	}
5604
5605	fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5606	const o = self.ng.object;
5607	const pt = self.ng.pt;
5608	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5609	const list = try self.resolveInst(ty_op.operand);
5610	const arg_ty = ty_op.ty.toType();
5611	const llvm_arg_ty = try o.lowerType(pt, arg_ty);
5612
5613	return self.wip.vaArg(list, llvm_arg_ty, "");
5614	}
5615
5616	fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5617	const o = self.ng.object;
5618	const pt = self.ng.pt;
5619	const zcu = pt.zcu;
5620	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5621	const src_list = try self.resolveInst(ty_op.operand);
5622	const va_list_ty = ty_op.ty.toType();
5623	const llvm_va_list_ty = try o.lowerType(pt, va_list_ty);
5624
5625	const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
5626	const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
5627
5628	_ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
5629	return if (isByRef(va_list_ty, zcu))
5630	dest_list
5631	else
5632	try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
5633	}
5634
5635	fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5636	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5637	const src_list = try self.resolveInst(un_op);
5638
5639	_ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, "");
5640	return .none;
5641	}
5642
5643	fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5644	const o = self.ng.object;
5645	const pt = self.ng.pt;
5646	const zcu = pt.zcu;
5647	const va_list_ty = self.typeOfIndex(inst);
5648	const llvm_va_list_ty = try o.lowerType(pt, va_list_ty);
5649
5650	const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
5651	const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
5652
5653	_ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
5654	return if (isByRef(va_list_ty, zcu))
5655	dest_list
5656	else
5657	try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
5658	}
5659
5660	fn airCmp(
5661	self: *FuncGen,
5662	inst: Air.Inst.Index,
5663	op: math.CompareOperator,
5664	fast: Builder.FastMathKind,
5665	) !Builder.Value {
5666	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5667	const lhs = try self.resolveInst(bin_op.lhs);
5668	const rhs = try self.resolveInst(bin_op.rhs);
5669	const operand_ty = self.typeOf(bin_op.lhs);
5670
5671	return self.cmp(fast, op, operand_ty, lhs, rhs);
5672	}
5673
5674	fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
5675	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5676	const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
5677
5678	const lhs = try self.resolveInst(extra.lhs);
5679	const rhs = try self.resolveInst(extra.rhs);
5680	const vec_ty = self.typeOf(extra.lhs);
5681	const cmp_op = extra.compareOperator();
5682
5683	return self.cmp(fast, cmp_op, vec_ty, lhs, rhs);
5684	}
5685
5686	fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5687	const o = self.ng.object;
5688	const pt = self.ng.pt;
5689	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5690	const operand = try self.resolveInst(un_op);
5691	const llvm_fn = try o.getCmpLtErrorsLenFunction(pt);
5692	return self.wip.call(
5693	.normal,
5694	.fastcc,
5695	.none,
5696	llvm_fn.typeOf(&o.builder),
5697	llvm_fn.toValue(&o.builder),
5698	&.{operand},
5699	"",
5700	);
5701	}
5702
5703	fn cmp(
5704	self: *FuncGen,
5705	fast: Builder.FastMathKind,
5706	op: math.CompareOperator,
5707	operand_ty: Type,
5708	lhs: Builder.Value,
5709	rhs: Builder.Value,
5710	) Allocator.Error!Builder.Value {
5711	const o = self.ng.object;
5712	const pt = self.ng.pt;
5713	const zcu = pt.zcu;
5714	const scalar_ty = operand_ty.scalarType(zcu);
5715	const int_ty = switch (scalar_ty.zigTypeTag(zcu)) {
5716	.@"enum" => scalar_ty.intTagType(zcu),
5717	.int, .bool, .pointer, .error_set => scalar_ty,
5718	.optional => blk: {
5719	const payload_ty = operand_ty.optionalChild(zcu);
5720	if (!payload_ty.hasRuntimeBits(zcu) or
5721	operand_ty.optionalReprIsPayload(zcu))
5722	{
5723	break :blk operand_ty;
5724	}
5725	// We need to emit instructions to check for equality/inequality
5726	// of optionals that are not pointers.
5727	const is_by_ref = isByRef(scalar_ty, zcu);
5728	const opt_llvm_ty = try o.lowerType(pt, scalar_ty);
5729	const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal);
5730	const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal);
5731	const llvm_i2 = try o.builder.intType(2);
5732	const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, "");
5733	const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, "");
5734	const lhs_shifted = try self.wip.bin(.shl, lhs_non_null_i2, try o.builder.intValue(llvm_i2, 1), "");
5735	const lhs_rhs_ored = try self.wip.bin(.@"or", lhs_shifted, rhs_non_null_i2, "");
5736	const both_null_block = try self.wip.block(1, "BothNull");
5737	const mixed_block = try self.wip.block(1, "Mixed");
5738	const both_pl_block = try self.wip.block(1, "BothNonNull");
5739	const end_block = try self.wip.block(3, "End");
5740	var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none);
5741	defer wip_switch.finish(&self.wip);
5742	try wip_switch.addCase(
5743	try o.builder.intConst(llvm_i2, 0b00),
5744	both_null_block,
5745	&self.wip,
5746	);
5747	try wip_switch.addCase(
5748	try o.builder.intConst(llvm_i2, 0b11),
5749	both_pl_block,
5750	&self.wip,
5751	);
5752
5753	self.wip.cursor = .{ .block = both_null_block };
5754	_ = try self.wip.br(end_block);
5755
5756	self.wip.cursor = .{ .block = mixed_block };
5757	_ = try self.wip.br(end_block);
5758
5759	self.wip.cursor = .{ .block = both_pl_block };
5760	const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true);
5761	const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true);
5762	const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload);
5763	_ = try self.wip.br(end_block);
5764	const both_pl_block_end = self.wip.cursor.block;
5765
5766	self.wip.cursor = .{ .block = end_block };
5767	const llvm_i1_0 = Builder.Value.false;
5768	const llvm_i1_1 = Builder.Value.true;
5769	const incoming_values: [3]Builder.Value = .{
5770	switch (op) {
5771	.eq => llvm_i1_1,
5772	.neq => llvm_i1_0,
5773	else => unreachable,
5774	},
5775	switch (op) {
5776	.eq => llvm_i1_0,
5777	.neq => llvm_i1_1,
5778	else => unreachable,
5779	},
5780	payload_cmp,
5781	};
5782
5783	const phi = try self.wip.phi(.i1, "");
5784	phi.finish(
5785	&incoming_values,
5786	&.{ both_null_block, mixed_block, both_pl_block_end },
5787	&self.wip,
5788	);
5789	return phi.toValue();
5790	},
5791	.float => return self.buildFloatCmp(fast, op, operand_ty, .{ lhs, rhs }),
5792	.@"struct", .@"union" => scalar_ty.bitpackBackingInt(zcu),
5793	else => unreachable,
5794	};
5795	const is_signed = int_ty.isSignedInt(zcu);
5796	const cond: Builder.IntegerCondition = switch (op) {
5797	.eq => .eq,
5798	.neq => .ne,
5799	.lt => if (is_signed) .slt else .ult,
5800	.lte => if (is_signed) .sle else .ule,
5801	.gt => if (is_signed) .sgt else .ugt,
5802	.gte => if (is_signed) .sge else .uge,
5803	};
5804	return self.wip.icmp(cond, lhs, rhs, "");
5805	}
5806
5807	fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5808	const block = self.air.unwrapBlock(inst);
5809	return self.lowerBlock(inst, null, block.body);
5810	}
5811
5812	fn lowerBlock(
5813	self: *FuncGen,
5814	inst: Air.Inst.Index,
5815	maybe_inline_func: ?InternPool.Index,
5816	body: []const Air.Inst.Index,
5817	) !Builder.Value {
5818	const o = self.ng.object;
5819	const pt = self.ng.pt;
5820	const zcu = pt.zcu;
5821	const inst_ty = self.typeOfIndex(inst);
5822
5823	if (inst_ty.isNoReturn(zcu)) {
5824	try self.genBodyDebugScope(maybe_inline_func, body, .none);
5825	return .none;
5826	}
5827
5828	const have_block_result = inst_ty.hasRuntimeBits(zcu);
5829
5830	var breaks: BreakList = if (have_block_result) .{ .list = .{} } else .{ .len = 0 };
5831	defer if (have_block_result) breaks.list.deinit(self.gpa);
5832
5833	const parent_bb = try self.wip.block(0, "Block");
5834	try self.blocks.putNoClobber(self.gpa, inst, .{
5835	.parent_bb = parent_bb,
5836	.breaks = &breaks,
5837	});
5838	defer assert(self.blocks.remove(inst));
5839
5840	try self.genBodyDebugScope(maybe_inline_func, body, .none);
5841
5842	self.wip.cursor = .{ .block = parent_bb };
5843
5844	// Create a phi node only if the block returns a value.
5845	if (have_block_result) {
5846	const raw_llvm_ty = try o.lowerType(pt, inst_ty);
5847	const llvm_ty: Builder.Type = ty: {
5848	// If the zig tag type is a function, this represents an actual function body; not
5849	// a pointer to it. LLVM IR allows the call instruction to use function bodies instead
5850	// of function pointers, however the phi makes it a runtime value and therefore
5851	// the LLVM type has to be wrapped in a pointer.
5852	if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) {
5853	break :ty .ptr;
5854	}
5855	break :ty raw_llvm_ty;
5856	};
5857
5858	parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);
5859	const phi = try self.wip.phi(llvm_ty, "");
5860	phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);
5861	return phi.toValue();
5862	} else {
5863	parent_bb.ptr(&self.wip).incoming = @intCast(breaks.len);
5864	return .none;
5865	}
5866	}
5867
5868	fn airBr(self: *FuncGen, inst: Air.Inst.Index) !void {
5869	const zcu = self.ng.pt.zcu;
5870	const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
5871	const block = self.blocks.get(branch.block_inst).?;
5872
5873	// Add the values to the lists only if the break provides a value.
5874	const operand_ty = self.typeOf(branch.operand);
5875	if (operand_ty.hasRuntimeBits(zcu)) {
5876	const val = try self.resolveInst(branch.operand);
5877
5878	// For the phi node, we need the basic blocks and the values of the
5879	// break instructions.
5880	try block.breaks.list.append(self.gpa, .{ .bb = self.wip.cursor.block, .val = val });
5881	} else block.breaks.len += 1;
5882	_ = try self.wip.br(block.parent_bb);
5883	}
5884
5885	fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) !void {
5886	const repeat = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat;
5887	const loop_bb = self.loops.get(repeat.loop_inst).?;
5888	loop_bb.ptr(&self.wip).incoming += 1;
5889	_ = try self.wip.br(loop_bb);
5890	}
5891
5892	fn lowerSwitchDispatch(
5893	self: *FuncGen,
5894	switch_inst: Air.Inst.Index,
5895	cond_ref: Air.Inst.Ref,
5896	dispatch_info: SwitchDispatchInfo,
5897	) !void {
5898	const o = self.ng.object;
5899	const pt = self.ng.pt;
5900	const zcu = pt.zcu;
5901	const cond_ty = self.typeOf(cond_ref);
5902	const switch_br = self.air.unwrapSwitch(switch_inst);
5903
5904	if (try self.air.value(cond_ref, pt)) \|cond_val\| {
5905	// Comptime-known dispatch. Iterate the cases to find the correct
5906	// one, and branch to the corresponding element of `case_blocks`.
5907	var it = switch_br.iterateCases();
5908	const target_case_idx = target: while (it.next()) \|case\| {
5909	for (case.items) \|item\| {
5910	const val = Value.fromInterned(item.toInterned().?);
5911	if (cond_val.compareHetero(.eq, val, zcu)) break :target case.idx;
5912	}
5913	for (case.ranges) \|range\| {
5914	const low = Value.fromInterned(range[0].toInterned().?);
5915	const high = Value.fromInterned(range[1].toInterned().?);
5916	if (cond_val.compareHetero(.gte, low, zcu) and
5917	cond_val.compareHetero(.lte, high, zcu))
5918	{
5919	break :target case.idx;
5920	}
5921	}
5922	} else dispatch_info.case_blocks.len - 1;
5923	const target_block = dispatch_info.case_blocks[target_case_idx];
5924	target_block.ptr(&self.wip).incoming += 1;
5925	_ = try self.wip.br(target_block);
5926	return;
5927	}
5928
5929	// Runtime-known dispatch.
5930	const cond = try self.resolveInst(cond_ref);
5931
5932	if (dispatch_info.jmp_table) \|jmp_table\| {
5933	// We should use the constructed jump table.
5934	// First, check the bounds to branch to the `else` case if needed.
5935	const inbounds = try self.wip.bin(
5936	.@"and",
5937	try self.cmp(.normal, .gte, cond_ty, cond, jmp_table.min.toValue()),
5938	try self.cmp(.normal, .lte, cond_ty, cond, jmp_table.max.toValue()),
5939	"",
5940	);
5941	const jmp_table_block = try self.wip.block(1, "Then");
5942	const else_block = dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1];
5943	else_block.ptr(&self.wip).incoming += 1;
5944	_ = try self.wip.brCond(inbounds, jmp_table_block, else_block, switch (jmp_table.in_bounds_hint) {
5945	.none => .none,
5946	.unpredictable => .unpredictable,
5947	.likely => .then_likely,
5948	.unlikely => .else_likely,
5949	});
5950
5951	self.wip.cursor = .{ .block = jmp_table_block };
5952
5953	// Figure out the list of blocks we might branch to.
5954	// This includes all case blocks, but it might not include the `else` block if
5955	// the table is dense.
5956	const target_blocks_len = dispatch_info.case_blocks.len - @intFromBool(!jmp_table.table_includes_else);
5957	const target_blocks = dispatch_info.case_blocks[0..target_blocks_len];
5958
5959	// Make sure to cast the index to a usize so it's not treated as negative!
5960	const table_index = try self.wip.conv(
5961	.unsigned,
5962	try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),
5963	try o.lowerType(pt, .usize),
5964	"",
5965	);
5966	const target_ptr_ptr = try self.wip.gep(
5967	.inbounds,
5968	.ptr,
5969	jmp_table.table.toValue(),
5970	&.{table_index},
5971	"",
5972	);
5973	const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, "");
5974
5975	// Do the branch!
5976	_ = try self.wip.indirectbr(target_ptr, target_blocks);
5977
5978	// Mark all target blocks as having one more incoming branch.
5979	for (target_blocks) \|case_block\| {
5980	case_block.ptr(&self.wip).incoming += 1;
5981	}
5982
5983	return;
5984	}
5985
5986	// We must lower to an actual LLVM `switch` instruction.
5987	// The switch prongs will correspond to our scalar cases. Ranges will
5988	// be handled by conditional branches in the `else` prong.
5989
5990	const llvm_usize = try o.lowerType(pt, Type.usize);
5991	const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder))
5992	try self.wip.cast(.ptrtoint, cond, llvm_usize, "")
5993	else
5994	cond;
5995
5996	const llvm_cases_len, const last_range_case = info: {
5997	var llvm_cases_len: u32 = 0;
5998	var last_range_case: ?u32 = null;
5999	var it = switch_br.iterateCases();
6000	while (it.next()) \|case\| {
6001	if (case.ranges.len > 0) last_range_case = case.idx;
6002	llvm_cases_len += @intCast(case.items.len);
6003	}
6004	break :info .{ llvm_cases_len, last_range_case };
6005	};
6006
6007	// The `else` of the LLVM `switch` is the actual `else` prong only
6008	// if there are no ranges. Otherwise, the `else` will have a
6009	// conditional chain before the "true" `else` prong.
6010	const llvm_else_block = if (last_range_case == null)
6011	dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
6012	else
6013	try self.wip.block(0, "RangeTest");
6014
6015	llvm_else_block.ptr(&self.wip).incoming += 1;
6016
6017	var wip_switch = try self.wip.@"switch"(cond_int, llvm_else_block, llvm_cases_len, dispatch_info.switch_weights);
6018	defer wip_switch.finish(&self.wip);
6019
6020	// Construct the actual cases. Set the cursor to the `else` block so
6021	// we can construct ranges at the same time as scalar cases.
6022	self.wip.cursor = .{ .block = llvm_else_block };
6023
6024	var it = switch_br.iterateCases();
6025	while (it.next()) \|case\| {
6026	const case_block = dispatch_info.case_blocks[case.idx];
6027
6028	for (case.items) \|item\| {
6029	const llvm_item = (try self.resolveInst(item)).toConst().?;
6030	const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder))
6031	try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize)
6032	else
6033	llvm_item;
6034	try wip_switch.addCase(llvm_int_item, case_block, &self.wip);
6035	}
6036	case_block.ptr(&self.wip).incoming += @intCast(case.items.len);
6037
6038	if (case.ranges.len == 0) continue;
6039
6040	// Add a conditional for the ranges, directing to the relevant bb.
6041	// We don't need to consider `cold` branch hints since that information is stored
6042	// in the target bb body, but we do care about likely/unlikely/unpredictable.
6043
6044	const hint = switch_br.getHint(case.idx);
6045
6046	var range_cond: ?Builder.Value = null;
6047	for (case.ranges) \|range\| {
6048	const llvm_min = try self.resolveInst(range[0]);
6049	const llvm_max = try self.resolveInst(range[1]);
6050	const cond_part = try self.wip.bin(
6051	.@"and",
6052	try self.cmp(.normal, .gte, cond_ty, cond, llvm_min),
6053	try self.cmp(.normal, .lte, cond_ty, cond, llvm_max),
6054	"",
6055	);
6056	if (range_cond) \|prev\| {
6057	range_cond = try self.wip.bin(.@"or", prev, cond_part, "");
6058	} else range_cond = cond_part;
6059	}
6060
6061	// If the check fails, we either branch to the "true" `else` case,
6062	// or to the next range condition.
6063	const range_else_block = if (case.idx == last_range_case.?)
6064	dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
6065	else
6066	try self.wip.block(0, "RangeTest");
6067
6068	_ = try self.wip.brCond(range_cond.?, case_block, range_else_block, switch (hint) {
6069	.none, .cold => .none,
6070	.unpredictable => .unpredictable,
6071	.likely => .then_likely,
6072	.unlikely => .else_likely,
6073	});
6074	case_block.ptr(&self.wip).incoming += 1;
6075	range_else_block.ptr(&self.wip).incoming += 1;
6076
6077	// Construct the next range conditional (if any) in the false branch.
6078	self.wip.cursor = .{ .block = range_else_block };
6079	}
6080	}
6081
6082	fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) !void {
6083	const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
6084	const dispatch_info = self.switch_dispatch_info.get(br.block_inst).?;
6085	return self.lowerSwitchDispatch(br.block_inst, br.operand, dispatch_info);
6086	}
6087
6088	fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void {
6089	const cond_br = self.air.unwrapCondBr(inst);
6090	const cond = try self.resolveInst(cond_br.condition);
6091	const then_body = cond_br.then_body;
6092	const else_body = cond_br.else_body;
6093
6094	const Hint = enum {
6095	none,
6096	unpredictable,
6097	then_likely,
6098	else_likely,
6099	then_cold,
6100	else_cold,
6101	};
6102	const hint: Hint = switch (cond_br.branch_hints.true) {
6103	.none => switch (cond_br.branch_hints.false) {
6104	.none => .none,
6105	.likely => .else_likely,
6106	.unlikely => .then_likely,
6107	.cold => .else_cold,
6108	.unpredictable => .unpredictable,
6109	},
6110	.likely => switch (cond_br.branch_hints.false) {
6111	.none => .then_likely,
6112	.likely => .unpredictable,
6113	.unlikely => .then_likely,
6114	.cold => .else_cold,
6115	.unpredictable => .unpredictable,
6116	},
6117	.unlikely => switch (cond_br.branch_hints.false) {
6118	.none => .else_likely,
6119	.likely => .else_likely,
6120	.unlikely => .unpredictable,
6121	.cold => .else_cold,
6122	.unpredictable => .unpredictable,
6123	},
6124	.cold => .then_cold,
6125	.unpredictable => .unpredictable,
6126	};
6127
6128	const then_block = try self.wip.block(1, "Then");
6129	const else_block = try self.wip.block(1, "Else");
6130	_ = try self.wip.brCond(cond, then_block, else_block, switch (hint) {
6131	.none, .then_cold, .else_cold => .none,
6132	.unpredictable => .unpredictable,
6133	.then_likely => .then_likely,
6134	.else_likely => .else_likely,
6135	});
6136
6137	self.wip.cursor = .{ .block = then_block };
6138	if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold();
6139	try self.genBodyDebugScope(null, then_body, cond_br.branch_hints.then_cov);
6140
6141	self.wip.cursor = .{ .block = else_block };
6142	if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold();
6143	try self.genBodyDebugScope(null, else_body, cond_br.branch_hints.else_cov);
6144
6145	// No need to reset the insert cursor since this instruction is noreturn.
6146	}
6147
6148	fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
6149	const unwrapped_try = self.air.unwrapTry(inst);
6150	const err_union = try self.resolveInst(unwrapped_try.error_union);
6151	const body = unwrapped_try.else_body;
6152	const err_union_ty = self.typeOf(unwrapped_try.error_union);
6153	const is_unused = self.liveness.isUnused(inst);
6154	return lowerTry(self, err_union, body, err_union_ty, false, .none, false, is_unused, err_cold);
6155	}
6156
6157	fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
6158	const zcu = self.ng.pt.zcu;
6159	const unwrapped_try = self.air.unwrapTryPtr(inst);
6160	const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr);
6161	const body = unwrapped_try.else_body;
6162	const err_union_ptr_ty = self.typeOf(unwrapped_try.error_union_ptr);
6163	const err_union_ty = err_union_ptr_ty.childType(zcu);
6164	const is_unused = self.liveness.isUnused(inst);
6165
6166	self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu));
6167
6168	return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), true, is_unused, err_cold);
6169	}
6170
6171	fn lowerTry(
6172	fg: *FuncGen,
6173	err_union: Builder.Value,
6174	body: []const Air.Inst.Index,
6175	err_union_ty: Type,
6176	operand_is_ptr: bool,
6177	operand_ptr_align: InternPool.Alignment,
6178	can_elide_load: bool,
6179	is_unused: bool,
6180	err_cold: bool,
6181	) !Builder.Value {
6182	const o = fg.ng.object;
6183	const pt = fg.ng.pt;
6184	const zcu = pt.zcu;
6185	const payload_ty = err_union_ty.errorUnionPayload(zcu);
6186	const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
6187	const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
6188	const error_type = try o.errorIntType(pt);
6189
6190	const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{
6191	operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)),
6192	operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)),
6193	} else .{ .none, .none };
6194
6195	if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
6196	const loaded = loaded: {
6197	const access_kind: Builder.MemoryAccessKind =
6198	if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
6199
6200	if (!payload_has_bits) {
6201	break :loaded if (operand_is_ptr)
6202	try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "")
6203	else
6204	err_union;
6205	}
6206	const err_field_index = try errUnionErrorOffset(payload_ty, pt);
6207	if (operand_is_ptr or isByRef(err_union_ty, zcu)) {
6208	const err_field_ptr =
6209	try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, "");
6210	break :loaded try fg.wip.load(
6211	if (operand_is_ptr) access_kind else .normal,
6212	error_type,
6213	err_field_ptr,
6214	err_set_align.toLlvm(),
6215	"",
6216	);
6217	}
6218	break :loaded try fg.wip.extractValue(err_union, &.{err_field_index}, "");
6219	};
6220	const zero = try o.builder.intValue(error_type, 0);
6221	const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
6222
6223	const return_block = try fg.wip.block(1, "TryRet");
6224	const continue_block = try fg.wip.block(1, "TryCont");
6225	_ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely);
6226
6227	fg.wip.cursor = .{ .block = return_block };
6228	if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold();
6229	try fg.genBodyDebugScope(null, body, .poi);
6230
6231	fg.wip.cursor = .{ .block = continue_block };
6232	}
6233	if (is_unused) return .none;
6234	if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;
6235	const offset = try errUnionPayloadOffset(payload_ty, pt);
6236	if (operand_is_ptr) {
6237	return fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");
6238	} else if (isByRef(err_union_ty, zcu)) {
6239	const payload_ptr = try fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");
6240	if (isByRef(payload_ty, zcu)) {
6241	if (can_elide_load)
6242	return payload_ptr;
6243
6244	return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
6245	}
6246	const load_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
6247	return fg.wip.load(.normal, load_ty, payload_ptr, payload_align.toLlvm(), "");
6248	}
6249	return fg.wip.extractValue(err_union, &.{offset}, "");
6250	}
6251
6252	fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void {
6253	const o = self.ng.object;
6254	const pt = self.ng.pt;
6255	const zcu = pt.zcu;
6256
6257	const switch_br = self.air.unwrapSwitch(inst);
6258
6259	// For `loop_switch_br`, we need these BBs prepared ahead of time to generate dispatches.
6260	// For `switch_br`, they allow us to sometimes generate better IR by sharing a BB between
6261	// scalar and range cases in the same prong.
6262	// +1 for `else` case. This is not the same as the LLVM `else` prong, as that may first contain
6263	// conditionals to handle ranges.
6264	const case_blocks = try self.gpa.alloc(Builder.Function.Block.Index, switch_br.cases_len + 1);
6265	defer self.gpa.free(case_blocks);
6266	// We set incoming as 0 for now, and increment it as we construct dispatches.
6267	for (case_blocks[0 .. case_blocks.len - 1]) \|b\| b. = try self.wip.block(0, "Case");
6268	case_blocks[case_blocks.len - 1] = try self.wip.block(0, "Default");
6269
6270	// There's a special case here to manually generate a jump table in some cases.
6271	//
6272	// Labeled switch in Zig is intended to follow the "direct threading" pattern. We would ideally use a jump
6273	// table, and each `continue` has its own indirect `jmp`, to allow the branch predictor to more accurately
6274	// use data patterns to predict future dispatches. The problem, however, is that LLVM emits fascinatingly
6275	// bad asm for this. Not only does it not share the jump table -- which we really need it to do to prevent
6276	// destroying the cache -- but it also actually generates slightly different jump tables for each case,
6277	// and a separate conditional branch beforehand to handle dispatching back to the case we're currently
6278	// within(!!).
6279	//
6280	// This asm is really, really, not what we want. As such, we will construct the jump table manually where
6281	// appropriate (the values are dense and relatively few), and use it when lowering dispatches.
6282
6283	const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: {
6284	if (!is_dispatch_loop) break :jmp_table null;
6285
6286	// Workaround for:
6287	// * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/lib/MC/WasmObjectWriter.cpp#L560
6288	// * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll
6289	if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null;
6290
6291	// On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just
6292	// about acceptable - it won't fill L1d cache on most CPUs.
6293	const max_table_len = 1024;
6294
6295	const cond_ty = self.typeOf(switch_br.operand);
6296	switch (cond_ty.zigTypeTag(zcu)) {
6297	.bool, .pointer => break :jmp_table null,
6298	.@"enum", .int, .error_set, .@"struct", .@"union" => {},
6299	else => unreachable,
6300	}
6301
6302	if (cond_ty.intInfo(zcu).signedness == .signed) break :jmp_table null;
6303
6304	// Don't worry about the size of the type -- it's irrelevant, because the prong values could be fairly dense.
6305	// If they are, then we will construct a jump table.
6306	const min, const max = self.switchCaseItemRange(switch_br) orelse break :jmp_table null;
6307	const min_int = min.getUnsignedInt(zcu) orelse break :jmp_table null;
6308	const max_int = max.getUnsignedInt(zcu) orelse break :jmp_table null;
6309	const table_len = max_int - min_int + 1;
6310	if (table_len > max_table_len) break :jmp_table null;
6311
6312	const table_elems = try self.gpa.alloc(Builder.Constant, @intCast(table_len));
6313	defer self.gpa.free(table_elems);
6314
6315	// Set them all to the `else` branch, then iterate over the AIR switch
6316	// and replace all values which correspond to other prongs.
6317	@memset(table_elems, try o.builder.blockAddrConst(
6318	self.wip.function,
6319	case_blocks[case_blocks.len - 1],
6320	));
6321	var item_count: u32 = 0;
6322	var it = switch_br.iterateCases();
6323	while (it.next()) \|case\| {
6324	const case_block = case_blocks[case.idx];
6325	const case_block_addr = try o.builder.blockAddrConst(
6326	self.wip.function,
6327	case_block,
6328	);
6329	for (case.items) \|item\| {
6330	const val = Value.fromInterned(item.toInterned().?);
6331	const table_idx = val.toUnsignedInt(zcu) - min_int;
6332	table_elems[@intCast(table_idx)] = case_block_addr;
6333	item_count += 1;
6334	}
6335	for (case.ranges) \|range\| {
6336	const low = Value.fromInterned(range[0].toInterned().?);
6337	const high = Value.fromInterned(range[1].toInterned().?);
6338	const low_idx = low.toUnsignedInt(zcu) - min_int;
6339	const high_idx = high.toUnsignedInt(zcu) - min_int;
6340	@memset(table_elems[@intCast(low_idx)..@intCast(high_idx + 1)], case_block_addr);
6341	item_count += @intCast(high_idx + 1 - low_idx);
6342	}
6343	}
6344
6345	const table_llvm_ty = try o.builder.arrayType(table_elems.len, .ptr);
6346	const table_val = try o.builder.arrayConst(table_llvm_ty, table_elems);
6347
6348	const table_variable = try o.builder.addVariable(
6349	try o.builder.strtabStringFmt("__jmptab_{d}", .{@intFromEnum(inst)}),
6350	table_llvm_ty,
6351	.default,
6352	);
6353	try table_variable.setInitializer(table_val, &o.builder);
6354	table_variable.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
6355	table_variable.setUnnamedAddr(.unnamed_addr, &o.builder);
6356
6357	const table_includes_else = item_count != table_len;
6358
6359	break :jmp_table .{
6360	.min = try o.lowerValue(pt, min.toIntern()),
6361	.max = try o.lowerValue(pt, max.toIntern()),
6362	.in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {
6363	.none, .cold => .none,
6364	.unpredictable => .unpredictable,
6365	.likely => .likely,
6366	.unlikely => .unlikely,
6367	},
6368	.table = table_variable.toConst(&o.builder),
6369	.table_includes_else = table_includes_else,
6370	};
6371	};
6372
6373	const weights: Builder.Function.Instruction.BrCond.Weights = weights: {
6374	if (jmp_table != null) break :weights .none; // not used
6375
6376	// First pass. If any weights are `.unpredictable`, unpredictable.
6377	// If all are `.none` or `.cold`, none.
6378	var any_likely = false;
6379	for (0..switch_br.cases_len) \|case_idx\| {
6380	switch (switch_br.getHint(@intCast(case_idx))) {
6381	.none, .cold => {},
6382	.likely, .unlikely => any_likely = true,
6383	.unpredictable => break :weights .unpredictable,
6384	}
6385	}
6386	switch (switch_br.getElseHint()) {
6387	.none, .cold => {},
6388	.likely, .unlikely => any_likely = true,
6389	.unpredictable => break :weights .unpredictable,
6390	}
6391	if (!any_likely) break :weights .none;
6392
6393	const llvm_cases_len = llvm_cases_len: {
6394	var len: u32 = 0;
6395	var it = switch_br.iterateCases();
6396	while (it.next()) \|case\| len += @intCast(case.items.len);
6397	break :llvm_cases_len len;
6398	};
6399
6400	var weights = try self.gpa.alloc(Builder.Metadata, 1 + llvm_cases_len + 1);
6401	defer self.gpa.free(weights);
6402	var weight_idx: usize = 0;
6403
6404	const branch_weights_str = try o.builder.metadataString("branch_weights");
6405	weights[weight_idx] = branch_weights_str.toMetadata();
6406	weight_idx += 1;
6407
6408	const else_weight: u32 = switch (switch_br.getElseHint()) {
6409	.unpredictable => unreachable,
6410	.none, .cold => 1000,
6411	.likely => 2000,
6412	.unlikely => 1,
6413	};
6414	weights[weight_idx] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight));
6415	weight_idx += 1;
6416
6417	var it = switch_br.iterateCases();
6418	while (it.next()) \|case\| {
6419	const weight_val: u32 = switch (switch_br.getHint(case.idx)) {
6420	.unpredictable => unreachable,
6421	.none, .cold => 1000,
6422	.likely => 2000,
6423	.unlikely => 1,
6424	};
6425	const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val));
6426	@memset(weights[weight_idx..][0..case.items.len], weight_meta);
6427	weight_idx += case.items.len;
6428	}
6429
6430	assert(weight_idx == weights.len);
6431	break :weights .fromMetadata(try o.builder.metadataTuple(weights));
6432	};
6433
6434	const dispatch_info: SwitchDispatchInfo = .{
6435	.case_blocks = case_blocks,
6436	.switch_weights = weights,
6437	.jmp_table = jmp_table,
6438	};
6439
6440	if (is_dispatch_loop) {
6441	try self.switch_dispatch_info.putNoClobber(self.gpa, inst, dispatch_info);
6442	}
6443	defer if (is_dispatch_loop) {
6444	assert(self.switch_dispatch_info.remove(inst));
6445	};
6446
6447	// Generate the initial dispatch.
6448	// If this is a simple `switch_br`, this is the only dispatch.
6449	try self.lowerSwitchDispatch(inst, switch_br.operand, dispatch_info);
6450
6451	// Iterate the cases and generate their bodies.
6452	var it = switch_br.iterateCases();
6453	while (it.next()) \|case\| {
6454	const case_block = case_blocks[case.idx];
6455	self.wip.cursor = .{ .block = case_block };
6456	if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold();
6457	try self.genBodyDebugScope(null, case.body, .none);
6458	}
6459	self.wip.cursor = .{ .block = case_blocks[case_blocks.len - 1] };
6460	const else_body = it.elseBody();
6461	if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold();
6462	if (else_body.len > 0) {
6463	try self.genBodyDebugScope(null, it.elseBody(), .none);
6464	} else {
6465	_ = try self.wip.@"unreachable"();
6466	}
6467	}
6468
6469	fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) ?[2]Value {
6470	const zcu = self.ng.pt.zcu;
6471	var it = switch_br.iterateCases();
6472	var min: ?Value = null;
6473	var max: ?Value = null;
6474	while (it.next()) \|case\| {
6475	for (case.items) \|item\| {
6476	const val = Value.fromInterned(item.toInterned().?);
6477	const low = if (min) \|m\| val.compareHetero(.lt, m, zcu) else true;
6478	const high = if (max) \|m\| val.compareHetero(.gt, m, zcu) else true;
6479	if (low) min = val;
6480	if (high) max = val;
6481	}
6482	for (case.ranges) \|range\| {
6483	const vals: [2]Value = .{
6484	Value.fromInterned(range[0].toInterned().?),
6485	Value.fromInterned(range[1].toInterned().?),
6486	};
6487	const low = if (min) \|m\| vals[0].compareHetero(.lt, m, zcu) else true;
6488	const high = if (max) \|m\| vals[1].compareHetero(.gt, m, zcu) else true;
6489	if (low) min = vals[0];
6490	if (high) max = vals[1];
6491	}
6492	}
6493	if (min == null) {
6494	assert(max == null);
6495	return null;
6496	}
6497	return .{ min.?, max.? };
6498	}
6499
6500	fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void {
6501	const block = self.air.unwrapBlock(inst);
6502	const body = block.body;
6503	const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time
6504	_ = try self.wip.br(loop_block);
6505
6506	try self.loops.putNoClobber(self.gpa, inst, loop_block);
6507	defer assert(self.loops.remove(inst));
6508
6509	self.wip.cursor = .{ .block = loop_block };
6510	try self.genBodyDebugScope(null, body, .none);
6511	}
6512
6513	fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6514	const o = self.ng.object;
6515	const pt = self.ng.pt;
6516	const zcu = pt.zcu;
6517	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6518	const operand_ty = self.typeOf(ty_op.operand);
6519	const array_ty = operand_ty.childType(zcu);
6520	const llvm_usize = try o.lowerType(pt, Type.usize);
6521	const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
6522	const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));
6523	const operand = try self.resolveInst(ty_op.operand);
6524	if (!array_ty.hasRuntimeBits(zcu))
6525	return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
6526	const ptr = try self.wip.gep(.inbounds, try o.lowerType(pt, array_ty), operand, &.{
6527	try o.builder.intValue(llvm_usize, 0), try o.builder.intValue(llvm_usize, 0),
6528	}, "");
6529	return self.wip.buildAggregate(slice_llvm_ty, &.{ ptr, len }, "");
6530	}
6531
6532	fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6533	const o = self.ng.object;
6534	const pt = self.ng.pt;
6535	const zcu = pt.zcu;
6536	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6537
6538	const operand = try self.resolveInst(ty_op.operand);
6539	const operand_ty = self.typeOf(ty_op.operand);
6540	const operand_scalar_ty = operand_ty.scalarType(zcu);
6541	const is_signed_int = operand_scalar_ty.isSignedInt(zcu);
6542
6543	const dest_ty = self.typeOfIndex(inst);
6544	const dest_scalar_ty = dest_ty.scalarType(zcu);
6545	const dest_llvm_ty = try o.lowerType(pt, dest_ty);
6546	const target = zcu.getTarget();
6547
6548	if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(
6549	if (is_signed_int) .signed else .unsigned,
6550	operand,
6551	dest_llvm_ty,
6552	"",
6553	);
6554
6555	const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu))) orelse {
6556	return self.todo("float_from_int from '{f}' without intrinsics", .{operand_scalar_ty.fmt(pt)});
6557	};
6558	const rt_int_ty = try o.builder.intType(rt_int_bits);
6559	var extended = try self.wip.conv(
6560	if (is_signed_int) .signed else .unsigned,
6561	operand,
6562	rt_int_ty,
6563	"",
6564	);
6565	const dest_bits = dest_scalar_ty.floatBits(target);
6566	const compiler_rt_operand_abbrev = compilerRtIntAbbrev(rt_int_bits);
6567	const compiler_rt_dest_abbrev = compilerRtFloatAbbrev(dest_bits);
6568	const sign_prefix = if (is_signed_int) "" else "un";
6569	const fn_name = try o.builder.strtabStringFmt("__float{s}{s}i{s}f", .{
6570	sign_prefix,
6571	compiler_rt_operand_abbrev,
6572	compiler_rt_dest_abbrev,
6573	});
6574
6575	var param_type = rt_int_ty;
6576	if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) {
6577	// On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
6578	// i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have.
6579	param_type = try o.builder.vectorType(.normal, 2, .i64);
6580	extended = try self.wip.cast(.bitcast, extended, param_type, "");
6581	}
6582
6583	const libc_fn = try self.getLibcFunction(fn_name, &.{param_type}, dest_llvm_ty);
6584	return self.wip.call(
6585	.normal,
6586	.ccc,
6587	.none,
6588	libc_fn.typeOf(&o.builder),
6589	libc_fn.toValue(&o.builder),
6590	&.{extended},
6591	"",
6592	);
6593	}
6594
6595	fn airIntFromFloat(
6596	self: *FuncGen,
6597	inst: Air.Inst.Index,
6598	fast: Builder.FastMathKind,
6599	) !Builder.Value {
6600	_ = fast;
6601
6602	const o = self.ng.object;
6603	const pt = self.ng.pt;
6604	const zcu = pt.zcu;
6605	const target = zcu.getTarget();
6606	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6607
6608	const operand = try self.resolveInst(ty_op.operand);
6609	const operand_ty = self.typeOf(ty_op.operand);
6610	const operand_scalar_ty = operand_ty.scalarType(zcu);
6611
6612	const dest_ty = self.typeOfIndex(inst);
6613	const dest_scalar_ty = dest_ty.scalarType(zcu);
6614	const dest_llvm_ty = try o.lowerType(pt, dest_ty);
6615
6616	if (intrinsicsAllowed(operand_scalar_ty, target)) {
6617	// TODO set fast math flag
6618	return self.wip.conv(
6619	if (dest_scalar_ty.isSignedInt(zcu)) .signed else .unsigned,
6620	operand,
6621	dest_llvm_ty,
6622	"",
6623	);
6624	}
6625
6626	const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu))) orelse {
6627	return self.todo("int_from_float to '{f}' without intrinsics", .{dest_scalar_ty.fmt(pt)});
6628	};
6629	const ret_ty = try o.builder.intType(rt_int_bits);
6630	const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: {
6631	// On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
6632	// i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have.
6633	break :b try o.builder.vectorType(.normal, 2, .i64);
6634	} else ret_ty;
6635
6636	const operand_bits = operand_scalar_ty.floatBits(target);
6637	const compiler_rt_operand_abbrev = compilerRtFloatAbbrev(operand_bits);
6638
6639	const compiler_rt_dest_abbrev = compilerRtIntAbbrev(rt_int_bits);
6640	const sign_prefix = if (dest_scalar_ty.isSignedInt(zcu)) "" else "uns";
6641
6642	const fn_name = try o.builder.strtabStringFmt("__fix{s}{s}f{s}i", .{
6643	sign_prefix,
6644	compiler_rt_operand_abbrev,
6645	compiler_rt_dest_abbrev,
6646	});
6647
6648	const operand_llvm_ty = try o.lowerType(pt, operand_ty);
6649	const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);
6650	var result = try self.wip.call(
6651	.normal,
6652	.ccc,
6653	.none,
6654	libc_fn.typeOf(&o.builder),
6655	libc_fn.toValue(&o.builder),
6656	&.{operand},
6657	"",
6658	);
6659
6660	if (libc_ret_ty != ret_ty) result = try self.wip.cast(.bitcast, result, ret_ty, "");
6661	if (ret_ty != dest_llvm_ty) result = try self.wip.cast(.trunc, result, dest_llvm_ty, "");
6662	return result;
6663	}
6664
6665	fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
6666	const zcu = fg.ng.pt.zcu;
6667	return if (ty.isSlice(zcu)) fg.wip.extractValue(ptr, &.{0}, "") else ptr;
6668	}
6669
6670	fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
6671	const o = fg.ng.object;
6672	const pt = fg.ng.pt;
6673	const zcu = pt.zcu;
6674	const llvm_usize = try o.lowerType(pt, Type.usize);
6675	switch (ty.ptrSize(zcu)) {
6676	.slice => {
6677	const len = try fg.wip.extractValue(ptr, &.{1}, "");
6678	const elem_ty = ty.childType(zcu);
6679	const abi_size = elem_ty.abiSize(zcu);
6680	if (abi_size == 1) return len;
6681	const abi_size_llvm_val = try o.builder.intValue(llvm_usize, abi_size);
6682	return fg.wip.bin(.@"mul nuw", len, abi_size_llvm_val, "");
6683	},
6684	.one => {
6685	const array_ty = ty.childType(zcu);
6686	const elem_ty = array_ty.childType(zcu);
6687	const abi_size = elem_ty.abiSize(zcu);
6688	return o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu) * abi_size);
6689	},
6690	.many, .c => unreachable,
6691	}
6692	}
6693
6694	fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Value {
6695	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6696	const operand = try self.resolveInst(ty_op.operand);
6697	return self.wip.extractValue(operand, &.{index}, "");
6698	}
6699
6700	fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: c_uint) !Builder.Value {
6701	const o = self.ng.object;
6702	const pt = self.ng.pt;
6703	const zcu = pt.zcu;
6704	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6705	const slice_ptr = try self.resolveInst(ty_op.operand);
6706	const slice_ptr_ty = self.typeOf(ty_op.operand);
6707	const slice_llvm_ty = try o.lowerType(pt, slice_ptr_ty.childType(zcu));
6708
6709	return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, "");
6710	}
6711
6712	fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6713	const o = self.ng.object;
6714	const pt = self.ng.pt;
6715	const zcu = pt.zcu;
6716	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6717	const slice_ty = self.typeOf(bin_op.lhs);
6718	const slice = try self.resolveInst(bin_op.lhs);
6719	const index = try self.resolveInst(bin_op.rhs);
6720	const slice_info = slice_ty.ptrInfo(zcu);
6721	assert(slice_info.flags.size == .slice);
6722	const elem_ty: Type = .fromInterned(slice_info.child);
6723	const llvm_elem_ty = try o.lowerType(pt, elem_ty);
6724	const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
6725	const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
6726	const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
6727	const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
6728	self.maybeMarkAllowZeroAccess(slice_info);
6729	if (isByRef(elem_ty, zcu)) {
6730	return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind);
6731	} else {
6732	return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm());
6733	}
6734	}
6735
6736	fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6737	const o = self.ng.object;
6738	const pt = self.ng.pt;
6739	const zcu = pt.zcu;
6740	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6741	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
6742	const slice_ty = self.typeOf(bin_op.lhs);
6743
6744	const slice = try self.resolveInst(bin_op.lhs);
6745	const index = try self.resolveInst(bin_op.rhs);
6746	const llvm_elem_ty = try o.lowerType(pt, slice_ty.childType(zcu));
6747	const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
6748	return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
6749	}
6750
6751	fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6752	const o = self.ng.object;
6753	const pt = self.ng.pt;
6754	const zcu = pt.zcu;
6755
6756	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6757	const array_ty = self.typeOf(bin_op.lhs);
6758	const array_llvm_val = try self.resolveInst(bin_op.lhs);
6759	const rhs = try self.resolveInst(bin_op.rhs);
6760	const array_llvm_ty = try o.lowerType(pt, array_ty);
6761	const elem_ty = array_ty.childType(zcu);
6762	if (isByRef(array_ty, zcu)) {
6763	const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{
6764	try o.builder.intValue(try o.lowerType(pt, Type.usize), 0),
6765	rhs,
6766	}, "");
6767	if (isByRef(elem_ty, zcu)) {
6768	const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();
6769	return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);
6770	} else {
6771	return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
6772	}
6773	}
6774
6775	// This branch can be reached for vectors, which are always by-value.
6776	return self.wip.extractElement(array_llvm_val, rhs, "");
6777	}
6778
6779	fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6780	const o = self.ng.object;
6781	const pt = self.ng.pt;
6782	const zcu = pt.zcu;
6783	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6784	const ptr_ty = self.typeOf(bin_op.lhs);
6785	const elem_ty = ptr_ty.indexableElem(zcu);
6786	const llvm_elem_ty = try o.lowerType(pt, elem_ty);
6787	const base_ptr = try self.resolveInst(bin_op.lhs);
6788	const rhs = try self.resolveInst(bin_op.rhs);
6789	const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, "");
6790	if (isByRef(elem_ty, zcu)) {
6791	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
6792	const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
6793	return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal);
6794	}
6795
6796	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
6797
6798	return self.load(ptr, ptr_ty);
6799	}
6800
6801	fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6802	const o = self.ng.object;
6803	const pt = self.ng.pt;
6804	const zcu = pt.zcu;
6805	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6806	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
6807	const ptr_ty = self.typeOf(bin_op.lhs);
6808	const elem_ty = ptr_ty.indexableElem(zcu);
6809	assert(elem_ty.hasRuntimeBits(zcu));
6810
6811	const base_ptr = try self.resolveInst(bin_op.lhs);
6812	const rhs = try self.resolveInst(bin_op.rhs);
6813
6814	const elem_ptr = ty_pl.ty.toType();
6815	if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
6816
6817	const llvm_elem_ty = try o.lowerType(pt, elem_ty);
6818	return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, "");
6819	}
6820
6821	fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6822	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6823	const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
6824	const struct_ptr = try self.resolveInst(struct_field.struct_operand);
6825	const struct_ptr_ty = self.typeOf(struct_field.struct_operand);
6826	return self.fieldPtr(struct_ptr, struct_ptr_ty, struct_field.field_index);
6827	}
6828
6829	fn airStructFieldPtrIndex(
6830	self: *FuncGen,
6831	inst: Air.Inst.Index,
6832	field_index: u32,
6833	) !Builder.Value {
6834	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6835	const struct_ptr = try self.resolveInst(ty_op.operand);
6836	const struct_ptr_ty = self.typeOf(ty_op.operand);
6837	return self.fieldPtr(struct_ptr, struct_ptr_ty, field_index);
6838	}
6839
6840	fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6841	const o = self.ng.object;
6842	const pt = self.ng.pt;
6843	const zcu = pt.zcu;
6844	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6845	const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
6846	const struct_ty = self.typeOf(struct_field.struct_operand);
6847	const struct_llvm_val = try self.resolveInst(struct_field.struct_operand);
6848	const field_index = struct_field.field_index;
6849	const field_ty = struct_ty.fieldType(field_index, zcu);
6850	if (!field_ty.hasRuntimeBits(zcu)) return .none;
6851
6852	if (!isByRef(struct_ty, zcu)) {
6853	assert(!isByRef(field_ty, zcu));
6854	switch (struct_ty.zigTypeTag(zcu)) {
6855	.@"struct" => switch (struct_ty.containerLayout(zcu)) {
6856	.@"packed" => {
6857	const struct_type = zcu.typeToStruct(struct_ty).?;
6858	const bit_offset = zcu.structPackedFieldBitOffset(struct_type, field_index);
6859	const containing_int = struct_llvm_val;
6860	const shift_amt =
6861	try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset);
6862	const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, "");
6863	const elem_llvm_ty = try o.lowerType(pt, field_ty);
6864	if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
6865	const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
6866	const truncated_int =
6867	try self.wip.cast(.trunc, shifted_value, same_size_int, "");
6868	return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
6869	}
6870	return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
6871	},
6872	else => {
6873	const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
6874	return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, "");
6875	},
6876	},
6877	.@"union" => {
6878	assert(struct_ty.containerLayout(zcu) == .@"packed");
6879	const containing_int = struct_llvm_val;
6880	const elem_llvm_ty = try o.lowerType(pt, field_ty);
6881	if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
6882	const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
6883	const truncated_int =
6884	try self.wip.cast(.trunc, containing_int, same_size_int, "");
6885	return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
6886	}
6887	return self.wip.cast(.trunc, containing_int, elem_llvm_ty, "");
6888	},
6889	else => unreachable,
6890	}
6891	}
6892
6893	switch (struct_ty.zigTypeTag(zcu)) {
6894	.@"struct" => {
6895	const layout = struct_ty.containerLayout(zcu);
6896	assert(layout != .@"packed");
6897	const struct_llvm_ty = try o.lowerType(pt, struct_ty);
6898	const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
6899	const field_ptr =
6900	try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, "");
6901	const explicit_alignment = struct_ty.explicitFieldAlignment(field_index, zcu);
6902	const field_ptr_ty = try pt.ptrType(.{
6903	.child = field_ty.toIntern(),
6904	.flags = .{ .alignment = explicit_alignment },
6905	});
6906	if (isByRef(field_ty, zcu)) {
6907	const alignment = switch (explicit_alignment) {
6908	.none => field_ty.abiAlignment(zcu),
6909	else => \|a\| a,
6910	};
6911	return self.loadByRef(field_ptr, field_ty, alignment.toLlvm(), .normal);
6912	} else {
6913	return self.load(field_ptr, field_ptr_ty);
6914	}
6915	},
6916	.@"union" => {
6917	const union_llvm_ty = try o.lowerType(pt, struct_ty);
6918	const layout = struct_ty.unionGetLayout(zcu);
6919	const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align));
6920	const field_ptr =
6921	try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, "");
6922	const payload_alignment = layout.payload_align.toLlvm();
6923	if (isByRef(field_ty, zcu)) {
6924	return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal);
6925	} else {
6926	return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment);
6927	}
6928	},
6929	else => unreachable,
6930	}
6931	}
6932
6933	fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6934	const o = self.ng.object;
6935	const pt = self.ng.pt;
6936	const zcu = pt.zcu;
6937	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6938	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
6939
6940	const field_ptr = try self.resolveInst(extra.field_ptr);
6941
6942	const parent_ty = ty_pl.ty.toType().childType(zcu);
6943	const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
6944	if (field_offset == 0) return field_ptr;
6945
6946	const res_ty = try o.lowerType(pt, ty_pl.ty.toType());
6947	const llvm_usize = try o.lowerType(pt, Type.usize);
6948
6949	const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");
6950	const base_ptr_int = try self.wip.bin(
6951	.@"sub nuw",
6952	field_ptr_int,
6953	try o.builder.intValue(llvm_usize, field_offset),
6954	"",
6955	);
6956	return self.wip.cast(.inttoptr, base_ptr_int, res_ty, "");
6957	}
6958
6959	fn airNot(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6960	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6961	const operand = try self.resolveInst(ty_op.operand);
6962
6963	return self.wip.not(operand, "");
6964	}
6965
6966	fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) !void {
6967	_ = inst;
6968	_ = try self.wip.@"unreachable"();
6969	}
6970
6971	fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6972	const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
6973	self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1);
6974	self.prev_dbg_column = @intCast(dbg_stmt.column + 1);
6975
6976	self.wip.debug_location = .{ .location = .{
6977	.line = self.prev_dbg_line,
6978	.column = self.prev_dbg_column,
6979	.scope = self.scope.toOptional(),
6980	.inlined_at = self.inlined_at,
6981	} };
6982
6983	return .none;
6984	}
6985
6986	fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6987	_ = self;
6988	_ = inst;
6989	return .none;
6990	}
6991
6992	fn airDbgInlineBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6993	const block = self.air.unwrapDbgBlock(inst);
6994	self.arg_inline_index = 0;
6995	return self.lowerBlock(inst, block.func, block.body);
6996	}
6997
6998	fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6999	const o = self.ng.object;
7000	const pt = self.ng.pt;
7001	const zcu = pt.zcu;
7002	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7003	const operand = try self.resolveInst(pl_op.operand);
7004	const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
7005	const ptr_ty = self.typeOf(pl_op.operand);
7006
7007	const debug_local_var = try o.builder.debugLocalVar(
7008	try o.builder.metadataString(name.toSlice(self.air)),
7009	self.file,
7010	self.scope,
7011	self.prev_dbg_line,
7012	try o.getDebugType(pt, ptr_ty.childType(zcu)),
7013	);
7014
7015	_ = try self.wip.callIntrinsic(
7016	.normal,
7017	.none,
7018	.@"dbg.declare",
7019	&.{},
7020	&.{
7021	(try self.wip.debugValue(operand)).toValue(),
7022	debug_local_var.toValue(),
7023	(try o.builder.debugExpression(&.{})).toValue(),
7024	},
7025	"",
7026	);
7027
7028	return .none;
7029	}
7030
7031	fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) !Builder.Value {
7032	const o = self.ng.object;
7033	const pt = self.ng.pt;
7034	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7035	const operand = try self.resolveInst(pl_op.operand);
7036	const operand_ty = self.typeOf(pl_op.operand);
7037	const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
7038	const name_slice = name.toSlice(self.air);
7039	const metadata_name = if (name_slice.len > 0) try o.builder.metadataString(name_slice) else null;
7040	const debug_local_var = if (is_arg) try o.builder.debugParameter(
7041	metadata_name,
7042	self.file,
7043	self.scope,
7044	self.prev_dbg_line,
7045	try o.getDebugType(pt, operand_ty),
7046	arg_no: {
7047	self.arg_inline_index += 1;
7048	break :arg_no self.arg_inline_index;
7049	},
7050	) else try o.builder.debugLocalVar(
7051	metadata_name,
7052	self.file,
7053	self.scope,
7054	self.prev_dbg_line,
7055	try o.getDebugType(pt, operand_ty),
7056	);
7057
7058	const zcu = pt.zcu;
7059	const owner_mod = self.ng.ownerModule();
7060	if (isByRef(operand_ty, zcu)) {
7061	_ = try self.wip.callIntrinsic(
7062	.normal,
7063	.none,
7064	.@"dbg.declare",
7065	&.{},
7066	&.{
7067	(try self.wip.debugValue(operand)).toValue(),
7068	debug_local_var.toValue(),
7069	(try o.builder.debugExpression(&.{})).toValue(),
7070	},
7071	"",
7072	);
7073	} else if (owner_mod.optimize_mode == .Debug and !self.is_naked) {
7074	// We avoid taking this path for naked functions because there's no guarantee that such
7075	// functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
7076
7077	const alignment = operand_ty.abiAlignment(zcu).toLlvm();
7078	const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment);
7079	_ = try self.wip.store(.normal, operand, alloca, alignment);
7080	_ = try self.wip.callIntrinsic(
7081	.normal,
7082	.none,
7083	.@"dbg.declare",
7084	&.{},
7085	&.{
7086	(try self.wip.debugValue(alloca)).toValue(),
7087	debug_local_var.toValue(),
7088	(try o.builder.debugExpression(&.{})).toValue(),
7089	},
7090	"",
7091	);
7092	} else {
7093	_ = try self.wip.callIntrinsic(
7094	.normal,
7095	.none,
7096	.@"dbg.value",
7097	&.{},
7098	&.{
7099	(try self.wip.debugValue(operand)).toValue(),
7100	debug_local_var.toValue(),
7101	(try o.builder.debugExpression(&.{})).toValue(),
7102	},
7103	"",
7104	);
7105	}
7106	return .none;
7107	}
7108
7109	fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7110	// Eventually, the Zig compiler needs to be reworked to have inline
7111	// assembly go through the same parsing code regardless of backend, and
7112	// have LLVM-flavored inline assembly be output from that assembler.
7113	// We don't have such an assembler implemented yet though. For now,
7114	// this implementation feeds the inline assembly code directly to LLVM.
7115
7116	const o = self.ng.object;
7117	const unwrapped_asm = self.air.unwrapAsm(inst);
7118	const is_volatile = unwrapped_asm.is_volatile;
7119	const gpa = self.gpa;
7120
7121	const outputs = unwrapped_asm.outputs;
7122	const inputs = unwrapped_asm.inputs;
7123
7124	var llvm_constraints: std.ArrayList(u8) = .empty;
7125	defer llvm_constraints.deinit(gpa);
7126
7127	var arena_allocator = std.heap.ArenaAllocator.init(gpa);
7128	defer arena_allocator.deinit();
7129	const arena = arena_allocator.allocator();
7130
7131	// The exact number of return / parameter values depends on which output values
7132	// are passed by reference as indirect outputs (determined below).
7133	const max_return_count = outputs.len;
7134	const llvm_ret_types = try arena.alloc(Builder.Type, max_return_count);
7135	const llvm_ret_indirect = try arena.alloc(bool, max_return_count);
7136	const llvm_rw_vals = try arena.alloc(Builder.Value, max_return_count);
7137
7138	const max_param_count = max_return_count + inputs.len + outputs.len;
7139	const llvm_param_types = try arena.alloc(Builder.Type, max_param_count);
7140	const llvm_param_values = try arena.alloc(Builder.Value, max_param_count);
7141	// This stores whether we need to add an elementtype attribute and
7142	// if so, the element type itself.
7143	const llvm_param_attrs = try arena.alloc(Builder.Type, max_param_count);
7144	const pt = self.ng.pt;
7145	const zcu = pt.zcu;
7146	const ip = &zcu.intern_pool;
7147	const target = zcu.getTarget();
7148
7149	var llvm_ret_i: usize = 0;
7150	var llvm_param_i: usize = 0;
7151	var total_i: usize = 0;
7152
7153	var name_map: std.StringArrayHashMapUnmanaged(u16) = .empty;
7154	try name_map.ensureUnusedCapacity(arena, max_param_count);
7155
7156	var it = unwrapped_asm.iterateOutputs();
7157	while (it.next()) \|output\| {
7158	const constraint = output.constraint;
7159	const name = output.name;
7160
7161	try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 3);
7162	if (total_i != 0) {
7163	llvm_constraints.appendAssumeCapacity(',');
7164	}
7165	llvm_constraints.appendAssumeCapacity('=');
7166
7167	if (output.operand != .none) {
7168	const output_inst = try self.resolveInst(output.operand);
7169	const output_ty = self.typeOf(output.operand);
7170	assert(output_ty.zigTypeTag(zcu) == .pointer);
7171	const elem_llvm_ty = try o.lowerType(pt, output_ty.childType(zcu));
7172
7173	switch (constraint[0]) {
7174	'=' => {},
7175	'+' => llvm_rw_vals[output.index] = output_inst,
7176	else => return self.todo("unsupported output constraint on output type '{c}'", .{
7177	constraint[0],
7178	}),
7179	}
7180
7181	self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu));
7182
7183	// Pass any non-return outputs indirectly, if the constraint accepts a memory location
7184	llvm_ret_indirect[output.index] = constraintAllowsMemory(constraint);
7185	if (llvm_ret_indirect[output.index]) {
7186	// Pass the result by reference as an indirect output (e.g. "=*m")
7187	llvm_constraints.appendAssumeCapacity('*');
7188
7189	llvm_param_values[llvm_param_i] = output_inst;
7190	llvm_param_types[llvm_param_i] = output_inst.typeOfWip(&self.wip);
7191	llvm_param_attrs[llvm_param_i] = elem_llvm_ty;
7192	llvm_param_i += 1;
7193	} else {
7194	// Pass the result directly (e.g. "=r")
7195	llvm_ret_types[llvm_ret_i] = elem_llvm_ty;
7196	llvm_ret_i += 1;
7197	}
7198	} else {
7199	switch (constraint[0]) {
7200	'=' => {},
7201	else => return self.todo("unsupported output constraint on result type '{s}'", .{
7202	constraint,
7203	}),
7204	}
7205
7206	llvm_ret_indirect[output.index] = false;
7207
7208	const ret_ty = self.typeOfIndex(inst);
7209	llvm_ret_types[llvm_ret_i] = try o.lowerType(pt, ret_ty);
7210	llvm_ret_i += 1;
7211	}
7212
7213	// LLVM uses commas internally to separate different constraints,
7214	// alternative constraints are achieved with pipes.
7215	// We still allow the user to use commas in a way that is similar
7216	// to GCC's inline assembly.
7217	// http://llvm.org/docs/LangRef.html#constraint-codes
7218	for (constraint[1..]) \|byte\| {
7219	switch (byte) {
7220	',' => llvm_constraints.appendAssumeCapacity('\|'),
7221	'*' => {}, // Indirect outputs are handled above
7222	else => llvm_constraints.appendAssumeCapacity(byte),
7223	}
7224	}
7225
7226	if (!std.mem.eql(u8, name, "_")) {
7227	const gop = name_map.getOrPutAssumeCapacity(name);
7228	if (gop.found_existing) return self.todo("duplicate asm output name '{s}'", .{name});
7229	gop.value_ptr.* = @intCast(total_i);
7230	}
7231	total_i += 1;
7232	}
7233
7234	it = unwrapped_asm.iterateInputs();
7235	while (it.next()) \|input\| {
7236	const constraint = input.constraint;
7237	const name = input.name;
7238
7239	const arg_llvm_value = try self.resolveInst(input.operand);
7240	const arg_ty = self.typeOf(input.operand);
7241	const is_by_ref = isByRef(arg_ty, zcu);
7242	if (is_by_ref) {
7243	if (constraintAllowsMemory(constraint)) {
7244	llvm_param_values[llvm_param_i] = arg_llvm_value;
7245	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
7246	} else {
7247	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
7248	const arg_llvm_ty = try o.lowerType(pt, arg_ty);
7249	const load_inst =
7250	try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
7251	llvm_param_values[llvm_param_i] = load_inst;
7252	llvm_param_types[llvm_param_i] = arg_llvm_ty;
7253	}
7254	} else {
7255	if (constraintAllowsRegister(constraint)) {
7256	llvm_param_values[llvm_param_i] = arg_llvm_value;
7257	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
7258	} else {
7259	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
7260	const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);
7261	_ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment);
7262	llvm_param_values[llvm_param_i] = arg_ptr;
7263	llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);
7264	}
7265	}
7266
7267	try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 1);
7268	if (total_i != 0) {
7269	llvm_constraints.appendAssumeCapacity(',');
7270	}
7271	for (constraint) \|byte\| {
7272	llvm_constraints.appendAssumeCapacity(switch (byte) {
7273	',' => '\|',
7274	else => byte,
7275	});
7276	}
7277
7278	if (!std.mem.eql(u8, name, "_")) {
7279	const gop = name_map.getOrPutAssumeCapacity(name);
7280	if (gop.found_existing) return self.todo("duplicate asm input name '{s}'", .{name});
7281	gop.value_ptr.* = @intCast(total_i);
7282	}
7283
7284	// In the case of indirect inputs, LLVM requires the callsite to have
7285	// an elementtype(<ty>) attribute.
7286	llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {
7287	if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));
7288
7289	break :blk try o.lowerType(pt, if (is_by_ref) arg_ty else arg_ty.childType(zcu));
7290	} else .none;
7291
7292	llvm_param_i += 1;
7293	total_i += 1;
7294	}
7295
7296	it = unwrapped_asm.iterateOutputs();
7297	while (it.next()) \|output\| {
7298	const constraint = output.constraint;
7299
7300	if (constraint[0] != '+') continue;
7301
7302	const rw_ty = self.typeOf(output.operand);
7303	const llvm_elem_ty = try o.lowerType(pt, rw_ty.childType(zcu));
7304	if (llvm_ret_indirect[output.index]) {
7305	llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
7306	llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
7307	} else {
7308	const alignment = rw_ty.abiAlignment(zcu).toLlvm();
7309	const loaded = try self.wip.load(
7310	if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
7311	llvm_elem_ty,
7312	llvm_rw_vals[output.index],
7313	alignment,
7314	"",
7315	);
7316	llvm_param_values[llvm_param_i] = loaded;
7317	llvm_param_types[llvm_param_i] = llvm_elem_ty;
7318	}
7319
7320	try llvm_constraints.print(gpa, ",{d}", .{output.index});
7321
7322	// In the case of indirect inputs, LLVM requires the callsite to have
7323	// an elementtype(<ty>) attribute.
7324	llvm_param_attrs[llvm_param_i] = if (llvm_ret_indirect[output.index]) llvm_elem_ty else .none;
7325
7326	llvm_param_i += 1;
7327	total_i += 1;
7328	}
7329
7330	if (total_i != 0) try llvm_constraints.append(gpa, ',');
7331	const clobbers_val: Value = .fromInterned(unwrapped_asm.clobbers);
7332	const clobbers_ty = clobbers_val.typeOf(zcu);
7333	var clobbers_bigint_buf: Value.BigIntSpace = undefined;
7334	const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu);
7335	for (0..clobbers_ty.structFieldCount(zcu)) \|field_index\| {
7336	assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type);
7337	const limb_bits = @bitSizeOf(std.math.big.Limb);
7338	if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false
7339	switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) {
7340	0 => continue, // field is false
7341	1 => {}, // field is true
7342	}
7343	const name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
7344	total_i += try appendConstraints(gpa, &llvm_constraints, name, target);
7345	}
7346
7347	// We have finished scanning through all inputs/outputs, so the number of
7348	// parameters and return values is known.
7349	const param_count = llvm_param_i;
7350	const return_count = llvm_ret_i;
7351
7352	// For some targets, Clang unconditionally adds some clobbers to all inline assembly.
7353	// While this is probably not strictly necessary, if we don't follow Clang's lead
7354	// here then we may risk tripping LLVM bugs since anything not used by Clang tends
7355	// to be buggy and regress often.
7356	switch (target.cpu.arch) {
7357	.x86_64, .x86 => {
7358	try llvm_constraints.appendSlice(gpa, "~{dirflag},~{fpsr},~{flags},");
7359	total_i += 3;
7360	},
7361	.mips, .mipsel, .mips64, .mips64el => {
7362	try llvm_constraints.appendSlice(gpa, "~{$1},");
7363	total_i += 1;
7364	},
7365	else => {},
7366	}
7367
7368	if (std.mem.endsWith(u8, llvm_constraints.items, ",")) llvm_constraints.items.len -= 1;
7369
7370	const asm_source = unwrapped_asm.source;
7371
7372	// hackety hacks until stage2 has proper inline asm in the frontend.
7373	var rendered_template = std.array_list.Managed(u8).init(gpa);
7374	defer rendered_template.deinit();
7375
7376	const State = enum { start, percent, input, modifier };
7377
7378	var state: State = .start;
7379
7380	var name_start: usize = undefined;
7381	var modifier_start: usize = undefined;
7382	for (asm_source, 0..) \|byte, i\| {
7383	switch (state) {
7384	.start => switch (byte) {
7385	'%' => state = .percent,
7386	'$' => try rendered_template.appendSlice("$$"),
7387	else => try rendered_template.append(byte),
7388	},
7389	.percent => switch (byte) {
7390	'%' => {
7391	try rendered_template.append('%');
7392	state = .start;
7393	},
7394	'[' => {
7395	try rendered_template.append('$');
7396	try rendered_template.append('{');
7397	name_start = i + 1;
7398	state = .input;
7399	},
7400	'=' => {
7401	try rendered_template.appendSlice("${:uid}");
7402	state = .start;
7403	},
7404	else => {
7405	try rendered_template.append('%');
7406	try rendered_template.append(byte);
7407	state = .start;
7408	},
7409	},
7410	.input => switch (byte) {
7411	']', ':' => {
7412	const name = asm_source[name_start..i];
7413
7414	const index = name_map.get(name) orelse {
7415	// we should validate the assembly in Sema; by now it is too late
7416	return self.todo("unknown input or output name: '{s}'", .{name});
7417	};
7418	try rendered_template.print("{d}", .{index});
7419	if (byte == ':') {
7420	try rendered_template.append(':');
7421	modifier_start = i + 1;
7422	state = .modifier;
7423	} else {
7424	try rendered_template.append('}');
7425	state = .start;
7426	}
7427	},
7428	else => {},
7429	},
7430	.modifier => switch (byte) {
7431	']' => {
7432	try rendered_template.appendSlice(asm_source[modifier_start..i]);
7433	try rendered_template.append('}');
7434	state = .start;
7435	},
7436	else => {},
7437	},
7438	}
7439	}
7440
7441	var attributes: Builder.FunctionAttributes.Wip = .{};
7442	defer attributes.deinit(&o.builder);
7443	for (llvm_param_attrs[0..param_count], 0..) \|llvm_elem_ty, i\| if (llvm_elem_ty != .none)
7444	try attributes.addParamAttr(i, .{ .elementtype = llvm_elem_ty }, &o.builder);
7445
7446	const ret_llvm_ty = switch (return_count) {
7447	0 => .void,
7448	1 => llvm_ret_types[0],
7449	else => try o.builder.structType(.normal, llvm_ret_types),
7450	};
7451	const llvm_fn_ty = try o.builder.fnType(ret_llvm_ty, llvm_param_types[0..param_count], .normal);
7452	const call = try self.wip.callAsm(
7453	try attributes.finish(&o.builder),
7454	llvm_fn_ty,
7455	.{ .sideeffect = is_volatile },
7456	try o.builder.string(rendered_template.items),
7457	try o.builder.string(llvm_constraints.items),
7458	llvm_param_values[0..param_count],
7459	"",
7460	);
7461
7462	var ret_val = call;
7463	llvm_ret_i = 0;
7464	for (outputs, 0..) \|output, i\| {
7465	if (llvm_ret_indirect[i]) continue;
7466
7467	const output_value = if (return_count > 1)
7468	try self.wip.extractValue(call, &[_]u32{@intCast(llvm_ret_i)}, "")
7469	else
7470	call;
7471
7472	if (output != .none) {
7473	const output_ptr = try self.resolveInst(output);
7474	const output_ptr_ty = self.typeOf(output);
7475	const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm();
7476	_ = try self.wip.store(
7477	if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
7478	output_value,
7479	output_ptr,
7480	alignment,
7481	);
7482	} else {
7483	ret_val = output_value;
7484	}
7485	llvm_ret_i += 1;
7486	}
7487
7488	return ret_val;
7489	}
7490
7491	fn airIsNonNull(
7492	self: *FuncGen,
7493	inst: Air.Inst.Index,
7494	operand_is_ptr: bool,
7495	cond: Builder.IntegerCondition,
7496	) !Builder.Value {
7497	const o = self.ng.object;
7498	const pt = self.ng.pt;
7499	const zcu = pt.zcu;
7500	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7501	const operand = try self.resolveInst(un_op);
7502	const operand_ty = self.typeOf(un_op);
7503	const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7504	const optional_llvm_ty = try o.lowerType(pt, optional_ty);
7505	const payload_ty = optional_ty.optionalChild(zcu);
7506
7507	const access_kind: Builder.MemoryAccessKind =
7508	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7509
7510	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7511
7512	if (optional_ty.optionalReprIsPayload(zcu)) {
7513	const loaded = if (operand_is_ptr)
7514	try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
7515	else
7516	operand;
7517	if (payload_ty.isSlice(zcu)) {
7518	const slice_ptr = try self.wip.extractValue(loaded, &.{0}, "");
7519	const ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(
7520	payload_ty.ptrAddressSpace(zcu),
7521	zcu.getTarget(),
7522	));
7523	return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
7524	}
7525	return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), "");
7526	}
7527
7528	comptime assert(optional_layout_version == 3);
7529
7530	if (!payload_ty.hasRuntimeBits(zcu)) {
7531	const loaded = if (operand_is_ptr)
7532	try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
7533	else
7534	operand;
7535	return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
7536	}
7537
7538	const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu);
7539	return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind);
7540	}
7541
7542	fn airIsErr(
7543	self: *FuncGen,
7544	inst: Air.Inst.Index,
7545	cond: Builder.IntegerCondition,
7546	operand_is_ptr: bool,
7547	) !Builder.Value {
7548	const o = self.ng.object;
7549	const pt = self.ng.pt;
7550	const zcu = pt.zcu;
7551	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7552	const operand = try self.resolveInst(un_op);
7553	const operand_ty = self.typeOf(un_op);
7554	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7555	const payload_ty = err_union_ty.errorUnionPayload(zcu);
7556	const error_type = try o.errorIntType(pt);
7557	const zero = try o.builder.intValue(error_type, 0);
7558
7559	const access_kind: Builder.MemoryAccessKind =
7560	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7561
7562	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7563	const val: Builder.Constant = switch (cond) {
7564	.eq => .true, // 0 == 0
7565	.ne => .false, // 0 != 0
7566	else => unreachable,
7567	};
7568	return val.toValue();
7569	}
7570
7571	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7572
7573	if (!payload_ty.hasRuntimeBits(zcu)) {
7574	const loaded = if (operand_is_ptr)
7575	try self.wip.load(access_kind, try o.lowerType(pt, err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
7576	else
7577	operand;
7578	return self.wip.icmp(cond, loaded, zero, "");
7579	}
7580
7581	const err_field_index = try errUnionErrorOffset(payload_ty, pt);
7582
7583	const loaded = if (operand_is_ptr or isByRef(err_union_ty, zcu)) loaded: {
7584	const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7585	const err_alignment = if (operand_is_ptr)
7586	operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu))
7587	else
7588	.none;
7589	const err_field_ptr =
7590	try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, "");
7591	break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, err_alignment.toLlvm(), "");
7592	} else try self.wip.extractValue(operand, &.{err_field_index}, "");
7593	return self.wip.icmp(cond, loaded, zero, "");
7594	}
7595
7596	fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7597	const o = self.ng.object;
7598	const pt = self.ng.pt;
7599	const zcu = pt.zcu;
7600	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7601	const operand = try self.resolveInst(ty_op.operand);
7602	const optional_ty = self.typeOf(ty_op.operand).childType(zcu);
7603	const payload_ty = optional_ty.optionalChild(zcu);
7604	if (!payload_ty.hasRuntimeBits(zcu)) {
7605	// We have a pointer to a zero-bit value and we need to return
7606	// a pointer to a zero-bit value.
7607	return operand;
7608	}
7609	if (optional_ty.optionalReprIsPayload(zcu)) {
7610	// The payload and the optional are the same value.
7611	return operand;
7612	}
7613	return self.wip.gepStruct(try o.lowerType(pt, optional_ty), operand, 0, "");
7614	}
7615
7616	fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7617	comptime assert(optional_layout_version == 3);
7618
7619	const o = self.ng.object;
7620	const pt = self.ng.pt;
7621	const zcu = pt.zcu;
7622	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7623	const operand = try self.resolveInst(ty_op.operand);
7624	const optional_ptr_ty = self.typeOf(ty_op.operand);
7625	const optional_ty = optional_ptr_ty.childType(zcu);
7626	const payload_ty = optional_ty.optionalChild(zcu);
7627	const non_null_bit = try o.builder.intValue(.i8, 1);
7628
7629	const access_kind: Builder.MemoryAccessKind =
7630	if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7631
7632	if (!payload_ty.hasRuntimeBits(zcu)) {
7633	self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
7634
7635	// We have a pointer to a i8. We need to set it to 1 and then return the same pointer.
7636	// Default alignment store because align of the non null bit is 1 anyway.
7637	_ = try self.wip.store(access_kind, non_null_bit, operand, .default);
7638	return operand;
7639	}
7640	if (optional_ty.optionalReprIsPayload(zcu)) {
7641	// The payload and the optional are the same value.
7642	// Setting to non-null will be done when the payload is set.
7643	return operand;
7644	}
7645
7646	// First set the non-null bit.
7647	const optional_llvm_ty = try o.lowerType(pt, optional_ty);
7648	const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, "");
7649
7650	self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
7651
7652	// Default alignment store because align of the non null bit is 1 anyway.
7653	_ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default);
7654
7655	// Then return the payload pointer (only if it's used).
7656	if (self.liveness.isUnused(inst)) return .none;
7657
7658	return self.wip.gepStruct(optional_llvm_ty, operand, 0, "");
7659	}
7660
7661	fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7662	const o = self.ng.object;
7663	const pt = self.ng.pt;
7664	const zcu = pt.zcu;
7665	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7666	const operand = try self.resolveInst(ty_op.operand);
7667	const optional_ty = self.typeOf(ty_op.operand);
7668	const payload_ty = self.typeOfIndex(inst);
7669	if (!payload_ty.hasRuntimeBits(zcu)) return .none;
7670
7671	if (optional_ty.optionalReprIsPayload(zcu)) {
7672	// Payload value is the same as the optional value.
7673	return operand;
7674	}
7675
7676	const opt_llvm_ty = try o.lowerType(pt, optional_ty);
7677	return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false);
7678	}
7679
7680	fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value {
7681	const o = self.ng.object;
7682	const pt = self.ng.pt;
7683	const zcu = pt.zcu;
7684	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7685	const operand = try self.resolveInst(ty_op.operand);
7686	const operand_ty = self.typeOf(ty_op.operand);
7687	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7688	const result_ty = self.typeOfIndex(inst);
7689	const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty;
7690
7691	if (!payload_ty.hasRuntimeBits(zcu)) {
7692	return if (operand_is_ptr) operand else .none;
7693	}
7694	const offset = try errUnionPayloadOffset(payload_ty, pt);
7695	const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7696	if (operand_is_ptr) {
7697	return self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
7698	} else if (isByRef(err_union_ty, zcu)) {
7699	const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
7700	const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
7701	if (isByRef(payload_ty, zcu)) {
7702	return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
7703	}
7704	const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
7705	return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");
7706	}
7707	return self.wip.extractValue(operand, &.{offset}, "");
7708	}
7709
7710	fn airErrUnionErr(
7711	self: *FuncGen,
7712	inst: Air.Inst.Index,
7713	operand_is_ptr: bool,
7714	) !Builder.Value {
7715	const o = self.ng.object;
7716	const pt = self.ng.pt;
7717	const zcu = pt.zcu;
7718	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7719	const operand = try self.resolveInst(ty_op.operand);
7720	const operand_ty = self.typeOf(ty_op.operand);
7721	const error_type = try o.errorIntType(pt);
7722	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7723	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7724	if (operand_is_ptr) {
7725	return operand;
7726	} else {
7727	return o.builder.intValue(error_type, 0);
7728	}
7729	}
7730
7731	const access_kind: Builder.MemoryAccessKind =
7732	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7733
7734	const payload_ty = err_union_ty.errorUnionPayload(zcu);
7735	if (!payload_ty.hasRuntimeBits(zcu)) {
7736	if (!operand_is_ptr) return operand;
7737
7738	self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7739
7740	return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");
7741	}
7742
7743	const offset = try errUnionErrorOffset(payload_ty, pt);
7744
7745	if (operand_is_ptr or isByRef(err_union_ty, zcu)) {
7746	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7747
7748	const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7749	const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
7750	return self.wip.load(access_kind, error_type, err_field_ptr, .default, "");
7751	}
7752
7753	return self.wip.extractValue(operand, &.{offset}, "");
7754	}
7755
7756	fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7757	const o = self.ng.object;
7758	const pt = self.ng.pt;
7759	const zcu = pt.zcu;
7760	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7761	const operand = try self.resolveInst(ty_op.operand);
7762	const err_union_ptr_ty = self.typeOf(ty_op.operand);
7763	const err_union_ty = err_union_ptr_ty.childType(zcu);
7764	const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);
7765
7766	const payload_ty = err_union_ty.errorUnionPayload(zcu);
7767	const non_error_val = try o.builder.intValue(try o.errorIntType(pt), 0);
7768
7769	const access_kind: Builder.MemoryAccessKind =
7770	if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7771
7772	if (!payload_ty.hasRuntimeBits(zcu)) {
7773	self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
7774	_ = try self.wip.store(access_kind, non_error_val, operand, err_union_ptr_align.toLlvm());
7775	return operand;
7776	}
7777	const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7778	{
7779	self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
7780
7781	const err_int_ty = try pt.errorIntType();
7782	const error_alignment = err_int_ty.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
7783	const error_offset = try errUnionErrorOffset(payload_ty, pt);
7784	// First set the non-error value.
7785	const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, "");
7786	_ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment);
7787	}
7788	// Then return the payload pointer (only if it is used).
7789	if (self.liveness.isUnused(inst)) return .none;
7790
7791	const payload_offset = try errUnionPayloadOffset(payload_ty, pt);
7792	return self.wip.gepStruct(err_union_llvm_ty, operand, payload_offset, "");
7793	}
7794
7795	fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value {
7796	assert(self.err_ret_trace != .none);
7797	return self.err_ret_trace;
7798	}
7799
7800	fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7801	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7802	self.err_ret_trace = try self.resolveInst(un_op);
7803	return .none;
7804	}
7805
7806	fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7807	const o = self.ng.object;
7808	const pt = self.ng.pt;
7809	const zcu = pt.zcu;
7810
7811	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7812	const struct_ty = ty_pl.ty.toType();
7813	const field_index = ty_pl.payload;
7814
7815	const struct_llvm_ty = try o.lowerType(pt, struct_ty);
7816	const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
7817	assert(self.err_ret_trace != .none);
7818	const field_ptr = try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, "");
7819	const field_alignment = struct_ty.explicitFieldAlignment(field_index, zcu);
7820	const field_ty = struct_ty.fieldType(field_index, zcu);
7821	const field_ptr_ty = try pt.ptrType(.{
7822	.child = field_ty.toIntern(),
7823	.flags = .{ .alignment = field_alignment },
7824	});
7825	return self.load(field_ptr, field_ptr_ty);
7826	}
7827
7828	/// As an optimization, we want to avoid unnecessary copies of
7829	/// error union/optional types when returning from a function.
7830	/// Here, we scan forward in the current block, looking to see
7831	/// if the next instruction is a return (ignoring debug instructions).
7832	///
7833	/// The first instruction of `body_tail` is a wrap instruction.
7834	fn isNextRet(
7835	self: *FuncGen,
7836	body_tail: []const Air.Inst.Index,
7837	) bool {
7838	const air_tags = self.air.instructions.items(.tag);
7839	for (body_tail[1..]) \|body_inst\| {
7840	switch (air_tags[@intFromEnum(body_inst)]) {
7841	.ret => return true,
7842	.dbg_stmt => continue,
7843	else => return false,
7844	}
7845	}
7846	// The only way to get here is to hit the end of a loop instruction
7847	// (implicit repeat).
7848	return false;
7849	}
7850
7851	fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
7852	const o = self.ng.object;
7853	const pt = self.ng.pt;
7854	const zcu = pt.zcu;
7855	const inst = body_tail[0];
7856	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7857	const payload_ty = self.typeOf(ty_op.operand);
7858	const non_null_bit = try o.builder.intValue(.i8, 1);
7859	comptime assert(optional_layout_version == 3);
7860	assert(payload_ty.hasRuntimeBits(zcu));
7861	const operand = try self.resolveInst(ty_op.operand);
7862	const optional_ty = self.typeOfIndex(inst);
7863	if (optional_ty.optionalReprIsPayload(zcu)) return operand;
7864	const llvm_optional_ty = try o.lowerType(pt, optional_ty);
7865	if (isByRef(optional_ty, zcu)) {
7866	const directReturn = self.isNextRet(body_tail);
7867	const optional_ptr = if (directReturn)
7868	self.ret_ptr
7869	else brk: {
7870	const alignment = optional_ty.abiAlignment(zcu).toLlvm();
7871	const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
7872	break :brk optional_ptr;
7873	};
7874
7875	const payload_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 0, "");
7876	const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
7877	try self.store(payload_ptr, payload_ptr_ty, operand, .none);
7878	const non_null_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 1, "");
7879	_ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);
7880	return optional_ptr;
7881	}
7882	return self.wip.buildAggregate(llvm_optional_ty, &.{ operand, non_null_bit }, "");
7883	}
7884
7885	fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
7886	const o = self.ng.object;
7887	const pt = self.ng.pt;
7888	const zcu = pt.zcu;
7889	const inst = body_tail[0];
7890	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7891	const err_un_ty = self.typeOfIndex(inst);
7892	const operand = try self.resolveInst(ty_op.operand);
7893	const payload_ty = self.typeOf(ty_op.operand);
7894	assert(payload_ty.hasRuntimeBits(zcu));
7895	const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0);
7896	const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
7897
7898	const payload_offset = try errUnionPayloadOffset(payload_ty, pt);
7899	const error_offset = try errUnionErrorOffset(payload_ty, pt);
7900	if (isByRef(err_un_ty, zcu)) {
7901	const directReturn = self.isNextRet(body_tail);
7902	const result_ptr = if (directReturn)
7903	self.ret_ptr
7904	else brk: {
7905	const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm();
7906	const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
7907	break :brk result_ptr;
7908	};
7909
7910	const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
7911	const err_int_ty = try pt.errorIntType();
7912	const error_alignment = err_int_ty.abiAlignment(pt.zcu).toLlvm();
7913	_ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);
7914	const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
7915	const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
7916	try self.store(payload_ptr, payload_ptr_ty, operand, .none);
7917	return result_ptr;
7918	}
7919	var fields: [2]Builder.Value = undefined;
7920	fields[payload_offset] = operand;
7921	fields[error_offset] = ok_err_code;
7922	return self.wip.buildAggregate(err_un_llvm_ty, &fields, "");
7923	}
7924
7925	fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
7926	const o = self.ng.object;
7927	const pt = self.ng.pt;
7928	const zcu = pt.zcu;
7929	const inst = body_tail[0];
7930	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7931	const err_un_ty = self.typeOfIndex(inst);
7932	const payload_ty = err_un_ty.errorUnionPayload(zcu);
7933	const operand = try self.resolveInst(ty_op.operand);
7934	if (!payload_ty.hasRuntimeBits(zcu)) return operand;
7935	const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
7936
7937	const payload_offset = try errUnionPayloadOffset(payload_ty, pt);
7938	const error_offset = try errUnionErrorOffset(payload_ty, pt);
7939	if (isByRef(err_un_ty, zcu)) {
7940	const directReturn = self.isNextRet(body_tail);
7941	const result_ptr = if (directReturn)
7942	self.ret_ptr
7943	else brk: {
7944	const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
7945	const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
7946	break :brk result_ptr;
7947	};
7948
7949	const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
7950	const err_int_ty = try pt.errorIntType();
7951	const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm();
7952	_ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
7953	const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
7954	const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
7955	// TODO store undef to payload_ptr
7956	_ = payload_ptr;
7957	_ = payload_ptr_ty;
7958	return result_ptr;
7959	}
7960
7961	// TODO set payload bytes to undef
7962	const undef = try o.builder.undefValue(err_un_llvm_ty);
7963	return self.wip.insertValue(undef, operand, &.{error_offset}, "");
7964	}
7965
7966	fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7967	const o = self.ng.object;
7968	const pt = self.ng.pt;
7969	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7970	const index = pl_op.payload;
7971	const llvm_usize = try o.lowerType(pt, Type.usize);
7972	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{
7973	try o.builder.intValue(.i32, index),
7974	}, "");
7975	}
7976
7977	fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7978	const o = self.ng.object;
7979	const pt = self.ng.pt;
7980	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7981	const index = pl_op.payload;
7982	const llvm_isize = try o.lowerType(pt, Type.isize);
7983	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{
7984	try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),
7985	}, "");
7986	}
7987
7988	fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7989	const o = fg.ng.object;
7990	const pt = fg.ng.pt;
7991	const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav;
7992	const llvm_ptr_const = try o.lowerNavRefValue(pt, ty_nav.nav);
7993	return llvm_ptr_const.toValue();
7994	}
7995
7996	fn airMin(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7997	const o = self.ng.object;
7998	const pt = self.ng.pt;
7999	const zcu = pt.zcu;
8000	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8001	const lhs = try self.resolveInst(bin_op.lhs);
8002	const rhs = try self.resolveInst(bin_op.rhs);
8003	const inst_ty = self.typeOfIndex(inst);
8004	const scalar_ty = inst_ty.scalarType(zcu);
8005
8006	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, .normal, inst_ty, 2, .{ lhs, rhs });
8007	return self.wip.callIntrinsic(
8008	.normal,
8009	.none,
8010	if (scalar_ty.isSignedInt(zcu)) .smin else .umin,
8011	&.{try o.lowerType(pt, inst_ty)},
8012	&.{ lhs, rhs },
8013	"",
8014	);
8015	}
8016
8017	fn airMax(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8018	const o = self.ng.object;
8019	const pt = self.ng.pt;
8020	const zcu = pt.zcu;
8021	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8022	const lhs = try self.resolveInst(bin_op.lhs);
8023	const rhs = try self.resolveInst(bin_op.rhs);
8024	const inst_ty = self.typeOfIndex(inst);
8025	const scalar_ty = inst_ty.scalarType(zcu);
8026
8027	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, .normal, inst_ty, 2, .{ lhs, rhs });
8028	return self.wip.callIntrinsic(
8029	.normal,
8030	.none,
8031	if (scalar_ty.isSignedInt(zcu)) .smax else .umax,
8032	&.{try o.lowerType(pt, inst_ty)},
8033	&.{ lhs, rhs },
8034	"",
8035	);
8036	}
8037
8038	fn airSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8039	const o = self.ng.object;
8040	const pt = self.ng.pt;
8041	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8042	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
8043	const ptr = try self.resolveInst(bin_op.lhs);
8044	const len = try self.resolveInst(bin_op.rhs);
8045	const inst_ty = self.typeOfIndex(inst);
8046	return self.wip.buildAggregate(try o.lowerType(pt, inst_ty), &.{ ptr, len }, "");
8047	}
8048
8049	fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8050	const zcu = self.ng.pt.zcu;
8051	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8052	const lhs = try self.resolveInst(bin_op.lhs);
8053	const rhs = try self.resolveInst(bin_op.rhs);
8054	const inst_ty = self.typeOfIndex(inst);
8055	const scalar_ty = inst_ty.scalarType(zcu);
8056
8057	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, fast, inst_ty, 2, .{ lhs, rhs });
8058	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"add nsw" else .@"add nuw", lhs, rhs, "");
8059	}
8060
8061	fn airSafeArithmetic(
8062	fg: *FuncGen,
8063	inst: Air.Inst.Index,
8064	signed_intrinsic: Builder.Intrinsic,
8065	unsigned_intrinsic: Builder.Intrinsic,
8066	) !Builder.Value {
8067	const o = fg.ng.object;
8068	const pt = fg.ng.pt;
8069	const zcu = pt.zcu;
8070
8071	const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8072	const lhs = try fg.resolveInst(bin_op.lhs);
8073	const rhs = try fg.resolveInst(bin_op.rhs);
8074	const inst_ty = fg.typeOfIndex(inst);
8075	const scalar_ty = inst_ty.scalarType(zcu);
8076
8077	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
8078	const llvm_inst_ty = try o.lowerType(pt, inst_ty);
8079	const results =
8080	try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");
8081
8082	const overflow_bits = try fg.wip.extractValue(results, &.{1}, "");
8083	const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip);
8084	const overflow_bit = if (overflow_bits_ty.isVector(&o.builder))
8085	try fg.wip.callIntrinsic(
8086	.normal,
8087	.none,
8088	.@"vector.reduce.or",
8089	&.{overflow_bits_ty},
8090	&.{overflow_bits},
8091	"",
8092	)
8093	else
8094	overflow_bits;
8095
8096	const fail_block = try fg.wip.block(1, "OverflowFail");
8097	const ok_block = try fg.wip.block(1, "OverflowOk");
8098	_ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none);
8099
8100	fg.wip.cursor = .{ .block = fail_block };
8101	try fg.buildSimplePanic(.integer_overflow);
8102
8103	fg.wip.cursor = .{ .block = ok_block };
8104	return fg.wip.extractValue(results, &.{0}, "");
8105	}
8106
8107	fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8108	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8109	const lhs = try self.resolveInst(bin_op.lhs);
8110	const rhs = try self.resolveInst(bin_op.rhs);
8111
8112	return self.wip.bin(.add, lhs, rhs, "");
8113	}
8114
8115	fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8116	const o = self.ng.object;
8117	const pt = self.ng.pt;
8118	const zcu = pt.zcu;
8119	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8120	const lhs = try self.resolveInst(bin_op.lhs);
8121	const rhs = try self.resolveInst(bin_op.rhs);
8122	const inst_ty = self.typeOfIndex(inst);
8123	const scalar_ty = inst_ty.scalarType(zcu);
8124	assert(scalar_ty.zigTypeTag(zcu) == .int);
8125	return self.wip.callIntrinsic(
8126	.normal,
8127	.none,
8128	if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",
8129	&.{try o.lowerType(pt, inst_ty)},
8130	&.{ lhs, rhs },
8131	"",
8132	);
8133	}
8134
8135	fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8136	const zcu = self.ng.pt.zcu;
8137	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8138	const lhs = try self.resolveInst(bin_op.lhs);
8139	const rhs = try self.resolveInst(bin_op.rhs);
8140	const inst_ty = self.typeOfIndex(inst);
8141	const scalar_ty = inst_ty.scalarType(zcu);
8142
8143	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, fast, inst_ty, 2, .{ lhs, rhs });
8144	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"sub nsw" else .@"sub nuw", lhs, rhs, "");
8145	}
8146
8147	fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8148	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8149	const lhs = try self.resolveInst(bin_op.lhs);
8150	const rhs = try self.resolveInst(bin_op.rhs);
8151
8152	return self.wip.bin(.sub, lhs, rhs, "");
8153	}
8154
8155	fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8156	const o = self.ng.object;
8157	const pt = self.ng.pt;
8158	const zcu = pt.zcu;
8159	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8160	const lhs = try self.resolveInst(bin_op.lhs);
8161	const rhs = try self.resolveInst(bin_op.rhs);
8162	const inst_ty = self.typeOfIndex(inst);
8163	const scalar_ty = inst_ty.scalarType(zcu);
8164	assert(scalar_ty.zigTypeTag(zcu) == .int);
8165	return self.wip.callIntrinsic(
8166	.normal,
8167	.none,
8168	if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",
8169	&.{try o.lowerType(pt, inst_ty)},
8170	&.{ lhs, rhs },
8171	"",
8172	);
8173	}
8174
8175	fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8176	const zcu = self.ng.pt.zcu;
8177	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8178	const lhs = try self.resolveInst(bin_op.lhs);
8179	const rhs = try self.resolveInst(bin_op.rhs);
8180	const inst_ty = self.typeOfIndex(inst);
8181	const scalar_ty = inst_ty.scalarType(zcu);
8182
8183	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, fast, inst_ty, 2, .{ lhs, rhs });
8184	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"mul nsw" else .@"mul nuw", lhs, rhs, "");
8185	}
8186
8187	fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8188	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8189	const lhs = try self.resolveInst(bin_op.lhs);
8190	const rhs = try self.resolveInst(bin_op.rhs);
8191
8192	return self.wip.bin(.mul, lhs, rhs, "");
8193	}
8194
8195	fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8196	const o = self.ng.object;
8197	const pt = self.ng.pt;
8198	const zcu = pt.zcu;
8199	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8200	const lhs = try self.resolveInst(bin_op.lhs);
8201	const rhs = try self.resolveInst(bin_op.rhs);
8202	const inst_ty = self.typeOfIndex(inst);
8203	const scalar_ty = inst_ty.scalarType(zcu);
8204	assert(scalar_ty.zigTypeTag(zcu) == .int);
8205	return self.wip.callIntrinsic(
8206	.normal,
8207	.none,
8208	if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",
8209	&.{try o.lowerType(pt, inst_ty)},
8210	&.{ lhs, rhs, .@"0" },
8211	"",
8212	);
8213	}
8214
8215	fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8216	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8217	const lhs = try self.resolveInst(bin_op.lhs);
8218	const rhs = try self.resolveInst(bin_op.rhs);
8219	const inst_ty = self.typeOfIndex(inst);
8220
8221	return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8222	}
8223
8224	fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8225	const zcu = self.ng.pt.zcu;
8226	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8227	const lhs = try self.resolveInst(bin_op.lhs);
8228	const rhs = try self.resolveInst(bin_op.rhs);
8229	const inst_ty = self.typeOfIndex(inst);
8230	const scalar_ty = inst_ty.scalarType(zcu);
8231
8232	if (scalar_ty.isRuntimeFloat()) {
8233	const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8234	return self.buildFloatOp(.trunc, fast, inst_ty, 1, .{result});
8235	}
8236	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .sdiv else .udiv, lhs, rhs, "");
8237	}
8238
8239	fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8240	const o = self.ng.object;
8241	const pt = self.ng.pt;
8242	const zcu = pt.zcu;
8243	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8244	const lhs = try self.resolveInst(bin_op.lhs);
8245	const rhs = try self.resolveInst(bin_op.rhs);
8246	const inst_ty = self.typeOfIndex(inst);
8247	const scalar_ty = inst_ty.scalarType(zcu);
8248
8249	if (scalar_ty.isRuntimeFloat()) {
8250	const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8251	return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});
8252	}
8253	if (scalar_ty.isSignedInt(zcu)) {
8254	const inst_llvm_ty = try o.lowerType(pt, inst_ty);
8255
8256	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
8257	var stack align(@max(
8258	@alignOf(std.heap.StackFallbackAllocator(0)),
8259	@alignOf(ExpectedContents),
8260	)) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
8261	const allocator = stack.get();
8262
8263	const scalar_bits = inst_llvm_ty.scalarBits(&o.builder);
8264	var smin_big_int: std.math.big.int.Mutable = .{
8265	.limbs = try allocator.alloc(
8266	std.math.big.Limb,
8267	std.math.big.int.calcTwosCompLimbCount(scalar_bits),
8268	),
8269	.len = undefined,
8270	.positive = undefined,
8271	};
8272	defer allocator.free(smin_big_int.limbs);
8273	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
8274	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
8275	inst_llvm_ty.scalarType(&o.builder),
8276	smin_big_int.toConst(),
8277	));
8278
8279	const div = try self.wip.bin(.sdiv, lhs, rhs, "divFloor.div");
8280	const rem = try self.wip.bin(.srem, lhs, rhs, "divFloor.rem");
8281	const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "divFloor.rhs_sign");
8282	const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "divFloor.rem_xor_rhs_sign");
8283	const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "divFloor.need_correction");
8284	const correction = try self.wip.cast(.sext, need_correction, inst_llvm_ty, "divFloor.correction");
8285	return self.wip.bin(.@"add nsw", div, correction, "divFloor");
8286	}
8287	return self.wip.bin(.udiv, lhs, rhs, "");
8288	}
8289
8290	fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8291	const zcu = self.ng.pt.zcu;
8292	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8293	const lhs = try self.resolveInst(bin_op.lhs);
8294	const rhs = try self.resolveInst(bin_op.rhs);
8295	const inst_ty = self.typeOfIndex(inst);
8296	const scalar_ty = inst_ty.scalarType(zcu);
8297
8298	if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8299	return self.wip.bin(
8300	if (scalar_ty.isSignedInt(zcu)) .@"sdiv exact" else .@"udiv exact",
8301	lhs,
8302	rhs,
8303	"",
8304	);
8305	}
8306
8307	fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8308	const zcu = self.ng.pt.zcu;
8309	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8310	const lhs = try self.resolveInst(bin_op.lhs);
8311	const rhs = try self.resolveInst(bin_op.rhs);
8312	const inst_ty = self.typeOfIndex(inst);
8313	const scalar_ty = inst_ty.scalarType(zcu);
8314
8315	if (scalar_ty.isRuntimeFloat())
8316	return self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
8317	return self.wip.bin(if (scalar_ty.isSignedInt(zcu))
8318	.srem
8319	else
8320	.urem, lhs, rhs, "");
8321	}
8322
8323	fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8324	const o = self.ng.object;
8325	const pt = self.ng.pt;
8326	const zcu = pt.zcu;
8327	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8328	const lhs = try self.resolveInst(bin_op.lhs);
8329	const rhs = try self.resolveInst(bin_op.rhs);
8330	const inst_ty = self.typeOfIndex(inst);
8331	const inst_llvm_ty = try o.lowerType(pt, inst_ty);
8332	const scalar_ty = inst_ty.scalarType(zcu);
8333
8334	if (scalar_ty.isRuntimeFloat()) {
8335	const a = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
8336	const b = try self.buildFloatOp(.add, fast, inst_ty, 2, .{ a, rhs });
8337	const c = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ b, rhs });
8338	const zero = try o.builder.zeroInitValue(inst_llvm_ty);
8339	const ltz = try self.buildFloatCmp(fast, .lt, inst_ty, .{ lhs, zero });
8340	return self.wip.select(fast, ltz, c, a, "");
8341	}
8342	if (scalar_ty.isSignedInt(zcu)) {
8343	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
8344	var stack align(@max(
8345	@alignOf(std.heap.StackFallbackAllocator(0)),
8346	@alignOf(ExpectedContents),
8347	)) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
8348	const allocator = stack.get();
8349
8350	const scalar_bits = inst_llvm_ty.scalarBits(&o.builder);
8351	var smin_big_int: std.math.big.int.Mutable = .{
8352	.limbs = try allocator.alloc(
8353	std.math.big.Limb,
8354	std.math.big.int.calcTwosCompLimbCount(scalar_bits),
8355	),
8356	.len = undefined,
8357	.positive = undefined,
8358	};
8359	defer allocator.free(smin_big_int.limbs);
8360	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
8361	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
8362	inst_llvm_ty.scalarType(&o.builder),
8363	smin_big_int.toConst(),
8364	));
8365
8366	const rem = try self.wip.bin(.srem, lhs, rhs, "mod.rem");
8367	const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "mod.rhs_sign");
8368	const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "mod.rem_xor_rhs_sign");
8369	const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "mod.need_correction");
8370	const zero = try o.builder.zeroInitValue(inst_llvm_ty);
8371	const correction = try self.wip.select(.normal, need_correction, rhs, zero, "mod.correction");
8372	return self.wip.bin(.@"add nsw", correction, rem, "mod");
8373	}
8374	return self.wip.bin(.urem, lhs, rhs, "");
8375	}
8376
8377	fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8378	const o = self.ng.object;
8379	const pt = self.ng.pt;
8380	const zcu = pt.zcu;
8381	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8382	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
8383	const ptr = try self.resolveInst(bin_op.lhs);
8384	const offset = try self.resolveInst(bin_op.rhs);
8385	const ptr_ty = self.typeOf(bin_op.lhs);
8386	const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu));
8387	switch (ptr_ty.ptrSize(zcu)) {
8388	// It's a pointer to an array, so according to LLVM we need an extra GEP index.
8389	.one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{
8390	try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), offset,
8391	}, ""),
8392	.c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{offset}, ""),
8393	.slice => {
8394	const base = try self.wip.extractValue(ptr, &.{0}, "");
8395	return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{offset}, "");
8396	},
8397	}
8398	}
8399
8400	fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8401	const o = self.ng.object;
8402	const pt = self.ng.pt;
8403	const zcu = pt.zcu;
8404	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8405	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
8406	const ptr = try self.resolveInst(bin_op.lhs);
8407	const offset = try self.resolveInst(bin_op.rhs);
8408	const negative_offset = try self.wip.neg(offset, "");
8409	const ptr_ty = self.typeOf(bin_op.lhs);
8410	const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu));
8411	switch (ptr_ty.ptrSize(zcu)) {
8412	// It's a pointer to an array, so according to LLVM we need an extra GEP index.
8413	.one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{
8414	try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), negative_offset,
8415	}, ""),
8416	.c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{negative_offset}, ""),
8417	.slice => {
8418	const base = try self.wip.extractValue(ptr, &.{0}, "");
8419	return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{negative_offset}, "");
8420	},
8421	}
8422	}
8423
8424	fn airOverflow(
8425	self: *FuncGen,
8426	inst: Air.Inst.Index,
8427	signed_intrinsic: Builder.Intrinsic,
8428	unsigned_intrinsic: Builder.Intrinsic,
8429	) !Builder.Value {
8430	const o = self.ng.object;
8431	const pt = self.ng.pt;
8432	const zcu = pt.zcu;
8433	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8434	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
8435
8436	const lhs = try self.resolveInst(extra.lhs);
8437	const rhs = try self.resolveInst(extra.rhs);
8438
8439	const lhs_ty = self.typeOf(extra.lhs);
8440	const scalar_ty = lhs_ty.scalarType(zcu);
8441	const inst_ty = self.typeOfIndex(inst);
8442
8443	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
8444	const llvm_inst_ty = try o.lowerType(pt, inst_ty);
8445	const llvm_lhs_ty = try o.lowerType(pt, lhs_ty);
8446	const results =
8447	try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");
8448
8449	const result_val = try self.wip.extractValue(results, &.{0}, "");
8450	const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
8451
8452	const result_index = o.llvmFieldIndex(inst_ty, 0).?;
8453	const overflow_index = o.llvmFieldIndex(inst_ty, 1).?;
8454
8455	if (isByRef(inst_ty, zcu)) {
8456	const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();
8457	const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
8458	{
8459	const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, "");
8460	_ = try self.wip.store(.normal, result_val, field_ptr, result_alignment);
8461	}
8462	{
8463	const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, overflow_index, "");
8464	_ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
8465	}
8466
8467	return alloca_inst;
8468	}
8469
8470	var fields: [2]Builder.Value = undefined;
8471	fields[result_index] = result_val;
8472	fields[overflow_index] = overflow_bit;
8473	return self.wip.buildAggregate(llvm_inst_ty, &fields, "");
8474	}
8475
8476	fn buildElementwiseCall(
8477	self: *FuncGen,
8478	llvm_fn: Builder.Function.Index,
8479	args_vectors: []const Builder.Value,
8480	result_vector: Builder.Value,
8481	vector_len: usize,
8482	) !Builder.Value {
8483	const o = self.ng.object;
8484	assert(args_vectors.len <= 3);
8485
8486	var i: usize = 0;
8487	var result = result_vector;
8488	while (i < vector_len) : (i += 1) {
8489	const index_i32 = try o.builder.intValue(.i32, i);
8490
8491	var args: [3]Builder.Value = undefined;
8492	for (args[0..args_vectors.len], args_vectors) \|*arg_elem, arg_vector\| {
8493	arg_elem.* = try self.wip.extractElement(arg_vector, index_i32, "");
8494	}
8495	const result_elem = try self.wip.call(
8496	.normal,
8497	.ccc,
8498	.none,
8499	llvm_fn.typeOf(&o.builder),
8500	llvm_fn.toValue(&o.builder),
8501	args[0..args_vectors.len],
8502	"",
8503	);
8504	result = try self.wip.insertElement(result, result_elem, index_i32, "");
8505	}
8506	return result;
8507	}
8508
8509	fn getLibcFunction(
8510	self: *FuncGen,
8511	fn_name: Builder.StrtabString,
8512	param_types: []const Builder.Type,
8513	return_type: Builder.Type,
8514	) Allocator.Error!Builder.Function.Index {
8515	const o = self.ng.object;
8516	if (o.builder.getGlobal(fn_name)) \|global\| return switch (global.ptrConst(&o.builder).kind) {
8517	.alias => \|alias\| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function,
8518	.function => \|function\| function,
8519	.variable, .replaced => unreachable,
8520	};
8521	return o.builder.addFunction(
8522	try o.builder.fnType(return_type, param_types, .normal),
8523	fn_name,
8524	toLlvmAddressSpace(.generic, self.ng.pt.zcu.getTarget()),
8525	);
8526	}
8527
8528	/// Creates a floating point comparison by lowering to the appropriate
8529	/// hardware instruction or softfloat routine for the target
8530	fn buildFloatCmp(
8531	self: *FuncGen,
8532	fast: Builder.FastMathKind,
8533	pred: math.CompareOperator,
8534	ty: Type,
8535	params: [2]Builder.Value,
8536	) !Builder.Value {
8537	const o = self.ng.object;
8538	const pt = self.ng.pt;
8539	const zcu = pt.zcu;
8540	const target = zcu.getTarget();
8541	const scalar_ty = ty.scalarType(zcu);
8542	const scalar_llvm_ty = try o.lowerType(pt, scalar_ty);
8543
8544	if (intrinsicsAllowed(scalar_ty, target)) {
8545	const cond: Builder.FloatCondition = switch (pred) {
8546	.eq => .oeq,
8547	.neq => .une,
8548	.lt => .olt,
8549	.lte => .ole,
8550	.gt => .ogt,
8551	.gte => .oge,
8552	};
8553	return self.wip.fcmp(fast, cond, params[0], params[1], "");
8554	}
8555
8556	const float_bits = scalar_ty.floatBits(target);
8557	const compiler_rt_float_abbrev = compilerRtFloatAbbrev(float_bits);
8558	const fn_base_name = switch (pred) {
8559	.neq => "ne",
8560	.eq => "eq",
8561	.lt => "lt",
8562	.lte => "le",
8563	.gt => "gt",
8564	.gte => "ge",
8565	};
8566	const fn_name = try o.builder.strtabStringFmt("__{s}{s}f2", .{ fn_base_name, compiler_rt_float_abbrev });
8567
8568	const libc_fn = try self.getLibcFunction(fn_name, &.{ scalar_llvm_ty, scalar_llvm_ty }, .i32);
8569
8570	const int_cond: Builder.IntegerCondition = switch (pred) {
8571	.eq => .eq,
8572	.neq => .ne,
8573	.lt => .slt,
8574	.lte => .sle,
8575	.gt => .sgt,
8576	.gte => .sge,
8577	};
8578
8579	if (ty.zigTypeTag(zcu) == .vector) {
8580	const vec_len = ty.vectorLen(zcu);
8581	const vector_result_ty = try o.builder.vectorType(.normal, vec_len, .i32);
8582
8583	const init = try o.builder.poisonValue(vector_result_ty);
8584	const result = try self.buildElementwiseCall(libc_fn, &params, init, vec_len);
8585
8586	const zero_vector = try o.builder.splatValue(vector_result_ty, .@"0");
8587	return self.wip.icmp(int_cond, result, zero_vector, "");
8588	}
8589
8590	const result = try self.wip.call(
8591	.normal,
8592	.ccc,
8593	.none,
8594	libc_fn.typeOf(&o.builder),
8595	libc_fn.toValue(&o.builder),
8596	&params,
8597	"",
8598	);
8599	return self.wip.icmp(int_cond, result, .@"0", "");
8600	}
8601
8602	const FloatOp = enum {
8603	add,
8604	ceil,
8605	cos,
8606	div,
8607	exp,
8608	exp2,
8609	fabs,
8610	floor,
8611	fma,
8612	fmax,
8613	fmin,
8614	fmod,
8615	log,
8616	log10,
8617	log2,
8618	mul,
8619	neg,
8620	round,
8621	sin,
8622	sqrt,
8623	sub,
8624	tan,
8625	trunc,
8626	};
8627
8628	const FloatOpStrat = union(enum) {
8629	intrinsic: []const u8,
8630	libc: Builder.String,
8631	};
8632
8633	/// Creates a floating point operation (add, sub, fma, sqrt, exp, etc.)
8634	/// by lowering to the appropriate hardware instruction or softfloat
8635	/// routine for the target
8636	fn buildFloatOp(
8637	self: *FuncGen,
8638	comptime op: FloatOp,
8639	fast: Builder.FastMathKind,
8640	ty: Type,
8641	comptime params_len: usize,
8642	params: [params_len]Builder.Value,
8643	) !Builder.Value {
8644	const o = self.ng.object;
8645	const pt = self.ng.pt;
8646	const zcu = pt.zcu;
8647	const target = zcu.getTarget();
8648	const scalar_ty = ty.scalarType(zcu);
8649	const llvm_ty = try o.lowerType(pt, ty);
8650
8651	if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {
8652	// Some operations are dedicated LLVM instructions, not available as intrinsics
8653	.neg => return self.wip.un(.fneg, params[0], ""),
8654	.add, .sub, .mul, .div, .fmod => return self.wip.bin(switch (fast) {
8655	.normal => switch (op) {
8656	.add => .fadd,
8657	.sub => .fsub,
8658	.mul => .fmul,
8659	.div => .fdiv,
8660	.fmod => .frem,
8661	else => unreachable,
8662	},
8663	.fast => switch (op) {
8664	.add => .@"fadd fast",
8665	.sub => .@"fsub fast",
8666	.mul => .@"fmul fast",
8667	.div => .@"fdiv fast",
8668	.fmod => .@"frem fast",
8669	else => unreachable,
8670	},
8671	}, params[0], params[1], ""),
8672	.fmax,
8673	.fmin,
8674	.ceil,
8675	.cos,
8676	.exp,
8677	.exp2,
8678	.fabs,
8679	.floor,
8680	.log,
8681	.log10,
8682	.log2,
8683	.round,
8684	.sin,
8685	.sqrt,
8686	.trunc,
8687	.fma,
8688	=> return self.wip.callIntrinsic(fast, .none, switch (op) {
8689	.fmax => .maxnum,
8690	.fmin => .minnum,
8691	.ceil => .ceil,
8692	.cos => .cos,
8693	.exp => .exp,
8694	.exp2 => .exp2,
8695	.fabs => .fabs,
8696	.floor => .floor,
8697	.log => .log,
8698	.log10 => .log10,
8699	.log2 => .log2,
8700	.round => .round,
8701	.sin => .sin,
8702	.sqrt => .sqrt,
8703	.trunc => .trunc,
8704	.fma => .fma,
8705	else => unreachable,
8706	}, &.{llvm_ty}, &params, ""),
8707	.tan => unreachable,
8708	};
8709
8710	const float_bits = scalar_ty.floatBits(target);
8711	const fn_name = switch (op) {
8712	.neg => {
8713	// In this case we can generate a softfloat negation by XORing the
8714	// bits with a constant.
8715	const int_ty = try o.builder.intType(@intCast(float_bits));
8716	const cast_ty = try llvm_ty.changeScalar(int_ty, &o.builder);
8717	const sign_mask = try o.builder.splatValue(
8718	cast_ty,
8719	try o.builder.intConst(int_ty, @as(u128, 1) << @intCast(float_bits - 1)),
8720	);
8721	const bitcasted_operand = try self.wip.cast(.bitcast, params[0], cast_ty, "");
8722	const result = try self.wip.bin(.xor, bitcasted_operand, sign_mask, "");
8723	return self.wip.cast(.bitcast, result, llvm_ty, "");
8724	},
8725	.add, .sub, .div, .mul => try o.builder.strtabStringFmt("__{s}{s}f3", .{
8726	@tagName(op), compilerRtFloatAbbrev(float_bits),
8727	}),
8728	.ceil,
8729	.cos,
8730	.exp,
8731	.exp2,
8732	.fabs,
8733	.floor,
8734	.fma,
8735	.fmax,
8736	.fmin,
8737	.fmod,
8738	.log,
8739	.log10,
8740	.log2,
8741	.round,
8742	.sin,
8743	.sqrt,
8744	.tan,
8745	.trunc,
8746	=> try o.builder.strtabStringFmt("{s}{s}{s}", .{
8747	libcFloatPrefix(float_bits), @tagName(op), libcFloatSuffix(float_bits),
8748	}),
8749	};
8750
8751	const scalar_llvm_ty = llvm_ty.scalarType(&o.builder);
8752	const libc_fn = try self.getLibcFunction(
8753	fn_name,
8754	([1]Builder.Type{scalar_llvm_ty} ** 3)[0..params.len],
8755	scalar_llvm_ty,
8756	);
8757	if (ty.zigTypeTag(zcu) == .vector) {
8758	const result = try o.builder.poisonValue(llvm_ty);
8759	return self.buildElementwiseCall(libc_fn, &params, result, ty.vectorLen(zcu));
8760	}
8761
8762	return self.wip.call(
8763	fast.toCallKind(),
8764	.ccc,
8765	.none,
8766	libc_fn.typeOf(&o.builder),
8767	libc_fn.toValue(&o.builder),
8768	&params,
8769	"",
8770	);
8771	}
8772
8773	fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8774	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
8775	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
8776
8777	const mulend1 = try self.resolveInst(extra.lhs);
8778	const mulend2 = try self.resolveInst(extra.rhs);
8779	const addend = try self.resolveInst(pl_op.operand);
8780
8781	const ty = self.typeOfIndex(inst);
8782	return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend });
8783	}
8784
8785	fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8786	const o = self.ng.object;
8787	const pt = self.ng.pt;
8788	const zcu = pt.zcu;
8789	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8790	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
8791
8792	const lhs = try self.resolveInst(extra.lhs);
8793	const rhs = try self.resolveInst(extra.rhs);
8794
8795	const lhs_ty = self.typeOf(extra.lhs);
8796	if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu))
8797	return self.ng.todo("implement vector shifts with scalar rhs", .{});
8798	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
8799
8800	const dest_ty = self.typeOfIndex(inst);
8801	const llvm_dest_ty = try o.lowerType(pt, dest_ty);
8802
8803	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
8804
8805	const result = try self.wip.bin(.shl, lhs, casted_rhs, "");
8806	const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
8807	.ashr
8808	else
8809	.lshr, result, casted_rhs, "");
8810
8811	const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
8812
8813	const result_index = o.llvmFieldIndex(dest_ty, 0).?;
8814	const overflow_index = o.llvmFieldIndex(dest_ty, 1).?;
8815
8816	if (isByRef(dest_ty, zcu)) {
8817	const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();
8818	const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
8819	{
8820	const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, "");
8821	_ = try self.wip.store(.normal, result, field_ptr, result_alignment);
8822	}
8823	{
8824	const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, overflow_index, "");
8825	_ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
8826	}
8827	return alloca_inst;
8828	}
8829
8830	var fields: [2]Builder.Value = undefined;
8831	fields[result_index] = result;
8832	fields[overflow_index] = overflow_bit;
8833	return self.wip.buildAggregate(llvm_dest_ty, &fields, "");
8834	}
8835
8836	fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8837	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8838	const lhs = try self.resolveInst(bin_op.lhs);
8839	const rhs = try self.resolveInst(bin_op.rhs);
8840	return self.wip.bin(.@"and", lhs, rhs, "");
8841	}
8842
8843	fn airOr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8844	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8845	const lhs = try self.resolveInst(bin_op.lhs);
8846	const rhs = try self.resolveInst(bin_op.rhs);
8847	return self.wip.bin(.@"or", lhs, rhs, "");
8848	}
8849
8850	fn airXor(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8851	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8852	const lhs = try self.resolveInst(bin_op.lhs);
8853	const rhs = try self.resolveInst(bin_op.rhs);
8854	return self.wip.bin(.xor, lhs, rhs, "");
8855	}
8856
8857	fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8858	const o = self.ng.object;
8859	const pt = self.ng.pt;
8860	const zcu = pt.zcu;
8861	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8862
8863	const lhs = try self.resolveInst(bin_op.lhs);
8864	const rhs = try self.resolveInst(bin_op.rhs);
8865
8866	const lhs_ty = self.typeOf(bin_op.lhs);
8867	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
8868	return self.ng.todo("implement vector shifts with scalar rhs", .{});
8869	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
8870
8871	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
8872	return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
8873	.@"shl nsw"
8874	else
8875	.@"shl nuw", lhs, casted_rhs, "");
8876	}
8877
8878	fn airShl(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8879	const o = self.ng.object;
8880	const pt = self.ng.pt;
8881	const zcu = pt.zcu;
8882	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8883
8884	const lhs = try self.resolveInst(bin_op.lhs);
8885	const rhs = try self.resolveInst(bin_op.rhs);
8886
8887	const lhs_ty = self.typeOf(bin_op.lhs);
8888	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
8889	return self.ng.todo("implement vector shifts with scalar rhs", .{});
8890
8891	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
8892	return self.wip.bin(.shl, lhs, casted_rhs, "");
8893	}
8894
8895	fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8896	const o = self.ng.object;
8897	const pt = self.ng.pt;
8898	const zcu = pt.zcu;
8899	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8900
8901	const lhs = try self.resolveInst(bin_op.lhs);
8902	const rhs = try self.resolveInst(bin_op.rhs);
8903
8904	const lhs_ty = self.typeOf(bin_op.lhs);
8905	const lhs_info = lhs_ty.intInfo(zcu);
8906	const llvm_lhs_ty = try o.lowerType(pt, lhs_ty);
8907	const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder);
8908
8909	const rhs_ty = self.typeOf(bin_op.rhs);
8910	if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu))
8911	return self.ng.todo("implement vector shifts with scalar rhs", .{});
8912	const rhs_info = rhs_ty.intInfo(zcu);
8913	assert(rhs_info.signedness == .unsigned);
8914	const llvm_rhs_ty = try o.lowerType(pt, rhs_ty);
8915	const llvm_rhs_scalar_ty = llvm_rhs_ty.scalarType(&o.builder);
8916
8917	const result = try self.wip.callIntrinsic(
8918	.normal,
8919	.none,
8920	switch (lhs_info.signedness) {
8921	.signed => .@"sshl.sat",
8922	.unsigned => .@"ushl.sat",
8923	},
8924	&.{llvm_lhs_ty},
8925	&.{ lhs, try self.wip.conv(.unsigned, rhs, llvm_lhs_ty, "") },
8926	"",
8927	);
8928
8929	// LLVM langref says "If b is (statically or dynamically) equal to or
8930	// larger than the integer bit width of the arguments, the result is a
8931	// poison value."
8932	// However Zig semantics says that saturating shift left can never produce
8933	// undefined; instead it saturates.
8934	if (rhs_info.bits <= math.log2_int(u16, lhs_info.bits)) return result;
8935	const bits = try o.builder.splatValue(
8936	llvm_rhs_ty,
8937	try o.builder.intConst(llvm_rhs_scalar_ty, lhs_info.bits),
8938	);
8939	const in_range = try self.wip.icmp(.ult, rhs, bits, "");
8940	const lhs_sat = lhs_sat: switch (lhs_info.signedness) {
8941	.signed => {
8942	const zero = try o.builder.splatValue(
8943	llvm_lhs_ty,
8944	try o.builder.intConst(llvm_lhs_scalar_ty, 0),
8945	);
8946	const smin = try o.builder.splatValue(
8947	llvm_lhs_ty,
8948	try minIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
8949	);
8950	const smax = try o.builder.splatValue(
8951	llvm_lhs_ty,
8952	try maxIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
8953	);
8954	const lhs_lt_zero = try self.wip.icmp(.slt, lhs, zero, "");
8955	const slimit = try self.wip.select(.normal, lhs_lt_zero, smin, smax, "");
8956	const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
8957	break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, slimit, "");
8958	},
8959	.unsigned => {
8960	const zero = try o.builder.splatValue(
8961	llvm_lhs_ty,
8962	try o.builder.intConst(llvm_lhs_scalar_ty, 0),
8963	);
8964	const umax = try o.builder.splatValue(
8965	llvm_lhs_ty,
8966	try o.builder.intConst(llvm_lhs_scalar_ty, -1),
8967	);
8968	const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
8969	break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, umax, "");
8970	},
8971	};
8972	return self.wip.select(.normal, in_range, result, lhs_sat, "");
8973	}
8974
8975	fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value {
8976	const o = self.ng.object;
8977	const pt = self.ng.pt;
8978	const zcu = pt.zcu;
8979	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8980
8981	const lhs = try self.resolveInst(bin_op.lhs);
8982	const rhs = try self.resolveInst(bin_op.rhs);
8983
8984	const lhs_ty = self.typeOf(bin_op.lhs);
8985	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
8986	return self.ng.todo("implement vector shifts with scalar rhs", .{});
8987	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
8988
8989	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
8990	const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);
8991
8992	return self.wip.bin(if (is_exact)
8993	if (is_signed_int) .@"ashr exact" else .@"lshr exact"
8994	else if (is_signed_int) .ashr else .lshr, lhs, casted_rhs, "");
8995	}
8996
8997	fn airAbs(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8998	const o = self.ng.object;
8999	const pt = self.ng.pt;
9000	const zcu = pt.zcu;
9001	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9002	const operand = try self.resolveInst(ty_op.operand);
9003	const operand_ty = self.typeOf(ty_op.operand);
9004	const scalar_ty = operand_ty.scalarType(zcu);
9005
9006	switch (scalar_ty.zigTypeTag(zcu)) {
9007	.int => return self.wip.callIntrinsic(
9008	.normal,
9009	.none,
9010	.abs,
9011	&.{try o.lowerType(pt, operand_ty)},
9012	&.{ operand, try o.builder.intValue(.i1, 0) },
9013	"",
9014	),
9015	.float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),
9016	else => unreachable,
9017	}
9018	}
9019
9020	fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
9021	const o = fg.ng.object;
9022	const pt = fg.ng.pt;
9023	const zcu = pt.zcu;
9024	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9025	const dest_ty = fg.typeOfIndex(inst);
9026	const dest_llvm_ty = try o.lowerType(pt, dest_ty);
9027	const operand = try fg.resolveInst(ty_op.operand);
9028	const operand_ty = fg.typeOf(ty_op.operand);
9029	const operand_info = operand_ty.intInfo(zcu);
9030
9031	const dest_is_enum = dest_ty.zigTypeTag(zcu) == .@"enum";
9032
9033	bounds_check: {
9034	const dest_scalar = dest_ty.scalarType(zcu);
9035	const operand_scalar = operand_ty.scalarType(zcu);
9036
9037	const dest_info = dest_ty.intInfo(zcu);
9038
9039	const have_min_check, const have_max_check = c: {
9040	const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed);
9041	const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed);
9042
9043	const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0;
9044	const operand_maybe_neg = operand_info.signedness == .signed and operand_info.bits > 0;
9045
9046	break :c .{
9047	operand_maybe_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits),
9048	dest_pos_bits < operand_pos_bits,
9049	};
9050	};
9051
9052	if (!have_min_check and !have_max_check) break :bounds_check;
9053
9054	const operand_llvm_ty = try o.lowerType(pt, operand_ty);
9055	const operand_scalar_llvm_ty = try o.lowerType(pt, operand_scalar);
9056
9057	const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
9058	assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));
9059
9060	const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .integer_out_of_bounds;
9061
9062	if (have_min_check) {
9063	const min_const_scalar = try minIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
9064	const min_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, min_const_scalar) else min_const_scalar.toValue();
9065	const ok_maybe_vec = try fg.cmp(.normal, .gte, operand_ty, operand, min_val);
9066	const ok = if (is_vector) ok: {
9067	const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
9068	break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
9069	} else ok_maybe_vec;
9070	if (safety) {
9071	const fail_block = try fg.wip.block(1, "IntMinFail");
9072	const ok_block = try fg.wip.block(1, "IntMinOk");
9073	_ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
9074	fg.wip.cursor = .{ .block = fail_block };
9075	try fg.buildSimplePanic(panic_id);
9076	fg.wip.cursor = .{ .block = ok_block };
9077	} else {
9078	_ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
9079	}
9080	}
9081
9082	if (have_max_check) {
9083	const max_const_scalar = try maxIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
9084	const max_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, max_const_scalar) else max_const_scalar.toValue();
9085	const ok_maybe_vec = try fg.cmp(.normal, .lte, operand_ty, operand, max_val);
9086	const ok = if (is_vector) ok: {
9087	const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
9088	break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
9089	} else ok_maybe_vec;
9090	if (safety) {
9091	const fail_block = try fg.wip.block(1, "IntMaxFail");
9092	const ok_block = try fg.wip.block(1, "IntMaxOk");
9093	_ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
9094	fg.wip.cursor = .{ .block = fail_block };
9095	try fg.buildSimplePanic(panic_id);
9096	fg.wip.cursor = .{ .block = ok_block };
9097	} else {
9098	_ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
9099	}
9100	}
9101	}
9102
9103	const result = try fg.wip.conv(switch (operand_info.signedness) {
9104	.signed => .signed,
9105	.unsigned => .unsigned,
9106	}, operand, dest_llvm_ty, "");
9107
9108	if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) {
9109	const llvm_fn = try fg.getIsNamedEnumValueFunction(dest_ty);
9110	const is_valid_enum_val = try fg.wip.call(
9111	.normal,
9112	.fastcc,
9113	.none,
9114	llvm_fn.typeOf(&o.builder),
9115	llvm_fn.toValue(&o.builder),
9116	&.{result},
9117	"",
9118	);
9119	const fail_block = try fg.wip.block(1, "ValidEnumFail");
9120	const ok_block = try fg.wip.block(1, "ValidEnumOk");
9121	_ = try fg.wip.brCond(is_valid_enum_val, ok_block, fail_block, .none);
9122	fg.wip.cursor = .{ .block = fail_block };
9123	try fg.buildSimplePanic(.invalid_enum_value);
9124	fg.wip.cursor = .{ .block = ok_block };
9125	}
9126
9127	return result;
9128	}
9129
9130	fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9131	const o = self.ng.object;
9132	const pt = self.ng.pt;
9133	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9134	const operand = try self.resolveInst(ty_op.operand);
9135	const dest_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));
9136	return self.wip.cast(.trunc, operand, dest_llvm_ty, "");
9137	}
9138
9139	fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9140	const o = self.ng.object;
9141	const pt = self.ng.pt;
9142	const zcu = pt.zcu;
9143	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9144	const operand = try self.resolveInst(ty_op.operand);
9145	const operand_ty = self.typeOf(ty_op.operand);
9146	const dest_ty = self.typeOfIndex(inst);
9147	const target = zcu.getTarget();
9148
9149	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
9150	return self.wip.cast(.fptrunc, operand, try o.lowerType(pt, dest_ty), "");
9151	} else {
9152	const operand_llvm_ty = try o.lowerType(pt, operand_ty);
9153	const dest_llvm_ty = try o.lowerType(pt, dest_ty);
9154
9155	const dest_bits = dest_ty.floatBits(target);
9156	const src_bits = operand_ty.floatBits(target);
9157	const fn_name = try o.builder.strtabStringFmt("__trunc{s}f{s}f2", .{
9158	compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
9159	});
9160
9161	const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
9162	return self.wip.call(
9163	.normal,
9164	.ccc,
9165	.none,
9166	libc_fn.typeOf(&o.builder),
9167	libc_fn.toValue(&o.builder),
9168	&.{operand},
9169	"",
9170	);
9171	}
9172	}
9173
9174	fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9175	const o = self.ng.object;
9176	const pt = self.ng.pt;
9177	const zcu = pt.zcu;
9178	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9179	const operand = try self.resolveInst(ty_op.operand);
9180	const operand_ty = self.typeOf(ty_op.operand);
9181	const dest_ty = self.typeOfIndex(inst);
9182	const target = zcu.getTarget();
9183
9184	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
9185	return self.wip.cast(.fpext, operand, try o.lowerType(pt, dest_ty), "");
9186	} else {
9187	const operand_llvm_ty = try o.lowerType(pt, operand_ty);
9188	const dest_llvm_ty = try o.lowerType(pt, dest_ty);
9189
9190	const dest_bits = dest_ty.scalarType(zcu).floatBits(target);
9191	const src_bits = operand_ty.scalarType(zcu).floatBits(target);
9192	const fn_name = try o.builder.strtabStringFmt("__extend{s}f{s}f2", .{
9193	compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
9194	});
9195
9196	const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
9197	if (dest_ty.isVector(zcu)) return self.buildElementwiseCall(
9198	libc_fn,
9199	&.{operand},
9200	try o.builder.poisonValue(dest_llvm_ty),
9201	dest_ty.vectorLen(zcu),
9202	);
9203	return self.wip.call(
9204	.normal,
9205	.ccc,
9206	.none,
9207	libc_fn.typeOf(&o.builder),
9208	libc_fn.toValue(&o.builder),
9209	&.{operand},
9210	"",
9211	);
9212	}
9213	}
9214
9215	fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9216	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9217	const operand_ty = self.typeOf(ty_op.operand);
9218	const inst_ty = self.typeOfIndex(inst);
9219	const operand = try self.resolveInst(ty_op.operand);
9220	return self.bitCast(operand, operand_ty, inst_ty);
9221	}
9222
9223	fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) !Builder.Value {
9224	const o = self.ng.object;
9225	const pt = self.ng.pt;
9226	const zcu = pt.zcu;
9227	const operand_is_ref = isByRef(operand_ty, zcu);
9228	const result_is_ref = isByRef(inst_ty, zcu);
9229	const llvm_dest_ty = try o.lowerType(pt, inst_ty);
9230
9231	if (operand_is_ref and result_is_ref) {
9232	// They are both pointers, so just return the same opaque pointer :)
9233	return operand;
9234	}
9235
9236	if (llvm_dest_ty.isInteger(&o.builder) and
9237	operand.typeOfWip(&self.wip).isInteger(&o.builder))
9238	{
9239	return self.wip.conv(.unsigned, operand, llvm_dest_ty, "");
9240	}
9241
9242	const operand_scalar_ty = operand_ty.scalarType(zcu);
9243	const inst_scalar_ty = inst_ty.scalarType(zcu);
9244	if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) {
9245	return self.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
9246	}
9247	if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) {
9248	return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
9249	}
9250
9251	if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
9252	const elem_ty = operand_ty.childType(zcu);
9253	if (!result_is_ref) {
9254	return self.ng.todo("implement bitcast vector to non-ref array", .{});
9255	}
9256	const alignment = inst_ty.abiAlignment(zcu).toLlvm();
9257	const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
9258	const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
9259	if (bitcast_ok) {
9260	_ = try self.wip.store(.normal, operand, array_ptr, alignment);
9261	} else {
9262	// If the ABI size of the element type is not evenly divisible by size in bits;
9263	// a simple bitcast will not work, and we fall back to extractelement.
9264	const llvm_usize = try o.lowerType(pt, Type.usize);
9265	const usize_zero = try o.builder.intValue(llvm_usize, 0);
9266	const vector_len = operand_ty.arrayLen(zcu);
9267	var i: u64 = 0;
9268	while (i < vector_len) : (i += 1) {
9269	const elem_ptr = try self.wip.gep(.inbounds, llvm_dest_ty, array_ptr, &.{
9270	usize_zero, try o.builder.intValue(llvm_usize, i),
9271	}, "");
9272	const elem =
9273	try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
9274	_ = try self.wip.store(.normal, elem, elem_ptr, .default);
9275	}
9276	}
9277	return array_ptr;
9278	} else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) {
9279	const elem_ty = operand_ty.childType(zcu);
9280	const llvm_vector_ty = try o.lowerType(pt, inst_ty);
9281	if (!operand_is_ref) return self.ng.todo("implement bitcast non-ref array to vector", .{});
9282
9283	const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
9284	if (bitcast_ok) {
9285	// The array is aligned to the element's alignment, while the vector might have a completely
9286	// different alignment. This means we need to enforce the alignment of this load.
9287	const alignment = elem_ty.abiAlignment(zcu).toLlvm();
9288	return self.wip.load(.normal, llvm_vector_ty, operand, alignment, "");
9289	} else {
9290	// If the ABI size of the element type is not evenly divisible by size in bits;
9291	// a simple bitcast will not work, and we fall back to extractelement.
9292	const array_llvm_ty = try o.lowerType(pt, operand_ty);
9293	const elem_llvm_ty = try o.lowerType(pt, elem_ty);
9294	const llvm_usize = try o.lowerType(pt, Type.usize);
9295	const usize_zero = try o.builder.intValue(llvm_usize, 0);
9296	const vector_len = operand_ty.arrayLen(zcu);
9297	var vector = try o.builder.poisonValue(llvm_vector_ty);
9298	var i: u64 = 0;
9299	while (i < vector_len) : (i += 1) {
9300	const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, operand, &.{
9301	usize_zero, try o.builder.intValue(llvm_usize, i),
9302	}, "");
9303	const elem = try self.wip.load(.normal, elem_llvm_ty, elem_ptr, .default, "");
9304	vector =
9305	try self.wip.insertElement(vector, elem, try o.builder.intValue(.i32, i), "");
9306	}
9307	return vector;
9308	}
9309	}
9310
9311	if (operand_is_ref) {
9312	const alignment = operand_ty.abiAlignment(zcu).toLlvm();
9313	return self.wip.load(.normal, llvm_dest_ty, operand, alignment, "");
9314	}
9315
9316	if (result_is_ref) {
9317	const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
9318	const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
9319	_ = try self.wip.store(.normal, operand, result_ptr, alignment);
9320	return result_ptr;
9321	}
9322
9323	if (llvm_dest_ty.isStruct(&o.builder) or
9324	((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and
9325	operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu)))
9326	{
9327	// Both our operand and our result are values, not pointers,
9328	// but LLVM won't let us bitcast struct values or vectors with padding bits.
9329	// Therefore, we store operand to alloca, then load for result.
9330	const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
9331	const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
9332	_ = try self.wip.store(.normal, operand, result_ptr, alignment);
9333	return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");
9334	}
9335
9336	return self.wip.cast(.bitcast, operand, llvm_dest_ty, "");
9337	}
9338
9339	fn airArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9340	const o = self.ng.object;
9341	const pt = self.ng.pt;
9342	const zcu = pt.zcu;
9343	const arg_val = self.args[self.arg_index];
9344	self.arg_index += 1;
9345
9346	// llvm does not support debug info for naked function arguments
9347	if (self.is_naked) return arg_val;
9348
9349	const inst_ty = self.typeOfIndex(inst);
9350
9351	const func = zcu.funcInfo(zcu.navValue(self.ng.nav_index).toIntern());
9352	const func_zir = func.zir_body_inst.resolveFull(&zcu.intern_pool).?;
9353	const file = zcu.fileByIndex(func_zir.file);
9354
9355	const mod = file.mod.?;
9356	if (mod.strip) return arg_val;
9357	const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg;
9358	const zir = &file.zir.?;
9359	const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
9360
9361	const lbrace_line = zcu.navSrcLine(func.owner_nav) + func.lbrace_line + 1;
9362	const lbrace_col = func.lbrace_column + 1;
9363
9364	const debug_parameter = try o.builder.debugParameter(
9365	if (name.len > 0) try o.builder.metadataString(name) else null,
9366	self.file,
9367	self.scope,
9368	lbrace_line,
9369	try o.getDebugType(pt, inst_ty),
9370	self.arg_index,
9371	);
9372
9373	const old_location = self.wip.debug_location;
9374	self.wip.debug_location = .{ .location = .{
9375	.line = lbrace_line,
9376	.column = lbrace_col,
9377	.scope = self.scope.toOptional(),
9378	.inlined_at = .none,
9379	} };
9380
9381	if (isByRef(inst_ty, zcu)) {
9382	_ = try self.wip.callIntrinsic(
9383	.normal,
9384	.none,
9385	.@"dbg.declare",
9386	&.{},
9387	&.{
9388	(try self.wip.debugValue(arg_val)).toValue(),
9389	debug_parameter.toValue(),
9390	(try o.builder.debugExpression(&.{})).toValue(),
9391	},
9392	"",
9393	);
9394	} else if (mod.optimize_mode == .Debug) {
9395	const alignment = inst_ty.abiAlignment(zcu).toLlvm();
9396	const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment);
9397	_ = try self.wip.store(.normal, arg_val, alloca, alignment);
9398	_ = try self.wip.callIntrinsic(
9399	.normal,
9400	.none,
9401	.@"dbg.declare",
9402	&.{},
9403	&.{
9404	(try self.wip.debugValue(alloca)).toValue(),
9405	debug_parameter.toValue(),
9406	(try o.builder.debugExpression(&.{})).toValue(),
9407	},
9408	"",
9409	);
9410	} else {
9411	_ = try self.wip.callIntrinsic(
9412	.normal,
9413	.none,
9414	.@"dbg.value",
9415	&.{},
9416	&.{
9417	(try self.wip.debugValue(arg_val)).toValue(),
9418	debug_parameter.toValue(),
9419	(try o.builder.debugExpression(&.{})).toValue(),
9420	},
9421	"",
9422	);
9423	}
9424
9425	self.wip.debug_location = old_location;
9426	return arg_val;
9427	}
9428
9429	fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9430	const o = self.ng.object;
9431	const pt = self.ng.pt;
9432	const zcu = pt.zcu;
9433	const ptr_ty = self.typeOfIndex(inst);
9434	const pointee_type = ptr_ty.childType(zcu);
9435	if (!pointee_type.hasRuntimeBits(zcu))
9436	return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue();
9437
9438	const pointee_llvm_ty = try o.lowerType(pt, pointee_type);
9439	const alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
9440	return self.buildAlloca(pointee_llvm_ty, alignment);
9441	}
9442
9443	fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9444	const o = self.ng.object;
9445	const pt = self.ng.pt;
9446	const zcu = pt.zcu;
9447	const ptr_ty = self.typeOfIndex(inst);
9448	const ret_ty = ptr_ty.childType(zcu);
9449	if (!ret_ty.hasRuntimeBits(zcu))
9450	return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue();
9451	if (self.ret_ptr != .none) return self.ret_ptr;
9452	const ret_llvm_ty = try o.lowerType(pt, ret_ty);
9453	const alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
9454	return self.buildAlloca(ret_llvm_ty, alignment);
9455	}
9456
9457	/// Use this instead of builder.buildAlloca, because this function makes sure to
9458	/// put the alloca instruction at the top of the function!
9459	fn buildAlloca(
9460	self: *FuncGen,
9461	llvm_ty: Builder.Type,
9462	alignment: Builder.Alignment,
9463	) Allocator.Error!Builder.Value {
9464	const target = self.ng.pt.zcu.getTarget();
9465	return buildAllocaInner(&self.wip, llvm_ty, alignment, target);
9466	}
9467
9468	fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
9469	const o = self.ng.object;
9470	const pt = self.ng.pt;
9471	const zcu = pt.zcu;
9472	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9473	const dest_ptr = try self.resolveInst(bin_op.lhs);
9474	const ptr_ty = self.typeOf(bin_op.lhs);
9475	const operand_ty = ptr_ty.childType(zcu);
9476
9477	const val_is_undef = if (try self.air.value(bin_op.rhs, pt)) \|val\| val.isUndef(zcu) else false;
9478	if (val_is_undef) {
9479	const owner_mod = self.ng.ownerModule();
9480
9481	// Even if safety is disabled, we still emit a memset to undefined since it conveys
9482	// extra information to LLVM, and LLVM will optimize it out. Safety makes the difference
9483	// between using 0xaa or actual undefined for the fill byte.
9484	//
9485	// However, for Debug builds specifically, we avoid emitting the memset because LLVM
9486	// will neither use the information nor get rid of the memset, thus leaving an
9487	// unexpected call in the user's code. This is problematic if the code in question is
9488	// not ready to correctly make calls yet, such as in our early PIE startup code, or in
9489	// the early stages of a dynamic linker, etc.
9490	if (!safety and owner_mod.optimize_mode == .Debug) {
9491	return .none;
9492	}
9493
9494	const ptr_info = ptr_ty.ptrInfo(zcu);
9495	const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
9496	if (needs_bitmask) {
9497	// TODO: only some bits are to be undef, we cannot write with a simple memset.
9498	// meanwhile, ignore the write rather than stomping over valid bits.
9499	// https://github.com/ziglang/zig/issues/15337
9500	return .none;
9501	}
9502
9503	self.maybeMarkAllowZeroAccess(ptr_info);
9504
9505	const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), operand_ty.abiSize(zcu));
9506	_ = try self.wip.callMemSet(
9507	dest_ptr,
9508	ptr_ty.ptrAlignment(zcu).toLlvm(),
9509	if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
9510	len,
9511	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
9512	self.disable_intrinsics,
9513	);
9514	if (safety and owner_mod.valgrind) {
9515	try self.valgrindMarkUndef(dest_ptr, len);
9516	}
9517	return .none;
9518	}
9519
9520	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9521
9522	const src_operand = try self.resolveInst(bin_op.rhs);
9523	try self.store(dest_ptr, ptr_ty, src_operand, .none);
9524	return .none;
9525	}
9526
9527	fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9528	const pt = fg.ng.pt;
9529	const zcu = pt.zcu;
9530	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9531	const ptr_ty = fg.typeOf(ty_op.operand);
9532	const ptr_info = ptr_ty.ptrInfo(zcu);
9533	const ptr = try fg.resolveInst(ty_op.operand);
9534	fg.maybeMarkAllowZeroAccess(ptr_info);
9535	return fg.load(ptr, ptr_ty);
9536	}
9537
9538	fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !void {
9539	_ = inst;
9540	const target = self.ng.object.target;
9541	if ((target.cpu.arch == .mips or target.cpu.arch == .mipsel) and
9542	target.cpu.has(.mips, .notraps))
9543	{
9544	// Emit a MIPS `break` instruction followed by an infinite loop (to fulfill the noreturn)
9545	// since this CPU does not support trap instructions.
9546	const o = self.ng.object;
9547	_ = try self.wip.callAsm(
9548	.none,
9549	try o.builder.fnType(.void, &.{}, .normal),
9550	.{ .sideeffect = true },
9551	try o.builder.string("break\n0:\nj 0b\nnop"),
9552	try o.builder.string("~{memory}"),
9553	&.{},
9554	"",
9555	);
9556	} else {
9557	_ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, "");
9558	}
9559	_ = try self.wip.@"unreachable"();
9560	}
9561
9562	fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9563	_ = inst;
9564	_ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, "");
9565	return .none;
9566	}
9567
9568	fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9569	_ = inst;
9570	const o = self.ng.object;
9571	const pt = self.ng.pt;
9572	const llvm_usize = try o.lowerType(pt, Type.usize);
9573	if (!target_util.supportsReturnAddress(self.ng.pt.zcu.getTarget(), self.ng.ownerModule().optimize_mode)) {
9574	// https://github.com/ziglang/zig/issues/11946
9575	return o.builder.intValue(llvm_usize, 0);
9576	}
9577	const result = try self.wip.callIntrinsic(.normal, .none, .returnaddress, &.{}, &.{.@"0"}, "");
9578	return self.wip.cast(.ptrtoint, result, llvm_usize, "");
9579	}
9580
9581	fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9582	_ = inst;
9583	const o = self.ng.object;
9584	const pt = self.ng.pt;
9585	const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");
9586	return self.wip.cast(.ptrtoint, result, try o.lowerType(pt, Type.usize), "");
9587	}
9588
9589	fn airCmpxchg(
9590	self: *FuncGen,
9591	inst: Air.Inst.Index,
9592	kind: Builder.Function.Instruction.CmpXchg.Kind,
9593	) !Builder.Value {
9594	const o = self.ng.object;
9595	const pt = self.ng.pt;
9596	const zcu = pt.zcu;
9597	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
9598	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
9599	const ptr = try self.resolveInst(extra.ptr);
9600	const ptr_ty = self.typeOf(extra.ptr);
9601	var expected_value = try self.resolveInst(extra.expected_value);
9602	var new_value = try self.resolveInst(extra.new_value);
9603	const operand_ty = ptr_ty.childType(zcu);
9604	const llvm_operand_ty = try o.lowerType(pt, operand_ty);
9605	const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, false);
9606	if (llvm_abi_ty != .none) {
9607	// operand needs widening and truncating
9608	const signedness: Builder.Function.Instruction.Cast.Signedness =
9609	if (operand_ty.isSignedInt(zcu)) .signed else .unsigned;
9610	expected_value = try self.wip.conv(signedness, expected_value, llvm_abi_ty, "");
9611	new_value = try self.wip.conv(signedness, new_value, llvm_abi_ty, "");
9612	}
9613
9614	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9615
9616	const result = try self.wip.cmpxchg(
9617	kind,
9618	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
9619	ptr,
9620	expected_value,
9621	new_value,
9622	self.sync_scope,
9623	toLlvmAtomicOrdering(extra.successOrder()),
9624	toLlvmAtomicOrdering(extra.failureOrder()),
9625	ptr_ty.ptrAlignment(zcu).toLlvm(),
9626	"",
9627	);
9628
9629	const optional_ty = self.typeOfIndex(inst);
9630
9631	var payload = try self.wip.extractValue(result, &.{0}, "");
9632	if (llvm_abi_ty != .none) payload = try self.wip.cast(.trunc, payload, llvm_operand_ty, "");
9633	const success_bit = try self.wip.extractValue(result, &.{1}, "");
9634
9635	if (optional_ty.optionalReprIsPayload(zcu)) {
9636	const zero = try o.builder.zeroInitValue(payload.typeOfWip(&self.wip));
9637	return self.wip.select(.normal, success_bit, zero, payload, "");
9638	}
9639
9640	comptime assert(optional_layout_version == 3);
9641
9642	const non_null_bit = try self.wip.not(success_bit, "");
9643	return buildOptional(self, optional_ty, payload, non_null_bit);
9644	}
9645
9646	fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9647	const o = self.ng.object;
9648	const pt = self.ng.pt;
9649	const zcu = pt.zcu;
9650	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
9651	const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
9652	const ptr = try self.resolveInst(pl_op.operand);
9653	const ptr_ty = self.typeOf(pl_op.operand);
9654	const operand_ty = ptr_ty.childType(zcu);
9655	const operand = try self.resolveInst(extra.operand);
9656	const is_signed_int = operand_ty.isSignedInt(zcu);
9657	const is_float = operand_ty.isRuntimeFloat();
9658	const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
9659	const ordering = toLlvmAtomicOrdering(extra.ordering());
9660	const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, op == .xchg);
9661	const llvm_operand_ty = try o.lowerType(pt, operand_ty);
9662
9663	const access_kind: Builder.MemoryAccessKind =
9664	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9665	const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
9666
9667	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9668
9669	if (llvm_abi_ty != .none) {
9670	// operand needs widening and truncating or bitcasting.
9671	return self.wip.cast(if (is_float) .bitcast else .trunc, try self.wip.atomicrmw(
9672	access_kind,
9673	op,
9674	ptr,
9675	try self.wip.cast(
9676	if (is_float) .bitcast else if (is_signed_int) .sext else .zext,
9677	operand,
9678	llvm_abi_ty,
9679	"",
9680	),
9681	self.sync_scope,
9682	ordering,
9683	ptr_alignment,
9684	"",
9685	), llvm_operand_ty, "");
9686	}
9687
9688	if (!llvm_operand_ty.isPointer(&o.builder)) return self.wip.atomicrmw(
9689	access_kind,
9690	op,
9691	ptr,
9692	operand,
9693	self.sync_scope,
9694	ordering,
9695	ptr_alignment,
9696	"",
9697	);
9698
9699	// It's a pointer but we need to treat it as an int.
9700	return self.wip.cast(.inttoptr, try self.wip.atomicrmw(
9701	access_kind,
9702	op,
9703	ptr,
9704	try self.wip.cast(.ptrtoint, operand, try o.lowerType(pt, Type.usize), ""),
9705	self.sync_scope,
9706	ordering,
9707	ptr_alignment,
9708	"",
9709	), llvm_operand_ty, "");
9710	}
9711
9712	fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9713	const o = self.ng.object;
9714	const pt = self.ng.pt;
9715	const zcu = pt.zcu;
9716	const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
9717	const ptr = try self.resolveInst(atomic_load.ptr);
9718	const ptr_ty = self.typeOf(atomic_load.ptr);
9719	const info = ptr_ty.ptrInfo(zcu);
9720	const elem_ty = Type.fromInterned(info.child);
9721	if (!elem_ty.hasRuntimeBits(zcu)) return .none;
9722	const ordering = toLlvmAtomicOrdering(atomic_load.order);
9723	const llvm_abi_ty = try o.getAtomicAbiType(pt, elem_ty, false);
9724	const ptr_alignment = (if (info.flags.alignment != .none)
9725	@as(InternPool.Alignment, info.flags.alignment)
9726	else
9727	Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();
9728	const access_kind: Builder.MemoryAccessKind =
9729	if (info.flags.is_volatile) .@"volatile" else .normal;
9730	const elem_llvm_ty = try o.lowerType(pt, elem_ty);
9731
9732	self.maybeMarkAllowZeroAccess(info);
9733
9734	if (llvm_abi_ty != .none) {
9735	// operand needs widening and truncating
9736	const loaded = try self.wip.loadAtomic(
9737	access_kind,
9738	llvm_abi_ty,
9739	ptr,
9740	self.sync_scope,
9741	ordering,
9742	ptr_alignment,
9743	"",
9744	);
9745	return self.wip.cast(.trunc, loaded, elem_llvm_ty, "");
9746	}
9747	return self.wip.loadAtomic(
9748	access_kind,
9749	elem_llvm_ty,
9750	ptr,
9751	self.sync_scope,
9752	ordering,
9753	ptr_alignment,
9754	"",
9755	);
9756	}
9757
9758	fn airAtomicStore(
9759	self: *FuncGen,
9760	inst: Air.Inst.Index,
9761	ordering: Builder.AtomicOrdering,
9762	) !Builder.Value {
9763	const o = self.ng.object;
9764	const pt = self.ng.pt;
9765	const zcu = pt.zcu;
9766	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9767	const ptr_ty = self.typeOf(bin_op.lhs);
9768	const operand_ty = ptr_ty.childType(zcu);
9769	if (!operand_ty.hasRuntimeBits(zcu)) return .none;
9770	const ptr = try self.resolveInst(bin_op.lhs);
9771	var element = try self.resolveInst(bin_op.rhs);
9772	const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, false);
9773
9774	if (llvm_abi_ty != .none) {
9775	// operand needs widening
9776	element = try self.wip.conv(
9777	if (operand_ty.isSignedInt(zcu)) .signed else .unsigned,
9778	element,
9779	llvm_abi_ty,
9780	"",
9781	);
9782	}
9783
9784	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9785
9786	try self.store(ptr, ptr_ty, element, ordering);
9787	return .none;
9788	}
9789
9790	fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
9791	const o = self.ng.object;
9792	const pt = self.ng.pt;
9793	const zcu = pt.zcu;
9794	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9795	const dest_slice = try self.resolveInst(bin_op.lhs);
9796	const ptr_ty = self.typeOf(bin_op.lhs);
9797	const elem_ty = self.typeOf(bin_op.rhs);
9798	const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();
9799	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);
9800	const access_kind: Builder.MemoryAccessKind =
9801	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9802
9803	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9804
9805	if (try self.air.value(bin_op.rhs, pt)) \|elem_val\| {
9806	if (elem_val.isUndef(zcu)) {
9807	// Even if safety is disabled, we still emit a memset to undefined since it conveys
9808	// extra information to LLVM. However, safety makes the difference between using
9809	// 0xaa or actual undefined for the fill byte.
9810	const fill_byte = if (safety)
9811	try o.builder.intValue(.i8, 0xaa)
9812	else
9813	try o.builder.undefValue(.i8);
9814	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
9815	_ = try self.wip.callMemSet(
9816	dest_ptr,
9817	dest_ptr_align,
9818	fill_byte,
9819	len,
9820	access_kind,
9821	self.disable_intrinsics,
9822	);
9823	const owner_mod = self.ng.ownerModule();
9824	if (safety and owner_mod.valgrind) {
9825	try self.valgrindMarkUndef(dest_ptr, len);
9826	}
9827	return .none;
9828	}
9829
9830	// Test if the element value is compile-time known to be a
9831	// repeating byte pattern, for example, `@as(u64, 0)` has a
9832	// repeating byte pattern of 0 bytes. In such case, the memset
9833	// intrinsic can be used.
9834	if (try elem_val.hasRepeatedByteRepr(pt)) \|byte_val\| {
9835	const fill_byte = try o.builder.intValue(.i8, byte_val);
9836	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
9837	_ = try self.wip.callMemSet(
9838	dest_ptr,
9839	dest_ptr_align,
9840	fill_byte,
9841	len,
9842	access_kind,
9843	self.disable_intrinsics,
9844	);
9845	return .none;
9846	}
9847	}
9848
9849	const value = try self.resolveInst(bin_op.rhs);
9850	const elem_abi_size = elem_ty.abiSize(zcu);
9851
9852	if (elem_abi_size == 1) {
9853	// In this case we can take advantage of LLVM's intrinsic.
9854	const fill_byte = try self.bitCast(value, elem_ty, Type.u8);
9855	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
9856
9857	_ = try self.wip.callMemSet(
9858	dest_ptr,
9859	dest_ptr_align,
9860	fill_byte,
9861	len,
9862	access_kind,
9863	self.disable_intrinsics,
9864	);
9865	return .none;
9866	}
9867
9868	// non-byte-sized element. lower with a loop. something like this:
9869
9870	// entry:
9871	// ...
9872	// %end_ptr = getelementptr %ptr, %len
9873	// br %loop
9874	// loop:
9875	// %it_ptr = phi body %next_ptr, entry %ptr
9876	// %end = cmp eq %it_ptr, %end_ptr
9877	// br %end, %body, %end
9878	// body:
9879	// store %it_ptr, %value
9880	// %next_ptr = getelementptr %it_ptr, 1
9881	// br %loop
9882	// end:
9883	// ...
9884	const entry_block = self.wip.cursor.block;
9885	const loop_block = try self.wip.block(2, "InlineMemsetLoop");
9886	const body_block = try self.wip.block(1, "InlineMemsetBody");
9887	const end_block = try self.wip.block(1, "InlineMemsetEnd");
9888
9889	const llvm_usize_ty = try o.lowerType(pt, Type.usize);
9890	const len = switch (ptr_ty.ptrSize(zcu)) {
9891	.slice => try self.wip.extractValue(dest_slice, &.{1}, ""),
9892	.one => try o.builder.intValue(llvm_usize_ty, ptr_ty.childType(zcu).arrayLen(zcu)),
9893	.many, .c => unreachable,
9894	};
9895	const elem_llvm_ty = try o.lowerType(pt, elem_ty);
9896	const end_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, dest_ptr, &.{len}, "");
9897	_ = try self.wip.br(loop_block);
9898
9899	self.wip.cursor = .{ .block = loop_block };
9900	const it_ptr = try self.wip.phi(.ptr, "");
9901	const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, "");
9902	_ = try self.wip.brCond(end, body_block, end_block, .none);
9903
9904	self.wip.cursor = .{ .block = body_block };
9905	const elem_abi_align = elem_ty.abiAlignment(zcu);
9906	const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm();
9907	if (isByRef(elem_ty, zcu)) {
9908	_ = try self.wip.callMemCpy(
9909	it_ptr.toValue(),
9910	it_ptr_align,
9911	value,
9912	elem_abi_align.toLlvm(),
9913	try o.builder.intValue(llvm_usize_ty, elem_abi_size),
9914	access_kind,
9915	self.disable_intrinsics,
9916	);
9917	} else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
9918	const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{
9919	try o.builder.intValue(llvm_usize_ty, 1),
9920	}, "");
9921	_ = try self.wip.br(loop_block);
9922
9923	self.wip.cursor = .{ .block = end_block };
9924	it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip);
9925	return .none;
9926	}
9927
9928	fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9929	const pt = self.ng.pt;
9930	const zcu = pt.zcu;
9931	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9932	const dest_slice = try self.resolveInst(bin_op.lhs);
9933	const dest_ptr_ty = self.typeOf(bin_op.lhs);
9934	const src_slice = try self.resolveInst(bin_op.rhs);
9935	const src_ptr_ty = self.typeOf(bin_op.rhs);
9936	const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
9937	const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
9938	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
9939	const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
9940	dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9941
9942	self.maybeMarkAllowZeroAccess(dest_ptr_ty.ptrInfo(zcu));
9943	self.maybeMarkAllowZeroAccess(src_ptr_ty.ptrInfo(zcu));
9944
9945	_ = try self.wip.callMemCpy(
9946	dest_ptr,
9947	dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
9948	src_ptr,
9949	src_ptr_ty.ptrAlignment(zcu).toLlvm(),
9950	len,
9951	access_kind,
9952	self.disable_intrinsics,
9953	);
9954	return .none;
9955	}
9956
9957	fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9958	const pt = self.ng.pt;
9959	const zcu = pt.zcu;
9960	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9961	const dest_slice = try self.resolveInst(bin_op.lhs);
9962	const dest_ptr_ty = self.typeOf(bin_op.lhs);
9963	const src_slice = try self.resolveInst(bin_op.rhs);
9964	const src_ptr_ty = self.typeOf(bin_op.rhs);
9965	const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
9966	const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
9967	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
9968	const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
9969	dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9970
9971	_ = try self.wip.callMemMove(
9972	dest_ptr,
9973	dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
9974	src_ptr,
9975	src_ptr_ty.ptrAlignment(zcu).toLlvm(),
9976	len,
9977	access_kind,
9978	);
9979	return .none;
9980	}
9981
9982	fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9983	const o = self.ng.object;
9984	const pt = self.ng.pt;
9985	const zcu = pt.zcu;
9986	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9987	const un_ptr_ty = self.typeOf(bin_op.lhs);
9988	const un_ty = un_ptr_ty.childType(zcu);
9989	const layout = un_ty.unionGetLayout(zcu);
9990	if (layout.tag_size == 0) return .none;
9991
9992	const access_kind: Builder.MemoryAccessKind =
9993	if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9994
9995	self.maybeMarkAllowZeroAccess(un_ptr_ty.ptrInfo(zcu));
9996
9997	const union_ptr = try self.resolveInst(bin_op.lhs);
9998	const new_tag = try self.resolveInst(bin_op.rhs);
9999	const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);
10000	if (layout.payload_size == 0) {
10001	_ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());
10002	return .none;
10003	}
10004	const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10005	const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(pt, un_ty), union_ptr, tag_index, "");
10006	const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) {
10007	0 => union_ptr_align,
10008	else => \|off\| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))),
10009	};
10010	_ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm());
10011	return .none;
10012	}
10013
10014	fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10015	const o = self.ng.object;
10016	const pt = self.ng.pt;
10017	const zcu = pt.zcu;
10018	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10019	const un_ty = self.typeOf(ty_op.operand);
10020	const layout = un_ty.unionGetLayout(zcu);
10021	if (layout.tag_size == 0) return .none;
10022	const union_handle = try self.resolveInst(ty_op.operand);
10023	if (isByRef(un_ty, zcu)) {
10024	const llvm_un_ty = try o.lowerType(pt, un_ty);
10025	if (layout.payload_size == 0)
10026	return self.wip.load(.normal, llvm_un_ty, union_handle, .default, "");
10027	const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10028	const tag_field_ptr = try self.wip.gepStruct(llvm_un_ty, union_handle, tag_index, "");
10029	const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index];
10030	return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");
10031	} else {
10032	if (layout.payload_size == 0) return union_handle;
10033	const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10034	return self.wip.extractValue(union_handle, &.{tag_index}, "");
10035	}
10036	}
10037
10038	fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) !Builder.Value {
10039	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10040	const operand = try self.resolveInst(un_op);
10041	const operand_ty = self.typeOf(un_op);
10042
10043	return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand});
10044	}
10045
10046	fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
10047	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10048	const operand = try self.resolveInst(un_op);
10049	const operand_ty = self.typeOf(un_op);
10050
10051	return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand});
10052	}
10053
10054	fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value {
10055	const o = self.ng.object;
10056	const pt = self.ng.pt;
10057	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10058	const inst_ty = self.typeOfIndex(inst);
10059	const operand_ty = self.typeOf(ty_op.operand);
10060	const operand = try self.resolveInst(ty_op.operand);
10061
10062	const result = try self.wip.callIntrinsic(
10063	.normal,
10064	.none,
10065	intrinsic,
10066	&.{try o.lowerType(pt, operand_ty)},
10067	&.{ operand, .false },
10068	"",
10069	);
10070	return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
10071	}
10072
10073	fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value {
10074	const o = self.ng.object;
10075	const pt = self.ng.pt;
10076	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10077	const inst_ty = self.typeOfIndex(inst);
10078	const operand_ty = self.typeOf(ty_op.operand);
10079	const operand = try self.resolveInst(ty_op.operand);
10080
10081	const result = try self.wip.callIntrinsic(
10082	.normal,
10083	.none,
10084	intrinsic,
10085	&.{try o.lowerType(pt, operand_ty)},
10086	&.{operand},
10087	"",
10088	);
10089	return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
10090	}
10091
10092	fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10093	const o = self.ng.object;
10094	const pt = self.ng.pt;
10095	const zcu = pt.zcu;
10096	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10097	const operand_ty = self.typeOf(ty_op.operand);
10098	var bits = operand_ty.intInfo(zcu).bits;
10099	assert(bits % 8 == 0);
10100
10101	const inst_ty = self.typeOfIndex(inst);
10102	var operand = try self.resolveInst(ty_op.operand);
10103	var llvm_operand_ty = try o.lowerType(pt, operand_ty);
10104
10105	if (bits % 16 == 8) {
10106	// If not an even byte-multiple, we need zero-extend + shift-left 1 byte
10107	// The truncated result at the end will be the correct bswap
10108	const scalar_ty = try o.builder.intType(@intCast(bits + 8));
10109	if (operand_ty.zigTypeTag(zcu) == .vector) {
10110	const vec_len = operand_ty.vectorLen(zcu);
10111	llvm_operand_ty = try o.builder.vectorType(.normal, vec_len, scalar_ty);
10112	} else llvm_operand_ty = scalar_ty;
10113
10114	const shift_amt =
10115	try o.builder.splatValue(llvm_operand_ty, try o.builder.intConst(scalar_ty, 8));
10116	const extended = try self.wip.cast(.zext, operand, llvm_operand_ty, "");
10117	operand = try self.wip.bin(.shl, extended, shift_amt, "");
10118
10119	bits = bits + 8;
10120	}
10121
10122	const result =
10123	try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");
10124	return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
10125	}
10126
10127	fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10128	const o = self.ng.object;
10129	const pt = self.ng.pt;
10130	const zcu = pt.zcu;
10131	const ip = &zcu.intern_pool;
10132	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10133	const operand = try self.resolveInst(ty_op.operand);
10134	const error_set_ty = ty_op.ty.toType();
10135
10136	const names = error_set_ty.errorSetNames(zcu);
10137	const valid_block = try self.wip.block(@intCast(names.len), "Valid");
10138	const invalid_block = try self.wip.block(1, "Invalid");
10139	const end_block = try self.wip.block(2, "End");
10140	var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none);
10141	defer wip_switch.finish(&self.wip);
10142
10143	for (0..names.len) \|name_index\| {
10144	const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
10145	const this_tag_int_value = try o.builder.intConst(try o.errorIntType(pt), err_int);
10146	try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);
10147	}
10148	self.wip.cursor = .{ .block = valid_block };
10149	_ = try self.wip.br(end_block);
10150
10151	self.wip.cursor = .{ .block = invalid_block };
10152	_ = try self.wip.br(end_block);
10153
10154	self.wip.cursor = .{ .block = end_block };
10155	const phi = try self.wip.phi(.i1, "");
10156	phi.finish(&.{ .true, .false }, &.{ valid_block, invalid_block }, &self.wip);
10157	return phi.toValue();
10158	}
10159
10160	fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10161	const o = self.ng.object;
10162	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10163	const operand = try self.resolveInst(un_op);
10164	const enum_ty = self.typeOf(un_op);
10165
10166	const llvm_fn = try self.getIsNamedEnumValueFunction(enum_ty);
10167	return self.wip.call(
10168	.normal,
10169	.fastcc,
10170	.none,
10171	llvm_fn.typeOf(&o.builder),
10172	llvm_fn.toValue(&o.builder),
10173	&.{operand},
10174	"",
10175	);
10176	}
10177
10178	fn getIsNamedEnumValueFunction(self: *FuncGen, enum_ty: Type) !Builder.Function.Index {
10179	const o = self.ng.object;
10180	const pt = self.ng.pt;
10181	const zcu = pt.zcu;
10182	const ip = &zcu.intern_pool;
10183
10184	const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern());
10185	if (gop.found_existing) return gop.value_ptr.*;
10186	errdefer assert(o.named_enum_map.remove(enum_ty.toIntern()));
10187	const function_index = try o.builder.addFunction(
10188	// Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type.
10189	// TODO: change the builder API so we don't need to do this.
10190	try o.builder.fnType(.void, &.{}, .normal),
10191	try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}),
10192	toLlvmAddressSpace(.generic, zcu.getTarget()),
10193	);
10194	gop.value_ptr.* = function_index;
10195	try o.updateIsNamedEnumValueFunction(pt, enum_ty, function_index);
10196	return function_index;
10197	}
10198
10199	fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10200	const o = self.ng.object;
10201	const pt = self.ng.pt;
10202	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10203	const operand = try self.resolveInst(un_op);
10204	const enum_ty = self.typeOf(un_op);
10205
10206	const llvm_fn = try o.getEnumTagNameFunction(pt, enum_ty);
10207	return self.wip.call(
10208	.normal,
10209	.fastcc,
10210	.none,
10211	llvm_fn.typeOf(&o.builder),
10212	llvm_fn.toValue(&o.builder),
10213	&.{operand},
10214	"",
10215	);
10216	}
10217
10218	fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10219	const o = self.ng.object;
10220	const pt = self.ng.pt;
10221	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10222	const operand = try self.resolveInst(un_op);
10223	const slice_ty = self.typeOfIndex(inst);
10224	const slice_llvm_ty = try o.lowerType(pt, slice_ty);
10225
10226	// If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
10227	const extended_operand = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), "");
10228
10229	const error_name_table_ptr = try self.getErrorNameTable();
10230	const error_name_table =
10231	try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, "");
10232	const error_name_ptr =
10233	try self.wip.gep(.inbounds, slice_llvm_ty, error_name_table, &.{extended_operand}, "");
10234	return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");
10235	}
10236
10237	fn airSplat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10238	const o = self.ng.object;
10239	const pt = self.ng.pt;
10240	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10241	const scalar = try self.resolveInst(ty_op.operand);
10242	const vector_ty = self.typeOfIndex(inst);
10243	return self.wip.splatVector(try o.lowerType(pt, vector_ty), scalar, "");
10244	}
10245
10246	fn airSelect(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10247	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
10248	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
10249	const pred = try self.resolveInst(pl_op.operand);
10250	const a = try self.resolveInst(extra.lhs);
10251	const b = try self.resolveInst(extra.rhs);
10252
10253	return self.wip.select(.normal, pred, a, b, "");
10254	}
10255
10256	fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10257	const o = fg.ng.object;
10258	const pt = fg.ng.pt;
10259	const zcu = pt.zcu;
10260	const gpa = zcu.gpa;
10261
10262	const unwrapped = fg.air.unwrapShuffleOne(zcu, inst);
10263
10264	const operand = try fg.resolveInst(unwrapped.operand);
10265	const mask = unwrapped.mask;
10266	const operand_ty = fg.typeOf(unwrapped.operand);
10267	const llvm_operand_ty = try o.lowerType(pt, operand_ty);
10268	const llvm_result_ty = try o.lowerType(pt, unwrapped.result_ty);
10269	const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu));
10270	const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
10271	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
10272	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
10273
10274	// LLVM requires that the two input vectors have the same length, so lowering isn't trivial.
10275	// And, in the words of jacobly0: "llvm sucks at shuffles so we do have to hold its hand at
10276	// least a bit". So, there are two cases here.
10277	//
10278	// If the operand length equals the mask length, we do just the one `shufflevector`, where
10279	// the second operand is a constant vector with comptime-known elements at the right indices
10280	// and poison values elsewhere (in the indices which won't be selected).
10281	//
10282	// Otherwise, we lower to two `shufflevector` instructions. The first shuffles the runtime
10283	// operand with an all-poison vector to extract and correctly position all of the runtime
10284	// elements. We also make a constant vector with all of the comptime elements correctly
10285	// positioned. Then, our second instruction selects elements from those "runtime-or-poison"
10286	// and "comptime-or-poison" vectors to compute the result.
10287
10288	// This buffer is used primarily for the mask constants.
10289	const llvm_elem_buf = try gpa.alloc(Builder.Constant, mask.len);
10290	defer gpa.free(llvm_elem_buf);
10291
10292	// ...but first, we'll collect all of the comptime-known values.
10293	var any_defined_comptime_value = false;
10294	for (mask, llvm_elem_buf) \|mask_elem, *llvm_elem\| {
10295	llvm_elem.* = switch (mask_elem.unwrap()) {
10296	.elem => llvm_poison_elem,
10297	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
10298	any_defined_comptime_value = true;
10299	break :elem try o.lowerValue(pt, val);
10300	} else llvm_poison_elem,
10301	};
10302	}
10303	// This vector is like the result, but runtime elements are replaced with poison.
10304	const comptime_and_poison: Builder.Value = if (any_defined_comptime_value) vec: {
10305	break :vec try o.builder.vectorValue(llvm_result_ty, llvm_elem_buf);
10306	} else try o.builder.poisonValue(llvm_result_ty);
10307
10308	if (operand_ty.vectorLen(zcu) == mask.len) {
10309	// input length equals mask/output length, so we lower to one instruction
10310	for (mask, llvm_elem_buf, 0..) \|mask_elem, *llvm_elem, elem_idx\| {
10311	llvm_elem.* = switch (mask_elem.unwrap()) {
10312	.elem => \|idx\| try o.builder.intConst(.i32, idx),
10313	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
10314	break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
10315	} else llvm_poison_mask_elem,
10316	};
10317	}
10318	return fg.wip.shuffleVector(
10319	operand,
10320	comptime_and_poison,
10321	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10322	"",
10323	);
10324	}
10325
10326	for (mask, llvm_elem_buf) \|mask_elem, *llvm_elem\| {
10327	llvm_elem.* = switch (mask_elem.unwrap()) {
10328	.elem => \|idx\| try o.builder.intConst(.i32, idx),
10329	.value => llvm_poison_mask_elem,
10330	};
10331	}
10332	// This vector is like our result, but all comptime-known elements are poison.
10333	const runtime_and_poison = try fg.wip.shuffleVector(
10334	operand,
10335	try o.builder.poisonValue(llvm_operand_ty),
10336	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10337	"",
10338	);
10339
10340	if (!any_defined_comptime_value) {
10341	// `comptime_and_poison` is just poison; a second shuffle would be a nop.
10342	return runtime_and_poison;
10343	}
10344
10345	// In this second shuffle, the inputs, the mask, and the output all have the same length.
10346	for (mask, llvm_elem_buf, 0..) \|mask_elem, *llvm_elem, elem_idx\| {
10347	llvm_elem.* = switch (mask_elem.unwrap()) {
10348	.elem => try o.builder.intConst(.i32, elem_idx),
10349	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
10350	break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
10351	} else llvm_poison_mask_elem,
10352	};
10353	}
10354	// Merge the runtime and comptime elements with the mask we just built.
10355	return fg.wip.shuffleVector(
10356	runtime_and_poison,
10357	comptime_and_poison,
10358	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10359	"",
10360	);
10361	}
10362
10363	fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10364	const o = fg.ng.object;
10365	const pt = fg.ng.pt;
10366	const zcu = pt.zcu;
10367	const gpa = zcu.gpa;
10368
10369	const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
10370
10371	const mask = unwrapped.mask;
10372	const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu));
10373	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
10374	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
10375
10376	// This is kind of simpler than in `airShuffleOne`. We extend the shorter vector to the
10377	// length of the longer one with an initial `shufflevector` if necessary, and then do the
10378	// actual computation with a second `shufflevector`.
10379
10380	const operand_a_len = fg.typeOf(unwrapped.operand_a).vectorLen(zcu);
10381	const operand_b_len = fg.typeOf(unwrapped.operand_b).vectorLen(zcu);
10382	const operand_len: u32 = @max(operand_a_len, operand_b_len);
10383
10384	// If we need to extend an operand, this is the type that mask will have.
10385	const llvm_operand_mask_ty = try o.builder.vectorType(.normal, operand_len, .i32);
10386
10387	const llvm_elem_buf = try gpa.alloc(Builder.Constant, @max(mask.len, operand_len));
10388	defer gpa.free(llvm_elem_buf);
10389
10390	const operand_a: Builder.Value = extend: {
10391	const raw = try fg.resolveInst(unwrapped.operand_a);
10392	if (operand_a_len == operand_len) break :extend raw;
10393	// Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
10394	const mask_elems = llvm_elem_buf[0..operand_len];
10395	for (mask_elems[0..operand_a_len], 0..) \|*llvm_elem, elem_idx\| {
10396	llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
10397	}
10398	@memset(mask_elems[operand_a_len..], llvm_poison_mask_elem);
10399	const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_a_len, llvm_elem_ty);
10400	break :extend try fg.wip.shuffleVector(
10401	raw,
10402	try o.builder.poisonValue(llvm_this_operand_ty),
10403	try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
10404	"",
10405	);
10406	};
10407	const operand_b: Builder.Value = extend: {
10408	const raw = try fg.resolveInst(unwrapped.operand_b);
10409	if (operand_b_len == operand_len) break :extend raw;
10410	// Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
10411	const mask_elems = llvm_elem_buf[0..operand_len];
10412	for (mask_elems[0..operand_b_len], 0..) \|*llvm_elem, elem_idx\| {
10413	llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
10414	}
10415	@memset(mask_elems[operand_b_len..], llvm_poison_mask_elem);
10416	const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_b_len, llvm_elem_ty);
10417	break :extend try fg.wip.shuffleVector(
10418	raw,
10419	try o.builder.poisonValue(llvm_this_operand_ty),
10420	try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
10421	"",
10422	);
10423	};
10424
10425	// `operand_a` and `operand_b` now have the same length (we've extended the shorter one with
10426	// an initial shuffle if necessary). Now for the easy bit.
10427
10428	const mask_elems = llvm_elem_buf[0..mask.len];
10429	for (mask, mask_elems) \|mask_elem, *llvm_mask_elem\| {
10430	llvm_mask_elem.* = switch (mask_elem.unwrap()) {
10431	.a_elem => \|idx\| try o.builder.intConst(.i32, idx),
10432	.b_elem => \|idx\| try o.builder.intConst(.i32, operand_len + idx),
10433	.undef => llvm_poison_mask_elem,
10434	};
10435	}
10436	return fg.wip.shuffleVector(
10437	operand_a,
10438	operand_b,
10439	try o.builder.vectorValue(llvm_mask_ty, mask_elems),
10440	"",
10441	);
10442	}
10443
10444	/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.
10445	///
10446	/// Equivalent to:
10447	/// reduce: {
10448	/// var i: usize = 0;
10449	/// var accum: T = init;
10450	/// while (i < vec.len) : (i += 1) {
10451	/// accum = llvm_fn(accum, vec[i]);
10452	/// }
10453	/// break :reduce accum;
10454	/// }
10455	///
10456	fn buildReducedCall(
10457	self: *FuncGen,
10458	llvm_fn: Builder.Function.Index,
10459	operand_vector: Builder.Value,
10460	vector_len: usize,
10461	accum_init: Builder.Value,
10462	) !Builder.Value {
10463	const o = self.ng.object;
10464	const pt = self.ng.pt;
10465	const usize_ty = try o.lowerType(pt, Type.usize);
10466	const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len);
10467	const llvm_result_ty = accum_init.typeOfWip(&self.wip);
10468
10469	// Allocate and initialize our mutable variables
10470	const i_ptr = try self.buildAlloca(usize_ty, .default);
10471	_ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
10472	const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
10473	_ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
10474
10475	// Setup the loop
10476	const loop = try self.wip.block(2, "ReduceLoop");
10477	const loop_exit = try self.wip.block(1, "AfterReduce");
10478	_ = try self.wip.br(loop);
10479	{
10480	self.wip.cursor = .{ .block = loop };
10481
10482	// while (i < vec.len)
10483	const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, "");
10484	const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");
10485	const loop_then = try self.wip.block(1, "ReduceLoopThen");
10486
10487	_ = try self.wip.brCond(cond, loop_then, loop_exit, .none);
10488
10489	{
10490	self.wip.cursor = .{ .block = loop_then };
10491
10492	// accum = f(accum, vec[i]);
10493	const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
10494	const element = try self.wip.extractElement(operand_vector, i, "");
10495	const new_accum = try self.wip.call(
10496	.normal,
10497	.ccc,
10498	.none,
10499	llvm_fn.typeOf(&o.builder),
10500	llvm_fn.toValue(&o.builder),
10501	&.{ accum, element },
10502	"",
10503	);
10504	_ = try self.wip.store(.normal, new_accum, accum_ptr, .default);
10505
10506	// i += 1
10507	const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), "");
10508	_ = try self.wip.store(.normal, new_i, i_ptr, .default);
10509	_ = try self.wip.br(loop);
10510	}
10511	}
10512
10513	self.wip.cursor = .{ .block = loop_exit };
10514	return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
10515	}
10516
10517	fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
10518	const o = self.ng.object;
10519	const pt = self.ng.pt;
10520	const zcu = pt.zcu;
10521	const target = zcu.getTarget();
10522
10523	const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
10524	const operand = try self.resolveInst(reduce.operand);
10525	const operand_ty = self.typeOf(reduce.operand);
10526	const llvm_operand_ty = try o.lowerType(pt, operand_ty);
10527	const scalar_ty = self.typeOfIndex(inst);
10528	const llvm_scalar_ty = try o.lowerType(pt, scalar_ty);
10529
10530	switch (reduce.operation) {
10531	.And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
10532	.And => .@"vector.reduce.and",
10533	.Or => .@"vector.reduce.or",
10534	.Xor => .@"vector.reduce.xor",
10535	else => unreachable,
10536	}, &.{llvm_operand_ty}, &.{operand}, ""),
10537	.Min, .Max => switch (scalar_ty.zigTypeTag(zcu)) {
10538	.int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
10539	.Min => if (scalar_ty.isSignedInt(zcu))
10540	.@"vector.reduce.smin"
10541	else
10542	.@"vector.reduce.umin",
10543	.Max => if (scalar_ty.isSignedInt(zcu))
10544	.@"vector.reduce.smax"
10545	else
10546	.@"vector.reduce.umax",
10547	else => unreachable,
10548	}, &.{llvm_operand_ty}, &.{operand}, ""),
10549	.float => if (intrinsicsAllowed(scalar_ty, target))
10550	return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
10551	.Min => .@"vector.reduce.fmin",
10552	.Max => .@"vector.reduce.fmax",
10553	else => unreachable,
10554	}, &.{llvm_operand_ty}, &.{operand}, ""),
10555	else => unreachable,
10556	},
10557	.Add, .Mul => switch (scalar_ty.zigTypeTag(zcu)) {
10558	.int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
10559	.Add => .@"vector.reduce.add",
10560	.Mul => .@"vector.reduce.mul",
10561	else => unreachable,
10562	}, &.{llvm_operand_ty}, &.{operand}, ""),
10563	.float => if (intrinsicsAllowed(scalar_ty, target))
10564	return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
10565	.Add => .@"vector.reduce.fadd",
10566	.Mul => .@"vector.reduce.fmul",
10567	else => unreachable,
10568	}, &.{llvm_operand_ty}, &.{ switch (reduce.operation) {
10569	.Add => try o.builder.fpValue(llvm_scalar_ty, -0.0),
10570	.Mul => try o.builder.fpValue(llvm_scalar_ty, 1.0),
10571	else => unreachable,
10572	}, operand }, ""),
10573	else => unreachable,
10574	},
10575	}
10576
10577	// Reduction could not be performed with intrinsics.
10578	// Use a manual loop over a softfloat call instead.
10579	const float_bits = scalar_ty.floatBits(target);
10580	const fn_name = switch (reduce.operation) {
10581	.Min => try o.builder.strtabStringFmt("{s}fmin{s}", .{
10582	libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
10583	}),
10584	.Max => try o.builder.strtabStringFmt("{s}fmax{s}", .{
10585	libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
10586	}),
10587	.Add => try o.builder.strtabStringFmt("__add{s}f3", .{
10588	compilerRtFloatAbbrev(float_bits),
10589	}),
10590	.Mul => try o.builder.strtabStringFmt("__mul{s}f3", .{
10591	compilerRtFloatAbbrev(float_bits),
10592	}),
10593	else => unreachable,
10594	};
10595
10596	const libc_fn =
10597	try self.getLibcFunction(fn_name, &.{ llvm_scalar_ty, llvm_scalar_ty }, llvm_scalar_ty);
10598	const init_val = switch (llvm_scalar_ty) {
10599	.i16 => try o.builder.intValue(.i16, @as(i16, @bitCast(
10600	@as(f16, switch (reduce.operation) {
10601	.Min, .Max => std.math.nan(f16),
10602	.Add => -0.0,
10603	.Mul => 1.0,
10604	else => unreachable,
10605	}),
10606	))),
10607	.i80 => try o.builder.intValue(.i80, @as(i80, @bitCast(
10608	@as(f80, switch (reduce.operation) {
10609	.Min, .Max => std.math.nan(f80),
10610	.Add => -0.0,
10611	.Mul => 1.0,
10612	else => unreachable,
10613	}),
10614	))),
10615	.i128 => try o.builder.intValue(.i128, @as(i128, @bitCast(
10616	@as(f128, switch (reduce.operation) {
10617	.Min, .Max => std.math.nan(f128),
10618	.Add => -0.0,
10619	.Mul => 1.0,
10620	else => unreachable,
10621	}),
10622	))),
10623	else => unreachable,
10624	};
10625	return self.buildReducedCall(libc_fn, operand, operand_ty.vectorLen(zcu), init_val);
10626	}
10627
10628	fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10629	const o = self.ng.object;
10630	const pt = self.ng.pt;
10631	const zcu = pt.zcu;
10632	const ip = &zcu.intern_pool;
10633	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
10634	const result_ty = self.typeOfIndex(inst);
10635	const len: usize = @intCast(result_ty.arrayLen(zcu));
10636	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
10637	const llvm_result_ty = try o.lowerType(pt, result_ty);
10638
10639	switch (result_ty.zigTypeTag(zcu)) {
10640	.vector => {
10641	var vector = try o.builder.poisonValue(llvm_result_ty);
10642	for (elements, 0..) \|elem, i\| {
10643	const index_u32 = try o.builder.intValue(.i32, i);
10644	const llvm_elem = try self.resolveInst(elem);
10645	vector = try self.wip.insertElement(vector, llvm_elem, index_u32, "");
10646	}
10647	return vector;
10648	},
10649	.@"struct" => {
10650	if (zcu.typeToPackedStruct(result_ty)) \|struct_type\| {
10651	const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type);
10652	const big_bits = backing_int_ty.bitSize(zcu);
10653	const int_ty = try o.builder.intType(@intCast(big_bits));
10654	comptime assert(Type.packed_struct_layout_version == 2);
10655	var running_int = try o.builder.intValue(int_ty, 0);
10656	var running_bits: u16 = 0;
10657	for (elements, struct_type.field_types.get(ip)) \|elem, field_ty\| {
10658	if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
10659
10660	const non_int_val = try self.resolveInst(elem);
10661	const ty_bit_size: u16 = @intCast(Type.fromInterned(field_ty).bitSize(zcu));
10662	const small_int_ty = try o.builder.intType(ty_bit_size);
10663	const small_int_val = if (Type.fromInterned(field_ty).isPtrAtRuntime(zcu))
10664	try self.wip.cast(.ptrtoint, non_int_val, small_int_ty, "")
10665	else
10666	try self.wip.cast(.bitcast, non_int_val, small_int_ty, "");
10667	const shift_rhs = try o.builder.intValue(int_ty, running_bits);
10668	const extended_int_val =
10669	try self.wip.conv(.unsigned, small_int_val, int_ty, "");
10670	const shifted = try self.wip.bin(.shl, extended_int_val, shift_rhs, "");
10671	running_int = try self.wip.bin(.@"or", running_int, shifted, "");
10672	running_bits += ty_bit_size;
10673	}
10674	return running_int;
10675	}
10676
10677	assert(result_ty.containerLayout(zcu) != .@"packed");
10678
10679	if (isByRef(result_ty, zcu)) {
10680	// TODO in debug builds init to undef so that the padding will be 0xaa
10681	// even if we fully populate the fields.
10682	const alignment = result_ty.abiAlignment(zcu).toLlvm();
10683	const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
10684
10685	for (elements, 0..) \|elem, i\| {
10686	if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
10687
10688	const llvm_elem = try self.resolveInst(elem);
10689	const llvm_i = o.llvmFieldIndex(result_ty, i).?;
10690	const field_ptr = try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, "");
10691
10692	const field_ptr_ty = try pt.ptrType(.{
10693	.child = self.typeOf(elem).toIntern(),
10694	.flags = .{
10695	.alignment = result_ty.explicitFieldAlignment(i, zcu),
10696	},
10697	});
10698	try self.store(field_ptr, field_ptr_ty, llvm_elem, .none);
10699	}
10700
10701	return alloca_inst;
10702	} else {
10703	var result = try o.builder.poisonValue(llvm_result_ty);
10704	for (elements, 0..) \|elem, i\| {
10705	if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
10706
10707	const llvm_elem = try self.resolveInst(elem);
10708	const llvm_i = o.llvmFieldIndex(result_ty, i).?;
10709	result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, "");
10710	}
10711	return result;
10712	}
10713	},
10714	.array => {
10715	assert(isByRef(result_ty, zcu));
10716
10717	const llvm_usize = try o.lowerType(pt, Type.usize);
10718	const usize_zero = try o.builder.intValue(llvm_usize, 0);
10719	const alignment = result_ty.abiAlignment(zcu).toLlvm();
10720	const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
10721
10722	const array_info = result_ty.arrayInfo(zcu);
10723	const elem_ptr_ty = try pt.ptrType(.{
10724	.child = array_info.elem_type.toIntern(),
10725	});
10726
10727	for (elements, 0..) \|elem, i\| {
10728	const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{
10729	usize_zero, try o.builder.intValue(llvm_usize, i),
10730	}, "");
10731	const llvm_elem = try self.resolveInst(elem);
10732	try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none);
10733	}
10734	if (array_info.sentinel) \|sent_val\| {
10735	const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{
10736	usize_zero, try o.builder.intValue(llvm_usize, array_info.len),
10737	}, "");
10738	const llvm_elem = try self.resolveValue(sent_val);
10739	try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none);
10740	}
10741
10742	return alloca_inst;
10743	},
10744	else => unreachable,
10745	}
10746	}
10747
10748	fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10749	const o = self.ng.object;
10750	const pt = self.ng.pt;
10751	const zcu = pt.zcu;
10752	const ip = &zcu.intern_pool;
10753	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
10754	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
10755	const union_ty = self.typeOfIndex(inst);
10756	const union_llvm_ty = try o.lowerType(pt, union_ty);
10757	const union_obj = zcu.typeToUnion(union_ty).?;
10758
10759	assert(union_obj.layout != .@"packed");
10760
10761	const layout = Type.getUnionLayout(union_obj, zcu);
10762
10763	const tag_int_val = blk: {
10764	const tag_ty = union_ty.unionTagTypeHypothetical(zcu);
10765	const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index);
10766	break :blk tag_val.intFromEnum(zcu);
10767	};
10768	if (layout.payload_size == 0) {
10769	if (layout.tag_size == 0) {
10770	return .none;
10771	}
10772	assert(!isByRef(union_ty, zcu));
10773	var big_int_space: Value.BigIntSpace = undefined;
10774	const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
10775	return try o.builder.bigIntValue(union_llvm_ty, tag_big_int);
10776	}
10777	assert(isByRef(union_ty, zcu));
10778	// The llvm type of the alloca will be the named LLVM union type, and will not
10779	// necessarily match the format that we need, depending on which tag is active.
10780	// We must construct the correct unnamed struct type here, in order to then set
10781	// the fields appropriately.
10782	const alignment = layout.abi_align.toLlvm();
10783	const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
10784	const llvm_payload = try self.resolveInst(extra.init);
10785	const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
10786	const field_llvm_ty = try o.lowerType(pt, field_ty);
10787	const field_size = field_ty.abiSize(zcu);
10788	const field_align = union_ty.explicitFieldAlignment(extra.field_index, zcu);
10789	const llvm_usize = try o.lowerType(pt, Type.usize);
10790	const usize_zero = try o.builder.intValue(llvm_usize, 0);
10791
10792	assert(field_ty.hasRuntimeBits(zcu));
10793
10794	const llvm_union_ty = t: {
10795	const payload_ty = p: {
10796	if (field_size == layout.payload_size) {
10797	break :p field_llvm_ty;
10798	}
10799	const padding_len = layout.payload_size - field_size;
10800	break :p try o.builder.structType(.@"packed", &.{
10801	field_llvm_ty, try o.builder.arrayType(padding_len, .i8),
10802	});
10803	};
10804	if (layout.tag_size == 0) break :t try o.builder.structType(.normal, &.{payload_ty});
10805	const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
10806	var fields: [3]Builder.Type = undefined;
10807	var fields_len: usize = 2;
10808	if (layout.tag_align.compare(.gte, layout.payload_align)) {
10809	fields = .{ tag_ty, payload_ty, undefined };
10810	} else {
10811	fields = .{ payload_ty, tag_ty, undefined };
10812	}
10813	if (layout.padding != 0) {
10814	fields[fields_len] = try o.builder.arrayType(layout.padding, .i8);
10815	fields_len += 1;
10816	}
10817	break :t try o.builder.structType(.normal, fields[0..fields_len]);
10818	};
10819
10820	// Now we follow the layout as expressed above with GEP instructions to set the
10821	// tag and the payload.
10822	const field_ptr_ty = try pt.ptrType(.{
10823	.child = field_ty.toIntern(),
10824	.flags = .{ .alignment = field_align },
10825	});
10826	if (layout.tag_size == 0) {
10827	const indices = [3]Builder.Value{ usize_zero, .@"0", .@"0" };
10828	const len: usize = if (field_size == layout.payload_size) 2 else 3;
10829	const field_ptr =
10830	try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
10831	try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
10832	return result_ptr;
10833	}
10834
10835	{
10836	const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align));
10837	const indices: [3]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, payload_index), .@"0" };
10838	const len: usize = if (field_size == layout.payload_size) 2 else 3;
10839	const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
10840	try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
10841	}
10842	{
10843	const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10844	const indices: [2]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, tag_index) };
10845	const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, &indices, "");
10846	const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
10847	var big_int_space: Value.BigIntSpace = undefined;
10848	const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
10849	const llvm_tag = try o.builder.bigIntValue(tag_ty, tag_big_int);
10850	const tag_alignment = Type.fromInterned(union_obj.enum_tag_type).abiAlignment(zcu).toLlvm();
10851	_ = try self.wip.store(.normal, llvm_tag, field_ptr, tag_alignment);
10852	}
10853
10854	return result_ptr;
10855	}
10856
10857	fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10858	const o = self.ng.object;
10859	const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
10860
10861	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.read) == 0);
10862	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.write) == 1);
10863
10864	comptime assert(prefetch.locality >= 0);
10865	comptime assert(prefetch.locality <= 3);
10866
10867	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.instruction) == 0);
10868	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.data) == 1);
10869
10870	// LLVM fails during codegen of instruction cache prefetchs for these architectures.
10871	// This is an LLVM bug as the prefetch intrinsic should be a noop if not supported
10872	// by the target.
10873	// To work around this, don't emit llvm.prefetch in this case.
10874	// See https://bugs.llvm.org/show_bug.cgi?id=21037
10875	const zcu = self.ng.pt.zcu;
10876	const target = zcu.getTarget();
10877	switch (prefetch.cache) {
10878	.instruction => switch (target.cpu.arch) {
10879	.x86_64,
10880	.x86,
10881	.powerpc,
10882	.powerpcle,
10883	.powerpc64,
10884	.powerpc64le,
10885	=> return .none,
10886	.arm, .armeb, .thumb, .thumbeb => {
10887	switch (prefetch.rw) {
10888	.write => return .none,
10889	else => {},
10890	}
10891	},
10892	else => {},
10893	},
10894	.data => {},
10895	}
10896
10897	_ = try self.wip.callIntrinsic(.normal, .none, .prefetch, &.{.ptr}, &.{
10898	try self.sliceOrArrayPtr(try self.resolveInst(prefetch.ptr), self.typeOf(prefetch.ptr)),
10899	try o.builder.intValue(.i32, prefetch.rw),
10900	try o.builder.intValue(.i32, prefetch.locality),
10901	try o.builder.intValue(.i32, prefetch.cache),
10902	}, "");
10903	return .none;
10904	}
10905
10906	fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10907	const o = self.ng.object;
10908	const pt = self.ng.pt;
10909	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10910	const inst_ty = self.typeOfIndex(inst);
10911	const operand = try self.resolveInst(ty_op.operand);
10912
10913	return self.wip.cast(.addrspacecast, operand, try o.lowerType(pt, inst_ty), "");
10914	}
10915
10916	fn workIntrinsic(
10917	self: *FuncGen,
10918	dimension: u32,
10919	default: u32,
10920	comptime basename: []const u8,
10921	) !Builder.Value {
10922	return self.wip.callIntrinsic(.normal, .none, switch (dimension) {
10923	0 => @field(Builder.Intrinsic, basename ++ ".x"),
10924	1 => @field(Builder.Intrinsic, basename ++ ".y"),
10925	2 => @field(Builder.Intrinsic, basename ++ ".z"),
10926	else => return self.ng.object.builder.intValue(.i32, default),
10927	}, &.{}, &.{}, "");
10928	}
10929
10930	fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10931	const target = self.ng.pt.zcu.getTarget();
10932
10933	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
10934	const dimension = pl_op.payload;
10935
10936	return switch (target.cpu.arch) {
10937	.amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workitem.id"),
10938	.nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.tid"),
10939	else => unreachable,
10940	};
10941	}
10942
10943	fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10944	const o = self.ng.object;
10945	const pt = self.ng.pt;
10946	const target = pt.zcu.getTarget();
10947
10948	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
10949	const dimension = pl_op.payload;
10950
10951	switch (target.cpu.arch) {
10952	.amdgcn => {
10953	if (dimension >= 3) return .@"1";
10954
10955	// Fetch the dispatch pointer, which points to this structure:
10956	// https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/adae6c61e10d371f7cbc3d0e94ae2c070cab18a4/src/inc/hsa.h#L2913
10957	const dispatch_ptr =
10958	try self.wip.callIntrinsic(.normal, .none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, "");
10959
10960	// Load the work_group_* member from the struct as u16.
10961	// Just treat the dispatch pointer as an array of u16 to keep things simple.
10962	const workgroup_size_ptr = try self.wip.gep(.inbounds, .i16, dispatch_ptr, &.{
10963	try o.builder.intValue(try o.lowerType(pt, Type.usize), 2 + dimension),
10964	}, "");
10965	const workgroup_size_alignment = comptime Builder.Alignment.fromByteUnits(2);
10966	return self.wip.load(.normal, .i16, workgroup_size_ptr, workgroup_size_alignment, "");
10967	},
10968	.nvptx, .nvptx64 => {
10969	return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
10970	},
10971	else => unreachable,
10972	}
10973	}
10974
10975	fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10976	const target = self.ng.pt.zcu.getTarget();
10977
10978	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
10979	const dimension = pl_op.payload;
10980
10981	return switch (target.cpu.arch) {
10982	.amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workgroup.id"),
10983	.nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.ctaid"),
10984	else => unreachable,
10985	};
10986	}
10987
10988	fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index {
10989	const o = self.ng.object;
10990	const pt = self.ng.pt;
10991
10992	const table = o.error_name_table;
10993	if (table != .none) return table;
10994
10995	// TODO: Address space
10996	const variable_index =
10997	try o.builder.addVariable(try o.builder.strtabString("__zig_err_name_table"), .ptr, .default);
10998	variable_index.setLinkage(.private, &o.builder);
10999	variable_index.setMutability(.constant, &o.builder);
11000	variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
11001	variable_index.setAlignment(
11002	Type.slice_const_u8_sentinel_0.abiAlignment(pt.zcu).toLlvm(),
11003	&o.builder,
11004	);
11005
11006	o.error_name_table = variable_index;
11007	return variable_index;
11008	}
11009
11010	/// Assumes the optional is not pointer-like and payload has bits.
11011	fn optCmpNull(
11012	self: *FuncGen,
11013	cond: Builder.IntegerCondition,
11014	opt_llvm_ty: Builder.Type,
11015	opt_handle: Builder.Value,
11016	is_by_ref: bool,
11017	access_kind: Builder.MemoryAccessKind,
11018	) Allocator.Error!Builder.Value {
11019	const o = self.ng.object;
11020	const field = b: {
11021	if (is_by_ref) {
11022	const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, "");
11023	break :b try self.wip.load(access_kind, .i8, field_ptr, .default, "");
11024	}
11025	break :b try self.wip.extractValue(opt_handle, &.{1}, "");
11026	};
11027	comptime assert(optional_layout_version == 3);
11028
11029	return self.wip.icmp(cond, field, try o.builder.intValue(.i8, 0), "");
11030	}
11031
11032	/// Assumes the optional is not pointer-like and payload has bits.
11033	fn optPayloadHandle(
11034	fg: *FuncGen,
11035	opt_llvm_ty: Builder.Type,
11036	opt_handle: Builder.Value,
11037	opt_ty: Type,
11038	can_elide_load: bool,
11039	) !Builder.Value {
11040	const pt = fg.ng.pt;
11041	const zcu = pt.zcu;
11042	const payload_ty = opt_ty.optionalChild(zcu);
11043
11044	if (isByRef(opt_ty, zcu)) {
11045	// We have a pointer and we need to return a pointer to the first field.
11046	const payload_ptr = try fg.wip.gepStruct(opt_llvm_ty, opt_handle, 0, "");
11047
11048	const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
11049	if (isByRef(payload_ty, zcu)) {
11050	if (can_elide_load)
11051	return payload_ptr;
11052
11053	return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
11054	}
11055	return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_alignment);
11056	}
11057
11058	assert(!isByRef(payload_ty, zcu));
11059	return fg.wip.extractValue(opt_handle, &.{0}, "");
11060	}
11061
11062	fn buildOptional(
11063	self: *FuncGen,
11064	optional_ty: Type,
11065	payload: Builder.Value,
11066	non_null_bit: Builder.Value,
11067	) !Builder.Value {
11068	const o = self.ng.object;
11069	const pt = self.ng.pt;
11070	const zcu = pt.zcu;
11071	const optional_llvm_ty = try o.lowerType(pt, optional_ty);
11072	const non_null_field = try self.wip.cast(.zext, non_null_bit, .i8, "");
11073
11074	if (isByRef(optional_ty, zcu)) {
11075	const payload_alignment = optional_ty.abiAlignment(pt.zcu).toLlvm();
11076	const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment);
11077
11078	{
11079	const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, "");
11080	_ = try self.wip.store(.normal, payload, field_ptr, payload_alignment);
11081	}
11082	{
11083	const non_null_alignment = comptime Builder.Alignment.fromByteUnits(1);
11084	const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 1, "");
11085	_ = try self.wip.store(.normal, non_null_field, field_ptr, non_null_alignment);
11086	}
11087
11088	return alloca_inst;
11089	}
11090
11091	return self.wip.buildAggregate(optional_llvm_ty, &.{ payload, non_null_field }, "");
11092	}
11093
11094	fn fieldPtr(
11095	self: *FuncGen,
11096	aggregate_ptr: Builder.Value,
11097	aggregate_ptr_ty: Type,
11098	field_index: u32,
11099	) !Builder.Value {
11100	const o = self.ng.object;
11101	const pt = self.ng.pt;
11102	const zcu = pt.zcu;
11103	const aggregate_ty = aggregate_ptr_ty.childType(zcu);
11104	if (aggregate_ty.containerLayout(zcu) == .@"packed") {
11105	// A pointer to a bitpack field is equivalent to a pointer to the whole bitpack; the
11106	// bit offset is represented in the pointer type.
11107	return aggregate_ptr;
11108	}
11109	switch (aggregate_ty.zigTypeTag(zcu)) {
11110	.@"struct" => {
11111	if (!aggregate_ty.hasRuntimeBits(zcu)) {
11112	return aggregate_ptr;
11113	}
11114	const struct_llvm_ty = try o.lowerType(pt, aggregate_ty);
11115	if (o.llvmFieldIndex(aggregate_ty, field_index)) \|llvm_field_index\| {
11116	return self.wip.gepStruct(struct_llvm_ty, aggregate_ptr, llvm_field_index, "");
11117	} else {
11118	// If we found no index then this means this is a zero sized field at the
11119	// end of the struct. Treat our struct pointer as an array of two and get
11120	// the index to the element at index `1` to get a pointer to the end of
11121	// the struct.
11122	const llvm_index = try o.builder.intValue(
11123	try o.lowerType(pt, Type.usize),
11124	@intFromBool(aggregate_ty.hasRuntimeBits(zcu)),
11125	);
11126	return self.wip.gep(.inbounds, struct_llvm_ty, aggregate_ptr, &.{llvm_index}, "");
11127	}
11128	},
11129	.@"union" => {
11130	const layout = aggregate_ty.unionGetLayout(zcu);
11131	if (layout.payload_size == 0) return aggregate_ptr;
11132	const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align));
11133	const union_llvm_ty = try o.lowerType(pt, aggregate_ty);
11134	return self.wip.gepStruct(union_llvm_ty, aggregate_ptr, payload_index, "");
11135	},
11136	else => unreachable,
11137	}
11138	}
11139
11140	/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.
11141	fn loadTruncate(
11142	fg: *FuncGen,
11143	access_kind: Builder.MemoryAccessKind,
11144	payload_ty: Type,
11145	payload_ptr: Builder.Value,
11146	payload_alignment: Builder.Alignment,
11147	) !Builder.Value {
11148	// from https://llvm.org/docs/LangRef.html#load-instruction :
11149	// "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. "
11150	// => so load the byte aligned value and trunc the unwanted bits.
11151
11152	const o = fg.ng.object;
11153	const pt = fg.ng.pt;
11154	const zcu = pt.zcu;
11155	const payload_llvm_ty = try o.lowerType(pt, payload_ty);
11156	const abi_size = payload_ty.abiSize(zcu);
11157
11158	const load_llvm_ty = if (payload_ty.isAbiInt(zcu))
11159	try o.builder.intType(@intCast(abi_size * 8))
11160	else
11161	payload_llvm_ty;
11162	const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, "");
11163	const shifted = if (payload_llvm_ty != load_llvm_ty and o.target.cpu.arch.endian() == .big)
11164	try fg.wip.bin(.lshr, loaded, try o.builder.intValue(
11165	load_llvm_ty,
11166	(payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8,
11167	), "")
11168	else
11169	loaded;
11170
11171	return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, "");
11172	}
11173
11174	/// Load a by-ref type by constructing a new alloca and performing a memcpy.
11175	fn loadByRef(
11176	fg: *FuncGen,
11177	ptr: Builder.Value,
11178	pointee_type: Type,
11179	ptr_alignment: Builder.Alignment,
11180	access_kind: Builder.MemoryAccessKind,
11181	) !Builder.Value {
11182	const o = fg.ng.object;
11183	const pt = fg.ng.pt;
11184	const pointee_llvm_ty = try o.lowerType(pt, pointee_type);
11185	const result_align = InternPool.Alignment.fromLlvm(ptr_alignment)
11186	.max(pointee_type.abiAlignment(pt.zcu)).toLlvm();
11187	const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);
11188	const size_bytes = pointee_type.abiSize(pt.zcu);
11189	_ = try fg.wip.callMemCpy(
11190	result_ptr,
11191	result_align,
11192	ptr,
11193	ptr_alignment,
11194	try o.builder.intValue(try o.lowerType(pt, Type.usize), size_bytes),
11195	access_kind,
11196	fg.disable_intrinsics,
11197	);
11198	return result_ptr;
11199	}
11200
11201	/// This function always performs a copy. For isByRef=true types, it creates a new
11202	/// alloca and copies the value into it, then returns the alloca instruction.
11203	/// For isByRef=false types, it creates a load instruction and returns it.
11204	fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) !Builder.Value {
11205	const o = self.ng.object;
11206	const pt = self.ng.pt;
11207	const zcu = pt.zcu;
11208	const info = ptr_ty.ptrInfo(zcu);
11209	const elem_ty = Type.fromInterned(info.child);
11210	if (!elem_ty.hasRuntimeBits(zcu)) return .none;
11211
11212	const ptr_alignment = (if (info.flags.alignment != .none)
11213	@as(InternPool.Alignment, info.flags.alignment)
11214	else
11215	elem_ty.abiAlignment(zcu)).toLlvm();
11216
11217	const access_kind: Builder.MemoryAccessKind =
11218	if (info.flags.is_volatile) .@"volatile" else .normal;
11219
11220	if (info.flags.vector_index != .none) {
11221	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
11222	const vec_elem_ty = try o.lowerType(pt, elem_ty);
11223	const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
11224
11225	const loaded_vector = try self.wip.load(access_kind, vec_ty, ptr, ptr_alignment, "");
11226	return self.wip.extractElement(loaded_vector, index_u32, "");
11227	}
11228
11229	if (info.packed_offset.host_size == 0) {
11230	if (isByRef(elem_ty, zcu)) {
11231	return self.loadByRef(ptr, elem_ty, ptr_alignment, access_kind);
11232	}
11233	return self.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment);
11234	}
11235
11236	const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
11237	const containing_int =
11238	try self.wip.load(access_kind, containing_int_ty, ptr, ptr_alignment, "");
11239
11240	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
11241	const shift_amt = try o.builder.intValue(containing_int_ty, info.packed_offset.bit_offset);
11242	const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, "");
11243	const elem_llvm_ty = try o.lowerType(pt, elem_ty);
11244
11245	if (isByRef(elem_ty, zcu)) {
11246	const result_align = elem_ty.abiAlignment(zcu).toLlvm();
11247	const result_ptr = try self.buildAlloca(elem_llvm_ty, result_align);
11248
11249	const same_size_int = try o.builder.intType(@intCast(elem_bits));
11250	const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
11251	_ = try self.wip.store(.normal, truncated_int, result_ptr, result_align);
11252	return result_ptr;
11253	}
11254
11255	if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
11256	const same_size_int = try o.builder.intType(@intCast(elem_bits));
11257	const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
11258	return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
11259	}
11260
11261	if (elem_ty.isPtrAtRuntime(zcu)) {
11262	const same_size_int = try o.builder.intType(@intCast(elem_bits));
11263	const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
11264	return self.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, "");
11265	}
11266
11267	return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
11268	}
11269
11270	fn store(
11271	self: *FuncGen,
11272	ptr: Builder.Value,
11273	ptr_ty: Type,
11274	elem: Builder.Value,
11275	ordering: Builder.AtomicOrdering,
11276	) !void {
11277	const o = self.ng.object;
11278	const pt = self.ng.pt;
11279	const zcu = pt.zcu;
11280	const info = ptr_ty.ptrInfo(zcu);
11281	const elem_ty = Type.fromInterned(info.child);
11282	if (!elem_ty.hasRuntimeBits(zcu)) {
11283	return;
11284	}
11285	const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
11286	const access_kind: Builder.MemoryAccessKind =
11287	if (info.flags.is_volatile) .@"volatile" else .normal;
11288
11289	if (info.flags.vector_index != .none) {
11290	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
11291	const vec_elem_ty = try o.lowerType(pt, elem_ty);
11292	const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
11293
11294	const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, "");
11295
11296	const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, "");
11297
11298	assert(ordering == .none);
11299	_ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment);
11300	return;
11301	}
11302
11303	if (info.packed_offset.host_size != 0) {
11304	const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
11305	assert(ordering == .none);
11306	const containing_int =
11307	try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, "");
11308	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
11309	const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset);
11310	// Convert to equally-sized integer type in order to perform the bit
11311	// operations on the value to store
11312	const value_bits_type = try o.builder.intType(@intCast(elem_bits));
11313	const value_bits = if (elem_ty.isPtrAtRuntime(zcu))
11314	try self.wip.cast(.ptrtoint, elem, value_bits_type, "")
11315	else
11316	try self.wip.cast(.bitcast, elem, value_bits_type, "");
11317
11318	const mask_val = blk: {
11319	const zext = try self.wip.cast(
11320	.zext,
11321	try o.builder.intValue(value_bits_type, -1),
11322	containing_int_ty,
11323	"",
11324	);
11325	const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), "");
11326	break :blk try self.wip.bin(
11327	.xor,
11328	shl,
11329	try o.builder.intValue(containing_int_ty, -1),
11330	"",
11331	);
11332	};
11333
11334	const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, "");
11335	const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, "");
11336	const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), "");
11337	const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, "");
11338
11339	assert(ordering == .none);
11340	_ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment);
11341	return;
11342	}
11343	if (!isByRef(elem_ty, zcu)) {
11344	_ = try self.wip.storeAtomic(
11345	access_kind,
11346	elem,
11347	ptr,
11348	self.sync_scope,
11349	ordering,
11350	ptr_alignment,
11351	);
11352	return;
11353	}
11354	assert(ordering == .none);
11355	_ = try self.wip.callMemCpy(
11356	ptr,
11357	ptr_alignment,
11358	elem,
11359	elem_ty.abiAlignment(zcu).toLlvm(),
11360	try o.builder.intValue(try o.lowerType(pt, Type.usize), elem_ty.abiSize(zcu)),
11361	access_kind,
11362	self.disable_intrinsics,
11363	);
11364	}
11365
11366	fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
11367	const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
11368	const o = fg.ng.object;
11369	const pt = fg.ng.pt;
11370	const usize_ty = try o.lowerType(pt, Type.usize);
11371	const zero = try o.builder.intValue(usize_ty, 0);
11372	const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);
11373	const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");
11374	_ = try valgrindClientRequest(fg, zero, req, ptr_as_usize, len, zero, zero, zero);
11375	}
11376
11377	fn valgrindClientRequest(
11378	fg: *FuncGen,
11379	default_value: Builder.Value,
11380	request: Builder.Value,
11381	a1: Builder.Value,
11382	a2: Builder.Value,
11383	a3: Builder.Value,
11384	a4: Builder.Value,
11385	a5: Builder.Value,
11386	) Allocator.Error!Builder.Value {
11387	const o = fg.ng.object;
11388	const pt = fg.ng.pt;
11389	const zcu = pt.zcu;
11390	const target = zcu.getTarget();
11391	if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
11392
11393	const llvm_usize = try o.lowerType(pt, Type.usize);
11394	const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm();
11395
11396	const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
11397	const array_ptr = if (fg.valgrind_client_request_array == .none) a: {
11398	const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment);
11399	fg.valgrind_client_request_array = array_ptr;
11400	break :a array_ptr;
11401	} else fg.valgrind_client_request_array;
11402	const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
11403	const zero = try o.builder.intValue(llvm_usize, 0);
11404	for (array_elements, 0..) \|elem, i\| {
11405	const elem_ptr = try fg.wip.gep(.inbounds, array_llvm_ty, array_ptr, &.{
11406	zero, try o.builder.intValue(llvm_usize, i),
11407	}, "");
11408	_ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);
11409	}
11410
11411	const arch_specific: struct {
11412	template: [:0]const u8,
11413	constraints: [:0]const u8,
11414	} = switch (target.cpu.arch) {
11415	.arm, .armeb, .thumb, .thumbeb => .{
11416	.template =
11417	\\ mov r12, r12, ror #3 ; mov r12, r12, ror #13
11418	\\ mov r12, r12, ror #29 ; mov r12, r12, ror #19
11419	\\ orr r10, r10, r10
11420	,
11421	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
11422	},
11423	.aarch64, .aarch64_be => .{
11424	.template =
11425	\\ ror x12, x12, #3 ; ror x12, x12, #13
11426	\\ ror x12, x12, #51 ; ror x12, x12, #61
11427	\\ orr x10, x10, x10
11428	,
11429	.constraints = "={x3},{x4},{x3},~{cc},~{memory}",
11430	},
11431	.mips, .mipsel => .{
11432	.template =
11433	\\ srl $$0, $$0, 13
11434	\\ srl $$0, $$0, 29
11435	\\ srl $$0, $$0, 3
11436	\\ srl $$0, $$0, 19
11437	\\ or $$13, $$13, $$13
11438	,
11439	.constraints = "={$11},{$12},{$11},~{memory},~{$1}",
11440	},
11441	.mips64, .mips64el => .{
11442	.template =
11443	\\ dsll $$0, $$0, 3 ; dsll $$0, $$0, 13
11444	\\ dsll $$0, $$0, 29 ; dsll $$0, $$0, 19
11445	\\ or $$13, $$13, $$13
11446	,
11447	.constraints = "={$11},{$12},{$11},~{memory},~{$1}",
11448	},
11449	.powerpc, .powerpcle => .{
11450	.template =
11451	\\ rlwinm 0, 0, 3, 0, 31 ; rlwinm 0, 0, 13, 0, 31
11452	\\ rlwinm 0, 0, 29, 0, 31 ; rlwinm 0, 0, 19, 0, 31
11453	\\ or 1, 1, 1
11454	,
11455	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
11456	},
11457	.powerpc64, .powerpc64le => .{
11458	.template =
11459	\\ rotldi 0, 0, 3 ; rotldi 0, 0, 13
11460	\\ rotldi 0, 0, 61 ; rotldi 0, 0, 51
11461	\\ or 1, 1, 1
11462	,
11463	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
11464	},
11465	.riscv64 => .{
11466	.template =
11467	\\ .option push
11468	\\ .option norvc
11469	\\ srli zero, zero, 3
11470	\\ srli zero, zero, 13
11471	\\ srli zero, zero, 51
11472	\\ srli zero, zero, 61
11473	\\ or a0, a0, a0
11474	\\ .option pop
11475	,
11476	.constraints = "={a3},{a4},{a3},~{cc},~{memory}",
11477	},
11478	.s390x => .{
11479	.template =
11480	\\ lr %r15, %r15
11481	\\ lr %r1, %r1
11482	\\ lr %r2, %r2
11483	\\ lr %r3, %r3
11484	\\ lr %r2, %r2
11485	,
11486	.constraints = "={r3},{r2},{r3},~{cc},~{memory}",
11487	},
11488	.x86 => .{
11489	.template =
11490	\\ roll $$3, %edi ; roll $$13, %edi
11491	\\ roll $$61, %edi ; roll $$51, %edi
11492	\\ xchgl %ebx, %ebx
11493	,
11494	.constraints = "={edx},{eax},{edx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
11495	},
11496	.x86_64 => .{
11497	.template =
11498	\\ rolq $$3, %rdi ; rolq $$13, %rdi
11499	\\ rolq $$61, %rdi ; rolq $$51, %rdi
11500	\\ xchgq %rbx, %rbx
11501	,
11502	.constraints = "={rdx},{rax},{rdx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
11503	},
11504	else => unreachable,
11505	};
11506
11507	return fg.wip.callAsm(
11508	.none,
11509	try o.builder.fnType(llvm_usize, &.{ llvm_usize, llvm_usize }, .normal),
11510	.{ .sideeffect = true },
11511	try o.builder.string(arch_specific.template),
11512	try o.builder.string(arch_specific.constraints),
11513	&.{ try fg.wip.cast(.ptrtoint, array_ptr, llvm_usize, ""), default_value },
11514	"",
11515	);
11516	}
11517
11518	fn typeOf(fg: *FuncGen, inst: Air.Inst.Ref) Type {
11519	const zcu = fg.ng.pt.zcu;
11520	return fg.air.typeOf(inst, &zcu.intern_pool);
11521	}
11522
11523	fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type {
11524	const zcu = fg.ng.pt.zcu;
11525	return fg.air.typeOfIndex(inst, &zcu.intern_pool);
11526	}
11527	};
11528
11529	fn toLlvmAtomicOrdering(atomic_order: std.builtin.AtomicOrder) Builder.AtomicOrdering {
11530	return switch (atomic_order) {
11531	.unordered => .unordered,
11532	.monotonic => .monotonic,
11533	.acquire => .acquire,
11534	.release => .release,
11535	.acq_rel => .acq_rel,
11536	.seq_cst => .seq_cst,
11537	};
11538	}
11539
11540	fn toLlvmAtomicRmwBinOp(
11541	op: std.builtin.AtomicRmwOp,
11542	is_signed: bool,
11543	is_float: bool,
11544	) Builder.Function.Instruction.AtomicRmw.Operation {
11545	return switch (op) {
11546	.Xchg => .xchg,
11547	.Add => if (is_float) .fadd else return .add,
11548	.Sub => if (is_float) .fsub else return .sub,
11549	.And => .@"and",
11550	.Nand => .nand,
11551	.Or => .@"or",
11552	.Xor => .xor,
11553	.Max => if (is_float) .fmax else if (is_signed) .max else return .umax,
11554	.Min => if (is_float) .fmin else if (is_signed) .min else return .umin,
11555	};
11556	}
11557
11558	const CallingConventionInfo = struct {
11559	/// The LLVM calling convention to use.
11560	llvm_cc: Builder.CallConv,
11561	/// Whether to use an `alignstack` attribute to forcibly re-align the stack pointer in the function's prologue.
11562	align_stack: bool,
11563	/// Whether the function needs a `naked` attribute.
11564	naked: bool,
11565	/// How many leading parameters to apply the `inreg` attribute to.
11566	inreg_param_count: u2 = 0,
11567	};
11568
11569	pub fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: *const std.Target) ?CallingConventionInfo {
11570	const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null;
11571	const incoming_stack_alignment: ?u64, const register_params: u2 = switch (cc) {
11572	inline else => \|pl\| switch (@TypeOf(pl)) {
11573	void => .{ null, 0 },
11574	std.builtin.CallingConvention.ArcInterruptOptions,
11575	std.builtin.CallingConvention.ArmInterruptOptions,
11576	std.builtin.CallingConvention.RiscvInterruptOptions,
11577	std.builtin.CallingConvention.ShInterruptOptions,
11578	std.builtin.CallingConvention.MicroblazeInterruptOptions,
11579	std.builtin.CallingConvention.MipsInterruptOptions,
11580	std.builtin.CallingConvention.CommonOptions,
11581	=> .{ pl.incoming_stack_alignment, 0 },
11582	std.builtin.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params },
11583	else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)),
11584	},
11585	};
11586	return .{
11587	.llvm_cc = llvm_cc,
11588	.align_stack = if (incoming_stack_alignment) \|a\| need_align: {
11589	const normal_stack_align = target.stackAlignment();
11590	break :need_align a < normal_stack_align;
11591	} else false,
11592	.naked = cc == .naked,
11593	.inreg_param_count = register_params,
11594	};
11595	}
11596	fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv {
11597	if (target.cCallingConvention()) \|default_c\| {
11598	if (cc_tag == default_c) {
11599	return .ccc;
11600	}
11601	}
11602	return switch (cc_tag) {
11603	.@"inline" => unreachable,
11604	.auto, .async => .fastcc,
11605	.naked => .ccc,
11606	.x86_64_sysv => .x86_64_sysvcc,
11607	.x86_64_win => .win64cc,
11608	.x86_64_regcall_v3_sysv => if (target.cpu.arch == .x86_64 and target.os.tag != .windows)
11609	.x86_regcallcc
11610	else
11611	null,
11612	.x86_64_regcall_v4_win => if (target.cpu.arch == .x86_64 and target.os.tag == .windows)
11613	.x86_regcallcc // we use the "RegCallv4" module flag to make this correct
11614	else
11615	null,
11616	.x86_64_vectorcall => .x86_vectorcallcc,
11617	.x86_64_interrupt => .x86_intrcc,
11618	.x86_stdcall => .x86_stdcallcc,
11619	.x86_fastcall => .x86_fastcallcc,
11620	.x86_thiscall => .x86_thiscallcc,
11621	.x86_regcall_v3 => if (target.cpu.arch == .x86 and target.os.tag != .windows)
11622	.x86_regcallcc
11623	else
11624	null,
11625	.x86_regcall_v4_win => if (target.cpu.arch == .x86 and target.os.tag == .windows)
11626	.x86_regcallcc // we use the "RegCallv4" module flag to make this correct
11627	else
11628	null,
11629	.x86_vectorcall => .x86_vectorcallcc,
11630	.x86_interrupt => .x86_intrcc,
11631	.aarch64_vfabi => .aarch64_vector_pcs,
11632	.aarch64_vfabi_sve => .aarch64_sve_vector_pcs,
11633	.arm_aapcs => .arm_aapcscc,
11634	.arm_aapcs_vfp => .arm_aapcs_vfpcc,
11635	.riscv64_lp64_v => .riscv_vectorcallcc,
11636	.riscv32_ilp32_v => .riscv_vectorcallcc,
11637	.avr_builtin => .avr_builtincc,
11638	.avr_signal => .avr_signalcc,
11639	.avr_interrupt => .avr_intrcc,
11640	.m68k_rtd => .m68k_rtdcc,
11641	.m68k_interrupt => .m68k_intrcc,
11642	.msp430_interrupt => .msp430_intrcc,
11643	.amdgcn_kernel => .amdgpu_kernel,
11644	.amdgcn_cs => .amdgpu_cs,
11645	.nvptx_device => .ptx_device,
11646	.nvptx_kernel => .ptx_kernel,
11647
11648	// Calling conventions which LLVM uses function attributes for.
11649	.riscv64_interrupt,
11650	.riscv32_interrupt,
11651	.arm_interrupt,
11652	.mips64_interrupt,
11653	.mips_interrupt,
11654	.csky_interrupt,
11655	=> .ccc,
11656		4605
11657	// All the calling conventions which LLVM does not have a general representation for.	4606	// All the calling conventions which LLVM does not have a general representation for.
11658	// Note that these are often still supported through the `cCallingConvention` path above via `ccc`.	4607	// Note that these are often still supported through the `cCallingConvention` path above via `ccc`.
...	@@ -11724,7 +4673,7 @@ fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const s	...	@@ -11724,7 +4673,7 @@ fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const s
11724	}	4673	}
11725		4674
11726	/// Convert a zig-address space to an llvm address space.	4675	/// Convert a zig-address space to an llvm address space.
11727	fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace {	4676	pub fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace {
11728	for (llvmAddrSpaceInfo(target)) \|info\| if (info.zig == address_space) return info.llvm;	4677	for (llvmAddrSpaceInfo(target)) \|info\| if (info.zig == address_space) return info.llvm;
11729	unreachable;	4678	unreachable;
11730	}	4679	}
...	@@ -11811,20 +4760,6 @@ fn llvmAddrSpaceInfo(target: *const std.Target) []const AddrSpaceInfo {	...	@@ -11811,20 +4760,6 @@ fn llvmAddrSpaceInfo(target: *const std.Target) []const AddrSpaceInfo {
11811	};	4760	};
11812	}	4761	}
11813		4762
11814	/// On some targets, local values that are in the generic address space must be generated into a
11815	/// different address, space and then cast back to the generic address space.
11816	/// For example, on GPUs local variable declarations must be generated into the local address space.
11817	/// This function returns the address space local values should be generated into.
11818	fn llvmAllocaAddressSpace(target: *const std.Target) Builder.AddrSpace {
11819	return switch (target.cpu.arch) {
11820	// On amdgcn, locals should be generated into the private address space.
11821	// To make Zig not impossible to use, these are then converted to addresses in the
11822	// generic address space and treates as regular pointers. This is the way that HIP also does it.
11823	.amdgcn => Builder.AddrSpace.amdgpu.private,
11824	else => .default,
11825	};
11826	}
11827
11828	/// On some targets, global values that are in the generic address space must be generated into a	4763	/// On some targets, global values that are in the generic address space must be generated into a
11829	/// different address space, and then cast back to the generic address space.	4764	/// different address space, and then cast back to the generic address space.
11830	fn llvmDefaultGlobalAddressSpace(target: *const std.Target) Builder.AddrSpace {	4765	fn llvmDefaultGlobalAddressSpace(target: *const std.Target) Builder.AddrSpace {
...	@@ -11845,731 +4780,10 @@ fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, targ	...	@@ -11845,731 +4780,10 @@ fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, targ
11845	};	4780	};
11846	}	4781	}
11847		4782
11848	fn returnTypeByRef(zcu: Zcu, target: const std.Target, ty: Type) bool {
11849	if (isByRef(ty, zcu)) {
11850	return true;
11851	} else if (target.cpu.arch.isX86() and
11852	!target.cpu.has(.x86, .evex512) and
11853	ty.totalVectorBits(zcu) >= 512)
11854	{
11855	// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
11856	// "512-bit vector arguments require 'evex512' for AVX512"
11857	return true;
11858	} else {
11859	return false;
11860	}
11861	}
11862
11863	fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: Zcu, target: const std.Target) bool {
11864	const return_type = Type.fromInterned(fn_info.return_type);
11865	if (!return_type.hasRuntimeBits(zcu)) return false;
11866
11867	return switch (fn_info.cc) {
11868	.auto => returnTypeByRef(zcu, target, return_type),
11869	.x86_64_sysv => firstParamSRetSystemV(return_type, zcu, target),
11870	.x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory,
11871	.x86_sysv, .x86_win => isByRef(return_type, zcu),
11872	.x86_stdcall => !isScalar(zcu, return_type),
11873	.wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect,
11874	.aarch64_aapcs,
11875	.aarch64_aapcs_darwin,
11876	.aarch64_aapcs_win,
11877	=> aarch64_c_abi.classifyType(return_type, zcu) == .memory,
11878	.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
11879	.memory, .i64_array => true,
11880	.i32_array => \|size\| size != 1,
11881	.byval => false,
11882	},
11883	.riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory,
11884	.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
11885	.memory, .i32_array => true,
11886	.byval => false,
11887	},
11888	else => false, // TODO: investigate other targets/callconvs
11889	};
11890	}
11891
11892	fn firstParamSRetSystemV(ty: Type, zcu: Zcu, target: const std.Target) bool {
11893	const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret);
11894	if (class[0] == .memory) return true;
11895	if (class[0] == .x87 and class[2] != .none) return true;
11896	return false;
11897	}
11898
11899	/// In order to support the C calling convention, some return types need to be lowered
11900	/// completely differently in the function prototype to honor the C ABI, and then
11901	/// be effectively bitcasted to the actual return type.
11902	fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
11903	const zcu = pt.zcu;
11904	const return_type = Type.fromInterned(fn_info.return_type);
11905	if (!return_type.hasRuntimeBits(zcu)) {
11906	assert(!return_type.isError(zcu));
11907	return .void;
11908	}
11909	const target = zcu.getTarget();
11910	switch (fn_info.cc) {
11911	.@"inline" => unreachable,
11912	.auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(pt, return_type),
11913
11914	.x86_64_sysv => return lowerSystemVFnRetTy(o, pt, fn_info),
11915	.x86_64_win => return lowerWin64FnRetTy(o, pt, fn_info),
11916	.x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(pt, return_type) else .void,
11917	.x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(pt, return_type),
11918	.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) {
11919	.memory => return .void,
11920	.float_array => return o.lowerType(pt, return_type),
11921	.byval => return o.lowerType(pt, return_type),
11922	.integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
11923	.double_integer => return o.builder.arrayType(2, .i64),
11924	},
11925	.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
11926	.memory, .i64_array => return .void,
11927	.i32_array => \|len\| return if (len == 1) .i32 else .void,
11928	.byval => return o.lowerType(pt, return_type),
11929	},
11930	.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
11931	.memory, .i32_array => return .void,
11932	.byval => return o.lowerType(pt, return_type),
11933	},
11934	.riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) {
11935	.memory => return .void,
11936	.integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
11937	.double_integer => {
11938	const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {
11939	.riscv64, .riscv64be => .i64,
11940	.riscv32, .riscv32be => .i32,
11941	else => unreachable,
11942	};
11943	return o.builder.structType(.normal, &.{ integer, integer });
11944	},
11945	.byval => return o.lowerType(pt, return_type),
11946	.fields => {
11947	var types_len: usize = 0;
11948	var types: [8]Builder.Type = undefined;
11949	for (0..return_type.structFieldCount(zcu)) \|field_index\| {
11950	const field_ty = return_type.fieldType(field_index, zcu);
11951	if (!field_ty.hasRuntimeBits(zcu)) continue;
11952	types[types_len] = try o.lowerType(pt, field_ty);
11953	types_len += 1;
11954	}
11955	return o.builder.structType(.normal, types[0..types_len]);
11956	},
11957	},
11958	.wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) {
11959	.direct => \|scalar_ty\| return o.lowerType(pt, scalar_ty),
11960	.indirect => return .void,
11961	},
11962	// TODO investigate other callconvs
11963	else => return o.lowerType(pt, return_type),
11964	}
11965	}
11966
11967	fn lowerWin64FnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
11968	const zcu = pt.zcu;
11969	const return_type = Type.fromInterned(fn_info.return_type);
11970	switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) {
11971	.integer => {
11972	if (isScalar(zcu, return_type)) {
11973	return o.lowerType(pt, return_type);
11974	} else {
11975	return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8));
11976	}
11977	},
11978	.win_i128 => return o.builder.vectorType(.normal, 2, .i64),
11979	.memory => return .void,
11980	.sse => return o.lowerType(pt, return_type),
11981	else => unreachable,
11982	}
11983	}
11984
11985	fn lowerSystemVFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
11986	const zcu = pt.zcu;
11987	const ip = &zcu.intern_pool;
11988	const return_type = Type.fromInterned(fn_info.return_type);
11989	return_type.assertHasLayout(zcu);
11990	if (isScalar(zcu, return_type)) {
11991	return o.lowerType(pt, return_type);
11992	}
11993	const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret);
11994	var types_index: u32 = 0;
11995	var types_buffer: [8]Builder.Type = undefined;
11996	for (classes) \|class\| {
11997	switch (class) {
11998	.integer => {
11999	types_buffer[types_index] = .i64;
12000	types_index += 1;
12001	},
12002	.sse => {
12003	types_buffer[types_index] = .double;
12004	types_index += 1;
12005	},
12006	.sseup => {
12007	if (types_buffer[types_index - 1] == .double) {
12008	types_buffer[types_index - 1] = .fp128;
12009	} else {
12010	types_buffer[types_index] = .double;
12011	types_index += 1;
12012	}
12013	},
12014	.float => {
12015	types_buffer[types_index] = .float;
12016	types_index += 1;
12017	},
12018	.float_combine => {
12019	types_buffer[types_index] = try o.builder.vectorType(.normal, 2, .float);
12020	types_index += 1;
12021	},
12022	.x87 => {
12023	if (types_index != 0 or classes[2] != .none) return .void;
12024	types_buffer[types_index] = .x86_fp80;
12025	types_index += 1;
12026	},
12027	.x87up => continue,
12028	.none => break,
12029	.memory, .integer_per_element => return .void,
12030	.win_i128 => unreachable, // windows only
12031	}
12032	}
12033	const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
12034	if (first_non_integer == null or classes[first_non_integer.?] == .none) {
12035	assert(first_non_integer orelse classes.len == types_index);
12036	switch (ip.indexToKey(return_type.toIntern())) {
12037	.struct_type => {
12038	const size = return_type.abiSize(zcu);
12039	assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
12040	if (size % 8 > 0) {
12041	types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));
12042	}
12043	},
12044	else => {},
12045	}
12046	if (types_index == 1) return types_buffer[0];
12047	}
12048	return o.builder.structType(.normal, types_buffer[0..types_index]);
12049	}
12050
12051	const ParamTypeIterator = struct {
12052	object: *Object,
12053	pt: Zcu.PerThread,
12054	fn_info: InternPool.Key.FuncType,
12055	zig_index: u32,
12056	llvm_index: u32,
12057	types_len: u32,
12058	types_buffer: [8]Builder.Type,
12059	byval_attr: bool,
12060
12061	const Lowering = union(enum) {
12062	no_bits,
12063	byval,
12064	byref,
12065	byref_mut,
12066	abi_sized_int,
12067	multiple_llvm_types,
12068	slice,
12069	float_array: u8,
12070	i32_array: u8,
12071	i64_array: u8,
12072	};
12073
12074	fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering {
12075	if (it.zig_index >= it.fn_info.param_types.len) return null;
12076	const ip = &it.pt.zcu.intern_pool;
12077	const ty = it.fn_info.param_types.get(ip)[it.zig_index];
12078	it.byval_attr = false;
12079	return nextInner(it, Type.fromInterned(ty));
12080	}
12081
12082	/// `airCall` uses this instead of `next` so that it can take into account variadic functions.
12083	fn nextCall(it: ParamTypeIterator, fg: FuncGen, args: []const Air.Inst.Ref) Allocator.Error!?Lowering {
12084	assert(std.meta.eql(it.pt, fg.ng.pt));
12085	const ip = &it.pt.zcu.intern_pool;
12086	if (it.zig_index >= it.fn_info.param_types.len) {
12087	if (it.zig_index >= args.len) {
12088	return null;
12089	} else {
12090	return nextInner(it, fg.typeOf(args[it.zig_index]));
12091	}
12092	} else {
12093	return nextInner(it, Type.fromInterned(it.fn_info.param_types.get(ip)[it.zig_index]));
12094	}
12095	}
12096
12097	fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
12098	const pt = it.pt;
12099	const zcu = pt.zcu;
12100	const target = zcu.getTarget();
12101
12102	if (!ty.hasRuntimeBits(zcu)) {
12103	it.zig_index += 1;
12104	return .no_bits;
12105	}
12106	switch (it.fn_info.cc) {
12107	.@"inline" => unreachable,
12108	.auto => {
12109	it.zig_index += 1;
12110	it.llvm_index += 1;
12111	if (ty.isSlice(zcu) or
12112	(ty.zigTypeTag(zcu) == .optional and ty.optionalChild(zcu).isSlice(zcu) and !ty.ptrAllowsZero(zcu)))
12113	{
12114	it.llvm_index += 1;
12115	return .slice;
12116	} else if (isByRef(ty, zcu)) {
12117	return .byref;
12118	} else if (target.cpu.arch.isX86() and
12119	!target.cpu.has(.x86, .evex512) and
12120	ty.totalVectorBits(zcu) >= 512)
12121	{
12122	// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
12123	// "512-bit vector arguments require 'evex512' for AVX512"
12124	return .byref;
12125	} else {
12126	return .byval;
12127	}
12128	},
12129	.async => {
12130	@panic("TODO implement async function lowering in the LLVM backend");
12131	},
12132	.x86_64_sysv => return it.nextSystemV(ty),
12133	.x86_64_win => return it.nextWin64(ty),
12134	.x86_stdcall => {
12135	it.zig_index += 1;
12136	it.llvm_index += 1;
12137
12138	if (isScalar(zcu, ty)) {
12139	return .byval;
12140	} else {
12141	it.byval_attr = true;
12142	return .byref;
12143	}
12144	},
12145	.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => {
12146	it.zig_index += 1;
12147	it.llvm_index += 1;
12148	switch (aarch64_c_abi.classifyType(ty, zcu)) {
12149	.memory => return .byref_mut,
12150	.float_array => \|len\| return Lowering{ .float_array = len },
12151	.byval => return .byval,
12152	.integer => {
12153	it.types_len = 1;
12154	it.types_buffer[0] = .i64;
12155	return .multiple_llvm_types;
12156	},
12157	.double_integer => return Lowering{ .i64_array = 2 },
12158	}
12159	},
12160	.arm_aapcs, .arm_aapcs_vfp => {
12161	it.zig_index += 1;
12162	it.llvm_index += 1;
12163	switch (arm_c_abi.classifyType(ty, zcu, .arg)) {
12164	.memory => {
12165	it.byval_attr = true;
12166	return .byref;
12167	},
12168	.byval => return .byval,
12169	.i32_array => \|size\| return Lowering{ .i32_array = size },
12170	.i64_array => \|size\| return Lowering{ .i64_array = size },
12171	}
12172	},
12173	.mips_o32 => {
12174	it.zig_index += 1;
12175	it.llvm_index += 1;
12176	switch (mips_c_abi.classifyType(ty, zcu, .arg)) {
12177	.memory => {
12178	it.byval_attr = true;
12179	return .byref;
12180	},
12181	.byval => return .byval,
12182	.i32_array => \|size\| return Lowering{ .i32_array = size },
12183	}
12184	},
12185	.riscv64_lp64, .riscv32_ilp32 => {
12186	it.zig_index += 1;
12187	it.llvm_index += 1;
12188	switch (riscv_c_abi.classifyType(ty, zcu)) {
12189	.memory => return .byref_mut,
12190	.byval => return .byval,
12191	.integer => return .abi_sized_int,
12192	.double_integer => return Lowering{ .i64_array = 2 },
12193	.fields => {
12194	it.types_len = 0;
12195	for (0..ty.structFieldCount(zcu)) \|field_index\| {
12196	const field_ty = ty.fieldType(field_index, zcu);
12197	if (!field_ty.hasRuntimeBits(zcu)) continue;
12198	it.types_buffer[it.types_len] = try it.object.lowerType(pt, field_ty);
12199	it.types_len += 1;
12200	}
12201	it.llvm_index += it.types_len - 1;
12202	return .multiple_llvm_types;
12203	},
12204	}
12205	},
12206	.wasm_mvp => switch (wasm_c_abi.classifyType(ty, zcu)) {
12207	.direct => \|scalar_ty\| {
12208	if (isScalar(zcu, ty)) {
12209	it.zig_index += 1;
12210	it.llvm_index += 1;
12211	return .byval;
12212	} else {
12213	var types_buffer: [8]Builder.Type = undefined;
12214	types_buffer[0] = try it.object.lowerType(pt, scalar_ty);
12215	it.types_buffer = types_buffer;
12216	it.types_len = 1;
12217	it.llvm_index += 1;
12218	it.zig_index += 1;
12219	return .multiple_llvm_types;
12220	}
12221	},
12222	.indirect => {
12223	it.zig_index += 1;
12224	it.llvm_index += 1;
12225	it.byval_attr = true;
12226	return .byref;
12227	},
12228	},
12229	// TODO investigate other callconvs
12230	else => {
12231	it.zig_index += 1;
12232	it.llvm_index += 1;
12233	return .byval;
12234	},
12235	}
12236	}
12237
12238	fn nextWin64(it: *ParamTypeIterator, ty: Type) ?Lowering {
12239	const zcu = it.pt.zcu;
12240	switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) {
12241	.integer => {
12242	if (isScalar(zcu, ty)) {
12243	it.zig_index += 1;
12244	it.llvm_index += 1;
12245	return .byval;
12246	} else {
12247	it.zig_index += 1;
12248	it.llvm_index += 1;
12249	return .abi_sized_int;
12250	}
12251	},
12252	.win_i128 => {
12253	it.zig_index += 1;
12254	it.llvm_index += 1;
12255	return .byref;
12256	},
12257	.memory => {
12258	it.zig_index += 1;
12259	it.llvm_index += 1;
12260	return .byref_mut;
12261	},
12262	.sse => {
12263	it.zig_index += 1;
12264	it.llvm_index += 1;
12265	return .byval;
12266	},
12267	else => unreachable,
12268	}
12269	}
12270
12271	fn nextSystemV(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
12272	const zcu = it.pt.zcu;
12273	const ip = &zcu.intern_pool;
12274	ty.assertHasLayout(zcu);
12275	const classes = x86_64_abi.classifySystemV(ty, zcu, zcu.getTarget(), .arg);
12276	if (classes[0] == .memory) {
12277	it.zig_index += 1;
12278	it.llvm_index += 1;
12279	it.byval_attr = true;
12280	return .byref;
12281	}
12282	if (isScalar(zcu, ty)) {
12283	it.zig_index += 1;
12284	it.llvm_index += 1;
12285	return .byval;
12286	}
12287	var types_index: u32 = 0;
12288	var types_buffer: [8]Builder.Type = undefined;
12289	for (classes) \|class\| {
12290	switch (class) {
12291	.integer => {
12292	types_buffer[types_index] = .i64;
12293	types_index += 1;
12294	},
12295	.sse => {
12296	types_buffer[types_index] = .double;
12297	types_index += 1;
12298	},
12299	.sseup => {
12300	if (types_buffer[types_index - 1] == .double) {
12301	types_buffer[types_index - 1] = .fp128;
12302	} else {
12303	types_buffer[types_index] = .double;
12304	types_index += 1;
12305	}
12306	},
12307	.float => {
12308	types_buffer[types_index] = .float;
12309	types_index += 1;
12310	},
12311	.float_combine => {
12312	types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float);
12313	types_index += 1;
12314	},
12315	.x87 => {
12316	it.zig_index += 1;
12317	it.llvm_index += 1;
12318	it.byval_attr = true;
12319	return .byref;
12320	},
12321	.x87up => unreachable,
12322	.none => break,
12323	.memory => unreachable, // handled above
12324	.win_i128 => unreachable, // windows only
12325	.integer_per_element => {
12326	@panic("TODO");
12327	},
12328	}
12329	}
12330	const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
12331	if (first_non_integer == null or classes[first_non_integer.?] == .none) {
12332	assert(first_non_integer orelse classes.len == types_index);
12333	if (types_index == 1) {
12334	it.zig_index += 1;
12335	it.llvm_index += 1;
12336	return .abi_sized_int;
12337	}
12338	if (it.llvm_index + types_index > 6) {
12339	it.zig_index += 1;
12340	it.llvm_index += 1;
12341	it.byval_attr = true;
12342	return .byref;
12343	}
12344	switch (ip.indexToKey(ty.toIntern())) {
12345	.struct_type => {
12346	const size = ty.abiSize(zcu);
12347	assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
12348	if (size % 8 > 0) {
12349	types_buffer[types_index - 1] =
12350	try it.object.builder.intType(@intCast(size % 8 * 8));
12351	}
12352	},
12353	else => {},
12354	}
12355	}
12356	it.types_len = types_index;
12357	it.types_buffer = types_buffer;
12358	it.llvm_index += types_index;
12359	it.zig_index += 1;
12360	return .multiple_llvm_types;
12361	}
12362	};
12363
12364	fn iterateParamTypes(object: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) ParamTypeIterator {
12365	return .{
12366	.object = object,
12367	.pt = pt,
12368	.fn_info = fn_info,
12369	.zig_index = 0,
12370	.llvm_index = 0,
12371	.types_len = 0,
12372	.types_buffer = undefined,
12373	.byval_attr = false,
12374	};
12375	}
12376
12377	/// This function deliberately does not handle `_BitInt` because it typically
12378	/// has different ABI than regular integer types, and there is no currently no
12379	/// way to determine whether a Zig integer type is meant to represent e.g. `int`
12380	/// or `_BitInt(32)`.
12381	fn ccAbiPromoteInt(cc: std.builtin.CallingConvention, zcu: *Zcu, ty: Type) ?std.builtin.Signedness {
12382	switch (cc) {
12383	.auto, .@"inline", .async => return null,
12384	else => {},
12385	}
12386
12387	const int_info = switch (ty.zigTypeTag(zcu)) {
12388	.bool => Type.u1.intInfo(zcu),
12389	else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return null,
12390	};
12391	assert(int_info.bits >= 0);
12392
12393	const target = zcu.getTarget();
12394	return switch (target.cpu.arch) {
12395	.aarch64,
12396	.aarch64_be,
12397	=> switch (target.os.tag) {
12398	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => switch (int_info.bits) {
12399	8, 16 => int_info.signedness,
12400	else => null,
12401	},
12402	else => null,
12403	},
12404
12405	.avr,
12406	=> switch (int_info.bits) {
12407	8 => int_info.signedness,
12408	else => null,
12409	},
12410
12411	.lanai,
12412	=> null,
12413
12414	.loongarch64,
12415	.riscv64,
12416	.riscv64be,
12417	=> switch (int_info.bits) {
12418	8, 16 => int_info.signedness,
12419	32 => .signed,
12420	else => null,
12421	},
12422
12423	.mips,
12424	.mipsel,
12425	.mips64,
12426	.mips64el,
12427	=> switch (int_info.bits) {
12428	8, 16, 64 => int_info.signedness,
12429	// https://github.com/llvm/llvm-project/issues/179088
12430	// 32 => .signed,
12431	else => null,
12432	},
12433
12434	.powerpc64,
12435	.powerpc64le,
12436	.s390x,
12437	.sparc64,
12438	.ve,
12439	=> switch (int_info.bits) {
12440	8, 16, 32 => int_info.signedness,
12441	else => null,
12442	},
12443
12444	else => switch (int_info.bits) {
12445	8, 16 => int_info.signedness,
12446	else => null,
12447	},
12448	};
12449	}
12450
12451	/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
12452	/// or as an LLVM value.
12453	fn isByRef(ty: Type, zcu: *Zcu) bool {
12454	// For tuples and structs, if there are more than this many non-void
12455	// fields, then we make it byref, otherwise byval.
12456	const max_fields_byval = 0;
12457	const ip = &zcu.intern_pool;
12458
12459	switch (ty.zigTypeTag(zcu)) {
12460	.type,
12461	.comptime_int,
12462	.comptime_float,
12463	.enum_literal,
12464	.undefined,
12465	.null,
12466	.@"opaque",
12467	=> unreachable,
12468
12469	.noreturn,
12470	.void,
12471	.bool,
12472	.int,
12473	.float,
12474	.pointer,
12475	.error_set,
12476	.@"fn",
12477	.@"enum",
12478	.vector,
12479	.@"anyframe",
12480	=> return false,
12481
12482	.array, .frame => return ty.hasRuntimeBits(zcu),
12483	.@"struct" => {
12484	const struct_type = switch (ip.indexToKey(ty.toIntern())) {
12485	.tuple_type => \|tuple\| {
12486	var count: usize = 0;
12487	for (tuple.types.get(ip), tuple.values.get(ip)) \|field_ty, field_val\| {
12488	if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
12489
12490	count += 1;
12491	if (count > max_fields_byval) return true;
12492	if (isByRef(Type.fromInterned(field_ty), zcu)) return true;
12493	}
12494	return false;
12495	},
12496	.struct_type => ip.loadStructType(ty.toIntern()),
12497	else => unreachable,
12498	};
12499
12500	// Packed structs are represented to LLVM as integers.
12501	if (struct_type.layout == .@"packed") return false;
12502
12503	const field_types = struct_type.field_types.get(ip);
12504	var it = struct_type.iterateRuntimeOrder(ip);
12505	var count: usize = 0;
12506	while (it.next()) \|field_index\| {
12507	count += 1;
12508	if (count > max_fields_byval) return true;
12509	const field_ty = Type.fromInterned(field_types[field_index]);
12510	if (isByRef(field_ty, zcu)) return true;
12511	}
12512	return false;
12513	},
12514	.@"union" => switch (ty.containerLayout(zcu)) {
12515	.@"packed" => return false,
12516	else => return ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),
12517	},
12518	.error_union => {
12519	const payload_ty = ty.errorUnionPayload(zcu);
12520	if (!payload_ty.hasRuntimeBits(zcu)) {
12521	return false;
12522	}
12523	return true;
12524	},
12525	.optional => {
12526	const payload_ty = ty.optionalChild(zcu);
12527	if (!payload_ty.hasRuntimeBits(zcu)) {
12528	return false;
12529	}
12530	if (ty.optionalReprIsPayload(zcu)) {
12531	return false;
12532	}
12533	return true;
12534	},
12535	}
12536	}
12537
12538	fn isScalar(zcu: *Zcu, ty: Type) bool {
12539	return switch (ty.zigTypeTag(zcu)) {
12540	.void,
12541	.bool,
12542	.noreturn,
12543	.int,
12544	.float,
12545	.pointer,
12546	.optional,
12547	.error_set,
12548	.@"enum",
12549	.@"anyframe",
12550	.vector,
12551	=> true,
12552
12553	.@"struct" => ty.containerLayout(zcu) == .@"packed",
12554	.@"union" => ty.containerLayout(zcu) == .@"packed",
12555	else => false,
12556	};
12557	}
12558
12559	/// This function returns true if we expect LLVM to lower x86_fp80 correctly
12560	/// and false if we expect LLVM to crash if it encounters an x86_fp80 type,
12561	/// or if it produces miscompilations.
12562	fn backendSupportsF80(target: *const std.Target) bool {
12563	return switch (target.cpu.arch) {
12564	.x86, .x86_64 => !target.cpu.has(.x86, .soft_float),
12565	else => false,
12566	};
12567	}
12568
12569	/// This function returns true if we expect LLVM to lower f16 correctly	4783	/// This function returns true if we expect LLVM to lower f16 correctly
12570	/// and false if we expect LLVM to crash if it encounters an f16 type,	4784	/// and false if we expect LLVM to crash if it encounters an f16 type,
12571	/// or if it produces miscompilations.	4785	/// or if it produces miscompilations.
12572	fn backendSupportsF16(target: *const std.Target) bool {	4786	pub fn backendSupportsF16(target: *const std.Target) bool {
12573	return switch (target.cpu.arch) {	4787	return switch (target.cpu.arch) {
12574	// https://github.com/llvm/llvm-project/issues/97981	4788	// https://github.com/llvm/llvm-project/issues/97981
12575	.csky,	4789	.csky,
...	@@ -12594,10 +4808,20 @@ fn backendSupportsF16(target: *const std.Target) bool {	...	@@ -12594,10 +4808,20 @@ fn backendSupportsF16(target: *const std.Target) bool {
12594	};	4808	};
12595	}	4809	}
12596		4810
		4811	/// This function returns true if we expect LLVM to lower x86_fp80 correctly
		4812	/// and false if we expect LLVM to crash if it encounters an x86_fp80 type,
		4813	/// or if it produces miscompilations.
		4814	pub fn backendSupportsF80(target: *const std.Target) bool {
		4815	return switch (target.cpu.arch) {
		4816	.x86, .x86_64 => !target.cpu.has(.x86, .soft_float),
		4817	else => false,
		4818	};
		4819	}
		4820
12597	/// This function returns true if we expect LLVM to lower f128 correctly,	4821	/// This function returns true if we expect LLVM to lower f128 correctly,
12598	/// and false if we expect LLVM to crash if it encounters an f128 type,	4822	/// and false if we expect LLVM to crash if it encounters an f128 type,
12599	/// or if it produces miscompilations.	4823	/// or if it produces miscompilations.
12600	fn backendSupportsF128(target: *const std.Target) bool {	4824	pub fn backendSupportsF128(target: *const std.Target) bool {
12601	return switch (target.cpu.arch) {	4825	return switch (target.cpu.arch) {
12602	// https://github.com/llvm/llvm-project/issues/121122	4826	// https://github.com/llvm/llvm-project/issues/121122
12603	.amdgcn,	4827	.amdgcn,
...	@@ -12616,17 +4840,6 @@ fn backendSupportsF128(target: *const std.Target) bool {	...	@@ -12616,17 +4840,6 @@ fn backendSupportsF128(target: *const std.Target) bool {
12616	};	4840	};
12617	}	4841	}
12618		4842
12619	/// LLVM does not support all relevant intrinsics for all targets, so we
12620	/// may need to manually generate a compiler-rt call.
12621	fn intrinsicsAllowed(scalar_ty: Type, target: *const std.Target) bool {
12622	return switch (scalar_ty.toIntern()) {
12623	.f16_type => backendSupportsF16(target),
12624	.f80_type => (target.cTypeBitSize(.longdouble) == 80) and backendSupportsF80(target),
12625	.f128_type => (target.cTypeBitSize(.longdouble) == 128) and backendSupportsF128(target),
12626	else => true,
12627	};
12628	}
12629
12630	/// We need to insert extra padding if LLVM's isn't enough.	4843	/// We need to insert extra padding if LLVM's isn't enough.
12631	/// However we don't want to ever call LLVMABIAlignmentOfType or	4844	/// However we don't want to ever call LLVMABIAlignmentOfType or
12632	/// LLVMABISizeOfType because these functions will trip assertions	4845	/// LLVMABISizeOfType because these functions will trip assertions
...	@@ -12638,264 +4851,188 @@ const struct_layout_version = 2;	...	@@ -12638,264 +4851,188 @@ const struct_layout_version = 2;
12638		4851
12639	// TODO: Restore the non_null field to i1 once	4852	// TODO: Restore the non_null field to i1 once
12640	// https://github.com/llvm/llvm-project/issues/56585/ is fixed	4853	// https://github.com/llvm/llvm-project/issues/56585/ is fixed
12641	const optional_layout_version = 3;	4854	pub const optional_layout_version = 3;
12642		4855
12643	const lt_errors_fn_name = "__zig_lt_errors_len";	4856	const lt_errors_fn_name = "__zig_lt_errors_len";
12644		4857
12645	fn compilerRtIntBits(bits: u16) ?u16 {
12646	inline for (.{ 32, 64, 128 }) \|b\| {
12647	if (bits <= b) {
12648	return b;
12649	}
12650	}
12651	return null;
12652	}
12653
12654	fn buildAllocaInner(
12655	wip: *Builder.WipFunction,
12656	llvm_ty: Builder.Type,
12657	alignment: Builder.Alignment,
12658	target: *const std.Target,
12659	) Allocator.Error!Builder.Value {
12660	const address_space = llvmAllocaAddressSpace(target);
12661
12662	const alloca = blk: {
12663	const prev_cursor = wip.cursor;
12664	const prev_debug_location = wip.debug_location;
12665	defer {
12666	wip.cursor = prev_cursor;
12667	if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
12668	wip.debug_location = prev_debug_location;
12669	}
12670
12671	wip.cursor = .{ .block = .entry };
12672	wip.debug_location = .no_location;
12673	break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
12674	};
12675
12676	// The pointer returned from this function should have the generic address space,
12677	// if this isn't the case then cast it to the generic address space.
12678	return wip.conv(.unneeded, alloca, .ptr, "");
12679	}
12680
12681	fn errUnionPayloadOffset(payload_ty: Type, pt: Zcu.PerThread) !u1 {
12682	const zcu = pt.zcu;
12683	const err_int_ty = try pt.errorIntType();
12684	return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.gt, payload_ty.abiAlignment(zcu)));
12685	}
12686
12687	fn errUnionErrorOffset(payload_ty: Type, pt: Zcu.PerThread) !u1 {
12688	const zcu = pt.zcu;
12689	const err_int_ty = try pt.errorIntType();
12690	return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.lte, payload_ty.abiAlignment(zcu)));
12691	}
12692
12693	/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location
12694	///
12695	/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
12696	fn constraintAllowsMemory(constraint: []const u8) bool {
12697	// TODO: This implementation is woefully incomplete.
12698	for (constraint) \|byte\| {
12699	switch (byte) {
12700	'=', '*', ',', '&' => {},
12701	'm', 'o', 'X', 'g' => return true,
12702	else => {},
12703	}
12704	} else return false;
12705	}
12706
12707	/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register
12708	///
12709	/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
12710	fn constraintAllowsRegister(constraint: []const u8) bool {
12711	// TODO: This implementation is woefully incomplete.
12712	for (constraint) \|byte\| {
12713	switch (byte) {
12714	'=', '*', ',', '&' => {},
12715	'm', 'o' => {},
12716	else => return true,
12717	}
12718	} else return false;
12719	}
12720
12721	pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void {	4858	pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void {
12722	switch (arch) {	4859	switch (arch) {
12723	.aarch64, .aarch64_be => {	4860	.aarch64, .aarch64_be => {
12724	llvm.LLVMInitializeAArch64Target();	4861	bindings.LLVMInitializeAArch64Target();
12725	llvm.LLVMInitializeAArch64TargetInfo();	4862	bindings.LLVMInitializeAArch64TargetInfo();
12726	llvm.LLVMInitializeAArch64TargetMC();	4863	bindings.LLVMInitializeAArch64TargetMC();
12727	llvm.LLVMInitializeAArch64AsmPrinter();	4864	bindings.LLVMInitializeAArch64AsmPrinter();
12728	llvm.LLVMInitializeAArch64AsmParser();	4865	bindings.LLVMInitializeAArch64AsmParser();
12729	},	4866	},
12730	.amdgcn => {	4867	.amdgcn => {
12731	llvm.LLVMInitializeAMDGPUTarget();	4868	bindings.LLVMInitializeAMDGPUTarget();
12732	llvm.LLVMInitializeAMDGPUTargetInfo();	4869	bindings.LLVMInitializeAMDGPUTargetInfo();
12733	llvm.LLVMInitializeAMDGPUTargetMC();	4870	bindings.LLVMInitializeAMDGPUTargetMC();
12734	llvm.LLVMInitializeAMDGPUAsmPrinter();	4871	bindings.LLVMInitializeAMDGPUAsmPrinter();
12735	llvm.LLVMInitializeAMDGPUAsmParser();	4872	bindings.LLVMInitializeAMDGPUAsmParser();
12736	},	4873	},
12737	.thumb, .thumbeb, .arm, .armeb => {	4874	.thumb, .thumbeb, .arm, .armeb => {
12738	llvm.LLVMInitializeARMTarget();	4875	bindings.LLVMInitializeARMTarget();
12739	llvm.LLVMInitializeARMTargetInfo();	4876	bindings.LLVMInitializeARMTargetInfo();
12740	llvm.LLVMInitializeARMTargetMC();	4877	bindings.LLVMInitializeARMTargetMC();
12741	llvm.LLVMInitializeARMAsmPrinter();	4878	bindings.LLVMInitializeARMAsmPrinter();
12742	llvm.LLVMInitializeARMAsmParser();	4879	bindings.LLVMInitializeARMAsmParser();
12743	},	4880	},
12744	.avr => {	4881	.avr => {
12745	llvm.LLVMInitializeAVRTarget();	4882	bindings.LLVMInitializeAVRTarget();
12746	llvm.LLVMInitializeAVRTargetInfo();	4883	bindings.LLVMInitializeAVRTargetInfo();
12747	llvm.LLVMInitializeAVRTargetMC();	4884	bindings.LLVMInitializeAVRTargetMC();
12748	llvm.LLVMInitializeAVRAsmPrinter();	4885	bindings.LLVMInitializeAVRAsmPrinter();
12749	llvm.LLVMInitializeAVRAsmParser();	4886	bindings.LLVMInitializeAVRAsmParser();
12750	},	4887	},
12751	.bpfel, .bpfeb => {	4888	.bpfel, .bpfeb => {
12752	llvm.LLVMInitializeBPFTarget();	4889	bindings.LLVMInitializeBPFTarget();
12753	llvm.LLVMInitializeBPFTargetInfo();	4890	bindings.LLVMInitializeBPFTargetInfo();
12754	llvm.LLVMInitializeBPFTargetMC();	4891	bindings.LLVMInitializeBPFTargetMC();
12755	llvm.LLVMInitializeBPFAsmPrinter();	4892	bindings.LLVMInitializeBPFAsmPrinter();
12756	llvm.LLVMInitializeBPFAsmParser();	4893	bindings.LLVMInitializeBPFAsmParser();
12757	},	4894	},
12758	.hexagon => {	4895	.hexagon => {
12759	llvm.LLVMInitializeHexagonTarget();	4896	bindings.LLVMInitializeHexagonTarget();
12760	llvm.LLVMInitializeHexagonTargetInfo();	4897	bindings.LLVMInitializeHexagonTargetInfo();
12761	llvm.LLVMInitializeHexagonTargetMC();	4898	bindings.LLVMInitializeHexagonTargetMC();
12762	llvm.LLVMInitializeHexagonAsmPrinter();	4899	bindings.LLVMInitializeHexagonAsmPrinter();
12763	llvm.LLVMInitializeHexagonAsmParser();	4900	bindings.LLVMInitializeHexagonAsmParser();
12764	},	4901	},
12765	.lanai => {	4902	.lanai => {
12766	llvm.LLVMInitializeLanaiTarget();	4903	bindings.LLVMInitializeLanaiTarget();
12767	llvm.LLVMInitializeLanaiTargetInfo();	4904	bindings.LLVMInitializeLanaiTargetInfo();
12768	llvm.LLVMInitializeLanaiTargetMC();	4905	bindings.LLVMInitializeLanaiTargetMC();
12769	llvm.LLVMInitializeLanaiAsmPrinter();	4906	bindings.LLVMInitializeLanaiAsmPrinter();
12770	llvm.LLVMInitializeLanaiAsmParser();	4907	bindings.LLVMInitializeLanaiAsmParser();
12771	},	4908	},
12772	.mips, .mipsel, .mips64, .mips64el => {	4909	.mips, .mipsel, .mips64, .mips64el => {
12773	llvm.LLVMInitializeMipsTarget();	4910	bindings.LLVMInitializeMipsTarget();
12774	llvm.LLVMInitializeMipsTargetInfo();	4911	bindings.LLVMInitializeMipsTargetInfo();
12775	llvm.LLVMInitializeMipsTargetMC();	4912	bindings.LLVMInitializeMipsTargetMC();
12776	llvm.LLVMInitializeMipsAsmPrinter();	4913	bindings.LLVMInitializeMipsAsmPrinter();
12777	llvm.LLVMInitializeMipsAsmParser();	4914	bindings.LLVMInitializeMipsAsmParser();
12778	},	4915	},
12779	.msp430 => {	4916	.msp430 => {
12780	llvm.LLVMInitializeMSP430Target();	4917	bindings.LLVMInitializeMSP430Target();
12781	llvm.LLVMInitializeMSP430TargetInfo();	4918	bindings.LLVMInitializeMSP430TargetInfo();
12782	llvm.LLVMInitializeMSP430TargetMC();	4919	bindings.LLVMInitializeMSP430TargetMC();
12783	llvm.LLVMInitializeMSP430AsmPrinter();	4920	bindings.LLVMInitializeMSP430AsmPrinter();
12784	llvm.LLVMInitializeMSP430AsmParser();	4921	bindings.LLVMInitializeMSP430AsmParser();
12785	},	4922	},
12786	.nvptx, .nvptx64 => {	4923	.nvptx, .nvptx64 => {
12787	llvm.LLVMInitializeNVPTXTarget();	4924	bindings.LLVMInitializeNVPTXTarget();
12788	llvm.LLVMInitializeNVPTXTargetInfo();	4925	bindings.LLVMInitializeNVPTXTargetInfo();
12789	llvm.LLVMInitializeNVPTXTargetMC();	4926	bindings.LLVMInitializeNVPTXTargetMC();
12790	llvm.LLVMInitializeNVPTXAsmPrinter();	4927	bindings.LLVMInitializeNVPTXAsmPrinter();
12791	// There is no LLVMInitializeNVPTXAsmParser function available.	4928	// There is no LLVMInitializeNVPTXAsmParser function available.
12792	},	4929	},
12793	.powerpc, .powerpcle, .powerpc64, .powerpc64le => {	4930	.powerpc, .powerpcle, .powerpc64, .powerpc64le => {
12794	llvm.LLVMInitializePowerPCTarget();	4931	bindings.LLVMInitializePowerPCTarget();
12795	llvm.LLVMInitializePowerPCTargetInfo();	4932	bindings.LLVMInitializePowerPCTargetInfo();
12796	llvm.LLVMInitializePowerPCTargetMC();	4933	bindings.LLVMInitializePowerPCTargetMC();
12797	llvm.LLVMInitializePowerPCAsmPrinter();	4934	bindings.LLVMInitializePowerPCAsmPrinter();
12798	llvm.LLVMInitializePowerPCAsmParser();	4935	bindings.LLVMInitializePowerPCAsmParser();
12799	},	4936	},
12800	.riscv32, .riscv32be, .riscv64, .riscv64be => {	4937	.riscv32, .riscv32be, .riscv64, .riscv64be => {
12801	llvm.LLVMInitializeRISCVTarget();	4938	bindings.LLVMInitializeRISCVTarget();
12802	llvm.LLVMInitializeRISCVTargetInfo();	4939	bindings.LLVMInitializeRISCVTargetInfo();
12803	llvm.LLVMInitializeRISCVTargetMC();	4940	bindings.LLVMInitializeRISCVTargetMC();
12804	llvm.LLVMInitializeRISCVAsmPrinter();	4941	bindings.LLVMInitializeRISCVAsmPrinter();
12805	llvm.LLVMInitializeRISCVAsmParser();	4942	bindings.LLVMInitializeRISCVAsmParser();
12806	},	4943	},
12807	.sparc, .sparc64 => {	4944	.sparc, .sparc64 => {
12808	llvm.LLVMInitializeSparcTarget();	4945	bindings.LLVMInitializeSparcTarget();
12809	llvm.LLVMInitializeSparcTargetInfo();	4946	bindings.LLVMInitializeSparcTargetInfo();
12810	llvm.LLVMInitializeSparcTargetMC();	4947	bindings.LLVMInitializeSparcTargetMC();
12811	llvm.LLVMInitializeSparcAsmPrinter();	4948	bindings.LLVMInitializeSparcAsmPrinter();
12812	llvm.LLVMInitializeSparcAsmParser();	4949	bindings.LLVMInitializeSparcAsmParser();
12813	},	4950	},
12814	.s390x => {	4951	.s390x => {
12815	llvm.LLVMInitializeSystemZTarget();	4952	bindings.LLVMInitializeSystemZTarget();
12816	llvm.LLVMInitializeSystemZTargetInfo();	4953	bindings.LLVMInitializeSystemZTargetInfo();
12817	llvm.LLVMInitializeSystemZTargetMC();	4954	bindings.LLVMInitializeSystemZTargetMC();
12818	llvm.LLVMInitializeSystemZAsmPrinter();	4955	bindings.LLVMInitializeSystemZAsmPrinter();
12819	llvm.LLVMInitializeSystemZAsmParser();	4956	bindings.LLVMInitializeSystemZAsmParser();
12820	},	4957	},
12821	.wasm32, .wasm64 => {	4958	.wasm32, .wasm64 => {
12822	llvm.LLVMInitializeWebAssemblyTarget();	4959	bindings.LLVMInitializeWebAssemblyTarget();
12823	llvm.LLVMInitializeWebAssemblyTargetInfo();	4960	bindings.LLVMInitializeWebAssemblyTargetInfo();
12824	llvm.LLVMInitializeWebAssemblyTargetMC();	4961	bindings.LLVMInitializeWebAssemblyTargetMC();
12825	llvm.LLVMInitializeWebAssemblyAsmPrinter();	4962	bindings.LLVMInitializeWebAssemblyAsmPrinter();
12826	llvm.LLVMInitializeWebAssemblyAsmParser();	4963	bindings.LLVMInitializeWebAssemblyAsmParser();
12827	},	4964	},
12828	.x86, .x86_64 => {	4965	.x86, .x86_64 => {
12829	llvm.LLVMInitializeX86Target();	4966	bindings.LLVMInitializeX86Target();
12830	llvm.LLVMInitializeX86TargetInfo();	4967	bindings.LLVMInitializeX86TargetInfo();
12831	llvm.LLVMInitializeX86TargetMC();	4968	bindings.LLVMInitializeX86TargetMC();
12832	llvm.LLVMInitializeX86AsmPrinter();	4969	bindings.LLVMInitializeX86AsmPrinter();
12833	llvm.LLVMInitializeX86AsmParser();	4970	bindings.LLVMInitializeX86AsmParser();
12834	},	4971	},
12835	.xtensa => {	4972	.xtensa => {
12836	if (build_options.llvm_has_xtensa) {	4973	if (build_options.llvm_has_xtensa) {
12837	llvm.LLVMInitializeXtensaTarget();	4974	bindings.LLVMInitializeXtensaTarget();
12838	llvm.LLVMInitializeXtensaTargetInfo();	4975	bindings.LLVMInitializeXtensaTargetInfo();
12839	llvm.LLVMInitializeXtensaTargetMC();	4976	bindings.LLVMInitializeXtensaTargetMC();
12840	// There is no LLVMInitializeXtensaAsmPrinter function.	4977	// There is no LLVMInitializeXtensaAsmPrinter function.
12841	llvm.LLVMInitializeXtensaAsmParser();	4978	bindings.LLVMInitializeXtensaAsmParser();
12842	}	4979	}
12843	},	4980	},
12844	.xcore => {	4981	.xcore => {
12845	llvm.LLVMInitializeXCoreTarget();	4982	bindings.LLVMInitializeXCoreTarget();
12846	llvm.LLVMInitializeXCoreTargetInfo();	4983	bindings.LLVMInitializeXCoreTargetInfo();
12847	llvm.LLVMInitializeXCoreTargetMC();	4984	bindings.LLVMInitializeXCoreTargetMC();
12848	llvm.LLVMInitializeXCoreAsmPrinter();	4985	bindings.LLVMInitializeXCoreAsmPrinter();
12849	// There is no LLVMInitializeXCoreAsmParser function.	4986	// There is no LLVMInitializeXCoreAsmParser function.
12850	},	4987	},
12851	.m68k => {	4988	.m68k => {
12852	if (build_options.llvm_has_m68k) {	4989	if (build_options.llvm_has_m68k) {
12853	llvm.LLVMInitializeM68kTarget();	4990	bindings.LLVMInitializeM68kTarget();
12854	llvm.LLVMInitializeM68kTargetInfo();	4991	bindings.LLVMInitializeM68kTargetInfo();
12855	llvm.LLVMInitializeM68kTargetMC();	4992	bindings.LLVMInitializeM68kTargetMC();
12856	llvm.LLVMInitializeM68kAsmPrinter();	4993	bindings.LLVMInitializeM68kAsmPrinter();
12857	llvm.LLVMInitializeM68kAsmParser();	4994	bindings.LLVMInitializeM68kAsmParser();
12858	}	4995	}
12859	},	4996	},
12860	.csky => {	4997	.csky => {
12861	if (build_options.llvm_has_csky) {	4998	if (build_options.llvm_has_csky) {
12862	llvm.LLVMInitializeCSKYTarget();	4999	bindings.LLVMInitializeCSKYTarget();
12863	llvm.LLVMInitializeCSKYTargetInfo();	5000	bindings.LLVMInitializeCSKYTargetInfo();
12864	llvm.LLVMInitializeCSKYTargetMC();	5001	bindings.LLVMInitializeCSKYTargetMC();
12865	// There is no LLVMInitializeCSKYAsmPrinter function.	5002	// There is no LLVMInitializeCSKYAsmPrinter function.
12866	llvm.LLVMInitializeCSKYAsmParser();	5003	bindings.LLVMInitializeCSKYAsmParser();
12867	}	5004	}
12868	},	5005	},
12869	.ve => {	5006	.ve => {
12870	llvm.LLVMInitializeVETarget();	5007	bindings.LLVMInitializeVETarget();
12871	llvm.LLVMInitializeVETargetInfo();	5008	bindings.LLVMInitializeVETargetInfo();
12872	llvm.LLVMInitializeVETargetMC();	5009	bindings.LLVMInitializeVETargetMC();
12873	llvm.LLVMInitializeVEAsmPrinter();	5010	bindings.LLVMInitializeVEAsmPrinter();
12874	llvm.LLVMInitializeVEAsmParser();	5011	bindings.LLVMInitializeVEAsmParser();
12875	},	5012	},
12876	.arc => {	5013	.arc => {
12877	if (build_options.llvm_has_arc) {	5014	if (build_options.llvm_has_arc) {
12878	llvm.LLVMInitializeARCTarget();	5015	bindings.LLVMInitializeARCTarget();
12879	llvm.LLVMInitializeARCTargetInfo();	5016	bindings.LLVMInitializeARCTargetInfo();
12880	llvm.LLVMInitializeARCTargetMC();	5017	bindings.LLVMInitializeARCTargetMC();
12881	llvm.LLVMInitializeARCAsmPrinter();	5018	bindings.LLVMInitializeARCAsmPrinter();
12882	// There is no LLVMInitializeARCAsmParser function.	5019	// There is no LLVMInitializeARCAsmParser function.
12883	}	5020	}
12884	},	5021	},
12885	.loongarch32, .loongarch64 => {	5022	.loongarch32, .loongarch64 => {
12886	llvm.LLVMInitializeLoongArchTarget();	5023	bindings.LLVMInitializeLoongArchTarget();
12887	llvm.LLVMInitializeLoongArchTargetInfo();	5024	bindings.LLVMInitializeLoongArchTargetInfo();
12888	llvm.LLVMInitializeLoongArchTargetMC();	5025	bindings.LLVMInitializeLoongArchTargetMC();
12889	llvm.LLVMInitializeLoongArchAsmPrinter();	5026	bindings.LLVMInitializeLoongArchAsmPrinter();
12890	llvm.LLVMInitializeLoongArchAsmParser();	5027	bindings.LLVMInitializeLoongArchAsmParser();
12891	},	5028	},
12892	.spirv32,	5029	.spirv32,
12893	.spirv64,	5030	.spirv64,
12894	=> {	5031	=> {
12895	llvm.LLVMInitializeSPIRVTarget();	5032	bindings.LLVMInitializeSPIRVTarget();
12896	llvm.LLVMInitializeSPIRVTargetInfo();	5033	bindings.LLVMInitializeSPIRVTargetInfo();
12897	llvm.LLVMInitializeSPIRVTargetMC();	5034	bindings.LLVMInitializeSPIRVTargetMC();
12898	llvm.LLVMInitializeSPIRVAsmPrinter();	5035	bindings.LLVMInitializeSPIRVAsmPrinter();
12899	},	5036	},
12900		5037
12901	// LLVM does does not have a backend for these.	5038	// LLVM does does not have a backend for these.
...	@@ -12916,296 +5053,3 @@ pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void {	...	@@ -12916,296 +5053,3 @@ pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void {
12916	=> unreachable,	5053	=> unreachable,
12917	}	5054	}
12918	}	5055	}
12919
12920	fn minIntConst(b: Builder, min_ty: Type, as_ty: Builder.Type, zcu: const Zcu) Allocator.Error!Builder.Constant {
12921	const info = min_ty.intInfo(zcu);
12922	if (info.signedness == .unsigned or info.bits == 0) {
12923	return b.intConst(as_ty, 0);
12924	}
12925	if (std.math.cast(u6, info.bits - 1)) \|shift\| {
12926	const min_val: i64 = @as(i64, std.math.minInt(i64)) >> (63 - shift);
12927	return b.intConst(as_ty, min_val);
12928	}
12929	var res: std.math.big.int.Managed = try .init(zcu.gpa);
12930	defer res.deinit();
12931	try res.setTwosCompIntLimit(.min, info.signedness, info.bits);
12932	return b.bigIntConst(as_ty, res.toConst());
12933	}
12934
12935	fn maxIntConst(b: Builder, max_ty: Type, as_ty: Builder.Type, zcu: const Zcu) Allocator.Error!Builder.Constant {
12936	const info = max_ty.intInfo(zcu);
12937	switch (info.bits) {
12938	0 => return b.intConst(as_ty, 0),
12939	1 => switch (info.signedness) {
12940	.signed => return b.intConst(as_ty, 0),
12941	.unsigned => return b.intConst(as_ty, 1),
12942	},
12943	else => {},
12944	}
12945	const unsigned_bits = switch (info.signedness) {
12946	.unsigned => info.bits,
12947	.signed => info.bits - 1,
12948	};
12949	if (std.math.cast(u6, unsigned_bits)) \|shift\| {
12950	const max_val: u64 = (@as(u64, 1) << shift) - 1;
12951	return b.intConst(as_ty, max_val);
12952	}
12953	var res: std.math.big.int.Managed = try .init(zcu.gpa);
12954	defer res.deinit();
12955	try res.setTwosCompIntLimit(.max, info.signedness, info.bits);
12956	return b.bigIntConst(as_ty, res.toConst());
12957	}
12958
12959	/// Appends zero or more LLVM constraints to `llvm_constraints`, returning how many were added.
12960	fn appendConstraints(
12961	gpa: Allocator,
12962	llvm_constraints: *std.ArrayList(u8),
12963	zig_name: []const u8,
12964	target: *const std.Target,
12965	) error{OutOfMemory}!usize {
12966	switch (target.cpu.arch) {
12967	.mips, .mipsel, .mips64, .mips64el => if (mips_clobber_overrides.get(zig_name)) \|llvm_tag\| {
12968	const llvm_name = @tagName(llvm_tag);
12969	try llvm_constraints.ensureUnusedCapacity(gpa, llvm_name.len + 4);
12970	llvm_constraints.appendSliceAssumeCapacity("~{");
12971	llvm_constraints.appendSliceAssumeCapacity(llvm_name);
12972	llvm_constraints.appendSliceAssumeCapacity("},");
12973	return 1;
12974	},
12975	else => {},
12976	}
12977
12978	try llvm_constraints.ensureUnusedCapacity(gpa, zig_name.len + 4);
12979	llvm_constraints.appendSliceAssumeCapacity("~{");
12980	llvm_constraints.appendSliceAssumeCapacity(zig_name);
12981	llvm_constraints.appendSliceAssumeCapacity("},");
12982	return 1;
12983	}
12984
12985	const mips_clobber_overrides = std.StaticStringMap(enum {
12986	@"$msair",
12987	@"$msacsr",
12988	@"$msaaccess",
12989	@"$msasave",
12990	@"$msamodify",
12991	@"$msarequest",
12992	@"$msamap",
12993	@"$msaunmap",
12994	@"$f0",
12995	@"$f1",
12996	@"$f2",
12997	@"$f3",
12998	@"$f4",
12999	@"$f5",
13000	@"$f6",
13001	@"$f7",
13002	@"$f8",
13003	@"$f9",
13004	@"$f10",
13005	@"$f11",
13006	@"$f12",
13007	@"$f13",
13008	@"$f14",
13009	@"$f15",
13010	@"$f16",
13011	@"$f17",
13012	@"$f18",
13013	@"$f19",
13014	@"$f20",
13015	@"$f21",
13016	@"$f22",
13017	@"$f23",
13018	@"$f24",
13019	@"$f25",
13020	@"$f26",
13021	@"$f27",
13022	@"$f28",
13023	@"$f29",
13024	@"$f30",
13025	@"$f31",
13026	@"$fcc0",
13027	@"$fcc1",
13028	@"$fcc2",
13029	@"$fcc3",
13030	@"$fcc4",
13031	@"$fcc5",
13032	@"$fcc6",
13033	@"$fcc7",
13034	@"$w0",
13035	@"$w1",
13036	@"$w2",
13037	@"$w3",
13038	@"$w4",
13039	@"$w5",
13040	@"$w6",
13041	@"$w7",
13042	@"$w8",
13043	@"$w9",
13044	@"$w10",
13045	@"$w11",
13046	@"$w12",
13047	@"$w13",
13048	@"$w14",
13049	@"$w15",
13050	@"$w16",
13051	@"$w17",
13052	@"$w18",
13053	@"$w19",
13054	@"$w20",
13055	@"$w21",
13056	@"$w22",
13057	@"$w23",
13058	@"$w24",
13059	@"$w25",
13060	@"$w26",
13061	@"$w27",
13062	@"$w28",
13063	@"$w29",
13064	@"$w30",
13065	@"$w31",
13066	@"$0",
13067	@"$1",
13068	@"$2",
13069	@"$3",
13070	@"$4",
13071	@"$5",
13072	@"$6",
13073	@"$7",
13074	@"$8",
13075	@"$9",
13076	@"$10",
13077	@"$11",
13078	@"$12",
13079	@"$13",
13080	@"$14",
13081	@"$15",
13082	@"$16",
13083	@"$17",
13084	@"$18",
13085	@"$19",
13086	@"$20",
13087	@"$21",
13088	@"$22",
13089	@"$23",
13090	@"$24",
13091	@"$25",
13092	@"$26",
13093	@"$27",
13094	@"$28",
13095	@"$29",
13096	@"$30",
13097	@"$31",
13098	}).initComptime(.{
13099	.{ "msa_ir", .@"$msair" },
13100	.{ "msa_csr", .@"$msacsr" },
13101	.{ "msa_access", .@"$msaaccess" },
13102	.{ "msa_save", .@"$msasave" },
13103	.{ "msa_modify", .@"$msamodify" },
13104	.{ "msa_request", .@"$msarequest" },
13105	.{ "msa_map", .@"$msamap" },
13106	.{ "msa_unmap", .@"$msaunmap" },
13107	.{ "f0", .@"$f0" },
13108	.{ "f1", .@"$f1" },
13109	.{ "f2", .@"$f2" },
13110	.{ "f3", .@"$f3" },
13111	.{ "f4", .@"$f4" },
13112	.{ "f5", .@"$f5" },
13113	.{ "f6", .@"$f6" },
13114	.{ "f7", .@"$f7" },
13115	.{ "f8", .@"$f8" },
13116	.{ "f9", .@"$f9" },
13117	.{ "f10", .@"$f10" },
13118	.{ "f11", .@"$f11" },
13119	.{ "f12", .@"$f12" },
13120	.{ "f13", .@"$f13" },
13121	.{ "f14", .@"$f14" },
13122	.{ "f15", .@"$f15" },
13123	.{ "f16", .@"$f16" },
13124	.{ "f17", .@"$f17" },
13125	.{ "f18", .@"$f18" },
13126	.{ "f19", .@"$f19" },
13127	.{ "f20", .@"$f20" },
13128	.{ "f21", .@"$f21" },
13129	.{ "f22", .@"$f22" },
13130	.{ "f23", .@"$f23" },
13131	.{ "f24", .@"$f24" },
13132	.{ "f25", .@"$f25" },
13133	.{ "f26", .@"$f26" },
13134	.{ "f27", .@"$f27" },
13135	.{ "f28", .@"$f28" },
13136	.{ "f29", .@"$f29" },
13137	.{ "f30", .@"$f30" },
13138	.{ "f31", .@"$f31" },
13139	.{ "fcc0", .@"$fcc0" },
13140	.{ "fcc1", .@"$fcc1" },
13141	.{ "fcc2", .@"$fcc2" },
13142	.{ "fcc3", .@"$fcc3" },
13143	.{ "fcc4", .@"$fcc4" },
13144	.{ "fcc5", .@"$fcc5" },
13145	.{ "fcc6", .@"$fcc6" },
13146	.{ "fcc7", .@"$fcc7" },
13147	.{ "w0", .@"$w0" },
13148	.{ "w1", .@"$w1" },
13149	.{ "w2", .@"$w2" },
13150	.{ "w3", .@"$w3" },
13151	.{ "w4", .@"$w4" },
13152	.{ "w5", .@"$w5" },
13153	.{ "w6", .@"$w6" },
13154	.{ "w7", .@"$w7" },
13155	.{ "w8", .@"$w8" },
13156	.{ "w9", .@"$w9" },
13157	.{ "w10", .@"$w10" },
13158	.{ "w11", .@"$w11" },
13159	.{ "w12", .@"$w12" },
13160	.{ "w13", .@"$w13" },
13161	.{ "w14", .@"$w14" },
13162	.{ "w15", .@"$w15" },
13163	.{ "w16", .@"$w16" },
13164	.{ "w17", .@"$w17" },
13165	.{ "w18", .@"$w18" },
13166	.{ "w19", .@"$w19" },
13167	.{ "w20", .@"$w20" },
13168	.{ "w21", .@"$w21" },
13169	.{ "w22", .@"$w22" },
13170	.{ "w23", .@"$w23" },
13171	.{ "w24", .@"$w24" },
13172	.{ "w25", .@"$w25" },
13173	.{ "w26", .@"$w26" },
13174	.{ "w27", .@"$w27" },
13175	.{ "w28", .@"$w28" },
13176	.{ "w29", .@"$w29" },
13177	.{ "w30", .@"$w30" },
13178	.{ "w31", .@"$w31" },
13179	.{ "r0", .@"$0" },
13180	.{ "r1", .@"$1" },
13181	.{ "r2", .@"$2" },
13182	.{ "r3", .@"$3" },
13183	.{ "r4", .@"$4" },
13184	.{ "r5", .@"$5" },
13185	.{ "r6", .@"$6" },
13186	.{ "r7", .@"$7" },
13187	.{ "r8", .@"$8" },
13188	.{ "r9", .@"$9" },
13189	.{ "r10", .@"$10" },
13190	.{ "r11", .@"$11" },
13191	.{ "r12", .@"$12" },
13192	.{ "r13", .@"$13" },
13193	.{ "r14", .@"$14" },
13194	.{ "r15", .@"$15" },
13195	.{ "r16", .@"$16" },
13196	.{ "r17", .@"$17" },
13197	.{ "r18", .@"$18" },
13198	.{ "r19", .@"$19" },
13199	.{ "r20", .@"$20" },
13200	.{ "r21", .@"$21" },
13201	.{ "r22", .@"$22" },
13202	.{ "r23", .@"$23" },
13203	.{ "r24", .@"$24" },
13204	.{ "r25", .@"$25" },
13205	.{ "r26", .@"$26" },
13206	.{ "r27", .@"$27" },
13207	.{ "r28", .@"$28" },
13208	.{ "r29", .@"$29" },
13209	.{ "r30", .@"$30" },
13210	.{ "r31", .@"$31" },
13211	});

src/codegen/llvm/FuncGen.zig created+8145

...	@@ -0,0 +1,8145 @@
1	const FuncGen = @This();
2
3	pub const Error = Zcu.CodegenFailError;
4
5	object: *Object,
6	nav_index: InternPool.Nav.Index,
7	pt: Zcu.PerThread,
8	gpa: Allocator,
9	air: Air,
10	liveness: Air.Liveness,
11	wip: Builder.WipFunction,
12	is_naked: bool,
13	fuzz: ?Fuzz,
14
15	file: Builder.Metadata,
16	scope: Builder.Metadata,
17
18	inlined_at: Builder.Metadata.Optional,
19
20	base_line: u32,
21	prev_dbg_line: c_uint,
22	prev_dbg_column: c_uint,
23
24	/// This stores the LLVM values used in a function, such that they can be referred to
25	/// in other instructions. This table is cleared before every function is generated.
26	func_inst_table: std.AutoHashMapUnmanaged(Air.Inst.Ref, Builder.Value),
27
28	/// If the return type is sret, this is the result pointer. Otherwise null.
29	/// Note that this can disagree with isByRef for the return type in the case
30	/// of C ABI functions.
31	ret_ptr: Builder.Value,
32	/// Any function that needs to perform Valgrind client requests needs an array alloca
33	/// instruction, however a maximum of one per function is needed.
34	valgrind_client_request_array: Builder.Value = .none,
35	/// These fields are used to refer to the LLVM value of the function parameters
36	/// in an Arg instruction.
37	/// This list may be shorter than the list according to the zig type system;
38	/// it omits 0-bit types. If the function uses sret as the first parameter,
39	/// this slice does not include it.
40	args: []const Builder.Value,
41	arg_index: u32,
42	arg_inline_index: u32,
43
44	err_ret_trace: Builder.Value,
45
46	/// This data structure is used to implement breaking to blocks.
47	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
48	parent_bb: Builder.Function.Block.Index,
49	breaks: *BreakList,
50	}),
51
52	/// Maps `loop` instructions to the bb to branch to to repeat the loop.
53	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, Builder.Function.Block.Index),
54
55	/// Maps `loop_switch_br` instructions to the information required to lower
56	/// dispatches (`switch_dispatch` instructions).
57	switch_dispatch_info: std.AutoHashMapUnmanaged(Air.Inst.Index, SwitchDispatchInfo),
58
59	sync_scope: Builder.SyncScope,
60
61	disable_intrinsics: bool,
62
63	/// Have we seen loads or stores involving `allowzero` pointers?
64	allowzero_access: bool,
65
66	fn todo(fg: *FuncGen, comptime format: []const u8, args: anytype) Error {
67	@branchHint(.cold);
68	return fg.pt.zcu.codegenFail(
69	fg.nav_index,
70	"TODO (LLVM): " ++ format,
71	args,
72	);
73	}
74
75	fn ownerModule(fg: const FuncGen) Package.Module {
76	return fg.pt.zcu.navFileScope(fg.nav_index).mod.?;
77	}
78
79	fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void {
80	// LLVM already considers null pointers to be valid in non-generic address spaces, so avoid
81	// pessimizing optimization for functions with accesses to such pointers.
82	if (info.flags.address_space == .generic and info.flags.is_allowzero) self.allowzero_access = true;
83	}
84
85	pub const Fuzz = struct {
86	counters_variable: Builder.Variable.Index,
87	pcs: std.ArrayList(Builder.Constant),
88
89	fn deinit(f: *Fuzz, gpa: Allocator) void {
90	f.pcs.deinit(gpa);
91	f.* = undefined;
92	}
93	};
94
95	const SwitchDispatchInfo = struct {
96	/// These are the blocks corresponding to each switch case.
97	/// The final element corresponds to the `else` case.
98	/// Slices allocated into `gpa`.
99	case_blocks: []Builder.Function.Block.Index,
100	/// This is `.none` if `jmp_table` is set, since we won't use a `switch` instruction to dispatch.
101	switch_weights: Builder.Function.Instruction.BrCond.Weights,
102	/// If not `null`, we have manually constructed a jump table to reach the desired block.
103	/// `table` can be used if the value is between `min` and `max` inclusive.
104	/// We perform this lowering manually to avoid some questionable behavior from LLVM.
105	/// See `airSwitchBr` for details.
106	jmp_table: ?JmpTable,
107
108	const JmpTable = struct {
109	min: Builder.Constant,
110	max: Builder.Constant,
111	in_bounds_hint: enum { none, unpredictable, likely, unlikely },
112	/// Pointer to the jump table itself, to be used with `indirectbr`.
113	/// The index into the jump table is the dispatch condition minus `min`.
114	/// The table values are `blockaddress` constants corresponding to blocks in `case_blocks`.
115	table: Builder.Constant,
116	/// `true` if `table` conatins a reference to the `else` block.
117	/// In this case, the `indirectbr` must include the `else` block in its target list.
118	table_includes_else: bool,
119	};
120	};
121
122	const BreakList = union {
123	list: std.MultiArrayList(struct {
124	bb: Builder.Function.Block.Index,
125	val: Builder.Value,
126	}),
127	len: usize,
128	};
129
130	pub fn deinit(self: *FuncGen) void {
131	const gpa = self.gpa;
132	if (self.fuzz) \|*f\| f.deinit(self.gpa);
133	self.wip.deinit();
134	self.func_inst_table.deinit(gpa);
135	self.blocks.deinit(gpa);
136	self.loops.deinit(gpa);
137	var it = self.switch_dispatch_info.valueIterator();
138	while (it.next()) \|info\| {
139	self.gpa.free(info.case_blocks);
140	}
141	self.switch_dispatch_info.deinit(gpa);
142	}
143
144	fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value {
145	const gpa = self.gpa;
146	const gop = try self.func_inst_table.getOrPut(gpa, inst);
147	if (gop.found_existing) return gop.value_ptr.*;
148
149	const llvm_val = try self.resolveValue((try self.air.value(inst, self.pt)).?);
150	gop.value_ptr.* = llvm_val.toValue();
151	return llvm_val.toValue();
152	}
153
154	fn resolveValue(self: *FuncGen, val: Value) Error!Builder.Constant {
155	const o = self.object;
156	const pt = self.pt;
157	const zcu = pt.zcu;
158	const ty = val.typeOf(zcu);
159	if (!isByRef(ty, zcu)) {
160	return o.lowerValue(pt, val.toIntern());
161	} else {
162	// We need a pointer to a global constant, i.e. a UAV.
163	return o.lowerUavRef(pt, .{
164	.val = val.toIntern(),
165	.orig_ty = (try pt.singleConstPtrType(ty)).toIntern(),
166	});
167	}
168	}
169
170	pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) Error!void {
171	const o = self.object;
172	const zcu = self.pt.zcu;
173	const ip = &zcu.intern_pool;
174	const air_tags = self.air.instructions.items(.tag);
175	switch (coverage_point) {
176	.none => {},
177	.poi => if (self.fuzz) \|*fuzz\| {
178	const poi_index = fuzz.pcs.items.len;
179	const base_ptr = fuzz.counters_variable.toValue(&o.builder);
180	const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{
181	try o.builder.intValue(.i32, poi_index),
182	}, "");
183	const one = try o.builder.intValue(.i8, 1);
184	_ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, "");
185
186	// LLVM does not allow blockaddress on the entry block.
187	const pc = if (self.wip.cursor.block == .entry)
188	self.wip.function.toConst(&o.builder)
189	else
190	try o.builder.blockAddrConst(self.wip.function, self.wip.cursor.block);
191	const gpa = self.gpa;
192	try fuzz.pcs.append(gpa, pc);
193	},
194	}
195	for (body, 0..) \|inst, i\| {
196	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
197
198	const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
199	// zig fmt: off
200
201	// No "scalarize" legalizations are enabled, so these instructions never appear.
202	.legalize_vec_elem_val => unreachable,
203	.legalize_vec_store_elem => unreachable,
204	// No soft float legalizations are enabled.
205	.legalize_compiler_rt_call => unreachable,
206
207	.add => try self.airAdd(inst, .normal),
208	.add_optimized => try self.airAdd(inst, .fast),
209	.add_wrap => try self.airAddWrap(inst),
210	.add_sat => try self.airAddSat(inst),
211
212	.sub => try self.airSub(inst, .normal),
213	.sub_optimized => try self.airSub(inst, .fast),
214	.sub_wrap => try self.airSubWrap(inst),
215	.sub_sat => try self.airSubSat(inst),
216
217	.mul => try self.airMul(inst, .normal),
218	.mul_optimized => try self.airMul(inst, .fast),
219	.mul_wrap => try self.airMulWrap(inst),
220	.mul_sat => try self.airMulSat(inst),
221
222	.add_safe => try self.airSafeArithmetic(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
223	.sub_safe => try self.airSafeArithmetic(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
224	.mul_safe => try self.airSafeArithmetic(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
225
226	.div_float => try self.airDivFloat(inst, .normal),
227	.div_trunc => try self.airDivTrunc(inst, .normal),
228	.div_floor => try self.airDivFloor(inst, .normal),
229	.div_exact => try self.airDivExact(inst, .normal),
230	.rem => try self.airRem(inst, .normal),
231	.mod => try self.airMod(inst, .normal),
232	.abs => try self.airAbs(inst),
233	.ptr_add => try self.airPtrAdd(inst),
234	.ptr_sub => try self.airPtrSub(inst),
235	.shl => try self.airShl(inst),
236	.shl_sat => try self.airShlSat(inst),
237	.shl_exact => try self.airShlExact(inst),
238	.min => try self.airMin(inst),
239	.max => try self.airMax(inst),
240	.slice => try self.airSlice(inst),
241	.mul_add => try self.airMulAdd(inst),
242
243	.div_float_optimized => try self.airDivFloat(inst, .fast),
244	.div_trunc_optimized => try self.airDivTrunc(inst, .fast),
245	.div_floor_optimized => try self.airDivFloor(inst, .fast),
246	.div_exact_optimized => try self.airDivExact(inst, .fast),
247	.rem_optimized => try self.airRem(inst, .fast),
248	.mod_optimized => try self.airMod(inst, .fast),
249
250	.add_with_overflow => try self.airOverflow(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
251	.sub_with_overflow => try self.airOverflow(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
252	.mul_with_overflow => try self.airOverflow(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
253	.shl_with_overflow => try self.airShlWithOverflow(inst),
254
255	.bit_and, .bool_and => try self.airAnd(inst),
256	.bit_or, .bool_or => try self.airOr(inst),
257	.xor => try self.airXor(inst),
258	.shr => try self.airShr(inst, false),
259	.shr_exact => try self.airShr(inst, true),
260
261	.sqrt => try self.airUnaryOp(inst, .sqrt),
262	.sin => try self.airUnaryOp(inst, .sin),
263	.cos => try self.airUnaryOp(inst, .cos),
264	.tan => try self.airUnaryOp(inst, .tan),
265	.exp => try self.airUnaryOp(inst, .exp),
266	.exp2 => try self.airUnaryOp(inst, .exp2),
267	.log => try self.airUnaryOp(inst, .log),
268	.log2 => try self.airUnaryOp(inst, .log2),
269	.log10 => try self.airUnaryOp(inst, .log10),
270	.floor => try self.airUnaryOp(inst, .floor),
271	.ceil => try self.airUnaryOp(inst, .ceil),
272	.round => try self.airUnaryOp(inst, .round),
273	.trunc_float => try self.airUnaryOp(inst, .trunc),
274
275	.neg => try self.airNeg(inst, .normal),
276	.neg_optimized => try self.airNeg(inst, .fast),
277
278	.cmp_eq => try self.airCmp(inst, .eq, .normal),
279	.cmp_gt => try self.airCmp(inst, .gt, .normal),
280	.cmp_gte => try self.airCmp(inst, .gte, .normal),
281	.cmp_lt => try self.airCmp(inst, .lt, .normal),
282	.cmp_lte => try self.airCmp(inst, .lte, .normal),
283	.cmp_neq => try self.airCmp(inst, .neq, .normal),
284
285	.cmp_eq_optimized => try self.airCmp(inst, .eq, .fast),
286	.cmp_gt_optimized => try self.airCmp(inst, .gt, .fast),
287	.cmp_gte_optimized => try self.airCmp(inst, .gte, .fast),
288	.cmp_lt_optimized => try self.airCmp(inst, .lt, .fast),
289	.cmp_lte_optimized => try self.airCmp(inst, .lte, .fast),
290	.cmp_neq_optimized => try self.airCmp(inst, .neq, .fast),
291
292	.cmp_vector => try self.airCmpVector(inst, .normal),
293	.cmp_vector_optimized => try self.airCmpVector(inst, .fast),
294	.cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),
295
296	.is_non_null => try self.airIsNonNull(inst, false, .ne),
297	.is_non_null_ptr => try self.airIsNonNull(inst, true , .ne),
298	.is_null => try self.airIsNonNull(inst, false, .eq),
299	.is_null_ptr => try self.airIsNonNull(inst, true , .eq),
300
301	.is_non_err => try self.airIsErr(inst, .eq, false),
302	.is_non_err_ptr => try self.airIsErr(inst, .eq, true),
303	.is_err => try self.airIsErr(inst, .ne, false),
304	.is_err_ptr => try self.airIsErr(inst, .ne, true),
305
306	.alloc => try self.airAlloc(inst),
307	.ret_ptr => try self.airRetPtr(inst),
308	.arg => try self.airArg(inst),
309	.bitcast => try self.airBitCast(inst),
310	.breakpoint => try self.airBreakpoint(inst),
311	.ret_addr => try self.airRetAddr(inst),
312	.frame_addr => try self.airFrameAddress(inst),
313	.@"try" => try self.airTry(inst, false),
314	.try_cold => try self.airTry(inst, true),
315	.try_ptr => try self.airTryPtr(inst, false),
316	.try_ptr_cold => try self.airTryPtr(inst, true),
317	.intcast => try self.airIntCast(inst, false),
318	.intcast_safe => try self.airIntCast(inst, true),
319	.trunc => try self.airTrunc(inst),
320	.fptrunc => try self.airFptrunc(inst),
321	.fpext => try self.airFpext(inst),
322	.load => try self.airLoad(inst),
323	.not => try self.airNot(inst),
324	.store => try self.airStore(inst, false),
325	.store_safe => try self.airStore(inst, true),
326	.assembly => try self.airAssembly(inst),
327	.slice_ptr => try self.airSliceField(inst, 0),
328	.slice_len => try self.airSliceField(inst, 1),
329
330	.ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
331	.ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
332
333	.int_from_float => try self.airIntFromFloat(inst, .normal),
334	.int_from_float_optimized => try self.airIntFromFloat(inst, .fast),
335	.int_from_float_safe => unreachable, // handled by `legalizeFeatures`
336	.int_from_float_optimized_safe => unreachable, // handled by `legalizeFeatures`
337
338	.array_to_slice => try self.airArrayToSlice(inst),
339	.float_from_int => try self.airFloatFromInt(inst),
340	.cmpxchg_weak => try self.airCmpxchg(inst, .weak),
341	.cmpxchg_strong => try self.airCmpxchg(inst, .strong),
342	.atomic_rmw => try self.airAtomicRmw(inst),
343	.atomic_load => try self.airAtomicLoad(inst),
344	.memset => try self.airMemset(inst, false),
345	.memset_safe => try self.airMemset(inst, true),
346	.memcpy => try self.airMemcpy(inst),
347	.memmove => try self.airMemmove(inst),
348	.set_union_tag => try self.airSetUnionTag(inst),
349	.get_union_tag => try self.airGetUnionTag(inst),
350	.clz => try self.airClzCtz(inst, .ctlz),
351	.ctz => try self.airClzCtz(inst, .cttz),
352	.popcount => try self.airBitOp(inst, .ctpop),
353	.byte_swap => try self.airByteSwap(inst),
354	.bit_reverse => try self.airBitOp(inst, .bitreverse),
355	.tag_name => try self.airTagName(inst),
356	.error_name => try self.airErrorName(inst),
357	.splat => try self.airSplat(inst),
358	.select => try self.airSelect(inst),
359	.shuffle_one => try self.airShuffleOne(inst),
360	.shuffle_two => try self.airShuffleTwo(inst),
361	.aggregate_init => try self.airAggregateInit(inst),
362	.union_init => try self.airUnionInit(inst),
363	.prefetch => try self.airPrefetch(inst),
364	.addrspace_cast => try self.airAddrSpaceCast(inst),
365
366	.is_named_enum_value => try self.airIsNamedEnumValue(inst),
367	.error_set_has_value => try self.airErrorSetHasValue(inst),
368
369	.reduce => try self.airReduce(inst, .normal),
370	.reduce_optimized => try self.airReduce(inst, .fast),
371
372	.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
373	.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
374	.atomic_store_release => try self.airAtomicStore(inst, .release),
375	.atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
376
377	.struct_field_ptr => try self.airStructFieldPtr(inst),
378	.struct_field_val => try self.airStructFieldVal(inst),
379
380	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
381	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
382	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
383	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
384
385	.field_parent_ptr => try self.airFieldParentPtr(inst),
386
387	.array_elem_val => try self.airArrayElemVal(inst),
388	.slice_elem_val => try self.airSliceElemVal(inst),
389	.slice_elem_ptr => try self.airSliceElemPtr(inst),
390	.ptr_elem_val => try self.airPtrElemVal(inst),
391	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
392
393	.optional_payload => try self.airOptionalPayload(inst),
394	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
395	.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
396
397	.unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),
398	.unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),
399	.unwrap_errunion_err => try self.airErrUnionErr(inst, false),
400	.unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
401	.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
402	.err_return_trace => try self.airErrReturnTrace(inst),
403	.set_err_return_trace => try self.airSetErrReturnTrace(inst),
404	.save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),
405
406	.wrap_optional => try self.airWrapOptional(body[i..]),
407	.wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]),
408	.wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]),
409
410	.wasm_memory_size => try self.airWasmMemorySize(inst),
411	.wasm_memory_grow => try self.airWasmMemoryGrow(inst),
412
413	.runtime_nav_ptr => try self.airRuntimeNavPtr(inst),
414
415	.inferred_alloc, .inferred_alloc_comptime => unreachable,
416
417	.dbg_stmt => try self.airDbgStmt(inst),
418	.dbg_empty_stmt => try self.airDbgEmptyStmt(inst),
419	.dbg_var_ptr => try self.airDbgVarPtr(inst),
420	.dbg_var_val => try self.airDbgVarVal(inst, false),
421	.dbg_arg_inline => try self.airDbgVarVal(inst, true),
422
423	.c_va_arg => try self.airCVaArg(inst),
424	.c_va_copy => try self.airCVaCopy(inst),
425	.c_va_end => try self.airCVaEnd(inst),
426	.c_va_start => try self.airCVaStart(inst),
427
428	.work_item_id => try self.airWorkItemId(inst),
429	.work_group_size => try self.airWorkGroupSize(inst),
430	.work_group_id => try self.airWorkGroupId(inst),
431
432	// Instructions that are known to always be `noreturn` based on their tag.
433	.br => return self.airBr(inst),
434	.repeat => return self.airRepeat(inst),
435	.switch_dispatch => return self.airSwitchDispatch(inst),
436	.cond_br => return self.airCondBr(inst),
437	.switch_br => return self.airSwitchBr(inst, false),
438	.loop_switch_br => return self.airSwitchBr(inst, true),
439	.loop => return self.airLoop(inst),
440	.ret => return self.airRet(inst, false),
441	.ret_safe => return self.airRet(inst, true),
442	.ret_load => return self.airRetLoad(inst),
443	.trap => return self.airTrap(inst),
444	.unreach => return self.airUnreach(inst),
445
446	// Instructions which may be `noreturn`.
447	.block => res: {
448	const res = try self.airBlock(inst);
449	if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
450	break :res res;
451	},
452	.dbg_inline_block => res: {
453	const res = try self.airDbgInlineBlock(inst);
454	if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
455	break :res res;
456	},
457	.call, .call_always_tail, .call_never_tail, .call_never_inline => \|tag\| res: {
458	const res = try self.airCall(inst, switch (tag) {
459	.call => .auto,
460	.call_always_tail => .always_tail,
461	.call_never_tail => .never_tail,
462	.call_never_inline => .never_inline,
463	else => unreachable,
464	});
465	// TODO: the AIR we emit for calls is a bit weird - the instruction has
466	// type `noreturn`, but there are instructions (and maybe a safety check) following
467	// nonetheless. The `unreachable` or safety check should be emitted by backends instead.
468	//if (self.typeOfIndex(inst).isNoReturn(mod)) return;
469	break :res res;
470	},
471
472	// zig fmt: on
473	};
474	if (val != .none) try self.func_inst_table.putNoClobber(self.gpa, inst.toRef(), val);
475	}
476	unreachable;
477	}
478
479	fn genBodyDebugScope(
480	self: *FuncGen,
481	maybe_inline_func: ?InternPool.Index,
482	body: []const Air.Inst.Index,
483	coverage_point: Air.CoveragePoint,
484	) Error!void {
485	const o = self.object;
486
487	if (self.wip.strip) return self.genBody(body, coverage_point);
488
489	const old_debug_location = self.wip.debug_location;
490	const old_file = self.file;
491	const old_inlined_at = self.inlined_at;
492	const old_base_line = self.base_line;
493	defer if (maybe_inline_func) \|_\| {
494	self.wip.debug_location = old_debug_location;
495	self.file = old_file;
496	self.inlined_at = old_inlined_at;
497	self.base_line = old_base_line;
498	};
499
500	const old_scope = self.scope;
501	defer self.scope = old_scope;
502
503	if (maybe_inline_func) \|inline_func\| {
504	const pt = self.pt;
505	const zcu = pt.zcu;
506	const ip = &zcu.intern_pool;
507
508	const func = zcu.funcInfo(inline_func);
509	const nav = ip.getNav(func.owner_nav);
510	const file_scope = zcu.navFileScopeIndex(func.owner_nav);
511	const mod = zcu.fileByIndex(file_scope).mod.?;
512
513	self.file = try o.getDebugFile(pt, file_scope);
514
515	self.base_line = zcu.navSrcLine(func.owner_nav);
516	const line_number = self.base_line + 1;
517	self.inlined_at = try self.wip.debug_location.toMetadata(&o.builder);
518
519	const fn_ty = try pt.funcType(.{
520	.param_types = &.{},
521	.return_type = .void_type,
522	});
523
524	self.scope = try o.builder.debugSubprogram(
525	self.file,
526	try o.builder.metadataString(nav.name.toSlice(&zcu.intern_pool)),
527	try o.builder.metadataString(nav.fqn.toSlice(&zcu.intern_pool)),
528	line_number,
529	line_number + func.lbrace_line,
530	try o.getDebugType(pt, fn_ty),
531	.{
532	.di_flags = .{ .StaticMember = true },
533	.sp_flags = .{
534	.Optimized = mod.optimize_mode != .Debug,
535	.Definition = true,
536	.LocalToUnit = true, // inline functions cannot be exported
537	},
538	},
539	o.debug_compile_unit.unwrap().?,
540	);
541	}
542
543	self.scope = try o.builder.debugLexicalBlock(
544	self.scope,
545	self.file,
546	self.prev_dbg_line,
547	self.prev_dbg_column,
548	);
549	self.wip.debug_location = .{ .location = .{
550	.line = self.prev_dbg_line,
551	.column = self.prev_dbg_column,
552	.scope = self.scope.toOptional(),
553	.inlined_at = self.inlined_at,
554	} };
555
556	try self.genBody(body, coverage_point);
557	}
558
559	const CallAttr = enum {
560	Auto,
561	NeverTail,
562	NeverInline,
563	AlwaysTail,
564	AlwaysInline,
565	};
566
567	fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value {
568	const air_call = self.air.unwrapCall(inst);
569	const args = air_call.args;
570	const o = self.object;
571	const pt = self.pt;
572	const zcu = pt.zcu;
573	const ip = &zcu.intern_pool;
574	const callee_ty = self.typeOf(air_call.callee);
575	const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
576	.@"fn" => callee_ty,
577	.pointer => callee_ty.childType(zcu),
578	else => unreachable,
579	};
580	const fn_info = zcu.typeToFunc(zig_fn_ty).?;
581	const return_type = Type.fromInterned(fn_info.return_type);
582	const llvm_fn = try self.resolveInst(air_call.callee);
583	const target = zcu.getTarget();
584	const sret = firstParamSRet(fn_info, zcu, target);
585
586	var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);
587	defer llvm_args.deinit();
588
589	var attributes: Builder.FunctionAttributes.Wip = .{};
590	defer attributes.deinit(&o.builder);
591
592	if (self.disable_intrinsics) {
593	try attributes.addFnAttr(.nobuiltin, &o.builder);
594	}
595
596	switch (modifier) {
597	.auto, .always_tail => {},
598	.never_tail, .never_inline => try attributes.addFnAttr(.@"noinline", &o.builder),
599	.no_suspend, .always_inline, .compile_time => unreachable,
600	}
601
602	const ret_ptr = if (!sret) null else blk: {
603	const llvm_ret_ty = try o.lowerType(pt, return_type);
604	try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);
605
606	const alignment = return_type.abiAlignment(zcu).toLlvm();
607	const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);
608	try llvm_args.append(ret_ptr);
609	break :blk ret_ptr;
610	};
611
612	const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
613	if (err_return_tracing) {
614	assert(self.err_ret_trace != .none);
615	try llvm_args.append(self.err_ret_trace);
616	}
617
618	var it = iterateParamTypes(o, pt, fn_info);
619	while (try it.nextCall(self, args)) \|lowering\| switch (lowering) {
620	.no_bits => continue,
621	.byval => {
622	const arg = args[it.zig_index - 1];
623	const param_ty = self.typeOf(arg);
624	const llvm_arg = try self.resolveInst(arg);
625	const llvm_param_ty = try o.lowerType(pt, param_ty);
626	if (isByRef(param_ty, zcu)) {
627	const alignment = param_ty.abiAlignment(zcu).toLlvm();
628	const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
629	try llvm_args.append(loaded);
630	} else {
631	try llvm_args.append(llvm_arg);
632	}
633	},
634	.byref => {
635	const arg = args[it.zig_index - 1];
636	const param_ty = self.typeOf(arg);
637	const llvm_arg = try self.resolveInst(arg);
638	if (isByRef(param_ty, zcu)) {
639	try llvm_args.append(llvm_arg);
640	} else {
641	const alignment = param_ty.abiAlignment(zcu).toLlvm();
642	const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);
643	const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
644	_ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
645	try llvm_args.append(arg_ptr);
646	}
647	},
648	.byref_mut => {
649	const arg = args[it.zig_index - 1];
650	const param_ty = self.typeOf(arg);
651	const llvm_arg = try self.resolveInst(arg);
652
653	const alignment = param_ty.abiAlignment(zcu).toLlvm();
654	const param_llvm_ty = try o.lowerType(pt, param_ty);
655	const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
656	if (isByRef(param_ty, zcu)) {
657	const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
658	_ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
659	} else {
660	_ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
661	}
662	try llvm_args.append(arg_ptr);
663	},
664	.abi_sized_int => {
665	const arg = args[it.zig_index - 1];
666	const param_ty = self.typeOf(arg);
667	const llvm_arg = try self.resolveInst(arg);
668	const int_llvm_ty = try o.builder.intType(@intCast(param_ty.abiSize(zcu) * 8));
669
670	if (isByRef(param_ty, zcu)) {
671	const alignment = param_ty.abiAlignment(zcu).toLlvm();
672	const loaded = try self.wip.load(.normal, int_llvm_ty, llvm_arg, alignment, "");
673	try llvm_args.append(loaded);
674	} else {
675	// LLVM does not allow bitcasting structs so we must allocate
676	// a local, store as one type, and then load as another type.
677	const alignment = param_ty.abiAlignment(zcu).toLlvm();
678	const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);
679	_ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);
680	const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");
681	try llvm_args.append(loaded);
682	}
683	},
684	.slice => {
685	const arg = args[it.zig_index - 1];
686	const llvm_arg = try self.resolveInst(arg);
687	const ptr = try self.wip.extractValue(llvm_arg, &.{0}, "");
688	const len = try self.wip.extractValue(llvm_arg, &.{1}, "");
689	try llvm_args.appendSlice(&.{ ptr, len });
690	},
691	.multiple_llvm_types => {
692	const arg = args[it.zig_index - 1];
693	const param_ty = self.typeOf(arg);
694	const llvm_types = it.types_buffer[0..it.types_len];
695	const llvm_arg = try self.resolveInst(arg);
696	const is_by_ref = isByRef(param_ty, zcu);
697	const arg_ptr = if (is_by_ref) llvm_arg else ptr: {
698	const alignment = param_ty.abiAlignment(zcu).toLlvm();
699	const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
700	_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
701	break :ptr ptr;
702	};
703
704	const llvm_ty = try o.builder.structType(.normal, llvm_types);
705	try llvm_args.ensureUnusedCapacity(it.types_len);
706	for (llvm_types, 0..) \|field_ty, i\| {
707	const alignment =
708	Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8));
709	const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, "");
710	const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, "");
711	llvm_args.appendAssumeCapacity(loaded);
712	}
713	},
714	.float_array => \|count\| {
715	const arg = args[it.zig_index - 1];
716	const arg_ty = self.typeOf(arg);
717	var llvm_arg = try self.resolveInst(arg);
718	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
719	if (!isByRef(arg_ty, zcu)) {
720	const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
721	_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
722	llvm_arg = ptr;
723	}
724
725	const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);
726	const array_ty = try o.builder.arrayType(count, float_ty);
727
728	const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
729	try llvm_args.append(loaded);
730	},
731	.i32_array, .i64_array => \|arr_len\| {
732	const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
733	const arg = args[it.zig_index - 1];
734	const arg_ty = self.typeOf(arg);
735	var llvm_arg = try self.resolveInst(arg);
736	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
737	if (!isByRef(arg_ty, zcu)) {
738	const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
739	_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
740	llvm_arg = ptr;
741	}
742
743	const array_ty =
744	try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
745	const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
746	try llvm_args.append(loaded);
747	},
748	};
749
750	{
751	// Add argument attributes.
752	it = iterateParamTypes(o, pt, fn_info);
753	it.llvm_index += @intFromBool(sret);
754	it.llvm_index += @intFromBool(err_return_tracing);
755	while (try it.next()) \|lowering\| switch (lowering) {
756	.byval => {
757	const param_index = it.zig_index - 1;
758	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
759	if (!isByRef(param_ty, zcu)) {
760	try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
761	}
762	},
763	.byref => {
764	const param_index = it.zig_index - 1;
765	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
766	const param_llvm_ty = try o.lowerType(pt, param_ty);
767	const alignment = param_ty.abiAlignment(zcu).toLlvm();
768	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
769	},
770	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
771	// No attributes needed for these.
772	.no_bits,
773	.abi_sized_int,
774	.multiple_llvm_types,
775	.float_array,
776	.i32_array,
777	.i64_array,
778	=> continue,
779
780	.slice => {
781	assert(!it.byval_attr);
782	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
783	const ptr_info = param_ty.ptrInfo(zcu);
784	const llvm_arg_i = it.llvm_index - 2;
785
786	if (math.cast(u5, it.zig_index - 1)) \|i\| {
787	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
788	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
789	}
790	}
791	if (param_ty.zigTypeTag(zcu) != .optional and
792	!ptr_info.flags.is_allowzero and
793	ptr_info.flags.address_space == .generic)
794	{
795	try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
796	}
797	if (ptr_info.flags.is_const) {
798	try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
799	}
800	const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) {
801	else => \|a\| .wrap(a.toLlvm()),
802	.none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)),
803	};
804	try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder);
805	},
806	};
807	}
808
809	const call = try self.wip.call(
810	switch (modifier) {
811	.auto, .never_inline => .normal,
812	.never_tail => .notail,
813	.always_tail => .musttail,
814	.no_suspend, .always_inline, .compile_time => unreachable,
815	},
816	toLlvmCallConvTag(fn_info.cc, target).?,
817	try attributes.finish(&o.builder),
818	try o.lowerType(pt, zig_fn_ty),
819	llvm_fn,
820	llvm_args.items,
821	"",
822	);
823
824	if (fn_info.return_type == .noreturn_type and modifier != .always_tail) {
825	return .none;
826	}
827
828	if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) {
829	return .none;
830	}
831
832	const llvm_ret_ty = try o.lowerType(pt, return_type);
833	if (ret_ptr) \|rp\| {
834	if (isByRef(return_type, zcu)) {
835	return rp;
836	} else {
837	// our by-ref status disagrees with sret so we must load.
838	const return_alignment = return_type.abiAlignment(zcu).toLlvm();
839	return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, "");
840	}
841	}
842
843	const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
844
845	if (abi_ret_ty != llvm_ret_ty) {
846	// In this case the function return type is honoring the calling convention by having
847	// a different LLVM type than the usual one. We solve this here at the callsite
848	// by using our canonical type, then loading it if necessary.
849	const alignment = return_type.abiAlignment(zcu).toLlvm();
850	const rp = try self.buildAlloca(abi_ret_ty, alignment);
851	_ = try self.wip.store(.normal, call, rp, alignment);
852	return if (isByRef(return_type, zcu))
853	rp
854	else
855	try self.wip.load(.normal, llvm_ret_ty, rp, alignment, "");
856	}
857
858	if (isByRef(return_type, zcu)) {
859	// our by-ref status disagrees with sret so we must allocate, store,
860	// and return the allocation pointer.
861	const alignment = return_type.abiAlignment(zcu).toLlvm();
862	const rp = try self.buildAlloca(llvm_ret_ty, alignment);
863	_ = try self.wip.store(.normal, call, rp, alignment);
864	return rp;
865	} else {
866	return call;
867	}
868	}
869
870	fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) !void {
871	const o = fg.object;
872	const pt = fg.pt;
873	const zcu = pt.zcu;
874	const target = zcu.getTarget();
875	const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin()));
876	const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;
877	const panic_global = try o.resolveLlvmFunction(pt, panic_func.owner_nav);
878
879	const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto;
880	if (has_err_trace) assert(fg.err_ret_trace != .none);
881	_ = try fg.wip.callIntrinsicAssumeCold();
882	_ = try fg.wip.call(
883	.normal,
884	toLlvmCallConvTag(fn_info.cc, target).?,
885	.none,
886	panic_global.typeOf(&o.builder),
887	panic_global.toValue(&o.builder),
888	if (has_err_trace) &.{fg.err_ret_trace} else &.{},
889	"",
890	);
891	_ = try fg.wip.@"unreachable"();
892	}
893
894	fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void {
895	const o = self.object;
896	const pt = self.pt;
897	const zcu = pt.zcu;
898	const ip = &zcu.intern_pool;
899	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
900	const ret_ty = self.typeOf(un_op);
901
902	if (self.ret_ptr != .none) {
903	const ptr_ty = try pt.singleMutPtrType(ret_ty);
904
905	const operand = try self.resolveInst(un_op);
906	const val_is_undef = if (try self.air.value(un_op, pt)) \|val\| val.isUndef(zcu) else false;
907	if (val_is_undef and safety) undef: {
908	const ptr_info = ptr_ty.ptrInfo(zcu);
909	const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
910	if (needs_bitmask) {
911	// TODO: only some bits are to be undef, we cannot write with a simple memset.
912	// meanwhile, ignore the write rather than stomping over valid bits.
913	// https://github.com/ziglang/zig/issues/15337
914	break :undef;
915	}
916	const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu));
917	_ = try self.wip.callMemSet(
918	self.ret_ptr,
919	ptr_ty.ptrAlignment(zcu).toLlvm(),
920	try o.builder.intValue(.i8, 0xaa),
921	len,
922	.normal,
923	self.disable_intrinsics,
924	);
925	const owner_mod = self.ownerModule();
926	if (owner_mod.valgrind) {
927	try self.valgrindMarkUndef(self.ret_ptr, len);
928	}
929	_ = try self.wip.retVoid();
930	return;
931	}
932
933	const unwrapped_operand = operand.unwrap();
934	const unwrapped_ret = self.ret_ptr.unwrap();
935
936	// Return value was stored previously
937	if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) {
938	_ = try self.wip.retVoid();
939	return;
940	}
941
942	try self.store(self.ret_ptr, ptr_ty, operand, .none);
943	_ = try self.wip.retVoid();
944	return;
945	}
946	const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
947	if (!ret_ty.hasRuntimeBits(zcu)) {
948	if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
949	// Functions with an empty error set are emitted with an error code
950	// return type and return zero so they can be function pointers coerced
951	// to functions that return anyerror.
952	_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0));
953	} else {
954	_ = try self.wip.retVoid();
955	}
956	return;
957	}
958
959	const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
960	const operand = try self.resolveInst(un_op);
961	const val_is_undef = if (try self.air.value(un_op, pt)) \|val\| val.isUndef(zcu) else false;
962	const alignment = ret_ty.abiAlignment(zcu).toLlvm();
963
964	if (val_is_undef and safety) {
965	const llvm_ret_ty = operand.typeOfWip(&self.wip);
966	const rp = try self.buildAlloca(llvm_ret_ty, alignment);
967	const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu));
968	_ = try self.wip.callMemSet(
969	rp,
970	alignment,
971	try o.builder.intValue(.i8, 0xaa),
972	len,
973	.normal,
974	self.disable_intrinsics,
975	);
976	const owner_mod = self.ownerModule();
977	if (owner_mod.valgrind) {
978	try self.valgrindMarkUndef(rp, len);
979	}
980	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
981	return;
982	}
983
984	if (isByRef(ret_ty, zcu)) {
985	// operand is a pointer however self.ret_ptr is null so that means
986	// we need to return a value.
987	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, ""));
988	return;
989	}
990
991	const llvm_ret_ty = operand.typeOfWip(&self.wip);
992	if (abi_ret_ty == llvm_ret_ty) {
993	_ = try self.wip.ret(operand);
994	return;
995	}
996
997	const rp = try self.buildAlloca(llvm_ret_ty, alignment);
998	_ = try self.wip.store(.normal, operand, rp, alignment);
999	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
1000	return;
1001	}
1002
1003	fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void {
1004	const o = self.object;
1005	const pt = self.pt;
1006	const zcu = pt.zcu;
1007	const ip = &zcu.intern_pool;
1008	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1009	const ptr_ty = self.typeOf(un_op);
1010	const ret_ty = ptr_ty.childType(zcu);
1011	const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1012	if (!ret_ty.hasRuntimeBits(zcu)) {
1013	if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
1014	// Functions with an empty error set are emitted with an error code
1015	// return type and return zero so they can be function pointers coerced
1016	// to functions that return anyerror.
1017	_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0));
1018	} else {
1019	_ = try self.wip.retVoid();
1020	}
1021	return;
1022	}
1023	if (self.ret_ptr != .none) {
1024	_ = try self.wip.retVoid();
1025	return;
1026	}
1027	const ptr = try self.resolveInst(un_op);
1028	const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
1029	const alignment = ret_ty.abiAlignment(zcu).toLlvm();
1030	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, ""));
1031	return;
1032	}
1033
1034	fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
1035	const o = self.object;
1036	const pt = self.pt;
1037	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1038	const list = try self.resolveInst(ty_op.operand);
1039	const arg_ty = ty_op.ty.toType();
1040	const llvm_arg_ty = try o.lowerType(pt, arg_ty);
1041
1042	return self.wip.vaArg(list, llvm_arg_ty, "");
1043	}
1044
1045	fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
1046	const o = self.object;
1047	const pt = self.pt;
1048	const zcu = pt.zcu;
1049	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1050	const src_list = try self.resolveInst(ty_op.operand);
1051	const va_list_ty = ty_op.ty.toType();
1052	const llvm_va_list_ty = try o.lowerType(pt, va_list_ty);
1053
1054	const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
1055	const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
1056
1057	_ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
1058	return if (isByRef(va_list_ty, zcu))
1059	dest_list
1060	else
1061	try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
1062	}
1063
1064	fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
1065	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1066	const src_list = try self.resolveInst(un_op);
1067
1068	_ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, "");
1069	return .none;
1070	}
1071
1072	fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
1073	const o = self.object;
1074	const pt = self.pt;
1075	const zcu = pt.zcu;
1076	const va_list_ty = self.typeOfIndex(inst);
1077	const llvm_va_list_ty = try o.lowerType(pt, va_list_ty);
1078
1079	const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
1080	const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
1081
1082	_ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
1083	return if (isByRef(va_list_ty, zcu))
1084	dest_list
1085	else
1086	try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
1087	}
1088
1089	fn airCmp(
1090	self: *FuncGen,
1091	inst: Air.Inst.Index,
1092	op: math.CompareOperator,
1093	fast: Builder.FastMathKind,
1094	) !Builder.Value {
1095	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
1096	const lhs = try self.resolveInst(bin_op.lhs);
1097	const rhs = try self.resolveInst(bin_op.rhs);
1098	const operand_ty = self.typeOf(bin_op.lhs);
1099
1100	return self.cmp(fast, op, operand_ty, lhs, rhs);
1101	}
1102
1103	fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
1104	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
1105	const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
1106
1107	const lhs = try self.resolveInst(extra.lhs);
1108	const rhs = try self.resolveInst(extra.rhs);
1109	const vec_ty = self.typeOf(extra.lhs);
1110	const cmp_op = extra.compareOperator();
1111
1112	return self.cmp(fast, cmp_op, vec_ty, lhs, rhs);
1113	}
1114
1115	fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
1116	const o = self.object;
1117	const pt = self.pt;
1118	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1119	const operand = try self.resolveInst(un_op);
1120	const llvm_fn = try o.getCmpLtErrorsLenFunction(pt);
1121	return self.wip.call(
1122	.normal,
1123	.fastcc,
1124	.none,
1125	llvm_fn.typeOf(&o.builder),
1126	llvm_fn.toValue(&o.builder),
1127	&.{operand},
1128	"",
1129	);
1130	}
1131
1132	fn cmp(
1133	self: *FuncGen,
1134	fast: Builder.FastMathKind,
1135	op: math.CompareOperator,
1136	operand_ty: Type,
1137	lhs: Builder.Value,
1138	rhs: Builder.Value,
1139	) Allocator.Error!Builder.Value {
1140	const o = self.object;
1141	const pt = self.pt;
1142	const zcu = pt.zcu;
1143	const scalar_ty = operand_ty.scalarType(zcu);
1144	const int_ty = switch (scalar_ty.zigTypeTag(zcu)) {
1145	.@"enum" => scalar_ty.intTagType(zcu),
1146	.int, .bool, .pointer, .error_set => scalar_ty,
1147	.optional => blk: {
1148	const payload_ty = operand_ty.optionalChild(zcu);
1149	if (!payload_ty.hasRuntimeBits(zcu) or
1150	operand_ty.optionalReprIsPayload(zcu))
1151	{
1152	break :blk operand_ty;
1153	}
1154	// We need to emit instructions to check for equality/inequality
1155	// of optionals that are not pointers.
1156	const is_by_ref = isByRef(scalar_ty, zcu);
1157	const opt_llvm_ty = try o.lowerType(pt, scalar_ty);
1158	const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal);
1159	const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal);
1160	const llvm_i2 = try o.builder.intType(2);
1161	const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, "");
1162	const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, "");
1163	const lhs_shifted = try self.wip.bin(.shl, lhs_non_null_i2, try o.builder.intValue(llvm_i2, 1), "");
1164	const lhs_rhs_ored = try self.wip.bin(.@"or", lhs_shifted, rhs_non_null_i2, "");
1165	const both_null_block = try self.wip.block(1, "BothNull");
1166	const mixed_block = try self.wip.block(1, "Mixed");
1167	const both_pl_block = try self.wip.block(1, "BothNonNull");
1168	const end_block = try self.wip.block(3, "End");
1169	var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none);
1170	defer wip_switch.finish(&self.wip);
1171	try wip_switch.addCase(
1172	try o.builder.intConst(llvm_i2, 0b00),
1173	both_null_block,
1174	&self.wip,
1175	);
1176	try wip_switch.addCase(
1177	try o.builder.intConst(llvm_i2, 0b11),
1178	both_pl_block,
1179	&self.wip,
1180	);
1181
1182	self.wip.cursor = .{ .block = both_null_block };
1183	_ = try self.wip.br(end_block);
1184
1185	self.wip.cursor = .{ .block = mixed_block };
1186	_ = try self.wip.br(end_block);
1187
1188	self.wip.cursor = .{ .block = both_pl_block };
1189	const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true);
1190	const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true);
1191	const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload);
1192	_ = try self.wip.br(end_block);
1193	const both_pl_block_end = self.wip.cursor.block;
1194
1195	self.wip.cursor = .{ .block = end_block };
1196	const llvm_i1_0 = Builder.Value.false;
1197	const llvm_i1_1 = Builder.Value.true;
1198	const incoming_values: [3]Builder.Value = .{
1199	switch (op) {
1200	.eq => llvm_i1_1,
1201	.neq => llvm_i1_0,
1202	else => unreachable,
1203	},
1204	switch (op) {
1205	.eq => llvm_i1_0,
1206	.neq => llvm_i1_1,
1207	else => unreachable,
1208	},
1209	payload_cmp,
1210	};
1211
1212	const phi = try self.wip.phi(.i1, "");
1213	phi.finish(
1214	&incoming_values,
1215	&.{ both_null_block, mixed_block, both_pl_block_end },
1216	&self.wip,
1217	);
1218	return phi.toValue();
1219	},
1220	.float => return self.buildFloatCmp(fast, op, operand_ty, .{ lhs, rhs }),
1221	.@"struct", .@"union" => scalar_ty.bitpackBackingInt(zcu),
1222	else => unreachable,
1223	};
1224	const is_signed = int_ty.isSignedInt(zcu);
1225	const cond: Builder.IntegerCondition = switch (op) {
1226	.eq => .eq,
1227	.neq => .ne,
1228	.lt => if (is_signed) .slt else .ult,
1229	.lte => if (is_signed) .sle else .ule,
1230	.gt => if (is_signed) .sgt else .ugt,
1231	.gte => if (is_signed) .sge else .uge,
1232	};
1233	return self.wip.icmp(cond, lhs, rhs, "");
1234	}
1235
1236	fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
1237	const block = self.air.unwrapBlock(inst);
1238	return self.lowerBlock(inst, null, block.body);
1239	}
1240
1241	fn lowerBlock(
1242	self: *FuncGen,
1243	inst: Air.Inst.Index,
1244	maybe_inline_func: ?InternPool.Index,
1245	body: []const Air.Inst.Index,
1246	) !Builder.Value {
1247	const o = self.object;
1248	const pt = self.pt;
1249	const zcu = pt.zcu;
1250	const inst_ty = self.typeOfIndex(inst);
1251
1252	if (inst_ty.isNoReturn(zcu)) {
1253	try self.genBodyDebugScope(maybe_inline_func, body, .none);
1254	return .none;
1255	}
1256
1257	const have_block_result = inst_ty.hasRuntimeBits(zcu);
1258
1259	var breaks: BreakList = if (have_block_result) .{ .list = .{} } else .{ .len = 0 };
1260	defer if (have_block_result) breaks.list.deinit(self.gpa);
1261
1262	const parent_bb = try self.wip.block(0, "Block");
1263	try self.blocks.putNoClobber(self.gpa, inst, .{
1264	.parent_bb = parent_bb,
1265	.breaks = &breaks,
1266	});
1267	defer assert(self.blocks.remove(inst));
1268
1269	try self.genBodyDebugScope(maybe_inline_func, body, .none);
1270
1271	self.wip.cursor = .{ .block = parent_bb };
1272
1273	// Create a phi node only if the block returns a value.
1274	if (have_block_result) {
1275	const raw_llvm_ty = try o.lowerType(pt, inst_ty);
1276	const llvm_ty: Builder.Type = ty: {
1277	// If the zig tag type is a function, this represents an actual function body; not
1278	// a pointer to it. LLVM IR allows the call instruction to use function bodies instead
1279	// of function pointers, however the phi makes it a runtime value and therefore
1280	// the LLVM type has to be wrapped in a pointer.
1281	if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) {
1282	break :ty .ptr;
1283	}
1284	break :ty raw_llvm_ty;
1285	};
1286
1287	parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);
1288	const phi = try self.wip.phi(llvm_ty, "");
1289	phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);
1290	return phi.toValue();
1291	} else {
1292	parent_bb.ptr(&self.wip).incoming = @intCast(breaks.len);
1293	return .none;
1294	}
1295	}
1296
1297	fn airBr(self: *FuncGen, inst: Air.Inst.Index) !void {
1298	const zcu = self.pt.zcu;
1299	const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
1300	const block = self.blocks.get(branch.block_inst).?;
1301
1302	// Add the values to the lists only if the break provides a value.
1303	const operand_ty = self.typeOf(branch.operand);
1304	if (operand_ty.hasRuntimeBits(zcu)) {
1305	const val = try self.resolveInst(branch.operand);
1306
1307	// For the phi node, we need the basic blocks and the values of the
1308	// break instructions.
1309	try block.breaks.list.append(self.gpa, .{ .bb = self.wip.cursor.block, .val = val });
1310	} else block.breaks.len += 1;
1311	_ = try self.wip.br(block.parent_bb);
1312	}
1313
1314	fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) !void {
1315	const repeat = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat;
1316	const loop_bb = self.loops.get(repeat.loop_inst).?;
1317	loop_bb.ptr(&self.wip).incoming += 1;
1318	_ = try self.wip.br(loop_bb);
1319	}
1320
1321	fn lowerSwitchDispatch(
1322	self: *FuncGen,
1323	switch_inst: Air.Inst.Index,
1324	cond_ref: Air.Inst.Ref,
1325	dispatch_info: SwitchDispatchInfo,
1326	) !void {
1327	const o = self.object;
1328	const pt = self.pt;
1329	const zcu = pt.zcu;
1330	const cond_ty = self.typeOf(cond_ref);
1331	const switch_br = self.air.unwrapSwitch(switch_inst);
1332
1333	if (try self.air.value(cond_ref, pt)) \|cond_val\| {
1334	// Comptime-known dispatch. Iterate the cases to find the correct
1335	// one, and branch to the corresponding element of `case_blocks`.
1336	var it = switch_br.iterateCases();
1337	const target_case_idx = target: while (it.next()) \|case\| {
1338	for (case.items) \|item\| {
1339	const val = Value.fromInterned(item.toInterned().?);
1340	if (cond_val.compareHetero(.eq, val, zcu)) break :target case.idx;
1341	}
1342	for (case.ranges) \|range\| {
1343	const low = Value.fromInterned(range[0].toInterned().?);
1344	const high = Value.fromInterned(range[1].toInterned().?);
1345	if (cond_val.compareHetero(.gte, low, zcu) and
1346	cond_val.compareHetero(.lte, high, zcu))
1347	{
1348	break :target case.idx;
1349	}
1350	}
1351	} else dispatch_info.case_blocks.len - 1;
1352	const target_block = dispatch_info.case_blocks[target_case_idx];
1353	target_block.ptr(&self.wip).incoming += 1;
1354	_ = try self.wip.br(target_block);
1355	return;
1356	}
1357
1358	// Runtime-known dispatch.
1359	const cond = try self.resolveInst(cond_ref);
1360
1361	if (dispatch_info.jmp_table) \|jmp_table\| {
1362	// We should use the constructed jump table.
1363	// First, check the bounds to branch to the `else` case if needed.
1364	const inbounds = try self.wip.bin(
1365	.@"and",
1366	try self.cmp(.normal, .gte, cond_ty, cond, jmp_table.min.toValue()),
1367	try self.cmp(.normal, .lte, cond_ty, cond, jmp_table.max.toValue()),
1368	"",
1369	);
1370	const jmp_table_block = try self.wip.block(1, "Then");
1371	const else_block = dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1];
1372	else_block.ptr(&self.wip).incoming += 1;
1373	_ = try self.wip.brCond(inbounds, jmp_table_block, else_block, switch (jmp_table.in_bounds_hint) {
1374	.none => .none,
1375	.unpredictable => .unpredictable,
1376	.likely => .then_likely,
1377	.unlikely => .else_likely,
1378	});
1379
1380	self.wip.cursor = .{ .block = jmp_table_block };
1381
1382	// Figure out the list of blocks we might branch to.
1383	// This includes all case blocks, but it might not include the `else` block if
1384	// the table is dense.
1385	const target_blocks_len = dispatch_info.case_blocks.len - @intFromBool(!jmp_table.table_includes_else);
1386	const target_blocks = dispatch_info.case_blocks[0..target_blocks_len];
1387
1388	// Make sure to cast the index to a usize so it's not treated as negative!
1389	const table_index = try self.wip.conv(
1390	.unsigned,
1391	try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),
1392	try o.lowerType(pt, .usize),
1393	"",
1394	);
1395	const target_ptr_ptr = try self.wip.gep(
1396	.inbounds,
1397	.ptr,
1398	jmp_table.table.toValue(),
1399	&.{table_index},
1400	"",
1401	);
1402	const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, "");
1403
1404	// Do the branch!
1405	_ = try self.wip.indirectbr(target_ptr, target_blocks);
1406
1407	// Mark all target blocks as having one more incoming branch.
1408	for (target_blocks) \|case_block\| {
1409	case_block.ptr(&self.wip).incoming += 1;
1410	}
1411
1412	return;
1413	}
1414
1415	// We must lower to an actual LLVM `switch` instruction.
1416	// The switch prongs will correspond to our scalar cases. Ranges will
1417	// be handled by conditional branches in the `else` prong.
1418
1419	const llvm_usize = try o.lowerType(pt, Type.usize);
1420	const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder))
1421	try self.wip.cast(.ptrtoint, cond, llvm_usize, "")
1422	else
1423	cond;
1424
1425	const llvm_cases_len, const last_range_case = info: {
1426	var llvm_cases_len: u32 = 0;
1427	var last_range_case: ?u32 = null;
1428	var it = switch_br.iterateCases();
1429	while (it.next()) \|case\| {
1430	if (case.ranges.len > 0) last_range_case = case.idx;
1431	llvm_cases_len += @intCast(case.items.len);
1432	}
1433	break :info .{ llvm_cases_len, last_range_case };
1434	};
1435
1436	// The `else` of the LLVM `switch` is the actual `else` prong only
1437	// if there are no ranges. Otherwise, the `else` will have a
1438	// conditional chain before the "true" `else` prong.
1439	const llvm_else_block = if (last_range_case == null)
1440	dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
1441	else
1442	try self.wip.block(0, "RangeTest");
1443
1444	llvm_else_block.ptr(&self.wip).incoming += 1;
1445
1446	var wip_switch = try self.wip.@"switch"(cond_int, llvm_else_block, llvm_cases_len, dispatch_info.switch_weights);
1447	defer wip_switch.finish(&self.wip);
1448
1449	// Construct the actual cases. Set the cursor to the `else` block so
1450	// we can construct ranges at the same time as scalar cases.
1451	self.wip.cursor = .{ .block = llvm_else_block };
1452
1453	var it = switch_br.iterateCases();
1454	while (it.next()) \|case\| {
1455	const case_block = dispatch_info.case_blocks[case.idx];
1456
1457	for (case.items) \|item\| {
1458	const llvm_item = (try self.resolveInst(item)).toConst().?;
1459	const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder))
1460	try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize)
1461	else
1462	llvm_item;
1463	try wip_switch.addCase(llvm_int_item, case_block, &self.wip);
1464	}
1465	case_block.ptr(&self.wip).incoming += @intCast(case.items.len);
1466
1467	if (case.ranges.len == 0) continue;
1468
1469	// Add a conditional for the ranges, directing to the relevant bb.
1470	// We don't need to consider `cold` branch hints since that information is stored
1471	// in the target bb body, but we do care about likely/unlikely/unpredictable.
1472
1473	const hint = switch_br.getHint(case.idx);
1474
1475	var range_cond: ?Builder.Value = null;
1476	for (case.ranges) \|range\| {
1477	const llvm_min = try self.resolveInst(range[0]);
1478	const llvm_max = try self.resolveInst(range[1]);
1479	const cond_part = try self.wip.bin(
1480	.@"and",
1481	try self.cmp(.normal, .gte, cond_ty, cond, llvm_min),
1482	try self.cmp(.normal, .lte, cond_ty, cond, llvm_max),
1483	"",
1484	);
1485	if (range_cond) \|prev\| {
1486	range_cond = try self.wip.bin(.@"or", prev, cond_part, "");
1487	} else range_cond = cond_part;
1488	}
1489
1490	// If the check fails, we either branch to the "true" `else` case,
1491	// or to the next range condition.
1492	const range_else_block = if (case.idx == last_range_case.?)
1493	dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
1494	else
1495	try self.wip.block(0, "RangeTest");
1496
1497	_ = try self.wip.brCond(range_cond.?, case_block, range_else_block, switch (hint) {
1498	.none, .cold => .none,
1499	.unpredictable => .unpredictable,
1500	.likely => .then_likely,
1501	.unlikely => .else_likely,
1502	});
1503	case_block.ptr(&self.wip).incoming += 1;
1504	range_else_block.ptr(&self.wip).incoming += 1;
1505
1506	// Construct the next range conditional (if any) in the false branch.
1507	self.wip.cursor = .{ .block = range_else_block };
1508	}
1509	}
1510
1511	fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) !void {
1512	const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
1513	const dispatch_info = self.switch_dispatch_info.get(br.block_inst).?;
1514	return self.lowerSwitchDispatch(br.block_inst, br.operand, dispatch_info);
1515	}
1516
1517	fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void {
1518	const cond_br = self.air.unwrapCondBr(inst);
1519	const cond = try self.resolveInst(cond_br.condition);
1520	const then_body = cond_br.then_body;
1521	const else_body = cond_br.else_body;
1522
1523	const Hint = enum {
1524	none,
1525	unpredictable,
1526	then_likely,
1527	else_likely,
1528	then_cold,
1529	else_cold,
1530	};
1531	const hint: Hint = switch (cond_br.branch_hints.true) {
1532	.none => switch (cond_br.branch_hints.false) {
1533	.none => .none,
1534	.likely => .else_likely,
1535	.unlikely => .then_likely,
1536	.cold => .else_cold,
1537	.unpredictable => .unpredictable,
1538	},
1539	.likely => switch (cond_br.branch_hints.false) {
1540	.none => .then_likely,
1541	.likely => .unpredictable,
1542	.unlikely => .then_likely,
1543	.cold => .else_cold,
1544	.unpredictable => .unpredictable,
1545	},
1546	.unlikely => switch (cond_br.branch_hints.false) {
1547	.none => .else_likely,
1548	.likely => .else_likely,
1549	.unlikely => .unpredictable,
1550	.cold => .else_cold,
1551	.unpredictable => .unpredictable,
1552	},
1553	.cold => .then_cold,
1554	.unpredictable => .unpredictable,
1555	};
1556
1557	const then_block = try self.wip.block(1, "Then");
1558	const else_block = try self.wip.block(1, "Else");
1559	_ = try self.wip.brCond(cond, then_block, else_block, switch (hint) {
1560	.none, .then_cold, .else_cold => .none,
1561	.unpredictable => .unpredictable,
1562	.then_likely => .then_likely,
1563	.else_likely => .else_likely,
1564	});
1565
1566	self.wip.cursor = .{ .block = then_block };
1567	if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold();
1568	try self.genBodyDebugScope(null, then_body, cond_br.branch_hints.then_cov);
1569
1570	self.wip.cursor = .{ .block = else_block };
1571	if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold();
1572	try self.genBodyDebugScope(null, else_body, cond_br.branch_hints.else_cov);
1573
1574	// No need to reset the insert cursor since this instruction is noreturn.
1575	}
1576
1577	fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
1578	const unwrapped_try = self.air.unwrapTry(inst);
1579	const err_union = try self.resolveInst(unwrapped_try.error_union);
1580	const body = unwrapped_try.else_body;
1581	const err_union_ty = self.typeOf(unwrapped_try.error_union);
1582	const is_unused = self.liveness.isUnused(inst);
1583	return lowerTry(self, err_union, body, err_union_ty, false, .none, false, is_unused, err_cold);
1584	}
1585
1586	fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
1587	const zcu = self.pt.zcu;
1588	const unwrapped_try = self.air.unwrapTryPtr(inst);
1589	const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr);
1590	const body = unwrapped_try.else_body;
1591	const err_union_ptr_ty = self.typeOf(unwrapped_try.error_union_ptr);
1592	const err_union_ty = err_union_ptr_ty.childType(zcu);
1593	const is_unused = self.liveness.isUnused(inst);
1594
1595	self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu));
1596
1597	return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), true, is_unused, err_cold);
1598	}
1599
1600	fn lowerTry(
1601	fg: *FuncGen,
1602	err_union: Builder.Value,
1603	body: []const Air.Inst.Index,
1604	err_union_ty: Type,
1605	operand_is_ptr: bool,
1606	operand_ptr_align: InternPool.Alignment,
1607	can_elide_load: bool,
1608	is_unused: bool,
1609	err_cold: bool,
1610	) !Builder.Value {
1611	const o = fg.object;
1612	const pt = fg.pt;
1613	const zcu = pt.zcu;
1614	const payload_ty = err_union_ty.errorUnionPayload(zcu);
1615	const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
1616	const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
1617	const error_type = try o.errorIntType(pt);
1618
1619	const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{
1620	operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)),
1621	operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)),
1622	} else .{ .none, .none };
1623
1624	if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
1625	const loaded = loaded: {
1626	const access_kind: Builder.MemoryAccessKind =
1627	if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
1628
1629	if (!payload_has_bits) {
1630	break :loaded if (operand_is_ptr)
1631	try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "")
1632	else
1633	err_union;
1634	}
1635	const err_field_index = try errUnionErrorFieldIndex(payload_ty, pt);
1636	if (operand_is_ptr or isByRef(err_union_ty, zcu)) {
1637	const err_field_ptr =
1638	try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, "");
1639	break :loaded try fg.wip.load(
1640	if (operand_is_ptr) access_kind else .normal,
1641	error_type,
1642	err_field_ptr,
1643	err_set_align.toLlvm(),
1644	"",
1645	);
1646	}
1647	break :loaded try fg.wip.extractValue(err_union, &.{err_field_index}, "");
1648	};
1649	const zero = try o.builder.intValue(error_type, 0);
1650	const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
1651
1652	const return_block = try fg.wip.block(1, "TryRet");
1653	const continue_block = try fg.wip.block(1, "TryCont");
1654	_ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely);
1655
1656	fg.wip.cursor = .{ .block = return_block };
1657	if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold();
1658	try fg.genBodyDebugScope(null, body, .poi);
1659
1660	fg.wip.cursor = .{ .block = continue_block };
1661	}
1662	if (is_unused) return .none;
1663	if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;
1664	const offset = try errUnionPayloadFieldIndex(payload_ty, pt);
1665	if (operand_is_ptr) {
1666	return fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");
1667	} else if (isByRef(err_union_ty, zcu)) {
1668	const payload_ptr = try fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");
1669	if (isByRef(payload_ty, zcu)) {
1670	if (can_elide_load)
1671	return payload_ptr;
1672
1673	return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
1674	}
1675	const load_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
1676	return fg.wip.load(.normal, load_ty, payload_ptr, payload_align.toLlvm(), "");
1677	}
1678	return fg.wip.extractValue(err_union, &.{offset}, "");
1679	}
1680
1681	fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void {
1682	const o = self.object;
1683	const pt = self.pt;
1684	const zcu = pt.zcu;
1685
1686	const switch_br = self.air.unwrapSwitch(inst);
1687
1688	// For `loop_switch_br`, we need these BBs prepared ahead of time to generate dispatches.
1689	// For `switch_br`, they allow us to sometimes generate better IR by sharing a BB between
1690	// scalar and range cases in the same prong.
1691	// +1 for `else` case. This is not the same as the LLVM `else` prong, as that may first contain
1692	// conditionals to handle ranges.
1693	const case_blocks = try self.gpa.alloc(Builder.Function.Block.Index, switch_br.cases_len + 1);
1694	defer self.gpa.free(case_blocks);
1695	// We set incoming as 0 for now, and increment it as we construct dispatches.
1696	for (case_blocks[0 .. case_blocks.len - 1]) \|b\| b. = try self.wip.block(0, "Case");
1697	case_blocks[case_blocks.len - 1] = try self.wip.block(0, "Default");
1698
1699	// There's a special case here to manually generate a jump table in some cases.
1700	//
1701	// Labeled switch in Zig is intended to follow the "direct threading" pattern. We would ideally use a jump
1702	// table, and each `continue` has its own indirect `jmp`, to allow the branch predictor to more accurately
1703	// use data patterns to predict future dispatches. The problem, however, is that LLVM emits fascinatingly
1704	// bad asm for this. Not only does it not share the jump table -- which we really need it to do to prevent
1705	// destroying the cache -- but it also actually generates slightly different jump tables for each case,
1706	// and a separate conditional branch beforehand to handle dispatching back to the case we're currently
1707	// within(!!).
1708	//
1709	// This asm is really, really, not what we want. As such, we will construct the jump table manually where
1710	// appropriate (the values are dense and relatively few), and use it when lowering dispatches.
1711
1712	const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: {
1713	if (!is_dispatch_loop) break :jmp_table null;
1714
1715	// Workaround for:
1716	// * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/lib/MC/WasmObjectWriter.cpp#L560
1717	// * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll
1718	if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null;
1719
1720	// On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just
1721	// about acceptable - it won't fill L1d cache on most CPUs.
1722	const max_table_len = 1024;
1723
1724	const cond_ty = self.typeOf(switch_br.operand);
1725	switch (cond_ty.zigTypeTag(zcu)) {
1726	.bool, .pointer => break :jmp_table null,
1727	.@"enum", .int, .error_set, .@"struct", .@"union" => {},
1728	else => unreachable,
1729	}
1730
1731	if (cond_ty.intInfo(zcu).signedness == .signed) break :jmp_table null;
1732
1733	// Don't worry about the size of the type -- it's irrelevant, because the prong values could be fairly dense.
1734	// If they are, then we will construct a jump table.
1735	const min, const max = self.switchCaseItemRange(switch_br) orelse break :jmp_table null;
1736	const min_int = min.getUnsignedInt(zcu) orelse break :jmp_table null;
1737	const max_int = max.getUnsignedInt(zcu) orelse break :jmp_table null;
1738	const table_len = max_int - min_int + 1;
1739	if (table_len > max_table_len) break :jmp_table null;
1740
1741	const table_elems = try self.gpa.alloc(Builder.Constant, @intCast(table_len));
1742	defer self.gpa.free(table_elems);
1743
1744	// Set them all to the `else` branch, then iterate over the AIR switch
1745	// and replace all values which correspond to other prongs.
1746	@memset(table_elems, try o.builder.blockAddrConst(
1747	self.wip.function,
1748	case_blocks[case_blocks.len - 1],
1749	));
1750	var item_count: u32 = 0;
1751	var it = switch_br.iterateCases();
1752	while (it.next()) \|case\| {
1753	const case_block = case_blocks[case.idx];
1754	const case_block_addr = try o.builder.blockAddrConst(
1755	self.wip.function,
1756	case_block,
1757	);
1758	for (case.items) \|item\| {
1759	const val = Value.fromInterned(item.toInterned().?);
1760	const table_idx = val.toUnsignedInt(zcu) - min_int;
1761	table_elems[@intCast(table_idx)] = case_block_addr;
1762	item_count += 1;
1763	}
1764	for (case.ranges) \|range\| {
1765	const low = Value.fromInterned(range[0].toInterned().?);
1766	const high = Value.fromInterned(range[1].toInterned().?);
1767	const low_idx = low.toUnsignedInt(zcu) - min_int;
1768	const high_idx = high.toUnsignedInt(zcu) - min_int;
1769	@memset(table_elems[@intCast(low_idx)..@intCast(high_idx + 1)], case_block_addr);
1770	item_count += @intCast(high_idx + 1 - low_idx);
1771	}
1772	}
1773
1774	const table_llvm_ty = try o.builder.arrayType(table_elems.len, .ptr);
1775	const table_val = try o.builder.arrayConst(table_llvm_ty, table_elems);
1776
1777	const table_variable = try o.builder.addVariable(
1778	try o.builder.strtabStringFmt("__jmptab_{d}", .{@intFromEnum(inst)}),
1779	table_llvm_ty,
1780	.default,
1781	);
1782	try table_variable.setInitializer(table_val, &o.builder);
1783	table_variable.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
1784	table_variable.setUnnamedAddr(.unnamed_addr, &o.builder);
1785
1786	const table_includes_else = item_count != table_len;
1787
1788	break :jmp_table .{
1789	.min = try o.lowerValue(pt, min.toIntern()),
1790	.max = try o.lowerValue(pt, max.toIntern()),
1791	.in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {
1792	.none, .cold => .none,
1793	.unpredictable => .unpredictable,
1794	.likely => .likely,
1795	.unlikely => .unlikely,
1796	},
1797	.table = table_variable.toConst(&o.builder),
1798	.table_includes_else = table_includes_else,
1799	};
1800	};
1801
1802	const weights: Builder.Function.Instruction.BrCond.Weights = weights: {
1803	if (jmp_table != null) break :weights .none; // not used
1804
1805	// First pass. If any weights are `.unpredictable`, unpredictable.
1806	// If all are `.none` or `.cold`, none.
1807	var any_likely = false;
1808	for (0..switch_br.cases_len) \|case_idx\| {
1809	switch (switch_br.getHint(@intCast(case_idx))) {
1810	.none, .cold => {},
1811	.likely, .unlikely => any_likely = true,
1812	.unpredictable => break :weights .unpredictable,
1813	}
1814	}
1815	switch (switch_br.getElseHint()) {
1816	.none, .cold => {},
1817	.likely, .unlikely => any_likely = true,
1818	.unpredictable => break :weights .unpredictable,
1819	}
1820	if (!any_likely) break :weights .none;
1821
1822	const llvm_cases_len = llvm_cases_len: {
1823	var len: u32 = 0;
1824	var it = switch_br.iterateCases();
1825	while (it.next()) \|case\| len += @intCast(case.items.len);
1826	break :llvm_cases_len len;
1827	};
1828
1829	var weights = try self.gpa.alloc(Builder.Metadata, 1 + llvm_cases_len + 1);
1830	defer self.gpa.free(weights);
1831	var weight_idx: usize = 0;
1832
1833	const branch_weights_str = try o.builder.metadataString("branch_weights");
1834	weights[weight_idx] = branch_weights_str.toMetadata();
1835	weight_idx += 1;
1836
1837	const else_weight: u32 = switch (switch_br.getElseHint()) {
1838	.unpredictable => unreachable,
1839	.none, .cold => 1000,
1840	.likely => 2000,
1841	.unlikely => 1,
1842	};
1843	weights[weight_idx] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight));
1844	weight_idx += 1;
1845
1846	var it = switch_br.iterateCases();
1847	while (it.next()) \|case\| {
1848	const weight_val: u32 = switch (switch_br.getHint(case.idx)) {
1849	.unpredictable => unreachable,
1850	.none, .cold => 1000,
1851	.likely => 2000,
1852	.unlikely => 1,
1853	};
1854	const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val));
1855	@memset(weights[weight_idx..][0..case.items.len], weight_meta);
1856	weight_idx += case.items.len;
1857	}
1858
1859	assert(weight_idx == weights.len);
1860	break :weights .fromMetadata(try o.builder.metadataTuple(weights));
1861	};
1862
1863	const dispatch_info: SwitchDispatchInfo = .{
1864	.case_blocks = case_blocks,
1865	.switch_weights = weights,
1866	.jmp_table = jmp_table,
1867	};
1868
1869	if (is_dispatch_loop) {
1870	try self.switch_dispatch_info.putNoClobber(self.gpa, inst, dispatch_info);
1871	}
1872	defer if (is_dispatch_loop) {
1873	assert(self.switch_dispatch_info.remove(inst));
1874	};
1875
1876	// Generate the initial dispatch.
1877	// If this is a simple `switch_br`, this is the only dispatch.
1878	try self.lowerSwitchDispatch(inst, switch_br.operand, dispatch_info);
1879
1880	// Iterate the cases and generate their bodies.
1881	var it = switch_br.iterateCases();
1882	while (it.next()) \|case\| {
1883	const case_block = case_blocks[case.idx];
1884	self.wip.cursor = .{ .block = case_block };
1885	if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold();
1886	try self.genBodyDebugScope(null, case.body, .none);
1887	}
1888	self.wip.cursor = .{ .block = case_blocks[case_blocks.len - 1] };
1889	const else_body = it.elseBody();
1890	if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold();
1891	if (else_body.len > 0) {
1892	try self.genBodyDebugScope(null, it.elseBody(), .none);
1893	} else {
1894	_ = try self.wip.@"unreachable"();
1895	}
1896	}
1897
1898	fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) ?[2]Value {
1899	const zcu = self.pt.zcu;
1900	var it = switch_br.iterateCases();
1901	var min: ?Value = null;
1902	var max: ?Value = null;
1903	while (it.next()) \|case\| {
1904	for (case.items) \|item\| {
1905	const val = Value.fromInterned(item.toInterned().?);
1906	const low = if (min) \|m\| val.compareHetero(.lt, m, zcu) else true;
1907	const high = if (max) \|m\| val.compareHetero(.gt, m, zcu) else true;
1908	if (low) min = val;
1909	if (high) max = val;
1910	}
1911	for (case.ranges) \|range\| {
1912	const vals: [2]Value = .{
1913	Value.fromInterned(range[0].toInterned().?),
1914	Value.fromInterned(range[1].toInterned().?),
1915	};
1916	const low = if (min) \|m\| vals[0].compareHetero(.lt, m, zcu) else true;
1917	const high = if (max) \|m\| vals[1].compareHetero(.gt, m, zcu) else true;
1918	if (low) min = vals[0];
1919	if (high) max = vals[1];
1920	}
1921	}
1922	if (min == null) {
1923	assert(max == null);
1924	return null;
1925	}
1926	return .{ min.?, max.? };
1927	}
1928
1929	fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void {
1930	const block = self.air.unwrapBlock(inst);
1931	const body = block.body;
1932	const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time
1933	_ = try self.wip.br(loop_block);
1934
1935	try self.loops.putNoClobber(self.gpa, inst, loop_block);
1936	defer assert(self.loops.remove(inst));
1937
1938	self.wip.cursor = .{ .block = loop_block };
1939	try self.genBodyDebugScope(null, body, .none);
1940	}
1941
1942	fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
1943	const o = self.object;
1944	const pt = self.pt;
1945	const zcu = pt.zcu;
1946	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1947	const operand_ty = self.typeOf(ty_op.operand);
1948	const array_ty = operand_ty.childType(zcu);
1949	const llvm_usize = try o.lowerType(pt, Type.usize);
1950	const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
1951	const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));
1952	const operand = try self.resolveInst(ty_op.operand);
1953	if (!array_ty.hasRuntimeBits(zcu))
1954	return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
1955	const ptr = try self.wip.gep(.inbounds, try o.lowerType(pt, array_ty), operand, &.{
1956	try o.builder.intValue(llvm_usize, 0), try o.builder.intValue(llvm_usize, 0),
1957	}, "");
1958	return self.wip.buildAggregate(slice_llvm_ty, &.{ ptr, len }, "");
1959	}
1960
1961	fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
1962	const o = self.object;
1963	const pt = self.pt;
1964	const zcu = pt.zcu;
1965	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1966
1967	const operand = try self.resolveInst(ty_op.operand);
1968	const operand_ty = self.typeOf(ty_op.operand);
1969	const operand_scalar_ty = operand_ty.scalarType(zcu);
1970	const is_signed_int = operand_scalar_ty.isSignedInt(zcu);
1971
1972	const dest_ty = self.typeOfIndex(inst);
1973	const dest_scalar_ty = dest_ty.scalarType(zcu);
1974	const dest_llvm_ty = try o.lowerType(pt, dest_ty);
1975	const target = zcu.getTarget();
1976
1977	if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(
1978	if (is_signed_int) .signed else .unsigned,
1979	operand,
1980	dest_llvm_ty,
1981	"",
1982	);
1983
1984	const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu))) orelse {
1985	return self.todo("float_from_int from '{f}' without intrinsics", .{operand_scalar_ty.fmt(pt)});
1986	};
1987	const rt_int_ty = try o.builder.intType(rt_int_bits);
1988	var extended = try self.wip.conv(
1989	if (is_signed_int) .signed else .unsigned,
1990	operand,
1991	rt_int_ty,
1992	"",
1993	);
1994	const dest_bits = dest_scalar_ty.floatBits(target);
1995	const compiler_rt_operand_abbrev = compilerRtIntAbbrev(rt_int_bits);
1996	const compiler_rt_dest_abbrev = compilerRtFloatAbbrev(dest_bits);
1997	const sign_prefix = if (is_signed_int) "" else "un";
1998	const fn_name = try o.builder.strtabStringFmt("__float{s}{s}i{s}f", .{
1999	sign_prefix,
2000	compiler_rt_operand_abbrev,
2001	compiler_rt_dest_abbrev,
2002	});
2003
2004	var param_type = rt_int_ty;
2005	if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) {
2006	// On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
2007	// i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have.
2008	param_type = try o.builder.vectorType(.normal, 2, .i64);
2009	extended = try self.wip.cast(.bitcast, extended, param_type, "");
2010	}
2011
2012	const libc_fn = try self.getLibcFunction(fn_name, &.{param_type}, dest_llvm_ty);
2013	return self.wip.call(
2014	.normal,
2015	.ccc,
2016	.none,
2017	libc_fn.typeOf(&o.builder),
2018	libc_fn.toValue(&o.builder),
2019	&.{extended},
2020	"",
2021	);
2022	}
2023
2024	fn airIntFromFloat(
2025	self: *FuncGen,
2026	inst: Air.Inst.Index,
2027	fast: Builder.FastMathKind,
2028	) !Builder.Value {
2029	_ = fast;
2030
2031	const o = self.object;
2032	const pt = self.pt;
2033	const zcu = pt.zcu;
2034	const target = zcu.getTarget();
2035	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2036
2037	const operand = try self.resolveInst(ty_op.operand);
2038	const operand_ty = self.typeOf(ty_op.operand);
2039	const operand_scalar_ty = operand_ty.scalarType(zcu);
2040
2041	const dest_ty = self.typeOfIndex(inst);
2042	const dest_scalar_ty = dest_ty.scalarType(zcu);
2043	const dest_llvm_ty = try o.lowerType(pt, dest_ty);
2044
2045	if (intrinsicsAllowed(operand_scalar_ty, target)) {
2046	// TODO set fast math flag
2047	return self.wip.conv(
2048	if (dest_scalar_ty.isSignedInt(zcu)) .signed else .unsigned,
2049	operand,
2050	dest_llvm_ty,
2051	"",
2052	);
2053	}
2054
2055	const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu))) orelse {
2056	return self.todo("int_from_float to '{f}' without intrinsics", .{dest_scalar_ty.fmt(pt)});
2057	};
2058	const ret_ty = try o.builder.intType(rt_int_bits);
2059	const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: {
2060	// On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
2061	// i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have.
2062	break :b try o.builder.vectorType(.normal, 2, .i64);
2063	} else ret_ty;
2064
2065	const operand_bits = operand_scalar_ty.floatBits(target);
2066	const compiler_rt_operand_abbrev = compilerRtFloatAbbrev(operand_bits);
2067
2068	const compiler_rt_dest_abbrev = compilerRtIntAbbrev(rt_int_bits);
2069	const sign_prefix = if (dest_scalar_ty.isSignedInt(zcu)) "" else "uns";
2070
2071	const fn_name = try o.builder.strtabStringFmt("__fix{s}{s}f{s}i", .{
2072	sign_prefix,
2073	compiler_rt_operand_abbrev,
2074	compiler_rt_dest_abbrev,
2075	});
2076
2077	const operand_llvm_ty = try o.lowerType(pt, operand_ty);
2078	const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);
2079	var result = try self.wip.call(
2080	.normal,
2081	.ccc,
2082	.none,
2083	libc_fn.typeOf(&o.builder),
2084	libc_fn.toValue(&o.builder),
2085	&.{operand},
2086	"",
2087	);
2088
2089	if (libc_ret_ty != ret_ty) result = try self.wip.cast(.bitcast, result, ret_ty, "");
2090	if (ret_ty != dest_llvm_ty) result = try self.wip.cast(.trunc, result, dest_llvm_ty, "");
2091	return result;
2092	}
2093
2094	fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
2095	const zcu = fg.pt.zcu;
2096	return if (ty.isSlice(zcu)) fg.wip.extractValue(ptr, &.{0}, "") else ptr;
2097	}
2098
2099	fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
2100	const o = fg.object;
2101	const pt = fg.pt;
2102	const zcu = pt.zcu;
2103	const llvm_usize = try o.lowerType(pt, Type.usize);
2104	switch (ty.ptrSize(zcu)) {
2105	.slice => {
2106	const len = try fg.wip.extractValue(ptr, &.{1}, "");
2107	const elem_ty = ty.childType(zcu);
2108	const abi_size = elem_ty.abiSize(zcu);
2109	if (abi_size == 1) return len;
2110	const abi_size_llvm_val = try o.builder.intValue(llvm_usize, abi_size);
2111	return fg.wip.bin(.@"mul nuw", len, abi_size_llvm_val, "");
2112	},
2113	.one => {
2114	const array_ty = ty.childType(zcu);
2115	const elem_ty = array_ty.childType(zcu);
2116	const abi_size = elem_ty.abiSize(zcu);
2117	return o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu) * abi_size);
2118	},
2119	.many, .c => unreachable,
2120	}
2121	}
2122
2123	fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Value {
2124	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2125	const operand = try self.resolveInst(ty_op.operand);
2126	return self.wip.extractValue(operand, &.{index}, "");
2127	}
2128
2129	fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: c_uint) !Builder.Value {
2130	const o = self.object;
2131	const pt = self.pt;
2132	const zcu = pt.zcu;
2133	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2134	const slice_ptr = try self.resolveInst(ty_op.operand);
2135	const slice_ptr_ty = self.typeOf(ty_op.operand);
2136	const slice_llvm_ty = try o.lowerType(pt, slice_ptr_ty.childType(zcu));
2137
2138	return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, "");
2139	}
2140
2141	fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2142	const o = self.object;
2143	const pt = self.pt;
2144	const zcu = pt.zcu;
2145	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2146	const slice_ty = self.typeOf(bin_op.lhs);
2147	const slice = try self.resolveInst(bin_op.lhs);
2148	const index = try self.resolveInst(bin_op.rhs);
2149	const slice_info = slice_ty.ptrInfo(zcu);
2150	assert(slice_info.flags.size == .slice);
2151	const elem_ty: Type = .fromInterned(slice_info.child);
2152	const llvm_elem_ty = try o.lowerType(pt, elem_ty);
2153	const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
2154	const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
2155	const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
2156	const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
2157	self.maybeMarkAllowZeroAccess(slice_info);
2158	if (isByRef(elem_ty, zcu)) {
2159	return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind);
2160	} else {
2161	return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm());
2162	}
2163	}
2164
2165	fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2166	const o = self.object;
2167	const pt = self.pt;
2168	const zcu = pt.zcu;
2169	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2170	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
2171	const slice_ty = self.typeOf(bin_op.lhs);
2172
2173	const slice = try self.resolveInst(bin_op.lhs);
2174	const index = try self.resolveInst(bin_op.rhs);
2175	const llvm_elem_ty = try o.lowerType(pt, slice_ty.childType(zcu));
2176	const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
2177	return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
2178	}
2179
2180	fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2181	const o = self.object;
2182	const pt = self.pt;
2183	const zcu = pt.zcu;
2184
2185	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2186	const array_ty = self.typeOf(bin_op.lhs);
2187	const array_llvm_val = try self.resolveInst(bin_op.lhs);
2188	const rhs = try self.resolveInst(bin_op.rhs);
2189	const array_llvm_ty = try o.lowerType(pt, array_ty);
2190	const elem_ty = array_ty.childType(zcu);
2191	if (isByRef(array_ty, zcu)) {
2192	const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{
2193	try o.builder.intValue(try o.lowerType(pt, Type.usize), 0),
2194	rhs,
2195	}, "");
2196	if (isByRef(elem_ty, zcu)) {
2197	const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();
2198	return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);
2199	} else {
2200	return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
2201	}
2202	}
2203
2204	// This branch can be reached for vectors, which are always by-value.
2205	return self.wip.extractElement(array_llvm_val, rhs, "");
2206	}
2207
2208	fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2209	const o = self.object;
2210	const pt = self.pt;
2211	const zcu = pt.zcu;
2212	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2213	const ptr_ty = self.typeOf(bin_op.lhs);
2214	const elem_ty = ptr_ty.indexableElem(zcu);
2215	const llvm_elem_ty = try o.lowerType(pt, elem_ty);
2216	const base_ptr = try self.resolveInst(bin_op.lhs);
2217	const rhs = try self.resolveInst(bin_op.rhs);
2218	const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, "");
2219	if (isByRef(elem_ty, zcu)) {
2220	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
2221	const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
2222	return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal);
2223	}
2224
2225	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
2226
2227	return self.load(ptr, ptr_ty);
2228	}
2229
2230	fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2231	const o = self.object;
2232	const pt = self.pt;
2233	const zcu = pt.zcu;
2234	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2235	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
2236	const ptr_ty = self.typeOf(bin_op.lhs);
2237	const elem_ty = ptr_ty.indexableElem(zcu);
2238	assert(elem_ty.hasRuntimeBits(zcu));
2239
2240	const base_ptr = try self.resolveInst(bin_op.lhs);
2241	const rhs = try self.resolveInst(bin_op.rhs);
2242
2243	const elem_ptr = ty_pl.ty.toType();
2244	if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
2245
2246	const llvm_elem_ty = try o.lowerType(pt, elem_ty);
2247	return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, "");
2248	}
2249
2250	fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2251	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2252	const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
2253	const struct_ptr = try self.resolveInst(struct_field.struct_operand);
2254	const struct_ptr_ty = self.typeOf(struct_field.struct_operand);
2255	return self.fieldPtr(struct_ptr, struct_ptr_ty, struct_field.field_index);
2256	}
2257
2258	fn airStructFieldPtrIndex(
2259	self: *FuncGen,
2260	inst: Air.Inst.Index,
2261	field_index: u32,
2262	) !Builder.Value {
2263	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2264	const struct_ptr = try self.resolveInst(ty_op.operand);
2265	const struct_ptr_ty = self.typeOf(ty_op.operand);
2266	return self.fieldPtr(struct_ptr, struct_ptr_ty, field_index);
2267	}
2268
2269	fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2270	const o = self.object;
2271	const pt = self.pt;
2272	const zcu = pt.zcu;
2273	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2274	const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
2275	const struct_ty = self.typeOf(struct_field.struct_operand);
2276	const struct_llvm_val = try self.resolveInst(struct_field.struct_operand);
2277	const field_index = struct_field.field_index;
2278	const field_ty = struct_ty.fieldType(field_index, zcu);
2279	if (!field_ty.hasRuntimeBits(zcu)) return .none;
2280
2281	if (!isByRef(struct_ty, zcu)) {
2282	assert(!isByRef(field_ty, zcu));
2283	switch (struct_ty.zigTypeTag(zcu)) {
2284	.@"struct" => switch (struct_ty.containerLayout(zcu)) {
2285	.@"packed" => {
2286	const struct_type = zcu.typeToStruct(struct_ty).?;
2287	const bit_offset = zcu.structPackedFieldBitOffset(struct_type, field_index);
2288	const containing_int = struct_llvm_val;
2289	const shift_amt =
2290	try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset);
2291	const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, "");
2292	const elem_llvm_ty = try o.lowerType(pt, field_ty);
2293	if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
2294	const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
2295	const truncated_int =
2296	try self.wip.cast(.trunc, shifted_value, same_size_int, "");
2297	return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
2298	}
2299	return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
2300	},
2301	else => {
2302	const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
2303	return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, "");
2304	},
2305	},
2306	.@"union" => {
2307	assert(struct_ty.containerLayout(zcu) == .@"packed");
2308	const containing_int = struct_llvm_val;
2309	const elem_llvm_ty = try o.lowerType(pt, field_ty);
2310	if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
2311	const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
2312	const truncated_int =
2313	try self.wip.cast(.trunc, containing_int, same_size_int, "");
2314	return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
2315	}
2316	return self.wip.cast(.trunc, containing_int, elem_llvm_ty, "");
2317	},
2318	else => unreachable,
2319	}
2320	}
2321
2322	switch (struct_ty.zigTypeTag(zcu)) {
2323	.@"struct" => {
2324	const layout = struct_ty.containerLayout(zcu);
2325	assert(layout != .@"packed");
2326	const struct_llvm_ty = try o.lowerType(pt, struct_ty);
2327	const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
2328	const field_ptr =
2329	try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, "");
2330	const explicit_alignment = struct_ty.explicitFieldAlignment(field_index, zcu);
2331	const field_ptr_ty = try pt.ptrType(.{
2332	.child = field_ty.toIntern(),
2333	.flags = .{ .alignment = explicit_alignment },
2334	});
2335	if (isByRef(field_ty, zcu)) {
2336	const alignment = switch (explicit_alignment) {
2337	.none => field_ty.abiAlignment(zcu),
2338	else => \|a\| a,
2339	};
2340	return self.loadByRef(field_ptr, field_ty, alignment.toLlvm(), .normal);
2341	} else {
2342	return self.load(field_ptr, field_ptr_ty);
2343	}
2344	},
2345	.@"union" => {
2346	const union_llvm_ty = try o.lowerType(pt, struct_ty);
2347	const layout = struct_ty.unionGetLayout(zcu);
2348	const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align));
2349	const field_ptr =
2350	try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, "");
2351	const payload_alignment = layout.payload_align.toLlvm();
2352	if (isByRef(field_ty, zcu)) {
2353	return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal);
2354	} else {
2355	return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment);
2356	}
2357	},
2358	else => unreachable,
2359	}
2360	}
2361
2362	fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2363	const o = self.object;
2364	const pt = self.pt;
2365	const zcu = pt.zcu;
2366	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
2367	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
2368
2369	const field_ptr = try self.resolveInst(extra.field_ptr);
2370
2371	const parent_ty = ty_pl.ty.toType().childType(zcu);
2372	const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
2373	if (field_offset == 0) return field_ptr;
2374
2375	const res_ty = try o.lowerType(pt, ty_pl.ty.toType());
2376	const llvm_usize = try o.lowerType(pt, Type.usize);
2377
2378	const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");
2379	const base_ptr_int = try self.wip.bin(
2380	.@"sub nuw",
2381	field_ptr_int,
2382	try o.builder.intValue(llvm_usize, field_offset),
2383	"",
2384	);
2385	return self.wip.cast(.inttoptr, base_ptr_int, res_ty, "");
2386	}
2387
2388	fn airNot(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2389	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2390	const operand = try self.resolveInst(ty_op.operand);
2391
2392	return self.wip.not(operand, "");
2393	}
2394
2395	fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) !void {
2396	_ = inst;
2397	_ = try self.wip.@"unreachable"();
2398	}
2399
2400	fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2401	const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
2402	self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1);
2403	self.prev_dbg_column = @intCast(dbg_stmt.column + 1);
2404
2405	self.wip.debug_location = .{ .location = .{
2406	.line = self.prev_dbg_line,
2407	.column = self.prev_dbg_column,
2408	.scope = self.scope.toOptional(),
2409	.inlined_at = self.inlined_at,
2410	} };
2411
2412	return .none;
2413	}
2414
2415	fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2416	_ = self;
2417	_ = inst;
2418	return .none;
2419	}
2420
2421	fn airDbgInlineBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2422	const block = self.air.unwrapDbgBlock(inst);
2423	self.arg_inline_index = 0;
2424	return self.lowerBlock(inst, block.func, block.body);
2425	}
2426
2427	fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2428	const o = self.object;
2429	const pt = self.pt;
2430	const zcu = pt.zcu;
2431	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
2432	const operand = try self.resolveInst(pl_op.operand);
2433	const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
2434	const ptr_ty = self.typeOf(pl_op.operand);
2435
2436	const debug_local_var = try o.builder.debugLocalVar(
2437	try o.builder.metadataString(name.toSlice(self.air)),
2438	self.file,
2439	self.scope,
2440	self.prev_dbg_line,
2441	try o.getDebugType(pt, ptr_ty.childType(zcu)),
2442	);
2443
2444	_ = try self.wip.callIntrinsic(
2445	.normal,
2446	.none,
2447	.@"dbg.declare",
2448	&.{},
2449	&.{
2450	(try self.wip.debugValue(operand)).toValue(),
2451	debug_local_var.toValue(),
2452	(try o.builder.debugExpression(&.{})).toValue(),
2453	},
2454	"",
2455	);
2456
2457	return .none;
2458	}
2459
2460	fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) !Builder.Value {
2461	const o = self.object;
2462	const pt = self.pt;
2463	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
2464	const operand = try self.resolveInst(pl_op.operand);
2465	const operand_ty = self.typeOf(pl_op.operand);
2466	const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
2467	const name_slice = name.toSlice(self.air);
2468	const metadata_name = if (name_slice.len > 0) try o.builder.metadataString(name_slice) else null;
2469	const debug_local_var = if (is_arg) try o.builder.debugParameter(
2470	metadata_name,
2471	self.file,
2472	self.scope,
2473	self.prev_dbg_line,
2474	try o.getDebugType(pt, operand_ty),
2475	arg_no: {
2476	self.arg_inline_index += 1;
2477	break :arg_no self.arg_inline_index;
2478	},
2479	) else try o.builder.debugLocalVar(
2480	metadata_name,
2481	self.file,
2482	self.scope,
2483	self.prev_dbg_line,
2484	try o.getDebugType(pt, operand_ty),
2485	);
2486
2487	const zcu = pt.zcu;
2488	const owner_mod = self.ownerModule();
2489	if (isByRef(operand_ty, zcu)) {
2490	_ = try self.wip.callIntrinsic(
2491	.normal,
2492	.none,
2493	.@"dbg.declare",
2494	&.{},
2495	&.{
2496	(try self.wip.debugValue(operand)).toValue(),
2497	debug_local_var.toValue(),
2498	(try o.builder.debugExpression(&.{})).toValue(),
2499	},
2500	"",
2501	);
2502	} else if (owner_mod.optimize_mode == .Debug and !self.is_naked) {
2503	// We avoid taking this path for naked functions because there's no guarantee that such
2504	// functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
2505
2506	const alignment = operand_ty.abiAlignment(zcu).toLlvm();
2507	const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment);
2508	_ = try self.wip.store(.normal, operand, alloca, alignment);
2509	_ = try self.wip.callIntrinsic(
2510	.normal,
2511	.none,
2512	.@"dbg.declare",
2513	&.{},
2514	&.{
2515	(try self.wip.debugValue(alloca)).toValue(),
2516	debug_local_var.toValue(),
2517	(try o.builder.debugExpression(&.{})).toValue(),
2518	},
2519	"",
2520	);
2521	} else {
2522	_ = try self.wip.callIntrinsic(
2523	.normal,
2524	.none,
2525	.@"dbg.value",
2526	&.{},
2527	&.{
2528	(try self.wip.debugValue(operand)).toValue(),
2529	debug_local_var.toValue(),
2530	(try o.builder.debugExpression(&.{})).toValue(),
2531	},
2532	"",
2533	);
2534	}
2535	return .none;
2536	}
2537
2538	fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2539	// Eventually, the Zig compiler needs to be reworked to have inline
2540	// assembly go through the same parsing code regardless of backend, and
2541	// have LLVM-flavored inline assembly be output from that assembler.
2542	// We don't have such an assembler implemented yet though. For now,
2543	// this implementation feeds the inline assembly code directly to LLVM.
2544
2545	const o = self.object;
2546	const unwrapped_asm = self.air.unwrapAsm(inst);
2547	const is_volatile = unwrapped_asm.is_volatile;
2548	const gpa = self.gpa;
2549
2550	const outputs = unwrapped_asm.outputs;
2551	const inputs = unwrapped_asm.inputs;
2552
2553	var llvm_constraints: std.ArrayList(u8) = .empty;
2554	defer llvm_constraints.deinit(gpa);
2555
2556	var arena_allocator = std.heap.ArenaAllocator.init(gpa);
2557	defer arena_allocator.deinit();
2558	const arena = arena_allocator.allocator();
2559
2560	// The exact number of return / parameter values depends on which output values
2561	// are passed by reference as indirect outputs (determined below).
2562	const max_return_count = outputs.len;
2563	const llvm_ret_types = try arena.alloc(Builder.Type, max_return_count);
2564	const llvm_ret_indirect = try arena.alloc(bool, max_return_count);
2565	const llvm_rw_vals = try arena.alloc(Builder.Value, max_return_count);
2566
2567	const max_param_count = max_return_count + inputs.len + outputs.len;
2568	const llvm_param_types = try arena.alloc(Builder.Type, max_param_count);
2569	const llvm_param_values = try arena.alloc(Builder.Value, max_param_count);
2570	// This stores whether we need to add an elementtype attribute and
2571	// if so, the element type itself.
2572	const llvm_param_attrs = try arena.alloc(Builder.Type, max_param_count);
2573	const pt = self.pt;
2574	const zcu = pt.zcu;
2575	const ip = &zcu.intern_pool;
2576	const target = zcu.getTarget();
2577
2578	var llvm_ret_i: usize = 0;
2579	var llvm_param_i: usize = 0;
2580	var total_i: usize = 0;
2581
2582	var name_map: std.StringArrayHashMapUnmanaged(u16) = .empty;
2583	try name_map.ensureUnusedCapacity(arena, max_param_count);
2584
2585	var it = unwrapped_asm.iterateOutputs();
2586	while (it.next()) \|output\| {
2587	const constraint = output.constraint;
2588	const name = output.name;
2589
2590	try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 3);
2591	if (total_i != 0) {
2592	llvm_constraints.appendAssumeCapacity(',');
2593	}
2594	llvm_constraints.appendAssumeCapacity('=');
2595
2596	if (output.operand != .none) {
2597	const output_inst = try self.resolveInst(output.operand);
2598	const output_ty = self.typeOf(output.operand);
2599	assert(output_ty.zigTypeTag(zcu) == .pointer);
2600	const elem_llvm_ty = try o.lowerType(pt, output_ty.childType(zcu));
2601
2602	switch (constraint[0]) {
2603	'=' => {},
2604	'+' => llvm_rw_vals[output.index] = output_inst,
2605	else => return self.todo("unsupported output constraint on output type '{c}'", .{
2606	constraint[0],
2607	}),
2608	}
2609
2610	self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu));
2611
2612	// Pass any non-return outputs indirectly, if the constraint accepts a memory location
2613	llvm_ret_indirect[output.index] = constraintAllowsMemory(constraint);
2614	if (llvm_ret_indirect[output.index]) {
2615	// Pass the result by reference as an indirect output (e.g. "=*m")
2616	llvm_constraints.appendAssumeCapacity('*');
2617
2618	llvm_param_values[llvm_param_i] = output_inst;
2619	llvm_param_types[llvm_param_i] = output_inst.typeOfWip(&self.wip);
2620	llvm_param_attrs[llvm_param_i] = elem_llvm_ty;
2621	llvm_param_i += 1;
2622	} else {
2623	// Pass the result directly (e.g. "=r")
2624	llvm_ret_types[llvm_ret_i] = elem_llvm_ty;
2625	llvm_ret_i += 1;
2626	}
2627	} else {
2628	switch (constraint[0]) {
2629	'=' => {},
2630	else => return self.todo("unsupported output constraint on result type '{s}'", .{
2631	constraint,
2632	}),
2633	}
2634
2635	llvm_ret_indirect[output.index] = false;
2636
2637	const ret_ty = self.typeOfIndex(inst);
2638	llvm_ret_types[llvm_ret_i] = try o.lowerType(pt, ret_ty);
2639	llvm_ret_i += 1;
2640	}
2641
2642	// LLVM uses commas internally to separate different constraints,
2643	// alternative constraints are achieved with pipes.
2644	// We still allow the user to use commas in a way that is similar
2645	// to GCC's inline assembly.
2646	// http://llvm.org/docs/LangRef.html#constraint-codes
2647	for (constraint[1..]) \|byte\| {
2648	switch (byte) {
2649	',' => llvm_constraints.appendAssumeCapacity('\|'),
2650	'*' => {}, // Indirect outputs are handled above
2651	else => llvm_constraints.appendAssumeCapacity(byte),
2652	}
2653	}
2654
2655	if (!std.mem.eql(u8, name, "_")) {
2656	const gop = name_map.getOrPutAssumeCapacity(name);
2657	if (gop.found_existing) return self.todo("duplicate asm output name '{s}'", .{name});
2658	gop.value_ptr.* = @intCast(total_i);
2659	}
2660	total_i += 1;
2661	}
2662
2663	it = unwrapped_asm.iterateInputs();
2664	while (it.next()) \|input\| {
2665	const constraint = input.constraint;
2666	const name = input.name;
2667
2668	const arg_llvm_value = try self.resolveInst(input.operand);
2669	const arg_ty = self.typeOf(input.operand);
2670	const is_by_ref = isByRef(arg_ty, zcu);
2671	if (is_by_ref) {
2672	if (constraintAllowsMemory(constraint)) {
2673	llvm_param_values[llvm_param_i] = arg_llvm_value;
2674	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
2675	} else {
2676	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2677	const arg_llvm_ty = try o.lowerType(pt, arg_ty);
2678	const load_inst =
2679	try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
2680	llvm_param_values[llvm_param_i] = load_inst;
2681	llvm_param_types[llvm_param_i] = arg_llvm_ty;
2682	}
2683	} else {
2684	if (constraintAllowsRegister(constraint)) {
2685	llvm_param_values[llvm_param_i] = arg_llvm_value;
2686	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
2687	} else {
2688	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2689	const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);
2690	_ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment);
2691	llvm_param_values[llvm_param_i] = arg_ptr;
2692	llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);
2693	}
2694	}
2695
2696	try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 1);
2697	if (total_i != 0) {
2698	llvm_constraints.appendAssumeCapacity(',');
2699	}
2700	for (constraint) \|byte\| {
2701	llvm_constraints.appendAssumeCapacity(switch (byte) {
2702	',' => '\|',
2703	else => byte,
2704	});
2705	}
2706
2707	if (!std.mem.eql(u8, name, "_")) {
2708	const gop = name_map.getOrPutAssumeCapacity(name);
2709	if (gop.found_existing) return self.todo("duplicate asm input name '{s}'", .{name});
2710	gop.value_ptr.* = @intCast(total_i);
2711	}
2712
2713	// In the case of indirect inputs, LLVM requires the callsite to have
2714	// an elementtype(<ty>) attribute.
2715	llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {
2716	if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));
2717
2718	break :blk try o.lowerType(pt, if (is_by_ref) arg_ty else arg_ty.childType(zcu));
2719	} else .none;
2720
2721	llvm_param_i += 1;
2722	total_i += 1;
2723	}
2724
2725	it = unwrapped_asm.iterateOutputs();
2726	while (it.next()) \|output\| {
2727	const constraint = output.constraint;
2728
2729	if (constraint[0] != '+') continue;
2730
2731	const rw_ty = self.typeOf(output.operand);
2732	const llvm_elem_ty = try o.lowerType(pt, rw_ty.childType(zcu));
2733	if (llvm_ret_indirect[output.index]) {
2734	llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
2735	llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
2736	} else {
2737	const alignment = rw_ty.abiAlignment(zcu).toLlvm();
2738	const loaded = try self.wip.load(
2739	if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2740	llvm_elem_ty,
2741	llvm_rw_vals[output.index],
2742	alignment,
2743	"",
2744	);
2745	llvm_param_values[llvm_param_i] = loaded;
2746	llvm_param_types[llvm_param_i] = llvm_elem_ty;
2747	}
2748
2749	try llvm_constraints.print(gpa, ",{d}", .{output.index});
2750
2751	// In the case of indirect inputs, LLVM requires the callsite to have
2752	// an elementtype(<ty>) attribute.
2753	llvm_param_attrs[llvm_param_i] = if (llvm_ret_indirect[output.index]) llvm_elem_ty else .none;
2754
2755	llvm_param_i += 1;
2756	total_i += 1;
2757	}
2758
2759	if (total_i != 0) try llvm_constraints.append(gpa, ',');
2760	const clobbers_val: Value = .fromInterned(unwrapped_asm.clobbers);
2761	const clobbers_ty = clobbers_val.typeOf(zcu);
2762	var clobbers_bigint_buf: Value.BigIntSpace = undefined;
2763	const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu);
2764	for (0..clobbers_ty.structFieldCount(zcu)) \|field_index\| {
2765	assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type);
2766	const limb_bits = @bitSizeOf(std.math.big.Limb);
2767	if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false
2768	switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) {
2769	0 => continue, // field is false
2770	1 => {}, // field is true
2771	}
2772	const name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
2773	total_i += try appendConstraints(gpa, &llvm_constraints, name, target);
2774	}
2775
2776	// We have finished scanning through all inputs/outputs, so the number of
2777	// parameters and return values is known.
2778	const param_count = llvm_param_i;
2779	const return_count = llvm_ret_i;
2780
2781	// For some targets, Clang unconditionally adds some clobbers to all inline assembly.
2782	// While this is probably not strictly necessary, if we don't follow Clang's lead
2783	// here then we may risk tripping LLVM bugs since anything not used by Clang tends
2784	// to be buggy and regress often.
2785	switch (target.cpu.arch) {
2786	.x86_64, .x86 => {
2787	try llvm_constraints.appendSlice(gpa, "~{dirflag},~{fpsr},~{flags},");
2788	total_i += 3;
2789	},
2790	.mips, .mipsel, .mips64, .mips64el => {
2791	try llvm_constraints.appendSlice(gpa, "~{$1},");
2792	total_i += 1;
2793	},
2794	else => {},
2795	}
2796
2797	if (std.mem.endsWith(u8, llvm_constraints.items, ",")) llvm_constraints.items.len -= 1;
2798
2799	const asm_source = unwrapped_asm.source;
2800
2801	// hackety hacks until stage2 has proper inline asm in the frontend.
2802	var rendered_template = std.array_list.Managed(u8).init(gpa);
2803	defer rendered_template.deinit();
2804
2805	const State = enum { start, percent, input, modifier };
2806
2807	var state: State = .start;
2808
2809	var name_start: usize = undefined;
2810	var modifier_start: usize = undefined;
2811	for (asm_source, 0..) \|byte, i\| {
2812	switch (state) {
2813	.start => switch (byte) {
2814	'%' => state = .percent,
2815	'$' => try rendered_template.appendSlice("$$"),
2816	else => try rendered_template.append(byte),
2817	},
2818	.percent => switch (byte) {
2819	'%' => {
2820	try rendered_template.append('%');
2821	state = .start;
2822	},
2823	'[' => {
2824	try rendered_template.append('$');
2825	try rendered_template.append('{');
2826	name_start = i + 1;
2827	state = .input;
2828	},
2829	'=' => {
2830	try rendered_template.appendSlice("${:uid}");
2831	state = .start;
2832	},
2833	else => {
2834	try rendered_template.append('%');
2835	try rendered_template.append(byte);
2836	state = .start;
2837	},
2838	},
2839	.input => switch (byte) {
2840	']', ':' => {
2841	const name = asm_source[name_start..i];
2842
2843	const index = name_map.get(name) orelse {
2844	// we should validate the assembly in Sema; by now it is too late
2845	return self.todo("unknown input or output name: '{s}'", .{name});
2846	};
2847	try rendered_template.print("{d}", .{index});
2848	if (byte == ':') {
2849	try rendered_template.append(':');
2850	modifier_start = i + 1;
2851	state = .modifier;
2852	} else {
2853	try rendered_template.append('}');
2854	state = .start;
2855	}
2856	},
2857	else => {},
2858	},
2859	.modifier => switch (byte) {
2860	']' => {
2861	try rendered_template.appendSlice(asm_source[modifier_start..i]);
2862	try rendered_template.append('}');
2863	state = .start;
2864	},
2865	else => {},
2866	},
2867	}
2868	}
2869
2870	var attributes: Builder.FunctionAttributes.Wip = .{};
2871	defer attributes.deinit(&o.builder);
2872	for (llvm_param_attrs[0..param_count], 0..) \|llvm_elem_ty, i\| if (llvm_elem_ty != .none)
2873	try attributes.addParamAttr(i, .{ .elementtype = llvm_elem_ty }, &o.builder);
2874
2875	const ret_llvm_ty = switch (return_count) {
2876	0 => .void,
2877	1 => llvm_ret_types[0],
2878	else => try o.builder.structType(.normal, llvm_ret_types),
2879	};
2880	const llvm_fn_ty = try o.builder.fnType(ret_llvm_ty, llvm_param_types[0..param_count], .normal);
2881	const call = try self.wip.callAsm(
2882	try attributes.finish(&o.builder),
2883	llvm_fn_ty,
2884	.{ .sideeffect = is_volatile },
2885	try o.builder.string(rendered_template.items),
2886	try o.builder.string(llvm_constraints.items),
2887	llvm_param_values[0..param_count],
2888	"",
2889	);
2890
2891	var ret_val = call;
2892	llvm_ret_i = 0;
2893	for (outputs, 0..) \|output, i\| {
2894	if (llvm_ret_indirect[i]) continue;
2895
2896	const output_value = if (return_count > 1)
2897	try self.wip.extractValue(call, &[_]u32{@intCast(llvm_ret_i)}, "")
2898	else
2899	call;
2900
2901	if (output != .none) {
2902	const output_ptr = try self.resolveInst(output);
2903	const output_ptr_ty = self.typeOf(output);
2904	const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm();
2905	_ = try self.wip.store(
2906	if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2907	output_value,
2908	output_ptr,
2909	alignment,
2910	);
2911	} else {
2912	ret_val = output_value;
2913	}
2914	llvm_ret_i += 1;
2915	}
2916
2917	return ret_val;
2918	}
2919
2920	fn airIsNonNull(
2921	self: *FuncGen,
2922	inst: Air.Inst.Index,
2923	operand_is_ptr: bool,
2924	cond: Builder.IntegerCondition,
2925	) !Builder.Value {
2926	const o = self.object;
2927	const pt = self.pt;
2928	const zcu = pt.zcu;
2929	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2930	const operand = try self.resolveInst(un_op);
2931	const operand_ty = self.typeOf(un_op);
2932	const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
2933	const optional_llvm_ty = try o.lowerType(pt, optional_ty);
2934	const payload_ty = optional_ty.optionalChild(zcu);
2935
2936	const access_kind: Builder.MemoryAccessKind =
2937	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
2938
2939	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
2940
2941	if (optional_ty.optionalReprIsPayload(zcu)) {
2942	const loaded = if (operand_is_ptr)
2943	try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
2944	else
2945	operand;
2946	if (payload_ty.isSlice(zcu)) {
2947	const slice_ptr = try self.wip.extractValue(loaded, &.{0}, "");
2948	const ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(
2949	payload_ty.ptrAddressSpace(zcu),
2950	zcu.getTarget(),
2951	));
2952	return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
2953	}
2954	return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), "");
2955	}
2956
2957	comptime assert(optional_layout_version == 3);
2958
2959	if (!payload_ty.hasRuntimeBits(zcu)) {
2960	const loaded = if (operand_is_ptr)
2961	try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
2962	else
2963	operand;
2964	return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
2965	}
2966
2967	const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu);
2968	return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind);
2969	}
2970
2971	fn airIsErr(
2972	self: *FuncGen,
2973	inst: Air.Inst.Index,
2974	cond: Builder.IntegerCondition,
2975	operand_is_ptr: bool,
2976	) !Builder.Value {
2977	const o = self.object;
2978	const pt = self.pt;
2979	const zcu = pt.zcu;
2980	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
2981	const operand = try self.resolveInst(un_op);
2982	const operand_ty = self.typeOf(un_op);
2983	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
2984	const payload_ty = err_union_ty.errorUnionPayload(zcu);
2985	const error_type = try o.errorIntType(pt);
2986	const zero = try o.builder.intValue(error_type, 0);
2987
2988	const access_kind: Builder.MemoryAccessKind =
2989	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
2990
2991	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
2992	const val: Builder.Constant = switch (cond) {
2993	.eq => .true, // 0 == 0
2994	.ne => .false, // 0 != 0
2995	else => unreachable,
2996	};
2997	return val.toValue();
2998	}
2999
3000	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
3001
3002	if (!payload_ty.hasRuntimeBits(zcu)) {
3003	const loaded = if (operand_is_ptr)
3004	try self.wip.load(access_kind, try o.lowerType(pt, err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
3005	else
3006	operand;
3007	return self.wip.icmp(cond, loaded, zero, "");
3008	}
3009
3010	const err_field_index = try errUnionErrorFieldIndex(payload_ty, pt);
3011
3012	const loaded = if (operand_is_ptr or isByRef(err_union_ty, zcu)) loaded: {
3013	const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
3014	const err_alignment = if (operand_is_ptr)
3015	operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu))
3016	else
3017	.none;
3018	const err_field_ptr =
3019	try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, "");
3020	break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, err_alignment.toLlvm(), "");
3021	} else try self.wip.extractValue(operand, &.{err_field_index}, "");
3022	return self.wip.icmp(cond, loaded, zero, "");
3023	}
3024
3025	fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3026	const o = self.object;
3027	const pt = self.pt;
3028	const zcu = pt.zcu;
3029	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3030	const operand = try self.resolveInst(ty_op.operand);
3031	const optional_ty = self.typeOf(ty_op.operand).childType(zcu);
3032	const payload_ty = optional_ty.optionalChild(zcu);
3033	if (!payload_ty.hasRuntimeBits(zcu)) {
3034	// We have a pointer to a zero-bit value and we need to return
3035	// a pointer to a zero-bit value.
3036	return operand;
3037	}
3038	if (optional_ty.optionalReprIsPayload(zcu)) {
3039	// The payload and the optional are the same value.
3040	return operand;
3041	}
3042	return self.wip.gepStruct(try o.lowerType(pt, optional_ty), operand, 0, "");
3043	}
3044
3045	fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3046	comptime assert(optional_layout_version == 3);
3047
3048	const o = self.object;
3049	const pt = self.pt;
3050	const zcu = pt.zcu;
3051	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3052	const operand = try self.resolveInst(ty_op.operand);
3053	const optional_ptr_ty = self.typeOf(ty_op.operand);
3054	const optional_ty = optional_ptr_ty.childType(zcu);
3055	const payload_ty = optional_ty.optionalChild(zcu);
3056	const non_null_bit = try o.builder.intValue(.i8, 1);
3057
3058	const access_kind: Builder.MemoryAccessKind =
3059	if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
3060
3061	if (!payload_ty.hasRuntimeBits(zcu)) {
3062	self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
3063
3064	// We have a pointer to a i8. We need to set it to 1 and then return the same pointer.
3065	// Default alignment store because align of the non null bit is 1 anyway.
3066	_ = try self.wip.store(access_kind, non_null_bit, operand, .default);
3067	return operand;
3068	}
3069	if (optional_ty.optionalReprIsPayload(zcu)) {
3070	// The payload and the optional are the same value.
3071	// Setting to non-null will be done when the payload is set.
3072	return operand;
3073	}
3074
3075	// First set the non-null bit.
3076	const optional_llvm_ty = try o.lowerType(pt, optional_ty);
3077	const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, "");
3078
3079	self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
3080
3081	// Default alignment store because align of the non null bit is 1 anyway.
3082	_ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default);
3083
3084	// Then return the payload pointer (only if it's used).
3085	if (self.liveness.isUnused(inst)) return .none;
3086
3087	return self.wip.gepStruct(optional_llvm_ty, operand, 0, "");
3088	}
3089
3090	fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3091	const o = self.object;
3092	const pt = self.pt;
3093	const zcu = pt.zcu;
3094	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3095	const operand = try self.resolveInst(ty_op.operand);
3096	const optional_ty = self.typeOf(ty_op.operand);
3097	const payload_ty = self.typeOfIndex(inst);
3098	if (!payload_ty.hasRuntimeBits(zcu)) return .none;
3099
3100	if (optional_ty.optionalReprIsPayload(zcu)) {
3101	// Payload value is the same as the optional value.
3102	return operand;
3103	}
3104
3105	const opt_llvm_ty = try o.lowerType(pt, optional_ty);
3106	return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false);
3107	}
3108
3109	fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value {
3110	const o = self.object;
3111	const pt = self.pt;
3112	const zcu = pt.zcu;
3113	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3114	const operand = try self.resolveInst(ty_op.operand);
3115	const operand_ty = self.typeOf(ty_op.operand);
3116	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3117	const result_ty = self.typeOfIndex(inst);
3118	const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty;
3119
3120	if (!payload_ty.hasRuntimeBits(zcu)) {
3121	return if (operand_is_ptr) operand else .none;
3122	}
3123	const offset = try errUnionPayloadFieldIndex(payload_ty, pt);
3124	const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
3125	if (operand_is_ptr) {
3126	return self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
3127	} else if (isByRef(err_union_ty, zcu)) {
3128	const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
3129	const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
3130	if (isByRef(payload_ty, zcu)) {
3131	return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
3132	}
3133	const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
3134	return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");
3135	}
3136	return self.wip.extractValue(operand, &.{offset}, "");
3137	}
3138
3139	fn airErrUnionErr(
3140	self: *FuncGen,
3141	inst: Air.Inst.Index,
3142	operand_is_ptr: bool,
3143	) !Builder.Value {
3144	const o = self.object;
3145	const pt = self.pt;
3146	const zcu = pt.zcu;
3147	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3148	const operand = try self.resolveInst(ty_op.operand);
3149	const operand_ty = self.typeOf(ty_op.operand);
3150	const error_type = try o.errorIntType(pt);
3151	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3152	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
3153	if (operand_is_ptr) {
3154	return operand;
3155	} else {
3156	return o.builder.intValue(error_type, 0);
3157	}
3158	}
3159
3160	const access_kind: Builder.MemoryAccessKind =
3161	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
3162
3163	const payload_ty = err_union_ty.errorUnionPayload(zcu);
3164	if (!payload_ty.hasRuntimeBits(zcu)) {
3165	if (!operand_is_ptr) return operand;
3166
3167	self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
3168
3169	return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");
3170	}
3171
3172	const offset = try errUnionErrorFieldIndex(payload_ty, pt);
3173
3174	if (operand_is_ptr or isByRef(err_union_ty, zcu)) {
3175	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
3176
3177	const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
3178	const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
3179	return self.wip.load(access_kind, error_type, err_field_ptr, .default, "");
3180	}
3181
3182	return self.wip.extractValue(operand, &.{offset}, "");
3183	}
3184
3185	fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3186	const o = self.object;
3187	const pt = self.pt;
3188	const zcu = pt.zcu;
3189	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3190	const operand = try self.resolveInst(ty_op.operand);
3191	const err_union_ptr_ty = self.typeOf(ty_op.operand);
3192	const err_union_ty = err_union_ptr_ty.childType(zcu);
3193	const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);
3194
3195	const payload_ty = err_union_ty.errorUnionPayload(zcu);
3196	const non_error_val = try o.builder.intValue(try o.errorIntType(pt), 0);
3197
3198	const access_kind: Builder.MemoryAccessKind =
3199	if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
3200
3201	if (!payload_ty.hasRuntimeBits(zcu)) {
3202	self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
3203	_ = try self.wip.store(access_kind, non_error_val, operand, err_union_ptr_align.toLlvm());
3204	return operand;
3205	}
3206	const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
3207	{
3208	self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
3209
3210	const err_int_ty = try pt.errorIntType();
3211	const error_alignment = err_int_ty.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
3212	const error_offset = try errUnionErrorFieldIndex(payload_ty, pt);
3213	// First set the non-error value.
3214	const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, "");
3215	_ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment);
3216	}
3217	// Then return the payload pointer (only if it is used).
3218	if (self.liveness.isUnused(inst)) return .none;
3219
3220	const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt);
3221	return self.wip.gepStruct(err_union_llvm_ty, operand, payload_offset, "");
3222	}
3223
3224	fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value {
3225	assert(self.err_ret_trace != .none);
3226	return self.err_ret_trace;
3227	}
3228
3229	fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3230	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3231	self.err_ret_trace = try self.resolveInst(un_op);
3232	return .none;
3233	}
3234
3235	fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3236	const o = self.object;
3237	const pt = self.pt;
3238	const zcu = pt.zcu;
3239
3240	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3241	const struct_ty = ty_pl.ty.toType();
3242	const field_index = ty_pl.payload;
3243
3244	const struct_llvm_ty = try o.lowerType(pt, struct_ty);
3245	const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
3246	assert(self.err_ret_trace != .none);
3247	const field_ptr = try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, "");
3248	const field_alignment = struct_ty.explicitFieldAlignment(field_index, zcu);
3249	const field_ty = struct_ty.fieldType(field_index, zcu);
3250	const field_ptr_ty = try pt.ptrType(.{
3251	.child = field_ty.toIntern(),
3252	.flags = .{ .alignment = field_alignment },
3253	});
3254	return self.load(field_ptr, field_ptr_ty);
3255	}
3256
3257	/// As an optimization, we want to avoid unnecessary copies of
3258	/// error union/optional types when returning from a function.
3259	/// Here, we scan forward in the current block, looking to see
3260	/// if the next instruction is a return (ignoring debug instructions).
3261	///
3262	/// The first instruction of `body_tail` is a wrap instruction.
3263	fn isNextRet(
3264	self: *FuncGen,
3265	body_tail: []const Air.Inst.Index,
3266	) bool {
3267	const air_tags = self.air.instructions.items(.tag);
3268	for (body_tail[1..]) \|body_inst\| {
3269	switch (air_tags[@intFromEnum(body_inst)]) {
3270	.ret => return true,
3271	.dbg_stmt => continue,
3272	else => return false,
3273	}
3274	}
3275	// The only way to get here is to hit the end of a loop instruction
3276	// (implicit repeat).
3277	return false;
3278	}
3279
3280	fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
3281	const o = self.object;
3282	const pt = self.pt;
3283	const zcu = pt.zcu;
3284	const inst = body_tail[0];
3285	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3286	const payload_ty = self.typeOf(ty_op.operand);
3287	const non_null_bit = try o.builder.intValue(.i8, 1);
3288	comptime assert(optional_layout_version == 3);
3289	assert(payload_ty.hasRuntimeBits(zcu));
3290	const operand = try self.resolveInst(ty_op.operand);
3291	const optional_ty = self.typeOfIndex(inst);
3292	if (optional_ty.optionalReprIsPayload(zcu)) return operand;
3293	const llvm_optional_ty = try o.lowerType(pt, optional_ty);
3294	if (isByRef(optional_ty, zcu)) {
3295	const directReturn = self.isNextRet(body_tail);
3296	const optional_ptr = if (directReturn)
3297	self.ret_ptr
3298	else brk: {
3299	const alignment = optional_ty.abiAlignment(zcu).toLlvm();
3300	const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
3301	break :brk optional_ptr;
3302	};
3303
3304	const payload_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 0, "");
3305	const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3306	try self.store(payload_ptr, payload_ptr_ty, operand, .none);
3307	const non_null_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 1, "");
3308	_ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);
3309	return optional_ptr;
3310	}
3311	return self.wip.buildAggregate(llvm_optional_ty, &.{ operand, non_null_bit }, "");
3312	}
3313
3314	fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
3315	const o = self.object;
3316	const pt = self.pt;
3317	const zcu = pt.zcu;
3318	const inst = body_tail[0];
3319	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3320	const err_un_ty = self.typeOfIndex(inst);
3321	const operand = try self.resolveInst(ty_op.operand);
3322	const payload_ty = self.typeOf(ty_op.operand);
3323	assert(payload_ty.hasRuntimeBits(zcu));
3324	const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0);
3325	const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
3326
3327	const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt);
3328	const error_offset = try errUnionErrorFieldIndex(payload_ty, pt);
3329	if (isByRef(err_un_ty, zcu)) {
3330	const directReturn = self.isNextRet(body_tail);
3331	const result_ptr = if (directReturn)
3332	self.ret_ptr
3333	else brk: {
3334	const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm();
3335	const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3336	break :brk result_ptr;
3337	};
3338
3339	const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
3340	const err_int_ty = try pt.errorIntType();
3341	const error_alignment = err_int_ty.abiAlignment(pt.zcu).toLlvm();
3342	_ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);
3343	const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
3344	const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3345	try self.store(payload_ptr, payload_ptr_ty, operand, .none);
3346	return result_ptr;
3347	}
3348	var fields: [2]Builder.Value = undefined;
3349	fields[payload_offset] = operand;
3350	fields[error_offset] = ok_err_code;
3351	return self.wip.buildAggregate(err_un_llvm_ty, &fields, "");
3352	}
3353
3354	fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
3355	const o = self.object;
3356	const pt = self.pt;
3357	const zcu = pt.zcu;
3358	const inst = body_tail[0];
3359	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3360	const err_un_ty = self.typeOfIndex(inst);
3361	const payload_ty = err_un_ty.errorUnionPayload(zcu);
3362	const operand = try self.resolveInst(ty_op.operand);
3363	if (!payload_ty.hasRuntimeBits(zcu)) return operand;
3364	const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
3365
3366	const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt);
3367	const error_offset = try errUnionErrorFieldIndex(payload_ty, pt);
3368	if (isByRef(err_un_ty, zcu)) {
3369	const directReturn = self.isNextRet(body_tail);
3370	const result_ptr = if (directReturn)
3371	self.ret_ptr
3372	else brk: {
3373	const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
3374	const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3375	break :brk result_ptr;
3376	};
3377
3378	const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
3379	const err_int_ty = try pt.errorIntType();
3380	const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm();
3381	_ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
3382	const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
3383	const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3384	// TODO store undef to payload_ptr
3385	_ = payload_ptr;
3386	_ = payload_ptr_ty;
3387	return result_ptr;
3388	}
3389
3390	// TODO set payload bytes to undef
3391	const undef = try o.builder.undefValue(err_un_llvm_ty);
3392	return self.wip.insertValue(undef, operand, &.{error_offset}, "");
3393	}
3394
3395	fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3396	const o = self.object;
3397	const pt = self.pt;
3398	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3399	const index = pl_op.payload;
3400	const llvm_usize = try o.lowerType(pt, Type.usize);
3401	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{
3402	try o.builder.intValue(.i32, index),
3403	}, "");
3404	}
3405
3406	fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3407	const o = self.object;
3408	const pt = self.pt;
3409	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3410	const index = pl_op.payload;
3411	const llvm_isize = try o.lowerType(pt, Type.isize);
3412	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{
3413	try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),
3414	}, "");
3415	}
3416
3417	fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3418	const o = fg.object;
3419	const pt = fg.pt;
3420	const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav;
3421	const llvm_ptr_const = try o.lowerNavRefValue(pt, ty_nav.nav);
3422	return llvm_ptr_const.toValue();
3423	}
3424
3425	fn airMin(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3426	const o = self.object;
3427	const pt = self.pt;
3428	const zcu = pt.zcu;
3429	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3430	const lhs = try self.resolveInst(bin_op.lhs);
3431	const rhs = try self.resolveInst(bin_op.rhs);
3432	const inst_ty = self.typeOfIndex(inst);
3433	const scalar_ty = inst_ty.scalarType(zcu);
3434
3435	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, .normal, inst_ty, 2, .{ lhs, rhs });
3436	return self.wip.callIntrinsic(
3437	.normal,
3438	.none,
3439	if (scalar_ty.isSignedInt(zcu)) .smin else .umin,
3440	&.{try o.lowerType(pt, inst_ty)},
3441	&.{ lhs, rhs },
3442	"",
3443	);
3444	}
3445
3446	fn airMax(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3447	const o = self.object;
3448	const pt = self.pt;
3449	const zcu = pt.zcu;
3450	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3451	const lhs = try self.resolveInst(bin_op.lhs);
3452	const rhs = try self.resolveInst(bin_op.rhs);
3453	const inst_ty = self.typeOfIndex(inst);
3454	const scalar_ty = inst_ty.scalarType(zcu);
3455
3456	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, .normal, inst_ty, 2, .{ lhs, rhs });
3457	return self.wip.callIntrinsic(
3458	.normal,
3459	.none,
3460	if (scalar_ty.isSignedInt(zcu)) .smax else .umax,
3461	&.{try o.lowerType(pt, inst_ty)},
3462	&.{ lhs, rhs },
3463	"",
3464	);
3465	}
3466
3467	fn airSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3468	const o = self.object;
3469	const pt = self.pt;
3470	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3471	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3472	const ptr = try self.resolveInst(bin_op.lhs);
3473	const len = try self.resolveInst(bin_op.rhs);
3474	const inst_ty = self.typeOfIndex(inst);
3475	return self.wip.buildAggregate(try o.lowerType(pt, inst_ty), &.{ ptr, len }, "");
3476	}
3477
3478	fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
3479	const zcu = self.pt.zcu;
3480	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3481	const lhs = try self.resolveInst(bin_op.lhs);
3482	const rhs = try self.resolveInst(bin_op.rhs);
3483	const inst_ty = self.typeOfIndex(inst);
3484	const scalar_ty = inst_ty.scalarType(zcu);
3485
3486	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, fast, inst_ty, 2, .{ lhs, rhs });
3487	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"add nsw" else .@"add nuw", lhs, rhs, "");
3488	}
3489
3490	fn airSafeArithmetic(
3491	fg: *FuncGen,
3492	inst: Air.Inst.Index,
3493	signed_intrinsic: Builder.Intrinsic,
3494	unsigned_intrinsic: Builder.Intrinsic,
3495	) !Builder.Value {
3496	const o = fg.object;
3497	const pt = fg.pt;
3498	const zcu = pt.zcu;
3499
3500	const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3501	const lhs = try fg.resolveInst(bin_op.lhs);
3502	const rhs = try fg.resolveInst(bin_op.rhs);
3503	const inst_ty = fg.typeOfIndex(inst);
3504	const scalar_ty = inst_ty.scalarType(zcu);
3505
3506	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3507	const llvm_inst_ty = try o.lowerType(pt, inst_ty);
3508	const results =
3509	try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");
3510
3511	const overflow_bits = try fg.wip.extractValue(results, &.{1}, "");
3512	const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip);
3513	const overflow_bit = if (overflow_bits_ty.isVector(&o.builder))
3514	try fg.wip.callIntrinsic(
3515	.normal,
3516	.none,
3517	.@"vector.reduce.or",
3518	&.{overflow_bits_ty},
3519	&.{overflow_bits},
3520	"",
3521	)
3522	else
3523	overflow_bits;
3524
3525	const fail_block = try fg.wip.block(1, "OverflowFail");
3526	const ok_block = try fg.wip.block(1, "OverflowOk");
3527	_ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none);
3528
3529	fg.wip.cursor = .{ .block = fail_block };
3530	try fg.buildSimplePanic(.integer_overflow);
3531
3532	fg.wip.cursor = .{ .block = ok_block };
3533	return fg.wip.extractValue(results, &.{0}, "");
3534	}
3535
3536	fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3537	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3538	const lhs = try self.resolveInst(bin_op.lhs);
3539	const rhs = try self.resolveInst(bin_op.rhs);
3540
3541	return self.wip.bin(.add, lhs, rhs, "");
3542	}
3543
3544	fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3545	const o = self.object;
3546	const pt = self.pt;
3547	const zcu = pt.zcu;
3548	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3549	const lhs = try self.resolveInst(bin_op.lhs);
3550	const rhs = try self.resolveInst(bin_op.rhs);
3551	const inst_ty = self.typeOfIndex(inst);
3552	const scalar_ty = inst_ty.scalarType(zcu);
3553	assert(scalar_ty.zigTypeTag(zcu) == .int);
3554	return self.wip.callIntrinsic(
3555	.normal,
3556	.none,
3557	if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",
3558	&.{try o.lowerType(pt, inst_ty)},
3559	&.{ lhs, rhs },
3560	"",
3561	);
3562	}
3563
3564	fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
3565	const zcu = self.pt.zcu;
3566	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3567	const lhs = try self.resolveInst(bin_op.lhs);
3568	const rhs = try self.resolveInst(bin_op.rhs);
3569	const inst_ty = self.typeOfIndex(inst);
3570	const scalar_ty = inst_ty.scalarType(zcu);
3571
3572	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, fast, inst_ty, 2, .{ lhs, rhs });
3573	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"sub nsw" else .@"sub nuw", lhs, rhs, "");
3574	}
3575
3576	fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3577	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3578	const lhs = try self.resolveInst(bin_op.lhs);
3579	const rhs = try self.resolveInst(bin_op.rhs);
3580
3581	return self.wip.bin(.sub, lhs, rhs, "");
3582	}
3583
3584	fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3585	const o = self.object;
3586	const pt = self.pt;
3587	const zcu = pt.zcu;
3588	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3589	const lhs = try self.resolveInst(bin_op.lhs);
3590	const rhs = try self.resolveInst(bin_op.rhs);
3591	const inst_ty = self.typeOfIndex(inst);
3592	const scalar_ty = inst_ty.scalarType(zcu);
3593	assert(scalar_ty.zigTypeTag(zcu) == .int);
3594	return self.wip.callIntrinsic(
3595	.normal,
3596	.none,
3597	if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",
3598	&.{try o.lowerType(pt, inst_ty)},
3599	&.{ lhs, rhs },
3600	"",
3601	);
3602	}
3603
3604	fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
3605	const zcu = self.pt.zcu;
3606	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3607	const lhs = try self.resolveInst(bin_op.lhs);
3608	const rhs = try self.resolveInst(bin_op.rhs);
3609	const inst_ty = self.typeOfIndex(inst);
3610	const scalar_ty = inst_ty.scalarType(zcu);
3611
3612	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, fast, inst_ty, 2, .{ lhs, rhs });
3613	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"mul nsw" else .@"mul nuw", lhs, rhs, "");
3614	}
3615
3616	fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3617	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3618	const lhs = try self.resolveInst(bin_op.lhs);
3619	const rhs = try self.resolveInst(bin_op.rhs);
3620
3621	return self.wip.bin(.mul, lhs, rhs, "");
3622	}
3623
3624	fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3625	const o = self.object;
3626	const pt = self.pt;
3627	const zcu = pt.zcu;
3628	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3629	const lhs = try self.resolveInst(bin_op.lhs);
3630	const rhs = try self.resolveInst(bin_op.rhs);
3631	const inst_ty = self.typeOfIndex(inst);
3632	const scalar_ty = inst_ty.scalarType(zcu);
3633	assert(scalar_ty.zigTypeTag(zcu) == .int);
3634	return self.wip.callIntrinsic(
3635	.normal,
3636	.none,
3637	if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",
3638	&.{try o.lowerType(pt, inst_ty)},
3639	&.{ lhs, rhs, .@"0" },
3640	"",
3641	);
3642	}
3643
3644	fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
3645	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3646	const lhs = try self.resolveInst(bin_op.lhs);
3647	const rhs = try self.resolveInst(bin_op.rhs);
3648	const inst_ty = self.typeOfIndex(inst);
3649
3650	return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
3651	}
3652
3653	fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
3654	const zcu = self.pt.zcu;
3655	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3656	const lhs = try self.resolveInst(bin_op.lhs);
3657	const rhs = try self.resolveInst(bin_op.rhs);
3658	const inst_ty = self.typeOfIndex(inst);
3659	const scalar_ty = inst_ty.scalarType(zcu);
3660
3661	if (scalar_ty.isRuntimeFloat()) {
3662	const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
3663	return self.buildFloatOp(.trunc, fast, inst_ty, 1, .{result});
3664	}
3665	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .sdiv else .udiv, lhs, rhs, "");
3666	}
3667
3668	fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
3669	const o = self.object;
3670	const pt = self.pt;
3671	const zcu = pt.zcu;
3672	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3673	const lhs = try self.resolveInst(bin_op.lhs);
3674	const rhs = try self.resolveInst(bin_op.rhs);
3675	const inst_ty = self.typeOfIndex(inst);
3676	const scalar_ty = inst_ty.scalarType(zcu);
3677
3678	if (scalar_ty.isRuntimeFloat()) {
3679	const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
3680	return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});
3681	}
3682	if (scalar_ty.isSignedInt(zcu)) {
3683	const inst_llvm_ty = try o.lowerType(pt, inst_ty);
3684
3685	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
3686	var stack align(@max(
3687	@alignOf(std.heap.StackFallbackAllocator(0)),
3688	@alignOf(ExpectedContents),
3689	)) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
3690	const allocator = stack.get();
3691
3692	const scalar_bits = inst_llvm_ty.scalarBits(&o.builder);
3693	var smin_big_int: std.math.big.int.Mutable = .{
3694	.limbs = try allocator.alloc(
3695	std.math.big.Limb,
3696	std.math.big.int.calcTwosCompLimbCount(scalar_bits),
3697	),
3698	.len = undefined,
3699	.positive = undefined,
3700	};
3701	defer allocator.free(smin_big_int.limbs);
3702	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
3703	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
3704	inst_llvm_ty.scalarType(&o.builder),
3705	smin_big_int.toConst(),
3706	));
3707
3708	const div = try self.wip.bin(.sdiv, lhs, rhs, "divFloor.div");
3709	const rem = try self.wip.bin(.srem, lhs, rhs, "divFloor.rem");
3710	const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "divFloor.rhs_sign");
3711	const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "divFloor.rem_xor_rhs_sign");
3712	const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "divFloor.need_correction");
3713	const correction = try self.wip.cast(.sext, need_correction, inst_llvm_ty, "divFloor.correction");
3714	return self.wip.bin(.@"add nsw", div, correction, "divFloor");
3715	}
3716	return self.wip.bin(.udiv, lhs, rhs, "");
3717	}
3718
3719	fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
3720	const zcu = self.pt.zcu;
3721	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3722	const lhs = try self.resolveInst(bin_op.lhs);
3723	const rhs = try self.resolveInst(bin_op.rhs);
3724	const inst_ty = self.typeOfIndex(inst);
3725	const scalar_ty = inst_ty.scalarType(zcu);
3726
3727	if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
3728	return self.wip.bin(
3729	if (scalar_ty.isSignedInt(zcu)) .@"sdiv exact" else .@"udiv exact",
3730	lhs,
3731	rhs,
3732	"",
3733	);
3734	}
3735
3736	fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
3737	const zcu = self.pt.zcu;
3738	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3739	const lhs = try self.resolveInst(bin_op.lhs);
3740	const rhs = try self.resolveInst(bin_op.rhs);
3741	const inst_ty = self.typeOfIndex(inst);
3742	const scalar_ty = inst_ty.scalarType(zcu);
3743
3744	if (scalar_ty.isRuntimeFloat())
3745	return self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
3746	return self.wip.bin(if (scalar_ty.isSignedInt(zcu))
3747	.srem
3748	else
3749	.urem, lhs, rhs, "");
3750	}
3751
3752	fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
3753	const o = self.object;
3754	const pt = self.pt;
3755	const zcu = pt.zcu;
3756	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3757	const lhs = try self.resolveInst(bin_op.lhs);
3758	const rhs = try self.resolveInst(bin_op.rhs);
3759	const inst_ty = self.typeOfIndex(inst);
3760	const inst_llvm_ty = try o.lowerType(pt, inst_ty);
3761	const scalar_ty = inst_ty.scalarType(zcu);
3762
3763	if (scalar_ty.isRuntimeFloat()) {
3764	const a = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
3765	const b = try self.buildFloatOp(.add, fast, inst_ty, 2, .{ a, rhs });
3766	const c = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ b, rhs });
3767	const zero = try o.builder.zeroInitValue(inst_llvm_ty);
3768	const ltz = try self.buildFloatCmp(fast, .lt, inst_ty, .{ lhs, zero });
3769	return self.wip.select(fast, ltz, c, a, "");
3770	}
3771	if (scalar_ty.isSignedInt(zcu)) {
3772	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
3773	var stack align(@max(
3774	@alignOf(std.heap.StackFallbackAllocator(0)),
3775	@alignOf(ExpectedContents),
3776	)) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
3777	const allocator = stack.get();
3778
3779	const scalar_bits = inst_llvm_ty.scalarBits(&o.builder);
3780	var smin_big_int: std.math.big.int.Mutable = .{
3781	.limbs = try allocator.alloc(
3782	std.math.big.Limb,
3783	std.math.big.int.calcTwosCompLimbCount(scalar_bits),
3784	),
3785	.len = undefined,
3786	.positive = undefined,
3787	};
3788	defer allocator.free(smin_big_int.limbs);
3789	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
3790	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
3791	inst_llvm_ty.scalarType(&o.builder),
3792	smin_big_int.toConst(),
3793	));
3794
3795	const rem = try self.wip.bin(.srem, lhs, rhs, "mod.rem");
3796	const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "mod.rhs_sign");
3797	const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "mod.rem_xor_rhs_sign");
3798	const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "mod.need_correction");
3799	const zero = try o.builder.zeroInitValue(inst_llvm_ty);
3800	const correction = try self.wip.select(.normal, need_correction, rhs, zero, "mod.correction");
3801	return self.wip.bin(.@"add nsw", correction, rem, "mod");
3802	}
3803	return self.wip.bin(.urem, lhs, rhs, "");
3804	}
3805
3806	fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3807	const o = self.object;
3808	const pt = self.pt;
3809	const zcu = pt.zcu;
3810	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3811	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3812	const ptr = try self.resolveInst(bin_op.lhs);
3813	const offset = try self.resolveInst(bin_op.rhs);
3814	const ptr_ty = self.typeOf(bin_op.lhs);
3815	const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu));
3816	switch (ptr_ty.ptrSize(zcu)) {
3817	// It's a pointer to an array, so according to LLVM we need an extra GEP index.
3818	.one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{
3819	try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), offset,
3820	}, ""),
3821	.c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{offset}, ""),
3822	.slice => {
3823	const base = try self.wip.extractValue(ptr, &.{0}, "");
3824	return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{offset}, "");
3825	},
3826	}
3827	}
3828
3829	fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3830	const o = self.object;
3831	const pt = self.pt;
3832	const zcu = pt.zcu;
3833	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3834	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3835	const ptr = try self.resolveInst(bin_op.lhs);
3836	const offset = try self.resolveInst(bin_op.rhs);
3837	const negative_offset = try self.wip.neg(offset, "");
3838	const ptr_ty = self.typeOf(bin_op.lhs);
3839	const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu));
3840	switch (ptr_ty.ptrSize(zcu)) {
3841	// It's a pointer to an array, so according to LLVM we need an extra GEP index.
3842	.one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{
3843	try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), negative_offset,
3844	}, ""),
3845	.c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{negative_offset}, ""),
3846	.slice => {
3847	const base = try self.wip.extractValue(ptr, &.{0}, "");
3848	return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{negative_offset}, "");
3849	},
3850	}
3851	}
3852
3853	fn airOverflow(
3854	self: *FuncGen,
3855	inst: Air.Inst.Index,
3856	signed_intrinsic: Builder.Intrinsic,
3857	unsigned_intrinsic: Builder.Intrinsic,
3858	) !Builder.Value {
3859	const o = self.object;
3860	const pt = self.pt;
3861	const zcu = pt.zcu;
3862	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3863	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
3864
3865	const lhs = try self.resolveInst(extra.lhs);
3866	const rhs = try self.resolveInst(extra.rhs);
3867
3868	const lhs_ty = self.typeOf(extra.lhs);
3869	const scalar_ty = lhs_ty.scalarType(zcu);
3870	const inst_ty = self.typeOfIndex(inst);
3871
3872	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3873	const llvm_inst_ty = try o.lowerType(pt, inst_ty);
3874	const llvm_lhs_ty = try o.lowerType(pt, lhs_ty);
3875	const results =
3876	try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");
3877
3878	const result_val = try self.wip.extractValue(results, &.{0}, "");
3879	const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
3880
3881	const result_index = o.llvmFieldIndex(inst_ty, 0).?;
3882	const overflow_index = o.llvmFieldIndex(inst_ty, 1).?;
3883
3884	if (isByRef(inst_ty, zcu)) {
3885	const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();
3886	const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
3887	{
3888	const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, "");
3889	_ = try self.wip.store(.normal, result_val, field_ptr, result_alignment);
3890	}
3891	{
3892	const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, overflow_index, "");
3893	_ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
3894	}
3895
3896	return alloca_inst;
3897	}
3898
3899	var fields: [2]Builder.Value = undefined;
3900	fields[result_index] = result_val;
3901	fields[overflow_index] = overflow_bit;
3902	return self.wip.buildAggregate(llvm_inst_ty, &fields, "");
3903	}
3904
3905	fn buildElementwiseCall(
3906	self: *FuncGen,
3907	llvm_fn: Builder.Function.Index,
3908	args_vectors: []const Builder.Value,
3909	result_vector: Builder.Value,
3910	vector_len: usize,
3911	) !Builder.Value {
3912	const o = self.object;
3913	assert(args_vectors.len <= 3);
3914
3915	var i: usize = 0;
3916	var result = result_vector;
3917	while (i < vector_len) : (i += 1) {
3918	const index_i32 = try o.builder.intValue(.i32, i);
3919
3920	var args: [3]Builder.Value = undefined;
3921	for (args[0..args_vectors.len], args_vectors) \|*arg_elem, arg_vector\| {
3922	arg_elem.* = try self.wip.extractElement(arg_vector, index_i32, "");
3923	}
3924	const result_elem = try self.wip.call(
3925	.normal,
3926	.ccc,
3927	.none,
3928	llvm_fn.typeOf(&o.builder),
3929	llvm_fn.toValue(&o.builder),
3930	args[0..args_vectors.len],
3931	"",
3932	);
3933	result = try self.wip.insertElement(result, result_elem, index_i32, "");
3934	}
3935	return result;
3936	}
3937
3938	fn getLibcFunction(
3939	self: *FuncGen,
3940	fn_name: Builder.StrtabString,
3941	param_types: []const Builder.Type,
3942	return_type: Builder.Type,
3943	) Allocator.Error!Builder.Function.Index {
3944	const o = self.object;
3945	if (o.builder.getGlobal(fn_name)) \|global\| return switch (global.ptrConst(&o.builder).kind) {
3946	.alias => \|alias\| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function,
3947	.function => \|function\| function,
3948	.variable, .replaced => unreachable,
3949	};
3950	return o.builder.addFunction(
3951	try o.builder.fnType(return_type, param_types, .normal),
3952	fn_name,
3953	toLlvmAddressSpace(.generic, self.pt.zcu.getTarget()),
3954	);
3955	}
3956
3957	/// Creates a floating point comparison by lowering to the appropriate
3958	/// hardware instruction or softfloat routine for the target
3959	fn buildFloatCmp(
3960	self: *FuncGen,
3961	fast: Builder.FastMathKind,
3962	pred: math.CompareOperator,
3963	ty: Type,
3964	params: [2]Builder.Value,
3965	) !Builder.Value {
3966	const o = self.object;
3967	const pt = self.pt;
3968	const zcu = pt.zcu;
3969	const target = zcu.getTarget();
3970	const scalar_ty = ty.scalarType(zcu);
3971	const scalar_llvm_ty = try o.lowerType(pt, scalar_ty);
3972
3973	if (intrinsicsAllowed(scalar_ty, target)) {
3974	const cond: Builder.FloatCondition = switch (pred) {
3975	.eq => .oeq,
3976	.neq => .une,
3977	.lt => .olt,
3978	.lte => .ole,
3979	.gt => .ogt,
3980	.gte => .oge,
3981	};
3982	return self.wip.fcmp(fast, cond, params[0], params[1], "");
3983	}
3984
3985	const float_bits = scalar_ty.floatBits(target);
3986	const compiler_rt_float_abbrev = compilerRtFloatAbbrev(float_bits);
3987	const fn_base_name = switch (pred) {
3988	.neq => "ne",
3989	.eq => "eq",
3990	.lt => "lt",
3991	.lte => "le",
3992	.gt => "gt",
3993	.gte => "ge",
3994	};
3995	const fn_name = try o.builder.strtabStringFmt("__{s}{s}f2", .{ fn_base_name, compiler_rt_float_abbrev });
3996
3997	const libc_fn = try self.getLibcFunction(fn_name, &.{ scalar_llvm_ty, scalar_llvm_ty }, .i32);
3998
3999	const int_cond: Builder.IntegerCondition = switch (pred) {
4000	.eq => .eq,
4001	.neq => .ne,
4002	.lt => .slt,
4003	.lte => .sle,
4004	.gt => .sgt,
4005	.gte => .sge,
4006	};
4007
4008	if (ty.zigTypeTag(zcu) == .vector) {
4009	const vec_len = ty.vectorLen(zcu);
4010	const vector_result_ty = try o.builder.vectorType(.normal, vec_len, .i32);
4011
4012	const init = try o.builder.poisonValue(vector_result_ty);
4013	const result = try self.buildElementwiseCall(libc_fn, &params, init, vec_len);
4014
4015	const zero_vector = try o.builder.splatValue(vector_result_ty, .@"0");
4016	return self.wip.icmp(int_cond, result, zero_vector, "");
4017	}
4018
4019	const result = try self.wip.call(
4020	.normal,
4021	.ccc,
4022	.none,
4023	libc_fn.typeOf(&o.builder),
4024	libc_fn.toValue(&o.builder),
4025	&params,
4026	"",
4027	);
4028	return self.wip.icmp(int_cond, result, .@"0", "");
4029	}
4030
4031	const FloatOp = enum {
4032	add,
4033	ceil,
4034	cos,
4035	div,
4036	exp,
4037	exp2,
4038	fabs,
4039	floor,
4040	fma,
4041	fmax,
4042	fmin,
4043	fmod,
4044	log,
4045	log10,
4046	log2,
4047	mul,
4048	neg,
4049	round,
4050	sin,
4051	sqrt,
4052	sub,
4053	tan,
4054	trunc,
4055	};
4056
4057	const FloatOpStrat = union(enum) {
4058	intrinsic: []const u8,
4059	libc: Builder.String,
4060	};
4061
4062	/// Creates a floating point operation (add, sub, fma, sqrt, exp, etc.)
4063	/// by lowering to the appropriate hardware instruction or softfloat
4064	/// routine for the target
4065	fn buildFloatOp(
4066	self: *FuncGen,
4067	comptime op: FloatOp,
4068	fast: Builder.FastMathKind,
4069	ty: Type,
4070	comptime params_len: usize,
4071	params: [params_len]Builder.Value,
4072	) !Builder.Value {
4073	const o = self.object;
4074	const pt = self.pt;
4075	const zcu = pt.zcu;
4076	const target = zcu.getTarget();
4077	const scalar_ty = ty.scalarType(zcu);
4078	const llvm_ty = try o.lowerType(pt, ty);
4079
4080	if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {
4081	// Some operations are dedicated LLVM instructions, not available as intrinsics
4082	.neg => return self.wip.un(.fneg, params[0], ""),
4083	.add, .sub, .mul, .div, .fmod => return self.wip.bin(switch (fast) {
4084	.normal => switch (op) {
4085	.add => .fadd,
4086	.sub => .fsub,
4087	.mul => .fmul,
4088	.div => .fdiv,
4089	.fmod => .frem,
4090	else => unreachable,
4091	},
4092	.fast => switch (op) {
4093	.add => .@"fadd fast",
4094	.sub => .@"fsub fast",
4095	.mul => .@"fmul fast",
4096	.div => .@"fdiv fast",
4097	.fmod => .@"frem fast",
4098	else => unreachable,
4099	},
4100	}, params[0], params[1], ""),
4101	.fmax,
4102	.fmin,
4103	.ceil,
4104	.cos,
4105	.exp,
4106	.exp2,
4107	.fabs,
4108	.floor,
4109	.log,
4110	.log10,
4111	.log2,
4112	.round,
4113	.sin,
4114	.sqrt,
4115	.trunc,
4116	.fma,
4117	=> return self.wip.callIntrinsic(fast, .none, switch (op) {
4118	.fmax => .maxnum,
4119	.fmin => .minnum,
4120	.ceil => .ceil,
4121	.cos => .cos,
4122	.exp => .exp,
4123	.exp2 => .exp2,
4124	.fabs => .fabs,
4125	.floor => .floor,
4126	.log => .log,
4127	.log10 => .log10,
4128	.log2 => .log2,
4129	.round => .round,
4130	.sin => .sin,
4131	.sqrt => .sqrt,
4132	.trunc => .trunc,
4133	.fma => .fma,
4134	else => unreachable,
4135	}, &.{llvm_ty}, &params, ""),
4136	.tan => unreachable,
4137	};
4138
4139	const float_bits = scalar_ty.floatBits(target);
4140	const fn_name = switch (op) {
4141	.neg => {
4142	// In this case we can generate a softfloat negation by XORing the
4143	// bits with a constant.
4144	const int_ty = try o.builder.intType(@intCast(float_bits));
4145	const cast_ty = try llvm_ty.changeScalar(int_ty, &o.builder);
4146	const sign_mask = try o.builder.splatValue(
4147	cast_ty,
4148	try o.builder.intConst(int_ty, @as(u128, 1) << @intCast(float_bits - 1)),
4149	);
4150	const bitcasted_operand = try self.wip.cast(.bitcast, params[0], cast_ty, "");
4151	const result = try self.wip.bin(.xor, bitcasted_operand, sign_mask, "");
4152	return self.wip.cast(.bitcast, result, llvm_ty, "");
4153	},
4154	.add, .sub, .div, .mul => try o.builder.strtabStringFmt("__{s}{s}f3", .{
4155	@tagName(op), compilerRtFloatAbbrev(float_bits),
4156	}),
4157	.ceil,
4158	.cos,
4159	.exp,
4160	.exp2,
4161	.fabs,
4162	.floor,
4163	.fma,
4164	.fmax,
4165	.fmin,
4166	.fmod,
4167	.log,
4168	.log10,
4169	.log2,
4170	.round,
4171	.sin,
4172	.sqrt,
4173	.tan,
4174	.trunc,
4175	=> try o.builder.strtabStringFmt("{s}{s}{s}", .{
4176	libcFloatPrefix(float_bits), @tagName(op), libcFloatSuffix(float_bits),
4177	}),
4178	};
4179
4180	const scalar_llvm_ty = llvm_ty.scalarType(&o.builder);
4181	const libc_fn = try self.getLibcFunction(
4182	fn_name,
4183	([1]Builder.Type{scalar_llvm_ty} ** 3)[0..params.len],
4184	scalar_llvm_ty,
4185	);
4186	if (ty.zigTypeTag(zcu) == .vector) {
4187	const result = try o.builder.poisonValue(llvm_ty);
4188	return self.buildElementwiseCall(libc_fn, &params, result, ty.vectorLen(zcu));
4189	}
4190
4191	return self.wip.call(
4192	fast.toCallKind(),
4193	.ccc,
4194	.none,
4195	libc_fn.typeOf(&o.builder),
4196	libc_fn.toValue(&o.builder),
4197	&params,
4198	"",
4199	);
4200	}
4201
4202	fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4203	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4204	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
4205
4206	const mulend1 = try self.resolveInst(extra.lhs);
4207	const mulend2 = try self.resolveInst(extra.rhs);
4208	const addend = try self.resolveInst(pl_op.operand);
4209
4210	const ty = self.typeOfIndex(inst);
4211	return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend });
4212	}
4213
4214	fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4215	const o = self.object;
4216	const pt = self.pt;
4217	const zcu = pt.zcu;
4218	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
4219	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
4220
4221	const lhs = try self.resolveInst(extra.lhs);
4222	const rhs = try self.resolveInst(extra.rhs);
4223
4224	const lhs_ty = self.typeOf(extra.lhs);
4225	if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu))
4226	return self.todo("implement vector shifts with scalar rhs", .{});
4227	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
4228
4229	const dest_ty = self.typeOfIndex(inst);
4230	const llvm_dest_ty = try o.lowerType(pt, dest_ty);
4231
4232	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
4233
4234	const result = try self.wip.bin(.shl, lhs, casted_rhs, "");
4235	const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
4236	.ashr
4237	else
4238	.lshr, result, casted_rhs, "");
4239
4240	const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
4241
4242	const result_index = o.llvmFieldIndex(dest_ty, 0).?;
4243	const overflow_index = o.llvmFieldIndex(dest_ty, 1).?;
4244
4245	if (isByRef(dest_ty, zcu)) {
4246	const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();
4247	const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
4248	{
4249	const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, "");
4250	_ = try self.wip.store(.normal, result, field_ptr, result_alignment);
4251	}
4252	{
4253	const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, overflow_index, "");
4254	_ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
4255	}
4256	return alloca_inst;
4257	}
4258
4259	var fields: [2]Builder.Value = undefined;
4260	fields[result_index] = result;
4261	fields[overflow_index] = overflow_bit;
4262	return self.wip.buildAggregate(llvm_dest_ty, &fields, "");
4263	}
4264
4265	fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4266	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4267	const lhs = try self.resolveInst(bin_op.lhs);
4268	const rhs = try self.resolveInst(bin_op.rhs);
4269	return self.wip.bin(.@"and", lhs, rhs, "");
4270	}
4271
4272	fn airOr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4273	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4274	const lhs = try self.resolveInst(bin_op.lhs);
4275	const rhs = try self.resolveInst(bin_op.rhs);
4276	return self.wip.bin(.@"or", lhs, rhs, "");
4277	}
4278
4279	fn airXor(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4280	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4281	const lhs = try self.resolveInst(bin_op.lhs);
4282	const rhs = try self.resolveInst(bin_op.rhs);
4283	return self.wip.bin(.xor, lhs, rhs, "");
4284	}
4285
4286	fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4287	const o = self.object;
4288	const pt = self.pt;
4289	const zcu = pt.zcu;
4290	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4291
4292	const lhs = try self.resolveInst(bin_op.lhs);
4293	const rhs = try self.resolveInst(bin_op.rhs);
4294
4295	const lhs_ty = self.typeOf(bin_op.lhs);
4296	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
4297	return self.todo("implement vector shifts with scalar rhs", .{});
4298	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
4299
4300	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
4301	return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
4302	.@"shl nsw"
4303	else
4304	.@"shl nuw", lhs, casted_rhs, "");
4305	}
4306
4307	fn airShl(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4308	const o = self.object;
4309	const pt = self.pt;
4310	const zcu = pt.zcu;
4311	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4312
4313	const lhs = try self.resolveInst(bin_op.lhs);
4314	const rhs = try self.resolveInst(bin_op.rhs);
4315
4316	const lhs_ty = self.typeOf(bin_op.lhs);
4317	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
4318	return self.todo("implement vector shifts with scalar rhs", .{});
4319
4320	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
4321	return self.wip.bin(.shl, lhs, casted_rhs, "");
4322	}
4323
4324	fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4325	const o = self.object;
4326	const pt = self.pt;
4327	const zcu = pt.zcu;
4328	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4329
4330	const lhs = try self.resolveInst(bin_op.lhs);
4331	const rhs = try self.resolveInst(bin_op.rhs);
4332
4333	const lhs_ty = self.typeOf(bin_op.lhs);
4334	const lhs_info = lhs_ty.intInfo(zcu);
4335	const llvm_lhs_ty = try o.lowerType(pt, lhs_ty);
4336	const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder);
4337
4338	const rhs_ty = self.typeOf(bin_op.rhs);
4339	if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu))
4340	return self.todo("implement vector shifts with scalar rhs", .{});
4341	const rhs_info = rhs_ty.intInfo(zcu);
4342	assert(rhs_info.signedness == .unsigned);
4343	const llvm_rhs_ty = try o.lowerType(pt, rhs_ty);
4344	const llvm_rhs_scalar_ty = llvm_rhs_ty.scalarType(&o.builder);
4345
4346	const result = try self.wip.callIntrinsic(
4347	.normal,
4348	.none,
4349	switch (lhs_info.signedness) {
4350	.signed => .@"sshl.sat",
4351	.unsigned => .@"ushl.sat",
4352	},
4353	&.{llvm_lhs_ty},
4354	&.{ lhs, try self.wip.conv(.unsigned, rhs, llvm_lhs_ty, "") },
4355	"",
4356	);
4357
4358	// LLVM langref says "If b is (statically or dynamically) equal to or
4359	// larger than the integer bit width of the arguments, the result is a
4360	// poison value."
4361	// However Zig semantics says that saturating shift left can never produce
4362	// undefined; instead it saturates.
4363	if (rhs_info.bits <= math.log2_int(u16, lhs_info.bits)) return result;
4364	const bits = try o.builder.splatValue(
4365	llvm_rhs_ty,
4366	try o.builder.intConst(llvm_rhs_scalar_ty, lhs_info.bits),
4367	);
4368	const in_range = try self.wip.icmp(.ult, rhs, bits, "");
4369	const lhs_sat = lhs_sat: switch (lhs_info.signedness) {
4370	.signed => {
4371	const zero = try o.builder.splatValue(
4372	llvm_lhs_ty,
4373	try o.builder.intConst(llvm_lhs_scalar_ty, 0),
4374	);
4375	const smin = try o.builder.splatValue(
4376	llvm_lhs_ty,
4377	try minIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
4378	);
4379	const smax = try o.builder.splatValue(
4380	llvm_lhs_ty,
4381	try maxIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
4382	);
4383	const lhs_lt_zero = try self.wip.icmp(.slt, lhs, zero, "");
4384	const slimit = try self.wip.select(.normal, lhs_lt_zero, smin, smax, "");
4385	const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
4386	break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, slimit, "");
4387	},
4388	.unsigned => {
4389	const zero = try o.builder.splatValue(
4390	llvm_lhs_ty,
4391	try o.builder.intConst(llvm_lhs_scalar_ty, 0),
4392	);
4393	const umax = try o.builder.splatValue(
4394	llvm_lhs_ty,
4395	try o.builder.intConst(llvm_lhs_scalar_ty, -1),
4396	);
4397	const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
4398	break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, umax, "");
4399	},
4400	};
4401	return self.wip.select(.normal, in_range, result, lhs_sat, "");
4402	}
4403
4404	fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value {
4405	const o = self.object;
4406	const pt = self.pt;
4407	const zcu = pt.zcu;
4408	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4409
4410	const lhs = try self.resolveInst(bin_op.lhs);
4411	const rhs = try self.resolveInst(bin_op.rhs);
4412
4413	const lhs_ty = self.typeOf(bin_op.lhs);
4414	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
4415	return self.todo("implement vector shifts with scalar rhs", .{});
4416	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
4417
4418	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
4419	const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);
4420
4421	return self.wip.bin(if (is_exact)
4422	if (is_signed_int) .@"ashr exact" else .@"lshr exact"
4423	else if (is_signed_int) .ashr else .lshr, lhs, casted_rhs, "");
4424	}
4425
4426	fn airAbs(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4427	const o = self.object;
4428	const pt = self.pt;
4429	const zcu = pt.zcu;
4430	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4431	const operand = try self.resolveInst(ty_op.operand);
4432	const operand_ty = self.typeOf(ty_op.operand);
4433	const scalar_ty = operand_ty.scalarType(zcu);
4434
4435	switch (scalar_ty.zigTypeTag(zcu)) {
4436	.int => return self.wip.callIntrinsic(
4437	.normal,
4438	.none,
4439	.abs,
4440	&.{try o.lowerType(pt, operand_ty)},
4441	&.{ operand, try o.builder.intValue(.i1, 0) },
4442	"",
4443	),
4444	.float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),
4445	else => unreachable,
4446	}
4447	}
4448
4449	fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
4450	const o = fg.object;
4451	const pt = fg.pt;
4452	const zcu = pt.zcu;
4453	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4454	const dest_ty = fg.typeOfIndex(inst);
4455	const dest_llvm_ty = try o.lowerType(pt, dest_ty);
4456	const operand = try fg.resolveInst(ty_op.operand);
4457	const operand_ty = fg.typeOf(ty_op.operand);
4458	const operand_info = operand_ty.intInfo(zcu);
4459
4460	const dest_is_enum = dest_ty.zigTypeTag(zcu) == .@"enum";
4461
4462	bounds_check: {
4463	const dest_scalar = dest_ty.scalarType(zcu);
4464	const operand_scalar = operand_ty.scalarType(zcu);
4465
4466	const dest_info = dest_ty.intInfo(zcu);
4467
4468	const have_min_check, const have_max_check = c: {
4469	const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed);
4470	const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed);
4471
4472	const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0;
4473	const operand_maybe_neg = operand_info.signedness == .signed and operand_info.bits > 0;
4474
4475	break :c .{
4476	operand_maybe_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits),
4477	dest_pos_bits < operand_pos_bits,
4478	};
4479	};
4480
4481	if (!have_min_check and !have_max_check) break :bounds_check;
4482
4483	const operand_llvm_ty = try o.lowerType(pt, operand_ty);
4484	const operand_scalar_llvm_ty = try o.lowerType(pt, operand_scalar);
4485
4486	const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
4487	assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));
4488
4489	const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .integer_out_of_bounds;
4490
4491	if (have_min_check) {
4492	const min_const_scalar = try minIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
4493	const min_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, min_const_scalar) else min_const_scalar.toValue();
4494	const ok_maybe_vec = try fg.cmp(.normal, .gte, operand_ty, operand, min_val);
4495	const ok = if (is_vector) ok: {
4496	const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
4497	break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
4498	} else ok_maybe_vec;
4499	if (safety) {
4500	const fail_block = try fg.wip.block(1, "IntMinFail");
4501	const ok_block = try fg.wip.block(1, "IntMinOk");
4502	_ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
4503	fg.wip.cursor = .{ .block = fail_block };
4504	try fg.buildSimplePanic(panic_id);
4505	fg.wip.cursor = .{ .block = ok_block };
4506	} else {
4507	_ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
4508	}
4509	}
4510
4511	if (have_max_check) {
4512	const max_const_scalar = try maxIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
4513	const max_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, max_const_scalar) else max_const_scalar.toValue();
4514	const ok_maybe_vec = try fg.cmp(.normal, .lte, operand_ty, operand, max_val);
4515	const ok = if (is_vector) ok: {
4516	const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
4517	break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
4518	} else ok_maybe_vec;
4519	if (safety) {
4520	const fail_block = try fg.wip.block(1, "IntMaxFail");
4521	const ok_block = try fg.wip.block(1, "IntMaxOk");
4522	_ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
4523	fg.wip.cursor = .{ .block = fail_block };
4524	try fg.buildSimplePanic(panic_id);
4525	fg.wip.cursor = .{ .block = ok_block };
4526	} else {
4527	_ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
4528	}
4529	}
4530	}
4531
4532	const result = try fg.wip.conv(switch (operand_info.signedness) {
4533	.signed => .signed,
4534	.unsigned => .unsigned,
4535	}, operand, dest_llvm_ty, "");
4536
4537	if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) {
4538	const llvm_fn = try fg.getIsNamedEnumValueFunction(dest_ty);
4539	const is_valid_enum_val = try fg.wip.call(
4540	.normal,
4541	.fastcc,
4542	.none,
4543	llvm_fn.typeOf(&o.builder),
4544	llvm_fn.toValue(&o.builder),
4545	&.{result},
4546	"",
4547	);
4548	const fail_block = try fg.wip.block(1, "ValidEnumFail");
4549	const ok_block = try fg.wip.block(1, "ValidEnumOk");
4550	_ = try fg.wip.brCond(is_valid_enum_val, ok_block, fail_block, .none);
4551	fg.wip.cursor = .{ .block = fail_block };
4552	try fg.buildSimplePanic(.invalid_enum_value);
4553	fg.wip.cursor = .{ .block = ok_block };
4554	}
4555
4556	return result;
4557	}
4558
4559	fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4560	const o = self.object;
4561	const pt = self.pt;
4562	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4563	const operand = try self.resolveInst(ty_op.operand);
4564	const dest_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));
4565	return self.wip.cast(.trunc, operand, dest_llvm_ty, "");
4566	}
4567
4568	fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4569	const o = self.object;
4570	const pt = self.pt;
4571	const zcu = pt.zcu;
4572	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4573	const operand = try self.resolveInst(ty_op.operand);
4574	const operand_ty = self.typeOf(ty_op.operand);
4575	const dest_ty = self.typeOfIndex(inst);
4576	const target = zcu.getTarget();
4577
4578	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
4579	return self.wip.cast(.fptrunc, operand, try o.lowerType(pt, dest_ty), "");
4580	} else {
4581	const operand_llvm_ty = try o.lowerType(pt, operand_ty);
4582	const dest_llvm_ty = try o.lowerType(pt, dest_ty);
4583
4584	const dest_bits = dest_ty.floatBits(target);
4585	const src_bits = operand_ty.floatBits(target);
4586	const fn_name = try o.builder.strtabStringFmt("__trunc{s}f{s}f2", .{
4587	compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
4588	});
4589
4590	const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
4591	return self.wip.call(
4592	.normal,
4593	.ccc,
4594	.none,
4595	libc_fn.typeOf(&o.builder),
4596	libc_fn.toValue(&o.builder),
4597	&.{operand},
4598	"",
4599	);
4600	}
4601	}
4602
4603	fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4604	const o = self.object;
4605	const pt = self.pt;
4606	const zcu = pt.zcu;
4607	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4608	const operand = try self.resolveInst(ty_op.operand);
4609	const operand_ty = self.typeOf(ty_op.operand);
4610	const dest_ty = self.typeOfIndex(inst);
4611	const target = zcu.getTarget();
4612
4613	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
4614	return self.wip.cast(.fpext, operand, try o.lowerType(pt, dest_ty), "");
4615	} else {
4616	const operand_llvm_ty = try o.lowerType(pt, operand_ty);
4617	const dest_llvm_ty = try o.lowerType(pt, dest_ty);
4618
4619	const dest_bits = dest_ty.scalarType(zcu).floatBits(target);
4620	const src_bits = operand_ty.scalarType(zcu).floatBits(target);
4621	const fn_name = try o.builder.strtabStringFmt("__extend{s}f{s}f2", .{
4622	compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
4623	});
4624
4625	const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
4626	if (dest_ty.isVector(zcu)) return self.buildElementwiseCall(
4627	libc_fn,
4628	&.{operand},
4629	try o.builder.poisonValue(dest_llvm_ty),
4630	dest_ty.vectorLen(zcu),
4631	);
4632	return self.wip.call(
4633	.normal,
4634	.ccc,
4635	.none,
4636	libc_fn.typeOf(&o.builder),
4637	libc_fn.toValue(&o.builder),
4638	&.{operand},
4639	"",
4640	);
4641	}
4642	}
4643
4644	fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4645	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4646	const operand_ty = self.typeOf(ty_op.operand);
4647	const inst_ty = self.typeOfIndex(inst);
4648	const operand = try self.resolveInst(ty_op.operand);
4649	return self.bitCast(operand, operand_ty, inst_ty);
4650	}
4651
4652	fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) !Builder.Value {
4653	const o = self.object;
4654	const pt = self.pt;
4655	const zcu = pt.zcu;
4656	const operand_is_ref = isByRef(operand_ty, zcu);
4657	const result_is_ref = isByRef(inst_ty, zcu);
4658	const llvm_dest_ty = try o.lowerType(pt, inst_ty);
4659
4660	if (operand_is_ref and result_is_ref) {
4661	// They are both pointers, so just return the same opaque pointer :)
4662	return operand;
4663	}
4664
4665	if (llvm_dest_ty.isInteger(&o.builder) and
4666	operand.typeOfWip(&self.wip).isInteger(&o.builder))
4667	{
4668	return self.wip.conv(.unsigned, operand, llvm_dest_ty, "");
4669	}
4670
4671	const operand_scalar_ty = operand_ty.scalarType(zcu);
4672	const inst_scalar_ty = inst_ty.scalarType(zcu);
4673	if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) {
4674	return self.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
4675	}
4676	if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) {
4677	return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
4678	}
4679
4680	if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
4681	const elem_ty = operand_ty.childType(zcu);
4682	if (!result_is_ref) {
4683	return self.todo("implement bitcast vector to non-ref array", .{});
4684	}
4685	const alignment = inst_ty.abiAlignment(zcu).toLlvm();
4686	const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4687	const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
4688	if (bitcast_ok) {
4689	_ = try self.wip.store(.normal, operand, array_ptr, alignment);
4690	} else {
4691	// If the ABI size of the element type is not evenly divisible by size in bits;
4692	// a simple bitcast will not work, and we fall back to extractelement.
4693	const llvm_usize = try o.lowerType(pt, Type.usize);
4694	const usize_zero = try o.builder.intValue(llvm_usize, 0);
4695	const vector_len = operand_ty.arrayLen(zcu);
4696	var i: u64 = 0;
4697	while (i < vector_len) : (i += 1) {
4698	const elem_ptr = try self.wip.gep(.inbounds, llvm_dest_ty, array_ptr, &.{
4699	usize_zero, try o.builder.intValue(llvm_usize, i),
4700	}, "");
4701	const elem =
4702	try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
4703	_ = try self.wip.store(.normal, elem, elem_ptr, .default);
4704	}
4705	}
4706	return array_ptr;
4707	} else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) {
4708	const elem_ty = operand_ty.childType(zcu);
4709	const llvm_vector_ty = try o.lowerType(pt, inst_ty);
4710	if (!operand_is_ref) return self.todo("implement bitcast non-ref array to vector", .{});
4711
4712	const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
4713	if (bitcast_ok) {
4714	// The array is aligned to the element's alignment, while the vector might have a completely
4715	// different alignment. This means we need to enforce the alignment of this load.
4716	const alignment = elem_ty.abiAlignment(zcu).toLlvm();
4717	return self.wip.load(.normal, llvm_vector_ty, operand, alignment, "");
4718	} else {
4719	// If the ABI size of the element type is not evenly divisible by size in bits;
4720	// a simple bitcast will not work, and we fall back to extractelement.
4721	const array_llvm_ty = try o.lowerType(pt, operand_ty);
4722	const elem_llvm_ty = try o.lowerType(pt, elem_ty);
4723	const llvm_usize = try o.lowerType(pt, Type.usize);
4724	const usize_zero = try o.builder.intValue(llvm_usize, 0);
4725	const vector_len = operand_ty.arrayLen(zcu);
4726	var vector = try o.builder.poisonValue(llvm_vector_ty);
4727	var i: u64 = 0;
4728	while (i < vector_len) : (i += 1) {
4729	const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, operand, &.{
4730	usize_zero, try o.builder.intValue(llvm_usize, i),
4731	}, "");
4732	const elem = try self.wip.load(.normal, elem_llvm_ty, elem_ptr, .default, "");
4733	vector =
4734	try self.wip.insertElement(vector, elem, try o.builder.intValue(.i32, i), "");
4735	}
4736	return vector;
4737	}
4738	}
4739
4740	if (operand_is_ref) {
4741	const alignment = operand_ty.abiAlignment(zcu).toLlvm();
4742	return self.wip.load(.normal, llvm_dest_ty, operand, alignment, "");
4743	}
4744
4745	if (result_is_ref) {
4746	const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
4747	const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4748	_ = try self.wip.store(.normal, operand, result_ptr, alignment);
4749	return result_ptr;
4750	}
4751
4752	if (llvm_dest_ty.isStruct(&o.builder) or
4753	((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and
4754	operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu)))
4755	{
4756	// Both our operand and our result are values, not pointers,
4757	// but LLVM won't let us bitcast struct values or vectors with padding bits.
4758	// Therefore, we store operand to alloca, then load for result.
4759	const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
4760	const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4761	_ = try self.wip.store(.normal, operand, result_ptr, alignment);
4762	return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");
4763	}
4764
4765	return self.wip.cast(.bitcast, operand, llvm_dest_ty, "");
4766	}
4767
4768	fn airArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4769	const o = self.object;
4770	const pt = self.pt;
4771	const zcu = pt.zcu;
4772	const arg_val = self.args[self.arg_index];
4773	self.arg_index += 1;
4774
4775	// llvm does not support debug info for naked function arguments
4776	if (self.is_naked) return arg_val;
4777
4778	const inst_ty = self.typeOfIndex(inst);
4779
4780	const func = zcu.funcInfo(zcu.navValue(self.nav_index).toIntern());
4781	const func_zir = func.zir_body_inst.resolveFull(&zcu.intern_pool).?;
4782	const file = zcu.fileByIndex(func_zir.file);
4783
4784	const mod = file.mod.?;
4785	if (mod.strip) return arg_val;
4786	const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg;
4787	const zir = &file.zir.?;
4788	const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
4789
4790	const lbrace_line = zcu.navSrcLine(func.owner_nav) + func.lbrace_line + 1;
4791	const lbrace_col = func.lbrace_column + 1;
4792
4793	const debug_parameter = try o.builder.debugParameter(
4794	if (name.len > 0) try o.builder.metadataString(name) else null,
4795	self.file,
4796	self.scope,
4797	lbrace_line,
4798	try o.getDebugType(pt, inst_ty),
4799	self.arg_index,
4800	);
4801
4802	const old_location = self.wip.debug_location;
4803	self.wip.debug_location = .{ .location = .{
4804	.line = lbrace_line,
4805	.column = lbrace_col,
4806	.scope = self.scope.toOptional(),
4807	.inlined_at = .none,
4808	} };
4809
4810	if (isByRef(inst_ty, zcu)) {
4811	_ = try self.wip.callIntrinsic(
4812	.normal,
4813	.none,
4814	.@"dbg.declare",
4815	&.{},
4816	&.{
4817	(try self.wip.debugValue(arg_val)).toValue(),
4818	debug_parameter.toValue(),
4819	(try o.builder.debugExpression(&.{})).toValue(),
4820	},
4821	"",
4822	);
4823	} else if (mod.optimize_mode == .Debug) {
4824	const alignment = inst_ty.abiAlignment(zcu).toLlvm();
4825	const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment);
4826	_ = try self.wip.store(.normal, arg_val, alloca, alignment);
4827	_ = try self.wip.callIntrinsic(
4828	.normal,
4829	.none,
4830	.@"dbg.declare",
4831	&.{},
4832	&.{
4833	(try self.wip.debugValue(alloca)).toValue(),
4834	debug_parameter.toValue(),
4835	(try o.builder.debugExpression(&.{})).toValue(),
4836	},
4837	"",
4838	);
4839	} else {
4840	_ = try self.wip.callIntrinsic(
4841	.normal,
4842	.none,
4843	.@"dbg.value",
4844	&.{},
4845	&.{
4846	(try self.wip.debugValue(arg_val)).toValue(),
4847	debug_parameter.toValue(),
4848	(try o.builder.debugExpression(&.{})).toValue(),
4849	},
4850	"",
4851	);
4852	}
4853
4854	self.wip.debug_location = old_location;
4855	return arg_val;
4856	}
4857
4858	fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4859	const o = self.object;
4860	const pt = self.pt;
4861	const zcu = pt.zcu;
4862	const ptr_ty = self.typeOfIndex(inst);
4863	const pointee_type = ptr_ty.childType(zcu);
4864	if (!pointee_type.hasRuntimeBits(zcu))
4865	return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue();
4866
4867	const pointee_llvm_ty = try o.lowerType(pt, pointee_type);
4868	const alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
4869	return self.buildAlloca(pointee_llvm_ty, alignment);
4870	}
4871
4872	fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4873	const o = self.object;
4874	const pt = self.pt;
4875	const zcu = pt.zcu;
4876	const ptr_ty = self.typeOfIndex(inst);
4877	const ret_ty = ptr_ty.childType(zcu);
4878	if (!ret_ty.hasRuntimeBits(zcu))
4879	return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue();
4880	if (self.ret_ptr != .none) return self.ret_ptr;
4881	const ret_llvm_ty = try o.lowerType(pt, ret_ty);
4882	const alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
4883	return self.buildAlloca(ret_llvm_ty, alignment);
4884	}
4885
4886	/// Use this instead of builder.buildAlloca, because this function makes sure to
4887	/// put the alloca instruction at the top of the function!
4888	fn buildAlloca(
4889	self: *FuncGen,
4890	llvm_ty: Builder.Type,
4891	alignment: Builder.Alignment,
4892	) Allocator.Error!Builder.Value {
4893	const target = self.pt.zcu.getTarget();
4894	return buildAllocaInner(&self.wip, llvm_ty, alignment, target);
4895	}
4896
4897	fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
4898	const o = self.object;
4899	const pt = self.pt;
4900	const zcu = pt.zcu;
4901	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4902	const dest_ptr = try self.resolveInst(bin_op.lhs);
4903	const ptr_ty = self.typeOf(bin_op.lhs);
4904	const operand_ty = ptr_ty.childType(zcu);
4905
4906	const val_is_undef = if (try self.air.value(bin_op.rhs, pt)) \|val\| val.isUndef(zcu) else false;
4907	if (val_is_undef) {
4908	const owner_mod = self.ownerModule();
4909
4910	// Even if safety is disabled, we still emit a memset to undefined since it conveys
4911	// extra information to LLVM, and LLVM will optimize it out. Safety makes the difference
4912	// between using 0xaa or actual undefined for the fill byte.
4913	//
4914	// However, for Debug builds specifically, we avoid emitting the memset because LLVM
4915	// will neither use the information nor get rid of the memset, thus leaving an
4916	// unexpected call in the user's code. This is problematic if the code in question is
4917	// not ready to correctly make calls yet, such as in our early PIE startup code, or in
4918	// the early stages of a dynamic linker, etc.
4919	if (!safety and owner_mod.optimize_mode == .Debug) {
4920	return .none;
4921	}
4922
4923	const ptr_info = ptr_ty.ptrInfo(zcu);
4924	const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
4925	if (needs_bitmask) {
4926	// TODO: only some bits are to be undef, we cannot write with a simple memset.
4927	// meanwhile, ignore the write rather than stomping over valid bits.
4928	// https://github.com/ziglang/zig/issues/15337
4929	return .none;
4930	}
4931
4932	self.maybeMarkAllowZeroAccess(ptr_info);
4933
4934	const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), operand_ty.abiSize(zcu));
4935	_ = try self.wip.callMemSet(
4936	dest_ptr,
4937	ptr_ty.ptrAlignment(zcu).toLlvm(),
4938	if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
4939	len,
4940	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
4941	self.disable_intrinsics,
4942	);
4943	if (safety and owner_mod.valgrind) {
4944	try self.valgrindMarkUndef(dest_ptr, len);
4945	}
4946	return .none;
4947	}
4948
4949	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
4950
4951	const src_operand = try self.resolveInst(bin_op.rhs);
4952	try self.store(dest_ptr, ptr_ty, src_operand, .none);
4953	return .none;
4954	}
4955
4956	fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4957	const pt = fg.pt;
4958	const zcu = pt.zcu;
4959	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4960	const ptr_ty = fg.typeOf(ty_op.operand);
4961	const ptr_info = ptr_ty.ptrInfo(zcu);
4962	const ptr = try fg.resolveInst(ty_op.operand);
4963	fg.maybeMarkAllowZeroAccess(ptr_info);
4964	return fg.load(ptr, ptr_ty);
4965	}
4966
4967	fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !void {
4968	_ = inst;
4969	const target = self.pt.zcu.getTarget();
4970	if ((target.cpu.arch == .mips or target.cpu.arch == .mipsel) and
4971	target.cpu.has(.mips, .notraps))
4972	{
4973	// Emit a MIPS `break` instruction followed by an infinite loop (to fulfil the noreturn)
4974	// since this CPU does not support trap instructions.
4975	const o = self.object;
4976	_ = try self.wip.callAsm(
4977	.none,
4978	try o.builder.fnType(.void, &.{}, .normal),
4979	.{ .sideeffect = true },
4980	try o.builder.string("break\n0:\nj 0b\nnop"),
4981	try o.builder.string("~{memory}"),
4982	&.{},
4983	"",
4984	);
4985	} else {
4986	_ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, "");
4987	}
4988	_ = try self.wip.@"unreachable"();
4989	}
4990
4991	fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4992	_ = inst;
4993	_ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, "");
4994	return .none;
4995	}
4996
4997	fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4998	_ = inst;
4999	const o = self.object;
5000	const pt = self.pt;
5001	const llvm_usize = try o.lowerType(pt, Type.usize);
5002	if (!target_util.supportsReturnAddress(self.pt.zcu.getTarget(), self.ownerModule().optimize_mode)) {
5003	// https://github.com/ziglang/zig/issues/11946
5004	return o.builder.intValue(llvm_usize, 0);
5005	}
5006	const result = try self.wip.callIntrinsic(.normal, .none, .returnaddress, &.{}, &.{.@"0"}, "");
5007	return self.wip.cast(.ptrtoint, result, llvm_usize, "");
5008	}
5009
5010	fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5011	_ = inst;
5012	const o = self.object;
5013	const pt = self.pt;
5014	const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");
5015	return self.wip.cast(.ptrtoint, result, try o.lowerType(pt, Type.usize), "");
5016	}
5017
5018	fn airCmpxchg(
5019	self: *FuncGen,
5020	inst: Air.Inst.Index,
5021	kind: Builder.Function.Instruction.CmpXchg.Kind,
5022	) !Builder.Value {
5023	const o = self.object;
5024	const pt = self.pt;
5025	const zcu = pt.zcu;
5026	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5027	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
5028	const ptr = try self.resolveInst(extra.ptr);
5029	const ptr_ty = self.typeOf(extra.ptr);
5030	var expected_value = try self.resolveInst(extra.expected_value);
5031	var new_value = try self.resolveInst(extra.new_value);
5032	const operand_ty = ptr_ty.childType(zcu);
5033	const llvm_operand_ty = try o.lowerType(pt, operand_ty);
5034	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);
5035	if (llvm_abi_ty != .none) {
5036	// operand needs widening and truncating
5037	const signedness: Builder.Function.Instruction.Cast.Signedness =
5038	if (operand_ty.isSignedInt(zcu)) .signed else .unsigned;
5039	expected_value = try self.wip.conv(signedness, expected_value, llvm_abi_ty, "");
5040	new_value = try self.wip.conv(signedness, new_value, llvm_abi_ty, "");
5041	}
5042
5043	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
5044
5045	const result = try self.wip.cmpxchg(
5046	kind,
5047	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
5048	ptr,
5049	expected_value,
5050	new_value,
5051	self.sync_scope,
5052	toLlvmAtomicOrdering(extra.successOrder()),
5053	toLlvmAtomicOrdering(extra.failureOrder()),
5054	ptr_ty.ptrAlignment(zcu).toLlvm(),
5055	"",
5056	);
5057
5058	const optional_ty = self.typeOfIndex(inst);
5059
5060	var payload = try self.wip.extractValue(result, &.{0}, "");
5061	if (llvm_abi_ty != .none) payload = try self.wip.cast(.trunc, payload, llvm_operand_ty, "");
5062	const success_bit = try self.wip.extractValue(result, &.{1}, "");
5063
5064	if (optional_ty.optionalReprIsPayload(zcu)) {
5065	const zero = try o.builder.zeroInitValue(payload.typeOfWip(&self.wip));
5066	return self.wip.select(.normal, success_bit, zero, payload, "");
5067	}
5068
5069	comptime assert(optional_layout_version == 3);
5070
5071	const non_null_bit = try self.wip.not(success_bit, "");
5072	return buildOptional(self, optional_ty, payload, non_null_bit);
5073	}
5074
5075	fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5076	const o = self.object;
5077	const pt = self.pt;
5078	const zcu = pt.zcu;
5079	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5080	const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
5081	const ptr = try self.resolveInst(pl_op.operand);
5082	const ptr_ty = self.typeOf(pl_op.operand);
5083	const operand_ty = ptr_ty.childType(zcu);
5084	const operand = try self.resolveInst(extra.operand);
5085	const is_signed_int = operand_ty.isSignedInt(zcu);
5086	const is_float = operand_ty.isRuntimeFloat();
5087	const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
5088	const ordering = toLlvmAtomicOrdering(extra.ordering());
5089	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg);
5090	const llvm_operand_ty = try o.lowerType(pt, operand_ty);
5091
5092	const access_kind: Builder.MemoryAccessKind =
5093	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
5094	const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
5095
5096	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
5097
5098	if (llvm_abi_ty != .none) {
5099	// operand needs widening and truncating or bitcasting.
5100	return self.wip.cast(if (is_float) .bitcast else .trunc, try self.wip.atomicrmw(
5101	access_kind,
5102	op,
5103	ptr,
5104	try self.wip.cast(
5105	if (is_float) .bitcast else if (is_signed_int) .sext else .zext,
5106	operand,
5107	llvm_abi_ty,
5108	"",
5109	),
5110	self.sync_scope,
5111	ordering,
5112	ptr_alignment,
5113	"",
5114	), llvm_operand_ty, "");
5115	}
5116
5117	if (!llvm_operand_ty.isPointer(&o.builder)) return self.wip.atomicrmw(
5118	access_kind,
5119	op,
5120	ptr,
5121	operand,
5122	self.sync_scope,
5123	ordering,
5124	ptr_alignment,
5125	"",
5126	);
5127
5128	// It's a pointer but we need to treat it as an int.
5129	return self.wip.cast(.inttoptr, try self.wip.atomicrmw(
5130	access_kind,
5131	op,
5132	ptr,
5133	try self.wip.cast(.ptrtoint, operand, try o.lowerType(pt, Type.usize), ""),
5134	self.sync_scope,
5135	ordering,
5136	ptr_alignment,
5137	"",
5138	), llvm_operand_ty, "");
5139	}
5140
5141	fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5142	const o = self.object;
5143	const pt = self.pt;
5144	const zcu = pt.zcu;
5145	const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
5146	const ptr = try self.resolveInst(atomic_load.ptr);
5147	const ptr_ty = self.typeOf(atomic_load.ptr);
5148	const info = ptr_ty.ptrInfo(zcu);
5149	const elem_ty = Type.fromInterned(info.child);
5150	if (!elem_ty.hasRuntimeBits(zcu)) return .none;
5151	const ordering = toLlvmAtomicOrdering(atomic_load.order);
5152	const llvm_abi_ty = try self.getAtomicAbiType(elem_ty, false);
5153	const ptr_alignment = (if (info.flags.alignment != .none)
5154	@as(InternPool.Alignment, info.flags.alignment)
5155	else
5156	Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();
5157	const access_kind: Builder.MemoryAccessKind =
5158	if (info.flags.is_volatile) .@"volatile" else .normal;
5159	const elem_llvm_ty = try o.lowerType(pt, elem_ty);
5160
5161	self.maybeMarkAllowZeroAccess(info);
5162
5163	if (llvm_abi_ty != .none) {
5164	// operand needs widening and truncating
5165	const loaded = try self.wip.loadAtomic(
5166	access_kind,
5167	llvm_abi_ty,
5168	ptr,
5169	self.sync_scope,
5170	ordering,
5171	ptr_alignment,
5172	"",
5173	);
5174	return self.wip.cast(.trunc, loaded, elem_llvm_ty, "");
5175	}
5176	return self.wip.loadAtomic(
5177	access_kind,
5178	elem_llvm_ty,
5179	ptr,
5180	self.sync_scope,
5181	ordering,
5182	ptr_alignment,
5183	"",
5184	);
5185	}
5186
5187	fn airAtomicStore(
5188	self: *FuncGen,
5189	inst: Air.Inst.Index,
5190	ordering: Builder.AtomicOrdering,
5191	) !Builder.Value {
5192	const pt = self.pt;
5193	const zcu = pt.zcu;
5194	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5195	const ptr_ty = self.typeOf(bin_op.lhs);
5196	const operand_ty = ptr_ty.childType(zcu);
5197	if (!operand_ty.hasRuntimeBits(zcu)) return .none;
5198	const ptr = try self.resolveInst(bin_op.lhs);
5199	var element = try self.resolveInst(bin_op.rhs);
5200	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);
5201
5202	if (llvm_abi_ty != .none) {
5203	// operand needs widening
5204	element = try self.wip.conv(
5205	if (operand_ty.isSignedInt(zcu)) .signed else .unsigned,
5206	element,
5207	llvm_abi_ty,
5208	"",
5209	);
5210	}
5211
5212	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
5213
5214	try self.store(ptr, ptr_ty, element, ordering);
5215	return .none;
5216	}
5217
5218	fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
5219	const o = self.object;
5220	const pt = self.pt;
5221	const zcu = pt.zcu;
5222	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5223	const dest_slice = try self.resolveInst(bin_op.lhs);
5224	const ptr_ty = self.typeOf(bin_op.lhs);
5225	const elem_ty = self.typeOf(bin_op.rhs);
5226	const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();
5227	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);
5228	const access_kind: Builder.MemoryAccessKind =
5229	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
5230
5231	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
5232
5233	if (try self.air.value(bin_op.rhs, pt)) \|elem_val\| {
5234	if (elem_val.isUndef(zcu)) {
5235	// Even if safety is disabled, we still emit a memset to undefined since it conveys
5236	// extra information to LLVM. However, safety makes the difference between using
5237	// 0xaa or actual undefined for the fill byte.
5238	const fill_byte = if (safety)
5239	try o.builder.intValue(.i8, 0xaa)
5240	else
5241	try o.builder.undefValue(.i8);
5242	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5243	_ = try self.wip.callMemSet(
5244	dest_ptr,
5245	dest_ptr_align,
5246	fill_byte,
5247	len,
5248	access_kind,
5249	self.disable_intrinsics,
5250	);
5251	const owner_mod = self.ownerModule();
5252	if (safety and owner_mod.valgrind) {
5253	try self.valgrindMarkUndef(dest_ptr, len);
5254	}
5255	return .none;
5256	}
5257
5258	// Test if the element value is compile-time known to be a
5259	// repeating byte pattern, for example, `@as(u64, 0)` has a
5260	// repeating byte pattern of 0 bytes. In such case, the memset
5261	// intrinsic can be used.
5262	if (try elem_val.hasRepeatedByteRepr(pt)) \|byte_val\| {
5263	const fill_byte = try o.builder.intValue(.i8, byte_val);
5264	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5265	_ = try self.wip.callMemSet(
5266	dest_ptr,
5267	dest_ptr_align,
5268	fill_byte,
5269	len,
5270	access_kind,
5271	self.disable_intrinsics,
5272	);
5273	return .none;
5274	}
5275	}
5276
5277	const value = try self.resolveInst(bin_op.rhs);
5278	const elem_abi_size = elem_ty.abiSize(zcu);
5279
5280	if (elem_abi_size == 1) {
5281	// In this case we can take advantage of LLVM's intrinsic.
5282	const fill_byte = try self.bitCast(value, elem_ty, Type.u8);
5283	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5284
5285	_ = try self.wip.callMemSet(
5286	dest_ptr,
5287	dest_ptr_align,
5288	fill_byte,
5289	len,
5290	access_kind,
5291	self.disable_intrinsics,
5292	);
5293	return .none;
5294	}
5295
5296	// non-byte-sized element. lower with a loop. something like this:
5297
5298	// entry:
5299	// ...
5300	// %end_ptr = getelementptr %ptr, %len
5301	// br %loop
5302	// loop:
5303	// %it_ptr = phi body %next_ptr, entry %ptr
5304	// %end = cmp eq %it_ptr, %end_ptr
5305	// br %end, %body, %end
5306	// body:
5307	// store %it_ptr, %value
5308	// %next_ptr = getelementptr %it_ptr, 1
5309	// br %loop
5310	// end:
5311	// ...
5312	const entry_block = self.wip.cursor.block;
5313	const loop_block = try self.wip.block(2, "InlineMemsetLoop");
5314	const body_block = try self.wip.block(1, "InlineMemsetBody");
5315	const end_block = try self.wip.block(1, "InlineMemsetEnd");
5316
5317	const llvm_usize_ty = try o.lowerType(pt, Type.usize);
5318	const len = switch (ptr_ty.ptrSize(zcu)) {
5319	.slice => try self.wip.extractValue(dest_slice, &.{1}, ""),
5320	.one => try o.builder.intValue(llvm_usize_ty, ptr_ty.childType(zcu).arrayLen(zcu)),
5321	.many, .c => unreachable,
5322	};
5323	const elem_llvm_ty = try o.lowerType(pt, elem_ty);
5324	const end_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, dest_ptr, &.{len}, "");
5325	_ = try self.wip.br(loop_block);
5326
5327	self.wip.cursor = .{ .block = loop_block };
5328	const it_ptr = try self.wip.phi(.ptr, "");
5329	const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, "");
5330	_ = try self.wip.brCond(end, body_block, end_block, .none);
5331
5332	self.wip.cursor = .{ .block = body_block };
5333	const elem_abi_align = elem_ty.abiAlignment(zcu);
5334	const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm();
5335	if (isByRef(elem_ty, zcu)) {
5336	_ = try self.wip.callMemCpy(
5337	it_ptr.toValue(),
5338	it_ptr_align,
5339	value,
5340	elem_abi_align.toLlvm(),
5341	try o.builder.intValue(llvm_usize_ty, elem_abi_size),
5342	access_kind,
5343	self.disable_intrinsics,
5344	);
5345	} else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
5346	const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{
5347	try o.builder.intValue(llvm_usize_ty, 1),
5348	}, "");
5349	_ = try self.wip.br(loop_block);
5350
5351	self.wip.cursor = .{ .block = end_block };
5352	it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip);
5353	return .none;
5354	}
5355
5356	fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5357	const pt = self.pt;
5358	const zcu = pt.zcu;
5359	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5360	const dest_slice = try self.resolveInst(bin_op.lhs);
5361	const dest_ptr_ty = self.typeOf(bin_op.lhs);
5362	const src_slice = try self.resolveInst(bin_op.rhs);
5363	const src_ptr_ty = self.typeOf(bin_op.rhs);
5364	const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
5365	const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
5366	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
5367	const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
5368	dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
5369
5370	self.maybeMarkAllowZeroAccess(dest_ptr_ty.ptrInfo(zcu));
5371	self.maybeMarkAllowZeroAccess(src_ptr_ty.ptrInfo(zcu));
5372
5373	_ = try self.wip.callMemCpy(
5374	dest_ptr,
5375	dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
5376	src_ptr,
5377	src_ptr_ty.ptrAlignment(zcu).toLlvm(),
5378	len,
5379	access_kind,
5380	self.disable_intrinsics,
5381	);
5382	return .none;
5383	}
5384
5385	fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5386	const pt = self.pt;
5387	const zcu = pt.zcu;
5388	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5389	const dest_slice = try self.resolveInst(bin_op.lhs);
5390	const dest_ptr_ty = self.typeOf(bin_op.lhs);
5391	const src_slice = try self.resolveInst(bin_op.rhs);
5392	const src_ptr_ty = self.typeOf(bin_op.rhs);
5393	const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
5394	const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
5395	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
5396	const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
5397	dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
5398
5399	_ = try self.wip.callMemMove(
5400	dest_ptr,
5401	dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
5402	src_ptr,
5403	src_ptr_ty.ptrAlignment(zcu).toLlvm(),
5404	len,
5405	access_kind,
5406	);
5407	return .none;
5408	}
5409
5410	fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5411	const o = self.object;
5412	const pt = self.pt;
5413	const zcu = pt.zcu;
5414	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5415	const un_ptr_ty = self.typeOf(bin_op.lhs);
5416	const un_ty = un_ptr_ty.childType(zcu);
5417	const layout = un_ty.unionGetLayout(zcu);
5418	if (layout.tag_size == 0) return .none;
5419
5420	const access_kind: Builder.MemoryAccessKind =
5421	if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
5422
5423	self.maybeMarkAllowZeroAccess(un_ptr_ty.ptrInfo(zcu));
5424
5425	const union_ptr = try self.resolveInst(bin_op.lhs);
5426	const new_tag = try self.resolveInst(bin_op.rhs);
5427	const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);
5428	if (layout.payload_size == 0) {
5429	_ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());
5430	return .none;
5431	}
5432	const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
5433	const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(pt, un_ty), union_ptr, tag_index, "");
5434	const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) {
5435	0 => union_ptr_align,
5436	else => \|off\| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))),
5437	};
5438	_ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm());
5439	return .none;
5440	}
5441
5442	fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5443	const o = self.object;
5444	const pt = self.pt;
5445	const zcu = pt.zcu;
5446	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5447	const un_ty = self.typeOf(ty_op.operand);
5448	const layout = un_ty.unionGetLayout(zcu);
5449	if (layout.tag_size == 0) return .none;
5450	const union_handle = try self.resolveInst(ty_op.operand);
5451	if (isByRef(un_ty, zcu)) {
5452	const llvm_un_ty = try o.lowerType(pt, un_ty);
5453	if (layout.payload_size == 0)
5454	return self.wip.load(.normal, llvm_un_ty, union_handle, .default, "");
5455	const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
5456	const tag_field_ptr = try self.wip.gepStruct(llvm_un_ty, union_handle, tag_index, "");
5457	const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index];
5458	return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");
5459	} else {
5460	if (layout.payload_size == 0) return union_handle;
5461	const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
5462	return self.wip.extractValue(union_handle, &.{tag_index}, "");
5463	}
5464	}
5465
5466	fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) !Builder.Value {
5467	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5468	const operand = try self.resolveInst(un_op);
5469	const operand_ty = self.typeOf(un_op);
5470
5471	return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand});
5472	}
5473
5474	fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
5475	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5476	const operand = try self.resolveInst(un_op);
5477	const operand_ty = self.typeOf(un_op);
5478
5479	return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand});
5480	}
5481
5482	fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value {
5483	const o = self.object;
5484	const pt = self.pt;
5485	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5486	const inst_ty = self.typeOfIndex(inst);
5487	const operand_ty = self.typeOf(ty_op.operand);
5488	const operand = try self.resolveInst(ty_op.operand);
5489
5490	const result = try self.wip.callIntrinsic(
5491	.normal,
5492	.none,
5493	intrinsic,
5494	&.{try o.lowerType(pt, operand_ty)},
5495	&.{ operand, .false },
5496	"",
5497	);
5498	return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
5499	}
5500
5501	fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value {
5502	const o = self.object;
5503	const pt = self.pt;
5504	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5505	const inst_ty = self.typeOfIndex(inst);
5506	const operand_ty = self.typeOf(ty_op.operand);
5507	const operand = try self.resolveInst(ty_op.operand);
5508
5509	const result = try self.wip.callIntrinsic(
5510	.normal,
5511	.none,
5512	intrinsic,
5513	&.{try o.lowerType(pt, operand_ty)},
5514	&.{operand},
5515	"",
5516	);
5517	return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
5518	}
5519
5520	fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5521	const o = self.object;
5522	const pt = self.pt;
5523	const zcu = pt.zcu;
5524	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5525	const operand_ty = self.typeOf(ty_op.operand);
5526	var bits = operand_ty.intInfo(zcu).bits;
5527	assert(bits % 8 == 0);
5528
5529	const inst_ty = self.typeOfIndex(inst);
5530	var operand = try self.resolveInst(ty_op.operand);
5531	var llvm_operand_ty = try o.lowerType(pt, operand_ty);
5532
5533	if (bits % 16 == 8) {
5534	// If not an even byte-multiple, we need zero-extend + shift-left 1 byte
5535	// The truncated result at the end will be the correct bswap
5536	const scalar_ty = try o.builder.intType(@intCast(bits + 8));
5537	if (operand_ty.zigTypeTag(zcu) == .vector) {
5538	const vec_len = operand_ty.vectorLen(zcu);
5539	llvm_operand_ty = try o.builder.vectorType(.normal, vec_len, scalar_ty);
5540	} else llvm_operand_ty = scalar_ty;
5541
5542	const shift_amt =
5543	try o.builder.splatValue(llvm_operand_ty, try o.builder.intConst(scalar_ty, 8));
5544	const extended = try self.wip.cast(.zext, operand, llvm_operand_ty, "");
5545	operand = try self.wip.bin(.shl, extended, shift_amt, "");
5546
5547	bits = bits + 8;
5548	}
5549
5550	const result =
5551	try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");
5552	return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
5553	}
5554
5555	fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5556	const o = self.object;
5557	const pt = self.pt;
5558	const zcu = pt.zcu;
5559	const ip = &zcu.intern_pool;
5560	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5561	const operand = try self.resolveInst(ty_op.operand);
5562	const error_set_ty = ty_op.ty.toType();
5563
5564	const names = error_set_ty.errorSetNames(zcu);
5565	const valid_block = try self.wip.block(@intCast(names.len), "Valid");
5566	const invalid_block = try self.wip.block(1, "Invalid");
5567	const end_block = try self.wip.block(2, "End");
5568	var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none);
5569	defer wip_switch.finish(&self.wip);
5570
5571	for (0..names.len) \|name_index\| {
5572	const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
5573	const this_tag_int_value = try o.builder.intConst(try o.errorIntType(pt), err_int);
5574	try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);
5575	}
5576	self.wip.cursor = .{ .block = valid_block };
5577	_ = try self.wip.br(end_block);
5578
5579	self.wip.cursor = .{ .block = invalid_block };
5580	_ = try self.wip.br(end_block);
5581
5582	self.wip.cursor = .{ .block = end_block };
5583	const phi = try self.wip.phi(.i1, "");
5584	phi.finish(&.{ .true, .false }, &.{ valid_block, invalid_block }, &self.wip);
5585	return phi.toValue();
5586	}
5587
5588	fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5589	const o = self.object;
5590	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5591	const operand = try self.resolveInst(un_op);
5592	const enum_ty = self.typeOf(un_op);
5593
5594	const llvm_fn = try self.getIsNamedEnumValueFunction(enum_ty);
5595	return self.wip.call(
5596	.normal,
5597	.fastcc,
5598	.none,
5599	llvm_fn.typeOf(&o.builder),
5600	llvm_fn.toValue(&o.builder),
5601	&.{operand},
5602	"",
5603	);
5604	}
5605
5606	fn getIsNamedEnumValueFunction(self: *FuncGen, enum_ty: Type) !Builder.Function.Index {
5607	const o = self.object;
5608	const pt = self.pt;
5609	const zcu = pt.zcu;
5610	const ip = &zcu.intern_pool;
5611
5612	const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern());
5613	if (gop.found_existing) return gop.value_ptr.*;
5614	errdefer assert(o.named_enum_map.remove(enum_ty.toIntern()));
5615	const function_index = try o.builder.addFunction(
5616	// Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type.
5617	// TODO: change the builder API so we don't need to do this.
5618	try o.builder.fnType(.void, &.{}, .normal),
5619	try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}),
5620	toLlvmAddressSpace(.generic, zcu.getTarget()),
5621	);
5622	gop.value_ptr.* = function_index;
5623	try o.updateIsNamedEnumValueFunction(pt, enum_ty, function_index);
5624	return function_index;
5625	}
5626
5627	fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5628	const o = self.object;
5629	const pt = self.pt;
5630	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5631	const operand = try self.resolveInst(un_op);
5632	const enum_ty = self.typeOf(un_op);
5633
5634	const llvm_fn = try o.getEnumTagNameFunction(pt, enum_ty);
5635	return self.wip.call(
5636	.normal,
5637	.fastcc,
5638	.none,
5639	llvm_fn.typeOf(&o.builder),
5640	llvm_fn.toValue(&o.builder),
5641	&.{operand},
5642	"",
5643	);
5644	}
5645
5646	fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5647	const o = self.object;
5648	const pt = self.pt;
5649	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5650	const operand = try self.resolveInst(un_op);
5651	const slice_ty = self.typeOfIndex(inst);
5652	const slice_llvm_ty = try o.lowerType(pt, slice_ty);
5653
5654	// If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
5655	const extended_operand = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), "");
5656
5657	const error_name_table_ptr = try self.getErrorNameTable();
5658	const error_name_table =
5659	try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, "");
5660	const error_name_ptr =
5661	try self.wip.gep(.inbounds, slice_llvm_ty, error_name_table, &.{extended_operand}, "");
5662	return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");
5663	}
5664
5665	fn airSplat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5666	const o = self.object;
5667	const pt = self.pt;
5668	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5669	const scalar = try self.resolveInst(ty_op.operand);
5670	const vector_ty = self.typeOfIndex(inst);
5671	return self.wip.splatVector(try o.lowerType(pt, vector_ty), scalar, "");
5672	}
5673
5674	fn airSelect(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5675	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
5676	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
5677	const pred = try self.resolveInst(pl_op.operand);
5678	const a = try self.resolveInst(extra.lhs);
5679	const b = try self.resolveInst(extra.rhs);
5680
5681	return self.wip.select(.normal, pred, a, b, "");
5682	}
5683
5684	fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5685	const o = fg.object;
5686	const pt = fg.pt;
5687	const zcu = pt.zcu;
5688	const gpa = zcu.gpa;
5689
5690	const unwrapped = fg.air.unwrapShuffleOne(zcu, inst);
5691
5692	const operand = try fg.resolveInst(unwrapped.operand);
5693	const mask = unwrapped.mask;
5694	const operand_ty = fg.typeOf(unwrapped.operand);
5695	const llvm_operand_ty = try o.lowerType(pt, operand_ty);
5696	const llvm_result_ty = try o.lowerType(pt, unwrapped.result_ty);
5697	const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu));
5698	const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
5699	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
5700	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
5701
5702	// LLVM requires that the two input vectors have the same length, so lowering isn't trivial.
5703	// And, in the words of jacobly0: "llvm sucks at shuffles so we do have to hold its hand at
5704	// least a bit". So, there are two cases here.
5705	//
5706	// If the operand length equals the mask length, we do just the one `shufflevector`, where
5707	// the second operand is a constant vector with comptime-known elements at the right indices
5708	// and poison values elsewhere (in the indices which won't be selected).
5709	//
5710	// Otherwise, we lower to two `shufflevector` instructions. The first shuffles the runtime
5711	// operand with an all-poison vector to extract and correctly position all of the runtime
5712	// elements. We also make a constant vector with all of the comptime elements correctly
5713	// positioned. Then, our second instruction selects elements from those "runtime-or-poison"
5714	// and "comptime-or-poison" vectors to compute the result.
5715
5716	// This buffer is used primarily for the mask constants.
5717	const llvm_elem_buf = try gpa.alloc(Builder.Constant, mask.len);
5718	defer gpa.free(llvm_elem_buf);
5719
5720	// ...but first, we'll collect all of the comptime-known values.
5721	var any_defined_comptime_value = false;
5722	for (mask, llvm_elem_buf) \|mask_elem, *llvm_elem\| {
5723	llvm_elem.* = switch (mask_elem.unwrap()) {
5724	.elem => llvm_poison_elem,
5725	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
5726	any_defined_comptime_value = true;
5727	break :elem try o.lowerValue(pt, val);
5728	} else llvm_poison_elem,
5729	};
5730	}
5731	// This vector is like the result, but runtime elements are replaced with poison.
5732	const comptime_and_poison: Builder.Value = if (any_defined_comptime_value) vec: {
5733	break :vec try o.builder.vectorValue(llvm_result_ty, llvm_elem_buf);
5734	} else try o.builder.poisonValue(llvm_result_ty);
5735
5736	if (operand_ty.vectorLen(zcu) == mask.len) {
5737	// input length equals mask/output length, so we lower to one instruction
5738	for (mask, llvm_elem_buf, 0..) \|mask_elem, *llvm_elem, elem_idx\| {
5739	llvm_elem.* = switch (mask_elem.unwrap()) {
5740	.elem => \|idx\| try o.builder.intConst(.i32, idx),
5741	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
5742	break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
5743	} else llvm_poison_mask_elem,
5744	};
5745	}
5746	return fg.wip.shuffleVector(
5747	operand,
5748	comptime_and_poison,
5749	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
5750	"",
5751	);
5752	}
5753
5754	for (mask, llvm_elem_buf) \|mask_elem, *llvm_elem\| {
5755	llvm_elem.* = switch (mask_elem.unwrap()) {
5756	.elem => \|idx\| try o.builder.intConst(.i32, idx),
5757	.value => llvm_poison_mask_elem,
5758	};
5759	}
5760	// This vector is like our result, but all comptime-known elements are poison.
5761	const runtime_and_poison = try fg.wip.shuffleVector(
5762	operand,
5763	try o.builder.poisonValue(llvm_operand_ty),
5764	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
5765	"",
5766	);
5767
5768	if (!any_defined_comptime_value) {
5769	// `comptime_and_poison` is just poison; a second shuffle would be a nop.
5770	return runtime_and_poison;
5771	}
5772
5773	// In this second shuffle, the inputs, the mask, and the output all have the same length.
5774	for (mask, llvm_elem_buf, 0..) \|mask_elem, *llvm_elem, elem_idx\| {
5775	llvm_elem.* = switch (mask_elem.unwrap()) {
5776	.elem => try o.builder.intConst(.i32, elem_idx),
5777	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
5778	break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
5779	} else llvm_poison_mask_elem,
5780	};
5781	}
5782	// Merge the runtime and comptime elements with the mask we just built.
5783	return fg.wip.shuffleVector(
5784	runtime_and_poison,
5785	comptime_and_poison,
5786	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
5787	"",
5788	);
5789	}
5790
5791	fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5792	const o = fg.object;
5793	const pt = fg.pt;
5794	const zcu = pt.zcu;
5795	const gpa = zcu.gpa;
5796
5797	const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
5798
5799	const mask = unwrapped.mask;
5800	const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu));
5801	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
5802	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
5803
5804	// This is kind of simpler than in `airShuffleOne`. We extend the shorter vector to the
5805	// length of the longer one with an initial `shufflevector` if necessary, and then do the
5806	// actual computation with a second `shufflevector`.
5807
5808	const operand_a_len = fg.typeOf(unwrapped.operand_a).vectorLen(zcu);
5809	const operand_b_len = fg.typeOf(unwrapped.operand_b).vectorLen(zcu);
5810	const operand_len: u32 = @max(operand_a_len, operand_b_len);
5811
5812	// If we need to extend an operand, this is the type that mask will have.
5813	const llvm_operand_mask_ty = try o.builder.vectorType(.normal, operand_len, .i32);
5814
5815	const llvm_elem_buf = try gpa.alloc(Builder.Constant, @max(mask.len, operand_len));
5816	defer gpa.free(llvm_elem_buf);
5817
5818	const operand_a: Builder.Value = extend: {
5819	const raw = try fg.resolveInst(unwrapped.operand_a);
5820	if (operand_a_len == operand_len) break :extend raw;
5821	// Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
5822	const mask_elems = llvm_elem_buf[0..operand_len];
5823	for (mask_elems[0..operand_a_len], 0..) \|*llvm_elem, elem_idx\| {
5824	llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
5825	}
5826	@memset(mask_elems[operand_a_len..], llvm_poison_mask_elem);
5827	const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_a_len, llvm_elem_ty);
5828	break :extend try fg.wip.shuffleVector(
5829	raw,
5830	try o.builder.poisonValue(llvm_this_operand_ty),
5831	try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
5832	"",
5833	);
5834	};
5835	const operand_b: Builder.Value = extend: {
5836	const raw = try fg.resolveInst(unwrapped.operand_b);
5837	if (operand_b_len == operand_len) break :extend raw;
5838	// Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
5839	const mask_elems = llvm_elem_buf[0..operand_len];
5840	for (mask_elems[0..operand_b_len], 0..) \|*llvm_elem, elem_idx\| {
5841	llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
5842	}
5843	@memset(mask_elems[operand_b_len..], llvm_poison_mask_elem);
5844	const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_b_len, llvm_elem_ty);
5845	break :extend try fg.wip.shuffleVector(
5846	raw,
5847	try o.builder.poisonValue(llvm_this_operand_ty),
5848	try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
5849	"",
5850	);
5851	};
5852
5853	// `operand_a` and `operand_b` now have the same length (we've extended the shorter one with
5854	// an initial shuffle if necessary). Now for the easy bit.
5855
5856	const mask_elems = llvm_elem_buf[0..mask.len];
5857	for (mask, mask_elems) \|mask_elem, *llvm_mask_elem\| {
5858	llvm_mask_elem.* = switch (mask_elem.unwrap()) {
5859	.a_elem => \|idx\| try o.builder.intConst(.i32, idx),
5860	.b_elem => \|idx\| try o.builder.intConst(.i32, operand_len + idx),
5861	.undef => llvm_poison_mask_elem,
5862	};
5863	}
5864	return fg.wip.shuffleVector(
5865	operand_a,
5866	operand_b,
5867	try o.builder.vectorValue(llvm_mask_ty, mask_elems),
5868	"",
5869	);
5870	}
5871
5872	/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.
5873	///
5874	/// Equivalent to:
5875	/// reduce: {
5876	/// var i: usize = 0;
5877	/// var accum: T = init;
5878	/// while (i < vec.len) : (i += 1) {
5879	/// accum = llvm_fn(accum, vec[i]);
5880	/// }
5881	/// break :reduce accum;
5882	/// }
5883	///
5884	fn buildReducedCall(
5885	self: *FuncGen,
5886	llvm_fn: Builder.Function.Index,
5887	operand_vector: Builder.Value,
5888	vector_len: usize,
5889	accum_init: Builder.Value,
5890	) !Builder.Value {
5891	const o = self.object;
5892	const pt = self.pt;
5893	const usize_ty = try o.lowerType(pt, Type.usize);
5894	const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len);
5895	const llvm_result_ty = accum_init.typeOfWip(&self.wip);
5896
5897	// Allocate and initialize our mutable variables
5898	const i_ptr = try self.buildAlloca(usize_ty, .default);
5899	_ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
5900	const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
5901	_ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
5902
5903	// Setup the loop
5904	const loop = try self.wip.block(2, "ReduceLoop");
5905	const loop_exit = try self.wip.block(1, "AfterReduce");
5906	_ = try self.wip.br(loop);
5907	{
5908	self.wip.cursor = .{ .block = loop };
5909
5910	// while (i < vec.len)
5911	const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, "");
5912	const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");
5913	const loop_then = try self.wip.block(1, "ReduceLoopThen");
5914
5915	_ = try self.wip.brCond(cond, loop_then, loop_exit, .none);
5916
5917	{
5918	self.wip.cursor = .{ .block = loop_then };
5919
5920	// accum = f(accum, vec[i]);
5921	const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
5922	const element = try self.wip.extractElement(operand_vector, i, "");
5923	const new_accum = try self.wip.call(
5924	.normal,
5925	.ccc,
5926	.none,
5927	llvm_fn.typeOf(&o.builder),
5928	llvm_fn.toValue(&o.builder),
5929	&.{ accum, element },
5930	"",
5931	);
5932	_ = try self.wip.store(.normal, new_accum, accum_ptr, .default);
5933
5934	// i += 1
5935	const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), "");
5936	_ = try self.wip.store(.normal, new_i, i_ptr, .default);
5937	_ = try self.wip.br(loop);
5938	}
5939	}
5940
5941	self.wip.cursor = .{ .block = loop_exit };
5942	return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
5943	}
5944
5945	fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
5946	const o = self.object;
5947	const pt = self.pt;
5948	const zcu = pt.zcu;
5949	const target = zcu.getTarget();
5950
5951	const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
5952	const operand = try self.resolveInst(reduce.operand);
5953	const operand_ty = self.typeOf(reduce.operand);
5954	const llvm_operand_ty = try o.lowerType(pt, operand_ty);
5955	const scalar_ty = self.typeOfIndex(inst);
5956	const llvm_scalar_ty = try o.lowerType(pt, scalar_ty);
5957
5958	switch (reduce.operation) {
5959	.And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
5960	.And => .@"vector.reduce.and",
5961	.Or => .@"vector.reduce.or",
5962	.Xor => .@"vector.reduce.xor",
5963	else => unreachable,
5964	}, &.{llvm_operand_ty}, &.{operand}, ""),
5965	.Min, .Max => switch (scalar_ty.zigTypeTag(zcu)) {
5966	.int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
5967	.Min => if (scalar_ty.isSignedInt(zcu))
5968	.@"vector.reduce.smin"
5969	else
5970	.@"vector.reduce.umin",
5971	.Max => if (scalar_ty.isSignedInt(zcu))
5972	.@"vector.reduce.smax"
5973	else
5974	.@"vector.reduce.umax",
5975	else => unreachable,
5976	}, &.{llvm_operand_ty}, &.{operand}, ""),
5977	.float => if (intrinsicsAllowed(scalar_ty, target))
5978	return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
5979	.Min => .@"vector.reduce.fmin",
5980	.Max => .@"vector.reduce.fmax",
5981	else => unreachable,
5982	}, &.{llvm_operand_ty}, &.{operand}, ""),
5983	else => unreachable,
5984	},
5985	.Add, .Mul => switch (scalar_ty.zigTypeTag(zcu)) {
5986	.int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
5987	.Add => .@"vector.reduce.add",
5988	.Mul => .@"vector.reduce.mul",
5989	else => unreachable,
5990	}, &.{llvm_operand_ty}, &.{operand}, ""),
5991	.float => if (intrinsicsAllowed(scalar_ty, target))
5992	return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
5993	.Add => .@"vector.reduce.fadd",
5994	.Mul => .@"vector.reduce.fmul",
5995	else => unreachable,
5996	}, &.{llvm_operand_ty}, &.{ switch (reduce.operation) {
5997	.Add => try o.builder.fpValue(llvm_scalar_ty, -0.0),
5998	.Mul => try o.builder.fpValue(llvm_scalar_ty, 1.0),
5999	else => unreachable,
6000	}, operand }, ""),
6001	else => unreachable,
6002	},
6003	}
6004
6005	// Reduction could not be performed with intrinsics.
6006	// Use a manual loop over a softfloat call instead.
6007	const float_bits = scalar_ty.floatBits(target);
6008	const fn_name = switch (reduce.operation) {
6009	.Min => try o.builder.strtabStringFmt("{s}fmin{s}", .{
6010	libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
6011	}),
6012	.Max => try o.builder.strtabStringFmt("{s}fmax{s}", .{
6013	libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
6014	}),
6015	.Add => try o.builder.strtabStringFmt("__add{s}f3", .{
6016	compilerRtFloatAbbrev(float_bits),
6017	}),
6018	.Mul => try o.builder.strtabStringFmt("__mul{s}f3", .{
6019	compilerRtFloatAbbrev(float_bits),
6020	}),
6021	else => unreachable,
6022	};
6023
6024	const libc_fn =
6025	try self.getLibcFunction(fn_name, &.{ llvm_scalar_ty, llvm_scalar_ty }, llvm_scalar_ty);
6026	const init_val = switch (llvm_scalar_ty) {
6027	.i16 => try o.builder.intValue(.i16, @as(i16, @bitCast(
6028	@as(f16, switch (reduce.operation) {
6029	.Min, .Max => std.math.nan(f16),
6030	.Add => -0.0,
6031	.Mul => 1.0,
6032	else => unreachable,
6033	}),
6034	))),
6035	.i80 => try o.builder.intValue(.i80, @as(i80, @bitCast(
6036	@as(f80, switch (reduce.operation) {
6037	.Min, .Max => std.math.nan(f80),
6038	.Add => -0.0,
6039	.Mul => 1.0,
6040	else => unreachable,
6041	}),
6042	))),
6043	.i128 => try o.builder.intValue(.i128, @as(i128, @bitCast(
6044	@as(f128, switch (reduce.operation) {
6045	.Min, .Max => std.math.nan(f128),
6046	.Add => -0.0,
6047	.Mul => 1.0,
6048	else => unreachable,
6049	}),
6050	))),
6051	else => unreachable,
6052	};
6053	return self.buildReducedCall(libc_fn, operand, operand_ty.vectorLen(zcu), init_val);
6054	}
6055
6056	fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6057	const o = self.object;
6058	const pt = self.pt;
6059	const zcu = pt.zcu;
6060	const ip = &zcu.intern_pool;
6061	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6062	const result_ty = self.typeOfIndex(inst);
6063	const len: usize = @intCast(result_ty.arrayLen(zcu));
6064	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
6065	const llvm_result_ty = try o.lowerType(pt, result_ty);
6066
6067	switch (result_ty.zigTypeTag(zcu)) {
6068	.vector => {
6069	var vector = try o.builder.poisonValue(llvm_result_ty);
6070	for (elements, 0..) \|elem, i\| {
6071	const index_u32 = try o.builder.intValue(.i32, i);
6072	const llvm_elem = try self.resolveInst(elem);
6073	vector = try self.wip.insertElement(vector, llvm_elem, index_u32, "");
6074	}
6075	return vector;
6076	},
6077	.@"struct" => {
6078	if (zcu.typeToPackedStruct(result_ty)) \|struct_type\| {
6079	const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type);
6080	const big_bits = backing_int_ty.bitSize(zcu);
6081	const int_ty = try o.builder.intType(@intCast(big_bits));
6082	comptime assert(Type.packed_struct_layout_version == 2);
6083	var running_int = try o.builder.intValue(int_ty, 0);
6084	var running_bits: u16 = 0;
6085	for (elements, struct_type.field_types.get(ip)) \|elem, field_ty\| {
6086	if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
6087
6088	const non_int_val = try self.resolveInst(elem);
6089	const ty_bit_size: u16 = @intCast(Type.fromInterned(field_ty).bitSize(zcu));
6090	const small_int_ty = try o.builder.intType(ty_bit_size);
6091	const small_int_val = if (Type.fromInterned(field_ty).isPtrAtRuntime(zcu))
6092	try self.wip.cast(.ptrtoint, non_int_val, small_int_ty, "")
6093	else
6094	try self.wip.cast(.bitcast, non_int_val, small_int_ty, "");
6095	const shift_rhs = try o.builder.intValue(int_ty, running_bits);
6096	const extended_int_val =
6097	try self.wip.conv(.unsigned, small_int_val, int_ty, "");
6098	const shifted = try self.wip.bin(.shl, extended_int_val, shift_rhs, "");
6099	running_int = try self.wip.bin(.@"or", running_int, shifted, "");
6100	running_bits += ty_bit_size;
6101	}
6102	return running_int;
6103	}
6104
6105	assert(result_ty.containerLayout(zcu) != .@"packed");
6106
6107	if (isByRef(result_ty, zcu)) {
6108	// TODO in debug builds init to undef so that the padding will be 0xaa
6109	// even if we fully populate the fields.
6110	const alignment = result_ty.abiAlignment(zcu).toLlvm();
6111	const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
6112
6113	for (elements, 0..) \|elem, i\| {
6114	if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
6115
6116	const llvm_elem = try self.resolveInst(elem);
6117	const llvm_i = o.llvmFieldIndex(result_ty, i).?;
6118	const field_ptr = try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, "");
6119
6120	const field_ptr_ty = try pt.ptrType(.{
6121	.child = self.typeOf(elem).toIntern(),
6122	.flags = .{
6123	.alignment = result_ty.explicitFieldAlignment(i, zcu),
6124	},
6125	});
6126	try self.store(field_ptr, field_ptr_ty, llvm_elem, .none);
6127	}
6128
6129	return alloca_inst;
6130	} else {
6131	var result = try o.builder.poisonValue(llvm_result_ty);
6132	for (elements, 0..) \|elem, i\| {
6133	if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
6134
6135	const llvm_elem = try self.resolveInst(elem);
6136	const llvm_i = o.llvmFieldIndex(result_ty, i).?;
6137	result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, "");
6138	}
6139	return result;
6140	}
6141	},
6142	.array => {
6143	assert(isByRef(result_ty, zcu));
6144
6145	const llvm_usize = try o.lowerType(pt, Type.usize);
6146	const usize_zero = try o.builder.intValue(llvm_usize, 0);
6147	const alignment = result_ty.abiAlignment(zcu).toLlvm();
6148	const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
6149
6150	const array_info = result_ty.arrayInfo(zcu);
6151	const elem_ptr_ty = try pt.ptrType(.{
6152	.child = array_info.elem_type.toIntern(),
6153	});
6154
6155	for (elements, 0..) \|elem, i\| {
6156	const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{
6157	usize_zero, try o.builder.intValue(llvm_usize, i),
6158	}, "");
6159	const llvm_elem = try self.resolveInst(elem);
6160	try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none);
6161	}
6162	if (array_info.sentinel) \|sent_val\| {
6163	const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{
6164	usize_zero, try o.builder.intValue(llvm_usize, array_info.len),
6165	}, "");
6166	const llvm_elem = try self.resolveValue(sent_val);
6167	try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none);
6168	}
6169
6170	return alloca_inst;
6171	},
6172	else => unreachable,
6173	}
6174	}
6175
6176	fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6177	const o = self.object;
6178	const pt = self.pt;
6179	const zcu = pt.zcu;
6180	const ip = &zcu.intern_pool;
6181	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6182	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
6183	const union_ty = self.typeOfIndex(inst);
6184	const union_llvm_ty = try o.lowerType(pt, union_ty);
6185	const union_obj = zcu.typeToUnion(union_ty).?;
6186
6187	assert(union_obj.layout != .@"packed");
6188
6189	const layout = Type.getUnionLayout(union_obj, zcu);
6190
6191	const tag_int_val = blk: {
6192	const tag_ty = union_ty.unionTagTypeHypothetical(zcu);
6193	const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index);
6194	break :blk tag_val.intFromEnum(zcu);
6195	};
6196	if (layout.payload_size == 0) {
6197	if (layout.tag_size == 0) {
6198	return .none;
6199	}
6200	assert(!isByRef(union_ty, zcu));
6201	var big_int_space: Value.BigIntSpace = undefined;
6202	const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
6203	return try o.builder.bigIntValue(union_llvm_ty, tag_big_int);
6204	}
6205	assert(isByRef(union_ty, zcu));
6206	// The llvm type of the alloca will be the named LLVM union type, and will not
6207	// necessarily match the format that we need, depending on which tag is active.
6208	// We must construct the correct unnamed struct type here, in order to then set
6209	// the fields appropriately.
6210	const alignment = layout.abi_align.toLlvm();
6211	const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
6212	const llvm_payload = try self.resolveInst(extra.init);
6213	const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6214	const field_llvm_ty = try o.lowerType(pt, field_ty);
6215	const field_size = field_ty.abiSize(zcu);
6216	const field_align = union_ty.explicitFieldAlignment(extra.field_index, zcu);
6217	const llvm_usize = try o.lowerType(pt, Type.usize);
6218	const usize_zero = try o.builder.intValue(llvm_usize, 0);
6219
6220	assert(field_ty.hasRuntimeBits(zcu));
6221
6222	const llvm_union_ty = t: {
6223	const payload_ty = p: {
6224	if (field_size == layout.payload_size) {
6225	break :p field_llvm_ty;
6226	}
6227	const padding_len = layout.payload_size - field_size;
6228	break :p try o.builder.structType(.@"packed", &.{
6229	field_llvm_ty, try o.builder.arrayType(padding_len, .i8),
6230	});
6231	};
6232	if (layout.tag_size == 0) break :t try o.builder.structType(.normal, &.{payload_ty});
6233	const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
6234	var fields: [3]Builder.Type = undefined;
6235	var fields_len: usize = 2;
6236	if (layout.tag_align.compare(.gte, layout.payload_align)) {
6237	fields = .{ tag_ty, payload_ty, undefined };
6238	} else {
6239	fields = .{ payload_ty, tag_ty, undefined };
6240	}
6241	if (layout.padding != 0) {
6242	fields[fields_len] = try o.builder.arrayType(layout.padding, .i8);
6243	fields_len += 1;
6244	}
6245	break :t try o.builder.structType(.normal, fields[0..fields_len]);
6246	};
6247
6248	// Now we follow the layout as expressed above with GEP instructions to set the
6249	// tag and the payload.
6250	const field_ptr_ty = try pt.ptrType(.{
6251	.child = field_ty.toIntern(),
6252	.flags = .{ .alignment = field_align },
6253	});
6254	if (layout.tag_size == 0) {
6255	const indices = [3]Builder.Value{ usize_zero, .@"0", .@"0" };
6256	const len: usize = if (field_size == layout.payload_size) 2 else 3;
6257	const field_ptr =
6258	try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
6259	try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
6260	return result_ptr;
6261	}
6262
6263	{
6264	const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align));
6265	const indices: [3]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, payload_index), .@"0" };
6266	const len: usize = if (field_size == layout.payload_size) 2 else 3;
6267	const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
6268	try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
6269	}
6270	{
6271	const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
6272	const indices: [2]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, tag_index) };
6273	const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, &indices, "");
6274	const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
6275	var big_int_space: Value.BigIntSpace = undefined;
6276	const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
6277	const llvm_tag = try o.builder.bigIntValue(tag_ty, tag_big_int);
6278	const tag_alignment = Type.fromInterned(union_obj.enum_tag_type).abiAlignment(zcu).toLlvm();
6279	_ = try self.wip.store(.normal, llvm_tag, field_ptr, tag_alignment);
6280	}
6281
6282	return result_ptr;
6283	}
6284
6285	fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6286	const o = self.object;
6287	const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
6288
6289	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.read) == 0);
6290	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.write) == 1);
6291
6292	comptime assert(prefetch.locality >= 0);
6293	comptime assert(prefetch.locality <= 3);
6294
6295	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.instruction) == 0);
6296	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.data) == 1);
6297
6298	// LLVM fails during codegen of instruction cache prefetchs for these architectures.
6299	// This is an LLVM bug as the prefetch intrinsic should be a noop if not supported
6300	// by the target.
6301	// To work around this, don't emit llvm.prefetch in this case.
6302	// See https://bugs.llvm.org/show_bug.cgi?id=21037
6303	const zcu = self.pt.zcu;
6304	const target = zcu.getTarget();
6305	switch (prefetch.cache) {
6306	.instruction => switch (target.cpu.arch) {
6307	.x86_64,
6308	.x86,
6309	.powerpc,
6310	.powerpcle,
6311	.powerpc64,
6312	.powerpc64le,
6313	=> return .none,
6314	.arm, .armeb, .thumb, .thumbeb => {
6315	switch (prefetch.rw) {
6316	.write => return .none,
6317	else => {},
6318	}
6319	},
6320	else => {},
6321	},
6322	.data => {},
6323	}
6324
6325	_ = try self.wip.callIntrinsic(.normal, .none, .prefetch, &.{.ptr}, &.{
6326	try self.sliceOrArrayPtr(try self.resolveInst(prefetch.ptr), self.typeOf(prefetch.ptr)),
6327	try o.builder.intValue(.i32, prefetch.rw),
6328	try o.builder.intValue(.i32, prefetch.locality),
6329	try o.builder.intValue(.i32, prefetch.cache),
6330	}, "");
6331	return .none;
6332	}
6333
6334	fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6335	const o = self.object;
6336	const pt = self.pt;
6337	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6338	const inst_ty = self.typeOfIndex(inst);
6339	const operand = try self.resolveInst(ty_op.operand);
6340
6341	return self.wip.cast(.addrspacecast, operand, try o.lowerType(pt, inst_ty), "");
6342	}
6343
6344	fn workIntrinsic(
6345	self: *FuncGen,
6346	dimension: u32,
6347	default: u32,
6348	comptime basename: []const u8,
6349	) !Builder.Value {
6350	return self.wip.callIntrinsic(.normal, .none, switch (dimension) {
6351	0 => @field(Builder.Intrinsic, basename ++ ".x"),
6352	1 => @field(Builder.Intrinsic, basename ++ ".y"),
6353	2 => @field(Builder.Intrinsic, basename ++ ".z"),
6354	else => return self.object.builder.intValue(.i32, default),
6355	}, &.{}, &.{}, "");
6356	}
6357
6358	fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6359	const target = self.pt.zcu.getTarget();
6360
6361	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6362	const dimension = pl_op.payload;
6363
6364	return switch (target.cpu.arch) {
6365	.amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workitem.id"),
6366	.nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.tid"),
6367	else => unreachable,
6368	};
6369	}
6370
6371	fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6372	const o = self.object;
6373	const pt = self.pt;
6374	const target = pt.zcu.getTarget();
6375
6376	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6377	const dimension = pl_op.payload;
6378
6379	switch (target.cpu.arch) {
6380	.amdgcn => {
6381	if (dimension >= 3) return .@"1";
6382
6383	// Fetch the dispatch pointer, which points to this structure:
6384	// https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/adae6c61e10d371f7cbc3d0e94ae2c070cab18a4/src/inc/hsa.h#L2913
6385	const dispatch_ptr =
6386	try self.wip.callIntrinsic(.normal, .none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, "");
6387
6388	// Load the work_group_* member from the struct as u16.
6389	// Just treat the dispatch pointer as an array of u16 to keep things simple.
6390	const workgroup_size_ptr = try self.wip.gep(.inbounds, .i16, dispatch_ptr, &.{
6391	try o.builder.intValue(try o.lowerType(pt, Type.usize), 2 + dimension),
6392	}, "");
6393	const workgroup_size_alignment = comptime Builder.Alignment.fromByteUnits(2);
6394	return self.wip.load(.normal, .i16, workgroup_size_ptr, workgroup_size_alignment, "");
6395	},
6396	.nvptx, .nvptx64 => {
6397	return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
6398	},
6399	else => unreachable,
6400	}
6401	}
6402
6403	fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6404	const target = self.pt.zcu.getTarget();
6405
6406	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
6407	const dimension = pl_op.payload;
6408
6409	return switch (target.cpu.arch) {
6410	.amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workgroup.id"),
6411	.nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.ctaid"),
6412	else => unreachable,
6413	};
6414	}
6415
6416	fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index {
6417	const o = self.object;
6418	const pt = self.pt;
6419
6420	const table = o.error_name_table;
6421	if (table != .none) return table;
6422
6423	// TODO: Address space
6424	const variable_index =
6425	try o.builder.addVariable(try o.builder.strtabString("__zig_err_name_table"), .ptr, .default);
6426	variable_index.setLinkage(.private, &o.builder);
6427	variable_index.setMutability(.constant, &o.builder);
6428	variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
6429	variable_index.setAlignment(
6430	Type.slice_const_u8_sentinel_0.abiAlignment(pt.zcu).toLlvm(),
6431	&o.builder,
6432	);
6433
6434	o.error_name_table = variable_index;
6435	return variable_index;
6436	}
6437
6438	/// Assumes the optional is not pointer-like and payload has bits.
6439	fn optCmpNull(
6440	self: *FuncGen,
6441	cond: Builder.IntegerCondition,
6442	opt_llvm_ty: Builder.Type,
6443	opt_handle: Builder.Value,
6444	is_by_ref: bool,
6445	access_kind: Builder.MemoryAccessKind,
6446	) Allocator.Error!Builder.Value {
6447	const o = self.object;
6448	const field = b: {
6449	if (is_by_ref) {
6450	const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, "");
6451	break :b try self.wip.load(access_kind, .i8, field_ptr, .default, "");
6452	}
6453	break :b try self.wip.extractValue(opt_handle, &.{1}, "");
6454	};
6455	comptime assert(optional_layout_version == 3);
6456
6457	return self.wip.icmp(cond, field, try o.builder.intValue(.i8, 0), "");
6458	}
6459
6460	/// Assumes the optional is not pointer-like and payload has bits.
6461	fn optPayloadHandle(
6462	fg: *FuncGen,
6463	opt_llvm_ty: Builder.Type,
6464	opt_handle: Builder.Value,
6465	opt_ty: Type,
6466	can_elide_load: bool,
6467	) !Builder.Value {
6468	const pt = fg.pt;
6469	const zcu = pt.zcu;
6470	const payload_ty = opt_ty.optionalChild(zcu);
6471
6472	if (isByRef(opt_ty, zcu)) {
6473	// We have a pointer and we need to return a pointer to the first field.
6474	const payload_ptr = try fg.wip.gepStruct(opt_llvm_ty, opt_handle, 0, "");
6475
6476	const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
6477	if (isByRef(payload_ty, zcu)) {
6478	if (can_elide_load)
6479	return payload_ptr;
6480
6481	return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
6482	}
6483	return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_alignment);
6484	}
6485
6486	assert(!isByRef(payload_ty, zcu));
6487	return fg.wip.extractValue(opt_handle, &.{0}, "");
6488	}
6489
6490	fn buildOptional(
6491	self: *FuncGen,
6492	optional_ty: Type,
6493	payload: Builder.Value,
6494	non_null_bit: Builder.Value,
6495	) !Builder.Value {
6496	const o = self.object;
6497	const pt = self.pt;
6498	const zcu = pt.zcu;
6499	const optional_llvm_ty = try o.lowerType(pt, optional_ty);
6500	const non_null_field = try self.wip.cast(.zext, non_null_bit, .i8, "");
6501
6502	if (isByRef(optional_ty, zcu)) {
6503	const payload_alignment = optional_ty.abiAlignment(pt.zcu).toLlvm();
6504	const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment);
6505
6506	{
6507	const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, "");
6508	_ = try self.wip.store(.normal, payload, field_ptr, payload_alignment);
6509	}
6510	{
6511	const non_null_alignment = comptime Builder.Alignment.fromByteUnits(1);
6512	const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 1, "");
6513	_ = try self.wip.store(.normal, non_null_field, field_ptr, non_null_alignment);
6514	}
6515
6516	return alloca_inst;
6517	}
6518
6519	return self.wip.buildAggregate(optional_llvm_ty, &.{ payload, non_null_field }, "");
6520	}
6521
6522	fn fieldPtr(
6523	self: *FuncGen,
6524	aggregate_ptr: Builder.Value,
6525	aggregate_ptr_ty: Type,
6526	field_index: u32,
6527	) !Builder.Value {
6528	const o = self.object;
6529	const pt = self.pt;
6530	const zcu = pt.zcu;
6531	const aggregate_ty = aggregate_ptr_ty.childType(zcu);
6532	if (aggregate_ty.containerLayout(zcu) == .@"packed") {
6533	// A pointer to a bitpack field is equivalent to a pointer to the whole bitpack; the
6534	// bit offset is represented in the pointer type.
6535	return aggregate_ptr;
6536	}
6537	switch (aggregate_ty.zigTypeTag(zcu)) {
6538	.@"struct" => {
6539	if (!aggregate_ty.hasRuntimeBits(zcu)) {
6540	return aggregate_ptr;
6541	}
6542	const struct_llvm_ty = try o.lowerType(pt, aggregate_ty);
6543	if (o.llvmFieldIndex(aggregate_ty, field_index)) \|llvm_field_index\| {
6544	return self.wip.gepStruct(struct_llvm_ty, aggregate_ptr, llvm_field_index, "");
6545	} else {
6546	// If we found no index then this means this is a zero sized field at the
6547	// end of the struct. Treat our struct pointer as an array of two and get
6548	// the index to the element at index `1` to get a pointer to the end of
6549	// the struct.
6550	const llvm_index = try o.builder.intValue(
6551	try o.lowerType(pt, Type.usize),
6552	@intFromBool(aggregate_ty.hasRuntimeBits(zcu)),
6553	);
6554	return self.wip.gep(.inbounds, struct_llvm_ty, aggregate_ptr, &.{llvm_index}, "");
6555	}
6556	},
6557	.@"union" => {
6558	const layout = aggregate_ty.unionGetLayout(zcu);
6559	if (layout.payload_size == 0) return aggregate_ptr;
6560	const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align));
6561	const union_llvm_ty = try o.lowerType(pt, aggregate_ty);
6562	return self.wip.gepStruct(union_llvm_ty, aggregate_ptr, payload_index, "");
6563	},
6564	else => unreachable,
6565	}
6566	}
6567
6568	/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.
6569	fn loadTruncate(
6570	fg: *FuncGen,
6571	access_kind: Builder.MemoryAccessKind,
6572	payload_ty: Type,
6573	payload_ptr: Builder.Value,
6574	payload_alignment: Builder.Alignment,
6575	) !Builder.Value {
6576	// from https://llvm.org/docs/LangRef.html#load-instruction :
6577	// "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. "
6578	// => so load the byte aligned value and trunc the unwanted bits.
6579
6580	const o = fg.object;
6581	const pt = fg.pt;
6582	const zcu = pt.zcu;
6583	const payload_llvm_ty = try o.lowerType(pt, payload_ty);
6584	const abi_size = payload_ty.abiSize(zcu);
6585
6586	const load_llvm_ty = if (payload_ty.isAbiInt(zcu))
6587	try o.builder.intType(@intCast(abi_size * 8))
6588	else
6589	payload_llvm_ty;
6590	const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, "");
6591	const shifted = if (payload_llvm_ty != load_llvm_ty and o.target.cpu.arch.endian() == .big)
6592	try fg.wip.bin(.lshr, loaded, try o.builder.intValue(
6593	load_llvm_ty,
6594	(payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8,
6595	), "")
6596	else
6597	loaded;
6598
6599	return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, "");
6600	}
6601
6602	/// Load a by-ref type by constructing a new alloca and performing a memcpy.
6603	fn loadByRef(
6604	fg: *FuncGen,
6605	ptr: Builder.Value,
6606	pointee_type: Type,
6607	ptr_alignment: Builder.Alignment,
6608	access_kind: Builder.MemoryAccessKind,
6609	) !Builder.Value {
6610	const o = fg.object;
6611	const pt = fg.pt;
6612	const pointee_llvm_ty = try o.lowerType(pt, pointee_type);
6613	const result_align = InternPool.Alignment.fromLlvm(ptr_alignment)
6614	.max(pointee_type.abiAlignment(pt.zcu)).toLlvm();
6615	const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);
6616	const size_bytes = pointee_type.abiSize(pt.zcu);
6617	_ = try fg.wip.callMemCpy(
6618	result_ptr,
6619	result_align,
6620	ptr,
6621	ptr_alignment,
6622	try o.builder.intValue(try o.lowerType(pt, Type.usize), size_bytes),
6623	access_kind,
6624	fg.disable_intrinsics,
6625	);
6626	return result_ptr;
6627	}
6628
6629	/// This function always performs a copy. For isByRef=true types, it creates a new
6630	/// alloca and copies the value into it, then returns the alloca instruction.
6631	/// For isByRef=false types, it creates a load instruction and returns it.
6632	fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) !Builder.Value {
6633	const o = self.object;
6634	const pt = self.pt;
6635	const zcu = pt.zcu;
6636	const info = ptr_ty.ptrInfo(zcu);
6637	const elem_ty = Type.fromInterned(info.child);
6638	if (!elem_ty.hasRuntimeBits(zcu)) return .none;
6639
6640	const ptr_alignment = (if (info.flags.alignment != .none)
6641	@as(InternPool.Alignment, info.flags.alignment)
6642	else
6643	elem_ty.abiAlignment(zcu)).toLlvm();
6644
6645	const access_kind: Builder.MemoryAccessKind =
6646	if (info.flags.is_volatile) .@"volatile" else .normal;
6647
6648	if (info.flags.vector_index != .none) {
6649	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
6650	const vec_elem_ty = try o.lowerType(pt, elem_ty);
6651	const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
6652
6653	const loaded_vector = try self.wip.load(access_kind, vec_ty, ptr, ptr_alignment, "");
6654	return self.wip.extractElement(loaded_vector, index_u32, "");
6655	}
6656
6657	if (info.packed_offset.host_size == 0) {
6658	if (isByRef(elem_ty, zcu)) {
6659	return self.loadByRef(ptr, elem_ty, ptr_alignment, access_kind);
6660	}
6661	return self.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment);
6662	}
6663
6664	const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
6665	const containing_int =
6666	try self.wip.load(access_kind, containing_int_ty, ptr, ptr_alignment, "");
6667
6668	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
6669	const shift_amt = try o.builder.intValue(containing_int_ty, info.packed_offset.bit_offset);
6670	const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, "");
6671	const elem_llvm_ty = try o.lowerType(pt, elem_ty);
6672
6673	if (isByRef(elem_ty, zcu)) {
6674	const result_align = elem_ty.abiAlignment(zcu).toLlvm();
6675	const result_ptr = try self.buildAlloca(elem_llvm_ty, result_align);
6676
6677	const same_size_int = try o.builder.intType(@intCast(elem_bits));
6678	const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
6679	_ = try self.wip.store(.normal, truncated_int, result_ptr, result_align);
6680	return result_ptr;
6681	}
6682
6683	if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
6684	const same_size_int = try o.builder.intType(@intCast(elem_bits));
6685	const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
6686	return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
6687	}
6688
6689	if (elem_ty.isPtrAtRuntime(zcu)) {
6690	const same_size_int = try o.builder.intType(@intCast(elem_bits));
6691	const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
6692	return self.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, "");
6693	}
6694
6695	return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
6696	}
6697
6698	fn store(
6699	self: *FuncGen,
6700	ptr: Builder.Value,
6701	ptr_ty: Type,
6702	elem: Builder.Value,
6703	ordering: Builder.AtomicOrdering,
6704	) !void {
6705	const o = self.object;
6706	const pt = self.pt;
6707	const zcu = pt.zcu;
6708	const info = ptr_ty.ptrInfo(zcu);
6709	const elem_ty = Type.fromInterned(info.child);
6710	if (!elem_ty.hasRuntimeBits(zcu)) {
6711	return;
6712	}
6713	const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
6714	const access_kind: Builder.MemoryAccessKind =
6715	if (info.flags.is_volatile) .@"volatile" else .normal;
6716
6717	if (info.flags.vector_index != .none) {
6718	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
6719	const vec_elem_ty = try o.lowerType(pt, elem_ty);
6720	const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
6721
6722	const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, "");
6723
6724	const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, "");
6725
6726	assert(ordering == .none);
6727	_ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment);
6728	return;
6729	}
6730
6731	if (info.packed_offset.host_size != 0) {
6732	const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
6733	assert(ordering == .none);
6734	const containing_int =
6735	try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, "");
6736	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
6737	const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset);
6738	// Convert to equally-sized integer type in order to perform the bit
6739	// operations on the value to store
6740	const value_bits_type = try o.builder.intType(@intCast(elem_bits));
6741	const value_bits = if (elem_ty.isPtrAtRuntime(zcu))
6742	try self.wip.cast(.ptrtoint, elem, value_bits_type, "")
6743	else
6744	try self.wip.cast(.bitcast, elem, value_bits_type, "");
6745
6746	const mask_val = blk: {
6747	const zext = try self.wip.cast(
6748	.zext,
6749	try o.builder.intValue(value_bits_type, -1),
6750	containing_int_ty,
6751	"",
6752	);
6753	const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), "");
6754	break :blk try self.wip.bin(
6755	.xor,
6756	shl,
6757	try o.builder.intValue(containing_int_ty, -1),
6758	"",
6759	);
6760	};
6761
6762	const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, "");
6763	const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, "");
6764	const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), "");
6765	const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, "");
6766
6767	assert(ordering == .none);
6768	_ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment);
6769	return;
6770	}
6771	if (!isByRef(elem_ty, zcu)) {
6772	_ = try self.wip.storeAtomic(
6773	access_kind,
6774	elem,
6775	ptr,
6776	self.sync_scope,
6777	ordering,
6778	ptr_alignment,
6779	);
6780	return;
6781	}
6782	assert(ordering == .none);
6783	_ = try self.wip.callMemCpy(
6784	ptr,
6785	ptr_alignment,
6786	elem,
6787	elem_ty.abiAlignment(zcu).toLlvm(),
6788	try o.builder.intValue(try o.lowerType(pt, Type.usize), elem_ty.abiSize(zcu)),
6789	access_kind,
6790	self.disable_intrinsics,
6791	);
6792	}
6793
6794	fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
6795	const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
6796	const o = fg.object;
6797	const pt = fg.pt;
6798	const usize_ty = try o.lowerType(pt, Type.usize);
6799	const zero = try o.builder.intValue(usize_ty, 0);
6800	const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);
6801	const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");
6802	_ = try valgrindClientRequest(fg, zero, req, ptr_as_usize, len, zero, zero, zero);
6803	}
6804
6805	fn valgrindClientRequest(
6806	fg: *FuncGen,
6807	default_value: Builder.Value,
6808	request: Builder.Value,
6809	a1: Builder.Value,
6810	a2: Builder.Value,
6811	a3: Builder.Value,
6812	a4: Builder.Value,
6813	a5: Builder.Value,
6814	) Allocator.Error!Builder.Value {
6815	const o = fg.object;
6816	const pt = fg.pt;
6817	const zcu = pt.zcu;
6818	const target = zcu.getTarget();
6819	if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
6820
6821	const llvm_usize = try o.lowerType(pt, Type.usize);
6822	const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm();
6823
6824	const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
6825	const array_ptr = if (fg.valgrind_client_request_array == .none) a: {
6826	const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment);
6827	fg.valgrind_client_request_array = array_ptr;
6828	break :a array_ptr;
6829	} else fg.valgrind_client_request_array;
6830	const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
6831	const zero = try o.builder.intValue(llvm_usize, 0);
6832	for (array_elements, 0..) \|elem, i\| {
6833	const elem_ptr = try fg.wip.gep(.inbounds, array_llvm_ty, array_ptr, &.{
6834	zero, try o.builder.intValue(llvm_usize, i),
6835	}, "");
6836	_ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);
6837	}
6838
6839	const arch_specific: struct {
6840	template: [:0]const u8,
6841	constraints: [:0]const u8,
6842	} = switch (target.cpu.arch) {
6843	.arm, .armeb, .thumb, .thumbeb => .{
6844	.template =
6845	\\ mov r12, r12, ror #3 ; mov r12, r12, ror #13
6846	\\ mov r12, r12, ror #29 ; mov r12, r12, ror #19
6847	\\ orr r10, r10, r10
6848	,
6849	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
6850	},
6851	.aarch64, .aarch64_be => .{
6852	.template =
6853	\\ ror x12, x12, #3 ; ror x12, x12, #13
6854	\\ ror x12, x12, #51 ; ror x12, x12, #61
6855	\\ orr x10, x10, x10
6856	,
6857	.constraints = "={x3},{x4},{x3},~{cc},~{memory}",
6858	},
6859	.mips, .mipsel => .{
6860	.template =
6861	\\ srl $$0, $$0, 13
6862	\\ srl $$0, $$0, 29
6863	\\ srl $$0, $$0, 3
6864	\\ srl $$0, $$0, 19
6865	\\ or $$13, $$13, $$13
6866	,
6867	.constraints = "={$11},{$12},{$11},~{memory},~{$1}",
6868	},
6869	.mips64, .mips64el => .{
6870	.template =
6871	\\ dsll $$0, $$0, 3 ; dsll $$0, $$0, 13
6872	\\ dsll $$0, $$0, 29 ; dsll $$0, $$0, 19
6873	\\ or $$13, $$13, $$13
6874	,
6875	.constraints = "={$11},{$12},{$11},~{memory},~{$1}",
6876	},
6877	.powerpc, .powerpcle => .{
6878	.template =
6879	\\ rlwinm 0, 0, 3, 0, 31 ; rlwinm 0, 0, 13, 0, 31
6880	\\ rlwinm 0, 0, 29, 0, 31 ; rlwinm 0, 0, 19, 0, 31
6881	\\ or 1, 1, 1
6882	,
6883	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
6884	},
6885	.powerpc64, .powerpc64le => .{
6886	.template =
6887	\\ rotldi 0, 0, 3 ; rotldi 0, 0, 13
6888	\\ rotldi 0, 0, 61 ; rotldi 0, 0, 51
6889	\\ or 1, 1, 1
6890	,
6891	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
6892	},
6893	.riscv64 => .{
6894	.template =
6895	\\ .option push
6896	\\ .option norvc
6897	\\ srli zero, zero, 3
6898	\\ srli zero, zero, 13
6899	\\ srli zero, zero, 51
6900	\\ srli zero, zero, 61
6901	\\ or a0, a0, a0
6902	\\ .option pop
6903	,
6904	.constraints = "={a3},{a4},{a3},~{cc},~{memory}",
6905	},
6906	.s390x => .{
6907	.template =
6908	\\ lr %r15, %r15
6909	\\ lr %r1, %r1
6910	\\ lr %r2, %r2
6911	\\ lr %r3, %r3
6912	\\ lr %r2, %r2
6913	,
6914	.constraints = "={r3},{r2},{r3},~{cc},~{memory}",
6915	},
6916	.x86 => .{
6917	.template =
6918	\\ roll $$3, %edi ; roll $$13, %edi
6919	\\ roll $$61, %edi ; roll $$51, %edi
6920	\\ xchgl %ebx, %ebx
6921	,
6922	.constraints = "={edx},{eax},{edx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
6923	},
6924	.x86_64 => .{
6925	.template =
6926	\\ rolq $$3, %rdi ; rolq $$13, %rdi
6927	\\ rolq $$61, %rdi ; rolq $$51, %rdi
6928	\\ xchgq %rbx, %rbx
6929	,
6930	.constraints = "={rdx},{rax},{rdx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
6931	},
6932	else => unreachable,
6933	};
6934
6935	return fg.wip.callAsm(
6936	.none,
6937	try o.builder.fnType(llvm_usize, &.{ llvm_usize, llvm_usize }, .normal),
6938	.{ .sideeffect = true },
6939	try o.builder.string(arch_specific.template),
6940	try o.builder.string(arch_specific.constraints),
6941	&.{ try fg.wip.cast(.ptrtoint, array_ptr, llvm_usize, ""), default_value },
6942	"",
6943	);
6944	}
6945
6946	fn typeOf(fg: *FuncGen, inst: Air.Inst.Ref) Type {
6947	const zcu = fg.pt.zcu;
6948	return fg.air.typeOf(inst, &zcu.intern_pool);
6949	}
6950
6951	fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type {
6952	const zcu = fg.pt.zcu;
6953	return fg.air.typeOfIndex(inst, &zcu.intern_pool);
6954	}
6955
6956	const ParamTypeIterator = struct {
6957	object: *Object,
6958	pt: Zcu.PerThread,
6959	fn_info: InternPool.Key.FuncType,
6960	zig_index: u32,
6961	llvm_index: u32,
6962	types_len: u32,
6963	types_buffer: [8]Builder.Type,
6964	byval_attr: bool,
6965
6966	const Lowering = union(enum) {
6967	no_bits,
6968	byval,
6969	byref,
6970	byref_mut,
6971	abi_sized_int,
6972	multiple_llvm_types,
6973	slice,
6974	float_array: u8,
6975	i32_array: u8,
6976	i64_array: u8,
6977	};
6978
6979	pub fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering {
6980	if (it.zig_index >= it.fn_info.param_types.len) return null;
6981	const ip = &it.pt.zcu.intern_pool;
6982	const ty = it.fn_info.param_types.get(ip)[it.zig_index];
6983	it.byval_attr = false;
6984	return nextInner(it, Type.fromInterned(ty));
6985	}
6986
6987	/// `airCall` uses this instead of `next` so that it can take into account variadic functions.
6988	fn nextCall(it: ParamTypeIterator, fg: FuncGen, args: []const Air.Inst.Ref) Allocator.Error!?Lowering {
6989	assert(std.meta.eql(it.pt, fg.pt));
6990	const ip = &it.pt.zcu.intern_pool;
6991	if (it.zig_index >= it.fn_info.param_types.len) {
6992	if (it.zig_index >= args.len) {
6993	return null;
6994	} else {
6995	return nextInner(it, fg.typeOf(args[it.zig_index]));
6996	}
6997	} else {
6998	return nextInner(it, Type.fromInterned(it.fn_info.param_types.get(ip)[it.zig_index]));
6999	}
7000	}
7001
7002	fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
7003	const pt = it.pt;
7004	const zcu = pt.zcu;
7005	const target = zcu.getTarget();
7006
7007	if (!ty.hasRuntimeBits(zcu)) {
7008	it.zig_index += 1;
7009	return .no_bits;
7010	}
7011	switch (it.fn_info.cc) {
7012	.@"inline" => unreachable,
7013	.auto => {
7014	it.zig_index += 1;
7015	it.llvm_index += 1;
7016	if (ty.isSlice(zcu) or
7017	(ty.zigTypeTag(zcu) == .optional and ty.optionalChild(zcu).isSlice(zcu) and !ty.ptrAllowsZero(zcu)))
7018	{
7019	it.llvm_index += 1;
7020	return .slice;
7021	} else if (isByRef(ty, zcu)) {
7022	return .byref;
7023	} else if (target.cpu.arch.isX86() and
7024	!target.cpu.has(.x86, .evex512) and
7025	ty.totalVectorBits(zcu) >= 512)
7026	{
7027	// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
7028	// "512-bit vector arguments require 'evex512' for AVX512"
7029	return .byref;
7030	} else {
7031	return .byval;
7032	}
7033	},
7034	.async => {
7035	@panic("TODO implement async function lowering in the LLVM backend");
7036	},
7037	.x86_64_sysv => return it.nextSystemV(ty),
7038	.x86_64_win => return it.nextWin64(ty),
7039	.x86_stdcall => {
7040	it.zig_index += 1;
7041	it.llvm_index += 1;
7042
7043	if (isScalar(zcu, ty)) {
7044	return .byval;
7045	} else {
7046	it.byval_attr = true;
7047	return .byref;
7048	}
7049	},
7050	.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => {
7051	it.zig_index += 1;
7052	it.llvm_index += 1;
7053	switch (aarch64_c_abi.classifyType(ty, zcu)) {
7054	.memory => return .byref_mut,
7055	.float_array => \|len\| return Lowering{ .float_array = len },
7056	.byval => return .byval,
7057	.integer => {
7058	it.types_len = 1;
7059	it.types_buffer[0] = .i64;
7060	return .multiple_llvm_types;
7061	},
7062	.double_integer => return Lowering{ .i64_array = 2 },
7063	}
7064	},
7065	.arm_aapcs, .arm_aapcs_vfp => {
7066	it.zig_index += 1;
7067	it.llvm_index += 1;
7068	switch (arm_c_abi.classifyType(ty, zcu, .arg)) {
7069	.memory => {
7070	it.byval_attr = true;
7071	return .byref;
7072	},
7073	.byval => return .byval,
7074	.i32_array => \|size\| return Lowering{ .i32_array = size },
7075	.i64_array => \|size\| return Lowering{ .i64_array = size },
7076	}
7077	},
7078	.mips_o32 => {
7079	it.zig_index += 1;
7080	it.llvm_index += 1;
7081	switch (mips_c_abi.classifyType(ty, zcu, .arg)) {
7082	.memory => {
7083	it.byval_attr = true;
7084	return .byref;
7085	},
7086	.byval => return .byval,
7087	.i32_array => \|size\| return Lowering{ .i32_array = size },
7088	}
7089	},
7090	.riscv64_lp64, .riscv32_ilp32 => {
7091	it.zig_index += 1;
7092	it.llvm_index += 1;
7093	switch (riscv_c_abi.classifyType(ty, zcu)) {
7094	.memory => return .byref_mut,
7095	.byval => return .byval,
7096	.integer => return .abi_sized_int,
7097	.double_integer => return Lowering{ .i64_array = 2 },
7098	.fields => {
7099	it.types_len = 0;
7100	for (0..ty.structFieldCount(zcu)) \|field_index\| {
7101	const field_ty = ty.fieldType(field_index, zcu);
7102	if (!field_ty.hasRuntimeBits(zcu)) continue;
7103	it.types_buffer[it.types_len] = try it.object.lowerType(pt, field_ty);
7104	it.types_len += 1;
7105	}
7106	it.llvm_index += it.types_len - 1;
7107	return .multiple_llvm_types;
7108	},
7109	}
7110	},
7111	.wasm_mvp => switch (wasm_c_abi.classifyType(ty, zcu)) {
7112	.direct => \|scalar_ty\| {
7113	if (isScalar(zcu, ty)) {
7114	it.zig_index += 1;
7115	it.llvm_index += 1;
7116	return .byval;
7117	} else {
7118	var types_buffer: [8]Builder.Type = undefined;
7119	types_buffer[0] = try it.object.lowerType(pt, scalar_ty);
7120	it.types_buffer = types_buffer;
7121	it.types_len = 1;
7122	it.llvm_index += 1;
7123	it.zig_index += 1;
7124	return .multiple_llvm_types;
7125	}
7126	},
7127	.indirect => {
7128	it.zig_index += 1;
7129	it.llvm_index += 1;
7130	it.byval_attr = true;
7131	return .byref;
7132	},
7133	},
7134	// TODO investigate other callconvs
7135	else => {
7136	it.zig_index += 1;
7137	it.llvm_index += 1;
7138	return .byval;
7139	},
7140	}
7141	}
7142
7143	fn nextWin64(it: *ParamTypeIterator, ty: Type) ?Lowering {
7144	const zcu = it.pt.zcu;
7145	switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) {
7146	.integer => {
7147	if (isScalar(zcu, ty)) {
7148	it.zig_index += 1;
7149	it.llvm_index += 1;
7150	return .byval;
7151	} else {
7152	it.zig_index += 1;
7153	it.llvm_index += 1;
7154	return .abi_sized_int;
7155	}
7156	},
7157	.win_i128 => {
7158	it.zig_index += 1;
7159	it.llvm_index += 1;
7160	return .byref;
7161	},
7162	.memory => {
7163	it.zig_index += 1;
7164	it.llvm_index += 1;
7165	return .byref_mut;
7166	},
7167	.sse => {
7168	it.zig_index += 1;
7169	it.llvm_index += 1;
7170	return .byval;
7171	},
7172	else => unreachable,
7173	}
7174	}
7175
7176	fn nextSystemV(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
7177	const zcu = it.pt.zcu;
7178	const ip = &zcu.intern_pool;
7179	ty.assertHasLayout(zcu);
7180	const classes = x86_64_abi.classifySystemV(ty, zcu, zcu.getTarget(), .arg);
7181	if (classes[0] == .memory) {
7182	it.zig_index += 1;
7183	it.llvm_index += 1;
7184	it.byval_attr = true;
7185	return .byref;
7186	}
7187	if (isScalar(zcu, ty)) {
7188	it.zig_index += 1;
7189	it.llvm_index += 1;
7190	return .byval;
7191	}
7192	var types_index: u32 = 0;
7193	var types_buffer: [8]Builder.Type = undefined;
7194	for (classes) \|class\| {
7195	switch (class) {
7196	.integer => {
7197	types_buffer[types_index] = .i64;
7198	types_index += 1;
7199	},
7200	.sse => {
7201	types_buffer[types_index] = .double;
7202	types_index += 1;
7203	},
7204	.sseup => {
7205	if (types_buffer[types_index - 1] == .double) {
7206	types_buffer[types_index - 1] = .fp128;
7207	} else {
7208	types_buffer[types_index] = .double;
7209	types_index += 1;
7210	}
7211	},
7212	.float => {
7213	types_buffer[types_index] = .float;
7214	types_index += 1;
7215	},
7216	.float_combine => {
7217	types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float);
7218	types_index += 1;
7219	},
7220	.x87 => {
7221	it.zig_index += 1;
7222	it.llvm_index += 1;
7223	it.byval_attr = true;
7224	return .byref;
7225	},
7226	.x87up => unreachable,
7227	.none => break,
7228	.memory => unreachable, // handled above
7229	.win_i128 => unreachable, // windows only
7230	.integer_per_element => {
7231	@panic("TODO");
7232	},
7233	}
7234	}
7235	const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
7236	if (first_non_integer == null or classes[first_non_integer.?] == .none) {
7237	assert(first_non_integer orelse classes.len == types_index);
7238	if (types_index == 1) {
7239	it.zig_index += 1;
7240	it.llvm_index += 1;
7241	return .abi_sized_int;
7242	}
7243	if (it.llvm_index + types_index > 6) {
7244	it.zig_index += 1;
7245	it.llvm_index += 1;
7246	it.byval_attr = true;
7247	return .byref;
7248	}
7249	switch (ip.indexToKey(ty.toIntern())) {
7250	.struct_type => {
7251	const size = ty.abiSize(zcu);
7252	assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
7253	if (size % 8 > 0) {
7254	types_buffer[types_index - 1] =
7255	try it.object.builder.intType(@intCast(size % 8 * 8));
7256	}
7257	},
7258	else => {},
7259	}
7260	}
7261	it.types_len = types_index;
7262	it.types_buffer = types_buffer;
7263	it.llvm_index += types_index;
7264	it.zig_index += 1;
7265	return .multiple_llvm_types;
7266	}
7267	};
7268	pub fn iterateParamTypes(object: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) ParamTypeIterator {
7269	return .{
7270	.object = object,
7271	.pt = pt,
7272	.fn_info = fn_info,
7273	.zig_index = 0,
7274	.llvm_index = 0,
7275	.types_len = 0,
7276	.types_buffer = undefined,
7277	.byval_attr = false,
7278	};
7279	}
7280
7281	fn returnTypeByRef(zcu: Zcu, target: const std.Target, ty: Type) bool {
7282	if (isByRef(ty, zcu)) {
7283	return true;
7284	} else if (target.cpu.arch.isX86() and
7285	!target.cpu.has(.x86, .evex512) and
7286	ty.totalVectorBits(zcu) >= 512)
7287	{
7288	// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
7289	// "512-bit vector arguments require 'evex512' for AVX512"
7290	return true;
7291	} else {
7292	return false;
7293	}
7294	}
7295
7296	pub fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: Zcu, target: const std.Target) bool {
7297	const return_type = Type.fromInterned(fn_info.return_type);
7298	if (!return_type.hasRuntimeBits(zcu)) return false;
7299
7300	return switch (fn_info.cc) {
7301	.auto => returnTypeByRef(zcu, target, return_type),
7302	.x86_64_sysv => firstParamSRetSystemV(return_type, zcu, target),
7303	.x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory,
7304	.x86_sysv, .x86_win => isByRef(return_type, zcu),
7305	.x86_stdcall => !isScalar(zcu, return_type),
7306	.wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect,
7307	.aarch64_aapcs,
7308	.aarch64_aapcs_darwin,
7309	.aarch64_aapcs_win,
7310	=> aarch64_c_abi.classifyType(return_type, zcu) == .memory,
7311	.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
7312	.memory, .i64_array => true,
7313	.i32_array => \|size\| size != 1,
7314	.byval => false,
7315	},
7316	.riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory,
7317	.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
7318	.memory, .i32_array => true,
7319	.byval => false,
7320	},
7321	else => false, // TODO: investigate other targets/callconvs
7322	};
7323	}
7324
7325	fn firstParamSRetSystemV(ty: Type, zcu: Zcu, target: const std.Target) bool {
7326	const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret);
7327	if (class[0] == .memory) return true;
7328	if (class[0] == .x87 and class[2] != .none) return true;
7329	return false;
7330	}
7331
7332	/// In order to support the C calling convention, some return types need to be lowered
7333	/// completely differently in the function prototype to honor the C ABI, and then
7334	/// be effectively bitcasted to the actual return type.
7335	pub fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
7336	const zcu = pt.zcu;
7337	const return_type = Type.fromInterned(fn_info.return_type);
7338	if (!return_type.hasRuntimeBits(zcu)) {
7339	assert(!return_type.isError(zcu));
7340	return .void;
7341	}
7342	const target = zcu.getTarget();
7343	switch (fn_info.cc) {
7344	.@"inline" => unreachable,
7345	.auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(pt, return_type),
7346
7347	.x86_64_sysv => return lowerSystemVFnRetTy(o, pt, fn_info),
7348	.x86_64_win => return lowerWin64FnRetTy(o, pt, fn_info),
7349	.x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(pt, return_type) else .void,
7350	.x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(pt, return_type),
7351	.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) {
7352	.memory => return .void,
7353	.float_array => return o.lowerType(pt, return_type),
7354	.byval => return o.lowerType(pt, return_type),
7355	.integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
7356	.double_integer => return o.builder.arrayType(2, .i64),
7357	},
7358	.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
7359	.memory, .i64_array => return .void,
7360	.i32_array => \|len\| return if (len == 1) .i32 else .void,
7361	.byval => return o.lowerType(pt, return_type),
7362	},
7363	.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
7364	.memory, .i32_array => return .void,
7365	.byval => return o.lowerType(pt, return_type),
7366	},
7367	.riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) {
7368	.memory => return .void,
7369	.integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
7370	.double_integer => {
7371	const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {
7372	.riscv64, .riscv64be => .i64,
7373	.riscv32, .riscv32be => .i32,
7374	else => unreachable,
7375	};
7376	return o.builder.structType(.normal, &.{ integer, integer });
7377	},
7378	.byval => return o.lowerType(pt, return_type),
7379	.fields => {
7380	var types_len: usize = 0;
7381	var types: [8]Builder.Type = undefined;
7382	for (0..return_type.structFieldCount(zcu)) \|field_index\| {
7383	const field_ty = return_type.fieldType(field_index, zcu);
7384	if (!field_ty.hasRuntimeBits(zcu)) continue;
7385	types[types_len] = try o.lowerType(pt, field_ty);
7386	types_len += 1;
7387	}
7388	return o.builder.structType(.normal, types[0..types_len]);
7389	},
7390	},
7391	.wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) {
7392	.direct => \|scalar_ty\| return o.lowerType(pt, scalar_ty),
7393	.indirect => return .void,
7394	},
7395	// TODO investigate other callconvs
7396	else => return o.lowerType(pt, return_type),
7397	}
7398	}
7399
7400	fn lowerWin64FnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
7401	const zcu = pt.zcu;
7402	const return_type = Type.fromInterned(fn_info.return_type);
7403	switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) {
7404	.integer => {
7405	if (isScalar(zcu, return_type)) {
7406	return o.lowerType(pt, return_type);
7407	} else {
7408	return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8));
7409	}
7410	},
7411	.win_i128 => return o.builder.vectorType(.normal, 2, .i64),
7412	.memory => return .void,
7413	.sse => return o.lowerType(pt, return_type),
7414	else => unreachable,
7415	}
7416	}
7417
7418	fn lowerSystemVFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
7419	const zcu = pt.zcu;
7420	const ip = &zcu.intern_pool;
7421	const return_type = Type.fromInterned(fn_info.return_type);
7422	return_type.assertHasLayout(zcu);
7423	if (isScalar(zcu, return_type)) {
7424	return o.lowerType(pt, return_type);
7425	}
7426	const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret);
7427	var types_index: u32 = 0;
7428	var types_buffer: [8]Builder.Type = undefined;
7429	for (classes) \|class\| {
7430	switch (class) {
7431	.integer => {
7432	types_buffer[types_index] = .i64;
7433	types_index += 1;
7434	},
7435	.sse => {
7436	types_buffer[types_index] = .double;
7437	types_index += 1;
7438	},
7439	.sseup => {
7440	if (types_buffer[types_index - 1] == .double) {
7441	types_buffer[types_index - 1] = .fp128;
7442	} else {
7443	types_buffer[types_index] = .double;
7444	types_index += 1;
7445	}
7446	},
7447	.float => {
7448	types_buffer[types_index] = .float;
7449	types_index += 1;
7450	},
7451	.float_combine => {
7452	types_buffer[types_index] = try o.builder.vectorType(.normal, 2, .float);
7453	types_index += 1;
7454	},
7455	.x87 => {
7456	if (types_index != 0 or classes[2] != .none) return .void;
7457	types_buffer[types_index] = .x86_fp80;
7458	types_index += 1;
7459	},
7460	.x87up => continue,
7461	.none => break,
7462	.memory, .integer_per_element => return .void,
7463	.win_i128 => unreachable, // windows only
7464	}
7465	}
7466	const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
7467	if (first_non_integer == null or classes[first_non_integer.?] == .none) {
7468	assert(first_non_integer orelse classes.len == types_index);
7469	switch (ip.indexToKey(return_type.toIntern())) {
7470	.struct_type => {
7471	const size = return_type.abiSize(zcu);
7472	assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
7473	if (size % 8 > 0) {
7474	types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));
7475	}
7476	},
7477	else => {},
7478	}
7479	if (types_index == 1) return types_buffer[0];
7480	}
7481	return o.builder.structType(.normal, types_buffer[0..types_index]);
7482	}
7483
7484	/// This function deliberately does not handle `_BitInt` because it typically
7485	/// has different ABI than regular integer types, and there is no currently no
7486	/// way to determine whether a Zig integer type is meant to represent e.g. `int`
7487	/// or `_BitInt(32)`.
7488	pub fn ccAbiPromoteInt(cc: std.builtin.CallingConvention, zcu: *Zcu, ty: Type) ?std.builtin.Signedness {
7489	switch (cc) {
7490	.auto, .@"inline", .async => return null,
7491	else => {},
7492	}
7493
7494	const int_info = switch (ty.zigTypeTag(zcu)) {
7495	.bool => Type.u1.intInfo(zcu),
7496	else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return null,
7497	};
7498	assert(int_info.bits >= 0);
7499
7500	const target = zcu.getTarget();
7501	return switch (target.cpu.arch) {
7502	.aarch64,
7503	.aarch64_be,
7504	=> switch (target.os.tag) {
7505	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => switch (int_info.bits) {
7506	8, 16 => int_info.signedness,
7507	else => null,
7508	},
7509	else => null,
7510	},
7511
7512	.avr,
7513	=> switch (int_info.bits) {
7514	8 => int_info.signedness,
7515	else => null,
7516	},
7517
7518	.lanai,
7519	=> null,
7520
7521	.loongarch64,
7522	.riscv64,
7523	.riscv64be,
7524	=> switch (int_info.bits) {
7525	8, 16 => int_info.signedness,
7526	32 => .signed,
7527	else => null,
7528	},
7529
7530	.mips,
7531	.mipsel,
7532	.mips64,
7533	.mips64el,
7534	=> switch (int_info.bits) {
7535	8, 16, 64 => int_info.signedness,
7536	// https://github.com/llvm/llvm-project/issues/179088
7537	// 32 => .signed,
7538	else => null,
7539	},
7540
7541	.powerpc64,
7542	.powerpc64le,
7543	.s390x,
7544	.sparc64,
7545	.ve,
7546	=> switch (int_info.bits) {
7547	8, 16, 32 => int_info.signedness,
7548	else => null,
7549	},
7550
7551	else => switch (int_info.bits) {
7552	8, 16 => int_info.signedness,
7553	else => null,
7554	},
7555	};
7556	}
7557
7558	fn isScalar(zcu: *Zcu, ty: Type) bool {
7559	return switch (ty.zigTypeTag(zcu)) {
7560	.void,
7561	.bool,
7562	.noreturn,
7563	.int,
7564	.float,
7565	.pointer,
7566	.optional,
7567	.error_set,
7568	.@"enum",
7569	.@"anyframe",
7570	.vector,
7571	=> true,
7572
7573	.@"struct" => ty.containerLayout(zcu) == .@"packed",
7574	.@"union" => ty.containerLayout(zcu) == .@"packed",
7575	else => false,
7576	};
7577	}
7578
7579	pub fn buildAllocaInner(
7580	wip: *Builder.WipFunction,
7581	llvm_ty: Builder.Type,
7582	alignment: Builder.Alignment,
7583	target: *const std.Target,
7584	) Allocator.Error!Builder.Value {
7585	const address_space = llvmAllocaAddressSpace(target);
7586
7587	const alloca = blk: {
7588	const prev_cursor = wip.cursor;
7589	const prev_debug_location = wip.debug_location;
7590	defer {
7591	wip.cursor = prev_cursor;
7592	if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
7593	wip.debug_location = prev_debug_location;
7594	}
7595
7596	wip.cursor = .{ .block = .entry };
7597	wip.debug_location = .no_location;
7598	break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
7599	};
7600
7601	// The pointer returned from this function should have the generic address space,
7602	// if this isn't the case then cast it to the generic address space.
7603	return wip.conv(.unneeded, alloca, .ptr, "");
7604	}
7605
7606	/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
7607	/// or as an LLVM value.
7608	pub fn isByRef(ty: Type, zcu: *Zcu) bool {
7609	// For tuples and structs, if there are more than this many non-void
7610	// fields, then we make it byref, otherwise byval.
7611	const max_fields_byval = 0;
7612	const ip = &zcu.intern_pool;
7613
7614	switch (ty.zigTypeTag(zcu)) {
7615	.type,
7616	.comptime_int,
7617	.comptime_float,
7618	.enum_literal,
7619	.undefined,
7620	.null,
7621	.@"opaque",
7622	=> unreachable,
7623
7624	.noreturn,
7625	.void,
7626	.bool,
7627	.int,
7628	.float,
7629	.pointer,
7630	.error_set,
7631	.@"fn",
7632	.@"enum",
7633	.vector,
7634	.@"anyframe",
7635	=> return false,
7636
7637	.array, .frame => return ty.hasRuntimeBits(zcu),
7638	.@"struct" => {
7639	const struct_type = switch (ip.indexToKey(ty.toIntern())) {
7640	.tuple_type => \|tuple\| {
7641	var count: usize = 0;
7642	for (tuple.types.get(ip), tuple.values.get(ip)) \|field_ty, field_val\| {
7643	if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
7644
7645	count += 1;
7646	if (count > max_fields_byval) return true;
7647	if (isByRef(Type.fromInterned(field_ty), zcu)) return true;
7648	}
7649	return false;
7650	},
7651	.struct_type => ip.loadStructType(ty.toIntern()),
7652	else => unreachable,
7653	};
7654
7655	// Packed structs are represented to LLVM as integers.
7656	if (struct_type.layout == .@"packed") return false;
7657
7658	const field_types = struct_type.field_types.get(ip);
7659	var it = struct_type.iterateRuntimeOrder(ip);
7660	var count: usize = 0;
7661	while (it.next()) \|field_index\| {
7662	count += 1;
7663	if (count > max_fields_byval) return true;
7664	const field_ty = Type.fromInterned(field_types[field_index]);
7665	if (isByRef(field_ty, zcu)) return true;
7666	}
7667	return false;
7668	},
7669	.@"union" => switch (ty.containerLayout(zcu)) {
7670	.@"packed" => return false,
7671	else => return ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),
7672	},
7673	.error_union => {
7674	const payload_ty = ty.errorUnionPayload(zcu);
7675	if (!payload_ty.hasRuntimeBits(zcu)) {
7676	return false;
7677	}
7678	return true;
7679	},
7680	.optional => {
7681	const payload_ty = ty.optionalChild(zcu);
7682	if (!payload_ty.hasRuntimeBits(zcu)) {
7683	return false;
7684	}
7685	if (ty.optionalReprIsPayload(zcu)) {
7686	return false;
7687	}
7688	return true;
7689	},
7690	}
7691	}
7692
7693	/// If the operand type of an atomic operation is not byte sized we need to
7694	/// widen it before using it and then truncate the result.
7695	/// RMW exchange of floating-point values is bitcasted to same-sized integer
7696	/// types to work around a LLVM deficiency when targeting ARM/AArch64.
7697	fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type {
7698	const zcu = fg.pt.zcu;
7699	switch (ty.zigTypeTag(zcu)) {
7700	.int, .@"enum", .@"struct", .@"union" => {},
7701	.float => {
7702	if (!is_rmw_xchg) return .none;
7703	return fg.object.builder.intType(@intCast(ty.abiSize(zcu) * 8));
7704	},
7705	.bool => return .i8,
7706	else => return .none,
7707	}
7708	const bit_count = ty.bitSize(zcu);
7709	if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) {
7710	return fg.object.builder.intType(@intCast(ty.abiSize(zcu) * 8));
7711	} else {
7712	return .none;
7713	}
7714	}
7715
7716	fn errUnionPayloadFieldIndex(payload_ty: Type, pt: Zcu.PerThread) !u1 {
7717	const zcu = pt.zcu;
7718	const err_int_ty = try pt.errorIntType();
7719	return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.gt, payload_ty.abiAlignment(zcu)));
7720	}
7721
7722	fn errUnionErrorFieldIndex(payload_ty: Type, pt: Zcu.PerThread) !u1 {
7723	const zcu = pt.zcu;
7724	const err_int_ty = try pt.errorIntType();
7725	return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.lte, payload_ty.abiAlignment(zcu)));
7726	}
7727
7728	fn compilerRtIntBits(bits: u16) ?u16 {
7729	inline for (.{ 32, 64, 128 }) \|b\| {
7730	if (bits <= b) {
7731	return b;
7732	}
7733	}
7734	return null;
7735	}
7736
7737	/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location
7738	///
7739	/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
7740	fn constraintAllowsMemory(constraint: []const u8) bool {
7741	// TODO: This implementation is woefully incomplete.
7742	for (constraint) \|byte\| {
7743	switch (byte) {
7744	'=', '*', ',', '&' => {},
7745	'm', 'o', 'X', 'g' => return true,
7746	else => {},
7747	}
7748	} else return false;
7749	}
7750
7751	/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register
7752	///
7753	/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
7754	fn constraintAllowsRegister(constraint: []const u8) bool {
7755	// TODO: This implementation is woefully incomplete.
7756	for (constraint) \|byte\| {
7757	switch (byte) {
7758	'=', '*', ',', '&' => {},
7759	'm', 'o' => {},
7760	else => return true,
7761	}
7762	} else return false;
7763	}
7764
7765	/// Appends zero or more LLVM constraints to `llvm_constraints`, returning how many were added.
7766	fn appendConstraints(
7767	gpa: Allocator,
7768	llvm_constraints: *std.ArrayList(u8),
7769	zig_name: []const u8,
7770	target: *const std.Target,
7771	) error{OutOfMemory}!usize {
7772	switch (target.cpu.arch) {
7773	.mips, .mipsel, .mips64, .mips64el => if (mips_clobber_overrides.get(zig_name)) \|llvm_tag\| {
7774	const llvm_name = @tagName(llvm_tag);
7775	try llvm_constraints.ensureUnusedCapacity(gpa, llvm_name.len + 4);
7776	llvm_constraints.appendSliceAssumeCapacity("~{");
7777	llvm_constraints.appendSliceAssumeCapacity(llvm_name);
7778	llvm_constraints.appendSliceAssumeCapacity("},");
7779	return 1;
7780	},
7781	else => {},
7782	}
7783
7784	try llvm_constraints.ensureUnusedCapacity(gpa, zig_name.len + 4);
7785	llvm_constraints.appendSliceAssumeCapacity("~{");
7786	llvm_constraints.appendSliceAssumeCapacity(zig_name);
7787	llvm_constraints.appendSliceAssumeCapacity("},");
7788	return 1;
7789	}
7790
7791	/// LLVM does not support all relevant intrinsics for all targets, so we
7792	/// may need to manually generate a compiler-rt call.
7793	fn intrinsicsAllowed(scalar_ty: Type, target: *const std.Target) bool {
7794	return switch (scalar_ty.toIntern()) {
7795	.f16_type => backendSupportsF16(target),
7796	.f80_type => (target.cTypeBitSize(.longdouble) == 80) and backendSupportsF80(target),
7797	.f128_type => (target.cTypeBitSize(.longdouble) == 128) and backendSupportsF128(target),
7798	else => true,
7799	};
7800	}
7801
7802	fn toLlvmAtomicOrdering(atomic_order: std.builtin.AtomicOrder) Builder.AtomicOrdering {
7803	return switch (atomic_order) {
7804	.unordered => .unordered,
7805	.monotonic => .monotonic,
7806	.acquire => .acquire,
7807	.release => .release,
7808	.acq_rel => .acq_rel,
7809	.seq_cst => .seq_cst,
7810	};
7811	}
7812
7813	fn toLlvmAtomicRmwBinOp(
7814	op: std.builtin.AtomicRmwOp,
7815	is_signed: bool,
7816	is_float: bool,
7817	) Builder.Function.Instruction.AtomicRmw.Operation {
7818	return switch (op) {
7819	.Xchg => .xchg,
7820	.Add => if (is_float) .fadd else return .add,
7821	.Sub => if (is_float) .fsub else return .sub,
7822	.And => .@"and",
7823	.Nand => .nand,
7824	.Or => .@"or",
7825	.Xor => .xor,
7826	.Max => if (is_float) .fmax else if (is_signed) .max else return .umax,
7827	.Min => if (is_float) .fmin else if (is_signed) .min else return .umin,
7828	};
7829	}
7830
7831	fn minIntConst(b: Builder, min_ty: Type, as_ty: Builder.Type, zcu: const Zcu) Allocator.Error!Builder.Constant {
7832	const info = min_ty.intInfo(zcu);
7833	if (info.signedness == .unsigned or info.bits == 0) {
7834	return b.intConst(as_ty, 0);
7835	}
7836	if (std.math.cast(u6, info.bits - 1)) \|shift\| {
7837	const min_val: i64 = @as(i64, std.math.minInt(i64)) >> (63 - shift);
7838	return b.intConst(as_ty, min_val);
7839	}
7840	var res: std.math.big.int.Managed = try .init(zcu.gpa);
7841	defer res.deinit();
7842	try res.setTwosCompIntLimit(.min, info.signedness, info.bits);
7843	return b.bigIntConst(as_ty, res.toConst());
7844	}
7845
7846	fn maxIntConst(b: Builder, max_ty: Type, as_ty: Builder.Type, zcu: const Zcu) Allocator.Error!Builder.Constant {
7847	const info = max_ty.intInfo(zcu);
7848	switch (info.bits) {
7849	0 => return b.intConst(as_ty, 0),
7850	1 => switch (info.signedness) {
7851	.signed => return b.intConst(as_ty, 0),
7852	.unsigned => return b.intConst(as_ty, 1),
7853	},
7854	else => {},
7855	}
7856	const unsigned_bits = switch (info.signedness) {
7857	.unsigned => info.bits,
7858	.signed => info.bits - 1,
7859	};
7860	if (std.math.cast(u6, unsigned_bits)) \|shift\| {
7861	const max_val: u64 = (@as(u64, 1) << shift) - 1;
7862	return b.intConst(as_ty, max_val);
7863	}
7864	var res: std.math.big.int.Managed = try .init(zcu.gpa);
7865	defer res.deinit();
7866	try res.setTwosCompIntLimit(.max, info.signedness, info.bits);
7867	return b.bigIntConst(as_ty, res.toConst());
7868	}
7869
7870	/// On some targets, local values that are in the generic address space must be generated into a
7871	/// different address, space and then cast back to the generic address space.
7872	/// For example, on GPUs local variable declarations must be generated into the local address space.
7873	/// This function returns the address space local values should be generated into.
7874	fn llvmAllocaAddressSpace(target: *const std.Target) Builder.AddrSpace {
7875	return switch (target.cpu.arch) {
7876	// On amdgcn, locals should be generated into the private address space.
7877	// To make Zig not impossible to use, these are then converted to addresses in the
7878	// generic address space and treates as regular pointers. This is the way that HIP also does it.
7879	.amdgcn => Builder.AddrSpace.amdgpu.private,
7880	else => .default,
7881	};
7882	}
7883
7884	const mips_clobber_overrides = std.StaticStringMap(enum {
7885	@"$msair",
7886	@"$msacsr",
7887	@"$msaaccess",
7888	@"$msasave",
7889	@"$msamodify",
7890	@"$msarequest",
7891	@"$msamap",
7892	@"$msaunmap",
7893	@"$f0",
7894	@"$f1",
7895	@"$f2",
7896	@"$f3",
7897	@"$f4",
7898	@"$f5",
7899	@"$f6",
7900	@"$f7",
7901	@"$f8",
7902	@"$f9",
7903	@"$f10",
7904	@"$f11",
7905	@"$f12",
7906	@"$f13",
7907	@"$f14",
7908	@"$f15",
7909	@"$f16",
7910	@"$f17",
7911	@"$f18",
7912	@"$f19",
7913	@"$f20",
7914	@"$f21",
7915	@"$f22",
7916	@"$f23",
7917	@"$f24",
7918	@"$f25",
7919	@"$f26",
7920	@"$f27",
7921	@"$f28",
7922	@"$f29",
7923	@"$f30",
7924	@"$f31",
7925	@"$fcc0",
7926	@"$fcc1",
7927	@"$fcc2",
7928	@"$fcc3",
7929	@"$fcc4",
7930	@"$fcc5",
7931	@"$fcc6",
7932	@"$fcc7",
7933	@"$w0",
7934	@"$w1",
7935	@"$w2",
7936	@"$w3",
7937	@"$w4",
7938	@"$w5",
7939	@"$w6",
7940	@"$w7",
7941	@"$w8",
7942	@"$w9",
7943	@"$w10",
7944	@"$w11",
7945	@"$w12",
7946	@"$w13",
7947	@"$w14",
7948	@"$w15",
7949	@"$w16",
7950	@"$w17",
7951	@"$w18",
7952	@"$w19",
7953	@"$w20",
7954	@"$w21",
7955	@"$w22",
7956	@"$w23",
7957	@"$w24",
7958	@"$w25",
7959	@"$w26",
7960	@"$w27",
7961	@"$w28",
7962	@"$w29",
7963	@"$w30",
7964	@"$w31",
7965	@"$0",
7966	@"$1",
7967	@"$2",
7968	@"$3",
7969	@"$4",
7970	@"$5",
7971	@"$6",
7972	@"$7",
7973	@"$8",
7974	@"$9",
7975	@"$10",
7976	@"$11",
7977	@"$12",
7978	@"$13",
7979	@"$14",
7980	@"$15",
7981	@"$16",
7982	@"$17",
7983	@"$18",
7984	@"$19",
7985	@"$20",
7986	@"$21",
7987	@"$22",
7988	@"$23",
7989	@"$24",
7990	@"$25",
7991	@"$26",
7992	@"$27",
7993	@"$28",
7994	@"$29",
7995	@"$30",
7996	@"$31",
7997	}).initComptime(.{
7998	.{ "msa_ir", .@"$msair" },
7999	.{ "msa_csr", .@"$msacsr" },
8000	.{ "msa_access", .@"$msaaccess" },
8001	.{ "msa_save", .@"$msasave" },
8002	.{ "msa_modify", .@"$msamodify" },
8003	.{ "msa_request", .@"$msarequest" },
8004	.{ "msa_map", .@"$msamap" },
8005	.{ "msa_unmap", .@"$msaunmap" },
8006	.{ "f0", .@"$f0" },
8007	.{ "f1", .@"$f1" },
8008	.{ "f2", .@"$f2" },
8009	.{ "f3", .@"$f3" },
8010	.{ "f4", .@"$f4" },
8011	.{ "f5", .@"$f5" },
8012	.{ "f6", .@"$f6" },
8013	.{ "f7", .@"$f7" },
8014	.{ "f8", .@"$f8" },
8015	.{ "f9", .@"$f9" },
8016	.{ "f10", .@"$f10" },
8017	.{ "f11", .@"$f11" },
8018	.{ "f12", .@"$f12" },
8019	.{ "f13", .@"$f13" },
8020	.{ "f14", .@"$f14" },
8021	.{ "f15", .@"$f15" },
8022	.{ "f16", .@"$f16" },
8023	.{ "f17", .@"$f17" },
8024	.{ "f18", .@"$f18" },
8025	.{ "f19", .@"$f19" },
8026	.{ "f20", .@"$f20" },
8027	.{ "f21", .@"$f21" },
8028	.{ "f22", .@"$f22" },
8029	.{ "f23", .@"$f23" },
8030	.{ "f24", .@"$f24" },
8031	.{ "f25", .@"$f25" },
8032	.{ "f26", .@"$f26" },
8033	.{ "f27", .@"$f27" },
8034	.{ "f28", .@"$f28" },
8035	.{ "f29", .@"$f29" },
8036	.{ "f30", .@"$f30" },
8037	.{ "f31", .@"$f31" },
8038	.{ "fcc0", .@"$fcc0" },
8039	.{ "fcc1", .@"$fcc1" },
8040	.{ "fcc2", .@"$fcc2" },
8041	.{ "fcc3", .@"$fcc3" },
8042	.{ "fcc4", .@"$fcc4" },
8043	.{ "fcc5", .@"$fcc5" },
8044	.{ "fcc6", .@"$fcc6" },
8045	.{ "fcc7", .@"$fcc7" },
8046	.{ "w0", .@"$w0" },
8047	.{ "w1", .@"$w1" },
8048	.{ "w2", .@"$w2" },
8049	.{ "w3", .@"$w3" },
8050	.{ "w4", .@"$w4" },
8051	.{ "w5", .@"$w5" },
8052	.{ "w6", .@"$w6" },
8053	.{ "w7", .@"$w7" },
8054	.{ "w8", .@"$w8" },
8055	.{ "w9", .@"$w9" },
8056	.{ "w10", .@"$w10" },
8057	.{ "w11", .@"$w11" },
8058	.{ "w12", .@"$w12" },
8059	.{ "w13", .@"$w13" },
8060	.{ "w14", .@"$w14" },
8061	.{ "w15", .@"$w15" },
8062	.{ "w16", .@"$w16" },
8063	.{ "w17", .@"$w17" },
8064	.{ "w18", .@"$w18" },
8065	.{ "w19", .@"$w19" },
8066	.{ "w20", .@"$w20" },
8067	.{ "w21", .@"$w21" },
8068	.{ "w22", .@"$w22" },
8069	.{ "w23", .@"$w23" },
8070	.{ "w24", .@"$w24" },
8071	.{ "w25", .@"$w25" },
8072	.{ "w26", .@"$w26" },
8073	.{ "w27", .@"$w27" },
8074	.{ "w28", .@"$w28" },
8075	.{ "w29", .@"$w29" },
8076	.{ "w30", .@"$w30" },
8077	.{ "w31", .@"$w31" },
8078	.{ "r0", .@"$0" },
8079	.{ "r1", .@"$1" },
8080	.{ "r2", .@"$2" },
8081	.{ "r3", .@"$3" },
8082	.{ "r4", .@"$4" },
8083	.{ "r5", .@"$5" },
8084	.{ "r6", .@"$6" },
8085	.{ "r7", .@"$7" },
8086	.{ "r8", .@"$8" },
8087	.{ "r9", .@"$9" },
8088	.{ "r10", .@"$10" },
8089	.{ "r11", .@"$11" },
8090	.{ "r12", .@"$12" },
8091	.{ "r13", .@"$13" },
8092	.{ "r14", .@"$14" },
8093	.{ "r15", .@"$15" },
8094	.{ "r16", .@"$16" },
8095	.{ "r17", .@"$17" },
8096	.{ "r18", .@"$18" },
8097	.{ "r19", .@"$19" },
8098	.{ "r20", .@"$20" },
8099	.{ "r21", .@"$21" },
8100	.{ "r22", .@"$22" },
8101	.{ "r23", .@"$23" },
8102	.{ "r24", .@"$24" },
8103	.{ "r25", .@"$25" },
8104	.{ "r26", .@"$26" },
8105	.{ "r27", .@"$27" },
8106	.{ "r28", .@"$28" },
8107	.{ "r29", .@"$29" },
8108	.{ "r30", .@"$30" },
8109	.{ "r31", .@"$31" },
8110	});
8111
8112	const std = @import("std");
8113	const Allocator = std.mem.Allocator;
8114	const Builder = std.zig.llvm.Builder;
8115	const assert = std.debug.assert;
8116	const math = std.math;
8117
8118	const x86_64_abi = @import("../x86_64/abi.zig");
8119	const wasm_c_abi = @import("../wasm/abi.zig");
8120	const aarch64_c_abi = @import("../aarch64/abi.zig");
8121	const arm_c_abi = @import("../arm/abi.zig");
8122	const riscv_c_abi = @import("../riscv64/abi.zig");
8123	const mips_c_abi = @import("../mips/abi.zig");
8124
8125	const Zcu = @import("../../Zcu.zig");
8126	const Air = @import("../../Air.zig");
8127	const Package = @import("../../Package.zig");
8128	const InternPool = @import("../../InternPool.zig");
8129	const Value = @import("../../Value.zig");
8130	const Type = @import("../../Type.zig");
8131
8132	const target_util = @import("../../target.zig");
8133	const libcFloatPrefix = target_util.libcFloatPrefix;
8134	const libcFloatSuffix = target_util.libcFloatSuffix;
8135	const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
8136	const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev;
8137
8138	const llvm = @import("../llvm.zig");
8139	const Object = llvm.Object;
8140	const toLlvmCallConvTag = llvm.toLlvmCallConvTag;
8141	const toLlvmAddressSpace = llvm.toLlvmAddressSpace;
8142	const optional_layout_version = llvm.optional_layout_version;
8143	const backendSupportsF16 = llvm.backendSupportsF16;
8144	const backendSupportsF80 = llvm.backendSupportsF80;
8145	const backendSupportsF128 = llvm.backendSupportsF128;