Commit d167bd4b568e2d5808fc37a8c683e4c06317a434 | zig

author	Jacob Young <jacobly@ziglang.org> 2023-07-08 04:38:31-04:00
committer	Jacob Young <jacobly@ziglang.org> 2023-07-19 23:38:40-04:00
log	d167bd4b568e2d5808fc37a8c683e4c06317a434
tree	c099f307438d8eeba89a7831a148fa1a3281047e
parent	3314fd83af5a6a269926a6f8ad3fcb62b04242c9

llvm: finish converting `lowerType`

3 files changed, 1308 insertions(+), 950 deletions(-)

src
- codegen
  - llvm.zig
  - llvm
    Builder.zig
    bindings.zig

src/codegen/llvm.zig+722-858

...	...	@@ -374,12 +374,11 @@ const DataLayoutBuilder = struct {
374	374	.pref = 64,
375	375	.idx = 64,
376	376	};
377		const address_space_info = llvmAddressSpaceInfo(self.target);
378		assert(address_space_info[0].llvm == llvm.address_space.default);
379		for (address_space_info) \|info\| {
380		const is_default = info.llvm == llvm.address_space.default;
	377	const addr_space_info = llvmAddrSpaceInfo(self.target);
	378	for (addr_space_info, 0..) \|info, i\| {
	379	assert((info.llvm == .default) == (i == 0));
381	380	if (info.non_integral) {
382		assert(!is_default);
	381	assert(info.llvm != .default);
383	382	any_non_integral = true;
384	383	}
385	384	const size = info.size orelse ptr_bit_width;
...	...	@@ -391,7 +390,7 @@ const DataLayoutBuilder = struct {
391	390	abi == default_info.abi and
392	391	pref == default_info.pref and
393	392	idx == default_info.idx;
394		if (is_default) default_info = .{
	393	if (info.llvm == .default) default_info = .{
395	394	.size = size,
396	395	.abi = abi,
397	396	.pref = pref,
...	...	@@ -400,7 +399,7 @@ const DataLayoutBuilder = struct {
400	399	if (!info.force_in_data_layout and matches_default and
401	400	self.target.cpu.arch != .riscv64 and !is_aarch64_windows) continue;
402	401	try writer.writeAll("-p");
403		if (!is_default) try writer.print("{d}", .{info.llvm});
	402	if (info.llvm != .default) try writer.print("{d}", .{@intFromEnum(info.llvm)});
404	403	try writer.print(":{d}:{d}", .{ size, abi });
405	404	if (pref != abi or idx != size) {
406	405	try writer.print(":{d}", .{pref});
...	...	@@ -459,8 +458,8 @@ const DataLayoutBuilder = struct {
459	458	try self.typeAlignment(.vector, 512, 128, 128, true, writer);
460	459	if (any_non_integral) {
461	460	try writer.writeAll("-ni");
462		for (address_space_info) \|info\| if (info.non_integral)
463		try writer.print(":{d}", .{info.llvm});
	461	for (addr_space_info) \|info\| if (info.non_integral)
	462	try writer.print(":{d}", .{@intFromEnum(info.llvm)});
464	463	}
465	464	}
466	465
...	...	@@ -589,7 +588,7 @@ pub const Object = struct {
589	588	/// Memoizes a null `?usize` value.
590	589	null_opt_addr: ?*llvm.Value,
591	590
592		pub const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, *llvm.Type);
	591	pub const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);
593	592
594	593	/// This is an ArrayHashMap as opposed to a HashMap because in `flushModule` we
595	594	/// want to iterate over it while adding entries to it.
...	...	@@ -809,15 +808,10 @@ pub const Object = struct {
809	808	const error_name_table_ptr_global = o.error_name_table orelse return;
810	809
811	810	const mod = o.module;
812		const target = mod.getTarget();
813	811
814		const llvm_ptr_ty = o.context.pointerType(0); // TODO: Address space
815		const llvm_usize_ty = o.context.intType(target.ptrBitWidth());
816		const type_fields = [_]*llvm.Type{
817		llvm_ptr_ty,
818		llvm_usize_ty,
819		};
820		const llvm_slice_ty = o.context.structType(&type_fields, type_fields.len, .False);
	812	// TODO: Address space
	813	const llvm_usize_ty = try o.lowerType(Type.usize);
	814	const llvm_slice_ty = (try o.builder.structType(.normal, &.{ .ptr, llvm_usize_ty })).toLlvm(&o.builder);
821	815	const slice_ty = Type.slice_const_u8_sentinel_0;
822	816	const slice_alignment = slice_ty.abiAlignment(mod);
823	817
...	...	@@ -838,7 +832,7 @@ pub const Object = struct {
838	832
839	833	const slice_fields = [_]*llvm.Value{
840	834	str_global,
841		llvm_usize_ty.constInt(name.len, .False),
	835	llvm_usize_ty.toLlvm(&o.builder).constInt(name.len, .False),
842	836	};
843	837	llvm_error.* = llvm_slice_ty.constNamedStruct(&slice_fields, slice_fields.len);
844	838	}
...	...	@@ -1204,7 +1198,7 @@ pub const Object = struct {
1204	1198	{
1205	1199	var llvm_arg_i = @as(c_uint, @intFromBool(ret_ptr != null)) + @intFromBool(err_return_tracing);
1206	1200	var it = iterateParamTypes(o, fn_info);
1207		while (it.next()) \|lowering\| switch (lowering) {
	1201	while (try it.next()) \|lowering\| switch (lowering) {
1208	1202	.no_bits => continue,
1209	1203	.byval => {
1210	1204	assert(!it.byval_attr);
...	...	@@ -1229,7 +1223,7 @@ pub const Object = struct {
1229	1223	},
1230	1224	.byref => {
1231	1225	const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
1232		const param_llvm_ty = try o.lowerLlvmType(param_ty);
	1226	const param_llvm_ty = try o.lowerType(param_ty);
1233	1227	const param = llvm_func.getParam(llvm_arg_i);
1234	1228	const alignment = param_ty.abiAlignment(mod);
1235	1229
...	...	@@ -1241,14 +1235,14 @@ pub const Object = struct {
1241	1235	if (isByRef(param_ty, mod)) {
1242	1236	args.appendAssumeCapacity(param);
1243	1237	} else {
1244		const load_inst = builder.buildLoad(param_llvm_ty, param, "");
	1238	const load_inst = builder.buildLoad(param_llvm_ty.toLlvm(&o.builder), param, "");
1245	1239	load_inst.setAlignment(alignment);
1246	1240	args.appendAssumeCapacity(load_inst);
1247	1241	}
1248	1242	},
1249	1243	.byref_mut => {
1250	1244	const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
1251		const param_llvm_ty = try o.lowerLlvmType(param_ty);
	1245	const param_llvm_ty = (try o.lowerType(param_ty)).toLlvm(&o.builder);
1252	1246	const param = llvm_func.getParam(llvm_arg_i);
1253	1247	const alignment = param_ty.abiAlignment(mod);
1254	1248
...	...	@@ -1271,7 +1265,7 @@ pub const Object = struct {
1271	1265	const param = llvm_func.getParam(llvm_arg_i);
1272	1266	llvm_arg_i += 1;
1273	1267
1274		const param_llvm_ty = try o.lowerLlvmType(param_ty);
	1268	const param_llvm_ty = (try o.lowerType(param_ty)).toLlvm(&o.builder);
1275	1269	const abi_size = @as(c_uint, @intCast(param_ty.abiSize(mod)));
1276	1270	const int_llvm_ty = o.context.intType(abi_size * 8);
1277	1271	const alignment = @max(
...	...	@@ -1316,16 +1310,16 @@ pub const Object = struct {
1316	1310	const len_param = llvm_func.getParam(llvm_arg_i);
1317	1311	llvm_arg_i += 1;
1318	1312
1319		const slice_llvm_ty = try o.lowerLlvmType(param_ty);
	1313	const slice_llvm_ty = (try o.lowerType(param_ty)).toLlvm(&o.builder);
1320	1314	const partial = builder.buildInsertValue(slice_llvm_ty.getUndef(), ptr_param, 0, "");
1321	1315	const aggregate = builder.buildInsertValue(partial, len_param, 1, "");
1322	1316	try args.append(aggregate);
1323	1317	},
1324	1318	.multiple_llvm_types => {
1325	1319	assert(!it.byval_attr);
1326		const field_types = it.llvm_types_buffer[0..it.llvm_types_len];
	1320	const field_types = it.llvm_types_buffer[0..it.types_len];
1327	1321	const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
1328		const param_llvm_ty = try o.lowerLlvmType(param_ty);
	1322	const param_llvm_ty = (try o.lowerType(param_ty)).toLlvm(&o.builder);
1329	1323	const param_alignment = param_ty.abiAlignment(mod);
1330	1324	const arg_ptr = buildAllocaInner(o.context, builder, llvm_func, false, param_llvm_ty, param_alignment, target);
1331	1325	const llvm_ty = o.context.structType(field_types.ptr, @as(c_uint, @intCast(field_types.len)), .False);
...	...	@@ -1356,7 +1350,7 @@ pub const Object = struct {
1356	1350	},
1357	1351	.float_array => {
1358	1352	const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
1359		const param_llvm_ty = try o.lowerLlvmType(param_ty);
	1353	const param_llvm_ty = (try o.lowerType(param_ty)).toLlvm(&o.builder);
1360	1354	const param = llvm_func.getParam(llvm_arg_i);
1361	1355	llvm_arg_i += 1;
1362	1356
...	...	@@ -1374,7 +1368,7 @@ pub const Object = struct {
1374	1368	},
1375	1369	.i32_array, .i64_array => {
1376	1370	const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
1377		const param_llvm_ty = try o.lowerLlvmType(param_ty);
	1371	const param_llvm_ty = (try o.lowerType(param_ty)).toLlvm(&o.builder);
1378	1372	const param = llvm_func.getParam(llvm_arg_i);
1379	1373	llvm_arg_i += 1;
1380	1374
...	...	@@ -2678,7 +2672,7 @@ pub const Object = struct {
2678	2672	const global = o.llvm_module.addGlobalInAddressSpace(
2679	2673	llvm_init.typeOf(),
2680	2674	"",
2681		llvm_actual_addrspace,
	2675	@intFromEnum(llvm_actual_addrspace),
2682	2676	);
2683	2677	global.setLinkage(.Internal);
2684	2678	global.setUnnamedAddr(.True);
...	...	@@ -2686,7 +2680,7 @@ pub const Object = struct {
2686	2680	global.setInitializer(llvm_init);
2687	2681
2688	2682	const addrspace_casted_global = if (llvm_wanted_addrspace != llvm_actual_addrspace)
2689		global.constAddrSpaceCast(o.context.pointerType(llvm_wanted_addrspace))
	2683	global.constAddrSpaceCast(o.context.pointerType(@intFromEnum(llvm_wanted_addrspace)))
2690	2684	else
2691	2685	global;
2692	2686
...	...	@@ -2710,16 +2704,16 @@ pub const Object = struct {
2710	2704	const target = mod.getTarget();
2711	2705	const sret = firstParamSRet(fn_info, mod);
2712	2706
2713		const fn_type = try o.lowerLlvmType(zig_fn_type);
	2707	const fn_type = try o.lowerType(zig_fn_type);
2714	2708
2715	2709	const ip = &mod.intern_pool;
2716	2710	const fqn = try o.builder.string(ip.stringToSlice(try decl.getFullyQualifiedName(mod)));
2717	2711
2718	2712	const llvm_addrspace = toLlvmAddressSpace(decl.@"addrspace", target);
2719		const llvm_fn = o.llvm_module.addFunctionInAddressSpace(fqn.toSlice(&o.builder).?, fn_type, llvm_addrspace);
	2713	const llvm_fn = o.llvm_module.addFunctionInAddressSpace(fqn.toSlice(&o.builder).?, fn_type.toLlvm(&o.builder), @intFromEnum(llvm_addrspace));
2720	2714
2721	2715	var global = Builder.Global{
2722		.type = .void,
	2716	.type = fn_type,
2723	2717	.kind = .{ .function = @enumFromInt(o.builder.functions.items.len) },
2724	2718	};
2725	2719	var function = Builder.Function{
...	...	@@ -2745,7 +2739,7 @@ pub const Object = struct {
2745	2739	o.addArgAttr(llvm_fn, 0, "nonnull"); // Sret pointers must not be address 0
2746	2740	o.addArgAttr(llvm_fn, 0, "noalias");
2747	2741
2748		const raw_llvm_ret_ty = try o.lowerLlvmType(fn_info.return_type.toType());
	2742	const raw_llvm_ret_ty = (try o.lowerType(fn_info.return_type.toType())).toLlvm(&o.builder);
2749	2743	llvm_fn.addSretAttr(raw_llvm_ret_ty);
2750	2744	}
2751	2745
...	...	@@ -2789,7 +2783,7 @@ pub const Object = struct {
2789	2783	var it = iterateParamTypes(o, fn_info);
2790	2784	it.llvm_index += @intFromBool(sret);
2791	2785	it.llvm_index += @intFromBool(err_return_tracing);
2792		while (it.next()) \|lowering\| switch (lowering) {
	2786	while (try it.next()) \|lowering\| switch (lowering) {
2793	2787	.byval => {
2794	2788	const param_index = it.zig_index - 1;
2795	2789	const param_ty = fn_info.param_types.get(ip)[param_index].toType();
...	...	@@ -2799,7 +2793,7 @@ pub const Object = struct {
2799	2793	},
2800	2794	.byref => {
2801	2795	const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1];
2802		const param_llvm_ty = try o.lowerLlvmType(param_ty.toType());
	2796	const param_llvm_ty = try o.lowerType(param_ty.toType());
2803	2797	const alignment = param_ty.toType().abiAlignment(mod);
2804	2798	o.addByRefParamAttrs(llvm_fn, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
2805	2799	},
...	...	@@ -2883,15 +2877,9 @@ pub const Object = struct {
2883	2877
2884	2878	const target = mod.getTarget();
2885	2879
2886		const ty = try o.lowerType(decl.ty);
2887		const llvm_type = if (ty != .none)
2888		o.builder.llvm_types.items[@intFromEnum(ty)]
2889		else
2890		try o.lowerLlvmType(decl.ty);
2891		const llvm_actual_addrspace = toLlvmGlobalAddressSpace(decl.@"addrspace", target);
2892
2893	2880	var global = Builder.Global{
2894		.type = if (ty != .none) ty else .void,
	2881	.addr_space = toLlvmGlobalAddressSpace(decl.@"addrspace", target),
	2882	.type = try o.lowerType(decl.ty),
2895	2883	.kind = .{ .object = @enumFromInt(o.builder.objects.items.len) },
2896	2884	};
2897	2885	var object = Builder.Object{
...	...	@@ -2904,9 +2892,9 @@ pub const Object = struct {
2904	2892	else
2905	2893	fqn;
2906	2894	const llvm_global = o.llvm_module.addGlobalInAddressSpace(
2907		llvm_type,
	2895	global.type.toLlvm(&o.builder),
2908	2896	fqn.toSlice(&o.builder).?,
2909		llvm_actual_addrspace,
	2897	@intFromEnum(global.addr_space),
2910	2898	);
2911	2899
2912	2900	// This is needed for declarations created by `@extern`.
...	...	@@ -2943,22 +2931,19 @@ pub const Object = struct {
2943	2931	}
2944	2932
2945	2933	fn isUnnamedType(o: Object, ty: Type, val: llvm.Value) bool {
2946		// Once `lowerLlvmType` succeeds, successive calls to it with the same Zig type
2947		// are guaranteed to succeed. So if a call to `lowerLlvmType` fails here it means
	2934	// Once `lowerType` succeeds, successive calls to it with the same Zig type
	2935	// are guaranteed to succeed. So if a call to `lowerType` fails here it means
2948	2936	// it is the first time lowering the type, which means the value can't possible
2949	2937	// have that type.
2950		const llvm_ty = o.lowerLlvmType(ty) catch return true;
	2938	const llvm_ty = (o.lowerType(ty) catch return true).toLlvm(&o.builder);
2951	2939	return val.typeOf() != llvm_ty;
2952	2940	}
2953	2941
2954		fn lowerLlvmType(o: Object, t: Type) Allocator.Error!llvm.Type {
2955		const ty = try o.lowerType(t);
2956		const llvm_ty = if (ty != .none)
2957		o.builder.llvm_types.items[@intFromEnum(ty)]
2958		else
2959		try o.lowerLlvmTypeInner(t);
	2942	fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type {
	2943	const ty = try o.lowerTypeInner(t);
2960	2944	const mod = o.module;
2961	2945	if (std.debug.runtime_safety and false) check: {
	2946	const llvm_ty = ty.toLlvm(&o.builder);
2962	2947	if (t.zigTypeTag(mod) == .Opaque) break :check;
2963	2948	if (!t.hasRuntimeBits(mod)) break :check;
2964	2949	if (!llvm_ty.isSized().toBool()) break :check;
...	...	@@ -2971,445 +2956,10 @@ pub const Object = struct {
2971	2956	});
2972	2957	}
2973	2958	}
2974		return llvm_ty;
2975		}
2976
2977		fn lowerLlvmTypeInner(o: Object, t: Type) Allocator.Error!llvm.Type {
2978		const gpa = o.gpa;
2979		const mod = o.module;
2980		const target = mod.getTarget();
2981		switch (t.zigTypeTag(mod)) {
2982		.Void, .NoReturn => return o.context.voidType(),
2983		.Int => {
2984		const info = t.intInfo(mod);
2985		assert(info.bits != 0);
2986		return o.context.intType(info.bits);
2987		},
2988		.Enum => {
2989		const int_ty = t.intTagType(mod);
2990		const bit_count = int_ty.intInfo(mod).bits;
2991		assert(bit_count != 0);
2992		return o.context.intType(bit_count);
2993		},
2994		.Float => switch (t.floatBits(target)) {
2995		16 => return if (backendSupportsF16(target)) o.context.halfType() else o.context.intType(16),
2996		32 => return o.context.floatType(),
2997		64 => return o.context.doubleType(),
2998		80 => return if (backendSupportsF80(target)) o.context.x86_fp80Type() else o.context.intType(80),
2999		128 => return o.context.fp128Type(),
3000		else => unreachable,
3001		},
3002		.Bool => return o.context.intType(1),
3003		.Pointer => {
3004		if (t.isSlice(mod)) {
3005		const ptr_type = t.slicePtrFieldType(mod);
3006
3007		const fields: [2]*llvm.Type = .{
3008		try o.lowerLlvmType(ptr_type),
3009		try o.lowerLlvmType(Type.usize),
3010		};
3011		return o.context.structType(&fields, fields.len, .False);
3012		}
3013		const ptr_info = t.ptrInfo(mod);
3014		const llvm_addrspace = toLlvmAddressSpace(ptr_info.flags.address_space, target);
3015		return o.context.pointerType(llvm_addrspace);
3016		},
3017		.Opaque => {
3018		if (t.toIntern() == .anyopaque_type) return o.context.intType(8);
3019
3020		const gop = try o.type_map.getOrPut(gpa, t.toIntern());
3021		if (gop.found_existing) return gop.value_ptr.*;
3022
3023		const opaque_type = mod.intern_pool.indexToKey(t.toIntern()).opaque_type;
3024		const name = mod.intern_pool.stringToSlice(try mod.opaqueFullyQualifiedName(opaque_type));
3025
3026		const llvm_struct_ty = o.context.structCreateNamed(name);
3027		gop.value_ptr.* = llvm_struct_ty; // must be done before any recursive calls
3028		return llvm_struct_ty;
3029		},
3030		.Array => {
3031		const elem_ty = t.childType(mod);
3032		if (std.debug.runtime_safety) assert((try elem_ty.onePossibleValue(mod)) == null);
3033		const elem_llvm_ty = try o.lowerLlvmType(elem_ty);
3034		const total_len = t.arrayLen(mod) + @intFromBool(t.sentinel(mod) != null);
3035		return elem_llvm_ty.arrayType(@as(c_uint, @intCast(total_len)));
3036		},
3037		.Vector => {
3038		const elem_type = try o.lowerLlvmType(t.childType(mod));
3039		return elem_type.vectorType(t.vectorLen(mod));
3040		},
3041		.Optional => {
3042		const child_ty = t.optionalChild(mod);
3043		if (!child_ty.hasRuntimeBitsIgnoreComptime(mod)) {
3044		return o.context.intType(8);
3045		}
3046		const payload_llvm_ty = try o.lowerLlvmType(child_ty);
3047		if (t.optionalReprIsPayload(mod)) {
3048		return payload_llvm_ty;
3049		}
3050
3051		comptime assert(optional_layout_version == 3);
3052		var fields_buf: [3]*llvm.Type = .{
3053		payload_llvm_ty, o.context.intType(8), undefined,
3054		};
3055		const offset = child_ty.abiSize(mod) + 1;
3056		const abi_size = t.abiSize(mod);
3057		const padding = @as(c_uint, @intCast(abi_size - offset));
3058		if (padding == 0) {
3059		return o.context.structType(&fields_buf, 2, .False);
3060		}
3061		fields_buf[2] = o.context.intType(8).arrayType(padding);
3062		return o.context.structType(&fields_buf, 3, .False);
3063		},
3064		.ErrorUnion => {
3065		const payload_ty = t.errorUnionPayload(mod);
3066		if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {
3067		return try o.lowerLlvmType(Type.anyerror);
3068		}
3069		const llvm_error_type = try o.lowerLlvmType(Type.anyerror);
3070		const llvm_payload_type = try o.lowerLlvmType(payload_ty);
3071
3072		const payload_align = payload_ty.abiAlignment(mod);
3073		const error_align = Type.anyerror.abiAlignment(mod);
3074
3075		const payload_size = payload_ty.abiSize(mod);
3076		const error_size = Type.anyerror.abiSize(mod);
3077
3078		var fields_buf: [3]*llvm.Type = undefined;
3079		if (error_align > payload_align) {
3080		fields_buf[0] = llvm_error_type;
3081		fields_buf[1] = llvm_payload_type;
3082		const payload_end =
3083		std.mem.alignForward(u64, error_size, payload_align) +
3084		payload_size;
3085		const abi_size = std.mem.alignForward(u64, payload_end, error_align);
3086		const padding = @as(c_uint, @intCast(abi_size - payload_end));
3087		if (padding == 0) {
3088		return o.context.structType(&fields_buf, 2, .False);
3089		}
3090		fields_buf[2] = o.context.intType(8).arrayType(padding);
3091		return o.context.structType(&fields_buf, 3, .False);
3092		} else {
3093		fields_buf[0] = llvm_payload_type;
3094		fields_buf[1] = llvm_error_type;
3095		const error_end =
3096		std.mem.alignForward(u64, payload_size, error_align) +
3097		error_size;
3098		const abi_size = std.mem.alignForward(u64, error_end, payload_align);
3099		const padding = @as(c_uint, @intCast(abi_size - error_end));
3100		if (padding == 0) {
3101		return o.context.structType(&fields_buf, 2, .False);
3102		}
3103		fields_buf[2] = o.context.intType(8).arrayType(padding);
3104		return o.context.structType(&fields_buf, 3, .False);
3105		}
3106		},
3107		.ErrorSet => return o.context.intType(16),
3108		.Struct => {
3109		const gop = try o.type_map.getOrPut(gpa, t.toIntern());
3110		if (gop.found_existing) return gop.value_ptr.*;
3111
3112		const struct_type = switch (mod.intern_pool.indexToKey(t.toIntern())) {
3113		.anon_struct_type => \|tuple\| {
3114		const llvm_struct_ty = o.context.structCreateNamed("");
3115		gop.value_ptr.* = llvm_struct_ty; // must be done before any recursive calls
3116
3117		var llvm_field_types: std.ArrayListUnmanaged(*llvm.Type) = .{};
3118		defer llvm_field_types.deinit(gpa);
3119
3120		try llvm_field_types.ensureUnusedCapacity(gpa, tuple.types.len);
3121
3122		comptime assert(struct_layout_version == 2);
3123		var offset: u64 = 0;
3124		var big_align: u32 = 0;
3125
3126		for (tuple.types, tuple.values) \|field_ty, field_val\| {
3127		if (field_val != .none or !field_ty.toType().hasRuntimeBits(mod)) continue;
3128
3129		const field_align = field_ty.toType().abiAlignment(mod);
3130		big_align = @max(big_align, field_align);
3131		const prev_offset = offset;
3132		offset = std.mem.alignForward(u64, offset, field_align);
3133
3134		const padding_len = offset - prev_offset;
3135		if (padding_len > 0) {
3136		const llvm_array_ty = o.context.intType(8).arrayType(@as(c_uint, @intCast(padding_len)));
3137		try llvm_field_types.append(gpa, llvm_array_ty);
3138		}
3139		const field_llvm_ty = try o.lowerLlvmType(field_ty.toType());
3140		try llvm_field_types.append(gpa, field_llvm_ty);
3141
3142		offset += field_ty.toType().abiSize(mod);
3143		}
3144		{
3145		const prev_offset = offset;
3146		offset = std.mem.alignForward(u64, offset, big_align);
3147		const padding_len = offset - prev_offset;
3148		if (padding_len > 0) {
3149		const llvm_array_ty = o.context.intType(8).arrayType(@as(c_uint, @intCast(padding_len)));
3150		try llvm_field_types.append(gpa, llvm_array_ty);
3151		}
3152		}
3153
3154		llvm_struct_ty.structSetBody(
3155		llvm_field_types.items.ptr,
3156		@as(c_uint, @intCast(llvm_field_types.items.len)),
3157		.False,
3158		);
3159
3160		return llvm_struct_ty;
3161		},
3162		.struct_type => \|struct_type\| struct_type,
3163		else => unreachable,
3164		};
3165
3166		const struct_obj = mod.structPtrUnwrap(struct_type.index).?;
3167
3168		if (struct_obj.layout == .Packed) {
3169		assert(struct_obj.haveLayout());
3170		const int_llvm_ty = try o.lowerLlvmType(struct_obj.backing_int_ty);
3171		gop.value_ptr.* = int_llvm_ty;
3172		return int_llvm_ty;
3173		}
3174
3175		const name = try o.builder.string(mod.intern_pool.stringToSlice(
3176		try struct_obj.getFullyQualifiedName(mod),
3177		));
3178		const ty = try o.builder.opaqueType(name);
3179
3180		const llvm_struct_ty = o.builder.llvm_types.items[@intFromEnum(ty)];
3181		gop.value_ptr.* = llvm_struct_ty; // must be done before any recursive calls
3182
3183		assert(struct_obj.haveFieldTypes());
3184
3185		var llvm_field_types: std.ArrayListUnmanaged(*llvm.Type) = .{};
3186		defer llvm_field_types.deinit(gpa);
3187
3188		try llvm_field_types.ensureUnusedCapacity(gpa, struct_obj.fields.count());
3189
3190		comptime assert(struct_layout_version == 2);
3191		var offset: u64 = 0;
3192		var big_align: u32 = 1;
3193		var any_underaligned_fields = false;
3194
3195		var it = struct_obj.runtimeFieldIterator(mod);
3196		while (it.next()) \|field_and_index\| {
3197		const field = field_and_index.field;
3198		const field_align = field.alignment(mod, struct_obj.layout);
3199		const field_ty_align = field.ty.abiAlignment(mod);
3200		any_underaligned_fields = any_underaligned_fields or
3201		field_align < field_ty_align;
3202		big_align = @max(big_align, field_align);
3203		const prev_offset = offset;
3204		offset = std.mem.alignForward(u64, offset, field_align);
3205
3206		const padding_len = offset - prev_offset;
3207		if (padding_len > 0) {
3208		const llvm_array_ty = o.context.intType(8).arrayType(@as(c_uint, @intCast(padding_len)));
3209		try llvm_field_types.append(gpa, llvm_array_ty);
3210		}
3211		const field_llvm_ty = try o.lowerLlvmType(field.ty);
3212		try llvm_field_types.append(gpa, field_llvm_ty);
3213
3214		offset += field.ty.abiSize(mod);
3215		}
3216		{
3217		const prev_offset = offset;
3218		offset = std.mem.alignForward(u64, offset, big_align);
3219		const padding_len = offset - prev_offset;
3220		if (padding_len > 0) {
3221		const llvm_array_ty = o.context.intType(8).arrayType(@as(c_uint, @intCast(padding_len)));
3222		try llvm_field_types.append(gpa, llvm_array_ty);
3223		}
3224		}
3225
3226		llvm_struct_ty.structSetBody(
3227		llvm_field_types.items.ptr,
3228		@as(c_uint, @intCast(llvm_field_types.items.len)),
3229		llvm.Bool.fromBool(any_underaligned_fields),
3230		);
3231
3232		return llvm_struct_ty;
3233		},
3234		.Union => {
3235		const gop = try o.type_map.getOrPut(gpa, t.toIntern());
3236		if (gop.found_existing) return gop.value_ptr.*;
3237
3238		const layout = t.unionGetLayout(mod);
3239		const union_obj = mod.typeToUnion(t).?;
3240
3241		if (union_obj.layout == .Packed) {
3242		const bitsize = @as(c_uint, @intCast(t.bitSize(mod)));
3243		const int_llvm_ty = o.context.intType(bitsize);
3244		gop.value_ptr.* = int_llvm_ty;
3245		return int_llvm_ty;
3246		}
3247
3248		if (layout.payload_size == 0) {
3249		const enum_tag_llvm_ty = try o.lowerLlvmType(union_obj.tag_ty);
3250		gop.value_ptr.* = enum_tag_llvm_ty;
3251		return enum_tag_llvm_ty;
3252		}
3253
3254		const name = mod.intern_pool.stringToSlice(try union_obj.getFullyQualifiedName(mod));
3255
3256		const llvm_union_ty = o.context.structCreateNamed(name);
3257		gop.value_ptr.* = llvm_union_ty; // must be done before any recursive calls
3258
3259		const aligned_field = union_obj.fields.values()[layout.most_aligned_field];
3260		const llvm_aligned_field_ty = try o.lowerLlvmType(aligned_field.ty);
3261
3262		const llvm_payload_ty = t: {
3263		if (layout.most_aligned_field_size == layout.payload_size) {
3264		break :t llvm_aligned_field_ty;
3265		}
3266		const padding_len = if (layout.tag_size == 0)
3267		@as(c_uint, @intCast(layout.abi_size - layout.most_aligned_field_size))
3268		else
3269		@as(c_uint, @intCast(layout.payload_size - layout.most_aligned_field_size));
3270		const fields: [2]*llvm.Type = .{
3271		llvm_aligned_field_ty,
3272		o.context.intType(8).arrayType(padding_len),
3273		};
3274		break :t o.context.structType(&fields, fields.len, .True);
3275		};
3276
3277		if (layout.tag_size == 0) {
3278		var llvm_fields: [1]*llvm.Type = .{llvm_payload_ty};
3279		llvm_union_ty.structSetBody(&llvm_fields, llvm_fields.len, .False);
3280		return llvm_union_ty;
3281		}
3282		const enum_tag_llvm_ty = try o.lowerLlvmType(union_obj.tag_ty);
3283
3284		// Put the tag before or after the payload depending on which one's
3285		// alignment is greater.
3286		var llvm_fields: [3]*llvm.Type = undefined;
3287		var llvm_fields_len: c_uint = 2;
3288
3289		if (layout.tag_align >= layout.payload_align) {
3290		llvm_fields = .{ enum_tag_llvm_ty, llvm_payload_ty, undefined };
3291		} else {
3292		llvm_fields = .{ llvm_payload_ty, enum_tag_llvm_ty, undefined };
3293		}
3294
3295		// Insert padding to make the LLVM struct ABI size match the Zig union ABI size.
3296		if (layout.padding != 0) {
3297		llvm_fields[2] = o.context.intType(8).arrayType(layout.padding);
3298		llvm_fields_len = 3;
3299		}
3300
3301		llvm_union_ty.structSetBody(&llvm_fields, llvm_fields_len, .False);
3302		return llvm_union_ty;
3303		},
3304		.Fn => return lowerLlvmTypeFn(o, t),
3305		.ComptimeInt => unreachable,
3306		.ComptimeFloat => unreachable,
3307		.Type => unreachable,
3308		.Undefined => unreachable,
3309		.Null => unreachable,
3310		.EnumLiteral => unreachable,
3311
3312		.Frame => @panic("TODO implement llvmType for Frame types"),
3313		.AnyFrame => @panic("TODO implement llvmType for AnyFrame types"),
3314		}
3315		}
3316
3317		fn lowerLlvmTypeFn(o: Object, fn_ty: Type) Allocator.Error!llvm.Type {
3318		const mod = o.module;
3319		const ip = &mod.intern_pool;
3320		const fn_info = mod.typeToFunc(fn_ty).?;
3321		const llvm_ret_ty = try lowerFnRetTy(o, fn_info);
3322
3323		var llvm_params = std.ArrayList(*llvm.Type).init(o.gpa);
3324		defer llvm_params.deinit();
3325
3326		if (firstParamSRet(fn_info, mod)) {
3327		try llvm_params.append(o.context.pointerType(0));
3328		}
3329
3330		if (fn_info.return_type.toType().isError(mod) and
3331		mod.comp.bin_file.options.error_return_tracing)
3332		{
3333		const ptr_ty = try mod.singleMutPtrType(try o.getStackTraceType());
3334		try llvm_params.append(try o.lowerLlvmType(ptr_ty));
3335		}
3336
3337		var it = iterateParamTypes(o, fn_info);
3338		while (it.next()) \|lowering\| switch (lowering) {
3339		.no_bits => continue,
3340		.byval => {
3341		const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
3342		try llvm_params.append(try o.lowerLlvmType(param_ty));
3343		},
3344		.byref, .byref_mut => {
3345		try llvm_params.append(o.context.pointerType(0));
3346		},
3347		.abi_sized_int => {
3348		const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
3349		const abi_size = @as(c_uint, @intCast(param_ty.abiSize(mod)));
3350		try llvm_params.append(o.context.intType(abi_size * 8));
3351		},
3352		.slice => {
3353		const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
3354		const ptr_ty = if (param_ty.zigTypeTag(mod) == .Optional)
3355		param_ty.optionalChild(mod).slicePtrFieldType(mod)
3356		else
3357		param_ty.slicePtrFieldType(mod);
3358		const ptr_llvm_ty = try o.lowerLlvmType(ptr_ty);
3359		const len_llvm_ty = try o.lowerLlvmType(Type.usize);
3360
3361		try llvm_params.ensureUnusedCapacity(2);
3362		llvm_params.appendAssumeCapacity(ptr_llvm_ty);
3363		llvm_params.appendAssumeCapacity(len_llvm_ty);
3364		},
3365		.multiple_llvm_types => {
3366		try llvm_params.appendSlice(it.llvm_types_buffer[0..it.llvm_types_len]);
3367		},
3368		.as_u16 => {
3369		try llvm_params.append(o.context.intType(16));
3370		},
3371		.float_array => \|count\| {
3372		const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
3373		const float_ty = try o.lowerLlvmType(aarch64_c_abi.getFloatArrayType(param_ty, mod).?);
3374		const field_count = @as(c_uint, @intCast(count));
3375		const arr_ty = float_ty.arrayType(field_count);
3376		try llvm_params.append(arr_ty);
3377		},
3378		.i32_array, .i64_array => \|arr_len\| {
3379		const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
3380		const arr_ty = o.context.intType(elem_size).arrayType(arr_len);
3381		try llvm_params.append(arr_ty);
3382		},
3383		};
3384
3385		return llvm.functionType(
3386		llvm_ret_ty,
3387		llvm_params.items.ptr,
3388		@as(c_uint, @intCast(llvm_params.items.len)),
3389		llvm.Bool.fromBool(fn_info.is_var_args),
3390		);
	2959	return ty;
3391	2960	}
3392	2961
3393		/// Use this instead of lowerLlvmType when you want to handle correctly the case of elem_ty
3394		/// being a zero bit type, but it should still be lowered as an i8 in such case.
3395		/// There are other similar cases handled here as well.
3396		fn lowerPtrElemTy(o: Object, elem_ty: Type) Allocator.Error!llvm.Type {
3397		const mod = o.module;
3398		const lower_elem_ty = switch (elem_ty.zigTypeTag(mod)) {
3399		.Opaque => true,
3400		.Fn => !mod.typeToFunc(elem_ty).?.is_generic,
3401		.Array => elem_ty.childType(mod).hasRuntimeBitsIgnoreComptime(mod),
3402		else => elem_ty.hasRuntimeBitsIgnoreComptime(mod),
3403		};
3404		const llvm_elem_ty = if (lower_elem_ty)
3405		try o.lowerLlvmType(elem_ty)
3406		else
3407		o.context.intType(8);
3408
3409		return llvm_elem_ty;
3410		}
3411
3412		fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type {
	2962	fn lowerTypeInner(o: *Object, t: Type) Allocator.Error!Builder.Type {
3413	2963	const mod = o.module;
3414	2964	const target = mod.getTarget();
3415	2965	return switch (t.toIntern()) {
...	...	@@ -3459,8 +3009,11 @@ pub const Object = struct {
3459	3009	.bool_type => .i1,
3460	3010	.void_type => .void,
3461	3011	.type_type => unreachable,
3462		.anyerror_type => .i16,
3463		.comptime_int_type, .comptime_float_type, .noreturn_type => unreachable,
	3012	.anyerror_type => Builder.Type.err_int,
	3013	.comptime_int_type,
	3014	.comptime_float_type,
	3015	.noreturn_type,
	3016	=> unreachable,
3464	3017	.anyframe_type => @panic("TODO implement lowerType for AnyFrame types"),
3465	3018	.null_type,
3466	3019	.undefined_type,
...	...	@@ -3482,10 +3035,16 @@ pub const Object = struct {
3482	3035	.manyptr_const_u8_sentinel_0_type,
3483	3036	.single_const_pointer_to_comptime_int_type,
3484	3037	=> .ptr,
3485		.slice_const_u8_type, .slice_const_u8_sentinel_0_type => .none,
	3038	.slice_const_u8_type,
	3039	.slice_const_u8_sentinel_0_type,
	3040	=> try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(Type.usize) }),
3486	3041	.optional_noreturn_type => unreachable,
3487		.anyerror_void_error_union_type => .i16,
3488		.generic_poison_type, .empty_struct_type => unreachable,
	3042	.anyerror_void_error_union_type,
	3043	.adhoc_inferred_error_set_type,
	3044	=> Builder.Type.err_int,
	3045	.generic_poison_type,
	3046	.empty_struct_type,
	3047	=> unreachable,
3489	3048	// values, not types
3490	3049	.undef,
3491	3050	.zero,
...	...	@@ -3511,22 +3070,265 @@ pub const Object = struct {
3511	3070	else => switch (mod.intern_pool.indexToKey(t.toIntern())) {
3512	3071	.int_type => \|int_type\| try o.builder.intType(int_type.bits),
3513	3072	.ptr_type => \|ptr_type\| switch (ptr_type.flags.size) {
3514		.One, .Many, .C => try o.builder.pointerType(@enumFromInt(
	3073	.One, .Many, .C => try o.builder.ptrType(
3515	3074	toLlvmAddressSpace(ptr_type.flags.address_space, target),
3516		)),
3517		.Slice => .none,
	3075	),
	3076	.Slice => try o.builder.structType(.normal, &.{
	3077	.ptr,
	3078	try o.lowerType(Type.usize),
	3079	}),
	3080	},
	3081	.array_type => \|array_type\| o.builder.arrayType(
	3082	array_type.len + @intFromBool(array_type.sentinel != .none),
	3083	try o.lowerType(array_type.child.toType()),
	3084	),
	3085	.vector_type => \|vector_type\| o.builder.vectorType(
	3086	.normal,
	3087	vector_type.len,
	3088	try o.lowerType(vector_type.child.toType()),
	3089	),
	3090	.opt_type => \|child_ty\| {
	3091	if (!child_ty.toType().hasRuntimeBitsIgnoreComptime(mod)) return .i8;
	3092
	3093	const payload_ty = try o.lowerType(child_ty.toType());
	3094	if (t.optionalReprIsPayload(mod)) return payload_ty;
	3095
	3096	comptime assert(optional_layout_version == 3);
	3097	var fields_buf: [3]Builder.Type = .{ payload_ty, .i8, .none };
	3098	const offset = child_ty.toType().abiSize(mod) + 1;
	3099	const abi_size = t.abiSize(mod);
	3100	const padding = abi_size - offset;
	3101	if (padding == 0) return o.builder.structType(.normal, fields_buf[0..2]);
	3102	fields_buf[2] = try o.builder.arrayType(padding, .i8);
	3103	return o.builder.structType(.normal, fields_buf[0..3]);
3518	3104	},
3519		.array_type, .vector_type, .opt_type => .none,
3520	3105	.anyframe_type => @panic("TODO implement lowerType for AnyFrame types"),
3521		.error_union_type => .none,
	3106	.error_union_type => \|error_union_type\| {
	3107	const error_type = Builder.Type.err_int;
	3108	if (!error_union_type.payload_type.toType().hasRuntimeBitsIgnoreComptime(mod))
	3109	return error_type;
	3110	const payload_type = try o.lowerType(error_union_type.payload_type.toType());
	3111
	3112	const payload_align = error_union_type.payload_type.toType().abiAlignment(mod);
	3113	const error_align = Type.err_int.abiAlignment(mod);
	3114
	3115	const payload_size = error_union_type.payload_type.toType().abiSize(mod);
	3116	const error_size = Type.err_int.abiSize(mod);
	3117
	3118	var fields_buf: [3]Builder.Type = undefined;
	3119	if (error_align > payload_align) {
	3120	fields_buf[0] = error_type;
	3121	fields_buf[1] = payload_type;
	3122	const payload_end =
	3123	std.mem.alignForward(u64, error_size, payload_align) +
	3124	payload_size;
	3125	const abi_size = std.mem.alignForward(u64, payload_end, error_align);
	3126	const padding = abi_size - payload_end;
	3127	if (padding == 0) return o.builder.structType(.normal, fields_buf[0..2]);
	3128	fields_buf[2] = try o.builder.arrayType(padding, .i8);
	3129	return o.builder.structType(.normal, fields_buf[0..3]);
	3130	} else {
	3131	fields_buf[0] = payload_type;
	3132	fields_buf[1] = error_type;
	3133	const error_end =
	3134	std.mem.alignForward(u64, payload_size, error_align) +
	3135	error_size;
	3136	const abi_size = std.mem.alignForward(u64, error_end, payload_align);
	3137	const padding = abi_size - error_end;
	3138	if (padding == 0) return o.builder.structType(.normal, fields_buf[0..2]);
	3139	fields_buf[2] = try o.builder.arrayType(padding, .i8);
	3140	return o.builder.structType(.normal, fields_buf[0..3]);
	3141	}
	3142	},
3522	3143	.simple_type => unreachable,
3523		.struct_type,
3524		.anon_struct_type,
3525		.union_type,
3526		.opaque_type,
3527		=> .none,
	3144	.struct_type => \|struct_type\| {
	3145	const gop = try o.type_map.getOrPut(o.gpa, t.toIntern());
	3146	if (gop.found_existing) return gop.value_ptr.*;
	3147
	3148	const struct_obj = mod.structPtrUnwrap(struct_type.index).?;
	3149	if (struct_obj.layout == .Packed) {
	3150	assert(struct_obj.haveLayout());
	3151	const int_ty = try o.lowerType(struct_obj.backing_int_ty);
	3152	gop.value_ptr.* = int_ty;
	3153	return int_ty;
	3154	}
	3155
	3156	const name = try o.builder.string(mod.intern_pool.stringToSlice(
	3157	try struct_obj.getFullyQualifiedName(mod),
	3158	));
	3159	const ty = try o.builder.opaqueType(name);
	3160	gop.value_ptr.* = ty; // must be done before any recursive calls
	3161
	3162	assert(struct_obj.haveFieldTypes());
	3163
	3164	var llvm_field_types = std.ArrayListUnmanaged(Builder.Type){};
	3165	defer llvm_field_types.deinit(o.gpa);
	3166	try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_obj.fields.count());
	3167
	3168	comptime assert(struct_layout_version == 2);
	3169	var offset: u64 = 0;
	3170	var big_align: u32 = 1;
	3171	var struct_kind: Builder.Type.Structure.Kind = .normal;
	3172
	3173	var it = struct_obj.runtimeFieldIterator(mod);
	3174	while (it.next()) \|field_and_index\| {
	3175	const field = field_and_index.field;
	3176	const field_align = field.alignment(mod, struct_obj.layout);
	3177	const field_ty_align = field.ty.abiAlignment(mod);
	3178	if (field_align < field_ty_align) struct_kind = .@"packed";
	3179	big_align = @max(big_align, field_align);
	3180	const prev_offset = offset;
	3181	offset = std.mem.alignForward(u64, offset, field_align);
	3182
	3183	const padding_len = offset - prev_offset;
	3184	if (padding_len > 0) try llvm_field_types.append(
	3185	o.gpa,
	3186	try o.builder.arrayType(padding_len, .i8),
	3187	);
	3188	try llvm_field_types.append(o.gpa, try o.lowerType(field.ty));
	3189
	3190	offset += field.ty.abiSize(mod);
	3191	}
	3192	{
	3193	const prev_offset = offset;
	3194	offset = std.mem.alignForward(u64, offset, big_align);
	3195	const padding_len = offset - prev_offset;
	3196	if (padding_len > 0) try llvm_field_types.append(
	3197	o.gpa,
	3198	try o.builder.arrayType(padding_len, .i8),
	3199	);
	3200	}
	3201
	3202	try o.builder.namedTypeSetBody(
	3203	ty,
	3204	try o.builder.structType(struct_kind, llvm_field_types.items),
	3205	);
	3206	return ty;
	3207	},
	3208	.anon_struct_type => \|anon_struct_type\| {
	3209	var llvm_field_types: std.ArrayListUnmanaged(Builder.Type) = .{};
	3210	defer llvm_field_types.deinit(o.gpa);
	3211	try llvm_field_types.ensureUnusedCapacity(o.gpa, anon_struct_type.types.len);
	3212
	3213	comptime assert(struct_layout_version == 2);
	3214	var offset: u64 = 0;
	3215	var big_align: u32 = 0;
	3216
	3217	for (anon_struct_type.types, anon_struct_type.values) \|field_ty, field_val\| {
	3218	if (field_val != .none or !field_ty.toType().hasRuntimeBits(mod)) continue;
	3219
	3220	const field_align = field_ty.toType().abiAlignment(mod);
	3221	big_align = @max(big_align, field_align);
	3222	const prev_offset = offset;
	3223	offset = std.mem.alignForward(u64, offset, field_align);
	3224
	3225	const padding_len = offset - prev_offset;
	3226	if (padding_len > 0) try llvm_field_types.append(
	3227	o.gpa,
	3228	try o.builder.arrayType(padding_len, .i8),
	3229	);
	3230	try llvm_field_types.append(o.gpa, try o.lowerType(field_ty.toType()));
	3231
	3232	offset += field_ty.toType().abiSize(mod);
	3233	}
	3234	{
	3235	const prev_offset = offset;
	3236	offset = std.mem.alignForward(u64, offset, big_align);
	3237	const padding_len = offset - prev_offset;
	3238	if (padding_len > 0) try llvm_field_types.append(
	3239	o.gpa,
	3240	try o.builder.arrayType(padding_len, .i8),
	3241	);
	3242	}
	3243	return o.builder.structType(.normal, llvm_field_types.items);
	3244	},
	3245	.union_type => \|union_type\| {
	3246	const gop = try o.type_map.getOrPut(o.gpa, t.toIntern());
	3247	if (gop.found_existing) return gop.value_ptr.*;
	3248
	3249	const union_obj = mod.unionPtr(union_type.index);
	3250	const layout = union_obj.getLayout(mod, union_type.hasTag());
	3251
	3252	if (union_obj.layout == .Packed) {
	3253	const int_ty = try o.builder.intType(@intCast(t.bitSize(mod)));
	3254	gop.value_ptr.* = int_ty;
	3255	return int_ty;
	3256	}
	3257
	3258	if (layout.payload_size == 0) {
	3259	const enum_tag_ty = try o.lowerType(union_obj.tag_ty);
	3260	gop.value_ptr.* = enum_tag_ty;
	3261	return enum_tag_ty;
	3262	}
	3263
	3264	const name = try o.builder.string(mod.intern_pool.stringToSlice(
	3265	try union_obj.getFullyQualifiedName(mod),
	3266	));
	3267	const ty = try o.builder.opaqueType(name);
	3268	gop.value_ptr.* = ty; // must be done before any recursive calls
	3269
	3270	const aligned_field = union_obj.fields.values()[layout.most_aligned_field];
	3271	const aligned_field_ty = try o.lowerType(aligned_field.ty);
	3272
	3273	const payload_ty = ty: {
	3274	if (layout.most_aligned_field_size == layout.payload_size) {
	3275	break :ty aligned_field_ty;
	3276	}
	3277	const padding_len = if (layout.tag_size == 0)
	3278	layout.abi_size - layout.most_aligned_field_size
	3279	else
	3280	layout.payload_size - layout.most_aligned_field_size;
	3281	break :ty try o.builder.structType(.@"packed", &.{
	3282	aligned_field_ty,
	3283	try o.builder.arrayType(padding_len, .i8),
	3284	});
	3285	};
	3286
	3287	if (layout.tag_size == 0) {
	3288	try o.builder.namedTypeSetBody(
	3289	ty,
	3290	try o.builder.structType(.normal, &.{payload_ty}),
	3291	);
	3292	return ty;
	3293	}
	3294	const enum_tag_ty = try o.lowerType(union_obj.tag_ty);
	3295
	3296	// Put the tag before or after the payload depending on which one's
	3297	// alignment is greater.
	3298	var llvm_fields: [3]Builder.Type = undefined;
	3299	var llvm_fields_len: usize = 2;
	3300
	3301	if (layout.tag_align >= layout.payload_align) {
	3302	llvm_fields = .{ enum_tag_ty, payload_ty, .none };
	3303	} else {
	3304	llvm_fields = .{ payload_ty, enum_tag_ty, .none };
	3305	}
	3306
	3307	// Insert padding to make the LLVM struct ABI size match the Zig union ABI size.
	3308	if (layout.padding != 0) {
	3309	llvm_fields[llvm_fields_len] = try o.builder.arrayType(layout.padding, .i8);
	3310	llvm_fields_len += 1;
	3311	}
	3312
	3313	try o.builder.namedTypeSetBody(
	3314	ty,
	3315	try o.builder.structType(.normal, llvm_fields[0..llvm_fields_len]),
	3316	);
	3317	return ty;
	3318	},
	3319	.opaque_type => \|opaque_type\| {
	3320	const gop = try o.type_map.getOrPut(o.gpa, t.toIntern());
	3321	if (!gop.found_existing) {
	3322	const name = try o.builder.string(mod.intern_pool.stringToSlice(
	3323	try mod.opaqueFullyQualifiedName(opaque_type),
	3324	));
	3325	gop.value_ptr.* = try o.builder.opaqueType(name);
	3326	}
	3327	return gop.value_ptr.*;
	3328	},
3528	3329	.enum_type => \|enum_type\| try o.lowerType(enum_type.tag_ty.toType()),
3529		.func_type, .error_set_type, .inferred_error_set_type => .none,
	3330	.func_type => \|func_type\| try o.lowerTypeFn(func_type),
	3331	.error_set_type, .inferred_error_set_type => Builder.Type.err_int,
3530	3332	// values, not types
3531	3333	.undef,
3532	3334	.runtime_value,
...	...	@@ -3552,6 +3354,85 @@ pub const Object = struct {
3552	3354	};
3553	3355	}
3554	3356
	3357	/// Use this instead of lowerType when you want to handle correctly the case of elem_ty
	3358	/// being a zero bit type, but it should still be lowered as an i8 in such case.
	3359	/// There are other similar cases handled here as well.
	3360	fn lowerPtrElemTy(o: *Object, elem_ty: Type) Allocator.Error!Builder.Type {
	3361	const mod = o.module;
	3362	const lower_elem_ty = switch (elem_ty.zigTypeTag(mod)) {
	3363	.Opaque => true,
	3364	.Fn => !mod.typeToFunc(elem_ty).?.is_generic,
	3365	.Array => elem_ty.childType(mod).hasRuntimeBitsIgnoreComptime(mod),
	3366	else => elem_ty.hasRuntimeBitsIgnoreComptime(mod),
	3367	};
	3368	return if (lower_elem_ty) try o.lowerType(elem_ty) else .i8;
	3369	}
	3370
	3371	fn lowerTypeFn(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	3372	const mod = o.module;
	3373	const ip = &mod.intern_pool;
	3374	const ret_ty = try lowerFnRetTy(o, fn_info);
	3375
	3376	var llvm_params = std.ArrayListUnmanaged(Builder.Type){};
	3377	defer llvm_params.deinit(o.gpa);
	3378
	3379	if (firstParamSRet(fn_info, mod)) {
	3380	try llvm_params.append(o.gpa, .ptr);
	3381	}
	3382
	3383	if (fn_info.return_type.toType().isError(mod) and
	3384	mod.comp.bin_file.options.error_return_tracing)
	3385	{
	3386	const ptr_ty = try mod.singleMutPtrType(try o.getStackTraceType());
	3387	try llvm_params.append(o.gpa, try o.lowerType(ptr_ty));
	3388	}
	3389
	3390	var it = iterateParamTypes(o, fn_info);
	3391	while (try it.next()) \|lowering\| switch (lowering) {
	3392	.no_bits => continue,
	3393	.byval => {
	3394	const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
	3395	try llvm_params.append(o.gpa, try o.lowerType(param_ty));
	3396	},
	3397	.byref, .byref_mut => {
	3398	try llvm_params.append(o.gpa, .ptr);
	3399	},
	3400	.abi_sized_int => {
	3401	const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
	3402	try llvm_params.append(o.gpa, try o.builder.intType(
	3403	@intCast(param_ty.abiSize(mod) * 8),
	3404	));
	3405	},
	3406	.slice => {
	3407	try llvm_params.appendSlice(o.gpa, &.{ .ptr, try o.lowerType(Type.usize) });
	3408	},
	3409	.multiple_llvm_types => {
	3410	try llvm_params.appendSlice(o.gpa, it.types_buffer[0..it.types_len]);
	3411	},
	3412	.as_u16 => {
	3413	try llvm_params.append(o.gpa, .i16);
	3414	},
	3415	.float_array => \|count\| {
	3416	const param_ty = fn_info.param_types.get(ip)[it.zig_index - 1].toType();
	3417	const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, mod).?);
	3418	try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));
	3419	},
	3420	.i32_array, .i64_array => \|arr_len\| {
	3421	try llvm_params.append(o.gpa, try o.builder.arrayType(arr_len, switch (lowering) {
	3422	.i32_array => .i32,
	3423	.i64_array => .i64,
	3424	else => unreachable,
	3425	}));
	3426	},
	3427	};
	3428
	3429	return o.builder.fnType(
	3430	ret_ty,
	3431	llvm_params.items,
	3432	if (fn_info.is_var_args) .vararg else .normal,
	3433	);
	3434	}
	3435
3555	3436	fn lowerValue(o: Object, arg_tv: TypedValue) Error!llvm.Value {
3556	3437	const mod = o.module;
3557	3438	const gpa = o.gpa;
...	...	@@ -3562,8 +3443,7 @@ pub const Object = struct {
3562	3443	else => {},
3563	3444	}
3564	3445	if (tv.val.isUndefDeep(mod)) {
3565		const llvm_type = try o.lowerLlvmType(tv.ty);
3566		return llvm_type.getUndef();
	3446	return (try o.lowerType(tv.ty)).toLlvm(&o.builder).getUndef();
3567	3447	}
3568	3448
3569	3449	switch (mod.intern_pool.indexToKey(tv.val.toIntern())) {
...	...	@@ -3595,7 +3475,7 @@ pub const Object = struct {
3595	3475	.generic_poison,
3596	3476	=> unreachable, // non-runtime values
3597	3477	.false, .true => {
3598		const llvm_type = try o.lowerLlvmType(tv.ty);
	3478	const llvm_type = (try o.lowerType(tv.ty)).toLlvm(&o.builder);
3599	3479	return if (tv.val.toBool()) llvm_type.constAllOnes() else llvm_type.constNull();
3600	3480	},
3601	3481	},
...	...	@@ -3623,7 +3503,7 @@ pub const Object = struct {
3623	3503	return lowerBigInt(o, tv.ty, bigint);
3624	3504	},
3625	3505	.err => \|err\| {
3626		const llvm_ty = try o.lowerLlvmType(Type.anyerror);
	3506	const llvm_ty = Builder.Type.err_int.toLlvm(&o.builder);
3627	3507	const int = try mod.getErrorValue(err.name);
3628	3508	return llvm_ty.constInt(int, .False);
3629	3509	},
...	...	@@ -3659,7 +3539,7 @@ pub const Object = struct {
3659	3539	});
3660	3540	var fields_buf: [3]*llvm.Value = undefined;
3661	3541
3662		const llvm_ty = try o.lowerLlvmType(tv.ty);
	3542	const llvm_ty = (try o.lowerType(tv.ty)).toLlvm(&o.builder);
3663	3543	const llvm_field_count = llvm_ty.countStructElementTypes();
3664	3544	if (llvm_field_count > 2) {
3665	3545	assert(llvm_field_count == 3);
...	...	@@ -3703,7 +3583,7 @@ pub const Object = struct {
3703	3583	return unsigned_val;
3704	3584	},
3705	3585	.float => {
3706		const llvm_ty = try o.lowerLlvmType(tv.ty);
	3586	const llvm_ty = (try o.lowerType(tv.ty)).toLlvm(&o.builder);
3707	3587	switch (tv.ty.floatBits(target)) {
3708	3588	16 => {
3709	3589	const repr = @as(u16, @bitCast(tv.val.toFloat(f16, mod)));
...	...	@@ -3788,7 +3668,7 @@ pub const Object = struct {
3788	3668	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {
3789	3669	return non_null_bit;
3790	3670	}
3791		const llvm_ty = try o.lowerLlvmType(tv.ty);
	3671	const llvm_ty = (try o.lowerType(tv.ty)).toLlvm(&o.builder);
3792	3672	if (tv.ty.optionalReprIsPayload(mod)) return switch (opt.val) {
3793	3673	.none => llvm_ty.constNull(),
3794	3674	else => \|payload\| o.lowerValue(.{ .ty = payload_ty, .val = payload.toValue() }),
...	...	@@ -3834,7 +3714,7 @@ pub const Object = struct {
3834	3714	.True,
3835	3715	);
3836	3716	} else {
3837		const llvm_elem_ty = try o.lowerLlvmType(elem_ty);
	3717	const llvm_elem_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
3838	3718	return llvm_elem_ty.constArray(
3839	3719	llvm_elems.ptr,
3840	3720	@as(c_uint, @intCast(llvm_elems.len)),
...	...	@@ -3869,7 +3749,7 @@ pub const Object = struct {
3869	3749	.True,
3870	3750	);
3871	3751	} else {
3872		const llvm_elem_ty = try o.lowerLlvmType(elem_ty);
	3752	const llvm_elem_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
3873	3753	return llvm_elem_ty.constArray(
3874	3754	llvm_elems.ptr,
3875	3755	@as(c_uint, @intCast(llvm_elems.len)),
...	...	@@ -3956,7 +3836,7 @@ pub const Object = struct {
3956	3836	.False,
3957	3837	);
3958	3838	} else {
3959		const llvm_struct_ty = try o.lowerLlvmType(tv.ty);
	3839	const llvm_struct_ty = (try o.lowerType(tv.ty)).toLlvm(&o.builder);
3960	3840	return llvm_struct_ty.constNamedStruct(
3961	3841	llvm_fields.items.ptr,
3962	3842	@as(c_uint, @intCast(llvm_fields.items.len)),
...	...	@@ -3965,7 +3845,7 @@ pub const Object = struct {
3965	3845	},
3966	3846	.struct_type => \|struct_type\| {
3967	3847	const struct_obj = mod.structPtrUnwrap(struct_type.index).?;
3968		const llvm_struct_ty = try o.lowerLlvmType(tv.ty);
	3848	const llvm_struct_ty = (try o.lowerType(tv.ty)).toLlvm(&o.builder);
3969	3849
3970	3850	if (struct_obj.layout == .Packed) {
3971	3851	assert(struct_obj.haveLayout());
...	...	@@ -4062,7 +3942,7 @@ pub const Object = struct {
4062	3942	else => unreachable,
4063	3943	},
4064	3944	.un => {
4065		const llvm_union_ty = try o.lowerLlvmType(tv.ty);
	3945	const llvm_union_ty = (try o.lowerType(tv.ty)).toLlvm(&o.builder);
4066	3946	const tag_and_val: Value.Payload.Union.Data = switch (tv.val.toIntern()) {
4067	3947	.none => tv.val.castTag(.@"union").?.data,
4068	3948	else => switch (mod.intern_pool.indexToKey(tv.val.toIntern())) {
...	...	@@ -4232,7 +4112,7 @@ pub const Object = struct {
4232	4112	llvm_u32.constInt(0, .False),
4233	4113	llvm_u32.constInt(payload_offset, .False),
4234	4114	};
4235		const eu_llvm_ty = try o.lowerLlvmType(eu_ty);
	4115	const eu_llvm_ty = (try o.lowerType(eu_ty)).toLlvm(&o.builder);
4236	4116	return eu_llvm_ty.constInBoundsGEP(parent_llvm_ptr, &indices, indices.len);
4237	4117	},
4238	4118	.opt_payload => \|opt_ptr\| {
...	...	@@ -4253,19 +4133,19 @@ pub const Object = struct {
4253	4133	llvm_u32.constInt(0, .False),
4254	4134	llvm_u32.constInt(0, .False),
4255	4135	};
4256		const opt_llvm_ty = try o.lowerLlvmType(opt_ty);
	4136	const opt_llvm_ty = (try o.lowerType(opt_ty)).toLlvm(&o.builder);
4257	4137	return opt_llvm_ty.constInBoundsGEP(parent_llvm_ptr, &indices, indices.len);
4258	4138	},
4259	4139	.comptime_field => unreachable,
4260	4140	.elem => \|elem_ptr\| {
4261	4141	const parent_llvm_ptr = try o.lowerParentPtr(elem_ptr.base.toValue(), true);
4262	4142
4263		const llvm_usize = try o.lowerLlvmType(Type.usize);
	4143	const llvm_usize = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
4264	4144	const indices: [1]*llvm.Value = .{
4265	4145	llvm_usize.constInt(elem_ptr.index, .False),
4266	4146	};
4267	4147	const elem_ty = mod.intern_pool.typeOf(elem_ptr.base).toType().elemType2(mod);
4268		const elem_llvm_ty = try o.lowerLlvmType(elem_ty);
	4148	const elem_llvm_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
4269	4149	return elem_llvm_ty.constInBoundsGEP(parent_llvm_ptr, &indices, indices.len);
4270	4150	},
4271	4151	.field => \|field_ptr\| {
...	...	@@ -4294,7 +4174,7 @@ pub const Object = struct {
4294	4174	llvm_u32.constInt(0, .False),
4295	4175	llvm_u32.constInt(llvm_pl_index, .False),
4296	4176	};
4297		const parent_llvm_ty = try o.lowerLlvmType(parent_ty);
	4177	const parent_llvm_ty = (try o.lowerType(parent_ty)).toLlvm(&o.builder);
4298	4178	return parent_llvm_ty.constInBoundsGEP(parent_llvm_ptr, &indices, indices.len);
4299	4179	},
4300	4180	.Struct => {
...	...	@@ -4317,7 +4197,7 @@ pub const Object = struct {
4317	4197	return field_addr.constIntToPtr(final_llvm_ty);
4318	4198	}
4319	4199
4320		const parent_llvm_ty = try o.lowerLlvmType(parent_ty);
	4200	const parent_llvm_ty = (try o.lowerType(parent_ty)).toLlvm(&o.builder);
4321	4201	if (llvmField(parent_ty, field_index, mod)) \|llvm_field\| {
4322	4202	const indices: [2]*llvm.Value = .{
4323	4203	llvm_u32.constInt(0, .False),
...	...	@@ -4336,7 +4216,7 @@ pub const Object = struct {
4336	4216	llvm_u32.constInt(0, .False),
4337	4217	llvm_u32.constInt(field_index, .False),
4338	4218	};
4339		const parent_llvm_ty = try o.lowerLlvmType(parent_ty);
	4219	const parent_llvm_ty = (try o.lowerType(parent_ty)).toLlvm(&o.builder);
4340	4220	return parent_llvm_ty.constInBoundsGEP(parent_llvm_ptr, &indices, indices.len);
4341	4221	},
4342	4222	else => unreachable,
...	...	@@ -4386,11 +4266,11 @@ pub const Object = struct {
4386	4266	const llvm_wanted_addrspace = toLlvmAddressSpace(decl.@"addrspace", target);
4387	4267	const llvm_actual_addrspace = toLlvmGlobalAddressSpace(decl.@"addrspace", target);
4388	4268	const llvm_val = if (llvm_wanted_addrspace != llvm_actual_addrspace) blk: {
4389		const llvm_decl_wanted_ptr_ty = o.context.pointerType(llvm_wanted_addrspace);
	4269	const llvm_decl_wanted_ptr_ty = o.context.pointerType(@intFromEnum(llvm_wanted_addrspace));
4390	4270	break :blk llvm_decl_val.constAddrSpaceCast(llvm_decl_wanted_ptr_ty);
4391	4271	} else llvm_decl_val;
4392	4272
4393		const llvm_type = try o.lowerLlvmType(tv.ty);
	4273	const llvm_type = (try o.lowerType(tv.ty)).toLlvm(&o.builder);
4394	4274	if (tv.ty.zigTypeTag(mod) == .Int) {
4395	4275	return llvm_val.constPtrToInt(llvm_type);
4396	4276	} else {
...	...	@@ -4405,8 +4285,8 @@ pub const Object = struct {
4405	4285	// The value cannot be undefined, because we use the `nonnull` annotation
4406	4286	// for non-optional pointers. We also need to respect the alignment, even though
4407	4287	// the address will never be dereferenced.
4408		const llvm_usize = try o.lowerLlvmType(Type.usize);
4409		const llvm_ptr_ty = try o.lowerLlvmType(ptr_ty);
	4288	const llvm_usize = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
	4289	const llvm_ptr_ty = (try o.lowerType(ptr_ty)).toLlvm(&o.builder);
4410	4290	if (ptr_ty.ptrInfo(mod).flags.alignment.toByteUnitsOptional()) \|alignment\| {
4411	4291	return llvm_usize.constInt(alignment, .False).constIntToPtr(llvm_ptr_ty);
4412	4292	}
...	...	@@ -4492,23 +4372,23 @@ pub const Object = struct {
4492	4372	/// widen it before using it and then truncate the result.
4493	4373	/// RMW exchange of floating-point values is bitcasted to same-sized integer
4494	4374	/// types to work around a LLVM deficiency when targeting ARM/AArch64.
4495		fn getAtomicAbiType(o: Object, ty: Type, is_rmw_xchg: bool) ?llvm.Type {
	4375	fn getAtomicAbiType(o: *Object, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type {
4496	4376	const mod = o.module;
4497	4377	const int_ty = switch (ty.zigTypeTag(mod)) {
4498	4378	.Int => ty,
4499	4379	.Enum => ty.intTagType(mod),
4500	4380	.Float => {
4501		if (!is_rmw_xchg) return null;
4502		return o.context.intType(@as(c_uint, @intCast(ty.abiSize(mod) * 8)));
	4381	if (!is_rmw_xchg) return .none;
	4382	return o.builder.intType(@intCast(ty.abiSize(mod) * 8));
4503	4383	},
4504		.Bool => return o.context.intType(8),
4505		else => return null,
	4384	.Bool => return .i8,
	4385	else => return .none,
4506	4386	};
4507	4387	const bit_count = int_ty.intInfo(mod).bits;
4508	4388	if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) {
4509		return o.context.intType(@as(c_uint, @intCast(int_ty.abiSize(mod) * 8)));
	4389	return o.builder.intType(@intCast(int_ty.abiSize(mod) * 8));
4510	4390	} else {
4511		return null;
	4391	return .none;
4512	4392	}
4513	4393	}
4514	4394
...	...	@@ -4549,13 +4429,13 @@ pub const Object = struct {
4549	4429	llvm_arg_i: u32,
4550	4430	alignment: u32,
4551	4431	byval_attr: bool,
4552		param_llvm_ty: *llvm.Type,
	4432	param_llvm_ty: Builder.Type,
4553	4433	) void {
4554	4434	o.addArgAttr(llvm_fn, llvm_arg_i, "nonnull");
4555	4435	o.addArgAttr(llvm_fn, llvm_arg_i, "readonly");
4556	4436	o.addArgAttrInt(llvm_fn, llvm_arg_i, "align", alignment);
4557	4437	if (byval_attr) {
4558		llvm_fn.addByValAttr(llvm_arg_i, param_llvm_ty);
	4438	llvm_fn.addByValAttr(llvm_arg_i, param_llvm_ty.toLlvm(&o.builder));
4559	4439	}
4560	4440	}
4561	4441	};
...	...	@@ -4626,7 +4506,7 @@ pub const DeclGen = struct {
4626	4506	const new_global = o.llvm_module.addGlobalInAddressSpace(
4627	4507	llvm_init.typeOf(),
4628	4508	"",
4629		llvm_global_addrspace,
	4509	@intFromEnum(llvm_global_addrspace),
4630	4510	);
4631	4511	new_global.setLinkage(llvm_global.getLinkage());
4632	4512	new_global.setUnnamedAddr(llvm_global.getUnnamedAddress());
...	...	@@ -4761,14 +4641,14 @@ pub const FuncGen = struct {
4761	4641	const target = mod.getTarget();
4762	4642	const llvm_wanted_addrspace = toLlvmAddressSpace(.generic, target);
4763	4643	const llvm_actual_addrspace = toLlvmGlobalAddressSpace(.generic, target);
4764		const global = o.llvm_module.addGlobalInAddressSpace(llvm_val.typeOf(), "", llvm_actual_addrspace);
	4644	const global = o.llvm_module.addGlobalInAddressSpace(llvm_val.typeOf(), "", @intFromEnum(llvm_actual_addrspace));
4765	4645	global.setInitializer(llvm_val);
4766	4646	global.setLinkage(.Private);
4767	4647	global.setGlobalConstant(.True);
4768	4648	global.setUnnamedAddr(.True);
4769	4649	global.setAlignment(tv.ty.abiAlignment(mod));
4770	4650	const addrspace_casted_ptr = if (llvm_actual_addrspace != llvm_wanted_addrspace)
4771		global.constAddrSpaceCast(self.context.pointerType(llvm_wanted_addrspace))
	4651	global.constAddrSpaceCast(self.context.pointerType(@intFromEnum(llvm_wanted_addrspace)))
4772	4652	else
4773	4653	global;
4774	4654	return addrspace_casted_ptr;
...	...	@@ -5053,7 +4933,7 @@ pub const FuncGen = struct {
5053	4933	defer llvm_args.deinit();
5054	4934
5055	4935	const ret_ptr = if (!sret) null else blk: {
5056		const llvm_ret_ty = try o.lowerLlvmType(return_type);
	4936	const llvm_ret_ty = (try o.lowerType(return_type)).toLlvm(&o.builder);
5057	4937	const ret_ptr = self.buildAlloca(llvm_ret_ty, return_type.abiAlignment(mod));
5058	4938	try llvm_args.append(ret_ptr);
5059	4939	break :blk ret_ptr;
...	...	@@ -5066,13 +4946,13 @@ pub const FuncGen = struct {
5066	4946	}
5067	4947
5068	4948	var it = iterateParamTypes(o, fn_info);
5069		while (it.nextCall(self, args)) \|lowering\| switch (lowering) {
	4949	while (try it.nextCall(self, args)) \|lowering\| switch (lowering) {
5070	4950	.no_bits => continue,
5071	4951	.byval => {
5072	4952	const arg = args[it.zig_index - 1];
5073	4953	const param_ty = self.typeOf(arg);
5074	4954	const llvm_arg = try self.resolveInst(arg);
5075		const llvm_param_ty = try o.lowerLlvmType(param_ty);
	4955	const llvm_param_ty = (try o.lowerType(param_ty)).toLlvm(&o.builder);
5076	4956	if (isByRef(param_ty, mod)) {
5077	4957	const alignment = param_ty.abiAlignment(mod);
5078	4958	const load_inst = self.builder.buildLoad(llvm_param_ty, llvm_arg, "");
...	...	@@ -5103,7 +4983,7 @@ pub const FuncGen = struct {
5103	4983	const llvm_arg = try self.resolveInst(arg);
5104	4984
5105	4985	const alignment = param_ty.abiAlignment(mod);
5106		const param_llvm_ty = try o.lowerLlvmType(param_ty);
	4986	const param_llvm_ty = (try o.lowerType(param_ty)).toLlvm(&o.builder);
5107	4987	const arg_ptr = self.buildAlloca(param_llvm_ty, alignment);
5108	4988	if (isByRef(param_ty, mod)) {
5109	4989	const load_inst = self.builder.buildLoad(param_llvm_ty, llvm_arg, "");
...	...	@@ -5157,7 +5037,7 @@ pub const FuncGen = struct {
5157	5037	.multiple_llvm_types => {
5158	5038	const arg = args[it.zig_index - 1];
5159	5039	const param_ty = self.typeOf(arg);
5160		const llvm_types = it.llvm_types_buffer[0..it.llvm_types_len];
	5040	const llvm_types = it.llvm_types_buffer[0..it.types_len];
5161	5041	const llvm_arg = try self.resolveInst(arg);
5162	5042	const is_by_ref = isByRef(param_ty, mod);
5163	5043	const arg_ptr = if (is_by_ref) llvm_arg else p: {
...	...	@@ -5168,7 +5048,7 @@ pub const FuncGen = struct {
5168	5048	};
5169	5049
5170	5050	const llvm_ty = self.context.structType(llvm_types.ptr, @as(c_uint, @intCast(llvm_types.len)), .False);
5171		try llvm_args.ensureUnusedCapacity(it.llvm_types_len);
	5051	try llvm_args.ensureUnusedCapacity(it.types_len);
5172	5052	for (llvm_types, 0..) \|field_ty, i_usize\| {
5173	5053	const i = @as(c_uint, @intCast(i_usize));
5174	5054	const field_ptr = self.builder.buildStructGEP(llvm_ty, arg_ptr, i, "");
...	...	@@ -5194,7 +5074,7 @@ pub const FuncGen = struct {
5194	5074	llvm_arg = store_inst;
5195	5075	}
5196	5076
5197		const float_ty = try o.lowerLlvmType(aarch64_c_abi.getFloatArrayType(arg_ty, mod).?);
	5077	const float_ty = (try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, mod).?)).toLlvm(&o.builder);
5198	5078	const array_llvm_ty = float_ty.arrayType(count);
5199	5079
5200	5080	const alignment = arg_ty.abiAlignment(mod);
...	...	@@ -5223,7 +5103,7 @@ pub const FuncGen = struct {
5223	5103	};
5224	5104
5225	5105	const call = self.builder.buildCall(
5226		try o.lowerLlvmType(zig_fn_ty),
	5106	(try o.lowerType(zig_fn_ty)).toLlvm(&o.builder),
5227	5107	llvm_fn,
5228	5108	llvm_args.items.ptr,
5229	5109	@as(c_uint, @intCast(llvm_args.items.len)),
...	...	@@ -5237,7 +5117,7 @@ pub const FuncGen = struct {
5237	5117	it = iterateParamTypes(o, fn_info);
5238	5118	it.llvm_index += @intFromBool(sret);
5239	5119	it.llvm_index += @intFromBool(err_return_tracing);
5240		while (it.next()) \|lowering\| switch (lowering) {
	5120	while (try it.next()) \|lowering\| switch (lowering) {
5241	5121	.byval => {
5242	5122	const param_index = it.zig_index - 1;
5243	5123	const param_ty = fn_info.param_types.get(ip)[param_index].toType();
...	...	@@ -5248,7 +5128,7 @@ pub const FuncGen = struct {
5248	5128	.byref => {
5249	5129	const param_index = it.zig_index - 1;
5250	5130	const param_ty = fn_info.param_types.get(ip)[param_index].toType();
5251		const param_llvm_ty = try o.lowerLlvmType(param_ty);
	5131	const param_llvm_ty = try o.lowerType(param_ty);
5252	5132	const alignment = param_ty.abiAlignment(mod);
5253	5133	o.addByRefParamAttrs(call, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
5254	5134	},
...	...	@@ -5297,7 +5177,7 @@ pub const FuncGen = struct {
5297	5177	return null;
5298	5178	}
5299	5179
5300		const llvm_ret_ty = try o.lowerLlvmType(return_type);
	5180	const llvm_ret_ty = (try o.lowerType(return_type)).toLlvm(&o.builder);
5301	5181
5302	5182	if (ret_ptr) \|rp\| {
5303	5183	call.setCallSret(llvm_ret_ty);
...	...	@@ -5311,7 +5191,7 @@ pub const FuncGen = struct {
5311	5191	}
5312	5192	}
5313	5193
5314		const abi_ret_ty = try lowerFnRetTy(o, fn_info);
	5194	const abi_ret_ty = (try lowerFnRetTy(o, fn_info)).toLlvm(&o.builder);
5315	5195
5316	5196	if (abi_ret_ty != llvm_ret_ty) {
5317	5197	// In this case the function return type is honoring the calling convention by having
...	...	@@ -5374,7 +5254,7 @@ pub const FuncGen = struct {
5374	5254	const fn_info = mod.typeToFunc(panic_decl.ty).?;
5375	5255	const panic_global = try o.resolveLlvmFunction(panic_func.owner_decl);
5376	5256	_ = fg.builder.buildCall(
5377		try o.lowerLlvmType(panic_decl.ty),
	5257	(try o.lowerType(panic_decl.ty)).toLlvm(&o.builder),
5378	5258	panic_global.toLlvm(&o.builder),
5379	5259	&args,
5380	5260	args.len,
...	...	@@ -5403,7 +5283,7 @@ pub const FuncGen = struct {
5403	5283	// Functions with an empty error set are emitted with an error code
5404	5284	// return type and return zero so they can be function pointers coerced
5405	5285	// to functions that return anyerror.
5406		const err_int = try o.lowerLlvmType(Type.anyerror);
	5286	const err_int = (try o.lowerType(Type.anyerror)).toLlvm(&o.builder);
5407	5287	_ = self.builder.buildRet(err_int.constInt(0, .False));
5408	5288	} else {
5409	5289	_ = self.builder.buildRetVoid();
...	...	@@ -5411,7 +5291,7 @@ pub const FuncGen = struct {
5411	5291	return null;
5412	5292	}
5413	5293
5414		const abi_ret_ty = try lowerFnRetTy(o, fn_info);
	5294	const abi_ret_ty = (try lowerFnRetTy(o, fn_info)).toLlvm(&o.builder);
5415	5295	const operand = try self.resolveInst(un_op);
5416	5296	const alignment = ret_ty.abiAlignment(mod);
5417	5297
...	...	@@ -5451,7 +5331,7 @@ pub const FuncGen = struct {
5451	5331	// Functions with an empty error set are emitted with an error code
5452	5332	// return type and return zero so they can be function pointers coerced
5453	5333	// to functions that return anyerror.
5454		const err_int = try o.lowerLlvmType(Type.anyerror);
	5334	const err_int = (try o.lowerType(Type.anyerror)).toLlvm(&o.builder);
5455	5335	_ = self.builder.buildRet(err_int.constInt(0, .False));
5456	5336	} else {
5457	5337	_ = self.builder.buildRetVoid();
...	...	@@ -5463,7 +5343,7 @@ pub const FuncGen = struct {
5463	5343	return null;
5464	5344	}
5465	5345	const ptr = try self.resolveInst(un_op);
5466		const abi_ret_ty = try lowerFnRetTy(o, fn_info);
	5346	const abi_ret_ty = (try lowerFnRetTy(o, fn_info)).toLlvm(&o.builder);
5467	5347	const loaded = self.builder.buildLoad(abi_ret_ty, ptr, "");
5468	5348	loaded.setAlignment(ret_ty.abiAlignment(mod));
5469	5349	_ = self.builder.buildRet(loaded);
...	...	@@ -5475,7 +5355,7 @@ pub const FuncGen = struct {
5475	5355	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
5476	5356	const list = try self.resolveInst(ty_op.operand);
5477	5357	const arg_ty = self.air.getRefType(ty_op.ty);
5478		const llvm_arg_ty = try o.lowerLlvmType(arg_ty);
	5358	const llvm_arg_ty = (try o.lowerType(arg_ty)).toLlvm(&o.builder);
5479	5359
5480	5360	return self.builder.buildVAArg(list, llvm_arg_ty, "");
5481	5361	}
...	...	@@ -5485,7 +5365,7 @@ pub const FuncGen = struct {
5485	5365	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
5486	5366	const src_list = try self.resolveInst(ty_op.operand);
5487	5367	const va_list_ty = self.air.getRefType(ty_op.ty);
5488		const llvm_va_list_ty = try o.lowerLlvmType(va_list_ty);
	5368	const llvm_va_list_ty = (try o.lowerType(va_list_ty)).toLlvm(&o.builder);
5489	5369	const mod = o.module;
5490	5370
5491	5371	const result_alignment = va_list_ty.abiAlignment(mod);
...	...	@@ -5533,7 +5413,7 @@ pub const FuncGen = struct {
5533	5413	const o = self.dg.object;
5534	5414	const mod = o.module;
5535	5415	const va_list_ty = self.typeOfIndex(inst);
5536		const llvm_va_list_ty = try o.lowerLlvmType(va_list_ty);
	5416	const llvm_va_list_ty = (try o.lowerType(va_list_ty)).toLlvm(&o.builder);
5537	5417
5538	5418	const result_alignment = va_list_ty.abiAlignment(mod);
5539	5419	const list = self.buildAlloca(llvm_va_list_ty, result_alignment);
...	...	@@ -5612,7 +5492,7 @@ pub const FuncGen = struct {
5612	5492	// We need to emit instructions to check for equality/inequality
5613	5493	// of optionals that are not pointers.
5614	5494	const is_by_ref = isByRef(scalar_ty, mod);
5615		const opt_llvm_ty = try o.lowerLlvmType(scalar_ty);
	5495	const opt_llvm_ty = (try o.lowerType(scalar_ty)).toLlvm(&o.builder);
5616	5496	const lhs_non_null = self.optIsNonNull(opt_llvm_ty, lhs, is_by_ref);
5617	5497	const rhs_non_null = self.optIsNonNull(opt_llvm_ty, rhs, is_by_ref);
5618	5498	const llvm_i2 = self.context.intType(2);
...	...	@@ -5722,7 +5602,7 @@ pub const FuncGen = struct {
5722	5602	const is_body = inst_ty.zigTypeTag(mod) == .Fn;
5723	5603	if (!is_body and !inst_ty.hasRuntimeBitsIgnoreComptime(mod)) return null;
5724	5604
5725		const raw_llvm_ty = try o.lowerLlvmType(inst_ty);
	5605	const raw_llvm_ty = (try o.lowerType(inst_ty)).toLlvm(&o.builder);
5726	5606
5727	5607	const llvm_ty = ty: {
5728	5608	// If the zig tag type is a function, this represents an actual function body; not
...	...	@@ -5827,11 +5707,11 @@ pub const FuncGen = struct {
5827	5707	const mod = o.module;
5828	5708	const payload_ty = err_union_ty.errorUnionPayload(mod);
5829	5709	const payload_has_bits = payload_ty.hasRuntimeBitsIgnoreComptime(mod);
5830		const err_union_llvm_ty = try o.lowerLlvmType(err_union_ty);
	5710	const err_union_llvm_ty = (try o.lowerType(err_union_ty)).toLlvm(&o.builder);
5831	5711
5832	5712	if (!err_union_ty.errorUnionSet(mod).errorSetIsEmpty(mod)) {
5833	5713	const is_err = err: {
5834		const err_set_ty = try o.lowerLlvmType(Type.anyerror);
	5714	const err_set_ty = (try o.lowerType(Type.anyerror)).toLlvm(&o.builder);
5835	5715	const zero = err_set_ty.constNull();
5836	5716	if (!payload_has_bits) {
5837	5717	// TODO add alignment to this load
...	...	@@ -5966,9 +5846,9 @@ pub const FuncGen = struct {
5966	5846	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
5967	5847	const operand_ty = self.typeOf(ty_op.operand);
5968	5848	const array_ty = operand_ty.childType(mod);
5969		const llvm_usize = try o.lowerLlvmType(Type.usize);
	5849	const llvm_usize = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
5970	5850	const len = llvm_usize.constInt(array_ty.arrayLen(mod), .False);
5971		const slice_llvm_ty = try o.lowerLlvmType(self.typeOfIndex(inst));
	5851	const slice_llvm_ty = (try o.lowerType(self.typeOfIndex(inst))).toLlvm(&o.builder);
5972	5852	const operand = try self.resolveInst(ty_op.operand);
5973	5853	if (!array_ty.hasRuntimeBitsIgnoreComptime(mod)) {
5974	5854	const partial = self.builder.buildInsertValue(slice_llvm_ty.getUndef(), operand, 0, "");
...	...	@@ -5977,7 +5857,7 @@ pub const FuncGen = struct {
5977	5857	const indices: [2]*llvm.Value = .{
5978	5858	llvm_usize.constNull(), llvm_usize.constNull(),
5979	5859	};
5980		const array_llvm_ty = try o.lowerLlvmType(array_ty);
	5860	const array_llvm_ty = (try o.lowerType(array_ty)).toLlvm(&o.builder);
5981	5861	const ptr = self.builder.buildInBoundsGEP(array_llvm_ty, operand, &indices, indices.len, "");
5982	5862	const partial = self.builder.buildInsertValue(slice_llvm_ty.getUndef(), ptr, 0, "");
5983	5863	return self.builder.buildInsertValue(partial, len, 1, "");
...	...	@@ -5994,7 +5874,7 @@ pub const FuncGen = struct {
5994	5874
5995	5875	const dest_ty = self.typeOfIndex(inst);
5996	5876	const dest_scalar_ty = dest_ty.scalarType(mod);
5997		const dest_llvm_ty = try o.lowerLlvmType(dest_ty);
	5877	const dest_llvm_ty = (try o.lowerType(dest_ty)).toLlvm(&o.builder);
5998	5878	const target = mod.getTarget();
5999	5879
6000	5880	if (intrinsicsAllowed(dest_scalar_ty, target)) {
...	...	@@ -6055,7 +5935,7 @@ pub const FuncGen = struct {
6055	5935
6056	5936	const dest_ty = self.typeOfIndex(inst);
6057	5937	const dest_scalar_ty = dest_ty.scalarType(mod);
6058		const dest_llvm_ty = try o.lowerLlvmType(dest_ty);
	5938	const dest_llvm_ty = (try o.lowerType(dest_ty)).toLlvm(&o.builder);
6059	5939
6060	5940	if (intrinsicsAllowed(operand_scalar_ty, target)) {
6061	5941	// TODO set fast math flag
...	...	@@ -6087,7 +5967,7 @@ pub const FuncGen = struct {
6087	5967	compiler_rt_dest_abbrev,
6088	5968	}) catch unreachable;
6089	5969
6090		const operand_llvm_ty = try o.lowerLlvmType(operand_ty);
	5970	const operand_llvm_ty = (try o.lowerType(operand_ty)).toLlvm(&o.builder);
6091	5971	const param_types = [1]*llvm.Type{operand_llvm_ty};
6092	5972	const libc_fn = try self.getLibcFunction(fn_name, &param_types, libc_ret_ty);
6093	5973	const params = [1]*llvm.Value{operand};
...	...	@@ -6145,7 +6025,7 @@ pub const FuncGen = struct {
6145	6025	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
6146	6026	const slice_ptr = try self.resolveInst(ty_op.operand);
6147	6027	const slice_ptr_ty = self.typeOf(ty_op.operand);
6148		const slice_llvm_ty = try o.lowerPtrElemTy(slice_ptr_ty.childType(mod));
	6028	const slice_llvm_ty = (try o.lowerPtrElemTy(slice_ptr_ty.childType(mod))).toLlvm(&o.builder);
6149	6029
6150	6030	return self.builder.buildStructGEP(slice_llvm_ty, slice_ptr, index, "");
6151	6031	}
...	...	@@ -6159,7 +6039,7 @@ pub const FuncGen = struct {
6159	6039	const slice = try self.resolveInst(bin_op.lhs);
6160	6040	const index = try self.resolveInst(bin_op.rhs);
6161	6041	const elem_ty = slice_ty.childType(mod);
6162		const llvm_elem_ty = try o.lowerPtrElemTy(elem_ty);
	6042	const llvm_elem_ty = (try o.lowerPtrElemTy(elem_ty)).toLlvm(&o.builder);
6163	6043	const base_ptr = self.builder.buildExtractValue(slice, 0, "");
6164	6044	const indices: [1]*llvm.Value = .{index};
6165	6045	const ptr = self.builder.buildInBoundsGEP(llvm_elem_ty, base_ptr, &indices, indices.len, "");
...	...	@@ -6182,7 +6062,7 @@ pub const FuncGen = struct {
6182	6062
6183	6063	const slice = try self.resolveInst(bin_op.lhs);
6184	6064	const index = try self.resolveInst(bin_op.rhs);
6185		const llvm_elem_ty = try o.lowerPtrElemTy(slice_ty.childType(mod));
	6065	const llvm_elem_ty = (try o.lowerPtrElemTy(slice_ty.childType(mod))).toLlvm(&o.builder);
6186	6066	const base_ptr = self.builder.buildExtractValue(slice, 0, "");
6187	6067	const indices: [1]*llvm.Value = .{index};
6188	6068	return self.builder.buildInBoundsGEP(llvm_elem_ty, base_ptr, &indices, indices.len, "");
...	...	@@ -6197,7 +6077,7 @@ pub const FuncGen = struct {
6197	6077	const array_ty = self.typeOf(bin_op.lhs);
6198	6078	const array_llvm_val = try self.resolveInst(bin_op.lhs);
6199	6079	const rhs = try self.resolveInst(bin_op.rhs);
6200		const array_llvm_ty = try o.lowerLlvmType(array_ty);
	6080	const array_llvm_ty = (try o.lowerType(array_ty)).toLlvm(&o.builder);
6201	6081	const elem_ty = array_ty.childType(mod);
6202	6082	if (isByRef(array_ty, mod)) {
6203	6083	const indices: [2]*llvm.Value = .{ self.context.intType(32).constNull(), rhs };
...	...	@@ -6208,7 +6088,7 @@ pub const FuncGen = struct {
6208	6088
6209	6089	return self.loadByRef(elem_ptr, elem_ty, elem_ty.abiAlignment(mod), false);
6210	6090	} else {
6211		const elem_llvm_ty = try o.lowerLlvmType(elem_ty);
	6091	const elem_llvm_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
6212	6092	if (Air.refToIndex(bin_op.lhs)) \|lhs_index\| {
6213	6093	if (self.air.instructions.items(.tag)[lhs_index] == .load) {
6214	6094	const load_data = self.air.instructions.items(.data)[lhs_index];
...	...	@@ -6242,7 +6122,7 @@ pub const FuncGen = struct {
6242	6122	const bin_op = self.air.instructions.items(.data)[inst].bin_op;
6243	6123	const ptr_ty = self.typeOf(bin_op.lhs);
6244	6124	const elem_ty = ptr_ty.childType(mod);
6245		const llvm_elem_ty = try o.lowerPtrElemTy(elem_ty);
	6125	const llvm_elem_ty = (try o.lowerPtrElemTy(elem_ty)).toLlvm(&o.builder);
6246	6126	const base_ptr = try self.resolveInst(bin_op.lhs);
6247	6127	const rhs = try self.resolveInst(bin_op.rhs);
6248	6128	// TODO: when we go fully opaque pointers in LLVM 16 we can remove this branch
...	...	@@ -6279,7 +6159,7 @@ pub const FuncGen = struct {
6279	6159	const elem_ptr = self.air.getRefType(ty_pl.ty);
6280	6160	if (elem_ptr.ptrInfo(mod).flags.vector_index != .none) return base_ptr;
6281	6161
6282		const llvm_elem_ty = try o.lowerPtrElemTy(elem_ty);
	6162	const llvm_elem_ty = (try o.lowerPtrElemTy(elem_ty)).toLlvm(&o.builder);
6283	6163	if (ptr_ty.isSinglePointer(mod)) {
6284	6164	// If this is a single-item pointer to an array, we need another index in the GEP.
6285	6165	const indices: [2]*llvm.Value = .{ self.context.intType(32).constNull(), rhs };
...	...	@@ -6333,7 +6213,7 @@ pub const FuncGen = struct {
6333	6213	const containing_int = struct_llvm_val;
6334	6214	const shift_amt = containing_int.typeOf().constInt(bit_offset, .False);
6335	6215	const shifted_value = self.builder.buildLShr(containing_int, shift_amt, "");
6336		const elem_llvm_ty = try o.lowerLlvmType(field_ty);
	6216	const elem_llvm_ty = (try o.lowerType(field_ty)).toLlvm(&o.builder);
6337	6217	if (field_ty.zigTypeTag(mod) == .Float or field_ty.zigTypeTag(mod) == .Vector) {
6338	6218	const elem_bits = @as(c_uint, @intCast(field_ty.bitSize(mod)));
6339	6219	const same_size_int = self.context.intType(elem_bits);
...	...	@@ -6355,7 +6235,7 @@ pub const FuncGen = struct {
6355	6235	.Union => {
6356	6236	assert(struct_ty.containerLayout(mod) == .Packed);
6357	6237	const containing_int = struct_llvm_val;
6358		const elem_llvm_ty = try o.lowerLlvmType(field_ty);
	6238	const elem_llvm_ty = (try o.lowerType(field_ty)).toLlvm(&o.builder);
6359	6239	if (field_ty.zigTypeTag(mod) == .Float or field_ty.zigTypeTag(mod) == .Vector) {
6360	6240	const elem_bits = @as(c_uint, @intCast(field_ty.bitSize(mod)));
6361	6241	const same_size_int = self.context.intType(elem_bits);
...	...	@@ -6377,7 +6257,7 @@ pub const FuncGen = struct {
6377	6257	.Struct => {
6378	6258	assert(struct_ty.containerLayout(mod) != .Packed);
6379	6259	const llvm_field = llvmField(struct_ty, field_index, mod).?;
6380		const struct_llvm_ty = try o.lowerLlvmType(struct_ty);
	6260	const struct_llvm_ty = (try o.lowerType(struct_ty)).toLlvm(&o.builder);
6381	6261	const field_ptr = self.builder.buildStructGEP(struct_llvm_ty, struct_llvm_val, llvm_field.index, "");
6382	6262	const field_ptr_ty = try mod.ptrType(.{
6383	6263	.child = llvm_field.ty.toIntern(),
...	...	@@ -6396,11 +6276,11 @@ pub const FuncGen = struct {
6396	6276	}
6397	6277	},
6398	6278	.Union => {
6399		const union_llvm_ty = try o.lowerLlvmType(struct_ty);
	6279	const union_llvm_ty = (try o.lowerType(struct_ty)).toLlvm(&o.builder);
6400	6280	const layout = struct_ty.unionGetLayout(mod);
6401	6281	const payload_index = @intFromBool(layout.tag_align >= layout.payload_align);
6402	6282	const field_ptr = self.builder.buildStructGEP(union_llvm_ty, struct_llvm_val, payload_index, "");
6403		const llvm_field_ty = try o.lowerLlvmType(field_ty);
	6283	const llvm_field_ty = (try o.lowerType(field_ty)).toLlvm(&o.builder);
6404	6284	if (isByRef(field_ty, mod)) {
6405	6285	if (canElideLoad(self, body_tail))
6406	6286	return field_ptr;
...	...	@@ -6426,7 +6306,7 @@ pub const FuncGen = struct {
6426	6306	const parent_ty = self.air.getRefType(ty_pl.ty).childType(mod);
6427	6307	const field_offset = parent_ty.structFieldOffset(extra.field_index, mod);
6428	6308
6429		const res_ty = try o.lowerLlvmType(self.air.getRefType(ty_pl.ty));
	6309	const res_ty = (try o.lowerType(self.air.getRefType(ty_pl.ty))).toLlvm(&o.builder);
6430	6310	if (field_offset == 0) {
6431	6311	return field_ptr;
6432	6312	}
...	...	@@ -6691,7 +6571,7 @@ pub const FuncGen = struct {
6691	6571	const output_inst = try self.resolveInst(output);
6692	6572	const output_ty = self.typeOf(output);
6693	6573	assert(output_ty.zigTypeTag(mod) == .Pointer);
6694		const elem_llvm_ty = try o.lowerPtrElemTy(output_ty.childType(mod));
	6574	const elem_llvm_ty = (try o.lowerPtrElemTy(output_ty.childType(mod))).toLlvm(&o.builder);
6695	6575
6696	6576	if (llvm_ret_indirect[i]) {
6697	6577	// Pass the result by reference as an indirect output (e.g. "=*m")
...	...	@@ -6708,7 +6588,7 @@ pub const FuncGen = struct {
6708	6588	}
6709	6589	} else {
6710	6590	const ret_ty = self.typeOfIndex(inst);
6711		llvm_ret_types[llvm_ret_i] = try o.lowerLlvmType(ret_ty);
	6591	llvm_ret_types[llvm_ret_i] = (try o.lowerType(ret_ty)).toLlvm(&o.builder);
6712	6592	llvm_ret_i += 1;
6713	6593	}
6714	6594
...	...	@@ -6745,13 +6625,13 @@ pub const FuncGen = struct {
6745	6625	const arg_ty = self.typeOf(input);
6746	6626	var llvm_elem_ty: ?*llvm.Type = null;
6747	6627	if (isByRef(arg_ty, mod)) {
6748		llvm_elem_ty = try o.lowerPtrElemTy(arg_ty);
	6628	llvm_elem_ty = (try o.lowerPtrElemTy(arg_ty)).toLlvm(&o.builder);
6749	6629	if (constraintAllowsMemory(constraint)) {
6750	6630	llvm_param_values[llvm_param_i] = arg_llvm_value;
6751	6631	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOf();
6752	6632	} else {
6753	6633	const alignment = arg_ty.abiAlignment(mod);
6754		const arg_llvm_ty = try o.lowerLlvmType(arg_ty);
	6634	const arg_llvm_ty = (try o.lowerType(arg_ty)).toLlvm(&o.builder);
6755	6635	const load_inst = self.builder.buildLoad(arg_llvm_ty, arg_llvm_value, "");
6756	6636	load_inst.setAlignment(alignment);
6757	6637	llvm_param_values[llvm_param_i] = load_inst;
...	...	@@ -6792,7 +6672,7 @@ pub const FuncGen = struct {
6792	6672	// an elementtype(<ty>) attribute.
6793	6673	if (constraint[0] == '*') {
6794	6674	llvm_param_attrs[llvm_param_i] = llvm_elem_ty orelse
6795		try o.lowerPtrElemTy(arg_ty.childType(mod));
	6675	(try o.lowerPtrElemTy(arg_ty.childType(mod))).toLlvm(&o.builder);
6796	6676	} else {
6797	6677	llvm_param_attrs[llvm_param_i] = null;
6798	6678	}
...	...	@@ -6989,7 +6869,7 @@ pub const FuncGen = struct {
6989	6869	const operand = try self.resolveInst(un_op);
6990	6870	const operand_ty = self.typeOf(un_op);
6991	6871	const optional_ty = if (operand_is_ptr) operand_ty.childType(mod) else operand_ty;
6992		const optional_llvm_ty = try o.lowerLlvmType(optional_ty);
	6872	const optional_llvm_ty = (try o.lowerType(optional_ty)).toLlvm(&o.builder);
6993	6873	const payload_ty = optional_ty.optionalChild(mod);
6994	6874	if (optional_ty.optionalReprIsPayload(mod)) {
6995	6875	const loaded = if (operand_is_ptr)
...	...	@@ -6998,7 +6878,7 @@ pub const FuncGen = struct {
6998	6878	operand;
6999	6879	if (payload_ty.isSlice(mod)) {
7000	6880	const slice_ptr = self.builder.buildExtractValue(loaded, 0, "");
7001		const ptr_ty = try o.lowerLlvmType(payload_ty.slicePtrFieldType(mod));
	6881	const ptr_ty = (try o.lowerType(payload_ty.slicePtrFieldType(mod))).toLlvm(&o.builder);
7002	6882	return self.builder.buildICmp(pred, slice_ptr, ptr_ty.constNull(), "");
7003	6883	}
7004	6884	return self.builder.buildICmp(pred, loaded, optional_llvm_ty.constNull(), "");
...	...	@@ -7037,7 +6917,7 @@ pub const FuncGen = struct {
7037	6917	const operand_ty = self.typeOf(un_op);
7038	6918	const err_union_ty = if (operand_is_ptr) operand_ty.childType(mod) else operand_ty;
7039	6919	const payload_ty = err_union_ty.errorUnionPayload(mod);
7040		const err_set_ty = try o.lowerLlvmType(Type.anyerror);
	6920	const err_set_ty = (try o.lowerType(Type.anyerror)).toLlvm(&o.builder);
7041	6921	const zero = err_set_ty.constNull();
7042	6922
7043	6923	if (err_union_ty.errorUnionSet(mod).errorSetIsEmpty(mod)) {
...	...	@@ -7051,7 +6931,7 @@ pub const FuncGen = struct {
7051	6931
7052	6932	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {
7053	6933	const loaded = if (operand_is_ptr)
7054		self.builder.buildLoad(try o.lowerLlvmType(err_union_ty), operand, "")
	6934	self.builder.buildLoad((try o.lowerType(err_union_ty)).toLlvm(&o.builder), operand, "")
7055	6935	else
7056	6936	operand;
7057	6937	return self.builder.buildICmp(op, loaded, zero, "");
...	...	@@ -7060,7 +6940,7 @@ pub const FuncGen = struct {
7060	6940	const err_field_index = errUnionErrorOffset(payload_ty, mod);
7061	6941
7062	6942	if (operand_is_ptr or isByRef(err_union_ty, mod)) {
7063		const err_union_llvm_ty = try o.lowerLlvmType(err_union_ty);
	6943	const err_union_llvm_ty = (try o.lowerType(err_union_ty)).toLlvm(&o.builder);
7064	6944	const err_field_ptr = self.builder.buildStructGEP(err_union_llvm_ty, operand, err_field_index, "");
7065	6945	const loaded = self.builder.buildLoad(err_set_ty, err_field_ptr, "");
7066	6946	return self.builder.buildICmp(op, loaded, zero, "");
...	...	@@ -7086,7 +6966,7 @@ pub const FuncGen = struct {
7086	6966	// The payload and the optional are the same value.
7087	6967	return operand;
7088	6968	}
7089		const optional_llvm_ty = try o.lowerLlvmType(optional_ty);
	6969	const optional_llvm_ty = (try o.lowerType(optional_ty)).toLlvm(&o.builder);
7090	6970	return self.builder.buildStructGEP(optional_llvm_ty, operand, 0, "");
7091	6971	}
7092	6972
...	...	@@ -7112,7 +6992,7 @@ pub const FuncGen = struct {
7112	6992	}
7113	6993
7114	6994	// First set the non-null bit.
7115		const optional_llvm_ty = try o.lowerLlvmType(optional_ty);
	6995	const optional_llvm_ty = (try o.lowerType(optional_ty)).toLlvm(&o.builder);
7116	6996	const non_null_ptr = self.builder.buildStructGEP(optional_llvm_ty, operand, 1, "");
7117	6997	// TODO set alignment on this store
7118	6998	_ = self.builder.buildStore(non_null_bit, non_null_ptr);
...	...	@@ -7139,7 +7019,7 @@ pub const FuncGen = struct {
7139	7019	return operand;
7140	7020	}
7141	7021
7142		const opt_llvm_ty = try o.lowerLlvmType(optional_ty);
	7022	const opt_llvm_ty = (try o.lowerType(optional_ty)).toLlvm(&o.builder);
7143	7023	const can_elide_load = if (isByRef(payload_ty, mod)) self.canElideLoad(body_tail) else false;
7144	7024	return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, can_elide_load);
7145	7025	}
...	...	@@ -7163,7 +7043,7 @@ pub const FuncGen = struct {
7163	7043	return if (operand_is_ptr) operand else null;
7164	7044	}
7165	7045	const offset = errUnionPayloadOffset(payload_ty, mod);
7166		const err_union_llvm_ty = try o.lowerLlvmType(err_union_ty);
	7046	const err_union_llvm_ty = (try o.lowerType(err_union_ty)).toLlvm(&o.builder);
7167	7047	if (operand_is_ptr) {
7168	7048	return self.builder.buildStructGEP(err_union_llvm_ty, operand, offset, "");
7169	7049	} else if (isByRef(err_union_ty, mod)) {
...	...	@@ -7193,7 +7073,7 @@ pub const FuncGen = struct {
7193	7073	const operand_ty = self.typeOf(ty_op.operand);
7194	7074	const err_union_ty = if (operand_is_ptr) operand_ty.childType(mod) else operand_ty;
7195	7075	if (err_union_ty.errorUnionSet(mod).errorSetIsEmpty(mod)) {
7196		const err_llvm_ty = try o.lowerLlvmType(Type.anyerror);
	7076	const err_llvm_ty = (try o.lowerType(Type.anyerror)).toLlvm(&o.builder);
7197	7077	if (operand_is_ptr) {
7198	7078	return operand;
7199	7079	} else {
...	...	@@ -7201,7 +7081,7 @@ pub const FuncGen = struct {
7201	7081	}
7202	7082	}
7203	7083
7204		const err_set_llvm_ty = try o.lowerLlvmType(Type.anyerror);
	7084	const err_set_llvm_ty = (try o.lowerType(Type.anyerror)).toLlvm(&o.builder);
7205	7085
7206	7086	const payload_ty = err_union_ty.errorUnionPayload(mod);
7207	7087	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {
...	...	@@ -7212,7 +7092,7 @@ pub const FuncGen = struct {
7212	7092	const offset = errUnionErrorOffset(payload_ty, mod);
7213	7093
7214	7094	if (operand_is_ptr or isByRef(err_union_ty, mod)) {
7215		const err_union_llvm_ty = try o.lowerLlvmType(err_union_ty);
	7095	const err_union_llvm_ty = (try o.lowerType(err_union_ty)).toLlvm(&o.builder);
7216	7096	const err_field_ptr = self.builder.buildStructGEP(err_union_llvm_ty, operand, offset, "");
7217	7097	return self.builder.buildLoad(err_set_llvm_ty, err_field_ptr, "");
7218	7098	}
...	...	@@ -7233,7 +7113,7 @@ pub const FuncGen = struct {
7233	7113	_ = self.builder.buildStore(non_error_val, operand);
7234	7114	return operand;
7235	7115	}
7236		const err_union_llvm_ty = try o.lowerLlvmType(err_union_ty);
	7116	const err_union_llvm_ty = (try o.lowerType(err_union_ty)).toLlvm(&o.builder);
7237	7117	{
7238	7118	const error_offset = errUnionErrorOffset(payload_ty, mod);
7239	7119	// First set the non-error value.
...	...	@@ -7269,7 +7149,7 @@ pub const FuncGen = struct {
7269	7149
7270	7150	const mod = o.module;
7271	7151	const llvm_field = llvmField(struct_ty, field_index, mod).?;
7272		const struct_llvm_ty = try o.lowerLlvmType(struct_ty);
	7152	const struct_llvm_ty = (try o.lowerType(struct_ty)).toLlvm(&o.builder);
7273	7153	const field_ptr = self.builder.buildStructGEP(struct_llvm_ty, self.err_ret_trace.?, llvm_field.index, "");
7274	7154	const field_ptr_ty = try mod.ptrType(.{
7275	7155	.child = llvm_field.ty.toIntern(),
...	...	@@ -7293,7 +7173,7 @@ pub const FuncGen = struct {
7293	7173	if (optional_ty.optionalReprIsPayload(mod)) {
7294	7174	return operand;
7295	7175	}
7296		const llvm_optional_ty = try o.lowerLlvmType(optional_ty);
	7176	const llvm_optional_ty = (try o.lowerType(optional_ty)).toLlvm(&o.builder);
7297	7177	if (isByRef(optional_ty, mod)) {
7298	7178	const optional_ptr = self.buildAlloca(llvm_optional_ty, optional_ty.abiAlignment(mod));
7299	7179	const payload_ptr = self.builder.buildStructGEP(llvm_optional_ty, optional_ptr, 0, "");
...	...	@@ -7317,8 +7197,8 @@ pub const FuncGen = struct {
7317	7197	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {
7318	7198	return operand;
7319	7199	}
7320		const ok_err_code = (try o.lowerLlvmType(Type.anyerror)).constNull();
7321		const err_un_llvm_ty = try o.lowerLlvmType(err_un_ty);
	7200	const ok_err_code = (try o.lowerType(Type.anyerror)).toLlvm(&o.builder).constNull();
	7201	const err_un_llvm_ty = (try o.lowerType(err_un_ty)).toLlvm(&o.builder);
7322	7202
7323	7203	const payload_offset = errUnionPayloadOffset(payload_ty, mod);
7324	7204	const error_offset = errUnionErrorOffset(payload_ty, mod);
...	...	@@ -7347,7 +7227,7 @@ pub const FuncGen = struct {
7347	7227	if (!payload_ty.hasRuntimeBitsIgnoreComptime(mod)) {
7348	7228	return operand;
7349	7229	}
7350		const err_un_llvm_ty = try o.lowerLlvmType(err_un_ty);
	7230	const err_un_llvm_ty = (try o.lowerType(err_un_ty)).toLlvm(&o.builder);
7351	7231
7352	7232	const payload_offset = errUnionPayloadOffset(payload_ty, mod);
7353	7233	const error_offset = errUnionErrorOffset(payload_ty, mod);
...	...	@@ -7403,7 +7283,7 @@ pub const FuncGen = struct {
7403	7283	const operand = try self.resolveInst(extra.rhs);
7404	7284
7405	7285	const loaded_vector = blk: {
7406		const elem_llvm_ty = try o.lowerLlvmType(vector_ptr_ty.childType(mod));
	7286	const elem_llvm_ty = (try o.lowerType(vector_ptr_ty.childType(mod))).toLlvm(&o.builder);
7407	7287	const load_inst = self.builder.buildLoad(elem_llvm_ty, vector_ptr, "");
7408	7288	load_inst.setAlignment(vector_ptr_ty.ptrAlignment(mod));
7409	7289	load_inst.setVolatile(llvm.Bool.fromBool(vector_ptr_ty.isVolatilePtr(mod)));
...	...	@@ -7447,7 +7327,7 @@ pub const FuncGen = struct {
7447	7327	const ptr = try self.resolveInst(bin_op.lhs);
7448	7328	const len = try self.resolveInst(bin_op.rhs);
7449	7329	const inst_ty = self.typeOfIndex(inst);
7450		const llvm_slice_ty = try o.lowerLlvmType(inst_ty);
	7330	const llvm_slice_ty = (try o.lowerType(inst_ty)).toLlvm(&o.builder);
7451	7331
7452	7332	// In case of slicing a global, the result type looks something like `{ i8*, i64 }`
7453	7333	// but `ptr` is pointing to the global directly.
...	...	@@ -7491,7 +7371,7 @@ pub const FuncGen = struct {
7491	7371	true => signed_intrinsic,
7492	7372	false => unsigned_intrinsic,
7493	7373	};
7494		const llvm_inst_ty = try o.lowerLlvmType(inst_ty);
	7374	const llvm_inst_ty = (try o.lowerType(inst_ty)).toLlvm(&o.builder);
7495	7375	const llvm_fn = fg.getIntrinsic(intrinsic_name, &.{llvm_inst_ty});
7496	7376	const result_struct = fg.builder.buildCall(
7497	7377	llvm_fn.globalGetValueType(),
...	...	@@ -7664,11 +7544,11 @@ pub const FuncGen = struct {
7664	7544	return self.buildFloatOp(.floor, inst_ty, 1, .{result});
7665	7545	}
7666	7546	if (scalar_ty.isSignedInt(mod)) {
7667		const inst_llvm_ty = try o.lowerLlvmType(inst_ty);
	7547	const inst_llvm_ty = (try o.lowerType(inst_ty)).toLlvm(&o.builder);
7668	7548	const scalar_bit_size_minus_one = scalar_ty.bitSize(mod) - 1;
7669	7549	const bit_size_minus_one = if (inst_ty.zigTypeTag(mod) == .Vector) const_vector: {
7670	7550	const vec_len = inst_ty.vectorLen(mod);
7671		const scalar_llvm_ty = try o.lowerLlvmType(scalar_ty);
	7551	const scalar_llvm_ty = (try o.lowerType(scalar_ty)).toLlvm(&o.builder);
7672	7552
7673	7553	const shifts = try self.gpa.alloc(*llvm.Value, vec_len);
7674	7554	defer self.gpa.free(shifts);
...	...	@@ -7730,7 +7610,7 @@ pub const FuncGen = struct {
7730	7610	const lhs = try self.resolveInst(bin_op.lhs);
7731	7611	const rhs = try self.resolveInst(bin_op.rhs);
7732	7612	const inst_ty = self.typeOfIndex(inst);
7733		const inst_llvm_ty = try o.lowerLlvmType(inst_ty);
	7613	const inst_llvm_ty = (try o.lowerType(inst_ty)).toLlvm(&o.builder);
7734	7614	const scalar_ty = inst_ty.scalarType(mod);
7735	7615
7736	7616	if (scalar_ty.isRuntimeFloat()) {
...	...	@@ -7745,7 +7625,7 @@ pub const FuncGen = struct {
7745	7625	const scalar_bit_size_minus_one = scalar_ty.bitSize(mod) - 1;
7746	7626	const bit_size_minus_one = if (inst_ty.zigTypeTag(mod) == .Vector) const_vector: {
7747	7627	const vec_len = inst_ty.vectorLen(mod);
7748		const scalar_llvm_ty = try o.lowerLlvmType(scalar_ty);
	7628	const scalar_llvm_ty = (try o.lowerType(scalar_ty)).toLlvm(&o.builder);
7749	7629
7750	7630	const shifts = try self.gpa.alloc(*llvm.Value, vec_len);
7751	7631	defer self.gpa.free(shifts);
...	...	@@ -7774,7 +7654,7 @@ pub const FuncGen = struct {
7774	7654	const ptr = try self.resolveInst(bin_op.lhs);
7775	7655	const offset = try self.resolveInst(bin_op.rhs);
7776	7656	const ptr_ty = self.typeOf(bin_op.lhs);
7777		const llvm_elem_ty = try o.lowerPtrElemTy(ptr_ty.childType(mod));
	7657	const llvm_elem_ty = (try o.lowerPtrElemTy(ptr_ty.childType(mod))).toLlvm(&o.builder);
7778	7658	switch (ptr_ty.ptrSize(mod)) {
7779	7659	.One => {
7780	7660	// It's a pointer to an array, so according to LLVM we need an extra GEP index.
...	...	@@ -7802,7 +7682,7 @@ pub const FuncGen = struct {
7802	7682	const offset = try self.resolveInst(bin_op.rhs);
7803	7683	const negative_offset = self.builder.buildNeg(offset, "");
7804	7684	const ptr_ty = self.typeOf(bin_op.lhs);
7805		const llvm_elem_ty = try o.lowerPtrElemTy(ptr_ty.childType(mod));
	7685	const llvm_elem_ty = (try o.lowerPtrElemTy(ptr_ty.childType(mod))).toLlvm(&o.builder);
7806	7686	switch (ptr_ty.ptrSize(mod)) {
7807	7687	.One => {
7808	7688	// It's a pointer to an array, so according to LLVM we need an extra GEP index.
...	...	@@ -7843,8 +7723,8 @@ pub const FuncGen = struct {
7843	7723
7844	7724	const intrinsic_name = if (scalar_ty.isSignedInt(mod)) signed_intrinsic else unsigned_intrinsic;
7845	7725
7846		const llvm_lhs_ty = try o.lowerLlvmType(lhs_ty);
7847		const llvm_dest_ty = try o.lowerLlvmType(dest_ty);
	7726	const llvm_lhs_ty = (try o.lowerType(lhs_ty)).toLlvm(&o.builder);
	7727	const llvm_dest_ty = (try o.lowerType(dest_ty)).toLlvm(&o.builder);
7848	7728
7849	7729	const llvm_fn = self.getIntrinsic(intrinsic_name, &.{llvm_lhs_ty});
7850	7730	const result_struct = self.builder.buildCall(llvm_fn.globalGetValueType(), llvm_fn, &[_]*llvm.Value{ lhs, rhs }, 2, .Fast, .Auto, "");
...	...	@@ -7920,7 +7800,7 @@ pub const FuncGen = struct {
7920	7800	const f = o.llvm_module.addFunction(name.toSlice(&o.builder).?, fn_type);
7921	7801
7922	7802	var global = Builder.Global{
7923		.type = .void,
	7803	.type = try o.builder.fnType(.void, &.{}, .normal),
7924	7804	.kind = .{ .function = @enumFromInt(o.builder.functions.items.len) },
7925	7805	};
7926	7806	var function = Builder.Function{
...	...	@@ -7947,7 +7827,7 @@ pub const FuncGen = struct {
7947	7827	const mod = o.module;
7948	7828	const target = o.module.getTarget();
7949	7829	const scalar_ty = ty.scalarType(mod);
7950		const scalar_llvm_ty = try o.lowerLlvmType(scalar_ty);
	7830	const scalar_llvm_ty = (try o.lowerType(scalar_ty)).toLlvm(&o.builder);
7951	7831
7952	7832	if (intrinsicsAllowed(scalar_ty, target)) {
7953	7833	const llvm_predicate: llvm.RealPredicate = switch (pred) {
...	...	@@ -8050,8 +7930,8 @@ pub const FuncGen = struct {
8050	7930	const mod = o.module;
8051	7931	const target = mod.getTarget();
8052	7932	const scalar_ty = ty.scalarType(mod);
8053		const llvm_ty = try o.lowerLlvmType(ty);
8054		const scalar_llvm_ty = try o.lowerLlvmType(scalar_ty);
	7933	const llvm_ty = (try o.lowerType(ty)).toLlvm(&o.builder);
	7934	const scalar_llvm_ty = (try o.lowerType(scalar_ty)).toLlvm(&o.builder);
8055	7935
8056	7936	const intrinsics_allowed = op != .tan and intrinsicsAllowed(scalar_ty, target);
8057	7937	var fn_name_buf: [64]u8 = undefined;
...	...	@@ -8161,10 +8041,10 @@ pub const FuncGen = struct {
8161	8041	const rhs_scalar_ty = rhs_ty.scalarType(mod);
8162	8042
8163	8043	const dest_ty = self.typeOfIndex(inst);
8164		const llvm_dest_ty = try o.lowerLlvmType(dest_ty);
	8044	const llvm_dest_ty = (try o.lowerType(dest_ty)).toLlvm(&o.builder);
8165	8045
8166	8046	const casted_rhs = if (rhs_scalar_ty.bitSize(mod) < lhs_scalar_ty.bitSize(mod))
8167		self.builder.buildZExt(rhs, try o.lowerLlvmType(lhs_ty), "")
	8047	self.builder.buildZExt(rhs, (try o.lowerType(lhs_ty)).toLlvm(&o.builder), "")
8168	8048	else
8169	8049	rhs;
8170	8050
...	...	@@ -8235,7 +8115,7 @@ pub const FuncGen = struct {
8235	8115	const rhs_scalar_ty = rhs_ty.scalarType(mod);
8236	8116
8237	8117	const casted_rhs = if (rhs_scalar_ty.bitSize(mod) < lhs_scalar_ty.bitSize(mod))
8238		self.builder.buildZExt(rhs, try o.lowerLlvmType(lhs_ty), "")
	8118	self.builder.buildZExt(rhs, (try o.lowerType(lhs_ty)).toLlvm(&o.builder), "")
8239	8119	else
8240	8120	rhs;
8241	8121	if (lhs_scalar_ty.isSignedInt(mod)) return self.builder.buildNSWShl(lhs, casted_rhs, "");
...	...	@@ -8256,7 +8136,7 @@ pub const FuncGen = struct {
8256	8136	const rhs_scalar_ty = rhs_type.scalarType(mod);
8257	8137
8258	8138	const casted_rhs = if (rhs_scalar_ty.bitSize(mod) < lhs_scalar_ty.bitSize(mod))
8259		self.builder.buildZExt(rhs, try o.lowerLlvmType(lhs_type), "")
	8139	self.builder.buildZExt(rhs, (try o.lowerType(lhs_type)).toLlvm(&o.builder), "")
8260	8140	else
8261	8141	rhs;
8262	8142	return self.builder.buildShl(lhs, casted_rhs, "");
...	...	@@ -8291,7 +8171,7 @@ pub const FuncGen = struct {
8291	8171	// poison value."
8292	8172	// However Zig semantics says that saturating shift left can never produce
8293	8173	// undefined; instead it saturates.
8294		const lhs_scalar_llvm_ty = try o.lowerLlvmType(lhs_scalar_ty);
	8174	const lhs_scalar_llvm_ty = (try o.lowerType(lhs_scalar_ty)).toLlvm(&o.builder);
8295	8175	const bits = lhs_scalar_llvm_ty.constInt(lhs_bits, .False);
8296	8176	const lhs_max = lhs_scalar_llvm_ty.constAllOnes();
8297	8177	if (rhs_ty.zigTypeTag(mod) == .Vector) {
...	...	@@ -8320,7 +8200,7 @@ pub const FuncGen = struct {
8320	8200	const rhs_scalar_ty = rhs_ty.scalarType(mod);
8321	8201
8322	8202	const casted_rhs = if (rhs_scalar_ty.bitSize(mod) < lhs_scalar_ty.bitSize(mod))
8323		self.builder.buildZExt(rhs, try o.lowerLlvmType(lhs_ty), "")
	8203	self.builder.buildZExt(rhs, (try o.lowerType(lhs_ty)).toLlvm(&o.builder), "")
8324	8204	else
8325	8205	rhs;
8326	8206	const is_signed_int = lhs_scalar_ty.isSignedInt(mod);
...	...	@@ -8346,7 +8226,7 @@ pub const FuncGen = struct {
8346	8226	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
8347	8227	const dest_ty = self.typeOfIndex(inst);
8348	8228	const dest_info = dest_ty.intInfo(mod);
8349		const dest_llvm_ty = try o.lowerLlvmType(dest_ty);
	8229	const dest_llvm_ty = (try o.lowerType(dest_ty)).toLlvm(&o.builder);
8350	8230	const operand = try self.resolveInst(ty_op.operand);
8351	8231	const operand_ty = self.typeOf(ty_op.operand);
8352	8232	const operand_info = operand_ty.intInfo(mod);
...	...	@@ -8367,7 +8247,7 @@ pub const FuncGen = struct {
8367	8247	const o = self.dg.object;
8368	8248	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
8369	8249	const operand = try self.resolveInst(ty_op.operand);
8370		const dest_llvm_ty = try o.lowerLlvmType(self.typeOfIndex(inst));
	8250	const dest_llvm_ty = (try o.lowerType(self.typeOfIndex(inst))).toLlvm(&o.builder);
8371	8251	return self.builder.buildTrunc(operand, dest_llvm_ty, "");
8372	8252	}
8373	8253
...	...	@@ -8383,11 +8263,11 @@ pub const FuncGen = struct {
8383	8263	const src_bits = operand_ty.floatBits(target);
8384	8264
8385	8265	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
8386		const dest_llvm_ty = try o.lowerLlvmType(dest_ty);
	8266	const dest_llvm_ty = (try o.lowerType(dest_ty)).toLlvm(&o.builder);
8387	8267	return self.builder.buildFPTrunc(operand, dest_llvm_ty, "");
8388	8268	} else {
8389		const operand_llvm_ty = try o.lowerLlvmType(operand_ty);
8390		const dest_llvm_ty = try o.lowerLlvmType(dest_ty);
	8269	const operand_llvm_ty = (try o.lowerType(operand_ty)).toLlvm(&o.builder);
	8270	const dest_llvm_ty = (try o.lowerType(dest_ty)).toLlvm(&o.builder);
8391	8271
8392	8272	var fn_name_buf: [64]u8 = undefined;
8393	8273	const fn_name = std.fmt.bufPrintZ(&fn_name_buf, "__trunc{s}f{s}f2", .{
...	...	@@ -8414,11 +8294,11 @@ pub const FuncGen = struct {
8414	8294	const src_bits = operand_ty.floatBits(target);
8415	8295
8416	8296	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
8417		const dest_llvm_ty = try o.lowerLlvmType(dest_ty);
	8297	const dest_llvm_ty = (try o.lowerType(dest_ty)).toLlvm(&o.builder);
8418	8298	return self.builder.buildFPExt(operand, dest_llvm_ty, "");
8419	8299	} else {
8420		const operand_llvm_ty = try o.lowerLlvmType(operand_ty);
8421		const dest_llvm_ty = try o.lowerLlvmType(dest_ty);
	8300	const operand_llvm_ty = (try o.lowerType(operand_ty)).toLlvm(&o.builder);
	8301	const dest_llvm_ty = (try o.lowerType(dest_ty)).toLlvm(&o.builder);
8422	8302
8423	8303	var fn_name_buf: [64]u8 = undefined;
8424	8304	const fn_name = std.fmt.bufPrintZ(&fn_name_buf, "__extend{s}f{s}f2", .{
...	...	@@ -8439,7 +8319,7 @@ pub const FuncGen = struct {
8439	8319	const operand = try self.resolveInst(un_op);
8440	8320	const ptr_ty = self.typeOf(un_op);
8441	8321	const operand_ptr = self.sliceOrArrayPtr(operand, ptr_ty);
8442		const dest_llvm_ty = try o.lowerLlvmType(self.typeOfIndex(inst));
	8322	const dest_llvm_ty = (try o.lowerType(self.typeOfIndex(inst))).toLlvm(&o.builder);
8443	8323	return self.builder.buildPtrToInt(operand_ptr, dest_llvm_ty, "");
8444	8324	}
8445	8325
...	...	@@ -8456,7 +8336,7 @@ pub const FuncGen = struct {
8456	8336	const mod = o.module;
8457	8337	const operand_is_ref = isByRef(operand_ty, mod);
8458	8338	const result_is_ref = isByRef(inst_ty, mod);
8459		const llvm_dest_ty = try o.lowerLlvmType(inst_ty);
	8339	const llvm_dest_ty = (try o.lowerType(inst_ty)).toLlvm(&o.builder);
8460	8340
8461	8341	if (operand_is_ref and result_is_ref) {
8462	8342	// They are both pointers, so just return the same opaque pointer :)
...	...	@@ -8486,7 +8366,7 @@ pub const FuncGen = struct {
8486	8366	} else {
8487	8367	// If the ABI size of the element type is not evenly divisible by size in bits;
8488	8368	// a simple bitcast will not work, and we fall back to extractelement.
8489		const llvm_usize = try o.lowerLlvmType(Type.usize);
	8369	const llvm_usize = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
8490	8370	const llvm_u32 = self.context.intType(32);
8491	8371	const zero = llvm_usize.constNull();
8492	8372	const vector_len = operand_ty.arrayLen(mod);
...	...	@@ -8503,7 +8383,7 @@ pub const FuncGen = struct {
8503	8383	return array_ptr;
8504	8384	} else if (operand_ty.zigTypeTag(mod) == .Array and inst_ty.zigTypeTag(mod) == .Vector) {
8505	8385	const elem_ty = operand_ty.childType(mod);
8506		const llvm_vector_ty = try o.lowerLlvmType(inst_ty);
	8386	const llvm_vector_ty = (try o.lowerType(inst_ty)).toLlvm(&o.builder);
8507	8387	if (!operand_is_ref) {
8508	8388	return self.dg.todo("implement bitcast non-ref array to vector", .{});
8509	8389	}
...	...	@@ -8518,9 +8398,9 @@ pub const FuncGen = struct {
8518	8398	} else {
8519	8399	// If the ABI size of the element type is not evenly divisible by size in bits;
8520	8400	// a simple bitcast will not work, and we fall back to extractelement.
8521		const array_llvm_ty = try o.lowerLlvmType(operand_ty);
8522		const elem_llvm_ty = try o.lowerLlvmType(elem_ty);
8523		const llvm_usize = try o.lowerLlvmType(Type.usize);
	8401	const array_llvm_ty = (try o.lowerType(operand_ty)).toLlvm(&o.builder);
	8402	const elem_llvm_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
	8403	const llvm_usize = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
8524	8404	const llvm_u32 = self.context.intType(32);
8525	8405	const zero = llvm_usize.constNull();
8526	8406	const vector_len = operand_ty.arrayLen(mod);
...	...	@@ -8629,7 +8509,7 @@ pub const FuncGen = struct {
8629	8509	if (!pointee_type.isFnOrHasRuntimeBitsIgnoreComptime(mod))
8630	8510	return o.lowerPtrToVoid(ptr_ty);
8631	8511
8632		const pointee_llvm_ty = try o.lowerLlvmType(pointee_type);
	8512	const pointee_llvm_ty = (try o.lowerType(pointee_type)).toLlvm(&o.builder);
8633	8513	const alignment = ptr_ty.ptrAlignment(mod);
8634	8514	return self.buildAlloca(pointee_llvm_ty, alignment);
8635	8515	}
...	...	@@ -8641,7 +8521,7 @@ pub const FuncGen = struct {
8641	8521	const ret_ty = ptr_ty.childType(mod);
8642	8522	if (!ret_ty.isFnOrHasRuntimeBitsIgnoreComptime(mod)) return o.lowerPtrToVoid(ptr_ty);
8643	8523	if (self.ret_ptr) \|ret_ptr\| return ret_ptr;
8644		const ret_llvm_ty = try o.lowerLlvmType(ret_ty);
	8524	const ret_llvm_ty = (try o.lowerType(ret_ty)).toLlvm(&o.builder);
8645	8525	return self.buildAlloca(ret_llvm_ty, ptr_ty.ptrAlignment(mod));
8646	8526	}
8647	8527
...	...	@@ -8673,7 +8553,7 @@ pub const FuncGen = struct {
8673	8553	else
8674	8554	u8_llvm_ty.getUndef();
8675	8555	const operand_size = operand_ty.abiSize(mod);
8676		const usize_llvm_ty = try o.lowerLlvmType(Type.usize);
	8556	const usize_llvm_ty = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
8677	8557	const len = usize_llvm_ty.constInt(operand_size, .False);
8678	8558	const dest_ptr_align = ptr_ty.ptrAlignment(mod);
8679	8559	_ = self.builder.buildMemSet(dest_ptr, fill_byte, len, dest_ptr_align, ptr_ty.isVolatilePtr(mod));
...	...	@@ -8746,7 +8626,7 @@ pub const FuncGen = struct {
8746	8626	_ = inst;
8747	8627	const o = self.dg.object;
8748	8628	const mod = o.module;
8749		const llvm_usize = try o.lowerLlvmType(Type.usize);
	8629	const llvm_usize = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
8750	8630	const target = mod.getTarget();
8751	8631	if (!target_util.supportsReturnAddress(target)) {
8752	8632	// https://github.com/ziglang/zig/issues/11946
...	...	@@ -8774,7 +8654,7 @@ pub const FuncGen = struct {
8774	8654
8775	8655	const params = [_]*llvm.Value{llvm_i32.constNull()};
8776	8656	const ptr_val = self.builder.buildCall(llvm_fn.globalGetValueType(), llvm_fn, &params, params.len, .Fast, .Auto, "");
8777		const llvm_usize = try o.lowerLlvmType(Type.usize);
	8657	const llvm_usize = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
8778	8658	return self.builder.buildPtrToInt(ptr_val, llvm_usize, "");
8779	8659	}
8780	8660
...	...	@@ -8795,15 +8675,16 @@ pub const FuncGen = struct {
8795	8675	var expected_value = try self.resolveInst(extra.expected_value);
8796	8676	var new_value = try self.resolveInst(extra.new_value);
8797	8677	const operand_ty = self.typeOf(extra.ptr).childType(mod);
8798		const opt_abi_ty = o.getAtomicAbiType(operand_ty, false);
8799		if (opt_abi_ty) \|abi_ty\| {
	8678	const abi_ty = try o.getAtomicAbiType(operand_ty, false);
	8679	if (abi_ty != .none) {
	8680	const llvm_abi_ty = abi_ty.toLlvm(&o.builder);
8800	8681	// operand needs widening and truncating
8801	8682	if (operand_ty.isSignedInt(mod)) {
8802		expected_value = self.builder.buildSExt(expected_value, abi_ty, "");
8803		new_value = self.builder.buildSExt(new_value, abi_ty, "");
	8683	expected_value = self.builder.buildSExt(expected_value, llvm_abi_ty, "");
	8684	new_value = self.builder.buildSExt(new_value, llvm_abi_ty, "");
8804	8685	} else {
8805		expected_value = self.builder.buildZExt(expected_value, abi_ty, "");
8806		new_value = self.builder.buildZExt(new_value, abi_ty, "");
	8686	expected_value = self.builder.buildZExt(expected_value, llvm_abi_ty, "");
	8687	new_value = self.builder.buildZExt(new_value, llvm_abi_ty, "");
8807	8688	}
8808	8689	}
8809	8690	const result = self.builder.buildAtomicCmpXchg(
...	...	@@ -8819,8 +8700,8 @@ pub const FuncGen = struct {
8819	8700	const optional_ty = self.typeOfIndex(inst);
8820	8701
8821	8702	var payload = self.builder.buildExtractValue(result, 0, "");
8822		if (opt_abi_ty != null) {
8823		payload = self.builder.buildTrunc(payload, try o.lowerLlvmType(operand_ty), "");
	8703	if (abi_ty != .none) {
	8704	payload = self.builder.buildTrunc(payload, (try o.lowerType(operand_ty)).toLlvm(&o.builder), "");
8824	8705	}
8825	8706	const success_bit = self.builder.buildExtractValue(result, 1, "");
8826	8707
...	...	@@ -8848,15 +8729,16 @@ pub const FuncGen = struct {
8848	8729	const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
8849	8730	const ordering = toLlvmAtomicOrdering(extra.ordering());
8850	8731	const single_threaded = llvm.Bool.fromBool(self.single_threaded);
8851		const opt_abi_ty = o.getAtomicAbiType(operand_ty, op == .Xchg);
8852		if (opt_abi_ty) \|abi_ty\| {
	8732	const abi_ty = try o.getAtomicAbiType(operand_ty, op == .Xchg);
	8733	if (abi_ty != .none) {
	8734	const llvm_abi_ty = abi_ty.toLlvm(&o.builder);
8853	8735	// operand needs widening and truncating or bitcasting.
8854	8736	const casted_operand = if (is_float)
8855		self.builder.buildBitCast(operand, abi_ty, "")
	8737	self.builder.buildBitCast(operand, llvm_abi_ty, "")
8856	8738	else if (is_signed_int)
8857		self.builder.buildSExt(operand, abi_ty, "")
	8739	self.builder.buildSExt(operand, llvm_abi_ty, "")
8858	8740	else
8859		self.builder.buildZExt(operand, abi_ty, "");
	8741	self.builder.buildZExt(operand, llvm_abi_ty, "");
8860	8742
8861	8743	const uncasted_result = self.builder.buildAtomicRmw(
8862	8744	op,
...	...	@@ -8865,7 +8747,7 @@ pub const FuncGen = struct {
8865	8747	ordering,
8866	8748	single_threaded,
8867	8749	);
8868		const operand_llvm_ty = try o.lowerLlvmType(operand_ty);
	8750	const operand_llvm_ty = (try o.lowerType(operand_ty)).toLlvm(&o.builder);
8869	8751	if (is_float) {
8870	8752	return self.builder.buildBitCast(uncasted_result, operand_llvm_ty, "");
8871	8753	} else {
...	...	@@ -8878,7 +8760,7 @@ pub const FuncGen = struct {
8878	8760	}
8879	8761
8880	8762	// It's a pointer but we need to treat it as an int.
8881		const usize_llvm_ty = try o.lowerLlvmType(Type.usize);
	8763	const usize_llvm_ty = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
8882	8764	const casted_operand = self.builder.buildPtrToInt(operand, usize_llvm_ty, "");
8883	8765	const uncasted_result = self.builder.buildAtomicRmw(
8884	8766	op,
...	...	@@ -8887,7 +8769,7 @@ pub const FuncGen = struct {
8887	8769	ordering,
8888	8770	single_threaded,
8889	8771	);
8890		const operand_llvm_ty = try o.lowerLlvmType(operand_ty);
	8772	const operand_llvm_ty = (try o.lowerType(operand_ty)).toLlvm(&o.builder);
8891	8773	return self.builder.buildIntToPtr(uncasted_result, operand_llvm_ty, "");
8892	8774	}
8893	8775
...	...	@@ -8902,15 +8784,16 @@ pub const FuncGen = struct {
8902	8784	if (!elem_ty.hasRuntimeBitsIgnoreComptime(mod))
8903	8785	return null;
8904	8786	const ordering = toLlvmAtomicOrdering(atomic_load.order);
8905		const opt_abi_llvm_ty = o.getAtomicAbiType(elem_ty, false);
	8787	const abi_ty = try o.getAtomicAbiType(elem_ty, false);
8906	8788	const ptr_alignment = @as(u32, @intCast(ptr_info.flags.alignment.toByteUnitsOptional() orelse
8907	8789	ptr_info.child.toType().abiAlignment(mod)));
8908	8790	const ptr_volatile = llvm.Bool.fromBool(ptr_info.flags.is_volatile);
8909		const elem_llvm_ty = try o.lowerLlvmType(elem_ty);
	8791	const elem_llvm_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
8910	8792
8911		if (opt_abi_llvm_ty) \|abi_llvm_ty\| {
	8793	if (abi_ty != .none) {
	8794	const llvm_abi_ty = abi_ty.toLlvm(&o.builder);
8912	8795	// operand needs widening and truncating
8913		const load_inst = self.builder.buildLoad(abi_llvm_ty, ptr, "");
	8796	const load_inst = self.builder.buildLoad(llvm_abi_ty, ptr, "");
8914	8797	load_inst.setAlignment(ptr_alignment);
8915	8798	load_inst.setVolatile(ptr_volatile);
8916	8799	load_inst.setOrdering(ordering);
...	...	@@ -8936,14 +8819,15 @@ pub const FuncGen = struct {
8936	8819	if (!operand_ty.isFnOrHasRuntimeBitsIgnoreComptime(mod)) return null;
8937	8820	const ptr = try self.resolveInst(bin_op.lhs);
8938	8821	var element = try self.resolveInst(bin_op.rhs);
8939		const opt_abi_ty = o.getAtomicAbiType(operand_ty, false);
	8822	const abi_ty = try o.getAtomicAbiType(operand_ty, false);
8940	8823
8941		if (opt_abi_ty) \|abi_ty\| {
	8824	if (abi_ty != .none) {
	8825	const llvm_abi_ty = abi_ty.toLlvm(&o.builder);
8942	8826	// operand needs widening
8943	8827	if (operand_ty.isSignedInt(mod)) {
8944		element = self.builder.buildSExt(element, abi_ty, "");
	8828	element = self.builder.buildSExt(element, llvm_abi_ty, "");
8945	8829	} else {
8946		element = self.builder.buildZExt(element, abi_ty, "");
	8830	element = self.builder.buildZExt(element, llvm_abi_ty, "");
8947	8831	}
8948	8832	}
8949	8833	try self.store(ptr, ptr_ty, element, ordering);
...	...	@@ -9056,7 +8940,7 @@ pub const FuncGen = struct {
9056	8940	.One => llvm_usize_ty.constInt(ptr_ty.childType(mod).arrayLen(mod), .False),
9057	8941	.Many, .C => unreachable,
9058	8942	};
9059		const elem_llvm_ty = try o.lowerLlvmType(elem_ty);
	8943	const elem_llvm_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
9060	8944	const len_gep = [_]*llvm.Value{len};
9061	8945	const end_ptr = self.builder.buildInBoundsGEP(elem_llvm_ty, dest_ptr, &len_gep, len_gep.len, "");
9062	8946	_ = self.builder.buildBr(loop_block);
...	...	@@ -9181,7 +9065,7 @@ pub const FuncGen = struct {
9181	9065	_ = self.builder.buildStore(new_tag, union_ptr);
9182	9066	return null;
9183	9067	}
9184		const un_llvm_ty = try o.lowerLlvmType(un_ty);
	9068	const un_llvm_ty = (try o.lowerType(un_ty)).toLlvm(&o.builder);
9185	9069	const tag_index = @intFromBool(layout.tag_align < layout.payload_align);
9186	9070	const tag_field_ptr = self.builder.buildStructGEP(un_llvm_ty, union_ptr, tag_index, "");
9187	9071	// TODO alignment on this store
...	...	@@ -9198,7 +9082,7 @@ pub const FuncGen = struct {
9198	9082	if (layout.tag_size == 0) return null;
9199	9083	const union_handle = try self.resolveInst(ty_op.operand);
9200	9084	if (isByRef(un_ty, mod)) {
9201		const llvm_un_ty = try o.lowerLlvmType(un_ty);
	9085	const llvm_un_ty = (try o.lowerType(un_ty)).toLlvm(&o.builder);
9202	9086	if (layout.payload_size == 0) {
9203	9087	return self.builder.buildLoad(llvm_un_ty, union_handle, "");
9204	9088	}
...	...	@@ -9240,13 +9124,13 @@ pub const FuncGen = struct {
9240	9124	const operand = try self.resolveInst(ty_op.operand);
9241	9125
9242	9126	const llvm_i1 = self.context.intType(1);
9243		const operand_llvm_ty = try o.lowerLlvmType(operand_ty);
	9127	const operand_llvm_ty = (try o.lowerType(operand_ty)).toLlvm(&o.builder);
9244	9128	const fn_val = self.getIntrinsic(llvm_fn_name, &.{operand_llvm_ty});
9245	9129
9246	9130	const params = [_]*llvm.Value{ operand, llvm_i1.constNull() };
9247	9131	const wrong_size_result = self.builder.buildCall(fn_val.globalGetValueType(), fn_val, &params, params.len, .C, .Auto, "");
9248	9132	const result_ty = self.typeOfIndex(inst);
9249		const result_llvm_ty = try o.lowerLlvmType(result_ty);
	9133	const result_llvm_ty = (try o.lowerType(result_ty)).toLlvm(&o.builder);
9250	9134
9251	9135	const bits = operand_ty.intInfo(mod).bits;
9252	9136	const result_bits = result_ty.intInfo(mod).bits;
...	...	@@ -9267,12 +9151,12 @@ pub const FuncGen = struct {
9267	9151	const operand = try self.resolveInst(ty_op.operand);
9268	9152
9269	9153	const params = [_]*llvm.Value{operand};
9270		const operand_llvm_ty = try o.lowerLlvmType(operand_ty);
	9154	const operand_llvm_ty = (try o.lowerType(operand_ty)).toLlvm(&o.builder);
9271	9155	const fn_val = self.getIntrinsic(llvm_fn_name, &.{operand_llvm_ty});
9272	9156
9273	9157	const wrong_size_result = self.builder.buildCall(fn_val.globalGetValueType(), fn_val, &params, params.len, .C, .Auto, "");
9274	9158	const result_ty = self.typeOfIndex(inst);
9275		const result_llvm_ty = try o.lowerLlvmType(result_ty);
	9159	const result_llvm_ty = (try o.lowerType(result_ty)).toLlvm(&o.builder);
9276	9160
9277	9161	const bits = operand_ty.intInfo(mod).bits;
9278	9162	const result_bits = result_ty.intInfo(mod).bits;
...	...	@@ -9294,7 +9178,7 @@ pub const FuncGen = struct {
9294	9178	assert(bits % 8 == 0);
9295	9179
9296	9180	var operand = try self.resolveInst(ty_op.operand);
9297		var operand_llvm_ty = try o.lowerLlvmType(operand_ty);
	9181	var operand_llvm_ty = (try o.lowerType(operand_ty)).toLlvm(&o.builder);
9298	9182
9299	9183	if (bits % 16 == 8) {
9300	9184	// If not an even byte-multiple, we need zero-extend + shift-left 1 byte
...	...	@@ -9328,7 +9212,7 @@ pub const FuncGen = struct {
9328	9212	const wrong_size_result = self.builder.buildCall(fn_val.globalGetValueType(), fn_val, &params, params.len, .C, .Auto, "");
9329	9213
9330	9214	const result_ty = self.typeOfIndex(inst);
9331		const result_llvm_ty = try o.lowerLlvmType(result_ty);
	9215	const result_llvm_ty = (try o.lowerType(result_ty)).toLlvm(&o.builder);
9332	9216	const result_bits = result_ty.intInfo(mod).bits;
9333	9217	if (bits > result_bits) {
9334	9218	return self.builder.buildTrunc(wrong_size_result, result_llvm_ty, "");
...	...	@@ -9407,9 +9291,9 @@ pub const FuncGen = struct {
9407	9291	const fqn = try mod.declPtr(enum_type.decl).getFullyQualifiedName(mod);
9408	9292	const llvm_fn_name = try std.fmt.allocPrintZ(arena, "__zig_is_named_enum_value_{}", .{fqn.fmt(&mod.intern_pool)});
9409	9293
9410		const param_types = [_]*llvm.Type{try o.lowerLlvmType(enum_type.tag_ty.toType())};
	9294	const param_types = [_]*llvm.Type{(try o.lowerType(enum_type.tag_ty.toType())).toLlvm(&o.builder)};
9411	9295
9412		const llvm_ret_ty = try o.lowerLlvmType(Type.bool);
	9296	const llvm_ret_ty = (try o.lowerType(Type.bool)).toLlvm(&o.builder);
9413	9297	const fn_type = llvm.functionType(llvm_ret_ty, &param_types, param_types.len, .False);
9414	9298	const fn_val = o.llvm_module.addFunction(llvm_fn_name, fn_type);
9415	9299	fn_val.setLinkage(.Internal);
...	...	@@ -9477,11 +9361,11 @@ pub const FuncGen = struct {
9477	9361	const llvm_fn_name = try o.builder.fmt("__zig_tag_name_{}", .{fqn.fmt(&mod.intern_pool)});
9478	9362
9479	9363	const slice_ty = Type.slice_const_u8_sentinel_0;
9480		const llvm_ret_ty = try o.lowerLlvmType(slice_ty);
9481		const usize_llvm_ty = try o.lowerLlvmType(Type.usize);
	9364	const llvm_ret_ty = (try o.lowerType(slice_ty)).toLlvm(&o.builder);
	9365	const usize_llvm_ty = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
9482	9366	const slice_alignment = slice_ty.abiAlignment(mod);
9483	9367
9484		const param_types = [_]*llvm.Type{try o.lowerLlvmType(enum_type.tag_ty.toType())};
	9368	const param_types = [_]*llvm.Type{(try o.lowerType(enum_type.tag_ty.toType())).toLlvm(&o.builder)};
9485	9369
9486	9370	const fn_type = llvm.functionType(llvm_ret_ty, &param_types, param_types.len, .False);
9487	9371	const fn_val = o.llvm_module.addFunction(llvm_fn_name.toSlice(&o.builder).?, fn_type);
...	...	@@ -9490,7 +9374,7 @@ pub const FuncGen = struct {
9490	9374	o.addCommonFnAttributes(fn_val);
9491	9375
9492	9376	var global = Builder.Global{
9493		.type = .void,
	9377	.type = try o.builder.fnType(.void, &.{}, .normal),
9494	9378	.kind = .{ .function = @enumFromInt(o.builder.functions.items.len) },
9495	9379	};
9496	9380	var function = Builder.Function{
...	...	@@ -9573,8 +9457,8 @@ pub const FuncGen = struct {
9573	9457
9574	9458	// Function signature: fn (anyerror) bool
9575	9459
9576		const ret_llvm_ty = try o.lowerLlvmType(Type.bool);
9577		const anyerror_llvm_ty = try o.lowerLlvmType(Type.anyerror);
	9460	const ret_llvm_ty = (try o.lowerType(Type.bool)).toLlvm(&o.builder);
	9461	const anyerror_llvm_ty = (try o.lowerType(Type.anyerror)).toLlvm(&o.builder);
9578	9462	const param_types = [_]*llvm.Type{anyerror_llvm_ty};
9579	9463
9580	9464	const fn_type = llvm.functionType(ret_llvm_ty, &param_types, param_types.len, .False);
...	...	@@ -9590,7 +9474,7 @@ pub const FuncGen = struct {
9590	9474	const un_op = self.air.instructions.items(.data)[inst].un_op;
9591	9475	const operand = try self.resolveInst(un_op);
9592	9476	const slice_ty = self.typeOfIndex(inst);
9593		const slice_llvm_ty = try o.lowerLlvmType(slice_ty);
	9477	const slice_llvm_ty = (try o.lowerType(slice_ty)).toLlvm(&o.builder);
9594	9478
9595	9479	const error_name_table_ptr = try self.getErrorNameTable();
9596	9480	const ptr_slice_llvm_ty = self.context.pointerType(0);
...	...	@@ -9676,7 +9560,7 @@ pub const FuncGen = struct {
9676	9560	accum_init: *llvm.Value,
9677	9561	) !*llvm.Value {
9678	9562	const o = self.dg.object;
9679		const llvm_usize_ty = try o.lowerLlvmType(Type.usize);
	9563	const llvm_usize_ty = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
9680	9564	const llvm_vector_len = llvm_usize_ty.constInt(vector_len, .False);
9681	9565	const llvm_result_ty = accum_init.typeOf();
9682	9566
...	...	@@ -9753,7 +9637,7 @@ pub const FuncGen = struct {
9753	9637	.Add => switch (scalar_ty.zigTypeTag(mod)) {
9754	9638	.Int => return self.builder.buildAddReduce(operand),
9755	9639	.Float => if (intrinsicsAllowed(scalar_ty, target)) {
9756		const scalar_llvm_ty = try o.lowerLlvmType(scalar_ty);
	9640	const scalar_llvm_ty = (try o.lowerType(scalar_ty)).toLlvm(&o.builder);
9757	9641	const neutral_value = scalar_llvm_ty.constReal(-0.0);
9758	9642	return self.builder.buildFPAddReduce(neutral_value, operand);
9759	9643	},
...	...	@@ -9762,7 +9646,7 @@ pub const FuncGen = struct {
9762	9646	.Mul => switch (scalar_ty.zigTypeTag(mod)) {
9763	9647	.Int => return self.builder.buildMulReduce(operand),
9764	9648	.Float => if (intrinsicsAllowed(scalar_ty, target)) {
9765		const scalar_llvm_ty = try o.lowerLlvmType(scalar_ty);
	9649	const scalar_llvm_ty = (try o.lowerType(scalar_ty)).toLlvm(&o.builder);
9766	9650	const neutral_value = scalar_llvm_ty.constReal(1.0);
9767	9651	return self.builder.buildFPMulReduce(neutral_value, operand);
9768	9652	},
...	...	@@ -9790,7 +9674,7 @@ pub const FuncGen = struct {
9790	9674	else => unreachable,
9791	9675	};
9792	9676
9793		const param_llvm_ty = try o.lowerLlvmType(scalar_ty);
	9677	const param_llvm_ty = (try o.lowerType(scalar_ty)).toLlvm(&o.builder);
9794	9678	const param_types = [2]*llvm.Type{ param_llvm_ty, param_llvm_ty };
9795	9679	const libc_fn = try self.getLibcFunction(fn_name, &param_types, param_llvm_ty);
9796	9680	const init_value = try o.lowerValue(.{
...	...	@@ -9813,7 +9697,7 @@ pub const FuncGen = struct {
9813	9697	const result_ty = self.typeOfIndex(inst);
9814	9698	const len = @as(usize, @intCast(result_ty.arrayLen(mod)));
9815	9699	const elements = @as([]const Air.Inst.Ref, @ptrCast(self.air.extra[ty_pl.payload..][0..len]));
9816		const llvm_result_ty = try o.lowerLlvmType(result_ty);
	9700	const llvm_result_ty = (try o.lowerType(result_ty)).toLlvm(&o.builder);
9817	9701
9818	9702	switch (result_ty.zigTypeTag(mod)) {
9819	9703	.Vector => {
...	...	@@ -9901,7 +9785,7 @@ pub const FuncGen = struct {
9901	9785	.Array => {
9902	9786	assert(isByRef(result_ty, mod));
9903	9787
9904		const llvm_usize = try o.lowerLlvmType(Type.usize);
	9788	const llvm_usize = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
9905	9789	const alloca_inst = self.buildAlloca(llvm_result_ty, result_ty.abiAlignment(mod));
9906	9790
9907	9791	const array_info = result_ty.arrayInfo(mod);
...	...	@@ -9944,7 +9828,7 @@ pub const FuncGen = struct {
9944	9828	const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
9945	9829	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
9946	9830	const union_ty = self.typeOfIndex(inst);
9947		const union_llvm_ty = try o.lowerLlvmType(union_ty);
	9831	const union_llvm_ty = (try o.lowerType(union_ty)).toLlvm(&o.builder);
9948	9832	const layout = union_ty.unionGetLayout(mod);
9949	9833	const union_obj = mod.typeToUnion(union_ty).?;
9950	9834
...	...	@@ -9986,7 +9870,7 @@ pub const FuncGen = struct {
9986	9870	const llvm_payload = try self.resolveInst(extra.init);
9987	9871	assert(union_obj.haveFieldTypes());
9988	9872	const field = union_obj.fields.values()[extra.field_index];
9989		const field_llvm_ty = try o.lowerLlvmType(field.ty);
	9873	const field_llvm_ty = (try o.lowerType(field.ty)).toLlvm(&o.builder);
9990	9874	const field_size = field.ty.abiSize(mod);
9991	9875	const field_align = field.normalAlignment(mod);
9992	9876
...	...	@@ -10009,7 +9893,7 @@ pub const FuncGen = struct {
10009	9893	const fields: [1]*llvm.Type = .{payload};
10010	9894	break :t self.context.structType(&fields, fields.len, .False);
10011	9895	}
10012		const tag_llvm_ty = try o.lowerLlvmType(union_obj.tag_ty);
	9896	const tag_llvm_ty = (try o.lowerType(union_obj.tag_ty)).toLlvm(&o.builder);
10013	9897	var fields: [3]*llvm.Type = undefined;
10014	9898	var fields_len: c_uint = 2;
10015	9899	if (layout.tag_align >= layout.payload_align) {
...	...	@@ -10062,7 +9946,7 @@ pub const FuncGen = struct {
10062	9946	index_type.constInt(@intFromBool(layout.tag_align < layout.payload_align), .False),
10063	9947	};
10064	9948	const field_ptr = self.builder.buildInBoundsGEP(llvm_union_ty, result_ptr, &indices, indices.len, "");
10065		const tag_llvm_ty = try o.lowerLlvmType(union_obj.tag_ty);
	9949	const tag_llvm_ty = (try o.lowerType(union_obj.tag_ty)).toLlvm(&o.builder);
10066	9950	const llvm_tag = tag_llvm_ty.constInt(tag_int, .False);
10067	9951	const store_inst = self.builder.buildStore(llvm_tag, field_ptr);
10068	9952	store_inst.setAlignment(union_obj.tag_ty.abiAlignment(mod));
...	...	@@ -10144,7 +10028,7 @@ pub const FuncGen = struct {
10144	10028	const inst_ty = self.typeOfIndex(inst);
10145	10029	const operand = try self.resolveInst(ty_op.operand);
10146	10030
10147		const llvm_dest_ty = try o.lowerLlvmType(inst_ty);
	10031	const llvm_dest_ty = (try o.lowerType(inst_ty)).toLlvm(&o.builder);
10148	10032	return self.builder.buildAddrSpaceCast(operand, llvm_dest_ty, "");
10149	10033	}
10150	10034
...	...	@@ -10278,7 +10162,7 @@ pub const FuncGen = struct {
10278	10162
10279	10163	return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, false);
10280	10164	}
10281		const payload_llvm_ty = try o.lowerLlvmType(payload_ty);
	10165	const payload_llvm_ty = (try o.lowerType(payload_ty)).toLlvm(&o.builder);
10282	10166	const load_inst = fg.builder.buildLoad(payload_llvm_ty, payload_ptr, "");
10283	10167	load_inst.setAlignment(payload_alignment);
10284	10168	return load_inst;
...	...	@@ -10295,7 +10179,7 @@ pub const FuncGen = struct {
10295	10179	non_null_bit: *llvm.Value,
10296	10180	) !?*llvm.Value {
10297	10181	const o = self.dg.object;
10298		const optional_llvm_ty = try o.lowerLlvmType(optional_ty);
	10182	const optional_llvm_ty = (try o.lowerType(optional_ty)).toLlvm(&o.builder);
10299	10183	const non_null_field = self.builder.buildZExt(non_null_bit, self.context.intType(8), "");
10300	10184	const mod = o.module;
10301	10185
...	...	@@ -10350,13 +10234,13 @@ pub const FuncGen = struct {
10350	10234	const byte_offset = struct_ty.packedStructFieldByteOffset(field_index, mod);
10351	10235	if (byte_offset == 0) return struct_ptr;
10352	10236	const byte_llvm_ty = self.context.intType(8);
10353		const llvm_usize = try o.lowerLlvmType(Type.usize);
	10237	const llvm_usize = (try o.lowerType(Type.usize)).toLlvm(&o.builder);
10354	10238	const llvm_index = llvm_usize.constInt(byte_offset, .False);
10355	10239	const indices: [1]*llvm.Value = .{llvm_index};
10356	10240	return self.builder.buildInBoundsGEP(byte_llvm_ty, struct_ptr, &indices, indices.len, "");
10357	10241	},
10358	10242	else => {
10359		const struct_llvm_ty = try o.lowerPtrElemTy(struct_ty);
	10243	const struct_llvm_ty = (try o.lowerPtrElemTy(struct_ty)).toLlvm(&o.builder);
10360	10244
10361	10245	if (llvmField(struct_ty, field_index, mod)) \|llvm_field\| {
10362	10246	return self.builder.buildStructGEP(struct_llvm_ty, struct_ptr, llvm_field.index, "");
...	...	@@ -10376,7 +10260,7 @@ pub const FuncGen = struct {
10376	10260	const layout = struct_ty.unionGetLayout(mod);
10377	10261	if (layout.payload_size == 0 or struct_ty.containerLayout(mod) == .Packed) return struct_ptr;
10378	10262	const payload_index = @intFromBool(layout.tag_align >= layout.payload_align);
10379		const union_llvm_ty = try o.lowerLlvmType(struct_ty);
	10263	const union_llvm_ty = (try o.lowerType(struct_ty)).toLlvm(&o.builder);
10380	10264	const union_field_ptr = self.builder.buildStructGEP(union_llvm_ty, struct_ptr, payload_index, "");
10381	10265	return union_field_ptr;
10382	10266	},
...	...	@@ -10401,7 +10285,7 @@ pub const FuncGen = struct {
10401	10285	) !*llvm.Value {
10402	10286	const o = fg.dg.object;
10403	10287	const mod = o.module;
10404		const pointee_llvm_ty = try o.lowerLlvmType(pointee_type);
	10288	const pointee_llvm_ty = (try o.lowerType(pointee_type)).toLlvm(&o.builder);
10405	10289	const result_align = @max(ptr_alignment, pointee_type.abiAlignment(mod));
10406	10290	const result_ptr = fg.buildAlloca(pointee_llvm_ty, result_align);
10407	10291	const llvm_usize = fg.context.intType(Type.usize.intInfo(mod).bits);
...	...	@@ -10434,7 +10318,7 @@ pub const FuncGen = struct {
10434	10318	assert(info.flags.vector_index != .runtime);
10435	10319	if (info.flags.vector_index != .none) {
10436	10320	const index_u32 = self.context.intType(32).constInt(@intFromEnum(info.flags.vector_index), .False);
10437		const vec_elem_ty = try o.lowerLlvmType(elem_ty);
	10321	const vec_elem_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
10438	10322	const vec_ty = vec_elem_ty.vectorType(info.packed_offset.host_size);
10439	10323
10440	10324	const loaded_vector = self.builder.buildLoad(vec_ty, ptr, "");
...	...	@@ -10448,7 +10332,7 @@ pub const FuncGen = struct {
10448	10332	if (isByRef(elem_ty, mod)) {
10449	10333	return self.loadByRef(ptr, elem_ty, ptr_alignment, info.flags.is_volatile);
10450	10334	}
10451		const elem_llvm_ty = try o.lowerLlvmType(elem_ty);
	10335	const elem_llvm_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
10452	10336	const llvm_inst = self.builder.buildLoad(elem_llvm_ty, ptr, "");
10453	10337	llvm_inst.setAlignment(ptr_alignment);
10454	10338	llvm_inst.setVolatile(ptr_volatile);
...	...	@@ -10463,7 +10347,7 @@ pub const FuncGen = struct {
10463	10347	const elem_bits = @as(c_uint, @intCast(ptr_ty.childType(mod).bitSize(mod)));
10464	10348	const shift_amt = containing_int.typeOf().constInt(info.packed_offset.bit_offset, .False);
10465	10349	const shifted_value = self.builder.buildLShr(containing_int, shift_amt, "");
10466		const elem_llvm_ty = try o.lowerLlvmType(elem_ty);
	10350	const elem_llvm_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
10467	10351
10468	10352	if (isByRef(elem_ty, mod)) {
10469	10353	const result_align = elem_ty.abiAlignment(mod);
...	...	@@ -10511,7 +10395,7 @@ pub const FuncGen = struct {
10511	10395	assert(info.flags.vector_index != .runtime);
10512	10396	if (info.flags.vector_index != .none) {
10513	10397	const index_u32 = self.context.intType(32).constInt(@intFromEnum(info.flags.vector_index), .False);
10514		const vec_elem_ty = try o.lowerLlvmType(elem_ty);
	10398	const vec_elem_ty = (try o.lowerType(elem_ty)).toLlvm(&o.builder);
10515	10399	const vec_ty = vec_elem_ty.vectorType(info.packed_offset.host_size);
10516	10400
10517	10401	const loaded_vector = self.builder.buildLoad(vec_ty, ptr, "");
...	...	@@ -10951,14 +10835,14 @@ fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: std.Target) llvm.Ca
10951	10835	}
10952	10836
10953	10837	/// Convert a zig-address space to an llvm address space.
10954		fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: std.Target) c_uint {
10955		for (llvmAddressSpaceInfo(target)) \|info\| if (info.zig == address_space) return info.llvm;
	10838	fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: std.Target) Builder.AddrSpace {
	10839	for (llvmAddrSpaceInfo(target)) \|info\| if (info.zig == address_space) return info.llvm;
10956	10840	unreachable;
10957	10841	}
10958	10842
10959		const AddressSpaceInfo = struct {
	10843	const AddrSpaceInfo = struct {
10960	10844	zig: ?std.builtin.AddressSpace,
10961		llvm: c_uint,
	10845	llvm: Builder.AddrSpace,
10962	10846	non_integral: bool = false,
10963	10847	size: ?u16 = null,
10964	10848	abi: ?u16 = null,
...	...	@@ -10966,49 +10850,49 @@ const AddressSpaceInfo = struct {
10966	10850	idx: ?u16 = null,
10967	10851	force_in_data_layout: bool = false,
10968	10852	};
10969		fn llvmAddressSpaceInfo(target: std.Target) []const AddressSpaceInfo {
	10853	fn llvmAddrSpaceInfo(target: std.Target) []const AddrSpaceInfo {
10970	10854	return switch (target.cpu.arch) {
10971	10855	.x86, .x86_64 => &.{
10972		.{ .zig = .generic, .llvm = llvm.address_space.default },
10973		.{ .zig = .gs, .llvm = llvm.address_space.x86.gs },
10974		.{ .zig = .fs, .llvm = llvm.address_space.x86.fs },
10975		.{ .zig = .ss, .llvm = llvm.address_space.x86.ss },
10976		.{ .zig = null, .llvm = llvm.address_space.x86.ptr32_sptr, .size = 32, .abi = 32, .force_in_data_layout = true },
10977		.{ .zig = null, .llvm = llvm.address_space.x86.ptr32_uptr, .size = 32, .abi = 32, .force_in_data_layout = true },
10978		.{ .zig = null, .llvm = llvm.address_space.x86.ptr64, .size = 64, .abi = 64, .force_in_data_layout = true },
	10856	.{ .zig = .generic, .llvm = .default },
	10857	.{ .zig = .gs, .llvm = Builder.AddrSpace.x86.gs },
	10858	.{ .zig = .fs, .llvm = Builder.AddrSpace.x86.fs },
	10859	.{ .zig = .ss, .llvm = Builder.AddrSpace.x86.ss },
	10860	.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_sptr, .size = 32, .abi = 32, .force_in_data_layout = true },
	10861	.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_uptr, .size = 32, .abi = 32, .force_in_data_layout = true },
	10862	.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr64, .size = 64, .abi = 64, .force_in_data_layout = true },
10979	10863	},
10980	10864	.nvptx, .nvptx64 => &.{
10981		.{ .zig = .generic, .llvm = llvm.address_space.default },
10982		.{ .zig = .global, .llvm = llvm.address_space.nvptx.global },
10983		.{ .zig = .constant, .llvm = llvm.address_space.nvptx.constant },
10984		.{ .zig = .param, .llvm = llvm.address_space.nvptx.param },
10985		.{ .zig = .shared, .llvm = llvm.address_space.nvptx.shared },
10986		.{ .zig = .local, .llvm = llvm.address_space.nvptx.local },
	10865	.{ .zig = .generic, .llvm = .default },
	10866	.{ .zig = .global, .llvm = Builder.AddrSpace.nvptx.global },
	10867	.{ .zig = .constant, .llvm = Builder.AddrSpace.nvptx.constant },
	10868	.{ .zig = .param, .llvm = Builder.AddrSpace.nvptx.param },
	10869	.{ .zig = .shared, .llvm = Builder.AddrSpace.nvptx.shared },
	10870	.{ .zig = .local, .llvm = Builder.AddrSpace.nvptx.local },
10987	10871	},
10988	10872	.amdgcn => &.{
10989		.{ .zig = .generic, .llvm = llvm.address_space.amdgpu.flat },
10990		.{ .zig = .global, .llvm = llvm.address_space.amdgpu.global },
10991		.{ .zig = .constant, .llvm = llvm.address_space.amdgpu.constant },
10992		.{ .zig = .shared, .llvm = llvm.address_space.amdgpu.local },
10993		.{ .zig = .local, .llvm = llvm.address_space.amdgpu.private },
	10873	.{ .zig = .generic, .llvm = Builder.AddrSpace.amdgpu.flat },
	10874	.{ .zig = .global, .llvm = Builder.AddrSpace.amdgpu.global },
	10875	.{ .zig = .constant, .llvm = Builder.AddrSpace.amdgpu.constant },
	10876	.{ .zig = .shared, .llvm = Builder.AddrSpace.amdgpu.local },
	10877	.{ .zig = .local, .llvm = Builder.AddrSpace.amdgpu.private },
10994	10878	},
10995	10879	.avr => &.{
10996		.{ .zig = .generic, .llvm = llvm.address_space.default },
10997		.{ .zig = .flash, .llvm = llvm.address_space.avr.flash },
10998		.{ .zig = .flash1, .llvm = llvm.address_space.avr.flash1 },
10999		.{ .zig = .flash2, .llvm = llvm.address_space.avr.flash2 },
11000		.{ .zig = .flash3, .llvm = llvm.address_space.avr.flash3 },
11001		.{ .zig = .flash4, .llvm = llvm.address_space.avr.flash4 },
11002		.{ .zig = .flash5, .llvm = llvm.address_space.avr.flash5 },
	10880	.{ .zig = .generic, .llvm = .default },
	10881	.{ .zig = .flash, .llvm = Builder.AddrSpace.avr.flash },
	10882	.{ .zig = .flash1, .llvm = Builder.AddrSpace.avr.flash1 },
	10883	.{ .zig = .flash2, .llvm = Builder.AddrSpace.avr.flash2 },
	10884	.{ .zig = .flash3, .llvm = Builder.AddrSpace.avr.flash3 },
	10885	.{ .zig = .flash4, .llvm = Builder.AddrSpace.avr.flash4 },
	10886	.{ .zig = .flash5, .llvm = Builder.AddrSpace.avr.flash5 },
11003	10887	},
11004	10888	.wasm32, .wasm64 => &.{
11005		.{ .zig = .generic, .llvm = llvm.address_space.default },
11006		.{ .zig = null, .llvm = llvm.address_space.wasm.variable, .non_integral = true },
11007		.{ .zig = null, .llvm = llvm.address_space.wasm.externref, .non_integral = true, .size = 8, .abi = 8 },
11008		.{ .zig = null, .llvm = llvm.address_space.wasm.funcref, .non_integral = true, .size = 8, .abi = 8 },
	10889	.{ .zig = .generic, .llvm = .default },
	10890	.{ .zig = null, .llvm = Builder.AddrSpace.wasm.variable, .non_integral = true },
	10891	.{ .zig = null, .llvm = Builder.AddrSpace.wasm.externref, .non_integral = true, .size = 8, .abi = 8 },
	10892	.{ .zig = null, .llvm = Builder.AddrSpace.wasm.funcref, .non_integral = true, .size = 8, .abi = 8 },
11009	10893	},
11010	10894	else => &.{
11011		.{ .zig = .generic, .llvm = llvm.address_space.default },
	10895	.{ .zig = .generic, .llvm = .default },
11012	10896	},
11013	10897	};
11014	10898	}
...	...	@@ -11017,30 +10901,30 @@ fn llvmAddressSpaceInfo(target: std.Target) []const AddressSpaceInfo {
11017	10901	/// different address, space and then cast back to the generic address space.
11018	10902	/// For example, on GPUs local variable declarations must be generated into the local address space.
11019	10903	/// This function returns the address space local values should be generated into.
11020		fn llvmAllocaAddressSpace(target: std.Target) c_uint {
	10904	fn llvmAllocaAddressSpace(target: std.Target) Builder.AddrSpace {
11021	10905	return switch (target.cpu.arch) {
11022	10906	// On amdgcn, locals should be generated into the private address space.
11023	10907	// To make Zig not impossible to use, these are then converted to addresses in the
11024	10908	// generic address space and treates as regular pointers. This is the way that HIP also does it.
11025		.amdgcn => llvm.address_space.amdgpu.private,
11026		else => llvm.address_space.default,
	10909	.amdgcn => Builder.AddrSpace.amdgpu.private,
	10910	else => .default,
11027	10911	};
11028	10912	}
11029	10913
11030	10914	/// On some targets, global values that are in the generic address space must be generated into a
11031	10915	/// different address space, and then cast back to the generic address space.
11032		fn llvmDefaultGlobalAddressSpace(target: std.Target) c_uint {
	10916	fn llvmDefaultGlobalAddressSpace(target: std.Target) Builder.AddrSpace {
11033	10917	return switch (target.cpu.arch) {
11034	10918	// On amdgcn, globals must be explicitly allocated and uploaded so that the program can access
11035	10919	// them.
11036		.amdgcn => llvm.address_space.amdgpu.global,
11037		else => llvm.address_space.default,
	10920	.amdgcn => Builder.AddrSpace.amdgpu.global,
	10921	else => .default,
11038	10922	};
11039	10923	}
11040	10924
11041	10925	/// Return the actual address space that a value should be stored in if its a global address space.
11042	10926	/// When a value is placed in the resulting address space, it needs to be cast back into wanted_address_space.
11043		fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, target: std.Target) c_uint {
	10927	fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, target: std.Target) Builder.AddrSpace {
11044	10928	return switch (wanted_address_space) {
11045	10929	.generic => llvmDefaultGlobalAddressSpace(target),
11046	10930	else => \|as\| toLlvmAddressSpace(as, target),
...	...	@@ -11170,161 +11054,129 @@ fn firstParamSRetSystemV(ty: Type, mod: *Module) bool {
11170	11054	/// In order to support the C calling convention, some return types need to be lowered
11171	11055	/// completely differently in the function prototype to honor the C ABI, and then
11172	11056	/// be effectively bitcasted to the actual return type.
11173		fn lowerFnRetTy(o: Object, fn_info: InternPool.Key.FuncType) !llvm.Type {
	11057	fn lowerFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
11174	11058	const mod = o.module;
11175	11059	const return_type = fn_info.return_type.toType();
11176	11060	if (!return_type.hasRuntimeBitsIgnoreComptime(mod)) {
11177	11061	// If the return type is an error set or an error union, then we make this
11178	11062	// anyerror return type instead, so that it can be coerced into a function
11179	11063	// pointer type which has anyerror as the return type.
11180		if (return_type.isError(mod)) {
11181		return o.lowerLlvmType(Type.anyerror);
11182		} else {
11183		return o.context.voidType();
11184		}
	11064	return if (return_type.isError(mod)) Builder.Type.err_int else .void;
11185	11065	}
11186	11066	const target = mod.getTarget();
11187	11067	switch (fn_info.cc) {
11188		.Unspecified, .Inline => {
11189		if (isByRef(return_type, mod)) {
11190		return o.context.voidType();
11191		} else {
11192		return o.lowerLlvmType(return_type);
11193		}
11194		},
	11068	.Unspecified,
	11069	.Inline,
	11070	=> return if (isByRef(return_type, mod)) .void else o.lowerType(return_type),
11195	11071	.C => {
11196	11072	switch (target.cpu.arch) {
11197		.mips, .mipsel => return o.lowerLlvmType(return_type),
	11073	.mips, .mipsel => return o.lowerType(return_type),
11198	11074	.x86_64 => switch (target.os.tag) {
11199	11075	.windows => return lowerWin64FnRetTy(o, fn_info),
11200	11076	else => return lowerSystemVFnRetTy(o, fn_info),
11201	11077	},
11202	11078	.wasm32 => {
11203	11079	if (isScalar(mod, return_type)) {
11204		return o.lowerLlvmType(return_type);
	11080	return o.lowerType(return_type);
11205	11081	}
11206	11082	const classes = wasm_c_abi.classifyType(return_type, mod);
11207	11083	if (classes[0] == .indirect or classes[0] == .none) {
11208		return o.context.voidType();
	11084	return .void;
11209	11085	}
11210	11086
11211	11087	assert(classes[0] == .direct and classes[1] == .none);
11212	11088	const scalar_type = wasm_c_abi.scalarType(return_type, mod);
11213		const abi_size = scalar_type.abiSize(mod);
11214		return o.context.intType(@as(c_uint, @intCast(abi_size * 8)));
	11089	return o.builder.intType(@intCast(scalar_type.abiSize(mod) * 8));
11215	11090	},
11216	11091	.aarch64, .aarch64_be => {
11217	11092	switch (aarch64_c_abi.classifyType(return_type, mod)) {
11218		.memory => return o.context.voidType(),
11219		.float_array => return o.lowerLlvmType(return_type),
11220		.byval => return o.lowerLlvmType(return_type),
11221		.integer => {
11222		const bit_size = return_type.bitSize(mod);
11223		return o.context.intType(@as(c_uint, @intCast(bit_size)));
11224		},
11225		.double_integer => return o.context.intType(64).arrayType(2),
	11093	.memory => return .void,
	11094	.float_array => return o.lowerType(return_type),
	11095	.byval => return o.lowerType(return_type),
	11096	.integer => return o.builder.intType(@intCast(return_type.bitSize(mod))),
	11097	.double_integer => return o.builder.arrayType(2, .i64),
11226	11098	}
11227	11099	},
11228	11100	.arm, .armeb => {
11229	11101	switch (arm_c_abi.classifyType(return_type, mod, .ret)) {
11230		.memory, .i64_array => return o.context.voidType(),
11231		.i32_array => \|len\| if (len == 1) {
11232		return o.context.intType(32);
11233		} else {
11234		return o.context.voidType();
11235		},
11236		.byval => return o.lowerLlvmType(return_type),
	11102	.memory, .i64_array => return .void,
	11103	.i32_array => \|len\| return if (len == 1) .i32 else .void,
	11104	.byval => return o.lowerType(return_type),
11237	11105	}
11238	11106	},
11239	11107	.riscv32, .riscv64 => {
11240	11108	switch (riscv_c_abi.classifyType(return_type, mod)) {
11241		.memory => return o.context.voidType(),
	11109	.memory => return .void,
11242	11110	.integer => {
11243		const bit_size = return_type.bitSize(mod);
11244		return o.context.intType(@as(c_uint, @intCast(bit_size)));
	11111	return o.builder.intType(@intCast(return_type.bitSize(mod)));
11245	11112	},
11246	11113	.double_integer => {
11247		var llvm_types_buffer: [2]*llvm.Type = .{
11248		o.context.intType(64),
11249		o.context.intType(64),
11250		};
11251		return o.context.structType(&llvm_types_buffer, 2, .False);
	11114	return o.builder.structType(.normal, &.{ .i64, .i64 });
11252	11115	},
11253		.byval => return o.lowerLlvmType(return_type),
	11116	.byval => return o.lowerType(return_type),
11254	11117	}
11255	11118	},
11256	11119	// TODO investigate C ABI for other architectures
11257		else => return o.lowerLlvmType(return_type),
	11120	else => return o.lowerType(return_type),
11258	11121	}
11259	11122	},
11260	11123	.Win64 => return lowerWin64FnRetTy(o, fn_info),
11261	11124	.SysV => return lowerSystemVFnRetTy(o, fn_info),
11262		.Stdcall => {
11263		if (isScalar(mod, return_type)) {
11264		return o.lowerLlvmType(return_type);
11265		} else {
11266		return o.context.voidType();
11267		}
11268		},
11269		else => return o.lowerLlvmType(return_type),
	11125	.Stdcall => return if (isScalar(mod, return_type)) o.lowerType(return_type) else .void,
	11126	else => return o.lowerType(return_type),
11270	11127	}
11271	11128	}
11272	11129
11273		fn lowerWin64FnRetTy(o: Object, fn_info: InternPool.Key.FuncType) !llvm.Type {
	11130	fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
11274	11131	const mod = o.module;
11275	11132	const return_type = fn_info.return_type.toType();
11276	11133	switch (x86_64_abi.classifyWindows(return_type, mod)) {
11277	11134	.integer => {
11278	11135	if (isScalar(mod, return_type)) {
11279		return o.lowerLlvmType(return_type);
	11136	return o.lowerType(return_type);
11280	11137	} else {
11281		const abi_size = return_type.abiSize(mod);
11282		return o.context.intType(@as(c_uint, @intCast(abi_size * 8)));
	11138	return o.builder.intType(@intCast(return_type.abiSize(mod) * 8));
11283	11139	}
11284	11140	},
11285		.win_i128 => return o.context.intType(64).vectorType(2),
11286		.memory => return o.context.voidType(),
11287		.sse => return o.lowerLlvmType(return_type),
	11141	.win_i128 => return o.builder.vectorType(.normal, 2, .i64),
	11142	.memory => return .void,
	11143	.sse => return o.lowerType(return_type),
11288	11144	else => unreachable,
11289	11145	}
11290	11146	}
11291	11147
11292		fn lowerSystemVFnRetTy(o: Object, fn_info: InternPool.Key.FuncType) !llvm.Type {
	11148	fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
11293	11149	const mod = o.module;
11294	11150	const return_type = fn_info.return_type.toType();
11295	11151	if (isScalar(mod, return_type)) {
11296		return o.lowerLlvmType(return_type);
	11152	return o.lowerType(return_type);
11297	11153	}
11298	11154	const classes = x86_64_abi.classifySystemV(return_type, mod, .ret);
11299		if (classes[0] == .memory) {
11300		return o.context.voidType();
11301		}
11302		var llvm_types_buffer: [8]*llvm.Type = undefined;
11303		var llvm_types_index: u32 = 0;
	11155	if (classes[0] == .memory) return .void;
	11156	var types_index: u32 = 0;
	11157	var types_buffer: [8]Builder.Type = undefined;
11304	11158	for (classes) \|class\| {
11305	11159	switch (class) {
11306	11160	.integer => {
11307		llvm_types_buffer[llvm_types_index] = o.context.intType(64);
11308		llvm_types_index += 1;
	11161	types_buffer[types_index] = .i64;
	11162	types_index += 1;
11309	11163	},
11310	11164	.sse, .sseup => {
11311		llvm_types_buffer[llvm_types_index] = o.context.doubleType();
11312		llvm_types_index += 1;
	11165	types_buffer[types_index] = .double;
	11166	types_index += 1;
11313	11167	},
11314	11168	.float => {
11315		llvm_types_buffer[llvm_types_index] = o.context.floatType();
11316		llvm_types_index += 1;
	11169	types_buffer[types_index] = .float;
	11170	types_index += 1;
11317	11171	},
11318	11172	.float_combine => {
11319		llvm_types_buffer[llvm_types_index] = o.context.floatType().vectorType(2);
11320		llvm_types_index += 1;
	11173	types_buffer[types_index] = try o.builder.vectorType(.normal, 2, .float);
	11174	types_index += 1;
11321	11175	},
11322	11176	.x87 => {
11323		if (llvm_types_index != 0 or classes[2] != .none) {
11324		return o.context.voidType();
11325		}
11326		llvm_types_buffer[llvm_types_index] = o.context.x86_fp80Type();
11327		llvm_types_index += 1;
	11177	if (types_index != 0 or classes[2] != .none) return .void;
	11178	types_buffer[types_index] = .x86_fp80;
	11179	types_index += 1;
11328	11180	},
11329	11181	.x87up => continue,
11330	11182	.complex_x87 => {
...	...	@@ -11336,10 +11188,9 @@ fn lowerSystemVFnRetTy(o: Object, fn_info: InternPool.Key.FuncType) !llvm.Type
11336	11188	}
11337	11189	}
11338	11190	if (classes[0] == .integer and classes[1] == .none) {
11339		const abi_size = return_type.abiSize(mod);
11340		return o.context.intType(@as(c_uint, @intCast(abi_size * 8)));
	11191	return o.builder.intType(@intCast(return_type.abiSize(mod) * 8));
11341	11192	}
11342		return o.context.structType(&llvm_types_buffer, llvm_types_index, .False);
	11193	return o.builder.structType(.normal, types_buffer[0..types_index]);
11343	11194	}
11344	11195
11345	11196	const ParamTypeIterator = struct {
...	...	@@ -11347,7 +11198,8 @@ const ParamTypeIterator = struct {
11347	11198	fn_info: InternPool.Key.FuncType,
11348	11199	zig_index: u32,
11349	11200	llvm_index: u32,
11350		llvm_types_len: u32,
	11201	types_len: u32,
	11202	types_buffer: [8]Builder.Type,
11351	11203	llvm_types_buffer: [8]*llvm.Type,
11352	11204	byval_attr: bool,
11353	11205
...	...	@@ -11365,7 +11217,7 @@ const ParamTypeIterator = struct {
11365	11217	i64_array: u8,
11366	11218	};
11367	11219
11368		pub fn next(it: *ParamTypeIterator) ?Lowering {
	11220	pub fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering {
11369	11221	if (it.zig_index >= it.fn_info.param_types.len) return null;
11370	11222	const mod = it.object.module;
11371	11223	const ip = &mod.intern_pool;
...	...	@@ -11375,7 +11227,7 @@ const ParamTypeIterator = struct {
11375	11227	}
11376	11228
11377	11229	/// `airCall` uses this instead of `next` so that it can take into account variadic functions.
11378		pub fn nextCall(it: ParamTypeIterator, fg: FuncGen, args: []const Air.Inst.Ref) ?Lowering {
	11230	pub fn nextCall(it: ParamTypeIterator, fg: FuncGen, args: []const Air.Inst.Ref) Allocator.Error!?Lowering {
11379	11231	const mod = it.object.module;
11380	11232	const ip = &mod.intern_pool;
11381	11233	if (it.zig_index >= it.fn_info.param_types.len) {
...	...	@@ -11389,7 +11241,7 @@ const ParamTypeIterator = struct {
11389	11241	}
11390	11242	}
11391	11243
11392		fn nextInner(it: *ParamTypeIterator, ty: Type) ?Lowering {
	11244	fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
11393	11245	const mod = it.object.module;
11394	11246	const target = mod.getTarget();
11395	11247
...	...	@@ -11444,8 +11296,9 @@ const ParamTypeIterator = struct {
11444	11296	.float_array => \|len\| return Lowering{ .float_array = len },
11445	11297	.byval => return .byval,
11446	11298	.integer => {
11447		it.llvm_types_len = 1;
11448		it.llvm_types_buffer[0] = it.object.context.intType(64);
	11299	it.types_len = 1;
	11300	it.types_buffer[0] = .i64;
	11301	it.llvm_types_buffer[0] = it.types_buffer[0].toLlvm(&it.object.builder);
11449	11302	return .multiple_llvm_types;
11450	11303	},
11451	11304	.double_integer => return Lowering{ .i64_array = 2 },
...	...	@@ -11539,7 +11392,7 @@ const ParamTypeIterator = struct {
11539	11392	}
11540	11393	}
11541	11394
11542		fn nextSystemV(it: *ParamTypeIterator, ty: Type) ?Lowering {
	11395	fn nextSystemV(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
11543	11396	const mod = it.object.module;
11544	11397	const classes = x86_64_abi.classifySystemV(ty, mod, .arg);
11545	11398	if (classes[0] == .memory) {
...	...	@@ -11553,25 +11406,34 @@ const ParamTypeIterator = struct {
11553	11406	it.llvm_index += 1;
11554	11407	return .byval;
11555	11408	}
	11409	var types_index: u32 = 0;
	11410	var types_buffer: [8]Builder.Type = undefined;
11556	11411	var llvm_types_buffer: [8]*llvm.Type = undefined;
11557		var llvm_types_index: u32 = 0;
11558	11412	for (classes) \|class\| {
11559	11413	switch (class) {
11560	11414	.integer => {
11561		llvm_types_buffer[llvm_types_index] = it.object.context.intType(64);
11562		llvm_types_index += 1;
	11415	types_buffer[types_index] = .i64;
	11416	llvm_types_buffer[types_index] =
	11417	types_buffer[types_index].toLlvm(&it.object.builder);
	11418	types_index += 1;
11563	11419	},
11564	11420	.sse, .sseup => {
11565		llvm_types_buffer[llvm_types_index] = it.object.context.doubleType();
11566		llvm_types_index += 1;
	11421	types_buffer[types_index] = .double;
	11422	llvm_types_buffer[types_index] =
	11423	types_buffer[types_index].toLlvm(&it.object.builder);
	11424	types_index += 1;
11567	11425	},
11568	11426	.float => {
11569		llvm_types_buffer[llvm_types_index] = it.object.context.floatType();
11570		llvm_types_index += 1;
	11427	types_buffer[types_index] = .float;
	11428	llvm_types_buffer[types_index] =
	11429	types_buffer[types_index].toLlvm(&it.object.builder);
	11430	types_index += 1;
11571	11431	},
11572	11432	.float_combine => {
11573		llvm_types_buffer[llvm_types_index] = it.object.context.floatType().vectorType(2);
11574		llvm_types_index += 1;
	11433	types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float);
	11434	llvm_types_buffer[types_index] =
	11435	types_buffer[types_index].toLlvm(&it.object.builder);
	11436	types_index += 1;
11575	11437	},
11576	11438	.x87 => {
11577	11439	it.zig_index += 1;
...	...	@@ -11593,9 +11455,10 @@ const ParamTypeIterator = struct {
11593	11455	it.llvm_index += 1;
11594	11456	return .abi_sized_int;
11595	11457	}
	11458	it.types_len = types_index;
	11459	it.types_buffer = types_buffer;
11596	11460	it.llvm_types_buffer = llvm_types_buffer;
11597		it.llvm_types_len = llvm_types_index;
11598		it.llvm_index += llvm_types_index;
	11461	it.llvm_index += types_index;
11599	11462	it.zig_index += 1;
11600	11463	return .multiple_llvm_types;
11601	11464	}
...	...	@@ -11607,8 +11470,9 @@ fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) ParamTyp
11607	11470	.fn_info = fn_info,
11608	11471	.zig_index = 0,
11609	11472	.llvm_index = 0,
	11473	.types_len = 0,
	11474	.types_buffer = undefined,
11610	11475	.llvm_types_buffer = undefined,
11611		.llvm_types_len = 0,
11612	11476	.byval_attr = false,
11613	11477	};
11614	11478	}
...	...	@@ -11905,7 +11769,7 @@ fn buildAllocaInner(
11905	11769	}
11906	11770	builder.clearCurrentDebugLocation();
11907	11771
11908		break :blk builder.buildAllocaInAddressSpace(llvm_ty, address_space, "");
	11772	break :blk builder.buildAllocaInAddressSpace(llvm_ty, @intFromEnum(address_space), "");
11909	11773	};
11910	11774
11911	11775	if (maybe_alignment) \|alignment\| {
...	...	@@ -11914,7 +11778,7 @@ fn buildAllocaInner(
11914	11778
11915	11779	// The pointer returned from this function should have the generic address space,
11916	11780	// if this isn't the case then cast it to the generic address space.
11917		if (address_space != llvm.address_space.default) {
	11781	if (address_space != .default) {
11918	11782	return builder.buildAddrSpaceCast(alloca, context.pointerType(llvm.address_space.default), "");
11919	11783	}
11920	11784

src/codegen/llvm/Builder.zig+580-89

...	...	@@ -19,7 +19,7 @@ types: std.AutoArrayHashMapUnmanaged(String, Type) = .{},
19	19	next_unnamed_type: String = @enumFromInt(0),
20	20	next_unique_type_id: std.AutoHashMapUnmanaged(String, u32) = .{},
21	21	type_map: std.AutoArrayHashMapUnmanaged(void, void) = .{},
22		type_data: std.ArrayListUnmanaged(Type.Data) = .{},
	22	type_items: std.ArrayListUnmanaged(Type.Item) = .{},
23	23	type_extra: std.ArrayListUnmanaged(u32) = .{},
24	24
25	25	globals: std.AutoArrayHashMapUnmanaged(String, Global) = .{},
...	...	@@ -62,10 +62,10 @@ pub const String = enum(u32) {
62	62	else
63	63	@compileError("invalid format string: '" ++ fmt_str ++ "'");
64	64	if (need_quotes) try writer.writeByte('\"');
65		for (slice) \|c\| switch (c) {
	65	for (slice) \|character\| switch (character) {
66	66	'\\' => try writer.writeAll("\\\\"),
67		' '...'"' - 1, '"' + 1...'\\' - 1, '\\' + 1...'~' => try writer.writeByte(c),
68		else => try writer.print("\\{X:0>2}", .{c}),
	67	' '...'"' - 1, '"' + 1...'\\' - 1, '\\' + 1...'~' => try writer.writeByte(character),
	68	else => try writer.print("\\{X:0>2}", .{character}),
69	69	};
70	70	if (need_quotes) try writer.writeByte('\"');
71	71	}
...	...	@@ -120,21 +120,25 @@ pub const Type = enum(u32) {
120	120	none = std.math.maxInt(u32),
121	121	_,
122	122
123		const Tag = enum(u4) {
	123	pub const err_int = Type.i16;
	124
	125	pub const Tag = enum(u4) {
124	126	simple,
125	127	function,
	128	vararg_function,
126	129	integer,
127	130	pointer,
128	131	target,
129	132	vector,
130		vscale_vector,
	133	scalable_vector,
	134	small_array,
131	135	array,
132	136	structure,
133	137	packed_structure,
134	138	named_structure,
135	139	};
136	140
137		const Simple = enum {
	141	pub const Simple = enum {
138	142	void,
139	143	half,
140	144	bfloat,
...	...	@@ -150,19 +154,55 @@ pub const Type = enum(u32) {
150	154	metadata,
151	155	};
152	156
153		const NamedStructure = struct {
154		id: String,
	157	pub const Function = struct {
	158	ret: Type,
	159	params_len: u32,
	160
	161	pub const Kind = enum { normal, vararg };
	162	};
	163
	164	pub const Target = extern struct {
	165	name: String,
	166	types_len: u32,
	167	ints_len: u32,
	168	};
	169
	170	pub const Vector = extern struct {
	171	len: u32,
	172	child: Type,
	173
	174	pub const Kind = enum { normal, scalable };
	175	};
	176
	177	pub const Array = extern struct {
	178	len_lo: u32,
	179	len_hi: u32,
155	180	child: Type,
	181
	182	fn len(self: Array) u64 {
	183	return @as(u64, self.len_hi) << 32 \| self.len_lo;
	184	}
	185	};
	186
	187	pub const Structure = struct {
	188	fields_len: u32,
	189
	190	pub const Kind = enum { normal, @"packed" };
156	191	};
157	192
158		const Data = packed struct(u32) {
	193	pub const NamedStructure = struct {
	194	id: String,
	195	body: Type,
	196	};
	197
	198	pub const Item = packed struct(u32) {
159	199	tag: Tag,
160	200	data: ExtraIndex,
161	201	};
162	202
163		const ExtraIndex = u28;
	203	pub const ExtraIndex = u28;
164	204
165		const FormatData = struct {
	205	pub const FormatData = struct {
166	206	type: Type,
167	207	builder: *const Builder,
168	208	};
...	...	@@ -174,27 +214,106 @@ pub const Type = enum(u32) {
174	214	) @TypeOf(writer).Error!void {
175	215	assert(data.type != .none);
176	216	if (std.enums.tagName(Type, data.type)) \|name\| return writer.writeAll(name);
177		const type_data = data.builder.type_data.items[@intFromEnum(data.type)];
178		switch (type_data.tag) {
179		.integer => try writer.print("i{d}", .{type_data.data}),
	217	const type_item = data.builder.type_items.items[@intFromEnum(data.type)];
	218	switch (type_item.tag) {
	219	.simple => unreachable,
	220	.function, .vararg_function => {
	221	const extra = data.builder.typeExtraDataTrail(Type.Function, type_item.data);
	222	const params: []const Type =
	223	@ptrCast(data.builder.type_extra.items[extra.end..][0..extra.data.params_len]);
	224	if (!comptime std.mem.eql(u8, fmt_str, ">"))
	225	try writer.print("{%} ", .{extra.data.ret.fmt(data.builder)});
	226	if (!comptime std.mem.eql(u8, fmt_str, "<")) {
	227	try writer.writeByte('(');
	228	for (params, 0..) \|param, index\| {
	229	if (index > 0) try writer.writeAll(", ");
	230	try writer.print("{%}", .{param.fmt(data.builder)});
	231	}
	232	switch (type_item.tag) {
	233	.function => {},
	234	.vararg_function => {
	235	if (params.len > 0) try writer.writeAll(", ");
	236	try writer.writeAll("...");
	237	},
	238	else => unreachable,
	239	}
	240	try writer.writeByte(')');
	241	}
	242	},
	243	.integer => try writer.print("i{d}", .{type_item.data}),
	244	.pointer => try writer.print("ptr{}", .{@as(AddrSpace, @enumFromInt(type_item.data))}),
	245	.target => {
	246	const extra = data.builder.typeExtraDataTrail(Type.Target, type_item.data);
	247	const types: []const Type =
	248	@ptrCast(data.builder.type_extra.items[extra.end..][0..extra.data.types_len]);
	249	const ints: []const u32 = @ptrCast(data.builder.type_extra.items[extra.end +
	250	extra.data.types_len ..][0..extra.data.ints_len]);
	251	try writer.print(
	252	\\target({"}
	253	, .{extra.data.name.fmt(data.builder)});
	254	for (types) \|ty\| try writer.print(", {%}", .{ty.fmt(data.builder)});
	255	for (ints) \|int\| try writer.print(", {d}", .{int});
	256	try writer.writeByte(')');
	257	},
	258	.vector => {
	259	const extra = data.builder.typeExtraData(Type.Vector, type_item.data);
	260	try writer.print("<{d} x {%}>", .{ extra.len, extra.child.fmt(data.builder) });
	261	},
	262	.scalable_vector => {
	263	const extra = data.builder.typeExtraData(Type.Vector, type_item.data);
	264	try writer.print("<vscale x {d} x {%}>", .{ extra.len, extra.child.fmt(data.builder) });
	265	},
	266	.small_array => {
	267	const extra = data.builder.typeExtraData(Type.Vector, type_item.data);
	268	try writer.print("[{d} x {%}]", .{ extra.len, extra.child.fmt(data.builder) });
	269	},
	270	.array => {
	271	const extra = data.builder.typeExtraData(Type.Array, type_item.data);
	272	try writer.print("[{d} x {%}]", .{ extra.len(), extra.child.fmt(data.builder) });
	273	},
	274	.structure, .packed_structure => {
	275	const extra = data.builder.typeExtraDataTrail(Type.Structure, type_item.data);
	276	const fields: []const Type =
	277	@ptrCast(data.builder.type_extra.items[extra.end..][0..extra.data.fields_len]);
	278	switch (type_item.tag) {
	279	.structure => {},
	280	.packed_structure => try writer.writeByte('<'),
	281	else => unreachable,
	282	}
	283	try writer.writeAll("{ ");
	284	for (fields, 0..) \|field, index\| {
	285	if (index > 0) try writer.writeAll(", ");
	286	try writer.print("{%}", .{field.fmt(data.builder)});
	287	}
	288	try writer.writeAll(" }");
	289	switch (type_item.tag) {
	290	.structure => {},
	291	.packed_structure => try writer.writeByte('>'),
	292	else => unreachable,
	293	}
	294	},
180	295	.named_structure => {
181		const extra = data.builder.typeExtraData(NamedStructure, type_data.data);
182		if (comptime std.mem.eql(u8, fmt_str, "")) switch (extra.child) {
	296	const extra = data.builder.typeExtraData(Type.NamedStructure, type_item.data);
	297	if (comptime std.mem.eql(u8, fmt_str, "%")) try writer.print("%{}", .{
	298	extra.id.fmt(data.builder),
	299	}) else switch (extra.body) {
183	300	.none => try writer.writeAll("opaque"),
184	301	else => try format(.{
185		.type = extra.child,
	302	.type = extra.body,
186	303	.builder = data.builder,
187	304	}, fmt_str, fmt_opts, writer),
188		} else if (comptime std.mem.eql(u8, fmt_str, "%")) try writer.print("%{}", .{
189		extra.id.fmt(data.builder),
190		}) else @compileError("invalid format string: '" ++ fmt_str ++ "'");
	305	}
191	306	},
192		else => try writer.print("<type 0x{X}>", .{@intFromEnum(data.type)}),
193	307	}
194	308	}
195	309	pub fn fmt(self: Type, builder: *const Builder) std.fmt.Formatter(format) {
196	310	return .{ .data = .{ .type = self, .builder = builder } };
197	311	}
	312
	313	pub fn toLlvm(self: Type, builder: const Builder) llvm.Type {
	314	assert(builder.useLibLlvm());
	315	return builder.llvm_types.items[@intFromEnum(self)];
	316	}
198	317	};
199	318
200	319	pub const Linkage = enum {
...	...	@@ -217,9 +336,7 @@ pub const Linkage = enum {
217	336	_: std.fmt.FormatOptions,
218	337	writer: anytype,
219	338	) @TypeOf(writer).Error!void {
220		if (self == .default) return;
221		try writer.writeAll(@tagName(self));
222		try writer.writeByte(' ');
	339	if (self != .default) try writer.print(" {s}", .{@tagName(self)});
223	340	}
224	341	};
225	342
...	...	@@ -234,9 +351,7 @@ pub const Preemption = enum {
234	351	_: std.fmt.FormatOptions,
235	352	writer: anytype,
236	353	) @TypeOf(writer).Error!void {
237		if (self == .default) return;
238		try writer.writeAll(@tagName(self));
239		try writer.writeByte(' ');
	354	if (self != .default) try writer.print(" {s}", .{@tagName(self)});
240	355	}
241	356	};
242	357
...	...	@@ -251,9 +366,7 @@ pub const Visibility = enum {
251	366	_: std.fmt.FormatOptions,
252	367	writer: anytype,
253	368	) @TypeOf(writer).Error!void {
254		if (self == .default) return;
255		try writer.writeAll(@tagName(self));
256		try writer.writeByte(' ');
	369	if (self != .default) try writer.print(" {s}", .{@tagName(self)});
257	370	}
258	371	};
259	372
...	...	@@ -268,9 +381,7 @@ pub const DllStorageClass = enum {
268	381	_: std.fmt.FormatOptions,
269	382	writer: anytype,
270	383	) @TypeOf(writer).Error!void {
271		if (self == .default) return;
272		try writer.writeAll(@tagName(self));
273		try writer.writeByte(' ');
	384	if (self != .default) try writer.print(" {s}", .{@tagName(self)});
274	385	}
275	386	};
276	387
...	...	@@ -288,13 +399,12 @@ pub const ThreadLocal = enum {
288	399	writer: anytype,
289	400	) @TypeOf(writer).Error!void {
290	401	if (self == .default) return;
291		try writer.writeAll("thread_local");
	402	try writer.writeAll(" thread_local");
292	403	if (self != .generaldynamic) {
293	404	try writer.writeByte('(');
294	405	try writer.writeAll(@tagName(self));
295	406	try writer.writeByte(')');
296	407	}
297		try writer.writeByte(' ');
298	408	}
299	409	};
300	410
...	...	@@ -309,9 +419,7 @@ pub const UnnamedAddr = enum {
309	419	_: std.fmt.FormatOptions,
310	420	writer: anytype,
311	421	) @TypeOf(writer).Error!void {
312		if (self == .default) return;
313		try writer.writeAll(@tagName(self));
314		try writer.writeByte(' ');
	422	if (self != .default) try writer.print(" {s}", .{@tagName(self)});
315	423	}
316	424	};
317	425
...	...	@@ -319,14 +427,82 @@ pub const AddrSpace = enum(u24) {
319	427	default,
320	428	_,
321	429
	430	// See llvm/lib/Target/X86/X86.h
	431	pub const x86 = struct {
	432	pub const gs: AddrSpace = @enumFromInt(256);
	433	pub const fs: AddrSpace = @enumFromInt(257);
	434	pub const ss: AddrSpace = @enumFromInt(258);
	435
	436	pub const ptr32_sptr: AddrSpace = @enumFromInt(270);
	437	pub const ptr32_uptr: AddrSpace = @enumFromInt(271);
	438	pub const ptr64: AddrSpace = @enumFromInt(272);
	439	};
	440	pub const x86_64 = x86;
	441
	442	// See llvm/lib/Target/AVR/AVR.h
	443	pub const avr = struct {
	444	pub const flash: AddrSpace = @enumFromInt(1);
	445	pub const flash1: AddrSpace = @enumFromInt(2);
	446	pub const flash2: AddrSpace = @enumFromInt(3);
	447	pub const flash3: AddrSpace = @enumFromInt(4);
	448	pub const flash4: AddrSpace = @enumFromInt(5);
	449	pub const flash5: AddrSpace = @enumFromInt(6);
	450	};
	451
	452	// See llvm/lib/Target/NVPTX/NVPTX.h
	453	pub const nvptx = struct {
	454	pub const generic: AddrSpace = @enumFromInt(0);
	455	pub const global: AddrSpace = @enumFromInt(1);
	456	pub const constant: AddrSpace = @enumFromInt(2);
	457	pub const shared: AddrSpace = @enumFromInt(3);
	458	pub const param: AddrSpace = @enumFromInt(4);
	459	pub const local: AddrSpace = @enumFromInt(5);
	460	};
	461
	462	// See llvm/lib/Target/AMDGPU/AMDGPU.h
	463	pub const amdgpu = struct {
	464	pub const flat: AddrSpace = @enumFromInt(0);
	465	pub const global: AddrSpace = @enumFromInt(1);
	466	pub const region: AddrSpace = @enumFromInt(2);
	467	pub const local: AddrSpace = @enumFromInt(3);
	468	pub const constant: AddrSpace = @enumFromInt(4);
	469	pub const private: AddrSpace = @enumFromInt(5);
	470	pub const constant_32bit: AddrSpace = @enumFromInt(6);
	471	pub const buffer_fat_pointer: AddrSpace = @enumFromInt(7);
	472	pub const param_d: AddrSpace = @enumFromInt(6);
	473	pub const param_i: AddrSpace = @enumFromInt(7);
	474	pub const constant_buffer_0: AddrSpace = @enumFromInt(8);
	475	pub const constant_buffer_1: AddrSpace = @enumFromInt(9);
	476	pub const constant_buffer_2: AddrSpace = @enumFromInt(10);
	477	pub const constant_buffer_3: AddrSpace = @enumFromInt(11);
	478	pub const constant_buffer_4: AddrSpace = @enumFromInt(12);
	479	pub const constant_buffer_5: AddrSpace = @enumFromInt(13);
	480	pub const constant_buffer_6: AddrSpace = @enumFromInt(14);
	481	pub const constant_buffer_7: AddrSpace = @enumFromInt(15);
	482	pub const constant_buffer_8: AddrSpace = @enumFromInt(16);
	483	pub const constant_buffer_9: AddrSpace = @enumFromInt(17);
	484	pub const constant_buffer_10: AddrSpace = @enumFromInt(18);
	485	pub const constant_buffer_11: AddrSpace = @enumFromInt(19);
	486	pub const constant_buffer_12: AddrSpace = @enumFromInt(20);
	487	pub const constant_buffer_13: AddrSpace = @enumFromInt(21);
	488	pub const constant_buffer_14: AddrSpace = @enumFromInt(22);
	489	pub const constant_buffer_15: AddrSpace = @enumFromInt(23);
	490	};
	491
	492	// See llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h
	493	pub const wasm = struct {
	494	pub const variable: AddrSpace = @enumFromInt(1);
	495	pub const externref: AddrSpace = @enumFromInt(10);
	496	pub const funcref: AddrSpace = @enumFromInt(20);
	497	};
	498
322	499	pub fn format(
323	500	self: AddrSpace,
324	501	comptime _: []const u8,
325	502	_: std.fmt.FormatOptions,
326	503	writer: anytype,
327	504	) @TypeOf(writer).Error!void {
328		if (self == .default) return;
329		try writer.print("addrspace({d}) ", .{@intFromEnum(self)});
	505	if (self != .default) try writer.print(" addrspace({d})", .{@intFromEnum(self)});
330	506	}
331	507	};
332	508
...	...	@@ -341,8 +517,8 @@ pub const ExternallyInitialized = enum {
341	517	writer: anytype,
342	518	) @TypeOf(writer).Error!void {
343	519	if (self == .default) return;
344		try writer.writeAll(@tagName(self));
345	520	try writer.writeByte(' ');
	521	try writer.writeAll(@tagName(self));
346	522	}
347	523	};
348	524
...	...	@@ -399,6 +575,7 @@ pub const Global = struct {
399	575	}
400	576
401	577	pub fn toLlvm(self: Index, builder: const Builder) llvm.Value {
	578	assert(builder.useLibLlvm());
402	579	return builder.llvm_globals.items[@intFromEnum(self)];
403	580	}
404	581
...	...	@@ -517,7 +694,7 @@ pub fn init(self: *Builder) Allocator.Error!void {
517	694	{
518	695	const static_len = @typeInfo(Type).Enum.fields.len - 1;
519	696	try self.type_map.ensureTotalCapacity(self.gpa, static_len);
520		try self.type_data.ensureTotalCapacity(self.gpa, static_len);
	697	try self.type_items.ensureTotalCapacity(self.gpa, static_len);
521	698	if (self.useLibLlvm()) try self.llvm_types.ensureTotalCapacity(self.gpa, static_len);
522	699	inline for (@typeInfo(Type.Simple).Enum.fields) \|simple_field\| {
523	700	const result = self.typeNoExtraAssumeCapacity(.{
...	...	@@ -532,7 +709,7 @@ pub fn init(self: *Builder) Allocator.Error!void {
532	709	inline for (.{ 1, 8, 16, 29, 32, 64, 80, 128 }) \|bits\| assert(self.intTypeAssumeCapacity(bits) ==
533	710	@field(Type, std.fmt.comptimePrint("i{d}", .{bits})));
534	711	inline for (.{0}) \|addr_space\|
535		assert(self.pointerTypeAssumeCapacity(@enumFromInt(addr_space)) == .ptr);
	712	assert(self.ptrTypeAssumeCapacity(@enumFromInt(addr_space)) == .ptr);
536	713	}
537	714	}
538	715
...	...	@@ -547,7 +724,7 @@ pub fn deinit(self: *Builder) void {
547	724	self.types.deinit(self.gpa);
548	725	self.next_unique_type_id.deinit(self.gpa);
549	726	self.type_map.deinit(self.gpa);
550		self.type_data.deinit(self.gpa);
	727	self.type_items.deinit(self.gpa);
551	728	self.type_extra.deinit(self.gpa);
552	729
553	730	self.globals.deinit(self.gpa);
...	...	@@ -600,25 +777,95 @@ pub fn fmtAssumeCapacity(self: *Builder, comptime fmt_str: []const u8, fmt_args:
600	777	return String.fromIndex(gop.index);
601	778	}
602	779
	780	pub fn fnType(
	781	self: *Builder,
	782	ret: Type,
	783	params: []const Type,
	784	kind: Type.Function.Kind,
	785	) Allocator.Error!Type {
	786	try self.ensureUnusedCapacityTypes(1, Type.Function, params.len);
	787	return switch (kind) {
	788	inline else => \|comptime_kind\| self.fnTypeAssumeCapacity(ret, params, comptime_kind),
	789	};
	790	}
	791
	792	pub fn intType(self: *Builder, bits: u24) Allocator.Error!Type {
	793	try self.ensureUnusedCapacityTypes(1, null, 0);
	794	return self.intTypeAssumeCapacity(bits);
	795	}
	796
	797	pub fn ptrType(self: *Builder, addr_space: AddrSpace) Allocator.Error!Type {
	798	try self.ensureUnusedCapacityTypes(1, null, 0);
	799	return self.ptrTypeAssumeCapacity(addr_space);
	800	}
	801
	802	pub fn vectorType(
	803	self: *Builder,
	804	kind: Type.Vector.Kind,
	805	len: u32,
	806	child: Type,
	807	) Allocator.Error!Type {
	808	try self.ensureUnusedCapacityTypes(1, Type.Vector, 0);
	809	return switch (kind) {
	810	inline else => \|comptime_kind\| self.vectorTypeAssumeCapacity(comptime_kind, len, child),
	811	};
	812	}
	813
	814	pub fn arrayType(self: *Builder, len: u64, child: Type) Allocator.Error!Type {
	815	comptime assert(@sizeOf(Type.Array) >= @sizeOf(Type.Vector));
	816	try self.ensureUnusedCapacityTypes(1, Type.Array, 0);
	817	return self.arrayTypeAssumeCapacity(len, child);
	818	}
	819
	820	pub fn structType(
	821	self: *Builder,
	822	kind: Type.Structure.Kind,
	823	fields: []const Type,
	824	) Allocator.Error!Type {
	825	try self.ensureUnusedCapacityTypes(1, Type.Structure, fields.len);
	826	return switch (kind) {
	827	inline else => \|comptime_kind\| self.structTypeAssumeCapacity(comptime_kind, fields),
	828	};
	829	}
	830
603	831	pub fn opaqueType(self: *Builder, name: String) Allocator.Error!Type {
604	832	try self.string_map.ensureUnusedCapacity(self.gpa, 1);
605		try self.string_bytes.ensureUnusedCapacity(self.gpa, name.toSlice(self).?.len +
	833	if (name.toSlice(self)) \|id\| try self.string_bytes.ensureUnusedCapacity(self.gpa, id.len +
606	834	comptime std.fmt.count("{d}" ++ .{0}, .{std.math.maxInt(u32)}));
607	835	try self.string_indices.ensureUnusedCapacity(self.gpa, 1);
608	836	try self.types.ensureUnusedCapacity(self.gpa, 1);
609	837	try self.next_unique_type_id.ensureUnusedCapacity(self.gpa, 1);
610		try self.ensureUnusedCapacityTypes(1, Type.NamedStructure);
	838	try self.ensureUnusedCapacityTypes(1, Type.NamedStructure, 0);
611	839	return self.opaqueTypeAssumeCapacity(name);
612	840	}
613	841
614		pub fn intType(self: *Builder, bits: u24) Allocator.Error!Type {
615		try self.ensureUnusedCapacityTypes(1, null);
616		return self.intTypeAssumeCapacity(bits);
617		}
618
619		pub fn pointerType(self: *Builder, addr_space: AddrSpace) Allocator.Error!Type {
620		try self.ensureUnusedCapacityTypes(1, null);
621		return self.pointerTypeAssumeCapacity(addr_space);
	842	pub fn namedTypeSetBody(
	843	self: *Builder,
	844	named_type: Type,
	845	body_type: Type,
	846	) if (build_options.have_llvm) Allocator.Error!void else void {
	847	const named_item = self.type_items.items[@intFromEnum(named_type)];
	848	self.type_extra.items[named_item.data + std.meta.fieldIndex(Type.NamedStructure, "body").?] =
	849	@intFromEnum(body_type);
	850	if (self.useLibLlvm()) {
	851	const body_item = self.type_items.items[@intFromEnum(body_type)];
	852	const body_extra = self.typeExtraDataTrail(Type.Structure, body_item.data);
	853	const body_fields: []const Type =
	854	@ptrCast(self.type_extra.items[body_extra.end..][0..body_extra.data.fields_len]);
	855	const llvm_fields = try self.gpa.alloc(*llvm.Type, body_fields.len);
	856	defer self.gpa.free(llvm_fields);
	857	for (llvm_fields, body_fields) \|*llvm_field, body_field\|
	858	llvm_field.* = self.llvm_types.items[@intFromEnum(body_field)];
	859	self.llvm_types.items[@intFromEnum(named_type)].structSetBody(
	860	llvm_fields.ptr,
	861	@intCast(llvm_fields.len),
	862	switch (body_item.tag) {
	863	.structure => .False,
	864	.packed_structure => .True,
	865	else => unreachable,
	866	},
	867	);
	868	}
622	869	}
623	870
624	871	pub fn addGlobal(self: *Builder, name: String, global: Global) Allocator.Error!Global.Index {
...	...	@@ -667,6 +914,7 @@ fn addTypeExtraAssumeCapacity(self: *Builder, extra: anytype) Type.ExtraIndex {
667	914	inline for (@typeInfo(@TypeOf(extra)).Struct.fields) \|field\| {
668	915	const value = @field(extra, field.name);
669	916	self.type_extra.appendAssumeCapacity(switch (field.type) {
	917	u32 => value,
670	918	String, Type => @intFromEnum(value),
671	919	else => @compileError("bad field type: " ++ @typeName(field.type)),
672	920	});
...	...	@@ -683,6 +931,7 @@ fn typeExtraDataTrail(
683	931	const fields = @typeInfo(T).Struct.fields;
684	932	inline for (fields, self.type_extra.items[index..][0..fields.len]) \|field, data\|
685	933	@field(result, field.name) = switch (field.type) {
	934	u32 => data,
686	935	String, Type => @enumFromInt(data),
687	936	else => @compileError("bad field type: " ++ @typeName(field.type)),
688	937	};
...	...	@@ -693,14 +942,254 @@ fn typeExtraData(self: *const Builder, comptime T: type, index: Type.ExtraIndex)
693	942	return self.typeExtraDataTrail(T, index).data;
694	943	}
695	944
	945	fn fnTypeAssumeCapacity(
	946	self: *Builder,
	947	ret: Type,
	948	params: []const Type,
	949	comptime kind: Type.Function.Kind,
	950	) if (build_options.have_llvm) Allocator.Error!Type else Type {
	951	const tag: Type.Tag = switch (kind) {
	952	.normal => .function,
	953	.vararg => .vararg_function,
	954	};
	955	const Key = struct { ret: Type, params: []const Type };
	956	const Adapter = struct {
	957	builder: *const Builder,
	958	pub fn hash(_: @This(), key: Key) u32 {
	959	var hasher = std.hash.Wyhash.init(comptime std.hash.uint32(@intFromEnum(tag)));
	960	hasher.update(std.mem.asBytes(&key.ret));
	961	hasher.update(std.mem.sliceAsBytes(key.params));
	962	return @truncate(hasher.final());
	963	}
	964	pub fn eql(ctx: @This(), lhs: Key, _: void, rhs_index: usize) bool {
	965	const rhs_data = ctx.builder.type_items.items[rhs_index];
	966	const rhs_extra = ctx.builder.typeExtraDataTrail(Type.Function, rhs_data.data);
	967	const rhs_params: []const Type =
	968	@ptrCast(ctx.builder.type_extra.items[rhs_extra.end..][0..rhs_extra.data.params_len]);
	969	return rhs_data.tag == tag and lhs.ret == rhs_extra.data.ret and
	970	std.mem.eql(Type, lhs.params, rhs_params);
	971	}
	972	};
	973	const data = Key{ .ret = ret, .params = params };
	974	const gop = self.type_map.getOrPutAssumeCapacityAdapted(data, Adapter{ .builder = self });
	975	if (!gop.found_existing) {
	976	gop.key_ptr.* = {};
	977	gop.value_ptr.* = {};
	978	self.type_items.appendAssumeCapacity(.{
	979	.tag = .function,
	980	.data = self.addTypeExtraAssumeCapacity(Type.Function{
	981	.ret = ret,
	982	.params_len = @intCast(params.len),
	983	}),
	984	});
	985	self.type_extra.appendSliceAssumeCapacity(@ptrCast(params));
	986	if (self.useLibLlvm()) {
	987	const llvm_params = try self.gpa.alloc(*llvm.Type, params.len);
	988	defer self.gpa.free(llvm_params);
	989	for (llvm_params, params) \|llvm_param, param\| llvm_param. = param.toLlvm(self);
	990	self.llvm_types.appendAssumeCapacity(llvm.functionType(
	991	ret.toLlvm(self),
	992	llvm_params.ptr,
	993	@intCast(llvm_params.len),
	994	switch (kind) {
	995	.normal => .False,
	996	.vararg => .True,
	997	},
	998	));
	999	}
	1000	}
	1001	return @enumFromInt(gop.index);
	1002	}
	1003
	1004	fn intTypeAssumeCapacity(self: *Builder, bits: u24) Type {
	1005	const result = self.typeNoExtraAssumeCapacity(.{ .tag = .integer, .data = bits });
	1006	if (self.useLibLlvm() and result.new)
	1007	self.llvm_types.appendAssumeCapacity(self.llvm_context.intType(bits));
	1008	return result.type;
	1009	}
	1010
	1011	fn ptrTypeAssumeCapacity(self: *Builder, addr_space: AddrSpace) Type {
	1012	const result = self.typeNoExtraAssumeCapacity(.{
	1013	.tag = .pointer,
	1014	.data = @intFromEnum(addr_space),
	1015	});
	1016	if (self.useLibLlvm() and result.new)
	1017	self.llvm_types.appendAssumeCapacity(self.llvm_context.pointerType(@intFromEnum(addr_space)));
	1018	return result.type;
	1019	}
	1020
	1021	fn vectorTypeAssumeCapacity(
	1022	self: *Builder,
	1023	comptime kind: Type.Vector.Kind,
	1024	len: u32,
	1025	child: Type,
	1026	) Type {
	1027	const tag: Type.Tag = switch (kind) {
	1028	.normal => .vector,
	1029	.scalable => .scalable_vector,
	1030	};
	1031	const Adapter = struct {
	1032	builder: *const Builder,
	1033	pub fn hash(_: @This(), key: Type.Vector) u32 {
	1034	return @truncate(std.hash.Wyhash.hash(
	1035	comptime std.hash.uint32(@intFromEnum(tag)),
	1036	std.mem.asBytes(&key),
	1037	));
	1038	}
	1039	pub fn eql(ctx: @This(), lhs: Type.Vector, _: void, rhs_index: usize) bool {
	1040	const rhs_data = ctx.builder.type_items.items[rhs_index];
	1041	return rhs_data.tag == tag and
	1042	std.meta.eql(lhs, ctx.builder.typeExtraData(Type.Vector, rhs_data.data));
	1043	}
	1044	};
	1045	const data = Type.Vector{ .len = len, .child = child };
	1046	const gop = self.type_map.getOrPutAssumeCapacityAdapted(data, Adapter{ .builder = self });
	1047	if (!gop.found_existing) {
	1048	gop.key_ptr.* = {};
	1049	gop.value_ptr.* = {};
	1050	self.type_items.appendAssumeCapacity(.{
	1051	.tag = tag,
	1052	.data = self.addTypeExtraAssumeCapacity(data),
	1053	});
	1054	if (self.useLibLlvm()) self.llvm_types.appendAssumeCapacity(switch (kind) {
	1055	.normal => &llvm.Type.vectorType,
	1056	.scalable => &llvm.Type.scalableVectorType,
	1057	}(child.toLlvm(self), @intCast(len)));
	1058	}
	1059	return @enumFromInt(gop.index);
	1060	}
	1061
	1062	fn arrayTypeAssumeCapacity(self: *Builder, len: u64, child: Type) Type {
	1063	if (std.math.cast(u32, len)) \|small_len\| {
	1064	const Adapter = struct {
	1065	builder: *const Builder,
	1066	pub fn hash(_: @This(), key: Type.Vector) u32 {
	1067	return @truncate(std.hash.Wyhash.hash(
	1068	comptime std.hash.uint32(@intFromEnum(Type.Tag.small_array)),
	1069	std.mem.asBytes(&key),
	1070	));
	1071	}
	1072	pub fn eql(ctx: @This(), lhs: Type.Vector, _: void, rhs_index: usize) bool {
	1073	const rhs_data = ctx.builder.type_items.items[rhs_index];
	1074	return rhs_data.tag == .small_array and
	1075	std.meta.eql(lhs, ctx.builder.typeExtraData(Type.Vector, rhs_data.data));
	1076	}
	1077	};
	1078	const data = Type.Vector{ .len = small_len, .child = child };
	1079	const gop = self.type_map.getOrPutAssumeCapacityAdapted(data, Adapter{ .builder = self });
	1080	if (!gop.found_existing) {
	1081	gop.key_ptr.* = {};
	1082	gop.value_ptr.* = {};
	1083	self.type_items.appendAssumeCapacity(.{
	1084	.tag = .small_array,
	1085	.data = self.addTypeExtraAssumeCapacity(data),
	1086	});
	1087	if (self.useLibLlvm()) self.llvm_types.appendAssumeCapacity(
	1088	child.toLlvm(self).arrayType(@intCast(len)),
	1089	);
	1090	}
	1091	return @enumFromInt(gop.index);
	1092	} else {
	1093	const Adapter = struct {
	1094	builder: *const Builder,
	1095	pub fn hash(_: @This(), key: Type.Array) u32 {
	1096	return @truncate(std.hash.Wyhash.hash(
	1097	comptime std.hash.uint32(@intFromEnum(Type.Tag.array)),
	1098	std.mem.asBytes(&key),
	1099	));
	1100	}
	1101	pub fn eql(ctx: @This(), lhs: Type.Array, _: void, rhs_index: usize) bool {
	1102	const rhs_data = ctx.builder.type_items.items[rhs_index];
	1103	return rhs_data.tag == .array and
	1104	std.meta.eql(lhs, ctx.builder.typeExtraData(Type.Array, rhs_data.data));
	1105	}
	1106	};
	1107	const data = Type.Array{
	1108	.len_lo = @truncate(len),
	1109	.len_hi = @intCast(len >> 32),
	1110	.child = child,
	1111	};
	1112	const gop = self.type_map.getOrPutAssumeCapacityAdapted(data, Adapter{ .builder = self });
	1113	if (!gop.found_existing) {
	1114	gop.key_ptr.* = {};
	1115	gop.value_ptr.* = {};
	1116	self.type_items.appendAssumeCapacity(.{
	1117	.tag = .array,
	1118	.data = self.addTypeExtraAssumeCapacity(data),
	1119	});
	1120	if (self.useLibLlvm()) self.llvm_types.appendAssumeCapacity(
	1121	child.toLlvm(self).arrayType(@intCast(len)),
	1122	);
	1123	}
	1124	return @enumFromInt(gop.index);
	1125	}
	1126	}
	1127
	1128	fn structTypeAssumeCapacity(
	1129	self: *Builder,
	1130	comptime kind: Type.Structure.Kind,
	1131	fields: []const Type,
	1132	) if (build_options.have_llvm) Allocator.Error!Type else Type {
	1133	const tag: Type.Tag = switch (kind) {
	1134	.normal => .structure,
	1135	.@"packed" => .packed_structure,
	1136	};
	1137	const Adapter = struct {
	1138	builder: *const Builder,
	1139	pub fn hash(_: @This(), key: []const Type) u32 {
	1140	return @truncate(std.hash.Wyhash.hash(
	1141	comptime std.hash.uint32(@intFromEnum(tag)),
	1142	std.mem.sliceAsBytes(key),
	1143	));
	1144	}
	1145	pub fn eql(ctx: @This(), lhs: []const Type, _: void, rhs_index: usize) bool {
	1146	const rhs_data = ctx.builder.type_items.items[rhs_index];
	1147	const rhs_extra = ctx.builder.typeExtraDataTrail(Type.Structure, rhs_data.data);
	1148	const rhs_fields: []const Type =
	1149	@ptrCast(ctx.builder.type_extra.items[rhs_extra.end..][0..rhs_extra.data.fields_len]);
	1150	return rhs_data.tag == tag and std.mem.eql(Type, lhs, rhs_fields);
	1151	}
	1152	};
	1153	const gop = self.type_map.getOrPutAssumeCapacityAdapted(fields, Adapter{ .builder = self });
	1154	if (!gop.found_existing) {
	1155	gop.key_ptr.* = {};
	1156	gop.value_ptr.* = {};
	1157	self.type_items.appendAssumeCapacity(.{
	1158	.tag = tag,
	1159	.data = self.addTypeExtraAssumeCapacity(Type.Structure{
	1160	.fields_len = @intCast(fields.len),
	1161	}),
	1162	});
	1163	self.type_extra.appendSliceAssumeCapacity(@ptrCast(fields));
	1164	if (self.useLibLlvm()) {
	1165	const llvm_fields = try self.gpa.alloc(*llvm.Type, fields.len);
	1166	defer self.gpa.free(llvm_fields);
	1167	for (llvm_fields, fields) \|*llvm_field, field\|
	1168	llvm_field.* = self.llvm_types.items[@intFromEnum(field)];
	1169	self.llvm_types.appendAssumeCapacity(self.llvm_context.structType(
	1170	llvm_fields.ptr,
	1171	@intCast(llvm_fields.len),
	1172	switch (kind) {
	1173	.normal => .False,
	1174	.@"packed" => .True,
	1175	},
	1176	));
	1177	}
	1178	}
	1179	return @enumFromInt(gop.index);
	1180	}
	1181
696	1182	fn opaqueTypeAssumeCapacity(self: *Builder, name: String) Type {
697	1183	const Adapter = struct {
698	1184	builder: *const Builder,
699	1185	pub fn hash(_: @This(), key: String) u32 {
700		return std.hash.uint32(@intFromEnum(key));
	1186	return @truncate(std.hash.Wyhash.hash(
	1187	comptime std.hash.uint32(@intFromEnum(Type.Tag.named_structure)),
	1188	std.mem.asBytes(&key),
	1189	));
701	1190	}
702	1191	pub fn eql(ctx: @This(), lhs: String, _: void, rhs_index: usize) bool {
703		const rhs_data = ctx.builder.type_data.items[rhs_index];
	1192	const rhs_data = ctx.builder.type_items.items[rhs_index];
704	1193	return rhs_data.tag == .named_structure and
705	1194	lhs == ctx.builder.typeExtraData(Type.NamedStructure, rhs_data.data).id;
706	1195	}
...	...	@@ -718,11 +1207,11 @@ fn opaqueTypeAssumeCapacity(self: *Builder, name: String) Type {
718	1207	assert(!gop.found_existing);
719	1208	gop.key_ptr.* = {};
720	1209	gop.value_ptr.* = {};
721		self.type_data.appendAssumeCapacity(.{
	1210	self.type_items.appendAssumeCapacity(.{
722	1211	.tag = .named_structure,
723	1212	.data = self.addTypeExtraAssumeCapacity(Type.NamedStructure{
724	1213	.id = id,
725		.child = .none,
	1214	.body = .none,
726	1215	}),
727	1216	});
728	1217	const result: Type = @enumFromInt(gop.index);
...	...	@@ -740,53 +1229,49 @@ fn opaqueTypeAssumeCapacity(self: *Builder, name: String) Type {
740	1229	}
741	1230	}
742	1231
743		fn intTypeAssumeCapacity(self: *Builder, bits: u24) Type {
744		const result = self.typeNoExtraAssumeCapacity(.{ .tag = .integer, .data = bits });
745		if (self.useLibLlvm() and result.new)
746		self.llvm_types.appendAssumeCapacity(self.llvm_context.intType(bits));
747		return result.type;
748		}
749
750		fn pointerTypeAssumeCapacity(self: *Builder, addr_space: AddrSpace) Type {
751		const result = self.typeNoExtraAssumeCapacity(.{ .tag = .pointer, .data = @intFromEnum(addr_space) });
752		if (self.useLibLlvm() and result.new)
753		self.llvm_types.appendAssumeCapacity(self.llvm_context.pointerType(@intFromEnum(addr_space)));
754		return result.type;
755		}
756
757		fn ensureUnusedCapacityTypes(self: *Builder, count: usize, comptime Extra: ?type) Allocator.Error!void {
	1232	fn ensureUnusedCapacityTypes(
	1233	self: *Builder,
	1234	count: usize,
	1235	comptime Extra: ?type,
	1236	trail_len: usize,
	1237	) Allocator.Error!void {
758	1238	try self.type_map.ensureUnusedCapacity(self.gpa, count);
759		try self.type_data.ensureUnusedCapacity(self.gpa, count);
760		if (Extra) \|E\|
761		try self.type_extra.ensureUnusedCapacity(self.gpa, count * @typeInfo(E).Struct.fields.len);
	1239	try self.type_items.ensureUnusedCapacity(self.gpa, count);
	1240	if (Extra) \|E\| try self.type_extra.ensureUnusedCapacity(
	1241	self.gpa,
	1242	count * (@typeInfo(E).Struct.fields.len + trail_len),
	1243	);
762	1244	if (self.useLibLlvm()) try self.llvm_types.ensureUnusedCapacity(self.gpa, count);
763	1245	}
764	1246
765		fn typeNoExtraAssumeCapacity(self: *Builder, data: Type.Data) struct { new: bool, type: Type } {
	1247	fn typeNoExtraAssumeCapacity(self: *Builder, item: Type.Item) struct { new: bool, type: Type } {
766	1248	const Adapter = struct {
767	1249	builder: *const Builder,
768		pub fn hash(_: @This(), key: Type.Data) u32 {
769		return std.hash.uint32(@bitCast(key));
	1250	pub fn hash(_: @This(), key: Type.Item) u32 {
	1251	return @truncate(std.hash.Wyhash.hash(
	1252	comptime std.hash.uint32(@intFromEnum(Type.Tag.simple)),
	1253	std.mem.asBytes(&key),
	1254	));
770	1255	}
771		pub fn eql(ctx: @This(), lhs: Type.Data, _: void, rhs_index: usize) bool {
	1256	pub fn eql(ctx: @This(), lhs: Type.Item, _: void, rhs_index: usize) bool {
772	1257	const lhs_bits: u32 = @bitCast(lhs);
773		const rhs_bits: u32 = @bitCast(ctx.builder.type_data.items[rhs_index]);
	1258	const rhs_bits: u32 = @bitCast(ctx.builder.type_items.items[rhs_index]);
774	1259	return lhs_bits == rhs_bits;
775	1260	}
776	1261	};
777		const gop = self.type_map.getOrPutAssumeCapacityAdapted(data, Adapter{ .builder = self });
	1262	const gop = self.type_map.getOrPutAssumeCapacityAdapted(item, Adapter{ .builder = self });
778	1263	if (!gop.found_existing) {
779	1264	gop.key_ptr.* = {};
780	1265	gop.value_ptr.* = {};
781		self.type_data.appendAssumeCapacity(data);
	1266	self.type_items.appendAssumeCapacity(item);
782	1267	}
783	1268	return .{ .new = !gop.found_existing, .type = @enumFromInt(gop.index) };
784	1269	}
785	1270
786	1271	fn isValidIdentifier(id: []const u8) bool {
787		for (id, 0..) \|c, i\| switch (c) {
	1272	for (id, 0..) \|character, index\| switch (character) {
788	1273	'$', '-', '.', 'A'...'Z', '_', 'a'...'z' => {},
789		'0'...'9' => if (i == 0) return false,
	1274	'0'...'9' => if (index == 0) return false,
790	1275	else => return false,
791	1276	};
792	1277	return true;
...	...	@@ -815,7 +1300,7 @@ pub fn dump(self: *Builder, writer: anytype) @TypeOf(writer).Error!void {
815	1300	for (self.objects.items) \|object\| {
816	1301	const global = self.globals.entries.get(@intFromEnum(object.global));
817	1302	try writer.print(
818		\\@{} = {}{}{}{}{}{}{}{}{s} {}{,}
	1303	\\@{} ={}{}{}{}{}{}{}{} {s} {%}{,}
819	1304	\\
820	1305	, .{
821	1306	global.key.fmt(self),
...	...	@@ -836,8 +1321,8 @@ pub fn dump(self: *Builder, writer: anytype) @TypeOf(writer).Error!void {
836	1321	for (self.functions.items) \|function\| {
837	1322	const global = self.globals.entries.get(@intFromEnum(function.global));
838	1323	try writer.print(
839		\\{s} {}{}{}{}void @{}() {}{}{{
840		\\ ret void
	1324	\\{s} {}{}{}{}{<}@{}{>} {}{}{{
	1325	\\ ret {%}
841	1326	\\}}
842	1327	\\
843	1328	, .{
...	...	@@ -846,9 +1331,15 @@ pub fn dump(self: *Builder, writer: anytype) @TypeOf(writer).Error!void {
846	1331	global.value.preemption,
847	1332	global.value.visibility,
848	1333	global.value.dll_storage_class,
	1334	global.value.type.fmt(self),
849	1335	global.key.fmt(self),
	1336	global.value.type.fmt(self),
850	1337	global.value.unnamed_addr,
851	1338	global.value.alignment,
	1339	self.typeExtraData(
	1340	Type.Function,
	1341	self.type_items.items[@intFromEnum(global.value.type)].data,
	1342	).ret.fmt(self),
852	1343	});
853	1344	}
854	1345	try writer.writeByte('\n');

src/codegen/llvm/bindings.zig+6-3

...	...	@@ -317,6 +317,9 @@ pub const Type = opaque {
317	317	pub const vectorType = LLVMVectorType;
318	318	extern fn LLVMVectorType(ElementType: Type, ElementCount: c_uint) Type;
319	319
	320	pub const scalableVectorType = LLVMScalableVectorType;
	321	extern fn LLVMScalableVectorType(ElementType: Type, ElementCount: c_uint) Type;
	322
320	323	pub const structSetBody = LLVMStructSetBody;
321	324	extern fn LLVMStructSetBody(
322	325	StructTy: *Type,
...	...	@@ -1615,9 +1618,9 @@ pub const address_space = struct {
1615	1618
1616	1619	// See llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h
1617	1620	pub const wasm = struct {
1618		pub const variable = 1;
1619		pub const externref = 10;
1620		pub const funcref = 20;
	1621	pub const variable: c_uint = 1;
	1622	pub const externref: c_uint = 10;
	1623	pub const funcref: c_uint = 20;
1621	1624	};
1622	1625	};
1623	1626