Commit 65f860bef7995a6120e49606d549bdf154bca150 | zig

doc/langref.html.in+9

...	...	@@ -7956,6 +7956,15 @@ fn readFile(allocator: Allocator, filename: []const u8) ![]u8 {
7956	7956	The {#syntax#}comptime{#endsyntax#} keyword on a parameter means that the parameter must be known
7957	7957	at compile time.
7958	7958	</p>
	7959	{#header_open\|@addrSpaceCast#}
	7960	<pre>{#syntax#}@addrSpaceCast(comptime addrspace: std.builtin.AddressSpace, ptr: anytype) anytype{#endsyntax#}</pre>
	7961	<p>
	7962	Converts a pointer from one address space to another. Depending on the current target and
	7963	address spaces, this cast may be a no-op, a complex operation, or illegal. If the cast is
	7964	legal, then the resulting pointer points to the same memory location as the pointer operand.
	7965	It is always valid to cast a pointer between the same address spaces.
	7966	</p>
	7967	{#header_close#}
7959	7968	{#header_open\|@addWithOverflow#}
7960	7969	<pre>{#syntax#}@addWithOverflow(comptime T: type, a: T, b: T, result: *T) bool{#endsyntax#}</pre>
7961	7970	<p>

lib/c.zig+3-3

...	...	@@ -64,10 +64,10 @@ pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, _: ?
64	64	if (builtin.is_test) {
65	65	std.debug.panic("{s}", .{msg});
66	66	}
67		if (native_os != .freestanding and native_os != .other) {
68		std.os.abort();
	67	switch (native_os) {
	68	.freestanding, .other, .amdhsa, .amdpal => while (true) {},
	69	else => std.os.abort(),
69	70	}
70		while (true) {}
71	71	}
72	72
73	73	extern fn main(argc: c_int, argv: [:null]?[:0]u8) c_int;

lib/compiler_rt/atomics.zig+63-10

...	...	@@ -35,6 +35,17 @@ const largest_atomic_size = switch (arch) {
35	35	else => @sizeOf(usize),
36	36	};
37	37
	38	// The size (in bytes) of the smallest atomic object that the architecture can
	39	// perform fetch/exchange atomically. Note, this does not encompass load and store.
	40	// Objects smaller than this threshold are implemented in terms of compare-exchange
	41	// of a larger value.
	42	const smallest_atomic_fetch_exch_size = switch (arch) {
	43	// On AMDGPU, there are no instructions for atomic operations other than load and store
	44	// (as of LLVM 15), and so these need to be implemented in terms of atomic CAS.
	45	.amdgcn => @sizeOf(u32),
	46	else => @sizeOf(u8),
	47	};
	48
38	49	const cache_line_size = 64;
39	50
40	51	const SpinlockTable = struct {
...	...	@@ -206,6 +217,31 @@ fn __atomic_store_8(dst: *u64, value: u64, model: i32) callconv(.C) void {
206	217	return atomic_store_N(u64, dst, value, model);
207	218	}
208	219
	220	fn wideUpdate(comptime T: type, ptr: *T, val: T, update: anytype) T {
	221	const WideAtomic = std.meta.Int(.unsigned, smallest_atomic_fetch_exch_size * 8);
	222
	223	const addr = @ptrToInt(ptr);
	224	const wide_addr = addr & ~(@as(T, smallest_atomic_fetch_exch_size) - 1);
	225	const wide_ptr = @alignCast(smallest_atomic_fetch_exch_size, @intToPtr(*WideAtomic, wide_addr));
	226
	227	const inner_offset = addr & (@as(T, smallest_atomic_fetch_exch_size) - 1);
	228	const inner_shift = @intCast(std.math.Log2Int(T), inner_offset * 8);
	229
	230	const mask = @as(WideAtomic, std.math.maxInt(T)) << inner_shift;
	231
	232	var wide_old = @atomicLoad(WideAtomic, wide_ptr, .SeqCst);
	233	while (true) {
	234	const old = @truncate(T, (wide_old & mask) >> inner_shift);
	235	const new = update(val, old);
	236	const wide_new = wide_old & ~mask \| (@as(WideAtomic, new) << inner_shift);
	237	if (@cmpxchgWeak(WideAtomic, wide_ptr, wide_old, wide_new, .SeqCst, .SeqCst)) \|new_wide_old\| {
	238	wide_old = new_wide_old;
	239	} else {
	240	return old;
	241	}
	242	}
	243	}
	244
209	245	inline fn atomic_exchange_N(comptime T: type, ptr: *T, val: T, model: i32) T {
210	246	_ = model;
211	247	if (@sizeOf(T) > largest_atomic_size) {
...	...	@@ -214,6 +250,15 @@ inline fn atomic_exchange_N(comptime T: type, ptr: *T, val: T, model: i32) T {
214	250	const value = ptr.*;
215	251	ptr.* = val;
216	252	return value;
	253	} else if (@sizeOf(T) < smallest_atomic_fetch_exch_size) {
	254	// Machine does not support this type, but it does support a larger type.
	255	const Updater = struct {
	256	fn update(new: T, old: T) T {
	257	_ = old;
	258	return new;
	259	}
	260	};
	261	return wideUpdate(T, ptr, val, Updater.update);
217	262	} else {
218	263	return @atomicRmw(T, ptr, .Xchg, val, .SeqCst);
219	264	}
...	...	@@ -282,22 +327,30 @@ fn __atomic_compare_exchange_8(ptr: u64, expected: u64, desired: u64, success:
282	327
283	328	inline fn fetch_op_N(comptime T: type, comptime op: std.builtin.AtomicRmwOp, ptr: *T, val: T, model: i32) T {
284	329	_ = model;
	330	const Updater = struct {
	331	fn update(new: T, old: T) T {
	332	return switch (op) {
	333	.Add => old +% new,
	334	.Sub => old -% new,
	335	.And => old & new,
	336	.Nand => ~(old & new),
	337	.Or => old \| new,
	338	.Xor => old ^ new,
	339	else => @compileError("unsupported atomic op"),
	340	};
	341	}
	342	};
	343
285	344	if (@sizeOf(T) > largest_atomic_size) {
286	345	var sl = spinlocks.get(@ptrToInt(ptr));
287	346	defer sl.release();
288	347
289	348	const value = ptr.*;
290		ptr.* = switch (op) {
291		.Add => value +% val,
292		.Sub => value -% val,
293		.And => value & val,
294		.Nand => ~(value & val),
295		.Or => value \| val,
296		.Xor => value ^ val,
297		else => @compileError("unsupported atomic op"),
298		};
299
	349	ptr.* = Updater.update(val, value);
300	350	return value;
	351	} else if (@sizeOf(T) < smallest_atomic_fetch_exch_size) {
	352	// Machine does not support this type, but it does support a larger type.
	353	return wideUpdate(T, ptr, val, Updater.update);
301	354	}
302	355
303	356	return @atomicRmw(T, ptr, op, val, .SeqCst);

lib/std/builtin.zig+1

...	...	@@ -157,6 +157,7 @@ pub const CallingConvention = enum {
157	157	SysV,
158	158	Win64,
159	159	PtxKernel,
	160	AmdgpuKernel,
160	161	};
161	162
162	163	/// This data structure is used by the Zig language code generation and

lib/std/math/big/int.zig+1-1

...	...	@@ -1859,7 +1859,7 @@ pub const Mutable = struct {
1859	1859	/// [1, 2, 3, 4, 0] -> [1, 2, 3, 4]
1860	1860	/// [1, 2, 0, 0, 0] -> [1, 2]
1861	1861	/// [0, 0, 0, 0, 0] -> [0]
1862		fn normalize(r: *Mutable, length: usize) void {
	1862	pub fn normalize(r: *Mutable, length: usize) void {
1863	1863	r.len = llnormalize(r.limbs[0..length]);
1864	1864	}
1865	1865	};

lib/std/target.zig+11

...	...	@@ -1157,6 +1157,17 @@ pub const Target = struct {
1157	1157	};
1158	1158	}
1159	1159
	1160	/// Returns whether this architecture supports the address space
	1161	pub fn supportsAddressSpace(arch: Arch, address_space: std.builtin.AddressSpace) bool {
	1162	const is_nvptx = arch == .nvptx or arch == .nvptx64;
	1163	return switch (address_space) {
	1164	.generic => true,
	1165	.fs, .gs, .ss => arch == .x86_64 or arch == .i386,
	1166	.global, .constant, .local, .shared => arch == .amdgcn or is_nvptx,
	1167	.param => is_nvptx,
	1168	};
	1169	}
	1170
1160	1171	pub fn ptrBitWidth(arch: Arch) u16 {
1161	1172	switch (arch) {
1162	1173	.avr,

src/Air.zig+5

...	...	@@ -729,6 +729,10 @@ pub const Inst = struct {
729	729	/// Sets the operand as the current error return trace,
730	730	set_err_return_trace,
731	731
	732	/// Convert the address space of a pointer.
	733	/// Uses the `ty_op` field.
	734	addrspace_cast,
	735
732	736	pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {
733	737	switch (op) {
734	738	.lt => return if (optimized) .cmp_lt_optimized else .cmp_lt,
...	...	@@ -1138,6 +1142,7 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
1138	1142	.popcount,
1139	1143	.byte_swap,
1140	1144	.bit_reverse,
	1145	.addrspace_cast,
1141	1146	=> return air.getRefType(datas[inst].ty_op.ty),
1142	1147
1143	1148	.loop,

src/AstGen.zig+8

...	...	@@ -7789,6 +7789,14 @@ fn builtinCall(
7789	7789	});
7790	7790	return rvalue(gz, rl, result, node);
7791	7791	},
	7792	.addrspace_cast => {
	7793	const result = try gz.addExtendedPayload(.addrspace_cast, Zir.Inst.BinNode{
	7794	.lhs = try comptimeExpr(gz, scope, .{ .ty = .address_space_type }, params[0]),
	7795	.rhs = try expr(gz, scope, .none, params[1]),
	7796	.node = gz.nodeIndexToRelative(node),
	7797	});
	7798	return rvalue(gz, rl, result, node);
	7799	},
7792	7800
7793	7801	// zig fmt: off
7794	7802	.has_decl => return hasDeclOrField(gz, scope, rl, node, params[0], params[1], .has_decl),

src/BuiltinFn.zig+8

...	...	@@ -2,6 +2,7 @@ const std = @import("std");
2	2
3	3	pub const Tag = enum {
4	4	add_with_overflow,
	5	addrspace_cast,
5	6	align_cast,
6	7	align_of,
7	8	as,
...	...	@@ -152,6 +153,13 @@ pub const list = list: {
152	153	.param_count = 4,
153	154	},
154	155	},
	156	.{
	157	"@addrSpaceCast",
	158	.{
	159	.tag = .addrspace_cast,
	160	.param_count = 2,
	161	},
	162	},
155	163	.{
156	164	"@alignCast",
157	165	.{

src/Liveness.zig+2

...	...	@@ -268,6 +268,7 @@ pub fn categorizeOperand(
268	268	.bit_reverse,
269	269	.splat,
270	270	.error_set_has_value,
	271	.addrspace_cast,
271	272	=> {
272	273	const o = air_datas[inst].ty_op;
273	274	if (o.operand == operand_ref) return matchOperandSmallIndex(l, inst, 0, .none);
...	...	@@ -844,6 +845,7 @@ fn analyzeInst(
844	845	.bit_reverse,
845	846	.splat,
846	847	.error_set_has_value,
	848	.addrspace_cast,
847	849	=> {
848	850	const o = inst_datas[inst].ty_op;
849	851	return trackOperands(a, new_set, inst, main_tomb, .{ o.operand, .none, .none });

src/Module.zig+1-1

...	...	@@ -4617,7 +4617,7 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool {
4617	4617	.constant => target_util.defaultAddressSpace(target, .global_constant),
4618	4618	else => unreachable,
4619	4619	},
4620		else => \|addrspace_ref\| try sema.analyzeAddrspace(&block_scope, address_space_src, addrspace_ref, addrspace_ctx),
	4620	else => \|addrspace_ref\| try sema.analyzeAddressSpace(&block_scope, address_space_src, addrspace_ref, addrspace_ctx),
4621	4621	};
4622	4622	};
4623	4623

src/Sema.zig+65-6

...	...	@@ -975,8 +975,9 @@ fn analyzeBodyInner(
975	975	.reify => try sema.zirReify( block, extended, inst),
976	976	.builtin_async_call => try sema.zirBuiltinAsyncCall( block, extended),
977	977	.cmpxchg => try sema.zirCmpxchg( block, extended),
978
	978	.addrspace_cast => try sema.zirAddrSpaceCast( block, extended),
979	979	// zig fmt: on
	980
980	981	.fence => {
981	982	try sema.zirFence(block, extended);
982	983	i += 1;
...	...	@@ -5897,7 +5898,7 @@ fn analyzeCall(
5897	5898	},
5898	5899	else => {},
5899	5900	}
5900		return sema.fail(block, func_src, "type '{}' not a function", .{callee_ty.fmt(sema.mod)});
	5901	return sema.fail(block, func_src, "type '{}' is not a function", .{callee_ty.fmt(sema.mod)});
5901	5902	};
5902	5903
5903	5904	const func_ty_info = func_ty.fnInfo();
...	...	@@ -8141,6 +8142,10 @@ fn funcCommon(
8141	8142	.nvptx, .nvptx64 => null,
8142	8143	else => @as([]const u8, "nvptx and nvptx64"),
8143	8144	},
	8145	.AmdgpuKernel => switch (arch) {
	8146	.amdgcn => null,
	8147	else => @as([]const u8, "amdgcn"),
	8148	},
8144	8149	}) \|allowed_platform\| {
8145	8150	return sema.fail(block, cc_src, "callconv '{s}' is only available on {s}, not {s}", .{
8146	8151	@tagName(cc_workaround),
...	...	@@ -16246,7 +16251,7 @@ fn zirPtrType(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air
16246	16251	const address_space = if (inst_data.flags.has_addrspace) blk: {
16247	16252	const ref = @intToEnum(Zir.Inst.Ref, sema.code.extra[extra_i]);
16248	16253	extra_i += 1;
16249		break :blk try sema.analyzeAddrspace(block, addrspace_src, ref, .pointer);
	16254	break :blk try sema.analyzeAddressSpace(block, addrspace_src, ref, .pointer);
16250	16255	} else .generic;
16251	16256
16252	16257	const bit_offset = if (inst_data.flags.has_bit_range) blk: {
...	...	@@ -18166,6 +18171,55 @@ fn reifyStruct(
18166	18171	return sema.analyzeDeclVal(block, src, new_decl_index);
18167	18172	}
18168	18173
	18174	fn zirAddrSpaceCast(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref {
	18175	const extra = sema.code.extraData(Zir.Inst.BinNode, extended.operand).data;
	18176	const src = LazySrcLoc.nodeOffset(extra.node);
	18177	const addrspace_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = extra.node };
	18178	const ptr_src: LazySrcLoc = .{ .node_offset_builtin_call_arg1 = extra.node };
	18179
	18180	const dest_addrspace = try sema.analyzeAddressSpace(block, addrspace_src, extra.lhs, .pointer);
	18181	const ptr = try sema.resolveInst(extra.rhs);
	18182	const ptr_ty = sema.typeOf(ptr);
	18183
	18184	try sema.checkPtrOperand(block, ptr_src, ptr_ty);
	18185
	18186	var ptr_info = ptr_ty.ptrInfo().data;
	18187	const src_addrspace = ptr_info.@"addrspace";
	18188	if (!target_util.addrSpaceCastIsValid(sema.mod.getTarget(), src_addrspace, dest_addrspace)) {
	18189	const msg = msg: {
	18190	const msg = try sema.errMsg(block, src, "invalid address space cast", .{});
	18191	errdefer msg.destroy(sema.gpa);
	18192	try sema.errNote(block, src, msg, "address space '{s}' is not compatible with address space '{s}'", .{ @tagName(src_addrspace), @tagName(dest_addrspace) });
	18193	break :msg msg;
	18194	};
	18195	return sema.failWithOwnedErrorMsg(msg);
	18196	}
	18197
	18198	ptr_info.@"addrspace" = dest_addrspace;
	18199	const dest_ptr_ty = try Type.ptr(sema.arena, sema.mod, ptr_info);
	18200	const dest_ty = if (ptr_ty.zigTypeTag() == .Optional)
	18201	try Type.optional(sema.arena, dest_ptr_ty)
	18202	else
	18203	dest_ptr_ty;
	18204
	18205	if (try sema.resolveMaybeUndefVal(block, ptr_src, ptr)) \|val\| {
	18206	// Pointer value should compatible with both address spaces.
	18207	// TODO: Figure out why this generates an invalid bitcast.
	18208	return sema.addConstant(dest_ty, val);
	18209	}
	18210
	18211	try sema.requireRuntimeBlock(block, src, ptr_src);
	18212	// TODO: Address space cast safety?
	18213
	18214	return block.addInst(.{
	18215	.tag = .addrspace_cast,
	18216	.data = .{ .ty_op = .{
	18217	.ty = try sema.addType(dest_ty),
	18218	.operand = ptr,
	18219	} },
	18220	});
	18221	}
	18222
18169	18223	fn zirTypeName(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
18170	18224	const inst_data = sema.code.instructions.items(.data)[inst].un_node;
18171	18225	const ty_src: LazySrcLoc = .{ .node_offset_builtin_call_arg0 = inst_data.src_node };
...	...	@@ -18413,6 +18467,9 @@ fn zirPtrCast(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air
18413	18467	if (operand_info.@"volatile" and !dest_info.@"volatile") {
18414	18468	return sema.fail(block, src, "cast discards volatile qualifier", .{});
18415	18469	}
	18470	if (operand_info.@"addrspace" != dest_info.@"addrspace") {
	18471	return sema.fail(block, src, "cast changes pointer address space", .{});
	18472	}
18416	18473
18417	18474	const dest_is_slice = dest_ty.isSlice();
18418	18475	const operand_is_slice = operand_ty.isSlice();
...	...	@@ -30302,7 +30359,7 @@ pub const AddressSpaceContext = enum {
30302	30359	pointer,
30303	30360	};
30304	30361
30305		pub fn analyzeAddrspace(
	30362	pub fn analyzeAddressSpace(
30306	30363	sema: *Sema,
30307	30364	block: *Block,
30308	30365	src: LazySrcLoc,
...	...	@@ -30313,13 +30370,15 @@ pub fn analyzeAddrspace(
30313	30370	const address_space = addrspace_tv.val.toEnum(std.builtin.AddressSpace);
30314	30371	const target = sema.mod.getTarget();
30315	30372	const arch = target.cpu.arch;
30316		const is_gpu = arch == .nvptx or arch == .nvptx64;
	30373	const is_nv = arch == .nvptx or arch == .nvptx64;
	30374	const is_gpu = is_nv or arch == .amdgcn;
30317	30375
30318	30376	const supported = switch (address_space) {
30319	30377	.generic => true,
30320	30378	.gs, .fs, .ss => (arch == .i386 or arch == .x86_64) and ctx == .pointer,
30321	30379	// TODO: check that .shared and .local are left uninitialized
30322		.global, .param, .shared, .local => is_gpu,
	30380	.param => is_nv,
	30381	.global, .shared, .local => is_gpu,
30323	30382	.constant => is_gpu and (ctx == .constant),
30324	30383	};
30325	30384

src/Zir.zig+3

...	...	@@ -1969,6 +1969,9 @@ pub const Inst = struct {
1969	1969	/// `small` 0=>weak 1=>strong
1970	1970	/// `operand` is payload index to `Cmpxchg`.
1971	1971	cmpxchg,
	1972	/// Implement the builtin `@addrSpaceCast`
	1973	/// `Operand` is payload index to `BinNode`. `lhs` is dest type, `rhs` is operand.
	1974	addrspace_cast,
1972	1975
1973	1976	pub const InstData = struct {
1974	1977	opcode: Extended,

src/arch/aarch64/CodeGen.zig+1

...	...	@@ -677,6 +677,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
677	677	.union_init => try self.airUnionInit(inst),
678	678	.prefetch => try self.airPrefetch(inst),
679	679	.mul_add => try self.airMulAdd(inst),
	680	.addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
680	681
681	682	.@"try" => try self.airTry(inst),
682	683	.try_ptr => try self.airTryPtr(inst),

src/arch/arm/CodeGen.zig+1

...	...	@@ -690,6 +690,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
690	690	.union_init => try self.airUnionInit(inst),
691	691	.prefetch => try self.airPrefetch(inst),
692	692	.mul_add => try self.airMulAdd(inst),
	693	.addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
693	694
694	695	.@"try" => try self.airTry(inst),
695	696	.try_ptr => try self.airTryPtr(inst),

src/arch/riscv64/CodeGen.zig+1

...	...	@@ -604,6 +604,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
604	604	.union_init => try self.airUnionInit(inst),
605	605	.prefetch => try self.airPrefetch(inst),
606	606	.mul_add => try self.airMulAdd(inst),
	607	.addrspace_cast => @panic("TODO"),
607	608
608	609	.@"try" => @panic("TODO"),
609	610	.try_ptr => @panic("TODO"),

src/arch/sparc64/CodeGen.zig+1

...	...	@@ -618,6 +618,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
618	618	.union_init => @panic("TODO try self.airUnionInit(inst)"),
619	619	.prefetch => try self.airPrefetch(inst),
620	620	.mul_add => @panic("TODO try self.airMulAdd(inst)"),
	621	.addrspace_cast => @panic("TODO try self.airAddrSpaceCast(int)"),
621	622
622	623	.@"try" => try self.airTry(inst),
623	624	.try_ptr => @panic("TODO try self.airTryPtr(inst)"),

src/arch/wasm/CodeGen.zig+1

...	...	@@ -1699,6 +1699,7 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
1699	1699	.set_err_return_trace,
1700	1700	.is_named_enum_value,
1701	1701	.error_set_has_value,
	1702	.addrspace_cast,
1702	1703	=> \|tag\| return self.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),
1703	1704
1704	1705	.add_optimized,

src/arch/x86_64/CodeGen.zig+1

...	...	@@ -695,6 +695,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
695	695	.union_init => try self.airUnionInit(inst),
696	696	.prefetch => try self.airPrefetch(inst),
697	697	.mul_add => try self.airMulAdd(inst),
	698	.addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}),
698	699
699	700	.@"try" => try self.airTry(inst),
700	701	.try_ptr => try self.airTryPtr(inst),

src/codegen/c.zig+1

...	...	@@ -1871,6 +1871,7 @@ fn genBody(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, OutO
1871	1871	.aggregate_init => try airAggregateInit(f, inst),
1872	1872	.union_init => try airUnionInit(f, inst),
1873	1873	.prefetch => try airPrefetch(f, inst),
	1874	.addrspace_cast => return f.fail("TODO: C backend: implement addrspace_cast", .{}),
1874	1875
1875	1876	.@"try" => try airTry(f, inst),
1876	1877	.try_ptr => try airTryPtr(f, inst),

src/codegen/llvm.zig+212-120

...	...	@@ -956,8 +956,7 @@ pub const Object = struct {
956	956	if (isByRef(param_ty)) {
957	957	const alignment = param_ty.abiAlignment(target);
958	958	const param_llvm_ty = param.typeOf();
959		const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty);
960		arg_ptr.setAlignment(alignment);
	959	const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty, alignment, target);
961	960	const store_inst = builder.buildStore(param, arg_ptr);
962	961	store_inst.setAlignment(alignment);
963	962	args.appendAssumeCapacity(arg_ptr);
...	...	@@ -1001,8 +1000,7 @@ pub const Object = struct {
1001	1000	param_ty.abiAlignment(target),
1002	1001	dg.object.target_data.abiAlignmentOfType(int_llvm_ty),
1003	1002	);
1004		const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty);
1005		arg_ptr.setAlignment(alignment);
	1003	const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty, alignment, target);
1006	1004	const casted_ptr = builder.buildBitCast(arg_ptr, int_ptr_llvm_ty, "");
1007	1005	const store_inst = builder.buildStore(param, casted_ptr);
1008	1006	store_inst.setAlignment(alignment);
...	...	@@ -1053,8 +1051,7 @@ pub const Object = struct {
1053	1051	const param_ty = fn_info.param_types[it.zig_index - 1];
1054	1052	const param_llvm_ty = try dg.lowerType(param_ty);
1055	1053	const param_alignment = param_ty.abiAlignment(target);
1056		const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty);
1057		arg_ptr.setAlignment(param_alignment);
	1054	const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty, param_alignment, target);
1058	1055	var field_types_buf: [8]*llvm.Type = undefined;
1059	1056	const field_types = field_types_buf[0..llvm_ints.len];
1060	1057	for (llvm_ints) \|int_bits, i\| {
...	...	@@ -1085,8 +1082,7 @@ pub const Object = struct {
1085	1082	const param_ty = fn_info.param_types[it.zig_index - 1];
1086	1083	const param_llvm_ty = try dg.lowerType(param_ty);
1087	1084	const param_alignment = param_ty.abiAlignment(target);
1088		const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty);
1089		arg_ptr.setAlignment(param_alignment);
	1085	const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty, param_alignment, target);
1090	1086	var field_types_buf: [8]*llvm.Type = undefined;
1091	1087	const field_types = field_types_buf[0..llvm_floats.len];
1092	1088	for (llvm_floats) \|float_bits, i\| {
...	...	@@ -1130,8 +1126,7 @@ pub const Object = struct {
1130	1126	llvm_arg_i += 1;
1131	1127
1132	1128	const alignment = param_ty.abiAlignment(target);
1133		const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty);
1134		arg_ptr.setAlignment(alignment);
	1129	const arg_ptr = buildAllocaInner(builder, llvm_func, false, param_llvm_ty, alignment, target);
1135	1130	const casted_ptr = builder.buildBitCast(arg_ptr, param.typeOf().pointerType(0), "");
1136	1131	_ = builder.buildStore(param, casted_ptr);
1137	1132
...	...	@@ -2431,19 +2426,21 @@ pub const DeclGen = struct {
2431	2426	// mismatch, because we don't have the LLVM type until the value is created,
2432	2427	// whereas the global needs to be created based on the type alone, because
2433	2428	// lowering the value may reference the global as a pointer.
	2429	const llvm_global_addrspace = toLlvmGlobalAddressSpace(decl.@"addrspace", target);
2434	2430	const new_global = dg.object.llvm_module.addGlobalInAddressSpace(
2435	2431	llvm_init.typeOf(),
2436	2432	"",
2437		dg.llvmAddressSpace(decl.@"addrspace"),
	2433	llvm_global_addrspace,
2438	2434	);
2439	2435	new_global.setLinkage(global.getLinkage());
2440	2436	new_global.setUnnamedAddr(global.getUnnamedAddress());
2441	2437	new_global.setAlignment(global.getAlignment());
2442	2438	if (decl.@"linksection") \|section\| new_global.setSection(section);
2443	2439	new_global.setInitializer(llvm_init);
2444		// replaceAllUsesWith requires the type to be unchanged. So we bitcast
	2440	// replaceAllUsesWith requires the type to be unchanged. So we convert
2445	2441	// the new global to the old type and use that as the thing to replace
2446	2442	// old uses.
	2443	// TODO: How should this work then the address space of a global changed?
2447	2444	const new_global_ptr = new_global.constBitCast(global.typeOf());
2448	2445	global.replaceAllUsesWith(new_global_ptr);
2449	2446	dg.object.decl_map.putAssumeCapacity(decl_index, new_global);
...	...	@@ -2492,7 +2489,7 @@ pub const DeclGen = struct {
2492	2489	const fqn = try decl.getFullyQualifiedName(dg.module);
2493	2490	defer dg.gpa.free(fqn);
2494	2491
2495		const llvm_addrspace = dg.llvmAddressSpace(decl.@"addrspace");
	2492	const llvm_addrspace = toLlvmAddressSpace(decl.@"addrspace", target);
2496	2493	const llvm_fn = dg.llvmModule().addFunctionInAddressSpace(fqn, fn_type, llvm_addrspace);
2497	2494	gop.value_ptr.* = llvm_fn;
2498	2495
...	...	@@ -2640,9 +2637,16 @@ pub const DeclGen = struct {
2640	2637	const fqn = try decl.getFullyQualifiedName(dg.module);
2641	2638	defer dg.gpa.free(fqn);
2642	2639
	2640	const target = dg.module.getTarget();
	2641
2643	2642	const llvm_type = try dg.lowerType(decl.ty);
2644		const llvm_addrspace = dg.llvmAddressSpace(decl.@"addrspace");
2645		const llvm_global = dg.object.llvm_module.addGlobalInAddressSpace(llvm_type, fqn, llvm_addrspace);
	2643	const llvm_actual_addrspace = toLlvmGlobalAddressSpace(decl.@"addrspace", target);
	2644
	2645	const llvm_global = dg.object.llvm_module.addGlobalInAddressSpace(
	2646	llvm_type,
	2647	fqn,
	2648	llvm_actual_addrspace,
	2649	);
2646	2650	gop.value_ptr.* = llvm_global;
2647	2651
2648	2652	// This is needed for declarations created by `@extern`.
...	...	@@ -2667,32 +2671,6 @@ pub const DeclGen = struct {
2667	2671	return llvm_global;
2668	2672	}
2669	2673
2670		fn llvmAddressSpace(self: DeclGen, address_space: std.builtin.AddressSpace) c_uint {
2671		const target = self.module.getTarget();
2672		return switch (target.cpu.arch) {
2673		.i386, .x86_64 => switch (address_space) {
2674		.generic => llvm.address_space.default,
2675		.gs => llvm.address_space.x86.gs,
2676		.fs => llvm.address_space.x86.fs,
2677		.ss => llvm.address_space.x86.ss,
2678		else => unreachable,
2679		},
2680		.nvptx, .nvptx64 => switch (address_space) {
2681		.generic => llvm.address_space.default,
2682		.global => llvm.address_space.nvptx.global,
2683		.constant => llvm.address_space.nvptx.constant,
2684		.param => llvm.address_space.nvptx.param,
2685		.shared => llvm.address_space.nvptx.shared,
2686		.local => llvm.address_space.nvptx.local,
2687		else => unreachable,
2688		},
2689		else => switch (address_space) {
2690		.generic => llvm.address_space.default,
2691		else => unreachable,
2692		},
2693		};
2694		}
2695
2696	2674	fn isUnnamedType(dg: DeclGen, ty: Type, val: llvm.Value) bool {
2697	2675	// Once `lowerType` succeeds, successive calls to it with the same Zig type
2698	2676	// are guaranteed to succeed. So if a call to `lowerType` fails here it means
...	...	@@ -2758,7 +2736,7 @@ pub const DeclGen = struct {
2758	2736	return dg.context.structType(&fields, fields.len, .False);
2759	2737	}
2760	2738	const ptr_info = t.ptrInfo().data;
2761		const llvm_addrspace = dg.llvmAddressSpace(ptr_info.@"addrspace");
	2739	const llvm_addrspace = toLlvmAddressSpace(ptr_info.@"addrspace", target);
2762	2740	if (ptr_info.host_size != 0) {
2763	2741	return dg.context.intType(ptr_info.host_size * 8).pointerType(llvm_addrspace);
2764	2742	}
...	...	@@ -3295,11 +3273,20 @@ pub const DeclGen = struct {
3295	3273	const decl_index = tv.val.castTag(.variable).?.data.owner_decl;
3296	3274	const decl = dg.module.declPtr(decl_index);
3297	3275	dg.module.markDeclAlive(decl);
3298		const val = try dg.resolveGlobalDecl(decl_index);
	3276
	3277	const llvm_wanted_addrspace = toLlvmAddressSpace(decl.@"addrspace", target);
	3278	const llvm_actual_addrspace = toLlvmGlobalAddressSpace(decl.@"addrspace", target);
	3279
3299	3280	const llvm_var_type = try dg.lowerType(tv.ty);
3300		const llvm_addrspace = dg.llvmAddressSpace(decl.@"addrspace");
3301		const llvm_type = llvm_var_type.pointerType(llvm_addrspace);
3302		return val.constBitCast(llvm_type);
	3281	const llvm_actual_ptr_type = llvm_var_type.pointerType(llvm_actual_addrspace);
	3282
	3283	const val = try dg.resolveGlobalDecl(decl_index);
	3284	const val_ptr = val.constBitCast(llvm_actual_ptr_type);
	3285	if (llvm_actual_addrspace != llvm_wanted_addrspace) {
	3286	const llvm_wanted_ptr_type = llvm_var_type.pointerType(llvm_wanted_addrspace);
	3287	return val_ptr.constAddrSpaceCast(llvm_wanted_ptr_type);
	3288	}
	3289	return val_ptr;
3303	3290	},
3304	3291	.slice => {
3305	3292	const slice = tv.val.castTag(.slice).?.data;
...	...	@@ -4096,11 +4083,20 @@ pub const DeclGen = struct {
4096	4083
4097	4084	self.module.markDeclAlive(decl);
4098	4085
4099		const llvm_val = if (is_fn_body)
	4086	const llvm_decl_val = if (is_fn_body)
4100	4087	try self.resolveLlvmFunction(decl_index)
4101	4088	else
4102	4089	try self.resolveGlobalDecl(decl_index);
4103	4090
	4091	const target = self.module.getTarget();
	4092	const llvm_wanted_addrspace = toLlvmAddressSpace(decl.@"addrspace", target);
	4093	const llvm_actual_addrspace = toLlvmGlobalAddressSpace(decl.@"addrspace", target);
	4094	const llvm_val = if (llvm_wanted_addrspace != llvm_actual_addrspace) blk: {
	4095	const llvm_decl_ty = try self.lowerType(decl.ty);
	4096	const llvm_decl_wanted_ptr_ty = llvm_decl_ty.pointerType(llvm_wanted_addrspace);
	4097	break :blk llvm_decl_val.constAddrSpaceCast(llvm_decl_wanted_ptr_ty);
	4098	} else llvm_decl_val;
	4099
4104	4100	const llvm_type = try self.lowerType(tv.ty);
4105	4101	if (tv.ty.zigTypeTag() == .Int) {
4106	4102	return llvm_val.constPtrToInt(llvm_type);
...	...	@@ -4370,7 +4366,9 @@ pub const FuncGen = struct {
4370	4366	// We have an LLVM value but we need to create a global constant and
4371	4367	// set the value as its initializer, and then return a pointer to the global.
4372	4368	const target = self.dg.module.getTarget();
4373		const global = self.dg.object.llvm_module.addGlobal(llvm_val.typeOf(), "");
	4369	const llvm_wanted_addrspace = toLlvmAddressSpace(.generic, target);
	4370	const llvm_actual_addrspace = toLlvmGlobalAddressSpace(.generic, target);
	4371	const global = self.dg.object.llvm_module.addGlobalInAddressSpace(llvm_val.typeOf(), "", llvm_actual_addrspace);
4374	4372	global.setInitializer(llvm_val);
4375	4373	global.setLinkage(.Private);
4376	4374	global.setGlobalConstant(.True);
...	...	@@ -4380,8 +4378,14 @@ pub const FuncGen = struct {
4380	4378	// the type of global constants might not match the type it is supposed to
4381	4379	// be, and so we must bitcast the pointer at the usage sites.
4382	4380	const wanted_llvm_ty = try self.dg.lowerType(tv.ty);
4383		const wanted_llvm_ptr_ty = wanted_llvm_ty.pointerType(0);
4384		return global.constBitCast(wanted_llvm_ptr_ty);
	4381	const wanted_bitcasted_llvm_ptr_ty = wanted_llvm_ty.pointerType(llvm_actual_addrspace);
	4382	const bitcasted_ptr = global.constBitCast(wanted_bitcasted_llvm_ptr_ty);
	4383	const wanted_llvm_ptr_ty = wanted_llvm_ty.pointerType(llvm_wanted_addrspace);
	4384	const casted_ptr = if (llvm_wanted_addrspace != llvm_actual_addrspace)
	4385	bitcasted_ptr.constAddrSpaceCast(wanted_llvm_ptr_ty)
	4386	else
	4387	bitcasted_ptr;
	4388	return casted_ptr;
4385	4389	}
4386	4390
4387	4391	fn genBody(self: *FuncGen, body: []const Air.Inst.Index) Error!void {
...	...	@@ -4462,7 +4466,7 @@ pub const FuncGen = struct {
4462	4466	.cmp_lt => try self.airCmp(inst, .lt, false),
4463	4467	.cmp_lte => try self.airCmp(inst, .lte, false),
4464	4468	.cmp_neq => try self.airCmp(inst, .neq, false),
4465
	4469
4466	4470	.cmp_eq_optimized => try self.airCmp(inst, .eq, true),
4467	4471	.cmp_gt_optimized => try self.airCmp(inst, .gt, true),
4468	4472	.cmp_gte_optimized => try self.airCmp(inst, .gte, true),
...	...	@@ -4548,6 +4552,7 @@ pub const FuncGen = struct {
4548	4552	.aggregate_init => try self.airAggregateInit(inst),
4549	4553	.union_init => try self.airUnionInit(inst),
4550	4554	.prefetch => try self.airPrefetch(inst),
	4555	.addrspace_cast => try self.airAddrSpaceCast(inst),
4551	4556
4552	4557	.is_named_enum_value => try self.airIsNamedEnumValue(inst),
4553	4558	.error_set_has_value => try self.airErrorSetHasValue(inst),
...	...	@@ -4635,8 +4640,7 @@ pub const FuncGen = struct {
4635	4640
4636	4641	const ret_ptr = if (!sret) null else blk: {
4637	4642	const llvm_ret_ty = try self.dg.lowerType(return_type);
4638		const ret_ptr = self.buildAlloca(llvm_ret_ty);
4639		ret_ptr.setAlignment(return_type.abiAlignment(target));
	4643	const ret_ptr = self.buildAlloca(llvm_ret_ty, return_type.abiAlignment(target));
4640	4644	try llvm_args.append(ret_ptr);
4641	4645	break :blk ret_ptr;
4642	4646	};
...	...	@@ -4683,8 +4687,7 @@ pub const FuncGen = struct {
4683	4687	} else {
4684	4688	const alignment = param_ty.abiAlignment(target);
4685	4689	const param_llvm_ty = llvm_arg.typeOf();
4686		const arg_ptr = self.buildAlloca(param_llvm_ty);
4687		arg_ptr.setAlignment(alignment);
	4690	const arg_ptr = self.buildAlloca(param_llvm_ty, alignment);
4688	4691	const store_inst = self.builder.buildStore(llvm_arg, arg_ptr);
4689	4692	store_inst.setAlignment(alignment);
4690	4693	try llvm_args.append(arg_ptr);
...	...	@@ -4711,8 +4714,7 @@ pub const FuncGen = struct {
4711	4714	param_ty.abiAlignment(target),
4712	4715	self.dg.object.target_data.abiAlignmentOfType(int_llvm_ty),
4713	4716	);
4714		const int_ptr = self.buildAlloca(int_llvm_ty);
4715		int_ptr.setAlignment(alignment);
	4717	const int_ptr = self.buildAlloca(int_llvm_ty, alignment);
4716	4718	const param_llvm_ty = try self.dg.lowerType(param_ty);
4717	4719	const casted_ptr = self.builder.buildBitCast(int_ptr, param_llvm_ty.pointerType(0), "");
4718	4720	const store_inst = self.builder.buildStore(llvm_arg, casted_ptr);
...	...	@@ -4738,7 +4740,7 @@ pub const FuncGen = struct {
4738	4740	const llvm_arg = try self.resolveInst(arg);
4739	4741	const is_by_ref = isByRef(param_ty);
4740	4742	const arg_ptr = if (is_by_ref) llvm_arg else p: {
4741		const p = self.buildAlloca(llvm_arg.typeOf());
	4743	const p = self.buildAlloca(llvm_arg.typeOf(), null);
4742	4744	const store_inst = self.builder.buildStore(llvm_arg, p);
4743	4745	store_inst.setAlignment(param_ty.abiAlignment(target));
4744	4746	break :p p;
...	...	@@ -4767,7 +4769,7 @@ pub const FuncGen = struct {
4767	4769	const llvm_arg = try self.resolveInst(arg);
4768	4770	const is_by_ref = isByRef(param_ty);
4769	4771	const arg_ptr = if (is_by_ref) llvm_arg else p: {
4770		const p = self.buildAlloca(llvm_arg.typeOf());
	4772	const p = self.buildAlloca(llvm_arg.typeOf(), null);
4771	4773	const store_inst = self.builder.buildStore(llvm_arg, p);
4772	4774	store_inst.setAlignment(param_ty.abiAlignment(target));
4773	4775	break :p p;
...	...	@@ -4804,7 +4806,7 @@ pub const FuncGen = struct {
4804	4806	const arg_ty = self.air.typeOf(arg);
4805	4807	var llvm_arg = try self.resolveInst(arg);
4806	4808	if (!isByRef(arg_ty)) {
4807		const p = self.buildAlloca(llvm_arg.typeOf());
	4809	const p = self.buildAlloca(llvm_arg.typeOf(), null);
4808	4810	const store_inst = self.builder.buildStore(llvm_arg, p);
4809	4811	store_inst.setAlignment(arg_ty.abiAlignment(target));
4810	4812	llvm_arg = store_inst;
...	...	@@ -4861,9 +4863,8 @@ pub const FuncGen = struct {
4861	4863	// In this case the function return type is honoring the calling convention by having
4862	4864	// a different LLVM type than the usual one. We solve this here at the callsite
4863	4865	// by bitcasting a pointer to our canonical type, then loading it if necessary.
4864		const rp = self.buildAlloca(llvm_ret_ty);
4865	4866	const alignment = return_type.abiAlignment(target);
4866		rp.setAlignment(alignment);
	4867	const rp = self.buildAlloca(llvm_ret_ty, alignment);
4867	4868	const ptr_abi_ty = abi_ret_ty.pointerType(0);
4868	4869	const casted_ptr = self.builder.buildBitCast(rp, ptr_abi_ty, "");
4869	4870	const store_inst = self.builder.buildStore(call, casted_ptr);
...	...	@@ -4880,9 +4881,8 @@ pub const FuncGen = struct {
4880	4881	if (isByRef(return_type)) {
4881	4882	// our by-ref status disagrees with sret so we must allocate, store,
4882	4883	// and return the allocation pointer.
4883		const rp = self.buildAlloca(llvm_ret_ty);
4884	4884	const alignment = return_type.abiAlignment(target);
4885		rp.setAlignment(alignment);
	4885	const rp = self.buildAlloca(llvm_ret_ty, alignment);
4886	4886	const store_inst = self.builder.buildStore(call, rp);
4887	4887	store_inst.setAlignment(alignment);
4888	4888	return rp;
...	...	@@ -4941,8 +4941,7 @@ pub const FuncGen = struct {
4941	4941	return null;
4942	4942	}
4943	4943
4944		const rp = self.buildAlloca(llvm_ret_ty);
4945		rp.setAlignment(alignment);
	4944	const rp = self.buildAlloca(llvm_ret_ty, alignment);
4946	4945	const store_inst = self.builder.buildStore(operand, rp);
4947	4946	store_inst.setAlignment(alignment);
4948	4947	const casted_ptr = self.builder.buildBitCast(rp, ptr_abi_ty, "");
...	...	@@ -6060,8 +6059,7 @@ pub const FuncGen = struct {
6060	6059	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOf();
6061	6060	} else {
6062	6061	const alignment = arg_ty.abiAlignment(target);
6063		const arg_ptr = self.buildAlloca(arg_llvm_value.typeOf());
6064		arg_ptr.setAlignment(alignment);
	6062	const arg_ptr = self.buildAlloca(arg_llvm_value.typeOf(), alignment);
6065	6063	const store_inst = self.builder.buildStore(arg_llvm_value, arg_ptr);
6066	6064	store_inst.setAlignment(alignment);
6067	6065	llvm_param_values[llvm_param_i] = arg_ptr;
...	...	@@ -6562,8 +6560,7 @@ pub const FuncGen = struct {
6562	6560	const llvm_optional_ty = try self.dg.lowerType(optional_ty);
6563	6561	if (isByRef(optional_ty)) {
6564	6562	const target = self.dg.module.getTarget();
6565		const optional_ptr = self.buildAlloca(llvm_optional_ty);
6566		optional_ptr.setAlignment(optional_ty.abiAlignment(target));
	6563	const optional_ptr = self.buildAlloca(llvm_optional_ty, optional_ty.abiAlignment(target));
6567	6564	const payload_ptr = self.builder.buildStructGEP(llvm_optional_ty, optional_ptr, 0, "");
6568	6565	var ptr_ty_payload: Type.Payload.ElemType = .{
6569	6566	.base = .{ .tag = .single_mut_pointer },
...	...	@@ -6596,8 +6593,7 @@ pub const FuncGen = struct {
6596	6593	const payload_offset = errUnionPayloadOffset(payload_ty, target);
6597	6594	const error_offset = errUnionErrorOffset(payload_ty, target);
6598	6595	if (isByRef(err_un_ty)) {
6599		const result_ptr = self.buildAlloca(err_un_llvm_ty);
6600		result_ptr.setAlignment(err_un_ty.abiAlignment(target));
	6596	const result_ptr = self.buildAlloca(err_un_llvm_ty, err_un_ty.abiAlignment(target));
6601	6597	const err_ptr = self.builder.buildStructGEP(err_un_llvm_ty, result_ptr, error_offset, "");
6602	6598	const store_inst = self.builder.buildStore(ok_err_code, err_ptr);
6603	6599	store_inst.setAlignment(Type.anyerror.abiAlignment(target));
...	...	@@ -6631,8 +6627,7 @@ pub const FuncGen = struct {
6631	6627	const payload_offset = errUnionPayloadOffset(payload_ty, target);
6632	6628	const error_offset = errUnionErrorOffset(payload_ty, target);
6633	6629	if (isByRef(err_un_ty)) {
6634		const result_ptr = self.buildAlloca(err_un_llvm_ty);
6635		result_ptr.setAlignment(err_un_ty.abiAlignment(target));
	6630	const result_ptr = self.buildAlloca(err_un_llvm_ty, err_un_ty.abiAlignment(target));
6636	6631	const err_ptr = self.builder.buildStructGEP(err_un_llvm_ty, result_ptr, error_offset, "");
6637	6632	const store_inst = self.builder.buildStore(operand, err_ptr);
6638	6633	store_inst.setAlignment(Type.anyerror.abiAlignment(target));
...	...	@@ -7050,9 +7045,8 @@ pub const FuncGen = struct {
7050	7045
7051	7046	if (isByRef(dest_ty)) {
7052	7047	const target = self.dg.module.getTarget();
7053		const alloca_inst = self.buildAlloca(llvm_dest_ty);
7054	7048	const result_alignment = dest_ty.abiAlignment(target);
7055		alloca_inst.setAlignment(result_alignment);
	7049	const alloca_inst = self.buildAlloca(llvm_dest_ty, result_alignment);
7056	7050	{
7057	7051	const field_ptr = self.builder.buildStructGEP(llvm_dest_ty, alloca_inst, result_index, "");
7058	7052	const store_inst = self.builder.buildStore(result, field_ptr);
...	...	@@ -7402,9 +7396,8 @@ pub const FuncGen = struct {
7402	7396
7403	7397	if (isByRef(dest_ty)) {
7404	7398	const target = self.dg.module.getTarget();
7405		const alloca_inst = self.buildAlloca(llvm_dest_ty);
7406	7399	const result_alignment = dest_ty.abiAlignment(target);
7407		alloca_inst.setAlignment(result_alignment);
	7400	const alloca_inst = self.buildAlloca(llvm_dest_ty, result_alignment);
7408	7401	{
7409	7402	const field_ptr = self.builder.buildStructGEP(llvm_dest_ty, alloca_inst, result_index, "");
7410	7403	const store_inst = self.builder.buildStore(result, field_ptr);
...	...	@@ -7710,7 +7703,7 @@ pub const FuncGen = struct {
7710	7703	if (!result_is_ref) {
7711	7704	return self.dg.todo("implement bitcast vector to non-ref array", .{});
7712	7705	}
7713		const array_ptr = self.buildAlloca(llvm_dest_ty);
	7706	const array_ptr = self.buildAlloca(llvm_dest_ty, null);
7714	7707	const bitcast_ok = elem_ty.bitSize(target) == elem_ty.abiSize(target) * 8;
7715	7708	if (bitcast_ok) {
7716	7709	const llvm_vector_ty = try self.dg.lowerType(operand_ty);
...	...	@@ -7786,8 +7779,7 @@ pub const FuncGen = struct {
7786	7779	if (result_is_ref) {
7787	7780	// Bitcast the result pointer, then store.
7788	7781	const alignment = @maximum(operand_ty.abiAlignment(target), inst_ty.abiAlignment(target));
7789		const result_ptr = self.buildAlloca(llvm_dest_ty);
7790		result_ptr.setAlignment(alignment);
	7782	const result_ptr = self.buildAlloca(llvm_dest_ty, alignment);
7791	7783	const operand_llvm_ty = try self.dg.lowerType(operand_ty);
7792	7784	const casted_ptr = self.builder.buildBitCast(result_ptr, operand_llvm_ty.pointerType(0), "");
7793	7785	const store_inst = self.builder.buildStore(operand, casted_ptr);
...	...	@@ -7800,8 +7792,7 @@ pub const FuncGen = struct {
7800	7792	// but LLVM won't let us bitcast struct values.
7801	7793	// Therefore, we store operand to bitcasted alloca, then load for result.
7802	7794	const alignment = @maximum(operand_ty.abiAlignment(target), inst_ty.abiAlignment(target));
7803		const result_ptr = self.buildAlloca(llvm_dest_ty);
7804		result_ptr.setAlignment(alignment);
	7795	const result_ptr = self.buildAlloca(llvm_dest_ty, alignment);
7805	7796	const operand_llvm_ty = try self.dg.lowerType(operand_ty);
7806	7797	const casted_ptr = self.builder.buildBitCast(result_ptr, operand_llvm_ty.pointerType(0), "");
7807	7798	const store_inst = self.builder.buildStore(operand, casted_ptr);
...	...	@@ -7877,11 +7868,9 @@ pub const FuncGen = struct {
7877	7868	if (!pointee_type.isFnOrHasRuntimeBitsIgnoreComptime()) return self.dg.lowerPtrToVoid(ptr_ty);
7878	7869
7879	7870	const pointee_llvm_ty = try self.dg.lowerType(pointee_type);
7880		const alloca_inst = self.buildAlloca(pointee_llvm_ty);
7881	7871	const target = self.dg.module.getTarget();
7882	7872	const alignment = ptr_ty.ptrAlignment(target);
7883		alloca_inst.setAlignment(alignment);
7884		return alloca_inst;
	7873	return self.buildAlloca(pointee_llvm_ty, alignment);
7885	7874	}
7886	7875
7887	7876	fn airRetPtr(self: FuncGen, inst: Air.Inst.Index) !?llvm.Value {
...	...	@@ -7892,15 +7881,13 @@ pub const FuncGen = struct {
7892	7881	if (self.ret_ptr) \|ret_ptr\| return ret_ptr;
7893	7882	const ret_llvm_ty = try self.dg.lowerType(ret_ty);
7894	7883	const target = self.dg.module.getTarget();
7895		const alloca_inst = self.buildAlloca(ret_llvm_ty);
7896		alloca_inst.setAlignment(ptr_ty.ptrAlignment(target));
7897		return alloca_inst;
	7884	return self.buildAlloca(ret_llvm_ty, ptr_ty.ptrAlignment(target));
7898	7885	}
7899	7886
7900	7887	/// Use this instead of builder.buildAlloca, because this function makes sure to
7901	7888	/// put the alloca instruction at the top of the function!
7902		fn buildAlloca(self: FuncGen, llvm_ty: llvm.Type) *llvm.Value {
7903		return buildAllocaInner(self.builder, self.llvm_func, self.di_scope != null, llvm_ty);
	7889	fn buildAlloca(self: FuncGen, llvm_ty: llvm.Type, alignment: ?c_uint) *llvm.Value {
	7890	return buildAllocaInner(self.builder, self.llvm_func, self.di_scope != null, llvm_ty, alignment, self.dg.module.getTarget());
7904	7891	}
7905	7892
7906	7893	fn airStore(self: FuncGen, inst: Air.Inst.Index) !?llvm.Value {
...	...	@@ -8779,9 +8766,9 @@ pub const FuncGen = struct {
8779	8766	const llvm_result_ty = accum_init.typeOf();
8780	8767
8781	8768	// Allocate and initialize our mutable variables
8782		const i_ptr = self.buildAlloca(llvm_usize_ty);
	8769	const i_ptr = self.buildAlloca(llvm_usize_ty, null);
8783	8770	_ = self.builder.buildStore(llvm_usize_ty.constInt(0, .False), i_ptr);
8784		const accum_ptr = self.buildAlloca(llvm_result_ty);
	8771	const accum_ptr = self.buildAlloca(llvm_result_ty, null);
8785	8772	_ = self.builder.buildStore(accum_init, accum_ptr);
8786	8773
8787	8774	// Setup the loop
...	...	@@ -8966,10 +8953,9 @@ pub const FuncGen = struct {
8966	8953
8967	8954	if (isByRef(result_ty)) {
8968	8955	const llvm_u32 = self.context.intType(32);
8969		const alloca_inst = self.buildAlloca(llvm_result_ty);
8970	8956	// TODO in debug builds init to undef so that the padding will be 0xaa
8971	8957	// even if we fully populate the fields.
8972		alloca_inst.setAlignment(result_ty.abiAlignment(target));
	8958	const alloca_inst = self.buildAlloca(llvm_result_ty, result_ty.abiAlignment(target));
8973	8959
8974	8960	var indices: [2]*llvm.Value = .{ llvm_u32.constNull(), undefined };
8975	8961	for (elements) \|elem, i\| {
...	...	@@ -9007,8 +8993,7 @@ pub const FuncGen = struct {
9007	8993	assert(isByRef(result_ty));
9008	8994
9009	8995	const llvm_usize = try self.dg.lowerType(Type.usize);
9010		const alloca_inst = self.buildAlloca(llvm_result_ty);
9011		alloca_inst.setAlignment(result_ty.abiAlignment(target));
	8996	const alloca_inst = self.buildAlloca(llvm_result_ty, result_ty.abiAlignment(target));
9012	8997
9013	8998	const array_info = result_ty.arrayInfo();
9014	8999	var elem_ptr_payload: Type.Payload.Pointer = .{
...	...	@@ -9083,7 +9068,7 @@ pub const FuncGen = struct {
9083	9068	// necessarily match the format that we need, depending on which tag is active. We
9084	9069	// must construct the correct unnamed struct type here and bitcast, in order to
9085	9070	// then set the fields appropriately.
9086		const result_ptr = self.buildAlloca(union_llvm_ty);
	9071	const result_ptr = self.buildAlloca(union_llvm_ty, null);
9087	9072	const llvm_payload = try self.resolveInst(extra.init);
9088	9073	assert(union_obj.haveFieldTypes());
9089	9074	const field = union_obj.fields.values()[extra.field_index];
...	...	@@ -9243,6 +9228,17 @@ pub const FuncGen = struct {
9243	9228	return null;
9244	9229	}
9245	9230
	9231	fn airAddrSpaceCast(self: FuncGen, inst: Air.Inst.Index) !?llvm.Value {
	9232	if (self.liveness.isUnused(inst)) return null;
	9233
	9234	const ty_op = self.air.instructions.items(.data)[inst].ty_op;
	9235	const inst_ty = self.air.typeOfIndex(inst);
	9236	const operand = try self.resolveInst(ty_op.operand);
	9237
	9238	const llvm_dest_ty = try self.dg.lowerType(inst_ty);
	9239	return self.builder.buildAddrSpaceCast(operand, llvm_dest_ty, "");
	9240	}
	9241
9246	9242	fn getErrorNameTable(self: FuncGen) !llvm.Value {
9247	9243	if (self.dg.object.error_name_table) \|table\| {
9248	9244	return table;
...	...	@@ -9324,9 +9320,8 @@ pub const FuncGen = struct {
9324	9320
9325	9321	if (isByRef(optional_ty)) {
9326	9322	const target = self.dg.module.getTarget();
9327		const alloca_inst = self.buildAlloca(optional_llvm_ty);
9328	9323	const payload_alignment = optional_ty.abiAlignment(target);
9329		alloca_inst.setAlignment(payload_alignment);
	9324	const alloca_inst = self.buildAlloca(optional_llvm_ty, payload_alignment);
9330	9325
9331	9326	{
9332	9327	const field_ptr = self.builder.buildStructGEP(optional_llvm_ty, alloca_inst, 0, "");
...	...	@@ -9450,8 +9445,7 @@ pub const FuncGen = struct {
9450	9445	if (isByRef(info.pointee_type)) {
9451	9446	const result_align = info.pointee_type.abiAlignment(target);
9452	9447	const max_align = @maximum(result_align, ptr_alignment);
9453		const result_ptr = self.buildAlloca(elem_llvm_ty);
9454		result_ptr.setAlignment(max_align);
	9448	const result_ptr = self.buildAlloca(elem_llvm_ty, max_align);
9455	9449	const llvm_ptr_u8 = self.context.intType(8).pointerType(0);
9456	9450	const llvm_usize = self.context.intType(Type.usize.intInfo(target).bits);
9457	9451	const size_bytes = info.pointee_type.abiSize(target);
...	...	@@ -9484,8 +9478,7 @@ pub const FuncGen = struct {
9484	9478
9485	9479	if (isByRef(info.pointee_type)) {
9486	9480	const result_align = info.pointee_type.abiAlignment(target);
9487		const result_ptr = self.buildAlloca(elem_llvm_ty);
9488		result_ptr.setAlignment(result_align);
	9481	const result_ptr = self.buildAlloca(elem_llvm_ty, result_align);
9489	9482
9490	9483	const same_size_int = self.context.intType(elem_bits);
9491	9484	const truncated_int = self.builder.buildTrunc(shifted_value, same_size_int, "");
...	...	@@ -9609,8 +9602,7 @@ pub const FuncGen = struct {
9609	9602	.x86_64 => {
9610	9603	const array_llvm_ty = usize_llvm_ty.arrayType(6);
9611	9604	const array_ptr = fg.valgrind_client_request_array orelse a: {
9612		const array_ptr = fg.buildAlloca(array_llvm_ty);
9613		array_ptr.setAlignment(usize_alignment);
	9605	const array_ptr = fg.buildAlloca(array_llvm_ty, usize_alignment);
9614	9606	fg.valgrind_client_request_array = array_ptr;
9615	9607	break :a array_ptr;
9616	9608	};
...	...	@@ -9905,6 +9897,78 @@ fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: std.Target) llvm.Ca
9905	9897	.nvptx, .nvptx64 => .PTX_Kernel,
9906	9898	else => unreachable,
9907	9899	},
	9900	.AmdgpuKernel => return switch (target.cpu.arch) {
	9901	.amdgcn => .AMDGPU_KERNEL,
	9902	else => unreachable,
	9903	},
	9904	};
	9905	}
	9906
	9907	/// Convert a zig-address space to an llvm address space.
	9908	fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: std.Target) c_uint {
	9909	return switch (target.cpu.arch) {
	9910	.i386, .x86_64 => switch (address_space) {
	9911	.generic => llvm.address_space.default,
	9912	.gs => llvm.address_space.x86.gs,
	9913	.fs => llvm.address_space.x86.fs,
	9914	.ss => llvm.address_space.x86.ss,
	9915	else => unreachable,
	9916	},
	9917	.nvptx, .nvptx64 => switch (address_space) {
	9918	.generic => llvm.address_space.default,
	9919	.global => llvm.address_space.nvptx.global,
	9920	.constant => llvm.address_space.nvptx.constant,
	9921	.param => llvm.address_space.nvptx.param,
	9922	.shared => llvm.address_space.nvptx.shared,
	9923	.local => llvm.address_space.nvptx.local,
	9924	else => unreachable,
	9925	},
	9926	.amdgcn => switch (address_space) {
	9927	.generic => llvm.address_space.amdgpu.flat,
	9928	.global => llvm.address_space.amdgpu.global,
	9929	.constant => llvm.address_space.amdgpu.constant,
	9930	.shared => llvm.address_space.amdgpu.local,
	9931	.local => llvm.address_space.amdgpu.private,
	9932	else => unreachable,
	9933	},
	9934	else => switch (address_space) {
	9935	.generic => llvm.address_space.default,
	9936	else => unreachable,
	9937	},
	9938	};
	9939	}
	9940
	9941	/// On some targets, local values that are in the generic address space must be generated into a
	9942	/// different address, space and then cast back to the generic address space.
	9943	/// For example, on GPUs local variable declarations must be generated into the local address space.
	9944	/// This function returns the address space local values should be generated into.
	9945	fn llvmAllocaAddressSpace(target: std.Target) c_uint {
	9946	return switch (target.cpu.arch) {
	9947	// On amdgcn, locals should be generated into the private address space.
	9948	// To make Zig not impossible to use, these are then converted to addresses in the
	9949	// generic address space and treates as regular pointers. This is the way that HIP also does it.
	9950	.amdgcn => llvm.address_space.amdgpu.private,
	9951	else => llvm.address_space.default,
	9952	};
	9953	}
	9954
	9955	/// On some targets, global values that are in the generic address space must be generated into a
	9956	/// different address space, and then cast back to the generic address space.
	9957	fn llvmDefaultGlobalAddressSpace(target: std.Target) c_uint {
	9958	return switch (target.cpu.arch) {
	9959	// On amdgcn, globals must be explicitly allocated and uploaded so that the program can access
	9960	// them.
	9961	.amdgcn => llvm.address_space.amdgpu.global,
	9962	else => llvm.address_space.default,
	9963	};
	9964	}
	9965
	9966	/// Return the actual address space that a value should be stored in if its a global address space.
	9967	/// When a value is placed in the resulting address space, it needs to be cast back into wanted_address_space.
	9968	fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, target: std.Target) c_uint {
	9969	return switch (wanted_address_space) {
	9970	.generic => llvmDefaultGlobalAddressSpace(target),
	9971	else => \|as\| toLlvmAddressSpace(as, target),
9908	9972	};
9909	9973	}
9910	9974
...	...	@@ -10537,13 +10601,23 @@ fn backendSupportsF16(target: std.Target) bool {
10537	10601	};
10538	10602	}
10539	10603
	10604	/// This function returns true if we expect LLVM to lower f128 correctly,
	10605	/// and false if we expect LLVm to crash if it encounters and f128 type
	10606	/// or if it produces miscompilations.
	10607	fn backendSupportsF128(target: std.Target) bool {
	10608	return switch (target.cpu.arch) {
	10609	.amdgcn => false,
	10610	else => true,
	10611	};
	10612	}
	10613
10540	10614	/// LLVM does not support all relevant intrinsics for all targets, so we
10541	10615	/// may need to manually generate a libc call
10542	10616	fn intrinsicsAllowed(scalar_ty: Type, target: std.Target) bool {
10543	10617	return switch (scalar_ty.tag()) {
10544	10618	.f16 => backendSupportsF16(target),
10545	10619	.f80 => target.longDoubleIs(f80) and backendSupportsF80(target),
10546		.f128 => target.longDoubleIs(f128),
	10620	.f128 => target.longDoubleIs(f128) and backendSupportsF128(target),
10547	10621	else => true,
10548	10622	};
10549	10623	}
...	...	@@ -10620,25 +10694,43 @@ fn buildAllocaInner(
10620	10694	llvm_func: *llvm.Value,
10621	10695	di_scope_non_null: bool,
10622	10696	llvm_ty: *llvm.Type,
	10697	maybe_alignment: ?c_uint,
	10698	target: std.Target,
10623	10699	) *llvm.Value {
10624		const prev_block = builder.getInsertBlock();
10625		const prev_debug_location = builder.getCurrentDebugLocation2();
10626		defer {
10627		builder.positionBuilderAtEnd(prev_block);
10628		if (di_scope_non_null) {
10629		builder.setCurrentDebugLocation2(prev_debug_location);
	10700	const address_space = llvmAllocaAddressSpace(target);
	10701
	10702	const alloca = blk: {
	10703	const prev_block = builder.getInsertBlock();
	10704	const prev_debug_location = builder.getCurrentDebugLocation2();
	10705	defer {
	10706	builder.positionBuilderAtEnd(prev_block);
	10707	if (di_scope_non_null) {
	10708	builder.setCurrentDebugLocation2(prev_debug_location);
	10709	}
10630	10710	}
	10711
	10712	const entry_block = llvm_func.getFirstBasicBlock().?;
	10713	if (entry_block.getFirstInstruction()) \|first_inst\| {
	10714	builder.positionBuilder(entry_block, first_inst);
	10715	} else {
	10716	builder.positionBuilderAtEnd(entry_block);
	10717	}
	10718	builder.clearCurrentDebugLocation();
	10719
	10720	break :blk builder.buildAllocaInAddressSpace(llvm_ty, address_space, "");
	10721	};
	10722
	10723	if (maybe_alignment) \|alignment\| {
	10724	alloca.setAlignment(alignment);
10631	10725	}
10632	10726
10633		const entry_block = llvm_func.getFirstBasicBlock().?;
10634		if (entry_block.getFirstInstruction()) \|first_inst\| {
10635		builder.positionBuilder(entry_block, first_inst);
10636		} else {
10637		builder.positionBuilderAtEnd(entry_block);
	10727	// The pointer returned from this function should have the generic address space,
	10728	// if this isn't the case then cast it to the generic address space.
	10729	if (address_space != llvm.address_space.default) {
	10730	return builder.buildAddrSpaceCast(alloca, llvm_ty.pointerType(llvm.address_space.default), "");
10638	10731	}
10639		builder.clearCurrentDebugLocation();
10640	10732
10641		return builder.buildAlloca(llvm_ty, "");
	10733	return alloca;
10642	10734	}
10643	10735
10644	10736	fn errUnionPayloadOffset(payload_ty: Type, target: std.Target) u1 {

src/codegen/llvm/bindings.zig+9

...	...	@@ -171,6 +171,9 @@ pub const Value = opaque {
171	171	pub const constAdd = LLVMConstAdd;
172	172	extern fn LLVMConstAdd(LHSConstant: Value, RHSConstant: Value) *Value;
173	173
	174	pub const constAddrSpaceCast = LLVMConstAddrSpaceCast;
	175	extern fn LLVMConstAddrSpaceCast(ConstantVal: Value, ToType: Type) *Value;
	176
174	177	pub const setWeak = LLVMSetWeak;
175	178	extern fn LLVMSetWeak(CmpXchgInst: *Value, IsWeak: Bool) void;
176	179
...	...	@@ -956,6 +959,12 @@ pub const Builder = opaque {
956	959
957	960	pub const setFastMath = ZigLLVMSetFastMath;
958	961	extern fn ZigLLVMSetFastMath(B: *Builder, on_state: bool) void;
	962
	963	pub const buildAddrSpaceCast = LLVMBuildAddrSpaceCast;
	964	extern fn LLVMBuildAddrSpaceCast(B: Builder, Val: Value, DestTy: Type, Name: [:0]const u8) *Value;
	965
	966	pub const buildAllocaInAddressSpace = ZigLLVMBuildAllocaInAddressSpace;
	967	extern fn ZigLLVMBuildAllocaInAddressSpace(B: Builder, Ty: Type, AddressSpace: c_uint, Name: [:0]const u8) Value;
959	968	};
960	969
961	970	pub const MDString = opaque {

src/print_air.zig+1

...	...	@@ -244,6 +244,7 @@ const Writer = struct {
244	244	.byte_swap,
245	245	.bit_reverse,
246	246	.error_set_has_value,
	247	.addrspace_cast,
247	248	=> try w.writeTyOp(s, inst),
248	249
249	250	.block,

src/print_zir.zig+1

...	...	@@ -512,6 +512,7 @@ const Writer = struct {
512	512	.err_set_cast,
513	513	.wasm_memory_grow,
514	514	.prefetch,
	515	.addrspace_cast,
515	516	=> {
516	517	const inst_data = self.code.extraData(Zir.Inst.BinNode, extended.operand).data;
517	518	const src = LazySrcLoc.nodeOffset(inst_data.node);

src/stage1/all_types.hpp+16-1

...	...	@@ -85,7 +85,8 @@ enum CallingConvention {
85	85	CallingConventionAAPCSVFP,
86	86	CallingConventionSysV,
87	87	CallingConventionWin64,
88		CallingConventionPtxKernel
	88	CallingConventionPtxKernel,
	89	CallingConventionAmdgpuKernel
89	90	};
90	91
91	92	// Stage 1 supports only the generic address space
...	...	@@ -94,6 +95,11 @@ enum AddressSpace {
94	95	AddressSpaceGS,
95	96	AddressSpaceFS,
96	97	AddressSpaceSS,
	98	AddressSpaceGlobal,
	99	AddressSpaceConstant,
	100	AddressSpaceParam,
	101	AddressSpaceShared,
	102	AddressSpaceLocal
97	103	};
98	104
99	105	// This one corresponds to the builtin.zig enum.
...	...	@@ -1841,6 +1847,7 @@ enum BuiltinFnId {
1841	1847	BuiltinFnIdMaximum,
1842	1848	BuiltinFnIdMinimum,
1843	1849	BuiltinFnIdPrefetch,
	1850	BuiltinFnIdAddrSpaceCast,
1844	1851	};
1845	1852
1846	1853	struct BuiltinFnEntry {
...	...	@@ -2672,6 +2679,7 @@ enum Stage1ZirInstId : uint8_t {
2672	2679	Stage1ZirInstIdWasmMemoryGrow,
2673	2680	Stage1ZirInstIdSrc,
2674	2681	Stage1ZirInstIdPrefetch,
	2682	Stage1ZirInstIdAddrSpaceCast,
2675	2683	};
2676	2684
2677	2685	// ir_render_* functions in codegen.cpp consume Gen instructions and produce LLVM IR.
...	...	@@ -4168,6 +4176,13 @@ struct Stage1AirInstAlignCast {
4168	4176	Stage1AirInst *target;
4169	4177	};
4170	4178
	4179	struct Stage1ZirInstAddrSpaceCast {
	4180	Stage1ZirInst base;
	4181
	4182	Stage1ZirInst *addrspace;
	4183	Stage1ZirInst *ptr;
	4184	};
	4185
4171	4186	struct Stage1ZirInstSetAlignStack {
4172	4187	Stage1ZirInst base;
4173	4188

src/stage1/analyze.cpp+14-3

...	...	@@ -993,6 +993,7 @@ const char *calling_convention_name(CallingConvention cc) {
993	993	case CallingConventionSysV: return "SysV";
994	994	case CallingConventionWin64: return "Win64";
995	995	case CallingConventionPtxKernel: return "PtxKernel";
	996	case CallingConventionAmdgpuKernel: return "AmdgpuKernel";
996	997	}
997	998	zig_unreachable();
998	999	}
...	...	@@ -1017,6 +1018,7 @@ bool calling_convention_allows_zig_types(CallingConvention cc) {
1017	1018	case CallingConventionAAPCSVFP:
1018	1019	case CallingConventionSysV:
1019	1020	case CallingConventionWin64:
	1021	case CallingConventionAmdgpuKernel:
1020	1022	return false;
1021	1023	}
1022	1024	zig_unreachable();
...	...	@@ -1028,6 +1030,11 @@ const char *address_space_name(AddressSpace as) {
1028	1030	case AddressSpaceGS: return "gs";
1029	1031	case AddressSpaceFS: return "fs";
1030	1032	case AddressSpaceSS: return "ss";
	1033	case AddressSpaceGlobal: return "global";
	1034	case AddressSpaceConstant: return "constant";
	1035	case AddressSpaceParam: return "param";
	1036	case AddressSpaceShared: return "shared";
	1037	case AddressSpaceLocal: return "local";
1031	1038	}
1032	1039	zig_unreachable();
1033	1040	}
...	...	@@ -2019,6 +2026,9 @@ Error emit_error_unless_callconv_allowed_for_target(CodeGen g, AstNode source_
2019	2026	allowed_platforms = "nvptx and nvptx64";
2020	2027	}
2021	2028	break;
	2029	case CallingConventionAmdgpuKernel:
	2030	if (g->zig_target->arch != ZigLLVM_amdgcn)
	2031	allowed_platforms = "amdgcn and amdpal";
2022	2032
2023	2033	}
2024	2034	if (allowed_platforms != nullptr) {
...	...	@@ -3857,6 +3867,7 @@ static void resolve_decl_fn(CodeGen g, TldFn tld_fn) {
3857	3867	case CallingConventionSysV:
3858	3868	case CallingConventionWin64:
3859	3869	case CallingConventionPtxKernel:
	3870	case CallingConventionAmdgpuKernel:
3860	3871	add_fn_export(g, fn_table_entry, buf_ptr(&fn_table_entry->symbol_name),
3861	3872	GlobalLinkageIdStrong, fn_cc);
3862	3873	break;
...	...	@@ -6012,7 +6023,7 @@ Error type_has_bits2(CodeGen g, ZigType type_entry, bool *result) {
6012	6023
6013	6024	bool fn_returns_c_abi_small_struct(FnTypeId *fn_type_id) {
6014	6025	ZigType *type = fn_type_id->return_type;
6015		return !calling_convention_allows_zig_types(fn_type_id->cc) &&
	6026	return !calling_convention_allows_zig_types(fn_type_id->cc) &&
6016	6027	type->id == ZigTypeIdStruct && type->abi_size <= 16;
6017	6028	}
6018	6029
...	...	@@ -8700,7 +8711,7 @@ static LLVMTypeRef llvm_int_for_size(size_t size) {
8700	8711	static LLVMTypeRef llvm_sse_for_size(size_t size) {
8701	8712	if (size > 4)
8702	8713	return LLVMDoubleType();
8703		else
	8714	else
8704	8715	return LLVMFloatType();
8705	8716	}
8706	8717
...	...	@@ -8758,7 +8769,7 @@ static Error resolve_llvm_c_abi_type(CodeGen g, ZigType ty) {
8758	8769
8759	8770	LLVMTypeRef return_elem_types[] = {
8760	8771	LLVMVoidType(),
8761		LLVMVoidType(),
	8772	LLVMVoidType(),
8762	8773	};
8763	8774	for (uint32_t i = 0; i <= eightbyte_index; i += 1) {
8764	8775	if (type_classes[i] == X64CABIClass_INTEGER) {

src/stage1/astgen.cpp+34

...	...	@@ -351,6 +351,8 @@ void destroy_instruction_src(Stage1ZirInst *inst) {
351	351	return heap::c_allocator.destroy(reinterpret_cast<Stage1ZirInstSrc *>(inst));
352	352	case Stage1ZirInstIdPrefetch:
353	353	return heap::c_allocator.destroy(reinterpret_cast<Stage1ZirInstPrefetch *>(inst));
	354	case Stage1ZirInstIdAddrSpaceCast:
	355	return heap::c_allocator.destroy(reinterpret_cast<Stage1ZirInstAddrSpaceCast *>(inst));
354	356	}
355	357	zig_unreachable();
356	358	}
...	...	@@ -947,6 +949,10 @@ static constexpr Stage1ZirInstId ir_inst_id(Stage1ZirInstPrefetch *) {
947	949	return Stage1ZirInstIdPrefetch;
948	950	}
949	951
	952	static constexpr Stage1ZirInstId ir_inst_id(Stage1ZirInstAddrSpaceCast *) {
	953	return Stage1ZirInstIdAddrSpaceCast;
	954	}
	955
950	956	template<typename T>
951	957	static T ir_create_instruction(Stage1AstGen ag, Scope scope, AstNode source_node) {
952	958	T *special_instruction = heap::c_allocator.create<T>();
...	...	@@ -2572,6 +2578,19 @@ static Stage1ZirInst ir_build_align_cast_src(Stage1AstGen ag, Scope *scope, As
2572	2578	return &instruction->base;
2573	2579	}
2574	2580
	2581	static Stage1ZirInst ir_build_addrspace_cast(Stage1AstGen ag, Scope scope, AstNode source_node,
	2582	Stage1ZirInst addrspace, Stage1ZirInst ptr)
	2583	{
	2584	Stage1ZirInstAddrSpaceCast *instruction = ir_build_instruction<Stage1ZirInstAddrSpaceCast>(ag, scope, source_node);
	2585	instruction->addrspace = addrspace;
	2586	instruction->ptr = ptr;
	2587
	2588	ir_ref_instruction(addrspace, ag->current_basic_block);
	2589	ir_ref_instruction(ptr, ag->current_basic_block);
	2590
	2591	return &instruction->base;
	2592	}
	2593
2575	2594	static Stage1ZirInst ir_build_resolve_result(Stage1AstGen ag, Scope scope, AstNode source_node,
2576	2595	ResultLoc result_loc, Stage1ZirInst ty)
2577	2596	{
...	...	@@ -5459,6 +5478,21 @@ static Stage1ZirInst astgen_builtin_fn_call(Stage1AstGen ag, Scope *scope, Ast
5459	5478	Stage1ZirInst *ir_extern = ir_build_prefetch(ag, scope, node, ptr_value, casted_options_value);
5460	5479	return ir_lval_wrap(ag, scope, ir_extern, lval, result_loc);
5461	5480	}
	5481	case BuiltinFnIdAddrSpaceCast:
	5482	{
	5483	AstNode *arg0_node = node->data.fn_call_expr.params.at(0);
	5484	Stage1ZirInst *arg0_value = astgen_node(ag, arg0_node, scope);
	5485	if (arg0_value == ag->codegen->invalid_inst_src)
	5486	return arg0_value;
	5487
	5488	AstNode* arg1_node = node->data.fn_call_expr.params.at(1);
	5489	Stage1ZirInst *arg1_value = astgen_node(ag, arg1_node, scope);
	5490	if (arg1_value == ag->codegen->invalid_inst_src)
	5491	return arg1_value;
	5492
	5493	Stage1ZirInst *addrspace_cast = ir_build_addrspace_cast(ag, scope, node, arg0_value, arg1_value);
	5494	return ir_lval_wrap(ag, scope, addrspace_cast, lval, result_loc);
	5495	}
5462	5496	}
5463	5497	zig_unreachable();
5464	5498	}

src/stage1/codegen.cpp+7-2

...	...	@@ -217,6 +217,9 @@ static ZigLLVM_CallingConv get_llvm_cc(CodeGen *g, CallingConvention cc) {
217	217	assert(g->zig_target->arch == ZigLLVM_nvptx \|\|
218	218	g->zig_target->arch == ZigLLVM_nvptx64);
219	219	return ZigLLVM_PTX_Kernel;
	220	case CallingConventionAmdgpuKernel:
	221	assert(g->zig_target->arch == ZigLLVM_amdgcn);
	222	return ZigLLVM_AMDGPU_KERNEL;
220	223
221	224	}
222	225	zig_unreachable();
...	...	@@ -365,6 +368,7 @@ static bool cc_want_sret_attr(CallingConvention cc) {
365	368	case CallingConventionSysV:
366	369	case CallingConventionWin64:
367	370	case CallingConventionPtxKernel:
	371	case CallingConventionAmdgpuKernel:
368	372	return true;
369	373	case CallingConventionAsync:
370	374	case CallingConventionUnspecified:
...	...	@@ -3515,7 +3519,7 @@ static LLVMValueRef gen_soft_float_to_int_op(CodeGen *g, LLVMValueRef value_ref,
3515	3519
3516	3520	// Handle integers of non-pot bitsize by shortening them on the output
3517	3521	if (result_type != wider_type) {
3518		result = gen_widen_or_shorten(g, false, wider_type, result_type, result);
	3522	result = gen_widen_or_shorten(g, false, wider_type, result_type, result);
3519	3523	}
3520	3524
3521	3525	return result;
...	...	@@ -4370,7 +4374,7 @@ static LLVMValueRef ir_render_binary_not(CodeGen g, Stage1Air executable,
4370	4374
4371	4375	static LLVMValueRef gen_soft_float_neg(CodeGen g, ZigType operand_type, LLVMValueRef operand) {
4372	4376	uint32_t vector_len = operand_type->id == ZigTypeIdVector ? operand_type->data.vector.len : 0;
4373		uint16_t num_bits = operand_type->id == ZigTypeIdVector ?
	4377	uint16_t num_bits = operand_type->id == ZigTypeIdVector ?
4374	4378	operand_type->data.vector.elem_type->data.floating.bit_count :
4375	4379	operand_type->data.floating.bit_count;
4376	4380
...	...	@@ -10181,6 +10185,7 @@ static void define_builtin_fns(CodeGen *g) {
10181	10185	create_builtin_fn(g, BuiltinFnIdMaximum, "maximum", 2);
10182	10186	create_builtin_fn(g, BuiltinFnIdMinimum, "minimum", 2);
10183	10187	create_builtin_fn(g, BuiltinFnIdPrefetch, "prefetch", 2);
	10188	create_builtin_fn(g, BuiltinFnIdAddrSpaceCast, "addrSpaceCast", 2);
10184	10189	}
10185	10190
10186	10191	static const char *bool_to_str(bool b) {

src/stage1/ir.cpp+48

...	...	@@ -11753,6 +11753,7 @@ static Stage1AirInst ir_analyze_instruction_export(IrAnalyze ira, Stage1ZirIns
11753	11753	case CallingConventionSysV:
11754	11754	case CallingConventionWin64:
11755	11755	case CallingConventionPtxKernel:
	11756	case CallingConventionAmdgpuKernel:
11756	11757	add_fn_export(ira->codegen, fn_entry, buf_ptr(symbol_name), global_linkage_id, cc);
11757	11758	fn_entry->section_name = section_name;
11758	11759	break;
...	...	@@ -23745,6 +23746,50 @@ static Stage1AirInst ir_analyze_instruction_align_cast(IrAnalyze ira, Stage1Zi
23745	23746	return result;
23746	23747	}
23747	23748
	23749	static bool ir_resolve_addrspace(IrAnalyze ira, Stage1AirInst value, AddressSpace *out) {
	23750	if (type_is_invalid(value->value->type))
	23751	return false;
	23752
	23753	ZigType *addrspace_type = get_builtin_type(ira->codegen, "AddressSpace");
	23754
	23755	Stage1AirInst *casted_value = ir_implicit_cast(ira, value, addrspace_type);
	23756	if (type_is_invalid(casted_value->value->type))
	23757	return false;
	23758
	23759	ZigValue *const_val = ir_resolve_const(ira, casted_value, UndefBad);
	23760	if (!const_val)
	23761	return false;
	23762
	23763	*out = (AddressSpace)bigint_as_u32(&const_val->data.x_enum_tag);
	23764	return true;
	23765	}
	23766
	23767	static Stage1AirInst ir_analyze_instruction_addrspace_cast(IrAnalyze ira, Stage1ZirInstAddrSpaceCast *instruction) {
	23768	Stage1AirInst *ptr_inst = instruction->ptr->child;
	23769	ZigType *ptr_type = ptr_inst->value->type;
	23770	if (type_is_invalid(ptr_type))
	23771	return ira->codegen->invalid_inst_gen;
	23772
	23773	AddressSpace addrspace;
	23774	if (!ir_resolve_addrspace(ira, instruction->addrspace->child, &addrspace))
	23775	return ira->codegen->invalid_inst_gen;
	23776
	23777	if (addrspace != AddressSpaceGeneric) {
	23778	ir_add_error_node(ira, instruction->addrspace->source_node, buf_sprintf(
	23779	"address space '%s' not available in stage 1 compiler, must be .generic",
	23780	address_space_name(addrspace)));
	23781	return ira->codegen->invalid_inst_gen;
	23782	}
	23783
	23784	if (is_slice(ptr_type) \|\| get_src_ptr_type(ptr_type) != nullptr) {
	23785	ir_add_error_node(ira, instruction->ptr->source_node,
	23786	buf_sprintf("expected pointer or slice, found '%s'", buf_ptr(&ptr_type->name)));
	23787	return ira->codegen->invalid_inst_gen;
	23788	}
	23789
	23790	return ptr_inst;
	23791	}
	23792
23748	23793	static Stage1AirInst ir_analyze_instruction_set_align_stack(IrAnalyze ira, Stage1ZirInstSetAlignStack *instruction) {
23749	23794	uint32_t align_bytes;
23750	23795	Stage1AirInst *align_bytes_inst = instruction->align_bytes->child;
...	...	@@ -25450,6 +25495,8 @@ static Stage1AirInst ir_analyze_instruction_base(IrAnalyze ira, Stage1ZirInst
25450	25495	return ir_analyze_instruction_src(ira, (Stage1ZirInstSrc *)instruction);
25451	25496	case Stage1ZirInstIdPrefetch:
25452	25497	return ir_analyze_instruction_prefetch(ira, (Stage1ZirInstPrefetch *)instruction);
	25498	case Stage1ZirInstIdAddrSpaceCast:
	25499	return ir_analyze_instruction_addrspace_cast(ira, (Stage1ZirInstAddrSpaceCast *)instruction);
25453	25500	}
25454	25501	zig_unreachable();
25455	25502	}
...	...	@@ -25831,6 +25878,7 @@ bool ir_inst_src_has_side_effects(Stage1ZirInst *instruction) {
25831	25878	case Stage1ZirInstIdWasmMemorySize:
25832	25879	case Stage1ZirInstIdSrc:
25833	25880	case Stage1ZirInstIdReduce:
	25881	case Stage1ZirInstIdAddrSpaceCast:
25834	25882	return false;
25835	25883
25836	25884	case Stage1ZirInstIdAsm:

src/stage1/ir_print.cpp+13

...	...	@@ -373,6 +373,8 @@ const char* ir_inst_src_type_str(Stage1ZirInstId id) {
373	373	return "SrcSrc";
374	374	case Stage1ZirInstIdPrefetch:
375	375	return "SrcPrefetch";
	376	case Stage1ZirInstIdAddrSpaceCast:
	377	return "SrcAddrSpaceCast";
376	378	}
377	379	zig_unreachable();
378	380	}
...	...	@@ -2382,6 +2384,14 @@ static void ir_print_align_cast(IrPrintSrc irp, Stage1ZirInstAlignCast instruc
2382	2384	fprintf(irp->f, ")");
2383	2385	}
2384	2386
	2387	static void ir_print_addrspace_cast(IrPrintSrc irp, Stage1ZirInstAddrSpaceCast instruction) {
	2388	fprintf(irp->f, "@addrSpaceCast(");
	2389	ir_print_other_inst_src(irp, instruction->addrspace);
	2390	fprintf(irp->f, ",");
	2391	ir_print_other_inst_src(irp, instruction->ptr);
	2392	fprintf(irp->f, ")");
	2393	}
	2394
2385	2395	static void ir_print_align_cast(IrPrintGen irp, Stage1AirInstAlignCast instruction) {
2386	2396	fprintf(irp->f, "@alignCast(");
2387	2397	ir_print_other_inst_gen(irp, instruction->target);
...	...	@@ -3127,6 +3137,9 @@ static void ir_print_inst_src(IrPrintSrc irp, Stage1ZirInst instruction, bool
3127	3137	case Stage1ZirInstIdPrefetch:
3128	3138	ir_print_prefetch(irp, (Stage1ZirInstPrefetch *)instruction);
3129	3139	break;
	3140	case Stage1ZirInstIdAddrSpaceCast:
	3141	ir_print_addrspace_cast(irp, (Stage1ZirInstAddrSpaceCast *)instruction);
	3142	break;
3130	3143	}
3131	3144	fprintf(irp->f, "\n");
3132	3145	}

src/target.zig+20-1

...	...	@@ -1,5 +1,6 @@
1	1	const std = @import("std");
2	2	const Type = @import("type.zig").Type;
	3	const AddressSpace = std.builtin.AddressSpace;
3	4
4	5	pub const ArchOsAbi = struct {
5	6	arch: std.Target.Cpu.Arch,
...	...	@@ -635,12 +636,30 @@ pub fn defaultAddressSpace(
635	636	/// Query the default address space for functions themselves.
636	637	function,
637	638	},
638		) std.builtin.AddressSpace {
	639	) AddressSpace {
639	640	_ = target;
640	641	_ = context;
641	642	return .generic;
642	643	}
643	644
	645	/// Returns true if pointers in `from` can be converted to a pointer in `to`.
	646	pub fn addrSpaceCastIsValid(
	647	target: std.Target,
	648	from: AddressSpace,
	649	to: AddressSpace,
	650	) bool {
	651	const arch = target.cpu.arch;
	652	switch (arch) {
	653	.x86_64, .i386 => return arch.supportsAddressSpace(from) and arch.supportsAddressSpace(to),
	654	.amdgcn => {
	655	const to_generic = arch.supportsAddressSpace(from) and to == .generic;
	656	const from_generic = arch.supportsAddressSpace(to) and from == .generic;
	657	return to_generic or from_generic;
	658	},
	659	else => return from == .generic and to == .generic,
	660	}
	661	}
	662
644	663	pub fn llvmMachineAbi(target: std.Target) ?[:0]const u8 {
645	664	const have_float = switch (target.abi) {
646	665	.gnuilp32 => return "ilp32",

src/type.zig+13-2

...	...	@@ -2786,6 +2786,12 @@ pub const Type = extern union {
2786	2786
2787	2787	.pointer => self.castTag(.pointer).?.data.@"addrspace",
2788	2788
	2789	.optional => {
	2790	var buf: Payload.ElemType = undefined;
	2791	const child_type = self.optionalChild(&buf);
	2792	return child_type.ptrAddressSpace();
	2793	},
	2794
2789	2795	else => unreachable,
2790	2796	};
2791	2797	}
...	...	@@ -6768,6 +6774,13 @@ pub const CType = enum {
6768	6774	},
6769	6775	},
6770	6776
	6777	.amdhsa, .amdpal => switch (self) {
	6778	.short, .ushort => return 16,
	6779	.int, .uint => return 32,
	6780	.long, .ulong, .longlong, .ulonglong => return 64,
	6781	.longdouble => return 128,
	6782	},
	6783
6771	6784	.cloudabi,
6772	6785	.kfreebsd,
6773	6786	.lv2,
...	...	@@ -6777,13 +6790,11 @@ pub const CType = enum {
6777	6790	.aix,
6778	6791	.cuda,
6779	6792	.nvcl,
6780		.amdhsa,
6781	6793	.ps4,
6782	6794	.ps5,
6783	6795	.elfiamcu,
6784	6796	.mesa3d,
6785	6797	.contiki,
6786		.amdpal,
6787	6798	.hermit,
6788	6799	.hurd,
6789	6800	.opencl,

src/zig_llvm.cpp+11-6

...	...	@@ -512,22 +512,22 @@ LLVMValueRef ZigLLVMBuildUSubSat(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRe
512	512
513	513	LLVMValueRef ZigLLVMBuildSMulFixSat(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS, const char *name) {
514	514	llvm::Type* types[1] = {
515		unwrap(LHS)->getType(),
	515	unwrap(LHS)->getType(),
516	516	};
517	517	// pass scale = 0 as third argument
518	518	llvm::Value* values[3] = {unwrap(LHS), unwrap(RHS), unwrap(B)->getInt32(0)};
519
	519
520	520	CallInst *call_inst = unwrap(B)->CreateIntrinsic(Intrinsic::smul_fix_sat, types, values, nullptr, name);
521	521	return wrap(call_inst);
522	522	}
523	523
524	524	LLVMValueRef ZigLLVMBuildUMulFixSat(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS, const char *name) {
525	525	llvm::Type* types[1] = {
526		unwrap(LHS)->getType(),
	526	unwrap(LHS)->getType(),
527	527	};
528	528	// pass scale = 0 as third argument
529	529	llvm::Value* values[3] = {unwrap(LHS), unwrap(RHS), unwrap(B)->getInt32(0)};
530
	530
531	531	CallInst *call_inst = unwrap(B)->CreateIntrinsic(Intrinsic::umul_fix_sat, types, values, nullptr, name);
532	532	return wrap(call_inst);
533	533	}
...	...	@@ -808,7 +808,7 @@ void ZigLLVMSetCurrentDebugLocation2(LLVMBuilderRef builder, unsigned int line,
808	808	unsigned int column, ZigLLVMDIScope scope, ZigLLVMDILocation inlined_at)
809	809	{
810	810	DIScope* di_scope = reinterpret_cast<DIScope*>(scope);
811		DebugLoc debug_loc = DILocation::get(di_scope->getContext(), line, column, di_scope,
	811	DebugLoc debug_loc = DILocation::get(di_scope->getContext(), line, column, di_scope,
812	812	reinterpret_cast<DILocation *>(inlined_at), false);
813	813	unwrap(builder)->SetCurrentDebugLocation(debug_loc);
814	814	}
...	...	@@ -1177,9 +1177,14 @@ LLVMValueRef ZigLLVMBuildAShrExact(LLVMBuilderRef builder, LLVMValueRef LHS, LLV
1177	1177	return wrap(unwrap(builder)->CreateAShr(unwrap(LHS), unwrap(RHS), name, true));
1178	1178	}
1179	1179
	1180	LLVMValueRef ZigLLVMBuildAllocaInAddressSpace(LLVMBuilderRef builder, LLVMTypeRef Ty,
	1181	unsigned AddressSpace, const char *Name) {
	1182	return wrap(unwrap(builder)->CreateAlloca(unwrap(Ty), AddressSpace, nullptr, Name));
	1183	}
	1184
1180	1185	void ZigLLVMSetTailCall(LLVMValueRef Call) {
1181	1186	unwrap<CallInst>(Call)->setTailCallKind(CallInst::TCK_MustTail);
1182		}
	1187	}
1183	1188
1184	1189	void ZigLLVMSetCallSret(LLVMValueRef Call, LLVMTypeRef return_type) {
1185	1190	CallInst *call_inst = unwrap<CallInst>(Call);

src/zig_llvm.h+2

...	...	@@ -162,6 +162,8 @@ ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildLShrExact(LLVMBuilderRef builder, LLVMValu
162	162	ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildAShrExact(LLVMBuilderRef builder, LLVMValueRef LHS, LLVMValueRef RHS,
163	163	const char *name);
164	164
	165	ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildAllocaInAddressSpace(LLVMBuilderRef builder, LLVMTypeRef Ty, unsigned AddressSpace,
	166	const char *Name);
165	167
166	168	ZIG_EXTERN_C struct ZigLLVMDIType ZigLLVMCreateDebugPointerType(struct ZigLLVMDIBuilder dibuilder,
167	169	struct ZigLLVMDIType pointee_type, uint64_t size_in_bits, uint64_t align_in_bits, const char name);

author	Andrew Kelley <andrew@ziglang.org> 2022-10-15 13:36:43-04:00
committer	GitHub <noreply@github.com> 2022-10-15 13:36:43-04:00
log	65f860bef7995a6120e49606d549bdf154bca150
tree	c7d4973f437fde43735ac8c08cb8e12f5e66257f
parent	1087e677625b0846cf25dc43474a63f9a25f1e32
parent	66d6183001e135e36df06194e29f082eb63503ec
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #12879 from Snektron/amdgpu-improvements