Commit 39a9a4b2cae0ef94eb4aea482dabf275fdd0a910 | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2026-06-24 17:28:01+02:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2026-06-24 17:28:01+02:00
log	39a9a4b2cae0ef94eb4aea482dabf275fdd0a910
tree	8fbb6febdf32fcd315de6bd279ee8b9000da6e7f
parent	efe73c787a89f71de37bf876c3ef7bcfddebebba
parent	cbef1f5a013c664979e68d3ea8fa3ba378e3f0f6

Merge pull request 'llvm: only load/store ABI-sized integers to/from memory' (#35711) from mlugg/llvm-weird-int-in-memory into master

Resolves: https://github.com/ziglang/zig/issues/17768
Resolves: https://github.com/ziglang/zig/issues/18936
Resolves: https://github.com/ziglang/zig/issues/19755
Resolves: https://github.com/ziglang/zig/issues/24282
Resolves: https://github.com/ziglang/zig/issues/24883
Resolves: https://github.com/ziglang/zig/issues/25042
Resolves: https://github.com/ziglang/zig/issues/25555
Resolves: https://codeberg.org/ziglang/zig/issues/32036
Resolves: https://codeberg.org/ziglang/zig/issues/35560
Reviewed-on: https://codeberg.org/ziglang/zig/pulls/35711
Reviewed-by: Andrew Kelley <andrew@ziglang.org>

69 files changed, 4646 insertions(+), 4049 deletions(-)

CMakeLists.txt
doc
- langref
  - test_packed_structs.zig
  - test_pointer_casting.zig
lib
- compiler_rt
- std
  - Build
    Configuration.zig
  - Io
    Threaded.zig
    net.zig
  - Random
    RomuTrio.zig
    Xoshiro256.zig
  - crypto
    tls
    Client.zig
  - hash
    xxhash.zig
  - hash_map.zig
  - http
    HeadParser.zig
    Server.zig
  - mem.zig
  - os
    linux
    IoUring
    test.zig
    uefi.zig
    windows.zig
  - testing.zig
  - zig
    llvm
    Builder.zig
    ir.zig
src
- Air.zig
- Air
  - Legalize.zig
  - Liveness.zig
  - Liveness
    Verify.zig
  - Verify.zig
  - print.zig
- InternPool.zig
- Sema.zig
- Sema
- Type.zig
- Value.zig
- Zcu
  - PerThread.zig
- codegen
  - aarch64
    Select.zig
    abi.zig
  - arm
    abi.zig
  - c.zig
  - c
    type.zig
  - llvm.zig
  - llvm
    FuncGen.zig
  - mips
    abi.zig
  - riscv64
    CodeGen.zig
    abi.zig
  - sparc64
    CodeGen.zig
  - spirv
    CodeGen.zig
  - wasm
    CodeGen.zig
  - x86_64
    CodeGen.zig
stage1
- zig.h
- zig1.wasm
test

CMakeLists.txt+1-1

...	...	@@ -342,7 +342,7 @@ set(ZIG_STAGE2_SOURCES
342	342	src/Package/Module.zig
343	343	src/RangeSet.zig
344	344	src/Sema.zig
345		src/Sema/bitcast.zig
	345	src/Sema/reinterpret.zig
346	346	src/Sema/comptime_ptr_access.zig
347	347	src/Sema/type_resolution.zig
348	348	src/Type.zig

doc/langref/test_packed_structs.zig+2-10

...	...	@@ -26,16 +26,8 @@ fn doTheTest() !void {
26	26	try expectEqual(0x1, divided.quarter4);
27	27
28	28	const ordered: [2]u8 = @bitCast(full);
29		switch (native_endian) {
30		.big => {
31		try expectEqual(0x12, ordered[0]);
32		try expectEqual(0x34, ordered[1]);
33		},
34		.little => {
35		try expectEqual(0x34, ordered[0]);
36		try expectEqual(0x12, ordered[1]);
37		},
38		}
	29	try expectEqual(0x34, ordered[0]);
	30	try expectEqual(0x12, ordered[1]);
39	31	}
40	32
41	33	// test

doc/langref/test_pointer_casting.zig+12-8

...	...	@@ -1,18 +1,22 @@
1	1	const std = @import("std");
	2	const native_endian = @import("builtin").target.cpu.arch.endian();
2	3	const expectEqual = std.testing.expectEqual;
3	4
4	5	test "pointer casting" {
5		const bytes align(@alignOf(u32)) = [_]u8{ 0x12, 0x12, 0x12, 0x12 };
	6	const bytes: [4]u8 align(@alignOf(u32)) = .{ 0x10, 0x20, 0x30, 0x40 };
6	7	const u32_ptr: *const u32 = @ptrCast(&bytes);
7		try expectEqual(0x12121212, u32_ptr.*);
8	8
9		// Even this example is contrived - there are better ways to do the above than
10		// pointer casting. For example, using a slice narrowing cast:
11		const u32_value = std.mem.bytesAsSlice(u32, bytes[0..])[0];
12		try expectEqual(0x12121212, u32_value);
	9	// Because we directly reinterpreted bytes of memory, the `u32` value we
	10	// load from `u32_ptr` depends on the target endian:
	11	switch (native_endian) {
	12	.little => try expectEqual(0x40302010, u32_ptr.*),
	13	.big => try expectEqual(0x10203040, u32_ptr.*),
	14	}
13	15
14		// And even another way, the most straightforward way to do it:
15		try expectEqual(0x12121212, @as(u32, @bitCast(bytes)));
	16	// To instead reinterpret the logical bit representation of `bytes` with no
	17	// dependency on the target endian, use `@bitCast`, which always places
	18	// earlier array elements into less-significant bits:
	19	try expectEqual(0x40302010, @as(u32, @bitCast(bytes)));
16	20	}
17	21
18	22	test "pointer child type" {

lib/compiler_rt/float_from_int.zig+15-4

...	...	@@ -64,10 +64,21 @@ inline fn limb(limbs: []const u32, index: usize) u32 {
64	64	pub inline fn floatFromBigInt(comptime T: type, comptime signedness: std.builtin.Signedness, x: []const u32) T {
65	65	switch (x.len) {
66	66	0 => return 0,
67		inline 1...4 => \|limbs_len\| return @floatFromInt(@as(
68		@Int(signedness, 32 * limbs_len),
69		@bitCast(x[0..limbs_len].*),
70		)),
	67	inline 1...4 => \|limbs_len\| {
	68	const low_to_high: [limbs_len]u32 = switch (@import("builtin").cpu.arch.endian()) {
	69	.little => x[0..limbs_len].*,
	70	.big => switch (limbs_len) {
	71	1 => .{x[0]},
	72	2 => .{ x[1], x[0] },
	73	3 => .{ x[2], x[1], x[0] },
	74	4 => .{ x[3], x[2], x[1], x[0] },
	75	else => comptime unreachable,
	76	},
	77	};
	78	const I = @Int(signedness, 32 * limbs_len);
	79	const int: I = @bitCast(low_to_high);
	80	return @floatFromInt(int);
	81	},
71	82	else => {},
72	83	}
73	84

lib/compiler_rt/int_from_float.zig+12-4

...	...	@@ -80,10 +80,18 @@ pub inline fn bigIntFromFloat(comptime signedness: std.builtin.Signedness, resul
80	80	switch (result.len) {
81	81	0 => return,
82	82	inline 1...4 => \|limbs_len\| {
83		result[0..limbs_len].* = @bitCast(@as(
84		@Int(signedness, 32 * limbs_len),
85		@intFromFloat(a),
86		));
	83	const I = @Int(signedness, 32 * limbs_len);
	84	const low_to_high: [limbs_len]u32 = @bitCast(@as(I, @intFromFloat(a)));
	85	result[0..limbs_len].* = switch (@import("builtin").cpu.arch.endian()) {
	86	.little => low_to_high,
	87	.big => switch (limbs_len) {
	88	1 => .{low_to_high[0]},
	89	2 => .{ low_to_high[1], low_to_high[0] },
	90	3 => .{ low_to_high[2], low_to_high[1], low_to_high[0] },
	91	4 => .{ low_to_high[3], low_to_high[2], low_to_high[1], low_to_high[0] },
	92	else => comptime unreachable,
	93	},
	94	};
87	95	return;
88	96	},
89	97	else => {},

lib/compiler_rt/limb64.zig+12-6

...	...	@@ -75,7 +75,17 @@ fn asLimbs(v: anytype) Limbs(@TypeOf(v)) {
75	75	const int_info = @typeInfo(T).int;
76	76	const limb_cnt = comptime limbCount(int_info.bits);
77	77	const ET = @Int(int_info.signedness, limb_cnt * 64);
78		return @bitCast(@as(ET, v));
	78	const low_to_high: Limbs(T) = @bitCast(@as(ET, v));
	79	switch (endian) {
	80	.little => return low_to_high,
	81	.big => {
	82	var swapped: Limbs(T) = undefined;
	83	for (low_to_high, 0..) \|x, i\| {
	84	swapped[limb_cnt - i - 1] = x;
	85	}
	86	return swapped;
	87	},
	88	}
79	89	}
80	90
81	91	fn limbWrap(limb: u64, is_signed: bool, bits: u16) u64 {
...	...	@@ -944,11 +954,7 @@ inline fn add3(x: *[3]u64, start: usize, v0: u64) void {
944	954
945	955	fn mulwide(a: u64, b: u64) [2]u64 {
946	956	const muldXi = @import("mulXi3.zig").muldXi;
947		const limbs: [2]u64 = @bitCast(muldXi(u64, a, b));
948		return switch (endian) {
949		.little => limbs,
950		.big => .{ limbs[1], limbs[0] },
951		};
	957	return @bitCast(muldXi(u64, a, b));
952	958	}
953	959
954	960	fn __mulo_limb64(out_ptr: []u64, a_ptr: []const u64, b_ptr: [*]const u64, is_signed: bool, bits: u16) callconv(.c) bool {

lib/compiler_rt/udivmod.zig+32-38

...	...	@@ -61,12 +61,6 @@ pub fn __umodti3(a: u128, b: u128) callconv(.c) u128 {
61	61	return r;
62	62	}
63	63
64		const lo = switch (builtin.cpu.arch.endian()) {
65		.big => 1,
66		.little => 0,
67		};
68		const hi = 1 - lo;
69
70	64	// Let _u1 and _u0 be the high and low limbs of U respectively.
71	65	// Returns U / v_ and sets r = U % v_.
72	66	fn divwide_generic(comptime T: type, _u1: T, _u0: T, v_: T, r: *T) T {
...	...	@@ -158,22 +152,22 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
158	152	return 0;
159	153	}
160	154
161		const a: [2]HalfT = @bitCast(a_);
162		const b: [2]HalfT = @bitCast(b_);
	155	const a: [2]HalfT = @bitCast(a_); // [0] is low bits, [1] is high bits
	156	const b: [2]HalfT = @bitCast(b_); // [0] is low bits, [1] is high bits
163	157	var q: [2]HalfT = undefined;
164	158	var r: [2]HalfT = undefined;
165	159
166	160	// When the divisor fits in 64 bits, we can use an optimized path
167		if (b[hi] == 0) {
168		r[hi] = 0;
169		if (a[hi] < b[lo]) {
	161	if (b[1] == 0) {
	162	r[1] = 0;
	163	if (a[1] < b[0]) {
170	164	// The result fits in 64 bits
171		q[hi] = 0;
172		q[lo] = divwide(HalfT, a[hi], a[lo], b[lo], &r[lo]);
	165	q[1] = 0;
	166	q[0] = divwide(HalfT, a[1], a[0], b[0], &r[0]);
173	167	} else {
174	168	// First, divide with the high part to get the remainder. After that a_hi < b_lo.
175		q[hi] = a[hi] / b[lo];
176		q[lo] = divwide(HalfT, a[hi] % b[lo], a[lo], b[lo], &r[lo]);
	169	q[1] = a[1] / b[0];
	170	q[0] = divwide(HalfT, a[1] % b[0], a[0], b[0], &r[0]);
177	171	}
178	172	if (maybe_rem) \|rem\| {
179	173	rem.* = @bitCast(r);
...	...	@@ -181,21 +175,21 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
181	175	return @bitCast(q);
182	176	}
183	177
184		// Large-divisor case: b[hi] != 0, so the quotient fits in one HalfT word.
	178	// Large-divisor case: b[1] != 0, so the quotient fits in one HalfT word.
185	179	//
186	180	// Trial quotient via divwide (Knuth Vol 2, Section 4.3.1):
187	181	// Normalize the divisor so its high half has the MSB set, then use divwide
188	182	// on the top bits to get a trial quotient that is at most 1 too large.
189	183	// This replaces the O(shift) bit-by-bit loop with O(1) operations.
190		const s: Log2Int(HalfT) = @intCast(@clz(b[hi]));
	184	const s: Log2Int(HalfT) = @intCast(@clz(b[1]));
191	185
192	186	if (s == 0) {
193		// b[hi] already has its MSB set, so b >= 2^(T_bits - 1). Since a >= b
	187	// b[1] already has its MSB set, so b >= 2^(T_bits - 1). Since a >= b
194	188	// (we passed the b_ > a_ check), a >= 2^(T_bits - 1) too, meaning
195		// a[hi] also has its MSB set. Therefore a / b < 2, and the quotient
	189	// a[1] also has its MSB set. Therefore a / b < 2, and the quotient
196	190	// is exactly 1.
197	191	q = @bitCast(@as(T, 0));
198		q[lo] = 1;
	192	q[0] = 1;
199	193	if (maybe_rem) \|rem\| {
200	194	rem.* = a_ - b_;
201	195	}
...	...	@@ -207,12 +201,12 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
207	201	std.math.IntFittingRange(0, half_bits),
208	202	@intCast(s),
209	203	));
210		const bn_hi: HalfT = (b[hi] << s) \| (b[lo] >> sr);
	204	const bn_hi: HalfT = (b[1] << s) \| (b[0] >> sr);
211	205
212	206	// Trial numerator: the top (half_bits + s) bits of (a << s), as [a2:a1].
213	207	// a2 < bn_hi is guaranteed since a2 < 2^s and bn_hi >= 2^(half_bits - 1).
214		const a2: HalfT = a[hi] >> sr;
215		const a1: HalfT = (a[hi] << s) \| (a[lo] >> sr);
	208	const a2: HalfT = a[1] >> sr;
	209	const a1: HalfT = (a[1] << s) \| (a[0] >> sr);
216	210
217	211	// Trial quotient via divwide: q_hat = floor([a2:a1] / bn_hi).
218	212	// By Knuth's theorem (normalized divisor), q <= q_hat <= q + 1.
...	...	@@ -223,42 +217,42 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
223	217	// Compute the product using HalfT * HalfT -> T widening multiplications,
224	218	// which are native single-instruction ops when HalfT fits in a register
225	219	// (e.g. u64 * u64 -> u128 via mulq on x86_64, mul on aarch64).
226		// product = q_hat * [b[hi]:b[lo]] = [p_top : p_mid : p_lo] (3 half-words)
227		const prod_lo: T = @as(T, q_hat) * @as(T, b[lo]);
228		const prod_hi: T = @as(T, q_hat) * @as(T, b[hi]);
	220	// product = q_hat * [b[1]:b[0]] = [p_top : p_mid : p_lo] (3 half-words)
	221	const prod_lo: T = @as(T, q_hat) * @as(T, b[0]);
	222	const prod_hi: T = @as(T, q_hat) * @as(T, b[1]);
229	223
230	224	const prod_lo_parts: [2]HalfT = @bitCast(prod_lo);
231	225	const prod_hi_parts: [2]HalfT = @bitCast(prod_hi);
232	226
233		const mid_add = @addWithOverflow(prod_hi_parts[lo], prod_lo_parts[hi]);
	227	const mid_add = @addWithOverflow(prod_hi_parts[0], prod_lo_parts[1]);
234	228	var p_mid: HalfT = mid_add[0];
235		const p_top: HalfT = prod_hi_parts[hi] +% @as(HalfT, mid_add[1]);
236		var p_lo: HalfT = prod_lo_parts[lo];
	229	const p_top: HalfT = prod_hi_parts[1] +% @as(HalfT, mid_add[1]);
	230	var p_lo: HalfT = prod_lo_parts[0];
237	231
238	232	// If product > a, decrement q_hat (at most once, guaranteed by Knuth).
239		if (p_top > 0 or p_mid > a[hi] or (p_mid == a[hi] and p_lo > a[lo])) {
	233	if (p_top > 0 or p_mid > a[1] or (p_mid == a[1] and p_lo > a[0])) {
240	234	q_hat -= 1;
241	235	// Subtract b from the product for correct remainder computation.
242	236	// After correction, (q_hat * b) fits in T bits, so borrows into
243	237	// p_top cancel it to zero -- we only need [p_mid:p_lo].
244		const sub_lo = @subWithOverflow(p_lo, b[lo]);
	238	const sub_lo = @subWithOverflow(p_lo, b[0]);
245	239	p_lo = sub_lo[0];
246		const sub_mid = @subWithOverflow(p_mid, b[hi]);
	240	const sub_mid = @subWithOverflow(p_mid, b[1]);
247	241	const sub_mid2 = @subWithOverflow(sub_mid[0], @as(HalfT, sub_lo[1]));
248	242	p_mid = sub_mid2[0];
249	243	}
250	244
251	245	q = @bitCast(@as(T, 0));
252		q[lo] = q_hat;
	246	q[0] = q_hat;
253	247
254	248	if (maybe_rem) \|rem\| {
255		// remainder = a - q_hat * b = [a[hi]:a[lo]] - [p_mid:p_lo]
	249	// remainder = a - q_hat * b = [a[1]:a[0]] - [p_mid:p_lo]
256	250	// This subtraction is non-negative since q_hat <= true quotient.
257		const rem_lo = @subWithOverflow(a[lo], p_lo);
258		r[lo] = rem_lo[0];
259		const rem_hi = @subWithOverflow(a[hi], p_mid);
	251	const rem_lo = @subWithOverflow(a[0], p_lo);
	252	r[0] = rem_lo[0];
	253	const rem_hi = @subWithOverflow(a[1], p_mid);
260	254	const rem_hi2 = @subWithOverflow(rem_hi[0], @as(HalfT, rem_lo[1]));
261		r[hi] = rem_hi2[0];
	255	r[1] = rem_hi2[0];
262	256	rem.* = @bitCast(r);
263	257	}
264	258	return @bitCast(q);

lib/std/Build/Configuration.zig+4-2

...	...	@@ -3215,7 +3215,8 @@ pub const Storage = enum {
3215	3215	.@"extern" => {
3216	3216	const n = @divExact(@sizeOf(Field), @sizeOf(u32));
3217	3217	defer i.* += n;
3218		return @bitCast(buffer[i...][0..n].);
	3218	const ptr: align(@alignOf(u32)) const Field = @ptrCast(buffer[i...][0..n]);
	3219	return ptr.*;
3219	3220	},
3220	3221	},
3221	3222	else => comptime unreachable,
...	...	@@ -3381,7 +3382,8 @@ pub const Storage = enum {
3381	3382	},
3382	3383	.@"extern" => {
3383	3384	const n = @divExact(@sizeOf(Field), @sizeOf(u32));
3384		buffer[i..][0..n].* = @bitCast(value);
	3385	const ptr: *align(@alignOf(Field)) const [n]u32 = @ptrCast(&value);
	3386	buffer[i..][0..n].* = ptr.*;
3385	3387	return n;
3386	3388	},
3387	3389	},

lib/std/Io/Threaded.zig+6-2

...	...	@@ -14188,9 +14188,11 @@ fn addressUnixToPosix(a: const net.UnixAddress, storage: UnixAddress) posix.so
14188	14188	}
14189	14189
14190	14190	fn address4FromPosix(in: *const posix.sockaddr.in) net.Ip4Address {
	14191	// The network byte order address in `in.addr` is already the byte order we want.
	14192	const addr_bytes: *const [4]u8 = @ptrCast(&in.addr);
14191	14193	return .{
14192	14194	.port = std.mem.bigToNative(u16, in.port),
14193		.bytes = @bitCast(in.addr),
	14195	.bytes = addr_bytes.*,
14194	14196	};
14195	14197	}
14196	14198
...	...	@@ -14204,9 +14206,11 @@ fn address6FromPosix(in6: *const posix.sockaddr.in6) net.Ip6Address {
14204	14206	}
14205	14207
14206	14208	fn address4ToPosix(a: net.Ip4Address) posix.sockaddr.in {
	14209	// The byte order of `a.bytes` is already equivalent to a network byte order address.
	14210	const addr_raw: *align(1) const u32 = @ptrCast(&a.bytes);
14207	14211	return .{
14208	14212	.port = std.mem.nativeToBig(u16, a.port),
14209		.addr = @bitCast(a.bytes),
	14213	.addr = addr_raw.*,
14210	14214	};
14211	14215	}
14212	14216

lib/std/Io/net.zig+1-5

...	...	@@ -592,11 +592,7 @@ pub const Ip6Address = struct {
592	592	if (remaining != 0) return .incomplete;
593	593	}
594	594
595		// Workaround that can be removed when this proposal is
596		// implemented https://github.com/ziglang/zig/issues/19755
597		if ((comptime @import("builtin").cpu.arch.endian()) != .big) {
598		for (&parts) \|part\| part. = @byteSwap(part.*);
599		}
	595	for (&parts) \|part\| part. = @byteSwap(part.*);
600	596
601	597	return .{ .success = .{
602	598	.bytes = @bitCast(parts),

lib/std/Random/RomuTrio.zig+4-2

...	...	@@ -5,6 +5,7 @@
5	5	const std = @import("std");
6	6	const math = std.math;
7	7	const RomuTrio = @This();
	8	const builtin = @import("builtin");
8	9
9	10	x_state: u64,
10	11	y_state: u64,
...	...	@@ -33,7 +34,7 @@ fn next(self: *RomuTrio) u64 {
33	34	}
34	35
35	36	pub fn seedWithBuf(self: *RomuTrio, buf: [24]u8) void {
36		const seed_buf = @as([3]u64, @bitCast(buf));
	37	const seed_buf: [3]u64 = @bitCast(buf);
37	38	self.x_state = seed_buf[0];
38	39	self.y_state = seed_buf[1];
39	40	self.z_state = seed_buf[2];
...	...	@@ -121,7 +122,8 @@ test fill {
121	122	}
122	123
123	124	test "buf seeding test" {
124		const buf0 = @as([24]u8, @bitCast([3]u64{ 16294208416658607535, 13964609475759908645, 4703697494102998476 }));
	125	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
	126	const buf0: [24]u8 = @bitCast([3]u64{ 16294208416658607535, 13964609475759908645, 4703697494102998476 });
125	127	const resulting_state = .{ .x = 16294208416658607535, .y = 13964609475759908645, .z = 4703697494102998476 };
126	128	var r = RomuTrio.init(0);
127	129	r.seedWithBuf(buf0);

lib/std/Random/Xoshiro256.zig+2-2

...	...	@@ -46,12 +46,12 @@ pub fn jump(self: *Xoshiro256) void {
46	46
47	47	while (table != 0) : (table >>= 1) {
48	48	if (@as(u1, @truncate(table)) != 0) {
49		s ^= @as(u256, @bitCast(self.s));
	49	s ^= @bitCast(self.s);
50	50	}
51	51	_ = self.next();
52	52	}
53	53
54		self.s = @as([4]u64, @bitCast(s));
	54	self.s = @bitCast(s);
55	55	}
56	56
57	57	pub fn seed(self: *Xoshiro256, init_s: u64) void {

lib/std/crypto/tls/Client.zig+5-5

...	...	@@ -376,7 +376,7 @@ pub fn init(input: Reader, output: Writer, options: Options) InitError!Client
376	376	const nonce = nonce: {
377	377	const V = @Vector(P.AEAD.nonce_length, u8);
378	378	const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
379		const operand: V = pad ++ @as([8]u8, @bitCast(big(read_seq)));
	379	const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(read_seq)));
380	380	break :nonce @as(V, pv.server_handshake_iv) ^ operand;
381	381	};
382	382	P.AEAD.decrypt(cleartext, ciphertext, auth_tag, record_header, nonce, pv.server_handshake_key) catch
...	...	@@ -416,7 +416,7 @@ pub fn init(input: Reader, output: Writer, options: Options) InitError!Client
416	416	const nonce: [P.AEAD.nonce_length]u8 = nonce: {
417	417	const V = @Vector(P.AEAD.nonce_length, u8);
418	418	const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
419		const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
	419	const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(masked_read_seq)));
420	420	break :nonce @as(V, pv.app_cipher.server_write_IV ++ record_iv) ^ operand;
421	421	};
422	422	const ciphertext = record_decoder.slice(message_len);
...	...	@@ -792,7 +792,7 @@ pub fn init(input: Reader, output: Writer, options: Options) InitError!Client
792	792	const nonce: [P.AEAD.nonce_length]u8 = nonce: {
793	793	const V = @Vector(P.AEAD.nonce_length, u8);
794	794	const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
795		const operand: V = pad ++ @as([8]u8, @bitCast(big(write_seq)));
	795	const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(write_seq)));
796	796	break :nonce @as(V, pv.app_cipher.client_write_IV ++ pv.app_cipher.client_salt) ^ operand;
797	797	};
798	798	var client_verify_msg = .{@intFromEnum(tls.ContentType.handshake)} ++
...	...	@@ -1105,7 +1105,7 @@ fn prepareCiphertextRecord(
1105	1105	const nonce: [P.AEAD.nonce_length]u8 = nonce: {
1106	1106	const V = @Vector(P.AEAD.nonce_length, u8);
1107	1107	const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
1108		const operand: V = pad ++ @as([8]u8, @bitCast(big(c.write_seq)));
	1108	const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(c.write_seq)));
1109	1109	break :nonce @as(V, pv.client_write_IV ++ pv.client_salt) ^ operand;
1110	1110	};
1111	1111	record_iv.* = nonce[P.fixed_iv_length..].*;
...	...	@@ -1214,7 +1214,7 @@ fn readIndirect(c: *Client) Reader.Error!usize {
1214	1214	const nonce: [P.AEAD.nonce_length]u8 = nonce: {
1215	1215	const V = @Vector(P.AEAD.nonce_length, u8);
1216	1216	const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
1217		const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));
	1217	const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(masked_read_seq)));
1218	1218	break :nonce @as(V, pv.server_write_IV ++ record_iv) ^ operand;
1219	1219	};
1220	1220	const ciphertext = input.take(message_len) catch unreachable; // already peeked

lib/std/hash/xxhash.zig+68-57

...	...	@@ -2,7 +2,6 @@ const std = @import("std");
2	2	const builtin = @import("builtin");
3	3	const mem = std.mem;
4	4	const expectEqual = std.testing.expectEqual;
5		const native_endian = builtin.cpu.arch.endian();
6	5
7	6	const rotl = std.math.rotl;
8	7
...	...	@@ -421,7 +420,18 @@ pub const XxHash32 = struct {
421	420	};
422	421
423	422	pub const XxHash3 = struct {
	423	const block_bytes = 64;
424	424	const Block = @Vector(8, u64);
	425	const InputBlock = extern struct {
	426	raw: [block_bytes]u8,
	427	inline fn load(ptr: *const InputBlock) Block {
	428	return @bitCast(ptr.raw);
	429	}
	430	inline fn store(ptr: *InputBlock, val: Block) void {
	431	ptr.raw = @bitCast(val);
	432	}
	433	};
	434
425	435	const default_secret: [192]u8 = .{
426	436	0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
427	437	0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
...	...	@@ -464,10 +474,6 @@ pub const XxHash3 = struct {
464	474	return wide[0] ^ wide[1];
465	475	}
466	476
467		inline fn swap(x: anytype) @TypeOf(x) {
468		return if (native_endian == .big) @byteSwap(x) else x;
469		}
470
471	477	inline fn disableAutoVectorization(x: anytype) void {
472	478	if (!@inComptime()) asm volatile (""
473	479	:
...	...	@@ -476,20 +482,20 @@ pub const XxHash3 = struct {
476	482	}
477	483
478	484	inline fn mix16(seed: u64, input: []const u8, secret: []const u8) u64 {
479		const blk: [4]u64 = @bitCast([_][16]u8{ input[0..16]., secret[0..16]. });
	485	const blk: [4]u64 = @bitCast([2][16]u8{ input[0..16]., secret[0..16]. });
480	486	disableAutoVectorization(seed);
481	487
482	488	return fold(
483		swap(blk[0]) ^ (swap(blk[2]) +% seed),
484		swap(blk[1]) ^ (swap(blk[3]) -% seed),
	489	blk[0] ^ (blk[2] +% seed),
	490	blk[1] ^ (blk[3] -% seed),
485	491	);
486	492	}
487	493
488		const Accumulator = extern struct {
	494	const Accumulator = struct {
489	495	consumed: usize = 0,
490	496	seed: u64,
491		secret: [192]u8 = undefined,
492		state: Block = Block{
	497	secret: [192]u8,
	498	state: Block = .{
493	499	XxHash32.prime_3,
494	500	XxHash64.prime_1,
495	501	XxHash64.prime_2,
...	...	@@ -501,33 +507,35 @@ pub const XxHash3 = struct {
501	507	},
502	508
503	509	inline fn init(seed: u64) Accumulator {
504		var self = Accumulator{ .seed = seed };
505		for (
506		std.mem.bytesAsSlice(Block, &self.secret),
507		std.mem.bytesAsSlice(Block, &default_secret),
508		) \|*dst, src\| {
509		dst.* = swap(swap(src) +% Block{
510		seed, @as(u64, 0) -% seed,
511		seed, @as(u64, 0) -% seed,
512		seed, @as(u64, 0) -% seed,
513		seed, @as(u64, 0) -% seed,
514		});
	510	const seed_block: Block = .{
	511	seed, @as(u64, 0) -% seed,
	512	seed, @as(u64, 0) -% seed,
	513	seed, @as(u64, 0) -% seed,
	514	seed, @as(u64, 0) -% seed,
	515	};
	516
	517	var secret: [192]u8 = undefined;
	518	const secret_blocks: []InputBlock = @ptrCast(&secret);
	519	const default_secret_blocks: []const InputBlock = @ptrCast(&default_secret);
	520	for (secret_blocks, default_secret_blocks) \|dst, src\| {
	521	dst.store(src.load() +% seed_block);
515	522	}
516		return self;
	523
	524	return .{ .seed = seed, .secret = secret };
517	525	}
518	526
519	527	inline fn round(
520	528	noalias state: *Block,
521		noalias input_block: *align(1) const Block,
522		noalias secret_block: *align(1) const Block,
	529	noalias input_block: *const InputBlock,
	530	noalias secret_block: *const InputBlock,
523	531	) void {
524		const data = swap(input_block.*);
525		const mixed = data ^ swap(secret_block.*);
	532	const data = input_block.load();
	533	const mixed = data ^ secret_block.load();
526	534	state.* +%= (mixed & @as(Block, @splat(0xffffffff))) *% (mixed >> @splat(32));
527	535	state.* +%= @shuffle(u64, data, undefined, [_]i32{ 1, 0, 3, 2, 5, 4, 7, 6 });
528	536	}
529	537
530		fn accumulate(noalias self: *Accumulator, blocks: []align(1) const Block) void {
	538	fn accumulate(noalias self: *Accumulator, blocks: []const InputBlock) void {
531	539	const secret = std.mem.bytesAsSlice(u64, self.secret[self.consumed * 8 ..]);
532	540	for (blocks, secret[0..blocks.len]) \|input_block, secret_block\| {
533	541	@prefetch(@as([*]const u8, @ptrCast(input_block)) + 320, .{});
...	...	@@ -536,14 +544,14 @@ pub const XxHash3 = struct {
536	544	}
537	545
538	546	fn scramble(self: *Accumulator) void {
539		const secret_block: Block = @bitCast(self.secret[192 - @sizeOf(Block) .. 192].*);
	547	const secret_block: Block = @bitCast(self.secret[192 - block_bytes .. 192].*);
540	548	self.state ^= self.state >> @splat(47);
541		self.state ^= swap(secret_block);
	549	self.state ^= secret_block;
542	550	self.state *%= @as(Block, @splat(XxHash32.prime_1));
543	551	}
544	552
545		fn consume(noalias self: *Accumulator, input_blocks: []align(1) const Block) void {
546		const blocks_per_scramble = 1024 / @sizeOf(Block);
	553	fn consume(noalias self: *Accumulator, input_blocks: []const InputBlock) void {
	554	const blocks_per_scramble = 1024 / block_bytes;
547	555	std.debug.assert(self.consumed <= blocks_per_scramble);
548	556
549	557	var blocks = input_blocks;
...	...	@@ -561,12 +569,12 @@ pub const XxHash3 = struct {
561	569	self.consumed += blocks.len;
562	570	}
563	571
564		fn digest(noalias self: Accumulator, total_len: u64, noalias last_block: align(1) const Block) u64 {
565		const secret_block = self.secret[192 - @sizeOf(Block) - 7 ..][0..@sizeOf(Block)];
	572	fn digest(noalias self: Accumulator, total_len: u64, noalias last_block: const InputBlock) u64 {
	573	const secret_block = self.secret[192 - block_bytes - 7 ..][0..block_bytes];
566	574	round(&self.state, last_block, @ptrCast(secret_block));
567	575
568		const merge_block: Block = @bitCast(self.secret[11 .. 11 + @sizeOf(Block)].*);
569		self.state ^= swap(merge_block);
	576	const merge_block: Block = @bitCast(self.secret[11 .. 11 + block_bytes].*);
	577	self.state ^= merge_block;
570	578
571	579	var result = XxHash64.prime_1 *% total_len;
572	580	inline for (0..4) \|i\| {
...	...	@@ -588,7 +596,7 @@ pub const XxHash3 = struct {
588	596	if (input.len > 0) return hash3(seed, input, secret);
589	597
590	598	const flip: [2]u64 = @bitCast(secret[56..72].*);
591		const key = swap(flip[0]) ^ swap(flip[1]);
	599	const key = flip[0] ^ flip[1];
592	600	return avalanche(.h64, seed ^ key);
593	601	}
594	602
...	...	@@ -604,8 +612,8 @@ pub const XxHash3 = struct {
604	612	input[input.len / 2],
605	613	});
606	614
607		const key = @as(u64, swap(flip[0]) ^ swap(flip[1])) +% seed;
608		return avalanche(.h64, key ^ swap(blk));
	615	const key = @as(u64, flip[0] ^ flip[1]) +% seed;
	616	return avalanche(.h64, key ^ blk);
609	617	}
610	618
611	619	fn hash8(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 {
...	...	@@ -619,8 +627,8 @@ pub const XxHash3 = struct {
619	627	});
620	628
621	629	const mixed = seed ^ (@as(u64, @byteSwap(@as(u32, @truncate(seed)))) << 32);
622		const key = (swap(flip[0]) ^ swap(flip[1])) -% mixed;
623		const combined = (@as(u64, swap(blk[0])) << 32) +% swap(blk[1]);
	630	const key = (flip[0] ^ flip[1]) -% mixed;
	631	const combined = (@as(u64, blk[0]) << 32) +% blk[1];
624	632	return avalanche(.{ .rrmxmx = input.len }, key ^ combined);
625	633	}
626	634
...	...	@@ -634,8 +642,8 @@ pub const XxHash3 = struct {
634	642	input[input.len - 8 ..][0..8].*,
635	643	});
636	644
637		const lo = swap(blk[0]) ^ ((swap(flip[0]) ^ swap(flip[1])) +% seed);
638		const hi = swap(blk[1]) ^ ((swap(flip[2]) ^ swap(flip[3])) -% seed);
	645	const lo = blk[0] ^ ((flip[0] ^ flip[1]) +% seed);
	646	const hi = blk[1] ^ ((flip[2] ^ flip[3]) -% seed);
639	647	const combined = @as(u64, input.len) +% @byteSwap(lo) +% hi +% fold(lo, hi);
640	648	return avalanche(.h3, combined);
641	649	}
...	...	@@ -679,11 +687,11 @@ pub const XxHash3 = struct {
679	687	@branchHint(.unlikely);
680	688	std.debug.assert(input.len >= 240);
681	689
682		const block_count = ((input.len - 1) / @sizeOf(Block)) * @sizeOf(Block);
683		const last_block = input[input.len - @sizeOf(Block) ..][0..@sizeOf(Block)];
	690	const block_count = ((input.len - 1) / block_bytes) * block_bytes;
	691	const last_block = input[input.len - block_bytes ..][0..block_bytes];
684	692
685	693	var acc = Accumulator.init(seed);
686		acc.consume(std.mem.bytesAsSlice(Block, input[0..block_count]));
	694	acc.consume(std.mem.bytesAsSlice(InputBlock, input[0..block_count]));
687	695	return acc.digest(input.len, @ptrCast(last_block));
688	696	}
689	697
...	...	@@ -716,21 +724,21 @@ pub const XxHash3 = struct {
716	724	@memcpy(self.buffer[self.buffered..], consumable[0..remaining]);
717	725	consumable = consumable[remaining..];
718	726
719		self.accumulator.consume(std.mem.bytesAsSlice(Block, &self.buffer));
	727	self.accumulator.consume(std.mem.bytesAsSlice(InputBlock, &self.buffer));
720	728	self.buffered = 0;
721	729	}
722	730
723	731	// The input isn't small enough to fit in the buffer. Consume it directly.
724	732	if (consumable.len > self.buffer.len) {
725		const block_count = ((consumable.len - 1) / @sizeOf(Block)) * @sizeOf(Block);
726		self.accumulator.consume(std.mem.bytesAsSlice(Block, consumable[0..block_count]));
	733	const block_count = ((consumable.len - 1) / block_bytes) * block_bytes;
	734	self.accumulator.consume(std.mem.bytesAsSlice(InputBlock, consumable[0..block_count]));
727	735	consumable = consumable[block_count..];
728	736
729	737	// In case we consume all remaining input, write the last block to end of the buffer
730	738	// to populate the last_block_copy in final() similar to hashLong()'s last_block.
731	739	@memcpy(
732		self.buffer[self.buffer.len - @sizeOf(Block) .. self.buffer.len],
733		(consumable.ptr - @sizeOf(Block))[0..@sizeOf(Block)],
	740	self.buffer[self.buffer.len - block_bytes .. self.buffer.len],
	741	(consumable.ptr - block_bytes)[0..block_bytes],
734	742	);
735	743	}
736	744
...	...	@@ -751,16 +759,16 @@ pub const XxHash3 = struct {
751	759
752	760	// Make a copy of the Accumulator state in case `self` needs to update() / be used later.
753	761	var accumulator_copy = self.accumulator;
754		var last_block_copy: [@sizeOf(Block)]u8 = undefined;
	762	var last_block_copy: [block_bytes]u8 = undefined;
755	763
756	764	// Digest the last block onthe Accumulator copy.
757	765	return accumulator_copy.digest(self.total_len, last_block: {
758		if (self.buffered >= @sizeOf(Block)) {
759		const block_count = ((self.buffered - 1) / @sizeOf(Block)) * @sizeOf(Block);
760		accumulator_copy.consume(std.mem.bytesAsSlice(Block, self.buffer[0..block_count]));
761		break :last_block @ptrCast(self.buffer[self.buffered - @sizeOf(Block) ..][0..@sizeOf(Block)]);
	766	if (self.buffered >= block_bytes) {
	767	const block_count = ((self.buffered - 1) / block_bytes) * block_bytes;
	768	accumulator_copy.consume(std.mem.bytesAsSlice(InputBlock, self.buffer[0..block_count]));
	769	break :last_block @ptrCast(self.buffer[self.buffered - block_bytes ..][0..block_bytes]);
762	770	} else {
763		const remaining = @sizeOf(Block) - self.buffered;
	771	const remaining = block_bytes - self.buffered;
764	772	@memcpy(last_block_copy[0..remaining], self.buffer[self.buffer.len - remaining ..][0..remaining]);
765	773	@memcpy(last_block_copy[remaining..][0..self.buffered], self.buffer[0..self.buffered]);
766	774	break :last_block @ptrCast(&last_block_copy);
...	...	@@ -780,6 +788,7 @@ fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64)
780	788	}
781	789
782	790	test "xxhash3" {
	791	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
783	792	const H = XxHash3;
784	793	// Non-Seeded Tests
785	794	try testExpect(H, 0, "", 0x2d06800538d394c2);
...	...	@@ -811,6 +820,7 @@ test "xxhash3" {
811	820	}
812	821
813	822	test "xxhash3 smhasher" {
	823	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
814	824	const Test = struct {
815	825	fn do() !void {
816	826	try expectEqual(verify.smhasher(XxHash3.hash), 0x9a636405);
...	...	@@ -822,6 +832,7 @@ test "xxhash3 smhasher" {
822	832	}
823	833
824	834	test "xxhash3 iterative api" {
	835	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
825	836	const Test = struct {
826	837	fn do() !void {
827	838	try verify.iterativeApi(XxHash3);

lib/std/hash_map.zig+2-2

...	...	@@ -593,8 +593,8 @@ fn Custom(
593	593	fingerprint: FingerPrint = free,
594	594	used: u1 = 0,
595	595
596		const slot_free = @as(u8, @bitCast(Metadata{ .fingerprint = free }));
597		const slot_tombstone = @as(u8, @bitCast(Metadata{ .fingerprint = tombstone }));
	596	const slot_free: u8 = @bitCast(Metadata{ .fingerprint = free });
	597	const slot_tombstone: u8 = @bitCast(Metadata{ .fingerprint = tombstone });
598	598
599	599	pub fn isUsed(self: Metadata) bool {
600	600	return self.used == 1;

lib/std/http/HeadParser.zig+5-8

...	...	@@ -116,11 +116,8 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {
116	116
117	117	const chunk = bytes[index..][0..vector_len];
118	118	const v: Vector = chunk.*;
119		// depends on https://github.com/ziglang/zig/issues/19755
120		// const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
121		// const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
122		const matches_r: BitVector = @select(u1, v == @as(Vector, @splat('\r')), @as(Vector, @splat(1)), @as(Vector, @splat(0)));
123		const matches_n: BitVector = @select(u1, v == @as(Vector, @splat('\n')), @as(Vector, @splat(1)), @as(Vector, @splat(0)));
	119	const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
	120	const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
124	121	const matches_or: SizeVector = matches_r \| matches_n;
125	122
126	123	const matches = @reduce(.Add, matches_or);
...	...	@@ -331,15 +328,15 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {
331	328	}
332	329
333	330	inline fn int16(array: *const [2]u8) u16 {
334		return @bitCast(array.*);
	331	return std.mem.toNative(u16, @bitCast(array.*), .little);
335	332	}
336	333
337	334	inline fn int24(array: *const [3]u8) u24 {
338		return @bitCast(array.*);
	335	return std.mem.toNative(u24, @bitCast(array.*), .little);
339	336	}
340	337
341	338	inline fn int32(array: *const [4]u8) u32 {
342		return @bitCast(array.*);
	339	return std.mem.toNative(u32, @bitCast(array.*), .little);
343	340	}
344	341
345	342	inline fn intShift(comptime T: type, x: anytype) T {

lib/std/http/Server.zig+10-5

...	...	@@ -726,7 +726,7 @@ pub const WebSocket = struct {
726	726	else => @intFromEnum(h1.payload_len),
727	727	};
728	728	if (len > in.buffer.len) return error.MessageOversize;
729		const mask: u32 = @bitCast((try in.takeArray(4)).*);
	729	const mask: [4]u8 = (try in.takeArray(4)).*;
730	730	const payload = try in.take(len);
731	731
732	732	// Skip pongs.
...	...	@@ -734,11 +734,16 @@ pub const WebSocket = struct {
734	734
735	735	// The last item may contain a partial word of unused data.
736	736	const floored_len = (payload.len / 4) * 4;
737		const u32_payload: []align(1) u32 = @ptrCast(payload[0..floored_len]);
738		for (u32_payload) \|elem\| elem. ^= mask;
739		const mask_bytes: []const u8 = @ptrCast(&mask);
740		for (payload[floored_len..], mask_bytes[0 .. payload.len - floored_len]) \|*leftover, m\|
	737
	738	const payload_chunks: [][4]u8 = @ptrCast(payload[0..floored_len]);
	739	for (payload_chunks) \|*chunk\| {
	740	const mask_i: u32 = @bitCast(mask);
	741	const chunk_i: u32 = @bitCast(chunk.*);
	742	chunk.* = @bitCast(chunk_i ^ mask_i);
	743	}
	744	for (payload[floored_len..], mask[0 .. payload.len - floored_len]) \|*leftover, m\| {
741	745	leftover.* ^= m;
	746	}
742	747
743	748	return .{
744	749	.opcode = h0.opcode,

lib/std/mem.zig+80-58

...	...	@@ -1848,18 +1848,18 @@ pub fn readVarPackedInt(
1848	1848	if (@bitSizeOf(T) <= 8) {
1849	1849	// These are the same shifts/masks we perform below, but adds `@truncate`/`@intCast`
1850	1850	// where needed since int is smaller than a byte.
1851		const value = if (read_size == 1) b: {
1852		break :b @as(uN, @truncate(read_bytes[0] >> bit_shift));
	1851	const value: uN = if (read_size == 1) b: {
	1852	break :b @truncate(read_bytes[0] >> bit_shift);
1853	1853	} else b: {
1854	1854	const i: u1 = @intFromBool(endian == .big);
1855		const head = @as(uN, @truncate(read_bytes[i] >> bit_shift));
1856		const tail_shift = @as(Log2N, @intCast(@as(u4, 8) - bit_shift));
1857		const tail = @as(uN, @truncate(read_bytes[1 - i]));
	1855	const head: uN = @truncate(read_bytes[i] >> bit_shift);
	1856	const tail_shift: Log2N = @intCast(@as(u4, 8) - bit_shift);
	1857	const tail: uN = @truncate(read_bytes[1 - i]);
1858	1858	break :b (tail << tail_shift) \| head;
1859	1859	};
1860	1860	switch (signedness) {
1861		.signed => return @as(T, @intCast((@as(iN, @bitCast(value)) << pad) >> pad)),
1862		.unsigned => return @as(T, @intCast((@as(uN, @bitCast(value)) << pad) >> pad)),
	1861	.signed => return @intCast((@as(iN, @bitCast(value)) << pad) >> pad),
	1862	.unsigned => return @intCast((value << pad) >> pad),
1863	1863	}
1864	1864	}
1865	1865
...	...	@@ -1880,8 +1880,8 @@ pub fn readVarPackedInt(
1880	1880	},
1881	1881	}
1882	1882	switch (signedness) {
1883		.signed => return @as(T, @intCast((@as(iN, @bitCast(int)) << pad) >> pad)),
1884		.unsigned => return @as(T, @intCast((@as(uN, @bitCast(int)) << pad) >> pad)),
	1883	.signed => return @intCast((@as(iN, @bitCast(int)) << pad) >> pad),
	1884	.unsigned => return @intCast((int << pad) >> pad),
1885	1885	}
1886	1886	}
1887	1887
...	...	@@ -1896,8 +1896,13 @@ test readVarPackedInt {
1896	1896	/// The bit count of T must be evenly divisible by 8.
1897	1897	/// This function cannot fail and cannot cause undefined behavior.
1898	1898	pub inline fn readInt(comptime T: type, buffer: *const [@divExact(@typeInfo(T).int.bits, 8)]u8, endian: Endian) T {
1899		const value: T = @bitCast(buffer.*);
1900		return if (endian == native_endian) value else @byteSwap(value);
	1899	// Zig's logical bit order aligns with a little-endian byte array, so when reading in big-endian
	1900	// we must `@byteSwap` the int after we `@bitCast` to it.
	1901	const little_val: T = @bitCast(buffer.*);
	1902	return switch (endian) {
	1903	.little => little_val,
	1904	.big => @byteSwap(little_val),
	1905	};
1901	1906	}
1902	1907
1903	1908	test readInt {
...	...	@@ -1940,13 +1945,15 @@ fn readPackedIntLittle(comptime T: type, bytes: []const u8, bit_offset: usize) T
1940	1945	// Read by loading a LoadInt, and then follow it up with a 1-byte read
1941	1946	// of the tail if bit_offset pushed us over a byte boundary.
1942	1947	const read_bytes = bytes[bit_offset / 8 ..];
1943		const val = @as(uN, @truncate(readInt(LoadInt, read_bytes[0..load_size], .little) >> bit_shift));
	1948	const val: uN = @truncate(readInt(LoadInt, read_bytes[0..load_size], .little) >> bit_shift);
1944	1949	if (bit_shift > load_tail_bits) {
1945	1950	const tail_bits = @as(Log2N, @intCast(bit_shift - load_tail_bits));
1946	1951	const tail_byte = read_bytes[load_size];
1947	1952	const tail_truncated = if (bit_count < 8) @as(uN, @truncate(tail_byte)) else @as(uN, tail_byte);
1948		return @as(T, @bitCast(val \| (tail_truncated << (@as(Log2N, @truncate(bit_count)) -% tail_bits))));
1949		} else return @as(T, @bitCast(val));
	1953	return @bitCast(val \| (tail_truncated << (@as(Log2N, @truncate(bit_count)) -% tail_bits)));
	1954	} else {
	1955	return @bitCast(val);
	1956	}
1950	1957	}
1951	1958
1952	1959	fn readPackedIntBig(comptime T: type, bytes: []const u8, bit_offset: usize) T {
...	...	@@ -1972,8 +1979,10 @@ fn readPackedIntBig(comptime T: type, bytes: []const u8, bit_offset: usize) T {
1972	1979	if (bit_shift > load_tail_bits) {
1973	1980	const tail_bits = @as(Log2N, @intCast(bit_shift - load_tail_bits));
1974	1981	const tail_byte = if (bit_count < 8) @as(uN, @truncate(read_bytes[0])) else @as(uN, read_bytes[0]);
1975		return @as(T, @bitCast(val \| (tail_byte << (@as(Log2N, @truncate(bit_count)) -% tail_bits))));
1976		} else return @as(T, @bitCast(val));
	1982	return @bitCast(val \| (tail_byte << (@as(Log2N, @truncate(bit_count)) -% tail_bits)));
	1983	} else {
	1984	return @bitCast(val);
	1985	}
1977	1986	}
1978	1987
1979	1988	/// Loads an integer from packed memory.
...	...	@@ -2011,7 +2020,12 @@ test "comptime read/write int" {
2011	2020	/// This function always succeeds, has defined behavior for all inputs, but
2012	2021	/// the integer bit width must be divisible by 8.
2013	2022	pub inline fn writeInt(comptime T: type, buffer: *[@divExact(@typeInfo(T).int.bits, 8)]u8, value: T, endian: Endian) void {
2014		buffer.* = @bitCast(if (endian == native_endian) value else @byteSwap(value));
	2023	// Zig's logical bit order aligns with a little-endian byte array, so when writing in big-endian
	2024	// we must `@byteSwap` the int before we `@bitCast` to an array.
	2025	buffer.* = switch (endian) {
	2026	.little => @bitCast(value),
	2027	.big => @bitCast(@byteSwap(value)),
	2028	};
2015	2029	}
2016	2030
2017	2031	test writeInt {
...	...	@@ -2209,10 +2223,10 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void {
2209	2223	/// (Changing their endianness)
2210	2224	pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *align(a.toByteUnits()) S) void {
2211	2225	switch (@typeInfo(S)) {
2212		.@"struct" => \|struct_info\| {
2213		if (struct_info.backing_integer) \|Int\| {
	2226	.@"struct" => \|@"struct"\| {
	2227	if (@"struct".backing_integer) \|Int\| {
2214	2228	ptr.* = @bitCast(@byteSwap(@as(Int, @bitCast(ptr.*))));
2215		} else inline for (struct_info.field_types, struct_info.field_names, struct_info.field_attrs) \|f_type, f_name, f_attr\| {
	2229	} else inline for (@"struct".field_types, @"struct".field_names, @"struct".field_attrs) \|f_type, f_name, f_attr\| {
2216	2230	switch (@typeInfo(f_type)) {
2217	2231	.@"struct" => byteSwapAllFieldsAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),
2218	2232	.@"union", .array => byteSwapAllFieldsAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),
...	...	@@ -2220,8 +2234,8 @@ pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *a
2220	2234	@field(ptr, f_name) = @enumFromInt(@byteSwap(@intFromEnum(@field(ptr, f_name))));
2221	2235	},
2222	2236	.bool => {},
2223		.float => \|float_info\| {
2224		@field(ptr, f_name) = @bitCast(@byteSwap(@as(@Int(.unsigned, float_info.bits), @bitCast(@field(ptr, f_name)))));
	2237	.float => \|float\| {
	2238	@field(ptr, f_name) = @bitCast(@byteSwap(@as(@Int(.unsigned, float.bits), @bitCast(@field(ptr, f_name)))));
2225	2239	},
2226	2240	else => {
2227	2241	@field(ptr, f_name) = @byteSwap(@field(ptr, f_name));
...	...	@@ -2229,23 +2243,26 @@ pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *a
2229	2243	}
2230	2244	}
2231	2245	},
2232		.@"union" => \|union_info\| {
2233		if (union_info.tag_type != null) {
2234		@compileError("byteSwapAllFields expects an untagged union");
	2246	.@"union" => \|@"union"\| if (@"union".backing_integer) \|Int\| {
	2247	ptr.* = @bitCast(@byteSwap(@as(Int, @bitCast(ptr.*))));
	2248	} else {
	2249	if (@"union".layout != .@"extern") {
	2250	@compileError("byteSwapAllFields expects a packed or extern union");
2235	2251	}
2236	2252
2237		const first_size = @bitSizeOf(union_info.field_types[0]);
2238		inline for (union_info.field_types) \|field_type\| {
	2253	const first_size = @bitSizeOf(@"union".field_types[0]);
	2254	inline for (@"union".field_types) \|field_type\| {
2239	2255	if (@bitSizeOf(field_type) != first_size) {
2240	2256	@compileError("Unable to byte-swap unions with varying field sizes");
2241	2257	}
2242	2258	}
2243	2259
2244		const BackingInt = @Int(.unsigned, @bitSizeOf(S));
2245		ptr.* = @bitCast(@byteSwap(@as(BackingInt, @bitCast(ptr.*))));
	2260	const FieldInt = @Int(.unsigned, first_size);
	2261	const field_ptr = &@field(ptr, @"union".field_names[0]);
	2262	field_ptr.* = @bitCast(@byteSwap(@as(FieldInt, @bitCast(field_ptr.*))));
2246	2263	},
2247		.array => \|info\| {
2248		byteSwapAllElements(info.child, ptr);
	2264	.array => \|array\| {
	2265	byteSwapAllElements(array.child, ptr);
2249	2266	},
2250	2267	else => {
2251	2268	ptr.* = @byteSwap(ptr.*);
...	...	@@ -2291,7 +2308,7 @@ test byteSwapAllFields {
2291	2308	.f2 = 0x12345678,
2292	2309	.f3 = .{0x12},
2293	2310	.f4 = true,
2294		.f5 = @as(f32, @bitCast(@as(u32, 0x4640e400))),
	2311	.f5 = @bitCast(@as(u32, 0x4640e400)),
2295	2312	.f6 = .{ .f0 = 0x1234 },
2296	2313	};
2297	2314	var k = K{
...	...	@@ -2300,7 +2317,7 @@ test byteSwapAllFields {
2300	2317	.f2 = 0x1234,
2301	2318	.f3 = .{0x12},
2302	2319	.f4 = false,
2303		.f5 = @as(f32, @bitCast(@as(u32, 0x45d42800))),
	2320	.f5 = @bitCast(@as(u32, 0x45d42800)),
2304	2321	};
2305	2322	var p: P = @bitCast(@as(u32, 0x01234567));
2306	2323	var a: A = A{
...	...	@@ -2318,7 +2335,7 @@ test byteSwapAllFields {
2318	2335	.f2 = 0x78563412,
2319	2336	.f3 = .{0x12},
2320	2337	.f4 = true,
2321		.f5 = @as(f32, @bitCast(@as(u32, 0x00e44046))),
	2338	.f5 = @bitCast(@as(u32, 0x00e44046)),
2322	2339	.f6 = .{ .f0 = 0x3412 },
2323	2340	}, s);
2324	2341	try std.testing.expectEqual(K{
...	...	@@ -2327,7 +2344,7 @@ test byteSwapAllFields {
2327	2344	.f2 = 0x3412,
2328	2345	.f3 = .{0x12},
2329	2346	.f4 = false,
2330		.f5 = @as(f32, @bitCast(@as(u32, 0x0028d445))),
	2347	.f5 = @bitCast(@as(u32, 0x0028d445)),
2331	2348	}, k);
2332	2349	try std.testing.expectEqual(@as(P, @bitCast(@as(u32, 0x67452301))), p);
2333	2350	try std.testing.expectEqual(A{
...	...	@@ -2348,8 +2365,9 @@ pub fn byteSwapAllElements(comptime Elem: type, slice: []Elem) void {
2348	2365	elem.* = @enumFromInt(@byteSwap(@intFromEnum(elem.*)));
2349	2366	},
2350	2367	.bool => {},
2351		.float => \|float_info\| {
2352		elem.* = @bitCast(@byteSwap(@as(@Int(.unsigned, float_info.bits), @bitCast(elem.*))));
	2368	.float => \|float\| {
	2369	const int_repr: @Int(.unsigned, float.bits) = @bitCast(elem.*);
	2370	elem.* = @bitCast(@byteSwap(int_repr));
2353	2371	},
2354	2372	else => {
2355	2373	elem.* = @byteSwap(elem.*);
...	...	@@ -3870,25 +3888,29 @@ inline fn reverseVector(comptime N: usize, comptime T: type, a: []T) [N]T {
3870	3888	pub fn reverse(comptime T: type, items: []T) void {
3871	3889	var i: usize = 0;
3872	3890	const end = items.len / 2;
3873		if (use_vectors and
3874		!@inComptime() and
3875		@bitSizeOf(T) > 0 and
3876		std.math.isPowerOfTwo(@bitSizeOf(T)))
3877		{
3878		if (std.simd.suggestVectorLength(T)) \|simd_size\| {
3879		if (simd_size <= end) {
3880		const simd_end = end - (simd_size - 1);
3881		while (i < simd_end) : (i += simd_size) {
3882		const left_slice = items[i .. i + simd_size];
3883		const right_slice = items[items.len - i - simd_size .. items.len - i];
3884
3885		const left_shuffled: [simd_size]T = reverseVector(simd_size, T, left_slice);
3886		const right_shuffled: [simd_size]T = reverseVector(simd_size, T, right_slice);
3887
3888		@memcpy(right_slice, &left_shuffled);
3889		@memcpy(left_slice, &right_shuffled);
3890		}
3891		}
	3891
	3892	vec: {
	3893	if (!use_vectors) break :vec;
	3894	if (@inComptime()) break :vec;
	3895	switch (@typeInfo(T)) {
	3896	.int, .float => {},
	3897	.pointer => \|pointer\| if (pointer.size == .slice) break :vec,
	3898	else => break :vec,
	3899	}
	3900	if (@bitSizeOf(T) == 0 or !comptime std.math.isPowerOfTwo(@bitSizeOf(T))) break :vec;
	3901	const simd_size = std.simd.suggestVectorLength(T) orelse break :vec;
	3902	if (simd_size > end) break :vec;
	3903
	3904	const simd_end = end - (simd_size - 1);
	3905	while (i < simd_end) : (i += simd_size) {
	3906	const left_slice = items[i .. i + simd_size];
	3907	const right_slice = items[items.len - i - simd_size .. items.len - i];
	3908
	3909	const left_shuffled: [simd_size]T = reverseVector(simd_size, T, left_slice);
	3910	const right_shuffled: [simd_size]T = reverseVector(simd_size, T, right_slice);
	3911
	3912	@memcpy(right_slice, &left_shuffled);
	3913	@memcpy(left_slice, &right_shuffled);
3892	3914	}
3893	3915	}
3894	3916
...	...	@@ -5009,8 +5031,8 @@ test "read/write(Var)PackedInt" {
5009	5031	for ([_]PackedType{
5010	5032	~@as(PackedType, 0), // all ones: -1 iN / maxInt uN
5011	5033	@as(PackedType, 0), // all zeros: 0 iN / 0 uN
5012		@as(PackedType, @bitCast(@as(iPackedType, math.maxInt(iPackedType)))), // maxInt iN
5013		@as(PackedType, @bitCast(@as(iPackedType, math.minInt(iPackedType)))), // maxInt iN
	5034	@bitCast(@as(iPackedType, math.maxInt(iPackedType))), // maxInt iN
	5035	@bitCast(@as(iPackedType, math.minInt(iPackedType))), // maxInt iN
5014	5036	random.int(PackedType), // random
5015	5037	random.int(PackedType), // random
5016	5038	}) \|write_value\| {

lib/std/os/linux/IoUring/test.zig+24-8

...	...	@@ -526,7 +526,9 @@ test "sendmsg/recvmsg" {
526	526
527	527	var address_server: linux.sockaddr.in = .{
528	528	.port = 0,
529		.addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
	529	.addr = @as(*align(1) const u32, @ptrCast(
	530	&@as([4]u8, .{ 127, 0, 0, 1 }),
	531	)).*,
530	532	};
531	533
532	534	const server = try socket(address_server.family, posix.SOCK.DGRAM, 0);
...	...	@@ -1028,7 +1030,9 @@ test "shutdown" {
1028	1030
1029	1031	var address: linux.sockaddr.in = .{
1030	1032	.port = 0,
1031		.addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
	1033	.addr = @as(*align(1) const u32, @ptrCast(
	1034	&@as([4]u8, .{ 127, 0, 0, 1 }),
	1035	)).*,
1032	1036	};
1033	1037
1034	1038	// Socket bound, expect shutdown to work
...	...	@@ -1740,7 +1744,9 @@ test "accept multishot" {
1740	1744
1741	1745	var address: linux.sockaddr.in = .{
1742	1746	.port = 0,
1743		.addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
	1747	.addr = @as(*align(1) const u32, @ptrCast(
	1748	&@as([4]u8, .{ 127, 0, 0, 1 }),
	1749	)).*,
1744	1750	};
1745	1751	const listener_socket = try createListenerSocket(&address);
1746	1752	defer _ = linux.close(listener_socket);
...	...	@@ -1842,7 +1848,9 @@ test "accept_direct" {
1842	1848	defer ring.deinit();
1843	1849	var address: linux.sockaddr.in = .{
1844	1850	.port = 0,
1845		.addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
	1851	.addr = @as(*align(1) const u32, @ptrCast(
	1852	&@as([4]u8, .{ 127, 0, 0, 1 }),
	1853	)).*,
1846	1854	};
1847	1855
1848	1856	// register direct file descriptors
...	...	@@ -1931,7 +1939,9 @@ test "accept_multishot_direct" {
1931	1939
1932	1940	var address: linux.sockaddr.in = .{
1933	1941	.port = 0,
1934		.addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
	1942	.addr = @as(*align(1) const u32, @ptrCast(
	1943	&@as([4]u8, .{ 127, 0, 0, 1 }),
	1944	)).*,
1935	1945	};
1936	1946
1937	1947	var registered_fds: [2]linux.fd_t = @splat(-1);
...	...	@@ -2041,7 +2051,9 @@ test "socket_direct/socket_direct_alloc/close_direct" {
2041	2051	// use sockets from registered_fds in connect operation
2042	2052	var address: linux.sockaddr.in = .{
2043	2053	.port = 0,
2044		.addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
	2054	.addr = @as(*align(1) const u32, @ptrCast(
	2055	&@as([4]u8, .{ 127, 0, 0, 1 }),
	2056	)).*,
2045	2057	};
2046	2058	const listener_socket = try createListenerSocket(&address);
2047	2059	defer _ = linux.close(listener_socket);
...	...	@@ -2426,7 +2438,9 @@ test "bind/listen/connect" {
2426	2438
2427	2439	var addr: linux.sockaddr.in = .{
2428	2440	.port = 0,
2429		.addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
	2441	.addr = @as(*align(1) const u32, @ptrCast(
	2442	&@as([4]u8, .{ 127, 0, 0, 1 }),
	2443	)).*,
2430	2444	};
2431	2445	const proto: u32 = if (addr.family == linux.AF.UNIX) 0 else linux.IPPROTO.TCP;
2432	2446
...	...	@@ -2614,7 +2628,9 @@ pub fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness {
2614	2628	// Create a TCP server socket
2615	2629	var address: linux.sockaddr.in = .{
2616	2630	.port = 0,
2617		.addr = @bitCast([4]u8{ 127, 0, 0, 1 }),
	2631	.addr = @as(*align(1) const u32, @ptrCast(
	2632	&@as([4]u8, .{ 127, 0, 0, 1 }),
	2633	)).*,
2618	2634	};
2619	2635	const listener_socket = try createListenerSocket(&address);
2620	2636	errdefer _ = linux.close(listener_socket);

lib/std/os/uefi.zig+1-1

...	...	@@ -218,7 +218,7 @@ pub const TimeCapabilities = extern struct {
218	218	pub const FileHandle = *opaque {};
219	219
220	220	test "GUID formatting" {
221		const bytes = [_]u8{ 137, 60, 203, 50, 128, 128, 124, 66, 186, 19, 80, 73, 135, 59, 194, 135 };
	221	const bytes: [16]u8 = .{ 137, 60, 203, 50, 128, 128, 124, 66, 186, 19, 80, 73, 135, 59, 194, 135 };
222	222	const guid: Guid = @bitCast(bytes);
223	223
224	224	const str = try std.fmt.allocPrint(std.testing.allocator, "{f}", .{guid});

lib/std/os/windows.zig+37-36

...	...	@@ -4189,19 +4189,11 @@ pub const GUID = extern struct {
4189	4189	Data3: u16,
4190	4190	Data4: [8]u8,
4191	4191
4192		const hex_offsets = switch (builtin.target.cpu.arch.endian()) {
4193		.big => [16]u6{
4194		0, 2, 4, 6,
4195		9, 11, 14, 16,
4196		19, 21, 24, 26,
4197		28, 30, 32, 34,
4198		},
4199		.little => [16]u6{
4200		6, 4, 2, 0,
4201		11, 9, 16, 14,
4202		19, 21, 24, 26,
4203		28, 30, 32, 34,
4204		},
	4192	const hex_offsets: [16]u6 = .{
	4193	6, 4, 2, 0,
	4194	11, 9, 16, 14,
	4195	19, 21, 24, 26,
	4196	28, 30, 32, 34,
4205	4197	};
4206	4198
4207	4199	pub fn parse(s: []const u8) GUID {
...	...	@@ -4216,12 +4208,21 @@ pub const GUID = extern struct {
4216	4208	assert(s[13] == '-');
4217	4209	assert(s[18] == '-');
4218	4210	assert(s[23] == '-');
4219		var bytes: [16]u8 = undefined;
4220		for (hex_offsets, 0..) \|hex_offset, i\| {
4221		bytes[i] = (try std.fmt.charToDigit(s[hex_offset], 16)) << 4 \|
4222		try std.fmt.charToDigit(s[hex_offset + 1], 16);
4223		}
4224		return @as(GUID, @bitCast(bytes));
	4211	var raw1: [4]u8 = undefined;
	4212	var raw2: [2]u8 = undefined;
	4213	var raw3: [2]u8 = undefined;
	4214	var raw4: [8]u8 = undefined;
	4215	assert((try std.fmt.hexToBytes(&raw1, s[0..8])).len == raw1.len);
	4216	assert((try std.fmt.hexToBytes(&raw2, s[9..13])).len == raw2.len);
	4217	assert((try std.fmt.hexToBytes(&raw3, s[14..18])).len == raw3.len);
	4218	assert((try std.fmt.hexToBytes(raw4[0..2], s[19..23])).len == 2);
	4219	assert((try std.fmt.hexToBytes(raw4[2..8], s[24..36])).len == 6);
	4220	return .{
	4221	.Data1 = @byteSwap(@as(u32, @bitCast(raw1))),
	4222	.Data2 = @byteSwap(@as(u16, @bitCast(raw2))),
	4223	.Data3 = @byteSwap(@as(u16, @bitCast(raw3))),
	4224	.Data4 = raw4,
	4225	};
4225	4226	}
4226	4227
4227	4228	pub fn format(self: GUID, w: *std.Io.Writer) std.Io.Writer.Error!void {
...	...	@@ -4233,28 +4234,28 @@ pub const GUID = extern struct {
4233	4234	self.Data4[2..8],
4234	4235	});
4235	4236	}
4236		};
4237	4237
4238		test GUID {
4239		try std.testing.expectEqual(
4240		GUID{
	4238	test parse {
	4239	const expected: GUID = .{
4241	4240	.Data1 = 0x01234567,
4242	4241	.Data2 = 0x89ab,
4243	4242	.Data3 = 0xef10,
4244	4243	.Data4 = "\x32\x54\x76\x98\xba\xdc\xfe\x91".*,
4245		},
4246		GUID.parse("{01234567-89AB-EF10-3254-7698badcfe91}"),
4247		);
4248		try std.testing.expectFmt(
4249		"{01234567-89ab-ef10-3254-7698badcfe91}",
4250		"{f}",
4251		.{GUID.parse("{01234567-89AB-EF10-3254-7698badcfe91}")},
4252		);
4253		try std.testing.expectFmt(
4254		"{00000001-0001-0001-0001-000000000001}",
4255		"{f}",
4256		.{GUID{ .Data1 = 1, .Data2 = 1, .Data3 = 1, .Data4 = [_]u8{ 0, 1, 0, 0, 0, 0, 0, 1 } }},
4257		);
	4244	};
	4245	try std.testing.expectEqual(expected, GUID.parse("{01234567-89AB-EF10-3254-7698badcfe91}"));
	4246	}
	4247
	4248	test format {
	4249	const guid0: GUID = .{ .Data1 = 1, .Data2 = 1, .Data3 = 1, .Data4 = .{ 0, 1, 0, 0, 0, 0, 0, 1 } };
	4250	try std.testing.expectFmt("{00000001-0001-0001-0001-000000000001}", "{f}", .{guid0});
	4251
	4252	const guid1: GUID = .parse("{01234567-89AB-EF10-3254-7698badcfe91}");
	4253	try std.testing.expectFmt("{01234567-89ab-ef10-3254-7698badcfe91}", "{f}", .{guid1});
	4254	}
	4255	};
	4256
	4257	test {
	4258	_ = GUID;
4258	4259	}
4259	4260
4260	4261	pub const COORD = extern struct {

lib/std/testing.zig+30-24

...	...	@@ -141,39 +141,45 @@ fn expectEqualInner(comptime T: type, expected: T, actual: T) !void {
141	141	try expectEqualSlices(info.child, &expect_array, &actual_array);
142	142	},
143	143
144		.@"struct" => \|structType\| {
145		inline for (structType.field_names) \|field_name\| {
	144	.@"struct" => \|@"struct"\| {
	145	inline for (@"struct".field_names) \|field_name\| {
146	146	try expectEqual(@field(expected, field_name), @field(actual, field_name));
147	147	}
148	148	},
149	149
150		.@"union" => \|union_info\| {
151		if (union_info.tag_type == null) {
152		const first_size = @bitSizeOf(union_info.field_types[0]);
153		inline for (union_info.field_types) \|field_type\| {
	150	.@"union" => \|@"union"\| if (@"union".backing_integer) \|Int\| {
	151	try expectEqual(@as(Int, @bitCast(expected)), @as(Int, @bitCast(actual)));
	152	} else switch (@"union".layout) {
	153	.@"packed" => {
	154	const Int = @Int(.unsigned, @bitSizeOf(T));
	155	try expectEqual(@as(Int, @bitCast(expected)), @as(Int, @bitCast(actual)));
	156	},
	157	.@"extern" => {
	158	const first_size = @bitSizeOf(@"union".field_types[0]);
	159	inline for (@"union".field_types) \|field_type\| {
154	160	if (@bitSizeOf(field_type) != first_size) {
155		@compileError("Unable to compare untagged unions with varying field sizes for type " ++ @typeName(@TypeOf(actual)));
	161	@compileError("Unable to compare extern unions with varying field sizes for type " ++ @typeName(T));
156	162	}
157	163	}
158
159		const BackingInt = @Int(.unsigned, @bitSizeOf(T));
	164	const FieldInt = @Int(.unsigned, first_size);
	165	const expected_field = @field(expected, @"union".field_names[0]);
	166	const actual_field = @field(actual, @"union".field_names[0]);
160	167	return expectEqual(
161		@as(BackingInt, @bitCast(expected)),
162		@as(BackingInt, @bitCast(actual)),
	168	@as(FieldInt, @bitCast(expected_field)),
	169	@as(FieldInt, @bitCast(actual_field)),
163	170	);
164		}
165
166		const Tag = std.meta.Tag(@TypeOf(expected));
167
168		const expectedTag = @as(Tag, expected);
169		const actualTag = @as(Tag, actual);
170
171		try expectEqual(expectedTag, actualTag);
172
173		// we only reach this switch if the tags are equal
174		switch (expected) {
175		inline else => \|val, tag\| try expectEqual(val, @field(actual, @tagName(tag))),
176		}
	171	},
	172	.auto => {
	173	const Tag = @"union".tag_type orelse @compileError("byteSwapAllFields expects packed, extern, or tagged union");
	174
	175	try expectEqual(@as(Tag, expected), @as(Tag, actual));
	176	switch (expected) {
	177	inline else => \|expected_payload, tag\| {
	178	const actual_payload = @field(actual, @tagName(tag));
	179	try expectEqual(expected_payload, actual_payload);
	180	},
	181	}
	182	},
177	183	},
178	184
179	185	.optional => {

lib/std/zig/llvm/Builder.zig+50-4

...	...	@@ -1816,7 +1816,7 @@ pub const Linkage = enum(u4) {
1816	1816	}
1817	1817	};
1818	1818
1819		pub const Preemption = enum {
	1819	pub const Preemption = enum(u2) {
1820	1820	dso_preemptable,
1821	1821	dso_local,
1822	1822	implicit_dso_local,
...	...	@@ -2011,7 +2011,7 @@ pub const AddrSpace = enum(u24) {
2011	2011	}
2012	2012	};
2013	2013
2014		pub const ExternallyInitialized = enum {
	2014	pub const ExternallyInitialized = enum(u1) {
2015	2015	default,
2016	2016	externally_initialized,
2017	2017
...	...	@@ -2061,6 +2061,13 @@ pub const Alignment = enum(u6) {
2061	2061	};
2062	2062	}
2063	2063
	2064	/// Asserts that neither `a` nor `b` is `.default`.
	2065	pub fn max(a: Alignment, b: Alignment) Alignment {
	2066	assert(a != .default);
	2067	assert(b != .default);
	2068	return @enumFromInt(@max(@intFromEnum(a), @intFromEnum(b)));
	2069	}
	2070
2064	2071	pub fn toLlvm(self: Alignment) u6 {
2065	2072	return switch (self) {
2066	2073	.default => 0,
...	...	@@ -4314,6 +4321,9 @@ pub const Function = struct {
4314	4321	@"tail call",
4315	4322	@"tail call fast",
4316	4323	trunc,
	4324	@"trunc nuw",
	4325	@"trunc nsw",
	4326	@"trunc nuw nsw",
4317	4327	udiv,
4318	4328	@"udiv exact",
4319	4329	urem,
...	...	@@ -4377,7 +4387,10 @@ pub const Function = struct {
4377	4387	};
4378	4388	}
4379	4389
4380		pub fn toCastOpcode(self: Tag) CastOpcode {
	4390	/// Does not accept `.@"trunc nuw"`, `.@"trunc nsw"`, or `.@"trunc nuw nsw"`, because
	4391	/// they do not have distinct `CastOpcode` values, and are instead encoded in bitcode
	4392	/// using flags on a normal `trunc` operation.
	4393	fn toCastOpcode(self: Tag) CastOpcode {
4381	4394	return switch (self) {
4382	4395	.trunc => .trunc,
4383	4396	.zext => .zext,
...	...	@@ -4572,6 +4585,9 @@ pub const Function = struct {
4572	4585	.sext,
4573	4586	.sitofp,
4574	4587	.trunc,
	4588	.@"trunc nuw",
	4589	.@"trunc nsw",
	4590	.@"trunc nuw nsw",
4575	4591	.uitofp,
4576	4592	.zext,
4577	4593	=> wip.extraData(Cast, instruction.data).type,
...	...	@@ -4758,6 +4774,9 @@ pub const Function = struct {
4758	4774	.sext,
4759	4775	.sitofp,
4760	4776	.trunc,
	4777	.@"trunc nuw",
	4778	.@"trunc nsw",
	4779	.@"trunc nuw nsw",
4761	4780	.uitofp,
4762	4781	.zext,
4763	4782	=> function.extraData(Cast, instruction.data).type,
...	...	@@ -5975,6 +5994,9 @@ pub const WipFunction = struct {
5975	5994	.sext,
5976	5995	.sitofp,
5977	5996	.trunc,
	5997	.@"trunc nuw",
	5998	.@"trunc nsw",
	5999	.@"trunc nuw nsw",
5978	6000	.uitofp,
5979	6001	.zext,
5980	6002	=> {},
...	...	@@ -6583,6 +6605,9 @@ pub const WipFunction = struct {
6583	6605	.sext,
6584	6606	.sitofp,
6585	6607	.trunc,
	6608	.@"trunc nuw",
	6609	.@"trunc nsw",
	6610	.@"trunc nuw nsw",
6586	6611	.uitofp,
6587	6612	.zext,
6588	6613	=> {
...	...	@@ -9975,6 +10000,9 @@ pub fn print(self: Builder, w: Writer) (Writer.Error \|\| Allocator.Error)!void
9975	10000	.sext,
9976	10001	.sitofp,
9977	10002	.trunc,
	10003	.@"trunc nuw",
	10004	.@"trunc nsw",
	10005	.@"trunc nuw nsw",
9978	10006	.uitofp,
9979	10007	.zext,
9980	10008	=> \|tag\| {
...	...	@@ -11649,7 +11677,11 @@ fn convTag(
11649	11677	.unneeded => unreachable,
11650	11678	},
11651	11679	.eq => unreachable,
11652		.gt => .trunc,
	11680	.gt => switch (signedness) {
	11681	.unsigned => .@"trunc nuw",
	11682	.signed => .@"trunc nsw",
	11683	.unneeded => .trunc,
	11684	},
11653	11685	},
11654	11686	.pointer => .inttoptr,
11655	11687	else => unreachable,
...	...	@@ -14962,6 +14994,20 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
14962	14994	.opcode = kind.toCastOpcode(),
14963	14995	});
14964	14996	},
	14997	.@"trunc nuw",
	14998	.@"trunc nsw",
	14999	.@"trunc nuw nsw",
	15000	=> \|kind\| {
	15001	const extra = func.extraData(Function.Instruction.Cast, data);
	15002	try function_block.writeAbbrev(FunctionBlock.TruncNoWrap{
	15003	.val = adapter.getOffsetValueIndex(extra.val),
	15004	.type_index = extra.type,
	15005	.flags = .{
	15006	.no_unsigned_wrap = kind == .@"trunc nuw" or kind == .@"trunc nuw nsw",
	15007	.no_signed_wrap = kind == .@"trunc nsw" or kind == .@"trunc nuw nsw",
	15008	},
	15009	});
	15010	},
14965	15011	.@"fcmp false",
14966	15012	.@"fcmp oeq",
14967	15013	.@"fcmp oge",

lib/std/zig/llvm/ir.zig+19

...	...	@@ -696,6 +696,7 @@ pub const ModuleBlock = struct {
696	696	ModuleBlock.FunctionBlock.Select,
697	697	ModuleBlock.FunctionBlock.SelectFast,
698	698	ModuleBlock.FunctionBlock.Cast,
	699	ModuleBlock.FunctionBlock.TruncNoWrap,
699	700	ModuleBlock.FunctionBlock.Alloca,
700	701	ModuleBlock.FunctionBlock.GetElementPtr,
701	702	ModuleBlock.FunctionBlock.ExtractValue,
...	...	@@ -1086,6 +1087,24 @@ pub const ModuleBlock = struct {
1086	1087	opcode: CastOpcode,
1087	1088	};
1088	1089
	1090	pub const TruncNoWrap = struct {
	1091	pub const Flags = packed struct(u2) {
	1092	no_unsigned_wrap: bool,
	1093	no_signed_wrap: bool,
	1094	};
	1095	pub const ops = [_]AbbrevOp{
	1096	.{ .literal = @intFromEnum(ModuleBlock.FunctionBlock.Code.INST_CAST) },
	1097	ValueAbbrev,
	1098	.{ .fixed_runtime = Builder.Type },
	1099	.{ .literal = @intFromEnum(Builder.CastOpcode.trunc) },
	1100	.{ .fixed = @bitSizeOf(Flags) },
	1101	};
	1102
	1103	val: u32,
	1104	type_index: Builder.Type,
	1105	flags: Flags,
	1106	};
	1107
1089	1108	pub const Alloca = struct {
1090	1109	pub const Flags = packed struct(u11) {
1091	1110	align_lower: u5,

src/Air.zig+68-13

...	...	@@ -17,6 +17,7 @@ const print = @import("Air/print.zig");
17	17
18	18	pub const Legalize = @import("Air/Legalize.zig");
19	19	pub const Liveness = @import("Air/Liveness.zig");
	20	pub const Verify = @import("Air/Verify.zig");
20	21
21	22	instructions: std.MultiArrayList(Inst).Slice,
22	23	/// The meaning of this data is determined by `Inst.Tag` value.
...	...	@@ -276,10 +277,50 @@ pub const Inst = struct {
276	277	/// Boolean or binary NOT.
277	278	/// Uses the `ty_op` field.
278	279	not,
279		/// Reinterpret the bits of a value as a different type. This is like `@bitCast` but
280		/// also supports enums and pointers.
	280	/// Implements `@bitCast`.
	281	///
	282	/// Uses the `ty_op` field.
	283	bit_cast,
	284	/// Cast a pointer to a different pointer type. The result type is a slice iff the operand
	285	/// type is a slice (the length of the slice does not change). All other pointer attributes
	286	/// except for the address space may change.
	287	///
	288	/// Supports vectors of pointers.
	289	///
	290	/// Uses the `ty_op` field.
	291	ptr_cast,
	292	/// Cast an integer to a pointer (not a slice). Operand type is always `usize`.
	293	///
	294	/// Supports vectors of integers.
	295	///
	296	/// Uses the `ty_op` field.
	297	ptr_from_int,
	298	/// Cast a pointer (not a slice) to an integer. Result type is always `usize`.
	299	///
	300	/// Supports vectors of pointers.
	301	///
	302	/// Uses the `ty_op` field.
	303	int_from_ptr,
	304	/// Cast an error set `E1` to a different error set `E2`, or cast an error union `E1!T` to
	305	/// an error union `E2!T` with the same payload type but a different error set type.
	306	///
	307	/// Uses the `ty_op` field.
	308	error_cast,
	309	/// Cast an integer to an error set type. The integer operand type is unsigned and has bit
	310	/// width equal to `zcu.errorSetBits()`.
	311	///
	312	/// Uses the `ty_op` field.
	313	error_from_int,
	314	/// Cast an error set to an integer type. The integer destination type is unsigned and has
	315	/// bit width equal to `zcu.errorSetBits()`.
	316	///
	317	/// Uses the `ty_op` field.
	318	int_from_error,
	319	/// Cast an enum value to a tagged union, whose tag type is that enum, and which has no
	320	/// payload bits (i.e. all payloads are equivalent to `void`).
	321	///
281	322	/// Uses the `ty_op` field.
282		bitcast,
	323	union_from_enum,
283	324	/// A block runs its body which always ends with a `noreturn` instruction,
284	325	/// so the only way to proceed to the code after the `block` is to encounter a `br`
285	326	/// that targets this `block`. If the `block` type is `noreturn`,
...	...	@@ -589,13 +630,13 @@ pub const Inst = struct {
589	630	/// the integer tag type of the enum.
590	631	/// See `trunc` for integer truncation.
591	632	/// Uses the `ty_op` field.
592		intcast,
593		/// Like `intcast`, but includes two safety checks:
	633	int_cast,
	634	/// Like `int_cast`, but includes two safety checks:
594	635	/// * triggers a safety panic if the cast truncates bits
595	636	/// * triggers a safety panic if the destination type is an exhaustive enum
596	637	/// and the operand is not a valid value of this type; i.e. equivalent to
597	638	/// a safety check based on `.is_named_enum_value`
598		intcast_safe,
	639	int_cast_safe,
599	640	/// Truncate higher bits from an integer, resulting in an integer type with the same
600	641	/// sign but an equal or smaller number of bits.
601	642	/// Uses the `ty_op` field.
...	...	@@ -955,7 +996,7 @@ pub const Inst = struct {
955	996	/// here is runtime-known, which is usually not allowed for vectors. `Legalize` may emit
956	997	/// this instruction when scalarizing vector operations.
957	998	///
958		/// Uses the `bin_op` field. `lhs` is the vector pointer. `rhs` is the element index. Result
	999	/// Uses the `bin_op` field. `lhs` is the vector value. `rhs` is the element index. Result
959	1000	/// type is the vector element type.
960	1001	legalize_vec_elem_val,
961	1002
...	...	@@ -1667,12 +1708,19 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)
1667	1708	=> return datas[@intFromEnum(inst)].ty_pl.ty.toType(),
1668	1709
1669	1710	.not,
1670		.bitcast,
	1711	.bit_cast,
	1712	.ptr_cast,
	1713	.ptr_from_int,
	1714	.int_from_ptr,
	1715	.error_cast,
	1716	.error_from_int,
	1717	.int_from_error,
	1718	.union_from_enum,
1671	1719	.load,
1672	1720	.fpext,
1673	1721	.fptrunc,
1674		.intcast,
1675		.intcast_safe,
	1722	.int_cast,
	1723	.int_cast_safe,
1676	1724	.trunc,
1677	1725	.optional_payload,
1678	1726	.optional_payload_ptr,
...	...	@@ -1913,7 +1961,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1913	1961	.add_safe,
1914	1962	.sub_safe,
1915	1963	.mul_safe,
1916		.intcast_safe,
	1964	.int_cast_safe,
1917	1965	.int_from_float_safe,
1918	1966	.int_from_float_optimized_safe,
1919	1967	.legalize_vec_store_elem,
...	...	@@ -1965,7 +2013,14 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1965	2013	.shl_sat,
1966	2014	.xor,
1967	2015	.not,
1968		.bitcast,
	2016	.bit_cast,
	2017	.ptr_cast,
	2018	.ptr_from_int,
	2019	.int_from_ptr,
	2020	.error_cast,
	2021	.error_from_int,
	2022	.int_from_error,
	2023	.union_from_enum,
1969	2024	.ret_addr,
1970	2025	.frame_addr,
1971	2026	.clz,
...	...	@@ -2009,7 +2064,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
2009	2064	.is_non_err,
2010	2065	.fptrunc,
2011	2066	.fpext,
2012		.intcast,
	2067	.int_cast,
2013	2068	.trunc,
2014	2069	.optional_payload,
2015	2070	.optional_payload_ptr,

src/Air/Legalize.zig+238-95

...	...	@@ -75,13 +75,9 @@ pub const Feature = enum {
75	75	scalarize_shl_sat,
76	76	scalarize_xor,
77	77	scalarize_not,
78		/// Scalarize `bitcast` from or to an array or vector type to `bitcast`s of the elements.
79		/// This does not apply if `@bitSizeOf(Elem) == 8 * @sizeOf(Elem)`.
80		/// When this feature is enabled, all remaining `bitcast`s can be lowered using the old bitcast
81		/// semantics (reinterpret memory) instead of the new bitcast semantics (copy logical bits) and
82		/// the behavior will be equivalent. However, the behavior of `@bitSize` on arrays must be
83		/// changed in `Type.zig` before enabling this feature to conform to the new bitcast semantics.
84		scalarize_bitcast,
	78	scalarize_ptr_cast,
	79	scalarize_ptr_from_int,
	80	scalarize_int_from_ptr,
85	81	scalarize_clz,
86	82	scalarize_ctz,
87	83	scalarize_popcount,
...	...	@@ -107,8 +103,8 @@ pub const Feature = enum {
107	103	scalarize_cmp_vector_optimized,
108	104	scalarize_fptrunc,
109	105	scalarize_fpext,
110		scalarize_intcast,
111		scalarize_intcast_safe,
	106	scalarize_int_cast,
	107	scalarize_int_cast_safe,
112	108	scalarize_trunc,
113	109	scalarize_int_from_float,
114	110	scalarize_int_from_float_optimized,
...	...	@@ -122,16 +118,45 @@ pub const Feature = enum {
122	118	scalarize_select,
123	119	scalarize_mul_add,
124	120
	121	// Below are several different features for scalarizing `bit_cast` in different scenarios. It is
	122	// valid to enable any combination of these features.
	123
	124	/// Scalarize `bit_cast` where the operand or result type is an array.
	125	scalarize_bit_cast_array,
	126	/// Scalarize `bit_cast` where either:
	127	///
	128	/// * operand type is `@Vector(n, A), but result type is not `@Vector(n, B)`; or
	129	/// * result type is `@Vector(n, A), but operand type is not `@Vector(n, B)`
	130	///
	131	/// This effectively scalarizes any `bit_cast` to/from a vector, unless the operation can be
	132	/// performed by bitcasting each vector element and returning a vector of the results.
	133	///
	134	/// If this feature is enabled, the following AIR instruction tags may be emitted:
	135	/// * `.legalize_vec_elem_val`
	136	/// * `.legalize_vec_store_elem`
	137	scalarize_bit_cast_vector_non_elementwise,
	138	/// Scalarize `bit_cast` where the operand or result type is an array or vector whose element
	139	/// type `E` has `@bitSizeOf(E) != 8 * @sizeOf(E)`. These are the cases where the backend may
	140	/// need to sign- or zero-extend multiple elements to populate "padding" bits.
	141	///
	142	/// Enabling this feature requires changing the behavior of `@bitSize` on arrays in `Type.zig`
	143	/// to conform to the new bitcast semantics.
	144	///
	145	/// If this feature is enabled, the following AIR instruction tags may be emitted:
	146	/// * `.legalize_vec_elem_val`
	147	/// * `.legalize_vec_store_elem`
	148	scalarize_bit_cast_padded_elems,
	149
125	150	/// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)
126	151	unsplat_shift_rhs,
127	152	/// Legalize reduce of a one element vector to a bitcast.
128		reduce_one_elem_to_bitcast,
	153	reduce_one_elem_to_bit_cast,
129	154	/// Legalize splat to a one element vector to a bitcast.
130		splat_one_elem_to_bitcast,
	155	splat_one_elem_to_bit_cast,
131	156
132		/// Replace `intcast_safe` with an explicit safety check which `call`s the panic function on failure.
133		/// Not compatible with `scalarize_intcast_safe`.
134		expand_intcast_safe,
	157	/// Replace `int_cast_safe` with an explicit safety check which `call`s the panic function on failure.
	158	/// Not compatible with `scalarize_int_cast_safe`.
	159	expand_int_cast_safe,
135	160	/// Replace `int_from_float_safe` with an explicit safety check which `call`s the panic function on failure.
136	161	/// Not compatible with `scalarize_int_from_float_safe`.
137	162	expand_int_from_float_safe,
...	...	@@ -156,9 +181,9 @@ pub const Feature = enum {
156	181	/// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
157	182	/// first byte of memory until bit pointers know their backing type.
158	183	expand_packed_store,
159		/// Replace `struct_field_val` of a packed field with a `bitcast` to integer, `shr`, `trunc`, and `bitcast` to field type.
	184	/// Replace `struct_field_val` of a packed field with a `bit_cast` to integer, `shr`, `trunc`, and `bit_cast` to field type.
160	185	expand_packed_struct_field_val,
161		/// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.
	186	/// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bit_cast`, `int_cast`, and `bit_or`.
162	187	expand_packed_aggregate_init,
163	188
164	189	/// Replace all arithmetic operations on 16-bit floating-point types with calls to soft-float
...	...	@@ -227,7 +252,6 @@ pub const Feature = enum {
227	252	.shl_sat => .scalarize_shl_sat,
228	253	.xor => .scalarize_xor,
229	254	.not => .scalarize_not,
230		.bitcast => .scalarize_bitcast,
231	255	.clz => .scalarize_clz,
232	256	.ctz => .scalarize_ctz,
233	257	.popcount => .scalarize_popcount,
...	...	@@ -253,8 +277,11 @@ pub const Feature = enum {
253	277	.cmp_vector_optimized => .scalarize_cmp_vector_optimized,
254	278	.fptrunc => .scalarize_fptrunc,
255	279	.fpext => .scalarize_fpext,
256		.intcast => .scalarize_intcast,
257		.intcast_safe => .scalarize_intcast_safe,
	280	.int_cast => .scalarize_int_cast,
	281	.int_cast_safe => .scalarize_int_cast_safe,
	282	.ptr_cast => .scalarize_ptr_cast,
	283	.ptr_from_int => .scalarize_ptr_from_int,
	284	.int_from_ptr => .scalarize_int_from_ptr,
258	285	.trunc => .scalarize_trunc,
259	286	.int_from_float => .scalarize_int_from_float,
260	287	.int_from_float_optimized => .scalarize_int_from_float_optimized,
...	...	@@ -474,7 +501,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
474	501	.popcount,
475	502	.byte_swap,
476	503	.bit_reverse,
477		.intcast,
	504	.int_cast,
	505	.ptr_cast,
	506	.ptr_from_int,
	507	.int_from_ptr,
478	508	.trunc,
479	509	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
480	510	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	...	@@ -548,15 +578,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
548	578	},
549	579	}
550	580	},
551		.bitcast => if (l.features.has(.scalarize_bitcast)) {
	581	.bit_cast => if (l.features.hasAny(&.{
	582	.scalarize_bit_cast_array,
	583	.scalarize_bit_cast_vector_non_elementwise,
	584	.scalarize_bit_cast_padded_elems,
	585	})) {
552	586	if (try l.scalarizeBitcastBlockPayload(inst)) \|payload\| {
553	587	continue :inst l.replaceInst(inst, .block, payload);
554	588	}
555	589	},
556		.intcast_safe => if (l.features.has(.expand_intcast_safe)) {
557		assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both
	590	.int_cast_safe => if (l.features.has(.expand_int_cast_safe)) {
	591	assert(!l.features.has(.scalarize_int_cast_safe)); // it doesn't make sense to do both
558	592	continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
559		} else if (l.features.has(.scalarize_intcast_safe)) {
	593	} else if (l.features.has(.scalarize_int_cast_safe)) {
560	594	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
561	595	if (ty_op.ty.toType().isVector(zcu)) {
562	596	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
...	...	@@ -772,10 +806,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
772	806	inline .reduce, .reduce_optimized => \|air_tag\| {
773	807	const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;
774	808	const vector_ty = l.typeOf(reduce.operand);
775		if (l.features.has(.reduce_one_elem_to_bitcast)) {
	809	if (l.features.has(.reduce_one_elem_to_bit_cast)) {
776	810	switch (vector_ty.vectorLen(zcu)) {
777	811	0 => unreachable,
778		1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
	812	1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
779	813	.ty = .fromType(vector_ty.childType(zcu)),
780	814	.operand = reduce.operand,
781	815	} }),
...	...	@@ -792,11 +826,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
792	826	.soft_float => unreachable, // the operand is not a scalar
793	827	}
794	828	},
795		.splat => if (l.features.has(.splat_one_elem_to_bitcast)) {
	829	.splat => if (l.features.has(.splat_one_elem_to_bit_cast)) {
796	830	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
797	831	switch (ty_op.ty.toType().vectorLen(zcu)) {
798	832	0 => unreachable,
799		1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
	833	1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
800	834	.ty = ty_op.ty,
801	835	.operand = ty_op.operand,
802	836	} }),
...	...	@@ -862,7 +896,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
862	896	const field_bits = agg_ty.fieldType(field_index, zcu).bitSize(zcu);
863	897	if (field_bits == struct_bits) {
864	898	// Just bitcast this field.
865		continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
	899	continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
866	900	.ty = .fromType(agg_ty),
867	901	.operand = @enumFromInt(l.air_extra.items[ty_pl.payload + field_index]),
868	902	} });
...	...	@@ -909,6 +943,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
909	943	.legalize_vec_store_elem,
910	944	.legalize_compiler_rt_call,
911	945	.spirv_runtime_array_len,
	946	.error_cast,
	947	.error_from_int,
	948	.int_from_error,
	949	.union_from_enum,
912	950	=> {},
913	951	}
914	952	}
...	...	@@ -931,7 +969,7 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: Scalariz
931	969
932	970	if (result_is_array) {
933	971	// This is only allowed when legalizing an elementwise bitcast.
934		assert(orig.tag == .bitcast);
	972	assert(orig.tag == .bit_cast);
935	973	assert(form == .ty_op);
936	974	}
937	975
...	...	@@ -1423,35 +1461,94 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1423	1461	const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1424	1462
1425	1463	const dest_ty = ty_op.ty.toType();
1426		const dest_legal = switch (dest_ty.zigTypeTag(zcu)) {
1427		else => true,
1428		.array, .vector => legal: {
1429		if (dest_ty.arrayLen(zcu) == 1) break :legal true;
1430		const dest_elem_ty = dest_ty.childType(zcu);
1431		break :legal dest_elem_ty.bitSize(zcu) == 8 * dest_elem_ty.abiSize(zcu);
1432		},
1433		};
1434
1435	1464	const operand_ty = l.typeOf(ty_op.operand);
1436		const operand_legal = switch (operand_ty.zigTypeTag(zcu)) {
1437		else => true,
1438		.array, .vector => legal: {
1439		if (operand_ty.arrayLen(zcu) == 1) break :legal true;
1440		const operand_elem_ty = operand_ty.childType(zcu);
1441		break :legal operand_elem_ty.bitSize(zcu) == 8 * operand_elem_ty.abiSize(zcu);
1442		},
1443		};
1444	1465
1445		if (dest_legal and operand_legal) return null;
	1466	// We exit this block only if the scalarization is actually necessary. Otherwise we will return
	1467	// `null` from within the block.
	1468	const operand_to_int_ok: bool, const int_to_dest_ok: bool = int_ok: {
	1469	const operand_tag = operand_ty.zigTypeTag(zcu);
	1470	const dest_tag = dest_ty.zigTypeTag(zcu);
	1471
	1472	if (operand_tag != .array and
	1473	operand_tag != .vector and
	1474	dest_tag != .array and
	1475	dest_tag != .vector)
	1476	{
	1477	return null;
	1478	}
	1479
	1480	// We track the validity of 3 different bitcast operations:
	1481	// * operand -> dest
	1482	// * operand -> uint
	1483	// * uint -> dest
	1484	// If operand->dest turns out to be valid, we don't need to scalarize. Otherwise, knowing
	1485	// the validity of the other operations helps us lower the scalarization efficiently.
	1486	var operand_to_dest: bool = true;
	1487	var operand_to_int: bool = true;
	1488	var int_to_dest: bool = true;
	1489
	1490	if (l.features.has(.scalarize_bit_cast_array)) {
	1491	if (operand_tag == .array) {
	1492	operand_to_dest = false;
	1493	operand_to_int = false;
	1494	}
	1495	if (dest_tag == .array) {
	1496	operand_to_dest = false;
	1497	int_to_dest = false;
	1498	}
	1499	}
1446	1500
1447		if (!operand_legal and !dest_legal and operand_ty.arrayLen(zcu) == dest_ty.arrayLen(zcu)) {
1448		// from_ty and to_ty are both arrays or vectors of types with the same bit size,
1449		// so we can do an elementwise bitcast.
1450		return try l.scalarizeBlockPayload(orig_inst, .ty_op);
1451		}
	1501	if (l.features.has(.scalarize_bit_cast_vector_non_elementwise)) {
	1502	if (operand_tag == .vector) operand_to_int = false;
	1503	if (dest_tag == .vector) int_to_dest = false;
1452	1504
1453		// Fallback path. Our strategy is to use an unsigned integer type as an intermediate
1454		// "bag of bits" representation which can be manipulated by bitwise operations.
	1505	if (operand_tag == .vector or dest_tag == .vector) {
	1506	if (operand_tag != .vector or
	1507	dest_tag != .vector or
	1508	operand_ty.vectorLen(zcu) != dest_ty.vectorLen(zcu))
	1509	{
	1510	operand_to_dest = false;
	1511	}
	1512	}
	1513	}
	1514
	1515	if (l.features.has(.scalarize_bit_cast_padded_elems)) {
	1516	if (operand_tag == .array or operand_tag == .vector) {
	1517	const elem_ty = operand_ty.childType(zcu);
	1518	if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
	1519	operand_to_int = false;
	1520	operand_to_dest = false;
	1521	}
	1522	}
	1523	if (dest_tag == .array or dest_tag == .vector) {
	1524	const elem_ty = dest_ty.childType(zcu);
	1525	if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
	1526	int_to_dest = false;
	1527	operand_to_dest = false;
	1528	}
	1529	}
	1530	}
	1531
	1532	if (operand_to_dest) {
	1533	return null; // no scalarization needed!
	1534	}
	1535
	1536	// We need a scalarization, but before breaking from the block, check if we can do it
	1537	// elementwise---if we can, that's preferable to the generic lowering.
	1538	if ((operand_tag == .array or operand_tag == .vector) and
	1539	(dest_tag == .array or dest_tag == .vector) and
	1540	operand_ty.arrayLenIncludingSentinel(zcu) == dest_ty.arrayLenIncludingSentinel(zcu))
	1541	{
	1542	// Operand and result types are both arrays/vectors whose element types have the same
	1543	// bit size, so we can do an elementwise bitcast.
	1544	return try l.scalarizeBlockPayload(orig_inst, .ty_op);
	1545	}
	1546
	1547	break :int_ok .{ operand_to_int, int_to_dest };
	1548	};
	1549
	1550	// Generic scalarization implementation. Our strategy is to use an unsigned integer type as an
	1551	// intermediate "bag of bits" representation which can be manipulated by bitwise operations.
1455	1552
1456	1553	const num_bits: u16 = @intCast(dest_ty.bitSize(zcu));
1457	1554	assert(operand_ty.bitSize(zcu) == num_bits);
...	...	@@ -1465,11 +1562,17 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1465	1562	// First, convert `operand_ty` to `uint_ty` (`uN`).
1466	1563
1467	1564	const uint_val: Air.Inst.Ref = uint_val: {
1468		if (operand_legal) {
	1565	if (operand_to_int_ok) {
1469	1566	_ = main_block.stealCapacity(19);
1470	1567	break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
1471	1568	}
1472	1569
	1570	if (operand_ty.arrayLenIncludingSentinel(zcu) == 1) {
	1571	_ = main_block.stealCapacity(18);
	1572	const elem = main_block.addBinOp(l, .array_elem_val, ty_op.operand, .zero_usize).toRef();
	1573	break :uint_val main_block.addBitCast(l, uint_ty, elem);
	1574	}
	1575
1473	1576	// %1 = block({
1474	1577	// %2 = alloc(*usize)
1475	1578	// %3 = alloc(*uN)
...	...	@@ -1478,8 +1581,8 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1478	1581	// %6 = loop({
1479	1582	// %7 = load(%2)
1480	1583	// %8 = array_elem_val(orig_operand, %7)
1481		// %9 = bitcast(uE, %8)
1482		// %10 = intcast(uN, %9)
	1584	// %9 = bit_cast(uE, %8)
	1585	// %10 = int_cast(uN, %9)
1483	1586	// %11 = load(%3)
1484	1587	// %12 = shl_exact(%11, <uS, E>)
1485	1588	// %13 = bit_or(%12, %10)
...	...	@@ -1529,7 +1632,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1529	1632	index_val,
1530	1633	).toRef();
1531	1634	const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);
1532		const elem_extended = loop.block.addTyOp(l, .intcast, uint_ty, elem_uint).toRef();
	1635	const elem_extended = loop.block.addTyOp(l, .int_cast, uint_ty, elem_uint).toRef();
1533	1636	const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();
1534	1637	const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();
1535	1638	const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();
...	...	@@ -1560,10 +1663,23 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1560	1663
1561	1664	// Now convert `uint_ty` (`uN`) to `dest_ty`.
1562	1665
1563		if (dest_legal) {
	1666	if (int_to_dest_ok) {
1564	1667	_ = main_block.stealCapacity(17);
1565	1668	const result = main_block.addBitCast(l, dest_ty, uint_val);
1566	1669	main_block.addBr(l, orig_inst, result);
	1670	} else if (dest_ty.arrayLenIncludingSentinel(zcu) == 1) {
	1671	_ = main_block.stealCapacity(16);
	1672	const elem = main_block.addBitCast(l, dest_ty.childType(zcu), uint_val);
	1673	const aggregate_init_payload_start = l.air_extra.items.len;
	1674	try l.air_extra.append(zcu.gpa, @intFromEnum(elem));
	1675	const result = main_block.add(l, .{
	1676	.tag = .aggregate_init,
	1677	.data = .{ .ty_pl = .{
	1678	.ty = .fromType(dest_ty),
	1679	.payload = @intCast(aggregate_init_payload_start),
	1680	} },
	1681	}).toRef();
	1682	main_block.addBr(l, orig_inst, result);
1567	1683	} else {
1568	1684	// %1 = alloc(*usize)
1569	1685	// %2 = alloc(*@Vector(N, Result))
...	...	@@ -1571,10 +1687,10 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1571	1687	// %4 = loop({
1572	1688	// %5 = load(%1)
1573	1689	// %6 = mul(%5, <usize, E>)
1574		// %7 = intcast(uS, %6)
	1690	// %7 = int_cast(uS, %6)
1575	1691	// %8 = shr(uint_val, %7)
1576	1692	// %9 = trunc(uE, %8)
1577		// %10 = bitcast(Result, %9)
	1693	// %10 = bit_cast(Result, %9)
1578	1694	// %11 = legalize_vec_store_elem(%2, %5, %10)
1579	1695	// %12 = cmp_eq(%5, <usize, vec_len>)
1580	1696	// %13 = cond_br(%12, {
...	...	@@ -1603,7 +1719,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1603	1719
1604	1720	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1605	1721	const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();
1606		const casted_bit_offset = loop.block.addTyOp(l, .intcast, shift_ty, bit_offset).toRef();
	1722	const casted_bit_offset = loop.block.addTyOp(l, .int_cast, shift_ty, bit_offset).toRef();
1607	1723	const shifted_uint = loop.block.addBinOp(l, .shr, uint_val, casted_bit_offset).toRef();
1608	1724	const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();
1609	1725	const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);
...	...	@@ -1981,7 +2097,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
1981	2097	// %5 = call(@panic.invalidEnumValue, [])
1982	2098	// %6 = unreach()
1983	2099	// }, {
1984		// %7 = intcast(@res_ty, %y)
	2100	// %7 = int_cast(@res_ty, %y)
1985	2101	// %8 = is_named_enum_value(%7)
1986	2102	// %9 = cond_br(%8, {
1987	2103	// %10 = br(%x, %7)
...	...	@@ -2003,7 +2119,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2003	2119	// %6 = call(@panic.invalidEnumValue, [])
2004	2120	// %7 = unreach()
2005	2121	// }, {
2006		// %8 = intcast(@res_ty, %y)
	2122	// %8 = int_cast(@res_ty, %y)
2007	2123	// %9 = br(%x, %8)
2008	2124	// })
2009	2125	// })
...	...	@@ -2056,9 +2172,9 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2056	2172	cur_block = &condbr.else_block;
2057	2173	}
2058	2174
2059		// Now we know we're in-range, we can intcast:
	2175	// Now we know we're in-range, we can int_cast:
2060	2176	const cast_inst = cur_block.add(l, .{
2061		.tag = .intcast,
	2177	.tag = .int_cast,
2062	2178	.data = .{ .ty_op = .{
2063	2179	.ty = Air.internedToRef(dest_ty.toIntern()),
2064	2180	.operand = operand_ref,
...	...	@@ -2229,7 +2345,7 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
2229	2345	// %1 = add_with_overflow(%x, %y)
2230	2346	// %2 = struct_field_val(%1, .@"1")
2231	2347	// %3 = reduce(%2, .@"or")
2232		// %4 = bitcast(%3, @bool_type)
	2348	// %4 = bit_cast(%3, @bool_type)
2233	2349	// %5 = cond_br(%4, {
2234	2350	// %6 = call(@panic.integerOverflow, [])
2235	2351	// %7 = unreach()
...	...	@@ -2335,7 +2451,7 @@ fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Ins
2335	2451	.tag = .load,
2336	2452	.data = .{ .ty_op = .{
2337	2453	.ty = Air.internedToRef(load_ty.toIntern()),
2338		.operand = res_block.addBitCast(l, load_ptr_ty: {
	2454	.operand = res_block.addPtrCast(l, load_ptr_ty: {
2339	2455	var load_ptr_info = ptr_info;
2340	2456	load_ptr_info.child = load_ty.toIntern();
2341	2457	load_ptr_info.flags.vector_index = .none;
...	...	@@ -2378,23 +2494,17 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2378	2494
2379	2495	var res_block: Block = .init(&inst_buf);
2380	2496	{
2381		const backing_ptr_inst = res_block.add(l, .{
2382		.tag = .bitcast,
2383		.data = .{ .ty_op = .{
2384		.ty = Air.internedToRef((load_store_ptr_ty: {
2385		var load_ptr_info = ptr_info;
2386		load_ptr_info.child = load_store_ty.toIntern();
2387		load_ptr_info.flags.vector_index = .none;
2388		load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
2389		break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
2390		}).toIntern()),
2391		.operand = orig_bin_op.lhs,
2392		} },
2393		});
	2497	const backing_ptr = res_block.addPtrCast(l, load_store_ptr_ty: {
	2498	var load_ptr_info = ptr_info;
	2499	load_ptr_info.child = load_store_ty.toIntern();
	2500	load_ptr_info.flags.vector_index = .none;
	2501	load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
	2502	break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
	2503	}, orig_bin_op.lhs);
2394	2504	_ = res_block.add(l, .{
2395	2505	.tag = .store,
2396	2506	.data = .{ .bin_op = .{
2397		.lhs = backing_ptr_inst.toRef(),
	2507	.lhs = backing_ptr,
2398	2508	.rhs = res_block.add(l, .{
2399	2509	.tag = .bit_or,
2400	2510	.data = .{ .bin_op = .{
...	...	@@ -2405,7 +2515,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2405	2515	.tag = .load,
2406	2516	.data = .{ .ty_op = .{
2407	2517	.ty = Air.internedToRef(load_store_ty.toIntern()),
2408		.operand = backing_ptr_inst.toRef(),
	2518	.operand = backing_ptr,
2409	2519	} },
2410	2520	}).toRef(),
2411	2521	.rhs = Air.internedToRef((keep_mask: {
...	...	@@ -2434,7 +2544,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2434	2544	.tag = .shl_exact,
2435	2545	.data = .{ .bin_op = .{
2436	2546	.lhs = res_block.add(l, .{
2437		.tag = .intcast,
	2547	.tag = .int_cast,
2438	2548	.data = .{ .ty_op = .{
2439	2549	.ty = Air.internedToRef(load_store_ty.toIntern()),
2440	2550	.operand = res_block.addBitCast(l, operand_int_ty, orig_bin_op.rhs),
...	...	@@ -2532,7 +2642,7 @@ fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Erro
2532	2642
2533	2643	const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();
2534	2644	const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);
2535		const field_extended = main_block.addTyOp(l, .intcast, uint_ty, field_as_uint).toRef();
	2645	const field_extended = main_block.addTyOp(l, .int_cast, uint_ty, field_as_uint).toRef();
2536	2646	cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();
2537	2647	}
2538	2648
...	...	@@ -2721,18 +2831,51 @@ const Block = struct {
2721	2831	});
2722	2832	}
2723	2833
2724		/// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for
	2834	/// Adds a `bit_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
2725	2835	/// no-op casts.
2726	2836	fn addBitCast(
2727	2837	b: *Block,
2728	2838	l: *Legalize,
2729		ty: Type,
	2839	result_ty: Type,
	2840	operand: Air.Inst.Ref,
	2841	) Air.Inst.Ref {
	2842	const zcu = l.pt.zcu;
	2843	const operand_ty = l.typeOf(operand);
	2844	assert(!operand_ty.isPtrAtRuntime(zcu));
	2845	assert(!operand_ty.isSliceAtRuntime(zcu));
	2846	assert(!result_ty.isPtrAtRuntime(zcu));
	2847	assert(!result_ty.isSliceAtRuntime(zcu));
	2848	if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
	2849	.tag = .bit_cast,
	2850	.data = .{ .ty_op = .{
	2851	.ty = .fromType(result_ty),
	2852	.operand = operand,
	2853	} },
	2854	}).toRef();
	2855	_ = b.stealCapacity(1);
	2856	return operand;
	2857	}
	2858
	2859	/// Adds a `ptr_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
	2860	/// no-op casts.
	2861	fn addPtrCast(
	2862	b: *Block,
	2863	l: *Legalize,
	2864	result_ty: Type,
2730	2865	operand: Air.Inst.Ref,
2731	2866	) Air.Inst.Ref {
2732		if (ty.toIntern() != l.typeOf(operand).toIntern()) return b.add(l, .{
2733		.tag = .bitcast,
	2867	const zcu = l.pt.zcu;
	2868	const operand_ty = l.typeOf(operand);
	2869	if (operand_ty.isSliceAtRuntime(zcu)) {
	2870	assert(result_ty.isSliceAtRuntime(zcu));
	2871	} else {
	2872	assert(operand_ty.isPtrAtRuntime(zcu));
	2873	assert(result_ty.isPtrAtRuntime(zcu));
	2874	}
	2875	if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
	2876	.tag = .ptr_cast,
2734	2877	.data = .{ .ty_op = .{
2735		.ty = Air.internedToRef(ty.toIntern()),
	2878	.ty = .fromType(result_ty),
2736	2879	.operand = operand,
2737	2880	} },
2738	2881	}).toRef();
...	...	@@ -3073,7 +3216,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3073	3216	var main_block: Block = .init(&inst_buf);
3074	3217	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3075	3218
3076		const extended_val = main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();
	3219	const extended_val = main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
3077	3220	const call_inst = try main_block.addCompilerRtCall(l, func, &.{extended_val});
3078	3221	const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());
3079	3222	main_block.addBr(l, orig_inst, casted_result);
...	...	@@ -3100,7 +3243,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3100	3243	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3101	3244
3102	3245	const extended_val: Air.Inst.Ref = if (extended_ty.toIntern() != src_ty.toIntern()) ext: {
3103		break :ext main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();
	3246	break :ext main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
3104	3247	} else ext: {
3105	3248	_ = main_block.stealCapacity(1);
3106	3249	break :ext ty_op.operand;
...	...	@@ -3165,7 +3308,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3165	3308	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3166	3309
3167	3310	const call_inst = try main_block.addCompilerRtCall(l, func, &.{ty_op.operand});
3168		const casted_val = main_block.addTyOp(l, .intcast, dest_ty, call_inst.toRef()).toRef();
	3311	const casted_val = main_block.addTyOp(l, .int_cast, dest_ty, call_inst.toRef()).toRef();
3169	3312	main_block.addBr(l, orig_inst, casted_val);
3170	3313
3171	3314	return .{ .block_payload = .{ .ty_pl = .{
...	...	@@ -3189,7 +3332,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3189	3332	const bits_val = try pt.intValue(.usize, dest_info.bits);
3190	3333	_ = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val), ty_op.operand });
3191	3334	const extended_val = main_block.addTyOp(l, .load, extended_ty, extended_ptr).toRef();
3192		const result_val = main_block.addTyOp(l, .intcast, dest_ty, extended_val).toRef();
	3335	const result_val = main_block.addTyOp(l, .int_cast, dest_ty, extended_val).toRef();
3193	3336	main_block.addBr(l, orig_inst, result_val);
3194	3337
3195	3338	return .{ .block_payload = .{ .ty_pl = .{

src/Air/Liveness.zig+10-3

...	...	@@ -488,12 +488,19 @@ fn analyzeInst(
488	488	=> return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),
489	489
490	490	.not,
491		.bitcast,
	491	.bit_cast,
	492	.ptr_cast,
	493	.ptr_from_int,
	494	.int_from_ptr,
	495	.error_cast,
	496	.error_from_int,
	497	.int_from_error,
	498	.union_from_enum,
492	499	.load,
493	500	.fpext,
494	501	.fptrunc,
495		.intcast,
496		.intcast_safe,
	502	.int_cast,
	503	.int_cast_safe,
497	504	.trunc,
498	505	.optional_payload,
499	506	.optional_payload_ptr,

src/Air/Liveness/Verify.zig+10-3

...	...	@@ -78,12 +78,19 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
78	78
79	79	// unary
80	80	.not,
81		.bitcast,
	81	.bit_cast,
	82	.ptr_cast,
	83	.ptr_from_int,
	84	.int_from_ptr,
	85	.error_cast,
	86	.error_from_int,
	87	.int_from_error,
	88	.union_from_enum,
82	89	.load,
83	90	.fpext,
84	91	.fptrunc,
85		.intcast,
86		.intcast_safe,
	92	.int_cast,
	93	.int_cast_safe,
87	94	.trunc,
88	95	.optional_payload,
89	96	.optional_payload_ptr,

src/Air/Verify.zig created+465

...	...	@@ -0,0 +1,465 @@
	1	/// Verifies that AIR is valid, in that every instruction has valid operands and types. In compiler
	2	/// builds with debug extensions, this is run on all AIR, both before `Air.Legalize` is run and (if
	3	/// it is run) after it.
	4	///
	5	/// This verification pass is currently highly incomplete---expand it as needed.
	6	const Verify = @This();
	7
	8	zcu: *Zcu,
	9	func_index: InternPool.Index,
	10	ret_ty: Type,
	11	air: *const Air,
	12	cur_inst: Air.Inst.Index,
	13
	14	pub fn run(pt: Zcu.PerThread, func_index: InternPool.Index, air: *const Air) void {
	15	if (!@import("build_options").enable_debug_extensions) {
	16	// `Air.Verify` is a debugging feature---it should not be used in release builds because it
	17	// has little benefit and negatively affects compiler performance.
	18	return;
	19	}
	20
	21	const zcu = pt.zcu;
	22
	23	const func_ty: Type = Value.fromInterned(func_index).typeOf(zcu);
	24	const ret_ty = func_ty.fnReturnType(zcu);
	25
	26	var verify: Verify = .{
	27	.zcu = zcu,
	28	.func_index = func_index,
	29	.ret_ty = ret_ty,
	30	.air = air,
	31	.cur_inst = undefined, // populated by `body(...)`
	32	};
	33	verify.body(air.getMainBody()) catch \|verify_err\| switch (verify_err) {
	34	error.VerifyFail => {
	35	const ip = &zcu.intern_pool;
	36	const func_nav = ip.indexToKey(func_index).func.owner_nav;
	37	const func_fqn = ip.getNav(func_nav).fqn.toSlice(ip);
	38	log.info("AIR for '{s}':", .{func_fqn});
	39	const io = zcu.comp.io;
	40	const stderr = io.lockStderr(&.{}, null) catch \|err\| switch (err) {
	41	error.Canceled => return io.recancel(),
	42	};
	43	defer io.unlockStderr();
	44	air.write(&stderr.file_writer.interface, pt, null) catch \|err\| switch (err) {
	45	error.WriteFailed => switch (stderr.file_writer.err.?) {
	46	error.Canceled => return io.recancel(),
	47	else => {},
	48	},
	49	};
	50	},
	51	};
	52	}
	53
	54	const Error = error{VerifyFail};
	55
	56	fn fail(verify: *Verify, msg: []const u8) Error {
	57	const ip = &verify.zcu.intern_pool;
	58	const func_nav = ip.indexToKey(verify.func_index).func.owner_nav;
	59	const func_fqn = ip.getNav(func_nav).fqn.toSlice(ip);
	60	log.err("'{s}', %{d}: {s}", .{ func_fqn, verify.cur_inst, msg });
	61	return error.VerifyFail;
	62	}
	63
	64	fn body(verify: *Verify, body_insts: []const Air.Inst.Index) Error!void {
	65	const zcu = verify.zcu;
	66	const ip = &zcu.intern_pool;
	67	const air = verify.air;
	68	const tags = air.instructions.items(.tag);
	69	const data = air.instructions.items(.data);
	70	for (body_insts, 0..) \|inst, body_index\| {
	71	verify.cur_inst = inst;
	72	switch (tags[@intFromEnum(inst)]) {
	73	.block => {
	74	const block = air.unwrapBlock(inst);
	75	try verify.body(block.body);
	76	},
	77	.dbg_inline_block => {
	78	const block = air.unwrapDbgBlock(inst);
	79	try verify.body(block.body);
	80	},
	81	.@"try", .try_cold => {
	82	const @"try" = air.unwrapTry(inst);
	83	try verify.body(@"try".else_body);
	84	},
	85	.try_ptr, .try_ptr_cold => {
	86	const try_ptr = air.unwrapTryPtr(inst);
	87	try verify.body(try_ptr.else_body);
	88	},
	89	.loop => {
	90	const block = air.unwrapBlock(inst);
	91	try verify.body(block.body);
	92	},
	93	.cond_br => {
	94	const cond_br = air.unwrapCondBr(inst);
	95	try verify.body(cond_br.then_body);
	96	try verify.body(cond_br.else_body);
	97	},
	98	.switch_br, .loop_switch_br => {
	99	const switch_br = air.unwrapSwitch(inst);
	100	var it = switch_br.iterateCases();
	101	while (it.next()) \|case\| {
	102	try verify.body(case.body);
	103	}
	104	const else_body = it.elseBody();
	105	if (else_body.len > 0) {
	106	try verify.body(else_body);
	107	}
	108	},
	109	.ret, .ret_safe => {
	110	const operand = data[@intFromEnum(inst)].un_op;
	111	if (air.typeOf(operand, ip).toIntern() != verify.ret_ty.toIntern()) return verify.fail("bad return type");
	112	},
	113	.ret_load => {
	114	const operand = data[@intFromEnum(inst)].un_op;
	115	const ptr_ty = air.typeOf(operand, ip);
	116	if (ptr_ty.zigTypeTag(zcu) != .pointer) return verify.fail("operand is not a pointer");
	117	if (ptr_ty.ptrSize(zcu) != .one) return verify.fail("pointer size is not '.one'");
	118	if (ptr_ty.childType(zcu).toIntern() != verify.ret_ty.toIntern()) return verify.fail("bad return type");
	119	},
	120
	121	.bit_cast => {
	122	const ty_op = data[@intFromEnum(inst)].ty_op;
	123	const operand_ty = air.typeOf(ty_op.operand, ip);
	124	const result_ty = ty_op.ty.toType();
	125	// Enums are allowed here even if their backing type is implicit.
	126	if (!operand_ty.hasBitRepresentation(zcu) and operand_ty.zigTypeTag(zcu) != .@"enum") {
	127	return verify.fail("bad operand type");
	128	}
	129	if (!result_ty.hasBitRepresentation(zcu) and result_ty.zigTypeTag(zcu) != .@"enum") {
	130	return verify.fail("bad result type");
	131	}
	132	if (operand_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type (pointer)");
	133	if (result_ty.isPtrAtRuntime(zcu)) return verify.fail("bad result type (pointer)");
	134	if (operand_ty.bitSize(zcu) != result_ty.bitSize(zcu)) return verify.fail("bit size mismatch");
	135	},
	136	.ptr_cast => {
	137	const ty_op = data[@intFromEnum(inst)].ty_op;
	138	const operand_ty = air.typeOf(ty_op.operand, ip);
	139	const result_ty = ty_op.ty.toType();
	140	const operand_scalar_ty = operand_ty.scalarType(zcu);
	141	const result_scalar_ty = result_ty.scalarType(zcu);
	142	if (operand_ty.isSliceAtRuntime(zcu)) {
	143	if (!result_ty.isSliceAtRuntime(zcu)) return verify.fail("operand is slice, but result is not");
	144	} else {
	145	if (!operand_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type");
	146	if (!result_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("operand is pointer, but result is not");
	147	if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
	148	if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
	149	}
	150	if (operand_scalar_ty.ptrAddressSpace(zcu) != result_scalar_ty.ptrAddressSpace(zcu)) {
	151	return verify.fail("illegal change to address space");
	152	}
	153	},
	154	.ptr_from_int => {
	155	const ty_op = data[@intFromEnum(inst)].ty_op;
	156	const operand_ty = air.typeOf(ty_op.operand, ip);
	157	const result_ty = ty_op.ty.toType();
	158	const operand_scalar_ty = operand_ty.scalarType(zcu);
	159	const result_scalar_ty = result_ty.scalarType(zcu);
	160	if (operand_scalar_ty.toIntern() != .usize_type) return verify.fail("bad operand type");
	161	if (!result_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad result type");
	162	if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
	163	if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
	164	},
	165	.int_from_ptr => {
	166	const ty_op = data[@intFromEnum(inst)].ty_op;
	167	const operand_ty = air.typeOf(ty_op.operand, ip);
	168	const result_ty = ty_op.ty.toType();
	169	const operand_scalar_ty = operand_ty.scalarType(zcu);
	170	const result_scalar_ty = result_ty.scalarType(zcu);
	171	if (!operand_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type");
	172	if (result_scalar_ty.toIntern() != .usize_type) return verify.fail("bad result type");
	173	if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
	174	if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
	175	},
	176	.error_cast => {
	177	const ty_op = data[@intFromEnum(inst)].ty_op;
	178	const operand_ty = air.typeOf(ty_op.operand, ip);
	179	const result_ty = ty_op.ty.toType();
	180	switch (operand_ty.zigTypeTag(zcu)) {
	181	else => return verify.fail("bad operand type"),
	182	.error_union => {
	183	if (result_ty.zigTypeTag(zcu) != .error_union) {
	184	return verify.fail("operand is error union, but result is not");
	185	}
	186	if (operand_ty.errorUnionPayload(zcu).toIntern() != result_ty.errorUnionPayload(zcu).toIntern()) {
	187	return verify.fail("error union payload type differs");
	188	}
	189	},
	190	.error_set => if (result_ty.zigTypeTag(zcu) != .error_set) {
	191	return verify.fail("operand is error set, but result is not");
	192	},
	193	}
	194	},
	195	.error_from_int => {
	196	const ty_op = data[@intFromEnum(inst)].ty_op;
	197	const operand_ty = air.typeOf(ty_op.operand, ip);
	198	const result_ty = ty_op.ty.toType();
	199	if (!operand_ty.isUnsignedInt(zcu)) return verify.fail("bad operand type");
	200	if (operand_ty.bitSize(zcu) != zcu.errorSetBits()) return verify.fail("bad operand bit size");
	201	if (result_ty.zigTypeTag(zcu) != .error_set) return verify.fail("bad result type");
	202	},
	203	.int_from_error => {
	204	const ty_op = data[@intFromEnum(inst)].ty_op;
	205	const operand_ty = air.typeOf(ty_op.operand, ip);
	206	const result_ty = ty_op.ty.toType();
	207	if (operand_ty.zigTypeTag(zcu) != .error_set) return verify.fail("bad operand type");
	208	if (!result_ty.isUnsignedInt(zcu)) return verify.fail("bad result type");
	209	if (result_ty.bitSize(zcu) != zcu.errorSetBits()) return verify.fail("bad result bit size");
	210	},
	211	.union_from_enum => {
	212	const ty_op = data[@intFromEnum(inst)].ty_op;
	213	const operand_ty = air.typeOf(ty_op.operand, ip);
	214	const result_ty = ty_op.ty.toType();
	215	if (operand_ty.zigTypeTag(zcu) != .@"enum") return verify.fail("bad operand type");
	216	if (result_ty.zigTypeTag(zcu) != .@"union") return verify.fail("bad result type");
	217	const union_tag_ty = result_ty.unionTagType(zcu) orelse return verify.fail("union type is not tagged");
	218	if (union_tag_ty.toIntern() != operand_ty.toIntern()) return verify.fail("union tag type does not match operand type");
	219	},
	220
	221	.ptr_elem_ptr => {
	222	const ty_pl = data[@intFromEnum(inst)].ty_pl;
	223	const bin_op = air.extraData(Air.Bin, ty_pl.payload).data;
	224	const ptr_ty = air.typeOf(bin_op.lhs, ip);
	225	const result_ty = ty_pl.ty.toType();
	226	if (ptr_ty.zigTypeTag(zcu) != .pointer) return verify.fail("bad pointer type");
	227	if (result_ty.zigTypeTag(zcu) != .pointer) return verify.fail("bad result type");
	228	const ptr_info = ptr_ty.ptrInfo(zcu);
	229	const result_ptr_info = result_ty.ptrInfo(zcu);
	230	if (ptr_info.packed_offset.host_size != 0) return verify.fail("pointer type is bitpacked pointer");
	231	if (result_ptr_info.packed_offset.host_size != 0) return verify.fail("result type is bitpacked pointer");
	232	},
	233
	234	.arg,
	235	.add,
	236	.add_safe,
	237	.add_optimized,
	238	.add_wrap,
	239	.add_sat,
	240	.sub,
	241	.sub_safe,
	242	.sub_optimized,
	243	.sub_wrap,
	244	.sub_sat,
	245	.mul,
	246	.mul_safe,
	247	.mul_optimized,
	248	.mul_wrap,
	249	.mul_sat,
	250	.div_float,
	251	.div_float_optimized,
	252	.div_trunc,
	253	.div_trunc_optimized,
	254	.div_floor,
	255	.div_floor_optimized,
	256	.div_exact,
	257	.div_exact_optimized,
	258	.rem,
	259	.rem_optimized,
	260	.mod,
	261	.mod_optimized,
	262	.ptr_add,
	263	.ptr_sub,
	264	.max,
	265	.min,
	266	.add_with_overflow,
	267	.sub_with_overflow,
	268	.mul_with_overflow,
	269	.shl_with_overflow,
	270	.alloc,
	271	.inferred_alloc,
	272	.inferred_alloc_comptime,
	273	.ret_ptr,
	274	.assembly,
	275	.bit_and,
	276	.bit_or,
	277	.shr,
	278	.shr_exact,
	279	.shl,
	280	.shl_exact,
	281	.shl_sat,
	282	.xor,
	283	.not,
	284	.repeat,
	285	.br,
	286	.trap,
	287	.breakpoint,
	288	.ret_addr,
	289	.frame_addr,
	290	.call,
	291	.call_always_tail,
	292	.call_never_tail,
	293	.call_never_inline,
	294	.clz,
	295	.ctz,
	296	.popcount,
	297	.byte_swap,
	298	.bit_reverse,
	299	.sqrt,
	300	.sin,
	301	.cos,
	302	.tan,
	303	.exp,
	304	.exp2,
	305	.log,
	306	.log2,
	307	.log10,
	308	.abs,
	309	.floor,
	310	.ceil,
	311	.round,
	312	.trunc_float,
	313	.neg,
	314	.neg_optimized,
	315	.cmp_lt,
	316	.cmp_lt_optimized,
	317	.cmp_lte,
	318	.cmp_lte_optimized,
	319	.cmp_eq,
	320	.cmp_eq_optimized,
	321	.cmp_gte,
	322	.cmp_gte_optimized,
	323	.cmp_gt,
	324	.cmp_gt_optimized,
	325	.cmp_neq,
	326	.cmp_neq_optimized,
	327	.cmp_vector,
	328	.cmp_vector_optimized,
	329	.switch_dispatch,
	330	.dbg_stmt,
	331	.dbg_empty_stmt,
	332	.dbg_var_ptr,
	333	.dbg_var_val,
	334	.dbg_arg_inline,
	335	.is_null,
	336	.is_non_null,
	337	.is_null_ptr,
	338	.is_non_null_ptr,
	339	.is_err,
	340	.is_non_err,
	341	.is_err_ptr,
	342	.is_non_err_ptr,
	343	.load,
	344	.store,
	345	.store_safe,
	346	.unreach,
	347	.fptrunc,
	348	.fpext,
	349	.int_cast,
	350	.int_cast_safe,
	351	.trunc,
	352	.optional_payload,
	353	.optional_payload_ptr,
	354	.optional_payload_ptr_set,
	355	.wrap_optional,
	356	.unwrap_errunion_payload,
	357	.unwrap_errunion_err,
	358	.unwrap_errunion_payload_ptr,
	359	.unwrap_errunion_err_ptr,
	360	.errunion_payload_ptr_set,
	361	.wrap_errunion_payload,
	362	.wrap_errunion_err,
	363	.struct_field_ptr,
	364	.struct_field_ptr_index_0,
	365	.struct_field_ptr_index_1,
	366	.struct_field_ptr_index_2,
	367	.struct_field_ptr_index_3,
	368	.struct_field_val,
	369	.set_union_tag,
	370	.get_union_tag,
	371	.slice,
	372	.slice_len,
	373	.slice_ptr,
	374	.ptr_slice_len_ptr,
	375	.ptr_slice_ptr_ptr,
	376	.array_elem_val,
	377	.slice_elem_val,
	378	.slice_elem_ptr,
	379	.ptr_elem_val,
	380	.array_to_slice,
	381	.int_from_float,
	382	.int_from_float_optimized,
	383	.int_from_float_safe,
	384	.int_from_float_optimized_safe,
	385	.float_from_int,
	386	.reduce,
	387	.reduce_optimized,
	388	.splat,
	389	.shuffle_one,
	390	.shuffle_two,
	391	.select,
	392	.memset,
	393	.memset_safe,
	394	.memcpy,
	395	.memmove,
	396	.cmpxchg_weak,
	397	.cmpxchg_strong,
	398	.atomic_load,
	399	.atomic_store_unordered,
	400	.atomic_store_monotonic,
	401	.atomic_store_release,
	402	.atomic_store_seq_cst,
	403	.atomic_rmw,
	404	.is_named_enum_value,
	405	.tag_name,
	406	.error_name,
	407	.error_set_has_value,
	408	.aggregate_init,
	409	.union_init,
	410	.prefetch,
	411	.mul_add,
	412	.field_parent_ptr,
	413	.wasm_memory_size,
	414	.wasm_memory_grow,
	415	.cmp_lte_errors_len,
	416	.err_return_trace,
	417	.set_err_return_trace,
	418	.addrspace_cast,
	419	.save_err_return_trace_index,
	420	.runtime_nav_ptr,
	421	.c_va_arg,
	422	.c_va_copy,
	423	.c_va_end,
	424	.c_va_start,
	425	.spirv_runtime_array_len,
	426	.work_item_id,
	427	.work_group_size,
	428	.work_group_id,
	429	.legalize_vec_store_elem,
	430	.legalize_vec_elem_val,
	431	.legalize_compiler_rt_call,
	432	=> {},
	433	}
	434	if (air.typeOfIndex(inst, ip).isNoReturn(zcu)) {
	435	if (body_index == body_insts.len - 1) return;
	436
	437	// HACK: right now, we emit the safety check for noreturn functions returning in a weird
	438	// way, where the `call` instruction is `noreturn` but there are still instructions
	439	// following it. We need to figure out a better way to represent that! That safety check
	440	// probably just needs to live exclusively in backends; putting AIR instructions after a
	441	// call implies that we have e.g. a valid stack at that point, which we can't actually
	442	// assume when the user has gotten a function's ABI wrong.
	443	switch (tags[@intFromEnum(inst)]) {
	444	.call,
	445	.call_always_tail,
	446	.call_never_tail,
	447	.call_never_inline,
	448	=> continue,
	449	else => {},
	450	}
	451
	452	return verify.fail("body contains instructions after noreturn");
	453	}
	454	}
	455	return verify.fail("body does not terminate noreturn");
	456	}
	457
	458	const std = @import("std");
	459	const log = std.log.scoped(.air_verify);
	460
	461	const Zcu = @import("../Zcu.zig");
	462	const InternPool = @import("../InternPool.zig");
	463	const Air = @import("../Air.zig");
	464	const Type = @import("../Type.zig");
	465	const Value = @import("../Value.zig");

src/Air/print.zig+10-3

...	...	@@ -232,12 +232,19 @@ const Writer = struct {
232	232	.arg => try w.writeArg(s, inst),
233	233
234	234	.not,
235		.bitcast,
	235	.bit_cast,
	236	.ptr_cast,
	237	.ptr_from_int,
	238	.int_from_ptr,
	239	.error_cast,
	240	.error_from_int,
	241	.int_from_error,
	242	.union_from_enum,
236	243	.load,
237	244	.fptrunc,
238	245	.fpext,
239		.intcast,
240		.intcast_safe,
	246	.int_cast,
	247	.int_cast_safe,
241	248	.trunc,
242	249	.optional_payload,
243	250	.optional_payload_ptr,

src/InternPool.zig+1-7

...	...	@@ -6007,13 +6007,7 @@ pub const Alignment = enum(u6) {
6007	6007	return r;
6008	6008	}
6009	6009
6010		const LlvmBuilderAlignment = std.zig.llvm.Builder.Alignment;
6011
6012		pub fn toLlvm(a: Alignment) LlvmBuilderAlignment {
6013		return @enumFromInt(@intFromEnum(a));
6014		}
6015
6016		pub fn fromLlvm(a: LlvmBuilderAlignment) Alignment {
	6010	pub fn toLlvm(a: Alignment) std.zig.llvm.Builder.Alignment {
6017	6011	return @enumFromInt(@intFromEnum(a));
6018	6012	}
6019	6013	};

src/Sema.zig+376-478

...	...	@@ -583,16 +583,6 @@ pub const Block = struct {
583	583	});
584	584	}
585	585
586		fn addBitCast(block: *Block, ty: Type, operand: Air.Inst.Ref) Allocator.Error!Air.Inst.Ref {
587		return block.addInst(.{
588		.tag = .bitcast,
589		.data = .{ .ty_op = .{
590		.ty = Air.internedToRef(ty.toIntern()),
591		.operand = operand,
592		} },
593		});
594		}
595
596	586	fn addNoOp(block: *Block, tag: Air.Inst.Tag) error{OutOfMemory}!Air.Inst.Ref {
597	587	return block.addInst(.{
598	588	.tag = tag,
...	...	@@ -3113,14 +3103,14 @@ fn zirRefDeref(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
3113	3103	// https://github.com/ziglang/zig/issues/6597
3114	3104	if (sema.resolveValue(operand)) \|operand_val\| {
3115	3105	if (!operand_val.isNull(zcu)) {
3116		break :single_ptr try sema.coerceInMemory(operand_val, single_ptr_ty);
	3106	break :single_ptr .fromValue(try pt.getCoerced(operand_val, single_ptr_ty));
3117	3107	}
3118	3108	}
3119	3109	if (block.wantSafety()) {
3120	3110	const is_non_null = try block.addUnOp(.is_non_null, operand);
3121	3111	try sema.addSafetyCheck(block, src, is_non_null, .unwrap_null);
3122	3112	}
3123		const single_ptr = try block.addBitCast(single_ptr_ty, operand);
	3113	const single_ptr = try block.addTyOp(.ptr_cast, single_ptr_ty, operand);
3124	3114	try sema.checkKnownAllocPtr(block, operand, single_ptr);
3125	3115	break :single_ptr single_ptr;
3126	3116	},
...	...	@@ -3586,7 +3576,7 @@ fn resolveComptimeKnownAllocPtr(sema: Sema, block: Block, alloc: Air.Inst.Ref,
3586	3576	.{ .elem = idx_val.toUnsignedInt(zcu) },
3587	3577	};
3588	3578	},
3589		.bitcast => .{
	3579	.ptr_cast => .{
3590	3580	tmp_air.instructions.items(.data)[@intFromEnum(air_ptr)].ty_op.operand,
3591	3581	.same_addr,
3592	3582	},
...	...	@@ -3729,7 +3719,7 @@ fn finishResolveComptimeKnownAllocPtr(
3729	3719	// This instruction has type `alloc_ty`, meaning we can rewrite the `alloc` AIR instruction to
3730	3720	// this one to drop the side effect. We also need to rewrite the stores; we'll turn them to this
3731	3721	// too because it doesn't really matter what they become.
3732		const nop_inst: Air.Inst = .{ .tag = .bitcast, .data = .{ .ty_op = .{
	3722	const nop_inst: Air.Inst = .{ .tag = .ptr_from_int, .data = .{ .ty_op = .{
3733	3723	.ty = .fromIntern(alloc_ty.toIntern()),
3734	3724	.operand = .zero_usize,
3735	3725	} } };
...	...	@@ -3779,7 +3769,7 @@ fn makePtrConst(sema: Sema, block: Block, alloc: Air.Inst.Ref) CompileError!Ai
3779	3769	return Air.internedToRef((try sema.pt.getCoerced(val, const_ptr_ty)).toIntern());
3780	3770	}
3781	3771
3782		return block.addBitCast(const_ptr_ty, alloc);
	3772	return block.addTyOp(.ptr_cast, const_ptr_ty, alloc);
3783	3773	}
3784	3774
3785	3775	fn zirAllocInferredComptime(
...	...	@@ -7330,7 +7320,7 @@ fn analyzeCall(
7330	7320	if (resolved_ty == .none) break :r result_raw;
7331	7321	// TODO: mutate in place the previous instruction if possible
7332	7322	// rather than adding a bitcast instruction.
7333		break :r try block.addBitCast(.fromInterned(resolved_ty), result_raw);
	7323	break :r try block.addTyOp(.error_cast, .fromInterned(resolved_ty), result_raw);
7334	7324	};
7335	7325
7336	7326	if (block.isComptime()) {
...	...	@@ -7636,7 +7626,7 @@ fn zirIntFromError(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstD
7636	7626	}
7637	7627
7638	7628	try sema.requireRuntimeBlock(block, src, operand_src);
7639		return block.addBitCast(err_int_ty, operand);
	7629	return block.addTyOp(.int_from_error, err_int_ty, operand);
7640	7630	}
7641	7631
7642	7632	fn zirErrorFromInt(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref {
...	...	@@ -7674,13 +7664,7 @@ fn zirErrorFromInt(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstD
7674	7664	const ok = try block.addBinOp(.bit_and, is_lte_len, is_non_zero);
7675	7665	try sema.addSafetyCheck(block, src, ok, .invalid_error_code);
7676	7666	}
7677		return block.addInst(.{
7678		.tag = .bitcast,
7679		.data = .{ .ty_op = .{
7680		.ty = .anyerror_type,
7681		.operand = operand,
7682		} },
7683		});
	7667	return block.addTyOp(.error_from_int, .anyerror, operand);
7684	7668	}
7685	7669
7686	7670	fn zirMergeErrorSets(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -7863,7 +7847,7 @@ fn zirIntFromEnum(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError
7863	7847	}
7864	7848
7865	7849	try sema.requireRuntimeBlock(block, src, operand_src);
7866		return block.addBitCast(int_tag_ty, enum_tag);
	7850	return block.addTyOp(.bit_cast, int_tag_ty, enum_tag);
7867	7851	}
7868	7852
7869	7853	fn zirEnumFromInt(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -7922,9 +7906,9 @@ fn zirEnumFromInt(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError
7922	7906	try sema.requireRuntimeBlock(block, src, operand_src);
7923	7907	if (block.wantSafety()) {
7924	7908	try sema.preparePanicId(src, .invalid_enum_value);
7925		return block.addTyOp(.intcast_safe, dest_ty, operand);
	7909	return block.addTyOp(.int_cast_safe, dest_ty, operand);
7926	7910	}
7927		return block.addTyOp(.intcast, dest_ty, operand);
	7911	return block.addTyOp(.int_cast, dest_ty, operand);
7928	7912	}
7929	7913
7930	7914	/// Pointer in, pointer out.
...	...	@@ -9152,7 +9136,7 @@ fn zirIntFromPtr(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!
9152	9136	try sema.requireRuntimeBlock(block, block.nodeOffset(inst_data.src_node), ptr_src);
9153	9137	try sema.validateRuntimeValue(block, ptr_src, operand);
9154	9138	try sema.checkLogicalPtrOperation(block, ptr_src, ptr_ty);
9155		return block.addBitCast(dest_ty, operand);
	9139	return block.addTyOp(.int_from_ptr, dest_ty, operand);
9156	9140	}
9157	9141
9158	9142	fn zirFieldPtrLoad(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -9314,9 +9298,9 @@ fn intCast(
9314	9298	try sema.requireRuntimeBlock(block, src, operand_src);
9315	9299	if (block.wantSafety()) {
9316	9300	try sema.preparePanicId(src, .integer_out_of_bounds);
9317		return block.addTyOp(.intcast_safe, dest_ty, operand);
	9301	return block.addTyOp(.int_cast_safe, dest_ty, operand);
9318	9302	}
9319		return block.addTyOp(.intcast, dest_ty, operand);
	9303	return block.addTyOp(.int_cast, dest_ty, operand);
9320	9304	}
9321	9305
9322	9306	fn zirBitcast(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -9330,158 +9314,53 @@ fn zirBitcast(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air
9330	9314	const dest_ty = try sema.resolveDestType(block, src, extra.lhs, .remove_eu_opt, "@bitCast");
9331	9315	const operand = sema.resolveInst(extra.rhs);
9332	9316	const operand_ty = sema.typeOf(operand);
9333		switch (dest_ty.zigTypeTag(zcu)) {
9334		.@"anyframe",
9335		.comptime_float,
9336		.comptime_int,
9337		.enum_literal,
9338		.error_set,
9339		.error_union,
9340		.@"fn",
9341		.frame,
9342		.noreturn,
9343		.null,
9344		.@"opaque",
9345		.spirv,
9346		.optional,
9347		.type,
9348		.undefined,
9349		.void,
9350		=> return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)}),
9351
9352		.@"enum" => {
9353		const msg = msg: {
9354		const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9355		errdefer msg.destroy(sema.gpa);
9356		switch (operand_ty.zigTypeTag(zcu)) {
9357		.int, .comptime_int => try sema.errNote(src, msg, "use @enumFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),
9358		else => {},
9359		}
9360
9361		break :msg msg;
9362		};
9363		return sema.failWithOwnedErrorMsg(block, msg);
9364		},
9365	9317
9366		.pointer => {
9367		const msg = msg: {
9368		const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9369		errdefer msg.destroy(sema.gpa);
9370		switch (operand_ty.zigTypeTag(zcu)) {
9371		.int, .comptime_int => try sema.errNote(src, msg, "use @ptrFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),
9372		.pointer => try sema.errNote(src, msg, "use @ptrCast to cast from '{f}'", .{operand_ty.fmt(pt)}),
9373		else => {},
9374		}
9375
9376		break :msg msg;
9377		};
9378		return sema.failWithOwnedErrorMsg(block, msg);
9379		},
9380		.@"struct", .@"union" => if (dest_ty.containerLayout(zcu) == .auto) {
9381		const container = switch (dest_ty.zigTypeTag(zcu)) {
9382		.@"struct" => "struct",
9383		.@"union" => "union",
9384		else => unreachable,
9385		};
9386		return sema.fail(block, src, "cannot @bitCast to '{f}'; {s} does not have a guaranteed in-memory layout", .{
9387		dest_ty.fmt(pt), container,
9388		});
9389		},
9390		.array => {
9391		const elem_ty = dest_ty.childType(zcu);
9392		if (!elem_ty.hasWellDefinedLayout(zcu)) {
9393		const msg = msg: {
9394		const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9395		errdefer msg.destroy(sema.gpa);
9396		try sema.errNote(src, msg, "array element type '{f}' does not have a guaranteed in-memory layout", .{elem_ty.fmt(pt)});
9397		break :msg msg;
9398		};
9399		return sema.failWithOwnedErrorMsg(block, msg);
	9318	// Check for pointers before checking `hasBitRepresentation` so we can emit a better message for slices.
	9319	switch (dest_ty.scalarType(zcu).zigTypeTag(zcu)) {
	9320	.pointer, .optional => return sema.failWithOwnedErrorMsg(block, msg: {
	9321	const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
	9322	errdefer msg.destroy(sema.gpa);
	9323	switch (operand_ty.zigTypeTag(zcu)) {
	9324	.int, .comptime_int => try sema.errNote(src, msg, "use @ptrFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),
	9325	.pointer => try sema.errNote(src, msg, "use @ptrCast to cast from '{f}'", .{operand_ty.fmt(pt)}),
	9326	else => {},
9400	9327	}
9401		},
9402	9328
9403		.bool,
9404		.float,
9405		.int,
9406		.vector,
9407		=> {},
9408		}
9409		switch (operand_ty.zigTypeTag(zcu)) {
9410		.@"anyframe",
9411		.comptime_float,
9412		.comptime_int,
9413		.enum_literal,
9414		.error_set,
9415		.error_union,
9416		.@"fn",
9417		.frame,
9418		.noreturn,
9419		.null,
9420		.@"opaque",
9421		.spirv,
9422		.optional,
9423		.type,
9424		.undefined,
9425		.void,
9426		=> return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)}),
9427
9428		.@"enum" => {
9429		const msg = msg: {
9430		const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9431		errdefer msg.destroy(sema.gpa);
9432		switch (dest_ty.zigTypeTag(zcu)) {
9433		.int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromEnum to cast to '{f}'", .{dest_ty.fmt(pt)}),
9434		else => {},
9435		}
9436
9437		break :msg msg;
9438		};
9439		return sema.failWithOwnedErrorMsg(block, msg);
	9329	break :msg msg;
	9330	}),
	9331	.array => switch (dest_ty.arrayBase(zcu)[0].zigTypeTag(zcu)) {
	9332	.pointer, .optional => return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)}),
	9333	else => {},
9440	9334	},
9441		.pointer => {
9442		const msg = msg: {
9443		const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9444		errdefer msg.destroy(sema.gpa);
9445		switch (dest_ty.zigTypeTag(zcu)) {
9446		.int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromPtr to cast to '{f}'", .{dest_ty.fmt(pt)}),
9447		.pointer => try sema.errNote(operand_src, msg, "use @ptrCast to cast to '{f}'", .{dest_ty.fmt(pt)}),
9448		else => {},
9449		}
	9335	else => {},
	9336	}
	9337	if (!dest_ty.hasBitRepresentation(zcu)) {
	9338	return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
	9339	}
9450	9340
9451		break :msg msg;
9452		};
9453		return sema.failWithOwnedErrorMsg(block, msg);
9454		},
9455		.@"struct", .@"union" => if (operand_ty.containerLayout(zcu) == .auto) {
9456		const container = switch (operand_ty.zigTypeTag(zcu)) {
9457		.@"struct" => "struct",
9458		.@"union" => "union",
9459		else => unreachable,
9460		};
9461		return sema.fail(block, operand_src, "cannot @bitCast from '{f}'; {s} does not have a guaranteed in-memory layout", .{
9462		operand_ty.fmt(pt), container,
9463		});
9464		},
9465		.array => {
9466		const elem_ty = operand_ty.childType(zcu);
9467		if (!elem_ty.hasWellDefinedLayout(zcu)) {
9468		const msg = msg: {
9469		const msg = try sema.errMsg(src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9470		errdefer msg.destroy(sema.gpa);
9471		try sema.errNote(src, msg, "array element type '{f}' does not have a guaranteed in-memory layout", .{elem_ty.fmt(pt)});
9472		break :msg msg;
9473		};
9474		return sema.failWithOwnedErrorMsg(block, msg);
	9341	// Check for pointers before checking `hasBitRepresentation` so we can emit a better message for slices.
	9342	switch (operand_ty.scalarType(zcu).zigTypeTag(zcu)) {
	9343	.pointer, .optional => return sema.failWithOwnedErrorMsg(block, msg: {
	9344	const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
	9345	errdefer msg.destroy(sema.gpa);
	9346	switch (dest_ty.zigTypeTag(zcu)) {
	9347	.int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromPtr to cast to '{f}'", .{dest_ty.fmt(pt)}),
	9348	.pointer => try sema.errNote(operand_src, msg, "use @ptrCast to cast to '{f}'", .{dest_ty.fmt(pt)}),
	9349	else => {},
9475	9350	}
	9351	break :msg msg;
	9352	}),
	9353	.array => switch (operand_ty.arrayBase(zcu)[0].zigTypeTag(zcu)) {
	9354	.pointer, .optional => return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{dest_ty.fmt(pt)}),
	9355	else => {},
9476	9356	},
9477
9478		.bool,
9479		.float,
9480		.int,
9481		.vector,
9482		=> {},
	9357	else => {},
	9358	}
	9359	if (!operand_ty.hasBitRepresentation(zcu)) {
	9360	return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9483	9361	}
9484		return sema.bitCast(block, dest_ty, operand, block.nodeOffset(inst_data.src_node), operand_src);
	9362
	9363	return sema.bitCast(block, dest_ty, operand, block.nodeOffset(inst_data.src_node));
9485	9364	}
9486	9365
9487	9366	fn zirFloatCast(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -12026,11 +11905,17 @@ fn analyzeSwitchCaptures(
12026	11905	.@"inline" => unreachable, // handled above
12027	11906	.has_ranges => unreachable, // not possible for error set
12028	11907	.special => {
12029		if (else_err_ty) \|err_ty\| {
12030		break :payload_ref try sema.bitCast(case_block, err_ty, loaded_operand, operand_src, null);
12031		} else {
	11908	const capture_err_ty = else_err_ty orelse {
12032	11909	try sema.analyzeUnreachable(case_block, operand_src, false);
12033	11910	break :payload_ref .unreachable_value;
	11911	};
	11912	if (sema.resolveValue(loaded_operand)) \|err_val\| {
	11913	break :payload_ref .fromIntern(try pt.intern(.{ .err = .{
	11914	.ty = capture_err_ty.toIntern(),
	11915	.name = zcu.intern_pool.indexToKey(err_val.toIntern()).err.name,
	11916	} }));
	11917	} else {
	11918	break :payload_ref try case_block.addTyOp(.error_cast, capture_err_ty, loaded_operand);
12034	11919	}
12035	11920	},
12036	11921	.item_refs => \|item_refs\| {
...	...	@@ -12040,8 +11925,15 @@ fn analyzeSwitchCaptures(
12040	11925	const item_val = sema.resolveValue(item_ref).?;
12041	11926	names.putAssumeCapacityNoClobber(item_val.getErrorName(zcu).unwrap().?, {});
12042	11927	}
12043		const narrowed_ty = try pt.errorSetFromUnsortedNames(names.keys());
12044		break :payload_ref try sema.bitCast(case_block, narrowed_ty, loaded_operand, operand_src, null);
	11928	const capture_err_ty = try pt.errorSetFromUnsortedNames(names.keys());
	11929	if (sema.resolveValue(loaded_operand)) \|err_val\| {
	11930	break :payload_ref .fromIntern(try pt.intern(.{ .err = .{
	11931	.ty = capture_err_ty.toIntern(),
	11932	.name = zcu.intern_pool.indexToKey(err_val.toIntern()).err.name,
	11933	} }));
	11934	} else {
	11935	break :payload_ref try case_block.addTyOp(.error_cast, capture_err_ty, loaded_operand);
	11936	}
12045	11937	},
12046	11938	}
12047	11939	}
...	...	@@ -12259,40 +12151,9 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12259	12151	return case_block.addStructFieldVal(loaded_operand, first_field_index, capture_ty);
12260	12152	}
12261	12153
12262		// We may have to emit a switch block which coerces the operand to the capture type.
12263		// If we can, try to avoid that using in-memory coercions.
12264		const first_non_imc = in_mem: {
12265		for (field_indices, 0..) \|field_idx, i\| {
12266		const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);
12267		if (.ok != try sema.coerceInMemoryAllowed(case_block, capture_ty, field_ty, false, zcu.getTarget(), .unneeded, .unneeded, null)) {
12268		break :in_mem i;
12269		}
12270		}
12271		// All fields are in-memory coercible to the resolved type!
12272		// Just take the first field and bitcast the result.
12273		const uncoerced = try case_block.addStructFieldVal(loaded_operand, first_field_index, first_field_ty);
12274		return case_block.addBitCast(capture_ty, uncoerced);
12275		};
12276
12277	12154	// By-val capture with heterogeneous types which are not all in-memory coercible to
12278	12155	// the resolved capture type. We finally have to fall back to the ugly method.
12279	12156
12280		// However, let's first track which operands are in-memory coercible. There may well
12281		// be several, and we can squash all of these cases into the same switch prong using
12282		// a simple bitcast. We'll make this the 'else' prong.
12283
12284		var in_mem_coercible: std.bit_set.Dynamic = try .initFull(sema.arena, field_indices.len);
12285		in_mem_coercible.unset(first_non_imc);
12286		{
12287		const next = first_non_imc + 1;
12288		for (field_indices[next..], next..) \|field_idx, i\| {
12289		const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);
12290		if (.ok != try sema.coerceInMemoryAllowed(case_block, capture_ty, field_ty, false, zcu.getTarget(), .unneeded, .unneeded, null)) {
12291		in_mem_coercible.unset(i);
12292		}
12293		}
12294		}
12295
12296	12157	const capture_block_inst = try case_block.addInstAsIndex(.{
12297	12158	.tag = .block,
12298	12159	.data = .{
...	...	@@ -12303,23 +12164,19 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12303	12164	},
12304	12165	});
12305	12166
12306		const prong_count = field_indices.len - in_mem_coercible.count();
12307
12308		const estimated_extra = prong_count * 6 + (prong_count / 10); // 2 for Case, 1 item, probably 3 insts; plus hints
	12167	const estimated_extra = field_indices.len * 6 + (field_indices.len / 10); // 2 for Case, 1 item, probably 3 insts; plus hints
12309	12168	var cases_extra = try std.ArrayList(u32).initCapacity(gpa, estimated_extra);
12310	12169	defer cases_extra.deinit(gpa);
12311	12170
12312	12171	{
12313	12172	// All branch hints are `.none`, so just add zero elems.
12314	12173	comptime assert(@intFromEnum(std.lang.BranchHint.none) == 0);
12315		const need_elems = std.math.divCeil(usize, prong_count + 1, 10) catch unreachable;
	12174	const need_elems = std.math.divCeil(usize, field_indices.len + 1, 10) catch unreachable;
12316	12175	try cases_extra.appendNTimes(gpa, 0, need_elems);
12317	12176	}
12318	12177
12319	12178	{
12320		// Non-bitcast cases
12321		var it = in_mem_coercible.iterator(.{ .kind = .unset });
12322		while (it.next()) \|idx\| {
	12179	for (field_indices, item_refs, 0..) \|field_index, item, item_index\| {
12323	12180	var coerce_block = case_block.makeSubBlock();
12324	12181	defer coerce_block.instructions.deinit(sema.gpa);
12325	12182
...	...	@@ -12328,13 +12185,12 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12328	12185	.offset = .{ .switch_case_item = .{
12329	12186	.switch_node_offset = switch_node_offset,
12330	12187	.case_idx = capture_src.offset.switch_capture.case_idx,
12331		.item_idx = .{ .kind = .single, .value = @intCast(idx) },
	12188	.item_idx = .{ .kind = .single, .value = @intCast(item_index) },
12332	12189	} },
12333	12190	};
12334	12191
12335		const field_idx = field_indices[idx];
12336		const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);
12337		const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, field_idx, field_ty);
	12192	const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_index]);
	12193	const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, field_index, field_ty);
12338	12194	const coerced = try sema.coerce(&coerce_block, capture_ty, uncoerced, case_src);
12339	12195	_ = try coerce_block.addBr(capture_block_inst, coerced);
12340	12196
...	...	@@ -12346,24 +12202,16 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12346	12202	.ranges_len = 0,
12347	12203	.body_len = @intCast(coerce_block.instructions.items.len),
12348	12204	}));
12349		cases_extra.appendAssumeCapacity(@intFromEnum(item_refs[idx])); // item
	12205	cases_extra.appendAssumeCapacity(@intFromEnum(item)); // item
12350	12206	cases_extra.appendSliceAssumeCapacity(@ptrCast(coerce_block.instructions.items)); // body
12351	12207	}
12352	12208	}
12353	12209	const else_body_len = len: {
12354		// 'else' prong uses a bitcast
12355		var coerce_block = case_block.makeSubBlock();
12356		defer coerce_block.instructions.deinit(sema.gpa);
12357
12358		const first_imc_item_idx = in_mem_coercible.findFirstSet().?;
12359		const first_imc_field_idx = field_indices[first_imc_item_idx];
12360		const first_imc_field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[first_imc_field_idx]);
12361		const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, first_imc_field_idx, first_imc_field_ty);
12362		const coerced = try coerce_block.addBitCast(capture_ty, uncoerced);
12363		_ = try coerce_block.addBr(capture_block_inst, coerced);
12364
12365		try cases_extra.appendSlice(gpa, @ptrCast(coerce_block.instructions.items));
12366		break :len coerce_block.instructions.items.len;
	12210	// 'else' prong is unreachable
	12211	const result_index: Air.Inst.Index = @enumFromInt(sema.air_instructions.len);
	12212	try sema.air_instructions.append(gpa, .{ .tag = .unreach, .data = .{ .no_op = {} } });
	12213	try cases_extra.append(gpa, @intFromEnum(result_index));
	12214	break :len 1;
12367	12215	};
12368	12216
12369	12217	try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.SwitchBr).@"struct".field_names.len +
...	...	@@ -12378,7 +12226,7 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12378	12226	.pl_op = .{
12379	12227	.operand = undefined, // set by switch below
12380	12228	.payload = sema.addExtraAssumeCapacity(Air.SwitchBr{
12381		.cases_len = @intCast(prong_count),
	12229	.cases_len = @intCast(field_indices.len),
12382	12230	.else_body_len = @intCast(else_body_len),
12383	12231	}),
12384	12232	},
...	...	@@ -12488,7 +12336,7 @@ fn resolveSwitchItem(
12488	12336	// being switched on if their prong body is `=> comptime unreachable,`.
12489	12337	switch (try sema.coerceInMemoryAllowedErrorSets(block, item_ty, uncoerced_ty, item_src, item_src)) {
12490	12338	.ok => if (sema.resolveValue(uncoerced)) \|uncoerced_val\| {
12491		break :item_ref try sema.coerceInMemory(uncoerced_val, item_ty);
	12339	break :item_ref .fromValue(try pt.getCoerced(uncoerced_val, item_ty));
12492	12340	},
12493	12341	.missing_error => if (prong_is_comptime_unreach) {
12494	12342	break :item_ref uncoerced;
...	...	@@ -13394,8 +13242,8 @@ fn zirArrayCat(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
13394	13242	defer trash_block.instructions.deinit(sema.gpa);
13395	13243
13396	13244	const instructions = [_]Air.Inst.Ref{
13397		try trash_block.addBitCast(lhs_info.elem_type, .void_value),
13398		try trash_block.addBitCast(rhs_info.elem_type, .void_value),
	13245	try trash_block.addTyOp(.bit_cast, lhs_info.elem_type, .void_value),
	13246	try trash_block.addTyOp(.bit_cast, rhs_info.elem_type, .void_value),
13399	13247	};
13400	13248	break :t try sema.resolvePeerTypes(block, src, &instructions, .{
13401	13249	.override = &[_]?LazySrcLoc{ lhs_src, rhs_src },
...	...	@@ -13552,7 +13400,7 @@ fn zirArrayCat(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
13552	13400	});
13553	13401
13554	13402	const many_ty = slice_ty.slicePtrFieldType(zcu);
13555		const many_alloc = try block.addBitCast(many_ty, mutable_alloc);
	13403	const many_alloc = try block.addTyOp(.ptr_cast, many_ty, mutable_alloc);
13556	13404
13557	13405	// lhs_dest_slice = dest[0..lhs.len]
13558	13406	if (lhs_len > 0) {
...	...	@@ -13601,7 +13449,7 @@ fn zirArrayCat(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
13601	13449	try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);
13602	13450	}
13603	13451
13604		return block.addBitCast(constant_alloc_ty, mutable_alloc);
	13452	return block.addTyOp(.ptr_cast, constant_alloc_ty, mutable_alloc);
13605	13453	}
13606	13454
13607	13455	var elem_i: u32 = 0;
...	...	@@ -13634,7 +13482,7 @@ fn zirArrayCat(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
13634	13482	try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);
13635	13483	}
13636	13484
13637		return block.addBitCast(constant_alloc_ty, mutable_alloc);
	13485	return block.addTyOp(.ptr_cast, constant_alloc_ty, mutable_alloc);
13638	13486	}
13639	13487
13640	13488	const element_refs = try sema.arena.alloc(Air.Inst.Ref, result_len);
...	...	@@ -14917,8 +14765,8 @@ fn analyzeArithmetic(
14917	14765	try sema.requireRuntimeBlock(block, src, runtime_src);
14918	14766	try sema.checkLogicalPtrOperation(block, src, lhs_ty);
14919	14767	try sema.checkLogicalPtrOperation(block, src, rhs_ty);
14920		const lhs_int = try block.addBitCast(.usize, lhs);
14921		const rhs_int = try block.addBitCast(.usize, rhs);
	14768	const lhs_int = try block.addTyOp(.int_from_ptr, .usize, lhs);
	14769	const rhs_int = try block.addTyOp(.int_from_ptr, .usize, rhs);
14922	14770	const address = try block.addBinOp(.sub_wrap, lhs_int, rhs_int);
14923	14771	return try block.addBinOp(.div_exact, address, try pt.intRef(.usize, elem_size));
14924	14772	}
...	...	@@ -15186,15 +15034,76 @@ fn zirAsm(
15186	15034	break :out_ty sema.typeOf(inst).childType(zcu);
15187	15035	}
15188	15036	};
15189		if (!out_ty.hasWellDefinedLayout(zcu)) {
15190		return sema.failWithOwnedErrorMsg(block, msg: {
15191		const msg = try sema.errMsg(output_src, "invalid inline assembly output type; '{f}' does not have a guaranteed in-memory layout", .{
15192		out_ty.fmt(pt),
15193		});
	15037	switch (out_ty.zigTypeTag(zcu)) {
	15038	.int, .float, .bool, .vector => {},
	15039
	15040	.pointer => if (out_ty.isSlice(zcu)) return sema.failWithOwnedErrorMsg(block, msg: {
	15041	const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15194	15042	errdefer msg.destroy(gpa);
15195		try sema.addDeclaredHereNote(msg, out_ty);
	15043	try sema.errNote(output_src, msg, "consider separate outputs for 'ptr' and 'len'", .{});
15196	15044	break :msg msg;
15197		});
	15045	}),
	15046
	15047	.optional => if (!out_ty.isPtrLikeOptional(zcu)) {
	15048	return sema.fail(block, output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
	15049	},
	15050
	15051	.@"enum" => switch (ip.loadEnumType(out_ty.toIntern()).int_tag_mode) {
	15052	.explicit => {},
	15053	.auto => return sema.failWithOwnedErrorMsg(block, msg: {
	15054	const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
	15055	errdefer msg.destroy(gpa);
	15056	try sema.errNote(out_ty.srcLoc(zcu), msg, "integer tag type of enum is inferred", .{});
	15057	try sema.errNote(out_ty.srcLoc(zcu), msg, "consider explicitly specifying the integer tag type", .{});
	15058	break :msg msg;
	15059	}),
	15060	},
	15061
	15062	.@"struct" => switch (out_ty.containerLayout(zcu)) {
	15063	.@"packed" => {},
	15064	.auto, .@"extern" => return sema.failWithOwnedErrorMsg(block, msg: {
	15065	const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
	15066	errdefer msg.destroy(gpa);
	15067	try sema.errNote(output_src, msg, "struct types cannot be passed to inline assembly", .{});
	15068	try sema.addDeclaredHereNote(msg, out_ty);
	15069	break :msg msg;
	15070	}),
	15071	},
	15072
	15073	.@"union" => switch (out_ty.containerLayout(zcu)) {
	15074	.@"packed" => {},
	15075	.auto, .@"extern" => return sema.failWithOwnedErrorMsg(block, msg: {
	15076	const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
	15077	errdefer msg.destroy(gpa);
	15078	try sema.errNote(output_src, msg, "union types cannot be passed to inline assembly", .{});
	15079	try sema.addDeclaredHereNote(msg, out_ty);
	15080	break :msg msg;
	15081	}),
	15082	},
	15083
	15084	.array => return sema.failWithOwnedErrorMsg(block, msg: {
	15085	const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
	15086	errdefer msg.destroy(gpa);
	15087	try sema.errNote(output_src, msg, "array types cannot be passed to inline assembly", .{});
	15088	break :msg msg;
	15089	}),
	15090
	15091	.void,
	15092	.type,
	15093	.noreturn,
	15094	.comptime_float,
	15095	.comptime_int,
	15096	.undefined,
	15097	.null,
	15098	.error_union,
	15099	.error_set,
	15100	.@"fn",
	15101	.@"opaque",
	15102	.frame,
	15103	.@"anyframe",
	15104	.enum_literal,
	15105	.spirv,
	15106	=> return sema.fail(block, output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)}),
15198	15107	}
15199	15108
15200	15109	const constraint = sema.code.nullTerminatedString(output.data.constraint);
...	...	@@ -15598,37 +15507,13 @@ fn zirBitSizeOf(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!A
15598	15507	const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;
15599	15508	const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);
15600	15509	const operand_ty = try sema.resolveType(block, operand_src, inst_data.operand);
15601		switch (operand_ty.zigTypeTag(zcu)) {
15602		.@"fn",
15603		.noreturn,
15604		.undefined,
15605		.null,
15606		.@"opaque",
15607		.spirv,
15608		.type,
15609		.enum_literal,
15610		.comptime_float,
15611		.comptime_int,
15612		=> return sema.fail(block, operand_src, "no size available for type '{f}'", .{operand_ty.fmt(pt)}),
15613
15614		.void,
15615		=> return .zero,
15616
15617		.bool,
15618		.int,
15619		.float,
15620		.pointer,
15621		.array,
15622		.@"struct",
15623		.optional,
15624		.error_union,
15625		.error_set,
15626		.@"enum",
15627		.@"union",
15628		.vector,
15629		.frame,
15630		.@"anyframe",
15631		=> {},
	15510	if (!operand_ty.hasBitRepresentation(zcu) and
	15511	// TODO: allow these types too for now because this is used in some places. We need to
	15512	// figure out whether we think errors and auto-enums have bit representations!
	15513	operand_ty.zigTypeTag(zcu) != .error_set and
	15514	operand_ty.zigTypeTag(zcu) != .@"enum")
	15515	{
	15516	return sema.fail(block, operand_src, "no bit size available for type '{f}'", .{operand_ty.fmt(pt)});
15632	15517	}
15633	15518	try sema.ensureLayoutResolved(operand_ty, operand_src, .size_of);
15634	15519	return .fromValue(try pt.intValue(.comptime_int, operand_ty.bitSize(zcu)));
...	...	@@ -18179,13 +18064,19 @@ fn zirPtrType(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air
18179	18064	} else 0;
18180	18065
18181	18066	if (host_size != 0) {
	18067	try sema.ensureLayoutResolved(elem_ty, elem_ty_src, .bit_ptr_child);
	18068	if (elem_ty.unpackable(zcu)) \|reason\| return sema.failWithOwnedErrorMsg(block, msg: {
	18069	const msg = try sema.errMsg(elem_ty_src, "bit-pointer cannot refer to value of type '{f}'", .{elem_ty.fmt(pt)});
	18070	errdefer msg.destroy(sema.gpa);
	18071	try sema.explainWhyTypeIsUnpackable(msg, elem_ty_src, reason);
	18072	break :msg msg;
	18073	});
	18074	const elem_bit_size = elem_ty.bitSize(zcu);
18182	18075	if (bit_offset >= host_size * 8) {
18183	18076	return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} starts {d} bits after the end of a {d} byte host integer", .{
18184	18077	elem_ty.fmt(pt), bit_offset, bit_offset - host_size * 8, host_size,
18185	18078	});
18186	18079	}
18187		try sema.ensureLayoutResolved(elem_ty, elem_ty_src, .bit_ptr_child);
18188		const elem_bit_size = elem_ty.bitSize(zcu);
18189	18080	if (elem_bit_size > host_size * 8 - bit_offset) {
18190	18081	return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} ends {d} bits after the end of a {d} byte host integer", .{
18191	18082	elem_ty.fmt(pt), bit_offset, elem_bit_size - (host_size * 8 - bit_offset), host_size,
...	...	@@ -18201,15 +18092,6 @@ fn zirPtrType(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air
18201	18092	return sema.fail(block, elem_ty_src, "indexable pointer to opaque type '{f}' not allowed", .{elem_ty.fmt(pt)});
18202	18093	}
18203	18094
18204		if (host_size != 0) {
18205		if (elem_ty.unpackable(zcu)) \|reason\| return sema.failWithOwnedErrorMsg(block, msg: {
18206		const msg = try sema.errMsg(elem_ty_src, "bit-pointer cannot refer to value of type '{f}'", .{elem_ty.fmt(pt)});
18207		errdefer msg.destroy(sema.gpa);
18208		try sema.explainWhyTypeIsUnpackable(msg, elem_ty_src, reason);
18209		break :msg msg;
18210		});
18211		}
18212
18213	18095	const ty = try pt.ptrType(.{
18214	18096	.child = elem_ty.toIntern(),
18215	18097	.sentinel = sentinel,
...	...	@@ -18379,7 +18261,7 @@ fn zirUnionInit(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!A
18379	18261	const payload = try sema.coerce(block, field_ty, sema.resolveInst(extra.init), payload_src);
18380	18262
18381	18263	if (union_ty.containerLayout(zcu) == .@"packed") {
18382		return sema.bitCast(block, union_ty, payload, block.nodeOffset(inst_data.src_node), payload_src);
	18264	return sema.bitCast(block, union_ty, payload, block.nodeOffset(inst_data.src_node));
18383	18265	}
18384	18266
18385	18267	if (sema.resolveValue(payload)) \|payload_val\| {
...	...	@@ -18516,7 +18398,7 @@ fn zirStructInit(
18516	18398	const init_inst = try sema.coerce(block, field_ty, uncoerced_init_inst, field_src);
18517	18399
18518	18400	if (resolved_ty.containerLayout(zcu) == .@"packed") {
18519		const union_val = try sema.bitCast(block, resolved_ty, init_inst, src, field_src);
	18401	const union_val = try sema.bitCast(block, resolved_ty, init_inst, src);
18520	18402	const result_val = try sema.coerce(block, result_ty, union_val, src);
18521	18403	if (is_ref) {
18522	18404	return sema.analyzeRef(block, src, result_val, .none);
...	...	@@ -18680,20 +18562,15 @@ fn finishStructInit(
18680	18562	},
18681	18563	.@"packed" => {
18682	18564	const buf = try sema.arena.alloc(u8, @intCast((struct_ty.bitSize(zcu) + 7) / 8));
	18565	@memset(buf, 0);
18683	18566	var bit_offset: u16 = 0;
18684	18567	for (field_inits) \|field_init\| {
18685	18568	const field_val = sema.resolveValue(field_init).?;
18686		field_val.writeToPackedMemory(zcu, buf, bit_offset) catch \|err\| switch (err) {
18687		error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers
18688		error.OutOfMemory => \|e\| return e,
18689		};
	18569	field_val.writeToPackedMemory(zcu, buf, bit_offset);
18690	18570	bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));
18691	18571	}
18692	18572	assert(bit_offset == struct_ty.bitSize(zcu));
18693		const struct_val = Value.readFromPackedMemory(struct_ty, pt, buf, 0, sema.arena) catch \|err\| switch (err) {
18694		error.IllDefinedMemoryLayout => unreachable, // bitpacks have well-defined layout
18695		error.OutOfMemory => \|e\| return e,
18696		};
	18573	const struct_val: Value = try .readFromPackedMemory(struct_ty, pt, buf, 0);
18697	18574	const final_val_ref = try sema.coerce(block, result_ty, .fromValue(struct_val), init_src);
18698	18575	return sema.addConstantMaybeRef(sema.resolveValue(final_val_ref).?, is_ref);
18699	18576	},
...	...	@@ -19387,7 +19264,7 @@ fn zirIntFromBool(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError
19387	19264	}
19388	19265	return Air.internedToRef((try pt.aggregateValue(dest_ty, new_elems)).toIntern());
19389	19266	}
19390		return block.addBitCast(dest_ty, operand);
	19267	return block.addTyOp(.bit_cast, dest_ty, operand);
19391	19268	}
19392	19269
19393	19270	fn zirErrorName(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -21245,7 +21122,7 @@ fn zirPtrFromInt(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!
21245	21122	try sema.addSafetyCheck(block, src, is_aligned, .incorrect_alignment);
21246	21123	}
21247	21124	}
21248		return block.addBitCast(dest_ty, operand_coerced);
	21125	return block.addTyOp(.ptr_from_int, dest_ty, operand_coerced);
21249	21126	}
21250	21127
21251	21128	fn ptrFromIntVal(
...	...	@@ -21444,7 +21321,7 @@ fn zirErrorCast(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstData
21444	21321	.error_union => try block.addTyOp(.unwrap_errunion_err, operand_err_ty, operand),
21445	21322	else => unreachable,
21446	21323	};
21447		const err_int_inst = try block.addBitCast(err_int_ty, err_code_inst);
	21324	const err_int_inst = try block.addTyOp(.int_from_error, err_int_ty, err_code_inst);
21448	21325	if (dest_tag == .error_union) {
21449	21326	const zero_err = try pt.intRef(err_int_ty, 0);
21450	21327	const is_zero = try block.addBinOp(.cmp_eq, err_int_inst, zero_err);
...	...	@@ -21464,10 +21341,10 @@ fn zirErrorCast(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstData
21464	21341	}
21465	21342
21466	21343	if (operand_tag == .error_set and dest_tag == .error_union) {
21467		const err_val = try block.addBitCast(dest_err_ty, operand);
	21344	const err_val = try block.addTyOp(.error_cast, dest_err_ty, operand);
21468	21345	return block.addTyOp(.wrap_errunion_err, dest_ty, err_val);
21469	21346	} else {
21470		return block.addBitCast(dest_ty, operand);
	21347	return block.addTyOp(.error_cast, dest_ty, operand);
21471	21348	}
21472	21349	}
21473	21350
...	...	@@ -22015,7 +21892,7 @@ fn ptrCastFull(
22015	21892	// `operand_ptr` converted to an integer, for safety checks.
22016	21893	const operand_ptr_int: Air.Inst.Ref = if (need_null_check or need_align_check) i: {
22017	21894	assert(need_operand_ptr);
22018		break :i try block.addBitCast(.usize, operand_ptr);
	21895	break :i try block.addTyOp(.int_from_ptr, .usize, operand_ptr);
22019	21896	} else .none;
22020	21897
22021	21898	if (need_null_check) {
...	...	@@ -22042,8 +21919,8 @@ fn ptrCastFull(
22042	21919
22043	21920	if (dest_info.flags.size == .slice) {
22044	21921	if (src_info.flags.size == .slice and !flags.addrspace_cast and !slice_needs_len_change) {
22045		// Fast path: just bitcast!
22046		return block.addBitCast(dest_ty, operand);
	21922	// Fast path: just pointer cast!
	21923	return block.addTyOp(.ptr_cast, dest_ty, operand);
22047	21924	}
22048	21925
22049	21926	// We need to deconstruct the slice (if applicable) and reconstruct it.
...	...	@@ -22101,7 +21978,7 @@ fn ptrCastFull(
22101	21978	else => unreachable,
22102	21979	};
22103	21980	const coerced_ptr = if (operand_ptr_ty.toIntern() != want_ptr_ty.toIntern()) ptr: {
22104		break :ptr try block.addBitCast(want_ptr_ty, operand_ptr);
	21981	break :ptr try block.addTyOp(.ptr_cast, want_ptr_ty, operand_ptr);
22105	21982	} else operand_ptr;
22106	21983
22107	21984	return block.addInst(.{
...	...	@@ -22116,12 +21993,11 @@ fn ptrCastFull(
22116	21993	});
22117	21994	} else {
22118	21995	assert(need_operand_ptr);
22119		// We just need to bitcast the pointer, if necessary.
22120		// It might not be necessary, since we might have just needed the `addrspace_cast`.
	21996	// We just need a ptr_cast, if even that (we might only have needed the `addrspace_cast`).
22121	21997	const result = if (sema.typeOf(operand_ptr).toIntern() == dest_ty.toIntern())
22122	21998	operand_ptr
22123	21999	else
22124		try block.addBitCast(dest_ty, operand_ptr);
	22000	try block.addTyOp(.ptr_cast, dest_ty, operand_ptr);
22125	22001
22126	22002	try sema.checkKnownAllocPtr(block, operand, result);
22127	22003	return result;
...	...	@@ -22157,7 +22033,7 @@ fn zirPtrCastNoDest(sema: Sema, block: Block, extended: Zir.Inst.Extended.Inst
22157	22033	}
22158	22034
22159	22035	try sema.requireRuntimeBlock(block, src, null);
22160		const new_ptr = try block.addBitCast(dest_ty, operand);
	22036	const new_ptr = try block.addTyOp(.ptr_cast, dest_ty, operand);
22161	22037	try sema.checkKnownAllocPtr(block, operand, new_ptr);
22162	22038	return new_ptr;
22163	22039	}
...	...	@@ -24259,7 +24135,7 @@ fn analyzeMinMax(
24259	24135	// where we have refined the range, so we should be doing an intcast.
24260	24136	assert(intermediate_scalar_ty.zigTypeTag(zcu) == .int);
24261	24137	assert(result_scalar_ty.zigTypeTag(zcu) == .int);
24262		return block.addTyOp(.intcast, result_ty, cur_result);
	24138	return block.addTyOp(.int_cast, result_ty, cur_result);
24263	24139	}
24264	24140
24265	24141	fn upgradeToArrayPtr(sema: Sema, block: Block, ptr: Air.Inst.Ref, len: u64) !Air.Inst.Ref {
...	...	@@ -24289,7 +24165,7 @@ fn upgradeToArrayPtr(sema: Sema, block: Block, ptr: Air.Inst.Ref, len: u64) !A
24289	24165	try block.addTyOp(.slice_ptr, ptr_ty.slicePtrFieldType(zcu), ptr)
24290	24166	else
24291	24167	ptr;
24292		return block.addBitCast(new_ty, non_slice_ptr);
	24168	return block.addTyOp(.ptr_cast, new_ty, non_slice_ptr);
24293	24169	}
24294	24170
24295	24171	fn zirMemcpy(
...	...	@@ -25087,7 +24963,7 @@ fn zirBuiltinExtern(
25087	24963	const casted_ptr_val = try pt.getCoerced(uncasted_ptr_val, result_ptr_ty);
25088	24964	return Air.internedToRef(casted_ptr_val.toIntern());
25089	24965	} else {
25090		return block.addBitCast(result_ptr_ty, uncasted_ptr);
	24966	return block.addTyOp(.ptr_cast, result_ptr_ty, uncasted_ptr);
25091	24967	}
25092	24968	}
25093	24969
...	...	@@ -25533,7 +25409,7 @@ pub fn explainWhyTypeIsNotExtern(
25533	25409	.param_ty => try sema.errNote(src_loc, msg, "arrays are not allowed as a parameter type", .{}),
25534	25410	else => try sema.explainWhyTypeIsNotExtern(msg, src_loc, ty.childType(zcu), .element),
25535	25411	},
25536		.vector => try sema.explainWhyTypeIsNotExtern(msg, src_loc, ty.childType(zcu), .element),
	25412	.vector => try sema.errNote(src_loc, msg, "vectors have no guaranteed in-memory representation", .{}),
25537	25413	.optional => try sema.errNote(src_loc, msg, "non-pointer optionals have no guaranteed in-memory representation", .{}),
25538	25414	}
25539	25415	}
...	...	@@ -25801,7 +25677,7 @@ fn addSafetyCheckSentinelMismatch(
25801	25677	.address_space = ptr_info.flags.address_space,
25802	25678	},
25803	25679	});
25804		const many_ptr = try parent_block.addBitCast(many_ptr_ty, ptr);
	25680	const many_ptr = try parent_block.addTyOp(.ptr_cast, many_ptr_ty, ptr);
25805	25681	break :s try parent_block.addBinOp(.ptr_elem_val, many_ptr, sentinel_index);
25806	25682	},
25807	25683	.many => unreachable,
...	...	@@ -26278,7 +26154,7 @@ fn fieldPtr(
26278	26154	},
26279	26155	.packed_offset = ptr_ptr_info.packed_offset,
26280	26156	});
26281		return sema.bitCast(block, result_ty, object_ptr, src, null);
	26157	return block.addTyOp(.ptr_cast, result_ty, object_ptr);
26282	26158	} else {
26283	26159	return sema.fail(
26284	26160	block,
...	...	@@ -26996,16 +26872,13 @@ fn unionFieldVal(
26996	26872	break :msg msg;
26997	26873	});
26998	26874	},
26999		.@"extern" => if (try sema.bitCastVal(union_val, field_ty, 0, 0, 0)) \|field_val\| {
	26875	.@"extern" => if (try sema.castMemory(union_val, field_ty, 0)) \|field_val\| {
27000	26876	return .fromValue(field_val);
27001	26877	} else {
27002	26878	// Runtime-known due to a pointer-to-integer conversion.
27003	26879	},
27004	26880	.@"packed" => {
27005		const field_val = try sema.bitCastVal(union_val, field_ty, 0, union_ty.bitSize(zcu), 0) orelse {
27006		unreachable; // `null` is only possible if the input value contains a pointer, which a packed union cannot.
27007		};
27008		return .fromValue(field_val);
	26881	return .fromValue(try sema.bitCastVal(union_val, field_ty));
27009	26882	},
27010	26883	}
27011	26884	}
...	...	@@ -27104,7 +26977,7 @@ fn elemPtrOneLayerOnly(
27104	26977
27105	26978	if (child_ty.abiSize(zcu) == 0) {
27106	26979	// zero-bit child type; just bitcast the pointer
27107		return block.addBitCast(result_ty, indexable);
	26980	return block.addTyOp(.ptr_cast, result_ty, indexable);
27108	26981	}
27109	26982
27110	26983	return block.addPtrElemPtr(indexable, elem_index, result_ty);
...	...	@@ -27401,68 +27274,36 @@ fn elemPtrVector(
27401	27274	}
27402	27275
27403	27276	const elem_ty = vector_ty.childType(zcu);
27404		const elem_bits = elem_ty.bitSize(zcu);
27405		// Exiting this block means the operation is a runtime one.
27406		const elem_ptr_ty: Type = if (elem_bits < 8 or !std.math.isPowerOfTwo(elem_bits)) elem_ptr_ty: {
27407		// Use a packed pointer (i.e. vector_index != 0)
27408		const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);
27409		const elem_ptr_ty = try pt.ptrType(.{
27410		.child = elem_ty.toIntern(),
27411		.flags = .{
27412		.size = .one,
27413		.alignment = vector_ptr_info.flags.alignment,
27414		.is_const = vector_ptr_info.flags.is_const,
27415		.is_volatile = vector_ptr_info.flags.is_volatile,
27416		.is_allowzero = vector_ptr_info.flags.is_allowzero,
27417		.address_space = vector_ptr_info.flags.address_space,
27418		.vector_index = @enumFromInt(index),
27419		},
27420		.packed_offset = .{
27421		.host_size = @intCast(vector_len),
27422		.bit_offset = 0,
27423		},
27424		});
27425		if (maybe_vector_ptr_val) \|ptr_val\| {
27426		if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);
27427		return .fromValue(try pt.getCoerced(ptr_val, elem_ptr_ty));
27428		}
27429		break :elem_ptr_ty elem_ptr_ty;
27430		} else elem_ptr_ty: {
27431		// Use a normal pointer (i.e. vector_index == 0)
27432		const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);
27433		const elem_ptr_ty = try pt.ptrType(.{
27434		.child = elem_ty.toIntern(),
27435		.flags = .{
27436		.size = .one,
27437		// TODO: this logic was ported from old code, but it's bogus. This entire block will
27438		// go away when https://github.com/ziglang/zig/issues/24061 is implemented anyway.
27439		.alignment = switch (vector_ptr_info.flags.alignment) {
27440		.none => .none,
27441		else => \|vec_align\| switch (index * elem_ty.abiSize(zcu)) {
27442		0 => vec_align,
27443		else => \|byte_offset\| .minStrict(vec_align, .fromLog2Units(@ctz(byte_offset))),
27444		},
27445		},
27446		.is_const = vector_ptr_info.flags.is_const,
27447		.is_volatile = vector_ptr_info.flags.is_volatile,
27448		.is_allowzero = vector_ptr_info.flags.is_allowzero,
27449		.address_space = vector_ptr_info.flags.address_space,
27450		},
27451		});
27452		if (maybe_vector_ptr_val) \|ptr_val\| {
27453		if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);
27454		const bit_offset = index * @divExact(elem_ty.bitSize(zcu), 8);
27455		return .fromValue(try ptr_val.getOffsetPtr(bit_offset, elem_ptr_ty, pt));
27456		}
27457		break :elem_ptr_ty elem_ptr_ty;
27458		};
	27277
	27278	const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);
	27279	const elem_ptr_ty = try pt.ptrType(.{
	27280	.child = elem_ty.toIntern(),
	27281	.flags = .{
	27282	.size = .one,
	27283	.alignment = vector_ptr_info.flags.alignment,
	27284	.is_const = vector_ptr_info.flags.is_const,
	27285	.is_volatile = vector_ptr_info.flags.is_volatile,
	27286	.is_allowzero = vector_ptr_info.flags.is_allowzero,
	27287	.address_space = vector_ptr_info.flags.address_space,
	27288	.vector_index = @enumFromInt(index),
	27289	},
	27290	.packed_offset = .{
	27291	.host_size = @intCast(vector_len),
	27292	.bit_offset = 0,
	27293	},
	27294	});
	27295
	27296	if (maybe_vector_ptr_val) \|ptr_val\| {
	27297	if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);
	27298	return .fromValue(try pt.getCoerced(ptr_val, elem_ptr_ty));
	27299	}
27459	27300
27460	27301	if (!init) {
27461	27302	try sema.validateRuntimeElemAccess(block, elem_index_src, elem_ty, vector_ptr_src);
27462	27303	try sema.validateRuntimeValue(block, vector_ptr_src, vector_ptr);
27463	27304	}
27464	27305
27465		return block.addPtrElemPtr(vector_ptr, elem_index, elem_ptr_ty);
	27306	return block.addTyOp(.ptr_cast, elem_ptr_ty, vector_ptr);
27466	27307	}
27467	27308
27468	27309	fn elemPtrSpirvRuntimeArray(
...	...	@@ -27544,7 +27385,7 @@ fn elemPtrArray(
27544	27385
27545	27386	if (array_ty.childType(zcu).abiSize(zcu) == 0) {
27546	27387	// zero-bit child type; just bitcast the pointer
27547		return block.addBitCast(elem_ptr_ty, array_ptr);
	27388	return block.addTyOp(.ptr_cast, elem_ptr_ty, array_ptr);
27548	27389	}
27549	27390
27550	27391	return block.addPtrElemPtr(array_ptr, elem_index, elem_ptr_ty);
...	...	@@ -27670,7 +27511,7 @@ fn elemPtrSlice(
27670	27511	if (elem_ty.abiSize(zcu) == 0) {
27671	27512	// zero-bit child type; just extract the pointer and bitcast it
27672	27513	const slice_ptr = try block.addTyOp(.slice_ptr, slice_ty.slicePtrFieldType(zcu), slice);
27673		return block.addBitCast(elem_ptr_ty, slice_ptr);
	27514	return block.addTyOp(.ptr_cast, elem_ptr_ty, slice_ptr);
27674	27515	}
27675	27516	return block.addSliceElemPtr(slice, elem_index, elem_ptr_ty);
27676	27517	}
...	...	@@ -27753,12 +27594,31 @@ fn coerceExtra(
27753	27594	var in_memory_result = try sema.coerceInMemoryAllowed(block, dest_ty, inst_ty, false, target, dest_ty_src, inst_src, maybe_inst_val);
27754	27595	if (in_memory_result == .ok) {
27755	27596	if (maybe_inst_val) \|val\| {
27756		return sema.coerceInMemory(val, dest_ty);
27757		}
27758		try sema.requireRuntimeBlock(block, inst_src, null);
27759		const new_val = try block.addBitCast(dest_ty, inst);
27760		try sema.checkKnownAllocPtr(block, inst, new_val);
27761		return new_val;
	27597	return .fromValue(try pt.getCoerced(val, dest_ty));
	27598	}
	27599	const coerced: Air.Inst.Ref = switch (in_memory_result.ok) {
	27600	.none => coerced: {
	27601	const @"addrspace" = target_util.defaultAddressSpace(zcu.getTarget(), .local);
	27602	const src_ptr_ty = try pt.ptrType(.{
	27603	.child = inst_ty.toIntern(),
	27604	.flags = .{ .size = .one, .address_space = @"addrspace" },
	27605	});
	27606	const dest_ptr_ty = try pt.ptrType(.{
	27607	.child = dest_ty.toIntern(),
	27608	.flags = .{ .size = .one, .address_space = @"addrspace" },
	27609	});
	27610	const ptr = try block.addTy(.alloc, src_ptr_ty);
	27611	_ = try block.addBinOp(.store_safe, ptr, inst);
	27612	const casted_ptr = try block.addTyOp(.ptr_cast, dest_ptr_ty, ptr);
	27613	break :coerced try block.addTyOp(.load, dest_ty, casted_ptr);
	27614	},
	27615	.same_type => unreachable, // we checked for equal types just above
	27616	.bit_cast => try block.addTyOp(.bit_cast, dest_ty, inst),
	27617	.ptr_cast => try block.addTyOp(.ptr_cast, dest_ty, inst),
	27618	.error_cast => try block.addTyOp(.error_cast, dest_ty, inst),
	27619	};
	27620	try sema.checkKnownAllocPtr(block, inst, coerced);
	27621	return coerced;
27762	27622	}
27763	27623
27764	27624	switch (dest_ty.zigTypeTag(zcu)) {
...	...	@@ -27872,8 +27732,8 @@ fn coerceExtra(
27872	27732
27873	27733	if (dest_info.sentinel != .none) {
27874	27734	if (array_ty.sentinel(zcu)) \|inst_sent\| {
27875		if (Air.internedToRef(dest_info.sentinel) !=
27876		try sema.coerceInMemory(inst_sent, dst_elem_type))
	27735	if (dest_info.sentinel !=
	27736	(try pt.getCoerced(inst_sent, dst_elem_type)).toIntern())
27877	27737	{
27878	27738	in_memory_result = .{ .ptr_sentinel = .{
27879	27739	.actual = inst_sent,
...	...	@@ -28067,8 +27927,8 @@ fn coerceExtra(
28067	27927	}
28068	27928
28069	27929	if (dest_info.sentinel == .none or inst_info.sentinel == .none or
28070		Air.internedToRef(dest_info.sentinel) !=
28071		try sema.coerceInMemory(Value.fromInterned(inst_info.sentinel), .fromInterned(dest_info.child)))
	27930	dest_info.sentinel !=
	27931	(try pt.getCoerced(.fromInterned(inst_info.sentinel), .fromInterned(dest_info.child))).toIntern())
28072	27932	break :p;
28073	27933
28074	27934	const slice_ptr = try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty);
...	...	@@ -28117,7 +27977,7 @@ fn coerceExtra(
28117	27977	(dst_info.signedness == .signed and dst_info.bits > src_info.bits))
28118	27978	{
28119	27979	try sema.requireRuntimeBlock(block, inst_src, null);
28120		return block.addTyOp(.intcast, dest_ty, inst);
	27980	return block.addTyOp(.int_cast, dest_ty, inst);
28121	27981	}
28122	27982	},
28123	27983	else => {},
...	...	@@ -28379,16 +28239,8 @@ fn coerceExtra(
28379	28239	return sema.failWithOwnedErrorMsg(block, msg);
28380	28240	}
28381	28241
28382		fn coerceInMemory(
28383		sema: *Sema,
28384		val: Value,
28385		dst_ty: Type,
28386		) CompileError!Air.Inst.Ref {
28387		return Air.internedToRef((try sema.pt.getCoerced(val, dst_ty)).toIntern());
28388		}
28389
28390	28242	const InMemoryCoercionResult = union(enum) {
28391		ok,
	28243	ok: Strategy,
28392	28244	no_match: Pair,
28393	28245	int_not_coercible: Int,
28394	28246	comptime_int_not_coercible: TypeValuePair,
...	...	@@ -28424,6 +28276,21 @@ const InMemoryCoercionResult = union(enum) {
28424	28276	double_ptr_to_anyopaque: Pair,
28425	28277	slice_to_anyopaque: Pair,
28426	28278
	28279	const Strategy = enum {
	28280	/// There isn't a special strategy for this particular coercion---we'll just need to
	28281	/// reinterpret the bytes in memory.
	28282	none,
	28283
	28284	/// The source and destination types are equal, so no explicit cast operation is necessary.
	28285	same_type,
	28286	/// The coercion can be lowered to `Air.Inst.Tag.bit_cast`.
	28287	bit_cast,
	28288	/// The coercion can be lowered to `Air.Inst.Tag.ptr_cast`.
	28289	ptr_cast,
	28290	/// The coercion can be lowered to `Air.Inst.Tag.error_cast`.
	28291	error_cast,
	28292	};
	28293
28427	28294	const Pair = struct {
28428	28295	actual: Type,
28429	28296	wanted: Type,
...	...	@@ -28797,7 +28664,7 @@ pub fn coerceInMemoryAllowed(
28797	28664	}
28798	28665
28799	28666	if (dest_ty.eql(src_ty))
28800		return .ok;
	28667	return .{ .ok = .same_type };
28801	28668
28802	28669	const dest_tag = dest_ty.zigTypeTag(zcu);
28803	28670	const src_tag = src_ty.zigTypeTag(zcu);
...	...	@@ -28810,7 +28677,7 @@ pub fn coerceInMemoryAllowed(
28810	28677	if (dest_info.signedness == src_info.signedness and
28811	28678	dest_info.bits == src_info.bits)
28812	28679	{
28813		return .ok;
	28680	return .{ .ok = .bit_cast };
28814	28681	}
28815	28682
28816	28683	if ((src_info.signedness == dest_info.signedness and dest_info.bits < src_info.bits) or
...	...	@@ -28818,7 +28685,7 @@ pub fn coerceInMemoryAllowed(
28818	28685	(dest_info.signedness == .signed and src_info.signedness == .unsigned and dest_info.bits <= src_info.bits) or
28819	28686	(dest_info.signedness == .unsigned and src_info.signedness == .signed))
28820	28687	{
28821		return InMemoryCoercionResult{ .int_not_coercible = .{
	28688	return .{ .int_not_coercible = .{
28822	28689	.actual_signedness = src_info.signedness,
28823	28690	.wanted_signedness = dest_info.signedness,
28824	28691	.actual_bits = src_info.bits,
...	...	@@ -28841,7 +28708,7 @@ pub fn coerceInMemoryAllowed(
28841	28708	const dest_bits = dest_ty.floatBits(target);
28842	28709	const src_bits = src_ty.floatBits(target);
28843	28710	if (dest_bits == src_bits) {
28844		return .ok;
	28711	return .{ .ok = .bit_cast };
28845	28712	}
28846	28713	}
28847	28714
...	...	@@ -28864,24 +28731,38 @@ pub fn coerceInMemoryAllowed(
28864	28731	if (dest_tag == .error_union and src_tag == .error_union) {
28865	28732	const dest_payload = dest_ty.errorUnionPayload(zcu);
28866	28733	const src_payload = src_ty.errorUnionPayload(zcu);
28867		const child = try sema.coerceInMemoryAllowed(block, dest_payload, src_payload, dest_is_mut, target, dest_src, src_src, null);
28868		if (child != .ok) {
28869		return .{ .error_union_payload = .{
28870		.child = try child.dupe(sema.arena),
	28734	const payload_strat = switch (try sema.coerceInMemoryAllowed(block, dest_payload, src_payload, dest_is_mut, target, dest_src, src_src, null)) {
	28735	.ok => \|strat\| strat,
	28736	else => \|payload_result\| return .{ .error_union_payload = .{
	28737	.child = try payload_result.dupe(sema.arena),
28871	28738	.actual = src_payload,
28872	28739	.wanted = dest_payload,
28873		} };
	28740	} },
	28741	};
	28742	switch (try sema.coerceInMemoryAllowed(block, dest_ty.errorUnionSet(zcu), src_ty.errorUnionSet(zcu), dest_is_mut, target, dest_src, src_src, null)) {
	28743	.ok => {},
	28744	else => \|err_set_result\| return err_set_result,
28874	28745	}
28875		return try sema.coerceInMemoryAllowed(block, dest_ty.errorUnionSet(zcu), src_ty.errorUnionSet(zcu), dest_is_mut, target, dest_src, src_src, null);
	28746	return switch (payload_strat) {
	28747	.same_type => .{ .ok = .error_cast },
	28748	else => .{ .ok = .none },
	28749	};
28876	28750	}
28877	28751
28878	28752	// Error Sets
28879	28753	if (dest_tag == .error_set and src_tag == .error_set) {
28880		const res1 = try sema.coerceInMemoryAllowedErrorSets(block, dest_ty, src_ty, dest_src, src_src);
28881		if (!dest_is_mut or res1 != .ok) return res1;
28882		// src -> dest is okay, but `dest_is_mut`, so it needs to be allowed in the other direction.
28883		const res2 = try sema.coerceInMemoryAllowedErrorSets(block, src_ty, dest_ty, src_src, dest_src);
28884		return res2;
	28754	switch (try sema.coerceInMemoryAllowedErrorSets(block, dest_ty, src_ty, dest_src, src_src)) {
	28755	.ok => \|strat\| assert(strat == .error_cast),
	28756	else => \|result\| return result,
	28757	}
	28758	if (dest_is_mut) {
	28759	// src -> dest is okay, but `dest_is_mut`, so it needs to be allowed in the other direction.
	28760	switch (try sema.coerceInMemoryAllowedErrorSets(block, src_ty, dest_ty, src_src, dest_src)) {
	28761	.ok => \|strat\| assert(strat == .error_cast),
	28762	else => \|result\| return result,
	28763	}
	28764	}
	28765	return .{ .ok = .error_cast };
28885	28766	}
28886	28767
28887	28768	// Arrays
...	...	@@ -28896,9 +28777,9 @@ pub fn coerceInMemoryAllowed(
28896	28777	}
28897	28778
28898	28779	const child = try sema.coerceInMemoryAllowed(block, dest_info.elem_type, src_info.elem_type, dest_is_mut, target, dest_src, src_src, null);
28899		switch (child) {
28900		.ok => {},
28901		.no_match => return child,
	28780	const child_strat = switch (child) {
	28781	.ok => \|strat\| strat,
	28782	.no_match => \|no_match\| return .{ .no_match = no_match },
28902	28783	else => {
28903	28784	return .{ .array_elem = .{
28904	28785	.child = try child.dupe(sema.arena),
...	...	@@ -28906,7 +28787,7 @@ pub fn coerceInMemoryAllowed(
28906	28787	.wanted = dest_info.elem_type,
28907	28788	} };
28908	28789	},
28909		}
	28790	};
28910	28791	const ok_sent = (dest_info.sentinel == null and src_info.sentinel == null) or
28911	28792	(src_info.sentinel != null and
28912	28793	dest_info.sentinel != null and
...	...	@@ -28922,7 +28803,10 @@ pub fn coerceInMemoryAllowed(
28922	28803	.ty = dest_info.elem_type,
28923	28804	} };
28924	28805	}
28925		return .ok;
	28806	return .{ .ok = switch (child_strat) {
	28807	.bit_cast => .bit_cast,
	28808	else => .none,
	28809	} };
28926	28810	}
28927	28811
28928	28812	// Vectors
...	...	@@ -28938,16 +28822,18 @@ pub fn coerceInMemoryAllowed(
28938	28822
28939	28823	const dest_elem_ty = dest_ty.scalarType(zcu);
28940	28824	const src_elem_ty = src_ty.scalarType(zcu);
28941		const child = try sema.coerceInMemoryAllowed(block, dest_elem_ty, src_elem_ty, dest_is_mut, target, dest_src, src_src, null);
28942		if (child != .ok) {
28943		return .{ .vector_elem = .{
28944		.child = try child.dupe(sema.arena),
	28825	switch (try sema.coerceInMemoryAllowed(block, dest_elem_ty, src_elem_ty, dest_is_mut, target, dest_src, src_src, null)) {
	28826	.ok => \|child_strat\| return .{ .ok = switch (child_strat) {
	28827	.bit_cast => .bit_cast,
	28828	.ptr_cast => .ptr_cast,
	28829	else => .none,
	28830	} },
	28831	else => \|child_result\| return .{ .vector_elem = .{
	28832	.child = try child_result.dupe(sema.arena),
28945	28833	.actual = src_elem_ty,
28946	28834	.wanted = dest_elem_ty,
28947		} };
	28835	} },
28948	28836	}
28949
28950		return .ok;
28951	28837	}
28952	28838
28953	28839	// Optionals
...	...	@@ -28971,7 +28857,7 @@ pub fn coerceInMemoryAllowed(
28971	28857	} };
28972	28858	}
28973	28859
28974		return .ok;
	28860	return .{ .ok = .none };
28975	28861	}
28976	28862
28977	28863	// Tuples (with in-memory-coercible fields)
...	...	@@ -28985,7 +28871,7 @@ pub fn coerceInMemoryAllowed(
28985	28871	const field = try sema.coerceInMemoryAllowed(block, dest_field_ty, src_field_ty, dest_is_mut, target, dest_src, src_src, null);
28986	28872	if (field != .ok) break :tuple;
28987	28873	}
28988		return .ok;
	28874	return .{ .ok = .none };
28989	28875	}
28990	28876
28991	28877	return .{ .no_match = .{
...	...	@@ -29008,13 +28894,13 @@ fn coerceInMemoryAllowedErrorSets(
29008	28894	const ip = &zcu.intern_pool;
29009	28895
29010	28896	const dest_set: InternPool.Key.ErrorSetType = err_set: switch (dest_ty.toIntern()) {
29011		.anyerror_type => return .ok,
	28897	.anyerror_type => return .{ .ok = .error_cast },
29012	28898	.adhoc_inferred_error_set_type => {
29013	28899	// We are trying to coerce an error set to the current function's
29014	28900	// inferred error set.
29015	28901	const dst_ies = sema.fn_ret_ty_ies.?;
29016	28902	try dst_ies.addErrorSet(src_ty, ip, sema.arena);
29017		return .ok;
	28903	return .{ .ok = .error_cast };
29018	28904	},
29019	28905	else => \|err_set_ty\| switch (ip.indexToKey(err_set_ty)) {
29020	28906	.inferred_error_set_type => \|func_index\| {
...	...	@@ -29023,7 +28909,7 @@ fn coerceInMemoryAllowedErrorSets(
29023	28909	// We are trying to coerce an error set to the current function's
29024	28910	// inferred error set.
29025	28911	try dst_ies.addErrorSet(src_ty, ip, sema.arena);
29026		return .ok;
	28912	return .{ .ok = .error_cast };
29027	28913	}
29028	28914	}
29029	28915	try sema.ensureFuncIesResolved(block, dest_src, func_index);
...	...	@@ -29062,7 +28948,7 @@ fn coerceInMemoryAllowedErrorSets(
29062	28948	) };
29063	28949	}
29064	28950
29065		return .ok;
	28951	return .{ .ok = .error_cast };
29066	28952	}
29067	28953
29068	28954	fn coerceInMemoryAllowedFns(
...	...	@@ -29179,7 +29065,7 @@ fn coerceInMemoryAllowedFns(
29179	29065	}
29180	29066	}
29181	29067
29182		return .ok;
	29068	return .{ .ok = .none };
29183	29069	}
29184	29070
29185	29071	fn callconvCoerceAllowed(
...	...	@@ -29256,7 +29142,7 @@ fn coerceInMemoryAllowedPtrs(
29256	29142	const ok_ptr_size = src_info.flags.size == dest_info.flags.size or
29257	29143	src_info.flags.size == .c or dest_info.flags.size == .c;
29258	29144	if (!ok_ptr_size) {
29259		return InMemoryCoercionResult{ .ptr_size = .{
	29145	return .{ .ptr_size = .{
29260	29146	.actual = src_info.flags.size,
29261	29147	.wanted = dest_info.flags.size,
29262	29148	} };
...	...	@@ -29392,14 +29278,14 @@ fn coerceInMemoryAllowedPtrs(
29392	29278	break :a dest_child.abiAlignment(zcu);
29393	29279	} else dest_info.flags.alignment;
29394	29280	if (dest_align.compare(if (dest_is_mut) .neq else .gt, src_align)) {
29395		return InMemoryCoercionResult{ .ptr_alignment = .{
	29281	return .{ .ptr_alignment = .{
29396	29282	.actual = src_align,
29397	29283	.wanted = dest_align,
29398	29284	} };
29399	29285	}
29400	29286	}
29401	29287
29402		return .ok;
	29288	return .{ .ok = .ptr_cast };
29403	29289	}
29404	29290
29405	29291	fn coerceVarArgParam(
...	...	@@ -29703,7 +29589,6 @@ fn bitCast(
29703	29589	dest_ty: Type,
29704	29590	inst: Air.Inst.Ref,
29705	29591	inst_src: LazySrcLoc,
29706		operand_src: ?LazySrcLoc,
29707	29592	) CompileError!Air.Inst.Ref {
29708	29593	const pt = sema.pt;
29709	29594	const zcu = pt.zcu;
...	...	@@ -29712,6 +29597,11 @@ fn bitCast(
29712	29597	old_ty.assertHasLayout(zcu);
29713	29598	try sema.ensureLayoutResolved(dest_ty, inst_src, .init);
29714	29599
	29600	assert(old_ty.hasBitRepresentation(zcu));
	29601	assert(dest_ty.hasBitRepresentation(zcu));
	29602	assert(old_ty.scalarType(zcu).zigTypeTag(zcu) != .pointer);
	29603	assert(dest_ty.scalarType(zcu).zigTypeTag(zcu) != .pointer);
	29604
29715	29605	const dest_bits = dest_ty.bitSize(zcu);
29716	29606	const old_bits = old_ty.bitSize(zcu);
29717	29607
...	...	@@ -29725,20 +29615,30 @@ fn bitCast(
29725	29615	}
29726	29616
29727	29617	if (sema.resolveValue(inst)) \|val\| {
29728		if (val.isUndef(zcu))
29729		return pt.undefRef(dest_ty);
29730		if (old_ty.zigTypeTag(zcu) == .error_set and dest_ty.zigTypeTag(zcu) == .error_set) {
29731		// Special case: we sometimes call `bitCast` on error set values, but they
29732		// don't have a well-defined layout, so we can't use `bitCastVal` on them.
29733		return Air.internedToRef((try pt.getCoerced(val, dest_ty)).toIntern());
29734		}
29735		if (try sema.bitCastVal(val, dest_ty, 0, 0, 0)) \|result_val\| {
29736		return Air.internedToRef(result_val.toIntern());
29737		}
	29618	return .fromValue(try sema.bitCastVal(val, dest_ty));
29738	29619	}
29739		try sema.requireRuntimeBlock(block, inst_src, operand_src);
29740	29620	try sema.validateRuntimeValue(block, inst_src, inst);
29741		return block.addBitCast(dest_ty, inst);
	29621	return block.addTyOp(.bit_cast, dest_ty, inst);
	29622	}
	29623
	29624	/// Supports only types which `@bitCast` supports, so pointers are not supported.
	29625	pub fn bitCastVal(
	29626	sema: *Sema,
	29627	val: Value,
	29628	dest_ty: Type,
	29629	) Allocator.Error!Value {
	29630	const pt = sema.pt;
	29631	const zcu = pt.zcu;
	29632	const bit_size = dest_ty.bitSize(zcu);
	29633	assert(val.typeOf(zcu).bitSize(zcu) == bit_size);
	29634	if (val.isUndef(zcu)) {
	29635	return pt.undefValue(dest_ty);
	29636	} else {
	29637	const buf = try sema.arena.alloc(u8, @intCast((bit_size + 7) / 8));
	29638	@memset(buf, 0);
	29639	val.writeToPackedMemory(zcu, buf, 0);
	29640	return .readFromPackedMemory(dest_ty, pt, buf, 0);
	29641	}
29742	29642	}
29743	29643
29744	29644	fn coerceArrayPtrToSlice(
...	...	@@ -29855,14 +29755,17 @@ fn coerceCompatiblePtrs(
29855	29755	);
29856	29756	}
29857	29757	try sema.requireRuntimeBlock(block, inst_src, null);
29858		const inst_allows_zero = inst_ty.zigTypeTag(zcu) != .pointer or inst_ty.ptrAllowsZero(zcu);
29859		if (block.wantSafety() and inst_allows_zero and !dest_ty.ptrAllowsZero(zcu)) {
	29758	const maybe_zero: bool = switch (inst_ty.toIntern()) {
	29759	.usize_type, .isize_type => true,
	29760	else => inst_ty.ptrAllowsZero(zcu),
	29761	};
	29762	if (block.wantSafety() and maybe_zero and !dest_ty.ptrAllowsZero(zcu)) {
29860	29763	try sema.checkLogicalPtrOperation(block, inst_src, inst_ty);
29861	29764	const actual_ptr = if (inst_ty.isSlice(zcu))
29862	29765	try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty)
29863	29766	else
29864	29767	inst;
29865		const ptr_int = try block.addBitCast(.usize, actual_ptr);
	29768	const ptr_int = try block.addTyOp(.int_from_ptr, .usize, actual_ptr);
29866	29769	const is_non_zero = try block.addBinOp(.cmp_neq, ptr_int, .zero_usize);
29867	29770	const ok = if (inst_ty.isSlice(zcu)) ok: {
29868	29771	const len = try sema.analyzeSliceLen(block, inst_src, inst);
...	...	@@ -29871,7 +29774,14 @@ fn coerceCompatiblePtrs(
29871	29774	} else is_non_zero;
29872	29775	try sema.addSafetyCheck(block, inst_src, ok, .cast_to_null);
29873	29776	}
29874		const new_ptr = try sema.bitCast(block, dest_ty, inst, inst_src, null);
	29777	const new_ptr: Air.Inst.Ref = switch (inst_ty.toIntern()) {
	29778	.usize_type => try block.addTyOp(.ptr_from_int, dest_ty, inst),
	29779	.isize_type => new_ptr: {
	29780	const usize_inst = try block.addTyOp(.bit_cast, .usize, inst);
	29781	break :new_ptr try block.addTyOp(.ptr_from_int, dest_ty, usize_inst);
	29782	},
	29783	else => try block.addTyOp(.ptr_cast, dest_ty, inst),
	29784	};
29875	29785	try sema.checkKnownAllocPtr(block, inst, new_ptr);
29876	29786	return new_ptr;
29877	29787	}
...	...	@@ -29968,7 +29878,7 @@ fn coerceEnumToUnion(
29968	29878	return .fromValue(opv);
29969	29879	} else {
29970	29880	// The union layout is just the tag, so we can bitcast the enum straight to the union.
29971		return block.addBitCast(union_ty, enum_tag);
	29881	return block.addTyOp(.union_from_enum, union_ty, enum_tag);
29972	29882	}
29973	29883	}
29974	29884
...	...	@@ -30017,18 +29927,6 @@ fn coerceArrayLike(
30017	29927	const inst_ty = sema.typeOf(inst);
30018	29928	const target = zcu.getTarget();
30019	29929
30020		// try coercion of the whole array
30021		const in_memory_result = try sema.coerceInMemoryAllowed(block, dest_ty, inst_ty, false, target, dest_ty_src, inst_src, null);
30022		if (in_memory_result == .ok) {
30023		if (sema.resolveValue(inst)) \|inst_val\| {
30024		// These types share the same comptime value representation.
30025		return sema.coerceInMemory(inst_val, dest_ty);
30026		}
30027		try sema.requireRuntimeBlock(block, inst_src, null);
30028		return block.addBitCast(dest_ty, inst);
30029		}
30030
30031		// otherwise, try element by element
30032	29930	const inst_len = inst_ty.arrayLen(zcu);
30033	29931	const dest_len = try sema.usizeCast(block, dest_ty_src, dest_ty.arrayLen(zcu));
30034	29932	if (dest_len != inst_len) {
...	...	@@ -30055,7 +29953,7 @@ fn coerceArrayLike(
30055	29953	(dst_info.signedness == .signed and dst_info.bits > src_info.bits))
30056	29954	{
30057	29955	try sema.requireRuntimeBlock(block, inst_src, null);
30058		return block.addTyOp(.intcast, dest_ty, inst);
	29956	return block.addTyOp(.int_cast, dest_ty, inst);
30059	29957	}
30060	29958	},
30061	29959	.float => if (inst_elem_ty.isRuntimeFloat()) {
...	...	@@ -30582,7 +30480,7 @@ fn analyzeRef(
30582	30480
30583	30481	// Cast to the constant pointer type. We do this directly rather than going via `coerce` to
30584	30482	// avoid errors in the `block.isComptime()` case.
30585		return block.addBitCast(ptr_type, alloc);
	30483	return block.addTyOp(.ptr_cast, ptr_type, alloc);
30586	30484	}
30587	30485
30588	30486	fn analyzeLoad(
...	...	@@ -31334,7 +31232,7 @@ fn analyzeSlice(
31334	31232
31335	31233	const opt_new_ptr_val = sema.resolveValue(new_ptr);
31336	31234	const new_ptr_val = opt_new_ptr_val orelse {
31337		const result = try block.addBitCast(return_ty, new_ptr);
	31235	const result = try block.addTyOp(.ptr_cast, return_ty, new_ptr);
31338	31236	if (block.wantSafety()) {
31339	31237	// requirement: slicing C ptr is non-null
31340	31238	if (ptr_ptr_child_ty.isCPtr(zcu)) {
...	...	@@ -34366,8 +34264,8 @@ pub fn flushExports(sema: *Sema) !void {
34366	34264	}
34367	34265	}
34368	34266
34369		pub const bitCastVal = @import("Sema/bitcast.zig").bitCast;
34370		pub const bitCastSpliceVal = @import("Sema/bitcast.zig").bitCastSplice;
	34267	pub const castMemory = @import("Sema/reinterpret.zig").castMemory;
	34268	pub const spliceMemory = @import("Sema/reinterpret.zig").spliceMemory;
34371	34269
34372	34270	const loadComptimePtr = @import("Sema/comptime_ptr_access.zig").loadComptimePtr;
34373	34271	const ComptimeLoadResult = @import("Sema/comptime_ptr_access.zig").ComptimeLoadResult;

src/Sema/LowerZon.zig+4-11

...	...	@@ -815,20 +815,15 @@ fn lowerStruct(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool
815	815	.@"packed" => result: {
816	816	const arena = self.sema.arena;
817	817	const buf = try arena.alloc(u8, @intCast((res_ty.bitSize(zcu) + 7) / 8));
	818	@memset(buf, 0);
818	819	var bit_offset: u16 = 0;
819	820	for (field_values) \|field_ip\| {
820	821	const field_val: Value = .fromInterned(field_ip);
821		field_val.writeToPackedMemory(zcu, buf, bit_offset) catch \|err\| switch (err) {
822		error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers
823		error.OutOfMemory => \|e\| return e,
824		};
	822	field_val.writeToPackedMemory(zcu, buf, bit_offset);
825	823	bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));
826	824	}
827	825	assert(bit_offset == res_ty.bitSize(zcu));
828		break :result Value.readFromPackedMemory(res_ty, pt, buf, 0, arena) catch \|err\| switch (err) {
829		error.IllDefinedMemoryLayout => unreachable, // bitpacks have well-defined layout
830		error.OutOfMemory => \|e\| return e,
831		};
	826	break :result try .readFromPackedMemory(res_ty, pt, buf, 0);
832	827	},
833	828	};
834	829	return result.toIntern();
...	...	@@ -981,9 +976,7 @@ fn lowerUnion(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool.
981	976	};
982	977	const result: Value = switch (union_info.layout) {
983	978	.auto, .@"extern" => try pt.unionValue(res_ty, tag, val),
984		.@"packed" => try self.sema.bitCastVal(val, res_ty, 0, 0, 0) orelse {
985		unreachable; // `null` is only possible if the input value contains a pointer, which a packed union cannot.
986		},
	979	.@"packed" => try self.sema.bitCastVal(val, res_ty),
987	980	};
988	981	return result.toIntern();
989	982	}

src/Sema/bitcast.zig deleted-774

...	...	@@ -1,774 +0,0 @@
1		//! This file contains logic for bit-casting arbitrary values at comptime, including splicing
2		//! bits together for comptime stores of bit-pointers. The strategy is to "flatten" values to
3		//! a sequence of values in packed memory, and then unflatten through a combination of special
4		//! cases (particularly for pointers and `undefined` values) and in-memory buffer reinterprets.
5		//!
6		//! This is a little awkward on big-endian targets, as non-packed datastructures (e.g. `extern struct`)
7		//! have their fields reversed when represented as packed memory on such targets.
8
9		/// If `host_bits` is `0`, attempts to convert the memory at offset
10		/// `byte_offset` into `val` to a non-packed value of type `dest_ty`,
11		/// ignoring `bit_offset`.
12		///
13		/// Otherwise, `byte_offset` is an offset in bytes into `val` to a
14		/// non-packed value consisting of `host_bits` bits. A value of type
15		/// `dest_ty` will be interpreted at a packed offset of `bit_offset`
16		/// into this value.
17		///
18		/// Returns `null` if the operation must be performed at runtime.
19		pub fn bitCast(
20		sema: *Sema,
21		val: Value,
22		dest_ty: Type,
23		byte_offset: u64,
24		host_bits: u64,
25		bit_offset: u64,
26		) CompileError!?Value {
27		return bitCastInner(sema, val, dest_ty, byte_offset, host_bits, bit_offset) catch \|err\| switch (err) {
28		error.ReinterpretDeclRef => return null,
29		error.IllDefinedMemoryLayout => unreachable,
30		error.Unimplemented => @panic("unimplemented bitcast"),
31		else => \|e\| return e,
32		};
33		}
34
35		/// Uses bitcasting to splice the value `splice_val` into `val`,
36		/// replacing overlapping bits and returning the modified value.
37		///
38		/// If `host_bits` is `0`, splices `splice_val` at an offset
39		/// `byte_offset` bytes into the virtual memory of `val`, ignoring
40		/// `bit_offset`.
41		///
42		/// Otherwise, `byte_offset` is an offset into bytes into `val` to
43		/// a non-packed value consisting of `host_bits` bits. The value
44		/// `splice_val` will be placed at a packed offset of `bit_offset`
45		/// into this value.
46		pub fn bitCastSplice(
47		sema: *Sema,
48		val: Value,
49		splice_val: Value,
50		byte_offset: u64,
51		host_bits: u64,
52		bit_offset: u64,
53		) CompileError!?Value {
54		return bitCastSpliceInner(sema, val, splice_val, byte_offset, host_bits, bit_offset) catch \|err\| switch (err) {
55		error.ReinterpretDeclRef => return null,
56		error.IllDefinedMemoryLayout => unreachable,
57		error.Unimplemented => @panic("unimplemented bitcast"),
58		else => \|e\| return e,
59		};
60		}
61
62		const BitCastError = CompileError \|\| error{ ReinterpretDeclRef, IllDefinedMemoryLayout, Unimplemented };
63
64		fn bitCastInner(
65		sema: *Sema,
66		val: Value,
67		dest_ty: Type,
68		byte_offset: u64,
69		host_bits: u64,
70		bit_offset: u64,
71		) BitCastError!Value {
72		const pt = sema.pt;
73		const zcu = pt.zcu;
74		const endian = zcu.getTarget().cpu.arch.endian();
75
76		if (dest_ty.toIntern() == val.typeOf(zcu).toIntern() and bit_offset == 0) {
77		return val;
78		}
79
80		const val_ty = val.typeOf(zcu);
81
82		val_ty.assertHasLayout(zcu);
83		dest_ty.assertHasLayout(zcu);
84
85		assert(val_ty.hasWellDefinedLayout(zcu));
86
87		const abi_pad_bits, const host_pad_bits = if (host_bits > 0)
88		.{ val_ty.abiSize(zcu) * 8 - host_bits, host_bits - val_ty.bitSize(zcu) }
89		else
90		.{ val_ty.abiSize(zcu) * 8 - val_ty.bitSize(zcu), 0 };
91
92		const skip_bits = switch (endian) {
93		.little => bit_offset + byte_offset * 8,
94		.big => if (host_bits > 0)
95		val_ty.abiSize(zcu) * 8 - byte_offset * 8 - host_bits + bit_offset
96		else
97		val_ty.abiSize(zcu) * 8 - byte_offset * 8 - dest_ty.bitSize(zcu),
98		};
99
100		var unpack: UnpackValueBits = .{
101		.pt = sema.pt,
102		.arena = sema.arena,
103		.skip_bits = skip_bits,
104		.remaining_bits = dest_ty.bitSize(zcu),
105		.unpacked = std.array_list.Managed(InternPool.Index).init(sema.arena),
106		};
107		switch (endian) {
108		.little => {
109		try unpack.add(val);
110		try unpack.padding(abi_pad_bits);
111		},
112		.big => {
113		try unpack.padding(abi_pad_bits);
114		try unpack.add(val);
115		},
116		}
117		try unpack.padding(host_pad_bits);
118
119		var pack: PackValueBits = .{
120		.pt = sema.pt,
121		.arena = sema.arena,
122		.unpacked = unpack.unpacked.items,
123		};
124		return pack.get(dest_ty);
125		}
126
127		fn bitCastSpliceInner(
128		sema: *Sema,
129		val: Value,
130		splice_val: Value,
131		byte_offset: u64,
132		host_bits: u64,
133		bit_offset: u64,
134		) BitCastError!Value {
135		const pt = sema.pt;
136		const zcu = pt.zcu;
137		const endian = zcu.getTarget().cpu.arch.endian();
138		const val_ty = val.typeOf(zcu);
139		const splice_val_ty = splice_val.typeOf(zcu);
140
141		val_ty.assertHasLayout(zcu);
142		splice_val_ty.assertHasLayout(zcu);
143
144		const splice_bits = splice_val_ty.bitSize(zcu);
145
146		const splice_offset = switch (endian) {
147		.little => bit_offset + byte_offset * 8,
148		.big => if (host_bits > 0)
149		val_ty.abiSize(zcu) * 8 - byte_offset * 8 - host_bits + bit_offset
150		else
151		val_ty.abiSize(zcu) * 8 - byte_offset * 8 - splice_bits,
152		};
153
154		assert(splice_offset + splice_bits <= val_ty.abiSize(zcu) * 8);
155
156		const abi_pad_bits, const host_pad_bits = if (host_bits > 0)
157		.{ val_ty.abiSize(zcu) * 8 - host_bits, host_bits - val_ty.bitSize(zcu) }
158		else
159		.{ val_ty.abiSize(zcu) * 8 - val_ty.bitSize(zcu), 0 };
160
161		var unpack: UnpackValueBits = .{
162		.pt = pt,
163		.arena = sema.arena,
164		.skip_bits = 0,
165		.remaining_bits = splice_offset,
166		.unpacked = std.array_list.Managed(InternPool.Index).init(sema.arena),
167		};
168		switch (endian) {
169		.little => {
170		try unpack.add(val);
171		try unpack.padding(abi_pad_bits);
172		},
173		.big => {
174		try unpack.padding(abi_pad_bits);
175		try unpack.add(val);
176		},
177		}
178		try unpack.padding(host_pad_bits);
179
180		unpack.remaining_bits = splice_bits;
181		try unpack.add(splice_val);
182
183		unpack.skip_bits = splice_offset + splice_bits;
184		unpack.remaining_bits = val_ty.abiSize(zcu) * 8 - splice_offset - splice_bits;
185		switch (endian) {
186		.little => {
187		try unpack.add(val);
188		try unpack.padding(abi_pad_bits);
189		},
190		.big => {
191		try unpack.padding(abi_pad_bits);
192		try unpack.add(val);
193		},
194		}
195		try unpack.padding(host_pad_bits);
196
197		var pack: PackValueBits = .{
198		.pt = pt,
199		.arena = sema.arena,
200		.unpacked = unpack.unpacked.items,
201		};
202		switch (endian) {
203		.little => {},
204		.big => try pack.padding(abi_pad_bits),
205		}
206		return pack.get(val_ty);
207		}
208
209		/// Recurses through struct fields, array elements, etc, to get a sequence of "primitive" values
210		/// which are bit-packed in memory to represent a single value. `unpacked` represents a series
211		/// of values in packed memory - therefore, on big-endian targets, the first element of this
212		/// list contains bits from the final byte of the value.
213		const UnpackValueBits = struct {
214		pt: Zcu.PerThread,
215		arena: Allocator,
216		skip_bits: u64,
217		remaining_bits: u64,
218		extra_bits: u64 = undefined,
219		unpacked: std.array_list.Managed(InternPool.Index),
220
221		fn add(unpack: *UnpackValueBits, val: Value) BitCastError!void {
222		const pt = unpack.pt;
223		const zcu = pt.zcu;
224		const endian = zcu.getTarget().cpu.arch.endian();
225		const ip = &zcu.intern_pool;
226
227		if (unpack.remaining_bits == 0) {
228		return;
229		}
230
231		const ty = val.typeOf(zcu);
232		const bit_size = ty.bitSize(zcu);
233
234		if (unpack.skip_bits >= bit_size) {
235		unpack.skip_bits -= bit_size;
236		return;
237		}
238
239		switch (ip.indexToKey(val.toIntern())) {
240		.int_type,
241		.ptr_type,
242		.array_type,
243		.vector_type,
244		.opt_type,
245		.anyframe_type,
246		.error_union_type,
247		.simple_type,
248		.struct_type,
249		.tuple_type,
250		.union_type,
251		.opaque_type,
252		.spirv_type,
253		.enum_type,
254		.func_type,
255		.error_set_type,
256		.inferred_error_set_type,
257		.@"extern",
258		.func,
259		.err,
260		.error_union,
261		.enum_literal,
262		.slice,
263		.memoized_call,
264		=> unreachable, // ill-defined layout or not real values
265
266		.undef,
267		.int,
268		.enum_tag,
269		.simple_value,
270		.float,
271		.ptr,
272		.opt,
273		=> try unpack.primitive(val),
274
275		.bitpack => \|bitpack\| try unpack.primitive(.fromInterned(bitpack.backing_int_val)),
276
277		.aggregate => switch (ty.zigTypeTag(zcu)) {
278		.vector => {
279		const len: usize = @intCast(ty.arrayLen(zcu));
280		for (0..len) \|i\| {
281		// We reverse vector elements in packed memory on BE targets.
282		const real_idx = switch (endian) {
283		.little => i,
284		.big => len - i - 1,
285		};
286		const elem_val = try val.elemValue(pt, real_idx);
287		try unpack.add(elem_val);
288		}
289		},
290		.array => {
291		// Each element is padded up to its ABI size. Padding bits are undefined.
292		// The final element does not have trailing padding.
293		// Elements are reversed in packed memory on BE targets.
294		const elem_ty = ty.childType(zcu);
295		const pad_bits = elem_ty.abiSize(zcu) * 8 - elem_ty.bitSize(zcu);
296		const len = ty.arrayLen(zcu);
297		const maybe_sent = ty.sentinel(zcu);
298
299		if (endian == .big) if (maybe_sent) \|s\| {
300		try unpack.add(s);
301		if (len != 0) try unpack.padding(pad_bits);
302		};
303
304		for (0..@intCast(len)) \|i\| {
305		// We reverse array elements in packed memory on BE targets.
306		const real_idx = switch (endian) {
307		.little => i,
308		.big => len - i - 1,
309		};
310		const elem_val = try val.elemValue(pt, @intCast(real_idx));
311		try unpack.add(elem_val);
312		if (i != len - 1) try unpack.padding(pad_bits);
313		}
314
315		if (endian == .little) if (maybe_sent) \|s\| {
316		if (len != 0) try unpack.padding(pad_bits);
317		try unpack.add(s);
318		};
319		},
320		.@"struct" => switch (ty.containerLayout(zcu)) {
321		.auto => unreachable, // ill-defined layout
322		.@"extern" => switch (endian) {
323		.little => {
324		var cur_bit_off: u64 = 0;
325		var it = zcu.typeToStruct(ty).?.iterateRuntimeOrder(ip);
326		while (it.next()) \|field_idx\| {
327		const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8;
328		const pad_bits = want_bit_off - cur_bit_off;
329		const field_val = try val.fieldValue(pt, field_idx);
330		try unpack.padding(pad_bits);
331		try unpack.add(field_val);
332		cur_bit_off = want_bit_off + field_val.typeOf(zcu).bitSize(zcu);
333		}
334		// Add trailing padding bits.
335		try unpack.padding(bit_size - cur_bit_off);
336		},
337		.big => {
338		var cur_bit_off: u64 = bit_size;
339		var it = zcu.typeToStruct(ty).?.iterateRuntimeOrderReverse(ip);
340		while (it.next()) \|field_idx\| {
341		const field_val = try val.fieldValue(pt, field_idx);
342		const field_ty = field_val.typeOf(zcu);
343		const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8 + field_ty.bitSize(zcu);
344		const pad_bits = cur_bit_off - want_bit_off;
345		try unpack.padding(pad_bits);
346		try unpack.add(field_val);
347		cur_bit_off = want_bit_off - field_ty.bitSize(zcu);
348		}
349		assert(cur_bit_off == 0);
350		},
351		},
352		.@"packed" => {
353		// Just add all fields in order. There are no padding bits.
354		// This is identical between LE and BE targets.
355		for (0..ty.structFieldCount(zcu)) \|i\| {
356		const field_val = try val.fieldValue(pt, i);
357		try unpack.add(field_val);
358		}
359		},
360		},
361		else => unreachable,
362		},
363
364		.un => \|un\| {
365		// We actually don't care about the tag here!
366		// Instead, we just need to write the payload value, plus any necessary padding.
367		// This correctly handles the case where `tag == .none`, since the payload is then
368		// either an integer or a byte array, both of which we can unpack.
369		const payload_val = Value.fromInterned(un.val);
370		const pad_bits = bit_size - payload_val.typeOf(zcu).bitSize(zcu);
371		if (endian == .little or ty.containerLayout(zcu) == .@"packed") {
372		try unpack.add(payload_val);
373		try unpack.padding(pad_bits);
374		} else {
375		try unpack.padding(pad_bits);
376		try unpack.add(payload_val);
377		}
378		},
379		}
380		}
381
382		fn padding(unpack: *UnpackValueBits, pad_bits: u64) BitCastError!void {
383		if (pad_bits == 0) return;
384		const pt = unpack.pt;
385		// Figure out how many full bytes and leftover bits there are.
386		const bytes = pad_bits / 8;
387		const bits = pad_bits % 8;
388		// Add undef u8 values for the bytes...
389		const undef_u8 = try pt.undefValue(Type.u8);
390		for (0..@intCast(bytes)) \|_\| {
391		try unpack.primitive(undef_u8);
392		}
393		// ...and an undef int for the leftover bits.
394		if (bits == 0) return;
395		const bits_ty = try pt.intType(.unsigned, @intCast(bits));
396		const bits_val = try pt.undefValue(bits_ty);
397		try unpack.primitive(bits_val);
398		}
399
400		fn primitive(unpack: *UnpackValueBits, val: Value) BitCastError!void {
401		const pt = unpack.pt;
402		const zcu = pt.zcu;
403
404		if (unpack.remaining_bits == 0) {
405		return;
406		}
407
408		const ty = val.typeOf(pt.zcu);
409		const bit_size = ty.bitSize(zcu);
410
411		// Note that this skips all zero-bit types.
412		if (unpack.skip_bits >= bit_size) {
413		unpack.skip_bits -= bit_size;
414		return;
415		}
416
417		if (unpack.skip_bits > 0) {
418		const skip = unpack.skip_bits;
419		unpack.skip_bits = 0;
420		return unpack.splitPrimitive(val, skip, bit_size - skip);
421		}
422
423		if (unpack.remaining_bits < bit_size) {
424		return unpack.splitPrimitive(val, 0, unpack.remaining_bits);
425		}
426
427		unpack.remaining_bits -\|= bit_size;
428
429		try unpack.unpacked.append(val.toIntern());
430		}
431
432		fn splitPrimitive(unpack: *UnpackValueBits, val: Value, bit_offset: u64, bit_count: u64) BitCastError!void {
433		const pt = unpack.pt;
434		const zcu = pt.zcu;
435		const ty = val.typeOf(pt.zcu);
436
437		const val_bits = ty.bitSize(zcu);
438		assert(bit_offset + bit_count <= val_bits);
439
440		switch (pt.zcu.intern_pool.indexToKey(val.toIntern())) {
441		// In the `ptr` case, this will return `error.ReinterpretDeclRef`
442		// if we're trying to split a non-integer pointer value.
443		.int, .float, .enum_tag, .ptr, .opt => {
444		// This @intCast is okay because no primitive can exceed the size of a u16.
445		const int_ty = try unpack.pt.intType(.unsigned, @intCast(bit_count));
446		const buf = try unpack.arena.alloc(u8, @intCast((val_bits + 7) / 8));
447		try val.writeToPackedMemory(zcu, buf, 0);
448		const sub_val = try Value.readFromPackedMemory(int_ty, unpack.pt, buf, @intCast(bit_offset), unpack.arena);
449		try unpack.primitive(sub_val);
450		},
451		.undef => try unpack.padding(bit_count),
452		// The only values here with runtime bits are `true` and `false.
453		// These are both 1 bit, so will never need truncating.
454		.simple_value => unreachable,
455		else => unreachable, // zero-bit or not primitives
456		}
457		}
458		};
459
460		/// Given a sequence of bit-packed values in packed memory (see `UnpackValueBits`),
461		/// reconstructs a value of an arbitrary type, with correct handling of `undefined`
462		/// values and of pointers which align in virtual memory.
463		const PackValueBits = struct {
464		pt: Zcu.PerThread,
465		arena: Allocator,
466		bit_offset: u64 = 0,
467		unpacked: []const InternPool.Index,
468
469		fn get(pack: *PackValueBits, ty: Type) BitCastError!Value {
470		const pt = pack.pt;
471		const zcu = pt.zcu;
472		const endian = zcu.getTarget().cpu.arch.endian();
473		const ip = &zcu.intern_pool;
474		const arena = pack.arena;
475		switch (ty.zigTypeTag(zcu)) {
476		.vector => {
477		// Elements are bit-packed.
478		const len = ty.arrayLen(zcu);
479		const elem_ty = ty.childType(zcu);
480		const elems = try arena.alloc(InternPool.Index, @intCast(len));
481		// We reverse vector elements in packed memory on BE targets.
482		switch (endian) {
483		.little => for (elems) \|*elem\| {
484		elem.* = (try pack.get(elem_ty)).toIntern();
485		},
486		.big => {
487		var i = elems.len;
488		while (i > 0) {
489		i -= 1;
490		elems[i] = (try pack.get(elem_ty)).toIntern();
491		}
492		},
493		}
494		return pt.aggregateValue(ty, elems);
495		},
496		.array => {
497		// Each element is padded up to its ABI size. The final element does not have trailing padding.
498		const len = ty.arrayLen(zcu);
499		const elem_ty = ty.childType(zcu);
500		const maybe_sent = ty.sentinel(zcu);
501		const pad_bits = elem_ty.abiSize(zcu) * 8 - elem_ty.bitSize(zcu);
502		const elems = try arena.alloc(InternPool.Index, @intCast(len));
503
504		if (endian == .big and maybe_sent != null) {
505		// TODO: validate sentinel was preserved!
506		try pack.padding(elem_ty.bitSize(zcu));
507		if (len != 0) try pack.padding(pad_bits);
508		}
509
510		for (0..elems.len) \|i\| {
511		const real_idx = switch (endian) {
512		.little => i,
513		.big => len - i - 1,
514		};
515		elems[@intCast(real_idx)] = (try pack.get(elem_ty)).toIntern();
516		if (i != len - 1) try pack.padding(pad_bits);
517		}
518
519		if (endian == .little and maybe_sent != null) {
520		// TODO: validate sentinel was preserved!
521		if (len != 0) try pack.padding(pad_bits);
522		try pack.padding(elem_ty.bitSize(zcu));
523		}
524
525		return pt.aggregateValue(ty, elems);
526		},
527		.@"struct" => switch (ty.containerLayout(zcu)) {
528		.auto => unreachable, // ill-defined layout
529		.@"extern" => {
530		const elems = try arena.alloc(InternPool.Index, ty.structFieldCount(zcu));
531		@memset(elems, .none);
532		switch (endian) {
533		.little => {
534		var cur_bit_off: u64 = 0;
535		var it = zcu.typeToStruct(ty).?.iterateRuntimeOrder(ip);
536		while (it.next()) \|field_idx\| {
537		const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8;
538		try pack.padding(want_bit_off - cur_bit_off);
539		const field_ty = ty.fieldType(field_idx, zcu);
540		elems[field_idx] = (try pack.get(field_ty)).toIntern();
541		cur_bit_off = want_bit_off + field_ty.bitSize(zcu);
542		}
543		try pack.padding(ty.bitSize(zcu) - cur_bit_off);
544		},
545		.big => {
546		var cur_bit_off: u64 = ty.bitSize(zcu);
547		var it = zcu.typeToStruct(ty).?.iterateRuntimeOrderReverse(ip);
548		while (it.next()) \|field_idx\| {
549		const field_ty = ty.fieldType(field_idx, zcu);
550		const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8 + field_ty.bitSize(zcu);
551		try pack.padding(cur_bit_off - want_bit_off);
552		elems[field_idx] = (try pack.get(field_ty)).toIntern();
553		cur_bit_off = want_bit_off - field_ty.bitSize(zcu);
554		}
555		assert(cur_bit_off == 0);
556		},
557		}
558		// Any fields which do not have runtime bits should be OPV or comptime fields.
559		// Fill those values now.
560		for (elems, 0..) \|*elem, field_idx\| {
561		if (elem.* != .none) continue;
562		const val = (try ty.structFieldValueComptime(pt, field_idx)).?;
563		elem.* = val.toIntern();
564		}
565		return pt.aggregateValue(ty, elems);
566		},
567		.@"packed" => {
568		const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
569		if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
570		return pt.bitpackValue(ty, backing_int_val);
571		},
572		},
573		.@"union" => switch (ty.containerLayout(zcu)) {
574		.auto => unreachable, // ill-defined layout
575		.@"extern" => {
576		// We will attempt to read as the backing representation. If this emits
577		// `error.ReinterpretDeclRef`, we will try each union field, preferring larger ones.
578		// We will also attempt smaller fields when we get `undefined`, as if some bits are
579		// defined we want to include them.
580		// TODO: this is very very bad. We need a more sophisticated union representation.
581
582		const prev_unpacked = pack.unpacked;
583		const prev_bit_offset = pack.bit_offset;
584
585		const backing_ty = try ty.externUnionBackingType(pt);
586
587		backing: {
588		const backing_val = pack.get(backing_ty) catch \|err\| switch (err) {
589		error.ReinterpretDeclRef => {
590		pack.unpacked = prev_unpacked;
591		pack.bit_offset = prev_bit_offset;
592		break :backing;
593		},
594		else => \|e\| return e,
595		};
596		if (backing_val.isUndef(zcu)) {
597		pack.unpacked = prev_unpacked;
598		pack.bit_offset = prev_bit_offset;
599		break :backing;
600		}
601		return Value.fromInterned(try pt.internUnion(.{
602		.ty = ty.toIntern(),
603		.tag = .none,
604		.val = backing_val.toIntern(),
605		}));
606		}
607
608		const field_order = try pack.arena.alloc(u32, ty.unionTagTypeHypothetical(zcu).enumFieldCount(zcu));
609		for (field_order, 0..) \|f, i\| f. = @intCast(i);
610		// Sort `field_order` to put the fields with the largest bit sizes first.
611		const SizeSortCtx = struct {
612		zcu: *Zcu,
613		field_types: []const InternPool.Index,
614		fn lessThan(ctx: @This(), a_idx: u32, b_idx: u32) bool {
615		const a_ty = Type.fromInterned(ctx.field_types[a_idx]);
616		const b_ty = Type.fromInterned(ctx.field_types[b_idx]);
617		return a_ty.bitSize(ctx.zcu) > b_ty.bitSize(ctx.zcu);
618		}
619		};
620		std.mem.sortUnstable(u32, field_order, SizeSortCtx{
621		.zcu = zcu,
622		.field_types = zcu.typeToUnion(ty).?.field_types.get(ip),
623		}, SizeSortCtx.lessThan);
624
625		const padding_after = endian == .little or ty.containerLayout(zcu) == .@"packed";
626
627		for (field_order) \|field_idx\| {
628		const field_ty = Type.fromInterned(zcu.typeToUnion(ty).?.field_types.get(ip)[field_idx]);
629		const pad_bits = ty.bitSize(zcu) - field_ty.bitSize(zcu);
630		if (!padding_after) try pack.padding(pad_bits);
631		const field_val = pack.get(field_ty) catch \|err\| switch (err) {
632		error.ReinterpretDeclRef => {
633		pack.unpacked = prev_unpacked;
634		pack.bit_offset = prev_bit_offset;
635		continue;
636		},
637		else => \|e\| return e,
638		};
639		if (padding_after) try pack.padding(pad_bits);
640		if (field_val.isUndef(zcu)) {
641		pack.unpacked = prev_unpacked;
642		pack.bit_offset = prev_bit_offset;
643		continue;
644		}
645		const tag_val = try pt.enumValueFieldIndex(ty.unionTagTypeHypothetical(zcu), field_idx);
646		return Value.fromInterned(try pt.internUnion(.{
647		.ty = ty.toIntern(),
648		.tag = tag_val.toIntern(),
649		.val = field_val.toIntern(),
650		}));
651		}
652
653		// No field could represent the value. Just do whatever happens when we try to read
654		// the backing type - either `undefined` or `error.ReinterpretDeclRef`.
655		const backing_val = try pack.get(backing_ty);
656		return Value.fromInterned(try pt.internUnion(.{
657		.ty = ty.toIntern(),
658		.tag = .none,
659		.val = backing_val.toIntern(),
660		}));
661		},
662		.@"packed" => {
663		const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
664		if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
665		return pt.bitpackValue(ty, backing_int_val);
666		},
667		},
668		else => return pack.primitive(ty),
669		}
670		}
671
672		fn padding(pack: *PackValueBits, pad_bits: u64) BitCastError!void {
673		_ = pack.prepareBits(pad_bits);
674		}
675
676		fn primitive(pack: *PackValueBits, want_ty: Type) BitCastError!Value {
677		const pt = pack.pt;
678		const zcu = pt.zcu;
679
680		if (try want_ty.onePossibleValue(pt)) \|opv\| return opv;
681
682		const vals, const bit_offset = pack.prepareBits(want_ty.bitSize(zcu));
683
684		for (vals) \|val\| {
685		if (!Value.fromInterned(val).isUndef(zcu)) break;
686		} else {
687		// All bits of the value are `undefined`.
688		return pt.undefValue(want_ty);
689		}
690
691		// TODO: we need to decide how to handle partially-undef values here.
692		// Currently, a value with some undefined bits becomes `0xAA` so that we
693		// preserve the well-defined bits, because we can't currently represent
694		// a partially-undefined primitive (e.g. an int with some undef bits).
695		// In future, we probably want to take one of these two routes:
696		// * Define that if any bits are `undefined`, the entire value is `undefined`.
697		// This is a major breaking change, and probably a footgun.
698		// * Introduce tracking for partially-undef values at comptime.
699		// This would complicate a lot of operations in Sema, such as basic
700		// arithmetic.
701		// This design complexity is tracked by #19634.
702
703		ptr_cast: {
704		if (vals.len != 1) break :ptr_cast;
705		const val = Value.fromInterned(vals[0]);
706		if (!val.typeOf(zcu).isPtrAtRuntime(zcu)) break :ptr_cast;
707		if (!want_ty.isPtrAtRuntime(zcu)) break :ptr_cast;
708		return pt.getCoerced(val, want_ty);
709		}
710
711		// Reinterpret via an in-memory buffer.
712
713		var buf_bits: u64 = 0;
714		for (vals) \|ip_val\| {
715		const val = Value.fromInterned(ip_val);
716		const ty = val.typeOf(pt.zcu);
717		buf_bits += ty.bitSize(zcu);
718		}
719
720		const buf = try pack.arena.alloc(u8, @intCast((buf_bits + 7) / 8));
721		// We will skip writing undefined values, so mark the buffer as `0xAA` so we get "undefined" bits.
722		@memset(buf, 0xAA);
723		var cur_bit_off: usize = 0;
724		for (vals) \|ip_val\| {
725		const val = Value.fromInterned(ip_val);
726		const ty = val.typeOf(zcu);
727		if (!val.isUndef(zcu)) {
728		try val.writeToPackedMemory(zcu, buf, cur_bit_off);
729		}
730		cur_bit_off += @intCast(ty.bitSize(zcu));
731		}
732
733		return Value.readFromPackedMemory(want_ty, pt, buf, @intCast(bit_offset), pack.arena);
734		}
735
736		fn prepareBits(pack: *PackValueBits, need_bits: u64) struct { []const InternPool.Index, u64 } {
737		if (need_bits == 0) return .{ &.{}, 0 };
738
739		const pt = pack.pt;
740		const zcu = pt.zcu;
741
742		var bits: u64 = 0;
743		var len: usize = 0;
744		while (bits < pack.bit_offset + need_bits) {
745		bits += Value.fromInterned(pack.unpacked[len]).typeOf(pt.zcu).bitSize(zcu);
746		len += 1;
747		}
748
749		const result_vals = pack.unpacked[0..len];
750		const result_offset = pack.bit_offset;
751
752		const extra_bits = bits - pack.bit_offset - need_bits;
753		if (extra_bits == 0) {
754		pack.unpacked = pack.unpacked[len..];
755		pack.bit_offset = 0;
756		} else {
757		pack.unpacked = pack.unpacked[len - 1 ..];
758		pack.bit_offset = Value.fromInterned(pack.unpacked[0]).typeOf(pt.zcu).bitSize(zcu) - extra_bits;
759		}
760
761		return .{ result_vals, result_offset };
762		}
763		};
764
765		const std = @import("std");
766		const Allocator = std.mem.Allocator;
767		const assert = std.debug.assert;
768
769		const Sema = @import("../Sema.zig");
770		const Zcu = @import("../Zcu.zig");
771		const InternPool = @import("../InternPool.zig");
772		const Type = @import("../Type.zig");
773		const Value = @import("../Value.zig");
774		const CompileError = Zcu.CompileError;

src/Sema/comptime_ptr_access.zig+173-148

...	...	@@ -14,27 +14,46 @@ pub const ComptimeLoadResult = union(enum) {
14	14	pub fn loadComptimePtr(sema: Sema, block: Block, src: LazySrcLoc, ptr: Value) !ComptimeLoadResult {
15	15	const pt = sema.pt;
16	16	const zcu = pt.zcu;
	17
17	18	const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);
18		// TODO: host size for vectors is terrible
19		const host_bits = switch (ptr_info.flags.vector_index) {
20		.none => ptr_info.packed_offset.host_size * 8,
21		else => ptr_info.packed_offset.host_size * Type.fromInterned(ptr_info.child).bitSize(zcu),
22		};
23		const bit_offset = if (host_bits != 0) bit_offset: {
24		const child_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);
25		const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
26		.none => 0,
27		else => \|idx\| switch (pt.zcu.getTarget().cpu.arch.endian()) {
28		.little => child_bits * @intFromEnum(idx),
29		.big => host_bits - child_bits * (@intFromEnum(idx) + 1), // element order reversed on big endian
30		},
31		};
32		if (child_bits + bit_offset > host_bits) {
	19	const elem_ty: Type = .fromInterned(ptr_info.child);
	20	const host_size = ptr_info.packed_offset.host_size;
	21
	22	if (host_size == 0) {
	23	return loadComptimePtrInner(sema, block, src, ptr, elem_ty, 0);
	24	}
	25
	26	assert(elem_ty.hasBitRepresentation(zcu));
	27	if (ptr_info.flags.vector_index == .none) {
	28	if (ptr_info.packed_offset.bit_offset + elem_ty.bitSize(zcu) > host_size * 8) {
33	29	return .exceeds_host_size;
34	30	}
35		break :bit_offset bit_offset;
36		} else 0;
37		return loadComptimePtrInner(sema, block, src, ptr, bit_offset, host_bits, Type.fromInterned(ptr_info.child), 0);
	31	const load_ty: Type = try pt.intType(.unsigned, host_size * 8);
	32	const backing_int_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, load_ty, 0)) {
	33	else => \|result\| return result,
	34	.success => \|mv\| mv,
	35	};
	36	const backing_int_val = try backing_int_mv.intern(pt, sema.arena);
	37	const buf = try sema.arena.alloc(u8, host_size);
	38	@memset(buf, 0);
	39	backing_int_val.writeToPackedMemory(zcu, buf, 0);
	40	const result_val: Value = try .readFromPackedMemory(elem_ty, pt, buf, ptr_info.packed_offset.bit_offset);
	41	return .{ .success = .{ .interned = result_val.toIntern() } };
	42	}
	43	if (@intFromEnum(ptr_info.flags.vector_index) >= host_size) {
	44	return .exceeds_host_size;
	45	}
	46	const load_ty: Type = try pt.vectorType(.{
	47	.len = host_size,
	48	.child = elem_ty.toIntern(),
	49	});
	50	const vector_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, load_ty, 0)) {
	51	else => \|result\| return result,
	52	.success => \|mv\| mv,
	53	};
	54	const vector_val = try vector_mv.intern(pt, sema.arena);
	55	const result_val = try vector_val.elemValue(pt, @intFromEnum(ptr_info.flags.vector_index));
	56	return .{ .success = .{ .interned = result_val.toIntern() } };
38	57	}
39	58
40	59	pub const ComptimeStoreResult = union(enum) {
...	...	@@ -52,7 +71,8 @@ pub const ComptimeStoreResult = union(enum) {
52	71	};
53	72
54	73	/// Perform a comptime load of value `store_val` to a pointer.
55		/// The pointer's type is ignored.
	74	///
	75	/// Asserts that the type of `store_val` equals the element type of the pointer type.
56	76	pub fn storeComptimePtr(
57	77	sema: *Sema,
58	78	block: *Block,
...	...	@@ -62,42 +82,84 @@ pub fn storeComptimePtr(
62	82	) !ComptimeStoreResult {
63	83	const pt = sema.pt;
64	84	const zcu = pt.zcu;
65		const ptr_info = ptr.typeOf(zcu).ptrInfo(zcu);
66		assert(store_val.typeOf(zcu).toIntern() == ptr_info.child);
67	85
68		{
69		const store_ty: Type = .fromInterned(ptr_info.child);
70		if (!store_ty.comptimeOnly(zcu) and !store_ty.hasRuntimeBits(zcu)) {
71		// zero-bit store; nothing to do
72		return .success;
73		}
	86	const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);
	87	const elem_ty: Type = .fromInterned(ptr_info.child);
	88	const host_size = ptr_info.packed_offset.host_size;
	89	assert(store_val.typeOf(zcu).toIntern() == elem_ty.toIntern());
	90
	91	if (host_size == 0) {
	92	return storeComptimePtrInner(sema, block, src, ptr, store_val);
74	93	}
75	94
76		// TODO: host size for vectors is terrible
77		const host_bits = switch (ptr_info.flags.vector_index) {
78		.none => ptr_info.packed_offset.host_size * 8,
79		else => ptr_info.packed_offset.host_size * Type.fromInterned(ptr_info.child).bitSize(zcu),
80		};
81		const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
82		.none => 0,
83		else => \|idx\| switch (zcu.getTarget().cpu.arch.endian()) {
84		.little => Type.fromInterned(ptr_info.child).bitSize(zcu) * @intFromEnum(idx),
85		.big => host_bits - Type.fromInterned(ptr_info.child).bitSize(zcu) * (@intFromEnum(idx) + 1), // element order reversed on big endian
86		},
87		};
88		const pseudo_store_ty = if (host_bits > 0) t: {
89		const need_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);
90		if (need_bits + bit_offset > host_bits) {
	95	assert(elem_ty.hasBitRepresentation(zcu));
	96	if (ptr_info.flags.vector_index == .none) {
	97	if (ptr_info.packed_offset.bit_offset + elem_ty.bitSize(zcu) > host_size * 8) {
91	98	return .exceeds_host_size;
92	99	}
93		break :t try sema.pt.intType(.unsigned, @intCast(host_bits));
94		} else Type.fromInterned(ptr_info.child);
	100	const backing_ty: Type = try pt.intType(.unsigned, host_size * 8);
	101	const backing_int_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, backing_ty, 0)) {
	102	.success => \|mv\| mv,
	103	.runtime_load => return .runtime_store,
	104	inline else => \|payload, tag\| return @unionInit(ComptimeStoreResult, @tagName(tag), payload),
	105	};
	106	const old_backing_int_val = try backing_int_mv.intern(pt, sema.arena);
	107	const buf = try sema.arena.alloc(u8, host_size);
	108	@memset(buf, 0);
	109	old_backing_int_val.writeToPackedMemory(zcu, buf, 0);
	110	// Write the new element...
	111	store_val.writeToPackedMemory(zcu, buf, ptr_info.packed_offset.bit_offset);
	112	// ...then read the resulting backing integer value...
	113	const new_backing_int_val: Value = try .readFromPackedMemory(backing_ty, pt, buf, 0);
	114	// ...and store that back into memory
	115	return storeComptimePtrInner(sema, block, src, ptr, new_backing_int_val);
	116	}
	117
	118	if (@intFromEnum(ptr_info.flags.vector_index) >= host_size) {
	119	return .exceeds_host_size;
	120	}
	121	const vec_ty: Type = try pt.vectorType(.{
	122	.len = host_size,
	123	.child = elem_ty.toIntern(),
	124	});
	125	const vector_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, vec_ty, 0)) {
	126	.success => \|mv\| mv,
	127	.runtime_load => return .runtime_store,
	128	inline else => \|payload, tag\| return @unionInit(ComptimeStoreResult, @tagName(tag), payload),
	129	};
	130	const old_vector_val = try vector_mv.intern(pt, sema.arena);
	131	const elems_buf = try sema.arena.alloc(InternPool.Index, host_size);
	132	for (elems_buf, 0..) \|*elem, elem_index\| {
	133	const elem_val = try old_vector_val.elemValue(pt, elem_index);
	134	elem.* = elem_val.toIntern();
	135	}
	136	elems_buf[@intFromEnum(ptr_info.flags.vector_index)] = store_val.toIntern();
	137	const new_vector_val = try pt.aggregateValue(vec_ty, elems_buf);
	138	return storeComptimePtrInner(sema, block, src, ptr, new_vector_val);
	139	}
95	140
96		const strat = try prepareComptimePtrStore(sema, block, src, ptr, pseudo_store_ty, 0);
	141	/// Like `storeComptimePtr`, except ignores the type of `ptr`, instead treating it as a single-item
	142	/// pointer to `store_val.typeOf(zcu)`.
	143	fn storeComptimePtrInner(
	144	sema: *Sema,
	145	block: *Block,
	146	src: LazySrcLoc,
	147	ptr: Value,
	148	store_val: Value,
	149	) !ComptimeStoreResult {
	150	const pt = sema.pt;
	151	const zcu = pt.zcu;
	152	const store_ty = store_val.typeOf(zcu);
	153
	154	if (store_ty.classify(zcu) == .one_possible_value) {
	155	// zero-bit store; nothing to do
	156	return .success;
	157	}
	158
	159	const strat = try prepareComptimePtrStore(sema, block, src, ptr, store_ty, 0);
97	160
98	161	// Propagate errors and handle comptime fields.
99	162	switch (strat) {
100		.direct, .index, .flat_index, .reinterpret => {},
101	163	.comptime_field => {
102	164	// To "store" to a comptime field, just perform a load of the field
103	165	// and see if the store value matches.
...	...	@@ -125,79 +187,60 @@ pub fn storeComptimePtr(
125	187	.inactive_union_field => return .inactive_union_field,
126	188	.needed_well_defined => \|ty\| return .{ .needed_well_defined = ty },
127	189	.out_of_bounds => \|ty\| return .{ .out_of_bounds = ty },
128		}
129
130		// Check the store is not inside a runtime condition
131		try checkComptimeVarStore(sema, block, src, strat.alloc());
132
133		if (host_bits == 0) {
134		// We can attempt a direct store depending on the strategy.
135		switch (strat) {
136		.direct => \|direct\| {
137		const want_ty = direct.val.typeOf(zcu);
138		const coerced_store_val = try pt.getCoerced(store_val, want_ty);
139		direct.val.* = .{ .interned = coerced_store_val.toIntern() };
140		return .success;
141		},
142		.index => \|index\| {
143		const want_ty = index.val.typeOf(zcu).childType(zcu);
144		const coerced_store_val = try pt.getCoerced(store_val, want_ty);
145		try index.val.setElem(pt, sema.arena, @intCast(index.elem_index), .{ .interned = coerced_store_val.toIntern() });
146		return .success;
147		},
148		.flat_index => \|flat\| {
149		const store_elems = store_val.typeOf(zcu).arrayBase(zcu)[1];
150		const flat_elems = try sema.arena.alloc(InternPool.Index, @intCast(store_elems));
151		{
152		var next_idx: u64 = 0;
153		var skip: u64 = 0;
154		try flattenArray(sema, .{ .interned = store_val.toIntern() }, &skip, &next_idx, flat_elems);
155		}
156		for (flat_elems, 0..) \|elem, idx\| {
157		// TODO: recursiveIndex in a loop does a lot of redundant work!
158		// Better would be to gather all the store targets into an array.
159		var index: u64 = flat.flat_elem_index + idx;
160		const val_ptr, const final_idx = (try recursiveIndex(sema, flat.val, &index)).?;
161		try val_ptr.setElem(pt, sema.arena, @intCast(final_idx), .{ .interned = elem });
162		}
163		return .success;
164		},
165		.reinterpret => {},
166		else => unreachable,
167		}
168		}
169	190
170		// Either there is a bit offset, or the strategy required reinterpreting.
171		// Therefore, we must perform a bitcast.
	191	.direct => \|direct\| {
	192	try checkComptimeVarStore(sema, block, src, direct.alloc);
	193	const want_ty = direct.val.typeOf(zcu);
	194	const coerced_store_val = try pt.getCoerced(store_val, want_ty);
	195	direct.val.* = .{ .interned = coerced_store_val.toIntern() };
	196	return .success;
	197	},
172	198
173		const val_ptr: *MutableValue, const byte_offset: u64 = switch (strat) {
174		.direct => \|direct\| .{ direct.val, 0 },
175		.index => \|index\| .{
176		index.val,
177		index.elem_index * index.val.typeOf(zcu).childType(zcu).abiSize(zcu),
	199	.index => \|index\| {
	200	try checkComptimeVarStore(sema, block, src, index.alloc);
	201	const want_ty = index.val.typeOf(zcu).childType(zcu);
	202	const coerced_store_val = try pt.getCoerced(store_val, want_ty);
	203	try index.val.setElem(pt, sema.arena, @intCast(index.elem_index), .{ .interned = coerced_store_val.toIntern() });
	204	return .success;
178	205	},
179		.flat_index => \|flat\| .{ flat.val, flat.flat_elem_index * flat.val.typeOf(zcu).arrayBase(zcu)[0].abiSize(zcu) },
180		.reinterpret => \|reinterpret\| .{ reinterpret.val, reinterpret.byte_offset },
181		else => unreachable,
182		};
183	206
184		if (!val_ptr.typeOf(zcu).hasWellDefinedLayout(zcu)) {
185		return .{ .needed_well_defined = val_ptr.typeOf(zcu) };
186		}
	207	.flat_index => \|flat\| {
	208	try checkComptimeVarStore(sema, block, src, flat.alloc);
	209	const store_elems = store_val.typeOf(zcu).arrayBase(zcu)[1];
	210	const flat_elems = try sema.arena.alloc(InternPool.Index, @intCast(store_elems));
	211	{
	212	var next_idx: u64 = 0;
	213	var skip: u64 = 0;
	214	try flattenArray(sema, .{ .interned = store_val.toIntern() }, &skip, &next_idx, flat_elems);
	215	}
	216	for (flat_elems, 0..) \|elem, idx\| {
	217	// TODO: recursiveIndex in a loop does a lot of redundant work!
	218	// Better would be to gather all the store targets into an array.
	219	var index: u64 = flat.flat_elem_index + idx;
	220	const val_ptr, const final_idx = (try recursiveIndex(sema, flat.val, &index)).?;
	221	try val_ptr.setElem(pt, sema.arena, @intCast(final_idx), .{ .interned = elem });
	222	}
	223	return .success;
	224	},
187	225
188		if (!store_val.typeOf(zcu).hasWellDefinedLayout(zcu)) {
189		return .{ .needed_well_defined = store_val.typeOf(zcu) };
	226	.reinterpret => \|reinterpret\| {
	227	try checkComptimeVarStore(sema, block, src, reinterpret.alloc);
	228	if (!reinterpret.val.typeOf(zcu).hasWellDefinedLayout(zcu)) {
	229	return .{ .needed_well_defined = reinterpret.val.typeOf(zcu) };
	230	}
	231	if (!store_ty.hasWellDefinedLayout(zcu)) {
	232	return .{ .needed_well_defined = store_ty };
	233	}
	234	const old_val = try reinterpret.val.intern(pt, sema.arena);
	235	const new_val = try sema.spliceMemory(
	236	old_val,
	237	store_val,
	238	reinterpret.byte_offset,
	239	) orelse return .runtime_store;
	240	reinterpret.val.* = .{ .interned = new_val.toIntern() };
	241	return .success;
	242	},
190	243	}
191
192		const new_val = try sema.bitCastSpliceVal(
193		try val_ptr.intern(pt, sema.arena),
194		store_val,
195		byte_offset,
196		host_bits,
197		bit_offset,
198		) orelse return .runtime_store;
199		val_ptr.* = .{ .interned = new_val.toIntern() };
200		return .success;
201	244	}
202	245
203	246	/// Perform a comptime load of type `load_ty` from a pointer.
...	...	@@ -207,8 +250,6 @@ fn loadComptimePtrInner(
207	250	block: *Block,
208	251	src: LazySrcLoc,
209	252	ptr_val: Value,
210		bit_offset: u64,
211		host_bits: u64,
212	253	load_ty: Type,
213	254	/// If `load_ty` is an array, this is the number of array elements to skip
214	255	/// before `load_ty`. Otherwise, it is ignored and may be `undefined`.
...	...	@@ -244,7 +285,7 @@ fn loadComptimePtrInner(
244	285	.eu_payload => \|base_ptr_ip\| val: {
245	286	const base_ptr = Value.fromInterned(base_ptr_ip);
246	287	const base_ty = base_ptr.typeOf(zcu).childType(zcu);
247		switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {
	288	switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
248	289	.success => \|eu_val\| switch (eu_val.unpackErrorUnion(zcu)) {
249	290	.undef => return .undef,
250	291	.err => \|err\| return .{ .err_payload = err },
...	...	@@ -256,7 +297,7 @@ fn loadComptimePtrInner(
256	297	.opt_payload => \|base_ptr_ip\| val: {
257	298	const base_ptr = Value.fromInterned(base_ptr_ip);
258	299	const base_ty = base_ptr.typeOf(zcu).childType(zcu);
259		switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {
	300	switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
260	301	.success => \|eu_val\| switch (eu_val.unpackOptional(zcu)) {
261	302	.undef => return .undef,
262	303	.null => return .null_payload,
...	...	@@ -283,7 +324,7 @@ fn loadComptimePtrInner(
283	324	.child = base_ty.toIntern(),
284	325	});
285	326
286		switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, want_ty, base_index.index)) {
	327	switch (try loadComptimePtrInner(sema, block, src, base_ptr, want_ty, base_index.index)) {
287	328	.success => \|arr_val\| break :val arr_val,
288	329	else => \|err\| return err,
289	330	}
...	...	@@ -293,7 +334,7 @@ fn loadComptimePtrInner(
293	334	const base_ty = base_ptr.typeOf(zcu).childType(zcu);
294	335
295	336	// Field of a slice, or of an auto-layout struct or union.
296		const agg_val = switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {
	337	const agg_val = switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
297	338	.success => \|val\| val,
298	339	else => \|err\| return err,
299	340	};
...	...	@@ -324,7 +365,7 @@ fn loadComptimePtrInner(
324	365	},
325	366	};
326	367
327		if (ptr.byte_offset == 0 and host_bits == 0) {
	368	if (ptr.byte_offset == 0) {
328	369	if (load_ty.zigTypeTag(zcu) != .array or array_offset == 0) {
329	370	if (.ok == try sema.coerceInMemoryAllowed(
330	371	block,
...	...	@@ -343,8 +384,6 @@ fn loadComptimePtrInner(
343	384	}
344	385
345	386	restructure_array: {
346		if (host_bits != 0) break :restructure_array;
347
348	387	// We might also be changing the length of an array, or restructuring it.
349	388	// e.g. [1][2][3]T -> [3][2]T.
350	389	// This case is important because it's permitted for types with ill-defined layouts.
...	...	@@ -402,7 +441,7 @@ fn loadComptimePtrInner(
402	441	cur_offset += load_ty.childType(zcu).abiSize(zcu) * array_offset;
403	442	}
404	443
405		const need_bytes = if (host_bits > 0) (host_bits + 7) / 8 else load_ty.abiSize(zcu);
	444	const need_bytes = load_ty.abiSize(zcu);
406	445
407	446	if (cur_offset + need_bytes > cur_val.typeOf(zcu).abiSize(zcu)) {
408	447	return .{ .out_of_bounds = cur_val.typeOf(zcu) };
...	...	@@ -453,7 +492,7 @@ fn loadComptimePtrInner(
453	492	},
454	493	.@"struct" => switch (cur_ty.containerLayout(zcu)) {
455	494	.auto => unreachable, // ill-defined layout
456		.@"packed" => break, // let the bitcast logic handle this
	495	.@"packed" => break, // let the memory reinterpret logic handle this
457	496	.@"extern" => for (0..cur_ty.structFieldCount(zcu)) \|field_idx\| {
458	497	const start_off = cur_ty.structFieldOffset(field_idx, zcu);
459	498	const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);
...	...	@@ -466,9 +505,9 @@ fn loadComptimePtrInner(
466	505	},
467	506	.@"union" => switch (cur_ty.containerLayout(zcu)) {
468	507	.auto => unreachable, // ill-defined layout
469		.@"packed" => break, // let the bitcast logic handle this
	508	.@"packed" => break, // let the memory reinterpret logic handle this
470	509	.@"extern" => {
471		// TODO: we have to let bitcast logic handle this for now.
	510	// TODO: we have to let the memory reinterpret logic handle this for now.
472	511	// Otherwise, we might traverse into a union field which doesn't allow pointers.
473	512	// Figure out a solution!
474	513	if (true) break;
...	...	@@ -495,27 +534,13 @@ fn loadComptimePtrInner(
495	534
496	535	// Fast path: check again if we're now at the type we want to load.
497	536	// If so, just return the loaded value.
498		if (cur_offset == 0 and host_bits == 0 and cur_val.typeOf(zcu).toIntern() == load_ty.toIntern()) {
	537	if (cur_offset == 0 and cur_val.typeOf(zcu).toIntern() == load_ty.toIntern()) {
499	538	return .{ .success = cur_val };
500	539	}
501	540
502		var bitcast_src_val = try cur_val.intern(sema.pt, sema.arena);
503
504		if (host_bits != 0) {
505		const src_bit_size = bitcast_src_val.typeOf(zcu).bitSize(zcu);
506		if (src_bit_size > host_bits) {
507		const truncate_ty = try pt.intType(.unsigned, @intCast(host_bits));
508		bitcast_src_val = try pt.getCoerced(bitcast_src_val, truncate_ty);
509		}
510		}
511
512		const result_val = try sema.bitCastVal(
513		bitcast_src_val,
514		load_ty,
515		cur_offset,
516		host_bits,
517		bit_offset,
518		) orelse return .runtime_load;
	541	// Otherwise, use the memory reinterpretation logic to pull out the bytes we need.
	542	const reinterpret_val = try cur_val.intern(pt, sema.arena);
	543	const result_val = try sema.castMemory(reinterpret_val, load_ty, cur_offset) orelse return .runtime_load;
519	544	return .{ .success = .{ .interned = result_val.toIntern() } };
520	545	}
521	546
...	...	@@ -546,7 +571,7 @@ const ComptimeStoreStrategy = union(enum) {
546	571	val: *MutableValue,
547	572	flat_elem_index: u64,
548	573	},
549		/// This value should be reinterpreted using bitcast logic to perform the
	574	/// This value should be reinterpreted using `Sema.spliceMemory` to perform
550	575	/// store. Only returned if `store_ty` and the type of `val` both have
551	576	/// well-defined layouts.
552	577	reinterpret: struct {
...	...	@@ -886,7 +911,7 @@ fn prepareComptimePtrStore(
886	911	},
887	912	.@"struct" => switch (cur_ty.containerLayout(zcu)) {
888	913	.auto => unreachable, // ill-defined layout
889		.@"packed" => break, // let the bitcast logic handle this
	914	.@"packed" => break, // let the memory reinterp logic handle this
890	915	.@"extern" => for (0..cur_ty.structFieldCount(zcu)) \|field_idx\| {
891	916	const start_off = cur_ty.structFieldOffset(field_idx, zcu);
892	917	const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);
...	...	@@ -899,9 +924,9 @@ fn prepareComptimePtrStore(
899	924	},
900	925	.@"union" => switch (cur_ty.containerLayout(zcu)) {
901	926	.auto => unreachable, // ill-defined layout
902		.@"packed" => break, // let the bitcast logic handle this
	927	.@"packed" => break, // let the memory reinterp logic handle this
903	928	.@"extern" => {
904		// TODO: we have to let bitcast logic handle this for now.
	929	// TODO: we have to let the memory reinterp logic handle this for now.
905	930	// Otherwise, we might traverse into a union field which doesn't allow pointers.
906	931	// Figure out a solution!
907	932	if (true) break;

src/Sema/reinterpret.zig created+576

...	...	@@ -0,0 +1,576 @@
	1	//! This file contains logic for bit-casting arbitrary values at comptime, including splicing
	2	//! bits together for comptime stores of bit-pointers. The strategy is to "flatten" values to
	3	//! a sequence of values in packed memory, and then unflatten through a combination of special
	4	//! cases (particularly for pointers and `undefined` values) and in-memory buffer reinterprets.
	5	//!
	6	//! This is a little awkward on big-endian targets, as non-packed datastructures (e.g. `extern struct`)
	7	//! have their fields reversed when represented as packed memory on such targets.
	8
	9	/// If `host_bits` is `0`, attempts to convert the memory at offset
	10	/// `byte_offset` into `val` to a non-packed value of type `dest_ty`,
	11	/// ignoring `bit_offset`.
	12	///
	13	/// Otherwise, `byte_offset` is an offset in bytes into `val` to a
	14	/// non-packed value consisting of `host_bits` bits. A value of type
	15	/// `dest_ty` will be interpreted at a packed offset of `bit_offset`
	16	/// into this value.
	17	///
	18	/// Returns `null` if the operation must be performed at runtime.
	19	pub fn castMemory(
	20	sema: *Sema,
	21	val: Value,
	22	dest_ty: Type,
	23	byte_offset: u64,
	24	) CompileError!?Value {
	25	const pt = sema.pt;
	26	const zcu = pt.zcu;
	27
	28	const val_ty = val.typeOf(zcu);
	29
	30	if (dest_ty.toIntern() == val_ty.toIntern()) {
	31	assert(byte_offset == 0);
	32	return val;
	33	}
	34
	35	val_ty.assertHasLayout(zcu);
	36	dest_ty.assertHasLayout(zcu);
	37
	38	var unpack: UnpackValueBytes = .{
	39	.pt = pt,
	40	.arena = sema.arena,
	41	.skip_bytes = byte_offset,
	42	.remaining_bytes = dest_ty.abiSize(zcu),
	43	.unpacked = .init(sema.arena),
	44	};
	45	unpack.add(val) catch \|err\| switch (err) {
	46	error.ReinterpretDeclRef => return null,
	47	error.OutOfMemory => \|e\| return e,
	48	};
	49
	50	var pack: PackValueBytes = .{
	51	.pt = pt,
	52	.arena = sema.arena,
	53	.unpacked = unpack.unpacked.items,
	54	};
	55	return pack.get(dest_ty) catch \|err\| switch (err) {
	56	error.ReinterpretDeclRef => return null,
	57	error.OutOfMemory => \|e\| return e,
	58	};
	59	}
	60
	61	/// Splice the value `splice_val` into `val` at the given `byte_offset`, replacing overlapping bits
	62	/// and returning the modified value.
	63	pub fn spliceMemory(
	64	sema: *Sema,
	65	val: Value,
	66	splice_val: Value,
	67	byte_offset: u64,
	68	) CompileError!?Value {
	69	const pt = sema.pt;
	70	const zcu = pt.zcu;
	71	const val_ty = val.typeOf(zcu);
	72	const splice_val_ty = splice_val.typeOf(zcu);
	73
	74	val_ty.assertHasLayout(zcu);
	75	splice_val_ty.assertHasLayout(zcu);
	76
	77	var unpack: UnpackValueBytes = .{
	78	.pt = pt,
	79	.arena = sema.arena,
	80	.skip_bytes = 0,
	81	.remaining_bytes = byte_offset,
	82	.unpacked = .init(sema.arena),
	83	};
	84	unpack.add(val) catch \|err\| switch (err) {
	85	error.ReinterpretDeclRef => return null,
	86	error.OutOfMemory => \|e\| return e,
	87	};
	88
	89	const splice_len = splice_val_ty.abiSize(zcu);
	90
	91	unpack.remaining_bytes = splice_len;
	92	unpack.add(splice_val) catch \|err\| switch (err) {
	93	error.ReinterpretDeclRef => return null,
	94	error.OutOfMemory => \|e\| return e,
	95	};
	96
	97	unpack.skip_bytes = byte_offset + splice_len;
	98	unpack.remaining_bytes = val_ty.abiSize(zcu) * 8 - byte_offset - splice_len;
	99	unpack.add(val) catch \|err\| switch (err) {
	100	error.ReinterpretDeclRef => return null,
	101	error.OutOfMemory => \|e\| return e,
	102	};
	103
	104	var pack: PackValueBytes = .{
	105	.pt = pt,
	106	.arena = sema.arena,
	107	.unpacked = unpack.unpacked.items,
	108	};
	109	return pack.get(val_ty) catch \|err\| switch (err) {
	110	error.ReinterpretDeclRef => return null,
	111	error.OutOfMemory => \|e\| return e,
	112	};
	113	}
	114
	115	/// Recurses through struct fields, array elements, etc, to get a sequence of "primitive" values
	116	/// which are bit-packed in memory to represent a single value. `unpacked` represents a series
	117	/// of values in packed memory - therefore, on big-endian targets, the first element of this
	118	/// list contains bits from the final byte of the value.
	119	const UnpackValueBytes = struct {
	120	pt: Zcu.PerThread,
	121	arena: Allocator,
	122	skip_bytes: u64,
	123	remaining_bytes: u64,
	124	unpacked: std.array_list.Managed(InternPool.Index),
	125
	126	fn add(unpack: *UnpackValueBytes, val: Value) (error{ReinterpretDeclRef} \|\| Allocator.Error)!void {
	127	const pt = unpack.pt;
	128	const zcu = pt.zcu;
	129	const ip = &zcu.intern_pool;
	130
	131	if (unpack.remaining_bytes == 0) {
	132	return;
	133	}
	134
	135	const ty = val.typeOf(zcu);
	136	const size = ty.abiSize(zcu);
	137
	138	if (unpack.skip_bytes >= size) {
	139	unpack.skip_bytes -= size;
	140	return;
	141	}
	142
	143	switch (ip.indexToKey(val.toIntern())) {
	144	.int_type,
	145	.ptr_type,
	146	.array_type,
	147	.vector_type,
	148	.opt_type,
	149	.anyframe_type,
	150	.error_union_type,
	151	.simple_type,
	152	.struct_type,
	153	.tuple_type,
	154	.union_type,
	155	.opaque_type,
	156	.spirv_type,
	157	.enum_type,
	158	.func_type,
	159	.error_set_type,
	160	.inferred_error_set_type,
	161	.@"extern",
	162	.func,
	163	.err,
	164	.error_union,
	165	.enum_literal,
	166	.slice,
	167	.memoized_call,
	168	=> unreachable, // ill-defined layout or not real values
	169
	170	.undef,
	171	.int,
	172	.enum_tag,
	173	.simple_value,
	174	.float,
	175	.ptr,
	176	.opt,
	177	=> try unpack.primitive(val),
	178
	179	.bitpack => \|bitpack\| try unpack.primitive(.fromInterned(bitpack.backing_int_val)),
	180
	181	.aggregate => switch (ty.zigTypeTag(zcu)) {
	182	.vector => unreachable, // ill-defined layout
	183	.array => {
	184	for (0..@intCast(ty.arrayLen(zcu))) \|elem_index\| {
	185	const elem_val = try val.elemValue(pt, @intCast(elem_index));
	186	try unpack.add(elem_val);
	187	}
	188	if (ty.sentinel(zcu)) \|s\| {
	189	try unpack.add(s);
	190	}
	191	},
	192	.@"struct" => switch (ty.containerLayout(zcu)) {
	193	.auto => unreachable, // ill-defined layout
	194	.@"packed" => unreachable, // uses `.bitpack`, not `.aggregate`
	195	.@"extern" => {
	196	var it = ip.loadStructType(ty.toIntern()).iterateRuntimeOrder(ip);
	197	var offset: u64 = 0;
	198	while (it.next()) \|field_index\| {
	199	const pad_bytes = ty.structFieldOffset(field_index, zcu) - offset;
	200	const field_val = try val.fieldValue(pt, field_index);
	201	try unpack.padding(pad_bytes);
	202	try unpack.add(field_val);
	203	offset += pad_bytes + field_val.typeOf(zcu).abiSize(zcu);
	204	}
	205	try unpack.padding(size - offset);
	206	},
	207	},
	208	else => unreachable,
	209	},
	210
	211	.un => \|un\| {
	212	const payload_val = Value.fromInterned(un.val);
	213	const pad_bytes = size - payload_val.typeOf(zcu).abiSize(zcu);
	214	try unpack.add(payload_val);
	215	try unpack.padding(pad_bytes);
	216	},
	217	}
	218	}
	219
	220	fn padding(unpack: *UnpackValueBytes, num_bytes: u64) Allocator.Error!void {
	221	if (num_bytes == 0) return;
	222	const undef_u8 = try unpack.pt.undefValue(Type.u8);
	223	for (0..@intCast(num_bytes)) \|_\| {
	224	unpack.primitive(undef_u8) catch \|err\| switch (err) {
	225	error.OutOfMemory => \|e\| return e,
	226	error.ReinterpretDeclRef => unreachable,
	227	};
	228	}
	229	}
	230
	231	fn primitive(unpack: *UnpackValueBytes, val: Value) (error{ReinterpretDeclRef} \|\| Allocator.Error)!void {
	232	const pt = unpack.pt;
	233	const zcu = pt.zcu;
	234
	235	if (unpack.remaining_bytes == 0) {
	236	return;
	237	}
	238
	239	const ty = val.typeOf(pt.zcu);
	240	const size = ty.abiSize(zcu);
	241
	242	if (unpack.skip_bytes >= size) {
	243	unpack.skip_bytes -= size;
	244	return;
	245	}
	246
	247	if (unpack.skip_bytes > 0) {
	248	const offset = unpack.skip_bytes;
	249	unpack.skip_bytes = 0;
	250	return unpack.splitPrimitive(val, offset, @min(size - offset, unpack.remaining_bytes));
	251	}
	252
	253	if (unpack.remaining_bytes < size) {
	254	return unpack.splitPrimitive(val, 0, unpack.remaining_bytes);
	255	}
	256
	257	unpack.remaining_bytes -= size;
	258	try unpack.unpacked.append(val.toIntern());
	259	}
	260
	261	fn splitPrimitive(unpack: *UnpackValueBytes, val: Value, offset: u64, len: u64) (error{ReinterpretDeclRef} \|\| Allocator.Error)!void {
	262	const pt = unpack.pt;
	263	const zcu = pt.zcu;
	264	const ty = val.typeOf(pt.zcu);
	265
	266	assert(offset + len <= ty.abiSize(zcu));
	267
	268	try unpack.unpacked.ensureUnusedCapacity(@intCast(len));
	269	unpack.remaining_bytes -= len;
	270
	271	switch (pt.zcu.intern_pool.indexToKey(val.toIntern())) {
	272	// In the `ptr` case, this will return `error.ReinterpretDeclRef`
	273	// if we're trying to split a non-integer pointer value.
	274	.int, .float, .enum_tag, .ptr, .opt => {
	275	const buf = try unpack.arena.alloc(u8, @intCast(ty.abiSize(zcu)));
	276	val.writeToMemory(zcu, buf) catch \|err\| switch (err) {
	277	error.IllDefinedMemoryLayout => unreachable,
	278	else => \|e\| return e,
	279	};
	280	for (buf[@intCast(offset)..][0..@intCast(len)]) \|byte_raw\| {
	281	const byte_val = try pt.intValue(.u8, byte_raw);
	282	unpack.unpacked.appendAssumeCapacity(byte_val.toIntern());
	283	}
	284	},
	285	.undef => {
	286	const undef_u8 = try pt.undefValue(.u8);
	287	for (0..@intCast(len)) \|_\| {
	288	unpack.unpacked.appendAssumeCapacity(undef_u8.toIntern());
	289	}
	290	},
	291	// The only values here with runtime bits are `true` and `false`.
	292	// These are both 1 byte, so will never need splitting.
	293	.simple_value => unreachable,
	294	else => unreachable, // zero-bit or not primitives
	295	}
	296	}
	297	};
	298
	299	/// Given a sequence of bit-packed values in packed memory (see `UnpackValueBytes`),
	300	/// reconstructs a value of an arbitrary type, with correct handling of `undefined`
	301	/// values and of pointers which align in virtual memory.
	302	const PackValueBytes = struct {
	303	pt: Zcu.PerThread,
	304	arena: Allocator,
	305	byte_offset: u64 = 0,
	306	unpacked: []const InternPool.Index,
	307
	308	fn get(pack: *PackValueBytes, ty: Type) (Allocator.Error \|\| error{ReinterpretDeclRef})!Value {
	309	const pt = pack.pt;
	310	const zcu = pt.zcu;
	311	const ip = &zcu.intern_pool;
	312	const arena = pack.arena;
	313	switch (ty.zigTypeTag(zcu)) {
	314	.vector => unreachable, // ill-defined layout
	315	.array => {
	316	// Each element is padded up to its ABI size. The final element does not have trailing padding.
	317	const elem_ty = ty.childType(zcu);
	318	const elems = try arena.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
	319
	320	for (elems) \|*elem\| {
	321	elem.* = (try pack.get(elem_ty)).toIntern();
	322	}
	323
	324	if (ty.sentinel(zcu)) \|s\| {
	325	_ = s; // TODO: validate sentinel was preserved!
	326	pack.padding(elem_ty.abiSize(zcu));
	327	}
	328
	329	return pt.aggregateValue(ty, elems);
	330	},
	331	.@"struct" => switch (ty.containerLayout(zcu)) {
	332	.auto => unreachable, // ill-defined layout
	333	.@"extern" => {
	334	const elems = try arena.alloc(InternPool.Index, ty.structFieldCount(zcu));
	335	@memset(elems, .none);
	336	var offset: u64 = 0;
	337	var it = ip.loadStructType(ty.toIntern()).iterateRuntimeOrder(ip);
	338	while (it.next()) \|field_index\| {
	339	const field_ty = ty.fieldType(field_index, zcu);
	340	const pad_bytes = ty.structFieldOffset(field_index, zcu) - offset;
	341	pack.padding(pad_bytes);
	342	elems[field_index] = (try pack.get(field_ty)).toIntern();
	343	offset += pad_bytes + field_ty.abiSize(zcu);
	344	}
	345	pack.padding(ty.abiSize(zcu) - offset);
	346	// Any fields which do not have runtime bits should be OPV or comptime fields.
	347	// Fill those values now.
	348	for (elems, 0..) \|*elem, field_index\| {
	349	if (elem.* != .none) continue;
	350	const val = (try ty.structFieldValueComptime(pt, field_index)).?;
	351	elem.* = val.toIntern();
	352	}
	353	return pt.aggregateValue(ty, elems);
	354	},
	355	.@"packed" => {
	356	const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
	357	if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
	358	return pt.bitpackValue(ty, backing_int_val);
	359	},
	360	},
	361	.@"union" => switch (ty.containerLayout(zcu)) {
	362	.auto => unreachable, // ill-defined layout
	363	.@"extern" => {
	364	// We will attempt to read as the backing representation. If this emits
	365	// `error.ReinterpretDeclRef`, we will try each union field, preferring larger ones.
	366	// We will also attempt smaller fields when we get `undefined`, as if some bits are
	367	// defined we want to include them.
	368	// TODO: this is very very bad. We need a more sophisticated union representation.
	369
	370	const prev_unpacked = pack.unpacked;
	371	const prev_byte_offset = pack.byte_offset;
	372
	373	const backing_ty = try ty.externUnionBackingType(pt);
	374
	375	const backing_result: enum { undef, reinterpret_decl_ref } = backing: {
	376	const backing_val = pack.get(backing_ty) catch \|err\| switch (err) {
	377	error.ReinterpretDeclRef => break :backing .reinterpret_decl_ref,
	378	else => \|e\| return e,
	379	};
	380	if (backing_val.isUndef(zcu)) break :backing .undef;
	381	return .fromInterned(try pt.internUnion(.{
	382	.ty = ty.toIntern(),
	383	.tag = .none,
	384	.val = backing_val.toIntern(),
	385	}));
	386	};
	387
	388	const field_order = try pack.arena.alloc(u32, ty.unionTagTypeHypothetical(zcu).enumFieldCount(zcu));
	389	for (field_order, 0..) \|f, i\| f. = @intCast(i);
	390	// Sort `field_order` to put the fields with the largest ABI sizes first.
	391	const SizeSortCtx = struct {
	392	zcu: *const Zcu,
	393	field_types: []const InternPool.Index,
	394	fn lessThan(ctx: @This(), a_idx: u32, b_idx: u32) bool {
	395	const a_ty: Type = .fromInterned(ctx.field_types[a_idx]);
	396	const b_ty: Type = .fromInterned(ctx.field_types[b_idx]);
	397	return a_ty.abiSize(ctx.zcu) > b_ty.abiSize(ctx.zcu);
	398	}
	399	};
	400	std.mem.sortUnstable(u32, field_order, SizeSortCtx{
	401	.zcu = zcu,
	402	.field_types = zcu.typeToUnion(ty).?.field_types.get(ip),
	403	}, SizeSortCtx.lessThan);
	404
	405	for (field_order) \|field_index\| {
	406	pack.unpacked = prev_unpacked;
	407	pack.byte_offset = prev_byte_offset;
	408	const field_ty = ty.fieldType(field_index, zcu);
	409	const field_val = pack.get(field_ty) catch \|err\| switch (err) {
	410	error.ReinterpretDeclRef => continue,
	411	else => \|e\| return e,
	412	};
	413	if (field_val.isUndef(zcu)) continue;
	414	pack.padding(ty.abiSize(zcu) - field_ty.abiSize(zcu));
	415	const tag_val = try pt.enumValueFieldIndex(ty.unionTagTypeHypothetical(zcu), field_index);
	416	return pt.unionValue(ty, tag_val, field_val);
	417	}
	418
	419	// No field could represent the value. Just do whatever happens when we try to read
	420	// the backing type - either `undefined` or `error.ReinterpretDeclRef`.
	421	switch (backing_result) {
	422	.undef => return pt.undefValue(ty),
	423	.reinterpret_decl_ref => return error.ReinterpretDeclRef,
	424	}
	425	},
	426	.@"packed" => {
	427	const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
	428	if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
	429	return pt.bitpackValue(ty, backing_int_val);
	430	},
	431	},
	432	.@"enum" => {
	433	const tag_int_val = try pack.primitive(ty.intTagType(zcu));
	434	if (tag_int_val.isUndef(zcu)) return pt.undefValue(ty);
	435	return pt.enumValue(ty, tag_int_val.toIntern());
	436	},
	437	else => return pack.primitive(ty),
	438	}
	439	}
	440
	441	fn padding(pack: *PackValueBytes, num_bytes: u64) void {
	442	_ = pack.prepareBytes(num_bytes);
	443	}
	444
	445	fn primitive(pack: *PackValueBytes, want_ty: Type) (Allocator.Error \|\| error{ReinterpretDeclRef})!Value {
	446	const pt = pack.pt;
	447	const zcu = pt.zcu;
	448
	449	if (try want_ty.onePossibleValue(pt)) \|opv\| return opv;
	450
	451	const vals, const byte_offset = pack.prepareBytes(want_ty.abiSize(zcu));
	452
	453	for (vals) \|val\| {
	454	if (!Value.fromInterned(val).isUndef(zcu)) break;
	455	} else {
	456	// All bits of the value are `undefined`.
	457	return pt.undefValue(want_ty);
	458	}
	459
	460	// TODO: we need to decide how to handle partially-undef values here.
	461	// Currently, a value with some undefined bits becomes `0xAA` so that we
	462	// preserve the well-defined bits, because we can't currently represent
	463	// a partially-undefined primitive (e.g. an int with some undef bits).
	464	// In future, we probably want to take one of these two routes:
	465	// * Define that if any bits are `undefined`, the entire value is `undefined`.
	466	// This is a major breaking change, and probably a footgun.
	467	// * Introduce tracking for partially-undef values at comptime.
	468	// This would complicate a lot of operations in Sema, such as basic
	469	// arithmetic.
	470	// This design complexity is tracked by #19634.
	471
	472	if (vals.len == 1 and
	473	want_ty.isPtrAtRuntime(zcu) and
	474	Value.fromInterned(vals[0]).typeOf(zcu).isPtrAtRuntime(zcu))
	475	{
	476	return pt.getCoerced(.fromInterned(vals[0]), want_ty);
	477	}
	478
	479	// Reinterpret via an in-memory buffer.
	480
	481	var buf_len: u64 = 0;
	482	for (vals) \|ip_val\| {
	483	const val: Value = .fromInterned(ip_val);
	484	buf_len += val.typeOf(zcu).abiSize(zcu);
	485	}
	486
	487	const buf = try pack.arena.alloc(u8, @intCast(buf_len));
	488	{
	489	var offset: usize = 0;
	490	for (vals) \|ip_val\| {
	491	const val: Value = .fromInterned(ip_val);
	492	const ty = val.typeOf(zcu);
	493	const size = ty.abiSize(zcu);
	494	if (val.isUndef(zcu)) {
	495	@memset(buf[offset..][0..@intCast(size)], 0xAA);
	496	} else {
	497	val.writeToMemory(zcu, buf[offset..][0..@intCast(size)]) catch \|err\| switch (err) {
	498	error.IllDefinedMemoryLayout => unreachable,
	499	else => \|e\| return e,
	500	};
	501	}
	502	offset += @intCast(size);
	503	}
	504	}
	505	const bytes = buf[@intCast(byte_offset)..];
	506
	507	const target = zcu.getTarget();
	508	const endian = target.cpu.arch.endian();
	509	switch (want_ty.zigTypeTag(zcu)) {
	510	.bool => return .makeBool(bytes[0] != 0),
	511	.int => return .readIntFromMemory(want_ty, pt, bytes, pack.arena),
	512	.float => switch (want_ty.floatBits(target)) {
	513	16 => return pt.floatValue(want_ty, @as(f16, @bitCast(std.mem.readInt(u16, bytes[0..2], endian)))),
	514	32 => return pt.floatValue(want_ty, @as(f32, @bitCast(std.mem.readInt(u32, bytes[0..4], endian)))),
	515	64 => return pt.floatValue(want_ty, @as(f64, @bitCast(std.mem.readInt(u64, bytes[0..8], endian)))),
	516	80 => return pt.floatValue(want_ty, @as(f80, @bitCast(std.mem.readInt(u80, bytes[0..10], endian)))),
	517	128 => return pt.floatValue(want_ty, @as(f128, @bitCast(std.mem.readInt(u128, bytes[0..16], endian)))),
	518	else => unreachable,
	519	},
	520	.pointer => {
	521	assert(!want_ty.isSlice(zcu));
	522	const ptr_addr = std.mem.readVarInt(u64, bytes[0..@intCast(want_ty.abiSize(zcu))], endian);
	523	return pt.ptrIntValue(want_ty, ptr_addr);
	524	},
	525	.optional => {
	526	assert(want_ty.isPtrLikeOptional(zcu));
	527	const ptr_ty = want_ty.optionalChild(zcu);
	528	const ptr_addr = std.mem.readVarInt(u64, bytes[0..@intCast(want_ty.abiSize(zcu))], endian);
	529	return .fromInterned(try pt.intern(.{ .opt = .{
	530	.ty = want_ty.toIntern(),
	531	.val = if (ptr_addr == 0) .none else (try pt.ptrIntValue(ptr_ty, ptr_addr)).toIntern(),
	532	} }));
	533	},
	534	else => unreachable,
	535	}
	536	}
	537
	538	fn prepareBytes(pack: *PackValueBytes, need_bytes: u64) struct { []const InternPool.Index, u64 } {
	539	if (need_bytes == 0) return .{ &.{}, 0 };
	540
	541	const pt = pack.pt;
	542	const zcu = pt.zcu;
	543
	544	var bytes: u64 = 0;
	545	var len: usize = 0;
	546	while (bytes < pack.byte_offset + need_bytes) {
	547	bytes += Value.fromInterned(pack.unpacked[len]).typeOf(zcu).abiSize(zcu);
	548	len += 1;
	549	}
	550
	551	const result_vals = pack.unpacked[0..len];
	552	const result_offset = pack.byte_offset;
	553
	554	const extra_bytes = bytes - pack.byte_offset - need_bytes;
	555	if (extra_bytes == 0) {
	556	pack.unpacked = pack.unpacked[len..];
	557	pack.byte_offset = 0;
	558	} else {
	559	pack.unpacked = pack.unpacked[len - 1 ..];
	560	pack.byte_offset = Value.fromInterned(pack.unpacked[0]).typeOf(zcu).abiSize(zcu) - extra_bytes;
	561	}
	562
	563	return .{ result_vals, result_offset };
	564	}
	565	};
	566
	567	const std = @import("std");
	568	const Allocator = std.mem.Allocator;
	569	const assert = std.debug.assert;
	570
	571	const Sema = @import("../Sema.zig");
	572	const Zcu = @import("../Zcu.zig");
	573	const InternPool = @import("../InternPool.zig");
	574	const Type = @import("../Type.zig");
	575	const Value = @import("../Value.zig");
	576	const CompileError = Zcu.CompileError;

src/Type.zig+47-106

...	...	@@ -757,9 +757,9 @@ pub fn hasWellDefinedLayout(ty: Type, zcu: *const Zcu) bool {
757	757	const ip = &zcu.intern_pool;
758	758	return switch (ip.indexToKey(ty.toIntern())) {
759	759	.int_type,
760		.vector_type,
761	760	=> true,
762	761
	762	.vector_type,
763	763	.error_union_type,
764	764	.error_set_type,
765	765	.inferred_error_set_type,
...	...	@@ -1241,112 +1241,17 @@ pub fn errorAbiSize(zcu: *const Zcu) u64 {
1241	1241	}
1242	1242
1243	1243	/// Asserts that `ty` is not an opaque or comptime-only type.
1244		/// Once #19755 is implemented, this query will only work on types with a defined bit-level representation.
1245	1244	pub fn bitSize(ty: Type, zcu: *const Zcu) u64 {
1246		const target = zcu.getTarget();
1247		const ip = &zcu.intern_pool;
1248		assertHasLayout(ty, zcu);
1249		return switch (ip.indexToKey(ty.toIntern())) {
1250		.int_type => \|int_type\| int_type.bits,
1251		.ptr_type => \|ptr_type\| switch (ptr_type.flags.size) {
1252		.slice => target.ptrBitWidth() * 2,
1253		else => target.ptrBitWidth(),
1254		},
1255		.anyframe_type => target.ptrBitWidth(),
1256		.array_type => \|array_type\| {
1257		const elem_ty: Type = .fromInterned(array_type.child);
1258		const len = array_type.lenIncludingSentinel();
1259		return switch (zcu.comp.getZigBackend()) {
1260		.stage2_x86_64 => len * elem_ty.bitSize(zcu),
1261		// this case will be removed under #19755
1262		else => switch (len) {
1263		0 => 0,
1264		else => (len - 1) * 8 * elem_ty.abiSize(zcu) + elem_ty.bitSize(zcu),
1265		},
1266		};
1267		},
1268		.vector_type => \|vec\| vec.len * Type.fromInterned(vec.child).bitSize(zcu),
1269		.error_set_type, .inferred_error_set_type => zcu.errorSetBits(),
1270		.func_type => unreachable,
1271
1272		.simple_type => \|t\| switch (t) {
1273		.void => 0,
1274		.bool => 1,
1275		.anyerror, .adhoc_inferred_error_set => zcu.errorSetBits(),
1276		.usize, .isize => target.ptrBitWidth(),
1277
1278		.c_char => target.cTypeBitSize(.char),
1279		.c_short => target.cTypeBitSize(.short),
1280		.c_ushort => target.cTypeBitSize(.ushort),
1281		.c_int => target.cTypeBitSize(.int),
1282		.c_uint => target.cTypeBitSize(.uint),
1283		.c_long => target.cTypeBitSize(.long),
1284		.c_ulong => target.cTypeBitSize(.ulong),
1285		.c_longlong => target.cTypeBitSize(.longlong),
1286		.c_ulonglong => target.cTypeBitSize(.ulonglong),
1287		.c_longdouble => target.cTypeBitSize(.longdouble),
1288
1289		.f16 => 16,
1290		.f32 => 32,
1291		.f64 => 64,
1292		.f80 => 80,
1293		.f128 => 128,
1294
1295		.anyopaque => unreachable,
1296		.type => unreachable,
1297		.comptime_int => unreachable,
1298		.comptime_float => unreachable,
1299		.noreturn => unreachable,
1300		.null => unreachable,
1301		.undefined => unreachable,
1302		.enum_literal => unreachable,
1303		.generic_poison => unreachable,
1304		},
1305
1306		.struct_type => {
1307		const struct_obj = ip.loadStructType(ty.toIntern());
1308		switch (struct_obj.layout) {
1309		.@"packed" => return Type.fromInterned(struct_obj.packed_backing_int_type).bitSize(zcu),
1310		.auto, .@"extern" => return struct_obj.size * 8, // will be `unreachable` under #19755
1311		}
1312		},
1313		.union_type => {
1314		const union_obj = ip.loadUnionType(ty.toIntern());
1315		switch (union_obj.layout) {
1316		.@"packed" => return Type.fromInterned(union_obj.packed_backing_int_type).bitSize(zcu),
1317		.auto, .@"extern" => return union_obj.size * 8, // will be `unreachable` under #19755
1318		}
	1245	return switch (ty.zigTypeTag(zcu)) {
	1246	.void => 0,
	1247	.bool => 1,
	1248	.float => ty.floatBits(zcu.getTarget()),
	1249	.pointer, .optional => {
	1250	assert(ty.isPtrAtRuntime(zcu));
	1251	return zcu.getTarget().ptrBitWidth();
1319	1252	},
1320		.enum_type => Type.fromInterned(ip.loadEnumType(ty.toIntern()).int_tag_type).bitSize(zcu),
1321
1322		// will be `unreachable` under #19755
1323		.opt_type,
1324		.error_union_type,
1325		.tuple_type,
1326		=> ty.abiSize(zcu) * 8,
1327
1328		.opaque_type, .spirv_type => unreachable,
1329
1330		// values, not types
1331		.undef,
1332		.simple_value,
1333		.@"extern",
1334		.func,
1335		.int,
1336		.err,
1337		.error_union,
1338		.enum_literal,
1339		.enum_tag,
1340		.float,
1341		.ptr,
1342		.slice,
1343		.opt,
1344		.aggregate,
1345		.un,
1346		.bitpack,
1347		// memoization, not types
1348		.memoized_call,
1349		=> unreachable,
	1253	.array, .vector => ty.arrayLenIncludingSentinel(zcu) * ty.childType(zcu).bitSize(zcu),
	1254	else => ty.intInfo(zcu).bits,
1350	1255	};
1351	1256	}
1352	1257
...	...	@@ -1528,6 +1433,7 @@ pub fn nullablePtrElem(ty: Type, zcu: *const Zcu) Type {
1528	1433	/// * `[*]T`
1529	1434	/// * `[*c]T`
1530	1435	/// * `@SpirvType(.{ .runtime_array = T })`
	1436	/// * `*@SpirvType(.{ .runtime_array = T })`
1531	1437	pub fn indexableElem(ty: Type, zcu: *const Zcu) Type {
1532	1438	const ip = &zcu.intern_pool;
1533	1439	return switch (ip.indexToKey(ty.toIntern())) {
...	...	@@ -3181,6 +3087,8 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool
3181	3087	.frame,
3182	3088	=> false,
3183	3089
	3090	.vector => position == .param_ty or position == .ret_ty,
	3091
3184	3092	.void => switch (position) {
3185	3093	.ret_ty,
3186	3094	.union_field,
...	...	@@ -3259,7 +3167,6 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool
3259	3167	.other,
3260	3168	=> ty.childType(zcu).validateExtern(.element, zcu),
3261	3169	},
3262		.vector => ty.childType(zcu).validateExtern(.element, zcu),
3263	3170	.optional => ty.isPtrLikeOptional(zcu),
3264	3171	};
3265	3172	}
...	...	@@ -3272,6 +3179,40 @@ fn validateExternCallconv(cc: std.lang.CallingConvention) bool {
3272	3179	};
3273	3180	}
3274	3181
	3182	/// Returns whether `ty` is considered by Zig to have a bit-level representation, meaning it is
	3183	/// allowed as the operand to `@bitSizeOf`. This is a superset of packable types.
	3184	pub fn hasBitRepresentation(ty: Type, zcu: *const Zcu) bool {
	3185	return switch (ty.zigTypeTag(zcu)) {
	3186	.@"fn",
	3187	.noreturn,
	3188	.undefined,
	3189	.null,
	3190	.@"opaque",
	3191	.spirv,
	3192	.type,
	3193	.enum_literal,
	3194	.comptime_float,
	3195	.comptime_int,
	3196	.error_set,
	3197	.error_union,
	3198	.frame,
	3199	.@"anyframe",
	3200	=> false,
	3201
	3202	.void,
	3203	.bool,
	3204	.int,
	3205	.float,
	3206	=> true,
	3207
	3208	.@"enum" => zcu.intern_pool.loadEnumType(ty.toIntern()).int_tag_mode == .explicit,
	3209	.pointer, .optional => ty.isPtrAtRuntime(zcu),
	3210	.@"struct", .@"union" => ty.containerLayout(zcu) == .@"packed",
	3211
	3212	.array, .vector => ty.childType(zcu).hasBitRepresentation(zcu),
	3213	};
	3214	}
	3215
3275	3216	/// Asserts that `ty` has resolved layout.
3276	3217	pub fn assertHasLayout(ty: Type, zcu: *const Zcu) void {
3277	3218	if (!std.debug.runtime_safety) {

src/Value.zig+135-134

...	...	@@ -248,7 +248,6 @@ pub fn toBool(val: Value) bool {
248	248	pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
249	249	ReinterpretDeclRef,
250	250	IllDefinedMemoryLayout,
251		Unimplemented,
252	251	OutOfMemory,
253	252	}!void {
254	253	const target = zcu.getTarget();
...	...	@@ -257,35 +256,50 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
257	256	const ty = val.typeOf(zcu);
258	257	if (val.isUndef(zcu)) {
259	258	const size: usize = @intCast(ty.abiSize(zcu));
260		@memset(buffer[0..size], 0xaa);
	259	@memset(buffer[0..size], 0xAA);
261	260	return;
262	261	}
263		switch (ty.zigTypeTag(zcu)) {
	262	tag: switch (ty.zigTypeTag(zcu)) {
	263	.type => return error.IllDefinedMemoryLayout,
	264	.comptime_float => return error.IllDefinedMemoryLayout,
	265	.comptime_int => return error.IllDefinedMemoryLayout,
	266	.undefined => return error.IllDefinedMemoryLayout,
	267	.null => return error.IllDefinedMemoryLayout,
	268	.error_union => return error.IllDefinedMemoryLayout,
	269	.enum_literal => return error.IllDefinedMemoryLayout,
	270	.@"fn" => return error.IllDefinedMemoryLayout,
	271	.spirv => return error.IllDefinedMemoryLayout,
	272	.@"opaque" => unreachable,
	273	.frame => unreachable,
	274	.@"anyframe" => unreachable,
	275	.noreturn => unreachable,
264	276	.void => {},
265	277	.bool => {
266	278	buffer[0] = @intFromBool(val.toBool());
267	279	},
268		.int, .@"enum", .error_set, .pointer => \|tag\| {
269		const int_ty = if (tag == .pointer) int_ty: {
270		if (ty.isSlice(zcu)) return error.IllDefinedMemoryLayout;
271		if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
272		break :int_ty Type.usize;
273		} else ty;
274		const int_info = int_ty.intInfo(zcu);
275		const bits = int_info.bits;
276		const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
277
	280	.pointer => {
	281	if (ty.isSlice(zcu)) return error.IllDefinedMemoryLayout;
	282	if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
	283	continue :tag .int;
	284	},
	285	.int, .@"enum", .error_set => {
278	286	var bigint_buffer: BigIntSpace = undefined;
279	287	const bigint = val.toBigInt(&bigint_buffer, zcu);
280		bigint.writeTwosComplement(buffer[0..byte_count], endian);
281		},
282		.float => switch (ty.floatBits(target)) {
283		16 => std.mem.writeInt(u16, buffer[0..2], @bitCast(val.toFloat(f16, zcu)), endian),
284		32 => std.mem.writeInt(u32, buffer[0..4], @bitCast(val.toFloat(f32, zcu)), endian),
285		64 => std.mem.writeInt(u64, buffer[0..8], @bitCast(val.toFloat(f64, zcu)), endian),
286		80 => std.mem.writeInt(u80, buffer[0..10], @bitCast(val.toFloat(f80, zcu)), endian),
287		128 => std.mem.writeInt(u128, buffer[0..16], @bitCast(val.toFloat(f128, zcu)), endian),
288		else => unreachable,
	288	bigint.writeTwosComplement(buffer[0..@intCast(ty.abiSize(zcu))], endian);
	289	},
	290	.float => {
	291	const float_bits = ty.floatBits(target);
	292	switch (float_bits) {
	293	16 => std.mem.writeInt(u16, buffer[0..2], @bitCast(val.toFloat(f16, zcu)), endian),
	294	32 => std.mem.writeInt(u32, buffer[0..4], @bitCast(val.toFloat(f32, zcu)), endian),
	295	64 => std.mem.writeInt(u64, buffer[0..8], @bitCast(val.toFloat(f64, zcu)), endian),
	296	80 => std.mem.writeInt(u80, buffer[0..10], @bitCast(val.toFloat(f80, zcu)), endian),
	297	128 => std.mem.writeInt(u128, buffer[0..16], @bitCast(val.toFloat(f128, zcu)), endian),
	298	else => unreachable,
	299	}
	300	const float_bytes = @divExact(float_bits, 8);
	301	const total_bytes: usize = @intCast(ty.abiSize(zcu));
	302	@memset(buffer[float_bytes..total_bytes], 0); // padding
289	303	},
290	304	.array => {
291	305	const aggregate = ip.indexToKey(val.toIntern()).aggregate;
...	...	@@ -302,28 +316,33 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
302	316	}
303	317	buf_off += elem_size;
304	318	}
	319	if (ty.sentinel(zcu)) \|sentinel_val\| {
	320	try sentinel_val.writeToMemory(zcu, buffer[buf_off..]);
	321	}
305	322	},
306		.vector => {
307		// We use byte_count instead of abi_size here, so that any padding bytes
308		// follow the data bytes, on both big- and little-endian systems.
309		const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
310		return writeToPackedMemory(val, zcu, buffer[0..byte_count], 0);
311		},
	323	.vector => return error.IllDefinedMemoryLayout,
312	324	.@"struct" => {
313	325	const struct_type = zcu.typeToStruct(ty) orelse return error.IllDefinedMemoryLayout;
314	326	switch (struct_type.layout) {
315	327	.auto => return error.IllDefinedMemoryLayout,
316		.@"extern" => for (0..struct_type.field_types.len) \|field_index\| {
317		const off: usize = @intCast(ty.structFieldOffset(field_index, zcu));
318		const field_val = Value.fromInterned(switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
319		.bytes => \|bytes\| {
320		buffer[off] = bytes.at(field_index, ip);
321		continue;
322		},
323		.elems => \|elems\| elems[field_index],
324		.repeated_elem => \|elem\| elem,
325		});
326		try writeToMemory(field_val, zcu, buffer[off..]);
	328	.@"extern" => {
	329	var last_off: usize = 0;
	330	for (struct_type.field_types.get(ip), 0..) \|field_ty_ip, field_index\| {
	331	const off: usize = @intCast(ty.structFieldOffset(field_index, zcu));
	332	@memset(buffer[last_off..off], 0xAA);
	333	const field_val = Value.fromInterned(switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
	334	.bytes => \|bytes\| {
	335	buffer[off] = bytes.at(field_index, ip);
	336	continue;
	337	},
	338	.elems => \|elems\| elems[field_index],
	339	.repeated_elem => \|elem\| elem,
	340	});
	341	try writeToMemory(field_val, zcu, buffer[off..]);
	342	last_off = @intCast(off + Type.fromInterned(field_ty_ip).abiSize(zcu));
	343	}
	344	const struct_size: usize = @intCast(ty.abiSize(zcu));
	345	@memset(buffer[last_off..struct_size], 0xAA);
327	346	},
328	347	.@"packed" => {
329	348	const int_index = ip.indexToKey(val.toIntern()).bitpack.backing_int_val;
...	...	@@ -335,6 +354,9 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
335	354	.auto => return error.IllDefinedMemoryLayout, // Sema is supposed to have emitted a compile error already
336	355	.@"extern" => {
337	356	const payload_val = val.unionPayload(zcu);
	357	const payload_size: usize = @intCast(payload_val.typeOf(zcu).abiSize(zcu));
	358	const union_size: usize = @intCast(ty.abiSize(zcu));
	359	@memset(buffer[payload_size..union_size], 0xAA);
338	360	return writeToMemory(payload_val, zcu, buffer);
339	361	},
340	362	.@"packed" => {
...	...	@@ -352,7 +374,6 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
352	374	@memset(buffer[0..@intCast(byte_count)], 0); // null pointer
353	375	}
354	376	},
355		else => return error.Unimplemented,
356	377	}
357	378	}
358	379
...	...	@@ -360,12 +381,15 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
360	381	///
361	382	/// Both the start and the end of the provided buffer must be tight, since
362	383	/// big-endian packed memory layouts start at the end of the buffer.
	384	///
	385	/// Supports arrays and vectors, for which the value is written in logical bit
	386	/// order, i.e. with the first element at bit offset 0.
363	387	pub fn writeToPackedMemory(
364	388	val: Value,
365	389	zcu: *const Zcu,
366	390	buffer: []u8,
367	391	bit_offset: usize,
368		) error{ ReinterpretDeclRef, OutOfMemory }!void {
	392	) void {
369	393	const ip = &zcu.intern_pool;
370	394	const target = zcu.getTarget();
371	395	const endian = target.cpu.arch.endian();
...	...	@@ -392,13 +416,7 @@ pub fn writeToPackedMemory(
392	416	},
393	417	.@"enum" => {
394	418	const int_val = val.intFromEnum(zcu);
395		return int_val.writeToPackedMemory(zcu, buffer, bit_offset);
396		},
397		.pointer => {
398		assert(!ty.isSlice(zcu)); // No well defined layout.
399		if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
400		const addr = val.toUnsignedInt(zcu);
401		std.mem.writeVarPackedInt(buffer, bit_offset, zcu.getTarget().ptrBitWidth(), addr, endian);
	419	int_val.writeToPackedMemory(zcu, buffer, bit_offset);
402	420	},
403	421	.int => {
404	422	const bits = ty.intInfo(zcu).bits;
...	...	@@ -416,47 +434,46 @@ pub fn writeToPackedMemory(
416	434	128 => std.mem.writePackedInt(u128, buffer, bit_offset, @bitCast(val.toFloat(f128, zcu)), endian),
417	435	else => unreachable,
418	436	},
419		.vector => {
420		const elem_ty = ty.childType(zcu);
421		const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
422		const len: usize = @intCast(ty.arrayLen(zcu));
423
424		var bits: u16 = 0;
425		var elem_i: usize = 0;
426		const aggregate = ip.indexToKey(val.toIntern()).aggregate;
427		while (elem_i < len) : (elem_i += 1) {
428		// On big-endian systems, LLVM reverses the element order of vectors by default
429		const tgt_elem_i = if (endian == .big) len - elem_i - 1 else elem_i;
430		switch (aggregate.storage) {
431		.bytes => \|bytes\| std.mem.writePackedInt(u8, buffer, bit_offset + bits, bytes.at(tgt_elem_i, ip), endian),
432		.elems => \|elems\| try Value.fromInterned(elems[tgt_elem_i]).writeToPackedMemory(zcu, buffer, bit_offset + bits),
433		.repeated_elem => \|elem\| try Value.fromInterned(elem).writeToPackedMemory(zcu, buffer, bit_offset + bits),
434		}
435		bits += elem_bit_size;
436		}
437		},
438	437	.@"struct", .@"union" => {
439	438	assert(ty.containerLayout(zcu) == .@"packed");
440	439	const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val);
441		return int_val.writeToPackedMemory(zcu, buffer, bit_offset);
	440	int_val.writeToPackedMemory(zcu, buffer, bit_offset);
442	441	},
443		.optional => {
444		assert(ty.isPtrLikeOptional(zcu));
445		if (val.optionalValue(zcu)) \|ptr_val\| {
446		return ptr_val.writeToPackedMemory(zcu, buffer, bit_offset);
447		} else {
448		return Value.zero_usize.writeToPackedMemory(zcu, buffer, bit_offset);
	442	.array, .vector => {
	443	const elem_bits: usize = @intCast(ty.childType(zcu).bitSize(zcu));
	444	const len: usize = @intCast(ty.arrayLen(zcu));
	445	var elem_bit_off: usize = bit_offset;
	446	switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
	447	.repeated_elem => \|elem_val_ip\| {
	448	const elem_val: Value = .fromInterned(elem_val_ip);
	449	for (0..len) \|_\| {
	450	elem_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
	451	elem_bit_off += elem_bits;
	452	}
	453	},
	454	.elems => \|elems\| for (elems[0..len]) \|elem_val_ip\| {
	455	const elem_val: Value = .fromInterned(elem_val_ip);
	456	elem_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
	457	elem_bit_off += elem_bits;
	458	},
	459	.bytes => \|bytes\| for (bytes.toSlice(len, ip)) \|raw_byte\| {
	460	std.mem.writeVarPackedInt(buffer, elem_bit_off, elem_bits, raw_byte, endian);
	461	elem_bit_off += elem_bits;
	462	},
	463	}
	464	if (ty.sentinel(zcu)) \|sentinel_val\| {
	465	sentinel_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
449	466	}
450	467	},
451		else => @panic("TODO implement writeToPackedMemory for more types"),
	468	else => unreachable,
452	469	}
453	470	}
454	471
455		/// Load a Value from the contents of `buffer`, where `ty` is an unsigned integer type.
	472	/// Load a Value from the contents of `buffer`, where `ty` is any integer type.
456	473	///
457	474	/// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past
458	475	/// the end of the value in memory.
459		pub fn readUintFromMemory(
	476	pub fn readIntFromMemory(
460	477	ty: Type,
461	478	pt: Zcu.PerThread,
462	479	buffer: []const u8,
...	...	@@ -465,23 +482,28 @@ pub fn readUintFromMemory(
465	482	const zcu = pt.zcu;
466	483	const endian = zcu.getTarget().cpu.arch.endian();
467	484
468		assert(ty.isUnsignedInt(zcu));
469		const bits = ty.intInfo(zcu).bits;
470		const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
	485	const int = ty.intInfo(zcu);
	486	const abi_size: usize = @intCast(ty.abiSize(zcu));
	487	const exact_buf = buffer[0..abi_size];
471	488
472		assert(buffer.len >= byte_count);
473
474		if (bits <= 64) {
475		const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian);
476		const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
477		return pt.intValue(ty, result);
	489	if (abi_size <= 8) {
	490	const shift: u6 = @intCast(64 - int.bits);
	491	switch (int.signedness) {
	492	.unsigned => {
	493	const x = std.mem.readVarInt(u64, exact_buf, endian);
	494	return pt.intValue(ty, (x << shift) >> shift);
	495	},
	496	.signed => {
	497	const x = std.mem.readVarInt(i64, exact_buf, endian);
	498	return pt.intValue(ty, (x << shift) >> shift);
	499	},
	500	}
478	501	} else {
479		const Limb = std.math.big.Limb;
480		const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb);
481		const limbs_buffer = try arena.alloc(Limb, limb_count);
	502	const limb_count = std.math.big.int.calcTwosCompLimbCount(int.bits);
	503	const limbs_buffer = try arena.alloc(std.math.big.Limb, limb_count);
482	504
483	505	var bigint: BigIntMutable = .init(limbs_buffer, 0);
484		bigint.readTwosComplement(buffer[0..byte_count], bits, endian, .unsigned);
	506	bigint.readTwosComplement(exact_buf, int.bits, endian, int.signedness);
485	507	return pt.intValue_big(ty, bigint.toConst());
486	508	}
487	509	}
...	...	@@ -490,17 +512,17 @@ pub fn readUintFromMemory(
490	512	///
491	513	/// Both the start and the end of the provided buffer must be tight, since
492	514	/// big-endian packed memory layouts start at the end of the buffer.
	515	///
	516	/// Supports arrays and vectors, for which the value is read in logical bit
	517	/// order, i.e. with the first element at bit offset 0.
493	518	pub fn readFromPackedMemory(
494	519	ty: Type,
495	520	pt: Zcu.PerThread,
496	521	buffer: []const u8,
497	522	bit_offset: usize,
498		gpa: Allocator,
499		) error{
500		IllDefinedMemoryLayout,
501		OutOfMemory,
502		}!Value {
	523	) Allocator.Error!Value {
503	524	const zcu = pt.zcu;
	525	const gpa = zcu.comp.gpa;
504	526	const target = zcu.getTarget();
505	527	const endian = target.cpu.arch.endian();
506	528	switch (ty.zigTypeTag(zcu)) {
...	...	@@ -543,7 +565,7 @@ pub fn readFromPackedMemory(
543	565	},
544	566	.@"enum" => {
545	567	const int_ty = ty.intTagType(zcu);
546		const int_val = try Value.readFromPackedMemory(int_ty, pt, buffer, bit_offset, gpa);
	568	const int_val: Value = try .readFromPackedMemory(int_ty, pt, buffer, bit_offset);
547	569	return pt.getCoerced(int_val, ty);
548	570	},
549	571	.float => return Value.fromInterned(try pt.intern(.{ .float = .{
...	...	@@ -557,40 +579,25 @@ pub fn readFromPackedMemory(
557	579	else => unreachable,
558	580	},
559	581	} })),
560		.vector => {
561		const elem_ty = ty.childType(zcu);
562		const elems = try gpa.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
563		defer gpa.free(elems);
564
565		var bits: u16 = 0;
566		const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
567		for (elems, 0..) \|_, i\| {
568		// On big-endian systems, LLVM reverses the element order of vectors by default
569		const tgt_elem_i = if (endian == .big) elems.len - i - 1 else i;
570		elems[tgt_elem_i] = (try readFromPackedMemory(elem_ty, pt, buffer, bit_offset + bits, gpa)).toIntern();
571		bits += elem_bit_size;
572		}
573		return pt.aggregateValue(ty, elems);
574		},
575	582	.@"struct", .@"union" => {
576	583	assert(ty.containerLayout(zcu) == .@"packed");
577		const int_val: Value = try .readFromPackedMemory(ty.bitpackBackingInt(zcu), pt, buffer, bit_offset, gpa);
	584	const int_val: Value = try .readFromPackedMemory(ty.bitpackBackingInt(zcu), pt, buffer, bit_offset);
578	585	return pt.bitpackValue(ty, int_val);
579	586	},
580		.pointer => {
581		assert(!ty.isSlice(zcu)); // No well defined layout.
582		const addr = (try readFromPackedMemory(Type.usize, pt, buffer, bit_offset, gpa)).toUnsignedInt(zcu);
583		return pt.ptrIntValue(ty, addr);
584		},
585		.optional => {
586		assert(ty.isPtrLikeOptional(zcu));
587		const addr = (try readFromPackedMemory(Type.usize, pt, buffer, bit_offset, gpa)).toUnsignedInt(zcu);
588		return .fromInterned(try pt.intern(.{ .opt = .{
589		.ty = ty.toIntern(),
590		.val = if (addr == 0) .none else (try pt.ptrIntValue(ty.childType(zcu), addr)).toIntern(),
591		} }));
	587	.array, .vector => {
	588	const elem_ty = ty.childType(zcu);
	589	const elem_bits: usize = @intCast(elem_ty.bitSize(zcu));
	590	const elems_buf = try gpa.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
	591	defer gpa.free(elems_buf);
	592	var elem_bit_off: usize = bit_offset;
	593	for (elems_buf) \|*elem\| {
	594	const elem_val = try readFromPackedMemory(elem_ty, pt, buffer, elem_bit_off);
	595	elem.* = elem_val.toIntern();
	596	elem_bit_off += elem_bits;
	597	}
	598	return pt.aggregateValue(ty, elems_buf);
592	599	},
593		else => @panic("TODO implement readFromPackedMemory for more types"),
	600	else => unreachable,
594	601	}
595	602	}
596	603
...	...	@@ -887,14 +894,9 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value {
887	894	const bfa = bfa_state.allocator();
888	895	const buf = try bfa.alloc(u8, @intCast((ty.bitSize(zcu) + 7) / 8));
889	896	defer bfa.free(buf);
890		int_val.writeToPackedMemory(zcu, buf, 0) catch \|err\| switch (err) {
891		error.ReinterpretDeclRef => unreachable, // it's an integer
892		error.OutOfMemory => \|e\| return e,
893		};
894		return Value.readFromPackedMemory(field_ty, pt, buf, field_bit_offset, bfa) catch \|err\| switch (err) {
895		error.IllDefinedMemoryLayout => unreachable, // it's a bitpack
896		error.OutOfMemory => \|e\| return e,
897		};
	897	@memset(buf, 0);
	898	int_val.writeToPackedMemory(zcu, buf, 0);
	899	return .readFromPackedMemory(field_ty, pt, buf, field_bit_offset);
898	900	},
899	901	else => unreachable,
900	902	};
...	...	@@ -1619,7 +1621,6 @@ pub fn hasRepeatedByteRepr(val: Value, zcu: *const Zcu) !?u8 {
1619	1621	// code late in compilation. So, this error handling is too aggressive and
1620	1622	// causes some false negatives, causing less-than-ideal code generation.
1621	1623	error.IllDefinedMemoryLayout => return null,
1622		error.Unimplemented => return null,
1623	1624	};
1624	1625	const first_byte = byte_buffer[0];
1625	1626	for (byte_buffer[1..]) \|byte\| {

src/Zcu/PerThread.zig+4

...	...	@@ -4544,8 +4544,12 @@ fn runCodegenInner(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) e
4544	4544	tracy_trace.addText(fqn.toSlice(ip));
4545	4545	tracy_trace.addTextFmt("func_ip_index={d}", .{func_index});
4546	4546
	4547	Air.Verify.run(pt, func_index, air);
	4548
4547	4549	if (codegen.legalizeFeatures(pt, nav)) \|features\| {
4548	4550	try air.legalize(pt, features);
	4551	// Verify the AIR again post-legalization.
	4552	Air.Verify.run(pt, func_index, air);
4549	4553	}
4550	4554
4551	4555	var liveness: ?Air.Liveness = if (codegen.wantsLiveness(pt, nav))

src/codegen/aarch64/Select.zig+23-7

...	...	@@ -292,8 +292,8 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
292	292	.load,
293	293	.fptrunc,
294	294	.fpext,
295		.intcast,
296		.intcast_safe,
	295	.int_cast,
	296	.int_cast_safe,
297	297	.trunc,
298	298	.optional_payload,
299	299	.optional_payload_ptr,
...	...	@@ -334,7 +334,15 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
334	334	air_inst_index = air_body[air_body_index];
335	335	continue :air_tag air_tags[@intFromEnum(air_inst_index)];
336	336	},
337		.bitcast => {
	337	.bit_cast,
	338	.ptr_cast,
	339	.ptr_from_int,
	340	.int_from_ptr,
	341	.error_cast,
	342	.error_from_int,
	343	.int_from_error,
	344	.union_from_enum,
	345	=> {
338	346	const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;
339	347	maybe_noop: {
340	348	if (ty_op.ty.toInterned().? != isel.air.typeOf(ty_op.operand, ip).toIntern()) break :maybe_noop;
...	...	@@ -3190,7 +3198,15 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3190	3198	}
3191	3199	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
3192	3200	},
3193		.bitcast => \|air_tag\| {
	3201	.bit_cast,
	3202	.ptr_cast,
	3203	.ptr_from_int,
	3204	.int_from_ptr,
	3205	.error_cast,
	3206	.error_from_int,
	3207	.int_from_error,
	3208	.union_from_enum,
	3209	=> \|air_tag\| {
3194	3210	if (isel.live_values.fetchRemove(air.inst_index)) \|dst_vi\| unused: {
3195	3211	defer dst_vi.value.deref(isel);
3196	3212	const ty_op = air.data(air.inst_index).ty_op;
...	...	@@ -5221,7 +5237,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
5221	5237	}
5222	5238	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
5223	5239	},
5224		.intcast => \|air_tag\| {
	5240	.int_cast => \|air_tag\| {
5225	5241	if (isel.live_values.fetchRemove(air.inst_index)) \|dst_vi\| unused: {
5226	5242	defer dst_vi.value.deref(isel);
5227	5243
...	...	@@ -5312,7 +5328,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
5312	5328	}
5313	5329	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
5314	5330	},
5315		.intcast_safe => \|air_tag\| {
	5331	.int_cast_safe => \|air_tag\| {
5316	5332	if (isel.live_values.fetchRemove(air.inst_index)) \|dst_vi\| unused: {
5317	5333	defer dst_vi.value.deref(isel);
5318	5334
...	...	@@ -11355,7 +11371,7 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
11355	11371	if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;
11356	11372	constant.writeToMemory(zcu, buffer) catch \|err\| switch (err) {
11357	11373	error.OutOfMemory => \|e\| return e,
11358		error.ReinterpretDeclRef, error.Unimplemented, error.IllDefinedMemoryLayout => return false,
	11374	error.ReinterpretDeclRef, error.IllDefinedMemoryLayout => return false,
11359	11375	};
11360	11376	return true;
11361	11377	}

src/codegen/aarch64/abi.zig+2-2

...	...	@@ -21,7 +21,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
21	21	if (ty.containerLayout(zcu) == .@"packed") return .byval;
22	22	if (countFloats(ty, zcu)) \|float\| return .{ .float_array = float.count };
23	23
24		const bit_size = ty.bitSize(zcu);
	24	const bit_size = ty.abiSize(zcu) * 8;
25	25	if (bit_size > 128) return .memory;
26	26	if (bit_size > 64) return .double_integer;
27	27	return .integer;
...	...	@@ -30,7 +30,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
30	30	if (ty.containerLayout(zcu) == .@"packed") return .byval;
31	31	if (countFloats(ty, zcu)) \|float\| return .{ .float_array = float.count };
32	32
33		const bit_size = ty.bitSize(zcu);
	33	const bit_size = ty.abiSize(zcu) * 8;
34	34	if (bit_size > 128) return .memory;
35	35	if (bit_size > 64) return .double_integer;
36	36	return .integer;

src/codegen/arm/abi.zig+6-6

...	...	@@ -30,11 +30,11 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
30	30	const ip = &zcu.intern_pool;
31	31	switch (ty.zigTypeTag(zcu)) {
32	32	.@"struct" => {
33		const bit_size = ty.bitSize(zcu);
34	33	if (ty.containerLayout(zcu) == .@"packed") {
35		if (bit_size > 64) return .memory;
	34	if (ty.bitSize(zcu) > 64) return .memory;
36	35	return .byval;
37	36	}
	37	const bit_size = ty.abiSize(zcu) * 8;
38	38	if (bit_size > max_byval_size) return .memory;
39	39	const float_count = countFloats(ty, zcu, &maybe_float_bits);
40	40	if (float_count <= byval_float_count) return .byval;
...	...	@@ -47,17 +47,17 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
47	47	var i: u32 = 0;
48	48	while (i < fields) : (i += 1) {
49	49	const field_ty = ty.fieldType(i, zcu);
50		if (field_ty.bitSize(zcu) > 32) return Class.arrSize(bit_size, 64);
	50	if (field_ty.abiSize(zcu) > 4) return Class.arrSize(bit_size, 64);
51	51	}
52	52	return Class.arrSize(bit_size, 32);
53	53	},
54	54	.@"union" => {
55		const bit_size = ty.bitSize(zcu);
56	55	const union_obj = zcu.typeToUnion(ty).?;
57	56	if (union_obj.layout == .@"packed") {
58		if (bit_size > 64) return .memory;
	57	if (ty.bitSize(zcu) > 64) return .memory;
59	58	return .byval;
60	59	}
	60	const bit_size = ty.abiSize(zcu) * 8;
61	61	if (bit_size > max_byval_size) return .memory;
62	62	const float_count = countFloats(ty, zcu, &maybe_float_bits);
63	63	if (float_count <= byval_float_count) return .byval;
...	...	@@ -67,7 +67,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
67	67	}
68	68
69	69	for (union_obj.field_types.get(ip)) \|field_ty\| {
70		if (Type.fromInterned(field_ty).bitSize(zcu) > 32) {
	70	if (Type.fromInterned(field_ty).abiSize(zcu) > 4) {
71	71	return Class.arrSize(bit_size, 64);
72	72	}
73	73	}

src/codegen/c.zig+270-104

...	...	@@ -27,7 +27,7 @@ pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {
27	27	return comptime switch (dev.env.supports(.legalize)) {
28	28	inline false, true => \|supports_legalize\| &.init(.{
29	29	// we don't currently ask zig1 to use safe optimization modes
30		.expand_intcast_safe = supports_legalize,
	30	.expand_int_cast_safe = supports_legalize,
31	31	.expand_int_from_float_safe = supports_legalize,
32	32	.expand_int_from_float_optimized_safe = supports_legalize,
33	33	.expand_add_safe = supports_legalize,
...	...	@@ -38,6 +38,9 @@ pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {
38	38	.expand_packed_store = true,
39	39	.expand_packed_struct_field_val = true,
40	40	.expand_packed_aggregate_init = true,
	41
	42	.scalarize_bit_cast_array = true,
	43	.scalarize_bit_cast_vector_non_elementwise = true,
41	44	}),
42	45	};
43	46	}
...	...	@@ -2636,9 +2639,9 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2636	2639	// zig fmt: off
2637	2640	.inferred_alloc, .inferred_alloc_comptime => unreachable,
2638	2641
2639		// No "scalarize" legalizations are enabled, so these instructions never appear.
2640		.legalize_vec_elem_val => unreachable,
2641		.legalize_vec_store_elem => unreachable,
	2642	// Possible because `Air.Legalize.scalarize_bit_cast_vector_non_elementwise` is enabled.
	2643	.legalize_vec_elem_val => try airArrayElemVal(f, inst),
	2644	.legalize_vec_store_elem => try airLegalizeVecStoreElem(f, inst),
2642	2645	// No soft float legalizations are enabled.
2643	2646	.legalize_compiler_rt_call => unreachable,
2644	2647
...	...	@@ -2751,8 +2754,15 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2751	2754	.alloc => try airAlloc(f, inst),
2752	2755	.ret_ptr => try airRetPtr(f, inst),
2753	2756	.assembly => try airAsm(f, inst),
2754		.bitcast => try airBitcast(f, inst),
2755		.intcast => try airIntCast(f, inst),
	2757	.ptr_cast => try airPtrCast(f, inst),
	2758	.ptr_from_int => try airSimpleCast(f, inst),
	2759	.int_from_ptr => try airSimpleCast(f, inst),
	2760	.error_cast => try airNopCast(f, inst),
	2761	.error_from_int => try airNopCast(f, inst),
	2762	.int_from_error => try airNopCast(f, inst),
	2763	.union_from_enum => try airUnionFromEnum(f, inst),
	2764	.bit_cast => try airBitCast(f, inst),
	2765	.int_cast => try airIntCast(f, inst),
2756	2766	.trunc => try airTrunc(f, inst),
2757	2767	.load => try airLoad(f, inst),
2758	2768	.store => try airStore(f, inst, false),
...	...	@@ -2864,7 +2874,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2864	2874	.add_safe,
2865	2875	.sub_safe,
2866	2876	.mul_safe,
2867		.intcast_safe,
	2877	.int_cast_safe,
2868	2878	.int_from_float_safe,
2869	2879	.int_from_float_optimized_safe,
2870	2880	=> return f.fail("TODO implement safety_checked_instructions", .{}),
...	...	@@ -3083,6 +3093,28 @@ fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue {
3083	3093	return local;
3084	3094	}
3085	3095
	3096	fn airLegalizeVecStoreElem(f: *Function, inst: Air.Inst.Index) !CValue {
	3097	const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	3098	const extra = f.air.extraData(Air.Bin, pl_op.payload).data;
	3099
	3100	const vec_ptr = try f.resolveInst(pl_op.operand);
	3101	const index = try f.resolveInst(extra.lhs);
	3102	const elem = try f.resolveInst(extra.rhs);
	3103	try reap(f, inst, &.{ pl_op.operand, extra.lhs, extra.rhs });
	3104
	3105	const w = &f.code.writer;
	3106
	3107	try f.writeCValueDerefMember(w, vec_ptr, .{ .identifier = "array" });
	3108	try w.writeByte('[');
	3109	try f.writeCValue(w, index, .other);
	3110	try w.writeAll("] = ");
	3111	try f.writeCValue(w, elem, .other);
	3112	try w.writeByte(';');
	3113	try f.newline();
	3114
	3115	return .none;
	3116	}
	3117
3086	3118	fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {
3087	3119	const pt = f.dg.pt;
3088	3120	const zcu = pt.zcu;
...	...	@@ -3190,35 +3222,42 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
3190	3222
3191	3223	try reap(f, inst, &.{ty_op.operand});
3192	3224
3193		const is_aligned = if (ptr_info.flags.alignment != .none)
3194		ptr_info.flags.alignment.order(src_ty.abiAlignment(zcu)).compare(.gte)
3195		else
3196		true;
	3225	const is_aligned = switch (ptr_info.flags.alignment) {
	3226	.none => true,
	3227	else => \|ptr_align\| ptr_align.compare(.gte, src_ty.abiAlignment(zcu)),
	3228	};
3197	3229
3198	3230	const w = &f.code.writer;
3199	3231	const local = try f.allocLocal(inst, src_ty);
3200		const v = try Vectorize.start(f, inst, w, ptr_ty);
3201	3232
3202	3233	if (!is_aligned) {
3203	3234	try w.writeAll("memcpy(&");
3204	3235	try f.writeCValue(w, local, .other);
3205		try v.elem(f, w);
3206	3236	try w.writeAll(", (const char *)");
3207		try f.writeCValue(w, operand, .other);
3208		try v.elem(f, w);
	3237	switch (ptr_info.flags.vector_index) {
	3238	.none => try f.writeCValue(w, operand, .other),
	3239	else => \|index\| {
	3240	try w.writeByte('&');
	3241	try f.writeCValue(w, operand, .other);
	3242	try w.print("[{d}]", .{@intFromEnum(index)});
	3243	},
	3244	}
3209	3245	try w.writeAll(", sizeof(");
3210	3246	try f.renderType(w, src_ty);
3211	3247	try w.writeAll("))");
3212	3248	} else {
3213	3249	try f.writeCValue(w, local, .other);
3214		try v.elem(f, w);
3215	3250	try w.writeAll(" = ");
3216		try f.writeCValueDeref(w, operand);
3217		try v.elem(f, w);
	3251	switch (ptr_info.flags.vector_index) {
	3252	.none => try f.writeCValueDeref(w, operand),
	3253	else => \|index\| {
	3254	try f.writeCValue(w, operand, .other);
	3255	try w.print("[{d}]", .{@intFromEnum(index)});
	3256	},
	3257	}
3218	3258	}
3219	3259	try w.writeByte(';');
3220	3260	try f.newline();
3221		try v.end(f, inst, w);
3222	3261
3223	3262	return local;
3224	3263	}
...	...	@@ -3433,21 +3472,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
3433	3472	// underlying type as the lhs (i.e. they must both be arrays of the same underlying type).
3434	3473	assert(src_ty.eql(.fromInterned(ptr_info.child)));
3435	3474
3436		const v = try Vectorize.start(f, inst, w, ptr_ty);
3437	3475	try w.writeAll("memcpy((char *)");
3438		try f.writeCValue(w, ptr_val, .other);
3439		try v.elem(f, w);
	3476	switch (ptr_info.flags.vector_index) {
	3477	.none => try f.writeCValue(w, ptr_val, .other),
	3478	else => \|index\| {
	3479	try w.writeByte('&');
	3480	try f.writeCValue(w, ptr_val, .other);
	3481	try w.print("[{d}]", .{@intFromEnum(index)});
	3482	},
	3483	}
3440	3484	try w.writeAll(", &");
3441	3485	switch (src_val) {
3442	3486	.constant => \|val\| try f.dg.renderValueAsLvalue(w, val),
3443	3487	else => try f.writeCValue(w, src_val, .other),
3444	3488	}
3445		try v.elem(f, w);
3446	3489	try w.writeAll(", sizeof(");
3447	3490	try f.renderType(w, src_ty);
3448	3491	try w.writeAll("));");
3449	3492	try f.newline();
3450		try v.end(f, inst, w);
3451	3493	} else {
3452	3494	switch (ptr_val) {
3453	3495	.local_ref => \|ptr_local_index\| switch (src_val) {
...	...	@@ -3457,15 +3499,18 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
3457	3499	},
3458	3500	else => {},
3459	3501	}
3460		const v = try Vectorize.start(f, inst, w, ptr_ty);
3461		try f.writeCValueDeref(w, ptr_val);
3462		try v.elem(f, w);
	3502
	3503	switch (ptr_info.flags.vector_index) {
	3504	.none => try f.writeCValueDeref(w, ptr_val),
	3505	else => \|index\| {
	3506	try f.writeCValue(w, ptr_val, .other);
	3507	try w.print("[{d}]", .{@intFromEnum(index)});
	3508	},
	3509	}
3463	3510	try w.writeAll(" = ");
3464	3511	try f.writeCValue(w, src_val, .other);
3465		try v.elem(f, w);
3466	3512	try w.writeByte(';');
3467	3513	try f.newline();
3468		try v.end(f, inst, w);
3469	3514	}
3470	3515	return .none;
3471	3516	}
...	...	@@ -3613,9 +3658,9 @@ fn airCmpOp(
3613	3658	const lhs_ty = f.typeOf(data.lhs);
3614	3659	const scalar_ty = lhs_ty.scalarType(zcu);
3615	3660
3616		const scalar_bits = scalar_ty.bitSize(zcu);
3617		if (scalar_ty.isInt(zcu) and scalar_bits > 64)
3618		return airCmpBuiltinCall(
	3661	if (scalar_ty.isInt(zcu)) {
	3662	const scalar_bits = scalar_ty.bitSize(zcu);
	3663	if (scalar_bits > 64) return airCmpBuiltinCall(
3619	3664	f,
3620	3665	inst,
3621	3666	data,
...	...	@@ -3623,6 +3668,7 @@ fn airCmpOp(
3623	3668	.cmp,
3624	3669	if (scalar_bits > 128) .bits else .none,
3625	3670	);
	3671	}
3626	3672	if (scalar_ty.isRuntimeFloat())
3627	3673	return airCmpBuiltinCall(f, inst, data, operator, .operator, .none);
3628	3674
...	...	@@ -3668,9 +3714,9 @@ fn airEquality(
3668	3714	const bin_op = f.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3669	3715
3670	3716	const operand_ty = f.typeOf(bin_op.lhs);
3671		const operand_bits = operand_ty.bitSize(zcu);
3672		if (operand_ty.isAbiInt(zcu) and operand_bits > 64)
3673		return airCmpBuiltinCall(
	3717	if (operand_ty.isAbiInt(zcu)) {
	3718	const operand_bits = operand_ty.bitSize(zcu);
	3719	if (operand_bits > 64) return airCmpBuiltinCall(
3674	3720	f,
3675	3721	inst,
3676	3722	bin_op,
...	...	@@ -3678,6 +3724,7 @@ fn airEquality(
3678	3724	.cmp,
3679	3725	if (operand_bits > 128) .bits else .none,
3680	3726	);
	3727	}
3681	3728	if (operand_ty.isRuntimeFloat())
3682	3729	return airCmpBuiltinCall(f, inst, bin_op, operator, .operator, .none);
3683	3730
...	...	@@ -4258,125 +4305,240 @@ fn airSwitchDispatch(f: *Function, inst: Air.Inst.Index) !void {
4258	4305	try w.print("goto zig_switch_{d}_loop;\n", .{@intFromEnum(br.block_inst)});
4259	4306	}
4260	4307
4261		fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue {
	4308	fn airPtrCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
	4309	const zcu = f.dg.pt.zcu;
	4310
	4311	const dest_ty = f.typeOfIndex(inst);
	4312	const ptr_ty = switch (dest_ty.zigTypeTag(zcu)) {
	4313	.optional => dest_ty.childType(zcu),
	4314	.pointer => dest_ty,
	4315	else => unreachable,
	4316	};
	4317
	4318	if (!ptr_ty.isSlice(zcu)) {
	4319	return airSimpleCast(f, inst);
	4320	}
	4321
	4322	// For slice casts we need to assign both fields.
	4323
4262	4324	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4263		const inst_ty = f.typeOfIndex(inst);
	4325	const operand = try f.resolveInst(ty_op.operand);
	4326
	4327	const w = &f.code.writer;
	4328	const dest_local = try f.allocLocal(inst, dest_ty);
	4329
	4330	try f.writeCValueMember(w, dest_local, .{ .identifier = "ptr" });
	4331	try w.writeAll(" = (");
	4332	try f.renderType(w, ptr_ty.slicePtrFieldType(zcu));
	4333	try w.writeByte(')');
	4334	try f.writeCValueMember(w, operand, .{ .identifier = "ptr" });
	4335	try w.writeByte(';');
	4336	try f.newline();
	4337
	4338	try f.writeCValueMember(w, dest_local, .{ .identifier = "len" });
	4339	try w.writeAll(" = ");
	4340	try f.writeCValueMember(w, operand, .{ .identifier = "len" });
	4341	try w.writeByte(';');
	4342	try f.newline();
	4343
	4344	try reap(f, inst, &.{ty_op.operand});
	4345	return dest_local;
	4346	}
	4347
	4348	fn airSimpleCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
	4349	const zcu = f.dg.pt.zcu;
	4350
	4351	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4352	const dest_ty = f.typeOfIndex(inst);
	4353	const operand_ty = f.typeOf(ty_op.operand);
	4354	const operand = try f.resolveInst(ty_op.operand);
	4355
	4356	const w = &f.code.writer;
	4357	const dest_local = try f.allocLocal(inst, dest_ty);
	4358	const v: Vectorize = try .start(f, inst, w, operand_ty);
	4359	try f.writeCValue(w, dest_local, .other);
	4360	try v.elem(f, w);
	4361	try w.writeAll(" = (");
	4362	try f.renderType(w, dest_ty.scalarType(zcu));
	4363	try w.writeByte(')');
	4364	try f.writeCValue(w, operand, .other);
	4365	try v.elem(f, w);
	4366	try w.writeByte(';');
	4367	try f.newline();
	4368	try v.end(f, inst, w);
	4369
	4370	try reap(f, inst, &.{ty_op.operand});
	4371	return dest_local;
	4372	}
4264	4373
	4374	fn airNopCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
	4375	const zcu = f.dg.pt.zcu;
	4376
	4377	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4378	const dest_ty = f.typeOfIndex(inst);
	4379	const operand_ty = f.typeOf(ty_op.operand);
4265	4380	const operand = try f.resolveInst(ty_op.operand);
	4381
	4382	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
	4383	assert(operand_ty.isAbiInt(zcu) == dest_ty.isAbiInt(zcu));
	4384
	4385	try reap(f, inst, &.{ty_op.operand});
	4386	return f.moveCValue(inst, dest_ty, operand);
	4387	}
	4388
	4389	fn airUnionFromEnum(f: *Function, inst: Air.Inst.Index) Error!CValue {
	4390	const zcu = f.dg.pt.zcu;
	4391
	4392	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4393	const dest_ty = f.typeOfIndex(inst);
4266	4394	const operand_ty = f.typeOf(ty_op.operand);
	4395	const operand = try f.resolveInst(ty_op.operand);
	4396
	4397	assert(dest_ty.zigTypeTag(zcu) == .@"union");
	4398	assert(operand_ty.zigTypeTag(zcu) == .@"enum");
	4399
	4400	const w = &f.code.writer;
	4401	const dest_local = try f.allocLocal(inst, dest_ty);
	4402	try f.writeCValueMember(w, dest_local, .{ .identifier = "tag" });
	4403	try w.writeAll(" = ");
	4404	try f.writeCValue(w, operand, .other);
	4405	try w.writeByte(';');
	4406	try f.newline();
4267	4407
4268		const bitcasted = try bitcast(f, inst_ty, operand, operand_ty);
4269	4408	try reap(f, inst, &.{ty_op.operand});
4270		return f.moveCValue(inst, inst_ty, bitcasted);
	4409	return dest_local;
4271	4410	}
4272	4411
4273		fn bitcast(f: *Function, dest_ty: Type, operand: CValue, operand_ty: Type) !CValue {
	4412	fn airBitCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
4274	4413	const pt = f.dg.pt;
4275	4414	const zcu = pt.zcu;
4276		const target = &f.dg.mod.resolved_target.result;
4277	4415	const w = &f.code.writer;
4278	4416
4279		if (operand_ty.isAbiInt(zcu) and dest_ty.isAbiInt(zcu)) {
4280		const src_info = dest_ty.intInfo(zcu);
4281		const dest_info = operand_ty.intInfo(zcu);
4282		if (src_info.signedness == dest_info.signedness and
4283		src_info.bits == dest_info.bits) return operand;
4284		}
	4417	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4418	const dest_ty = f.typeOfIndex(inst);
4285	4419
4286		if (dest_ty.isPtrAtRuntime(zcu) or operand_ty.isPtrAtRuntime(zcu)) {
4287		const local = try f.allocLocal(null, dest_ty);
4288		try f.writeCValue(w, local, .other);
	4420	const operand = try f.resolveInst(ty_op.operand);
	4421	const operand_ty = f.typeOf(ty_op.operand);
	4422
	4423	const dest_local = try f.allocLocal(inst, dest_ty);
	4424
	4425	// Because we have `scalarize_bit_cast_array` and `scalarize_bit_cast_vector_non_elementwise`
	4426	// enabled, we usually only see scalars here. The only case in which we may see vectors is when
	4427	// the operation happens elementwise, which we can handle with `Vectorize`.
	4428	var v: Vectorize = try .start(f, inst, w, operand_ty);
	4429	const operand_scalar_ty = operand_ty.scalarType(zcu);
	4430	const dest_scalar_ty = dest_ty.scalarType(zcu);
	4431
	4432	// Some cases are handled with a simple cast:
	4433	// * float -> float
	4434	// * bool -> int
	4435	if ((operand_scalar_ty.isRuntimeFloat() and dest_scalar_ty.isRuntimeFloat()) or
	4436	(operand_scalar_ty.toIntern() == .bool_type and dest_scalar_ty.isAbiInt(zcu)))
	4437	{
	4438	try f.writeCValue(w, dest_local, .other);
	4439	try v.elem(f, w);
4289	4440	try w.writeAll(" = (");
4290		try f.renderType(w, dest_ty);
	4441	try f.renderType(w, dest_scalar_ty);
4291	4442	try w.writeByte(')');
4292	4443	try f.writeCValue(w, operand, .other);
	4444	try v.elem(f, w);
4293	4445	try w.writeByte(';');
4294	4446	try f.newline();
4295		return local;
4296		}
4297
4298		const local = try f.allocLocal(null, dest_ty);
4299		// On big-endian targets, copying ABI integers with padding bits is awkward, because the padding bits are at the low bytes of the value.
4300		// We need to offset the source or destination pointer appropriately and copy the right number of bytes.
4301		if (target.cpu.arch.endian() == .big and dest_ty.isAbiInt(zcu) and !operand_ty.isAbiInt(zcu)) {
4302		// e.g. [10]u8 -> u80. We need to offset the destination so that we copy to the least significant bits of the integer.
4303		const offset = dest_ty.abiSize(zcu) - operand_ty.abiSize(zcu);
4304		try w.writeAll("memcpy((char *)&");
4305		try f.writeCValue(w, local, .other);
4306		try w.print(" + {d}, &", .{offset});
4307		switch (operand) {
4308		.constant => \|val\| try f.dg.renderValueAsLvalue(w, val),
4309		else => try f.writeCValue(w, operand, .other),
4310		}
4311		try w.print(", {d});", .{operand_ty.abiSize(zcu)});
4312		} else if (target.cpu.arch.endian() == .big and operand_ty.isAbiInt(zcu) and !dest_ty.isAbiInt(zcu)) {
4313		// e.g. u80 -> [10]u8. We need to offset the source so that we copy from the least significant bits of the integer.
4314		const offset = operand_ty.abiSize(zcu) - dest_ty.abiSize(zcu);
	4447	} else if (dest_scalar_ty.toIntern() == .bool_type) {
	4448	// If the result is a boolean type, just check if the operand is non-zero.
	4449	assert(operand_scalar_ty.isAbiInt(zcu));
	4450	try f.writeCValue(w, dest_local, .other);
	4451	try v.elem(f, w);
	4452	try w.writeAll(" = ");
	4453	try f.writeCValue(w, operand, .other);
	4454	try v.elem(f, w);
	4455	try w.writeAll(" != 0;");
	4456	try f.newline();
	4457	} else if (dest_scalar_ty.isRuntimeFloat()) {
	4458	// For int->float, just do a memcpy.
	4459	assert(operand_scalar_ty.isAbiInt(zcu));
4315	4460	try w.writeAll("memcpy(&");
4316		try f.writeCValue(w, local, .other);
4317		try w.writeAll(", (const char *)&");
	4461	try f.writeCValue(w, dest_local, .other);
	4462	try v.elem(f, w);
	4463	try w.writeAll(", &");
4318	4464	switch (operand) {
4319	4465	.constant => \|val\| try f.dg.renderValueAsLvalue(w, val),
4320	4466	else => try f.writeCValue(w, operand, .other),
4321	4467	}
4322		try w.print(" + {d}, {d});", .{ offset, dest_ty.abiSize(zcu) });
	4468	try v.elem(f, w);
	4469	try w.print(", {d});", .{@min(operand_scalar_ty.abiSize(zcu), dest_scalar_ty.abiSize(zcu))});
	4470	try f.newline();
4323	4471	} else {
	4472	// The only remaining possibility is that the result is an integer. We will need to use
	4473	// `zig_wrap_*` to correct the "padding" bits after we populate the value bits.
	4474	assert(dest_scalar_ty.isAbiInt(zcu));
	4475	assert(operand_scalar_ty.isRuntimeFloat() or operand_scalar_ty.isAbiInt(zcu));
	4476
	4477	// memcpy the value...
4324	4478	try w.writeAll("memcpy(&");
4325		try f.writeCValue(w, local, .other);
	4479	try f.writeCValue(w, dest_local, .other);
	4480	try v.elem(f, w);
4326	4481	try w.writeAll(", &");
4327	4482	switch (operand) {
4328	4483	.constant => \|val\| try f.dg.renderValueAsLvalue(w, val),
4329	4484	else => try f.writeCValue(w, operand, .other),
4330	4485	}
4331		try w.print(", {d});", .{@min(dest_ty.abiSize(zcu), operand_ty.abiSize(zcu))});
4332		}
4333
4334		try f.newline();
	4486	try v.elem(f, w);
	4487	try w.print(", {d});", .{@min(operand_scalar_ty.abiSize(zcu), dest_scalar_ty.abiSize(zcu))});
	4488	try f.newline();
4335	4489
4336		// Ensure padding bits have the expected value.
4337		if (dest_ty.isAbiInt(zcu)) {
4338		switch (CType.classifyInt(dest_ty, zcu)) {
	4490	// ...and ensure padding bits have the correct value.
	4491	switch (CType.classifyInt(dest_scalar_ty, zcu)) {
4339	4492	.void => unreachable, // opv
4340	4493	.small => {
4341		try f.writeCValue(w, local, .other);
	4494	try f.writeCValue(w, dest_local, .other);
	4495	try v.elem(f, w);
4342	4496	try w.writeAll(" = zig_wrap_");
4343		try f.dg.renderTypeForBuiltinFnName(w, dest_ty);
	4497	try f.dg.renderTypeForBuiltinFnName(w, dest_scalar_ty);
4344	4498	try w.writeByte('(');
4345		try f.writeCValue(w, local, .other);
4346		try f.dg.renderBuiltinInfo(w, dest_ty, .bits);
	4499	try f.writeCValue(w, dest_local, .other);
	4500	try v.elem(f, w);
	4501	try f.dg.renderBuiltinInfo(w, dest_scalar_ty, .bits);
4347	4502	try w.writeAll(");");
4348	4503	try f.newline();
4349	4504	},
4350	4505	.big => \|big\| {
4351		const dest_info = dest_ty.intInfo(zcu);
4352		const padding_index: u16 = switch (target.cpu.arch.endian()) {
	4506	const dest_info = dest_scalar_ty.intInfo(zcu);
	4507	const padding_index: u16 = switch (f.dg.mod.resolved_target.result.cpu.arch.endian()) {
4353	4508	.little => big.limbs_len - 1,
4354	4509	.big => 0,
4355	4510	};
4356	4511	const wrap_bits = ((dest_info.bits - 1) % big.limb_size.bits()) + 1;
4357	4512	if (big.limb_size != .@"128" or dest_info.signedness == .unsigned) {
4358		try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
4359		try w.print("[{d}] = zig_wrap_{c}{d}(", .{
	4513	try f.writeCValue(w, dest_local, .other);
	4514	try v.elem(f, w);
	4515	try w.print(".limbs[{d}] = zig_wrap_{c}{d}(", .{
4360	4516	padding_index,
4361	4517	signAbbrev(dest_info.signedness),
4362	4518	big.limb_size.bits(),
4363	4519	});
4364		try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
4365		try w.print("[{d}], {d});", .{ padding_index, wrap_bits });
	4520	try f.writeCValue(w, dest_local, .other);
	4521	try v.elem(f, w);
	4522	try w.print(".limbs[{d}], {d});", .{ padding_index, wrap_bits });
4366	4523	} else {
4367		try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
4368		try w.print("[{d}] = zig_bitCast_u128(zig_wrap_i128(zig_bitCast_i128(", .{
	4524	try f.writeCValue(w, dest_local, .other);
	4525	try v.elem(f, w);
	4526	try w.print(".limbs[{d}] = zig_bitCast_u128(zig_wrap_i128(zig_bitCast_i128(", .{
4369	4527	padding_index,
4370	4528	});
4371		try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
4372		try w.print("[{d}]), {d}));", .{ padding_index, wrap_bits });
	4529	try f.writeCValue(w, dest_local, .other);
	4530	try v.elem(f, w);
	4531	try w.print(".limbs[{d}]), {d}));", .{ padding_index, wrap_bits });
4373	4532	try f.newline();
4374	4533	}
4375	4534	},
4376	4535	}
4377	4536	}
4378	4537
4379		return local;
	4538	try v.end(f, inst, w);
	4539
	4540	try reap(f, inst, &.{ty_op.operand});
	4541	return dest_local;
4380	4542	}
4381	4543
4382	4544	fn airTrap(f: *Function) !void {
...	...	@@ -6151,28 +6313,32 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
6151	6313	return .none;
6152	6314	}
6153	6315
6154		if (elem_abi_size == 1 and !dest_ty.isVolatilePtr(zcu)) {
6155		const bitcasted = try bitcast(f, .u8, value, elem_ty);
	6316	if (elem_abi_size == 1 and elem_ty.isAbiInt(zcu) and !dest_ty.isVolatilePtr(zcu)) {
6156	6317	try w.writeAll("memset(");
6157	6318	switch (dest_ty.ptrSize(zcu)) {
6158	6319	.slice => {
6159	6320	try f.writeCValueMember(w, dest_slice, .{ .identifier = "ptr" });
6160		try w.writeAll(", ");
6161		try f.writeCValue(w, bitcasted, .other);
	6321	try w.writeAll(", (const char )&");
	6322	switch (value) {
	6323	.constant => \|v\| try f.dg.renderValueAsLvalue(w, v),
	6324	else => try f.writeCValue(w, value, .other),
	6325	}
6162	6326	try w.writeAll(", ");
6163	6327	try f.writeCValueMember(w, dest_slice, .{ .identifier = "len" });
6164	6328	},
6165	6329	.one => {
6166	6330	try f.writeCValue(w, dest_slice, .other);
6167		try w.writeAll(", ");
6168		try f.writeCValue(w, bitcasted, .other);
	6331	try w.writeAll(", (const char )&");
	6332	switch (value) {
	6333	.constant => \|v\| try f.dg.renderValueAsLvalue(w, v),
	6334	else => try f.writeCValue(w, value, .other),
	6335	}
6169	6336	try w.print(", {d}", .{dest_ty.childType(zcu).arrayLen(zcu)});
6170	6337	},
6171	6338	.many, .c => unreachable,
6172	6339	}
6173	6340	try w.writeAll(");");
6174	6341	try f.newline();
6175		try f.freeCValue(inst, bitcasted);
6176	6342	try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });
6177	6343	return .none;
6178	6344	}

src/codegen/c/type.zig+15-19

...	...	@@ -514,40 +514,39 @@ pub const CType = union(enum) {
514	514	}
515	515	}
516	516	fn classifyBitInt(signedness: std.lang.Signedness, bits: u16, zcu: *const Zcu) IntClass {
517		const is_ez80 = zcu.getTarget().cpu.arch == .ez80;
518		return switch (bits) {
	517	const target = zcu.getTarget();
	518	return switch (std.zig.target.intByteSize(target, bits)) {
519	519	0 => .void,
520		1...8 => switch (signedness) {
	520	1 => switch (signedness) {
521	521	.unsigned => .{ .small = .uint8_t },
522	522	.signed => .{ .small = .int8_t },
523	523	},
524		9...16 => switch (signedness) {
	524	2 => switch (signedness) {
525	525	.unsigned => .{ .small = .uint16_t },
526	526	.signed => .{ .small = .int16_t },
527	527	},
528		17...24 => switch (signedness) {
529		.unsigned => .{ .small = if (is_ez80) .uint24_t else .uint32_t },
530		.signed => .{ .small = if (is_ez80) .int24_t else .int32_t },
	528	3 => switch (signedness) {
	529	.unsigned => .{ .small = .uint24_t },
	530	.signed => .{ .small = .int24_t },
531	531	},
532		25...32 => switch (signedness) {
	532	4 => switch (signedness) {
533	533	.unsigned => .{ .small = .uint32_t },
534	534	.signed => .{ .small = .int32_t },
535	535	},
536		33...48 => switch (signedness) {
537		.unsigned => .{ .small = if (is_ez80) .uint48_t else .uint64_t },
538		.signed => .{ .small = if (is_ez80) .int48_t else .int64_t },
	536	6 => switch (signedness) {
	537	.unsigned => .{ .small = .uint48_t },
	538	.signed => .{ .small = .int48_t },
539	539	},
540		49...64 => switch (signedness) {
	540	8 => switch (signedness) {
541	541	.unsigned => .{ .small = .uint64_t },
542	542	.signed => .{ .small = .int64_t },
543	543	},
544		65...128 => switch (signedness) {
	544	16 => switch (signedness) {
545	545	.unsigned => .{ .small = .zig_u128 },
546	546	.signed => .{ .small = .zig_i128 },
547	547	},
548		else => {
	548	else => \|n\| {
549	549	@branchHint(.unlikely);
550		const target = zcu.getTarget();
551	550	const limb_bytes = std.zig.target.intAlignment(target, bits);
552	551	return .{ .big = .{
553	552	.limb_size = switch (limb_bytes) {
...	...	@@ -558,10 +557,7 @@ pub const CType = union(enum) {
558	557	16 => .@"128",
559	558	else => unreachable,
560	559	},
561		.limbs_len = @divExact(
562		std.zig.target.intByteSize(target, bits),
563		limb_bytes,
564		),
	560	.limbs_len = @divExact(n, limb_bytes),
565	561	} };
566	562	},
567	563	};

src/codegen/llvm.zig+156-288

...	...	@@ -20,10 +20,8 @@ const Value = @import("../Value.zig");
20	20	const Zcu = @import("../Zcu.zig");
21	21	const aarch64_c_abi = @import("aarch64/abi.zig");
22	22	const FuncGen = @import("llvm/FuncGen.zig");
23		const buildAllocaInner = FuncGen.buildAllocaInner;
24	23	const isByRef = FuncGen.isByRef;
25		const firstParamSRet = FuncGen.firstParamSRet;
26		const lowerFnRetTy = FuncGen.lowerFnRetTy;
	24	const fnReturnStrat = FuncGen.fnReturnStrat;
27	25	const iterateParamTypes = FuncGen.iterateParamTypes;
28	26	const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt;
29	27
...	...	@@ -37,6 +35,11 @@ pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {
37	35	return comptime &.initMany(&.{
38	36	.expand_int_from_float_safe,
39	37	.expand_int_from_float_optimized_safe,
	38
	39	.scalarize_bit_cast_array,
	40	// Needed because LLVM's `bitcast` on vectors is endian-specific unless the source and dest
	41	// types are vectors with equal length (hence also with equal bits-per-element).
	42	.scalarize_bit_cast_vector_non_elementwise,
40	43	});
41	44	}
42	45
...	...	@@ -728,8 +731,8 @@ pub const Object = struct {
728	731
729	732	// TODO: Address space
730	733	const slice_ty = Type.slice_const_u8_sentinel_0;
731		const llvm_usize_ty = try o.lowerType(.usize);
732		const llvm_slice_ty = try o.lowerType(slice_ty);
	734	const llvm_usize_ty = try o.lowerType(.usize, .in_memory);
	735	const llvm_slice_ty = try o.lowerType(slice_ty, .in_memory);
733	736	const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty);
734	737
735	738	llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty);
...	...	@@ -795,7 +798,7 @@ pub const Object = struct {
795	798	{
796	799	if (o.errors_len_variable != .none) {
797	800	const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len;
798		const init_val = try o.builder.intConst(try o.errorIntType(), errors_len);
	801	const init_val = try o.builder.intConst(try o.errorIntType(.in_memory), errors_len);
799	802	try o.errors_len_variable.setInitializer(init_val, &o.builder);
800	803	}
801	804	try o.genErrorNameTable();
...	...	@@ -1187,7 +1190,7 @@ pub const Object = struct {
1187	1190	};
1188	1191	{
1189	1192	const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder);
1190		global.type = try o.lowerType(fn_ty);
	1193	global.type = try o.lowerType(fn_ty, .in_memory);
1191	1194	global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target);
1192	1195	global.linkage = if (o.builder.strip) .private else .internal;
1193	1196	global.visibility = .default;
...	...	@@ -1274,165 +1277,7 @@ pub const Object = struct {
1274	1277	} }, &o.builder);
1275	1278	}
1276	1279
1277		var deinit_wip = true;
1278		var wip = try Builder.WipFunction.init(&o.builder, .{
1279		.function = llvm_function,
1280		.strip = owner_mod.strip,
1281		});
1282		defer if (deinit_wip) wip.deinit();
1283		wip.cursor = .{ .block = try wip.block(0, "Entry") };
1284
1285		// This is the list of args we will use that correspond directly to the AIR arg
1286		// instructions. Depending on the calling convention, this list is not necessarily
1287		// a bijection with the actual LLVM parameters of the function.
1288		var args: std.ArrayList(Builder.Value) = .empty;
1289		defer args.deinit(gpa);
1290
1291		const ret_ptr: Builder.Value, const err_ret_trace: Builder.Value = implicit_args: {
1292		var it = iterateParamTypes(o, fn_info);
1293
1294		const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: {
1295		const param = wip.arg(it.llvm_index);
1296		it.llvm_index += 1;
1297		break :param param;
1298		} else .none;
1299
1300		const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing;
1301		const err_ret_trace: Builder.Value = if (err_return_tracing) param: {
1302		const param = wip.arg(it.llvm_index);
1303		it.llvm_index += 1;
1304		break :param param;
1305		} else .none;
1306
1307		while (try it.next()) \|lowering\| {
1308		try args.ensureUnusedCapacity(gpa, 1);
1309
1310		switch (lowering) {
1311		.no_bits => continue,
1312		.byval => {
1313		assert(!it.byval_attr);
1314		const param_index = it.zig_index - 1;
1315		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
1316		const param = wip.arg(it.llvm_index - 1);
1317
1318		if (isByRef(param_ty, zcu)) {
1319		const alignment = param_ty.abiAlignment(zcu).toLlvm();
1320		const param_llvm_ty = param.typeOfWip(&wip);
1321		const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1322		_ = try wip.store(.normal, param, arg_ptr, alignment);
1323		args.appendAssumeCapacity(arg_ptr);
1324		} else {
1325		args.appendAssumeCapacity(param);
1326		}
1327		},
1328		.byref => {
1329		const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1330		const param = wip.arg(it.llvm_index - 1);
1331
1332		if (isByRef(param_ty, zcu)) {
1333		args.appendAssumeCapacity(param);
1334		} else {
1335		const param_llvm_ty = try o.lowerType(param_ty);
1336		const alignment = param_ty.abiAlignment(zcu).toLlvm();
1337		args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1338		}
1339		},
1340		.byref_mut => {
1341		const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1342		const param = wip.arg(it.llvm_index - 1);
1343
1344		if (isByRef(param_ty, zcu)) {
1345		args.appendAssumeCapacity(param);
1346		} else {
1347		const param_llvm_ty = try o.lowerType(param_ty);
1348		const alignment = param_ty.abiAlignment(zcu).toLlvm();
1349		args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1350		}
1351		},
1352		.abi_sized_int => {
1353		assert(!it.byval_attr);
1354		const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1355		const param = wip.arg(it.llvm_index - 1);
1356
1357		const param_llvm_ty = try o.lowerType(param_ty);
1358		const alignment = param_ty.abiAlignment(zcu).toLlvm();
1359		const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1360		_ = try wip.store(.normal, param, arg_ptr, alignment);
1361
1362		if (isByRef(param_ty, zcu)) {
1363		args.appendAssumeCapacity(arg_ptr);
1364		} else {
1365		args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1366		}
1367		},
1368		.slice => {
1369		assert(!it.byval_attr);
1370		const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1371		assert(!isByRef(param_ty, zcu));
1372		const slice_val = try wip.buildAggregate(
1373		try o.lowerType(param_ty),
1374		&.{ wip.arg(it.llvm_index - 2), wip.arg(it.llvm_index - 1) },
1375		"",
1376		);
1377		args.appendAssumeCapacity(slice_val);
1378		},
1379		.multiple_llvm_types => {
1380		assert(!it.byval_attr);
1381		const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1382		const param_llvm_ty = try o.lowerType(param_ty);
1383		const param_alignment = param_ty.abiAlignment(zcu);
1384		const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
1385		const arg_ptr = try buildAllocaInner(&wip, llvm_ty, param_alignment.toLlvm(), target);
1386		const llvm_args_start = it.llvm_index - it.types_len;
1387		for (llvm_args_start.., it.offsets_buffer[0..it.types_len]) \|llvm_arg_index, offset\| {
1388		const param = wip.arg(@intCast(llvm_arg_index));
1389		const part_ptr = try o.ptraddConst(&wip, arg_ptr, offset);
1390		_ = try wip.store(.normal, param, part_ptr, param_alignment.offset(offset).toLlvm());
1391		}
1392
1393		if (isByRef(param_ty, zcu)) {
1394		args.appendAssumeCapacity(arg_ptr);
1395		} else {
1396		args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment.toLlvm(), ""));
1397		}
1398		},
1399		.float_array => {
1400		const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1401		const param_llvm_ty = try o.lowerType(param_ty);
1402		const param = wip.arg(it.llvm_index - 1);
1403
1404		const alignment = param_ty.abiAlignment(zcu).toLlvm();
1405		const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1406		_ = try wip.store(.normal, param, arg_ptr, alignment);
1407
1408		if (isByRef(param_ty, zcu)) {
1409		args.appendAssumeCapacity(arg_ptr);
1410		} else {
1411		args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1412		}
1413		},
1414		.i32_array, .i64_array => {
1415		const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1416		const param_llvm_ty = try o.lowerType(param_ty);
1417		const param = wip.arg(it.llvm_index - 1);
1418
1419		const alignment = param_ty.abiAlignment(zcu).toLlvm();
1420		const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target);
1421		_ = try wip.store(.normal, param, arg_ptr, alignment);
1422
1423		if (isByRef(param_ty, zcu)) {
1424		args.appendAssumeCapacity(arg_ptr);
1425		} else {
1426		args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1427		}
1428		},
1429		}
1430		}
1431
1432		break :implicit_args .{ ret_ptr, err_ret_trace };
1433		};
1434
1435		const file, const subprogram = if (!wip.strip) debug_info: {
	1280	const file, const subprogram = if (!owner_mod.strip) debug_info: {
1436	1281	const file = try o.getDebugFile(file_scope);
1437	1282
1438	1283	const line_number = zcu.navSrcLine(func.owner_nav) + 1;
...	...	@@ -1498,11 +1343,12 @@ pub const Object = struct {
1498	1343	.gpa = gpa,
1499	1344	.air = air.*,
1500	1345	.liveness = liveness.*.?,
1501		.wip = wip,
	1346	.wip = try .init(&o.builder, .{
	1347	.function = llvm_function,
	1348	.strip = owner_mod.strip,
	1349	}),
1502	1350	.is_naked = fn_info.cc == .naked,
1503	1351	.fuzz = fuzz,
1504		.ret_ptr = ret_ptr,
1505		.args = args.items,
1506	1352	.arg_index = 0,
1507	1353	.arg_inline_index = 0,
1508	1354	.func_inst_table = .empty,
...	...	@@ -1516,14 +1362,18 @@ pub const Object = struct {
1516	1362	.base_line = zcu.navSrcLine(func.owner_nav),
1517	1363	.prev_dbg_line = 0,
1518	1364	.prev_dbg_column = 0,
1519		.err_ret_trace = err_ret_trace,
1520	1365	.disable_intrinsics = disable_intrinsics,
1521	1366	.allowzero_access = false,
	1367
	1368	.ret_ptr = undefined, // populated by `genMainBody`
	1369	.err_ret_trace = undefined, // populated by `genMainBody`
	1370	.args = undefined, // populated by `genMainBody`
1522	1371	};
1523	1372	defer fg.deinit();
1524		deinit_wip = false;
1525	1373
1526		try fg.genBody(air.getMainBody(), .poi);
	1374	fg.wip.cursor = .{ .block = try fg.wip.block(0, "Entry") };
	1375
	1376	try fg.genMainBody();
1527	1377
1528	1378	// If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole
1529	1379	// function as considering null pointers valid so that LLVM's optimizers don't remove these
...	...	@@ -1589,10 +1439,10 @@ pub const Object = struct {
1589	1439	// represent (because it doesn't have runtime bits), we instead lower as the zero-size
1590	1440	// type `[0 x i8]`. I don't think the type on an extern declaration actually does much
1591	1441	// anyway.
1592		if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty);
	1442	if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty, .in_memory);
1593	1443	break :ty try o.builder.arrayType(0, .i8);
1594	1444	} else if (nav_ty.hasRuntimeBits(zcu)) ty: {
1595		break :ty try o.lowerType(nav_ty);
	1445	break :ty try o.lowerType(nav_ty, .in_memory);
1596	1446	} else {
1597	1447	// This is a non-extern zero-bit `Nav`---we're not interested in it.
1598	1448	// TODO: we might need to rethink this a little under incremental compilation. If a
...	...	@@ -1685,7 +1535,7 @@ pub const Object = struct {
1685	1535	llvm_variable.setAlignment(llvm_align, &o.builder);
1686	1536	llvm_variable.setSection(llvm_section, &o.builder);
1687	1537	llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder);
1688		try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value), &o.builder);
	1538	try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value, .in_memory), &o.builder);
1689	1539	llvm_variable.setThreadLocal(tl: {
1690	1540	if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic;
1691	1541	break :tl .default;
...	...	@@ -2283,7 +2133,7 @@ pub const Object = struct {
2283	2133	defer debug_param_types.deinit(gpa);
2284	2134
2285	2135	// Return type goes first.
2286		if (firstParamSRet(fn_info, zcu, target)) {
	2136	if (try fnReturnStrat(o, fn_info) == .sret) {
2287	2137	// Actual return type is void, then first arg is the sret pointer.
2288	2138	const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type));
2289	2139	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void));
...	...	@@ -2831,12 +2681,12 @@ pub const Object = struct {
2831	2681	if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder);
2832	2682
2833	2683	var it = iterateParamTypes(o, fn_info);
2834		if (firstParamSRet(fn_info, zcu, target)) {
	2684	if (try fnReturnStrat(o, fn_info) == .sret) {
2835	2685	// Sret pointers must not be address 0
2836	2686	try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
2837	2687	try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder);
2838	2688
2839		const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type));
	2689	const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type), .in_memory);
2840	2690	try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder);
2841	2691	it.llvm_index += 1;
2842	2692	} else if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) \|s\| switch (s) {
...	...	@@ -2878,9 +2728,7 @@ pub const Object = struct {
2878	2728	},
2879	2729	.byref => {
2880	2730	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
2881		const param_llvm_ty = try o.lowerType(param_ty);
2882		const alignment = param_ty.abiAlignment(zcu);
2883		try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty);
	2731	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, it.byval_attr, param_ty);
2884	2732	},
2885	2733	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
2886	2734	.slice => {
...	...	@@ -2990,14 +2838,31 @@ pub const Object = struct {
2990	2838	}
2991	2839	}
2992	2840
2993		pub fn errorIntType(o: *Object) Allocator.Error!Builder.Type {
2994		return o.builder.intType(o.zcu.errorSetBits());
	2841	pub const TypeRepr = enum {
	2842	/// The representation of the type when it is being manipulated as a value in a function.
	2843	/// e.g. Zig `u5` -> LLVM `i5`
	2844	by_value,
	2845	/// The representation of the type when it is stored in memory.
	2846	/// e.g. Zig `u5` -> LLVM `i8`
	2847	in_memory,
	2848	};
	2849
	2850	pub fn errorIntType(o: *Object, repr: TypeRepr) Allocator.Error!Builder.Type {
	2851	return o.builder.intType(switch (repr) {
	2852	.by_value => o.zcu.errorSetBits(),
	2853	.in_memory => @intCast(Type.anyerror.abiSize(o.zcu) * 8),
	2854	});
2995	2855	}
2996	2856
2997		pub fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type {
	2857	pub fn lowerType(o: *Object, t: Type, repr: TypeRepr) Allocator.Error!Builder.Type {
2998	2858	const zcu = o.zcu;
2999	2859	const target = zcu.getTarget();
3000	2860	const ip = &zcu.intern_pool;
	2861
	2862	if (repr == .by_value) {
	2863	assert(!isByRef(t, zcu)); // by-ref types must only be manipulated in memory
	2864	}
	2865
3001	2866	return switch (t.toIntern()) {
3002	2867	.u0_type => unreachable, // no runtime bits
3003	2868	inline .u1_type,
...	...	@@ -3013,7 +2878,10 @@ pub const Object = struct {
3013	2878	.u80_type,
3014	2879	.u128_type,
3015	2880	.i128_type,
3016		=> \|tag\| @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),
	2881	=> \|tag\| switch (repr) {
	2882	.by_value => @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),
	2883	.in_memory => try o.builder.intType(@intCast(t.abiSize(zcu) * 8)),
	2884	},
3017	2885	.usize_type, .isize_type => try o.builder.intType(target.ptrBitWidth()),
3018	2886	inline .c_char_type,
3019	2887	.c_short_type,
...	...	@@ -3048,7 +2916,7 @@ pub const Object = struct {
3048	2916	return .i8;
3049	2917	},
3050	2918	.bool_type => .i1,
3051		.anyerror_type => try o.errorIntType(),
	2919	.anyerror_type => try o.errorIntType(repr),
3052	2920	.void_type => unreachable, // no runtime bits
3053	2921	.type_type => unreachable, // no runtime bits
3054	2922	.comptime_int_type => unreachable, // no runtime bits
...	...	@@ -3068,10 +2936,10 @@ pub const Object = struct {
3068	2936	=> .ptr,
3069	2937	.slice_const_u8_type,
3070	2938	.slice_const_u8_sentinel_0_type,
3071		=> try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize) }),
	2939	=> try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize, repr) }),
3072	2940	.anyerror_void_error_union_type,
3073	2941	.adhoc_inferred_error_set_type,
3074		=> try o.errorIntType(),
	2942	=> try o.errorIntType(repr),
3075	2943	.generic_poison_type => unreachable,
3076	2944	// values, not types
3077	2945	.undef,
...	...	@@ -3097,7 +2965,10 @@ pub const Object = struct {
3097	2965	.none,
3098	2966	=> unreachable,
3099	2967	else => switch (ip.indexToKey(t.toIntern())) {
3100		.int_type => \|int_type\| try o.builder.intType(int_type.bits),
	2968	.int_type => \|int_type\| switch (repr) {
	2969	.by_value => try o.builder.intType(int_type.bits),
	2970	.in_memory => try o.builder.intType(@intCast(t.abiSize(zcu) * 8)),
	2971	},
3101	2972	.ptr_type => \|ptr_type\| type: {
3102	2973	const ptr_ty = try o.builder.ptrType(
3103	2974	toLlvmAddressSpace(ptr_type.flags.address_space, target),
...	...	@@ -3106,18 +2977,18 @@ pub const Object = struct {
3106	2977	.one, .many, .c => ptr_ty,
3107	2978	.slice => try o.builder.structType(.normal, &.{
3108	2979	ptr_ty,
3109		try o.lowerType(.usize),
	2980	try o.lowerType(.usize, repr),
3110	2981	}),
3111	2982	};
3112	2983	},
3113	2984	.array_type => \|array_type\| o.builder.arrayType(
3114	2985	array_type.lenIncludingSentinel(),
3115		try o.lowerType(.fromInterned(array_type.child)),
	2986	try o.lowerType(.fromInterned(array_type.child), repr),
3116	2987	),
3117	2988	.vector_type => \|vector_type\| o.builder.vectorType(
3118	2989	.normal,
3119	2990	vector_type.len,
3120		try o.lowerType(.fromInterned(vector_type.child)),
	2991	try o.lowerType(.fromInterned(vector_type.child), .by_value),
3121	2992	),
3122	2993	.opt_type => \|child_ty\| {
3123	2994	// Must stay in sync with `opt_payload` logic in `lowerPtr`.
...	...	@@ -3127,8 +2998,11 @@ pub const Object = struct {
3127	2998	.runtime, .partially_comptime => {},
3128	2999	}
3129	3000
3130		const payload_ty = try o.lowerType(.fromInterned(child_ty));
3131		if (t.optionalReprIsPayload(zcu)) return payload_ty;
	3001	if (t.optionalReprIsPayload(zcu)) {
	3002	return o.lowerType(.fromInterned(child_ty), repr);
	3003	}
	3004
	3005	const payload_ty = try o.lowerType(.fromInterned(child_ty), repr);
3132	3006
3133	3007	comptime assert(optional_layout_version == 3);
3134	3008	var fields: [3]Builder.Type = .{ payload_ty, .i8, undefined };
...	...	@@ -3146,7 +3020,7 @@ pub const Object = struct {
3146	3020	.error_union_type => \|error_union_type\| {
3147	3021	// Must stay in sync with `codegen.errUnionPayloadOffset`.
3148	3022	// See logic in `lowerPtr`.
3149		const error_type = try o.errorIntType();
	3023	const error_type = try o.errorIntType(repr);
3150	3024
3151	3025	switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) {
3152	3026	.fully_comptime => unreachable,
...	...	@@ -3154,7 +3028,7 @@ pub const Object = struct {
3154	3028	.runtime, .partially_comptime => {},
3155	3029	}
3156	3030
3157		const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type));
	3031	const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type), repr);
3158	3032
3159	3033	const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu);
3160	3034	const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits()));
...	...	@@ -3189,16 +3063,14 @@ pub const Object = struct {
3189	3063	},
3190	3064	.simple_type => unreachable,
3191	3065	.struct_type => {
3192		if (o.type_map.get(t.toIntern())) \|value\| return value;
3193
3194	3066	const struct_type = ip.loadStructType(t.toIntern());
3195	3067
3196	3068	if (struct_type.layout == .@"packed") {
3197		const int_ty = try o.lowerType(.fromInterned(struct_type.packed_backing_int_type));
3198		try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3199		return int_ty;
	3069	return o.lowerType(.fromInterned(struct_type.packed_backing_int_type), repr);
3200	3070	}
3201	3071
	3072	if (o.type_map.get(t.toIntern())) \|value\| return value;
	3073
3202	3074	assert(struct_type.size > 0);
3203	3075
3204	3076	var llvm_field_types: std.ArrayList(Builder.Type) = .empty;
...	...	@@ -3232,7 +3104,7 @@ pub const Object = struct {
3232	3104
3233	3105	if (!field_ty.hasRuntimeBits(zcu)) continue;
3234	3106
3235		try llvm_field_types.append(o.gpa, try o.lowerType(field_ty));
	3107	try llvm_field_types.append(o.gpa, try o.lowerType(field_ty, repr));
3236	3108
3237	3109	offset += field_ty.abiSize(zcu);
3238	3110	}
...	...	@@ -3288,7 +3160,7 @@ pub const Object = struct {
3288	3160	if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {
3289	3161	continue;
3290	3162	}
3291		try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty)));
	3163	try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty), repr));
3292	3164
3293	3165	offset += Type.fromInterned(field_ty).abiSize(zcu);
3294	3166	}
...	...	@@ -3305,28 +3177,24 @@ pub const Object = struct {
3305	3177	return o.builder.structType(.normal, llvm_field_types.items);
3306	3178	},
3307	3179	.union_type => {
3308		if (o.type_map.get(t.toIntern())) \|value\| return value;
3309
3310	3180	const union_obj = ip.loadUnionType(t.toIntern());
3311	3181
3312	3182	if (union_obj.layout == .@"packed") {
3313		const int_ty = try o.lowerType(.fromInterned(union_obj.packed_backing_int_type));
3314		try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3315		return int_ty;
	3183	return o.lowerType(.fromInterned(union_obj.packed_backing_int_type), repr);
3316	3184	}
3317	3185
3318		assert(union_obj.size > 0);
3319
3320	3186	const layout = Type.getUnionLayout(union_obj, zcu);
3321	3187
3322	3188	if (layout.payload_size == 0) {
3323		const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));
3324		try o.type_map.put(o.gpa, t.toIntern(), enum_tag_ty);
3325		return enum_tag_ty;
	3189	return o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
3326	3190	}
3327	3191
	3192	if (o.type_map.get(t.toIntern())) \|value\| return value;
	3193
	3194	assert(union_obj.size > 0);
	3195
3328	3196	const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]);
3329		const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty);
	3197	const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty, repr);
3330	3198
3331	3199	const payload_ty = ty: {
3332	3200	if (layout.most_aligned_field_size == layout.payload_size) {
...	...	@@ -3352,7 +3220,7 @@ pub const Object = struct {
3352	3220	);
3353	3221	return ty;
3354	3222	}
3355		const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));
	3223	const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
3356	3224
3357	3225	// Put the tag before or after the payload depending on which one's
3358	3226	// alignment is greater.
...	...	@@ -3381,9 +3249,9 @@ pub const Object = struct {
3381	3249	return ty;
3382	3250	},
3383	3251	.opaque_type, .spirv_type => unreachable, // no runtime bits
3384		.enum_type => try o.lowerType(t.intTagType(zcu)),
	3252	.enum_type => try o.lowerType(t.intTagType(zcu), repr),
3385	3253	.func_type => \|func_type\| try o.lowerFnType(t, func_type),
3386		.error_set_type, .inferred_error_set_type => try o.errorIntType(),
	3254	.error_set_type, .inferred_error_set_type => try o.errorIntType(repr),
3387	3255	// values, not types
3388	3256	.undef,
3389	3257	.simple_value,
...	...	@@ -3415,12 +3283,12 @@ pub const Object = struct {
3415	3283
3416	3284	assert(fn_ty.fnHasRuntimeBits(zcu));
3417	3285
3418		const ret_ty = try lowerFnRetTy(o, fn_info);
	3286	const ret_strat = try fnReturnStrat(o, fn_info);
3419	3287
3420	3288	var llvm_params: std.ArrayList(Builder.Type) = .empty;
3421	3289	defer llvm_params.deinit(o.gpa);
3422	3290
3423		if (firstParamSRet(fn_info, zcu, target)) {
	3291	if (ret_strat == .sret) {
3424	3292	try llvm_params.append(o.gpa, .ptr);
3425	3293	}
3426	3294
...	...	@@ -3435,7 +3303,7 @@ pub const Object = struct {
3435	3303	.no_bits => continue,
3436	3304	.byval => {
3437	3305	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3438		try llvm_params.append(o.gpa, try o.lowerType(param_ty));
	3306	try llvm_params.append(o.gpa, try o.lowerType(param_ty, if (isByRef(param_ty, zcu)) .in_memory else .by_value));
3439	3307	},
3440	3308	.byref, .byref_mut => {
3441	3309	try llvm_params.append(o.gpa, .ptr);
...	...	@@ -3450,7 +3318,7 @@ pub const Object = struct {
3450	3318	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3451	3319	try llvm_params.appendSlice(o.gpa, &.{
3452	3320	try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)),
3453		try o.lowerType(.usize),
	3321	try o.lowerType(.usize, .by_value),
3454	3322	});
3455	3323	},
3456	3324	.multiple_llvm_types => {
...	...	@@ -3458,7 +3326,7 @@ pub const Object = struct {
3458	3326	},
3459	3327	.float_array => \|count\| {
3460	3328	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3461		const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?);
	3329	const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?, .in_memory);
3462	3330	try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));
3463	3331	},
3464	3332	.i32_array, .i64_array => \|arr_len\| {
...	...	@@ -3470,14 +3338,19 @@ pub const Object = struct {
3470	3338	},
3471	3339	};
3472	3340
3473		return o.builder.fnType(
3474		ret_ty,
3475		llvm_params.items,
3476		if (fn_info.is_var_args) .vararg else .normal,
3477		);
	3341	const llvm_ret_ty: Builder.Type = switch (ret_strat) {
	3342	.void, .sret => .void,
	3343	.by_val => try o.lowerType(.fromInterned(fn_info.return_type), .by_value),
	3344	.mem_cast => \|llvm_ret_ty\| llvm_ret_ty,
	3345	};
	3346	const llvm_fn_kind: Builder.Type.Function.Kind = switch (fn_info.is_var_args) {
	3347	true => .vararg,
	3348	false => .normal,
	3349	};
	3350	return o.builder.fnType(llvm_ret_ty, llvm_params.items, llvm_fn_kind);
3478	3351	}
3479	3352
3480		pub fn lowerValue(o: *Object, arg_val: InternPool.Index) Allocator.Error!Builder.Constant {
	3353	pub fn lowerValue(o: *Object, arg_val: InternPool.Index, repr: TypeRepr) Allocator.Error!Builder.Constant {
3481	3354	const zcu = o.zcu;
3482	3355	const ip = &zcu.intern_pool;
3483	3356	const target = zcu.getTarget();
...	...	@@ -3509,7 +3382,7 @@ pub const Object = struct {
3509	3382	.inferred_error_set_type,
3510	3383	=> unreachable, // types, not values
3511	3384
3512		.undef => return o.builder.undefConst(try o.lowerType(ty)),
	3385	.undef => return o.builder.undefConst(try o.lowerType(ty, repr)),
3513	3386	.simple_value => \|simple_value\| switch (simple_value) {
3514	3387	.void => unreachable, // non-runtime value
3515	3388	.null => unreachable, // non-runtime value
...	...	@@ -3524,15 +3397,15 @@ pub const Object = struct {
3524	3397	.int => {
3525	3398	var bigint_space: Value.BigIntSpace = undefined;
3526	3399	const bigint = val.toBigInt(&bigint_space, zcu);
3527		const llvm_int_ty = try o.builder.intType(ty.intInfo(zcu).bits);
	3400	const llvm_int_ty = try o.lowerType(ty, repr);
3528	3401	return o.builder.bigIntConst(llvm_int_ty, bigint);
3529	3402	},
3530	3403	.err => \|err\| {
3531	3404	const int = zcu.intern_pool.getErrorValueIfExists(err.name).?;
3532		return o.builder.intConst(try o.errorIntType(), int);
	3405	return o.builder.intConst(try o.errorIntType(repr), int);
3533	3406	},
3534	3407	.error_union => \|error_union\| {
3535		const llvm_error_ty = try o.errorIntType();
	3408	const llvm_error_ty = try o.errorIntType(repr);
3536	3409	const llvm_error_value = switch (error_union.val) {
3537	3410	.err_name => \|name\| try o.builder.intConst(
3538	3411	llvm_error_ty,
...	...	@@ -3550,8 +3423,8 @@ pub const Object = struct {
3550	3423	const payload_align = payload_type.abiAlignment(zcu);
3551	3424	const error_align = Type.errorAbiAlignment(zcu);
3552	3425	const llvm_payload_value = switch (error_union.val) {
3553		.err_name => try o.builder.undefConst(try o.lowerType(payload_type)),
3554		.payload => \|payload\| try o.lowerValue(payload),
	3426	.err_name => try o.builder.undefConst(try o.lowerType(payload_type, repr)),
	3427	.payload => \|payload\| try o.lowerValue(payload, repr),
3555	3428	};
3556	3429
3557	3430	var fields: [3]Builder.Type = undefined;
...	...	@@ -3566,7 +3439,7 @@ pub const Object = struct {
3566	3439	fields[0] = vals[0].typeOf(&o.builder);
3567	3440	fields[1] = vals[1].typeOf(&o.builder);
3568	3441
3569		const llvm_ty = try o.lowerType(ty);
	3442	const llvm_ty = try o.lowerType(ty, repr);
3570	3443	const llvm_ty_fields = llvm_ty.structFields(&o.builder);
3571	3444	if (llvm_ty_fields.len > 2) {
3572	3445	assert(llvm_ty_fields.len == 3);
...	...	@@ -3578,7 +3451,7 @@ pub const Object = struct {
3578	3451	fields[0..llvm_ty_fields.len],
3579	3452	), vals[0..llvm_ty_fields.len]);
3580	3453	},
3581		.enum_tag => \|enum_tag\| o.lowerValue(enum_tag.int),
	3454	.enum_tag => \|enum_tag\| o.lowerValue(enum_tag.int, repr),
3582	3455	.float => switch (ty.floatBits(target)) {
3583	3456	16 => if (backendSupportsF16(target))
3584	3457	try o.builder.halfConst(val.toFloat(f16, zcu))
...	...	@@ -3594,9 +3467,9 @@ pub const Object = struct {
3594	3467	else => unreachable,
3595	3468	},
3596	3469	.ptr => try o.lowerPtr(arg_val, 0),
3597		.slice => \|slice\| return o.builder.structConst(try o.lowerType(ty), &.{
3598		try o.lowerValue(slice.ptr),
3599		try o.lowerValue(slice.len),
	3470	.slice => \|slice\| return o.builder.structConst(try o.lowerType(ty, repr), &.{
	3471	try o.lowerValue(slice.ptr, repr),
	3472	try o.lowerValue(slice.len, repr),
3600	3473	}),
3601	3474	.opt => \|opt\| {
3602	3475	comptime assert(optional_layout_version == 3);
...	...	@@ -3606,7 +3479,7 @@ pub const Object = struct {
3606	3479	if (!payload_ty.hasRuntimeBits(zcu)) {
3607	3480	return non_null_bit;
3608	3481	}
3609		const llvm_ty = try o.lowerType(ty);
	3482	const llvm_ty = try o.lowerType(ty, repr);
3610	3483	if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) {
3611	3484	.none => switch (llvm_ty.tag(&o.builder)) {
3612	3485	.integer => try o.builder.intConst(llvm_ty, 0),
...	...	@@ -3614,15 +3487,15 @@ pub const Object = struct {
3614	3487	.structure => try o.builder.zeroInitConst(llvm_ty),
3615	3488	else => unreachable,
3616	3489	},
3617		else => \|payload\| try o.lowerValue(payload),
	3490	else => \|payload\| try o.lowerValue(payload, repr),
3618	3491	};
3619	3492	assert(payload_ty.zigTypeTag(zcu) != .@"fn");
3620	3493
3621	3494	var fields: [3]Builder.Type = undefined;
3622	3495	var vals: [3]Builder.Constant = undefined;
3623	3496	vals[0] = switch (opt.val) {
3624		.none => try o.builder.undefConst(try o.lowerType(payload_ty)),
3625		else => \|payload\| try o.lowerValue(payload),
	3497	.none => try o.builder.undefConst(try o.lowerType(payload_ty, repr)),
	3498	else => \|payload\| try o.lowerValue(payload, repr),
3626	3499	};
3627	3500	vals[1] = non_null_bit;
3628	3501	fields[0] = vals[0].typeOf(&o.builder);
...	...	@@ -3639,14 +3512,14 @@ pub const Object = struct {
3639	3512	fields[0..llvm_ty_fields.len],
3640	3513	), vals[0..llvm_ty_fields.len]);
3641	3514	},
3642		.bitpack => \|bitpack\| return o.lowerValue(bitpack.backing_int_val),
	3515	.bitpack => \|bitpack\| return o.lowerValue(bitpack.backing_int_val, repr),
3643	3516	.aggregate => \|aggregate\| switch (ip.indexToKey(ty.toIntern())) {
3644	3517	.array_type => \|array_type\| switch (aggregate.storage) {
3645	3518	.bytes => \|bytes\| try o.builder.stringConst(try o.builder.string(
3646	3519	bytes.toSlice(array_type.lenIncludingSentinel(), ip),
3647	3520	)),
3648	3521	.elems => \|elems\| {
3649		const array_ty = try o.lowerType(ty);
	3522	const array_ty = try o.lowerType(ty, repr);
3650	3523	const elem_ty = array_ty.childType(&o.builder);
3651	3524	assert(elems.len == array_ty.aggregateLen(&o.builder));
3652	3525
...	...	@@ -3664,7 +3537,7 @@ pub const Object = struct {
3664	3537
3665	3538	var need_unnamed = false;
3666	3539	for (vals, fields, elems) \|result_val, result_field, elem\| {
3667		result_val.* = try o.lowerValue(elem);
	3540	result_val.* = try o.lowerValue(elem, repr);
3668	3541	result_field.* = result_val.typeOf(&o.builder);
3669	3542	if (result_field.* != elem_ty) need_unnamed = true;
3670	3543	}
...	...	@@ -3676,7 +3549,7 @@ pub const Object = struct {
3676	3549	.repeated_elem => \|elem\| {
3677	3550	const len: usize = @intCast(array_type.len);
3678	3551	const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
3679		const array_ty = try o.lowerType(ty);
	3552	const array_ty = try o.lowerType(ty, repr);
3680	3553	const elem_ty = array_ty.childType(&o.builder);
3681	3554
3682	3555	const ExpectedContents = extern struct {
...	...	@@ -3692,12 +3565,12 @@ pub const Object = struct {
3692	3565	defer allocator.free(fields);
3693	3566
3694	3567	var need_unnamed = false;
3695		@memset(vals[0..len], try o.lowerValue(elem));
	3568	@memset(vals[0..len], try o.lowerValue(elem, repr));
3696	3569	@memset(fields[0..len], vals[0].typeOf(&o.builder));
3697	3570	if (fields[0] != elem_ty) need_unnamed = true;
3698	3571
3699	3572	if (array_type.sentinel != .none) {
3700		vals[len] = try o.lowerValue(array_type.sentinel);
	3573	vals[len] = try o.lowerValue(array_type.sentinel, repr);
3701	3574	fields[len] = vals[len].typeOf(&o.builder);
3702	3575	if (fields[len] != elem_ty) need_unnamed = true;
3703	3576	}
...	...	@@ -3709,7 +3582,7 @@ pub const Object = struct {
3709	3582	},
3710	3583	},
3711	3584	.vector_type => \|vector_type\| {
3712		const vector_ty = try o.lowerType(ty);
	3585	const vector_ty = try o.lowerType(ty, repr);
3713	3586	switch (aggregate.storage) {
3714	3587	.bytes, .elems => {
3715	3588	const ExpectedContents = [Builder.expected_fields_len]Builder.Constant;
...	...	@@ -3724,7 +3597,7 @@ pub const Object = struct {
3724	3597	result_val.* = try o.builder.intConst(.i8, byte);
3725	3598	},
3726	3599	.elems => \|elems\| for (vals, elems) \|*result_val, elem\| {
3727		result_val.* = try o.lowerValue(elem);
	3600	result_val.* = try o.lowerValue(elem, .by_value);
3728	3601	},
3729	3602	.repeated_elem => unreachable,
3730	3603	}
...	...	@@ -3732,12 +3605,12 @@ pub const Object = struct {
3732	3605	},
3733	3606	.repeated_elem => \|elem\| return o.builder.splatConst(
3734	3607	vector_ty,
3735		try o.lowerValue(elem),
	3608	try o.lowerValue(elem, .by_value),
3736	3609	),
3737	3610	}
3738	3611	},
3739	3612	.tuple_type => \|tuple\| {
3740		const struct_ty = try o.lowerType(ty);
	3613	const struct_ty = try o.lowerType(ty, repr);
3741	3614	const llvm_len = struct_ty.aggregateLen(&o.builder);
3742	3615
3743	3616	const ExpectedContents = extern struct {
...	...	@@ -3782,8 +3655,8 @@ pub const Object = struct {
3782	3655
3783	3656	vals[llvm_index] = switch (aggregate.storage) {
3784	3657	.bytes => \|bytes\| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
3785		.elems => \|elems\| try o.lowerValue(elems[field_index]),
3786		.repeated_elem => \|elem\| try o.lowerValue(elem),
	3658	.elems => \|elems\| try o.lowerValue(elems[field_index], repr),
	3659	.repeated_elem => \|elem\| try o.lowerValue(elem, repr),
3787	3660	};
3788	3661	fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3789	3662	if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
...	...	@@ -3812,7 +3685,7 @@ pub const Object = struct {
3812	3685	},
3813	3686	.struct_type => {
3814	3687	const struct_type = ip.loadStructType(ty.toIntern());
3815		const struct_ty = try o.lowerType(ty);
	3688	const struct_ty = try o.lowerType(ty, repr);
3816	3689	assert(struct_type.layout != .@"packed");
3817	3690	const llvm_len = struct_ty.aggregateLen(&o.builder);
3818	3691
...	...	@@ -3856,8 +3729,8 @@ pub const Object = struct {
3856	3729
3857	3730	vals[llvm_index] = switch (aggregate.storage) {
3858	3731	.bytes => \|bytes\| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
3859		.elems => \|elems\| try o.lowerValue(elems[field_index]),
3860		.repeated_elem => \|elem\| try o.lowerValue(elem),
	3732	.elems => \|elems\| try o.lowerValue(elems[field_index], repr),
	3733	.repeated_elem => \|elem\| try o.lowerValue(elem, repr),
3861	3734	};
3862	3735	fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3863	3736	if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
...	...	@@ -3887,9 +3760,9 @@ pub const Object = struct {
3887	3760	else => unreachable,
3888	3761	},
3889	3762	.un => \|un\| {
3890		const union_ty = try o.lowerType(ty);
	3763	const union_ty = try o.lowerType(ty, repr);
3891	3764	const layout = ty.unionGetLayout(zcu);
3892		if (layout.payload_size == 0) return o.lowerValue(un.tag);
	3765	if (layout.payload_size == 0) return o.lowerValue(un.tag, repr);
3893	3766
3894	3767	const union_obj = zcu.typeToUnion(ty).?;
3895	3768	const container_layout = union_obj.layout;
...	...	@@ -3910,7 +3783,7 @@ pub const Object = struct {
3910	3783	const padding_len = layout.payload_size;
3911	3784	break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8));
3912	3785	}
3913		const payload = try o.lowerValue(un.val);
	3786	const payload = try o.lowerValue(un.val, repr);
3914	3787	const payload_ty = payload.typeOf(&o.builder);
3915	3788	if (payload_ty != union_ty.structFields(&o.builder)[
3916	3789	@intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align))
...	...	@@ -3925,7 +3798,7 @@ pub const Object = struct {
3925	3798	);
3926	3799	} else p: {
3927	3800	assert(layout.tag_size == 0);
3928		const union_val = try o.lowerValue(un.val);
	3801	const union_val = try o.lowerValue(un.val, repr);
3929	3802	need_unnamed = true;
3930	3803	break :p union_val;
3931	3804	};
...	...	@@ -3935,7 +3808,7 @@ pub const Object = struct {
3935	3808	try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty})
3936	3809	else
3937	3810	union_ty, &.{payload});
3938		const tag = try o.lowerValue(un.tag);
	3811	const tag = try o.lowerValue(un.tag, repr);
3939	3812	const tag_ty = tag.typeOf(&o.builder);
3940	3813	var fields: [3]Builder.Type = undefined;
3941	3814	var vals: [3]Builder.Constant = undefined;
...	...	@@ -3989,8 +3862,8 @@ pub const Object = struct {
3989	3862	},
3990	3863	.int => try o.builder.castConst(
3991	3864	.inttoptr,
3992		try o.builder.intConst(try o.lowerType(.usize), offset),
3993		try o.lowerType(.fromInterned(ptr.ty)),
	3865	try o.builder.intConst(try o.lowerType(.usize, .by_value), offset),
	3866	try o.lowerType(.fromInterned(ptr.ty), .by_value),
3994	3867	),
3995	3868	.eu_payload => \|eu_ptr\| try o.lowerPtr(
3996	3869	eu_ptr,
...	...	@@ -4037,7 +3910,7 @@ pub const Object = struct {
4037	3910	@"addrspace": std.lang.AddressSpace,
4038	3911	) Allocator.Error!Builder.Constant {
4039	3912	const addr: u64 = @"align".toByteUnits().?;
4040		const llvm_usize = try o.lowerType(.usize);
	3913	const llvm_usize = try o.lowerType(.usize, .by_value);
4041	3914	const llvm_addr = try o.builder.intConst(llvm_usize, addr);
4042	3915	const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget()));
4043	3916	return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty);
...	...	@@ -4074,17 +3947,18 @@ pub const Object = struct {
4074	3947	if (gop.found_existing) {
4075	3948	// Keep the greater of the two alignments.
4076	3949	const llvm_variable = gop.value_ptr.*;
4077		const old_align: InternPool.Alignment = .fromLlvm(llvm_variable.getAlignment(&o.builder));
4078		llvm_variable.setAlignment(old_align.maxStrict(@"align").toLlvm(), &o.builder);
	3950	const llvm_old_align = llvm_variable.getAlignment(&o.builder);
	3951	const llvm_new_align = llvm_old_align.max(@"align".toLlvm());
	3952	llvm_variable.setAlignment(llvm_new_align, &o.builder);
4079	3953	return llvm_variable.ptrConst(&o.builder).global.toConst();
4080	3954	}
4081	3955	errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));
4082	3956
4083		const llvm_ty = try o.lowerType(uav_ty);
	3957	const llvm_ty = try o.lowerType(uav_ty, .in_memory);
4084	3958	const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)});
4085	3959	const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace);
4086	3960	gop.value_ptr.* = llvm_variable;
4087		try llvm_variable.setInitializer(try o.lowerValue(uav_val), &o.builder);
	3961	try llvm_variable.setInitializer(try o.lowerValue(uav_val, .in_memory), &o.builder);
4088	3962	llvm_variable.setMutability(.constant, &o.builder);
4089	3963	llvm_variable.setAlignment(@"align".toLlvm(), &o.builder);
4090	3964	const llvm_global = llvm_variable.ptrConst(&o.builder).global;
...	...	@@ -4156,7 +4030,7 @@ pub const Object = struct {
4156	4030	.x86_64_interrupt,
4157	4031	.x86_interrupt,
4158	4032	=> {
4159		const child_type = try lowerType(o, Type.fromInterned(ptr_info.child));
	4033	const child_type = try o.lowerType(.fromInterned(ptr_info.child), .in_memory);
4160	4034	try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder);
4161	4035	},
4162	4036	}
...	...	@@ -4178,14 +4052,15 @@ pub const Object = struct {
4178	4052	o: *Object,
4179	4053	attributes: *Builder.FunctionAttributes.Wip,
4180	4054	llvm_arg_i: u32,
4181		alignment: Builder.Alignment,
4182	4055	byval: bool,
4183		param_llvm_ty: Builder.Type,
	4056	param_ty: Type,
4184	4057	) Allocator.Error!void {
	4058	const llvm_param_ty = try o.lowerType(param_ty, .in_memory);
	4059	const alignment = param_ty.abiAlignment(o.zcu).toLlvm();
4185	4060	try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
4186	4061	try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
4187	4062	try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = .wrap(alignment) }, &o.builder);
4188		if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);
	4063	if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = llvm_param_ty }, &o.builder);
4189	4064	}
4190	4065
4191	4066	pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index {
...	...	@@ -4210,7 +4085,7 @@ pub const Object = struct {
4210	4085	pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index {
4211	4086	const builder = &o.builder;
4212	4087	if (o.errors_len_variable == .none) {
4213		const llvm_err_int_ty = try o.errorIntType();
	4088	const llvm_err_int_ty = try o.errorIntType(.in_memory);
4214	4089	const name = try builder.strtabString("__zig_errors_len");
4215	4090	const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default);
4216	4091	variable_index.setMutability(.constant, builder);
...	...	@@ -4250,9 +4125,9 @@ pub const Object = struct {
4250	4125	const ip = &zcu.intern_pool;
4251	4126	const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
4252	4127
4253		const llvm_usize_ty = try o.lowerType(.usize);
4254		const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0);
4255		const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));
	4128	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
	4129	const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0, .by_value);
	4130	const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .by_value);
4256	4131
4257	4132	function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
4258	4133	try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal);
...	...	@@ -4301,7 +4176,7 @@ pub const Object = struct {
4301	4176	const return_block = try wip.block(1, "Name");
4302	4177	const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) {
4303	4178	.none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered
4304		else => \|tag_val_ip\| try o.lowerValue(tag_val_ip),
	4179	else => \|tag_val_ip\| try o.lowerValue(tag_val_ip, .by_value),
4305	4180	};
4306	4181	try wip_switch.addCase(llvm_tag_val, return_block, &wip);
4307	4182
...	...	@@ -4347,7 +4222,7 @@ pub const Object = struct {
4347	4222	const ip = &zcu.intern_pool;
4348	4223	const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
4349	4224
4350		const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));
	4225	const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .by_value);
4351	4226	function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
4352	4227	try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal);
4353	4228
...	...	@@ -4374,7 +4249,7 @@ pub const Object = struct {
4374	4249
4375	4250	if (loaded_enum.field_values.len > 0) {
4376	4251	for (loaded_enum.field_values.get(ip)) \|tag_val_ip\| {
4377		const llvm_tag_val = try o.lowerValue(tag_val_ip);
	4252	const llvm_tag_val = try o.lowerValue(tag_val_ip, .by_value);
4378	4253	try wip_switch.addCase(llvm_tag_val, named_block, &wip);
4379	4254	}
4380	4255	} else {
...	...	@@ -4411,13 +4286,6 @@ pub const Object = struct {
4411	4286	toLlvmAddressSpace(.generic, o.zcu.getTarget()),
4412	4287	);
4413	4288	}
4414
4415		pub fn ptraddConst(o: Object, wip: Builder.WipFunction, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
4416		if (offset == 0) return ptr;
4417		const llvm_usize_ty = try o.lowerType(.usize);
4418		const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
4419		return wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
4420		}
4421	4289	};
4422	4290
4423	4291	const CallingConventionInfo = struct {

src/codegen/llvm/FuncGen.zig+1097-1176

...	...	@@ -164,7 +164,7 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
164	164	const zcu = o.zcu;
165	165	const ty = val.typeOf(zcu);
166	166	if (!isByRef(ty, zcu)) {
167		return o.lowerValue(val.toIntern());
	167	return o.lowerValue(val.toIntern(), .by_value);
168	168	} else {
169	169	// We need a pointer to a global constant, i.e. a UAV.
170	170	return o.lowerUavRef(
...	...	@@ -175,7 +175,157 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
175	175	}
176	176	}
177	177
178		pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void {
	178	/// Populates `fg.ret_ptr`, `fg.err_ret_trace`, and `fg.args` based on the parameters of the
	179	/// function type, then generates the entire function body.
	180	///
	181	/// The caller may initialize `fg.ret_ptr`, `fg.err_ret_trace`, and `fg.args` to undefined.
	182	pub fn genMainBody(fg: *FuncGen) TodoError!void {
	183	const o = fg.object;
	184	const zcu = o.zcu;
	185	const ip = &zcu.intern_pool;
	186	const comp = zcu.comp;
	187	const gpa = comp.gpa;
	188
	189	const fn_ty: Type = .fromInterned(ip.getNav(fg.nav_index).resolved.?.type);
	190	const fn_info = zcu.typeToFunc(fn_ty).?;
	191	const param_types = fn_info.param_types.get(ip);
	192
	193	var it = iterateParamTypes(o, fn_info);
	194
	195	// Populate `fg.ret_ptr`...
	196	fg.ret_ptr = switch (try fnReturnStrat(o, fn_info)) {
	197	.sret => rp: {
	198	defer it.llvm_index += 1;
	199	break :rp fg.wip.arg(it.llvm_index);
	200	},
	201	else => .none,
	202	};
	203	// ...and `fg.err_ret_trace`...
	204	if (fn_info.cc == .auto and comp.config.any_error_tracing) {
	205	fg.err_ret_trace = fg.wip.arg(it.llvm_index);
	206	it.llvm_index += 1;
	207	} else {
	208	fg.err_ret_trace = .none;
	209	}
	210	// ...and as for `fg.args`, we'll put all of the arguments into this ArrayList, and once that's
	211	// done we'll use its buffer as `fg.args`.
	212	var args: std.ArrayList(Builder.Value) = .empty;
	213	defer args.deinit(gpa);
	214
	215	while (try it.next()) \|lowering\| {
	216	try args.ensureUnusedCapacity(gpa, 1);
	217
	218	switch (lowering) {
	219	.no_bits => continue,
	220	.byval => {
	221	assert(!it.byval_attr);
	222	const param_index = it.zig_index - 1;
	223	const param_ty: Type = .fromInterned(param_types[param_index]);
	224	const param = fg.wip.arg(it.llvm_index - 1);
	225
	226	if (isByRef(param_ty, zcu)) {
	227	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	228	const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
	229	// We don't need to handle non-ABI-sized integer types in memory here since they
	230	// are never by-ref.
	231	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
	232	args.appendAssumeCapacity(arg_ptr);
	233	} else {
	234	args.appendAssumeCapacity(param);
	235	}
	236	},
	237	.byref, .byref_mut => {
	238	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
	239	const param = fg.wip.arg(it.llvm_index - 1);
	240
	241	if (isByRef(param_ty, zcu)) {
	242	args.appendAssumeCapacity(param);
	243	} else {
	244	args.appendAssumeCapacity(try fg.load(param, .none, param_ty, .normal));
	245	}
	246	},
	247	.abi_sized_int => {
	248	assert(!it.byval_attr);
	249	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
	250	const param = fg.wip.arg(it.llvm_index - 1);
	251
	252	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	253	const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
	254	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
	255
	256	if (isByRef(param_ty, zcu)) {
	257	args.appendAssumeCapacity(arg_ptr);
	258	} else {
	259	args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
	260	}
	261	},
	262	.slice => {
	263	assert(!it.byval_attr);
	264	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
	265	assert(!isByRef(param_ty, zcu));
	266	const slice_val = try fg.wip.buildAggregate(
	267	try o.lowerType(param_ty, .by_value),
	268	&.{ fg.wip.arg(it.llvm_index - 2), fg.wip.arg(it.llvm_index - 1) },
	269	"",
	270	);
	271	args.appendAssumeCapacity(slice_val);
	272	},
	273	.multiple_llvm_types => {
	274	assert(!it.byval_attr);
	275	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
	276	const param_alignment = param_ty.abiAlignment(zcu);
	277	const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
	278	const arg_ptr = try fg.buildAlloca(llvm_ty, param_alignment.toLlvm());
	279	const llvm_args_start = it.llvm_index - it.types_len;
	280	for (llvm_args_start.., it.offsets_buffer[0..it.types_len]) \|llvm_arg_index, offset\| {
	281	const param = fg.wip.arg(@intCast(llvm_arg_index));
	282	const part_ptr = try fg.ptraddConst(arg_ptr, offset);
	283	_ = try fg.wip.store(.normal, param, part_ptr, param_alignment.offset(offset).toLlvm());
	284	}
	285
	286	if (isByRef(param_ty, zcu)) {
	287	args.appendAssumeCapacity(arg_ptr);
	288	} else {
	289	args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
	290	}
	291	},
	292	.float_array => {
	293	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
	294	const param = fg.wip.arg(it.llvm_index - 1);
	295
	296	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	297	const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
	298	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
	299
	300	if (isByRef(param_ty, zcu)) {
	301	args.appendAssumeCapacity(arg_ptr);
	302	} else {
	303	args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
	304	}
	305	},
	306	.i32_array, .i64_array => {
	307	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
	308	const param = fg.wip.arg(it.llvm_index - 1);
	309
	310	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	311	const arg_ptr = try fg.buildAlloca(param.typeOfWip(&fg.wip), alignment);
	312	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
	313
	314	if (isByRef(param_ty, zcu)) {
	315	args.appendAssumeCapacity(arg_ptr);
	316	} else {
	317	args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
	318	}
	319	},
	320	}
	321	}
	322
	323	fg.args = args.items;
	324
	325	try fg.genBody(fg.air.getMainBody(), .poi);
	326	}
	327
	328	fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void {
179	329	const o = self.object;
180	330	const zcu = self.object.zcu;
181	331	const ip = &zcu.intern_pool;
...	...	@@ -198,15 +348,16 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
198	348	try fuzz.pcs.append(gpa, pc);
199	349	},
200	350	}
201		for (body, 0..) \|inst, i\| {
	351	for (body) \|inst\| {
202	352	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
203	353
204	354	const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
205	355	// zig fmt: off
206	356
207		// No "scalarize" legalizations are enabled, so these instructions never appear.
208		.legalize_vec_elem_val => unreachable,
209		.legalize_vec_store_elem => unreachable,
	357	// Required due to `.scalarize_bit_cast_vector_non_elementwise` being enabled.
	358	.legalize_vec_elem_val => try self.airLegalizeVecElemVal(inst),
	359	.legalize_vec_store_elem => try self.airLegalizeVecStoreElem(inst),
	360
210	361	// No soft float legalizations are enabled.
211	362	.legalize_compiler_rt_call => unreachable,
212	363
...	...	@@ -309,29 +460,36 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
309	460	.is_err => try self.airIsErr(inst, .ne, false),
310	461	.is_err_ptr => try self.airIsErr(inst, .ne, true),
311	462
312		.alloc => try self.airAlloc(inst),
313		.ret_ptr => try self.airRetPtr(inst),
314		.arg => try self.airArg(inst),
315		.bitcast => try self.airBitCast(inst),
316		.breakpoint => try self.airBreakpoint(inst),
317		.ret_addr => try self.airRetAddr(inst),
318		.frame_addr => try self.airFrameAddress(inst),
319		.@"try" => try self.airTry(inst, false),
320		.try_cold => try self.airTry(inst, true),
321		.try_ptr => try self.airTryPtr(inst, false),
322		.try_ptr_cold => try self.airTryPtr(inst, true),
323		.intcast => try self.airIntCast(inst, false),
324		.intcast_safe => try self.airIntCast(inst, true),
325		.trunc => try self.airTrunc(inst),
326		.fptrunc => try self.airFptrunc(inst),
327		.fpext => try self.airFpext(inst),
328		.load => try self.airLoad(inst),
329		.not => try self.airNot(inst),
330		.store => try self.airStore(inst, false),
331		.store_safe => try self.airStore(inst, true),
332		.assembly => try self.airAssembly(inst),
333		.slice_ptr => try self.airSliceField(inst, 0),
334		.slice_len => try self.airSliceField(inst, 1),
	463	.alloc => try self.airAlloc(inst),
	464	.ret_ptr => try self.airRetPtr(inst),
	465	.arg => try self.airArg(inst),
	466	.bit_cast => try self.airBitCast(inst),
	467	.ptr_cast => try self.airNopCast(inst),
	468	.ptr_from_int => try self.airPtrFromInt(inst),
	469	.int_from_ptr => try self.airIntFromPtr(inst),
	470	.error_cast => try self.airNopCast(inst),
	471	.error_from_int => try self.airNopCast(inst),
	472	.int_from_error => try self.airNopCast(inst),
	473	.union_from_enum => try self.airUnionFromEnum(inst),
	474	.breakpoint => try self.airBreakpoint(inst),
	475	.ret_addr => try self.airRetAddr(inst),
	476	.frame_addr => try self.airFrameAddress(inst),
	477	.@"try" => try self.airTry(inst, false),
	478	.try_cold => try self.airTry(inst, true),
	479	.try_ptr => try self.airTryPtr(inst, false),
	480	.try_ptr_cold => try self.airTryPtr(inst, true),
	481	.int_cast => try self.airIntCast(inst, false),
	482	.int_cast_safe => try self.airIntCast(inst, true),
	483	.trunc => try self.airTrunc(inst),
	484	.fptrunc => try self.airFptrunc(inst),
	485	.fpext => try self.airFpext(inst),
	486	.load => try self.airLoad(inst),
	487	.not => try self.airNot(inst),
	488	.store => try self.airStore(inst, false),
	489	.store_safe => try self.airStore(inst, true),
	490	.assembly => try self.airAssembly(inst),
	491	.slice_ptr => try self.airSliceField(inst, 0),
	492	.slice_len => try self.airSliceField(inst, 1),
335	493
336	494	.ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
337	495	.ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
...	...	@@ -400,8 +558,8 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
400	558	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
401	559	.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
402	560
403		.unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),
404		.unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),
	561	.unwrap_errunion_payload => try self.airErrUnionPayload(inst),
	562	.unwrap_errunion_payload_ptr => try self.airErrUnionPayloadPtr(inst),
405	563	.unwrap_errunion_err => try self.airErrUnionErr(inst, false),
406	564	.unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
407	565	.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
...	...	@@ -409,9 +567,9 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
409	567	.set_err_return_trace => try self.airSetErrReturnTrace(inst),
410	568	.save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),
411	569
412		.wrap_optional => try self.airWrapOptional(body[i..]),
413		.wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]),
414		.wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]),
	570	.wrap_optional => try self.airWrapOptional(inst),
	571	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
	572	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
415	573
416	574	.wasm_memory_size => try self.airWasmMemorySize(inst),
417	575	.wasm_memory_grow => try self.airWasmMemoryGrow(inst),
...	...	@@ -594,7 +752,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
594	752	break :llvm_fn try self.resolveInst(air_call.callee);
595	753	};
596	754	const target = zcu.getTarget();
597		const sret = firstParamSRet(fn_info, zcu, target);
	755	const ret_strat = try fnReturnStrat(o, fn_info);
598	756
599	757	var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);
600	758	defer llvm_args.deinit();
...	...	@@ -612,20 +770,21 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
612	770	.no_suspend, .always_inline, .compile_time => unreachable,
613	771	}
614	772
615		const ret_ptr = if (sret) ret_ptr: {
616		const llvm_ret_ty = try o.lowerType(return_type);
617		try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);
	773	const sret_alloc: ?Builder.Value = switch (ret_strat) {
	774	.sret => sret_alloc: {
	775	try attributes.addParamAttr(0, .{ .sret = try o.lowerType(return_type, .in_memory) }, &o.builder);
618	776
619		const alignment = return_type.abiAlignment(zcu).toLlvm();
620		const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);
621		try llvm_args.append(ret_ptr);
622		break :ret_ptr ret_ptr;
623		} else ret_ptr: {
624		if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) \|s\| switch (s) {
625		.signed => try attributes.addRetAttr(.signext, &o.builder),
626		.unsigned => try attributes.addRetAttr(.zeroext, &o.builder),
627		};
628		break :ret_ptr null;
	777	const ptr = try self.buildZigAlloca(return_type, .none);
	778	try llvm_args.append(ptr);
	779	break :sret_alloc ptr;
	780	},
	781	else => sret_alloc: {
	782	if (ccAbiPromoteInt(fn_info.cc, zcu, .fromInterned(fn_info.return_type))) \|s\| switch (s) {
	783	.signed => try attributes.addRetAttr(.signext, &o.builder),
	784	.unsigned => try attributes.addRetAttr(.zeroext, &o.builder),
	785	};
	786	break :sret_alloc null;
	787	},
629	788	};
630	789
631	790	const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
...	...	@@ -641,9 +800,11 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
641	800	const arg = args[it.zig_index - 1];
642	801	const param_ty = self.typeOf(arg);
643	802	const llvm_arg = try self.resolveInst(arg);
644		const llvm_param_ty = try o.lowerType(param_ty);
645	803	if (isByRef(param_ty, zcu)) {
646	804	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	805	// We don't need to handle non-ABI-sized integer types in memory here since they are
	806	// never by-ref.
	807	const llvm_param_ty = try o.lowerType(param_ty, .in_memory);
647	808	const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
648	809	try llvm_args.append(loaded);
649	810	} else {
...	...	@@ -657,10 +818,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
657	818	if (isByRef(param_ty, zcu)) {
658	819	try llvm_args.append(llvm_arg);
659	820	} else {
660		const alignment = param_ty.abiAlignment(zcu).toLlvm();
661		const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);
662		const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
663		_ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
	821	const arg_ptr = try self.buildZigAlloca(param_ty, .none);
	822	try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
664	823	try llvm_args.append(arg_ptr);
665	824	}
666	825	},
...	...	@@ -669,15 +828,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
669	828	const param_ty = self.typeOf(arg);
670	829	const llvm_arg = try self.resolveInst(arg);
671	830
672		const alignment = param_ty.abiAlignment(zcu).toLlvm();
673		const param_llvm_ty = try o.lowerType(param_ty);
674		const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
675		if (isByRef(param_ty, zcu)) {
676		const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
677		_ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
678		} else {
679		_ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
680		}
	831	const arg_ptr = try self.buildZigAlloca(param_ty, .none);
	832	try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
681	833	try llvm_args.append(arg_ptr);
682	834	},
683	835	.abi_sized_int => {
...	...	@@ -694,9 +846,9 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
694	846	// LLVM does not allow bitcasting structs so we must allocate
695	847	// a local, store as one type, and then load as another type.
696	848	const alignment = param_ty.abiAlignment(zcu).toLlvm();
697		const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);
698		_ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);
699		const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");
	849	const ptr = try self.buildAlloca(int_llvm_ty, alignment);
	850	try self.store(ptr, .none, llvm_arg, param_ty, .normal);
	851	const loaded = try self.wip.load(.normal, int_llvm_ty, ptr, alignment, "");
700	852	try llvm_args.append(loaded);
701	853	}
702	854	},
...	...	@@ -711,19 +863,10 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
711	863	const arg = args[it.zig_index - 1];
712	864	const param_ty = self.typeOf(arg);
713	865	const llvm_arg = try self.resolveInst(arg);
714		const is_by_ref = isByRef(param_ty, zcu);
715	866	const param_alignment = param_ty.abiAlignment(zcu);
716	867	const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
717	868	const arg_ptr = try self.buildAlloca(llvm_ty, param_alignment.toLlvm());
718		if (is_by_ref) _ = try self.wip.callMemCpy(
719		arg_ptr,
720		param_alignment.toLlvm(),
721		llvm_arg,
722		param_alignment.toLlvm(),
723		try o.builder.intValue(try o.lowerType(.usize), param_ty.abiSize(zcu)),
724		.normal,
725		self.disable_intrinsics,
726		) else _ = try self.wip.store(.normal, llvm_arg, arg_ptr, param_alignment.toLlvm());
	869	try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
727	870
728	871	try llvm_args.ensureUnusedCapacity(it.types_len);
729	872	for (it.types_buffer[0..it.types_len], it.offsets_buffer[0..it.types_len]) \|field_ty, offset\| {
...	...	@@ -735,35 +878,34 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
735	878	.float_array => \|count\| {
736	879	const arg = args[it.zig_index - 1];
737	880	const arg_ty = self.typeOf(arg);
738		var llvm_arg = try self.resolveInst(arg);
739		const alignment = arg_ty.abiAlignment(zcu).toLlvm();
740		if (!isByRef(arg_ty, zcu)) {
741		const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
742		_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
743		llvm_arg = ptr;
744		}
	881	const arg_val = try self.resolveInst(arg);
745	882
746		const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);
	883	const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
	884	const ptr = try self.buildZigAlloca(arg_ty, .none);
	885	try self.store(ptr, .none, arg_val, arg_ty, .normal);
	886	break :ptr ptr;
	887	} else arg_val;
	888
	889	const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?, .in_memory);
747	890	const array_ty = try o.builder.arrayType(count, float_ty);
748	891
749		const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
	892	const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_ty.abiAlignment(zcu).toLlvm(), "");
750	893	try llvm_args.append(loaded);
751	894	},
752	895	.i32_array, .i64_array => \|arr_len\| {
753	896	const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
754	897	const arg = args[it.zig_index - 1];
755	898	const arg_ty = self.typeOf(arg);
756		var llvm_arg = try self.resolveInst(arg);
757		const alignment = arg_ty.abiAlignment(zcu).toLlvm();
758		if (!isByRef(arg_ty, zcu)) {
759		const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
760		_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
761		llvm_arg = ptr;
762		}
	899	const arg_val = try self.resolveInst(arg);
763	900
764		const array_ty =
765		try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
766		const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
	901	const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
	902	const ptr = try self.buildZigAlloca(arg_ty, .none);
	903	try self.store(ptr, .none, arg_val, arg_ty, .normal);
	904	break :ptr ptr;
	905	} else arg_val;
	906
	907	const array_ty = try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
	908	const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_ty.abiAlignment(zcu).toLlvm(), "");
767	909	try llvm_args.append(loaded);
768	910	},
769	911	};
...	...	@@ -773,7 +915,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
773	915	{
774	916	// Add argument attributes.
775	917	it = iterateParamTypes(o, fn_info);
776		it.llvm_index += @intFromBool(sret);
	918	it.llvm_index += @intFromBool(ret_strat == .sret);
777	919	it.llvm_index += @intFromBool(err_return_tracing);
778	920	var remaining_inreg_int = cc_info.inreg_int_params;
779	921	var remaining_inreg_float = cc_info.inreg_float_params;
...	...	@@ -802,10 +944,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
802	944	},
803	945	.byref => {
804	946	const param_index = it.zig_index - 1;
805		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
806		const param_llvm_ty = try o.lowerType(param_ty);
807		const alignment = param_ty.abiAlignment(zcu).toLlvm();
808		try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
	947	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[param_index]);
	948	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, it.byval_attr, param_ty);
809	949	},
810	950	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
811	951	// No attributes needed for these.
...	...	@@ -855,7 +995,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
855	995	},
856	996	cc_info.llvm_cc,
857	997	try attributes.finish(&o.builder),
858		try o.lowerType(zig_fn_ty),
	998	try o.lowerType(zig_fn_ty, .by_value),
859	999	llvm_fn,
860	1000	llvm_args.items,
861	1001	"",
...	...	@@ -865,45 +1005,27 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
865	1005	return .none;
866	1006	}
867	1007
868		if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) {
	1008	if (self.liveness.isUnused(inst)) {
869	1009	return .none;
870	1010	}
871	1011
872		const llvm_ret_ty = try o.lowerType(return_type);
873		if (ret_ptr) \|rp\| {
874		if (isByRef(return_type, zcu)) {
875		return rp;
876		} else {
877		// our by-ref status disagrees with sret so we must load.
878		const return_alignment = return_type.abiAlignment(zcu).toLlvm();
879		return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, "");
880		}
881		}
882
883		const abi_ret_ty = try lowerFnRetTy(o, fn_info);
884
885		if (abi_ret_ty != llvm_ret_ty) {
886		// In this case the function return type is honoring the calling convention by having
887		// a different LLVM type than the usual one. We solve this here at the callsite
888		// by using our canonical type, then loading it if necessary.
889		const alignment = return_type.abiAlignment(zcu).toLlvm();
890		const rp = try self.buildAlloca(abi_ret_ty, alignment);
891		_ = try self.wip.store(.normal, call, rp, alignment);
892		return if (isByRef(return_type, zcu))
893		rp
894		else
895		try self.wip.load(.normal, llvm_ret_ty, rp, alignment, "");
896		}
	1012	// We exit this `switch` if we have a pointer to the return value.
	1013	const ret_val_ptr: Builder.Value = switch (ret_strat) {
	1014	.void => return .none,
	1015	.by_val => return call,
897	1016
	1017	.sret => sret_alloc.?,
	1018	.mem_cast => \|llvm_ret_ty\| ret_val_ptr: {
	1019	const alignment = return_type.abiAlignment(zcu).toLlvm();
	1020	const ptr = try self.buildAlloca(llvm_ret_ty, alignment);
	1021	_ = try self.wip.store(.normal, call, ptr, alignment);
	1022	break :ret_val_ptr ptr;
	1023	},
	1024	};
898	1025	if (isByRef(return_type, zcu)) {
899		// our by-ref status disagrees with sret so we must allocate, store,
900		// and return the allocation pointer.
901		const alignment = return_type.abiAlignment(zcu).toLlvm();
902		const rp = try self.buildAlloca(llvm_ret_ty, alignment);
903		_ = try self.wip.store(.normal, call, rp, alignment);
904		return rp;
	1026	return ret_val_ptr;
905	1027	} else {
906		return call;
	1028	return self.load(ret_val_ptr, .none, return_type, .normal);
907	1029	}
908	1030	}
909	1031
...	...	@@ -913,7 +1035,7 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v
913	1035	const target = zcu.getTarget();
914	1036	const panic_func = zcu.funcInfo(zcu.std_lang_decl_values.get(panic_id.toStdLangDecl()));
915	1037	const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;
916		const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty));
	1038	const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty), .by_value);
917	1039
918	1040	const llvm_panic_fn_ref = try o.lowerNavRef(panic_func.owner_nav);
919	1041
...	...	@@ -937,72 +1059,24 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
937	1059	const zcu = o.zcu;
938	1060	const ip = &zcu.intern_pool;
939	1061	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
940		const ret_ty = self.typeOf(un_op);
941
942		if (self.ret_ptr != .none) {
943		const operand = try self.resolveInst(un_op);
944		const val_is_undef = if (un_op.toInterned()) \|i\| Value.fromInterned(i).isUndef(zcu) else false;
945		if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
946		const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));
947		_ = try self.wip.callMemSet(
948		self.ret_ptr,
949		ret_ty.abiAlignment(zcu).toLlvm(),
950		try o.builder.intValue(.i8, 0xaa),
951		len,
952		.normal,
953		self.disable_intrinsics,
954		);
955		const owner_mod = self.ownerModule();
956		if (owner_mod.valgrind) {
957		try self.valgrindMarkUndef(self.ret_ptr, len);
958		}
959		_ = try self.wip.retVoid();
960		return;
961		}
962	1062
963		const unwrapped_operand = operand.unwrap();
964		const unwrapped_ret = self.ret_ptr.unwrap();
965
966		// Return value was stored previously
967		if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) {
968		_ = try self.wip.retVoid();
969		return;
970		}
	1063	const ret_ty = self.typeOf(un_op);
971	1064
972		try self.store(
973		self.ret_ptr,
974		.none,
975		operand,
976		ret_ty,
977		);
978		_ = try self.wip.retVoid();
979		return;
980		}
981	1065	const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
982		if (!ret_ty.hasRuntimeBits(zcu)) {
983		if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
984		// Functions with an empty error set are emitted with an error code
985		// return type and return zero so they can be function pointers coerced
986		// to functions that return anyerror.
987		_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0));
988		} else {
989		_ = try self.wip.retVoid();
990		}
991		return;
992		}
993	1066
994		const abi_ret_ty = try lowerFnRetTy(o, fn_info);
995		const operand = try self.resolveInst(un_op);
	1067	const ret_strat = try fnReturnStrat(o, fn_info);
996	1068	const val_is_undef = if (un_op.toInterned()) \|i\| Value.fromInterned(i).isUndef(zcu) else false;
997		const alignment = ret_ty.abiAlignment(zcu).toLlvm();
	1069	const ret_ty_align = ret_ty.abiAlignment(zcu);
998	1070
999	1071	if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
1000		const llvm_ret_ty = operand.typeOfWip(&self.wip);
1001		const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1002		const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));
	1072	const rp = switch (self.ret_ptr) {
	1073	.none => try self.buildZigAlloca(ret_ty, .none),
	1074	else => \|rp\| rp,
	1075	};
	1076	const len = try o.builder.intValue(try o.lowerType(.usize, .by_value), ret_ty.abiSize(zcu));
1003	1077	_ = try self.wip.callMemSet(
1004	1078	rp,
1005		alignment,
	1079	ret_ty_align.toLlvm(),
1006	1080	try o.builder.intValue(.i8, 0xaa),
1007	1081	len,
1008	1082	.normal,
...	...	@@ -1012,27 +1086,47 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
1012	1086	if (owner_mod.valgrind) {
1013	1087	try self.valgrindMarkUndef(rp, len);
1014	1088	}
1015		_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
1016		return;
1017		}
1018
1019		if (isByRef(ret_ty, zcu)) {
1020		// operand is a pointer however self.ret_ptr is null so that means
1021		// we need to return a value.
1022		_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, ""));
	1089	switch (ret_strat) {
	1090	.void => unreachable, // value is undef so return type cannot be OPV
	1091	.sret => {
	1092	// We just stored directly to `self.ret_ptr`.
	1093	_ = try self.wip.retVoid();
	1094	},
	1095	.by_val => {
	1096	const loaded = try self.load(rp, .none, ret_ty, .normal);
	1097	_ = try self.wip.ret(loaded);
	1098	},
	1099	.mem_cast => \|llvm_abi_ret_ty\| {
	1100	const loaded = try self.wip.load(.normal, llvm_abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
	1101	_ = try self.wip.ret(loaded);
	1102	},
	1103	}
1023	1104	return;
1024	1105	}
1025	1106
1026		const llvm_ret_ty = operand.typeOfWip(&self.wip);
1027		if (abi_ret_ty == llvm_ret_ty) {
1028		_ = try self.wip.ret(operand);
1029		return;
	1107	switch (ret_strat) {
	1108	.void => _ = try self.wip.retVoid(),
	1109	.sret => {
	1110	const operand = try self.resolveInst(un_op);
	1111	try self.store(self.ret_ptr, .none, operand, ret_ty, .normal);
	1112	_ = try self.wip.retVoid();
	1113	},
	1114	.by_val => {
	1115	assert(!isByRef(ret_ty, zcu));
	1116	const operand = try self.resolveInst(un_op);
	1117	_ = try self.wip.ret(operand);
	1118	},
	1119	.mem_cast => \|llvm_ret_ty\| {
	1120	const operand = try self.resolveInst(un_op);
	1121	const ptr: Builder.Value = if (!isByRef(ret_ty, zcu)) ptr: {
	1122	const ptr = try self.buildZigAlloca(ret_ty, .none);
	1123	try self.store(ptr, .none, operand, ret_ty, .normal);
	1124	break :ptr ptr;
	1125	} else operand;
	1126	const ret_val = try self.wip.load(.normal, llvm_ret_ty, ptr, ret_ty_align.toLlvm(), "");
	1127	_ = try self.wip.ret(ret_val);
	1128	},
1030	1129	}
1031
1032		const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1033		_ = try self.wip.store(.normal, operand, rp, alignment);
1034		_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
1035		return;
1036	1130	}
1037	1131
1038	1132	fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
...	...	@@ -1043,33 +1137,32 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
1043	1137	const ptr_ty = self.typeOf(un_op);
1044	1138	const ret_ty = ptr_ty.childType(zcu);
1045	1139	const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1046		if (!ret_ty.hasRuntimeBits(zcu)) {
1047		if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
1048		// Functions with an empty error set are emitted with an error code
1049		// return type and return zero so they can be function pointers coerced
1050		// to functions that return anyerror.
1051		_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0));
1052		} else {
	1140	const ptr = try self.resolveInst(un_op);
	1141	switch (try fnReturnStrat(o, fn_info)) {
	1142	.void => _ = try self.wip.retVoid(),
	1143	.sret => {
	1144	assert(self.ret_ptr != .none);
1053	1145	_ = try self.wip.retVoid();
1054		}
1055		return;
1056		}
1057		if (self.ret_ptr != .none) {
1058		_ = try self.wip.retVoid();
1059		return;
	1146	},
	1147	.by_val => {
	1148	assert(self.ret_ptr == .none);
	1149	const loaded = try self.load(ptr, .none, ret_ty, .normal);
	1150	_ = try self.wip.ret(loaded);
	1151	},
	1152	.mem_cast => \|llvm_abi_ret_ty\| {
	1153	assert(self.ret_ptr == .none);
	1154	const ret_ty_align = ret_ty.abiAlignment(zcu);
	1155	const loaded = try self.wip.load(.normal, llvm_abi_ret_ty, ptr, ret_ty_align.toLlvm(), "");
	1156	_ = try self.wip.ret(loaded);
	1157	},
1060	1158	}
1061		const ptr = try self.resolveInst(un_op);
1062		const abi_ret_ty = try lowerFnRetTy(o, fn_info);
1063		const alignment = ret_ty.abiAlignment(zcu).toLlvm();
1064		_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, ""));
1065		return;
1066	1159	}
1067	1160
1068	1161	fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
1069	1162	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1070	1163	const list = try self.resolveInst(ty_op.operand);
1071	1164	const arg_ty = ty_op.ty.toType();
1072		const llvm_arg_ty = try self.object.lowerType(arg_ty);
	1165	const llvm_arg_ty = try self.object.lowerType(arg_ty, .by_value);
1073	1166
1074	1167	return self.wip.vaArg(list, llvm_arg_ty, "");
1075	1168	}
...	...	@@ -1080,16 +1173,14 @@ fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
1080	1173	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1081	1174	const src_list = try self.resolveInst(ty_op.operand);
1082	1175	const va_list_ty = ty_op.ty.toType();
1083		const llvm_va_list_ty = try o.lowerType(va_list_ty);
1084	1176
1085		const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm();
1086		const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
	1177	const dest_list = try self.buildZigAlloca(va_list_ty, .none);
1087	1178
1088	1179	_ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
1089	1180	return if (isByRef(va_list_ty, zcu))
1090	1181	dest_list
1091	1182	else
1092		try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
	1183	try self.load(dest_list, .none, va_list_ty, .normal);
1093	1184	}
1094	1185
1095	1186	fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -1104,16 +1195,14 @@ fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
1104	1195	const o = self.object;
1105	1196	const zcu = o.zcu;
1106	1197	const va_list_ty = self.typeOfIndex(inst);
1107		const llvm_va_list_ty = try o.lowerType(va_list_ty);
1108	1198
1109		const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm();
1110		const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
	1199	const dest_list = try self.buildZigAlloca(va_list_ty, .none);
1111	1200
1112	1201	_ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
1113	1202	return if (isByRef(va_list_ty, zcu))
1114	1203	dest_list
1115	1204	else
1116		try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
	1205	try self.load(dest_list, .none, va_list_ty, .normal);
1117	1206	}
1118	1207
1119	1208	fn airCmp(
...	...	@@ -1147,13 +1236,7 @@ fn airCmpLteErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
1147	1236	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1148	1237	const operand = try self.resolveInst(un_op);
1149	1238	const errors_len_ptr = try o.getErrorsLen();
1150		const errors_len_val = try self.wip.load(
1151		.normal,
1152		try o.errorIntType(),
1153		errors_len_ptr.toValue(&o.builder),
1154		Type.errorAbiAlignment(o.zcu).toLlvm(),
1155		"",
1156		);
	1239	const errors_len_val = try self.load(errors_len_ptr.toValue(&o.builder), .none, .anyerror, .normal);
1157	1240	return self.wip.icmp(.ule, operand, errors_len_val, "");
1158	1241	}
1159	1242
...	...	@@ -1291,18 +1374,10 @@ fn lowerBlock(
1291	1374
1292	1375	// Create a phi node only if the block returns a value.
1293	1376	if (have_block_result) {
1294		const raw_llvm_ty = try o.lowerType(inst_ty);
1295		const llvm_ty: Builder.Type = ty: {
1296		// If the zig tag type is a function, this represents an actual function body; not
1297		// a pointer to it. LLVM IR allows the call instruction to use function bodies instead
1298		// of function pointers, however the phi makes it a runtime value and therefore
1299		// the LLVM type has to be wrapped in a pointer.
1300		if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) {
1301		break :ty .ptr;
1302		}
1303		break :ty raw_llvm_ty;
	1377	const llvm_ty: Builder.Type = switch (isByRef(inst_ty, zcu)) {
	1378	true => .ptr,
	1379	false => try o.lowerType(inst_ty, .by_value),
1304	1380	};
1305
1306	1381	parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);
1307	1382	const phi = try self.wip.phi(llvm_ty, "");
1308	1383	phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);
...	...	@@ -1408,7 +1483,7 @@ fn lowerSwitchDispatch(
1408	1483	const table_index = try self.wip.conv(
1409	1484	.unsigned,
1410	1485	try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),
1411		try o.lowerType(.usize),
	1486	try o.lowerType(.usize, .by_value),
1412	1487	"",
1413	1488	);
1414	1489	const target_ptr_ptr = try self.ptraddScaled(
...	...	@@ -1433,7 +1508,7 @@ fn lowerSwitchDispatch(
1433	1508	// The switch prongs will correspond to our scalar cases. Ranges will
1434	1509	// be handled by conditional branches in the `else` prong.
1435	1510
1436		const llvm_usize = try o.lowerType(.usize);
	1511	const llvm_usize = try o.lowerType(.usize, .by_value);
1437	1512	const cond_int = if (cond_ty.zigTypeTag(zcu) == .pointer)
1438	1513	try self.wip.cast(.ptrtoint, cond, llvm_usize, "")
1439	1514	else
...	...	@@ -1628,37 +1703,27 @@ fn lowerTry(
1628	1703	const zcu = o.zcu;
1629	1704	const payload_ty = err_union_ty.errorUnionPayload(zcu);
1630	1705	const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
1631		const error_type = try o.errorIntType();
1632	1706
1633		const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{
1634		operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)),
1635		operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)),
1636		} else .{ .none, .none };
	1707	const operand_align: InternPool.Alignment = if (operand_is_ptr) operand_ptr_align else err_union_ty.abiAlignment(zcu);
1637	1708
1638	1709	if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
1639	1710	const loaded = loaded: {
1640		const access_kind: Builder.MemoryAccessKind =
1641		if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
1642
1643		if (!payload_has_bits) {
1644		break :loaded if (operand_is_ptr)
1645		try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "")
1646		else
1647		err_union;
	1711	if (payload_has_bits) {
	1712	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
	1713	} else if (!operand_is_ptr) {
	1714	break :loaded err_union;
1648	1715	}
1649	1716
1650		assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1651	1717	const offset = codegen.errUnionErrorOffset(payload_ty, zcu);
1652	1718	const err_field_ptr = try fg.ptraddConst(err_union, offset);
1653		break :loaded try fg.wip.load(
1654		if (operand_is_ptr) access_kind else .normal,
1655		error_type,
	1719	break :loaded try fg.load(
1656	1720	err_field_ptr,
1657		err_set_align.toLlvm(),
1658		"",
	1721	operand_align.offset(offset),
	1722	.anyerror,
	1723	if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
1659	1724	);
1660	1725	};
1661		const zero = try o.builder.intValue(error_type, 0);
	1726	const zero = try o.builder.intValue(try o.errorIntType(.by_value), 0);
1662	1727	const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
1663	1728
1664	1729	const return_block = try fg.wip.block(1, "TryRet");
...	...	@@ -1672,15 +1737,18 @@ fn lowerTry(
1672	1737	fg.wip.cursor = .{ .block = continue_block };
1673	1738	}
1674	1739	if (is_unused) return .none;
1675		if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;
1676		assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1677		const payload_ptr = try fg.ptraddConst(err_union, codegen.errUnionPayloadOffset(payload_ty, zcu));
	1740
	1741	if (!operand_is_ptr) {
	1742	assert(payload_has_bits); // otherwise the result should be comptime-known
	1743	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
	1744	}
	1745
	1746	const offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
	1747	const payload_ptr = try fg.ptraddConst(err_union, offset);
1678	1748	if (operand_is_ptr) {
1679	1749	return payload_ptr;
1680		} else if (isByRef(payload_ty, zcu)) {
1681		return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
1682	1750	} else {
1683		return fg.wip.load(.normal, try o.lowerType(payload_ty), payload_ptr, payload_align.toLlvm(), "");
	1751	return fg.load(payload_ptr, operand_align.offset(offset), payload_ty, .normal);
1684	1752	}
1685	1753	}
1686	1754
...	...	@@ -1792,8 +1860,8 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod
1792	1860	const table_includes_else = item_count != table_len;
1793	1861
1794	1862	break :jmp_table .{
1795		.min = try o.lowerValue(min.toIntern()),
1796		.max = try o.lowerValue(max.toIntern()),
	1863	.min = try o.lowerValue(min.toIntern(), .by_value),
	1864	.max = try o.lowerValue(max.toIntern(), .by_value),
1797	1865	.in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {
1798	1866	.none, .cold => .none,
1799	1867	.unpredictable => .unpredictable,
...	...	@@ -1951,9 +2019,9 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
1951	2019	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1952	2020	const operand_ty = self.typeOf(ty_op.operand);
1953	2021	const array_ty = operand_ty.childType(zcu);
1954		const llvm_usize = try o.lowerType(.usize);
	2022	const llvm_usize = try o.lowerType(.usize, .by_value);
1955	2023	const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
1956		const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst));
	2024	const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst), .by_value);
1957	2025	const operand = try self.resolveInst(ty_op.operand);
1958	2026	return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
1959	2027	}
...	...	@@ -1970,7 +2038,7 @@ fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value
1970	2038
1971	2039	const dest_ty = self.typeOfIndex(inst);
1972	2040	const dest_scalar_ty = dest_ty.scalarType(zcu);
1973		const dest_llvm_ty = try o.lowerType(dest_ty);
	2041	const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
1974	2042	const target = zcu.getTarget();
1975	2043
1976	2044	if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(
...	...	@@ -2038,7 +2106,7 @@ fn airIntFromFloat(
2038	2106
2039	2107	const dest_ty = self.typeOfIndex(inst);
2040	2108	const dest_scalar_ty = dest_ty.scalarType(zcu);
2041		const dest_llvm_ty = try o.lowerType(dest_ty);
	2109	const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
2042	2110
2043	2111	if (intrinsicsAllowed(operand_scalar_ty, target)) {
2044	2112	// TODO set fast math flag
...	...	@@ -2072,7 +2140,7 @@ fn airIntFromFloat(
2072	2140	compiler_rt_dest_abbrev,
2073	2141	});
2074	2142
2075		const operand_llvm_ty = try o.lowerType(operand_ty);
	2143	const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
2076	2144	const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);
2077	2145	var result = try self.wip.call(
2078	2146	.normal,
...	...	@@ -2097,7 +2165,7 @@ fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!B
2097	2165	fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
2098	2166	const o = fg.object;
2099	2167	const zcu = o.zcu;
2100		const llvm_usize = try o.lowerType(.usize);
	2168	const llvm_usize = try o.lowerType(.usize, .by_value);
2101	2169	switch (ty.ptrSize(zcu)) {
2102	2170	.slice => {
2103	2171	const len = try fg.wip.extractValue(ptr, &.{1}, "");
...	...	@@ -2144,11 +2212,7 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
2144	2212	const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
2145	2213	const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
2146	2214	self.maybeMarkAllowZeroAccess(slice_info);
2147		if (isByRef(elem_ty, zcu)) {
2148		return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind);
2149		} else {
2150		return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm());
2151		}
	2215	return self.load(ptr, elem_align, elem_ty, access_kind);
2152	2216	}
2153	2217
2154	2218	fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -2173,18 +2237,41 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
2173	2237	const elem_ty = array_ty.childType(zcu);
2174	2238	if (isByRef(array_ty, zcu)) {
2175	2239	const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu));
2176		if (isByRef(elem_ty, zcu)) {
2177		const elem_align = elem_ty.abiAlignment(zcu).toLlvm();
2178		return self.loadByRef(elem_ptr, elem_ty, elem_align, .normal);
2179		} else {
2180		return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
2181		}
	2240	return self.load(elem_ptr, .none, elem_ty, .normal);
2182	2241	}
2183	2242
2184	2243	// This branch can be reached for vectors, which are always by-value.
2185	2244	return self.wip.extractElement(array_llvm_val, rhs, "");
2186	2245	}
2187	2246
	2247	fn airLegalizeVecElemVal(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2248	const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	2249	const vec = try fg.resolveInst(bin_op.lhs);
	2250	const index = try fg.resolveInst(bin_op.rhs);
	2251	return fg.wip.extractElement(vec, index, "");
	2252	}
	2253	fn airLegalizeVecStoreElem(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2254	const zcu = fg.object.zcu;
	2255
	2256	const pl_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	2257	const extra = fg.air.extraData(Air.Bin, pl_op.payload).data;
	2258
	2259	const ptr_ty = fg.typeOf(pl_op.operand);
	2260	const vec_ty = ptr_ty.childType(zcu);
	2261
	2262	const ptr_align = ptr_ty.ptrAlignment(zcu);
	2263
	2264	const vec_ptr = try fg.resolveInst(pl_op.operand);
	2265	const index = try fg.resolveInst(extra.lhs);
	2266	const elem = try fg.resolveInst(extra.rhs);
	2267
	2268	const old_vec = try fg.load(vec_ptr, ptr_align, vec_ty, .normal);
	2269	const new_vec = try fg.wip.insertElement(old_vec, elem, index, "");
	2270	try fg.store(vec_ptr, ptr_align, new_vec, vec_ty, .normal);
	2271
	2272	return .none;
	2273	}
	2274
2188	2275	fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2189	2276	const zcu = self.object.zcu;
2190	2277	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...	...	@@ -2197,8 +2284,8 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
2197	2284
2198	2285	return self.load(
2199	2286	try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)),
	2287	ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)),
2200	2288	elem_ty,
2201		ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)).toLlvm(),
2202	2289	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2203	2290	);
2204	2291	}
...	...	@@ -2214,9 +2301,6 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
2214	2301	const base_ptr = try self.resolveInst(bin_op.lhs);
2215	2302	const rhs = try self.resolveInst(bin_op.rhs);
2216	2303
2217		const elem_ptr = ty_pl.ty.toType();
2218		if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
2219
2220	2304	return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));
2221	2305	}
2222	2306
...	...	@@ -2279,7 +2363,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2279	2363	},
2280	2364	.float => {
2281	2365	// bitcast int->float
2282		return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty), "");
	2366	return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty, .by_value), "");
2283	2367	},
2284	2368	}
2285	2369	}
...	...	@@ -2294,11 +2378,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2294	2378	const field_ptr = try self.ptraddConst(struct_llvm_val, offset);
2295	2379	const field_ptr_align = struct_ptr_align.offset(offset);
2296	2380
2297		if (isByRef(field_ty, zcu)) {
2298		return self.loadByRef(field_ptr, field_ty, field_ptr_align.toLlvm(), .normal);
2299		} else {
2300		return self.loadTruncate(.normal, field_ty, field_ptr, field_ptr_align.toLlvm());
2301		}
	2381	return self.load(field_ptr, field_ptr_align, field_ty, .normal);
2302	2382	}
2303	2383
2304	2384	fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -2313,8 +2393,8 @@ fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2313	2393	const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
2314	2394	if (field_offset == 0) return field_ptr;
2315	2395
2316		const res_ty = try o.lowerType(ty_pl.ty.toType());
2317		const llvm_usize = try o.lowerType(.usize);
	2396	const res_ty = try o.lowerType(ty_pl.ty.toType(), .by_value);
	2397	const llvm_usize = try o.lowerType(.usize, .by_value);
2318	2398
2319	2399	const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");
2320	2400	const base_ptr_int = try self.wip.bin(
...	...	@@ -2438,9 +2518,8 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Er
2438	2518	// We avoid taking this path for naked functions because there's no guarantee that such
2439	2519	// functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
2440	2520
2441		const alignment = operand_ty.abiAlignment(zcu).toLlvm();
2442		const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment);
2443		_ = try self.wip.store(.normal, operand, alloca, alignment);
	2521	const alloca = try self.buildZigAlloca(operand_ty, .none);
	2522	try self.store(alloca, .none, operand, operand_ty, .normal);
2444	2523	_ = try self.wip.callIntrinsic(
2445	2524	.normal,
2446	2525	.none,
...	...	@@ -2531,7 +2610,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2531	2610	const output_inst = try self.resolveInst(output.operand);
2532	2611	const output_ty = self.typeOf(output.operand);
2533	2612	assert(output_ty.zigTypeTag(zcu) == .pointer);
2534		const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu));
	2613	const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu), .by_value);
2535	2614
2536	2615	switch (constraint[0]) {
2537	2616	'=' => {},
...	...	@@ -2569,7 +2648,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2569	2648	llvm_ret_indirect[output.index] = false;
2570	2649
2571	2650	const ret_ty = self.typeOfIndex(inst);
2572		llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty);
	2651	llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty, .by_value);
2573	2652	llvm_ret_i += 1;
2574	2653	}
2575	2654
...	...	@@ -2608,9 +2687,8 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2608	2687	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
2609	2688	} else {
2610	2689	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2611		const arg_llvm_ty = try o.lowerType(arg_ty);
2612		const load_inst =
2613		try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
	2690	const arg_llvm_ty = try o.lowerType(arg_ty, .by_value);
	2691	const load_inst = try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
2614	2692	llvm_param_values[llvm_param_i] = load_inst;
2615	2693	llvm_param_types[llvm_param_i] = arg_llvm_ty;
2616	2694	}
...	...	@@ -2621,7 +2699,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2621	2699	} else {
2622	2700	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2623	2701	const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);
2624		_ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment);
	2702	try self.store(arg_ptr, .none, arg_llvm_value, arg_ty, .normal);
2625	2703	llvm_param_values[llvm_param_i] = arg_ptr;
2626	2704	llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);
2627	2705	}
...	...	@@ -2649,7 +2727,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2649	2727	llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {
2650	2728	if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));
2651	2729
2652		break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu));
	2730	break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu), .by_value);
2653	2731	} else .none;
2654	2732
2655	2733	llvm_param_i += 1;
...	...	@@ -2663,19 +2741,13 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2663	2741	if (constraint[0] != '+') continue;
2664	2742
2665	2743	const rw_ty = self.typeOf(output.operand);
2666		const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu));
	2744	const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu), .by_value);
2667	2745	if (llvm_ret_indirect[output.index]) {
2668	2746	llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
2669	2747	llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
2670	2748	} else {
2671		const alignment = rw_ty.abiAlignment(zcu).toLlvm();
2672		const loaded = try self.wip.load(
2673		if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2674		llvm_elem_ty,
2675		llvm_rw_vals[output.index],
2676		alignment,
2677		"",
2678		);
	2749	const access_kind: Builder.MemoryAccessKind = if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	2750	const loaded = try self.load(llvm_rw_vals[output.index], .none, rw_ty.childType(zcu), access_kind);
2679	2751	llvm_param_values[llvm_param_i] = loaded;
2680	2752	llvm_param_types[llvm_param_i] = llvm_elem_ty;
2681	2753	}
...	...	@@ -2835,12 +2907,12 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2835	2907	if (output != .none) {
2836	2908	const output_ptr = try self.resolveInst(output);
2837	2909	const output_ptr_ty = self.typeOf(output);
2838		const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm();
2839		_ = try self.wip.store(
2840		if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2841		output_value,
	2910	try self.store(
2842	2911	output_ptr,
2843		alignment,
	2912	output_ptr_ty.ptrAlignment(zcu),
	2913	output_value,
	2914	output_ptr_ty.childType(zcu),
	2915	if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2844	2916	);
2845	2917	} else {
2846	2918	ret_val = output_value;
...	...	@@ -2863,7 +2935,6 @@ fn airIsNonNull(
2863	2935	const operand = try self.resolveInst(un_op);
2864	2936	const operand_ty = self.typeOf(un_op);
2865	2937	const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
2866		const optional_llvm_ty = try o.lowerType(optional_ty);
2867	2938	const payload_ty = optional_ty.optionalChild(zcu);
2868	2939
2869	2940	const access_kind: Builder.MemoryAccessKind =
...	...	@@ -2873,7 +2944,7 @@ fn airIsNonNull(
2873	2944
2874	2945	if (optional_ty.optionalReprIsPayload(zcu)) {
2875	2946	const loaded = if (operand_is_ptr)
2876		try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
	2947	try self.load(operand, operand_ty.ptrAlignment(zcu), optional_ty, access_kind)
2877	2948	else
2878	2949	operand;
2879	2950	if (payload_ty.isSlice(zcu)) {
...	...	@@ -2884,14 +2955,14 @@ fn airIsNonNull(
2884	2955	));
2885	2956	return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
2886	2957	}
2887		return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), "");
	2958	return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(try o.lowerType(optional_ty, .by_value)), "");
2888	2959	}
2889	2960
2890	2961	comptime assert(optional_layout_version == 3);
2891	2962
2892	2963	if (!payload_ty.hasRuntimeBits(zcu)) {
2893	2964	const loaded = if (operand_is_ptr)
2894		try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
	2965	try self.load(operand, operand_ty.ptrAlignment(zcu), optional_ty, access_kind)
2895	2966	else
2896	2967	operand;
2897	2968	return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
...	...	@@ -2913,8 +2984,7 @@ fn airIsErr(
2913	2984	const operand_ty = self.typeOf(un_op);
2914	2985	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
2915	2986	const payload_ty = err_union_ty.errorUnionPayload(zcu);
2916		const error_type = try o.errorIntType();
2917		const zero = try o.builder.intValue(error_type, 0);
	2987	const zero_err = try o.builder.intValue(try o.errorIntType(.by_value), 0);
2918	2988
2919	2989	const access_kind: Builder.MemoryAccessKind =
2920	2990	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...	...	@@ -2932,10 +3002,10 @@ fn airIsErr(
2932	3002
2933	3003	if (!payload_ty.hasRuntimeBits(zcu)) {
2934	3004	const loaded = if (operand_is_ptr)
2935		try self.wip.load(access_kind, try o.lowerType(err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
	3005	try self.load(operand, operand_ty.ptrAlignment(zcu), err_union_ty, access_kind)
2936	3006	else
2937	3007	operand;
2938		return self.wip.icmp(cond, loaded, zero, "");
	3008	return self.wip.icmp(cond, loaded, zero_err, "");
2939	3009	}
2940	3010	assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref
2941	3011
...	...	@@ -2944,8 +3014,8 @@ fn airIsErr(
2944	3014	else
2945	3015	.none;
2946	3016	const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
2947		const loaded = try self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");
2948		return self.wip.icmp(cond, loaded, zero, "");
	3017	const loaded = try self.load(err_field_ptr, err_align, .anyerror, access_kind);
	3018	return self.wip.icmp(cond, loaded, zero_err, "");
2949	3019	}
2950	3020
2951	3021	fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -2966,7 +3036,6 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
2966	3036	const optional_ptr_ty = self.typeOf(ty_op.operand);
2967	3037	const optional_ty = optional_ptr_ty.childType(zcu);
2968	3038	const payload_ty = optional_ty.optionalChild(zcu);
2969		const non_null_bit = try o.builder.intValue(.i8, 1);
2970	3039
2971	3040	const access_kind: Builder.MemoryAccessKind =
2972	3041	if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...	...	@@ -2976,7 +3045,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
2976	3045
2977	3046	// We have a pointer to a i8. We need to set it to 1 and then return the same pointer.
2978	3047	// Default alignment store because align of the non null bit is 1 anyway.
2979		_ = try self.wip.store(access_kind, non_null_bit, operand, .default);
	3048	try self.store(operand, .@"1", .true, .bool, access_kind);
2980	3049	return operand;
2981	3050	}
2982	3051	if (optional_ty.optionalReprIsPayload(zcu)) {
...	...	@@ -2992,7 +3061,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
2992	3061	self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
2993	3062
2994	3063	// Default alignment store because align of the non null bit is 1 anyway.
2995		_ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default);
	3064	try self.store(non_null_ptr, .@"1", .true, .bool, access_kind);
2996	3065
2997	3066	// Then return the payload pointer (only if it's used).
2998	3067	if (self.liveness.isUnused(inst)) return .none;
...	...	@@ -3016,31 +3085,29 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
3016	3085	return self.optPayloadHandle(operand, optional_ty, false);
3017	3086	}
3018	3087
3019		fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) Allocator.Error!Builder.Value {
3020		const o = self.object;
	3088	fn airErrUnionPayload(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3089	const o = fg.object;
3021	3090	const zcu = o.zcu;
3022		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3023		const operand = try self.resolveInst(ty_op.operand);
3024		const operand_ty = self.typeOf(ty_op.operand);
3025		const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3026		const result_ty = self.typeOfIndex(inst);
3027		const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty;
	3091	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	3092	const operand = try fg.resolveInst(ty_op.operand);
	3093	const err_union_ty = fg.typeOf(ty_op.operand);
	3094	const payload_ty = fg.typeOfIndex(inst);
3028	3095
3029		if (!payload_ty.hasRuntimeBits(zcu)) {
3030		return if (operand_is_ptr) operand else .none;
3031		}
3032		const payload_ptr = try self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3033		if (operand_is_ptr) {
3034		return payload_ptr;
3035		}
	3096	assert(payload_ty.hasRuntimeBits(zcu));
3036	3097	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3037		const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
3038		if (isByRef(payload_ty, zcu)) {
3039		return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
3040		} else {
3041		const payload_llvm_ty = try o.lowerType(payload_ty);
3042		return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");
3043		}
	3098
	3099	const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
	3100	const payload_ptr = try fg.ptraddConst(operand, payload_offset);
	3101	return fg.load(payload_ptr, err_union_ty.abiAlignment(zcu).offset(payload_offset), payload_ty, .normal);
	3102	}
	3103
	3104	fn airErrUnionPayloadPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3105	const o = fg.object;
	3106	const zcu = o.zcu;
	3107	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	3108	const operand = try fg.resolveInst(ty_op.operand);
	3109	const payload_ty = fg.typeOfIndex(inst).childType(zcu);
	3110	return fg.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3044	3111	}
3045	3112
3046	3113	fn airErrUnionErr(
...	...	@@ -3053,40 +3120,28 @@ fn airErrUnionErr(
3053	3120	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3054	3121	const operand = try self.resolveInst(ty_op.operand);
3055	3122	const operand_ty = self.typeOf(ty_op.operand);
3056		const error_type = try o.errorIntType();
3057	3123	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3058		if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
3059		if (operand_is_ptr) {
3060		return operand;
3061		} else {
3062		return o.builder.intValue(error_type, 0);
3063		}
3064		}
3065	3124
3066	3125	const access_kind: Builder.MemoryAccessKind =
3067	3126	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
3068	3127
3069	3128	const payload_ty = err_union_ty.errorUnionPayload(zcu);
3070		if (!payload_ty.hasRuntimeBits(zcu)) {
3071		if (!operand_is_ptr) return operand;
3072	3129
3073		self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
3074
3075		return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");
	3130	if (payload_ty.hasRuntimeBits(zcu)) {
	3131	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
	3132	} else if (!operand_is_ptr) {
	3133	return operand;
3076	3134	}
3077	3135
3078		assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3079
3080	3136	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
3081	3137
3082		const err_align: InternPool.Alignment = a: {
3083		const err_abi_align = Type.anyerror.abiAlignment(zcu);
3084		if (!operand_is_ptr) break :a err_abi_align;
3085		break :a err_abi_align.minStrict(operand_ty.ptrAlignment(zcu));
3086		};
	3138	const ptr_align = if (operand_is_ptr) operand_ty.ptrAlignment(zcu) else err_union_ty.abiAlignment(zcu);
3087	3139
3088		const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3089		return self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");
	3140	const err_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
	3141	const err_align = ptr_align.offset(err_offset);
	3142	const err_ptr = try self.ptraddConst(operand, err_offset);
	3143
	3144	return self.load(err_ptr, err_align, .anyerror, access_kind);
3090	3145	}
3091	3146
3092	3147	fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -3099,7 +3154,7 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
3099	3154	const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);
3100	3155
3101	3156	const payload_ty = err_union_ty.errorUnionPayload(zcu);
3102		const non_error_val = try o.builder.intValue(try o.errorIntType(), 0);
	3157	const non_error_val = try o.builder.intValue(try o.errorIntType(.by_value), 0);
3103	3158
3104	3159	const access_kind: Builder.MemoryAccessKind =
3105	3160	if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...	...	@@ -3107,10 +3162,10 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
3107	3162	self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
3108	3163
3109	3164	{
3110		const error_align = Type.anyerror.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
3111	3165	// First set the non-error value.
3112		const error_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3113		_ = try self.wip.store(access_kind, non_error_val, error_ptr, error_align);
	3166	const error_off = codegen.errUnionErrorOffset(payload_ty, zcu);
	3167	const error_ptr = try self.ptraddConst(operand, error_off);
	3168	try self.store(error_ptr, err_union_ptr_align.offset(error_off), non_error_val, .anyerror, access_kind);
3114	3169	}
3115	3170
3116	3171	// Then return the payload pointer (only if it is used).
...	...	@@ -3142,127 +3197,73 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er
3142	3197	const field_offset = struct_ty.structFieldOffset(field_index, zcu);
3143	3198	const field_align = struct_ty.abiAlignment(zcu).offset(field_offset);
3144	3199	const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset);
3145		return self.load(field_ptr, field_ty, field_align.toLlvm(), .normal);
3146		}
3147
3148		/// As an optimization, we want to avoid unnecessary copies of
3149		/// error union/optional types when returning from a function.
3150		/// Here, we scan forward in the current block, looking to see
3151		/// if the next instruction is a return (ignoring debug instructions).
3152		///
3153		/// The first instruction of `body_tail` is a wrap instruction.
3154		fn isNextRet(
3155		self: *FuncGen,
3156		body_tail: []const Air.Inst.Index,
3157		) bool {
3158		const air_tags = self.air.instructions.items(.tag);
3159		for (body_tail[1..]) \|body_inst\| {
3160		switch (air_tags[@intFromEnum(body_inst)]) {
3161		.ret => return true,
3162		.dbg_stmt => continue,
3163		else => return false,
3164		}
3165		}
3166		// The only way to get here is to hit the end of a loop instruction
3167		// (implicit repeat).
3168		return false;
	3200	return self.load(field_ptr, field_align, field_ty, .normal);
3169	3201	}
3170	3202
3171		fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {
	3203	fn airWrapOptional(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3172	3204	const o = self.object;
3173	3205	const zcu = o.zcu;
3174		const inst = body_tail[0];
3175	3206	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3176	3207	const payload_ty = self.typeOf(ty_op.operand);
3177		const non_null_bit = try o.builder.intValue(.i8, 1);
3178	3208	comptime assert(optional_layout_version == 3);
3179	3209	assert(payload_ty.hasRuntimeBits(zcu));
3180	3210	const operand = try self.resolveInst(ty_op.operand);
3181	3211	const optional_ty = self.typeOfIndex(inst);
3182	3212	if (optional_ty.optionalReprIsPayload(zcu)) return operand;
3183	3213	assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload`
3184		const llvm_optional_ty = try o.lowerType(optional_ty);
3185		const optional_ptr = if (self.isNextRet(body_tail))
3186		self.ret_ptr
3187		else brk: {
3188		const alignment = optional_ty.abiAlignment(zcu).toLlvm();
3189		const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
3190		break :brk optional_ptr;
3191		};
	3214	const optional_ptr = try self.buildZigAlloca(optional_ty, .none);
3192	3215
3193	3216	const payload_ptr = optional_ptr; // payload always at offset 0
3194		try self.store(
3195		payload_ptr,
3196		.none,
3197		operand,
3198		payload_ty,
3199		);
	3217	try self.store(payload_ptr, .none, operand, payload_ty, .normal);
	3218
3200	3219	// Non-null bit immediately after payload (no padding because the bit has alignment 1).
3201	3220	const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu));
3202		_ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);
	3221	try self.store(non_null_ptr, .none, .true, .bool, .normal);
	3222
3203	3223	return optional_ptr;
3204	3224	}
3205	3225
3206		fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {
	3226	fn airWrapErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3207	3227	const o = self.object;
3208	3228	const zcu = o.zcu;
3209		const inst = body_tail[0];
3210	3229	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3211	3230	const err_un_ty = self.typeOfIndex(inst);
3212	3231	const operand = try self.resolveInst(ty_op.operand);
3213	3232	const payload_ty = self.typeOf(ty_op.operand);
3214	3233	assert(payload_ty.hasRuntimeBits(zcu));
3215	3234	assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3216		const ok_err_code = try o.builder.intValue(try o.errorIntType(), 0);
3217		const err_un_llvm_ty = try o.lowerType(err_un_ty);
3218
3219		const result_ptr = if (self.isNextRet(body_tail))
3220		self.ret_ptr
3221		else brk: {
3222		const alignment = err_un_ty.abiAlignment(o.zcu).toLlvm();
3223		const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3224		break :brk result_ptr;
3225		};
	3235	const ok_err_code = try o.builder.intValue(try o.errorIntType(.by_value), 0);
	3236
	3237	const result_ptr = try self.buildZigAlloca(err_un_ty, .none);
3226	3238
3227	3239	const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3228		const error_alignment = Type.anyerror.abiAlignment(o.zcu).toLlvm();
3229		_ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);
	3240	try self.store(err_ptr, .none, ok_err_code, .anyerror, .normal);
	3241
3230	3242	const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3231		try self.store(
3232		payload_ptr,
3233		.none,
3234		operand,
3235		payload_ty,
3236		);
	3243	try self.store(payload_ptr, .none, operand, payload_ty, .normal);
	3244
3237	3245	return result_ptr;
3238	3246	}
3239	3247
3240		fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {
	3248	fn airWrapErrUnionErr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3241	3249	const o = self.object;
3242	3250	const zcu = o.zcu;
3243		const inst = body_tail[0];
3244	3251	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3245	3252	const err_un_ty = self.typeOfIndex(inst);
3246	3253	const payload_ty = err_un_ty.errorUnionPayload(zcu);
3247	3254	const operand = try self.resolveInst(ty_op.operand);
3248	3255	if (!payload_ty.hasRuntimeBits(zcu)) return operand;
3249	3256	assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3250		const err_un_llvm_ty = try o.lowerType(err_un_ty);
3251
3252		const result_ptr = if (self.isNextRet(body_tail))
3253		self.ret_ptr
3254		else brk: {
3255		const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
3256		const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3257		break :brk result_ptr;
3258		};
	3257
	3258	const result_ptr = try self.buildZigAlloca(err_un_ty, .none);
3259	3259
3260	3260	const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3261		const error_alignment = Type.anyerror.abiAlignment(zcu).toLlvm();
3262		_ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
	3261	try self.store(err_ptr, .none, operand, .anyerror, .normal);
	3262
3263	3263	const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3264	3264	// TODO store undef to payload_ptr
3265	3265	_ = payload_ptr;
	3266
3266	3267	return result_ptr;
3267	3268	}
3268	3269
...	...	@@ -3270,7 +3271,7 @@ fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
3270	3271	const o = self.object;
3271	3272	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3272	3273	const index = pl_op.payload;
3273		const llvm_usize = try o.lowerType(.usize);
	3274	const llvm_usize = try o.lowerType(.usize, .by_value);
3274	3275	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{
3275	3276	try o.builder.intValue(.i32, index),
3276	3277	}, "");
...	...	@@ -3280,7 +3281,7 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
3280	3281	const o = self.object;
3281	3282	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3282	3283	const index = pl_op.payload;
3283		const llvm_isize = try o.lowerType(.isize);
	3284	const llvm_isize = try o.lowerType(.isize, .by_value);
3284	3285	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{
3285	3286	try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),
3286	3287	}, "");
...	...	@@ -3307,7 +3308,7 @@ fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3307	3308	.normal,
3308	3309	.none,
3309	3310	if (scalar_ty.isSignedInt(zcu)) .smin else .umin,
3310		&.{try o.lowerType(inst_ty)},
	3311	&.{try o.lowerType(inst_ty, .by_value)},
3311	3312	&.{ lhs, rhs },
3312	3313	"",
3313	3314	);
...	...	@@ -3327,7 +3328,7 @@ fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3327	3328	.normal,
3328	3329	.none,
3329	3330	if (scalar_ty.isSignedInt(zcu)) .smax else .umax,
3330		&.{try o.lowerType(inst_ty)},
	3331	&.{try o.lowerType(inst_ty, .by_value)},
3331	3332	&.{ lhs, rhs },
3332	3333	"",
3333	3334	);
...	...	@@ -3339,7 +3340,7 @@ fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3339	3340	const ptr = try self.resolveInst(bin_op.lhs);
3340	3341	const len = try self.resolveInst(bin_op.rhs);
3341	3342	const inst_ty = self.typeOfIndex(inst);
3342		return self.wip.buildAggregate(try self.object.lowerType(inst_ty), &.{ ptr, len }, "");
	3343	return self.wip.buildAggregate(try self.object.lowerType(inst_ty, .by_value), &.{ ptr, len }, "");
3343	3344	}
3344	3345
3345	3346	fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
...	...	@@ -3370,7 +3371,7 @@ fn airSafeArithmetic(
3370	3371	const scalar_ty = inst_ty.scalarType(zcu);
3371	3372
3372	3373	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3373		const llvm_inst_ty = try o.lowerType(inst_ty);
	3374	const llvm_inst_ty = try o.lowerType(inst_ty, .by_value);
3374	3375	const results =
3375	3376	try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");
3376	3377
...	...	@@ -3420,7 +3421,7 @@ fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3420	3421	.normal,
3421	3422	.none,
3422	3423	if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",
3423		&.{try o.lowerType(inst_ty)},
	3424	&.{try o.lowerType(inst_ty, .by_value)},
3424	3425	&.{ lhs, rhs },
3425	3426	"",
3426	3427	);
...	...	@@ -3459,7 +3460,7 @@ fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3459	3460	.normal,
3460	3461	.none,
3461	3462	if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",
3462		&.{try o.lowerType(inst_ty)},
	3463	&.{try o.lowerType(inst_ty, .by_value)},
3463	3464	&.{ lhs, rhs },
3464	3465	"",
3465	3466	);
...	...	@@ -3498,7 +3499,7 @@ fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3498	3499	.normal,
3499	3500	.none,
3500	3501	if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",
3501		&.{try o.lowerType(inst_ty)},
	3502	&.{try o.lowerType(inst_ty, .by_value)},
3502	3503	&.{ lhs, rhs, .@"0" },
3503	3504	"",
3504	3505	);
...	...	@@ -3542,8 +3543,8 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind)
3542	3543	return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});
3543	3544	}
3544	3545	if (scalar_ty.isSignedInt(zcu)) {
3545		const scalar_llvm_ty = try o.lowerType(scalar_ty);
3546		const inst_llvm_ty = try o.lowerType(inst_ty);
	3546	const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
	3547	const inst_llvm_ty = try o.lowerType(inst_ty, .by_value);
3547	3548
3548	3549	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
3549	3550	var bfa_buf: ExpectedContents = undefined;
...	...	@@ -3617,7 +3618,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo
3617	3618	const lhs = try self.resolveInst(bin_op.lhs);
3618	3619	const rhs = try self.resolveInst(bin_op.rhs);
3619	3620	const inst_ty = self.typeOfIndex(inst);
3620		const inst_llvm_ty = try o.lowerType(inst_ty);
	3621	const inst_llvm_ty = try o.lowerType(inst_ty, .by_value);
3621	3622	const scalar_ty = inst_ty.scalarType(zcu);
3622	3623
3623	3624	if (scalar_ty.isRuntimeFloat()) {
...	...	@@ -3646,7 +3647,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo
3646	3647	defer allocator.free(smin_big_int.limbs);
3647	3648	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
3648	3649	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
3649		try o.lowerType(scalar_ty),
	3650	try o.lowerType(scalar_ty, .by_value),
3650	3651	smin_big_int.toConst(),
3651	3652	));
3652	3653
...	...	@@ -3682,7 +3683,7 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3682	3683	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3683	3684	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3684	3685	const ptr_or_slice = try self.resolveInst(bin_op.lhs);
3685		const llvm_usize_ty = try o.lowerType(.usize);
	3686	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
3686	3687	const ptr_ty = self.typeOf(bin_op.lhs);
3687	3688	const elem_ty = ptr_ty.indexableElem(zcu);
3688	3689	const ptr = switch (ptr_ty.ptrSize(zcu)) {
...	...	@@ -3715,27 +3716,28 @@ fn airOverflow(
3715	3716	assert(isByRef(inst_ty, zcu)); // auto structs are by-ref
3716	3717
3717	3718	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3718		const llvm_inst_ty = try o.lowerType(inst_ty);
3719		const llvm_lhs_ty = try o.lowerType(lhs_ty);
	3719	const llvm_lhs_ty = try o.lowerType(lhs_ty, .by_value);
3720	3720	const results =
3721	3721	try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");
3722	3722
3723	3723	const result_val = try self.wip.extractValue(results, &.{0}, "");
3724	3724	const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
3725	3725
3726		const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();
3727		const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
	3726	const result_alignment = inst_ty.abiAlignment(zcu);
	3727	const alloca_inst = try self.buildZigAlloca(inst_ty, .none);
3728	3728
3729	3729	{
3730	3730	// Store to 'result: IntType' field
3731		const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(0, zcu));
3732		_ = try self.wip.store(.normal, result_val, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());
	3731	const field_off = inst_ty.structFieldOffset(0, zcu);
	3732	const field_ptr = try self.ptraddConst(alloca_inst, field_off);
	3733	try self.store(field_ptr, result_alignment.offset(field_off), result_val, lhs_ty, .normal);
3733	3734	}
3734	3735
3735	3736	{
3736	3737	// Store to 'overflow: u1' field
3737		const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(1, zcu));
3738		_ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
	3738	const field_off = inst_ty.structFieldOffset(1, zcu);
	3739	const field_ptr = try self.ptraddConst(alloca_inst, field_off);
	3740	try self.store(field_ptr, result_alignment.offset(field_off), overflow_bit, inst_ty.fieldType(1, zcu), .normal);
3739	3741	}
3740	3742
3741	3743	return alloca_inst;
...	...	@@ -3787,7 +3789,7 @@ fn buildFloatCmp(
3787	3789	const zcu = o.zcu;
3788	3790	const target = zcu.getTarget();
3789	3791	const scalar_ty = ty.scalarType(zcu);
3790		const scalar_llvm_ty = try o.lowerType(scalar_ty);
	3792	const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
3791	3793
3792	3794	if (intrinsicsAllowed(scalar_ty, target)) {
3793	3795	const cond: Builder.FloatCondition = switch (pred) {
...	...	@@ -3893,7 +3895,7 @@ fn buildFloatOp(
3893	3895	const zcu = o.zcu;
3894	3896	const target = zcu.getTarget();
3895	3897	const scalar_ty = ty.scalarType(zcu);
3896		const llvm_ty = try o.lowerType(ty);
	3898	const llvm_ty = try o.lowerType(ty, .by_value);
3897	3899
3898	3900	if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {
3899	3901	// Some operations are dedicated LLVM instructions, not available as intrinsics
...	...	@@ -3998,7 +4000,7 @@ fn buildFloatOp(
3998	4000	}),
3999	4001	};
4000	4002
4001		const scalar_llvm_ty = try o.lowerType(scalar_ty);
	4003	const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
4002	4004	const libc_fn = try o.getLibcFunction(
4003	4005	fn_name,
4004	4006	@as([3]Builder.Type, @splat(scalar_llvm_ty))[0..params.len],
...	...	@@ -4052,9 +4054,8 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
4052	4054
4053	4055	const dest_ty = self.typeOfIndex(inst);
4054	4056	assert(isByRef(dest_ty, zcu)); // auto structs are by-ref
4055		const llvm_dest_ty = try o.lowerType(dest_ty);
4056	4057
4057		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4058	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4058	4059
4059	4060	const result = try self.wip.bin(.shl, lhs, casted_rhs, "");
4060	4061	const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
...	...	@@ -4064,19 +4065,21 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
4064	4065
4065	4066	const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
4066	4067
4067		const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();
4068		const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
	4068	const result_alignment = dest_ty.abiAlignment(zcu);
	4069	const alloca_inst = try self.buildZigAlloca(dest_ty, .none);
4069	4070
4070	4071	{
4071	4072	// Store to 'result: IntType' field
4072		const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(0, zcu));
4073		_ = try self.wip.store(.normal, result, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());
	4073	const field_off = dest_ty.structFieldOffset(0, zcu);
	4074	const field_ptr = try self.ptraddConst(alloca_inst, field_off);
	4075	try self.store(field_ptr, result_alignment.offset(field_off), result, lhs_ty, .normal);
4074	4076	}
4075	4077
4076	4078	{
4077	4079	// Store to 'overflow: u1' field
4078		const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(1, zcu));
4079		_ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
	4080	const field_off = dest_ty.structFieldOffset(1, zcu);
	4081	const field_ptr = try self.ptraddConst(alloca_inst, field_off);
	4082	try self.store(field_ptr, result_alignment.offset(field_off), overflow_bit, dest_ty.fieldType(1, zcu), .normal);
4080	4083	}
4081	4084
4082	4085	return alloca_inst;
...	...	@@ -4119,7 +4122,7 @@ fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
4119	4122	}
4120	4123	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
4121	4124
4122		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4125	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4123	4126	return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
4124	4127	.@"shl nsw"
4125	4128	else
...	...	@@ -4140,7 +4143,7 @@ fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4140	4143	// features which we do not use. Therefore this branch is currently impossible.
4141	4144	unreachable;
4142	4145	}
4143		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4146	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4144	4147	return self.wip.bin(.shl, lhs, casted_rhs, "");
4145	4148	}
4146	4149
...	...	@@ -4154,8 +4157,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4154	4157
4155	4158	const lhs_ty = self.typeOf(bin_op.lhs);
4156	4159	const lhs_info = lhs_ty.intInfo(zcu);
4157		const llvm_lhs_ty = try o.lowerType(lhs_ty);
4158		const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu));
	4160	const llvm_lhs_ty = try o.lowerType(lhs_ty, .by_value);
	4161	const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu), .by_value);
4159	4162
4160	4163	const rhs_ty = self.typeOf(bin_op.rhs);
4161	4164	if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) {
...	...	@@ -4165,8 +4168,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4165	4168	}
4166	4169	const rhs_info = rhs_ty.intInfo(zcu);
4167	4170	assert(rhs_info.signedness == .unsigned);
4168		const llvm_rhs_ty = try o.lowerType(rhs_ty);
4169		const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu));
	4171	const llvm_rhs_ty = try o.lowerType(rhs_ty, .by_value);
	4172	const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu), .by_value);
4170	4173
4171	4174	const result = try self.wip.callIntrinsic(
4172	4175	.normal,
...	...	@@ -4242,7 +4245,7 @@ fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!
4242	4245	}
4243	4246	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
4244	4247
4245		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4248	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4246	4249	const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);
4247	4250
4248	4251	return self.wip.bin(if (is_exact)
...	...	@@ -4263,8 +4266,8 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4263	4266	.normal,
4264	4267	.none,
4265	4268	.abs,
4266		&.{try o.lowerType(operand_ty)},
4267		&.{ operand, try o.builder.intValue(.i1, 0) },
	4269	&.{try o.lowerType(operand_ty, .by_value)},
	4270	&.{ operand, .false },
4268	4271	"",
4269	4272	),
4270	4273	.float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),
...	...	@@ -4277,7 +4280,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4277	4280	const zcu = o.zcu;
4278	4281	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4279	4282	const dest_ty = fg.typeOfIndex(inst);
4280		const dest_llvm_ty = try o.lowerType(dest_ty);
	4283	const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
4281	4284	const operand = try fg.resolveInst(ty_op.operand);
4282	4285	const operand_ty = fg.typeOf(ty_op.operand);
4283	4286	const operand_info = operand_ty.intInfo(zcu);
...	...	@@ -4305,8 +4308,8 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4305	4308
4306	4309	if (!have_min_check and !have_max_check) break :bounds_check;
4307	4310
4308		const operand_llvm_ty = try o.lowerType(operand_ty);
4309		const operand_scalar_llvm_ty = try o.lowerType(operand_scalar);
	4311	const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
	4312	const operand_scalar_llvm_ty = try o.lowerType(operand_scalar, .by_value);
4310	4313
4311	4314	const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
4312	4315	assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));
...	...	@@ -4384,7 +4387,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4384	4387	fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4385	4388	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4386	4389	const operand = try self.resolveInst(ty_op.operand);
4387		const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst));
	4390	const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst), .by_value);
4388	4391	return self.wip.cast(.trunc, operand, dest_llvm_ty, "");
4389	4392	}
4390	4393
...	...	@@ -4398,10 +4401,10 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
4398	4401	const target = zcu.getTarget();
4399	4402
4400	4403	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
4401		return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty), "");
	4404	return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty, .by_value), "");
4402	4405	} else {
4403		const operand_llvm_ty = try o.lowerType(operand_ty);
4404		const dest_llvm_ty = try o.lowerType(dest_ty);
	4406	const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
	4407	const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
4405	4408
4406	4409	const dest_bits = dest_ty.floatBits(target);
4407	4410	const src_bits = operand_ty.floatBits(target);
...	...	@@ -4432,10 +4435,10 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4432	4435	const target = zcu.getTarget();
4433	4436
4434	4437	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
4435		return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty), "");
	4438	return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty, .by_value), "");
4436	4439	} else {
4437		const operand_llvm_ty = try o.lowerType(operand_ty);
4438		const dest_llvm_ty = try o.lowerType(dest_ty);
	4440	const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
	4441	const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
4439	4442
4440	4443	const dest_bits = dest_ty.scalarType(zcu).floatBits(target);
4441	4444	const src_bits = operand_ty.scalarType(zcu).floatBits(target);
...	...	@@ -4462,114 +4465,80 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4462	4465	}
4463	4466	}
4464	4467
4465		fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4466		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4467		const operand_ty = self.typeOf(ty_op.operand);
4468		const inst_ty = self.typeOfIndex(inst);
4469		const operand = try self.resolveInst(ty_op.operand);
4470		return self.bitCast(operand, operand_ty, inst_ty);
4471		}
4472
4473		fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) Allocator.Error!Builder.Value {
4474		const o = self.object;
	4468	fn airBitCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4469	const o = fg.object;
4475	4470	const zcu = o.zcu;
4476		const operand_is_ref = isByRef(operand_ty, zcu);
4477		const result_is_ref = isByRef(inst_ty, zcu);
4478		const llvm_dest_ty = try o.lowerType(inst_ty);
4479	4471
4480		if (operand_is_ref and result_is_ref) {
4481		// They are both pointers, so just return the same opaque pointer :)
4482		return operand;
4483		}
	4472	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4473	const operand_ty = fg.typeOf(ty_op.operand);
	4474	const dest_ty = fg.typeOfIndex(inst);
	4475	const operand = try fg.resolveInst(ty_op.operand);
4484	4476
4485		if (inst_ty.isAbiInt(zcu) and operand_ty.isAbiInt(zcu)) {
4486		return self.wip.conv(.unsigned, operand, llvm_dest_ty, "");
4487		}
	4477	// We have the following `Air.Legalize` features enabled:
	4478	//
	4479	// * `.scalarize_bit_cast_array`
	4480	// * `.scalarize_bit_cast_vector_non_elementwise`
	4481	//
	4482	// That means the `bit_cast` instructions we might see are limited to the following:
	4483	//
	4484	// * bool/int/float <-> bool/int/float
	4485	// * `@Vector(n, A)` <-> `@Vector(n, B)`
	4486	//
	4487	// All of these cases can be handled by LLVM's `bitcast` instruction.
4488	4488
4489		const operand_scalar_ty = operand_ty.scalarType(zcu);
4490		const inst_scalar_ty = inst_ty.scalarType(zcu);
4491		if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) {
4492		return self.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
4493		}
4494		if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) {
4495		return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
4496		}
4497
4498		if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
4499		const elem_ty = operand_scalar_ty;
4500		assert(result_is_ref); // arrays are always by-ref provided they have runtime bits
4501		const alignment = inst_ty.abiAlignment(zcu).toLlvm();
4502		const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4503		const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
4504		if (bitcast_ok) {
4505		_ = try self.wip.store(.normal, operand, array_ptr, alignment);
4506		} else {
4507		// If the ABI size of the element type is not evenly divisible by size in bits;
4508		// a simple bitcast will not work, and we fall back to extractelement.
4509		const elem_size = elem_ty.abiSize(zcu);
4510		const vector_len = operand_ty.arrayLen(zcu);
4511		var i: u64 = 0;
4512		while (i < vector_len) : (i += 1) {
4513		const arr_elem_ptr = try self.ptraddConst(array_ptr, i * elem_size);
4514		const vec_elem = try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
4515		_ = try self.wip.store(.normal, vec_elem, arr_elem_ptr, .default);
4516		}
4517		}
4518		return array_ptr;
4519		} else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) {
4520		const elem_ty = operand_ty.childType(zcu);
4521		assert(operand_is_ref); // arrays are always by-ref provided they have runtime bits
4522		const llvm_vector_ty = try o.lowerType(inst_ty);
4523
4524		const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
4525		if (bitcast_ok) {
4526		// The array is aligned to the element's alignment, while the vector might have a completely
4527		// different alignment. This means we need to enforce the alignment of this load.
4528		const alignment = elem_ty.abiAlignment(zcu).toLlvm();
4529		return self.wip.load(.normal, llvm_vector_ty, operand, alignment, "");
4530		} else {
4531		// If the ABI size of the element type is not evenly divisible by size in bits;
4532		// a simple bitcast will not work, and we fall back to extractelement.
4533		const elem_llvm_ty = try o.lowerType(elem_ty);
4534		const elem_size = elem_ty.abiSize(zcu);
4535		const vector_len = operand_ty.arrayLen(zcu);
4536		var vector = try o.builder.poisonValue(llvm_vector_ty);
4537		var i: u64 = 0;
4538		while (i < vector_len) : (i += 1) {
4539		const arr_elem_ptr = try self.ptraddConst(operand, i * elem_size);
4540		const arr_elem = try self.wip.load(.normal, elem_llvm_ty, arr_elem_ptr, .default, "");
4541		vector = try self.wip.insertElement(vector, arr_elem, try o.builder.intValue(.i32, i), "");
4542		}
4543		return vector;
4544		}
4545		}
	4489	assert(!isByRef(operand_ty, zcu));
	4490	assert(!isByRef(dest_ty, zcu));
4546	4491
4547		if (operand_is_ref) {
4548		const alignment = operand_ty.abiAlignment(zcu).toLlvm();
4549		return self.wip.load(.normal, llvm_dest_ty, operand, alignment, "");
4550		}
	4492	const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
	4493	return fg.wip.cast(.bitcast, operand, llvm_dest_ty, "");
	4494	}
4551	4495
4552		if (result_is_ref) {
4553		const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
4554		const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4555		_ = try self.wip.store(.normal, operand, result_ptr, alignment);
4556		return result_ptr;
4557		}
	4496	fn airNopCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4497	const zcu = fg.object.zcu;
	4498	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4499	const operand_ty = fg.typeOf(ty_op.operand);
	4500	const dest_ty = fg.typeOfIndex(inst);
	4501	assert(isByRef(operand_ty, zcu) == isByRef(dest_ty, zcu));
	4502	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
	4503	return fg.resolveInst(ty_op.operand);
	4504	}
4558	4505
4559		if (inst_ty.isSliceAtRuntime(zcu) or
4560		((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and
4561		operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu)))
4562		{
4563		// Both our operand and our result are values, not pointers,
4564		// but LLVM won't let us bitcast struct values or vectors with padding bits.
4565		// Therefore, we store operand to alloca, then load for result.
4566		const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
4567		const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4568		_ = try self.wip.store(.normal, operand, result_ptr, alignment);
4569		return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");
4570		}
	4506	fn airPtrFromInt(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4507	const o = fg.object;
	4508	const zcu = o.zcu;
	4509	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4510	const operand_ty = fg.typeOf(ty_op.operand);
	4511	const dest_ty = fg.typeOfIndex(inst);
	4512	assert(operand_ty.scalarType(zcu).toIntern() == .usize_type);
	4513	assert(dest_ty.scalarType(zcu).isPtrAtRuntime(zcu));
4571	4514
4572		return self.wip.cast(.bitcast, operand, llvm_dest_ty, "");
	4515	const operand = try fg.resolveInst(ty_op.operand);
	4516	const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
	4517	return fg.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
	4518	}
	4519
	4520	fn airIntFromPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4521	const o = fg.object;
	4522	const zcu = o.zcu;
	4523	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4524	const operand_ty = fg.typeOf(ty_op.operand);
	4525	const dest_ty = fg.typeOfIndex(inst);
	4526	assert(operand_ty.scalarType(zcu).isPtrAtRuntime(zcu));
	4527	assert(dest_ty.scalarType(zcu).toIntern() == .usize_type);
	4528
	4529	const operand = try fg.resolveInst(ty_op.operand);
	4530	const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
	4531	return fg.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
	4532	}
	4533
	4534	fn airUnionFromEnum(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4535	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4536	const enum_ty = fg.typeOf(ty_op.operand);
	4537	const union_ty = fg.typeOfIndex(inst);
	4538	const enum_val = try fg.resolveInst(ty_op.operand);
	4539	const union_ptr = try fg.buildZigAlloca(union_ty, .none);
	4540	try fg.store(union_ptr, .none, enum_val, enum_ty, .normal);
	4541	return union_ptr;
4573	4542	}
4574	4543
4575	4544	fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -4628,9 +4597,8 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4628	4597	"",
4629	4598	);
4630	4599	} else if (mod.optimize_mode == .Debug) {
4631		const alignment = inst_ty.abiAlignment(zcu).toLlvm();
4632		const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment);
4633		_ = try self.wip.store(.normal, arg_val, alloca, alignment);
	4600	const alloca = try self.buildZigAlloca(inst_ty, .none);
	4601	try self.store(alloca, .none, arg_val, inst_ty, .normal);
4634	4602	_ = try self.wip.callIntrinsic(
4635	4603	.normal,
4636	4604	.none,
...	...	@@ -4671,8 +4639,7 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4671	4639	if (!elem_ty.hasRuntimeBits(zcu)) {
4672	4640	return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();
4673	4641	}
4674		const llvm_elem_ty = try o.lowerType(elem_ty);
4675		return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
	4642	return self.buildZigAlloca(elem_ty, ptr_align);
4676	4643	}
4677	4644
4678	4645	fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -4685,32 +4652,71 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4685	4652	if (!elem_ty.hasRuntimeBits(zcu)) {
4686	4653	return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();
4687	4654	}
4688		const llvm_elem_ty = try o.lowerType(elem_ty);
4689		return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
	4655	return self.buildZigAlloca(elem_ty, ptr_align);
4690	4656	}
4691	4657
4692		/// Use this instead of builder.buildAlloca, because this function makes sure to
4693		/// put the alloca instruction at the top of the function!
	4658	fn buildZigAlloca(fg: *FuncGen, ty: Type, @"align": InternPool.Alignment) Allocator.Error!Builder.Value {
	4659	const o = fg.object;
	4660	const resolved_align: InternPool.Alignment = switch (@"align") {
	4661	.none => ty.abiAlignment(o.zcu),
	4662	else => \|a\| a,
	4663	};
	4664	return fg.buildAlloca(
	4665	try o.lowerType(ty, .in_memory),
	4666	resolved_align.toLlvm(),
	4667	);
	4668	}
	4669
	4670	/// Unlike `WipFunction.alloca`, this puts the alloca instruction at the top of the function.
4694	4671	fn buildAlloca(
4695		self: *FuncGen,
	4672	fg: *FuncGen,
4696	4673	llvm_ty: Builder.Type,
4697	4674	alignment: Builder.Alignment,
4698	4675	) Allocator.Error!Builder.Value {
4699		const target = self.object.zcu.getTarget();
4700		return buildAllocaInner(&self.wip, llvm_ty, alignment, target);
	4676	const wip = &fg.wip;
	4677
	4678	const alloca = blk: {
	4679	const prev_cursor = wip.cursor;
	4680	const prev_debug_location = wip.debug_location;
	4681	defer {
	4682	wip.cursor = prev_cursor;
	4683	if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
	4684	wip.debug_location = prev_debug_location;
	4685	}
	4686
	4687	wip.cursor = .{ .block = .entry };
	4688	wip.debug_location = .no_location;
	4689	const address_space = llvmAllocaAddressSpace(fg.object.zcu.getTarget());
	4690	break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
	4691	};
	4692
	4693	// The pointer returned from this function should have the generic address space,
	4694	// if this isn't the case then cast it to the generic address space.
	4695	return fg.wip.conv(.unneeded, alloca, .ptr, "");
4701	4696	}
4702	4697
4703		fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
4704		const o = self.object;
	4698	fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
	4699	const o = fg.object;
4705	4700	const zcu = o.zcu;
4706		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4707		const dest_ptr = try self.resolveInst(bin_op.lhs);
4708		const ptr_ty = self.typeOf(bin_op.lhs);
4709		const operand_ty = ptr_ty.childType(zcu);
	4701	const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	4702	const ptr = try fg.resolveInst(bin_op.lhs);
	4703	const ptr_ty = fg.typeOf(bin_op.lhs);
	4704	const ptr_info = ptr_ty.ptrInfo(zcu);
	4705	const ptr_alignment = ptr_ty.ptrAlignment(zcu);
	4706
	4707	const elem_ty = fg.typeOf(bin_op.rhs);
	4708	assert(elem_ty.hasRuntimeBits(zcu));
	4709
	4710	fg.maybeMarkAllowZeroAccess(ptr_info);
	4711
	4712	const access_kind: Builder.MemoryAccessKind = switch (ptr_info.flags.is_volatile) {
	4713	true => .@"volatile",
	4714	false => .normal,
	4715	};
4710	4716
4711	4717	const val_is_undef = if (bin_op.rhs.toInterned()) \|i\| Value.fromInterned(i).isUndef(zcu) else false;
4712		if (val_is_undef and !self.needMemsetWorkaround(operand_ty.abiSize(zcu))) {
4713		const owner_mod = self.ownerModule();
	4718	if (val_is_undef and !fg.needMemsetWorkaround(elem_ty.abiSize(zcu))) {
	4719	const owner_mod = fg.ownerModule();
4714	4720
4715	4721	// Even if safety is disabled, we still emit a memset to undefined since it conveys
4716	4722	// extra information to LLVM, and LLVM will optimize it out. Safety makes the difference
...	...	@@ -4725,7 +4731,6 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4725	4731	return .none;
4726	4732	}
4727	4733
4728		const ptr_info = ptr_ty.ptrInfo(zcu);
4729	4734	const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
4730	4735	if (needs_bitmask) {
4731	4736	// TODO: only some bits are to be undef, we cannot write with a simple memset.
...	...	@@ -4734,27 +4739,82 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4734	4739	return .none;
4735	4740	}
4736	4741
4737		self.maybeMarkAllowZeroAccess(ptr_info);
4738
4739		const len = try o.builder.intValue(try o.lowerType(.usize), operand_ty.abiSize(zcu));
4740		_ = try self.wip.callMemSet(
4741		dest_ptr,
4742		ptr_ty.ptrAlignment(zcu).toLlvm(),
	4742	const len = try o.builder.intValue(try o.lowerType(.usize, .by_value), elem_ty.abiSize(zcu));
	4743	_ = try fg.wip.callMemSet(
	4744	ptr,
	4745	ptr_alignment.toLlvm(),
4743	4746	if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
4744	4747	len,
4745		if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
4746		self.disable_intrinsics,
	4748	access_kind,
	4749	fg.disable_intrinsics,
4747	4750	);
4748	4751	if (safety and owner_mod.valgrind) {
4749		try self.valgrindMarkUndef(dest_ptr, len);
	4752	try fg.valgrindMarkUndef(ptr, len);
4750	4753	}
4751	4754	return .none;
4752	4755	}
4753	4756
4754		self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
	4757	const elem = try fg.resolveInst(bin_op.rhs);
	4758
	4759	if (ptr_info.flags.vector_index != .none) {
	4760	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
	4761	const vec_ty = try fg.pt.vectorType(.{
	4762	.len = ptr_info.packed_offset.host_size,
	4763	.child = elem_ty.toIntern(),
	4764	});
	4765
	4766	const loaded_vector = try fg.load(ptr, ptr_alignment, vec_ty, access_kind);
	4767	const index_val = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
	4768	const modified_vector = try fg.wip.insertElement(loaded_vector, elem, index_val, "");
	4769
	4770	try fg.store(ptr, ptr_alignment, modified_vector, vec_ty, access_kind);
	4771	return .none;
	4772	}
4755	4773
4756		const src_operand = try self.resolveInst(bin_op.rhs);
4757		try self.storeFull(dest_ptr, ptr_ty, src_operand, .none);
	4774	if (ptr_info.packed_offset.host_size != 0) {
	4775	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
	4776	const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
	4777	const llvm_backing_int_ty = try o.lowerType(backing_int_ty, .by_value);
	4778
	4779	const backing_int_val = try fg.load(ptr, ptr_alignment, backing_int_ty, access_kind);
	4780
	4781	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
	4782	const shift_amt = try o.builder.intConst(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
	4783
	4784	// Convert to equally-sized integer type in order to perform the bit
	4785	// operations on the value to store
	4786	const new_val_bits_type = try o.builder.intType(@intCast(elem_bits));
	4787	const new_val_bits = if (elem_ty.isPtrAtRuntime(zcu))
	4788	try fg.wip.cast(.ptrtoint, elem, new_val_bits_type, "")
	4789	else
	4790	try fg.wip.cast(.bitcast, elem, new_val_bits_type, "");
	4791
	4792	const mask_val = blk: {
	4793	const zext = try fg.wip.cast(
	4794	.zext,
	4795	try o.builder.intValue(new_val_bits_type, -1),
	4796	llvm_backing_int_ty,
	4797	"",
	4798	);
	4799	const shl = try fg.wip.bin(.shl, zext, shift_amt.toValue(), "");
	4800	break :blk try fg.wip.bin(
	4801	.xor,
	4802	shl,
	4803	try o.builder.intValue(llvm_backing_int_ty, -1),
	4804	"",
	4805	);
	4806	};
	4807
	4808	const masked_backing_int_val = try fg.wip.bin(.@"and", backing_int_val, mask_val, "");
	4809	const extended_new_val = try fg.wip.cast(.zext, new_val_bits, llvm_backing_int_ty, "");
	4810	const shifted_new_val = try fg.wip.bin(.shl, extended_new_val, shift_amt.toValue(), "");
	4811	const new_backing_int_val = try fg.wip.bin(.@"or", shifted_new_val, masked_backing_int_val, "");
	4812
	4813	try fg.store(ptr, ptr_alignment, new_backing_int_val, backing_int_ty, access_kind);
	4814	return .none;
	4815	}
	4816
	4817	try fg.store(ptr, ptr_alignment, elem, elem_ty, access_kind);
4758	4818	return .none;
4759	4819	}
4760	4820
...	...	@@ -4766,7 +4826,7 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4766	4826	const ptr_info = ptr_ty.ptrInfo(zcu);
4767	4827	const ptr = try fg.resolveInst(ty_op.operand);
4768	4828	const elem_ty = ptr_ty.childType(zcu);
4769		const llvm_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();
	4829	const ptr_align = ptr_ty.ptrAlignment(zcu);
4770	4830
4771	4831	fg.maybeMarkAllowZeroAccess(ptr_info);
4772	4832
...	...	@@ -4774,36 +4834,32 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4774	4834	if (ptr_info.flags.is_volatile) .@"volatile" else .normal;
4775	4835
4776	4836	if (ptr_info.flags.vector_index != .none) {
4777		const index_u32 = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
4778		const vec_elem_ty = try o.lowerType(elem_ty);
4779		const vec_ty = try o.builder.vectorType(.normal, ptr_info.packed_offset.host_size, vec_elem_ty);
4780
4781		const loaded_vector = try fg.wip.load(access_kind, vec_ty, ptr, llvm_ptr_align, "");
4782		return fg.wip.extractElement(loaded_vector, index_u32, "");
	4837	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
	4838	const vec_ty = try fg.pt.vectorType(.{
	4839	.len = ptr_info.packed_offset.host_size,
	4840	.child = elem_ty.toIntern(),
	4841	});
	4842	const vector_val = try fg.load(ptr, ptr_align, vec_ty, access_kind);
	4843	const index_val = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
	4844	return fg.wip.extractElement(vector_val, index_val, "");
4783	4845	}
4784	4846
4785	4847	if (ptr_info.packed_offset.host_size == 0) {
4786		return fg.load(ptr, elem_ty, llvm_ptr_align, access_kind);
	4848	return fg.load(ptr, ptr_align, elem_ty, access_kind);
4787	4849	}
4788	4850
4789		const containing_int_ty = try o.builder.intType(@intCast(ptr_info.packed_offset.host_size * 8));
4790		const containing_int =
4791		try fg.wip.load(access_kind, containing_int_ty, ptr, llvm_ptr_align, "");
	4851	assert(!isByRef(elem_ty, zcu)); // all packable types are by-val
4792	4852
4793		const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
4794		const shift_amt = try o.builder.intValue(containing_int_ty, ptr_info.packed_offset.bit_offset);
4795		const shifted_value = try fg.wip.bin(.lshr, containing_int, shift_amt, "");
4796		const elem_llvm_ty = try o.lowerType(elem_ty);
	4853	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
	4854	const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
	4855	const llvm_backing_int_ty = try o.lowerType(backing_int_ty, .by_value);
4797	4856
4798		if (isByRef(elem_ty, zcu)) {
4799		const result_align = elem_ty.abiAlignment(zcu).toLlvm();
4800		const result_ptr = try fg.buildAlloca(elem_llvm_ty, result_align);
	4857	const backing_int_val = try fg.load(ptr, ptr_align, backing_int_ty, .normal);
4801	4858
4802		const same_size_int = try o.builder.intType(@intCast(elem_bits));
4803		const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, "");
4804		_ = try fg.wip.store(.normal, truncated_int, result_ptr, result_align);
4805		return result_ptr;
4806		}
	4859	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
	4860	const shift_amt = try o.builder.intValue(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
	4861	const shifted_value = try fg.wip.bin(.lshr, backing_int_val, shift_amt, "");
	4862	const elem_llvm_ty = try o.lowerType(elem_ty, .by_value);
4807	4863
4808	4864	if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
4809	4865	const same_size_int = try o.builder.intType(@intCast(elem_bits));
...	...	@@ -4853,7 +4909,7 @@ fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
4853	4909	fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4854	4910	_ = inst;
4855	4911	const o = self.object;
4856		const llvm_usize = try o.lowerType(.usize);
	4912	const llvm_usize = try o.lowerType(.usize, .by_value);
4857	4913	if (!target_util.supportsReturnAddress(self.object.zcu.getTarget(), self.ownerModule().optimize_mode)) {
4858	4914	// https://github.com/ziglang/zig/issues/11946
4859	4915	return o.builder.intValue(llvm_usize, 0);
...	...	@@ -4865,7 +4921,7 @@ fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
4865	4921	fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4866	4922	_ = inst;
4867	4923	const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");
4868		return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize), "");
	4924	return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize, .by_value), "");
4869	4925	}
4870	4926
4871	4927	fn airCmpxchg(
...	...	@@ -4882,7 +4938,7 @@ fn airCmpxchg(
4882	4938	var expected_value = try self.resolveInst(extra.expected_value);
4883	4939	var new_value = try self.resolveInst(extra.new_value);
4884	4940	const operand_ty = ptr_ty.childType(zcu);
4885		const llvm_operand_ty = try o.lowerType(operand_ty);
	4941	const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
4886	4942	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);
4887	4943	if (llvm_abi_ty != .none) {
4888	4944	// operand needs widening and truncating
...	...	@@ -4918,21 +4974,22 @@ fn airCmpxchg(
4918	4974	return self.wip.select(.normal, success_bit, zero, payload, "");
4919	4975	}
4920	4976
4921		assert(isByRef(optional_ty, zcu));
	4977	assert(!isByRef(operand_ty, zcu)); // can only cmpxchg non-by-ref types
	4978	assert(isByRef(optional_ty, zcu)); // all optionals are by-ref
4922	4979
4923	4980	comptime assert(optional_layout_version == 3);
4924	4981
4925	4982	const non_null_bit = try self.wip.not(success_bit, "");
4926	4983
4927		const payload_align = operand_ty.abiAlignment(zcu).toLlvm();
4928		const alloca_inst = try self.buildAlloca(try o.lowerType(optional_ty), payload_align);
	4984	const payload_align = operand_ty.abiAlignment(zcu);
	4985	const alloca_inst = try self.buildZigAlloca(optional_ty, .none);
4929	4986
4930	4987	// Payload is always the first field at offset 0, so address is `alloca_inst`
4931		_ = try self.wip.store(.normal, payload, alloca_inst, payload_align);
	4988	try self.store(alloca_inst, .none, payload, operand_ty, .normal);
4932	4989
4933	4990	// Non-null bit is after payload with no padding because it has alignment 1
4934	4991	const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu));
4935		_ = try self.wip.store(.normal, non_null_bit, non_null_ptr, comptime .fromByteUnits(1));
	4992	try self.store(non_null_ptr, payload_align, non_null_bit, .bool, .normal);
4936	4993
4937	4994	return alloca_inst;
4938	4995	}
...	...	@@ -4951,7 +5008,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
4951	5008	const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
4952	5009	const ordering = toLlvmAtomicOrdering(extra.ordering());
4953	5010	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg);
4954		const llvm_operand_ty = try o.lowerType(operand_ty);
	5011	const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
4955	5012
4956	5013	const access_kind: Builder.MemoryAccessKind =
4957	5014	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...	...	@@ -4980,7 +5037,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
4980	5037
4981	5038	// If we are storing a pointer we need to convert to and from a plain old integer.
4982	5039	const non_ptr_operand = switch (operand_ty.zigTypeTag(zcu)) {
4983		.pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize), ""),
	5040	.pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize, .by_value), ""),
4984	5041	else => operand,
4985	5042	};
4986	5043
...	...	@@ -5019,7 +5076,7 @@ fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
5019	5076	Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();
5020	5077	const access_kind: Builder.MemoryAccessKind =
5021	5078	if (info.flags.is_volatile) .@"volatile" else .normal;
5022		const elem_llvm_ty = try o.lowerType(elem_ty);
	5079	const elem_llvm_ty = try o.lowerType(elem_ty, .by_value);
5023	5080
5024	5081	self.maybeMarkAllowZeroAccess(info);
5025	5082
...	...	@@ -5073,7 +5130,17 @@ fn airAtomicStore(
5073	5130
5074	5131	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
5075	5132
5076		try self.storeFull(ptr, ptr_ty, element, ordering);
	5133	assert(!isByRef(operand_ty, zcu));
	5134
	5135	_ = try self.wip.storeAtomic(
	5136	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
	5137	element,
	5138	ptr,
	5139	self.sync_scope,
	5140	ordering,
	5141	ptr_ty.ptrAlignment(zcu).toLlvm(),
	5142	);
	5143
5077	5144	return .none;
5078	5145	}
5079	5146
...	...	@@ -5084,7 +5151,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5084	5151	const dest_slice = try self.resolveInst(bin_op.lhs);
5085	5152	const ptr_ty = self.typeOf(bin_op.lhs);
5086	5153	const elem_ty = self.typeOf(bin_op.rhs);
5087		const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();
	5154	const dest_ptr_align = ptr_ty.ptrAlignment(zcu);
5088	5155	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);
5089	5156	const access_kind: Builder.MemoryAccessKind =
5090	5157	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...	...	@@ -5110,7 +5177,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5110	5177	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5111	5178	_ = try self.wip.callMemSet(
5112	5179	dest_ptr,
5113		dest_ptr_align,
	5180	dest_ptr_align.toLlvm(),
5114	5181	fill_byte,
5115	5182	len,
5116	5183	access_kind,
...	...	@@ -5132,7 +5199,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5132	5199	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5133	5200	_ = try self.wip.callMemSet(
5134	5201	dest_ptr,
5135		dest_ptr_align,
	5202	dest_ptr_align.toLlvm(),
5136	5203	fill_byte,
5137	5204	len,
5138	5205	access_kind,
...	...	@@ -5145,14 +5212,31 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5145	5212	const value = try self.resolveInst(bin_op.rhs);
5146	5213	const elem_abi_size = elem_ty.abiSize(zcu);
5147	5214
5148		if (allow_byte_memset and elem_abi_size == 1 and elem_ty.bitSize(zcu) == 8) {
5149		// In this case we can take advantage of LLVM's intrinsic.
5150		const fill_byte = try self.bitCast(value, elem_ty, Type.u8);
	5215	intrinsic: {
	5216	if (!allow_byte_memset) break :intrinsic;
	5217	if (elem_abi_size != 1) break :intrinsic;
	5218	// To use LLVM's intrinsic, we need to convert the operand to a raw 8-bit integer value.
	5219	const fill_byte: Builder.Value = byte: {
	5220	if (isByRef(elem_ty, zcu)) {
	5221	break :byte try self.load(value, elem_ty.abiAlignment(zcu), .u8, .normal);
	5222	}
	5223	if (elem_ty.isAbiInt(zcu)) {
	5224	const info = elem_ty.intInfo(zcu);
	5225	break :byte try self.wip.conv(switch (info.signedness) {
	5226	.unsigned => .unsigned,
	5227	.signed => .signed,
	5228	}, value, .i8, "");
	5229	}
	5230	if (elem_ty.toIntern() == .bool_type) {
	5231	break :byte try self.wip.cast(.zext, value, .i8, "");
	5232	}
	5233	break :intrinsic;
	5234	};
	5235	// Great, we can use the intrinsic!
5151	5236	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5152
5153	5237	_ = try self.wip.callMemSet(
5154	5238	dest_ptr,
5155		dest_ptr_align,
	5239	dest_ptr_align.toLlvm(),
5156	5240	fill_byte,
5157	5241	len,
5158	5242	access_kind,
...	...	@@ -5182,7 +5266,6 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5182	5266	const body_block = try self.wip.block(1, "InlineMemsetBody");
5183	5267	const end_block = try self.wip.block(1, "InlineMemsetEnd");
5184	5268
5185		const llvm_usize_ty = try o.lowerType(.usize);
5186	5269	const end_ptr = switch (ptr_ty.ptrSize(zcu)) {
5187	5270	.slice => try self.ptraddScaled(
5188	5271	dest_ptr,
...	...	@@ -5201,18 +5284,8 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5201	5284
5202	5285	self.wip.cursor = .{ .block = body_block };
5203	5286	const elem_abi_align = elem_ty.abiAlignment(zcu);
5204		const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm();
5205		if (isByRef(elem_ty, zcu)) {
5206		_ = try self.wip.callMemCpy(
5207		it_ptr.toValue(),
5208		it_ptr_align,
5209		value,
5210		elem_abi_align.toLlvm(),
5211		try o.builder.intValue(llvm_usize_ty, elem_abi_size),
5212		access_kind,
5213		self.disable_intrinsics,
5214		);
5215		} else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
	5287	const it_ptr_align: InternPool.Alignment = dest_ptr_align.min(elem_abi_align);
	5288	try self.store(it_ptr.toValue(), it_ptr_align, value, elem_ty, access_kind);
5216	5289	const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size);
5217	5290	_ = try self.wip.br(loop_block);
5218	5291
...	...	@@ -5289,14 +5362,11 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.
5289	5362
5290	5363	const union_ptr = try self.resolveInst(bin_op.lhs);
5291	5364	const new_tag = try self.resolveInst(bin_op.rhs);
	5365	const tag_ty = self.typeOf(bin_op.rhs);
5292	5366	const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);
5293		if (layout.payload_size == 0) {
5294		_ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());
5295		return .none;
5296		}
5297	5367	const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());
5298	5368	const tag_ptr_align = union_ptr_align.offset(layout.tagOffset());
5299		_ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm());
	5369	try self.store(tag_field_ptr, tag_ptr_align, new_tag, tag_ty, access_kind);
5300	5370	return .none;
5301	5371	}
5302	5372
...	...	@@ -5308,16 +5378,9 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.
5308	5378	const layout = un_ty.unionGetLayout(zcu);
5309	5379	assert(layout.tag_size != 0);
5310	5380	const operand = try self.resolveInst(ty_op.operand);
5311		if (isByRef(un_ty, zcu)) {
5312		const llvm_tag_ty = try o.lowerType(un_ty.unionTagTypeRuntime(zcu).?);
5313		const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());
5314		return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");
5315		} else {
5316		// This is only possible if all fields are zero-bit, in which case `operand` is already an
5317		// integer value (the union is lowered as its enum tag).
5318		assert(layout.payload_size == 0);
5319		return operand;
5320		}
	5381	assert(isByRef(un_ty, zcu));
	5382	const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());
	5383	return self.load(tag_field_ptr, .none, un_ty.unionTagTypeRuntime(zcu).?, .normal);
5321	5384	}
5322	5385
5323	5386	fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) Allocator.Error!Builder.Value {
...	...	@@ -5347,11 +5410,11 @@ fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)
5347	5410	.normal,
5348	5411	.none,
5349	5412	intrinsic,
5350		&.{try o.lowerType(operand_ty)},
	5413	&.{try o.lowerType(operand_ty, .by_value)},
5351	5414	&.{ operand, .false },
5352	5415	"",
5353	5416	);
5354		return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");
	5417	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5355	5418	}
5356	5419
5357	5420	fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value {
...	...	@@ -5365,11 +5428,11 @@ fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)
5365	5428	.normal,
5366	5429	.none,
5367	5430	intrinsic,
5368		&.{try o.lowerType(operand_ty)},
	5431	&.{try o.lowerType(operand_ty, .by_value)},
5369	5432	&.{operand},
5370	5433	"",
5371	5434	);
5372		return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");
	5435	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5373	5436	}
5374	5437
5375	5438	fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -5382,7 +5445,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5382	5445
5383	5446	const inst_ty = self.typeOfIndex(inst);
5384	5447	var operand = try self.resolveInst(ty_op.operand);
5385		var llvm_operand_ty = try o.lowerType(operand_ty);
	5448	var llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5386	5449
5387	5450	if (bits % 16 == 8) {
5388	5451	// If not an even byte-multiple, we need zero-extend + shift-left 1 byte
...	...	@@ -5403,7 +5466,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5403	5466
5404	5467	const result =
5405	5468	try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");
5406		return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");
	5469	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5407	5470	}
5408	5471
5409	5472	fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -5423,7 +5486,7 @@ fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Bui
5423	5486
5424	5487	for (0..names.len) \|name_index\| {
5425	5488	const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
5426		const this_tag_int_value = try o.builder.intConst(try o.errorIntType(), err_int);
	5489	const this_tag_int_value = try o.builder.intConst(try o.errorIntType(.by_value), err_int);
5427	5490	try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);
5428	5491	}
5429	5492	self.wip.cursor = .{ .block = valid_block };
...	...	@@ -5480,21 +5543,20 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
5480	5543	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5481	5544	const operand = try self.resolveInst(un_op);
5482	5545	const slice_ty = self.typeOfIndex(inst);
5483		const slice_llvm_ty = try o.lowerType(slice_ty);
5484	5546
5485	5547	// If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
5486		const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize), "");
	5548	const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize, .by_value), "");
5487	5549
5488	5550	const error_name_table_ptr = try o.getErrorNameTable();
5489	5551	const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));
5490		return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");
	5552	return self.load(error_name_ptr, .none, slice_ty, .normal);
5491	5553	}
5492	5554
5493	5555	fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5494	5556	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5495	5557	const scalar = try self.resolveInst(ty_op.operand);
5496	5558	const vector_ty = self.typeOfIndex(inst);
5497		return self.wip.splatVector(try self.object.lowerType(vector_ty), scalar, "");
	5559	return self.wip.splatVector(try self.object.lowerType(vector_ty, .by_value), scalar, "");
5498	5560	}
5499	5561
5500	5562	fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -5517,9 +5579,9 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5517	5579	const operand = try fg.resolveInst(unwrapped.operand);
5518	5580	const mask = unwrapped.mask;
5519	5581	const operand_ty = fg.typeOf(unwrapped.operand);
5520		const llvm_operand_ty = try o.lowerType(operand_ty);
5521		const llvm_result_ty = try o.lowerType(unwrapped.result_ty);
5522		const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));
	5582	const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
	5583	const llvm_result_ty = try o.lowerType(unwrapped.result_ty, .by_value);
	5584	const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu), .by_value);
5523	5585	const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
5524	5586	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
5525	5587	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
...	...	@@ -5549,7 +5611,7 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5549	5611	.elem => llvm_poison_elem,
5550	5612	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
5551	5613	any_defined_comptime_value = true;
5552		break :elem try o.lowerValue(val);
	5614	break :elem try o.lowerValue(val, .by_value);
5553	5615	} else llvm_poison_elem,
5554	5616	};
5555	5617	}
...	...	@@ -5621,7 +5683,7 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5621	5683	const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
5622	5684
5623	5685	const mask = unwrapped.mask;
5624		const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));
	5686	const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu), .by_value);
5625	5687	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
5626	5688	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
5627	5689
...	...	@@ -5696,15 +5758,13 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5696	5758	/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.
5697	5759	///
5698	5760	/// Equivalent to:
5699		/// reduce: {
5700		/// var i: usize = 0;
5701		/// var accum: T = init;
5702		/// while (i < vec.len) : (i += 1) {
5703		/// accum = llvm_fn(accum, vec[i]);
5704		/// }
5705		/// break :reduce accum;
5706		/// }
5707		///
	5761	/// ```
	5762	/// var accum: T = init;
	5763	/// for (0..i) \|i\| {
	5764	/// accum = llvm_fn(accum, vec[i]);
	5765	/// }
	5766	/// // result is 'accum'
	5767	/// ```
5708	5768	fn buildReducedCall(
5709	5769	self: *FuncGen,
5710	5770	llvm_fn: Builder.Function.Index,
...	...	@@ -5713,56 +5773,54 @@ fn buildReducedCall(
5713	5773	accum_init: Builder.Value,
5714	5774	) Allocator.Error!Builder.Value {
5715	5775	const o = self.object;
5716		const usize_ty = try o.lowerType(.usize);
5717		const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len);
	5776	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
	5777	const llvm_vector_len = try o.builder.intValue(llvm_usize_ty, vector_len);
5718	5778	const llvm_result_ty = accum_init.typeOfWip(&self.wip);
5719	5779
5720		// Allocate and initialize our mutable variables
5721		const i_ptr = try self.buildAlloca(usize_ty, .default);
5722		_ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
5723		const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
5724		_ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
	5780	const entry_block = self.wip.cursor.block;
5725	5781
5726		// Setup the loop
5727		const loop = try self.wip.block(2, "ReduceLoop");
5728		const loop_exit = try self.wip.block(1, "AfterReduce");
5729		_ = try self.wip.br(loop);
5730		{
5731		self.wip.cursor = .{ .block = loop };
5732
5733		// while (i < vec.len)
5734		const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, "");
5735		const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");
5736		const loop_then = try self.wip.block(1, "ReduceLoopThen");
5737
5738		_ = try self.wip.brCond(cond, loop_then, loop_exit, .none);
5739
5740		{
5741		self.wip.cursor = .{ .block = loop_then };
5742
5743		// accum = f(accum, vec[i]);
5744		const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
5745		const element = try self.wip.extractElement(operand_vector, i, "");
5746		const new_accum = try self.wip.call(
5747		.normal,
5748		.ccc,
5749		.none,
5750		llvm_fn.typeOf(&o.builder),
5751		llvm_fn.toValue(&o.builder),
5752		&.{ accum, element },
5753		"",
5754		);
5755		_ = try self.wip.store(.normal, new_accum, accum_ptr, .default);
	5782	const cond_block = try self.wip.block(2, "ReduceLoopCond");
	5783	const body_block = try self.wip.block(1, "ReduceLoopBody");
	5784	const exit_block = try self.wip.block(1, "ReduceLoopExit");
	5785
	5786	_ = try self.wip.br(cond_block);
	5787
	5788	// ReduceLoopCond:
	5789	// %index = phi iN [0, %Entry], [%new_index, %ReduceLoopBody]
	5790	// %accum = phi T [%accum_init, %Entry], [%new_accum, %ReduceLoopBody]
	5791	// %cond = icmp ult iN %index, %vector_len
	5792	// br i1 %cond, label %ReduceLoopBody, label %ReduceLoopExit
	5793	self.wip.cursor = .{ .block = cond_block };
	5794	const index = try self.wip.phi(llvm_usize_ty, "");
	5795	const accum = try self.wip.phi(llvm_result_ty, "");
	5796	const cond = try self.wip.icmp(.ult, index.toValue(), llvm_vector_len, "");
	5797	_ = try self.wip.brCond(cond, body_block, exit_block, .none);
	5798
	5799	// ReduceLoopBody:
	5800	// %elem = extractelement <n x T> %operand_vec, iN %index
	5801	// %new_accum = call T @llvm_fn(T %accum, T %elem)
	5802	// %new_index = add nuw iN %index, 1
	5803	// br label %ReduceLoopCond
	5804	self.wip.cursor = .{ .block = body_block };
	5805	const elem = try self.wip.extractElement(operand_vector, index.toValue(), "");
	5806	const new_accum = try self.wip.call(
	5807	.normal,
	5808	.ccc,
	5809	.none,
	5810	llvm_fn.typeOf(&o.builder),
	5811	llvm_fn.toValue(&o.builder),
	5812	&.{ accum.toValue(), elem },
	5813	"",
	5814	);
	5815	const new_index = try self.wip.bin(.@"add nuw", index.toValue(), try o.builder.intValue(llvm_usize_ty, 1), "");
	5816	_ = try self.wip.br(cond_block);
5756	5817
5757		// i += 1
5758		const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), "");
5759		_ = try self.wip.store(.normal, new_i, i_ptr, .default);
5760		_ = try self.wip.br(loop);
5761		}
5762		}
	5818	const index_init = try o.builder.intValue(llvm_usize_ty, 0);
	5819	index.finish(&.{ index_init, new_index }, &.{ entry_block, body_block }, &self.wip);
	5820	accum.finish(&.{ accum_init, new_accum }, &.{ entry_block, body_block }, &self.wip);
5763	5821
5764		self.wip.cursor = .{ .block = loop_exit };
5765		return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
	5822	self.wip.cursor = .{ .block = exit_block };
	5823	return accum.toValue();
5766	5824	}
5767	5825
5768	5826	fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
...	...	@@ -5773,9 +5831,9 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) A
5773	5831	const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
5774	5832	const operand = try self.resolveInst(reduce.operand);
5775	5833	const operand_ty = self.typeOf(reduce.operand);
5776		const llvm_operand_ty = try o.lowerType(operand_ty);
	5834	const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5777	5835	const scalar_ty = self.typeOfIndex(inst);
5778		const llvm_scalar_ty = try o.lowerType(scalar_ty);
	5836	const llvm_scalar_ty = try o.lowerType(scalar_ty, .by_value);
5779	5837
5780	5838	switch (reduce.operation) {
5781	5839	.And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
...	...	@@ -5882,10 +5940,10 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
5882	5940	const result_ty = self.typeOfIndex(inst);
5883	5941	const len: usize = @intCast(result_ty.arrayLen(zcu));
5884	5942	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
5885		const llvm_result_ty = try o.lowerType(result_ty);
5886	5943
5887	5944	switch (result_ty.zigTypeTag(zcu)) {
5888	5945	.vector => {
	5946	const llvm_result_ty = try o.lowerType(result_ty, .by_value);
5889	5947	var vector = try o.builder.poisonValue(llvm_result_ty);
5890	5948	for (elements, 0..) \|elem, i\| {
5891	5949	const index_u32 = try o.builder.intValue(.i32, i);
...	...	@@ -5927,7 +5985,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
5927	5985	// TODO in debug builds init to undef so that the padding will be 0xaa
5928	5986	// even if we fully populate the fields.
5929	5987	const struct_align = result_ty.abiAlignment(zcu);
5930		const alloca_inst = try self.buildAlloca(llvm_result_ty, struct_align.toLlvm());
	5988	const alloca_inst = try self.buildZigAlloca(result_ty, .none);
5931	5989
5932	5990	for (elements, 0..) \|elem, field_index\| {
5933	5991	if (result_ty.structFieldIsComptime(field_index, zcu)) continue;
...	...	@@ -5939,24 +5997,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
5939	5997
5940	5998	const llvm_field_val = try self.resolveInst(elem);
5941	5999
5942		if (isByRef(field_ty, zcu)) {
5943		_ = try self.wip.callMemCpy(
5944		field_ptr,
5945		field_ptr_align.toLlvm(),
5946		llvm_field_val,
5947		field_ty.abiAlignment(zcu).toLlvm(),
5948		try o.builder.intValue(try o.lowerType(.usize), field_ty.abiSize(zcu)),
5949		.normal,
5950		self.disable_intrinsics,
5951		);
5952		} else {
5953		_ = try self.wip.store(
5954		.normal,
5955		llvm_field_val,
5956		field_ptr,
5957		field_ptr_align.toLlvm(),
5958		);
5959		}
	6000	try self.store(field_ptr, field_ptr_align, llvm_field_val, field_ty, .normal);
5960	6001	}
5961	6002
5962	6003	return alloca_inst;
...	...	@@ -5965,8 +6006,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
5965	6006	.array => {
5966	6007	assert(isByRef(result_ty, zcu));
5967	6008
5968		const alignment = result_ty.abiAlignment(zcu).toLlvm();
5969		const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
	6009	const alloca_inst = try self.buildZigAlloca(result_ty, .none);
5970	6010
5971	6011	const array_info = result_ty.arrayInfo(zcu);
5972	6012
...	...	@@ -5975,12 +6015,12 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
5975	6015	for (elements, 0..) \|elem, i\| {
5976	6016	const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i);
5977	6017	const llvm_elem = try self.resolveInst(elem);
5978		try self.store(elem_ptr, .none, llvm_elem, array_info.elem_type);
	6018	try self.store(elem_ptr, .none, llvm_elem, array_info.elem_type, .normal);
5979	6019	}
5980	6020	if (array_info.sentinel) \|sent_val\| {
5981	6021	const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len);
5982	6022	const llvm_elem = try self.resolveValue(sent_val);
5983		try self.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type);
	6023	try self.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type, .normal);
5984	6024	}
5985	6025
5986	6026	return alloca_inst;
...	...	@@ -5996,7 +6036,6 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
5996	6036	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5997	6037	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
5998	6038	const union_ty = self.typeOfIndex(inst);
5999		const union_llvm_ty = try o.lowerType(union_ty);
6000	6039	const union_obj = zcu.typeToUnion(union_ty).?;
6001	6040
6002	6041	assert(union_obj.layout != .@"packed");
...	...	@@ -6006,28 +6045,28 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
6006	6045	assert(layout.payload_size != 0); // otherwise the value would be comptime-known
6007	6046	assert(isByRef(union_ty, zcu));
6008	6047
6009		const alignment = layout.abi_align.toLlvm();
6010		const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
	6048	const result_ptr = try self.buildZigAlloca(union_ty, layout.abi_align);
6011	6049	const llvm_payload = try self.resolveInst(extra.init);
6012	6050	const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6013	6051	assert(field_ty.hasRuntimeBits(zcu));
6014	6052
6015	6053	{
6016	6054	const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset());
6017		try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty);
	6055	try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty, .normal);
6018	6056	}
6019	6057
6020	6058	if (layout.tag_size != 0) {
6021		const loaded_enum = ip.loadEnumType(union_obj.enum_tag_type);
	6059	const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
	6060	const loaded_enum = ip.loadEnumType(tag_ty.toIntern());
6022	6061	const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {
6023	6062	.none => try o.builder.intConst(
6024		try o.lowerType(.fromInterned(union_obj.enum_tag_type)),
	6063	try o.lowerType(.fromInterned(union_obj.enum_tag_type), .by_value),
6025	6064	extra.field_index, // auto-numbered
6026	6065	),
6027		else => \|tag_val_ip\| try o.lowerValue(tag_val_ip),
	6066	else => \|tag_val_ip\| try o.lowerValue(tag_val_ip, .by_value),
6028	6067	};
6029	6068	const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());
6030		_ = try self.wip.store(.normal, llvm_tag_val.toValue(), tag_ptr, layout.tag_align.toLlvm());
	6069	try self.store(tag_ptr, layout.tag_align, llvm_tag_val.toValue(), tag_ty, .normal);
6031	6070	}
6032	6071
6033	6072	return result_ptr;
...	...	@@ -6086,7 +6125,7 @@ fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6086	6125	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6087	6126	const inst_ty = self.typeOfIndex(inst);
6088	6127	const operand = try self.resolveInst(ty_op.operand);
6089		return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty), "");
	6128	return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty, .by_value), "");
6090	6129	}
6091	6130
6092	6131	fn workIntrinsic(
...	...	@@ -6134,7 +6173,7 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6134	6173	// Load the work_group_* member from the struct as u16.
6135	6174	// Just treat the dispatch pointer as an array of u16 to keep things simple.
6136	6175	const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2);
6137		return self.wip.load(.normal, .i16, workgroup_size_ptr, comptime .fromByteUnits(2), "");
	6176	return self.load(workgroup_size_ptr, .@"2", .u16, .normal);
6138	6177	},
6139	6178	.nvptx, .nvptx64 => {
6140	6179	return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
...	...	@@ -6169,8 +6208,8 @@ fn optCmpNull(
6169	6208	comptime assert(optional_layout_version == 3);
6170	6209	// Non-null bit is always after the payload, with no padding because it has alignment 1.
6171	6210	const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu));
6172		const non_null = try self.wip.load(access_kind, .i8, non_null_ptr, .default, "");
6173		return self.wip.icmp(cond, non_null, try self.object.builder.intValue(.i8, 0), "");
	6211	const non_null = try self.load(non_null_ptr, .@"1", .bool, access_kind);
	6212	return self.wip.icmp(cond, non_null, .false, "");
6174	6213	}
6175	6214
6176	6215	/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
...	...	@@ -6187,13 +6226,9 @@ fn optPayloadHandle(
6187	6226	// Payload is first field so always at the same address as the optional itself.
6188	6227	const payload_ptr = opt_ptr;
6189	6228
6190		const payload_align = payload_ty.abiAlignment(zcu).toLlvm();
6191		if (isByRef(payload_ty, zcu)) {
6192		if (can_elide_load) return payload_ptr;
6193		return fg.loadByRef(payload_ptr, payload_ty, payload_align, .normal);
6194		} else {
6195		return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_align);
6196		}
	6229	if (can_elide_load and isByRef(payload_ty, zcu)) return payload_ptr;
	6230
	6231	return fg.load(payload_ptr, .none, payload_ty, .normal);
6197	6232	}
6198	6233
6199	6234	fn fieldPtr(
...	...	@@ -6217,220 +6252,156 @@ fn fieldPtr(
6217	6252	return self.ptraddConst(aggregate_ptr, offset);
6218	6253	}
6219	6254
6220		/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.
6221		fn loadTruncate(
6222		fg: *FuncGen,
6223		access_kind: Builder.MemoryAccessKind,
6224		payload_ty: Type,
6225		payload_ptr: Builder.Value,
6226		payload_alignment: Builder.Alignment,
6227		) Allocator.Error!Builder.Value {
6228		// from https://llvm.org/docs/LangRef.html#load-instruction :
6229		// "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. "
6230		// => so load the byte aligned value and trunc the unwanted bits.
6231
6232		const o = fg.object;
6233		const zcu = o.zcu;
6234		const payload_llvm_ty = try o.lowerType(payload_ty);
6235		const abi_size = payload_ty.abiSize(zcu);
6236
6237		const load_llvm_ty = if (payload_ty.isAbiInt(zcu))
6238		try o.builder.intType(@intCast(abi_size * 8))
6239		else
6240		payload_llvm_ty;
6241		const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, "");
6242		const shifted = if (payload_llvm_ty != load_llvm_ty and zcu.getTarget().cpu.arch.endian() == .big)
6243		try fg.wip.bin(.lshr, loaded, try o.builder.intValue(
6244		load_llvm_ty,
6245		(payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8,
6246		), "")
6247		else
6248		loaded;
6249
6250		return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, "");
6251		}
6252
6253		/// Load a by-ref type by constructing a new alloca and performing a memcpy.
6254		fn loadByRef(
6255		fg: *FuncGen,
6256		ptr: Builder.Value,
6257		pointee_type: Type,
6258		ptr_alignment: Builder.Alignment,
6259		access_kind: Builder.MemoryAccessKind,
6260		) Allocator.Error!Builder.Value {
6261		const o = fg.object;
6262		const pointee_llvm_ty = try o.lowerType(pointee_type);
6263		const result_align = InternPool.Alignment.fromLlvm(ptr_alignment)
6264		.max(pointee_type.abiAlignment(o.zcu)).toLlvm();
6265		const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);
6266		const size_bytes = pointee_type.abiSize(o.zcu);
6267		_ = try fg.wip.callMemCpy(
6268		result_ptr,
6269		result_align,
6270		ptr,
6271		ptr_alignment,
6272		try o.builder.intValue(try o.lowerType(.usize), size_bytes),
6273		access_kind,
6274		fg.disable_intrinsics,
6275		);
6276		return result_ptr;
6277		}
6278
6279		/// If `isByRef` returns `true` for `elem_ty`, this still performs a copy by memcpy'ing the value
6280		/// into a new alloca.
	6255	/// Non-atomic, non-bitpacked load of type `load_ty` from pointer `ptr`.
	6256	///
	6257	/// `ptr` has alignment `ptr_align`, or `load_ty.abiAlignment(zcu)` if `ptr_align` is `.none`.
	6258	///
	6259	/// If `load_ty` is a by-ref type, then the value is copied to a new alloca with a memcpy, and a
	6260	/// pointer to that alloca is returned.
6281	6261	fn load(
6282	6262	fg: *FuncGen,
6283	6263	ptr: Builder.Value,
6284		elem_ty: Type,
6285		ptr_alignment: Builder.Alignment,
	6264	ptr_align: InternPool.Alignment,
	6265	load_ty: Type,
6286	6266	access_kind: Builder.MemoryAccessKind,
6287	6267	) Allocator.Error!Builder.Value {
6288		const zcu = fg.object.zcu;
6289		if (isByRef(elem_ty, zcu)) {
6290		return fg.loadByRef(ptr, elem_ty, ptr_alignment, access_kind);
6291		} else {
6292		return fg.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment);
6293		}
6294		}
6295
6296		fn storeFull(
6297		self: *FuncGen,
6298		ptr: Builder.Value,
6299		ptr_ty: Type,
6300		elem: Builder.Value,
6301		ordering: Builder.AtomicOrdering,
6302		) Allocator.Error!void {
6303		const o = self.object;
	6268	const o = fg.object;
6304	6269	const zcu = o.zcu;
6305		const info = ptr_ty.ptrInfo(zcu);
6306		const elem_ty = Type.fromInterned(info.child);
6307		if (!elem_ty.hasRuntimeBits(zcu)) {
6308		return;
6309		}
6310		const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
6311		const access_kind: Builder.MemoryAccessKind =
6312		if (info.flags.is_volatile) .@"volatile" else .normal;
6313
6314		if (info.flags.vector_index != .none) {
6315		const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
6316		const vec_elem_ty = try o.lowerType(elem_ty);
6317		const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
6318	6270
6319		const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, "");
	6271	const abi_align = load_ty.abiAlignment(zcu);
	6272	const abi_size = load_ty.abiSize(zcu);
6320	6273
6321		const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, "");
	6274	const llvm_ptr_align: Builder.Alignment = switch (ptr_align) {
	6275	.none => abi_align.toLlvm(),
	6276	else => \|a\| a.toLlvm(),
	6277	};
6322	6278
6323		assert(ordering == .none);
6324		_ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment);
6325		return;
	6279	if (isByRef(load_ty, zcu)) {
	6280	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
	6281	const result_ptr = try fg.buildZigAlloca(load_ty, .none);
	6282	_ = try fg.wip.callMemCpy(
	6283	result_ptr,
	6284	abi_align.toLlvm(),
	6285	ptr,
	6286	llvm_ptr_align,
	6287	try o.builder.intValue(llvm_usize_ty, abi_size),
	6288	access_kind,
	6289	fg.disable_intrinsics,
	6290	);
	6291	return result_ptr;
6326	6292	}
6327	6293
6328		if (info.packed_offset.host_size != 0) {
6329		const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
6330		assert(ordering == .none);
6331		const containing_int =
6332		try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, "");
6333		const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
6334		const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset);
6335		// Convert to equally-sized integer type in order to perform the bit
6336		// operations on the value to store
6337		const value_bits_type = try o.builder.intType(@intCast(elem_bits));
6338		const value_bits = if (elem_ty.isPtrAtRuntime(zcu))
6339		try self.wip.cast(.ptrtoint, elem, value_bits_type, "")
6340		else
6341		try self.wip.cast(.bitcast, elem, value_bits_type, "");
6342
6343		const mask_val = blk: {
6344		const zext = try self.wip.cast(
6345		.zext,
6346		try o.builder.intValue(value_bits_type, -1),
6347		containing_int_ty,
6348		"",
6349		);
6350		const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), "");
6351		break :blk try self.wip.bin(
6352		.xor,
6353		shl,
6354		try o.builder.intValue(containing_int_ty, -1),
6355		"",
6356		);
6357		};
6358
6359		const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, "");
6360		const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, "");
6361		const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), "");
6362		const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, "");
	6294	const llvm_memory_ty = try o.lowerType(load_ty, .in_memory);
	6295	const llvm_value_ty = try o.lowerType(load_ty, .by_value);
6363	6296
6364		assert(ordering == .none);
6365		_ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment);
6366		return;
6367		}
6368		if (!isByRef(elem_ty, zcu)) {
6369		_ = try self.wip.storeAtomic(
6370		access_kind,
6371		elem,
6372		ptr,
6373		self.sync_scope,
6374		ordering,
6375		ptr_alignment,
6376		);
6377		return;
	6297	if (llvm_memory_ty != llvm_value_ty) {
	6298	assert(load_ty.isAbiInt(zcu));
	6299	// `load_ty` is an integer type with padding bits. In theory, we shouldn't need any special
	6300	// handling for these, as LLVM's documented semantics are a valid implementation of Zig's
	6301	// semantics. However:
	6302	//
	6303	// * LLVM's lowering for these integer types generally leads to poor codegen, as integers
	6304	// are only extended to the next byte, instead of to the next "natural" integer type.
	6305	//
	6306	// * Clang never emits loads or stores of these types, so LLVM's support for them is rather
	6307	// flaky---we have encountered several LLVM bugs caused by incorrect handling of them.
	6308	//
	6309	// Therefore, we handle these memory accesses specially: in this case we will actually load
	6310	// the next-largest "natural" integer type and then truncate to `load_ty`.
	6311	const loaded = try fg.wip.load(access_kind, llvm_memory_ty, ptr, llvm_ptr_align, "");
	6312	// For packed structs, current Zig semantics don't really allow us to make the padding bits
	6313	// well-defined. This should be solved once https://github.com/ziglang/zig/issues/24061 is
	6314	// implemented, but until then, do a normal trunc for packed types.
	6315	return fg.wip.cast(switch (load_ty.zigTypeTag(zcu)) {
	6316	.@"struct", .@"union" => .trunc,
	6317	else => switch (load_ty.intInfo(zcu).signedness) {
	6318	.unsigned => .@"trunc nuw",
	6319	.signed => .@"trunc nsw",
	6320	},
	6321	}, loaded, llvm_value_ty, "");
6378	6322	}
6379		assert(ordering == .none);
6380		_ = try self.wip.callMemCpy(
6381		ptr,
6382		ptr_alignment,
6383		elem,
6384		elem_ty.abiAlignment(zcu).toLlvm(),
6385		try o.builder.intValue(try o.lowerType(.usize), elem_ty.abiSize(zcu)),
6386		access_kind,
6387		self.disable_intrinsics,
6388		);
	6323
	6324	// `load_ty` is a simple by-val type which requires no special handling.
	6325	return fg.wip.load(access_kind, llvm_value_ty, ptr, llvm_ptr_align, "");
6389	6326	}
6390	6327
6391		/// Non-atomic, non-volatile, non-packed store.
	6328	/// Non-atomic, non-bitpacked store of `elem` to pointer `ptr`.
	6329	///
	6330	/// `ptr` has alignment `ptr_align`, or `elem_ty.abiAlignment(zcu)` if `ptr_align` is `.none`.
	6331	///
	6332	/// If `elem_ty` is a by-ref type, then `elem` is itself a pointer, and a memcpy is emitted.
6392	6333	fn store(
6393	6334	fg: *FuncGen,
6394	6335	ptr: Builder.Value,
6395	6336	ptr_align: InternPool.Alignment,
6396	6337	elem: Builder.Value,
6397	6338	elem_ty: Type,
	6339	access_kind: Builder.MemoryAccessKind,
6398	6340	) Allocator.Error!void {
6399	6341	const o = fg.object;
6400	6342	const zcu = o.zcu;
	6343
	6344	const abi_align = elem_ty.abiAlignment(zcu);
	6345	const abi_size = elem_ty.abiSize(zcu);
	6346
6401	6347	const llvm_ptr_align = switch (ptr_align) {
6402		.none => elem_ty.abiAlignment(zcu).toLlvm(),
	6348	.none => abi_align.toLlvm(),
6403	6349	else => ptr_align.toLlvm(),
6404	6350	};
	6351
6405	6352	if (isByRef(elem_ty, zcu)) {
	6353	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
6406	6354	_ = try fg.wip.callMemCpy(
6407	6355	ptr,
6408	6356	llvm_ptr_align,
6409	6357	elem,
6410		elem_ty.abiAlignment(zcu).toLlvm(),
6411		try o.builder.intValue(
6412		try o.lowerType(.usize),
6413		elem_ty.abiSize(zcu),
6414		),
6415		.normal,
	6358	abi_align.toLlvm(),
	6359	try o.builder.intValue(llvm_usize_ty, abi_size),
	6360	access_kind,
6416	6361	fg.disable_intrinsics,
6417	6362	);
6418		} else {
	6363	return;
	6364	}
	6365
	6366	assert(elem.typeOfWip(&fg.wip) == try o.lowerType(elem_ty, .by_value));
	6367
	6368	const llvm_memory_ty = try o.lowerType(elem_ty, .in_memory);
	6369	const llvm_value_ty = try o.lowerType(elem_ty, .by_value);
	6370
	6371	if (llvm_memory_ty != llvm_value_ty) {
	6372	assert(elem_ty.isAbiInt(zcu));
	6373	// `elem_ty` is an integer type with padding bits, so we need to handle it specially---see
	6374	// the corresponding comment in `FuncGen.load` for more details.
	6375	const extended = try fg.wip.cast(switch (elem_ty.intInfo(zcu).signedness) {
	6376	.unsigned => .zext,
	6377	.signed => .sext,
	6378	}, elem, llvm_memory_ty, "");
6419	6379	_ = try fg.wip.storeAtomic(
6420		.normal,
6421		elem,
	6380	access_kind,
	6381	extended,
6422	6382	ptr,
6423	6383	fg.sync_scope,
6424	6384	.none,
6425	6385	llvm_ptr_align,
6426	6386	);
	6387	return;
6427	6388	}
	6389
	6390	// `elem_ty` is a simple by-val type which requires no special handling.
	6391	_ = try fg.wip.storeAtomic(
	6392	access_kind,
	6393	elem,
	6394	ptr,
	6395	fg.sync_scope,
	6396	.none,
	6397	llvm_ptr_align,
	6398	);
6428	6399	}
6429	6400
6430	6401	fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
6431	6402	const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
6432	6403	const o = fg.object;
6433		const usize_ty = try o.lowerType(.usize);
	6404	const usize_ty = try o.lowerType(.usize, .by_value);
6434	6405	const zero = try o.builder.intValue(usize_ty, 0);
6435	6406	const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);
6436	6407	const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");
...	...	@@ -6452,19 +6423,19 @@ fn valgrindClientRequest(
6452	6423	const target = zcu.getTarget();
6453	6424	if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
6454	6425
6455		const llvm_usize = try o.lowerType(.usize);
6456		const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm();
	6426	const llvm_usize = try o.lowerType(.usize, .by_value);
	6427	const usize_align = Type.usize.abiAlignment(zcu).toLlvm();
6457	6428
6458	6429	const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
6459	6430	const array_ptr = if (fg.valgrind_client_request_array == .none) a: {
6460		const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment);
	6431	const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_align);
6461	6432	fg.valgrind_client_request_array = array_ptr;
6462	6433	break :a array_ptr;
6463	6434	} else fg.valgrind_client_request_array;
6464	6435	const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
6465	6436	for (array_elements, 0..) \|elem, i\| {
6466	6437	const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu));
6467		_ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);
	6438	try fg.store(elem_ptr, .none, elem, .usize, .normal);
6468	6439	}
6469	6440
6470	6441	const arch_specific: struct {
...	...	@@ -6734,7 +6705,7 @@ const ParamTypeIterator = struct {
6734	6705	while (field_it.next()) \|field_index\| {
6735	6706	const field_ty = ty.fieldType(field_index, zcu);
6736	6707	if (!field_ty.hasRuntimeBits(zcu)) continue;
6737		it.types_buffer[it.types_len] = try it.object.lowerType(field_ty);
	6708	it.types_buffer[it.types_len] = try it.object.lowerType(field_ty, .by_value);
6738	6709	it.offsets_buffer[it.types_len] = ty.structFieldOffset(field_index, zcu);
6739	6710	it.types_len += 1;
6740	6711	}
...	...	@@ -6751,7 +6722,7 @@ const ParamTypeIterator = struct {
6751	6722	it.llvm_index += 1;
6752	6723	return .byval;
6753	6724	} else {
6754		it.types_buffer[0..1].* = .{try it.object.lowerType(scalar_ty)};
	6725	it.types_buffer[0..1].* = .{try it.object.lowerType(scalar_ty, .by_value)};
6755	6726	it.offsets_buffer[0..2].* = .{ 0, scalar_ty.abiSize(zcu) };
6756	6727	it.types_len = 1;
6757	6728	it.llvm_index += 1;
...	...	@@ -6932,166 +6903,138 @@ pub fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) Para
6932	6903	};
6933	6904	}
6934	6905
6935		fn returnTypeByRef(zcu: Zcu, target: const std.Target, ty: Type) bool {
6936		if (isByRef(ty, zcu)) {
6937		return true;
6938		} else if (target.cpu.arch.isX86() and
6939		!target.cpu.has(.x86, .avx512f) and
6940		ty.totalVectorBits(zcu) >= 512)
6941		{
6942		// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
6943		// "512-bit vector arguments require 'avx512f' for AVX512"
6944		return true;
6945		} else {
6946		return false;
6947		}
6948		}
	6906	pub const FnReturnStrat = union(enum) {
	6907	/// The function return type is OPV (zero-bit), so the LLVM function return type is `void`.
	6908	void,
	6909	/// An sret parameter is used. The LLVM function return type is `void`.
	6910	sret,
	6911	/// The function's return type directly corresponds to the LLVM function return type.
	6912	///
	6913	/// The return type is by-val, i.e. `isByRef` returns `false`.
	6914	by_val,
	6915	/// The LLVM function returns the given `Builder.Type` by reinterpreting memory containing the
	6916	/// actual return value. The actual return type may be by-val or by-ref.
	6917	mem_cast: Builder.Type,
6949	6918
6950		pub fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: Zcu, target: const std.Target) bool {
6951		const return_type = Type.fromInterned(fn_info.return_type);
6952		if (!return_type.hasRuntimeBits(zcu)) return false;
6953
6954		return switch (fn_info.cc) {
6955		.auto => returnTypeByRef(zcu, target, return_type),
6956		.x86_64_sysv, .x86_64_x32 => firstParamSRetSystemV(return_type, zcu, target),
6957		.x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory,
6958		.x86_sysv, .x86_win => isByRef(return_type, zcu),
6959		.x86_stdcall => !isScalar(zcu, return_type),
6960		.x86_fastcall => firstParamSRetX86Fastcall(zcu, return_type),
6961		.wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect,
6962		.aarch64_aapcs,
6963		.aarch64_aapcs_darwin,
6964		.aarch64_aapcs_win,
6965		=> aarch64_c_abi.classifyType(return_type, zcu) == .memory,
6966		.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
6967		.memory, .i64_array => true,
6968		.i32_array => \|size\| size != 1,
6969		.byval => false,
6970		},
6971		.riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory,
6972		.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
6973		.memory, .i32_array => true,
6974		.byval => false,
6975		},
6976		else => false, // TODO: investigate other targets/callconvs
6977		};
6978		}
6979
6980		fn firstParamSRetX86Fastcall(zcu: *Zcu, ty: Type) bool {
6981		if (isScalar(zcu, ty)) {
6982		return false;
6983		}
6984		const tag = ty.zigTypeTag(zcu);
6985		if (tag == .@"struct" or tag == .@"union") {
6986		const size = ty.abiSize(zcu);
6987		if (size == 1 or size == 2 or size == 4 or size == 8) {
6988		return false;
6989		}
	6919	fn forceByVal(o: *Object, ret_ty: Type) Allocator.Error!FnReturnStrat {
	6920	if (!isByRef(ret_ty, o.zcu)) return .by_val;
	6921	return .{ .mem_cast = try o.lowerType(ret_ty, .in_memory) };
6990	6922	}
6991		return true;
6992		}
6993
6994		fn firstParamSRetSystemV(ty: Type, zcu: Zcu, target: const std.Target) bool {
6995		if (isScalar(zcu, ty)) return false;
6996		const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret);
6997		if (class[0] == .memory) return true;
6998		if (class[0] == .x87 and class[2] != .none) return true;
6999		return false;
7000		}
7001
	6923	};
7002	6924	/// In order to support the C calling convention, some return types need to be lowered
7003	6925	/// completely differently in the function prototype to honor the C ABI, and then
7004	6926	/// be effectively bitcasted to the actual return type.
7005		pub fn lowerFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	6927	pub fn fnReturnStrat(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7006	6928	const zcu = o.zcu;
7007		const return_type = Type.fromInterned(fn_info.return_type);
7008		if (!return_type.hasRuntimeBits(zcu)) {
7009		assert(!return_type.isError(zcu));
7010		return .void;
7011		}
7012		const target = zcu.getTarget();
	6929	const ret_ty: Type = .fromInterned(fn_info.return_type);
	6930	ret_ty.assertHasLayout(zcu);
	6931	if (!ret_ty.hasRuntimeBits(zcu)) return .void;
7013	6932	switch (fn_info.cc) {
7014	6933	.@"inline" => unreachable,
7015		.auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(return_type),
	6934	.auto => {
	6935	if (isByRef(ret_ty, zcu)) return .sret;
	6936
	6937	const target = zcu.getTarget();
	6938	if (target.cpu.arch.isX86() and
	6939	!target.cpu.has(.x86, .avx512f) and
	6940	ret_ty.totalVectorBits(zcu) >= 512)
	6941	{
	6942	// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
	6943	// "512-bit vector arguments require 'avx512f' for AVX512"
	6944	return .sret;
	6945	}
	6946
	6947	return .by_val;
	6948	},
7016	6949	.x86_64_sysv, .x86_64_x32 => return lowerSystemVFnRetTy(o, fn_info),
7017	6950	.x86_64_win => return lowerWin64FnRetTy(o, fn_info),
7018		.x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(return_type) else .void,
7019		.x86_fastcall => return lowerX86FastcallFnRetTy(o, zcu, return_type),
7020		.x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(return_type),
7021		.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) {
7022		.memory => return .void,
7023		.float_array => return o.lowerType(return_type),
7024		.byval => return o.lowerType(return_type),
7025		.integer => return .i64,
7026		.double_integer => return o.builder.arrayType(2, .i64),
	6951	.x86_stdcall => if (isScalar(zcu, ret_ty)) {
	6952	assert(!isByRef(ret_ty, zcu));
	6953	return .by_val;
	6954	} else return .sret,
	6955	.x86_fastcall => return lowerX86FastcallFnRetTy(o, zcu, ret_ty),
	6956	.x86_sysv, .x86_win => return if (isByRef(ret_ty, zcu)) .sret else .by_val,
	6957	.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(ret_ty, zcu)) {
	6958	.memory => return .sret,
	6959	.float_array, .byval => return .forceByVal(o, ret_ty),
	6960	.integer => return .{ .mem_cast = .i64 },
	6961	.double_integer => return .{ .mem_cast = try o.builder.arrayType(2, .i64) },
7027	6962	},
7028		.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
7029		.memory, .i64_array => return .void,
7030		.i32_array => \|len\| return if (len == 1) .i32 else .void,
7031		.byval => return o.lowerType(return_type),
	6963	.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(ret_ty, zcu, .ret)) {
	6964	.memory, .i64_array => return .sret,
	6965	.i32_array => \|len\| return if (len == 1) .{ .mem_cast = .i32 } else .sret,
	6966	.byval => return .forceByVal(o, ret_ty),
7032	6967	},
7033		.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
7034		.memory, .i32_array => return .void,
7035		.byval => return o.lowerType(return_type),
	6968	.mips_o32 => switch (mips_c_abi.classifyType(ret_ty, zcu, .ret)) {
	6969	.memory, .i32_array => return .sret,
	6970	.byval => return .forceByVal(o, ret_ty),
7036	6971	},
7037		.riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) {
7038		.memory => return .void,
7039		.integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
	6972	.riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(ret_ty, zcu)) {
	6973	.memory => return .sret,
	6974	.integer => return .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) },
7040	6975	.double_integer => {
7041	6976	const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {
7042	6977	.riscv64, .riscv64be => .i64,
7043	6978	.riscv32, .riscv32be => .i32,
7044	6979	else => unreachable,
7045	6980	};
7046		return o.builder.structType(.normal, &.{ integer, integer });
	6981	return .{ .mem_cast = try o.builder.structType(.normal, &.{ integer, integer }) };
7047	6982	},
7048		.byval => return o.lowerType(return_type),
	6983	.byval => return .forceByVal(o, ret_ty),
7049	6984	.fields => {
7050	6985	var types_len: usize = 0;
7051	6986	var types: [8]Builder.Type = undefined;
7052		for (0..return_type.structFieldCount(zcu)) \|field_index\| {
7053		const field_ty = return_type.fieldType(field_index, zcu);
	6987	for (0..ret_ty.structFieldCount(zcu)) \|field_index\| {
	6988	const field_ty = ret_ty.fieldType(field_index, zcu);
7054	6989	if (!field_ty.hasRuntimeBits(zcu)) continue;
7055		types[types_len] = try o.lowerType(field_ty);
	6990	types[types_len] = try o.lowerType(field_ty, .by_value);
7056	6991	types_len += 1;
7057	6992	}
7058		return o.builder.structType(.normal, types[0..types_len]);
	6993	return .{ .mem_cast = try o.builder.structType(.normal, types[0..types_len]) };
7059	6994	},
7060	6995	},
7061		.wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) {
7062		.direct => \|scalar_ty\| return o.lowerType(scalar_ty),
7063		.indirect => return .void,
	6996	.wasm_mvp => switch (wasm_c_abi.classifyType(ret_ty, zcu)) {
	6997	.direct => \|scalar_ty\| if (scalar_ty.toIntern() == ret_ty.toIntern()) {
	6998	assert(!isByRef(ret_ty, zcu));
	6999	return .by_val;
	7000	} else {
	7001	return .{ .mem_cast = try o.lowerType(scalar_ty, .by_value) };
	7002	},
	7003	.indirect => return .sret,
7064	7004	},
7065	7005	// TODO investigate other callconvs
7066		else => return o.lowerType(return_type),
	7006	else => return .forceByVal(o, ret_ty),
7067	7007	}
7068	7008	}
7069	7009
7070		fn lowerX86FastcallFnRetTy(o: Object, zcu: Zcu, ty: Type) Allocator.Error!Builder.Type {
	7010	fn lowerX86FastcallFnRetTy(o: Object, zcu: Zcu, ty: Type) Allocator.Error!FnReturnStrat {
7071	7011	if (isScalar(zcu, ty)) {
7072		return o.lowerType(ty);
	7012	assert(!isByRef(ty, zcu));
	7013	return .by_val;
7073	7014	}
7074	7015	const tag = ty.zigTypeTag(zcu);
7075	7016	if (tag == .@"struct" or tag == .@"union") {
7076	7017	const size = ty.abiSize(zcu);
7077	7018	if (size == 1 or size == 2 or size == 4 or size == 8) {
7078		return o.builder.intType(@intCast(size * 8));
	7019	return .{ .mem_cast = try o.builder.intType(@intCast(size * 8)) };
7079	7020	}
7080	7021	}
7081		return .void;
	7022	return .sret;
7082	7023	}
7083	7024
7084		fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	7025	fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7085	7026	const zcu = o.zcu;
7086		const return_type = Type.fromInterned(fn_info.return_type);
7087		switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) {
7088		.integer => {
7089		if (isScalar(zcu, return_type)) {
7090		return o.lowerType(return_type);
7091		} else {
7092		return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8));
7093		}
	7027	const ret_ty = Type.fromInterned(fn_info.return_type);
	7028	switch (x86_64_abi.classifyWindows(ret_ty, zcu, zcu.getTarget(), .ret)) {
	7029	.integer => if (isScalar(zcu, ret_ty)) {
	7030	assert(!isByRef(ret_ty, zcu));
	7031	return .by_val;
	7032	} else {
	7033	return .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) };
7094	7034	},
	7035	.win_i128 => return .{ .mem_cast = try o.builder.vectorType(.normal, 2, .i64) },
	7036	.memory => return .sret,
	7037
7095	7038	.sse,
7096	7039	.bool_vector_mask,
7097	7040	.integer_per_element,
...	...	@@ -7100,7 +7043,10 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err
7100	7043	.sse_per_xword,
7101	7044	.sse_per_yword,
7102	7045	.sse_per_zword,
7103		=> return o.lowerType(return_type),
	7046	=> {
	7047	assert(!isByRef(ret_ty, zcu));
	7048	return .by_val;
	7049	},
7104	7050	.sseup,
7105	7051	.x87,
7106	7052	.x87up,
...	...	@@ -7108,20 +7054,18 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err
7108	7054	.float,
7109	7055	.float_combine,
7110	7056	=> unreachable,
7111		.win_i128 => return o.builder.vectorType(.normal, 2, .i64),
7112		.memory => return .void,
7113	7057	}
7114	7058	}
7115	7059
7116		fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	7060	fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7117	7061	const zcu = o.zcu;
7118	7062	const ip = &zcu.intern_pool;
7119		const return_type = Type.fromInterned(fn_info.return_type);
7120		return_type.assertHasLayout(zcu);
7121		if (isScalar(zcu, return_type)) {
7122		return o.lowerType(return_type);
	7063	const ret_ty = Type.fromInterned(fn_info.return_type);
	7064	if (isScalar(zcu, ret_ty)) {
	7065	assert(!isByRef(ret_ty, zcu));
	7066	return .by_val;
7123	7067	}
7124		const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret);
	7068	const classes = x86_64_abi.classifySystemV(ret_ty, zcu, zcu.getTarget(), .ret);
7125	7069	var types_index: u32 = 0;
7126	7070	var types_buffer: [8]Builder.Type = undefined;
7127	7071	for (classes) \|class\| {
...	...	@@ -7151,13 +7095,13 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7151	7095	types_index += 1;
7152	7096	},
7153	7097	.x87 => {
7154		if (types_index != 0 or classes[2] != .none) return .void;
	7098	if (types_index != 0 or classes[2] != .none) return .sret;
7155	7099	types_buffer[types_index] = .x86_fp80;
7156	7100	types_index += 1;
7157	7101	},
7158	7102	.x87up => continue,
7159	7103	.none => break,
7160		.memory => return .void,
	7104	.memory => return .sret,
7161	7105	.win_i128 => unreachable, // windows only
7162	7106	.bool_vector_mask,
7163	7107	.integer_per_element,
...	...	@@ -7172,9 +7116,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7172	7116	const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
7173	7117	if (first_non_integer == null or classes[first_non_integer.?] == .none) {
7174	7118	assert(first_non_integer orelse classes.len == types_index);
7175		switch (ip.indexToKey(return_type.toIntern())) {
	7119	switch (ip.indexToKey(ret_ty.toIntern())) {
7176	7120	.struct_type => {
7177		const size = return_type.abiSize(zcu);
	7121	const size = ret_ty.abiSize(zcu);
7178	7122	assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
7179	7123	if (size % 8 > 0) {
7180	7124	types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));
...	...	@@ -7182,9 +7126,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7182	7126	},
7183	7127	else => {},
7184	7128	}
7185		if (types_index == 1) return types_buffer[0];
	7129	if (types_index == 1) return .{ .mem_cast = types_buffer[0] };
7186	7130	}
7187		return o.builder.structType(.normal, types_buffer[0..types_index]);
	7131	return .{ .mem_cast = try o.builder.structType(.normal, types_buffer[0..types_index]) };
7188	7132	}
7189	7133
7190	7134	/// This function deliberately does not handle `_BitInt` because it typically
...	...	@@ -7283,33 +7227,6 @@ fn isScalar(zcu: *Zcu, ty: Type) bool {
7283	7227	};
7284	7228	}
7285	7229
7286		pub fn buildAllocaInner(
7287		wip: *Builder.WipFunction,
7288		llvm_ty: Builder.Type,
7289		alignment: Builder.Alignment,
7290		target: *const std.Target,
7291		) Allocator.Error!Builder.Value {
7292		const address_space = llvmAllocaAddressSpace(target);
7293
7294		const alloca = blk: {
7295		const prev_cursor = wip.cursor;
7296		const prev_debug_location = wip.debug_location;
7297		defer {
7298		wip.cursor = prev_cursor;
7299		if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
7300		wip.debug_location = prev_debug_location;
7301		}
7302
7303		wip.cursor = .{ .block = .entry };
7304		wip.debug_location = .no_location;
7305		break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
7306		};
7307
7308		// The pointer returned from this function should have the generic address space,
7309		// if this isn't the case then cast it to the generic address space.
7310		return wip.conv(.unneeded, alloca, .ptr, "");
7311		}
7312
7313	7230	/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
7314	7231	/// or as an LLVM value.
7315	7232	pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
...	...	@@ -7351,7 +7268,7 @@ pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
7351	7268	},
7352	7269	.@"union" => switch (ty.containerLayout(zcu)) {
7353	7270	.@"packed" => false,
7354		else => ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),
	7271	else => ty.hasRuntimeBits(zcu),
7355	7272	},
7356	7273	};
7357	7274	}
...	...	@@ -7380,7 +7297,11 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E
7380	7297	}
7381	7298
7382	7299	fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
7383		return fg.object.ptraddConst(&fg.wip, ptr, offset);
	7300	if (offset == 0) return ptr;
	7301	const o = fg.object;
	7302	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
	7303	const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
	7304	return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
7384	7305	}
7385	7306	fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value {
7386	7307	if (scale == 0) return ptr;

src/codegen/mips/abi.zig+4-5

...	...	@@ -18,24 +18,23 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
18	18	const max_direct_size = target.ptrBitWidth() * 2;
19	19	switch (ty.zigTypeTag(zcu)) {
20	20	.@"struct" => {
21		const bit_size = ty.bitSize(zcu);
22	21	if (ty.containerLayout(zcu) == .@"packed") {
23		if (bit_size > max_direct_size) return .memory;
	22	if (ty.bitSize(zcu) > max_direct_size) return .memory;
24	23	return .byval;
25	24	}
	25	const bit_size = ty.abiSize(zcu) * 8;
26	26	if (bit_size > max_direct_size) return .memory;
27	27	// TODO: for bit_size <= 32 using byval is more correct, but that needs inreg argument attribute
28	28	const count = @as(u8, @intCast(std.mem.alignForward(u64, bit_size, 32) / 32));
29	29	return .{ .i32_array = count };
30	30	},
31	31	.@"union" => {
32		const bit_size = ty.bitSize(zcu);
33	32	if (ty.containerLayout(zcu) == .@"packed") {
34		if (bit_size > max_direct_size) return .memory;
	33	if (ty.bitSize(zcu) > max_direct_size) return .memory;
35	34	return .byval;
36	35	}
	36	const bit_size = ty.abiSize(zcu) * 8;
37	37	if (bit_size > max_direct_size) return .memory;
38
39	38	return .byval;
40	39	},
41	40	.bool => return .byval,

src/codegen/riscv64/CodeGen.zig+12-7

...	...	@@ -51,7 +51,7 @@ const InnerError = codegen.Error \|\| error{OutOfRegisters};
51	51
52	52	pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
53	53	return comptime &.initMany(&.{
54		.expand_intcast_safe,
	54	.expand_int_cast_safe,
55	55	.expand_int_from_float_safe,
56	56	.expand_int_from_float_optimized_safe,
57	57	.expand_add_safe,
...	...	@@ -1453,7 +1453,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1453	1453	.add_safe,
1454	1454	.sub_safe,
1455	1455	.mul_safe,
1456		.intcast_safe,
	1456	.int_cast_safe,
1457	1457	.int_from_float_safe,
1458	1458	.int_from_float_optimized_safe,
1459	1459	=> return func.fail("TODO implement safety_checked_instructions", .{}),
...	...	@@ -1479,7 +1479,14 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1479	1479	.ret_ptr => try func.airRetPtr(inst),
1480	1480	.arg => try func.airArg(inst),
1481	1481	.assembly => try func.airAsm(inst),
1482		.bitcast => try func.airBitCast(inst),
	1482	.bit_cast => try func.airBitCast(inst),
	1483	.ptr_cast => try func.airBitCast(inst),
	1484	.ptr_from_int => try func.airBitCast(inst),
	1485	.int_from_ptr => try func.airBitCast(inst),
	1486	.error_cast => try func.airBitCast(inst),
	1487	.error_from_int => try func.airBitCast(inst),
	1488	.int_from_error => try func.airBitCast(inst),
	1489	.union_from_enum => try func.airBitCast(inst),
1483	1490	.block => try func.airBlock(inst),
1484	1491	.br => try func.airBr(inst),
1485	1492	.repeat => try func.airRepeat(inst),
...	...	@@ -1493,7 +1500,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1493	1500	.dbg_empty_stmt => func.finishAirBookkeeping(),
1494	1501	.fptrunc => try func.airFptrunc(inst),
1495	1502	.fpext => try func.airFpext(inst),
1496		.intcast => try func.airIntCast(inst),
	1503	.int_cast => try func.airIntCast(inst),
1497	1504	.trunc => try func.airTrunc(inst),
1498	1505	.is_non_null => try func.airIsNonNull(inst),
1499	1506	.is_non_null_ptr => try func.airIsNonNullPtr(inst),
...	...	@@ -3953,9 +3960,7 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {
3953	3960	const elem_ptr_ty = func.typeOfIndex(inst);
3954	3961	const base_ptr_ty = func.typeOf(extra.lhs);
3955	3962
3956		if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {
3957		@panic("audit");
3958		}
	3963	assert(elem_ptr_ty.ptrInfo(zcu).flags.vector_index == .none);
3959	3964
3960	3965	const base_ptr_mcv = try func.resolveInst(extra.lhs);
3961	3966	const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {

src/codegen/riscv64/abi.zig+5-6

...	...	@@ -16,9 +16,8 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
16	16	const max_byval_size = target.ptrBitWidth() * 2;
17	17	switch (ty.zigTypeTag(zcu)) {
18	18	.@"struct" => {
19		const bit_size = ty.bitSize(zcu);
20	19	if (ty.containerLayout(zcu) == .@"packed") {
21		if (bit_size > max_byval_size) return .memory;
	20	if (ty.bitSize(zcu) > max_byval_size) return .memory;
22	21	return .byval;
23	22	}
24	23
...	...	@@ -40,17 +39,18 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
40	39	}
41	40
42	41	// TODO this doesn't exactly match what clang produces but its better than nothing
	42	const bit_size = ty.abiSize(zcu) * 8;
43	43	if (bit_size > max_byval_size) return .memory;
44	44	if (bit_size > max_byval_size / 2) return .double_integer;
45	45	return .integer;
46	46	},
47	47	.@"union" => {
48		const bit_size = ty.bitSize(zcu);
49	48	if (ty.containerLayout(zcu) == .@"packed") {
50		if (bit_size > max_byval_size) return .memory;
	49	if (ty.bitSize(zcu) > max_byval_size) return .memory;
51	50	return .byval;
52	51	}
53	52	// TODO this doesn't exactly match what clang produces but its better than nothing
	53	const bit_size = ty.abiSize(zcu) * 8;
54	54	if (bit_size > max_byval_size) return .memory;
55	55	if (bit_size > max_byval_size / 2) return .double_integer;
56	56	return .integer;
...	...	@@ -153,13 +153,12 @@ pub fn classifySystem(ty: Type, zcu: *Zcu) [8]SystemClass {
153	153	},
154	154	.error_union => {
155	155	const payload_ty = ty.errorUnionPayload(zcu);
156		const payload_bits = payload_ty.bitSize(zcu);
157	156
158	157	// the error union itself
159	158	result[0] = .integer;
160	159
161	160	// anyerror!void can fit into one register
162		if (payload_bits == 0) return result;
	161	if (!payload_ty.hasRuntimeBits(zcu)) return result;
163	162
164	163	return memory_class;
165	164	},

src/codegen/sparc64/CodeGen.zig+11-4

...	...	@@ -538,7 +538,14 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
538	538	.ret_ptr => try self.airRetPtr(inst),
539	539	.arg => try self.airArg(inst),
540	540	.assembly => try self.airAsm(inst),
541		.bitcast => try self.airBitCast(inst),
	541	.bit_cast => try self.airBitCast(inst),
	542	.ptr_cast => try self.airBitCast(inst),
	543	.ptr_from_int => try self.airBitCast(inst),
	544	.int_from_ptr => try self.airBitCast(inst),
	545	.error_cast => try self.airBitCast(inst),
	546	.error_from_int => try self.airBitCast(inst),
	547	.int_from_error => try self.airBitCast(inst),
	548	.union_from_enum => try self.airBitCast(inst),
542	549	.block => try self.airBlock(inst),
543	550	.br => try self.airBr(inst),
544	551	.repeat => return self.fail("TODO implement `repeat`", .{}),
...	...	@@ -550,7 +557,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
550	557	.cond_br => try self.airCondBr(inst),
551	558	.fptrunc => @panic("TODO try self.airFptrunc(inst)"),
552	559	.fpext => @panic("TODO try self.airFpext(inst)"),
553		.intcast => try self.airIntCast(inst),
	560	.int_cast => try self.airIntCast(inst),
554	561	.trunc => try self.airTrunc(inst),
555	562	.is_non_null => try self.airIsNonNull(inst),
556	563	.is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"),
...	...	@@ -689,7 +696,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
689	696	.add_safe,
690	697	.sub_safe,
691	698	.mul_safe,
692		.intcast_safe,
	699	.int_cast_safe,
693	700	.int_from_float_safe,
694	701	.int_from_float_optimized_safe,
695	702	=> @panic("TODO implement safety_checked_instructions"),
...	...	@@ -1659,7 +1666,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1659	1666	const info_a = operand_ty.intInfo(zcu);
1660	1667	const info_b = self.typeOfIndex(inst).intInfo(zcu);
1661	1668	if (info_a.signedness != info_b.signedness)
1662		return self.fail("TODO gen intcast sign safety in semantic analysis", .{});
	1669	return self.fail("TODO gen int_cast sign safety in semantic analysis", .{});
1663	1670
1664	1671	if (info_a.bits == info_b.bits)
1665	1672	return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });

src/codegen/spirv/CodeGen.zig+67-28

...	...	@@ -34,7 +34,7 @@ const CodeGen = @This();
34	34
35	35	pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
36	36	return comptime &.initMany(&.{
37		.expand_intcast_safe,
	37	.expand_int_cast_safe,
38	38	.expand_int_from_float_safe,
39	39	.expand_int_from_float_optimized_safe,
40	40	.expand_add_safe,
...	...	@@ -1848,7 +1848,17 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {
1848	1848	.pointer => {
1849	1849	const ptr_info = ty.ptrInfo(zcu);
1850	1850
1851		const child_ty: Type = .fromInterned(ptr_info.child);
	1851	const child_ty: Type = switch (ptr_info.packed_offset.host_size) {
	1852	0 => .fromInterned(ptr_info.child),
	1853	else => switch (ptr_info.flags.vector_index) {
	1854	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate these usages of `pt`.
	1855	.none => try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8),
	1856	else => try pt.vectorType(.{
	1857	.child = ptr_info.child,
	1858	.len = ptr_info.packed_offset.host_size,
	1859	}),
	1860	},
	1861	};
1852	1862	const child_ty_id = try cg.resolveType(child_ty, .indirect);
1853	1863	const storage_class = cg.module.storageClass(ptr_info.flags.address_space);
1854	1864	const ptr_ty_id = try cg.module.ptrType(child_ty_id, storage_class);
...	...	@@ -3847,12 +3857,19 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {
3847	3857	.min => try cg.airMinMax(inst, .min),
3848	3858	.max => try cg.airMinMax(inst, .max),
3849	3859
3850		.bitcast => try cg.airBitCast(inst),
3851		.intcast, .trunc => try cg.airIntCast(inst),
3852		.float_from_int => try cg.airFloatFromInt(inst),
3853		.int_from_float => try cg.airIntFromFloat(inst),
3854		.fpext, .fptrunc => try cg.airFloatCast(inst),
3855		.not => try cg.airNot(inst),
	3860	.bit_cast => try cg.airBitCast(inst),
	3861	.ptr_cast => try cg.airBitCast(inst),
	3862	.ptr_from_int => try cg.airBitCast(inst),
	3863	.int_from_ptr => try cg.airBitCast(inst),
	3864	.error_cast => try cg.airBitCast(inst),
	3865	.error_from_int => try cg.airBitCast(inst),
	3866	.int_from_error => try cg.airBitCast(inst),
	3867	.union_from_enum => try cg.airBitCast(inst),
	3868	.int_cast, .trunc => try cg.airIntCast(inst),
	3869	.float_from_int => try cg.airFloatFromInt(inst),
	3870	.int_from_float => try cg.airIntFromFloat(inst),
	3871	.fpext, .fptrunc => try cg.airFloatCast(inst),
	3872	.not => try cg.airNot(inst),
3856	3873
3857	3874	.array_to_slice => try cg.airArrayToSlice(inst),
3858	3875	.slice => try cg.airSlice(inst),
...	...	@@ -6913,13 +6930,15 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6913	6930	const zcu = cg.module.zcu;
6914	6931	const pt = cg.pt;
6915	6932	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6916		const ptr_ty = cg.typeOf(ty_op.operand);
6917		const ptr_info = ptr_ty.ptrInfo(zcu);
	6933
	6934	const ptr_info = cg.typeOf(ty_op.operand).ptrInfo(zcu);
	6935
6918	6936	const elem_ty = cg.typeOfIndex(inst);
6919		const operand = try cg.resolve(ty_op.operand);
6920		if (!ptr_ty.isVolatilePtr(zcu) and cg.liveness.isUnused(inst)) return null;
	6937	const operand_ptr_id = try cg.resolve(ty_op.operand);
6921	6938
6922		if (cg.virtual_allocas.get(operand)) \|stored\| return stored.?;
	6939	assert(ptr_info.child == elem_ty.toIntern());
	6940
	6941	if (cg.virtual_allocas.get(operand_ptr_id)) \|stored\| return stored.?;
6923	6942
6924	6943	if (ptr_info.packed_offset.host_size != 0 and
6925	6944	ptr_info.flags.vector_index == .none)
...	...	@@ -6927,7 +6946,7 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6927	6946	const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
6928	6947	const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
6929	6948	const host_int_ty = try pt.intType(.unsigned, host_bits);
6930		const host_val = try cg.load(host_int_ty, operand, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
	6949	const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6931	6950	const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
6932	6951	const field_int_ty = try pt.intType(signedness, elem_bit_size);
6933	6952	const narrowed = if (ptr_info.packed_offset.bit_offset > 0) blk: {
...	...	@@ -6946,21 +6965,30 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6946	6965	return try cg.bitCast(elem_ty, field_int_ty, result_id);
6947	6966	}
6948	6967
6949		return try cg.load(elem_ty, operand, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
	6968	const ptr_id = switch (ptr_info.flags.vector_index) {
	6969	.none => operand_ptr_id,
	6970	else => \|index\| ptr_id: {
	6971	const elem_ptr_ty_id = try cg.module.ptrType(
	6972	try cg.resolveType(elem_ty, .indirect),
	6973	cg.module.storageClass(ptr_info.flags.address_space),
	6974	);
	6975	break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@intFromEnum(index)});
	6976	},
	6977	};
	6978	return try cg.load(elem_ty, ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6950	6979	}
6951	6980
6952	6981	fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
6953	6982	const zcu = cg.module.zcu;
6954	6983	const pt = cg.pt;
6955	6984	const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6956		const ptr_ty = cg.typeOf(bin_op.lhs);
6957		const ptr_info = ptr_ty.ptrInfo(zcu);
6958		const elem_ty = ptr_ty.childType(zcu);
6959		const ptr = try cg.resolve(bin_op.lhs);
6960		const value = try cg.resolve(bin_op.rhs);
	6985	const ptr_info = cg.typeOf(bin_op.lhs).ptrInfo(zcu);
	6986	const elem_ty: Type = .fromInterned(ptr_info.child);
	6987	const operand_ptr_id = try cg.resolve(bin_op.lhs);
	6988	const value_id = try cg.resolve(bin_op.rhs);
6961	6989
6962		if (cg.virtual_allocas.getPtr(ptr)) \|slot\| {
6963		slot.* = value;
	6990	if (cg.virtual_allocas.getPtr(operand_ptr_id)) \|slot\| {
	6991	slot.* = value_id;
6964	6992	return;
6965	6993	}
6966	6994
...	...	@@ -6969,19 +6997,19 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
6969	6997	{
6970	6998	const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
6971	6999	const host_int_ty = try pt.intType(.unsigned, host_bits);
6972		const host_val = try cg.load(host_int_ty, ptr, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
	7000	const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6973	7001	const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
6974	7002	const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
6975	7003	const field_int_ty = try pt.intType(signedness, elem_bit_size);
6976	7004
6977	7005	var value_as_int: Id = undefined;
6978	7006	if (elem_ty.ip_index == .bool_type) {
6979		value_as_int = try cg.convertToIndirect(.bool, value);
	7007	value_as_int = try cg.convertToIndirect(.bool, value_id);
6980	7008	value_as_int = try cg.bitCast(field_int_ty, .u1, value_as_int);
6981	7009	} else if (elem_ty.isInt(zcu)) {
6982		value_as_int = value;
	7010	value_as_int = value_id;
6983	7011	} else {
6984		value_as_int = try cg.bitCast(field_int_ty, elem_ty, value);
	7012	value_as_int = try cg.bitCast(field_int_ty, elem_ty, value_id);
6985	7013	}
6986	7014
6987	7015	const extended = blk: {
...	...	@@ -7002,11 +7030,22 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
7002	7030	const combined = try cg.buildBinary(.OpBitwiseOr, cleared, shifted_val);
7003	7031	const combined_id = try combined.materialize(cg);
7004	7032
7005		try cg.store(host_int_ty, ptr, combined_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
	7033	try cg.store(host_int_ty, operand_ptr_id, combined_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7006	7034	return;
7007	7035	}
7008	7036
7009		try cg.store(elem_ty, ptr, value, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
	7037	const ptr_id = switch (ptr_info.flags.vector_index) {
	7038	.none => operand_ptr_id,
	7039	else => \|index\| ptr_id: {
	7040	const elem_ptr_ty_id = try cg.module.ptrType(
	7041	try cg.resolveType(elem_ty, .indirect),
	7042	cg.module.storageClass(ptr_info.flags.address_space),
	7043	);
	7044	break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@intFromEnum(index)});
	7045	},
	7046	};
	7047
	7048	try cg.store(elem_ty, ptr_id, value_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7010	7049	}
7011	7050
7012	7051	fn airRet(cg: *CodeGen, inst: Air.Inst.Index) !void {

src/codegen/wasm/CodeGen.zig+97-25

...	...	@@ -32,7 +32,7 @@ const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
32	32
33	33	pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
34	34	return comptime &.initMany(&.{
35		.expand_intcast_safe,
	35	.expand_int_cast_safe,
36	36	.expand_int_from_float_safe,
37	37	.expand_int_from_float_optimized_safe,
38	38	.expand_add_safe,
...	...	@@ -83,7 +83,6 @@ pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
83	83	.scalarize_shl_sat,
84	84	.scalarize_xor,
85	85	.scalarize_not,
86		.scalarize_bitcast,
87	86	.scalarize_clz,
88	87	.scalarize_ctz,
89	88	.scalarize_popcount,
...	...	@@ -109,7 +108,10 @@ pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
109	108	.scalarize_cmp_vector_optimized,
110	109	.scalarize_fptrunc,
111	110	.scalarize_fpext,
112		.scalarize_intcast,
	111	.scalarize_int_cast,
	112	.scalarize_ptr_cast,
	113	.scalarize_ptr_from_int,
	114	.scalarize_int_from_ptr,
113	115	.scalarize_trunc,
114	116	.scalarize_int_from_float,
115	117	.scalarize_int_from_float_optimized,
...	...	@@ -120,6 +122,8 @@ pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
120	122	.scalarize_shuffle_two,
121	123	.scalarize_select,
122	124	.scalarize_mul_add,
	125
	126	.scalarize_bit_cast_padded_elems,
123	127	});
124	128	}
125	129
...	...	@@ -1550,9 +1554,17 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1550	1554	try cg.finishAir(inst, result, &.{ty_op.operand});
1551	1555	},
1552	1556
1553		.bitcast => cg.airBitcast(inst),
	1557	.ptr_cast => cg.airNopCast(inst),
	1558	.error_cast => cg.airNopCast(inst),
	1559	.error_from_int => cg.airNopCast(inst),
	1560	.int_from_error => cg.airNopCast(inst),
	1561	.ptr_from_int => cg.airNopCast(inst),
	1562	.int_from_ptr => cg.airIntFromPtr(inst),
	1563
	1564	.bit_cast => cg.airBitcast(inst),
	1565	.union_from_enum => cg.airBitcast(inst),
1554	1566
1555		.intcast => {
	1567	.int_cast => {
1556	1568	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1557	1569
1558	1570	const dest_ty = ty_op.ty.toType();
...	...	@@ -1560,7 +1572,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1560	1572	const src_ty = cg.typeOf(ty_op.operand);
1561	1573
1562	1574	if (dest_ty.zigTypeTag(zcu) == .vector) {
1563		return cg.fail("TODO: implement AIR op: intcast for vectors", .{});
	1575	return cg.fail("TODO: implement AIR op: int_cast for vectors", .{});
1564	1576	}
1565	1577
1566	1578	const src_int_ty: IntType = .fromType(cg, src_ty);
...	...	@@ -1875,7 +1887,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1875	1887	.add_safe,
1876	1888	.sub_safe,
1877	1889	.mul_safe,
1878		.intcast_safe,
	1890	.int_cast_safe,
1879	1891	.int_from_float_safe,
1880	1892	.int_from_float_optimized_safe,
1881	1893	=> return cg.fail("TODO implement safety_checked_instructions", .{}),
...	...	@@ -2099,16 +2111,20 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2099	2111	const rhs = try cg.resolveInst(bin_op.rhs);
2100	2112	const ptr_ty = cg.typeOf(bin_op.lhs);
2101	2113	const ptr_info = ptr_ty.ptrInfo(zcu);
2102		const ty = ptr_ty.childType(zcu);
	2114	const elem_ty = ptr_ty.childType(zcu);
2103	2115
2104	2116	if (!safety and bin_op.rhs == .undef) {
2105	2117	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2106	2118	}
2107	2119
2108		assert(!(ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none)); // legalize .expand_packed_store
2109
2110		try cg.store(lhs, rhs, ty, 0);
2111
	2120	const offset: u32 = switch (ptr_info.flags.vector_index) {
	2121	.none => offset: {
	2122	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
	2123	break :offset 0;
	2124	},
	2125	else => \|index\| @intCast(@intFromEnum(index) * elem_ty.abiSize(zcu)),
	2126	};
	2127	try cg.store(lhs, rhs, elem_ty, offset);
2112	2128	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2113	2129	}
2114	2130
...	...	@@ -2121,7 +2137,16 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2121	2137	if (!ty.hasRuntimeBits(zcu)) return;
2122	2138
2123	2139	if (isByRef(ty, zcu, cg.target)) {
2124		return cg.memcpy(lhs, rhs, .{ .imm32 = @intCast(abi_size) });
	2140	const offset_ptr: WValue = switch (offset + lhs.offset()) {
	2141	0 => lhs,
	2142	else => \|total_offset\| ptr: {
	2143	try cg.emitWValue(lhs);
	2144	try cg.addImm32(total_offset);
	2145	try cg.addTag(.i32_add);
	2146	break :ptr .stack;
	2147	},
	2148	};
	2149	return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
2125	2150	}
2126	2151
2127	2152	if (ty.zigTypeTag(zcu) == .vector) {
...	...	@@ -2133,7 +2158,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2133	2158	try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2134	2159	@intFromEnum(std.wasm.SimdOpcode.v128_store),
2135	2160	offset + lhs.offset(),
2136		@intCast(ty.abiAlignment(zcu).toByteUnits() orelse 0),
	2161	@intCast(ty.abiAlignment(zcu).toByteUnits().?),
2137	2162	});
2138	2163	return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2139	2164	}
...	...	@@ -2174,15 +2199,20 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2174	2199	const zcu = pt.zcu;
2175	2200	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2176	2201	const operand = try cg.resolveInst(ty_op.operand);
2177		const ty = ty_op.ty.toType();
	2202	const elem_ty = ty_op.ty.toType();
2178	2203	const ptr_ty = cg.typeOf(ty_op.operand);
2179	2204	const ptr_info = ptr_ty.ptrInfo(zcu);
2180	2205
2181		if (!ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{ty_op.operand});
2182
2183		assert(!(ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none)); // legalize .expand_packed_load
	2206	assert(elem_ty.hasRuntimeBits(zcu));
2184	2207
2185		const result = try cg.load(operand, ty, 0);
	2208	const offset: u32 = switch (ptr_info.flags.vector_index) {
	2209	.none => offset: {
	2210	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
	2211	break :offset 0;
	2212	},
	2213	else => \|index\| @intCast(@intFromEnum(index) * elem_ty.abiSize(zcu)),
	2214	};
	2215	const result = try cg.load(operand, elem_ty, offset);
2186	2216	return cg.finishAir(inst, result, &.{ty_op.operand});
2187	2217	}
2188	2218
...	...	@@ -2191,9 +2221,19 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2191	2221	fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
2192	2222	const zcu = cg.pt.zcu;
2193	2223	if (isByRef(ty, zcu, cg.target)) {
2194		const val = try cg.allocStack(ty);
2195		try cg.store(val, try operand.toLocal(cg, .usize), ty, 0);
2196		return val;
	2224	const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {
	2225	0 => operand,
	2226	else => \|total_offset\| ptr: {
	2227	try cg.emitWValue(operand);
	2228	try cg.addImm32(total_offset);
	2229	try cg.addTag(.i32_add);
	2230	break :ptr .stack;
	2231	},
	2232	};
	2233	const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
	2234	const new_ptr = try cg.allocStack(ty);
	2235	try cg.store(new_ptr, src_ptr, ty, 0);
	2236	return new_ptr;
2197	2237	}
2198	2238
2199	2239	// load local's value from memory by its stack position
...	...	@@ -5235,6 +5275,39 @@ fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5235	5275	return cg.finishAir(inst, .none, &.{});
5236	5276	}
5237	5277
	5278	fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
	5279	const zcu = cg.pt.zcu;
	5280	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	5281
	5282	const operand_ty = cg.typeOf(ty_op.operand);
	5283	const dest_ty = cg.typeOfIndex(inst);
	5284	assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
	5285	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
	5286	assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
	5287
	5288	const operand = try cg.resolveInst(ty_op.operand);
	5289	const result = cg.reuseOperand(ty_op.operand, operand);
	5290	return cg.finishAir(inst, result, &.{ty_op.operand});
	5291	}
	5292
	5293	fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
	5294	const zcu = cg.pt.zcu;
	5295	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	5296
	5297	const operand_ty = cg.typeOf(ty_op.operand);
	5298	const dest_ty = cg.typeOfIndex(inst);
	5299	assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
	5300	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
	5301	assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
	5302
	5303	const operand = try cg.resolveInst(ty_op.operand);
	5304	const result = switch (operand) {
	5305	.stack_offset => try cg.buildPointerOffset(operand, 0, .new),
	5306	else => cg.reuseOperand(ty_op.operand, operand),
	5307	};
	5308	return cg.finishAir(inst, result, &.{ty_op.operand});
	5309	}
	5310
5238	5311	fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5239	5312	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5240	5313	const operand = try cg.resolveInst(ty_op.operand);
...	...	@@ -6340,12 +6413,11 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6340	6413	}
6341	6414	}
6342	6415
6343		const elem_result = if (isByRef(elem_ty, zcu, cg.target))
	6416	const result = if (isByRef(elem_ty, zcu, cg.target))
6344	6417	.stack
6345	6418	else
6346	6419	try cg.load(.stack, elem_ty, 0);
6347
6348		return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
	6420	return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6349	6421	}
6350	6422
6351	6423	fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {

src/codegen/x86_64/CodeGen.zig+33-19

...	...	@@ -47,7 +47,6 @@ pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
47	47	.scalarize_shl,
48	48	.scalarize_shl_exact,
49	49	.scalarize_shl_sat,
50		.scalarize_bitcast,
51	50	.scalarize_ctz,
52	51	.scalarize_popcount,
53	52	.scalarize_byte_swap,
...	...	@@ -58,11 +57,13 @@ pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
58	57	.scalarize_shuffle_two,
59	58	.scalarize_select,
60	59
	60	.scalarize_bit_cast_padded_elems,
	61
61	62	//.unsplat_shift_rhs,
62		.reduce_one_elem_to_bitcast,
63		.splat_one_elem_to_bitcast,
	63	.reduce_one_elem_to_bit_cast,
	64	.splat_one_elem_to_bit_cast,
64	65
65		.expand_intcast_safe,
	66	.expand_int_cast_safe,
66	67	.expand_int_from_float_safe,
67	68	.expand_int_from_float_optimized_safe,
68	69	.expand_add_safe,
...	...	@@ -67433,7 +67434,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67433	67434	};
67434	67435	try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
67435	67436	},
67436		.bitcast => try cg.airBitCast(inst),
	67437	.bit_cast,
	67438	.ptr_cast,
	67439	.ptr_from_int,
	67440	.int_from_ptr,
	67441	.error_cast,
	67442	.error_from_int,
	67443	.int_from_error,
	67444	.union_from_enum,
	67445	=> try cg.airBitCast(inst),
67437	67446	.block => {
67438	67447	const block = cg.air.unwrapBlock(inst);
67439	67448	if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none);
...	...	@@ -93374,7 +93383,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
93374	93383	};
93375	93384	try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
93376	93385	},
93377		.intcast => \|air_tag\| {
	93386	.int_cast => \|air_tag\| {
93378	93387	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
93379	93388	const dst_ty = ty_op.ty.toType();
93380	93389	const src_ty = cg.typeOf(ty_op.operand);
...	...	@@ -98132,7 +98141,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
98132	98141	};
98133	98142	try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
98134	98143	},
98135		.intcast_safe => unreachable,
	98144	.int_cast_safe => unreachable,
98136	98145	.trunc => \|air_tag\| {
98137	98146	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
98138	98147	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
...	...	@@ -104350,7 +104359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104350	104359	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
104351	104360	try ops[0].toSlicePtr(cg);
104352	104361	const dst_ty = ty_pl.ty.toType();
104353		if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: {
	104362	zero_offset: {
104354	104363	const elem_size = dst_ty.childType(zcu).abiSize(zcu);
104355	104364	if (hack_around_sema_opv_bugs and elem_size == 0) break :zero_offset;
104356	104365	while (true) for (&ops) \|*op\| {
...	...	@@ -179053,8 +179062,11 @@ fn genSetReg(
179053	179062	const zcu = pt.zcu;
179054	179063	const abi_size: u32 = @intCast(ty.abiSize(zcu));
179055	179064	const dst_alias = registerAlias(dst_reg, @intCast(cg.unalignedSize(ty)));
179056		if (ty.bitSize(zcu) > dst_alias.size().bitSize(cg.target))
179057		return cg.fail("genSetReg called with a value larger than dst_reg", .{});
	179065	{
	179066	const ty_bit_size = if (ty.hasBitRepresentation(zcu)) ty.bitSize(zcu) else 8 * abi_size;
	179067	if (ty_bit_size > dst_alias.size().bitSize(cg.target))
	179068	return cg.fail("genSetReg called with a value larger than dst_reg", .{});
	179069	}
179058	179070	switch (src_mcv) {
179059	179071	.none,
179060	179072	.unreach,
...	...	@@ -180127,14 +180139,19 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
180127	180139	break :dst dst_mcv;
180128	180140	};
180129	180141
180130		if (dst_ty.isRuntimeFloat()) break :result dst_mcv;
	180142	switch (dst_ty.zigTypeTag(zcu)) {
	180143	.float, .error_union, .error_set, .vector => break :result dst_mcv,
	180144	.@"struct", .@"union" => if (dst_ty.containerLayout(zcu) != .@"packed") break :result dst_mcv,
	180145	.optional, .pointer => if (!dst_ty.isPtrAtRuntime(zcu)) break :result dst_mcv,
	180146	else => {},
	180147	}
180131	180148
180132	180149	if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and src_ty.zigTypeTag(zcu) != .@"struct" and
180133	180150	dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;
180134	180151
180135	180152	const abi_size = dst_ty.abiSize(zcu);
180136	180153	const bit_size = dst_ty.bitSize(zcu);
180137		if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;
	180154	if (abi_size * 8 <= bit_size) break :result dst_mcv;
180138	180155
180139	180156	const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable;
180140	180157	const high_mcv: MCValue = switch (dst_mcv) {
...	...	@@ -182410,7 +182427,7 @@ fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
182410	182427	const zcu = pt.zcu;
182411	182428	const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{
182412	182429	.signedness = .unsigned,
182413		.bits = @intCast(ty.bitSize(zcu)),
	182430	.bits = @intCast(if (ty.hasBitRepresentation(zcu)) ty.bitSize(zcu) else ty.abiSize(zcu) * 8),
182414	182431	};
182415	182432	const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;
182416	182433	try self.spillEflagsIfOccupied();
...	...	@@ -182450,10 +182467,6 @@ fn regBitSize(self: *CodeGen, ty: Type) u64 {
182450	182467	};
182451	182468	}
182452	182469
182453		fn regExtraBits(self: *CodeGen, ty: Type) u64 {
182454		return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
182455		}
182456
182457	182470	fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool {
182458	182471	return switch (feature) {
182459	182472	.@"64bit" => switch (cg.target.cpu.arch) {
...	...	@@ -182569,7 +182582,7 @@ fn nonBoolScalarBitSize(cg: *CodeGen, ty: Type) u32 {
182569	182582	.bool_type => vector_type.len,
182570	182583	else => @intCast(Type.fromInterned(vector_type.child).bitSize(zcu)),
182571	182584	},
182572		else => @intCast(ty.bitSize(zcu)),
	182585	else => if (ty.hasBitRepresentation(zcu) or ty.isAbiInt(zcu)) @intCast(ty.bitSize(zcu)) else @intCast(ty.abiSize(zcu) * 8),
182573	182586	};
182574	182587	}
182575	182588
...	...	@@ -192242,7 +192255,8 @@ const Select = struct {
192242	192255	.src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))),
192243	192256	.@"8_size_sub_bit_size" => {
192244	192257	const ty = op.flags.base.ref.typeOf(s);
192245		break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - ty.bitSize(s.cg.pt.zcu));
	192258	const bit_size = s.cg.intInfo(ty).?.bits;
	192259	break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - bit_size);
192246	192260	},
192247	192261	.len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)),
192248	192262	.elem_limbs => @intCast(@divExact(

stage1/zig.h+55-19

...	...	@@ -11,8 +11,6 @@
11	11	#elif defined(__GNUC__)
12	12	#define zig_gcc
13	13	#define zig_gnuc
14		#elif defined(__IBMC__)
15		#define zig_xlc
16	14	#elif defined(__TINYC__)
17	15	#define zig_tinyc
18	16	#elif defined(__slimcc__)
...	...	@@ -28,8 +26,18 @@
28	26	#define zig_arm
29	27	#elif defined(__arm__)
30	28	#define zig_arm
	29	#elif defined(__arc__)
	30	#define zig_arc
	31	#elif defined(__csky__)
	32	#define zig_csky
31	33	#elif defined(__hexagon__)
32	34	#define zig_hexagon
	35	#elif defined(__hppa__) && defined(_LP64)
	36	#define zig_hppa64
	37	#define zig_hppa
	38	#elif defined(__hppa__)
	39	#define zig_hppa32
	40	#define zig_hppa
33	41	#elif defined(__kvx__)
34	42	#define zig_kvx
35	43	#elif defined(__loongarch32)
...	...	@@ -42,6 +50,8 @@
42	50	#define zig_m68k
43	51	#elif defined(__m88k__)
44	52	#define zig_m88k
	53	#elif defined(__microblaze__)
	54	#define zig_microblaze
45	55	#elif defined(__mips64)
46	56	#define zig_mips64
47	57	#define zig_mips
...	...	@@ -64,6 +74,8 @@
64	74	#define zig_riscv
65	75	#elif defined(__s390x__)
66	76	#define zig_s390x
	77	#elif defined(__sh__)
	78	#define zig_sh
67	79	#elif defined(__sparc__) && defined(__arch64__)
68	80	#define zig_sparc64
69	81	#define zig_sparc
...	...	@@ -100,32 +112,30 @@
100	112	#define zig_big_endian 1
101	113	#endif
102	114
103		#if defined(__MACH__)
	115	#if defined(__APPLE__)
104	116	#define zig_darwin
105	117	#elif defined(__DragonFly__)
106	118	#define zig_dragonfly
107		#define zig_bsd
108	119	#elif defined(__EMSCRIPTEN__)
109	120	#define zig_emscripten
110	121	#elif defined(__FreeBSD__)
111	122	#define zig_freebsd
112		#define zig_bsd
113	123	#elif defined(__Fuchsia__)
114	124	#define zig_fuchsia
115	125	#elif defined(__HAIKU__)
116	126	#define zig_haiku
117	127	#elif defined(__gnu_hurd__)
118	128	#define zig_hurd
	129	#elif defined(__illumos__)
	130	#define zig_illumos
119	131	#elif defined(__linux__)
120	132	#define zig_linux
121	133	#elif defined(__NetBSD__)
122	134	#define zig_netbsd
123		#define zig_bsd
124	135	#elif defined(__OpenBSD__)
125	136	#define zig_openbsd
126		#define zig_bsd
127		#elif defined(__SVR4)
128		#define zig_solaris
	137	#elif defined(__serenity__)
	138	#define zig_serenity
129	139	#elif defined(__wasi__)
130	140	#define zig_wasi
131	141	#elif defined(_WIN32)
...	...	@@ -404,14 +414,22 @@
404	414	#define zig_trap() __asm__ volatile("udf #0xfe")
405	415	#elif defined(zig_arm) \|\| defined(zig_aarch64)
406	416	#define zig_trap() __asm__ volatile("udf #0xfdee")
	417	#elif defined(zig_arc)
	418	#define zig_trap() __asm__ volatile("unimp_s")
	419	#elif defined(zig_csky)
	420	#define zig_trap() __asm__ volatile(".word 0x3fff")
407	421	#elif defined(zig_hexagon)
408	422	#define zig_trap() __asm__ volatile("r27:26 = memd(#0xbadc0fee)")
	423	#elif defined(zig_hppa)
	424	#define zig_trap() __asm__ volatile("iitlbp %r0, (%sr0, %r0)")
409	425	#elif defined(zig_kvx) \|\| defined(zig_loongarch) \|\| defined(zig_powerpc)
410	426	#define zig_trap() __asm__ volatile(".word 0x0")
411	427	#elif defined(zig_m68k)
412	428	#define zig_trap() __asm__ volatile("illegal")
413	429	#elif defined(zig_m88k)
414	430	#define zig_trap() __asm__ volatile("tb0 0, %%r0, 511")
	431	#elif defined(zig_microblaze)
	432	#define zig_trap() __asm__ volatile("getd r0, r0")
415	433	#elif defined(zig_mips)
416	434	#define zig_trap() __asm__ volatile(".word 0x3d")
417	435	#elif defined(zig_or1k)
...	...	@@ -420,6 +438,8 @@
420	438	#define zig_trap() __asm__ volatile("unimp")
421	439	#elif defined(zig_s390x)
422	440	#define zig_trap() __asm__ volatile("j 0x2")
	441	#elif defined(zig_sh)
	442	#define zig_trap() __asm__ volatile(".word 0x0001")
423	443	#elif defined(zig_sparc)
424	444	#define zig_trap() __asm__ volatile("illtrap")
425	445	#elif defined(zig_x86_16)
...	...	@@ -446,16 +466,22 @@
446	466
447	467	#if defined(zig_alpha)
448	468	#define zig_breakpoint() __asm__ volatile("call_pal 0x000080")
449		#elif defined(zig_arm)
	469	#elif defined(zig_arm) \|\| defined(zig_csky)
450	470	#define zig_breakpoint() __asm__ volatile("bkpt #0x0")
451	471	#elif defined(zig_aarch64)
452	472	#define zig_breakpoint() __asm__ volatile("brk #0xf000")
	473	#elif defined(zig_arc)
	474	#define zig_breakpoint() __asm__ volatile("brk_s")
453	475	#elif defined(zig_hexagon)
454	476	#define zig_breakpoint() __asm__ volatile("brkpt")
	477	#elif defined(zig_hppa)
	478	#define zig_breakpoint() __asm__ volatile("break 0x04, 0x0008")
455	479	#elif defined(zig_kvx) \|\| defined(zig_loongarch)
456	480	#define zig_breakpoint() __asm__ volatile("break 0x0")
457	481	#elif defined(zig_m88k)
458	482	#define zig_breakpoint() __asm__ volatile("illop1")
	483	#elif defined(zig_microblaze)
	484	#define zig_breakpoint() __asm__ volatile("brki r16, 0x0018")
459	485	#elif defined(zig_mips)
460	486	#define zig_breakpoint() __asm__ volatile("break")
461	487	#elif defined(zig_or1k)
...	...	@@ -466,6 +492,8 @@
466	492	#define zig_breakpoint() __asm__ volatile("ebreak")
467	493	#elif defined(zig_s390x)
468	494	#define zig_breakpoint() __asm__ volatile("j 0x6")
	495	#elif defined(zig_sh)
	496	#define zig_breakpoint() __asm__ volatile("trapa #0xc3")
469	497	#elif defined(zig_sparc)
470	498	#define zig_breakpoint() __asm__ volatile("ta 0x1")
471	499	#elif defined(zig_x86)
...	...	@@ -4529,9 +4557,12 @@ static inline void zig_msvc_atomic_store_i128(zig_i128 volatile* obj, zig_i128 a
4529	4557	#include <intrin.h>
4530	4558	#endif
4531	4559
	4560	static inline void* zig_e_zig_windows_teb(void) zig_mangled(zig_e_zig_windows_teb, "zig_windows_teb");
	4561	static inline void* zig_e_zig_windows_peb(void) zig_mangled(zig_e_zig_windows_peb, "zig_windows_peb");
	4562
4532	4563	#if defined(zig_thumb)
4533	4564
4534		static inline void* zig_windows_teb(void) {
	4565	static inline void* zig_e_zig_windows_teb(void) {
4535	4566	void* teb = 0;
4536	4567	#if defined(zig_msvc)
4537	4568	teb = (void*)_MoveFromCoprocessor(15, 0, 13, 0, 2);
...	...	@@ -4543,7 +4574,7 @@ static inline void* zig_windows_teb(void) {
4543	4574
4544	4575	#elif defined(zig_aarch64)
4545	4576
4546		static inline void* zig_windows_teb(void) {
	4577	static inline void* zig_e_zig_windows_teb(void) {
4547	4578	void* teb = 0;
4548	4579	#if defined(zig_msvc)
4549	4580	teb = (void*)__readx18qword(0x0);
...	...	@@ -4555,7 +4586,7 @@ static inline void* zig_windows_teb(void) {
4555	4586
4556	4587	#elif defined(zig_x86_32)
4557	4588
4558		static inline void* zig_windows_teb(void) {
	4589	static inline void* zig_e_zig_windows_teb(void) {
4559	4590	void* teb = 0;
4560	4591	#if defined(zig_msvc)
4561	4592	teb = (void*)__readfsdword(0x18);
...	...	@@ -4565,7 +4596,7 @@ static inline void* zig_windows_teb(void) {
4565	4596	return teb;
4566	4597	}
4567	4598
4568		static inline void* zig_windows_peb(void) {
	4599	static inline void* zig_e_zig_windows_peb(void) {
4569	4600	void* peb = 0;
4570	4601	#if defined(zig_msvc)
4571	4602	peb = (void*)__readfsdword(0x30);
...	...	@@ -4577,7 +4608,7 @@ static inline void* zig_windows_peb(void) {
4577	4608
4578	4609	#elif defined(zig_x86_64)
4579	4610
4580		static inline void* zig_windows_teb(void) {
	4611	static inline void* zig_e_zig_windows_teb(void) {
4581	4612	void* teb = 0;
4582	4613	#if defined(zig_msvc)
4583	4614	teb = (void*)__readgsqword(0x30);
...	...	@@ -4587,7 +4618,7 @@ static inline void* zig_windows_teb(void) {
4587	4618	return teb;
4588	4619	}
4589	4620
4590		static inline void* zig_windows_peb(void) {
	4621	static inline void* zig_e_zig_windows_peb(void) {
4591	4622	void* peb = 0;
4592	4623	#if defined(zig_msvc)
4593	4624	peb = (void*)__readgsqword(0x60);
...	...	@@ -4601,7 +4632,9 @@ static inline void* zig_windows_peb(void) {
4601	4632
4602	4633	#if defined(zig_loongarch)
4603	4634
4604		static inline void zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {
	4635	static inline void zig_e_zig_loongarch_cpucfg(uint32_t word, uint32_t* result) zig_mangled(zig_e_zig_loongarch_cpucfg, "zig_loongarch_cpucfg");
	4636
	4637	static inline void zig_e_zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {
4605	4638	#if defined(zig_gnuc_asm)
4606	4639	__asm__("cpucfg %[result], %[word]" : [result] "=r" (result) : [word] "r" (word));
4607	4640	#else
...	...	@@ -4611,7 +4644,10 @@ static inline void zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {
4611	4644
4612	4645	#elif defined(zig_x86) && !defined(zig_x86_16)
4613	4646
4614		static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) {
	4647	static inline void zig_e_zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) zig_mangled(zig_e_zig_x86_cpuid, "zig_x86_cpuid");
	4648	static inline uint32_t zig_e_zig_x86_get_xcr0(void) zig_mangled(zig_e_zig_x86_get_xcr0, "zig_x86_get_xcr0");
	4649
	4650	static inline void zig_e_zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) {
4615	4651	#if defined(zig_msvc)
4616	4652	int cpu_info[4];
4617	4653	__cpuidex(cpu_info, leaf_id, subid);
...	...	@@ -4629,7 +4665,7 @@ static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax
4629	4665	#endif
4630	4666	}
4631	4667
4632		static inline uint32_t zig_x86_get_xcr0(void) {
	4668	static inline uint32_t zig_e_zig_x86_get_xcr0(void) {
4633	4669	#if defined(zig_msvc)
4634	4670	return (uint32_t)_xgetbv(0);
4635	4671	#elif defined(zig_gnuc_asm)

stage1/zig1.wasm

Binary files a/stage1/zig1.wasm and b/stage1/zig1.wasm differ

test/behavior/asm.zig-36

...	...	@@ -209,42 +209,6 @@ test "packed output types (x86_64)" {
209	209	}
210	210	}
211	211
212		test "extern output types (x86_64)" {
213		if (builtin.target.cpu.arch != .x86_64) return error.SkipZigTest;
214		if (builtin.zig_backend == .stage2_c and builtin.os.tag == .windows) return error.SkipZigTest; // MSVC doesn't support inline assembly
215		if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://codeberg.org/ziglang/zig/issues/31531
216
217		const S = extern struct { x: u32 };
218		{
219		const s: S = asm volatile ("mov $123, %[ret]"
220		: [ret] "=r" (-> S),
221		);
222		try expect(s.x == 123);
223		}
224		{
225		var s: S = undefined;
226		asm volatile ("mov $123, %[ret]"
227		: [ret] "=r" (s),
228		);
229		try expect(s.x == 123);
230		}
231
232		const U = extern union { x: u32 };
233		{
234		const u: U = asm volatile ("mov $123, %[ret]"
235		: [ret] "=r" (-> U),
236		);
237		try expect(u.x == 123);
238		}
239		{
240		var u: U = undefined;
241		asm volatile ("mov $123, %[ret]"
242		: [ret] "=r" (u),
243		);
244		try expect(u.x == 123);
245		}
246		}
247
248	212	test "abi register aliases as clobbers (RISC-V)" {
249	213	if (!builtin.target.cpu.arch.isRISCV()) return error.SkipZigTest;
250	214	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;

test/behavior/atomics.zig-1

...	...	@@ -358,7 +358,6 @@ test "atomics with different types" {
358	358	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
359	359	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
360	360	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
361		if (builtin.target.cpu.arch.endian() == .big) return error.SkipZigTest; // #24282
362	361
363	362	try testAtomicsWithType(bool, true, false);
364	363

test/behavior/bitcast.zig+141-167

...	...	@@ -74,55 +74,6 @@ fn conv_uN(comptime N: usize, x: @Int(.unsigned, N)) @Int(.signed, N) {
74	74	return @as(@Int(.signed, N), @bitCast(x));
75	75	}
76	76
77		test "bitcast uX to bytes" {
78		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
79		if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
80		if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
81		if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
82		if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
83
84		const bit_values = [_]usize{ 1, 48, 27, 512, 493, 293, 125, 204, 112 };
85		inline for (bit_values) \|bits\| {
86		try testBitCast(bits);
87		try comptime testBitCast(bits);
88		}
89		}
90
91		fn testBitCastuXToBytes(comptime N: usize) !void {
92
93		// The location of padding bits in these layouts are technically not defined
94		// by LLVM, but we currently allow exotic integers to be cast (at comptime)
95		// to types that expose their padding bits anyway.
96		//
97		// This test at least makes sure those bits are matched by the runtime behavior
98		// on the platforms we target. If the above behavior is restricted after all,
99		// this test should be deleted.
100
101		const T = @Int(.unsigned, N);
102		for ([_]T{ 0, ~@as(T, 0) }) \|init_value\| {
103		var x: T = init_value;
104		const bytes = std.mem.asBytes(&x);
105
106		const byte_count = (N + 7) / 8;
107		switch (native_endian) {
108		.little => {
109		var byte_i = 0;
110		while (byte_i < (byte_count - 1)) : (byte_i += 1) {
111		try expect(bytes[byte_i] == 0xff);
112		}
113		try expect(((bytes[byte_i] ^ 0xff) << -%@as(u3, @truncate(N))) == 0);
114		},
115		.big => {
116		var byte_i = byte_count - 1;
117		while (byte_i > 0) : (byte_i -= 1) {
118		try expect(bytes[byte_i] == 0xff);
119		}
120		try expect(((bytes[byte_i] ^ 0xff) << -%@as(u3, @truncate(N))) == 0);
121		},
122		}
123		}
124		}
125
126	77	test "nested bitcast" {
127	78	const S = struct {
128	79	fn moo(x: isize) !void {
...	...	@@ -183,39 +134,6 @@ test "@bitCast packed structs at runtime and comptime" {
183	134	try comptime S.doTheTest();
184	135	}
185	136
186		test "@bitCast extern structs at runtime and comptime" {
187		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
188		if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
189		if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
190
191		const Full = extern struct {
192		number: u16,
193		};
194		const TwoHalves = extern struct {
195		half1: u8,
196		half2: u8,
197		};
198		const S = struct {
199		fn doTheTest() !void {
200		var full = Full{ .number = 0x1234 };
201		_ = &full;
202		const two_halves: TwoHalves = @bitCast(full);
203		switch (native_endian) {
204		.big => {
205		try expect(two_halves.half1 == 0x12);
206		try expect(two_halves.half2 == 0x34);
207		},
208		.little => {
209		try expect(two_halves.half1 == 0x34);
210		try expect(two_halves.half2 == 0x12);
211		},
212		}
213		}
214		};
215		try S.doTheTest();
216		try comptime S.doTheTest();
217		}
218
219	137	test "bitcast packed struct to integer and back" {
220	138	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
221	139	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
...	...	@@ -363,32 +281,12 @@ test "comptime @bitCast packed struct to int and back" {
363	281	}
364	282	}
365	283
366		test "comptime bitcast with fields following f80" {
367		if (true) {
368		// https://github.com/ziglang/zig/issues/19387
369		return error.SkipZigTest;
370		}
371
372		if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
373		if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
374		if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
375
376		const FloatT = extern struct { f: f80, x: u128 align(16) };
377		const x: FloatT = .{ .f = 0.5, .x = 123 };
378		var x_as_uint: u256 = comptime @as(u256, @bitCast(x));
379		_ = &x_as_uint;
380
381		try expect(x.f == @as(FloatT, @bitCast(x_as_uint)).f);
382		try expect(x.x == @as(FloatT, @bitCast(x_as_uint)).x);
383		}
384
385	284	test "bitcast vector to integer and back" {
386	285	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
387	286	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
388	287	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
389	288	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
390	289	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
391		if (builtin.cpu.arch.endian() == .big and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
392	290
393	291	var vec: @Vector(16, bool) = @splat(true);
394	292	vec[1] = false;
...	...	@@ -524,87 +422,163 @@ test "@bitCast of packed struct of bools all false" {
524	422	try expect(@as(u8, @as(u4, @bitCast(p))) == 0);
525	423	}
526	424
527		test "@bitCast of extern struct containing pointer" {
528		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
529		if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
530		if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
531		if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest; // TODO
532
533		const S = struct {
534		const A = extern struct {
535		ptr: *const u32,
536		};
537
538		const B = extern struct {
539		ptr: *const i32,
540		};
	425	test "@bitCast of packed struct with void field to integer" {
	426	const S = packed struct(u8) {
	427	v: void,
	428	x: u8,
541	429
542		fn doTheTest() !void {
543		const x: u32 = 123;
544		var a: A = undefined;
545		a = .{ .ptr = &x };
546		const b: B = @bitCast(a);
547		try expect(b.ptr.* == 123);
	430	fn doTheTest(x: u8) !void {
	431	// Intentionally using `@as` to avoid RLS which masks the bug
	432	const foo = @as(@This(), .{ .v = {}, .x = x });
	433	const as_int: u8 = @bitCast(foo);
	434	try expect(as_int == x);
548	435	}
549	436	};
550
551		try S.doTheTest();
552		try comptime S.doTheTest();
	437	try S.doTheTest(123);
	438	try comptime S.doTheTest(123);
553	439	}
554	440
555		test "@bitCast of extern struct to float" {
556		if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
557		if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
558
559		const S = struct {
560		const S = extern struct {
561		x: u16,
562		y: u16,
563		};
564		fn doTheTest() !void {
565		var s: S = .{ .x = 0, .y = 0 };
566		_ = &s;
567		const a: f32 = @bitCast(s);
568		try expect(a == 0);
	441	test "@bitCast vector to array with different element size" {
	442	const static = struct {
	443	fn doTheTest(v: @Vector(4, u5)) !void {
	444	const result: [5]u4 = @bitCast(v);
	445	// See the definition of `v` in the test proper for these values.
	446	try expect(result[0] == 0b0010);
	447	try expect(result[1] == 0b1110);
	448	try expect(result[2] == 0b0101);
	449	try expect(result[3] == 0b0110);
	450	try expect(result[4] == 0b0000);
	451	}
	452	};
	453	// The strange digit groupings here are to indicate how this maps to `expected` above.
	454	const v: @Vector(4, u5) = .{
	455	0b0_0010,
	456	0b01_111,
	457	0b110_01,
	458	0b0000_0,
	459	};
	460	try static.doTheTest(v);
	461	try comptime static.doTheTest(v);
	462	}
	463
	464	test "@bitCast packed struct to array of bits" {
	465	const S = packed struct(u16) {
	466	foo: u5,
	467	bar: i7,
	468	baz: u3,
	469	qux: bool,
	470	fn doTheTest(val: @This(), comptime Bits: type) !void {
	471	const bits: Bits = @bitCast(val);
	472
	473	// foo
	474	try expect(bits[0] == 1);
	475	try expect(bits[1] == 0);
	476	try expect(bits[2] == 0);
	477	try expect(bits[3] == 1);
	478	try expect(bits[4] == 0);
	479	// bar
	480	try expect(bits[5] == 0);
	481	try expect(bits[6] == 1);
	482	try expect(bits[7] == 1);
	483	try expect(bits[8] == 1);
	484	try expect(bits[9] == 1);
	485	try expect(bits[10] == 1);
	486	try expect(bits[11] == 1);
	487	// baz
	488	try expect(bits[12] == 0);
	489	try expect(bits[13] == 1);
	490	try expect(bits[14] == 0);
	491	// qux
	492	try expect(bits[15] == 1);
569	493	}
570	494	};
571	495
572		try S.doTheTest();
573		try comptime S.doTheTest();
	496	const val: S = .{
	497	.foo = 0b01001,
	498	.bar = -2,
	499	.baz = 0b010,
	500	.qux = true,
	501	};
	502
	503	try val.doTheTest(@Vector(16, u1));
	504	try val.doTheTest([16]u1);
	505
	506	try comptime val.doTheTest(@Vector(16, u1));
	507	try comptime val.doTheTest([16]u1);
574	508	}
575	509
576		test "@bitCast of float to extern struct" {
577		if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
578		if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
	510	test "@bitCast nested arrays of vectors" {
	511	const Src = [2][2]@Vector(4, u5);
	512	const Dest = [5]@Vector(2, u8);
579	513
580		const S = struct {
581		const S = extern struct {
582		x: u32,
583		};
584		fn doTheTest() !void {
585		var a: f32 = -0.0;
586		_ = &a;
587		const s: S = @bitCast(a);
588		try expect(s.x == 0x80000000);
	514	// The strange digit groupings here are to indicate how this maps to the output.
	515	const src: Src = .{ .{
	516	.{ 0b00011, 0b00_100, 0b11100, 0b0010_1 },
	517	.{ 0b1_0110, 0b11011, 0b101_10, 0b10101 },
	518	}, .{
	519	.{ 0b10101, 0b00_001, 0b01011, 0b0001_0 },
	520	.{ 0b0_0001, 0b01111, 0b111_10, 0b00001 },
	521	} };
	522
	523	const expected: Dest = .{
	524	.{ 0b10000011, 0b11110000 },
	525	.{ 0b01100010, 0b10110111 },
	526	.{ 0b10101101, 0b00110101 },
	527	.{ 0b00101100, 0b00010001 },
	528	.{ 0b10011110, 0b00001111 },
	529	};
	530
	531	const static = struct {
	532	fn doTheTest(src_arg: Src) !void {
	533	const actual: Dest = @bitCast(src_arg);
	534	for (actual, expected) \|actual_vec, expected_vec\| {
	535	try expect(actual_vec[0] == expected_vec[0]);
	536	try expect(actual_vec[1] == expected_vec[1]);
	537	}
589	538	}
590	539	};
591	540
592		try S.doTheTest();
593		try comptime S.doTheTest();
	541	try static.doTheTest(src);
	542	try comptime static.doTheTest(src);
594	543	}
595	544
596		test "@bitCast of packed struct with void field to integer" {
597		const S = packed struct(u8) {
598		v: void,
599		x: u8,
600
601		fn doTheTest(x: u8) !void {
602		// Intentionally using `@as` to avoid RLS which masks the bug
603		const foo = @as(@This(), .{ .v = {}, .x = x });
604		const as_int: u8 = @bitCast(foo);
605		try expect(as_int == x);
	545	test "@bitCast nested arrays of bool to scalar" {
	546	const static = struct {
	547	fn doTheTest(src: [4][4]bool) !void {
	548	const result: u16 = @bitCast(src);
	549	try expect(result == 0b1100_0101_1010_0011);
606	550	}
607	551	};
608		try S.doTheTest(123);
609		try comptime S.doTheTest(123);
	552	const src: [4][4]bool = .{
	553	.{ true, true, false, false }, // 0b0011
	554	.{ false, true, false, true }, // 0b1010
	555	.{ true, false, true, false }, // 0b0101
	556	.{ false, false, true, true }, // 0b1100
	557	};
	558	try static.doTheTest(src);
	559	try comptime static.doTheTest(src);
	560	}
	561
	562	test "@bitCast deeply nested arrays to scalar" {
	563	const static = struct {
	564	fn doTheTest(src: [2][1][3][5]u4) !void {
	565	const signed: i120 = @bitCast(src);
	566	try expect(signed < 0); // top nibble is 0x8 so sign bit is 1
	567	const unsigned: u120 = @bitCast(src);
	568	try expect(unsigned == 0x8873B_5BF6F_F4020_0E7AC_1EFED_40F51);
	569	try expect(@as(i120, @bitCast(unsigned)) == signed);
	570	try expect(@as(u120, @bitCast(signed)) == unsigned);
	571	}
	572	};
	573	const src: [2][1][3][5]u4 = .{ .{.{
	574	.{ 0x1, 0x5, 0xF, 0x0, 0x4 },
	575	.{ 0xD, 0xE, 0xF, 0xE, 0x1 },
	576	.{ 0xC, 0xA, 0x7, 0xE, 0x0 },
	577	}}, .{.{
	578	.{ 0x0, 0x2, 0x0, 0x4, 0xF },
	579	.{ 0xF, 0x6, 0xF, 0xB, 0x5 },
	580	.{ 0xB, 0x3, 0x7, 0x8, 0x8 },
	581	}} };
	582	try static.doTheTest(src);
	583	try comptime static.doTheTest(src);
610	584	}

test/behavior/comptime_memory.zig+16

...	...	@@ -583,3 +583,19 @@ test "comptime store to extern struct reinterpreted as byte array" {
583	583
584	584	comptime std.debug.assert(val.x == 0);
585	585	}
	586
	587	test "reinterpret sentinel-terminated array as packed struct" {
	588	const S = packed struct(u16) { lo: u8, hi: u8 };
	589	const data: [2:0]u8 = .{ 0x12, 0x34 };
	590	const ptr: *align(1) const S = @ptrCast(&data);
	591	switch (endian) {
	592	.little => {
	593	try testing.expect(ptr.lo == 0x12);
	594	try testing.expect(ptr.hi == 0x34);
	595	},
	596	.big => {
	597	try testing.expect(ptr.lo == 0x34);
	598	try testing.expect(ptr.hi == 0x12);
	599	},
	600	}
	601	}

test/behavior/sizeof_and_typeof.zig-11

...	...	@@ -151,9 +151,6 @@ test "branching logic inside @TypeOf" {
151	151	test "@bitSizeOf" {
152	152	try expect(@bitSizeOf(u2) == 2);
153	153	try expect(@bitSizeOf(u8) == @sizeOf(u8) * 8);
154		try expect(@bitSizeOf(struct {
155		a: u2,
156		}) == 8);
157	154	try expect(@bitSizeOf(packed struct {
158	155	a: u2,
159	156	}) == 2);
...	...	@@ -281,14 +278,6 @@ test "@offsetOf zero-bit field" {
281	278	try expect(@offsetOf(S, "b") == @offsetOf(S, "c"));
282	279	}
283	280
284		test "@bitSizeOf on array of structs" {
285		const S = struct {
286		foo: u64,
287		};
288
289		try expectEqual(128, @bitSizeOf([2]S));
290		}
291
292	281	test "lazy abi size used in comparison" {
293	282	const S = struct { a: usize };
294	283	var rhs: i32 = 100;

test/behavior/vector.zig+3-2

...	...	@@ -1536,12 +1536,12 @@ test "vector pointer is indexable" {
1536	1536	const V = @Vector(2, u32);
1537	1537
1538	1538	const x: V = .{ 123, 456 };
1539		comptime assert(@TypeOf(&(&x)[0]) == *const u32); // validate constness
	1539	comptime assert(@typeInfo(@TypeOf(&(&x)[0])).pointer.attrs.@"const");
1540	1540	try expectEqual(@as(u32, 123), (&x)[0]);
1541	1541	try expectEqual(@as(u32, 456), (&x)[1]);
1542	1542
1543	1543	var y: V = .{ 123, 456 };
1544		comptime assert(@TypeOf(&(&y)[0]) == *u32); // validate constness
	1544	comptime assert(!@typeInfo(@TypeOf(&(&y)[0])).pointer.attrs.@"const");
1545	1545	try expectEqual(@as(u32, 123), (&y)[0]);
1546	1546	try expectEqual(@as(u32, 456), (&y)[1]);
1547	1547
...	...	@@ -1620,6 +1620,7 @@ test "bitcast vector to array of smaller vectors" {
1620	1620	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1621	1621	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1622	1622	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
	1623	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
1623	1624
1624	1625	const u8x32 = @Vector(32, u8);
1625	1626	const u8x64 = @Vector(64, u8);

test/c_abi/main.zig+4-1

...	...	@@ -14501,6 +14501,7 @@ test "@Vector(4, f64)" {
14501	14501	if (builtin.cpu.arch.isMIPS32()) return error.SkipZigTest;
14502	14502	if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
14503	14503	if (builtin.cpu.arch == .s390x) return error.SkipZigTest;
	14504	if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // https://codeberg.org/ziglang/zig/issues/35899
14504	14505
14505	14506	const v = c_ret_vector_4_f64();
14506	14507	try expect(v[0] == 33);
...	...	@@ -14570,6 +14571,7 @@ test "@Vector(8, f64)" {
14570	14571	if (builtin.cpu.arch.isMIPS32()) return error.SkipZigTest;
14571	14572	if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
14572	14573	if (builtin.cpu.arch == .s390x) return error.SkipZigTest;
	14574	if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // https://codeberg.org/ziglang/zig/issues/35899
14573	14575
14574	14576	const v = c_ret_vector_8_f64();
14575	14577	try expect(v[0] == 81);
...	...	@@ -17109,7 +17111,8 @@ const byval_tail_callsite_attr = struct {
17109	17111	}
17110	17112
17111	17113	fn cast(self: MyRect) struct_Rect {
17112		return @bitCast(self);
	17114	const ptr: *const struct_Rect = @ptrCast(&self);
	17115	return ptr.*;
17113	17116	}
17114	17117
17115	17118	extern fn c_byval_tail_callsite_attr(struct_Rect) f64;

test/cases/compile_errors/asm_output_type_no_guaranteed_in_memory_layout.zig+28-4

...	...	@@ -25,14 +25,38 @@ export fn entry4() void {
25	25	: [_] "=r" (u),
26	26	);
27	27	}
	28	const ES = extern struct { x: u32 };
	29	export fn entry5() void {
	30	var es: ES = undefined;
	31	asm volatile (""
	32	: [_] "=r" (es),
	33	);
	34	}
	35	const EU = extern union { x: u32 };
	36	export fn entry6() void {
	37	var eu: EU = undefined;
	38	asm volatile (""
	39	: [_] "=r" (eu),
	40	);
	41	}
28	42
29	43	// error
30	44	//
31		// :4:24: error: invalid inline assembly output type; 'tmp.S' does not have a guaranteed in-memory layout
	45	// :4:24: error: invalid inline assembly output type 'tmp.S'
	46	// :4:24: note: struct types cannot be passed to inline assembly
32	47	// :1:11: note: struct declared here
33		// :11:21: error: invalid inline assembly output type; 'tmp.S' does not have a guaranteed in-memory layout
	48	// :11:21: error: invalid inline assembly output type 'tmp.S'
	49	// :11:21: note: struct types cannot be passed to inline assembly
34	50	// :1:11: note: struct declared here
35		// :18:24: error: invalid inline assembly output type; 'tmp.U' does not have a guaranteed in-memory layout
	51	// :18:24: error: invalid inline assembly output type 'tmp.U'
	52	// :18:24: note: union types cannot be passed to inline assembly
36	53	// :15:11: note: union declared here
37		// :25:21: error: invalid inline assembly output type; 'tmp.U' does not have a guaranteed in-memory layout
	54	// :25:21: error: invalid inline assembly output type 'tmp.U'
	55	// :25:21: note: union types cannot be passed to inline assembly
38	56	// :15:11: note: union declared here
	57	// :32:21: error: invalid inline assembly output type 'tmp.ES'
	58	// :32:21: note: struct types cannot be passed to inline assembly
	59	// :28:19: note: struct declared here
	60	// :39:21: error: invalid inline assembly output type 'tmp.EU'
	61	// :39:21: note: union types cannot be passed to inline assembly
	62	// :35:19: note: union declared here

test/cases/compile_errors/bitCast_extern_struct.zig created+10

...	...	@@ -0,0 +1,10 @@
	1	const S = extern struct { x: u32 };
	2	export fn foo(s: S) void {
	3	const as_int: u32 = @bitCast(s);
	4	_ = as_int;
	5	}
	6
	7	// error
	8	//
	9	// :3:34: error: cannot @bitCast from 'tmp.S'
	10	// :1:18: note: struct declared here

test/cases/compile_errors/bitCast_to_enum_type.zig deleted-10

...	...	@@ -1,10 +0,0 @@
1		export fn entry() void {
2		const E = enum(u32) { a, b };
3		const y: E = @bitCast(@as(u32, 3));
4		_ = y;
5		}
6
7		// error
8		//
9		// :3:18: error: cannot @bitCast to 'tmp.entry.E'
10		// :3:18: note: use @enumFromInt to cast from 'u32'

test/cases/compile_errors/bitCast_vector_of_pointer.zig created+9

...	...	@@ -0,0 +1,9 @@
	1	export fn foo(p: *u32) void {
	2	const vec: @Vector(2, *u32) = .{ p, p };
	3	const raw: [2]usize = @bitCast(vec);
	4	_ = raw;
	5	}
	6
	7	// error
	8	//
	9	// :3:36: error: cannot @bitCast from '@Vector(2, *u32)'

test/cases/compile_errors/bitCast_with_invalid_array_element_type.zig+2-5

...	...	@@ -18,9 +18,6 @@ export fn baz() void {
18	18
19	19	// error
20	20	//
21		// :5:29: error: cannot @bitCast from '[1]tmp.foo.S'
22		// :5:29: note: array element type 'tmp.foo.S' does not have a guaranteed in-memory layout
	21	// :5:42: error: cannot @bitCast from '[1]tmp.foo.S'
23	22	// :12:19: error: cannot @bitCast to '[1]tmp.bar.S'
24		// :12:19: note: array element type 'tmp.bar.S' does not have a guaranteed in-memory layout
25		// :16:21: error: cannot @bitCast from '[1]comptime_int'
26		// :16:21: note: array element type 'comptime_int' does not have a guaranteed in-memory layout
	23	// :16:45: error: cannot @bitCast from '[1]comptime_int'

test/cases/compile_errors/error_in_nested_declaration.zig created+29

...	...	@@ -0,0 +1,29 @@
	1	const S = struct {
	2	b: u32,
	3	c: i32,
	4	a: struct {
	5	pub fn str(_: @This(), extra: []u32) []i32 {
	6	return @bitCast(extra);
	7	}
	8	},
	9	};
	10
	11	pub export fn entry() void {
	12	var s: S = undefined;
	13	_ = s.a.str(undefined);
	14	}
	15
	16	const S2 = struct {
	17	a: [*c]anyopaque,
	18	};
	19
	20	pub export fn entry2() void {
	21	var s: S2 = undefined;
	22	_ = &s;
	23	}
	24
	25	// error
	26	//
	27	// :6:20: error: cannot @bitCast to '[]i32'
	28	// :6:20: note: use @ptrCast to cast from '[]u32'
	29	// :17:12: error: indexable pointer to opaque type 'anyopaque' not allowed

test/cases/error_in_nested_declaration.zig deleted-30

...	...	@@ -1,30 +0,0 @@
1		const S = struct {
2		b: u32,
3		c: i32,
4		a: struct {
5		pub fn str(_: @This(), extra: []u32) []i32 {
6		return @bitCast(extra);
7		}
8		},
9		};
10
11		pub export fn entry() void {
12		var s: S = undefined;
13		_ = s.a.str(undefined);
14		}
15
16		const S2 = struct {
17		a: [*c]anyopaque,
18		};
19
20		pub export fn entry2() void {
21		var s: S2 = undefined;
22		_ = &s;
23		}
24
25		// error
26		// backend=selfhosted,llvm
27		//
28		// :6:20: error: cannot @bitCast to '[]i32'
29		// :6:20: note: use @ptrCast to cast from '[]u32'
30		// :17:12: error: indexable pointer to opaque type 'anyopaque' not allowed