Commit d12892e8904bbb27aac7baa69f483348e516a30e | zig

author	Pavel Verigo <paul.verigo@gmail.com> 2026-08-21 12:43:10+02:00
committer	Pavel Verigo <paul.verigo@gmail.com> 2026-09-17 00:04:51+02:00
log	d12892e8904bbb27aac7baa69f483348e516a30e
tree	d129832a30921a505603594390425a2c45fdcdb1
parent	5ec1668eb4cee3ce3069f6ff7b2e3af0d52f43a6

wasm: store/load use memarg, support alignment

4 files changed, 288 insertions(+), 246 deletions(-)

src
- codegen
  - wasm
    CodeGen.zig
    Emit.zig
    Mir.zig
    assembly.zig

src/codegen/wasm/CodeGen.zig+274-237

...	...	@@ -584,7 +584,7 @@ fn addCallIntrinsic(cg: *CodeGen, intrinsic: Mir.Intrinsic) error{OutOfMemory}!v
584	584	/// Returns the index into `mir_extra`
585	585	fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
586	586	const field_count = @typeInfo(@TypeOf(extra)).@"struct".field_names.len;
587		try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count);
	587	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count * 2);
588	588	return cg.addExtraAssumeCapacity(extra);
589	589	}
590	590
...	...	@@ -594,14 +594,22 @@ fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
594	594	const info = @typeInfo(@TypeOf(extra)).@"struct";
595	595	const result: u32 = @intCast(cg.mir_extra.items.len);
596	596	inline for (info.field_names, info.field_types) \|field_name, field_type\| {
597		cg.mir_extra.appendAssumeCapacity(switch (field_type) {
598		u32 => @field(extra, field_name),
599		i32 => @bitCast(@field(extra, field_name)),
600		InternPool.Index,
601		InternPool.Nav.Index,
602		=> @backingInt(@field(extra, field_name)),
603		else => @compileError("Unsupported field type " ++ @typeName(field_type)),
604		});
	597	switch (field_type) {
	598	u64 => {
	599	const value = @field(extra, field_name);
	600	cg.mir_extra.appendAssumeCapacity(@truncate(value));
	601	cg.mir_extra.appendAssumeCapacity(@truncate(value >> 32));
	602	},
	603	else => cg.mir_extra.appendAssumeCapacity(switch (field_type) {
	604	u32 => @field(extra, field_name),
	605	i32 => @bitCast(@field(extra, field_name)),
	606	InternPool.Index,
	607	InternPool.Nav.Index,
	608	mem.Alignment,
	609	=> @backingInt(@field(extra, field_name)),
	610	else => @compileError("Unsupported field type " ++ @typeName(field_type)),
	611	}),
	612	}
605	613	}
606	614	return result;
607	615	}
...	...	@@ -991,26 +999,26 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu
991	999	if (!isByRef(ty, zcu, cg.target)) {
992	1000	return cg.lowerToStack(value);
993	1001	} else {
994		_ = try cg.load(value, scalar_ty, 0);
	1002	_ = try cg.load(value, scalar_ty, .{});
995	1003	}
996	1004	},
997	1005	.double_i64 => {
998	1006	assert(ty.abiSize(zcu) == 16);
999	1007	// in this case we have an integer or float that must be lowered as 2 i64's.
1000	1008	try cg.emitWValue(value);
1001		try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = 8 });
	1009	try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = .@"8" });
1002	1010	try cg.emitWValue(value);
1003		try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = 8 });
	1011	try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = .@"8" });
1004	1012	},
1005	1013	.indirect => {
1006	1014	const stack_copy = try cg.allocStack(ty);
1007		try cg.store(stack_copy, value, ty, 0);
	1015	try cg.store(stack_copy, value, ty, .{});
1008	1016	return cg.lowerToStack(stack_copy);
1009	1017	},
1010	1018	.unrolled => \|vector\| {
1011		const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));
	1019	const elem_size = vector.elem_type.abiSize(zcu);
1012	1020	for (0..vector.len) \|index\| {
1013		_ = try cg.load(value, vector.elem_type, @intCast(index * elem_size));
	1021	_ = try cg.load(value, vector.elem_type, .{ .offset = index * elem_size });
1014	1022	}
1015	1023	},
1016	1024	}
...	...	@@ -1330,7 +1338,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1330	1338	try cg.addTag(.i32_add);
1331	1339	const ptr = try WValue.toLocal(.stack, cg, Type.usize);
1332	1340
1333		try cg.store(ptr, elem_val, elem_ty, 0);
	1341	try cg.store(ptr, elem_val, elem_ty, .{});
1334	1342
1335	1343	return cg.finishAir(inst, .none, &.{ pl_op.operand, bin_op.lhs, bin_op.rhs });
1336	1344	},
...	...	@@ -1507,8 +1515,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1507	1515	const result = try cg.allocStack(cg.typeOfIndex(inst));
1508	1516	const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu));
1509	1517
1510		try cg.store(result, res_tmp, ty, 0);
1511		try cg.store(result, ov_tmp, Type.u1, offset);
	1518	try cg.store(result, res_tmp, ty, .{});
	1519	try cg.store(result, ov_tmp, .u1, .{ .offset = offset });
1512	1520
1513	1521	try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });
1514	1522	},
...	...	@@ -1939,20 +1947,20 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1939	1947	// result must be stored in the stack and we return a pointer
1940	1948	// to the stack instead
1941	1949	if (cg.return_value != .none) {
1942		try cg.store(cg.return_value, operand, ret_ty, 0);
	1950	try cg.store(cg.return_value, operand, ret_ty, .{});
1943	1951	} else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBits(zcu)) {
1944	1952	switch (abi.classifyType(ret_ty, zcu, cg.target)) {
1945	1953	.direct => \|scalar_type\| {
1946	1954	if (!isByRef(ret_ty, zcu, cg.target)) {
1947	1955	try cg.emitWValue(operand);
1948	1956	} else {
1949		_ = try cg.load(operand, scalar_type, 0);
	1957	_ = try cg.load(operand, scalar_type, .{});
1950	1958	}
1951	1959	},
1952	1960	.double_i64, .indirect => unreachable,
1953	1961	.unrolled => \|vector\| {
1954	1962	assert(vector.len == 1);
1955		_ = try cg.load(operand, vector.elem_type, 0);
	1963	_ = try cg.load(operand, vector.elem_type, .{});
1956	1964	},
1957	1965	}
1958	1966	} else {
...	...	@@ -2002,15 +2010,15 @@ fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2002	2010	} else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {
2003	2011	if (fn_info.cc == .wasm_mvp) {
2004	2012	switch (abi.classifyType(ret_ty, zcu, cg.target)) {
2005		.direct => \|scalar_type\| _ = try cg.load(operand, scalar_type, 0),
	2013	.direct => \|scalar_type\| _ = try cg.load(operand, scalar_type, .{}),
2006	2014	.double_i64, .indirect => unreachable,
2007	2015	.unrolled => \|vector\| {
2008	2016	assert(vector.len == 1);
2009		_ = try cg.load(operand, vector.elem_type, 0);
	2017	_ = try cg.load(operand, vector.elem_type, .{});
2010	2018	},
2011	2019	}
2012	2020	} else {
2013		_ = try cg.load(operand, ret_ty, 0);
	2021	_ = try cg.load(operand, ret_ty, .{});
2014	2022	}
2015	2023	}
2016	2024
...	...	@@ -2092,7 +2100,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2092	2100	const arg_align = arg_ty.abiAlignment(zcu);
2093	2101
2094	2102	offset = @intCast(arg_align.forward(offset));
2095		try cg.store(buffer, arg_val, arg_ty, offset);
	2103	try cg.store(buffer, arg_val, arg_ty, .{ .offset = offset });
2096	2104	offset += arg_size;
2097	2105	}
2098	2106
...	...	@@ -2149,7 +2157,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2149	2157	const result_local = try cg.allocLocal(scalar_type);
2150	2158	try cg.addLocal(.local_set, result_local.local.value);
2151	2159	const result = try cg.allocStack(ret_ty);
2152		try cg.store(result, result_local, scalar_type, 0);
	2160	try cg.store(result, result_local, scalar_type, .{});
2153	2161	break :result_value result;
2154	2162	}
2155	2163	},
...	...	@@ -2160,7 +2168,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2160	2168	// save call result from operand stack
2161	2169	try cg.addLocal(.local_set, result_local.local.value);
2162	2170	const result = try cg.allocStack(ret_ty);
2163		try cg.store(result, result_local, vector.elem_type, 0);
	2171	try cg.store(result, result_local, vector.elem_type, .{});
2164	2172	break :result_value result;
2165	2173	},
2166	2174	}
...	...	@@ -2191,7 +2199,7 @@ fn airVaCopy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2191	2199	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2192	2200	const operand = try cg.resolveInst(ty_op.operand);
2193	2201
2194		const result = try cg.load(operand, .usize, 0);
	2202	const result = try cg.load(operand, .usize, .{});
2195	2203
2196	2204	return cg.finishAir(inst, result, &.{ty_op.operand});
2197	2205	}
...	...	@@ -2214,7 +2222,7 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2214	2222	const abi_align: u32 = @intCast(load_ty.abiAlignment(zcu).toByteUnits().?);
2215	2223
2216	2224	const arg_ptr = try cg.allocLocal(.usize);
2217		_ = try cg.load(operand, .usize, 0);
	2225	_ = try cg.load(operand, .usize, .{});
2218	2226
2219	2227	if (abi_align > 1) {
2220	2228	switch (cg.ptr_size) {
...	...	@@ -2247,17 +2255,17 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2247	2255	try cg.addTag(.i64_add);
2248	2256	},
2249	2257	}
2250		try cg.store(.stack, .stack, .usize, 0);
	2258	try cg.store(.stack, .stack, .usize, .{});
2251	2259
2252	2260	const result = if (is_f32_va_arg) result: {
2253		const promoted = try cg.load(arg_ptr, Type.f64, 0);
	2261	const promoted = try cg.load(arg_ptr, .f64, .{});
2254	2262	try cg.emitWValue(promoted);
2255	2263	try cg.addTag(.f32_demote_f64);
2256	2264
2257	2265	const result_local = try cg.allocLocal(Type.f32);
2258	2266	try cg.addLocal(.local_set, result_local.local.value);
2259	2267	break :result result_local;
2260		} else try cg.load(arg_ptr, ty, 0);
	2268	} else try cg.load(arg_ptr, ty, .{});
2261	2269
2262	2270	return cg.finishAir(inst, result, &.{ty_op.operand});
2263	2271	}
...	...	@@ -2282,18 +2290,26 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2282	2290	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2283	2291	}
2284	2292
2285		const offset: u32 = switch (ptr_info.flags.vector_index) {
	2293	const offset: u64 = switch (ptr_info.flags.vector_index) {
2286	2294	.none => offset: {
2287	2295	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
2288	2296	break :offset 0;
2289	2297	},
2290		else => \|index\| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),
	2298	else => \|index\| @backingInt(index) * elem_ty.abiSize(zcu),
2291	2299	};
2292		try cg.store(lhs, rhs, elem_ty, offset);
	2300	try cg.store(lhs, rhs, elem_ty, .{
	2301	.offset = offset,
	2302	.alignment = ptr_ty.ptrAlignment(zcu),
	2303	});
2293	2304	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2294	2305	}
2295	2306
2296		fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void {
	2307	const MemArg = struct {
	2308	offset: u64 = 0,
	2309	alignment: Alignment = .none,
	2310	};
	2311
	2312	fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) InnerError!void {
2297	2313	assert(!(lhs != .stack and rhs == .stack));
2298	2314	const pt = cg.pt;
2299	2315	const zcu = pt.zcu;
...	...	@@ -2301,12 +2317,19 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2301	2317
2302	2318	if (!ty.hasRuntimeBits(zcu)) return;
2303	2319
	2320	const natural_alignment = ty.abiAlignment(zcu);
	2321	// wasm memory instruction alignment is <= access width
	2322	const alignment = if (memarg.alignment == .none)
	2323	natural_alignment
	2324	else
	2325	natural_alignment.minStrict(memarg.alignment);
	2326
2304	2327	if (isByRef(ty, zcu, cg.target)) {
2305		const offset_ptr: WValue = switch (offset + lhs.offset()) {
	2328	const offset_ptr: WValue = switch (memarg.offset + lhs.offset()) {
2306	2329	0 => lhs,
2307	2330	else => \|total_offset\| ptr: {
2308	2331	try cg.emitWValue(lhs);
2309		try cg.addImm32(total_offset);
	2332	try cg.addImm32(@intCast(total_offset));
2310	2333	try cg.addTag(.i32_add);
2311	2334	break :ptr .stack;
2312	2335	},
...	...	@@ -2320,10 +2343,11 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2320	2343	// TODO: Add helper functions for simd opcodes
2321	2344	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2322	2345	// stores as := opcode, offset, alignment (opcode::memarg)
2323		try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2324		@backingInt(std.wasm.SimdOpcode.v128_store),
2325		offset + lhs.offset(),
2326		@intCast(ty.abiAlignment(zcu).toByteUnits().?),
	2346	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
	2347	cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_store));
	2348	_ = try cg.addExtraAssumeCapacity(Mir.MemArg{
	2349	.offset = memarg.offset + lhs.offset(),
	2350	.alignment = alignment.toStdMem(),
2327	2351	});
2328	2352	return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2329	2353	}
...	...	@@ -2353,8 +2377,8 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2353	2377	try cg.addMemArg(
2354	2378	store_opcode,
2355	2379	.{
2356		.offset = offset + lhs.offset(),
2357		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	2380	.offset = memarg.offset + lhs.offset(),
	2381	.alignment = alignment.toStdMem(),
2358	2382	},
2359	2383	);
2360	2384	}
...	...	@@ -2370,34 +2394,45 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2370	2394
2371	2395	assert(elem_ty.hasRuntimeBits(zcu));
2372	2396
2373		const offset: u32 = switch (ptr_info.flags.vector_index) {
	2397	const offset: u64 = switch (ptr_info.flags.vector_index) {
2374	2398	.none => offset: {
2375	2399	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
2376	2400	break :offset 0;
2377	2401	},
2378		else => \|index\| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),
	2402	else => \|index\| @backingInt(index) * elem_ty.abiSize(zcu),
2379	2403	};
2380		const result = try cg.load(operand, elem_ty, offset);
	2404	const result = try cg.load(operand, elem_ty, .{
	2405	.offset = offset,
	2406	.alignment = ptr_ty.ptrAlignment(zcu),
	2407	});
2381	2408	return cg.finishAir(inst, result, &.{ty_op.operand});
2382	2409	}
2383	2410
2384	2411	/// Loads an operand from the linear memory section.
2385	2412	/// NOTE: Leaves the value on the stack, if isByRef == false.
2386		fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
	2413	fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WValue {
2387	2414	const zcu = cg.pt.zcu;
	2415
	2416	const natural_alignment = ty.abiAlignment(zcu);
	2417	// wasm memory instruction alignment is <= access width
	2418	const alignment = if (memarg.alignment == .none)
	2419	natural_alignment
	2420	else
	2421	natural_alignment.minStrict(memarg.alignment);
	2422
2388	2423	if (isByRef(ty, zcu, cg.target)) {
2389		const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {
	2424	const src_ptr_maybe_stack: WValue = switch (memarg.offset + operand.offset()) {
2390	2425	0 => operand,
2391	2426	else => \|total_offset\| ptr: {
2392	2427	try cg.emitWValue(operand);
2393		try cg.addImm32(total_offset);
	2428	try cg.addImm32(@intCast(total_offset));
2394	2429	try cg.addTag(.i32_add);
2395	2430	break :ptr .stack;
2396	2431	},
2397	2432	};
2398	2433	const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
2399	2434	const new_ptr = try cg.allocStack(ty);
2400		try cg.store(new_ptr, src_ptr, ty, 0);
	2435	try cg.store(new_ptr, src_ptr, ty, .{});
2401	2436	return new_ptr;
2402	2437	}
2403	2438
...	...	@@ -2408,10 +2443,11 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue
2408	2443	// TODO: Add helper functions for simd opcodes
2409	2444	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2410	2445	// stores as := opcode, offset, alignment (opcode::memarg)
2411		try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2412		@backingInt(std.wasm.SimdOpcode.v128_load),
2413		offset + operand.offset(),
2414		@intCast(ty.abiAlignment(zcu).toByteUnits().?),
	2446	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
	2447	cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_load));
	2448	_ = try cg.addExtraAssumeCapacity(Mir.MemArg{
	2449	.offset = memarg.offset + operand.offset(),
	2450	.alignment = alignment.toStdMem(),
2415	2451	});
2416	2452	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2417	2453	return .stack;
...	...	@@ -2441,8 +2477,8 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue
2441	2477	try cg.addMemArg(
2442	2478	load_opcode,
2443	2479	.{
2444		.offset = offset + operand.offset(),
2445		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	2480	.offset = memarg.offset + operand.offset(),
	2481	.alignment = alignment.toStdMem(),
2446	2482	},
2447	2483	);
2448	2484
...	...	@@ -2470,7 +2506,7 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2470	2506	cg.arg_index += 1;
2471	2507	if (isByRef(arg_ty, zcu, cg.target)) {
2472	2508	const result = try cg.allocStack(arg_ty);
2473		try cg.store(result, arg, scalar_type, 0);
	2509	try cg.store(result, arg, scalar_type, .{});
2474	2510	return cg.finishAir(inst, result, &.{});
2475	2511	}
2476	2512	},
...	...	@@ -2478,15 +2514,15 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2478	2514	.double_i64 => {
2479	2515	cg.arg_index += 2;
2480	2516	const result = try cg.allocStack(arg_ty);
2481		try cg.store(result, arg, Type.u64, 0);
2482		try cg.store(result, cg.args[arg_index + 1], Type.u64, 8);
	2517	try cg.store(result, arg, .u64, .{});
	2518	try cg.store(result, cg.args[arg_index + 1], .u64, .{ .offset = 8 });
2483	2519	return cg.finishAir(inst, result, &.{});
2484	2520	},
2485	2521	.unrolled => \|vector\| {
2486	2522	const result = try cg.allocStack(arg_ty);
2487		const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));
	2523	const elem_size = vector.elem_type.abiSize(zcu);
2488	2524	for (0..vector.len) \|index\| {
2489		try cg.store(result, cg.args[cg.arg_index], vector.elem_type, @intCast(index * elem_size));
	2525	try cg.store(result, cg.args[cg.arg_index], vector.elem_type, .{ .offset = index * elem_size });
2490	2526	cg.arg_index += 1;
2491	2527	}
2492	2528	return cg.finishAir(inst, result, &.{});
...	...	@@ -2588,15 +2624,15 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2588	2624	65...128 => {
2589	2625	const result = try cg.allocStack(Type.u128);
2590	2626
2591		var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
	2627	var lhs_lsb = try (try cg.load(lhs, .u64, .{})).toLocal(cg, Type.u64);
2592	2628	defer lhs_lsb.free(cg);
2593		var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
	2629	var rhs_lsb = try (try cg.load(rhs, .u64, .{})).toLocal(cg, Type.u64);
2594	2630	defer rhs_lsb.free(cg);
2595	2631	var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2596	2632	defer op_lsb.free(cg);
2597	2633
2598		const lhs_msb = try cg.load(lhs, Type.u64, 8);
2599		const rhs_msb = try cg.load(rhs, Type.u64, 8);
	2634	const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
	2635	const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
2600	2636	const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb);
2601	2637
2602	2638	const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb);
...	...	@@ -2604,8 +2640,8 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2604	2640	var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2605	2641	defer tmp_op.free(cg);
2606	2642
2607		try cg.store(result, op_lsb, Type.u64, 0);
2608		try cg.store(result, tmp_op, Type.u64, 8);
	2643	try cg.store(result, op_lsb, .u64, .{});
	2644	try cg.store(result, tmp_op, .u64, .{ .offset = 8 });
2609	2645	return result;
2610	2646	},
2611	2647	else => {
...	...	@@ -2641,15 +2677,15 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2641	2677	65...128 => {
2642	2678	const result = try cg.allocStack(Type.u128);
2643	2679
2644		var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
	2680	var lhs_lsb = try (try cg.load(lhs, .u64, .{})).toLocal(cg, Type.u64);
2645	2681	defer lhs_lsb.free(cg);
2646		var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
	2682	var rhs_lsb = try (try cg.load(rhs, .u64, .{})).toLocal(cg, Type.u64);
2647	2683	defer rhs_lsb.free(cg);
2648	2684	var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2649	2685	defer op_lsb.free(cg);
2650	2686
2651		const lhs_msb = try cg.load(lhs, Type.u64, 8);
2652		const rhs_msb = try cg.load(rhs, Type.u64, 8);
	2687	const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
	2688	const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
2653	2689	const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb);
2654	2690
2655	2691	const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb);
...	...	@@ -2657,8 +2693,8 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2657	2693	var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2658	2694	defer tmp_op.free(cg);
2659	2695
2660		try cg.store(result, op_lsb, Type.u64, 0);
2661		try cg.store(result, tmp_op, Type.u64, 8);
	2696	try cg.store(result, op_lsb, .u64, .{});
	2697	try cg.store(result, tmp_op, .u64, .{ .offset = 8 });
2662	2698	return result;
2663	2699	},
2664	2700	else => {
...	...	@@ -2990,15 +3026,15 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2990	3026	65...128 => {
2991	3027	const result = try cg.allocStack(Type.u128);
2992	3028
2993		const lhs_lsb = try cg.load(lhs, Type.u64, 0);
2994		const rhs_lsb = try cg.load(rhs, Type.u64, 0);
	3029	const lhs_lsb = try cg.load(lhs, .u64, .{});
	3030	const rhs_lsb = try cg.load(rhs, .u64, .{});
2995	3031	const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2996		try cg.store(result, and_lsb, Type.u64, 0);
	3032	try cg.store(result, and_lsb, .u64, .{});
2997	3033
2998		const lhs_msb = try cg.load(lhs, Type.u64, 8);
2999		const rhs_msb = try cg.load(rhs, Type.u64, 8);
	3034	const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
	3035	const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
3000	3036	const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3001		try cg.store(result, and_msb, Type.u64, 8);
	3037	try cg.store(result, and_msb, .u64, .{ .offset = 8 });
3002	3038
3003	3039	return result;
3004	3040	},
...	...	@@ -3034,15 +3070,15 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3034	3070	65...128 => {
3035	3071	const result = try cg.allocStack(Type.u128);
3036	3072
3037		const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3038		const rhs_lsb = try cg.load(rhs, Type.u64, 0);
	3073	const lhs_lsb = try cg.load(lhs, .u64, .{});
	3074	const rhs_lsb = try cg.load(rhs, .u64, .{});
3039	3075	const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3040		try cg.store(result, or_lsb, Type.u64, 0);
	3076	try cg.store(result, or_lsb, .u64, .{});
3041	3077
3042		const lhs_msb = try cg.load(lhs, Type.u64, 8);
3043		const rhs_msb = try cg.load(rhs, Type.u64, 8);
	3078	const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
	3079	const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
3044	3080	const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3045		try cg.store(result, or_msb, Type.u64, 8);
	3081	try cg.store(result, or_msb, .u64, .{ .offset = 8 });
3046	3082
3047	3083	return result;
3048	3084	},
...	...	@@ -3078,15 +3114,15 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3078	3114	65...128 => {
3079	3115	const result = try cg.allocStack(Type.u128);
3080	3116
3081		const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3082		const rhs_lsb = try cg.load(rhs, Type.u64, 0);
	3117	const lhs_lsb = try cg.load(lhs, .u64, .{});
	3118	const rhs_lsb = try cg.load(rhs, .u64, .{});
3083	3119	const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3084		try cg.store(result, xor_lsb, Type.u64, 0);
	3120	try cg.store(result, xor_lsb, .u64, .{});
3085	3121
3086		const lhs_msb = try cg.load(lhs, Type.u64, 8);
3087		const rhs_msb = try cg.load(rhs, Type.u64, 8);
	3122	const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
	3123	const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
3088	3124	const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3089		try cg.store(result, xor_msb, Type.u64, 8);
	3125	try cg.store(result, xor_msb, .u64, .{ .offset = 8 });
3090	3126
3091	3127	return result;
3092	3128	},
...	...	@@ -3141,20 +3177,20 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3141	3177	const result = try cg.allocStack(Type.u128);
3142	3178
3143	3179	try cg.emitWValue(result);
3144		_ = try cg.load(operand, Type.u64, 0);
	3180	_ = try cg.load(operand, .u64, .{});
3145	3181	try cg.addImm64(~@as(u64, 0));
3146	3182	try cg.addTag(.i64_xor);
3147		try cg.store(.stack, .stack, Type.u64, result.offset());
	3183	try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
3148	3184
3149	3185	try cg.emitWValue(result);
3150		_ = try cg.load(operand, Type.u64, 8);
	3186	_ = try cg.load(operand, .u64, .{ .offset = 8 });
3151	3187	const high_mask: u64 = if (ty.is_signed)
3152	3188	~@as(u64, 0)
3153	3189	else
3154	3190	~@as(u64, 0) >> @intCast(128 - ty.bits);
3155	3191	try cg.addImm64(high_mask);
3156	3192	try cg.addTag(.i64_xor);
3157		try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
	3193	try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 });
3158	3194
3159	3195	return result;
3160	3196	},
...	...	@@ -3287,16 +3323,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3287	3323	try cg.emitWValue(mask);
3288	3324	try cg.emitWValue(mask);
3289	3325
3290		_ = try cg.load(operand, Type.u64, 8);
	3326	_ = try cg.load(operand, .u64, .{ .offset = 8 });
3291	3327	try cg.addImm64(63);
3292	3328	try cg.addTag(.i64_shr_s);
3293	3329
3294	3330	var tmp = try cg.allocLocal(Type.u64);
3295	3331	defer tmp.free(cg);
3296	3332	try cg.addLocal(.local_tee, tmp.local.value);
3297		try cg.store(.stack, .stack, Type.u64, mask.offset() + 0);
	3333	try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 0 });
3298	3334	try cg.emitWValue(tmp);
3299		try cg.store(.stack, .stack, Type.u64, mask.offset() + 8);
	3335	try cg.store(.stack, .stack, .u64, .{ .offset = mask.offset() + 8 });
3300	3336
3301	3337	const a = try cg.intXor(u128_ty, operand, mask);
3302	3338	const b = try cg.intSub(u128_ty, a, mask);
...	...	@@ -3364,7 +3400,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3364	3400	return .stack;
3365	3401	},
3366	3402	65...128 => {
3367		var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64);
	3403	var msb = try (try cg.load(operand, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
3368	3404	defer msb.free(cg);
3369	3405
3370	3406	if (ty.is_signed and ty.bits < 128) {
...	...	@@ -3375,7 +3411,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3375	3411	}
3376	3412
3377	3413	try cg.addTag(.i64_clz);
3378		_ = try cg.load(operand, Type.u64, 0);
	3414	_ = try cg.load(operand, .u64, .{});
3379	3415	try cg.addTag(.i64_clz);
3380	3416	try cg.emitWValue(.{ .imm64 = 64 });
3381	3417	try cg.addTag(.i64_add);
...	...	@@ -3423,13 +3459,13 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3423	3459	return .stack;
3424	3460	},
3425	3461	65...128 => {
3426		var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64);
	3462	var lsb = try (try cg.load(operand, .u64, .{})).toLocal(cg, Type.u64);
3427	3463	defer lsb.free(cg);
3428	3464
3429	3465	try cg.emitWValue(lsb);
3430	3466	try cg.addTag(.i64_ctz);
3431	3467
3432		_ = try cg.load(operand, Type.u64, 8);
	3468	_ = try cg.load(operand, .u64, .{ .offset = 8 });
3433	3469	if (ty.bits < 128) {
3434	3470	try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64));
3435	3471	try cg.addTag(.i64_or);
...	...	@@ -3475,9 +3511,9 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3475	3511	return .stack;
3476	3512	},
3477	3513	65...128 => {
3478		_ = try cg.load(operand, Type.u64, 0);
	3514	_ = try cg.load(operand, .u64, .{});
3479	3515	try cg.addTag(.i64_popcnt);
3480		_ = try cg.load(operand, Type.u64, 8);
	3516	_ = try cg.load(operand, .u64, .{ .offset = 8 });
3481	3517	if (ty.is_signed and ty.bits < 128) {
3482	3518	try cg.addImm64(128 - ty.bits);
3483	3519	try cg.addTag(.i64_shl);
...	...	@@ -3525,7 +3561,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3525	3561	const tmp = try cg.allocStack(Type.u128);
3526	3562
3527	3563	try cg.emitWValue(tmp);
3528		const hi = try cg.load(operand, Type.u64, 8);
	3564	const hi = try cg.load(operand, .u64, .{ .offset = 8 });
3529	3565	const hi_rev = try cg.callIntrinsic(
3530	3566	.__bitreversedi2,
3531	3567	&.{.u64_type},
...	...	@@ -3533,10 +3569,10 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3533	3569	&.{hi},
3534	3570	);
3535	3571	try cg.emitWValue(hi_rev);
3536		try cg.store(.stack, .stack, Type.u64, tmp.offset());
	3572	try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() });
3537	3573
3538	3574	try cg.emitWValue(tmp);
3539		const lo = try cg.load(operand, Type.u64, 0);
	3575	const lo = try cg.load(operand, .u64, .{});
3540	3576	const lo_rev = try cg.callIntrinsic(
3541	3577	.__bitreversedi2,
3542	3578	&.{.u64_type},
...	...	@@ -3544,7 +3580,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3544	3580	&.{lo},
3545	3581	);
3546	3582	try cg.emitWValue(lo_rev);
3547		try cg.store(.stack, .stack, Type.u64, tmp.offset() + 8);
	3583	try cg.store(.stack, .stack, .u64, .{ .offset = tmp.offset() + 8 });
3548	3584
3549	3585	if (ty.bits < 128) {
3550	3586	const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
...	...	@@ -3595,25 +3631,25 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3595	3631
3596	3632	try cg.emitWValue(result);
3597	3633
3598		const low = try cg.load(operand, Type.u64, 0);
	3634	const low = try cg.load(operand, .u64, .{});
3599	3635	const swap_low = try cg.callIntrinsic(
3600	3636	.__bswapdi2,
3601	3637	&.{.u64_type},
3602	3638	Type.u64,
3603	3639	&.{low},
3604	3640	);
3605		try cg.store(.stack, swap_low, Type.u64, result.offset() + 8);
	3641	try cg.store(.stack, swap_low, .u64, .{ .offset = result.offset() + 8 });
3606	3642
3607	3643	try cg.emitWValue(result);
3608	3644
3609		const high = try cg.load(operand, Type.u64, 8);
	3645	const high = try cg.load(operand, .u64, .{ .offset = 8 });
3610	3646	const swap_high = try cg.callIntrinsic(
3611	3647	.__bswapdi2,
3612	3648	&.{.u64_type},
3613	3649	Type.u64,
3614	3650	&.{high},
3615	3651	);
3616		try cg.store(.stack, swap_high, Type.u64, result.offset());
	3652	try cg.store(.stack, swap_high, .u64, .{ .offset = result.offset() });
3617	3653
3618	3654	if (ty.bits < 128) {
3619	3655	const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
...	...	@@ -3671,11 +3707,11 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3671	3707	const result = try cg.allocStack(Type.u128);
3672	3708
3673	3709	try cg.emitWValue(result);
3674		_ = try cg.load(operand, Type.u64, 0);
3675		try cg.store(.stack, .stack, Type.u64, result.offset());
	3710	_ = try cg.load(operand, .u64, .{});
	3711	try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
3676	3712
3677	3713	try cg.emitWValue(result);
3678		_ = try cg.load(operand, Type.u64, 8);
	3714	_ = try cg.load(operand, .u64, .{ .offset = 8 });
3679	3715	if (ty.is_signed) {
3680	3716	try cg.addImm64(128 - ty.bits);
3681	3717	try cg.addTag(.i64_shl);
...	...	@@ -3685,7 +3721,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3685	3721	try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));
3686	3722	try cg.addTag(.i64_and);
3687	3723	}
3688		try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
	3724	try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + 8 });
3689	3725
3690	3726	return result;
3691	3727	},
...	...	@@ -3703,7 +3739,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3703	3739	const pad = 64 - ty.bits % 64;
3704	3740
3705	3741	try cg.emitWValue(result);
3706		_ = try cg.load(operand, Type.u64, used_len - 8);
	3742	_ = try cg.load(operand, .u64, .{ .offset = used_len - 8 });
3707	3743	if (ty.is_signed) {
3708	3744	try cg.addImm64(pad);
3709	3745	try cg.addTag(.i64_shl);
...	...	@@ -3713,7 +3749,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3713	3749	try cg.addImm64(~@as(u64, 0) >> @intCast(pad));
3714	3750	try cg.addTag(.i64_and);
3715	3751	}
3716		try cg.store(.stack, .stack, Type.u64, result.offset() + used_len - 8);
	3752	try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len - 8 });
3717	3753	} else {
3718	3754	try cg.memcpy(result, operand, .{ .imm32 = used_len });
3719	3755	}
...	...	@@ -3722,13 +3758,13 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3722	3758	if (used_len + 8 == full_len) { // last limb needs sign extended
3723	3759	try cg.emitWValue(result);
3724	3760	if (ty.is_signed) {
3725		_ = try cg.load(result, Type.u64, used_len - 8);
	3761	_ = try cg.load(result, .u64, .{ .offset = used_len - 8 });
3726	3762	try cg.addImm64(63);
3727	3763	try cg.addTag(.i64_shr_s);
3728	3764	} else {
3729	3765	try cg.addImm64(0);
3730	3766	}
3731		try cg.store(.stack, .stack, Type.u64, result.offset() + used_len);
	3767	try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() + used_len });
3732	3768	}
3733	3769
3734	3770	return result;
...	...	@@ -3751,12 +3787,12 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3751	3787	}
3752	3788	} else if (int_ty.bits <= 128) {
3753	3789	const result = try cg.allocInt(int_ty);
3754		try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, 0);
	3790	try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, .u64, .{});
3755	3791
3756	3792	if (int_ty.is_signed) {
3757		try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, Type.u64, 8);
	3793	try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, .u64, .{ .offset = 8 });
3758	3794	} else {
3759		try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, Type.u64, 8);
	3795	try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, .u64, .{ .offset = 8 });
3760	3796	}
3761	3797	return result;
3762	3798	} else {
...	...	@@ -3767,13 +3803,13 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3767	3803	try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0xFF });
3768	3804
3769	3805	if (int_ty.is_signed) {
3770		try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, Type.u64, used_len - 8);
	3806	try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, .u64, .{ .offset = used_len - 8 });
3771	3807	} else {
3772		try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, Type.u64, used_len - 8);
	3808	try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, .u64, .{ .offset = used_len - 8 });
3773	3809	}
3774	3810
3775	3811	if (used_len + 8 == full_len) {
3776		try cg.store(result, .{ .imm64 = 0 }, Type.u64, full_len - 8);
	3812	try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = full_len - 8 });
3777	3813	}
3778	3814
3779	3815	return result;
...	...	@@ -3790,8 +3826,8 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3790	3826	return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) };
3791	3827	} else if (int_ty.bits <= 128) {
3792	3828	const result = try cg.allocInt(int_ty);
3793		try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
3794		try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, Type.u64, 8);
	3829	try cg.store(result, .{ .imm64 = 0 }, .u64, .{});
	3830	try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, .u64, .{ .offset = 8 });
3795	3831	return result;
3796	3832	} else {
3797	3833	const result = try cg.allocInt(int_ty);
...	...	@@ -3799,10 +3835,10 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3799	3835	const used_len = @divCeil(int_ty.bits, 64) * 8;
3800	3836
3801	3837	try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0 });
3802		try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, Type.u64, used_len - 8);
	3838	try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, .u64, .{ .offset = used_len - 8 });
3803	3839
3804	3840	if (used_len + 8 == full_len) {
3805		try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, full_len - 8);
	3841	try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, .u64, .{ .offset = full_len - 8 });
3806	3842	}
3807	3843
3808	3844	return result;
...	...	@@ -3962,8 +3998,8 @@ fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3962	3998	33...64 => return .{ .imm64 = 0 },
3963	3999	65...128 => {
3964	4000	const result = try cg.allocInt(int_ty);
3965		try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
3966		try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
	4001	try cg.store(result, .{ .imm64 = 0 }, .u64, .{});
	4002	try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });
3967	4003	return result;
3968	4004	},
3969	4005	else => {
...	...	@@ -4103,7 +4139,7 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner
4103	4139	Type.i128,
4104	4140	&.{ lhs, rhs, overflow_ret },
4105	4141	);
4106		_ = try cg.load(overflow_ret, Type.i32, 0);
	4142	_ = try cg.load(overflow_ret, .i32, .{});
4107	4143	try cg.addLocal(.local_set, overflow_bit.local.value);
4108	4144	break :blk res;
4109	4145	} else {
...	...	@@ -4180,12 +4216,12 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4180	4216	try cg.emitWValue(result);
4181	4217	try cg.emitWValue(operand);
4182	4218	try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4183		try cg.store(.stack, .stack, Type.u64, result.offset());
	4219	try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
4184	4220	src_len = 8;
4185	4221	} else if (src_bits == 64) {
4186	4222	try cg.emitWValue(result);
4187	4223	try cg.emitWValue(operand);
4188		try cg.store(.stack, .stack, Type.u64, result.offset());
	4224	try cg.store(.stack, .stack, .u64, .{ .offset = result.offset() });
4189	4225	src_len = 8;
4190	4226	} else {
4191	4227	src_len = src_bits / 8;
...	...	@@ -4202,9 +4238,9 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4202	4238	try cg.emitWValue(operand);
4203	4239	} else unreachable;
4204	4240	const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4205		try cg.store(.stack, shr, Type.u64, 8 + result.offset());
	4241	try cg.store(.stack, shr, .u64, .{ .offset = 8 + result.offset() });
4206	4242	} else {
4207		try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
	4243	try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });
4208	4244	}
4209	4245	} else {
4210	4246	var pad = result;
...	...	@@ -4219,7 +4255,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4219	4255	_ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4220	4256	try cg.addTag(.i32_wrap_i64);
4221	4257	} else {
4222		_ = try cg.load(operand, Type.u64, src_len - 8);
	4258	_ = try cg.load(operand, .u64, .{ .offset = src_len - 8 });
4223	4259	_ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4224	4260	try cg.addTag(.i32_wrap_i64);
4225	4261	}
...	...	@@ -4237,7 +4273,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4237	4273	assert(dest_bits <= 64);
4238	4274	assert(src_bits >= 128);
4239	4275	const load_ty = if (dest_bits == 32) Type.u32 else Type.u64;
4240		return cg.load(operand, load_ty, 0);
	4276	return cg.load(operand, load_ty, .{});
4241	4277	}
4242	4278	}
4243	4279
...	...	@@ -4608,18 +4644,18 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4608	4644	const result = try cg.allocStack(Type.f128);
4609	4645	try cg.emitWValue(result);
4610	4646	try cg.emitWValue(arg);
4611		try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
4612		try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
	4647	try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" });
	4648	try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" });
4613	4649	try cg.emitWValue(result);
4614	4650	try cg.emitWValue(arg);
4615		try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
	4651	try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" });
4616	4652	if (ty == .f80) {
4617	4653	try cg.addImm64(0x8000);
4618	4654	} else {
4619	4655	try cg.addImm64(0x8000000000000000);
4620	4656	}
4621	4657	try cg.addTag(.i64_xor);
4622		try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
	4658	try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" });
4623	4659	return result;
4624	4660	},
4625	4661	}
...	...	@@ -4647,18 +4683,18 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4647	4683	const result = try cg.allocStack(Type.f128);
4648	4684	try cg.emitWValue(result);
4649	4685	try cg.emitWValue(arg);
4650		try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
4651		try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
	4686	try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = .@"2" });
	4687	try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = .@"2" });
4652	4688	try cg.emitWValue(result);
4653	4689	try cg.emitWValue(arg);
4654		try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
	4690	try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = .@"2" });
4655	4691	if (ty == .f80) {
4656	4692	try cg.addImm64(0x7FFF);
4657	4693	} else {
4658	4694	try cg.addImm64(0x7FFFFFFFFFFFFFFF);
4659	4695	}
4660	4696	try cg.addTag(.i64_and);
4661		try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
	4697	try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = .@"2" });
4662	4698	return result;
4663	4699	},
4664	4700	}
...	...	@@ -5309,8 +5345,8 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne
5309	5345	try cg.addLocal(.local_get, lhs_null.local.value);
5310	5346	try cg.addLabel(.br_if, 0);
5311	5347
5312		_ = try cg.load(lhs, payload_ty, 0);
5313		_ = try cg.load(rhs, payload_ty, 0);
	5348	_ = try cg.load(lhs, payload_ty, .{});
	5349	_ = try cg.load(rhs, payload_ty, .{});
5314	5350
5315	5351	if (payload_ty.isAnyFloat()) {
5316	5352	_ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack);
...	...	@@ -5370,16 +5406,16 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5370	5406	return .stack;
5371	5407	},
5372	5408	65...128 => {
5373		var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);
	5409	var lhs_msb = try (try cg.load(lhs, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
5374	5410	defer lhs_msb.free(cg);
5375		var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);
	5411	var rhs_msb = try (try cg.load(rhs, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
5376	5412	defer rhs_msb.free(cg);
5377	5413
5378	5414	switch (op) {
5379	5415	.eq, .neq => {
5380	5416	const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb);
5381		const lhs_lsb = try cg.load(lhs, Type.u64, 0);
5382		const rhs_lsb = try cg.load(rhs, Type.u64, 0);
	5417	const lhs_lsb = try cg.load(lhs, .u64, .{});
	5418	const rhs_lsb = try cg.load(rhs, .u64, .{});
5383	5419	const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb);
5384	5420	const or_result = try cg.intOr(.u64, xor_high, xor_low);
5385	5421
...	...	@@ -5392,8 +5428,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5392	5428	else => {
5393	5429	const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64;
5394	5430
5395		const lhs_lsb = try cg.load(lhs, Type.u64, 0);
5396		const rhs_lsb = try cg.load(rhs, Type.u64, 0);
	5431	const lhs_lsb = try cg.load(lhs, .u64, .{});
	5432	const rhs_lsb = try cg.load(rhs, .u64, .{});
5397	5433
5398	5434	// leave values on stack for 'select'
5399	5435	_ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb);
...	...	@@ -5599,7 +5635,7 @@ fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5599	5635
5600	5636	const enum_value = try cg.resolveInst(ty_op.operand);
5601	5637	const result = try cg.allocStack(union_ty);
5602		try cg.store(result, enum_value, enum_ty, @intCast(layout.tagOffset()));
	5638	try cg.store(result, enum_value, enum_ty, .{ .offset = layout.tagOffset() });
5603	5639
5604	5640	return cg.finishAir(inst, result, &.{ty_op.operand});
5605	5641	}
...	...	@@ -5662,12 +5698,12 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro
5662	5698
5663	5699	if (dest_by_ref) {
5664	5700	const result = try cg.allocStack(src_ty);
5665		try cg.store(result, operand, src_ty, 0);
	5701	try cg.store(result, operand, src_ty, .{});
5666	5702	return result;
5667	5703	}
5668	5704
5669	5705	if (src_by_ref) {
5670		return try cg.load(operand, dest_ty, 0);
	5706	return try cg.load(operand, dest_ty, .{});
5671	5707	}
5672	5708
5673	5709	switch (src_class) {
...	...	@@ -5803,7 +5839,7 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5803	5839	else => break :result try cg.buildPointerOffset(operand, offset, .new),
5804	5840	}
5805	5841	}
5806		break :result try cg.load(operand, field_ty, offset);
	5842	break :result try cg.load(operand, field_ty, .{ .offset = offset });
5807	5843	},
5808	5844	};
5809	5845
...	...	@@ -6048,7 +6084,7 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind
6048	6084	if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
6049	6085	try cg.addMemArg(.i32_load16_u, .{
6050	6086	.offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),
6051		.alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
	6087	.alignment = Type.anyerror.abiAlignment(zcu).toStdMem(),
6052	6088	});
6053	6089	}
6054	6090
...	...	@@ -6085,7 +6121,7 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool)
6085	6121	break :result try cg.buildPointerOffset(operand, pl_offset, .new);
6086	6122	} else {
6087	6123	assert(isByRef(eu_ty, zcu, cg.target));
6088		break :result try cg.load(operand, payload_ty, pl_offset);
	6124	break :result try cg.load(operand, payload_ty, .{ .offset = pl_offset });
6089	6125	}
6090	6126	};
6091	6127	return cg.finishAir(inst, result, &.{ty_op.operand});
...	...	@@ -6110,7 +6146,7 @@ fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) I
6110	6146
6111	6147	const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
6112	6148	if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {
6113		break :result try cg.load(operand, Type.anyerror, err_offset);
	6149	break :result try cg.load(operand, .anyerror, .{ .offset = err_offset });
6114	6150	} else {
6115	6151	assert(!payload_ty.hasRuntimeBits(zcu));
6116	6152	break :result operand;
...	...	@@ -6134,7 +6170,7 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6134	6170
6135	6171	const err_union = try cg.allocStack(err_ty);
6136	6172	const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
6137		try cg.store(payload_ptr, operand, pl_ty, 0);
	6173	try cg.store(payload_ptr, operand, pl_ty, .{});
6138	6174
6139	6175	// ensure we also write '0' to the error part, so any present stack value gets overwritten by it.
6140	6176	try cg.emitWValue(err_union);
...	...	@@ -6142,7 +6178,7 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6142	6178	const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
6143	6179	try cg.addMemArg(.i32_store16, .{
6144	6180	.offset = err_union.offset() + err_val_offset,
6145		.alignment = 2,
	6181	.alignment = .@"2",
6146	6182	});
6147	6183	break :result err_union;
6148	6184	};
...	...	@@ -6164,7 +6200,7 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6164	6200
6165	6201	const err_union = try cg.allocStack(err_ty);
6166	6202	// store error value
6167		try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu)));
	6203	try cg.store(err_union, operand, .anyerror, .{ .offset = errUnionErrorOffset(pl_ty, zcu) });
6168	6204
6169	6205	// write 'undefined' to the payload
6170	6206	const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
...	...	@@ -6205,18 +6241,18 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o
6205	6241	const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6206	6242	return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});
6207	6243	};
6208		try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });
	6244	try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = .@"1" });
6209	6245	}
6210	6246	} else if (payload_ty.isSlice(zcu)) {
6211	6247	switch (cg.ptr_size) {
6212		.wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }),
6213		.wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }),
	6248	.wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = .@"4" }),
	6249	.wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = .@"8" }),
6214	6250	}
6215	6251	} else {
6216	6252	if (op_kind == .ptr) {
6217	6253	try cg.addMemArg(.i32_load, .{
6218	6254	.offset = operand.offset(),
6219		.alignment = 4,
	6255	.alignment = .@"4",
6220	6256	});
6221	6257	}
6222	6258	}
...	...	@@ -6245,7 +6281,7 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6245	6281	break :result try cg.buildPointerOffset(operand, 0, .new);
6246	6282	}
6247	6283
6248		break :result try cg.load(operand, payload_ty, 0);
	6284	break :result try cg.load(operand, payload_ty, .{});
6249	6285	};
6250	6286	return cg.finishAir(inst, result, &.{ty_op.operand});
6251	6287	}
...	...	@@ -6285,7 +6321,7 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
6285	6321
6286	6322	try cg.emitWValue(operand);
6287	6323	try cg.addImm32(1);
6288		try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });
	6324	try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = .@"1" });
6289	6325
6290	6326	const result = try cg.buildPointerOffset(operand, 0, .new);
6291	6327	return cg.finishAir(inst, result, &.{ty_op.operand});
...	...	@@ -6302,7 +6338,7 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6302	6338	const non_null_bit = try cg.allocStack(Type.u1);
6303	6339	try cg.emitWValue(non_null_bit);
6304	6340	try cg.addImm32(1);
6305		try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 });
	6341	try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = .@"1" });
6306	6342	break :result non_null_bit;
6307	6343	}
6308	6344
...	...	@@ -6319,10 +6355,10 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6319	6355	const result_ptr = try cg.allocStack(op_ty);
6320	6356	try cg.emitWValue(result_ptr);
6321	6357	try cg.addImm32(1);
6322		try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });
	6358	try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = .@"1" });
6323	6359
6324	6360	const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);
6325		try cg.store(payload_ptr, operand, payload_ty, 0);
	6361	try cg.store(payload_ptr, operand, payload_ty, .{});
6326	6362	break :result result_ptr;
6327	6363	};
6328	6364
...	...	@@ -6338,8 +6374,8 @@ fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6338	6374	const slice_ty = cg.typeOfIndex(inst);
6339	6375
6340	6376	const slice = try cg.allocStack(slice_ty);
6341		try cg.store(slice, lhs, Type.usize, 0);
6342		try cg.store(slice, rhs, Type.usize, cg.ptrSize());
	6377	try cg.store(slice, lhs, .usize, .{});
	6378	try cg.store(slice, rhs, .usize, .{ .offset = cg.ptrSize() });
6343	6379
6344	6380	return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });
6345	6381	}
...	...	@@ -6362,7 +6398,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6362	6398	const elem_size = elem_ty.abiSize(zcu);
6363	6399
6364	6400	// load pointer onto stack
6365		_ = try cg.load(slice, Type.usize, 0);
	6401	_ = try cg.load(slice, .usize, .{});
6366	6402
6367	6403	// calculate index into slice
6368	6404	try cg.emitWValue(index);
...	...	@@ -6370,7 +6406,7 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6370	6406	try cg.addTag(.i32_mul);
6371	6407	try cg.addTag(.i32_add);
6372	6408
6373		const elem_result = try cg.load(.stack, elem_ty, 0);
	6409	const elem_result = try cg.load(.stack, elem_ty, .{});
6374	6410
6375	6411	return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6376	6412	}
...	...	@@ -6386,7 +6422,7 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6386	6422	const slice = try cg.resolveInst(bin_op.lhs);
6387	6423	const index = try cg.resolveInst(bin_op.rhs);
6388	6424
6389		_ = try cg.load(slice, Type.usize, 0);
	6425	_ = try cg.load(slice, .usize, .{});
6390	6426
6391	6427	// calculate index into slice
6392	6428	try cg.emitWValue(index);
...	...	@@ -6404,12 +6440,12 @@ fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6404	6440	}
6405	6441
6406	6442	fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {
6407		const ptr = try cg.load(operand, Type.usize, 0);
	6443	const ptr = try cg.load(operand, .usize, .{});
6408	6444	return ptr.toLocal(cg, Type.usize);
6409	6445	}
6410	6446
6411	6447	fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {
6412		const len = try cg.load(operand, Type.usize, cg.ptrSize());
	6448	const len = try cg.load(operand, .usize, .{ .offset = cg.ptrSize() });
6413	6449	return len.toLocal(cg, Type.usize);
6414	6450	}
6415	6451
...	...	@@ -6424,11 +6460,11 @@ fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6424	6460	// create a slice on the stack
6425	6461	const slice_local = try cg.allocStack(slice_ty);
6426	6462
6427		try cg.store(slice_local, operand, Type.usize, 0);
	6463	try cg.store(slice_local, operand, .usize, .{});
6428	6464
6429	6465	// store the length of the array in the slice
6430	6466	const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
6431		try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize());
	6467	try cg.store(slice_local, .{ .imm32 = array_len }, .usize, .{ .offset = cg.ptrSize() });
6432	6468
6433	6469	return cg.finishAir(inst, slice_local, &.{ty_op.operand});
6434	6470	}
...	...	@@ -6445,7 +6481,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6445	6481
6446	6482	// load pointer onto the stack
6447	6483	if (ptr_ty.isSlice(zcu)) {
6448		_ = try cg.load(ptr, Type.usize, 0);
	6484	_ = try cg.load(ptr, .usize, .{});
6449	6485	} else {
6450	6486	try cg.lowerToStack(ptr);
6451	6487	}
...	...	@@ -6456,7 +6492,7 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6456	6492	try cg.addTag(.i32_mul);
6457	6493	try cg.addTag(.i32_add);
6458	6494
6459		const elem_result = try cg.load(.stack, elem_ty, 0);
	6495	const elem_result = try cg.load(.stack, elem_ty, .{});
6460	6496
6461	6497	return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6462	6498	}
...	...	@@ -6475,7 +6511,7 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6475	6511
6476	6512	// load pointer onto the stack
6477	6513	if (ptr_ty.isSlice(zcu)) {
6478		_ = try cg.load(ptr, Type.usize, 0);
	6514	_ = try cg.load(ptr, .usize, .{});
6479	6515	} else {
6480	6516	try cg.lowerToStack(ptr);
6481	6517	}
...	...	@@ -6645,7 +6681,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6645	6681	try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
6646	6682
6647	6683	// store the value at the current position of the pointer
6648		try cg.store(new_ptr, value, elem_ty, 0);
	6684	try cg.store(new_ptr, value, elem_ty, .{});
6649	6685
6650	6686	// move the pointer to the next element
6651	6687	try cg.emitWValue(new_ptr);
...	...	@@ -6708,7 +6744,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6708	6744	},
6709	6745	else => {
6710	6746	const stack_vec = try cg.allocStack(array_ty);
6711		try cg.store(stack_vec, array, array_ty, 0);
	6747	try cg.store(stack_vec, array, array_ty, .{});
6712	6748
6713	6749	// Is a non-unrolled vector (v128)
6714	6750	try cg.lowerToStack(stack_vec);
...	...	@@ -6723,7 +6759,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6723	6759	const result = if (isByRef(elem_ty, zcu, cg.target))
6724	6760	.stack
6725	6761	else
6726		try cg.load(.stack, elem_ty, 0);
	6762	try cg.load(.stack, elem_ty, .{});
6727	6763	return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6728	6764	}
6729	6765
...	...	@@ -6750,10 +6786,11 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6750	6786	try cg.emitWValue(operand);
6751	6787	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6752	6788	// stores as := opcode, offset, alignment (opcode::memarg)
6753		try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
6754		opcode,
6755		operand.offset(),
6756		@intCast(elem_ty.abiAlignment(zcu).toByteUnits().?),
	6789	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
	6790	cg.mir_extra.appendAssumeCapacity(opcode);
	6791	_ = try cg.addExtraAssumeCapacity(Mir.MemArg{
	6792	.offset = operand.offset(),
	6793	.alignment = elem_ty.abiAlignment(zcu).toStdMem(),
6757	6794	});
6758	6795	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6759	6796	return cg.finishAir(inst, .stack, &.{ty_op.operand});
...	...	@@ -6782,7 +6819,7 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6782	6819	var index: usize = 0;
6783	6820	var offset: u32 = 0;
6784	6821	while (index < vector_len) : (index += 1) {
6785		try cg.store(result, operand, elem_ty, offset);
	6822	try cg.store(result, operand, elem_ty, .{ .offset = offset });
6786	6823	offset += elem_byte_size;
6787	6824	}
6788	6825
...	...	@@ -6821,10 +6858,10 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6821	6858	for (mask, 0..) \|mask_elem, out_idx\| {
6822	6859	try cg.emitWValue(dest_alloc);
6823	6860	const elem_val = switch (mask_elem.unwrap()) {
6824		.elem => \|idx\| try cg.load(operand, elem_ty, @intCast(elem_size * idx)),
	6861	.elem => \|idx\| try cg.load(operand, elem_ty, .{ .offset = elem_size * idx }),
6825	6862	.value => \|val\| try cg.lowerConstant(.fromInterned(val)),
6826	6863	};
6827		try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
	6864	try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx });
6828	6865	}
6829	6866	return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
6830	6867	}
...	...	@@ -6889,11 +6926,11 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6889	6926	for (mask, 0..) \|mask_elem, out_idx\| {
6890	6927	try cg.emitWValue(dest_alloc);
6891	6928	const elem_val = switch (mask_elem.unwrap()) {
6892		.a_elem => \|idx\| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)),
6893		.b_elem => \|idx\| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)),
	6929	.a_elem => \|idx\| try cg.load(operand_a, elem_ty, .{ .offset = elem_size * idx }),
	6930	.b_elem => \|idx\| try cg.load(operand_b, elem_ty, .{ .offset = elem_size * idx }),
6894	6931	.undef => try cg.emitUndefined(elem_ty),
6895	6932	};
6896		try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
	6933	try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.offset() + elem_size * out_idx });
6897	6934	}
6898	6935	return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
6899	6936	}
...	...	@@ -6931,7 +6968,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6931	6968	const offset = try cg.buildPointerOffset(result, 0, .new);
6932	6969	for (elements, 0..) \|elem, elem_index\| {
6933	6970	const elem_val = try cg.resolveInst(elem);
6934		try cg.store(offset, elem_val, elem_ty, 0);
	6971	try cg.store(offset, elem_val, elem_ty, .{});
6935	6972
6936	6973	if (elem_index < elements.len - 1 or sentinel != null) {
6937	6974	_ = try cg.buildPointerOffset(offset, elem_size, .modify);
...	...	@@ -6939,18 +6976,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6939	6976	}
6940	6977	if (sentinel) \|s\| {
6941	6978	const val = try cg.resolveValue(s);
6942		try cg.store(offset, val, elem_ty, 0);
	6979	try cg.store(offset, val, elem_ty, .{});
6943	6980	}
6944	6981	} else {
6945	6982	var offset: u32 = 0;
6946	6983	for (elements) \|elem\| {
6947	6984	const elem_val = try cg.resolveInst(elem);
6948		try cg.store(result, elem_val, elem_ty, offset);
	6985	try cg.store(result, elem_val, elem_ty, .{ .offset = offset });
6949	6986	offset += elem_size;
6950	6987	}
6951	6988	if (sentinel) \|s\| {
6952	6989	const val = try cg.resolveValue(s);
6953		try cg.store(result, val, elem_ty, offset);
	6990	try cg.store(result, val, elem_ty, .{ .offset = offset });
6954	6991	}
6955	6992	}
6956	6993	break :result_value result;
...	...	@@ -6967,7 +7004,7 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6967	7004	const offset = try cg.buildPointerOffset(result, field_offset, .new);
6968	7005
6969	7006	const value = try cg.resolveInst(elem);
6970		try cg.store(offset, value, elem_ty, 0);
	7007	try cg.store(offset, value, elem_ty, .{});
6971	7008	}
6972	7009
6973	7010	break :result_value result;
...	...	@@ -7016,22 +7053,22 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7016	7053	if (layout.tag_align.compare(.gte, layout.payload_align)) {
7017	7054	if (isByRef(field_ty, zcu, cg.target)) {
7018	7055	const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);
7019		try cg.store(payload_ptr, payload, field_ty, 0);
	7056	try cg.store(payload_ptr, payload, field_ty, .{});
7020	7057	} else {
7021		try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size));
	7058	try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size });
7022	7059	}
7023	7060
7024	7061	if (layout.tag_size > 0) {
7025		try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), 0);
	7062	try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), .{});
7026	7063	}
7027	7064	} else {
7028		try cg.store(result_ptr, payload, field_ty, 0);
	7065	try cg.store(result_ptr, payload, field_ty, .{});
7029	7066	if (layout.tag_size > 0) {
7030	7067	try cg.store(
7031	7068	result_ptr,
7032	7069	tag_int,
7033	7070	.fromInterned(union_obj.enum_tag_type),
7034		@intCast(layout.payload_size),
	7071	.{ .offset = layout.payload_size },
7035	7072	);
7036	7073	}
7037	7074	}
...	...	@@ -7077,7 +7114,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7077	7114	const union_ptr = try cg.resolveInst(bin_op.lhs);
7078	7115	const new_tag = try cg.resolveInst(bin_op.rhs);
7079	7116	if (layout.payload_size == 0) {
7080		try cg.store(union_ptr, new_tag, tag_ty, 0);
	7117	try cg.store(union_ptr, new_tag, tag_ty, .{});
7081	7118	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7082	7119	}
7083	7120
...	...	@@ -7086,7 +7123,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7086	7123	const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {
7087	7124	break :blk @intCast(layout.payload_size);
7088	7125	} else 0;
7089		try cg.store(union_ptr, new_tag, tag_ty, offset);
	7126	try cg.store(union_ptr, new_tag, tag_ty, .{ .offset = offset });
7090	7127	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7091	7128	}
7092	7129
...	...	@@ -7106,7 +7143,7 @@ fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7106	7143	@intCast(layout.payload_size)
7107	7144	else
7108	7145	0;
7109		const result = try cg.load(operand, tag_ty, offset);
	7146	const result = try cg.load(operand, tag_ty, .{ .offset = offset });
7110	7147	return cg.finishAir(inst, result, &.{ty_op.operand});
7111	7148	}
7112	7149
...	...	@@ -7122,8 +7159,8 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
7122	7159	try cg.store(
7123	7160	operand,
7124	7161	.{ .imm32 = 0 },
7125		Type.anyerror,
7126		@intCast(errUnionErrorOffset(payload_ty, zcu)),
	7162	.anyerror,
	7163	.{ .offset = errUnionErrorOffset(payload_ty, zcu) },
7127	7164	);
7128	7165
7129	7166	const result = result: {
...	...	@@ -7360,7 +7397,7 @@ fn lowerTry(
7360	7397	const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
7361	7398	try cg.addMemArg(.i32_load16_u, .{
7362	7399	.offset = err_union.offset() + err_offset,
7363		.alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
	7400	.alignment = Type.anyerror.abiAlignment(zcu).toStdMem(),
7364	7401	});
7365	7402	}
7366	7403	try cg.addTag(.i32_eqz);
...	...	@@ -7384,7 +7421,7 @@ fn lowerTry(
7384	7421	if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {
7385	7422	return buildPointerOffset(cg, err_union, pl_offset, .new);
7386	7423	}
7387		const payload = try cg.load(err_union, pl_ty, pl_offset);
	7424	const payload = try cg.load(err_union, pl_ty, .{ .offset = pl_offset });
7388	7425	return payload.toLocal(cg, pl_ty);
7389	7426	}
7390	7427
...	...	@@ -7588,7 +7625,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7588	7625	else => \|size\| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),
7589	7626	}, .{
7590	7627	.offset = ptr_operand.offset(),
7591		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	7628	.alignment = ty.abiAlignment(zcu).toStdMem(),
7592	7629	});
7593	7630	try cg.addLocal(.local_tee, val_local.local.value);
7594	7631	_ = try cg.intCmp(int_ty, .eq, .stack, expected_val);
...	...	@@ -7598,7 +7635,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7598	7635	if (ty.abiSize(zcu) > 8) {
7599	7636	return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{});
7600	7637	}
7601		const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, 0), cg, ty);
	7638	const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, .{}), cg, ty);
7602	7639
7603	7640	try cg.lowerToStack(ptr_operand);
7604	7641	try cg.lowerToStack(new_val);
...	...	@@ -7606,7 +7643,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7606	7643	_ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);
7607	7644	try cg.addLocal(.local_tee, cmp_result.local.value);
7608	7645	try cg.addTag(.select);
7609		try cg.store(.stack, .stack, ty, 0);
	7646	try cg.store(.stack, .stack, ty, .{});
7610	7647
7611	7648	break :val ptr_val;
7612	7649	};
...	...	@@ -7619,8 +7656,8 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7619	7656	try cg.addTag(.i32_and);
7620	7657	const and_result = try WValue.toLocal(.stack, cg, Type.bool);
7621	7658	const result_ptr = try cg.allocStack(result_ty);
7622		try cg.store(result_ptr, and_result, Type.bool, @as(u32, @intCast(ty.abiSize(zcu))));
7623		try cg.store(result_ptr, ptr_val, ty, 0);
	7659	try cg.store(result_ptr, and_result, .bool, .{ .offset = ty.abiSize(zcu) });
	7660	try cg.store(result_ptr, ptr_val, ty, .{});
7624	7661	break :val result_ptr;
7625	7662	} else val: {
7626	7663	try cg.addImm32(0);
...	...	@@ -7650,10 +7687,10 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7650	7687	try cg.emitWValue(ptr);
7651	7688	try cg.addAtomicMemArg(tag, .{
7652	7689	.offset = ptr.offset(),
7653		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	7690	.alignment = ty.abiAlignment(zcu).toStdMem(),
7654	7691	});
7655	7692	} else {
7656		_ = try cg.load(ptr, ty, 0);
	7693	_ = try cg.load(ptr, ty, .{});
7657	7694	}
7658	7695
7659	7696	return cg.finishAir(inst, .stack, &.{atomic_load.ptr});
...	...	@@ -7676,7 +7713,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7676	7713	.Min,
7677	7714	.Nand,
7678	7715	=> {
7679		const tmp = try cg.load(ptr, ty, 0);
	7716	const tmp = try cg.load(ptr, ty, .{});
7680	7717	const value = try tmp.toLocal(cg, ty);
7681	7718
7682	7719	// create a loop to cmpxchg the new value
...	...	@@ -7710,7 +7747,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7710	7747	},
7711	7748	.{
7712	7749	.offset = ptr.offset(),
7713		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	7750	.alignment = ty.abiAlignment(zcu).toStdMem(),
7714	7751	},
7715	7752	);
7716	7753	const select_res = try cg.allocLocal(ty);
...	...	@@ -7770,18 +7807,18 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7770	7807	};
7771	7808	try cg.addAtomicMemArg(tag, .{
7772	7809	.offset = ptr.offset(),
7773		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	7810	.alignment = ty.abiAlignment(zcu).toStdMem(),
7774	7811	});
7775	7812	return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });
7776	7813	},
7777	7814	}
7778	7815	} else {
7779		const loaded = try cg.load(ptr, ty, 0);
	7816	const loaded = try cg.load(ptr, ty, .{});
7780	7817	const result = try loaded.toLocal(cg, ty);
7781	7818
7782	7819	switch (op) {
7783	7820	.Xchg => {
7784		try cg.store(ptr, operand, ty, 0);
	7821	try cg.store(ptr, operand, ty, .{});
7785	7822	},
7786	7823	.Add,
7787	7824	.Sub,
...	...	@@ -7794,7 +7831,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7794	7831	.Sub => try cg.floatSub(float_ty, result, operand),
7795	7832	else => unreachable,
7796	7833	};
7797		try cg.store(.stack, .stack, ty, ptr.offset());
	7834	try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7798	7835	} else {
7799	7836	const int_ty: IntType = .fromType(cg, ty);
7800	7837	try cg.emitWValue(ptr);
...	...	@@ -7804,7 +7841,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7804	7841	else => unreachable,
7805	7842	};
7806	7843	_ = try cg.intWrap(int_ty, .stack);
7807		try cg.store(.stack, .stack, ty, ptr.offset());
	7844	try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7808	7845	}
7809	7846	},
7810	7847	.And,
...	...	@@ -7819,7 +7856,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7819	7856	.Xor => try cg.intXor(int_ty, result, operand),
7820	7857	else => unreachable,
7821	7858	};
7822		try cg.store(.stack, .stack, ty, ptr.offset());
	7859	try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7823	7860	},
7824	7861	.Max,
7825	7862	.Min,
...	...	@@ -7831,7 +7868,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7831	7868	try cg.emitWValue(operand);
7832	7869	_ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);
7833	7870	try cg.addTag(.select);
7834		try cg.store(.stack, .stack, ty, ptr.offset());
	7871	try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7835	7872	} else {
7836	7873	const int_ty: IntType = .fromType(cg, ty);
7837	7874	try cg.emitWValue(ptr);
...	...	@@ -7839,7 +7876,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7839	7876	try cg.emitWValue(operand);
7840	7877	_ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);
7841	7878	try cg.addTag(.select);
7842		try cg.store(.stack, .stack, ty, ptr.offset());
	7879	try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7843	7880	}
7844	7881	},
7845	7882	.Nand => {
...	...	@@ -7854,7 +7891,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7854	7891	return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7855	7892	}
7856	7893	_ = try cg.intXor(int_ty, and_res, .stack);
7857		try cg.store(.stack, .stack, ty, ptr.offset());
	7894	try cg.store(.stack, .stack, ty, .{ .offset = ptr.offset() });
7858	7895	},
7859	7896	}
7860	7897
...	...	@@ -7883,10 +7920,10 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7883	7920	try cg.lowerToStack(operand);
7884	7921	try cg.addAtomicMemArg(tag, .{
7885	7922	.offset = ptr.offset(),
7886		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	7923	.alignment = ty.abiAlignment(zcu).toStdMem(),
7887	7924	});
7888	7925	} else {
7889		try cg.store(ptr, operand, ty, 0);
	7926	try cg.store(ptr, operand, ty, .{});
7890	7927	}
7891	7928
7892	7929	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });

src/codegen/wasm/Emit.zig+1-3

...	...	@@ -988,9 +988,7 @@ pub fn lower(emit: *Emit) Error!void {
988	988	/// Asserts 20 unused capacity.
989	989	fn encodeMemArg(code: *ArrayList(u8), mem_arg: Mir.MemArg) void {
990	990	assert(code.unusedCapacitySlice().len >= 20);
991		// Wasm encodes alignment as power of 2, rather than natural alignment.
992		const encoded_alignment = @ctz(mem_arg.alignment);
993		writeUleb128(code, encoded_alignment);
	991	writeUleb128(code, @backingInt(mem_arg.alignment));
994	992	writeUleb128(code, mem_arg.offset);
995	993	}
996	994

src/codegen/wasm/Mir.zig+9-2

...	...	@@ -694,11 +694,18 @@ pub fn extraData(self: *const Mir, comptime T: type, index: usize) struct { data
694	694	inline for (info.field_names, info.field_types) \|field_name, field_type\| {
695	695	@field(result, field_name) = switch (field_type) {
696	696	u32 => self.extra[i],
	697	u64 => value: {
	698	const lo = @as(u64, self.extra[i]);
	699	const hi = @as(u64, self.extra[i + 1]) << 32;
	700	i += 1;
	701	break :value hi \| lo;
	702	},
697	703	i32 => @bitCast(self.extra[i]),
698	704	Wasm.UavsObjIndex,
699	705	Wasm.UavsExeIndex,
700	706	InternPool.Nav.Index,
701	707	InternPool.Index,
	708	std.mem.Alignment,
702	709	=> @fromBackingInt(@intCast(self.extra[i])),
703	710	else => @compileError("Unsupported field type " ++ @typeName(field_type)),
704	711	};
...	...	@@ -747,8 +754,8 @@ pub const Float64 = struct {
747	754	};
748	755
749	756	pub const MemArg = struct {
750		offset: u32,
751		alignment: u32,
	757	offset: u64,
	758	alignment: std.mem.Alignment,
752	759	};
753	760
754	761	pub const UavRefOff = struct {

src/codegen/wasm/assembly.zig+4-4

...	...	@@ -679,20 +679,20 @@ fn parseMemArg(cg: CodeGen, inst: []const u8, word_it: mem.TokenIterator(u8, .
679	679	const offset_str = oa[0];
680	680	const alignment_str = oa[1];
681	681
682		const offset = std.fmt.parseInt(i32, offset_str, 0) catch {
	682	const offset = std.fmt.parseInt(i64, offset_str, 0) catch {
683	683	return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});
684	684	};
685	685	const alignment_p2 = std.fmt.parseInt(u5, alignment_str, 0) catch {
686	686	return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{alignment_str});
687	687	};
688	688
689		return .{ .offset = @bitCast(offset), .alignment = @as(u32, 1) << alignment_p2 };
	689	return .{ .offset = @bitCast(offset), .alignment = @fromBackingInt(@intCast(alignment_p2)) };
690	690	} else {
691	691	const offset_str = mem_arg_str;
692		const offset = std.fmt.parseInt(i32, offset_str, 0) catch {
	692	const offset = std.fmt.parseInt(i64, offset_str, 0) catch {
693	693	return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});
694	694	};
695		return .{ .offset = @bitCast(offset), .alignment = 1 };
	695	return .{ .offset = @bitCast(offset), .alignment = .@"1" };
696	696	}
697	697	}
698	698