Commit d8147ba476217797d600b7e5f46f0a67a5b96188 | zig

author	Alex Rønne Petersen <alex@alexrp.com> 2026-09-17 00:09:47+02:00
committer	Alex Rønne Petersen <alex@alexrp.com> 2026-09-17 00:09:47+02:00
log	d8147ba476217797d600b7e5f46f0a67a5b96188
tree	3b72c66376a962ddf75ff319fecd8489633e8fb2
parent	39512c3e5547b411294369c8eec8093045f74d64
parent	5119f0a70de2565f22e1edc5d43d49b655c6f7a7

Merge pull request 'wasm: codegen improvements + better liveness handling' (#36846) from pavelverigo/zig:wasm-codegen-imporvements into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36846

4 files changed, 1080 insertions(+), 1036 deletions(-)

src
- codegen
  - wasm
    CodeGen.zig
    Emit.zig
    Mir.zig
    assembly.zig

src/codegen/wasm/CodeGen.zig+1057-1020

...	...	@@ -136,13 +136,10 @@ air: Air,
136	136	liveness: Air.Liveness,
137	137	gpa: mem.Allocator,
138	138	func_index: InternPool.Index,
139		/// Contains a list of current branches.
140		/// When we return from a branch, the branch will be popped from this list,
141		/// which means branches can only contain references from within its own branch,
142		/// or a branch higher (lower index) in the tree.
143		branches: std.ArrayList(Branch) = .empty,
	139	/// Contains a list of current branches. Stores LocalPool before entering branch.
	140	branches: std.ArrayList(LocalPool) = .empty,
144	141	/// Table to save `WValue`'s generated by an `Air.Inst`
145		// values: ValueTable,
	142	values: ValueTable = .empty,
146	143	/// Mapping from Air.Inst.Index to block ids
147	144	blocks: std.array_hash_map.Auto(Air.Inst.Index, struct {
148	145	label: u32,
...	...	@@ -150,10 +147,6 @@ blocks: std.array_hash_map.Auto(Air.Inst.Index, struct {
150	147	}) = .{},
151	148	/// Maps `loop` instructions to their label. `br` to here repeats the loop.
152	149	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, u32) = .empty,
153		/// The index the next local generated will have
154		/// NOTE: arguments share the index with locals therefore the first variable
155		/// will have the index that comes after the last argument's index
156		local_index: u32,
157	150	/// The index of the current argument.
158	151	/// Used to track which argument is being referenced in `airArg`.
159	152	arg_index: u32 = 0,
...	...	@@ -207,23 +200,8 @@ stack_size: u32 = 0,
207	200	/// However, local variables or the usage of `incoming_stack_alignment` in a `CallingConvention` can overwrite this default.
208	201	stack_alignment: Alignment = .@"16",
209	202
210		// For each individual Wasm valtype we store a seperate free list which
211		// allows us to re-use locals that are no longer used. e.g. a temporary local.
212		/// A list of indexes which represents a local of valtype `i32`.
213		/// It is illegal to store a non-i32 valtype in this list.
214		free_locals_i32: std.ArrayList(u32) = .empty,
215		/// A list of indexes which represents a local of valtype `i64`.
216		/// It is illegal to store a non-i64 valtype in this list.
217		free_locals_i64: std.ArrayList(u32) = .empty,
218		/// A list of indexes which represents a local of valtype `f32`.
219		/// It is illegal to store a non-f32 valtype in this list.
220		free_locals_f32: std.ArrayList(u32) = .empty,
221		/// A list of indexes which represents a local of valtype `f64`.
222		/// It is illegal to store a non-f64 valtype in this list.
223		free_locals_f64: std.ArrayList(u32) = .empty,
224		/// A list of indexes which represents a local of valtype `v127`.
225		/// It is illegal to store a non-v128 valtype in this list.
226		free_locals_v128: std.ArrayList(u32) = .empty,
	203	local_pool: LocalPool = .{},
	204	wasm_param_count: u32 = undefined,
227	205
228	206	/// When in debug mode, this tracks if no `finishAir` was missed.
229	207	/// Forgetting to call `finishAir` will cause the result to not be
...	...	@@ -239,13 +217,7 @@ const WValue = union(enum) {
239	217	none: void,
240	218	/// The value lives on top of the stack
241	219	stack: void,
242		/// Index of the local
243		local: struct {
244		/// Contains the index to the local
245		value: u32,
246		/// The amount of instructions referencing this `WValue`
247		references: u32,
248		},
	220	local: u32, // index
249	221	/// An immediate 32bit value
250	222	imm32: u32,
251	223	/// An immediate 64bit value
...	...	@@ -271,23 +243,16 @@ const WValue = union(enum) {
271	243	/// pointing to where the value lives.
272	244	stack_offset: struct {
273	245	/// Contains the actual value of the offset
274		value: u32,
275		/// The amount of instructions referencing this `WValue`
276		references: u32,
	246	value: u64,
277	247	},
278	248
279		/// Returns the offset from the bottom of the stack. This is useful when
280		/// we use the load or store instruction to ensure we retrieve the value
281		/// from the correct position, rather than the value that lives at the
282		/// bottom of the stack. For instances where `WValue` is not `stack_value`
283		/// this will return 0, which allows us to simply call this function for all
284		/// loads and stores without requiring checks everywhere.
285		fn offset(value: WValue) u32 {
286		switch (value) {
287		.stack_offset => \|stack_offset\| return stack_offset.value,
	249	/// Returns the offset from the bottom of the stack.
	250	fn memOffset(value: WValue) u64 {
	251	return switch (value) {
	252	.stack_offset => \|stack_offset\| stack_offset.value,
288	253	.dead => unreachable,
289		else => return 0,
290		}
	254	else => 0,
	255	};
291	256	}
292	257
293	258	/// Promotes a `WValue` to a local when given value is on top of the stack.
...	...	@@ -297,7 +262,7 @@ const WValue = union(enum) {
297	262	switch (value) {
298	263	.stack => {
299	264	const new_local = try gen.allocLocal(ty);
300		try gen.addLocal(.local_set, new_local.local.value);
	265	try gen.addLocal(.local_set, new_local.local);
301	266	return new_local;
302	267	},
303	268	else => return value,
...	...	@@ -306,23 +271,9 @@ const WValue = union(enum) {
306	271
307	272	/// Marks a local as no longer being referenced and essentially allows
308	273	/// us to re-use it somewhere else within the function.
309		/// The valtype of the local is deducted by using the index of the given `WValue`.
310		fn free(value: WValue, gen: CodeGen) void {
	274	fn free(value: WValue, cg: CodeGen) void {
311	275	if (value.* != .local) return;
312		const local_value = value.local.value;
313		const reserved = gen.args.len + @intFromBool(gen.return_value != .none) + @intFromBool(gen.varargs != .none);
314		if (local_value < reserved + 2) return; // reserved locals may never be re-used. Also accounts for 2 stack locals.
315
316		const index = local_value - reserved;
317		const valtype = gen.mir_locals.items[index];
318		switch (valtype) {
319		.i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead
320		.i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return,
321		.f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return,
322		.f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return,
323		.v128 => gen.free_locals_v128.append(gen.gpa, local_value) catch return,
324		}
325		log.debug("freed local ({d}) of type {}", .{ local_value, valtype });
	276	cg.releaseLocal(value.local);
326	277	value.* = .dead;
327	278	}
328	279	};
...	...	@@ -330,6 +281,77 @@ const WValue = union(enum) {
330	281	/// Hashmap to store generated `WValue` for each `Air.Inst.Ref`
331	282	const ValueTable = std.array_hash_map.Auto(Air.Inst.Ref, WValue);
332	283
	284	const LocalPool = struct {
	285	const OptionalIndex = enum(u32) {
	286	none = math.maxInt(u32),
	287	_,
	288
	289	fn init(index: u32) OptionalIndex {
	290	const result: OptionalIndex = @fromBackingInt(index);
	291	assert(result != .none);
	292	return result;
	293	}
	294
	295	fn unwrap(index: OptionalIndex) ?u32 {
	296	return if (index == .none) null else @backingInt(index);
	297	}
	298	};
	299
	300	const Local = union {
	301	ref_count: u32,
	302	next_free: OptionalIndex,
	303	};
	304
	305	locals: std.ArrayList(Local) = .empty,
	306	free_i32: OptionalIndex = .none,
	307	free_i64: OptionalIndex = .none,
	308	free_f32: OptionalIndex = .none,
	309	free_f64: OptionalIndex = .none,
	310	free_v128: OptionalIndex = .none,
	311
	312	fn deinit(pool: *LocalPool, gpa: Allocator) void {
	313	pool.locals.deinit(gpa);
	314	pool.* = undefined;
	315	}
	316
	317	fn clone(pool: *LocalPool, gpa: Allocator) Allocator.Error!LocalPool {
	318	return .{
	319	.locals = try pool.locals.clone(gpa),
	320	.free_i32 = pool.free_i32,
	321	.free_i64 = pool.free_i64,
	322	.free_f32 = pool.free_f32,
	323	.free_f64 = pool.free_f64,
	324	.free_v128 = pool.free_v128,
	325	};
	326	}
	327
	328	fn freeList(pool: LocalPool, valtype: std.wasm.Valtype) OptionalIndex {
	329	return switch (valtype) {
	330	.i32 => &pool.free_i32,
	331	.i64 => &pool.free_i64,
	332	.f32 => &pool.free_f32,
	333	.f64 => &pool.free_f64,
	334	.v128 => &pool.free_v128,
	335	};
	336	}
	337
	338	fn restore(snapshot: LocalPool, cg: CodeGen) void {
	339	const local_pool = &cg.local_pool;
	340	const old_local_count = snapshot.locals.items.len;
	341	assert(local_pool.locals.items.len >= old_local_count);
	342	@memcpy(local_pool.locals.items[0..old_local_count], snapshot.locals.items);
	343	local_pool.free_i32 = snapshot.free_i32;
	344	local_pool.free_i64 = snapshot.free_i64;
	345	local_pool.free_f32 = snapshot.free_f32;
	346	local_pool.free_f64 = snapshot.free_f64;
	347	local_pool.free_v128 = snapshot.free_v128;
	348
	349	for (old_local_count..local_pool.locals.items.len) \|index\| {
	350	cg.appendFreeLocal(cg.wasm_param_count + @as(u32, @intCast(index)));
	351	}
	352	}
	353	};
	354
333	355	const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
334	356
335	357	const InnerError = Error \|\| error{
...	...	@@ -343,14 +365,11 @@ pub fn deinit(cg: *CodeGen) void {
343	365	const gpa = cg.gpa;
344	366	for (cg.branches.items) \|*branch\| branch.deinit(gpa);
345	367	cg.branches.deinit(gpa);
	368	cg.values.deinit(gpa);
346	369	cg.blocks.deinit(gpa);
347	370	cg.loops.deinit(gpa);
348	371	cg.simd_immediates.deinit(gpa);
349		cg.free_locals_i32.deinit(gpa);
350		cg.free_locals_i64.deinit(gpa);
351		cg.free_locals_f32.deinit(gpa);
352		cg.free_locals_f64.deinit(gpa);
353		cg.free_locals_v128.deinit(gpa);
	372	cg.local_pool.deinit(gpa);
354	373	cg.mir_instructions.deinit(gpa);
355	374	cg.mir_extra.deinit(gpa);
356	375	cg.mir_locals.deinit(gpa);
...	...	@@ -369,19 +388,11 @@ pub fn fail(cg: *CodeGen, comptime fmt: []const u8, args: anytype) Error {
369	388	/// Resolves the `WValue` for the given instruction `inst`
370	389	/// When the given instruction has a `Value`, it returns a constant instead
371	390	fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue {
372		var branch_index = cg.branches.items.len;
373		while (branch_index > 0) : (branch_index -= 1) {
374		const branch = cg.branches.items[branch_index - 1];
375		if (branch.values.get(ref)) \|value\| {
376		return value;
377		}
378		}
	391	if (cg.values.get(ref)) \|value\| return value;
379	392
380	393	// when we did not find an existing instruction, it
381	394	// means we must generate it from a constant.
382		// We always store constants in the most outer branch as they must never
383		// be removed. The most outer branch is always at index 0.
384		const gop = try cg.branches.items[0].values.getOrPut(cg.gpa, ref);
	395	const gop = try cg.values.getOrPut(cg.gpa, ref);
385	396	assert(!gop.found_existing);
386	397
387	398	const pt = cg.pt;
...	...	@@ -420,7 +431,34 @@ fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue {
420	431
421	432	/// NOTE: if result == .stack, it will be stored in .local
422	433	fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void {
	434	return cg.finishAirWithBranchDeaths(inst, result, operands, &.{});
	435	}
	436
	437	/// Takes `branch_deaths` from continuing branch and calls `processDeath` after result retaining logic.
	438	fn finishAirWithBranchDeaths(
	439	cg: *CodeGen,
	440	inst: Air.Inst.Index,
	441	result: WValue,
	442	operands: []const Air.Inst.Ref,
	443	branch_deaths: []const Air.Inst.Index,
	444	) InnerError!void {
423	445	assert(operands.len <= Air.Liveness.bpi - 1);
	446
	447	if (result == .local) aliased: {
	448	for (operands) \|operand\| {
	449	if (operand.toIndex() == null) continue;
	450	const resolved_operand = cg.values.getPtr(operand) orelse unreachable;
	451	if (resolved_operand.* != .local or resolved_operand.local != result.local) continue;
	452
	453	if (result.local >= cg.wasm_param_count) {
	454	const local = &cg.local_pool.locals.items[result.local - cg.wasm_param_count];
	455	assert(local.ref_count > 0);
	456	local.ref_count += 1;
	457	}
	458	break :aliased;
	459	}
	460	}
	461
424	462	var tomb_bits = cg.liveness.getTombBits(inst);
425	463	for (operands) \|operand\| {
426	464	const dies = @as(u1, @truncate(tomb_bits)) != 0;
...	...	@@ -428,18 +466,27 @@ fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []con
428	466	if (!dies) continue;
429	467	processDeath(cg, operand);
430	468	}
	469	for (branch_deaths) \|death\| cg.processDeath(death.toRef());
431	470	try cg.finishAirResult(inst, result);
432	471	}
433	472
434	473	fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerError!void {
435	474	// results of `none` can never be referenced.
436	475	if (result != .none) {
437		const trackable_result = if (result != .stack)
438		result
439		else
440		try result.toLocal(cg, cg.typeOfIndex(inst));
441		const branch = cg.currentBranch();
442		branch.values.putAssumeCapacityNoClobber(inst.toRef(), trackable_result);
	476	if (cg.liveness.isUnused(inst)) {
	477	if (result == .stack) {
	478	try cg.addTag(.drop);
	479	} else {
	480	var unused_result = result;
	481	unused_result.free(cg);
	482	}
	483	} else {
	484	const trackable_result = if (result != .stack)
	485	result
	486	else
	487	try result.toLocal(cg, cg.typeOfIndex(inst));
	488	try cg.values.putNoClobber(cg.gpa, inst.toRef(), trackable_result);
	489	}
443	490	}
444	491
445	492	if (std.debug.runtime_safety) {
...	...	@@ -447,17 +494,16 @@ fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerErro
447	494	}
448	495	}
449	496
450		const Branch = struct {
451		values: ValueTable = .{},
452
453		fn deinit(branch: *Branch, gpa: Allocator) void {
454		branch.values.deinit(gpa);
455		branch.* = undefined;
456		}
457		};
	497	fn pushBranch(cg: *CodeGen) InnerError!void {
	498	var snapshot = try cg.local_pool.clone(cg.gpa);
	499	errdefer snapshot.deinit(cg.gpa);
	500	try cg.branches.append(cg.gpa, snapshot);
	501	}
458	502
459		inline fn currentBranch(cg: CodeGen) Branch {
460		return &cg.branches.items[cg.branches.items.len - 1];
	503	fn popBranch(cg: *CodeGen) void {
	504	var snapshot = cg.branches.pop().?;
	505	defer snapshot.deinit(cg.gpa);
	506	snapshot.restore(cg);
461	507	}
462	508
463	509	fn feed(cg: CodeGen, bt: Air.Liveness.BigTomb, operand: Air.Inst.Ref) void {
...	...	@@ -468,20 +514,30 @@ fn feed(cg: CodeGen, bt: Air.Liveness.BigTomb, operand: Air.Inst.Ref) void {
468	514
469	515	fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void {
470	516	if (ref.toIndex() == null) return;
471		// Branches are currently only allowed to free locals allocated
472		// within their own branch.
473		// TODO: Upon branch consolidation free any locals if needed.
474		const value = cg.currentBranch().values.getPtr(ref) orelse return;
	517	const value = cg.values.getPtr(ref) orelse unreachable;
475	518	if (value.* != .local) return;
476		const reserved_indexes = cg.args.len + @intFromBool(cg.return_value != .none);
477		if (value.local.value < reserved_indexes) {
478		return; // function arguments can never be re-used
479		}
480		log.debug("Decreasing reference for ref: %{d}, using local '{d}'", .{ @backingInt(ref.toIndex().?), value.local.value });
481		value.local.references -= 1; // if this panics, a call to `reuseOperand` was forgotten by the developer
482		if (value.local.references == 0) {
483		value.free(cg);
	519	cg.releaseLocal(value.local);
	520	}
	521
	522	fn verifyLocalPool(cg: *CodeGen) void {
	523	if (true) return; // TODO: test more
	524	if (!std.debug.runtime_safety) return;
	525
	526	var free_count: usize = 0;
	527	for ([_]std.wasm.Valtype{ .i32, .i64, .f32, .f64, .v128 }) \|valtype\| {
	528	var next = cg.local_pool.freeList(valtype).*;
	529	while (next.unwrap()) \|index\| {
	530	assert(index >= cg.wasm_param_count);
	531	const pool_index = index - cg.wasm_param_count;
	532	assert(pool_index < cg.local_pool.locals.items.len);
	533	assert(index != cg.initial_stack_value.local);
	534	assert(index != cg.bottom_stack_value.local);
	535	assert(cg.mir_locals.items[pool_index] == valtype);
	536	free_count += 1;
	537	next = cg.local_pool.locals.items[pool_index].next_free;
	538	}
484	539	}
	540	assert(free_count + 2 == cg.local_pool.locals.items.len);
485	541	}
486	542
487	543	pub fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void {
...	...	@@ -569,7 +625,7 @@ fn addCallIntrinsic(cg: *CodeGen, intrinsic: Mir.Intrinsic) error{OutOfMemory}!v
569	625	/// Returns the index into `mir_extra`
570	626	fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
571	627	const field_count = @typeInfo(@TypeOf(extra)).@"struct".field_names.len;
572		try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count);
	628	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count * 2);
573	629	return cg.addExtraAssumeCapacity(extra);
574	630	}
575	631
...	...	@@ -579,14 +635,22 @@ fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
579	635	const info = @typeInfo(@TypeOf(extra)).@"struct";
580	636	const result: u32 = @intCast(cg.mir_extra.items.len);
581	637	inline for (info.field_names, info.field_types) \|field_name, field_type\| {
582		cg.mir_extra.appendAssumeCapacity(switch (field_type) {
583		u32 => @field(extra, field_name),
584		i32 => @bitCast(@field(extra, field_name)),
585		InternPool.Index,
586		InternPool.Nav.Index,
587		=> @backingInt(@field(extra, field_name)),
588		else => @compileError("Unsupported field type " ++ @typeName(field_type)),
589		});
	638	switch (field_type) {
	639	u64 => {
	640	const value = @field(extra, field_name);
	641	cg.mir_extra.appendAssumeCapacity(@truncate(value));
	642	cg.mir_extra.appendAssumeCapacity(@truncate(value >> 32));
	643	},
	644	else => cg.mir_extra.appendAssumeCapacity(switch (field_type) {
	645	u32 => @field(extra, field_name),
	646	i32 => @bitCast(@field(extra, field_name)),
	647	InternPool.Index,
	648	InternPool.Nav.Index,
	649	Alignment,
	650	=> @backingInt(@field(extra, field_name)),
	651	else => @compileError("Unsupported field type " ++ @typeName(field_type)),
	652	}),
	653	}
590	654	}
591	655	return result;
592	656	}
...	...	@@ -622,18 +686,75 @@ pub fn typeToValtype(ty: Type, zcu: const Zcu, target: const std.Target) std.w
622	686	/// Differently from `typeToValtype` this also allows `void` to create a block
623	687	/// with no return type
624	688	fn genBlockType(ty: Type, zcu: const Zcu, target: const std.Target) std.wasm.BlockType {
625		return switch (ty.ip_index) {
626		.void_type, .noreturn_type => .empty,
627		else => .fromValtype(typeToValtype(ty, zcu, target)),
	689	if (!ty.hasRuntimeBits(zcu)) return .empty;
	690	return .fromValtype(typeToValtype(ty, zcu, target));
	691	}
	692
	693	/// Returns `ptr` with `offset` applied.
	694	fn offsetPointer(cg: *CodeGen, ptr: WValue, offset: u64) InnerError!WValue {
	695	if (offset == 0) return ptr;
	696	return switch (ptr) {
	697	.stack_offset => \|stack_offset\| .{ .stack_offset = .{
	698	.value = stack_offset.value + offset,
	699	} },
	700	else => result: {
	701	try cg.emitWValue(ptr);
	702	switch (cg.ptr_size) {
	703	.wasm32 => {
	704	try cg.addImm32(@intCast(offset));
	705	try cg.addTag(.i32_add);
	706	},
	707	.wasm64 => {
	708	try cg.addImm64(offset);
	709	try cg.addTag(.i64_add);
	710	},
	711	}
	712	break :result .stack;
	713	},
628	714	};
629	715	}
630	716
	717	const PointerOffsetOp = enum { add, sub };
	718
	719	/// The offset may only be known at runtime. Base pointer must be already on the stack. Result is left on it also.
	720	fn emitScaledPointerOffset(cg: *CodeGen, offset: WValue, scale: u64, op: PointerOffsetOp) InnerError!void {
	721	if (scale == 0 or switch (offset) {
	722	.imm32 => \|imm\| imm == 0,
	723	.imm64 => \|imm\| imm == 0,
	724	else => false,
	725	}) return;
	726
	727	try cg.emitWValue(offset);
	728	switch (cg.ptr_size) {
	729	.wasm32 => {
	730	if (scale != 1) {
	731	try cg.addImm32(@intCast(scale));
	732	try cg.addTag(.i32_mul);
	733	}
	734	try cg.addTag(switch (op) {
	735	.add => .i32_add,
	736	.sub => .i32_sub,
	737	});
	738	},
	739	.wasm64 => {
	740	if (scale != 1) {
	741	try cg.addImm64(scale);
	742	try cg.addTag(.i64_mul);
	743	}
	744	try cg.addTag(switch (op) {
	745	.add => .i64_add,
	746	.sub => .i64_sub,
	747	});
	748	},
	749	}
	750	}
	751
631	752	/// Writes the bytecode depending on the given `WValue` in `val`
632	753	fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
633	754	switch (value) {
634		.dead => unreachable, // reference to free'd `WValue` (missing reuseOperand?)
	755	.dead => unreachable, // reference to free'd `WValue`
635	756	.none, .stack => {}, // no-op
636		.local => \|idx\| try cg.addLocal(.local_get, idx.value),
	757	.local => \|idx\| try cg.addLocal(.local_get, idx),
637	758	.imm32 => \|val\| try cg.addImm32(val),
638	759	.imm64 => \|val\| try cg.addImm64(val),
639	760	.imm128 => \|val\| try cg.addImm128(val),
...	...	@@ -695,38 +816,38 @@ fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
695	816	});
696	817	}
697	818	},
698		.stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), // caller must ensure to address the offset
	819	.stack_offset => \|stack_offset\| {
	820	try cg.emitWValue(cg.bottom_stack_value);
	821	_ = try cg.offsetPointer(.stack, stack_offset.value);
	822	},
699	823	}
700	824	}
701	825
702		/// If given a local or stack-offset, increases the reference count by 1.
703		/// The old `WValue` found at instruction `ref` is then replaced by the
704		/// modified `WValue` and returned. When given a non-local or non-stack-offset,
705		/// returns the given `operand` itfunc instead.
706		fn reuseOperand(cg: *CodeGen, ref: Air.Inst.Ref, operand: WValue) WValue {
707		if (operand != .local and operand != .stack_offset) return operand;
708		var new_value = operand;
709		switch (new_value) {
710		.local => \|*local\| local.references += 1,
711		.stack_offset => \|*stack_offset\| stack_offset.references += 1,
712		else => unreachable,
	826	/// Emits the base address for a memory instruction.
	827	fn emitMemBase(cg: *CodeGen, ptr: WValue) InnerError!void {
	828	switch (ptr) {
	829	.stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local),
	830	else => try cg.emitWValue(ptr),
713	831	}
714		const old_value = cg.getResolvedInst(ref);
715		old_value.* = new_value;
716		return new_value;
717	832	}
718	833
719		/// From a reference, returns its resolved `WValue`.
720		/// It's illegal to provide a `Air.Inst.Ref` that hasn't been resolved yet.
721		fn getResolvedInst(cg: CodeGen, ref: Air.Inst.Ref) WValue {
722		var index = cg.branches.items.len;
723		while (index > 0) : (index -= 1) {
724		const branch = cg.branches.items[index - 1];
725		if (branch.values.getPtr(ref)) \|value\| {
726		return value;
727		}
728		}
729		unreachable; // developer-error: This can only be called on resolved instructions. Use `resolveInst` instead.
	834	fn appendFreeLocal(cg: *CodeGen, index: u32) void {
	835	assert(index >= cg.wasm_param_count);
	836	const mir_index = index - cg.wasm_param_count;
	837	const valtype = cg.mir_locals.items[mir_index];
	838	const free_list = cg.local_pool.freeList(valtype);
	839	cg.local_pool.locals.items[mir_index] = .{ .next_free = free_list.* };
	840	free_list.* = .init(index);
	841	log.debug("freed local ({d}) of type {}", .{ index, valtype });
	842	}
	843
	844	fn releaseLocal(cg: *CodeGen, index: u32) void {
	845	if (index < cg.wasm_param_count) return;
	846	const local = &cg.local_pool.locals.items[index - cg.wasm_param_count];
	847	assert(local.ref_count > 0);
	848	local.ref_count -= 1;
	849	if (local.ref_count > 0) return;
	850	cg.appendFreeLocal(index);
730	851	}
731	852
732	853	/// Creates one locals for a given `Type`.
...	...	@@ -734,16 +855,13 @@ fn getResolvedInst(cg: CodeGen, ref: Air.Inst.Ref) WValue {
734	855	fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
735	856	const zcu = cg.pt.zcu;
736	857	const valtype = typeToValtype(ty, zcu, cg.target);
737		const index_or_null = switch (valtype) {
738		.i32 => cg.free_locals_i32.pop(),
739		.i64 => cg.free_locals_i64.pop(),
740		.f32 => cg.free_locals_f32.pop(),
741		.f64 => cg.free_locals_f64.pop(),
742		.v128 => cg.free_locals_v128.pop(),
743		};
744		if (index_or_null) \|index\| {
	858	const free_list = cg.local_pool.freeList(valtype);
	859	if (free_list.unwrap()) \|index\| {
	860	const local = &cg.local_pool.locals.items[index - cg.wasm_param_count];
	861	free_list.* = local.next_free;
	862	local.* = .{ .ref_count = 1 };
745	863	log.debug("reusing local ({d}) of type {}", .{ index, valtype });
746		return .{ .local = .{ .value = index, .references = 1 } };
	864	return .{ .local = index };
747	865	}
748	866	log.debug("new local of type {}", .{valtype});
749	867	return cg.ensureAllocLocal(ty);
...	...	@@ -753,10 +871,13 @@ fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
753	871	/// to use a zero-initialized local.
754	872	fn ensureAllocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
755	873	const zcu = cg.pt.zcu;
756		try cg.mir_locals.append(cg.gpa, typeToValtype(ty, zcu, cg.target));
757		const initial_index = cg.local_index;
758		cg.local_index += 1;
759		return .{ .local = .{ .value = initial_index, .references = 1 } };
	874	try cg.mir_locals.ensureUnusedCapacity(cg.gpa, 1);
	875	try cg.local_pool.locals.ensureUnusedCapacity(cg.gpa, 1);
	876	assert(cg.mir_locals.items.len == cg.local_pool.locals.items.len);
	877	const initial_index = cg.wasm_param_count + @as(u32, @intCast(cg.local_pool.locals.items.len));
	878	cg.mir_locals.appendAssumeCapacity(typeToValtype(ty, zcu, cg.target));
	879	cg.local_pool.locals.appendAssumeCapacity(.{ .ref_count = 1 });
	880	return .{ .local = initial_index };
760	881	}
761	882
762	883	pub const Error = codegen.Error;
...	...	@@ -798,7 +919,6 @@ pub fn generate(
798	919	.args = cc_result.args,
799	920	.return_value = cc_result.return_value,
800	921	.varargs = cc_result.varargs,
801		.local_index = cc_result.local_index,
802	922	.mir_instructions = .empty,
803	923	.mir_extra = .empty,
804	924	.mir_locals = .empty,
...	...	@@ -821,17 +941,10 @@ pub fn generate(
821	941
822	942	fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir {
823	943	const zcu = cg.pt.zcu;
824		// branch used for const values
825		try cg.branches.append(cg.gpa, .{});
826		// func scope branch
827		try cg.branches.append(cg.gpa, .{});
828		defer {
829		var func_branch = cg.branches.pop().?;
830		func_branch.deinit(cg.gpa);
831		var const_branch = cg.branches.pop().?;
832		const_branch.deinit(cg.gpa);
833		assert(cg.branches.items.len == 0); // missing branch merge
834		}
	944	cg.wasm_param_count = @intCast(cg.args.len + @intFromBool(cg.return_value != .none) + @intFromBool(cg.varargs != .none));
	945	cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize);
	946	cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize);
	947	defer assert(cg.branches.items.len == 0);
835	948	// Generate MIR for function body
836	949	try cg.genBody(cg.air.getMainBody());
837	950
...	...	@@ -856,11 +969,11 @@ fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir {
856	969	.instructions = cg.mir_instructions.toOwnedSlice(),
857	970	.extra = cg.mir_extra.toOwnedSliceAssert(),
858	971	.locals = cg.mir_locals.toOwnedSliceAssert(),
859		.prologue = if (cg.initial_stack_value == .none) .none else .{
860		.sp_local = cg.initial_stack_value.local.value,
	972	.prologue = .{
	973	.sp_local = cg.initial_stack_value.local,
861	974	.flags = .{ .stack_alignment = cg.stack_alignment },
862	975	.stack_size = cg.stack_size,
863		.bottom_stack_local = cg.bottom_stack_value.local.value,
	976	.bottom_stack_local = cg.bottom_stack_value.local,
864	977	},
865	978	.uavs = cg.mir_uavs.move(),
866	979	.indirect_function_set = cg.mir_indirect_function_set.move(),
...	...	@@ -907,7 +1020,7 @@ fn resolveCallingConventionValues(
907	1020	if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, target)) {
908	1021	// the sret arg will be passed as first argument, therefore we
909	1022	// set the `return_value` before allocating locals for regular args.
910		result.return_value = .{ .local = .{ .value = result.local_index, .references = 1 } };
	1023	result.return_value = .{ .local = result.local_index };
911	1024	result.local_index += 1;
912	1025	}
913	1026
...	...	@@ -918,7 +1031,7 @@ fn resolveCallingConventionValues(
918	1031	continue;
919	1032	}
920	1033
921		try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
	1034	try args.append(.{ .local = result.local_index });
922	1035	result.local_index += 1;
923	1036	}
924	1037	},
...	...	@@ -931,17 +1044,17 @@ fn resolveCallingConventionValues(
931	1044
932	1045	switch (abi.classifyType(param_ty, zcu, target)) {
933	1046	.direct, .indirect => {
934		try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
	1047	try args.append(.{ .local = result.local_index });
935	1048	result.local_index += 1;
936	1049	},
937	1050	.double_i64 => {
938		try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
939		try args.append(.{ .local = .{ .value = result.local_index + 1, .references = 1 } });
	1051	try args.append(.{ .local = result.local_index });
	1052	try args.append(.{ .local = result.local_index + 1 });
940	1053	result.local_index += 2;
941	1054	},
942	1055	.unrolled => \|vector\| {
943	1056	for (0..vector.len) \|_\| {
944		try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
	1057	try args.append(.{ .local = result.local_index });
945	1058	result.local_index += 1;
946	1059	}
947	1060	},
...	...	@@ -952,7 +1065,7 @@ fn resolveCallingConventionValues(
952	1065	}
953	1066
954	1067	if (fn_info.is_var_args) {
955		result.varargs = .{ .local = .{ .value = result.local_index, .references = 1 } };
	1068	result.varargs = .{ .local = result.local_index };
956	1069	result.local_index += 1;
957	1070	}
958	1071
...	...	@@ -983,7 +1096,7 @@ pub fn firstParamSRet(
983	1096	/// it matches the ABI.
984	1097	fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValue) !void {
985	1098	if (cc != .wasm_mvp) {
986		return cg.lowerToStack(value);
	1099	return cg.emitWValue(value);
987	1100	}
988	1101
989	1102	const zcu = cg.pt.zcu;
...	...	@@ -991,74 +1104,37 @@ fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValu
991	1104	switch (abi.classifyType(ty, zcu, cg.target)) {
992	1105	.direct => \|scalar_ty\| {
993	1106	if (!isByRef(ty, zcu, cg.target)) {
994		return cg.lowerToStack(value);
	1107	return cg.emitWValue(value);
995	1108	} else {
996		_ = try cg.load(value, scalar_ty, 0);
	1109	_ = try cg.load(value, scalar_ty, .{});
997	1110	}
998	1111	},
999	1112	.double_i64 => {
1000	1113	assert(ty.abiSize(zcu) == 16);
1001	1114	// in this case we have an integer or float that must be lowered as 2 i64's.
1002		try cg.emitWValue(value);
1003		try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = 8 });
1004		try cg.emitWValue(value);
1005		try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = 8 });
	1115	const base_offset = value.memOffset();
	1116	try cg.emitMemBase(value);
	1117	try cg.addMemArg(.i64_load, .{ .offset = base_offset, .alignment = .@"8" });
	1118	try cg.emitMemBase(value);
	1119	try cg.addMemArg(.i64_load, .{ .offset = base_offset + 8, .alignment = .@"8" });
1006	1120	},
1007	1121	.indirect => {
1008	1122	const stack_copy = try cg.allocStack(ty);
1009		try cg.store(stack_copy, value, ty, 0);
1010		return cg.lowerToStack(stack_copy);
	1123	try cg.store(stack_copy, value, ty, .{});
	1124	return cg.emitWValue(stack_copy);
1011	1125	},
1012	1126	.unrolled => \|vector\| {
1013		const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));
	1127	const elem_size = vector.elem_type.abiSize(zcu);
1014	1128	for (0..vector.len) \|index\| {
1015		_ = try cg.load(value, vector.elem_type, @intCast(index * elem_size));
1016		}
1017		},
1018		}
1019		}
1020
1021		/// Lowers a `WValue` to the stack. This means when the `value` results in
1022		/// `.stack_offset` we calculate the pointer of this offset and use that.
1023		/// The value is left on the stack, and not stored in any temporary.
1024		fn lowerToStack(cg: *CodeGen, value: WValue) !void {
1025		switch (value) {
1026		.stack_offset => \|offset\| {
1027		try cg.emitWValue(value);
1028		if (offset.value > 0) {
1029		switch (cg.ptr_size) {
1030		.wasm32 => {
1031		try cg.addImm32(offset.value);
1032		try cg.addTag(.i32_add);
1033		},
1034		.wasm64 => {
1035		try cg.addImm64(offset.value);
1036		try cg.addTag(.i64_add);
1037		},
1038		}
	1129	_ = try cg.load(value, vector.elem_type, .{ .offset = index * elem_size });
1039	1130	}
1040	1131	},
1041		else => try cg.emitWValue(value),
1042	1132	}
1043	1133	}
1044	1134
1045		/// Creates a local for the initial stack value
1046		/// Asserts `initial_stack_value` is `.none`
1047		fn initializeStack(cg: *CodeGen) !void {
1048		assert(cg.initial_stack_value == .none);
1049		// Reserve a local to store the current stack pointer
1050		// We can later use this local to set the stack pointer back to the value
1051		// we have stored here.
1052		cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize);
1053		// Also reserve a local to store the bottom stack value
1054		cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize);
1055		}
1056
1057	1135	/// Reads the stack pointer from `Context.initial_stack_value` and writes it
1058	1136	/// to the global stack pointer variable
1059	1137	fn restoreStackPointer(cg: *CodeGen) !void {
1060		// only restore the pointer if it was initialized
1061		if (cg.initial_stack_value == .none) return;
1062	1138	// Get the original stack pointer's value
1063	1139	try cg.emitWValue(cg.initial_stack_value);
1064	1140
...	...	@@ -1098,16 +1174,12 @@ fn allocInt(cg: *CodeGen, int_ty: IntType) !WValue {
1098	1174	fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue {
1099	1175	assert(size > 0);
1100	1176
1101		if (cg.initial_stack_value == .none) {
1102		try cg.initializeStack();
1103		}
1104
1105	1177	cg.stack_alignment = cg.stack_alignment.max(alignment);
1106	1178
1107	1179	const offset: u32 = @intCast(alignment.forward(cg.stack_size));
1108	1180	defer cg.stack_size = offset + size;
1109	1181
1110		return .{ .stack_offset = .{ .value = offset, .references = 1 } };
	1182	return .{ .stack_offset = .{ .value = offset } };
1111	1183	}
1112	1184
1113	1185	/// From a given AIR instruction generates a pointer to the stack where
...	...	@@ -1120,10 +1192,6 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {
1120	1192	const ptr_ty = cg.typeOfIndex(inst);
1121	1193	const pointee_ty = ptr_ty.childType(zcu);
1122	1194
1123		if (cg.initial_stack_value == .none) {
1124		try cg.initializeStack();
1125		}
1126
1127	1195	const abi_alignment = ptr_ty.ptrAlignment(zcu);
1128	1196	const abi_size = std.math.cast(u32, pointee_ty.abiSize(zcu)) orelse {
1129	1197	return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{
...	...	@@ -1135,7 +1203,7 @@ fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {
1135	1203	const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size));
1136	1204	defer cg.stack_size = offset + abi_size;
1137	1205
1138		return .{ .stack_offset = .{ .value = offset, .references = 1 } };
	1206	return .{ .stack_offset = .{ .value = offset } };
1139	1207	}
1140	1208
1141	1209	fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
...	...	@@ -1165,8 +1233,8 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1165	1233	try cg.addLabel(.br_if, 0);
1166	1234	}
1167	1235
1168		try cg.lowerToStack(dst);
1169		try cg.lowerToStack(src);
	1236	try cg.emitWValue(dst);
	1237	try cg.emitWValue(src);
1170	1238	try cg.emitWValue(len);
1171	1239	try cg.addExtended(.memory_copy);
1172	1240
...	...	@@ -1176,26 +1244,26 @@ fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1176	1244	}
1177	1245
1178	1246	fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1179		if (cg.target.cpu.has(.wasm, .bulk_memory)) {
	1247	if (cg.target.cpu.has(.wasm, .bulk_memory_opt)) {
1180	1248	try cg.emitMemoryCopy(dst, src, len);
1181	1249	return;
1182	1250	}
1183	1251
1184		try cg.lowerToStack(dst);
1185		try cg.lowerToStack(src);
	1252	try cg.emitWValue(dst);
	1253	try cg.emitWValue(src);
1186	1254	try cg.emitWValue(len);
1187	1255	try cg.addCallIntrinsic(.memcpy);
1188	1256	try cg.addTag(.drop);
1189	1257	}
1190	1258
1191	1259	fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1192		if (cg.target.cpu.has(.wasm, .bulk_memory)) {
	1260	if (cg.target.cpu.has(.wasm, .bulk_memory_opt)) {
1193	1261	try cg.emitMemoryCopy(dst, src, len);
1194	1262	return;
1195	1263	}
1196	1264
1197		try cg.lowerToStack(dst);
1198		try cg.lowerToStack(src);
	1265	try cg.emitWValue(dst);
	1266	try cg.emitWValue(src);
1199	1267	try cg.emitWValue(len);
1200	1268	try cg.addCallIntrinsic(.memmove);
1201	1269	try cg.addTag(.drop);
...	...	@@ -1277,34 +1345,6 @@ pub fn determineSimdStoreStrategy(ty: Type, zcu: const Zcu, target: const std.
1277	1345	return .unrolled;
1278	1346	}
1279	1347
1280		/// Creates a new local for a pointer that points to memory with given offset.
1281		/// This can be used to get a pointer to a struct field, error payload, etc.
1282		/// By providing `modify` as action, it will modify the given `ptr_value` instead of making a new
1283		/// local value to store the pointer. This allows for local re-use and improves binary size.
1284		fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum { modify, new }) InnerError!WValue {
1285		// do not perform arithmetic when offset is 0.
1286		if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value;
1287		const result_ptr: WValue = switch (action) {
1288		.new => try cg.ensureAllocLocal(Type.usize),
1289		.modify => ptr_value,
1290		};
1291		try cg.emitWValue(ptr_value);
1292		if (offset + ptr_value.offset() > 0) {
1293		switch (cg.ptr_size) {
1294		.wasm32 => {
1295		try cg.addImm32(@intCast(offset + ptr_value.offset()));
1296		try cg.addTag(.i32_add);
1297		},
1298		.wasm64 => {
1299		try cg.addImm64(offset + ptr_value.offset());
1300		try cg.addTag(.i64_add);
1301		},
1302		}
1303		}
1304		try cg.addLocal(.local_set, result_ptr.local.value);
1305		return result_ptr;
1306		}
1307
1308	1348	fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1309	1349	const zcu = cg.pt.zcu;
1310	1350	const air_tags = cg.air.instructions.items(.tag);
...	...	@@ -1325,14 +1365,12 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1325	1365	const elem_ty = cg.typeOf(bin_op.rhs);
1326	1366	const elem_size = elem_ty.abiSize(zcu);
1327	1367
1328		try cg.lowerToStack(vec_ptr);
1329		try cg.emitWValue(elem_idx);
1330		try cg.addImm32(@intCast(elem_size));
1331		try cg.addTag(.i32_mul);
1332		try cg.addTag(.i32_add);
1333		const ptr = try WValue.toLocal(.stack, cg, Type.usize);
	1368	try cg.emitWValue(vec_ptr);
	1369	try cg.emitScaledPointerOffset(elem_idx, elem_size, .add);
	1370	var ptr = try WValue.toLocal(.stack, cg, Type.usize);
	1371	defer ptr.free(cg);
1334	1372
1335		try cg.store(ptr, elem_val, elem_ty, 0);
	1373	try cg.store(ptr, elem_val, elem_ty, .{});
1336	1374
1337	1375	return cg.finishAir(inst, .none, &.{ pl_op.operand, bin_op.lhs, bin_op.rhs });
1338	1376	},
...	...	@@ -1509,8 +1547,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1509	1547	const result = try cg.allocStack(cg.typeOfIndex(inst));
1510	1548	const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu));
1511	1549
1512		try cg.store(result, res_tmp, ty, 0);
1513		try cg.store(result, ov_tmp, Type.u1, offset);
	1550	try cg.store(result, res_tmp, ty, .{});
	1551	try cg.store(result, ov_tmp, .u1, .{ .offset = offset });
1514	1552
1515	1553	try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });
1516	1554	},
...	...	@@ -1598,17 +1636,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1598	1636	const src_int_ty: IntType = .fromType(cg, src_ty);
1599	1637	const dest_int_ty: IntType = .fromType(cg, dest_ty);
1600	1638
1601		const src_bits = src_int_ty.bits;
1602		const dest_bits = dest_int_ty.bits;
1603
1604		const same_class: bool = (src_bits <= 32 and dest_bits <= 32) or
1605		(src_bits >= 33 and src_bits <= 64 and dest_bits >= 33 and dest_bits <= 64) or
1606		(src_bits >= 65 and src_bits <= 128 and dest_bits >= 65 and dest_bits <= 128);
1607
1608		const result = if (same_class)
1609		cg.reuseOperand(ty_op.operand, operand)
1610		else
1611		try cg.intCast(dest_int_ty, src_int_ty, operand);
	1639	const result = try cg.intCast(dest_int_ty, src_int_ty, operand);
1612	1640
1613	1641	try cg.finishAir(inst, result, &.{ty_op.operand});
1614	1642	},
...	...	@@ -1625,12 +1653,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1625	1653
1626	1654	const src_int_ty: IntType = .fromType(cg, src_ty);
1627	1655	const dest_int_ty: IntType = .fromType(cg, dest_ty);
1628
1629		const result = if (src_int_ty.bits == dest_int_ty.bits)
1630		cg.reuseOperand(ty_op.operand, operand)
1631		else blk: {
1632		break :blk try cg.intTrunc(dest_int_ty, src_int_ty, operand);
1633		};
	1656	const result = try cg.intTrunc(dest_int_ty, src_int_ty, operand);
1634	1657
1635	1658	try cg.finishAir(inst, result, &.{ty_op.operand});
1636	1659	},
...	...	@@ -1934,7 +1957,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1934	1957	continue;
1935	1958	}
1936	1959	const old_bookkeeping_value = cg.air_bookkeeping;
1937		try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, 1);
1938	1960	try cg.genInst(inst);
1939	1961
1940	1962	if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) {
...	...	@@ -1956,20 +1978,20 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1956	1978	// result must be stored in the stack and we return a pointer
1957	1979	// to the stack instead
1958	1980	if (cg.return_value != .none) {
1959		try cg.store(cg.return_value, operand, ret_ty, 0);
	1981	try cg.store(cg.return_value, operand, ret_ty, .{});
1960	1982	} else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBits(zcu)) {
1961	1983	switch (abi.classifyType(ret_ty, zcu, cg.target)) {
1962	1984	.direct => \|scalar_type\| {
1963	1985	if (!isByRef(ret_ty, zcu, cg.target)) {
1964	1986	try cg.emitWValue(operand);
1965	1987	} else {
1966		_ = try cg.load(operand, scalar_type, 0);
	1988	_ = try cg.load(operand, scalar_type, .{});
1967	1989	}
1968	1990	},
1969	1991	.double_i64, .indirect => unreachable,
1970	1992	.unrolled => \|vector\| {
1971	1993	assert(vector.len == 1);
1972		_ = try cg.load(operand, vector.elem_type, 0);
	1994	_ = try cg.load(operand, vector.elem_type, .{});
1973	1995	},
1974	1996	}
1975	1997	} else {
...	...	@@ -1982,7 +2004,8 @@ fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1982	2004	try cg.restoreStackPointer();
1983	2005	try cg.addTag(.@"return");
1984	2006
1985		return cg.finishAir(inst, .none, &.{un_op});
	2007	try cg.finishAir(inst, .none, &.{un_op});
	2008	cg.verifyLocalPool();
1986	2009	}
1987	2010
1988	2011	fn airRetPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -2019,21 +2042,22 @@ fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2019	2042	} else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {
2020	2043	if (fn_info.cc == .wasm_mvp) {
2021	2044	switch (abi.classifyType(ret_ty, zcu, cg.target)) {
2022		.direct => \|scalar_type\| _ = try cg.load(operand, scalar_type, 0),
	2045	.direct => \|scalar_type\| _ = try cg.load(operand, scalar_type, .{}),
2023	2046	.double_i64, .indirect => unreachable,
2024	2047	.unrolled => \|vector\| {
2025	2048	assert(vector.len == 1);
2026		_ = try cg.load(operand, vector.elem_type, 0);
	2049	_ = try cg.load(operand, vector.elem_type, .{});
2027	2050	},
2028	2051	}
2029	2052	} else {
2030		_ = try cg.load(operand, ret_ty, 0);
	2053	_ = try cg.load(operand, ret_ty, .{});
2031	2054	}
2032	2055	}
2033	2056
2034	2057	try cg.restoreStackPointer();
2035	2058	try cg.addTag(.@"return");
2036		return cg.finishAir(inst, .none, &.{un_op});
	2059	try cg.finishAir(inst, .none, &.{un_op});
	2060	cg.verifyLocalPool();
2037	2061	}
2038	2062
2039	2063	fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) InnerError!void {
...	...	@@ -2109,7 +2133,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2109	2133	const arg_align = arg_ty.abiAlignment(zcu);
2110	2134
2111	2135	offset = @intCast(arg_align.forward(offset));
2112		try cg.store(buffer, arg_val, arg_ty, offset);
	2136	try cg.store(buffer, arg_val, arg_ty, .{ .offset = offset });
2113	2137	offset += arg_size;
2114	2138	}
2115	2139
...	...	@@ -2117,7 +2141,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2117	2141	} else .none;
2118	2142
2119	2143	if (first_param_sret) {
2120		try cg.lowerToStack(sret);
	2144	try cg.emitWValue(sret);
2121	2145	}
2122	2146
2123	2147	for (args, 0..) \|arg, arg_i\| {
...	...	@@ -2131,7 +2155,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2131	2155	}
2132	2156
2133	2157	if (fn_info.is_var_args) {
2134		try cg.lowerToStack(varargs_buf);
	2158	try cg.emitWValue(varargs_buf);
2135	2159	}
2136	2160
2137	2161	if (callee) \|nav_index\| {
...	...	@@ -2160,30 +2184,32 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier)
2160	2184	.direct => \|scalar_type\| {
2161	2185	if (!isByRef(ret_ty, zcu, cg.target)) {
2162	2186	const result_local = try cg.allocLocal(ret_ty);
2163		try cg.addLocal(.local_set, result_local.local.value);
	2187	try cg.addLocal(.local_set, result_local.local);
2164	2188	break :result_value result_local;
2165	2189	} else {
2166		const result_local = try cg.allocLocal(scalar_type);
2167		try cg.addLocal(.local_set, result_local.local.value);
	2190	var result_local = try cg.allocLocal(scalar_type);
	2191	defer result_local.free(cg);
	2192	try cg.addLocal(.local_set, result_local.local);
2168	2193	const result = try cg.allocStack(ret_ty);
2169		try cg.store(result, result_local, scalar_type, 0);
	2194	try cg.store(result, result_local, scalar_type, .{});
2170	2195	break :result_value result;
2171	2196	}
2172	2197	},
2173	2198	.double_i64, .indirect => unreachable,
2174	2199	.unrolled => \|vector\| {
2175	2200	assert(vector.len == 1);
2176		const result_local = try cg.allocLocal(vector.elem_type);
	2201	var result_local = try cg.allocLocal(vector.elem_type);
	2202	defer result_local.free(cg);
2177	2203	// save call result from operand stack
2178		try cg.addLocal(.local_set, result_local.local.value);
	2204	try cg.addLocal(.local_set, result_local.local);
2179	2205	const result = try cg.allocStack(ret_ty);
2180		try cg.store(result, result_local, vector.elem_type, 0);
	2206	try cg.store(result, result_local, vector.elem_type, .{});
2181	2207	break :result_value result;
2182	2208	},
2183	2209	}
2184	2210	} else {
2185	2211	const result_local = try cg.allocLocal(ret_ty);
2186		try cg.addLocal(.local_set, result_local.local.value);
	2212	try cg.addLocal(.local_set, result_local.local);
2187	2213	break :result_value result_local;
2188	2214	}
2189	2215	};
...	...	@@ -2208,7 +2234,7 @@ fn airVaCopy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2208	2234	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2209	2235	const operand = try cg.resolveInst(ty_op.operand);
2210	2236
2211		const result = try cg.load(operand, .usize, 0);
	2237	const result = try cg.load(operand, .usize, .{});
2212	2238
2213	2239	return cg.finishAir(inst, result, &.{ty_op.operand});
2214	2240	}
...	...	@@ -2230,8 +2256,9 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2230	2256	const abi_size: u32 = @intCast(load_ty.abiSize(zcu));
2231	2257	const abi_align: u32 = @intCast(load_ty.abiAlignment(zcu).toByteUnits().?);
2232	2258
2233		const arg_ptr = try cg.allocLocal(.usize);
2234		_ = try cg.load(operand, .usize, 0);
	2259	var arg_ptr = try cg.allocLocal(.usize);
	2260	defer arg_ptr.free(cg);
	2261	_ = try cg.load(operand, .usize, .{});
2235	2262
2236	2263	if (abi_align > 1) {
2237	2264	switch (cg.ptr_size) {
...	...	@@ -2250,10 +2277,10 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2250	2277	}
2251	2278	}
2252	2279
2253		try cg.addLocal(.local_set, arg_ptr.local.value);
	2280	try cg.addLocal(.local_set, arg_ptr.local);
2254	2281
2255		try cg.lowerToStack(operand);
2256		try cg.lowerToStack(arg_ptr);
	2282	try cg.emitWValue(operand);
	2283	try cg.emitWValue(arg_ptr);
2257	2284	switch (cg.ptr_size) {
2258	2285	.wasm32 => {
2259	2286	try cg.addImm32(abi_size);
...	...	@@ -2264,17 +2291,17 @@ fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2264	2291	try cg.addTag(.i64_add);
2265	2292	},
2266	2293	}
2267		try cg.store(.stack, .stack, .usize, 0);
	2294	try cg.store(.stack, .stack, .usize, .{});
2268	2295
2269	2296	const result = if (is_f32_va_arg) result: {
2270		const promoted = try cg.load(arg_ptr, Type.f64, 0);
	2297	const promoted = try cg.load(arg_ptr, .f64, .{});
2271	2298	try cg.emitWValue(promoted);
2272	2299	try cg.addTag(.f32_demote_f64);
2273	2300
2274	2301	const result_local = try cg.allocLocal(Type.f32);
2275		try cg.addLocal(.local_set, result_local.local.value);
	2302	try cg.addLocal(.local_set, result_local.local);
2276	2303	break :result result_local;
2277		} else try cg.load(arg_ptr, ty, 0);
	2304	} else try cg.load(arg_ptr, ty, .{});
2278	2305
2279	2306	return cg.finishAir(inst, result, &.{ty_op.operand});
2280	2307	}
...	...	@@ -2290,27 +2317,44 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2290	2317	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2291	2318
2292	2319	const lhs = try cg.resolveInst(bin_op.lhs);
2293		const rhs = try cg.resolveInst(bin_op.rhs);
2294	2320	const ptr_ty = cg.typeOf(bin_op.lhs);
2295	2321	const ptr_info = ptr_ty.ptrInfo(zcu);
2296	2322	const elem_ty = ptr_ty.childType(zcu);
	2323	const rhs_is_undef = if (bin_op.rhs.toInterned()) \|i\| Value.fromInterned(i).isUndef(zcu) else false;
2297	2324
2298		if (!safety and bin_op.rhs == .undef) {
	2325	if (!safety and rhs_is_undef) {
2299	2326	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2300	2327	}
2301	2328
2302		const offset: u32 = switch (ptr_info.flags.vector_index) {
	2329	const rhs = try cg.resolveInst(bin_op.rhs);
	2330
	2331	const offset: u64 = switch (ptr_info.flags.vector_index) {
2303	2332	.none => offset: {
2304	2333	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
2305	2334	break :offset 0;
2306	2335	},
2307		else => \|index\| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),
	2336	else => \|index\| @backingInt(index) * elem_ty.abiSize(zcu),
2308	2337	};
2309		try cg.store(lhs, rhs, elem_ty, offset);
	2338
	2339	if (safety and rhs_is_undef and isByRef(elem_ty, zcu, cg.target)) {
	2340	const dst = try cg.offsetPointer(lhs, offset);
	2341	try cg.memset(.u8, dst, .{ .imm32 = @intCast(elem_ty.abiSize(zcu)) }, .{ .imm32 = 0xaa });
	2342	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
	2343	}
	2344
	2345	try cg.store(lhs, rhs, elem_ty, .{
	2346	.offset = offset,
	2347	.alignment = ptr_ty.ptrAlignment(zcu),
	2348	});
2310	2349	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2311	2350	}
2312	2351
2313		fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void {
	2352	const MemArg = struct {
	2353	offset: u64 = 0,
	2354	alignment: Alignment = .none,
	2355	};
	2356
	2357	fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, memarg: MemArg) InnerError!void {
2314	2358	assert(!(lhs != .stack and rhs == .stack));
2315	2359	const pt = cg.pt;
2316	2360	const zcu = pt.zcu;
...	...	@@ -2318,29 +2362,31 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2318	2362
2319	2363	if (!ty.hasRuntimeBits(zcu)) return;
2320	2364
	2365	const natural_alignment = ty.abiAlignment(zcu);
	2366	// wasm memory instruction alignment is <= access width
	2367	const alignment = if (memarg.alignment == .none)
	2368	natural_alignment
	2369	else
	2370	natural_alignment.minStrict(memarg.alignment);
	2371
2321	2372	if (isByRef(ty, zcu, cg.target)) {
2322		const offset_ptr: WValue = switch (offset + lhs.offset()) {
2323		0 => lhs,
2324		else => \|total_offset\| ptr: {
2325		try cg.emitWValue(lhs);
2326		try cg.addImm32(total_offset);
2327		try cg.addTag(.i32_add);
2328		break :ptr .stack;
2329		},
2330		};
	2373	const offset_ptr = try cg.offsetPointer(lhs, memarg.offset);
2331	2374	return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
2332	2375	}
2333	2376
	2377	const base_offset = lhs.memOffset();
	2378	try cg.emitMemBase(lhs);
	2379
2334	2380	if (ty.zigTypeTag(zcu) == .vector) {
2335		try cg.emitWValue(lhs);
2336		try cg.lowerToStack(rhs);
	2381	try cg.emitWValue(rhs);
2337	2382	// TODO: Add helper functions for simd opcodes
2338	2383	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2339	2384	// stores as := opcode, offset, alignment (opcode::memarg)
2340		try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2341		@backingInt(std.wasm.SimdOpcode.v128_store),
2342		offset + lhs.offset(),
2343		@intCast(ty.abiAlignment(zcu).toByteUnits().?),
	2385	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
	2386	cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_store));
	2387	_ = try cg.addExtraAssumeCapacity(Mir.MemArg{
	2388	.offset = base_offset + memarg.offset,
	2389	.alignment = alignment,
2344	2390	});
2345	2391	return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2346	2392	}
...	...	@@ -2364,14 +2410,13 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2364	2410	}
2365	2411	};
2366	2412
2367		try cg.emitWValue(lhs);
2368		try cg.lowerToStack(rhs);
	2413	try cg.emitWValue(rhs);
2369	2414
2370	2415	try cg.addMemArg(
2371	2416	store_opcode,
2372	2417	.{
2373		.offset = offset + lhs.offset(),
2374		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	2418	.offset = base_offset + memarg.offset,
	2419	.alignment = alignment,
2375	2420	},
2376	2421	);
2377	2422	}
...	...	@@ -2387,48 +2432,55 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2387	2432
2388	2433	assert(elem_ty.hasRuntimeBits(zcu));
2389	2434
2390		const offset: u32 = switch (ptr_info.flags.vector_index) {
	2435	const offset: u64 = switch (ptr_info.flags.vector_index) {
2391	2436	.none => offset: {
2392	2437	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
2393	2438	break :offset 0;
2394	2439	},
2395		else => \|index\| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),
	2440	else => \|index\| @backingInt(index) * elem_ty.abiSize(zcu),
2396	2441	};
2397		const result = try cg.load(operand, elem_ty, offset);
	2442	const result = try cg.load(operand, elem_ty, .{
	2443	.offset = offset,
	2444	.alignment = ptr_ty.ptrAlignment(zcu),
	2445	});
2398	2446	return cg.finishAir(inst, result, &.{ty_op.operand});
2399	2447	}
2400	2448
2401	2449	/// Loads an operand from the linear memory section.
2402	2450	/// NOTE: Leaves the value on the stack, if isByRef == false.
2403		fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
	2451	fn load(cg: *CodeGen, operand: WValue, ty: Type, memarg: MemArg) InnerError!WValue {
2404	2452	const zcu = cg.pt.zcu;
	2453
	2454	const natural_alignment = ty.abiAlignment(zcu);
	2455	// wasm memory instruction alignment is <= access width
	2456	const alignment = if (memarg.alignment == .none)
	2457	natural_alignment
	2458	else
	2459	natural_alignment.minStrict(memarg.alignment);
	2460
2405	2461	if (isByRef(ty, zcu, cg.target)) {
2406		const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {
2407		0 => operand,
2408		else => \|total_offset\| ptr: {
2409		try cg.emitWValue(operand);
2410		try cg.addImm32(total_offset);
2411		try cg.addTag(.i32_add);
2412		break :ptr .stack;
2413		},
2414		};
2415		const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
	2462	var src_ptr = try cg.offsetPointer(operand, memarg.offset);
	2463	const owns_src_ptr = src_ptr == .stack;
	2464	src_ptr = try src_ptr.toLocal(cg, .usize);
	2465	defer if (owns_src_ptr) src_ptr.free(cg);
	2466
2416	2467	const new_ptr = try cg.allocStack(ty);
2417		try cg.store(new_ptr, src_ptr, ty, 0);
	2468	try cg.memcpy(new_ptr, src_ptr, .{ .imm32 = @intCast(ty.abiSize(zcu)) });
2418	2469	return new_ptr;
2419	2470	}
2420	2471
2421		// load local's value from memory by its stack position
2422		try cg.emitWValue(operand);
	2472	const base_offset = operand.memOffset();
	2473	try cg.emitMemBase(operand);
2423	2474
2424	2475	if (ty.zigTypeTag(zcu) == .vector) {
2425	2476	// TODO: Add helper functions for simd opcodes
2426	2477	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2427	2478	// stores as := opcode, offset, alignment (opcode::memarg)
2428		try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2429		@backingInt(std.wasm.SimdOpcode.v128_load),
2430		offset + operand.offset(),
2431		@intCast(ty.abiAlignment(zcu).toByteUnits().?),
	2479	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
	2480	cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_load));
	2481	_ = try cg.addExtraAssumeCapacity(Mir.MemArg{
	2482	.offset = base_offset + memarg.offset,
	2483	.alignment = alignment,
2432	2484	});
2433	2485	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2434	2486	return .stack;
...	...	@@ -2458,8 +2510,8 @@ fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue
2458	2510	try cg.addMemArg(
2459	2511	load_opcode,
2460	2512	.{
2461		.offset = offset + operand.offset(),
2462		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	2513	.offset = base_offset + memarg.offset,
	2514	.alignment = alignment,
2463	2515	},
2464	2516	);
2465	2517
...	...	@@ -2487,7 +2539,7 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2487	2539	cg.arg_index += 1;
2488	2540	if (isByRef(arg_ty, zcu, cg.target)) {
2489	2541	const result = try cg.allocStack(arg_ty);
2490		try cg.store(result, arg, scalar_type, 0);
	2542	try cg.store(result, arg, scalar_type, .{});
2491	2543	return cg.finishAir(inst, result, &.{});
2492	2544	}
2493	2545	},
...	...	@@ -2495,15 +2547,15 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2495	2547	.double_i64 => {
2496	2548	cg.arg_index += 2;
2497	2549	const result = try cg.allocStack(arg_ty);
2498		try cg.store(result, arg, Type.u64, 0);
2499		try cg.store(result, cg.args[arg_index + 1], Type.u64, 8);
	2550	try cg.store(result, arg, .u64, .{});
	2551	try cg.store(result, cg.args[arg_index + 1], .u64, .{ .offset = 8 });
2500	2552	return cg.finishAir(inst, result, &.{});
2501	2553	},
2502	2554	.unrolled => \|vector\| {
2503	2555	const result = try cg.allocStack(arg_ty);
2504		const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));
	2556	const elem_size = vector.elem_type.abiSize(zcu);
2505	2557	for (0..vector.len) \|index\| {
2506		try cg.store(result, cg.args[cg.arg_index], vector.elem_type, @intCast(index * elem_size));
	2558	try cg.store(result, cg.args[cg.arg_index], vector.elem_type, .{ .offset = index * elem_size });
2507	2559	cg.arg_index += 1;
2508	2560	}
2509	2561	return cg.finishAir(inst, result, &.{});
...	...	@@ -2605,15 +2657,15 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2605	2657	65...128 => {
2606	2658	const result = try cg.allocStack(Type.u128);
2607	2659
2608		var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
	2660	var lhs_lsb = try (try cg.load(lhs, .u64, .{})).toLocal(cg, Type.u64);
2609	2661	defer lhs_lsb.free(cg);
2610		var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
	2662	var rhs_lsb = try (try cg.load(rhs, .u64, .{})).toLocal(cg, Type.u64);
2611	2663	defer rhs_lsb.free(cg);
2612	2664	var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2613	2665	defer op_lsb.free(cg);
2614	2666
2615		const lhs_msb = try cg.load(lhs, Type.u64, 8);
2616		const rhs_msb = try cg.load(rhs, Type.u64, 8);
	2667	const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
	2668	const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
2617	2669	const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb);
2618	2670
2619	2671	const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb);
...	...	@@ -2621,16 +2673,16 @@ fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2621	2673	var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2622	2674	defer tmp_op.free(cg);
2623	2675
2624		try cg.store(result, op_lsb, Type.u64, 0);
2625		try cg.store(result, tmp_op, Type.u64, 8);
	2676	try cg.store(result, op_lsb, .u64, .{});
	2677	try cg.store(result, tmp_op, .u64, .{ .offset = 8 });
2626	2678	return result;
2627	2679	},
2628	2680	else => {
2629	2681	const result = try cg.allocInt(ty);
2630	2682
2631		try cg.lowerToStack(result);
2632		try cg.lowerToStack(lhs);
2633		try cg.lowerToStack(rhs);
	2683	try cg.emitWValue(result);
	2684	try cg.emitWValue(lhs);
	2685	try cg.emitWValue(rhs);
2634	2686	try cg.addImm32(@intFromBool(ty.is_signed));
2635	2687	try cg.addImm32(ty.bits);
2636	2688	try cg.addCallIntrinsic(.__addo_limb64);
...	...	@@ -2658,15 +2710,15 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2658	2710	65...128 => {
2659	2711	const result = try cg.allocStack(Type.u128);
2660	2712
2661		var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
	2713	var lhs_lsb = try (try cg.load(lhs, .u64, .{})).toLocal(cg, Type.u64);
2662	2714	defer lhs_lsb.free(cg);
2663		var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
	2715	var rhs_lsb = try (try cg.load(rhs, .u64, .{})).toLocal(cg, Type.u64);
2664	2716	defer rhs_lsb.free(cg);
2665	2717	var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2666	2718	defer op_lsb.free(cg);
2667	2719
2668		const lhs_msb = try cg.load(lhs, Type.u64, 8);
2669		const rhs_msb = try cg.load(rhs, Type.u64, 8);
	2720	const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
	2721	const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
2670	2722	const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb);
2671	2723
2672	2724	const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb);
...	...	@@ -2674,16 +2726,16 @@ fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2674	2726	var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2675	2727	defer tmp_op.free(cg);
2676	2728
2677		try cg.store(result, op_lsb, Type.u64, 0);
2678		try cg.store(result, tmp_op, Type.u64, 8);
	2729	try cg.store(result, op_lsb, .u64, .{});
	2730	try cg.store(result, tmp_op, .u64, .{ .offset = 8 });
2679	2731	return result;
2680	2732	},
2681	2733	else => {
2682	2734	const result = try cg.allocInt(ty);
2683	2735
2684		try cg.lowerToStack(result);
2685		try cg.lowerToStack(lhs);
2686		try cg.lowerToStack(rhs);
	2736	try cg.emitWValue(result);
	2737	try cg.emitWValue(lhs);
	2738	try cg.emitWValue(rhs);
2687	2739	try cg.addImm32(@intFromBool(ty.is_signed));
2688	2740	try cg.addImm32(ty.bits);
2689	2741	try cg.addCallIntrinsic(.__subo_limb64);
...	...	@@ -2713,9 +2765,9 @@ fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
2713	2765	else => {
2714	2766	const result = try cg.allocInt(ty);
2715	2767
2716		try cg.lowerToStack(result);
2717		try cg.lowerToStack(lhs);
2718		try cg.lowerToStack(rhs);
	2768	try cg.emitWValue(result);
	2769	try cg.emitWValue(lhs);
	2770	try cg.emitWValue(rhs);
2719	2771	try cg.addImm32(@intFromBool(ty.is_signed));
2720	2772	try cg.addImm32(ty.bits);
2721	2773	try cg.addCallIntrinsic(.__mulo_limb64);
...	...	@@ -3007,24 +3059,26 @@ fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3007	3059	65...128 => {
3008	3060	const result = try cg.allocStack(Type.u128);
3009	3061
3010		const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3011		const rhs_lsb = try cg.load(rhs, Type.u64, 0);
3012		const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3013		try cg.store(result, and_lsb, Type.u64, 0);
	3062	const lhs_lsb = try cg.load(lhs, .u64, .{});
	3063	const rhs_lsb = try cg.load(rhs, .u64, .{});
	3064	var and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
	3065	defer and_lsb.free(cg);
	3066	try cg.store(result, and_lsb, .u64, .{});
3014	3067
3015		const lhs_msb = try cg.load(lhs, Type.u64, 8);
3016		const rhs_msb = try cg.load(rhs, Type.u64, 8);
3017		const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3018		try cg.store(result, and_msb, Type.u64, 8);
	3068	const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
	3069	const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
	3070	var and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
	3071	defer and_msb.free(cg);
	3072	try cg.store(result, and_msb, .u64, .{ .offset = 8 });
3019	3073
3020	3074	return result;
3021	3075	},
3022	3076	else => {
3023	3077	const result = try cg.allocInt(ty);
3024	3078
3025		try cg.lowerToStack(result);
3026		try cg.lowerToStack(lhs);
3027		try cg.lowerToStack(rhs);
	3079	try cg.emitWValue(result);
	3080	try cg.emitWValue(lhs);
	3081	try cg.emitWValue(rhs);
3028	3082	try cg.addImm32(ty.bits);
3029	3083	try cg.addCallIntrinsic(.__and_limb64);
3030	3084
...	...	@@ -3051,24 +3105,26 @@ fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3051	3105	65...128 => {
3052	3106	const result = try cg.allocStack(Type.u128);
3053	3107
3054		const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3055		const rhs_lsb = try cg.load(rhs, Type.u64, 0);
3056		const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3057		try cg.store(result, or_lsb, Type.u64, 0);
	3108	const lhs_lsb = try cg.load(lhs, .u64, .{});
	3109	const rhs_lsb = try cg.load(rhs, .u64, .{});
	3110	var or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
	3111	defer or_lsb.free(cg);
	3112	try cg.store(result, or_lsb, .u64, .{});
3058	3113
3059		const lhs_msb = try cg.load(lhs, Type.u64, 8);
3060		const rhs_msb = try cg.load(rhs, Type.u64, 8);
3061		const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3062		try cg.store(result, or_msb, Type.u64, 8);
	3114	const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
	3115	const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
	3116	var or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
	3117	defer or_msb.free(cg);
	3118	try cg.store(result, or_msb, .u64, .{ .offset = 8 });
3063	3119
3064	3120	return result;
3065	3121	},
3066	3122	else => {
3067	3123	const result = try cg.allocInt(ty);
3068	3124
3069		try cg.lowerToStack(result);
3070		try cg.lowerToStack(lhs);
3071		try cg.lowerToStack(rhs);
	3125	try cg.emitWValue(result);
	3126	try cg.emitWValue(lhs);
	3127	try cg.emitWValue(rhs);
3072	3128	try cg.addImm32(ty.bits);
3073	3129	try cg.addCallIntrinsic(.__or_limb64);
3074	3130
...	...	@@ -3095,24 +3151,26 @@ fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3095	3151	65...128 => {
3096	3152	const result = try cg.allocStack(Type.u128);
3097	3153
3098		const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3099		const rhs_lsb = try cg.load(rhs, Type.u64, 0);
3100		const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3101		try cg.store(result, xor_lsb, Type.u64, 0);
	3154	const lhs_lsb = try cg.load(lhs, .u64, .{});
	3155	const rhs_lsb = try cg.load(rhs, .u64, .{});
	3156	var xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
	3157	defer xor_lsb.free(cg);
	3158	try cg.store(result, xor_lsb, .u64, .{});
3102	3159
3103		const lhs_msb = try cg.load(lhs, Type.u64, 8);
3104		const rhs_msb = try cg.load(rhs, Type.u64, 8);
3105		const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3106		try cg.store(result, xor_msb, Type.u64, 8);
	3160	const lhs_msb = try cg.load(lhs, .u64, .{ .offset = 8 });
	3161	const rhs_msb = try cg.load(rhs, .u64, .{ .offset = 8 });
	3162	var xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
	3163	defer xor_msb.free(cg);
	3164	try cg.store(result, xor_msb, .u64, .{ .offset = 8 });
3107	3165
3108	3166	return result;
3109	3167	},
3110	3168	else => {
3111	3169	const result = try cg.allocInt(ty);
3112	3170
3113		try cg.lowerToStack(result);
3114		try cg.lowerToStack(lhs);
3115		try cg.lowerToStack(rhs);
	3171	try cg.emitWValue(result);
	3172	try cg.emitWValue(lhs);
	3173	try cg.emitWValue(rhs);
3116	3174	try cg.addImm32(ty.bits);
3117	3175	try cg.addCallIntrinsic(.__xor_limb64);
3118	3176
...	...	@@ -3157,29 +3215,30 @@ fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3157	3215	65...128 => {
3158	3216	const result = try cg.allocStack(Type.u128);
3159	3217
3160		try cg.emitWValue(result);
3161		_ = try cg.load(operand, Type.u64, 0);
	3218	const result_offset = result.memOffset();
	3219	try cg.emitMemBase(result);
	3220	_ = try cg.load(operand, .u64, .{});
3162	3221	try cg.addImm64(~@as(u64, 0));
3163	3222	try cg.addTag(.i64_xor);
3164		try cg.store(.stack, .stack, Type.u64, result.offset());
	3223	try cg.store(.stack, .stack, .u64, .{ .offset = result_offset });
3165	3224
3166		try cg.emitWValue(result);
3167		_ = try cg.load(operand, Type.u64, 8);
	3225	try cg.emitMemBase(result);
	3226	_ = try cg.load(operand, .u64, .{ .offset = 8 });
3168	3227	const high_mask: u64 = if (ty.is_signed)
3169	3228	~@as(u64, 0)
3170	3229	else
3171	3230	~@as(u64, 0) >> @intCast(128 - ty.bits);
3172	3231	try cg.addImm64(high_mask);
3173	3232	try cg.addTag(.i64_xor);
3174		try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
	3233	try cg.store(.stack, .stack, .u64, .{ .offset = result_offset + 8 });
3175	3234
3176	3235	return result;
3177	3236	},
3178	3237	else => {
3179	3238	const result = try cg.allocInt(ty);
3180	3239
3181		try cg.lowerToStack(result);
3182		try cg.lowerToStack(operand);
	3240	try cg.emitWValue(result);
	3241	try cg.emitWValue(operand);
3183	3242	try cg.addImm32(@intFromBool(ty.is_signed));
3184	3243	try cg.addImm32(ty.bits);
3185	3244	try cg.addCallIntrinsic(.__not_limb64);
...	...	@@ -3211,9 +3270,9 @@ fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3211	3270	else => {
3212	3271	const result = try cg.allocInt(ty);
3213	3272
3214		try cg.lowerToStack(result);
3215		try cg.lowerToStack(lhs);
3216		try cg.lowerToStack(rhs);
	3273	try cg.emitWValue(result);
	3274	try cg.emitWValue(lhs);
	3275	try cg.emitWValue(rhs);
3217	3276	try cg.addImm32(@intFromBool(ty.is_signed));
3218	3277	try cg.addImm32(ty.bits);
3219	3278	try cg.addCallIntrinsic(.__shlo_limb64);
...	...	@@ -3251,9 +3310,9 @@ fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue
3251	3310	else => {
3252	3311	const result = try cg.allocInt(ty);
3253	3312
3254		try cg.lowerToStack(result);
3255		try cg.lowerToStack(lhs);
3256		try cg.lowerToStack(rhs);
	3313	try cg.emitWValue(result);
	3314	try cg.emitWValue(lhs);
	3315	try cg.emitWValue(rhs);
3257	3316	try cg.addImm32(@intFromBool(ty.is_signed));
3258	3317	try cg.addImm32(ty.bits);
3259	3318	try cg.addCallIntrinsic(.__shr_limb64);
...	...	@@ -3274,7 +3333,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3274	3333
3275	3334	var mask = try cg.allocLocal(Type.i32);
3276	3335	defer mask.free(cg);
3277		try cg.addLocal(.local_tee, mask.local.value);
	3336	try cg.addLocal(.local_tee, mask.local);
3278	3337
3279	3338	try cg.emitWValue(operand);
3280	3339	try cg.addTag(.i32_xor);
...	...	@@ -3289,7 +3348,7 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3289	3348
3290	3349	var mask = try cg.allocLocal(Type.i64);
3291	3350	defer mask.free(cg);
3292		try cg.addLocal(.local_tee, mask.local.value);
	3351	try cg.addLocal(.local_tee, mask.local);
3293	3352
3294	3353	try cg.emitWValue(operand);
3295	3354	try cg.addTag(.i64_xor);
...	...	@@ -3301,19 +3360,20 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3301	3360	const u128_ty: IntType = .{ .is_signed = false, .bits = 128 };
3302	3361
3303	3362	const mask = try cg.allocStack(Type.u128);
3304		try cg.emitWValue(mask);
3305		try cg.emitWValue(mask);
	3363	const mask_offset = mask.memOffset();
	3364	try cg.emitMemBase(mask);
	3365	try cg.emitMemBase(mask);
3306	3366
3307		_ = try cg.load(operand, Type.u64, 8);
	3367	_ = try cg.load(operand, .u64, .{ .offset = 8 });
3308	3368	try cg.addImm64(63);
3309	3369	try cg.addTag(.i64_shr_s);
3310	3370
3311	3371	var tmp = try cg.allocLocal(Type.u64);
3312	3372	defer tmp.free(cg);
3313		try cg.addLocal(.local_tee, tmp.local.value);
3314		try cg.store(.stack, .stack, Type.u64, mask.offset() + 0);
	3373	try cg.addLocal(.local_tee, tmp.local);
	3374	try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset });
3315	3375	try cg.emitWValue(tmp);
3316		try cg.store(.stack, .stack, Type.u64, mask.offset() + 8);
	3376	try cg.store(.stack, .stack, .u64, .{ .offset = mask_offset + 8 });
3317	3377
3318	3378	const a = try cg.intXor(u128_ty, operand, mask);
3319	3379	const b = try cg.intSub(u128_ty, a, mask);
...	...	@@ -3322,8 +3382,8 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3322	3382	else => {
3323	3383	const result = try cg.allocInt(ty);
3324	3384
3325		try cg.lowerToStack(result);
3326		try cg.lowerToStack(operand);
	3385	try cg.emitWValue(result);
	3386	try cg.emitWValue(operand);
3327	3387	try cg.addImm32(ty.bits);
3328	3388	try cg.addCallIntrinsic(.__abs_limb64);
3329	3389
...	...	@@ -3333,16 +3393,16 @@ fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3333	3393	}
3334	3394
3335	3395	fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3336		try cg.lowerToStack(lhs);
3337		try cg.lowerToStack(rhs);
	3396	try cg.emitWValue(lhs);
	3397	try cg.emitWValue(rhs);
3338	3398	_ = try cg.intCmp(ty, .gt, lhs, rhs);
3339	3399	try cg.addTag(.select);
3340	3400	return .stack;
3341	3401	}
3342	3402
3343	3403	fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3344		try cg.lowerToStack(lhs);
3345		try cg.lowerToStack(rhs);
	3404	try cg.emitWValue(lhs);
	3405	try cg.emitWValue(rhs);
3346	3406	_ = try cg.intCmp(ty, .lt, lhs, rhs);
3347	3407	try cg.addTag(.select);
3348	3408	return .stack;
...	...	@@ -3381,7 +3441,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3381	3441	return .stack;
3382	3442	},
3383	3443	65...128 => {
3384		var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64);
	3444	var msb = try (try cg.load(operand, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
3385	3445	defer msb.free(cg);
3386	3446
3387	3447	if (ty.is_signed and ty.bits < 128) {
...	...	@@ -3392,7 +3452,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3392	3452	}
3393	3453
3394	3454	try cg.addTag(.i64_clz);
3395		_ = try cg.load(operand, Type.u64, 0);
	3455	_ = try cg.load(operand, .u64, .{});
3396	3456	try cg.addTag(.i64_clz);
3397	3457	try cg.emitWValue(.{ .imm64 = 64 });
3398	3458	try cg.addTag(.i64_add);
...	...	@@ -3408,7 +3468,7 @@ fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3408	3468	return .stack;
3409	3469	},
3410	3470	else => {
3411		try cg.lowerToStack(operand);
	3471	try cg.emitWValue(operand);
3412	3472	try cg.addImm32(ty.bits);
3413	3473	try cg.addCallIntrinsic(.__clz_limb64);
3414	3474
...	...	@@ -3440,13 +3500,13 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3440	3500	return .stack;
3441	3501	},
3442	3502	65...128 => {
3443		var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64);
	3503	var lsb = try (try cg.load(operand, .u64, .{})).toLocal(cg, Type.u64);
3444	3504	defer lsb.free(cg);
3445	3505
3446	3506	try cg.emitWValue(lsb);
3447	3507	try cg.addTag(.i64_ctz);
3448	3508
3449		_ = try cg.load(operand, Type.u64, 8);
	3509	_ = try cg.load(operand, .u64, .{ .offset = 8 });
3450	3510	if (ty.bits < 128) {
3451	3511	try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64));
3452	3512	try cg.addTag(.i64_or);
...	...	@@ -3460,7 +3520,7 @@ fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3460	3520	return .stack;
3461	3521	},
3462	3522	else => {
3463		try cg.lowerToStack(operand);
	3523	try cg.emitWValue(operand);
3464	3524	try cg.addImm32(ty.bits);
3465	3525	try cg.addCallIntrinsic(.__ctz_limb64);
3466	3526
...	...	@@ -3492,9 +3552,9 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3492	3552	return .stack;
3493	3553	},
3494	3554	65...128 => {
3495		_ = try cg.load(operand, Type.u64, 0);
	3555	_ = try cg.load(operand, .u64, .{});
3496	3556	try cg.addTag(.i64_popcnt);
3497		_ = try cg.load(operand, Type.u64, 8);
	3557	_ = try cg.load(operand, .u64, .{ .offset = 8 });
3498	3558	if (ty.is_signed and ty.bits < 128) {
3499	3559	try cg.addImm64(128 - ty.bits);
3500	3560	try cg.addTag(.i64_shl);
...	...	@@ -3506,7 +3566,7 @@ fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3506	3566	return .stack;
3507	3567	},
3508	3568	else => {
3509		try cg.lowerToStack(operand);
	3569	try cg.emitWValue(operand);
3510	3570	try cg.addImm32(ty.bits);
3511	3571	try cg.addCallIntrinsic(.__popcount_limb64);
3512	3572
...	...	@@ -3541,8 +3601,9 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3541	3601	65...128 => {
3542	3602	const tmp = try cg.allocStack(Type.u128);
3543	3603
3544		try cg.emitWValue(tmp);
3545		const hi = try cg.load(operand, Type.u64, 8);
	3604	const tmp_offset = tmp.memOffset();
	3605	try cg.emitMemBase(tmp);
	3606	const hi = try cg.load(operand, .u64, .{ .offset = 8 });
3546	3607	const hi_rev = try cg.callIntrinsic(
3547	3608	.__bitreversedi2,
3548	3609	&.{.u64_type},
...	...	@@ -3550,10 +3611,10 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3550	3611	&.{hi},
3551	3612	);
3552	3613	try cg.emitWValue(hi_rev);
3553		try cg.store(.stack, .stack, Type.u64, tmp.offset());
	3614	try cg.store(.stack, .stack, .u64, .{ .offset = tmp_offset });
3554	3615
3555		try cg.emitWValue(tmp);
3556		const lo = try cg.load(operand, Type.u64, 0);
	3616	try cg.emitMemBase(tmp);
	3617	const lo = try cg.load(operand, .u64, .{});
3557	3618	const lo_rev = try cg.callIntrinsic(
3558	3619	.__bitreversedi2,
3559	3620	&.{.u64_type},
...	...	@@ -3561,7 +3622,7 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3561	3622	&.{lo},
3562	3623	);
3563	3624	try cg.emitWValue(lo_rev);
3564		try cg.store(.stack, .stack, Type.u64, tmp.offset() + 8);
	3625	try cg.store(.stack, .stack, .u64, .{ .offset = tmp_offset + 8 });
3565	3626
3566	3627	if (ty.bits < 128) {
3567	3628	const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
...	...	@@ -3573,8 +3634,8 @@ fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3573	3634	else => {
3574	3635	const result = try cg.allocInt(ty);
3575	3636
3576		try cg.lowerToStack(result);
3577		try cg.lowerToStack(operand);
	3637	try cg.emitWValue(result);
	3638	try cg.emitWValue(operand);
3578	3639	try cg.addImm32(@intFromBool(ty.is_signed));
3579	3640	try cg.addImm32(ty.bits);
3580	3641	try cg.addCallIntrinsic(.__bitreverse_limb64);
...	...	@@ -3610,27 +3671,28 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3610	3671	65...128 => {
3611	3672	const result = try cg.allocStack(Type.u128);
3612	3673
3613		try cg.emitWValue(result);
	3674	const result_offset = result.memOffset();
	3675	try cg.emitMemBase(result);
3614	3676
3615		const low = try cg.load(operand, Type.u64, 0);
	3677	const low = try cg.load(operand, .u64, .{});
3616	3678	const swap_low = try cg.callIntrinsic(
3617	3679	.__bswapdi2,
3618	3680	&.{.u64_type},
3619	3681	Type.u64,
3620	3682	&.{low},
3621	3683	);
3622		try cg.store(.stack, swap_low, Type.u64, result.offset() + 8);
	3684	try cg.store(.stack, swap_low, .u64, .{ .offset = result_offset + 8 });
3623	3685
3624		try cg.emitWValue(result);
	3686	try cg.emitMemBase(result);
3625	3687
3626		const high = try cg.load(operand, Type.u64, 8);
	3688	const high = try cg.load(operand, .u64, .{ .offset = 8 });
3627	3689	const swap_high = try cg.callIntrinsic(
3628	3690	.__bswapdi2,
3629	3691	&.{.u64_type},
3630	3692	Type.u64,
3631	3693	&.{high},
3632	3694	);
3633		try cg.store(.stack, swap_high, Type.u64, result.offset());
	3695	try cg.store(.stack, swap_high, .u64, .{ .offset = result_offset });
3634	3696
3635	3697	if (ty.bits < 128) {
3636	3698	const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
...	...	@@ -3642,8 +3704,8 @@ fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3642	3704	else => {
3643	3705	const result = try cg.allocInt(ty);
3644	3706
3645		try cg.lowerToStack(result);
3646		try cg.lowerToStack(operand);
	3707	try cg.emitWValue(result);
	3708	try cg.emitWValue(operand);
3647	3709	try cg.addImm32(@intFromBool(ty.is_signed));
3648	3710	try cg.addImm32(ty.bits);
3649	3711	try cg.addCallIntrinsic(.__byteswap_limb64);
...	...	@@ -3687,12 +3749,13 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3687	3749	65...127 => {
3688	3750	const result = try cg.allocStack(Type.u128);
3689	3751
3690		try cg.emitWValue(result);
3691		_ = try cg.load(operand, Type.u64, 0);
3692		try cg.store(.stack, .stack, Type.u64, result.offset());
	3752	const result_offset = result.memOffset();
	3753	try cg.emitMemBase(result);
	3754	_ = try cg.load(operand, .u64, .{});
	3755	try cg.store(.stack, .stack, .u64, .{ .offset = result_offset });
3693	3756
3694		try cg.emitWValue(result);
3695		_ = try cg.load(operand, Type.u64, 8);
	3757	try cg.emitMemBase(result);
	3758	_ = try cg.load(operand, .u64, .{ .offset = 8 });
3696	3759	if (ty.is_signed) {
3697	3760	try cg.addImm64(128 - ty.bits);
3698	3761	try cg.addTag(.i64_shl);
...	...	@@ -3702,7 +3765,7 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3702	3765	try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));
3703	3766	try cg.addTag(.i64_and);
3704	3767	}
3705		try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
	3768	try cg.store(.stack, .stack, .u64, .{ .offset = result_offset + 8 });
3706	3769
3707	3770	return result;
3708	3771	},
...	...	@@ -3719,8 +3782,8 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3719	3782	try cg.memcpy(result, operand, .{ .imm32 = used_len - 8 });
3720	3783	const pad = 64 - ty.bits % 64;
3721	3784
3722		try cg.emitWValue(result);
3723		_ = try cg.load(operand, Type.u64, used_len - 8);
	3785	try cg.emitMemBase(result);
	3786	_ = try cg.load(operand, .u64, .{ .offset = used_len - 8 });
3724	3787	if (ty.is_signed) {
3725	3788	try cg.addImm64(pad);
3726	3789	try cg.addTag(.i64_shl);
...	...	@@ -3730,22 +3793,22 @@ fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3730	3793	try cg.addImm64(~@as(u64, 0) >> @intCast(pad));
3731	3794	try cg.addTag(.i64_and);
3732	3795	}
3733		try cg.store(.stack, .stack, Type.u64, result.offset() + used_len - 8);
	3796	try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() + used_len - 8 });
3734	3797	} else {
3735	3798	try cg.memcpy(result, operand, .{ .imm32 = used_len });
3736	3799	}
3737	3800
3738	3801	const full_len = @divExact(bits, 8);
3739	3802	if (used_len + 8 == full_len) { // last limb needs sign extended
3740		try cg.emitWValue(result);
	3803	try cg.emitMemBase(result);
3741	3804	if (ty.is_signed) {
3742		_ = try cg.load(result, Type.u64, used_len - 8);
	3805	_ = try cg.load(result, .u64, .{ .offset = used_len - 8 });
3743	3806	try cg.addImm64(63);
3744	3807	try cg.addTag(.i64_shr_s);
3745	3808	} else {
3746	3809	try cg.addImm64(0);
3747	3810	}
3748		try cg.store(.stack, .stack, Type.u64, result.offset() + used_len);
	3811	try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() + used_len });
3749	3812	}
3750	3813
3751	3814	return result;
...	...	@@ -3768,12 +3831,12 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3768	3831	}
3769	3832	} else if (int_ty.bits <= 128) {
3770	3833	const result = try cg.allocInt(int_ty);
3771		try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, 0);
	3834	try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, .u64, .{});
3772	3835
3773	3836	if (int_ty.is_signed) {
3774		try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, Type.u64, 8);
	3837	try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, .u64, .{ .offset = 8 });
3775	3838	} else {
3776		try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, Type.u64, 8);
	3839	try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, .u64, .{ .offset = 8 });
3777	3840	}
3778	3841	return result;
3779	3842	} else {
...	...	@@ -3784,13 +3847,13 @@ fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3784	3847	try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0xFF });
3785	3848
3786	3849	if (int_ty.is_signed) {
3787		try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, Type.u64, used_len - 8);
	3850	try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, .u64, .{ .offset = used_len - 8 });
3788	3851	} else {
3789		try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, Type.u64, used_len - 8);
	3852	try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, .u64, .{ .offset = used_len - 8 });
3790	3853	}
3791	3854
3792	3855	if (used_len + 8 == full_len) {
3793		try cg.store(result, .{ .imm64 = 0 }, Type.u64, full_len - 8);
	3856	try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = full_len - 8 });
3794	3857	}
3795	3858
3796	3859	return result;
...	...	@@ -3807,8 +3870,8 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3807	3870	return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) };
3808	3871	} else if (int_ty.bits <= 128) {
3809	3872	const result = try cg.allocInt(int_ty);
3810		try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
3811		try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, Type.u64, 8);
	3873	try cg.store(result, .{ .imm64 = 0 }, .u64, .{});
	3874	try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, .u64, .{ .offset = 8 });
3812	3875	return result;
3813	3876	} else {
3814	3877	const result = try cg.allocInt(int_ty);
...	...	@@ -3816,10 +3879,10 @@ fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3816	3879	const used_len = @divCeil(int_ty.bits, 64) * 8;
3817	3880
3818	3881	try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0 });
3819		try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, Type.u64, used_len - 8);
	3882	try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, .u64, .{ .offset = used_len - 8 });
3820	3883
3821	3884	if (used_len + 8 == full_len) {
3822		try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, full_len - 8);
	3885	try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, .u64, .{ .offset = full_len - 8 });
3823	3886	}
3824	3887
3825	3888	return result;
...	...	@@ -3839,20 +3902,20 @@ fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3839	3902	defer rhs_is_neg.free(cg);
3840	3903	const min_val = try cg.intMinValue(int_ty);
3841	3904
3842		try cg.lowerToStack(min_val);
3843		try cg.lowerToStack(max_val);
	3905	try cg.emitWValue(min_val);
	3906	try cg.emitWValue(max_val);
3844	3907	try cg.emitWValue(rhs_is_neg);
3845	3908	try cg.addTag(.select);
3846	3909
3847		try cg.lowerToStack(op_val);
	3910	try cg.emitWValue(op_val);
3848	3911	const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs);
3849	3912	const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3850	3913	try cg.emitWValue(is_overflow);
3851	3914	try cg.addTag(.select);
3852	3915	return .stack;
3853	3916	} else {
3854		try cg.lowerToStack(max_val);
3855		try cg.lowerToStack(op_val);
	3917	try cg.emitWValue(max_val);
	3918	try cg.emitWValue(op_val);
3856	3919
3857	3920	const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs);
3858	3921	try cg.emitWValue(is_overflow);
...	...	@@ -3873,12 +3936,12 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3873	3936	const max_val = try cg.intMaxValue(int_ty);
3874	3937	const min_val = try cg.intMinValue(int_ty);
3875	3938
3876		try cg.lowerToStack(max_val);
3877		try cg.lowerToStack(min_val);
	3939	try cg.emitWValue(max_val);
	3940	try cg.emitWValue(min_val);
3878	3941	try cg.emitWValue(rhs_is_neg);
3879	3942	try cg.addTag(.select);
3880	3943
3881		try cg.lowerToStack(op_val);
	3944	try cg.emitWValue(op_val);
3882	3945	const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs);
3883	3946	const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3884	3947	try cg.emitWValue(is_overflow);
...	...	@@ -3887,8 +3950,8 @@ fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3887	3950	} else {
3888	3951	const zero = try cg.intZeroValue(int_ty);
3889	3952
3890		try cg.lowerToStack(zero);
3891		try cg.lowerToStack(op_val);
	3953	try cg.emitWValue(zero);
	3954	try cg.emitWValue(op_val);
3892	3955	const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs);
3893	3956	try cg.emitWValue(is_overflow);
3894	3957	try cg.addTag(.select);
...	...	@@ -3912,10 +3975,10 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3912	3975	if (int_ty.is_signed) {
3913	3976	const min_val = try cg.intMinValue(int_ty);
3914	3977
3915		try cg.lowerToStack(min_val);
	3978	try cg.emitWValue(min_val);
3916	3979
3917		try cg.lowerToStack(max_val);
3918		try cg.lowerToStack(op_val);
	3980	try cg.emitWValue(max_val);
	3981	try cg.emitWValue(op_val);
3919	3982	const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3920	3983	const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3921	3984	try cg.emitWValue(ov_pos);
...	...	@@ -3923,12 +3986,12 @@ fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3923	3986
3924	3987	const min_ext = try cg.intCast(ext_ty, int_ty, min_val);
3925	3988	const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext);
3926		try cg.lowerToStack(ov_neg);
	3989	try cg.emitWValue(ov_neg);
3927	3990	try cg.addTag(.select);
3928	3991	return .stack;
3929	3992	} else {
3930		try cg.lowerToStack(max_val);
3931		try cg.lowerToStack(op_val);
	3993	try cg.emitWValue(max_val);
	3994	try cg.emitWValue(op_val);
3932	3995	const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3933	3996	const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3934	3997	try cg.emitWValue(is_overflow);
...	...	@@ -3951,20 +4014,20 @@ fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError
3951	4014	const zero = try cg.intZeroValue(int_ty);
3952	4015	const min_val = try cg.intMinValue(int_ty);
3953	4016
3954		try cg.lowerToStack(min_val);
3955		try cg.lowerToStack(max_val);
	4017	try cg.emitWValue(min_val);
	4018	try cg.emitWValue(max_val);
3956	4019	const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero);
3957	4020	try cg.emitWValue(lhs_is_neg);
3958	4021	try cg.addTag(.select);
3959	4022
3960		try cg.lowerToStack(op_val);
	4023	try cg.emitWValue(op_val);
3961	4024	const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3962	4025	try cg.emitWValue(is_overflow);
3963	4026	try cg.addTag(.select);
3964	4027	return .stack;
3965	4028	} else {
3966		try cg.lowerToStack(max_val);
3967		try cg.lowerToStack(op_val);
	4029	try cg.emitWValue(max_val);
	4030	try cg.emitWValue(op_val);
3968	4031	const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3969	4032	try cg.emitWValue(is_overflow);
3970	4033	try cg.addTag(.select);
...	...	@@ -3979,8 +4042,8 @@ fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3979	4042	33...64 => return .{ .imm64 = 0 },
3980	4043	65...128 => {
3981	4044	const result = try cg.allocInt(int_ty);
3982		try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
3983		try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
	4045	try cg.store(result, .{ .imm64 = 0 }, .u64, .{});
	4046	try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });
3984	4047	return result;
3985	4048	},
3986	4049	else => {
...	...	@@ -4003,7 +4066,7 @@ fn toLocalInt(cg: *CodeGen, value: WValue, int_ty: IntType) InnerError!WValue {
4003	4066	else => return cg.fail("TODO: Support toLocalInt for integer bitsize: {d}", .{int_ty.bits}),
4004	4067	};
4005	4068	const new_local = try cg.allocLocal(ty);
4006		try cg.addLocal(.local_set, new_local.local.value);
	4069	try cg.addLocal(.local_set, new_local.local);
4007	4070	return new_local;
4008	4071	},
4009	4072	.local, .stack_offset => return value,
...	...	@@ -4036,9 +4099,9 @@ fn intAddOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4036	4099	else => {
4037	4100	const result = try cg.allocInt(ty);
4038	4101
4039		try cg.lowerToStack(result);
4040		try cg.lowerToStack(lhs);
4041		try cg.lowerToStack(rhs);
	4102	try cg.emitWValue(result);
	4103	try cg.emitWValue(lhs);
	4104	try cg.emitWValue(rhs);
4042	4105	try cg.addImm32(@intFromBool(ty.is_signed));
4043	4106	try cg.addImm32(ty.bits);
4044	4107	try cg.addCallIntrinsic(.__addo_limb64);
...	...	@@ -4068,9 +4131,9 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4068	4131	else => {
4069	4132	const result = try cg.allocInt(ty);
4070	4133
4071		try cg.lowerToStack(result);
4072		try cg.lowerToStack(lhs);
4073		try cg.lowerToStack(rhs);
	4134	try cg.emitWValue(result);
	4135	try cg.emitWValue(lhs);
	4136	try cg.emitWValue(rhs);
4074	4137	try cg.addImm32(@intFromBool(ty.is_signed));
4075	4138	try cg.addImm32(ty.bits);
4076	4139	try cg.addCallIntrinsic(.__subo_limb64);
...	...	@@ -4080,39 +4143,55 @@ fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4080	4143	}
4081	4144
4082	4145	fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
	4146	if (int_ty.bits > 64 and (int_ty.bits != 128 or !int_ty.is_signed)) {
	4147	const result = try cg.allocInt(int_ty);
	4148
	4149	try cg.emitWValue(result);
	4150	try cg.emitWValue(lhs);
	4151	try cg.emitWValue(rhs);
	4152	try cg.addImm32(@intFromBool(int_ty.is_signed));
	4153	try cg.addImm32(int_ty.bits);
	4154	try cg.addCallIntrinsic(.__mulo_limb64);
	4155
	4156	return .{ .result = result, .ov = .stack };
	4157	}
	4158
4083	4159	const overflow_bit = try cg.allocLocal(Type.u32);
4084	4160	try cg.addImm32(0);
4085		try cg.addLocal(.local_set, overflow_bit.local.value);
	4161	try cg.addLocal(.local_set, overflow_bit.local);
4086	4162
4087	4163	const result_val = if (int_ty.bits <= 32) blk: {
4088	4164	const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 64 };
4089	4165	const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
4090	4166	const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
4091	4167	const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
4092		const bin_op = try cg.toLocalInt(mul_raw, new_ty);
	4168	var bin_op = try cg.toLocalInt(mul_raw, new_ty);
	4169	defer bin_op.free(cg);
4093	4170
4094	4171	const res = try cg.intTrunc(int_ty, new_ty, bin_op);
4095	4172	const res_tmp = try cg.toLocalInt(res, int_ty);
4096	4173
4097	4174	const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
4098	4175	_ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
4099		try cg.addLocal(.local_set, overflow_bit.local.value);
	4176	try cg.addLocal(.local_set, overflow_bit.local);
4100	4177	break :blk res_tmp;
4101	4178	} else if (int_ty.bits <= 64) blk: {
4102	4179	const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 128 };
4103	4180	const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
4104	4181	const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
4105	4182	const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
4106		const bin_op = try cg.toLocalInt(mul_raw, new_ty);
	4183	var bin_op = try cg.toLocalInt(mul_raw, new_ty);
	4184	defer bin_op.free(cg);
4107	4185
4108	4186	const res = try cg.intTrunc(int_ty, new_ty, bin_op);
4109	4187	const res_tmp = try cg.toLocalInt(res, int_ty);
4110	4188
4111	4189	const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
4112	4190	_ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
4113		try cg.addLocal(.local_set, overflow_bit.local.value);
	4191	try cg.addLocal(.local_set, overflow_bit.local);
4114	4192	break :blk res_tmp;
4115		} else if (int_ty.bits == 128 and int_ty.is_signed) blk: {
	4193	} else blk: {
	4194	assert(int_ty.bits == 128 and int_ty.is_signed);
4116	4195	const overflow_ret = try cg.allocStack(Type.i32);
4117	4196	const res = try cg.callIntrinsic(
4118	4197	.__muloti4,
...	...	@@ -4120,20 +4199,9 @@ fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) Inner
4120	4199	Type.i128,
4121	4200	&.{ lhs, rhs, overflow_ret },
4122	4201	);
4123		_ = try cg.load(overflow_ret, Type.i32, 0);
4124		try cg.addLocal(.local_set, overflow_bit.local.value);
	4202	_ = try cg.load(overflow_ret, .i32, .{});
	4203	try cg.addLocal(.local_set, overflow_bit.local);
4125	4204	break :blk res;
4126		} else {
4127		const result = try cg.allocInt(int_ty);
4128
4129		try cg.lowerToStack(result);
4130		try cg.lowerToStack(lhs);
4131		try cg.lowerToStack(rhs);
4132		try cg.addImm32(@intFromBool(int_ty.is_signed));
4133		try cg.addImm32(int_ty.bits);
4134		try cg.addCallIntrinsic(.__mulo_limb64);
4135
4136		return .{ .result = result, .ov = .stack };
4137	4205	};
4138	4206
4139	4207	return .{ .result = result_val, .ov = .{ .local = overflow_bit.local } };
...	...	@@ -4155,9 +4223,9 @@ fn intShlOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerErro
4155	4223	else => {
4156	4224	const result = try cg.allocInt(ty);
4157	4225
4158		try cg.lowerToStack(result);
4159		try cg.lowerToStack(lhs);
4160		try cg.lowerToStack(rhs);
	4226	try cg.emitWValue(result);
	4227	try cg.emitWValue(lhs);
	4228	try cg.emitWValue(rhs);
4161	4229	try cg.addImm32(@intFromBool(ty.is_signed));
4162	4230	try cg.addImm32(ty.bits);
4163	4231	try cg.addCallIntrinsic(.__shlo_limb64);
...	...	@@ -4194,15 +4262,15 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4194	4262	} else {
4195	4263	var src_len: u32 = undefined;
4196	4264	if (src_bits == 32) {
4197		try cg.emitWValue(result);
	4265	try cg.emitMemBase(result);
4198	4266	try cg.emitWValue(operand);
4199	4267	try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4200		try cg.store(.stack, .stack, Type.u64, result.offset());
	4268	try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() });
4201	4269	src_len = 8;
4202	4270	} else if (src_bits == 64) {
4203		try cg.emitWValue(result);
	4271	try cg.emitMemBase(result);
4204	4272	try cg.emitWValue(operand);
4205		try cg.store(.stack, .stack, Type.u64, result.offset());
	4273	try cg.store(.stack, .stack, .u64, .{ .offset = result.memOffset() });
4206	4274	src_len = 8;
4207	4275	} else {
4208	4276	src_len = src_bits / 8;
...	...	@@ -4211,7 +4279,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4211	4279
4212	4280	if (dest_bits == 128) {
4213	4281	if (src_ty.is_signed) {
4214		try cg.emitWValue(result);
	4282	try cg.emitMemBase(result);
4215	4283	if (src_bits == 32) {
4216	4284	try cg.emitWValue(operand);
4217	4285	try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
...	...	@@ -4219,9 +4287,9 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4219	4287	try cg.emitWValue(operand);
4220	4288	} else unreachable;
4221	4289	const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4222		try cg.store(.stack, shr, Type.u64, 8 + result.offset());
	4290	try cg.store(.stack, shr, .u64, .{ .offset = result.memOffset() + 8 });
4223	4291	} else {
4224		try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
	4292	try cg.store(result, .{ .imm64 = 0 }, .u64, .{ .offset = 8 });
4225	4293	}
4226	4294	} else {
4227	4295	var pad = result;
...	...	@@ -4236,7 +4304,7 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4236	4304	_ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4237	4305	try cg.addTag(.i32_wrap_i64);
4238	4306	} else {
4239		_ = try cg.load(operand, Type.u64, src_len - 8);
	4307	_ = try cg.load(operand, .u64, .{ .offset = src_len - 8 });
4240	4308	_ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4241	4309	try cg.addTag(.i32_wrap_i64);
4242	4310	}
...	...	@@ -4254,11 +4322,12 @@ fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) Inn
4254	4322	assert(dest_bits <= 64);
4255	4323	assert(src_bits >= 128);
4256	4324	const load_ty = if (dest_bits == 32) Type.u32 else Type.u64;
4257		return cg.load(operand, load_ty, 0);
	4325	return cg.load(operand, load_ty, .{});
4258	4326	}
4259	4327	}
4260	4328
4261	4329	fn intTrunc(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {
	4330	assert(src_ty.bits > dest_ty.bits);
4262	4331	var result = try cg.intCast(dest_ty, src_ty, operand);
4263	4332
4264	4333	const dest_wasm_bits = cg.intBackingBits(dest_ty.bits);
...	...	@@ -4622,20 +4691,22 @@ fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4622	4691	},
4623	4692	.f80, .f128 => {
4624	4693	const result = try cg.allocStack(Type.f128);
4625		try cg.emitWValue(result);
4626		try cg.emitWValue(arg);
4627		try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
4628		try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
4629		try cg.emitWValue(result);
4630		try cg.emitWValue(arg);
4631		try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
	4694	const result_offset = result.memOffset();
	4695	const arg_offset = arg.memOffset();
	4696	try cg.emitMemBase(result);
	4697	try cg.emitMemBase(arg);
	4698	try cg.addMemArg(.i64_load, .{ .offset = arg_offset, .alignment = .@"2" });
	4699	try cg.addMemArg(.i64_store, .{ .offset = result_offset, .alignment = .@"2" });
	4700	try cg.emitMemBase(result);
	4701	try cg.emitMemBase(arg);
	4702	try cg.addMemArg(.i64_load, .{ .offset = arg_offset + 8, .alignment = .@"2" });
4632	4703	if (ty == .f80) {
4633	4704	try cg.addImm64(0x8000);
4634	4705	} else {
4635	4706	try cg.addImm64(0x8000000000000000);
4636	4707	}
4637	4708	try cg.addTag(.i64_xor);
4638		try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
	4709	try cg.addMemArg(.i64_store, .{ .offset = result_offset + 8, .alignment = .@"2" });
4639	4710	return result;
4640	4711	},
4641	4712	}
...	...	@@ -4661,20 +4732,22 @@ fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4661	4732	},
4662	4733	.f80, .f128 => {
4663	4734	const result = try cg.allocStack(Type.f128);
4664		try cg.emitWValue(result);
4665		try cg.emitWValue(arg);
4666		try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
4667		try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
4668		try cg.emitWValue(result);
4669		try cg.emitWValue(arg);
4670		try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
	4735	const result_offset = result.memOffset();
	4736	const arg_offset = arg.memOffset();
	4737	try cg.emitMemBase(result);
	4738	try cg.emitMemBase(arg);
	4739	try cg.addMemArg(.i64_load, .{ .offset = arg_offset, .alignment = .@"2" });
	4740	try cg.addMemArg(.i64_store, .{ .offset = result_offset, .alignment = .@"2" });
	4741	try cg.emitMemBase(result);
	4742	try cg.emitMemBase(arg);
	4743	try cg.addMemArg(.i64_load, .{ .offset = arg_offset + 8, .alignment = .@"2" });
4671	4744	if (ty == .f80) {
4672	4745	try cg.addImm64(0x7FFF);
4673	4746	} else {
4674	4747	try cg.addImm64(0x7FFFFFFFFFFFFFFF);
4675	4748	}
4676	4749	try cg.addTag(.i64_and);
4677		try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
	4750	try cg.addMemArg(.i64_store, .{ .offset = result_offset + 8, .alignment = .@"2" });
4678	4751	return result;
4679	4752	},
4680	4753	}
...	...	@@ -5184,30 +5257,36 @@ fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5184	5257
5185	5258	fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {
5186	5259	const zcu = cg.pt.zcu;
5187		// if wasm_block_ty is non-empty, we create a register to store the temporary value
	5260
	5261	if (block_ty.isNoReturn(zcu)) {
	5262	assert(cg.liveness.getBlock(inst).deaths.len == 0);
	5263	try cg.genBody(body);
	5264	return cg.finishAir(inst, .none, &.{});
	5265	}
	5266
5188	5267	const block_result: WValue = if (block_ty.hasRuntimeBits(zcu))
5189		try cg.allocLocal(block_ty)
	5268	.stack
5190	5269	else
5191	5270	.none;
5192	5271
5193		try cg.startBlock(.block, .empty);
	5272	try cg.startBlock(.block, genBlockType(block_ty, zcu, cg.target));
5194	5273	// Here we set the current block idx, so breaks know the depth to jump
5195	5274	// to when breaking out.
5196	5275	try cg.blocks.putNoClobber(cg.gpa, inst, .{
5197	5276	.label = cg.block_depth,
5198	5277	.value = block_result,
5199	5278	});
	5279	defer assert(cg.blocks.swapRemove(inst));
5200	5280
5201	5281	{
5202		try cg.branches.append(cg.gpa, .{});
5203		defer {
5204		var branch = cg.branches.pop().?;
5205		branch.deinit(cg.gpa);
5206		}
	5282	try cg.pushBranch();
	5283	defer cg.popBranch();
5207	5284	try cg.genBody(body);
5208	5285	try cg.endBlock();
5209	5286	}
5210	5287
	5288	for (cg.liveness.getBlock(inst).deaths) \|death\| cg.processDeath(death.toRef());
	5289
5211	5290	return cg.finishAir(inst, block_result, &.{});
5212	5291	}
5213	5292
...	...	@@ -5238,6 +5317,7 @@ fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5238	5317
5239	5318	try cg.genBody(block.body);
5240	5319	try cg.endBlock();
	5320	try cg.addTag(.@"unreachable");
5241	5321
5242	5322	return cg.finishAir(inst, .none, &.{});
5243	5323	}
...	...	@@ -5247,6 +5327,7 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5247	5327	const condition = try cg.resolveInst(cond_br.condition);
5248	5328	const then_body = cond_br.then_body;
5249	5329	const else_body = cond_br.else_body;
	5330	const branch_liveness = cg.liveness.getCondBr(inst);
5250	5331
5251	5332	// result type is always noreturn, so use `block_empty` as type.
5252	5333	try cg.startBlock(.block, .empty);
...	...	@@ -5258,24 +5339,21 @@ fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5258	5339	// and continue with the then codepath
5259	5340	try cg.addLabel(.br_if, 0);
5260	5341
5261		try cg.branches.ensureUnusedCapacity(cg.gpa, 2);
	5342	if (cg.liveness.operandDies(inst, 0)) cg.processDeath(cond_br.condition);
	5343
5262	5344	{
5263		cg.branches.appendAssumeCapacity(.{});
5264		defer {
5265		var else_stack = cg.branches.pop().?;
5266		else_stack.deinit(cg.gpa);
5267		}
	5345	try cg.pushBranch();
	5346	defer cg.popBranch();
	5347	for (branch_liveness.else_deaths) \|death\| cg.processDeath(death.toRef());
5268	5348	try cg.genBody(else_body);
5269	5349	try cg.endBlock();
5270	5350	}
5271	5351
5272	5352	// Outer block that matches the condition
5273	5353	{
5274		cg.branches.appendAssumeCapacity(.{});
5275		defer {
5276		var then_stack = cg.branches.pop().?;
5277		then_stack.deinit(cg.gpa);
5278		}
	5354	try cg.pushBranch();
	5355	defer cg.popBranch();
	5356	for (branch_liveness.then_deaths) \|death\| cg.processDeath(death.toRef());
5279	5357	try cg.genBody(then_body);
5280	5358	}
5281	5359
...	...	@@ -5311,22 +5389,22 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne
5311	5389	try cg.startBlock(.block, .empty);
5312	5390
5313	5391	try cg.addImm32(if (op == .eq) 0 else 1);
5314		try cg.addLocal(.local_set, result.local.value);
	5392	try cg.addLocal(.local_set, result.local);
5315	5393
5316	5394	_ = try cg.isNull(lhs, operand_ty, .i32_eq, .value);
5317		try cg.addLocal(.local_tee, lhs_null.local.value);
	5395	try cg.addLocal(.local_tee, lhs_null.local);
5318	5396	_ = try cg.isNull(rhs, operand_ty, .i32_eq, .value);
5319	5397	try cg.addTag(.i32_ne);
5320	5398	try cg.addLabel(.br_if, 0);
5321	5399
5322	5400	try cg.addImm32(if (op == .eq) 1 else 0);
5323		try cg.addLocal(.local_set, result.local.value);
	5401	try cg.addLocal(.local_set, result.local);
5324	5402
5325		try cg.addLocal(.local_get, lhs_null.local.value);
	5403	try cg.addLocal(.local_get, lhs_null.local);
5326	5404	try cg.addLabel(.br_if, 0);
5327	5405
5328		_ = try cg.load(lhs, payload_ty, 0);
5329		_ = try cg.load(rhs, payload_ty, 0);
	5406	_ = try cg.load(lhs, payload_ty, .{});
	5407	_ = try cg.load(rhs, payload_ty, .{});
5330	5408
5331	5409	if (payload_ty.isAnyFloat()) {
5332	5410	_ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack);
...	...	@@ -5334,10 +5412,10 @@ fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) Inne
5334	5412	_ = try cg.intCmp(.fromType(cg, payload_ty), op, .stack, .stack);
5335	5413	}
5336	5414
5337		try cg.addLocal(.local_set, result.local.value);
	5415	try cg.addLocal(.local_set, result.local);
5338	5416	try cg.endBlock();
5339	5417
5340		try cg.addLocal(.local_get, result.local.value);
	5418	try cg.addLocal(.local_get, result.local);
5341	5419	try cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
5342	5420	} else {
5343	5421	const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);
...	...	@@ -5357,8 +5435,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5357	5435	0 => unreachable,
5358	5436	1...32 => {
5359	5437	// lhs or rhs could be stack pointers
5360		try cg.lowerToStack(lhs);
5361		try cg.lowerToStack(rhs);
	5438	try cg.emitWValue(lhs);
	5439	try cg.emitWValue(rhs);
5362	5440	const opcode: Mir.Inst.Tag = switch (op) {
5363	5441	.eq => .i32_eq,
5364	5442	.neq => .i32_ne,
...	...	@@ -5372,8 +5450,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5372	5450	},
5373	5451	33...64 => {
5374	5452	// lhs or rhs could be stack pointers
5375		try cg.lowerToStack(lhs);
5376		try cg.lowerToStack(rhs);
	5453	try cg.emitWValue(lhs);
	5454	try cg.emitWValue(rhs);
5377	5455	const opcode: Mir.Inst.Tag = switch (op) {
5378	5456	.eq => .i64_eq,
5379	5457	.neq => .i64_ne,
...	...	@@ -5386,16 +5464,16 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5386	5464	return .stack;
5387	5465	},
5388	5466	65...128 => {
5389		var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);
	5467	var lhs_msb = try (try cg.load(lhs, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
5390	5468	defer lhs_msb.free(cg);
5391		var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);
	5469	var rhs_msb = try (try cg.load(rhs, .u64, .{ .offset = 8 })).toLocal(cg, Type.u64);
5392	5470	defer rhs_msb.free(cg);
5393	5471
5394	5472	switch (op) {
5395	5473	.eq, .neq => {
5396	5474	const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb);
5397		const lhs_lsb = try cg.load(lhs, Type.u64, 0);
5398		const rhs_lsb = try cg.load(rhs, Type.u64, 0);
	5475	const lhs_lsb = try cg.load(lhs, .u64, .{});
	5476	const rhs_lsb = try cg.load(rhs, .u64, .{});
5399	5477	const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb);
5400	5478	const or_result = try cg.intOr(.u64, xor_high, xor_low);
5401	5479
...	...	@@ -5408,8 +5486,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5408	5486	else => {
5409	5487	const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64;
5410	5488
5411		const lhs_lsb = try cg.load(lhs, Type.u64, 0);
5412		const rhs_lsb = try cg.load(rhs, Type.u64, 0);
	5489	const lhs_lsb = try cg.load(lhs, .u64, .{});
	5490	const rhs_lsb = try cg.load(rhs, .u64, .{});
5413	5491
5414	5492	// leave values on stack for 'select'
5415	5493	_ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb);
...	...	@@ -5422,8 +5500,8 @@ fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue,
5422	5500	return .stack;
5423	5501	},
5424	5502	else => {
5425		try cg.lowerToStack(lhs);
5426		try cg.lowerToStack(rhs);
	5503	try cg.emitWValue(lhs);
	5504	try cg.emitWValue(rhs);
5427	5505	try cg.addImm32(@intFromBool(ty.is_signed));
5428	5506	try cg.addImm32(ty.bits);
5429	5507	try cg.addCallIntrinsic(.__cmp_limb64);
...	...	@@ -5531,11 +5609,13 @@ fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5531	5609	const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;
5532	5610	const block = cg.blocks.get(br.block_inst).?;
5533	5611
5534		// if operand has codegen bits we should break with a value
5535		if (block.value != .none) {
5536		const operand = try cg.resolveInst(br.operand);
5537		try cg.lowerToStack(operand);
5538		try cg.addLocal(.local_set, block.value.local.value);
	5612	switch (block.value) {
	5613	.none => {},
	5614	.stack => {
	5615	const operand = try cg.resolveInst(br.operand);
	5616	try cg.emitWValue(operand);
	5617	},
	5618	else => unreachable,
5539	5619	}
5540	5620
5541	5621	// We map every block to its block index.
...	...	@@ -5558,7 +5638,8 @@ fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5558	5638
5559	5639	fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5560	5640	try cg.addTag(.@"unreachable");
5561		return cg.finishAir(inst, .none, &.{});
	5641	try cg.finishAir(inst, .none, &.{});
	5642	cg.verifyLocalPool();
5562	5643	}
5563	5644
5564	5645	fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -5570,7 +5651,8 @@ fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5570	5651
5571	5652	fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5572	5653	try cg.addTag(.@"unreachable");
5573		return cg.finishAir(inst, .none, &.{});
	5654	try cg.finishAir(inst, .none, &.{});
	5655	cg.verifyLocalPool();
5574	5656	}
5575	5657
5576	5658	fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -5584,8 +5666,7 @@ fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5584	5666	assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
5585	5667
5586	5668	const operand = try cg.resolveInst(ty_op.operand);
5587		const result = cg.reuseOperand(ty_op.operand, operand);
5588		return cg.finishAir(inst, result, &.{ty_op.operand});
	5669	return cg.finishAir(inst, operand, &.{ty_op.operand});
5589	5670	}
5590	5671
5591	5672	fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -5599,11 +5680,12 @@ fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5599	5680	assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
5600	5681
5601	5682	const operand = try cg.resolveInst(ty_op.operand);
5602		const result = switch (operand) {
5603		.stack_offset => try cg.buildPointerOffset(operand, 0, .new),
5604		else => cg.reuseOperand(ty_op.operand, operand),
5605		};
5606		return cg.finishAir(inst, result, &.{ty_op.operand});
	5683	if (operand == .stack_offset) {
	5684	// materialize operand, we assume precomputed value is more efficient
	5685	try cg.emitWValue(operand);
	5686	return cg.finishAir(inst, .stack, &.{ty_op.operand});
	5687	}
	5688	return cg.finishAir(inst, operand, &.{ty_op.operand});
5607	5689	}
5608	5690
5609	5691	fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -5616,7 +5698,7 @@ fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5616	5698
5617	5699	const enum_value = try cg.resolveInst(ty_op.operand);
5618	5700	const result = try cg.allocStack(union_ty);
5619		try cg.store(result, enum_value, enum_ty, @intCast(layout.tagOffset()));
	5701	try cg.store(result, enum_value, enum_ty, .{ .offset = layout.tagOffset() });
5620	5702
5621	5703	return cg.finishAir(inst, result, &.{ty_op.operand});
5622	5704	}
...	...	@@ -5627,7 +5709,7 @@ fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5627	5709	const dest_ty = cg.typeOfIndex(inst);
5628	5710	const src_ty = cg.typeOf(ty_op.operand);
5629	5711
5630		const result = (try cg.bitcast(dest_ty, src_ty, operand)) orelse cg.reuseOperand(ty_op.operand, operand);
	5712	const result = try cg.bitcast(dest_ty, src_ty, operand);
5631	5713
5632	5714	return cg.finishAir(inst, result, &.{ty_op.operand});
5633	5715	}
...	...	@@ -5654,8 +5736,8 @@ fn bitcastClass(cg: *CodeGen, ty: Type) BitcastClass {
5654	5736	};
5655	5737	}
5656	5738
5657		fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerError!?WValue {
5658		if (dest_ty.eql(src_ty)) return null;
	5739	fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerError!WValue {
	5740	if (dest_ty.eql(src_ty)) return operand;
5659	5741
5660	5742	const zcu = cg.pt.zcu;
5661	5743	const src_class = cg.bitcastClass(src_ty);
...	...	@@ -5674,17 +5756,17 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro
5674	5756
5675	5757	if (src_by_ref and dest_by_ref) {
5676	5758	if (needs_wrapping) return try cg.intWrap(dest_class.int, operand);
5677		return null;
	5759	return operand;
5678	5760	}
5679	5761
5680	5762	if (dest_by_ref) {
5681	5763	const result = try cg.allocStack(src_ty);
5682		try cg.store(result, operand, src_ty, 0);
	5764	try cg.store(result, operand, src_ty, .{});
5683	5765	return result;
5684	5766	}
5685	5767
5686	5768	if (src_by_ref) {
5687		return try cg.load(operand, dest_ty, 0);
	5769	return try cg.load(operand, dest_ty, .{});
5688	5770	}
5689	5771
5690	5772	switch (src_class) {
...	...	@@ -5708,7 +5790,7 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro
5708	5790
5709	5791	switch (dest_class) {
5710	5792	.float => \|float_ty\| switch (float_ty) {
5711		.f16 => return null,
	5793	.f16 => return operand,
5712	5794	.f32 => {
5713	5795	try cg.emitWValue(operand);
5714	5796	try cg.addTag(.f32_reinterpret_i32);
...	...	@@ -5729,7 +5811,7 @@ fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerErro
5729	5811	return try cg.intWrap(dest_class.int, operand);
5730	5812	}
5731	5813
5732		return null;
	5814	return operand;
5733	5815	}
5734	5816
5735	5817	fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -5740,7 +5822,7 @@ fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5740	5822	const struct_ptr = try cg.resolveInst(extra.data.struct_operand);
5741	5823	const struct_ptr_ty = cg.typeOf(extra.data.struct_operand);
5742	5824	const struct_ty = struct_ptr_ty.childType(zcu);
5743		const result = try cg.structFieldPtr(inst, extra.data.struct_operand, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index);
	5825	const result = try cg.structFieldPtr(inst, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index);
5744	5826	return cg.finishAir(inst, result, &.{extra.data.struct_operand});
5745	5827	}
5746	5828
...	...	@@ -5751,14 +5833,13 @@ fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerE
5751	5833	const struct_ptr_ty = cg.typeOf(ty_op.operand);
5752	5834	const struct_ty = struct_ptr_ty.childType(zcu);
5753	5835
5754		const result = try cg.structFieldPtr(inst, ty_op.operand, struct_ptr, struct_ptr_ty, struct_ty, index);
	5836	const result = try cg.structFieldPtr(inst, struct_ptr, struct_ptr_ty, struct_ty, index);
5755	5837	return cg.finishAir(inst, result, &.{ty_op.operand});
5756	5838	}
5757	5839
5758	5840	fn structFieldPtr(
5759	5841	cg: *CodeGen,
5760	5842	inst: Air.Inst.Index,
5761		ref: Air.Inst.Ref,
5762	5843	struct_ptr: WValue,
5763	5844	struct_ptr_ty: Type,
5764	5845	struct_ty: Type,
...	...	@@ -5783,16 +5864,7 @@ fn structFieldPtr(
5783	5864	},
5784	5865	else => struct_ty.structFieldOffset(index, zcu),
5785	5866	};
5786		// save a load and store when we can simply reuse the operand
5787		if (offset == 0) {
5788		return cg.reuseOperand(ref, struct_ptr);
5789		}
5790		switch (struct_ptr) {
5791		.stack_offset => \|stack_offset\| {
5792		return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } };
5793		},
5794		else => return cg.buildPointerOffset(struct_ptr, offset, .new),
5795		}
	5867	return cg.offsetPointer(struct_ptr, offset);
5796	5868	}
5797	5869
5798	5870	fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -5814,44 +5886,45 @@ fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5814	5886	return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});
5815	5887	};
5816	5888	if (isByRef(field_ty, zcu, cg.target)) {
5817		switch (operand) {
5818		.stack_offset => \|stack_offset\| {
5819		break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } };
5820		},
5821		else => break :result try cg.buildPointerOffset(operand, offset, .new),
5822		}
	5889	break :result try cg.offsetPointer(operand, offset);
5823	5890	}
5824		break :result try cg.load(operand, field_ty, offset);
	5891	break :result try cg.load(operand, field_ty, .{ .offset = offset });
5825	5892	},
5826	5893	};
5827	5894
5828	5895	return cg.finishAir(inst, result, &.{struct_field.struct_operand});
5829	5896	}
5830	5897
	5898	// Both `.switch_br` and `.loop_switch_br` are noreturn.
5831	5899	fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void {
5832	5900	const pt = cg.pt;
5833	5901	const zcu = pt.zcu;
5834	5902
5835	5903	const switch_br = cg.air.unwrapSwitch(inst);
5836	5904	const target_ty = cg.typeOf(switch_br.operand);
	5905	const branch_liveness = try cg.liveness.getSwitchBr(cg.gpa, inst, switch_br.cases_len + 1);
	5906	defer cg.gpa.free(branch_liveness.deaths);
5837	5907
5838	5908	assert(target_ty.hasRuntimeBits(zcu));
5839	5909
5840		// swap target value with placeholder local, for dispatching
	5910	const operand_dies = cg.liveness.operandDies(inst, 0);
	5911	var owned_target: WValue = .none;
	5912	errdefer owned_target.free(cg);
5841	5913	const target = if (is_dispatch_loop) target: {
5842	5914	const initial_target = try cg.resolveInst(switch_br.operand);
5843		const target: WValue = try cg.allocLocal(target_ty);
5844		try cg.lowerToStack(initial_target);
5845		try cg.addLocal(.local_set, target.local.value);
	5915	owned_target = try cg.allocLocal(target_ty);
	5916	try cg.emitWValue(initial_target);
	5917	try cg.addLocal(.local_set, owned_target.local);
5846	5918
5847	5919	try cg.startBlock(.loop, .empty); // dispatch loop start
5848	5920	try cg.blocks.putNoClobber(cg.gpa, inst, .{
5849	5921	.label = cg.block_depth,
5850		.value = target,
	5922	.value = owned_target,
5851	5923	});
5852	5924
5853		break :target target;
	5925	break :target owned_target;
5854	5926	} else try cg.resolveInst(switch_br.operand);
	5927	defer if (is_dispatch_loop) assert(cg.blocks.swapRemove(inst));
5855	5928
5856	5929	const has_else_body = switch_br.else_body_len != 0;
5857	5930	const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions
...	...	@@ -5863,15 +5936,19 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner
5863	5936	var it = switch_br.iterateCases();
5864	5937	const else_body = it.elseBody();
5865	5938
5866		cg.branches.appendAssumeCapacity(.{});
5867		defer {
5868		var else_branch = cg.branches.pop().?;
5869		else_branch.deinit(cg.gpa);
	5939	if (operand_dies) cg.processDeath(switch_br.operand);
	5940	owned_target.free(cg);
	5941
	5942	{
	5943	try cg.pushBranch();
	5944	defer cg.popBranch();
	5945	for (branch_liveness.deaths[0]) \|death\| cg.processDeath(death.toRef());
	5946	try cg.genBody(else_body);
5870	5947	}
5871		try cg.genBody(else_body);
5872	5948
5873	5949	if (is_dispatch_loop) {
5874	5950	try cg.endBlock(); // dispatch loop end
	5951	try cg.addTag(.@"unreachable");
5875	5952	}
5876	5953	return cg.finishAir(inst, .none, &.{});
5877	5954	}
...	...	@@ -5916,8 +5993,6 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner
5916	5993	break :width width_bigint.toConst().toInt(u32) catch null;
5917	5994	};
5918	5995
5919		try cg.startBlock(.block, .empty); // whole switch block start
5920
5921	5996	for (0..branch_count) \|_\| {
5922	5997	try cg.startBlock(.block, .empty);
5923	5998	}
...	...	@@ -5992,38 +6067,38 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Inner
5992	6067	try cg.addLabel(.br, branch_count - 1);
5993	6068	}
5994	6069
	6070	if (operand_dies) cg.processDeath(switch_br.operand);
	6071	owned_target.free(cg);
	6072
5995	6073	var cases_it = switch_br.iterateCases();
5996	6074	while (cases_it.next()) \|case\| {
5997	6075	try cg.endBlock();
5998	6076
5999		cg.branches.appendAssumeCapacity(.{});
6000		defer {
6001		var case_branch = cg.branches.pop().?;
6002		case_branch.deinit(cg.gpa);
	6077	{
	6078	try cg.pushBranch();
	6079	defer cg.popBranch();
	6080	for (branch_liveness.deaths[case.idx]) \|death\| cg.processDeath(death.toRef());
	6081	try cg.genBody(case.body);
6003	6082	}
6004		try cg.genBody(case.body);
6005
6006		try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch
6007	6083	}
6008	6084
6009	6085	try cg.endBlock();
6010	6086	if (has_else_body) {
6011	6087	const else_body = cases_it.elseBody();
6012	6088
6013		cg.branches.appendAssumeCapacity(.{});
6014		defer {
6015		var else_branch = cg.branches.pop().?;
6016		else_branch.deinit(cg.gpa);
	6089	{
	6090	try cg.pushBranch();
	6091	defer cg.popBranch();
	6092	for (branch_liveness.deaths[branch_liveness.deaths.len - 1]) \|death\| cg.processDeath(death.toRef());
	6093	try cg.genBody(else_body);
6017	6094	}
6018		try cg.genBody(else_body);
6019	6095	} else {
6020	6096	try cg.addTag(.@"unreachable");
6021	6097	}
6022	6098
6023		try cg.endBlock(); // whole switch block end
6024
6025	6099	if (is_dispatch_loop) {
6026	6100	try cg.endBlock(); // dispatch loop end
	6101	try cg.addTag(.@"unreachable");
6027	6102	}
6028	6103
6029	6104	return cg.finishAir(inst, .none, &.{});
...	...	@@ -6034,8 +6109,8 @@ fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6034	6109	const switch_loop = cg.blocks.get(br.block_inst).?;
6035	6110
6036	6111	const operand = try cg.resolveInst(br.operand);
6037		try cg.lowerToStack(operand);
6038		try cg.addLocal(.local_set, switch_loop.value.local.value);
	6112	try cg.emitWValue(operand);
	6113	try cg.addLocal(.local_set, switch_loop.value.local);
6039	6114
6040	6115	const idx: u32 = cg.block_depth - switch_loop.label;
6041	6116	try cg.addLabel(.br, idx);
...	...	@@ -6062,12 +6137,14 @@ fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind
6062	6137	}
6063	6138	}
6064	6139
6065		try cg.emitWValue(operand);
6066	6140	if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
	6141	try cg.emitMemBase(operand);
6067	6142	try cg.addMemArg(.i32_load16_u, .{
6068		.offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),
6069		.alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
	6143	.offset = operand.memOffset() + errUnionErrorOffset(pl_ty, zcu),
	6144	.alignment = Type.anyerror.abiAlignment(zcu),
6070	6145	});
	6146	} else {
	6147	try cg.emitWValue(operand);
6071	6148	}
6072	6149
6073	6150	// Compare the error value with '0'
...	...	@@ -6092,18 +6169,18 @@ fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool)
6092	6169	const result: WValue = result: {
6093	6170	if (!payload_ty.hasRuntimeBits(zcu)) {
6094	6171	if (op_is_ptr) {
6095		break :result cg.reuseOperand(ty_op.operand, operand);
	6172	break :result operand;
6096	6173	} else {
6097	6174	break :result .none;
6098	6175	}
6099	6176	}
6100	6177
6101		const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
	6178	const pl_offset = errUnionPayloadOffset(payload_ty, zcu);
6102	6179	if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {
6103		break :result try cg.buildPointerOffset(operand, pl_offset, .new);
	6180	break :result try cg.offsetPointer(operand, pl_offset);
6104	6181	} else {
6105	6182	assert(isByRef(eu_ty, zcu, cg.target));
6106		break :result try cg.load(operand, payload_ty, pl_offset);
	6183	break :result try cg.load(operand, payload_ty, .{ .offset = pl_offset });
6107	6184	}
6108	6185	};
6109	6186	return cg.finishAir(inst, result, &.{ty_op.operand});
...	...	@@ -6128,10 +6205,10 @@ fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) I
6128	6205
6129	6206	const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
6130	6207	if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {
6131		break :result try cg.load(operand, Type.anyerror, err_offset);
	6208	break :result try cg.load(operand, .anyerror, .{ .offset = err_offset });
6132	6209	} else {
6133	6210	assert(!payload_ty.hasRuntimeBits(zcu));
6134		break :result cg.reuseOperand(ty_op.operand, operand);
	6211	break :result operand;
6135	6212	}
6136	6213	};
6137	6214	return cg.finishAir(inst, result, &.{ty_op.operand});
...	...	@@ -6147,20 +6224,19 @@ fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6147	6224	const pl_ty = cg.typeOf(ty_op.operand);
6148	6225	const result = result: {
6149	6226	if (!pl_ty.hasRuntimeBits(zcu)) {
6150		break :result cg.reuseOperand(ty_op.operand, operand);
	6227	break :result operand;
6151	6228	}
6152	6229
6153	6230	const err_union = try cg.allocStack(err_ty);
6154		const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
6155		try cg.store(payload_ptr, operand, pl_ty, 0);
	6231	try cg.store(err_union, operand, pl_ty, .{ .offset = errUnionPayloadOffset(pl_ty, zcu) });
6156	6232
6157	6233	// ensure we also write '0' to the error part, so any present stack value gets overwritten by it.
6158		try cg.emitWValue(err_union);
	6234	try cg.emitMemBase(err_union);
6159	6235	try cg.addImm32(0);
6160		const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
	6236	const err_val_offset = errUnionErrorOffset(pl_ty, zcu);
6161	6237	try cg.addMemArg(.i32_store16, .{
6162		.offset = err_union.offset() + err_val_offset,
6163		.alignment = 2,
	6238	.offset = err_union.memOffset() + err_val_offset,
	6239	.alignment = .@"2",
6164	6240	});
6165	6241	break :result err_union;
6166	6242	};
...	...	@@ -6177,15 +6253,15 @@ fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6177	6253
6178	6254	const result = result: {
6179	6255	if (!pl_ty.hasRuntimeBits(zcu)) {
6180		break :result cg.reuseOperand(ty_op.operand, operand);
	6256	break :result operand;
6181	6257	}
6182	6258
6183	6259	const err_union = try cg.allocStack(err_ty);
6184	6260	// store error value
6185		try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu)));
	6261	try cg.store(err_union, operand, .anyerror, .{ .offset = errUnionErrorOffset(pl_ty, zcu) });
6186	6262
6187	6263	// write 'undefined' to the payload
6188		const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
	6264	const payload_ptr = try cg.offsetPointer(err_union, errUnionPayloadOffset(pl_ty, zcu));
6189	6265	const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));
6190	6266	try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });
6191	6267
...	...	@@ -6210,7 +6286,6 @@ fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kin
6210	6286	fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!WValue {
6211	6287	const pt = cg.pt;
6212	6288	const zcu = pt.zcu;
6213		try cg.emitWValue(operand);
6214	6289	const optional_ty = switch (op_kind) {
6215	6290	.value => op_ty,
6216	6291	.ptr => op_ty.childType(zcu),
...	...	@@ -6223,19 +6298,27 @@ fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, o
6223	6298	const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6224	6299	return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});
6225	6300	};
6226		try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });
	6301	try cg.emitMemBase(operand);
	6302	try cg.addMemArg(.i32_load8_u, .{ .offset = operand.memOffset() + offset, .alignment = .@"1" });
	6303	} else {
	6304	try cg.emitWValue(operand);
6227	6305	}
6228	6306	} else if (payload_ty.isSlice(zcu)) {
	6307	const base_offset = operand.memOffset();
	6308	try cg.emitMemBase(operand);
6229	6309	switch (cg.ptr_size) {
6230		.wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }),
6231		.wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }),
	6310	.wasm32 => try cg.addMemArg(.i32_load, .{ .offset = base_offset, .alignment = .@"4" }),
	6311	.wasm64 => try cg.addMemArg(.i64_load, .{ .offset = base_offset, .alignment = .@"8" }),
6232	6312	}
6233	6313	} else {
6234	6314	if (op_kind == .ptr) {
	6315	try cg.emitMemBase(operand);
6235	6316	try cg.addMemArg(.i32_load, .{
6236		.offset = operand.offset(),
6237		.alignment = 4,
	6317	.offset = operand.memOffset(),
	6318	.alignment = .@"4",
6238	6319	});
	6320	} else {
	6321	try cg.emitWValue(operand);
6239	6322	}
6240	6323	}
6241	6324
...	...	@@ -6257,13 +6340,13 @@ fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6257	6340
6258	6341	const result = result: {
6259	6342	const operand = try cg.resolveInst(ty_op.operand);
6260		if (opt_ty.optionalReprIsPayload(zcu)) break :result cg.reuseOperand(ty_op.operand, operand);
	6343	if (opt_ty.optionalReprIsPayload(zcu)) break :result operand;
6261	6344
6262	6345	if (isByRef(payload_ty, zcu, cg.target)) {
6263		break :result try cg.buildPointerOffset(operand, 0, .new);
	6346	break :result operand;
6264	6347	}
6265	6348
6266		break :result try cg.load(operand, payload_ty, 0);
	6349	break :result try cg.load(operand, payload_ty, .{});
6267	6350	};
6268	6351	return cg.finishAir(inst, result, &.{ty_op.operand});
6269	6352	}
...	...	@@ -6277,10 +6360,10 @@ fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6277	6360	const result = result: {
6278	6361	const payload_ty = opt_ty.optionalChild(zcu);
6279	6362	if (!payload_ty.hasRuntimeBits(zcu) or opt_ty.optionalReprIsPayload(zcu)) {
6280		break :result cg.reuseOperand(ty_op.operand, operand);
	6363	break :result operand;
6281	6364	}
6282	6365
6283		break :result try cg.buildPointerOffset(operand, 0, .new);
	6366	break :result operand;
6284	6367	};
6285	6368	return cg.finishAir(inst, result, &.{ty_op.operand});
6286	6369	}
...	...	@@ -6301,12 +6384,11 @@ fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
6301	6384	return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});
6302	6385	};
6303	6386
6304		try cg.emitWValue(operand);
	6387	try cg.emitMemBase(operand);
6305	6388	try cg.addImm32(1);
6306		try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });
	6389	try cg.addMemArg(.i32_store8, .{ .offset = operand.memOffset() + offset, .alignment = .@"1" });
6307	6390
6308		const result = try cg.buildPointerOffset(operand, 0, .new);
6309		return cg.finishAir(inst, result, &.{ty_op.operand});
	6391	return cg.finishAir(inst, operand, &.{ty_op.operand});
6310	6392	}
6311	6393
6312	6394	fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -6318,16 +6400,16 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6318	6400	const result = result: {
6319	6401	if (!payload_ty.hasRuntimeBits(zcu)) {
6320	6402	const non_null_bit = try cg.allocStack(Type.u1);
6321		try cg.emitWValue(non_null_bit);
	6403	try cg.emitMemBase(non_null_bit);
6322	6404	try cg.addImm32(1);
6323		try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 });
	6405	try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.memOffset(), .alignment = .@"1" });
6324	6406	break :result non_null_bit;
6325	6407	}
6326	6408
6327	6409	const operand = try cg.resolveInst(ty_op.operand);
6328	6410	const op_ty = cg.typeOfIndex(inst);
6329	6411	if (op_ty.optionalReprIsPayload(zcu)) {
6330		break :result cg.reuseOperand(ty_op.operand, operand);
	6412	break :result operand;
6331	6413	}
6332	6414	const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6333	6415	return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)});
...	...	@@ -6335,12 +6417,11 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6335	6417
6336	6418	// Create optional type, set the non-null bit, and store the operand inside the optional type
6337	6419	const result_ptr = try cg.allocStack(op_ty);
6338		try cg.emitWValue(result_ptr);
	6420	try cg.emitMemBase(result_ptr);
6339	6421	try cg.addImm32(1);
6340		try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });
	6422	try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.memOffset() + offset, .alignment = .@"1" });
6341	6423
6342		const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);
6343		try cg.store(payload_ptr, operand, payload_ty, 0);
	6424	try cg.store(result_ptr, operand, payload_ty, .{});
6344	6425	break :result result_ptr;
6345	6426	};
6346	6427
...	...	@@ -6356,8 +6437,8 @@ fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6356	6437	const slice_ty = cg.typeOfIndex(inst);
6357	6438
6358	6439	const slice = try cg.allocStack(slice_ty);
6359		try cg.store(slice, lhs, Type.usize, 0);
6360		try cg.store(slice, rhs, Type.usize, cg.ptrSize());
	6440	try cg.store(slice, lhs, .usize, .{});
	6441	try cg.store(slice, rhs, .usize, .{ .offset = cg.ptrSize() });
6361	6442
6362	6443	return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });
6363	6444	}
...	...	@@ -6380,15 +6461,12 @@ fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6380	6461	const elem_size = elem_ty.abiSize(zcu);
6381	6462
6382	6463	// load pointer onto stack
6383		_ = try cg.load(slice, Type.usize, 0);
	6464	_ = try cg.load(slice, .usize, .{});
6384	6465
6385	6466	// calculate index into slice
6386		try cg.emitWValue(index);
6387		try cg.addImm32(@intCast(elem_size));
6388		try cg.addTag(.i32_mul);
6389		try cg.addTag(.i32_add);
	6467	try cg.emitScaledPointerOffset(index, elem_size, .add);
6390	6468
6391		const elem_result = try cg.load(.stack, elem_ty, 0);
	6469	const elem_result = try cg.load(.stack, elem_ty, .{});
6392	6470
6393	6471	return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6394	6472	}
...	...	@@ -6404,13 +6482,10 @@ fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6404	6482	const slice = try cg.resolveInst(bin_op.lhs);
6405	6483	const index = try cg.resolveInst(bin_op.rhs);
6406	6484
6407		_ = try cg.load(slice, Type.usize, 0);
	6485	_ = try cg.load(slice, .usize, .{});
6408	6486
6409	6487	// calculate index into slice
6410		try cg.emitWValue(index);
6411		try cg.addImm32(@intCast(elem_size));
6412		try cg.addTag(.i32_mul);
6413		try cg.addTag(.i32_add);
	6488	try cg.emitScaledPointerOffset(index, elem_size, .add);
6414	6489
6415	6490	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6416	6491	}
...	...	@@ -6422,12 +6497,12 @@ fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6422	6497	}
6423	6498
6424	6499	fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {
6425		const ptr = try cg.load(operand, Type.usize, 0);
	6500	const ptr = try cg.load(operand, .usize, .{});
6426	6501	return ptr.toLocal(cg, Type.usize);
6427	6502	}
6428	6503
6429	6504	fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {
6430		const len = try cg.load(operand, Type.usize, cg.ptrSize());
	6505	const len = try cg.load(operand, .usize, .{ .offset = cg.ptrSize() });
6431	6506	return len.toLocal(cg, Type.usize);
6432	6507	}
6433	6508
...	...	@@ -6442,11 +6517,11 @@ fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6442	6517	// create a slice on the stack
6443	6518	const slice_local = try cg.allocStack(slice_ty);
6444	6519
6445		try cg.store(slice_local, operand, Type.usize, 0);
	6520	try cg.store(slice_local, operand, .usize, .{});
6446	6521
6447	6522	// store the length of the array in the slice
6448	6523	const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
6449		try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize());
	6524	try cg.store(slice_local, .{ .imm32 = array_len }, .usize, .{ .offset = cg.ptrSize() });
6450	6525
6451	6526	return cg.finishAir(inst, slice_local, &.{ty_op.operand});
6452	6527	}
...	...	@@ -6463,18 +6538,15 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6463	6538
6464	6539	// load pointer onto the stack
6465	6540	if (ptr_ty.isSlice(zcu)) {
6466		_ = try cg.load(ptr, Type.usize, 0);
	6541	_ = try cg.load(ptr, .usize, .{});
6467	6542	} else {
6468		try cg.lowerToStack(ptr);
	6543	try cg.emitWValue(ptr);
6469	6544	}
6470	6545
6471	6546	// calculate index into slice
6472		try cg.emitWValue(index);
6473		try cg.addImm32(@intCast(elem_size));
6474		try cg.addTag(.i32_mul);
6475		try cg.addTag(.i32_add);
	6547	try cg.emitScaledPointerOffset(index, elem_size, .add);
6476	6548
6477		const elem_result = try cg.load(.stack, elem_ty, 0);
	6549	const elem_result = try cg.load(.stack, elem_ty, .{});
6478	6550
6479	6551	return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6480	6552	}
...	...	@@ -6493,21 +6565,18 @@ fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6493	6565
6494	6566	// load pointer onto the stack
6495	6567	if (ptr_ty.isSlice(zcu)) {
6496		_ = try cg.load(ptr, Type.usize, 0);
	6568	_ = try cg.load(ptr, .usize, .{});
6497	6569	} else {
6498		try cg.lowerToStack(ptr);
	6570	try cg.emitWValue(ptr);
6499	6571	}
6500	6572
6501	6573	// calculate index into ptr
6502		try cg.emitWValue(index);
6503		try cg.addImm32(@intCast(elem_size));
6504		try cg.addTag(.i32_mul);
6505		try cg.addTag(.i32_add);
	6574	try cg.emitScaledPointerOffset(index, elem_size, .add);
6506	6575
6507	6576	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6508	6577	}
6509	6578
6510		fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerError!void {
	6579	fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: PointerOffsetOp) InnerError!void {
6511	6580	const zcu = cg.pt.zcu;
6512	6581	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6513	6582	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
...	...	@@ -6520,27 +6589,8 @@ fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerE
6520	6589	else => ptr_ty.childType(zcu),
6521	6590	};
6522	6591
6523		try cg.lowerToStack(ptr);
6524		try cg.emitWValue(offset);
6525
6526		switch (cg.ptr_size) {
6527		.wasm32 => {
6528		try cg.addImm32(@intCast(pointee_ty.abiSize(zcu)));
6529		try cg.addTag(.i32_mul);
6530		try cg.addTag(switch (op) {
6531		.add => .i32_add,
6532		.sub => .i32_sub,
6533		});
6534		},
6535		.wasm64 => {
6536		try cg.addImm64(pointee_ty.abiSize(zcu));
6537		try cg.addTag(.i64_mul);
6538		try cg.addTag(switch (op) {
6539		.add => .i64_add,
6540		.sub => .i64_sub,
6541		});
6542		},
6543		}
	6592	try cg.emitWValue(ptr);
	6593	try cg.emitScaledPointerOffset(offset, pointee_ty.abiSize(zcu), op);
6544	6594
6545	6595	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6546	6596	}
...	...	@@ -6552,11 +6602,6 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
6552	6602	const ptr = try cg.resolveInst(bin_op.lhs);
6553	6603	const ptr_ty = cg.typeOf(bin_op.lhs);
6554	6604	const value = try cg.resolveInst(bin_op.rhs);
6555		const len = switch (ptr_ty.ptrSize(zcu)) {
6556		.slice => try cg.sliceLen(ptr),
6557		.one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }),
6558		.c, .many => unreachable,
6559		};
6560	6605
6561	6606	const elem_ty = if (ptr_ty.ptrSize(zcu) == .one)
6562	6607	ptr_ty.childType(zcu).childType(zcu)
...	...	@@ -6567,26 +6612,38 @@ fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
6567	6612	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6568	6613	}
6569	6614
6570		const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty);
	6615	var dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty);
	6616	defer if (ptr_ty.isSlice(zcu)) dst_ptr.free(cg);
	6617	var len = switch (ptr_ty.ptrSize(zcu)) {
	6618	.slice => try cg.sliceLen(ptr),
	6619	.one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }),
	6620	.c, .many => unreachable,
	6621	};
	6622	defer len.free(cg);
6571	6623	try cg.memset(elem_ty, dst_ptr, len, value);
6572	6624
6573	6625	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6574	6626	}
6575	6627
6576	6628	/// Sets a region of memory at `ptr` to the value of `value`
6577		/// When the user has enabled the bulk_memory feature, we lower
	6629	/// When the user has enabled the bulk_memory_opt feature, we lower
6578	6630	/// this to wasm's memset instruction. When the feature is not present,
6579	6631	/// we implement it manually.
6580	6632	fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {
6581	6633	const zcu = cg.pt.zcu;
6582	6634	const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6583
6584		// When bulk_memory is enabled, we lower it to wasm's memset instruction.
	6635	const len_known_neq_0 = switch (len) {
	6636	.imm32 => \|imm\| if (imm != 0) true else return,
	6637	.imm64 => \|imm\| if (imm != 0) true else return,
	6638	else => false,
	6639	};
	6640	// When bulk_memory_opt is enabled, we lower it to wasm's memset instruction.
6585	6641	// If not, we lower it ourselves.
6586		if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) {
	6642	if (cg.target.cpu.has(.wasm, .bulk_memory_opt) and abi_size == 1) {
6587	6643	const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);
	6644	const emit_check = !(len0_ok or len_known_neq_0);
6588	6645
6589		if (!len0_ok) {
	6646	if (emit_check) {
6590	6647	try cg.startBlock(.block, .empty);
6591	6648
6592	6649	// Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is
...	...	@@ -6602,23 +6659,25 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6602	6659	try cg.addLabel(.br_if, 0);
6603	6660	}
6604	6661
6605		try cg.lowerToStack(ptr);
	6662	try cg.emitWValue(ptr);
6606	6663	try cg.emitWValue(value);
6607	6664	try cg.emitWValue(len);
6608	6665	try cg.addExtended(.memory_fill);
6609	6666
6610		if (!len0_ok) {
	6667	if (emit_check) {
6611	6668	try cg.endBlock();
6612	6669	}
6613	6670
6614	6671	return;
6615	6672	}
6616	6673
	6674	var owned_final_len: WValue = .none;
	6675	defer owned_final_len.free(cg);
6617	6676	const final_len: WValue = switch (len) {
6618	6677	.imm32 => \|val\| .{ .imm32 = val * abi_size },
6619	6678	.imm64 => \|val\| .{ .imm64 = val * abi_size },
6620	6679	else => if (abi_size != 1) blk: {
6621		const new_len = try cg.ensureAllocLocal(Type.usize);
	6680	owned_final_len = try cg.allocLocal(Type.usize);
6622	6681	try cg.emitWValue(len);
6623	6682	switch (cg.ptr_size) {
6624	6683	.wasm32 => {
...	...	@@ -6630,24 +6689,26 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6630	6689	try cg.addTag(.i64_mul);
6631	6690	},
6632	6691	}
6633		try cg.addLocal(.local_set, new_len.local.value);
6634		break :blk new_len;
	6692	try cg.addLocal(.local_set, owned_final_len.local);
	6693	break :blk owned_final_len;
6635	6694	} else len,
6636	6695	};
6637	6696
6638	6697	var end_ptr = try cg.allocLocal(Type.usize);
6639	6698	defer end_ptr.free(cg);
6640		var new_ptr = try cg.buildPointerOffset(ptr, 0, .new);
	6699	var new_ptr = try cg.allocLocal(Type.usize);
6641	6700	defer new_ptr.free(cg);
	6701	try cg.emitWValue(ptr);
	6702	try cg.addLocal(.local_set, new_ptr.local);
6642	6703
6643	6704	// get the loop conditional: if current pointer address equals final pointer's address
6644		try cg.lowerToStack(ptr);
	6705	try cg.emitWValue(ptr);
6645	6706	try cg.emitWValue(final_len);
6646	6707	switch (cg.ptr_size) {
6647	6708	.wasm32 => try cg.addTag(.i32_add),
6648	6709	.wasm64 => try cg.addTag(.i64_add),
6649	6710	}
6650		try cg.addLocal(.local_set, end_ptr.local.value);
	6711	try cg.addLocal(.local_set, end_ptr.local);
6651	6712
6652	6713	// outer block to jump to when loop is done
6653	6714	try cg.startBlock(.block, .empty);
...	...	@@ -6663,7 +6724,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6663	6724	try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
6664	6725
6665	6726	// store the value at the current position of the pointer
6666		try cg.store(new_ptr, value, elem_ty, 0);
	6727	try cg.store(new_ptr, value, elem_ty, .{});
6667	6728
6668	6729	// move the pointer to the next element
6669	6730	try cg.emitWValue(new_ptr);
...	...	@@ -6677,7 +6738,7 @@ fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue)
6677	6738	try cg.addTag(.i64_add);
6678	6739	},
6679	6740	}
6680		try cg.addLocal(.local_set, new_ptr.local.value);
	6741	try cg.addLocal(.local_set, new_ptr.local);
6681	6742
6682	6743	// end of loop
6683	6744	try cg.addLabel(.br, 0); // jump to start of loop
...	...	@@ -6696,11 +6757,8 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6696	6757	const elem_size = elem_ty.abiSize(zcu);
6697	6758
6698	6759	if (isByRef(array_ty, zcu, cg.target)) {
6699		try cg.lowerToStack(array);
6700		try cg.emitWValue(index);
6701		try cg.addImm32(@intCast(elem_size));
6702		try cg.addTag(.i32_mul);
6703		try cg.addTag(.i32_add);
	6760	try cg.emitWValue(array);
	6761	try cg.emitScaledPointerOffset(index, elem_size, .add);
6704	6762	} else {
6705	6763	assert(array_ty.zigTypeTag(zcu) == .vector);
6706	6764
...	...	@@ -6726,14 +6784,11 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6726	6784	},
6727	6785	else => {
6728	6786	const stack_vec = try cg.allocStack(array_ty);
6729		try cg.store(stack_vec, array, array_ty, 0);
	6787	try cg.store(stack_vec, array, array_ty, .{});
6730	6788
6731	6789	// Is a non-unrolled vector (v128)
6732		try cg.lowerToStack(stack_vec);
6733		try cg.emitWValue(index);
6734		try cg.addImm32(@intCast(elem_size));
6735		try cg.addTag(.i32_mul);
6736		try cg.addTag(.i32_add);
	6790	try cg.emitWValue(stack_vec);
	6791	try cg.emitScaledPointerOffset(index, elem_size, .add);
6737	6792	},
6738	6793	}
6739	6794	}
...	...	@@ -6741,7 +6796,7 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6741	6796	const result = if (isByRef(elem_ty, zcu, cg.target))
6742	6797	.stack
6743	6798	else
6744		try cg.load(.stack, elem_ty, 0);
	6799	try cg.load(.stack, elem_ty, .{});
6745	6800	return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6746	6801	}
6747	6802
...	...	@@ -6765,13 +6820,14 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6765	6820	64 => @backingInt(std.wasm.SimdOpcode.v128_load64_splat),
6766	6821	else => break :blk, // Cannot make use of simd-instructions
6767	6822	};
6768		try cg.emitWValue(operand);
	6823	try cg.emitMemBase(operand);
6769	6824	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6770	6825	// stores as := opcode, offset, alignment (opcode::memarg)
6771		try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
6772		opcode,
6773		operand.offset(),
6774		@intCast(elem_ty.abiAlignment(zcu).toByteUnits().?),
	6826	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 4);
	6827	cg.mir_extra.appendAssumeCapacity(opcode);
	6828	_ = try cg.addExtraAssumeCapacity(Mir.MemArg{
	6829	.offset = operand.memOffset(),
	6830	.alignment = elem_ty.abiAlignment(zcu),
6775	6831	});
6776	6832	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6777	6833	return cg.finishAir(inst, .stack, &.{ty_op.operand});
...	...	@@ -6800,7 +6856,7 @@ fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6800	6856	var index: usize = 0;
6801	6857	var offset: u32 = 0;
6802	6858	while (index < vector_len) : (index += 1) {
6803		try cg.store(result, operand, elem_ty, offset);
	6859	try cg.store(result, operand, elem_ty, .{ .offset = offset });
6804	6860	offset += elem_byte_size;
6805	6861	}
6806	6862
...	...	@@ -6837,12 +6893,12 @@ fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6837	6893
6838	6894	const dest_alloc = try cg.allocStack(result_ty);
6839	6895	for (mask, 0..) \|mask_elem, out_idx\| {
6840		try cg.emitWValue(dest_alloc);
	6896	try cg.emitMemBase(dest_alloc);
6841	6897	const elem_val = switch (mask_elem.unwrap()) {
6842		.elem => \|idx\| try cg.load(operand, elem_ty, @intCast(elem_size * idx)),
	6898	.elem => \|idx\| try cg.load(operand, elem_ty, .{ .offset = elem_size * idx }),
6843	6899	.value => \|val\| try cg.lowerConstant(.fromInterned(val)),
6844	6900	};
6845		try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
	6901	try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.memOffset() + elem_size * out_idx });
6846	6902	}
6847	6903	return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
6848	6904	}
...	...	@@ -6905,13 +6961,13 @@ fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6905	6961
6906	6962	const dest_alloc = try cg.allocStack(result_ty);
6907	6963	for (mask, 0..) \|mask_elem, out_idx\| {
6908		try cg.emitWValue(dest_alloc);
	6964	try cg.emitMemBase(dest_alloc);
6909	6965	const elem_val = switch (mask_elem.unwrap()) {
6910		.a_elem => \|idx\| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)),
6911		.b_elem => \|idx\| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)),
	6966	.a_elem => \|idx\| try cg.load(operand_a, elem_ty, .{ .offset = elem_size * idx }),
	6967	.b_elem => \|idx\| try cg.load(operand_b, elem_ty, .{ .offset = elem_size * idx }),
6912	6968	.undef => try cg.emitUndefined(elem_ty),
6913	6969	};
6914		try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
	6970	try cg.store(.stack, elem_val, elem_ty, .{ .offset = dest_alloc.memOffset() + elem_size * out_idx });
6915	6971	}
6916	6972	return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
6917	6973	}
...	...	@@ -6937,39 +6993,18 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6937	6993	.array, .vector => {
6938	6994	const result = try cg.allocStack(result_ty);
6939	6995	const elem_ty = result_ty.childType(zcu);
6940		const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
	6996	const elem_size = elem_ty.abiSize(zcu);
6941	6997	const sentinel = result_ty.sentinel(zcu);
6942	6998
6943		// When the element type is by reference, we must copy the entire
6944		// value. It is therefore safer to move the offset pointer and store
6945		// each value individually, instead of using store offsets.
6946		if (isByRef(elem_ty, zcu, cg.target)) {
6947		// copy stack pointer into a temporary local, which is
6948		// moved for each element to store each value in the right position.
6949		const offset = try cg.buildPointerOffset(result, 0, .new);
6950		for (elements, 0..) \|elem, elem_index\| {
6951		const elem_val = try cg.resolveInst(elem);
6952		try cg.store(offset, elem_val, elem_ty, 0);
6953
6954		if (elem_index < elements.len - 1 or sentinel != null) {
6955		_ = try cg.buildPointerOffset(offset, elem_size, .modify);
6956		}
6957		}
6958		if (sentinel) \|s\| {
6959		const val = try cg.resolveValue(s);
6960		try cg.store(offset, val, elem_ty, 0);
6961		}
6962		} else {
6963		var offset: u32 = 0;
6964		for (elements) \|elem\| {
6965		const elem_val = try cg.resolveInst(elem);
6966		try cg.store(result, elem_val, elem_ty, offset);
6967		offset += elem_size;
6968		}
6969		if (sentinel) \|s\| {
6970		const val = try cg.resolveValue(s);
6971		try cg.store(result, val, elem_ty, offset);
6972		}
	6999	var offset: u64 = 0;
	7000	for (elements) \|elem\| {
	7001	const elem_val = try cg.resolveInst(elem);
	7002	try cg.store(result, elem_val, elem_ty, .{ .offset = offset });
	7003	offset += elem_size;
	7004	}
	7005	if (sentinel) \|s\| {
	7006	const val = try cg.resolveValue(s);
	7007	try cg.store(result, val, elem_ty, .{ .offset = offset });
6973	7008	}
6974	7009	break :result_value result;
6975	7010	},
...	...	@@ -6982,10 +7017,9 @@ fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6982	7017
6983	7018	const elem_ty = result_ty.fieldType(elem_index, zcu);
6984	7019	const field_offset = result_ty.structFieldOffset(elem_index, zcu);
6985		const offset = try cg.buildPointerOffset(result, field_offset, .new);
6986	7020
6987	7021	const value = try cg.resolveInst(elem);
6988		try cg.store(offset, value, elem_ty, 0);
	7022	try cg.store(result, value, elem_ty, .{ .offset = field_offset });
6989	7023	}
6990	7024
6991	7025	break :result_value result;
...	...	@@ -7032,24 +7066,19 @@ fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7032	7066	const result_ptr = try cg.allocStack(union_ty);
7033	7067	const payload = try cg.resolveInst(extra.init);
7034	7068	if (layout.tag_align.compare(.gte, layout.payload_align)) {
7035		if (isByRef(field_ty, zcu, cg.target)) {
7036		const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);
7037		try cg.store(payload_ptr, payload, field_ty, 0);
7038		} else {
7039		try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size));
7040		}
	7069	try cg.store(result_ptr, payload, field_ty, .{ .offset = layout.tag_size });
7041	7070
7042	7071	if (layout.tag_size > 0) {
7043		try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), 0);
	7072	try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), .{});
7044	7073	}
7045	7074	} else {
7046		try cg.store(result_ptr, payload, field_ty, 0);
	7075	try cg.store(result_ptr, payload, field_ty, .{});
7047	7076	if (layout.tag_size > 0) {
7048	7077	try cg.store(
7049	7078	result_ptr,
7050	7079	tag_int,
7051	7080	.fromInterned(union_obj.enum_tag_type),
7052		@intCast(layout.payload_size),
	7081	.{ .offset = layout.payload_size },
7053	7082	);
7054	7083	}
7055	7084	}
...	...	@@ -7095,7 +7124,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7095	7124	const union_ptr = try cg.resolveInst(bin_op.lhs);
7096	7125	const new_tag = try cg.resolveInst(bin_op.rhs);
7097	7126	if (layout.payload_size == 0) {
7098		try cg.store(union_ptr, new_tag, tag_ty, 0);
	7127	try cg.store(union_ptr, new_tag, tag_ty, .{});
7099	7128	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7100	7129	}
7101	7130
...	...	@@ -7104,7 +7133,7 @@ fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7104	7133	const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {
7105	7134	break :blk @intCast(layout.payload_size);
7106	7135	} else 0;
7107		try cg.store(union_ptr, new_tag, tag_ty, offset);
	7136	try cg.store(union_ptr, new_tag, tag_ty, .{ .offset = offset });
7108	7137	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7109	7138	}
7110	7139
...	...	@@ -7124,7 +7153,7 @@ fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7124	7153	@intCast(layout.payload_size)
7125	7154	else
7126	7155	0;
7127		const result = try cg.load(operand, tag_ty, offset);
	7156	const result = try cg.load(operand, tag_ty, .{ .offset = offset });
7128	7157	return cg.finishAir(inst, result, &.{ty_op.operand});
7129	7158	}
7130	7159
...	...	@@ -7140,17 +7169,15 @@ fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void
7140	7169	try cg.store(
7141	7170	operand,
7142	7171	.{ .imm32 = 0 },
7143		Type.anyerror,
7144		@intCast(errUnionErrorOffset(payload_ty, zcu)),
	7172	.anyerror,
	7173	.{ .offset = errUnionErrorOffset(payload_ty, zcu) },
7145	7174	);
7146	7175
7147		const result = result: {
7148		if (!payload_ty.hasRuntimeBits(zcu)) {
7149		break :result cg.reuseOperand(ty_op.operand, operand);
7150		}
	7176	if (!payload_ty.hasRuntimeBits(zcu)) {
	7177	return cg.finishAir(inst, operand, &.{ty_op.operand});
	7178	}
7151	7179
7152		break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new);
7153		};
	7180	const result = try cg.offsetPointer(operand, errUnionPayloadOffset(payload_ty, zcu));
7154	7181	return cg.finishAir(inst, result, &.{ty_op.operand});
7155	7182	}
7156	7183
...	...	@@ -7175,16 +7202,23 @@ fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7175	7202	},
7176	7203	};
7177	7204
7178		const result = if (field_offset != 0) result: {
7179		const base = try cg.buildPointerOffset(field_ptr, 0, .new);
7180		try cg.addLocal(.local_get, base.local.value);
7181		try cg.addImm32(@intCast(field_offset));
7182		try cg.addTag(.i32_sub);
7183		try cg.addLocal(.local_set, base.local.value);
7184		break :result base;
7185		} else cg.reuseOperand(extra.field_ptr, field_ptr);
	7205	if (field_offset == 0) {
	7206	return cg.finishAir(inst, field_ptr, &.{extra.field_ptr});
	7207	}
7186	7208
7187		return cg.finishAir(inst, result, &.{extra.field_ptr});
	7209	// maybe use offsetPointer? sub looks better in disasm
	7210	try cg.emitWValue(field_ptr);
	7211	switch (cg.ptr_size) {
	7212	.wasm32 => {
	7213	try cg.addImm32(@intCast(field_offset));
	7214	try cg.addTag(.i32_sub);
	7215	},
	7216	.wasm64 => {
	7217	try cg.addImm64(field_offset);
	7218	try cg.addTag(.i64_sub);
	7219	},
	7220	}
	7221	return cg.finishAir(inst, .stack, &.{extra.field_ptr});
7188	7222	}
7189	7223
7190	7224	fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {
...	...	@@ -7204,14 +7238,14 @@ fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7204	7238	const ptr_elem_ty = dst_ty.childType(zcu);
7205	7239	const src = try cg.resolveInst(bin_op.rhs);
7206	7240	const src_ty = cg.typeOf(bin_op.rhs);
7207		const len = switch (dst_ty.ptrSize(zcu)) {
	7241	var len = switch (dst_ty.ptrSize(zcu)) {
7208	7242	.slice => blk: {
7209	7243	const slice_len = try cg.sliceLen(dst);
7210	7244	if (ptr_elem_ty.abiSize(zcu) != 1) {
7211	7245	try cg.emitWValue(slice_len);
7212	7246	try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
7213	7247	try cg.addTag(.i32_mul);
7214		try cg.addLocal(.local_set, slice_len.local.value);
	7248	try cg.addLocal(.local_set, slice_len.local);
7215	7249	}
7216	7250	break :blk slice_len;
7217	7251	},
...	...	@@ -7220,8 +7254,11 @@ fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7220	7254	}),
7221	7255	.c, .many => unreachable,
7222	7256	};
7223		const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
7224		const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
	7257	defer len.free(cg);
	7258	var dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
	7259	defer if (dst_ty.isSlice(zcu)) dst_ptr.free(cg);
	7260	var src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
	7261	defer if (src_ty.isSlice(zcu)) src_ptr.free(cg);
7225	7262	try cg.memcpy(dst_ptr, src_ptr, len);
7226	7263
7227	7264	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
...	...	@@ -7235,14 +7272,14 @@ fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7235	7272	const ptr_elem_ty = dst_ty.childType(zcu);
7236	7273	const src = try cg.resolveInst(bin_op.rhs);
7237	7274	const src_ty = cg.typeOf(bin_op.rhs);
7238		const len = switch (dst_ty.ptrSize(zcu)) {
	7275	var len = switch (dst_ty.ptrSize(zcu)) {
7239	7276	.slice => blk: {
7240	7277	const slice_len = try cg.sliceLen(dst);
7241	7278	if (ptr_elem_ty.abiSize(zcu) != 1) {
7242	7279	try cg.emitWValue(slice_len);
7243	7280	try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
7244	7281	try cg.addTag(.i32_mul);
7245		try cg.addLocal(.local_set, slice_len.local.value);
	7282	try cg.addLocal(.local_set, slice_len.local);
7246	7283	}
7247	7284	break :blk slice_len;
7248	7285	},
...	...	@@ -7251,8 +7288,11 @@ fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7251	7288	}),
7252	7289	.c, .many => unreachable,
7253	7290	};
7254		const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
7255		const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
	7291	defer len.free(cg);
	7292	var dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
	7293	defer if (dst_ty.isSlice(zcu)) dst_ptr.free(cg);
	7294	var src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
	7295	defer if (src_ty.isSlice(zcu)) src_ptr.free(cg);
7256	7296	try cg.memmove(dst_ptr, src_ptr, len);
7257	7297
7258	7298	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
...	...	@@ -7283,19 +7323,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7283	7323	// Lowers to a i32.const or i64.const with the error table memory address.
7284	7324	cg.error_name_table_ref_count += 1;
7285	7325	try cg.addTag(.error_name_table_ref);
7286		try cg.emitWValue(operand);
7287		switch (cg.ptr_size) {
7288		.wasm32 => {
7289		try cg.addImm32(@intCast(abi_size));
7290		try cg.addTag(.i32_mul);
7291		try cg.addTag(.i32_add);
7292		},
7293		.wasm64 => {
7294		try cg.addImm64(abi_size);
7295		try cg.addTag(.i64_mul);
7296		try cg.addTag(.i64_add);
7297		},
7298		}
	7326	try cg.emitScaledPointerOffset(operand, abi_size, .add);
7299	7327
7300	7328	return cg.finishAir(inst, .stack, &.{un_op});
7301	7329	}
...	...	@@ -7303,7 +7331,7 @@ fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7303	7331	fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {
7304	7332	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7305	7333	const slice_ptr = try cg.resolveInst(ty_op.operand);
7306		const result = try cg.buildPointerOffset(slice_ptr, offset, .new);
	7334	const result = try cg.offsetPointer(slice_ptr, offset);
7307	7335	return cg.finishAir(inst, result, &.{ty_op.operand});
7308	7336	}
7309	7337
...	...	@@ -7332,7 +7360,8 @@ fn airDbgVar(
7332	7360	) InnerError!void {
7333	7361	_ = is_ptr;
7334	7362	_ = local_tag;
7335		return cg.finishAir(inst, .none, &.{});
	7363	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
	7364	return cg.finishAir(inst, .none, &.{pl_op.operand});
7336	7365	}
7337	7366
7338	7367	fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -7340,8 +7369,13 @@ fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7340	7369	const body = unwrapped_try.else_body;
7341	7370	const err_union = try cg.resolveInst(unwrapped_try.error_union);
7342	7371	const err_union_ty = cg.typeOf(unwrapped_try.error_union);
7343		const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);
7344		return cg.finishAir(inst, result, &.{unwrapped_try.error_union});
	7372	const result = try lowerTry(cg, inst, unwrapped_try.error_union, err_union, body, err_union_ty, false);
	7373	return cg.finishAirWithBranchDeaths(
	7374	inst,
	7375	result,
	7376	&.{unwrapped_try.error_union},
	7377	cg.liveness.getCondBr(inst).then_deaths,
	7378	);
7345	7379	}
7346	7380
7347	7381	fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
...	...	@@ -7350,20 +7384,26 @@ fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7350	7384	const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr);
7351	7385	const body = unwrapped_try.else_body;
7352	7386	const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu);
7353		const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);
7354		return cg.finishAir(inst, result, &.{unwrapped_try.error_union_ptr});
	7387	const result = try lowerTry(cg, inst, unwrapped_try.error_union_ptr, err_union_ptr, body, err_union_ty, true);
	7388	return cg.finishAirWithBranchDeaths(
	7389	inst,
	7390	result,
	7391	&.{unwrapped_try.error_union_ptr},
	7392	cg.liveness.getCondBr(inst).then_deaths,
	7393	);
7355	7394	}
7356	7395
7357	7396	fn lowerTry(
7358	7397	cg: *CodeGen,
7359	7398	inst: Air.Inst.Index,
	7399	operand_ref: Air.Inst.Ref,
7360	7400	err_union: WValue,
7361	7401	body: []const Air.Inst.Index,
7362	7402	err_union_ty: Type,
7363	7403	operand_is_ptr: bool,
7364	7404	) InnerError!WValue {
7365		_ = inst;
7366	7405	const zcu = cg.pt.zcu;
	7406	const branch_liveness = cg.liveness.getCondBr(inst);
7367	7407
7368	7408	const pl_ty = err_union_ty.errorUnionPayload(zcu);
7369	7409	const pl_has_bits = pl_ty.hasRuntimeBits(zcu);
...	...	@@ -7373,23 +7413,26 @@ fn lowerTry(
7373	7413	try cg.startBlock(.block, .empty);
7374	7414
7375	7415	// check if the error tag is set for the error union.
7376		try cg.emitWValue(err_union);
7377	7416	if (pl_has_bits or operand_is_ptr) {
7378		const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
	7417	try cg.emitMemBase(err_union);
	7418	const err_offset = errUnionErrorOffset(pl_ty, zcu);
7379	7419	try cg.addMemArg(.i32_load16_u, .{
7380		.offset = err_union.offset() + err_offset,
7381		.alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
	7420	.offset = err_union.memOffset() + err_offset,
	7421	.alignment = Type.anyerror.abiAlignment(zcu),
7382	7422	});
	7423	} else {
	7424	try cg.emitWValue(err_union);
7383	7425	}
7384	7426	try cg.addTag(.i32_eqz);
7385	7427	try cg.addLabel(.br_if, 0); // jump out of block when error is '0'
7386	7428
7387		try cg.branches.append(cg.gpa, .{});
7388		defer {
7389		var branch = cg.branches.pop().?;
7390		branch.deinit(cg.gpa);
	7429	{
	7430	try cg.pushBranch();
	7431	defer cg.popBranch();
	7432	if (cg.liveness.operandDies(inst, 0)) cg.processDeath(operand_ref);
	7433	for (branch_liveness.else_deaths) \|death\| cg.processDeath(death.toRef());
	7434	try cg.genBody(body);
7391	7435	}
7392		try cg.genBody(body);
7393	7436	try cg.endBlock();
7394	7437	}
7395	7438
...	...	@@ -7398,12 +7441,11 @@ fn lowerTry(
7398	7441	return .none;
7399	7442	}
7400	7443
7401		const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
	7444	const pl_offset = errUnionPayloadOffset(pl_ty, zcu);
7402	7445	if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {
7403		return buildPointerOffset(cg, err_union, pl_offset, .new);
	7446	return cg.offsetPointer(err_union, pl_offset);
7404	7447	}
7405		const payload = try cg.load(err_union, pl_ty, pl_offset);
7406		return payload.toLocal(cg, pl_ty);
	7448	return cg.load(err_union, pl_ty, .{ .offset = pl_offset });
7407	7449	}
7408	7450
7409	7451	/// Calls a compiler-rt intrinsic by creating an undefined symbol,
...	...	@@ -7429,7 +7471,7 @@ fn callIntrinsic(
7429	7471	// and emit it as our first argument
7430	7472	const sret = if (want_sret_param) blk: {
7431	7473	const sret_local = try cg.allocStack(return_type);
7432		try cg.lowerToStack(sret_local);
	7474	try cg.emitWValue(sret_local);
7433	7475	break :blk sret_local;
7434	7476	} else .none;
7435	7477
...	...	@@ -7457,21 +7499,9 @@ fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7457	7499	const enum_ty = cg.typeOf(un_op);
7458	7500
7459	7501	try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });
7460		try cg.lowerToStack(operand);
	7502	try cg.emitWValue(operand);
7461	7503	try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
7462
7463		switch (cg.ptr_size) {
7464		.wasm32 => {
7465		try cg.addImm32(@intCast(8));
7466		try cg.addTag(.i32_mul);
7467		try cg.addTag(.i32_add);
7468		},
7469		.wasm64 => {
7470		try cg.addImm64(8);
7471		try cg.addTag(.i64_mul);
7472		try cg.addTag(.i64_add);
7473		},
7474		}
	7504	try cg.emitScaledPointerOffset(.stack, 8, .add);
7475	7505
7476	7506	return cg.finishAir(inst, .stack, &.{un_op});
7477	7507	}
...	...	@@ -7481,7 +7511,7 @@ fn airIsNamedEnumValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7481	7511	const operand = try cg.resolveInst(un_op);
7482	7512	const enum_ty = cg.typeOf(un_op);
7483	7513
7484		try cg.lowerToStack(operand);
	7514	try cg.emitWValue(operand);
7485	7515	try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
7486	7516	try cg.addImm32(~@as(u32, 0));
7487	7517	try cg.addTag(.i32_ne);
...	...	@@ -7559,13 +7589,13 @@ fn airErrorSetHasValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7559	7589	// 'false' branch (i.e. error set does not have value
7560	7590	// ensure we set local to 0 in case the local was re-used.
7561	7591	try cg.addImm32(0);
7562		try cg.addLocal(.local_set, result.local.value);
	7592	try cg.addLocal(.local_set, result.local);
7563	7593	try cg.addLabel(.br, 1);
7564	7594	try cg.endBlock();
7565	7595
7566	7596	// 'true' branch
7567	7597	try cg.addImm32(1);
7568		try cg.addLocal(.local_set, result.local.value);
	7598	try cg.addLocal(.local_set, result.local);
7569	7599	try cg.addLabel(.br, 0);
7570	7600	try cg.endBlock();
7571	7601
...	...	@@ -7591,13 +7621,14 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7591	7621	const expected_val = try cg.resolveInst(extra.expected_value);
7592	7622	const new_val = try cg.resolveInst(extra.new_value);
7593	7623
7594		const cmp_result = try cg.allocLocal(Type.bool);
	7624	var cmp_result = try cg.allocLocal(Type.bool);
	7625	defer cmp_result.free(cg);
7595	7626
7596		const ptr_val = if (cg.useAtomicFeature()) val: {
	7627	var ptr_val = if (cg.useAtomicFeature()) val: {
7597	7628	const val_local = try cg.allocLocal(ty);
7598		try cg.emitWValue(ptr_operand);
7599		try cg.lowerToStack(expected_val);
7600		try cg.lowerToStack(new_val);
	7629	try cg.emitMemBase(ptr_operand);
	7630	try cg.emitWValue(expected_val);
	7631	try cg.emitWValue(new_val);
7601	7632	try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) {
7602	7633	1 => .i32_atomic_rmw8_cmpxchg_u,
7603	7634	2 => .i32_atomic_rmw16_cmpxchg_u,
...	...	@@ -7605,26 +7636,26 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7605	7636	8 => .i32_atomic_rmw_cmpxchg,
7606	7637	else => \|size\| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),
7607	7638	}, .{
7608		.offset = ptr_operand.offset(),
7609		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	7639	.offset = ptr_operand.memOffset(),
	7640	.alignment = ty.abiAlignment(zcu),
7610	7641	});
7611		try cg.addLocal(.local_tee, val_local.local.value);
	7642	try cg.addLocal(.local_tee, val_local.local);
7612	7643	_ = try cg.intCmp(int_ty, .eq, .stack, expected_val);
7613		try cg.addLocal(.local_set, cmp_result.local.value);
	7644	try cg.addLocal(.local_set, cmp_result.local);
7614	7645	break :val val_local;
7615	7646	} else val: {
7616	7647	if (ty.abiSize(zcu) > 8) {
7617	7648	return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{});
7618	7649	}
7619		const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, 0), cg, ty);
	7650	const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, .{}), cg, ty);
7620	7651
7621		try cg.lowerToStack(ptr_operand);
7622		try cg.lowerToStack(new_val);
	7652	try cg.emitMemBase(ptr_operand);
	7653	try cg.emitWValue(new_val);
7623	7654	try cg.emitWValue(ptr_val);
7624	7655	_ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);
7625		try cg.addLocal(.local_tee, cmp_result.local.value);
	7656	try cg.addLocal(.local_tee, cmp_result.local);
7626	7657	try cg.addTag(.select);
7627		try cg.store(.stack, .stack, ty, 0);
	7658	try cg.store(.stack, .stack, ty, .{ .offset = ptr_operand.memOffset() });
7628	7659
7629	7660	break :val ptr_val;
7630	7661	};
...	...	@@ -7635,10 +7666,11 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7635	7666	try cg.addTag(.i32_xor);
7636	7667	try cg.addImm32(1);
7637	7668	try cg.addTag(.i32_and);
7638		const and_result = try WValue.toLocal(.stack, cg, Type.bool);
	7669	var and_result = try WValue.toLocal(.stack, cg, Type.bool);
	7670	defer and_result.free(cg);
7639	7671	const result_ptr = try cg.allocStack(result_ty);
7640		try cg.store(result_ptr, and_result, Type.bool, @as(u32, @intCast(ty.abiSize(zcu))));
7641		try cg.store(result_ptr, ptr_val, ty, 0);
	7672	try cg.store(result_ptr, and_result, .bool, .{ .offset = ty.abiSize(zcu) });
	7673	try cg.store(result_ptr, ptr_val, ty, .{});
7642	7674	break :val result_ptr;
7643	7675	} else val: {
7644	7676	try cg.addImm32(0);
...	...	@@ -7647,6 +7679,7 @@ fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7647	7679	try cg.addTag(.select);
7648	7680	break :val .stack;
7649	7681	};
	7682	defer ptr_val.free(cg);
7650	7683
7651	7684	return cg.finishAir(inst, result, &.{ extra.ptr, extra.expected_value, extra.new_value });
7652	7685	}
...	...	@@ -7665,13 +7698,13 @@ fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7665	7698	8 => .i64_atomic_load,
7666	7699	else => \|size\| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
7667	7700	};
7668		try cg.emitWValue(ptr);
	7701	try cg.emitMemBase(ptr);
7669	7702	try cg.addAtomicMemArg(tag, .{
7670		.offset = ptr.offset(),
7671		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	7703	.offset = ptr.memOffset(),
	7704	.alignment = ty.abiAlignment(zcu),
7672	7705	});
7673	7706	} else {
7674		_ = try cg.load(ptr, ty, 0);
	7707	_ = try cg.load(ptr, ty, .{});
7675	7708	}
7676	7709
7677	7710	return cg.finishAir(inst, .stack, &.{atomic_load.ptr});
...	...	@@ -7694,13 +7727,13 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7694	7727	.Min,
7695	7728	.Nand,
7696	7729	=> {
7697		const tmp = try cg.load(ptr, ty, 0);
	7730	const tmp = try cg.load(ptr, ty, .{});
7698	7731	const value = try tmp.toLocal(cg, ty);
7699	7732
7700	7733	// create a loop to cmpxchg the new value
7701	7734	try cg.startBlock(.loop, .empty);
7702	7735
7703		try cg.emitWValue(ptr);
	7736	try cg.emitMemBase(ptr);
7704	7737	try cg.emitWValue(value);
7705	7738	if (op == .Nand) {
7706	7739	const and_res = try cg.intAnd(int_ty, value, operand);
...	...	@@ -7727,16 +7760,17 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7727	7760	else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}),
7728	7761	},
7729	7762	.{
7730		.offset = ptr.offset(),
7731		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	7763	.offset = ptr.memOffset(),
	7764	.alignment = ty.abiAlignment(zcu),
7732	7765	},
7733	7766	);
7734		const select_res = try cg.allocLocal(ty);
7735		try cg.addLocal(.local_tee, select_res.local.value);
	7767	var select_res = try cg.allocLocal(ty);
	7768	defer select_res.free(cg);
	7769	try cg.addLocal(.local_tee, select_res.local);
7736	7770	_ = try cg.intCmp(int_ty, .neq, .stack, value); // leave on stack so we can use it for br_if
7737	7771
7738	7772	try cg.emitWValue(select_res);
7739		try cg.addLocal(.local_set, value.local.value);
	7773	try cg.addLocal(.local_set, value.local);
7740	7774
7741	7775	try cg.addLabel(.br_if, 0);
7742	7776	try cg.endBlock();
...	...	@@ -7745,7 +7779,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7745	7779
7746	7780	// the other operations have their own instructions for Wasm.
7747	7781	else => {
7748		try cg.emitWValue(ptr);
	7782	try cg.emitMemBase(ptr);
7749	7783	try cg.emitWValue(operand);
7750	7784	const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
7751	7785	1 => switch (op) {
...	...	@@ -7787,42 +7821,42 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7787	7821	else => \|size\| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}),
7788	7822	};
7789	7823	try cg.addAtomicMemArg(tag, .{
7790		.offset = ptr.offset(),
7791		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	7824	.offset = ptr.memOffset(),
	7825	.alignment = ty.abiAlignment(zcu),
7792	7826	});
7793	7827	return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });
7794	7828	},
7795	7829	}
7796	7830	} else {
7797		const loaded = try cg.load(ptr, ty, 0);
	7831	const loaded = try cg.load(ptr, ty, .{});
7798	7832	const result = try loaded.toLocal(cg, ty);
7799	7833
7800	7834	switch (op) {
7801	7835	.Xchg => {
7802		try cg.store(ptr, operand, ty, 0);
	7836	try cg.store(ptr, operand, ty, .{});
7803	7837	},
7804	7838	.Add,
7805	7839	.Sub,
7806	7840	=> {
7807	7841	if (ty.isAnyFloat()) {
7808	7842	const float_ty: FloatType = .fromType(cg, ty);
7809		try cg.emitWValue(ptr);
	7843	try cg.emitMemBase(ptr);
7810	7844	_ = switch (op) {
7811	7845	.Add => try cg.floatAdd(float_ty, result, operand),
7812	7846	.Sub => try cg.floatSub(float_ty, result, operand),
7813	7847	else => unreachable,
7814	7848	};
7815		try cg.store(.stack, .stack, ty, ptr.offset());
	7849	try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7816	7850	} else {
7817	7851	const int_ty: IntType = .fromType(cg, ty);
7818		try cg.emitWValue(ptr);
	7852	try cg.emitMemBase(ptr);
7819	7853	_ = switch (op) {
7820	7854	.Add => try cg.intAdd(int_ty, result, operand),
7821	7855	.Sub => try cg.intSub(int_ty, result, operand),
7822	7856	else => unreachable,
7823	7857	};
7824	7858	_ = try cg.intWrap(int_ty, .stack);
7825		try cg.store(.stack, .stack, ty, ptr.offset());
	7859	try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7826	7860	}
7827	7861	},
7828	7862	.And,
...	...	@@ -7830,39 +7864,39 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7830	7864	.Xor,
7831	7865	=> {
7832	7866	const int_ty: IntType = .fromType(cg, ty);
7833		try cg.emitWValue(ptr);
	7867	try cg.emitMemBase(ptr);
7834	7868	_ = switch (op) {
7835	7869	.And => try cg.intAnd(int_ty, result, operand),
7836	7870	.Or => try cg.intOr(int_ty, result, operand),
7837	7871	.Xor => try cg.intXor(int_ty, result, operand),
7838	7872	else => unreachable,
7839	7873	};
7840		try cg.store(.stack, .stack, ty, ptr.offset());
	7874	try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7841	7875	},
7842	7876	.Max,
7843	7877	.Min,
7844	7878	=> {
7845	7879	if (ty.isAnyFloat()) {
7846	7880	const float_ty: FloatType = .fromType(cg, ty);
7847		try cg.emitWValue(ptr);
	7881	try cg.emitMemBase(ptr);
7848	7882	try cg.emitWValue(result);
7849	7883	try cg.emitWValue(operand);
7850	7884	_ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);
7851	7885	try cg.addTag(.select);
7852		try cg.store(.stack, .stack, ty, ptr.offset());
	7886	try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7853	7887	} else {
7854	7888	const int_ty: IntType = .fromType(cg, ty);
7855		try cg.emitWValue(ptr);
	7889	try cg.emitMemBase(ptr);
7856	7890	try cg.emitWValue(result);
7857	7891	try cg.emitWValue(operand);
7858	7892	_ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);
7859	7893	try cg.addTag(.select);
7860		try cg.store(.stack, .stack, ty, ptr.offset());
	7894	try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7861	7895	}
7862	7896	},
7863	7897	.Nand => {
7864	7898	const int_ty: IntType = .fromType(cg, ty);
7865		try cg.emitWValue(ptr);
	7899	try cg.emitMemBase(ptr);
7866	7900	const and_res = try cg.intAnd(int_ty, result, operand);
7867	7901	if (int_ty.bits <= 32) {
7868	7902	try cg.addImm32(~@as(u32, 0));
...	...	@@ -7872,7 +7906,7 @@ fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7872	7906	return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7873	7907	}
7874	7908	_ = try cg.intXor(int_ty, and_res, .stack);
7875		try cg.store(.stack, .stack, ty, ptr.offset());
	7909	try cg.store(.stack, .stack, ty, .{ .offset = ptr.memOffset() });
7876	7910	},
7877	7911	}
7878	7912
...	...	@@ -7897,23 +7931,20 @@ fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7897	7931	8 => .i64_atomic_store,
7898	7932	else => \|size\| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
7899	7933	};
7900		try cg.emitWValue(ptr);
7901		try cg.lowerToStack(operand);
	7934	try cg.emitMemBase(ptr);
	7935	try cg.emitWValue(operand);
7902	7936	try cg.addAtomicMemArg(tag, .{
7903		.offset = ptr.offset(),
7904		.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
	7937	.offset = ptr.memOffset(),
	7938	.alignment = ty.abiAlignment(zcu),
7905	7939	});
7906	7940	} else {
7907		try cg.store(ptr, operand, ty, 0);
	7941	try cg.store(ptr, operand, ty, .{});
7908	7942	}
7909	7943
7910	7944	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7911	7945	}
7912	7946
7913	7947	fn airFrameAddress(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7914		if (cg.initial_stack_value == .none) {
7915		try cg.initializeStack();
7916		}
7917	7948	try cg.emitWValue(cg.bottom_stack_value);
7918	7949	return cg.finishAir(inst, .stack, &.{});
7919	7950	}
...	...	@@ -7947,6 +7978,11 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7947	7978	if (unwrapped_asm.source.len != 0) {
7948	7979	var local_map: assembly.LocalMap = .empty;
7949	7980	defer local_map.deinit(cg.gpa);
	7981	var input_locals: std.ArrayList(WValue) = try .initCapacity(cg.gpa, inputs.len);
	7982	defer {
	7983	for (input_locals.items) \|*input_local\| input_local.free(cg);
	7984	input_locals.deinit(cg.gpa);
	7985	}
7950	7986
7951	7987	{
7952	7988	var it = unwrapped_asm.iterateOutputs();
...	...	@@ -7959,7 +7995,7 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7959	7995	return cg.fail("Self-hosted wasm backend requires output constraint to be equal \"=r\"", .{});
7960	7996	}
7961	7997
7962		const gop = try local_map.getOrPutValue(cg.gpa, name, result.local.value);
	7998	const gop = try local_map.getOrPutValue(cg.gpa, name, result.local);
7963	7999	assert(!gop.found_existing); // first value
7964	8000
7965	8001	assert(it.next() == null);
...	...	@@ -7977,10 +8013,11 @@ fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7977	8013	return cg.fail("Self-hosted wasm backend requires input constraint to be equal \"r\"", .{});
7978	8014	}
7979	8015
7980		try cg.lowerToStack(operand);
	8016	try cg.emitWValue(operand);
7981	8017	const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand));
	8018	input_locals.appendAssumeCapacity(op_local);
7982	8019
7983		const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value);
	8020	const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local);
7984	8021	if (gop.found_existing) {
7985	8022	return cg.fail("Duplicate asm variable name \"{s}\"", .{name});
7986	8023	}

src/codegen/wasm/Emit.zig+10-10

...	...	@@ -56,13 +56,15 @@ pub fn lower(emit: *Emit) Error!void {
56	56	writeSleb128(code, aligned_stack);
57	57	// subtract it from the current stack pointer
58	58	code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_sub));
59		// Get negative stack alignment
60		const neg_stack_align = @as(i32, @intCast(align_bytes)) * -1;
61		code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_const));
62		writeSleb128(code, neg_stack_align);
63		// Bitwise-and the value to get the new stack pointer to ensure the
64		// pointers are aligned with the abi alignment.
65		code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_and));
	59	if (align_bytes != 16) {
	60	// Get negative stack alignment
	61	const neg_stack_align = @as(i32, @intCast(align_bytes)) * -1;
	62	code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_const));
	63	writeSleb128(code, neg_stack_align);
	64	// Bitwise-and the value to get the new stack pointer to ensure the
	65	// pointers are aligned with the abi alignment.
	66	code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.i32_and));
	67	}
66	68	// The bottom will be used to calculate all stack pointer offsets.
67	69	code.appendAssumeCapacity(@backingInt(std.wasm.Opcode.local_tee));
68	70	writeUleb128(code, mir.prologue.bottom_stack_local);
...	...	@@ -988,9 +990,7 @@ pub fn lower(emit: *Emit) Error!void {
988	990	/// Asserts 20 unused capacity.
989	991	fn encodeMemArg(code: *ArrayList(u8), mem_arg: Mir.MemArg) void {
990	992	assert(code.unusedCapacitySlice().len >= 20);
991		// Wasm encodes alignment as power of 2, rather than natural alignment.
992		const encoded_alignment = @ctz(mem_arg.alignment);
993		writeUleb128(code, encoded_alignment);
	993	writeUleb128(code, mem_arg.alignment.toLog2Units());
994	994	writeUleb128(code, mem_arg.offset);
995	995	}
996	996

src/codegen/wasm/Mir.zig+9-2

...	...	@@ -694,11 +694,18 @@ pub fn extraData(self: *const Mir, comptime T: type, index: usize) struct { data
694	694	inline for (info.field_names, info.field_types) \|field_name, field_type\| {
695	695	@field(result, field_name) = switch (field_type) {
696	696	u32 => self.extra[i],
	697	u64 => value: {
	698	const lo = @as(u64, self.extra[i]);
	699	const hi = @as(u64, self.extra[i + 1]) << 32;
	700	i += 1;
	701	break :value hi \| lo;
	702	},
697	703	i32 => @bitCast(self.extra[i]),
698	704	Wasm.UavsObjIndex,
699	705	Wasm.UavsExeIndex,
700	706	InternPool.Nav.Index,
701	707	InternPool.Index,
	708	Alignment,
702	709	=> @fromBackingInt(@intCast(self.extra[i])),
703	710	else => @compileError("Unsupported field type " ++ @typeName(field_type)),
704	711	};
...	...	@@ -747,8 +754,8 @@ pub const Float64 = struct {
747	754	};
748	755
749	756	pub const MemArg = struct {
750		offset: u32,
751		alignment: u32,
	757	offset: u64,
	758	alignment: Alignment,
752	759	};
753	760
754	761	pub const UavRefOff = struct {

src/codegen/wasm/assembly.zig+4-4

...	...	@@ -679,20 +679,20 @@ fn parseMemArg(cg: CodeGen, inst: []const u8, word_it: mem.TokenIterator(u8, .
679	679	const offset_str = oa[0];
680	680	const alignment_str = oa[1];
681	681
682		const offset = std.fmt.parseInt(i32, offset_str, 0) catch {
	682	const offset = std.fmt.parseInt(i64, offset_str, 0) catch {
683	683	return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});
684	684	};
685	685	const alignment_p2 = std.fmt.parseInt(u5, alignment_str, 0) catch {
686	686	return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{alignment_str});
687	687	};
688	688
689		return .{ .offset = @bitCast(offset), .alignment = @as(u32, 1) << alignment_p2 };
	689	return .{ .offset = @bitCast(offset), .alignment = .fromLog2Units(alignment_p2) };
690	690	} else {
691	691	const offset_str = mem_arg_str;
692		const offset = std.fmt.parseInt(i32, offset_str, 0) catch {
	692	const offset = std.fmt.parseInt(i64, offset_str, 0) catch {
693	693	return cg.fail("Malformed assembly, malformed offset \"{s}\"", .{offset_str});
694	694	};
695		return .{ .offset = @bitCast(offset), .alignment = 1 };
	695	return .{ .offset = @bitCast(offset), .alignment = .@"1" };
696	696	}
697	697	}
698	698