Meg's Mirrors

zig

1	const std = @import("std");
2	const builtin = @import("builtin");
3	const Allocator = std.mem.Allocator;
4	const assert = std.debug.assert;
5	const testing = std.testing;
6	const math = std.math;
7	const mem = std.mem;
8	const log = std.log.scoped(.codegen);
9
10	const CodeGen = @This();
11	const codegen = @import("../../codegen.zig");
12	const Zcu = @import("../../Zcu.zig");
13	const InternPool = @import("../../InternPool.zig");
14	const Decl = Zcu.Decl;
15	const Type = @import("../../Type.zig");
16	const Value = @import("../../Value.zig");
17	const Compilation = @import("../../Compilation.zig");
18	const link = @import("../../link.zig");
19	const Air = @import("../../Air.zig");
20	const Mir = @import("Mir.zig");
21	const assembly = @import("assembly.zig");
22	const abi = @import("../../codegen/wasm/abi.zig");
23	const Alignment = InternPool.Alignment;
24	const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
25	const errUnionErrorOffset = codegen.errUnionErrorOffset;
26
27	pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
28	return comptime &.initMany(&.{
29	.expand_bit_cast_safe,
30	.expand_int_cast_safe,
31	.expand_int_from_float_safe,
32	.expand_int_from_float_optimized_safe,
33	.expand_add_safe,
34	.expand_sub_safe,
35	.expand_mul_safe,
36
37	.expand_packed_load,
38	.expand_packed_store,
39	.expand_packed_agg_field_val,
40	.expand_packed_aggregate_init,
41	.expand_array_splat,
42	.expand_array_to_vector,
43
44	.scalarize_add,
45	.scalarize_add_optimized,
46	.scalarize_add_wrap,
47	.scalarize_add_sat,
48	.scalarize_sub,
49	.scalarize_sub_optimized,
50	.scalarize_sub_wrap,
51	.scalarize_sub_sat,
52	.scalarize_mul,
53	.scalarize_mul_optimized,
54	.scalarize_mul_wrap,
55	.scalarize_mul_sat,
56	.scalarize_div_float,
57	.scalarize_div_float_optimized,
58	.scalarize_div_trunc,
59	.scalarize_div_trunc_optimized,
60	.scalarize_div_floor,
61	.scalarize_div_floor_optimized,
62	.scalarize_div_ceil,
63	.scalarize_div_ceil_optimized,
64	.scalarize_div_exact,
65	.scalarize_div_exact_optimized,
66	.scalarize_rem,
67	.scalarize_rem_optimized,
68	.scalarize_mod,
69	.scalarize_mod_optimized,
70	.scalarize_max,
71	.scalarize_min,
72	.scalarize_add_with_overflow,
73	.scalarize_sub_with_overflow,
74	.scalarize_mul_with_overflow,
75	.scalarize_shl_with_overflow,
76	.scalarize_bit_and,
77	.scalarize_bit_or,
78	.scalarize_shr,
79	.scalarize_shr_exact,
80	.scalarize_shl,
81	.scalarize_shl_exact,
82	.scalarize_shl_sat,
83	.scalarize_xor,
84	.scalarize_not,
85	.scalarize_clz,
86	.scalarize_ctz,
87	.scalarize_popcount,
88	.scalarize_byte_swap,
89	.scalarize_bit_reverse,
90	.scalarize_sqrt,
91	.scalarize_sin,
92	.scalarize_cos,
93	.scalarize_tan,
94	.scalarize_exp,
95	.scalarize_exp2,
96	.scalarize_log,
97	.scalarize_log2,
98	.scalarize_log10,
99	.scalarize_abs,
100	.scalarize_floor,
101	.scalarize_ceil,
102	.scalarize_round,
103	.scalarize_trunc_float,
104	.scalarize_neg,
105	.scalarize_neg_optimized,
106	.scalarize_cmp_vector,
107	.scalarize_cmp_vector_optimized,
108	.scalarize_fptrunc,
109	.scalarize_fpext,
110	.scalarize_int_cast,
111	.scalarize_ptr_cast,
112	.scalarize_ptr_from_int,
113	.scalarize_int_from_ptr,
114	.scalarize_trunc,
115	.scalarize_int_from_float,
116	.scalarize_int_from_float_optimized,
117	.scalarize_float_from_int,
118	.scalarize_reduce,
119	.scalarize_reduce_optimized,
120	.scalarize_shuffle_one,
121	.scalarize_shuffle_two,
122	.scalarize_select,
123	.scalarize_mul_add,
124
125	.scalarize_bit_cast_padded_elems,
126	});
127	}
128
129	/// Reference to the function declaration the code
130	/// section belongs to
131	owner_nav: InternPool.Nav.Index,
132	/// Current block depth. Used to calculate the relative difference between a break
133	/// and block
134	block_depth: u32 = 0,
135	air: Air,
136	liveness: Air.Liveness,
137	gpa: mem.Allocator,
138	func_index: InternPool.Index,
139	/// Contains a list of current branches.
140	/// When we return from a branch, the branch will be popped from this list,
141	/// which means branches can only contain references from within its own branch,
142	/// or a branch higher (lower index) in the tree.
143	branches: std.ArrayList(Branch) = .empty,
144	/// Table to save `WValue`'s generated by an `Air.Inst`
145	// values: ValueTable,
146	/// Mapping from Air.Inst.Index to block ids
147	blocks: std.array_hash_map.Auto(Air.Inst.Index, struct {
148	label: u32,
149	value: WValue,
150	}) = .{},
151	/// Maps `loop` instructions to their label. `br` to here repeats the loop.
152	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, u32) = .empty,
153	/// The index the next local generated will have
154	/// NOTE: arguments share the index with locals therefore the first variable
155	/// will have the index that comes after the last argument's index
156	local_index: u32,
157	/// The index of the current argument.
158	/// Used to track which argument is being referenced in `airArg`.
159	arg_index: u32 = 0,
160	/// List of simd128 immediates. Each value is stored as an array of bytes.
161	/// This list will only be populated for 128bit-simd values when the target features
162	/// are enabled also.
163	simd_immediates: std.ArrayList([16]u8) = .empty,
164	/// The Target we're emitting (used to call intInfo)
165	target: *const std.Target,
166	ptr_size: enum { wasm32, wasm64 },
167	pt: Zcu.PerThread,
168	/// List of MIR Instructions
169	mir_instructions: std.MultiArrayList(Mir.Inst),
170	/// Contains extra data for MIR
171	mir_extra: std.ArrayList(u32),
172	/// List of all locals' types generated throughout this declaration
173	/// used to emit locals count at start of 'code' section.
174	mir_locals: std.ArrayList(std.wasm.Valtype),
175	/// Set of all UAVs referenced by this function. Key is the UAV value, value is the alignment.
176	/// `.none` means naturally aligned. An explicit alignment is never less than the natural alignment.
177	mir_uavs: std.array_hash_map.Auto(InternPool.Index, Alignment),
178	/// Set of all functions whose address this function has taken and which therefore might be called
179	/// via a `call_indirect` function.
180	mir_indirect_function_set: std.array_hash_map.Auto(InternPool.Nav.Index, void),
181	/// Set of all function types used by this function. These must be interned by the linker.
182	mir_func_tys: std.array_hash_map.Auto(InternPool.Index, void),
183	/// The number of `error_name_table_ref` instructions emitted.
184	error_name_table_ref_count: u32,
185	/// When a function is executing, we store the the current stack pointer's value within this local.
186	/// This value is then used to restore the stack pointer to the original value at the return of the function.
187	initial_stack_value: WValue = .none,
188	/// The current stack pointer subtracted with the stack size. From this value, we will calculate
189	/// all offsets of the stack values.
190	bottom_stack_value: WValue = .none,
191	/// Arguments of this function declaration
192	/// This will be set after `resolveCallingConventionValues`
193	args: []WValue,
194	/// This will only be `.none` if the function returns void, or returns an immediate.
195	/// When it returns a pointer to the stack, the `.local` tag will be active and must be populated
196	/// before this function returns its execution to the caller.
197	return_value: WValue,
198	/// Only populated for variadic functions.
199	/// Holds the hidden final parameter pointing to the varargs buffer.
200	varargs: WValue,
201	/// The size of the stack this function occupies. In the function prologue
202	/// we will move the stack pointer by this number, forward aligned with the `stack_alignment`.
203	stack_size: u32 = 0,
204	/// The stack alignment, which is 16 bytes by default. This is specified by the
205	/// tool-conventions: https://github.com/WebAssembly/tool-conventions/blob/main/BasicCABI.md
206	/// and also what the llvm backend will emit.
207	/// However, local variables or the usage of `incoming_stack_alignment` in a `CallingConvention` can overwrite this default.
208	stack_alignment: Alignment = .@"16",
209
210	// For each individual Wasm valtype we store a seperate free list which
211	// allows us to re-use locals that are no longer used. e.g. a temporary local.
212	/// A list of indexes which represents a local of valtype `i32`.
213	/// It is illegal to store a non-i32 valtype in this list.
214	free_locals_i32: std.ArrayList(u32) = .empty,
215	/// A list of indexes which represents a local of valtype `i64`.
216	/// It is illegal to store a non-i64 valtype in this list.
217	free_locals_i64: std.ArrayList(u32) = .empty,
218	/// A list of indexes which represents a local of valtype `f32`.
219	/// It is illegal to store a non-f32 valtype in this list.
220	free_locals_f32: std.ArrayList(u32) = .empty,
221	/// A list of indexes which represents a local of valtype `f64`.
222	/// It is illegal to store a non-f64 valtype in this list.
223	free_locals_f64: std.ArrayList(u32) = .empty,
224	/// A list of indexes which represents a local of valtype `v127`.
225	/// It is illegal to store a non-v128 valtype in this list.
226	free_locals_v128: std.ArrayList(u32) = .empty,
227
228	/// When in debug mode, this tracks if no `finishAir` was missed.
229	/// Forgetting to call `finishAir` will cause the result to not be
230	/// stored in our `values` map and therefore cause bugs.
231	air_bookkeeping: @TypeOf(bookkeeping_init) = bookkeeping_init,
232
233	/// Wasm Value, created when generating an instruction
234	const WValue = union(enum) {
235	/// `WValue` which has been freed and may no longer hold
236	/// any references.
237	dead: void,
238	/// May be referenced but is unused
239	none: void,
240	/// The value lives on top of the stack
241	stack: void,
242	/// Index of the local
243	local: struct {
244	/// Contains the index to the local
245	value: u32,
246	/// The amount of instructions referencing this `WValue`
247	references: u32,
248	},
249	/// An immediate 32bit value
250	imm32: u32,
251	/// An immediate 64bit value
252	imm64: u64,
253	/// Index into the list of simd128 immediates. This `WValue` is
254	/// only possible in very rare cases, therefore it would be
255	/// a waste of memory to store the value in a 128 bit integer.
256	imm128: u32,
257	/// A constant 32bit float value
258	float32: f32,
259	/// A constant 64bit float value
260	float64: f64,
261	nav_ref: struct {
262	nav_index: InternPool.Nav.Index,
263	offset: i32 = 0,
264	},
265	uav_ref: struct {
266	ip_index: InternPool.Index,
267	offset: i32 = 0,
268	orig_ptr_ty: InternPool.Index = .none,
269	},
270	/// Offset from the bottom of the virtual stack, with the offset
271	/// pointing to where the value lives.
272	stack_offset: struct {
273	/// Contains the actual value of the offset
274	value: u32,
275	/// The amount of instructions referencing this `WValue`
276	references: u32,
277	},
278
279	/// Returns the offset from the bottom of the stack. This is useful when
280	/// we use the load or store instruction to ensure we retrieve the value
281	/// from the correct position, rather than the value that lives at the
282	/// bottom of the stack. For instances where `WValue` is not `stack_value`
283	/// this will return 0, which allows us to simply call this function for all
284	/// loads and stores without requiring checks everywhere.
285	fn offset(value: WValue) u32 {
286	switch (value) {
287	.stack_offset => |stack_offset| return stack_offset.value,
288	.dead => unreachable,
289	else => return 0,
290	}
291	}
292
293	/// Promotes a `WValue` to a local when given value is on top of the stack.
294	/// When encountering a `local` or `stack_offset` this is essentially a no-op.
295	/// All other tags are illegal.
296	fn toLocal(value: WValue, gen: *CodeGen, ty: Type) InnerError!WValue {
297	switch (value) {
298	.stack => {
299	const new_local = try gen.allocLocal(ty);
300	try gen.addLocal(.local_set, new_local.local.value);
301	return new_local;
302	},
303	else => return value,
304	}
305	}
306
307	/// Marks a local as no longer being referenced and essentially allows
308	/// us to re-use it somewhere else within the function.
309	/// The valtype of the local is deducted by using the index of the given `WValue`.
310	fn free(value: *WValue, gen: *CodeGen) void {
311	if (value.* != .local) return;
312	const local_value = value.local.value;
313	const reserved = gen.args.len + @intFromBool(gen.return_value != .none) + @intFromBool(gen.varargs != .none);
314	if (local_value < reserved + 2) return; // reserved locals may never be re-used. Also accounts for 2 stack locals.
315
316	const index = local_value - reserved;
317	const valtype = gen.mir_locals.items[index];
318	switch (valtype) {
319	.i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead
320	.i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return,
321	.f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return,
322	.f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return,
323	.v128 => gen.free_locals_v128.append(gen.gpa, local_value) catch return,
324	}
325	log.debug("freed local ({d}) of type {}", .{ local_value, valtype });
326	value.* = .dead;
327	}
328	};
329
330	/// Hashmap to store generated `WValue` for each `Air.Inst.Ref`
331	const ValueTable = std.array_hash_map.Auto(Air.Inst.Ref, WValue);
332
333	const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
334
335	const InnerError = Error || error{
336	/// An error occurred when trying to lower AIR to MIR.
337	AlreadyReported,
338	/// Compiler implementation could not handle a large integer.
339	Overflow,
340	};
341
342	pub fn deinit(cg: *CodeGen) void {
343	const gpa = cg.gpa;
344	for (cg.branches.items) |*branch| branch.deinit(gpa);
345	cg.branches.deinit(gpa);
346	cg.blocks.deinit(gpa);
347	cg.loops.deinit(gpa);
348	cg.simd_immediates.deinit(gpa);
349	cg.free_locals_i32.deinit(gpa);
350	cg.free_locals_i64.deinit(gpa);
351	cg.free_locals_f32.deinit(gpa);
352	cg.free_locals_f64.deinit(gpa);
353	cg.free_locals_v128.deinit(gpa);
354	cg.mir_instructions.deinit(gpa);
355	cg.mir_extra.deinit(gpa);
356	cg.mir_locals.deinit(gpa);
357	cg.mir_uavs.deinit(gpa);
358	cg.mir_indirect_function_set.deinit(gpa);
359	cg.mir_func_tys.deinit(gpa);
360	cg.* = undefined;
361	}
362
363	pub fn fail(cg: *CodeGen, comptime fmt: []const u8, args: anytype) Error {
364	const zcu = cg.pt.zcu;
365	const func = zcu.funcInfo(cg.func_index);
366	return zcu.codegenFail(func.owner_nav, fmt, args);
367	}
368
369	/// Resolves the `WValue` for the given instruction `inst`
370	/// When the given instruction has a `Value`, it returns a constant instead
371	fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue {
372	var branch_index = cg.branches.items.len;
373	while (branch_index > 0) : (branch_index -= 1) {
374	const branch = cg.branches.items[branch_index - 1];
375	if (branch.values.get(ref)) |value| {
376	return value;
377	}
378	}
379
380	// when we did not find an existing instruction, it
381	// means we must generate it from a constant.
382	// We always store constants in the most outer branch as they must never
383	// be removed. The most outer branch is always at index 0.
384	const gop = try cg.branches.items[0].values.getOrPut(cg.gpa, ref);
385	assert(!gop.found_existing);
386
387	const pt = cg.pt;
388	const zcu = pt.zcu;
389	const val: Value = .fromInterned(ref.toInterned().?);
390	const ty = cg.typeOf(ref);
391	if (!ty.hasRuntimeBits(zcu) and !ty.isInt(zcu) and !ty.isError(zcu)) {
392	gop.value_ptr.* = .none;
393	return .none;
394	}
395
396	// When we need to pass the value by reference (such as a struct), we will
397	// leverage `generateSymbol` to lower the constant to bytes and emit it
398	// to the 'rodata' section. We then return the index into the section as `WValue`.
399	//
400	// In the other cases, we will simply lower the constant to a value that fits
401	// into a single local (such as a pointer, integer, bool, etc).
402	const result: WValue = if (isByRef(ty, zcu, cg.target))
403	.{ .uav_ref = .{ .ip_index = val.toIntern() } }
404	else
405	try cg.lowerConstant(val);
406
407	gop.value_ptr.* = result;
408	return result;
409	}
410
411	fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue {
412	const zcu = cg.pt.zcu;
413	const ty = val.typeOf(zcu);
414
415	return if (isByRef(ty, zcu, cg.target))
416	.{ .uav_ref = .{ .ip_index = val.toIntern() } }
417	else
418	try cg.lowerConstant(val);
419	}
420
421	/// NOTE: if result == .stack, it will be stored in .local
422	fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void {
423	assert(operands.len <= Air.Liveness.bpi - 1);
424	var tomb_bits = cg.liveness.getTombBits(inst);
425	for (operands) |operand| {
426	const dies = @as(u1, @truncate(tomb_bits)) != 0;
427	tomb_bits >>= 1;
428	if (!dies) continue;
429	processDeath(cg, operand);
430	}
431	try cg.finishAirResult(inst, result);
432	}
433
434	fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerError!void {
435	// results of `none` can never be referenced.
436	if (result != .none) {
437	const trackable_result = if (result != .stack)
438	result
439	else
440	try result.toLocal(cg, cg.typeOfIndex(inst));
441	const branch = cg.currentBranch();
442	branch.values.putAssumeCapacityNoClobber(inst.toRef(), trackable_result);
443	}
444
445	if (std.debug.runtime_safety) {
446	cg.air_bookkeeping += 1;
447	}
448	}
449
450	const Branch = struct {
451	values: ValueTable = .{},
452
453	fn deinit(branch: *Branch, gpa: Allocator) void {
454	branch.values.deinit(gpa);
455	branch.* = undefined;
456	}
457	};
458
459	inline fn currentBranch(cg: *CodeGen) *Branch {
460	return &cg.branches.items[cg.branches.items.len - 1];
461	}
462
463	fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void {
464	if (bt.feed()) {
465	cg.processDeath(operand);
466	}
467	}
468
469	fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void {
470	if (ref.toIndex() == null) return;
471	// Branches are currently only allowed to free locals allocated
472	// within their own branch.
473	// TODO: Upon branch consolidation free any locals if needed.
474	const value = cg.currentBranch().values.getPtr(ref) orelse return;
475	if (value.* != .local) return;
476	const reserved_indexes = cg.args.len + @intFromBool(cg.return_value != .none);
477	if (value.local.value < reserved_indexes) {
478	return; // function arguments can never be re-used
479	}
480	log.debug("Decreasing reference for ref: %{d}, using local '{d}'", .{ @backingInt(ref.toIndex().?), value.local.value });
481	value.local.references -= 1; // if this panics, a call to `reuseOperand` was forgotten by the developer
482	if (value.local.references == 0) {
483	value.free(cg);
484	}
485	}
486
487	pub fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void {
488	try cg.mir_instructions.append(cg.gpa, inst);
489	}
490
491	pub fn addTag(cg: *CodeGen, tag: Mir.Inst.Tag) error{OutOfMemory}!void {
492	try cg.addInst(.{ .tag = tag, .data = .{ .tag = {} } });
493	}
494
495	pub fn addExtended(cg: *CodeGen, opcode: std.wasm.MiscOpcode) error{OutOfMemory}!void {
496	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
497	try cg.mir_extra.append(cg.gpa, @backingInt(opcode));
498	try cg.addInst(.{ .tag = .misc_prefix, .data = .{ .payload = extra_index } });
499	}
500
501	pub fn addLabel(cg: *CodeGen, tag: Mir.Inst.Tag, label: u32) error{OutOfMemory}!void {
502	try cg.addInst(.{ .tag = tag, .data = .{ .label = label } });
503	}
504
505	pub fn addLocal(cg: *CodeGen, tag: Mir.Inst.Tag, local: u32) error{OutOfMemory}!void {
506	try cg.addInst(.{ .tag = tag, .data = .{ .local = local } });
507	}
508
509	/// Accepts an unsigned 32bit integer rather than a signed integer to
510	/// prevent us from having to bitcast multiple times as most values
511	/// within codegen are represented as unsigned rather than signed.
512	pub fn addImm32(cg: *CodeGen, imm: u32) error{OutOfMemory}!void {
513	try cg.addInst(.{ .tag = .i32_const, .data = .{ .imm32 = @bitCast(imm) } });
514	}
515
516	/// Accepts an unsigned 64bit integer rather than a signed integer to
517	/// prevent us from having to bitcast multiple times as most values
518	/// within codegen are represented as unsigned rather than signed.
519	pub fn addImm64(cg: *CodeGen, imm: u64) error{OutOfMemory}!void {
520	const extra_index = try cg.addExtra(Mir.Imm64.init(imm));
521	try cg.addInst(.{ .tag = .i64_const, .data = .{ .payload = extra_index } });
522	}
523
524	/// Accepts the index into the list of 128bit-immediates
525	pub fn addImm128(cg: *CodeGen, index: u32) error{OutOfMemory}!void {
526	const simd_values = cg.simd_immediates.items[index];
527	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
528	// tag + 128bit value
529	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 5);
530	cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_const));
531	cg.mir_extra.appendSliceAssumeCapacity(@alignCast(mem.bytesAsSlice(u32, &simd_values)));
532	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
533	}
534
535	pub fn addFloat32(cg: *CodeGen, float: f32) error{OutOfMemory}!void {
536	try cg.addInst(.{ .tag = .f32_const, .data = .{ .float32 = float } });
537	}
538
539	pub fn addFloat64(cg: *CodeGen, float: f64) error{OutOfMemory}!void {
540	const extra_index = try cg.addExtra(Mir.Float64.init(float));
541	try cg.addInst(.{ .tag = .f64_const, .data = .{ .payload = extra_index } });
542	}
543
544	/// Inserts an instruction to load/store from/to wasm's linear memory dependent on the given `tag`.
545	pub fn addMemArg(cg: *CodeGen, tag: Mir.Inst.Tag, mem_arg: Mir.MemArg) error{OutOfMemory}!void {
546	const extra_index = try cg.addExtra(mem_arg);
547	try cg.addInst(.{ .tag = tag, .data = .{ .payload = extra_index } });
548	}
549
550	/// Inserts an instruction from the 'atomics' feature which accesses wasm's linear memory dependent on the
551	/// given `tag`.
552	pub fn addAtomicMemArg(cg: *CodeGen, tag: std.wasm.AtomicsOpcode, mem_arg: Mir.MemArg) error{OutOfMemory}!void {
553	const extra_index = try cg.addExtra(@as(struct { val: u32 }, .{ .val = @backingInt(tag) }));
554	_ = try cg.addExtra(mem_arg);
555	try cg.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } });
556	}
557
558	/// Helper function to emit atomic mir opcodes.
559	pub fn addAtomicTag(cg: *CodeGen, tag: std.wasm.AtomicsOpcode) error{OutOfMemory}!void {
560	const extra_index = try cg.addExtra(@as(struct { val: u32 }, .{ .val = @backingInt(tag) }));
561	try cg.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } });
562	}
563
564	fn addCallIntrinsic(cg: *CodeGen, intrinsic: Mir.Intrinsic) error{OutOfMemory}!void {
565	try cg.addInst(.{ .tag = .call_intrinsic, .data = .{ .intrinsic = intrinsic } });
566	}
567
568	/// Appends entries to `mir_extra` based on the type of `extra`.
569	/// Returns the index into `mir_extra`
570	fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
571	const field_count = @typeInfo(@TypeOf(extra)).@"struct".field_names.len;
572	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count);
573	return cg.addExtraAssumeCapacity(extra);
574	}
575
576	/// Appends entries to `mir_extra` based on the type of `extra`.
577	/// Returns the index into `mir_extra`
578	fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
579	const info = @typeInfo(@TypeOf(extra)).@"struct";
580	const result: u32 = @intCast(cg.mir_extra.items.len);
581	inline for (info.field_names, info.field_types) |field_name, field_type| {
582	cg.mir_extra.appendAssumeCapacity(switch (field_type) {
583	u32 => @field(extra, field_name),
584	i32 => @bitCast(@field(extra, field_name)),
585	InternPool.Index,
586	InternPool.Nav.Index,
587	=> @backingInt(@field(extra, field_name)),
588	else => @compileError("Unsupported field type " ++ @typeName(field_type)),
589	});
590	}
591	return result;
592	}
593
594	/// For `std.lang.CallingConvention.auto`.
595	pub fn typeToValtype(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.Valtype {
596	return switch (ty.zigTypeTag(zcu)) {
597	.float => switch (ty.floatBits(target)) {
598	16 => .i32, // stored/loaded as u16
599	32 => .f32,
600	64 => .f64,
601	80, 128 => .i32,
602	else => unreachable,
603	},
604	.int, .@"enum" => switch (ty.intInfo(zcu).bits) {
605	0...32 => .i32,
606	33...64 => .i64,
607	else => .i32,
608	},
609	.vector => switch (CodeGen.determineSimdStoreStrategy(ty, zcu, target)) {
610	.direct => .v128,
611	.unrolled => .i32,
612	},
613	.@"union", .@"struct" => switch (ty.containerLayout(zcu)) {
614	.@"packed" => typeToValtype(ty.backingIntType(zcu), zcu, target),
615	.auto, .@"extern" => .i32,
616	},
617	else => .i32, // all represented as reference/immediate
618	};
619	}
620
621	/// Using a given `Type`, returns the corresponding wasm value type
622	/// Differently from `typeToValtype` this also allows `void` to create a block
623	/// with no return type
624	fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.BlockType {
625	return switch (ty.ip_index) {
626	.void_type, .noreturn_type => .empty,
627	else => .fromValtype(typeToValtype(ty, zcu, target)),
628	};
629	}
630
631	/// Writes the bytecode depending on the given `WValue` in `val`
632	fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
633	switch (value) {
634	.dead => unreachable, // reference to free'd `WValue` (missing reuseOperand?)
635	.none, .stack => {}, // no-op
636	.local => |idx| try cg.addLocal(.local_get, idx.value),
637	.imm32 => |val| try cg.addImm32(val),
638	.imm64 => |val| try cg.addImm64(val),
639	.imm128 => |val| try cg.addImm128(val),
640	.float32 => |val| try cg.addFloat32(val),
641	.float64 => |val| try cg.addFloat64(val),
642	.nav_ref => |nav_ref| {
643	const zcu = cg.pt.zcu;
644	const ip = &zcu.intern_pool;
645	if (ip.zigTypeTag(ip.getNav(nav_ref.nav_index).resolved.?.type) == .@"fn") {
646	assert(nav_ref.offset == 0);
647	try cg.mir_indirect_function_set.put(cg.gpa, nav_ref.nav_index, {});
648	try cg.addInst(.{ .tag = .func_ref, .data = .{ .nav_index = nav_ref.nav_index } });
649	} else if (nav_ref.offset == 0) {
650	try cg.addInst(.{ .tag = .nav_ref, .data = .{ .nav_index = nav_ref.nav_index } });
651	} else {
652	try cg.addInst(.{
653	.tag = .nav_ref_off,
654	.data = .{
655	.payload = try cg.addExtra(Mir.NavRefOff{
656	.nav_index = nav_ref.nav_index,
657	.offset = nav_ref.offset,
658	}),
659	},
660	});
661	}
662	},
663	.uav_ref => |uav| {
664	const zcu = cg.pt.zcu;
665	const ip = &zcu.intern_pool;
666	assert(!ip.isFunctionType(ip.typeOf(uav.ip_index)));
667	const gop = try cg.mir_uavs.getOrPut(cg.gpa, uav.ip_index);
668	const this_align: Alignment = a: {
669	if (uav.orig_ptr_ty == .none) break :a .none;
670	const ptr_type = ip.indexToKey(uav.orig_ptr_ty).ptr_type;
671	const this_align = ptr_type.flags.alignment;
672	if (this_align == .none) break :a .none;
673	const abi_align = Type.fromInterned(ptr_type.child).abiAlignment(zcu);
674	if (this_align.compare(.lte, abi_align)) break :a .none;
675	break :a this_align;
676	};
677	if (!gop.found_existing or
678	gop.value_ptr.* == .none or
679	(this_align != .none and this_align.compare(.gt, gop.value_ptr.*)))
680	{
681	gop.value_ptr.* = this_align;
682	}
683	if (uav.offset == 0) {
684	try cg.addInst(.{
685	.tag = .uav_ref,
686	.data = .{ .ip_index = uav.ip_index },
687	});
688	} else {
689	try cg.addInst(.{
690	.tag = .uav_ref_off,
691	.data = .{ .payload = try cg.addExtra(@as(Mir.UavRefOff, .{
692	.value = uav.ip_index,
693	.offset = uav.offset,
694	})) },
695	});
696	}
697	},
698	.stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), // caller must ensure to address the offset
699	}
700	}
701
702	/// If given a local or stack-offset, increases the reference count by 1.
703	/// The old `WValue` found at instruction `ref` is then replaced by the
704	/// modified `WValue` and returned. When given a non-local or non-stack-offset,
705	/// returns the given `operand` itfunc instead.
706	fn reuseOperand(cg: *CodeGen, ref: Air.Inst.Ref, operand: WValue) WValue {
707	if (operand != .local and operand != .stack_offset) return operand;
708	var new_value = operand;
709	switch (new_value) {
710	.local => |*local| local.references += 1,
711	.stack_offset => |*stack_offset| stack_offset.references += 1,
712	else => unreachable,
713	}
714	const old_value = cg.getResolvedInst(ref);
715	old_value.* = new_value;
716	return new_value;
717	}
718
719	/// From a reference, returns its resolved `WValue`.
720	/// It's illegal to provide a `Air.Inst.Ref` that hasn't been resolved yet.
721	fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue {
722	var index = cg.branches.items.len;
723	while (index > 0) : (index -= 1) {
724	const branch = cg.branches.items[index - 1];
725	if (branch.values.getPtr(ref)) |value| {
726	return value;
727	}
728	}
729	unreachable; // developer-error: This can only be called on resolved instructions. Use `resolveInst` instead.
730	}
731
732	/// Creates one locals for a given `Type`.
733	/// Returns a corresponding `Wvalue` with `local` as active tag
734	fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
735	const zcu = cg.pt.zcu;
736	const valtype = typeToValtype(ty, zcu, cg.target);
737	const index_or_null = switch (valtype) {
738	.i32 => cg.free_locals_i32.pop(),
739	.i64 => cg.free_locals_i64.pop(),
740	.f32 => cg.free_locals_f32.pop(),
741	.f64 => cg.free_locals_f64.pop(),
742	.v128 => cg.free_locals_v128.pop(),
743	};
744	if (index_or_null) |index| {
745	log.debug("reusing local ({d}) of type {}", .{ index, valtype });
746	return .{ .local = .{ .value = index, .references = 1 } };
747	}
748	log.debug("new local of type {}", .{valtype});
749	return cg.ensureAllocLocal(ty);
750	}
751
752	/// Ensures a new local will be created. This is useful when it's useful
753	/// to use a zero-initialized local.
754	fn ensureAllocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
755	const zcu = cg.pt.zcu;
756	try cg.mir_locals.append(cg.gpa, typeToValtype(ty, zcu, cg.target));
757	const initial_index = cg.local_index;
758	cg.local_index += 1;
759	return .{ .local = .{ .value = initial_index, .references = 1 } };
760	}
761
762	pub const Error = codegen.Error;
763
764	pub fn generate(
765	bin_file: *link.File,
766	pt: Zcu.PerThread,
767	func_index: InternPool.Index,
768	air: *const Air,
769	liveness: *const ?Air.Liveness,
770	) Error!Mir {
771	_ = bin_file;
772	const zcu = pt.zcu;
773	const gpa = zcu.gpa;
774	const cg = zcu.funcInfo(func_index);
775	const file_scope = zcu.navFileScope(cg.owner_nav);
776	const target = &file_scope.mod.?.resolved_target.result;
777	const fn_ty = zcu.navValue(cg.owner_nav).typeOf(zcu);
778	const fn_info = zcu.typeToFunc(fn_ty).?;
779	const ret_ty: Type = .fromInterned(fn_info.return_type);
780	const any_returns = !firstParamSRet(fn_info.cc, ret_ty, zcu, target) and ret_ty.hasRuntimeBits(zcu);
781
782	var cc_result = try resolveCallingConventionValues(zcu, fn_ty, target);
783	defer cc_result.deinit(gpa);
784
785	var code_gen: CodeGen = .{
786	.gpa = gpa,
787	.pt = pt,
788	.air = air.*,
789	.liveness = liveness.*.?,
790	.owner_nav = cg.owner_nav,
791	.target = target,
792	.ptr_size = switch (target.cpu.arch) {
793	.wasm32 => .wasm32,
794	.wasm64 => .wasm64,
795	else => unreachable,
796	},
797	.func_index = func_index,
798	.args = cc_result.args,
799	.return_value = cc_result.return_value,
800	.varargs = cc_result.varargs,
801	.local_index = cc_result.local_index,
802	.mir_instructions = .empty,
803	.mir_extra = .empty,
804	.mir_locals = .empty,
805	.mir_uavs = .empty,
806	.mir_indirect_function_set = .empty,
807	.mir_func_tys = .empty,
808	.error_name_table_ref_count = 0,
809	};
810	defer code_gen.deinit();
811
812	try code_gen.mir_func_tys.putNoClobber(gpa, fn_ty.toIntern(), {});
813
814	return generateInner(&code_gen, any_returns) catch |err| switch (err) {
815	error.AlreadyReported,
816	error.OutOfMemory,
817	=> |e| return e,
818	else => |e| return code_gen.fail("failed to generate function: {s}", .{@errorName(e)}),
819	};
820	}
821
822	fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir {
823	const zcu = cg.pt.zcu;
824	// branch used for const values
825	try cg.branches.append(cg.gpa, .{});
826	// func scope branch
827	try cg.branches.append(cg.gpa, .{});
828	defer {
829	var func_branch = cg.branches.pop().?;
830	func_branch.deinit(cg.gpa);
831	var const_branch = cg.branches.pop().?;
832	const_branch.deinit(cg.gpa);
833	assert(cg.branches.items.len == 0); // missing branch merge
834	}
835	// Generate MIR for function body
836	try cg.genBody(cg.air.getMainBody());
837
838	// In case we have a return value, but the last instruction is a noreturn (such as a while loop)
839	// we emit an unreachable instruction to tell the stack validator that part will never be reached.
840	if (any_returns and cg.air.instructions.len > 0) {
841	const main_body = cg.air.getMainBody();
842	const inst: Air.Inst.Index = main_body[main_body.len - 1];
843	const last_inst_ty = cg.typeOfIndex(inst);
844	if (!last_inst_ty.hasRuntimeBits(zcu)) {
845	try cg.addTag(.@"unreachable");
846	}
847	}
848	// End of function body
849	try cg.addTag(.end);
850	try cg.addTag(.dbg_epilogue_begin);
851
852	try cg.mir_extra.shrinkToLen(cg.gpa);
853	try cg.mir_locals.shrinkToLen(cg.gpa);
854
855	return .{
856	.instructions = cg.mir_instructions.toOwnedSlice(),
857	.extra = cg.mir_extra.toOwnedSliceAssert(),
858	.locals = cg.mir_locals.toOwnedSliceAssert(),
859	.prologue = if (cg.initial_stack_value == .none) .none else .{
860	.sp_local = cg.initial_stack_value.local.value,
861	.flags = .{ .stack_alignment = cg.stack_alignment },
862	.stack_size = cg.stack_size,
863	.bottom_stack_local = cg.bottom_stack_value.local.value,
864	},
865	.uavs = cg.mir_uavs.move(),
866	.indirect_function_set = cg.mir_indirect_function_set.move(),
867	.func_tys = cg.mir_func_tys.move(),
868	.error_name_table_ref_count = cg.error_name_table_ref_count,
869	};
870	}
871
872	const CallWValues = struct {
873	args: []WValue,
874	return_value: WValue,
875	varargs: WValue,
876	local_index: u32,
877
878	fn deinit(values: *CallWValues, gpa: Allocator) void {
879	gpa.free(values.args);
880	values.* = undefined;
881	}
882	};
883
884	fn resolveCallingConventionValues(
885	zcu: *const Zcu,
886	fn_ty: Type,
887	target: *const std.Target,
888	) Allocator.Error!CallWValues {
889	const gpa = zcu.gpa;
890	const ip = &zcu.intern_pool;
891	const fn_info = zcu.typeToFunc(fn_ty).?;
892	const cc = fn_info.cc;
893
894	var result: CallWValues = .{
895	.args = &.{},
896	.return_value = .none,
897	.varargs = .none,
898	.local_index = 0,
899	};
900	if (cc == .naked) return result;
901
902	var args = std.array_list.Managed(WValue).init(gpa);
903	defer args.deinit();
904
905	// Check if we store the result as a pointer to the stack rather than
906	// by value
907	if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, target)) {
908	// the sret arg will be passed as first argument, therefore we
909	// set the `return_value` before allocating locals for regular args.
910	result.return_value = .{ .local = .{ .value = result.local_index, .references = 1 } };
911	result.local_index += 1;
912	}
913
914	switch (cc) {
915	.auto => {
916	for (fn_info.param_types.get(ip)) |ty| {
917	if (!Type.fromInterned(ty).hasRuntimeBits(zcu)) {
918	continue;
919	}
920
921	try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
922	result.local_index += 1;
923	}
924	},
925	.wasm_mvp => {
926	for (fn_info.param_types.get(ip)) |ty| {
927	const param_ty: Type = .fromInterned(ty);
928	if (!param_ty.hasRuntimeBits(zcu)) {
929	continue;
930	}
931
932	switch (abi.classifyType(param_ty, zcu, target)) {
933	.direct, .indirect => {
934	try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
935	result.local_index += 1;
936	},
937	.double_i64 => {
938	try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
939	try args.append(.{ .local = .{ .value = result.local_index + 1, .references = 1 } });
940	result.local_index += 2;
941	},
942	.unrolled => |vector| {
943	for (0..vector.len) |_| {
944	try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
945	result.local_index += 1;
946	}
947	},
948	}
949	}
950	},
951	else => unreachable, // Frontend is responsible for emitting an error earlier.
952	}
953
954	if (fn_info.is_var_args) {
955	result.varargs = .{ .local = .{ .value = result.local_index, .references = 1 } };
956	result.local_index += 1;
957	}
958
959	result.args = try args.toOwnedSlice();
960	return result;
961	}
962
963	pub fn firstParamSRet(
964	cc: std.lang.CallingConvention,
965	return_type: Type,
966	zcu: *const Zcu,
967	target: *const std.Target,
968	) bool {
969	if (!return_type.hasRuntimeBits(zcu)) return false;
970	switch (cc) {
971	.@"inline" => unreachable,
972	.auto => return isByRef(return_type, zcu, target),
973	.wasm_mvp => switch (abi.classifyType(return_type, zcu, target)) {
974	.direct => return false,
975	.double_i64, .indirect => return true,
976	.unrolled => |vector| return vector.len > 1,
977	},
978	else => return false,
979	}
980	}
981
982	/// Lowers a Zig type and its value based on a given calling convention to ensure
983	/// it matches the ABI.
984	fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValue) !void {
985	if (cc != .wasm_mvp) {
986	return cg.lowerToStack(value);
987	}
988
989	const zcu = cg.pt.zcu;
990
991	switch (abi.classifyType(ty, zcu, cg.target)) {
992	.direct => |scalar_ty| {
993	if (!isByRef(ty, zcu, cg.target)) {
994	return cg.lowerToStack(value);
995	} else {
996	_ = try cg.load(value, scalar_ty, 0);
997	}
998	},
999	.double_i64 => {
1000	assert(ty.abiSize(zcu) == 16);
1001	// in this case we have an integer or float that must be lowered as 2 i64's.
1002	try cg.emitWValue(value);
1003	try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = 8 });
1004	try cg.emitWValue(value);
1005	try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = 8 });
1006	},
1007	.indirect => {
1008	const stack_copy = try cg.allocStack(ty);
1009	try cg.store(stack_copy, value, ty, 0);
1010	return cg.lowerToStack(stack_copy);
1011	},
1012	.unrolled => |vector| {
1013	const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));
1014	for (0..vector.len) |index| {
1015	_ = try cg.load(value, vector.elem_type, @intCast(index * elem_size));
1016	}
1017	},
1018	}
1019	}
1020
1021	/// Lowers a `WValue` to the stack. This means when the `value` results in
1022	/// `.stack_offset` we calculate the pointer of this offset and use that.
1023	/// The value is left on the stack, and not stored in any temporary.
1024	fn lowerToStack(cg: *CodeGen, value: WValue) !void {
1025	switch (value) {
1026	.stack_offset => |offset| {
1027	try cg.emitWValue(value);
1028	if (offset.value > 0) {
1029	switch (cg.ptr_size) {
1030	.wasm32 => {
1031	try cg.addImm32(offset.value);
1032	try cg.addTag(.i32_add);
1033	},
1034	.wasm64 => {
1035	try cg.addImm64(offset.value);
1036	try cg.addTag(.i64_add);
1037	},
1038	}
1039	}
1040	},
1041	else => try cg.emitWValue(value),
1042	}
1043	}
1044
1045	/// Creates a local for the initial stack value
1046	/// Asserts `initial_stack_value` is `.none`
1047	fn initializeStack(cg: *CodeGen) !void {
1048	assert(cg.initial_stack_value == .none);
1049	// Reserve a local to store the current stack pointer
1050	// We can later use this local to set the stack pointer back to the value
1051	// we have stored here.
1052	cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize);
1053	// Also reserve a local to store the bottom stack value
1054	cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize);
1055	}
1056
1057	/// Reads the stack pointer from `Context.initial_stack_value` and writes it
1058	/// to the global stack pointer variable
1059	fn restoreStackPointer(cg: *CodeGen) !void {
1060	// only restore the pointer if it was initialized
1061	if (cg.initial_stack_value == .none) return;
1062	// Get the original stack pointer's value
1063	try cg.emitWValue(cg.initial_stack_value);
1064
1065	try cg.addTag(.global_set_sp);
1066	}
1067
1068	/// From a given type, will create space on the virtual stack to store the value of such type.
1069	/// This returns a `WValue` with its active tag set to `local`, containing the index to the local
1070	/// that points to the position on the virtual stack. This function should be used instead of
1071	/// moveStack unless a local was already created to store the pointer.
1072	///
1073	/// Asserts Type has codegenbits
1074	fn allocStack(cg: *CodeGen, ty: Type) !WValue {
1075	const pt = cg.pt;
1076	const zcu = pt.zcu;
1077	assert(ty.hasRuntimeBits(zcu));
1078
1079	const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse {
1080	return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{
1081	ty.fmt(pt), ty.abiSize(zcu),
1082	});
1083	};
1084	const abi_align = ty.abiAlignment(zcu);
1085
1086	return cg.allocStackBytes(abi_size, abi_align);
1087	}
1088
1089	fn allocInt(cg: *CodeGen, int_ty: IntType) !WValue {
1090	const abi_size = std.math.cast(u32, std.zig.target.intByteSize(cg.target, int_ty.bits)) orelse {
1091	return cg.fail("Integer ABI size exceeds max stack size", .{});
1092	};
1093	const abi_align: Alignment = .fromByteUnits(std.zig.target.intAlignment(cg.target, int_ty.bits));
1094
1095	return cg.allocStackBytes(abi_size, abi_align);
1096	}
1097
1098	fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue {
1099	assert(size > 0);
1100
1101	if (cg.initial_stack_value == .none) {
1102	try cg.initializeStack();
1103	}
1104
1105	cg.stack_alignment = cg.stack_alignment.max(alignment);
1106
1107	const offset: u32 = @intCast(alignment.forward(cg.stack_size));
1108	defer cg.stack_size = offset + size;
1109
1110	return .{ .stack_offset = .{ .value = offset, .references = 1 } };
1111	}
1112
1113	/// From a given AIR instruction generates a pointer to the stack where
1114	/// the value of its type will live.
1115	/// This is different from allocStack where this will use the pointer's alignment
1116	/// if it is set, to ensure the stack alignment will be set correctly.
1117	fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {
1118	const pt = cg.pt;
1119	const zcu = pt.zcu;
1120	const ptr_ty = cg.typeOfIndex(inst);
1121	const pointee_ty = ptr_ty.childType(zcu);
1122
1123	if (cg.initial_stack_value == .none) {
1124	try cg.initializeStack();
1125	}
1126
1127	const abi_alignment = ptr_ty.ptrAlignment(zcu);
1128	const abi_size = std.math.cast(u32, pointee_ty.abiSize(zcu)) orelse {
1129	return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{
1130	pointee_ty.fmt(pt), pointee_ty.abiSize(zcu),
1131	});
1132	};
1133	cg.stack_alignment = cg.stack_alignment.max(abi_alignment);
1134
1135	const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size));
1136	defer cg.stack_size = offset + abi_size;
1137
1138	return .{ .stack_offset = .{ .value = offset, .references = 1 } };
1139	}
1140
1141	fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1142	const len_known_neq_0 = switch (len) {
1143	.imm32 => |val| if (val != 0) true else return,
1144	.imm64 => |val| if (val != 0) true else return,
1145	else => false,
1146	};
1147	const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);
1148	const emit_check = !(len0_ok or len_known_neq_0);
1149
1150	if (emit_check) {
1151	try cg.startBlock(.block, .empty);
1152
1153	// Even if `len` is zero, the spec requires an implementation to trap if `src + len` or
1154	// `dst + len` are out of memory bounds. This can easily happen in Zig in a case such
1155	// as:
1156	//
1157	// const dst: [*]u8 = undefined;
1158	// const src: [*]u8 = undefined;
1159	// var len: usize = runtime_zero();
1160	// @memcpy(dst[0..len], src[0..len]);
1161	//
1162	// So explicitly avoid using `memory.copy` in the `len == 0` case. Lovely design.
1163	try cg.emitWValue(len);
1164	try cg.addTag(.i32_eqz);
1165	try cg.addLabel(.br_if, 0);
1166	}
1167
1168	try cg.lowerToStack(dst);
1169	try cg.lowerToStack(src);
1170	try cg.emitWValue(len);
1171	try cg.addExtended(.memory_copy);
1172
1173	if (emit_check) {
1174	try cg.endBlock();
1175	}
1176	}
1177
1178	fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1179	if (cg.target.cpu.has(.wasm, .bulk_memory)) {
1180	try cg.emitMemoryCopy(dst, src, len);
1181	return;
1182	}
1183
1184	try cg.lowerToStack(dst);
1185	try cg.lowerToStack(src);
1186	try cg.emitWValue(len);
1187	try cg.addCallIntrinsic(.memcpy);
1188	try cg.addTag(.drop);
1189	}
1190
1191	fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1192	if (cg.target.cpu.has(.wasm, .bulk_memory)) {
1193	try cg.emitMemoryCopy(dst, src, len);
1194	return;
1195	}
1196
1197	try cg.lowerToStack(dst);
1198	try cg.lowerToStack(src);
1199	try cg.emitWValue(len);
1200	try cg.addCallIntrinsic(.memmove);
1201	try cg.addTag(.drop);
1202	}
1203
1204	fn ptrSize(cg: *const CodeGen) u16 {
1205	return @divExact(cg.target.ptrBitWidth(), 8);
1206	}
1207
1208	/// For a given `Type`, will return true when the type will be passed
1209	/// by reference, rather than by value
1210	fn isByRef(ty: Type, zcu: *const Zcu, target: *const std.Target) bool {
1211	switch (ty.zigTypeTag(zcu)) {
1212	.type,
1213	.comptime_int,
1214	.comptime_float,
1215	.enum_literal,
1216	.undefined,
1217	.null,
1218	.@"opaque",
1219	.spirv,
1220	=> unreachable,
1221
1222	.noreturn,
1223	.void,
1224	.bool,
1225	.error_set,
1226	.@"fn",
1227	.@"anyframe",
1228	=> return false,
1229
1230	.array,
1231	.frame,
1232	=> return ty.hasRuntimeBits(zcu),
1233	.@"struct", .@"union" => switch (ty.containerLayout(zcu)) {
1234	.@"packed" => return isByRef(ty.backingIntType(zcu), zcu, target),
1235	.@"extern", .auto => return ty.hasRuntimeBits(zcu),
1236	},
1237	.vector => return determineSimdStoreStrategy(ty, zcu, target) == .unrolled,
1238	.int => return ty.intInfo(zcu).bits > 64,
1239	.@"enum" => return ty.intInfo(zcu).bits > 64,
1240	.float => return ty.floatBits(target) > 64,
1241	.error_union => {
1242	const pl_ty = ty.errorUnionPayload(zcu);
1243	if (!pl_ty.hasRuntimeBits(zcu)) {
1244	return false;
1245	}
1246	return true;
1247	},
1248	.optional => {
1249	if (ty.isPtrLikeOptional(zcu)) return false;
1250	const pl_type = ty.optionalChild(zcu);
1251	if (pl_type.zigTypeTag(zcu) == .error_set) return false;
1252	return pl_type.hasRuntimeBits(zcu);
1253	},
1254	.pointer => {
1255	// Slices act like struct and will be passed by reference
1256	if (ty.isSlice(zcu)) return true;
1257	return false;
1258	},
1259	}
1260	}
1261
1262	const SimdStoreStrategy = enum {
1263	direct,
1264	unrolled,
1265	};
1266
1267	/// For a given vector type, returns the `SimdStoreStrategy`.
1268	/// This means when a given type is 128 bits and either the simd128 or relaxed-simd
1269	/// features are enabled, the function will return `.direct`. This would allow to store
1270	/// it using a instruction, rather than an unrolled version.
1271	pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std.Target) SimdStoreStrategy {
1272	assert(ty.zigTypeTag(zcu) == .vector);
1273	if (ty.bitSize(zcu) != 128) return .unrolled;
1274	if (target.cpu.has(.wasm, .relaxed_simd) or target.cpu.has(.wasm, .simd128)) {
1275	return .direct;
1276	}
1277	return .unrolled;
1278	}
1279
1280	/// Creates a new local for a pointer that points to memory with given offset.
1281	/// This can be used to get a pointer to a struct field, error payload, etc.
1282	/// By providing `modify` as action, it will modify the given `ptr_value` instead of making a new
1283	/// local value to store the pointer. This allows for local re-use and improves binary size.
1284	fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum { modify, new }) InnerError!WValue {
1285	// do not perform arithmetic when offset is 0.
1286	if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value;
1287	const result_ptr: WValue = switch (action) {
1288	.new => try cg.ensureAllocLocal(Type.usize),
1289	.modify => ptr_value,
1290	};
1291	try cg.emitWValue(ptr_value);
1292	if (offset + ptr_value.offset() > 0) {
1293	switch (cg.ptr_size) {
1294	.wasm32 => {
1295	try cg.addImm32(@intCast(offset + ptr_value.offset()));
1296	try cg.addTag(.i32_add);
1297	},
1298	.wasm64 => {
1299	try cg.addImm64(offset + ptr_value.offset());
1300	try cg.addTag(.i64_add);
1301	},
1302	}
1303	}
1304	try cg.addLocal(.local_set, result_ptr.local.value);
1305	return result_ptr;
1306	}
1307
1308	fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1309	const zcu = cg.pt.zcu;
1310	const air_tags = cg.air.instructions.items(.tag);
1311	return switch (air_tags[@backingInt(inst)]) {
1312	// No soft float legalizations are enabled.
1313	.legalize_compiler_rt_call => unreachable,
1314
1315	.inferred_alloc, .inferred_alloc_comptime => unreachable,
1316
1317	.legalize_vec_elem_val => cg.airArrayElemVal(inst),
1318	.legalize_vec_store_elem => {
1319	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
1320	const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data;
1321	const vec_ptr = try cg.resolveInst(pl_op.operand);
1322	const elem_idx = try cg.resolveInst(bin_op.lhs);
1323	const elem_val = try cg.resolveInst(bin_op.rhs);
1324
1325	const elem_ty = cg.typeOf(bin_op.rhs);
1326	const elem_size = elem_ty.abiSize(zcu);
1327
1328	try cg.lowerToStack(vec_ptr);
1329	try cg.emitWValue(elem_idx);
1330	try cg.addImm32(@intCast(elem_size));
1331	try cg.addTag(.i32_mul);
1332	try cg.addTag(.i32_add);
1333	const ptr = try WValue.toLocal(.stack, cg, Type.usize);
1334
1335	try cg.store(ptr, elem_val, elem_ty, 0);
1336
1337	return cg.finishAir(inst, .none, &.{ pl_op.operand, bin_op.lhs, bin_op.rhs });
1338	},
1339
1340	.add,
1341	.sub,
1342	.mul,
1343	.rem,
1344	.mod,
1345	.max,
1346	.min,
1347	.div_exact,
1348	.div_trunc,
1349	.div_floor,
1350	.div_ceil,
1351	=> |tag| {
1352	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1353	const lhs = try cg.resolveInst(bin_op.lhs);
1354	const rhs = try cg.resolveInst(bin_op.rhs);
1355
1356	const ty = cg.typeOfIndex(inst);
1357	const type_tag = ty.zigTypeTag(zcu);
1358
1359	if (type_tag == .vector) {
1360	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1361	}
1362
1363	if (type_tag == .float) {
1364	const float_ty: FloatType = .fromType(cg, ty);
1365
1366	const result = switch (tag) {
1367	.add => try cg.floatAdd(float_ty, lhs, rhs),
1368	.sub => try cg.floatSub(float_ty, lhs, rhs),
1369	.mul => try cg.floatMul(float_ty, lhs, rhs),
1370	.rem => try cg.floatRem(float_ty, lhs, rhs),
1371	.mod => try cg.floatMod(float_ty, lhs, rhs),
1372	.max => try cg.floatMax(float_ty, lhs, rhs),
1373	.min => try cg.floatMin(float_ty, lhs, rhs),
1374	.div_exact => try cg.floatDiv(float_ty, lhs, rhs),
1375	.div_trunc => try cg.floatDivTrunc(float_ty, lhs, rhs),
1376	.div_floor => try cg.floatDivFloor(float_ty, lhs, rhs),
1377	.div_ceil => try cg.floatDivCeil(float_ty, lhs, rhs),
1378	else => unreachable,
1379	};
1380
1381	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1382	} else if (type_tag == .int) {
1383	const int_ty: IntType = .fromType(cg, ty);
1384
1385	const result = switch (tag) {
1386	.add => try cg.intAdd(int_ty, lhs, rhs),
1387	.sub => try cg.intSub(int_ty, lhs, rhs),
1388	.mul => try cg.intMul(int_ty, lhs, rhs),
1389	.rem => try cg.intRem(int_ty, lhs, rhs),
1390	.mod => try cg.intMod(int_ty, lhs, rhs),
1391	.max => try cg.intMax(int_ty, lhs, rhs),
1392	.min => try cg.intMin(int_ty, lhs, rhs),
1393	.div_exact => try cg.intDiv(int_ty, lhs, rhs),
1394	.div_trunc => try cg.intDiv(int_ty, lhs, rhs),
1395	.div_floor => try cg.intDivFloor(int_ty, lhs, rhs),
1396	.div_ceil => try cg.intDivCeil(int_ty, lhs, rhs),
1397	else => unreachable,
1398	};
1399
1400	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1401	} else {
1402	unreachable;
1403	}
1404	},
1405	.div_float => {
1406	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1407	const lhs = try cg.resolveInst(bin_op.lhs);
1408	const rhs = try cg.resolveInst(bin_op.rhs);
1409	const ty = cg.typeOfIndex(inst);
1410
1411	if (ty.zigTypeTag(zcu) == .vector) {
1412	return cg.fail("TODO: implement AIR op: div_float for vectors", .{});
1413	}
1414
1415	const result = try cg.floatDiv(.fromType(cg, ty), lhs, rhs);
1416	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1417	},
1418	.abs => {
1419	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1420	const operand = try cg.resolveInst(ty_op.operand);
1421
1422	const ty = cg.typeOf(ty_op.operand);
1423	const type_tag = ty.zigTypeTag(zcu);
1424
1425	if (type_tag == .vector) {
1426	return cg.fail("TODO: implement AIR op: abs for vectors", .{});
1427	}
1428
1429	if (type_tag == .float) {
1430	const result = try cg.floatAbs(.fromType(cg, ty), operand);
1431	return cg.finishAir(inst, result, &.{ty_op.operand});
1432	} else if (type_tag == .int) {
1433	const result = try cg.intAbs(.fromType(cg, ty), operand);
1434	return cg.finishAir(inst, result, &.{ty_op.operand});
1435	} else {
1436	unreachable;
1437	}
1438	},
1439	.mul_add => {
1440	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
1441	const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data;
1442	const addend = try cg.resolveInst(pl_op.operand);
1443	const lhs = try cg.resolveInst(bin_op.lhs);
1444	const rhs = try cg.resolveInst(bin_op.rhs);
1445	const ty = cg.typeOfIndex(inst);
1446
1447	if (ty.zigTypeTag(cg.pt.zcu) == .vector) {
1448	return cg.fail("TODO: implement AIR op: mul_add for vectors", .{});
1449	}
1450
1451	const result = try cg.floatMulAdd(.fromType(cg, ty), lhs, rhs, addend);
1452	return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand });
1453	},
1454
1455	.add_sat,
1456	.sub_sat,
1457	.mul_sat,
1458	.shl_sat,
1459	=> |tag| {
1460	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1461	const lhs = try cg.resolveInst(bin_op.lhs);
1462	const rhs = try cg.resolveInst(bin_op.rhs);
1463	const ty = cg.typeOfIndex(inst);
1464
1465	if (ty.zigTypeTag(cg.pt.zcu) == .vector) {
1466	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1467	}
1468
1469	const int_ty: IntType = .fromType(cg, ty);
1470	const result = switch (tag) {
1471	.add_sat => try cg.intAddSat(int_ty, lhs, rhs),
1472	.sub_sat => try cg.intSubSat(int_ty, lhs, rhs),
1473	.mul_sat => try cg.intMulSat(int_ty, lhs, rhs),
1474	.shl_sat => try cg.intShlSat(int_ty, lhs, rhs),
1475	else => unreachable,
1476	};
1477
1478	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1479	},
1480
1481	.add_with_overflow,
1482	.sub_with_overflow,
1483	.mul_with_overflow,
1484	.shl_with_overflow,
1485	=> |tag| {
1486	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
1487	const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
1488
1489	const lhs = try cg.resolveInst(extra.lhs);
1490	const rhs = try cg.resolveInst(extra.rhs);
1491
1492	const ty = cg.typeOf(extra.lhs);
1493	const int_ty: IntType = .fromType(cg, ty);
1494
1495	const out = switch (tag) {
1496	.add_with_overflow => try cg.intAddOverflow(int_ty, lhs, rhs),
1497	.sub_with_overflow => try cg.intSubOverflow(int_ty, lhs, rhs),
1498	.mul_with_overflow => try cg.intMulOverflow(int_ty, lhs, rhs),
1499	.shl_with_overflow => try cg.intShlOverflow(int_ty, lhs, rhs),
1500	else => unreachable,
1501	};
1502
1503	var ov_tmp = try out.ov.toLocal(cg, Type.u1);
1504	defer ov_tmp.free(cg);
1505
1506	var res_tmp = try out.result.toLocal(cg, ty);
1507	defer res_tmp.free(cg);
1508
1509	const result = try cg.allocStack(cg.typeOfIndex(inst));
1510	const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu));
1511
1512	try cg.store(result, res_tmp, ty, 0);
1513	try cg.store(result, ov_tmp, Type.u1, offset);
1514
1515	try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });
1516	},
1517
1518	.add_wrap, .sub_wrap, .mul_wrap, .shl => |tag| {
1519	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1520	const lhs = try cg.resolveInst(bin_op.lhs);
1521	const rhs = try cg.resolveInst(bin_op.rhs);
1522	const ty = cg.typeOfIndex(inst);
1523
1524	if (ty.zigTypeTag(zcu) == .vector) {
1525	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1526	}
1527
1528	const int_ty: IntType = .fromType(cg, ty);
1529	const raw_result = switch (tag) {
1530	.add_wrap => try cg.intAdd(int_ty, lhs, rhs),
1531	.sub_wrap => try cg.intSub(int_ty, lhs, rhs),
1532	.mul_wrap => try cg.intMul(int_ty, lhs, rhs),
1533	.shl => try cg.intShl(int_ty, lhs, rhs),
1534	else => unreachable,
1535	};
1536	const result = try cg.intWrap(int_ty, raw_result);
1537
1538	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1539	},
1540
1541	.bit_and, .bit_or, .xor, .shl_exact, .shr, .shr_exact => |tag| {
1542	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1543	const lhs = try cg.resolveInst(bin_op.lhs);
1544	const rhs = try cg.resolveInst(bin_op.rhs);
1545	const ty = cg.typeOfIndex(inst);
1546
1547	if (ty.zigTypeTag(zcu) == .vector) {
1548	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1549	}
1550
1551	const int_ty: IntType = .fromType(cg, ty);
1552	const result = switch (tag) {
1553	.bit_and => try cg.intAnd(int_ty, lhs, rhs),
1554	.bit_or => try cg.intOr(int_ty, lhs, rhs),
1555	.xor => try cg.intXor(int_ty, lhs, rhs),
1556	.shl_exact => try cg.intShl(int_ty, lhs, rhs),
1557	.shr, .shr_exact => try cg.intShr(int_ty, lhs, rhs),
1558	else => unreachable,
1559	};
1560
1561	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1562	},
1563
1564	.not => {
1565	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1566	const operand = try cg.resolveInst(ty_op.operand);
1567	const ty = cg.typeOf(ty_op.operand);
1568
1569	if (ty.zigTypeTag(zcu) == .vector) {
1570	return cg.fail("TODO: implement AIR op: not for vectors", .{});
1571	}
1572
1573	const result = try cg.intNot(.fromType(cg, ty), operand);
1574	try cg.finishAir(inst, result, &.{ty_op.operand});
1575	},
1576
1577	.ptr_cast => cg.airNopCast(inst),
1578	.error_cast => cg.airNopCast(inst),
1579	.error_from_int => cg.airNopCast(inst),
1580	.int_from_error => cg.airNopCast(inst),
1581	.ptr_from_int => cg.airNopCast(inst),
1582	.int_from_ptr => cg.airIntFromPtr(inst),
1583
1584	.bit_cast => cg.airBitcast(inst),
1585	.union_from_enum => cg.airUnionFromEnum(inst),
1586
1587	.int_cast => {
1588	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1589
1590	const dest_ty = ty_op.ty;
1591	const operand = try cg.resolveInst(ty_op.operand);
1592	const src_ty = cg.typeOf(ty_op.operand);
1593
1594	if (dest_ty.zigTypeTag(zcu) == .vector) {
1595	return cg.fail("TODO: implement AIR op: int_cast for vectors", .{});
1596	}
1597
1598	const src_int_ty: IntType = .fromType(cg, src_ty);
1599	const dest_int_ty: IntType = .fromType(cg, dest_ty);
1600
1601	const src_bits = src_int_ty.bits;
1602	const dest_bits = dest_int_ty.bits;
1603
1604	const same_class: bool = (src_bits <= 32 and dest_bits <= 32) or
1605	(src_bits >= 33 and src_bits <= 64 and dest_bits >= 33 and dest_bits <= 64) or
1606	(src_bits >= 65 and src_bits <= 128 and dest_bits >= 65 and dest_bits <= 128);
1607
1608	const result = if (same_class)
1609	cg.reuseOperand(ty_op.operand, operand)
1610	else
1611	try cg.intCast(dest_int_ty, src_int_ty, operand);
1612
1613	try cg.finishAir(inst, result, &.{ty_op.operand});
1614	},
1615	.trunc => {
1616	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1617
1618	const operand = try cg.resolveInst(ty_op.operand);
1619	const dest_ty = ty_op.ty;
1620	const src_ty = cg.typeOf(ty_op.operand);
1621
1622	if (dest_ty.zigTypeTag(zcu) == .vector or src_ty.zigTypeTag(zcu) == .vector) {
1623	return cg.fail("TODO: implement AIR op: trunc for vectors", .{});
1624	}
1625
1626	const src_int_ty: IntType = .fromType(cg, src_ty);
1627	const dest_int_ty: IntType = .fromType(cg, dest_ty);
1628
1629	const result = if (src_int_ty.bits == dest_int_ty.bits)
1630	cg.reuseOperand(ty_op.operand, operand)
1631	else blk: {
1632	break :blk try cg.intTrunc(dest_int_ty, src_int_ty, operand);
1633	};
1634
1635	try cg.finishAir(inst, result, &.{ty_op.operand});
1636	},
1637
1638	.fptrunc, .fpext => |tag| {
1639	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1640
1641	const operand = try cg.resolveInst(ty_op.operand);
1642	const src_ty = cg.typeOf(ty_op.operand);
1643	const dest_ty = cg.typeOfIndex(inst);
1644
1645	if (dest_ty.zigTypeTag(cg.pt.zcu) == .vector) {
1646	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1647	}
1648
1649	const src_float_ty: FloatType = .fromType(cg, src_ty);
1650	const dest_float_ty: FloatType = .fromType(cg, dest_ty);
1651
1652	const result = switch (tag) {
1653	.fptrunc => try cg.floatTruncCast(dest_float_ty, src_float_ty, operand),
1654	.fpext => try cg.floatExtendCast(dest_float_ty, src_float_ty, operand),
1655	else => unreachable,
1656	};
1657
1658	try cg.finishAir(inst, result, &.{ty_op.operand});
1659	},
1660
1661	.int_from_float => {
1662	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1663	const operand = try cg.resolveInst(ty_op.operand);
1664	const src_ty = cg.typeOf(ty_op.operand);
1665	const dest_ty = cg.typeOfIndex(inst);
1666
1667	if (src_ty.zigTypeTag(zcu) == .vector) {
1668	return cg.fail("TODO: implement AIR op: int_from_float for vectors", .{});
1669	}
1670
1671	const result = try cg.intFromFloat(.fromType(cg, dest_ty), .fromType(cg, src_ty), operand);
1672	try cg.finishAir(inst, result, &.{ty_op.operand});
1673	},
1674	.float_from_int => {
1675	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1676	const operand = try cg.resolveInst(ty_op.operand);
1677	const src_ty = cg.typeOf(ty_op.operand);
1678	const dest_ty = cg.typeOfIndex(inst);
1679
1680	if (src_ty.zigTypeTag(zcu) == .vector) {
1681	return cg.fail("TODO: implement AIR op: float_from_int for vectors", .{});
1682	}
1683
1684	const result = try cg.floatFromInt(.fromType(cg, dest_ty), .fromType(cg, src_ty), operand);
1685	try cg.finishAir(inst, result, &.{ty_op.operand});
1686	},
1687
1688	.clz, .ctz, .popcount, .byte_swap, .bit_reverse => |tag| {
1689	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1690	const operand = try cg.resolveInst(ty_op.operand);
1691
1692	const ty = cg.typeOf(ty_op.operand);
1693
1694	if (ty.zigTypeTag(zcu) == .vector) {
1695	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1696	}
1697
1698	const int_ty: IntType = .fromType(cg, ty);
1699	const result = switch (tag) {
1700	.clz => try cg.intClz(int_ty, operand),
1701	.ctz => try cg.intCtz(int_ty, operand),
1702	.popcount => try cg.intPopCount(int_ty, operand),
1703	.byte_swap => try cg.intByteSwap(int_ty, operand),
1704	.bit_reverse => try cg.intBitReverse(int_ty, operand),
1705	else => unreachable,
1706	};
1707	try cg.finishAir(inst, result, &.{ty_op.operand});
1708	},
1709
1710	.sqrt, .sin, .cos, .tan, .exp, .exp2, .log, .log2, .log10, .floor, .ceil, .round, .trunc_float, .neg => |tag| {
1711	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
1712	const operand = try cg.resolveInst(un_op);
1713	const ty = cg.typeOfIndex(inst);
1714
1715	if (ty.zigTypeTag(zcu) == .vector) {
1716	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1717	}
1718
1719	const float_ty: FloatType = .fromType(cg, ty);
1720	const result = switch (tag) {
1721	.sqrt => try cg.floatSqrt(float_ty, operand),
1722	.sin => try cg.floatSin(float_ty, operand),
1723	.cos => try cg.floatCos(float_ty, operand),
1724	.tan => try cg.floatTan(float_ty, operand),
1725	.exp => try cg.floatExp(float_ty, operand),
1726	.exp2 => try cg.floatExp2(float_ty, operand),
1727	.log => try cg.floatLog(float_ty, operand),
1728	.log2 => try cg.floatLog2(float_ty, operand),
1729	.log10 => try cg.floatLog10(float_ty, operand),
1730	.floor => try cg.floatFloor(float_ty, operand),
1731	.ceil => try cg.floatCeil(float_ty, operand),
1732	.round => try cg.floatRound(float_ty, operand),
1733	.trunc_float => try cg.floatTrunc(float_ty, operand),
1734	.neg => try cg.floatNeg(float_ty, operand),
1735	else => unreachable,
1736	};
1737
1738	try cg.finishAir(inst, result, &.{un_op});
1739	},
1740
1741	.cmp_eq => cg.airCmp(inst, .eq),
1742	.cmp_gte => cg.airCmp(inst, .gte),
1743	.cmp_gt => cg.airCmp(inst, .gt),
1744	.cmp_lte => cg.airCmp(inst, .lte),
1745	.cmp_lt => cg.airCmp(inst, .lt),
1746	.cmp_neq => cg.airCmp(inst, .neq),
1747
1748	.cmp_vector => cg.airCmpVector(inst),
1749	.cmp_lte_errors_len => cg.airCmpLteErrorsLen(inst),
1750
1751	.array_elem_val => cg.airArrayElemVal(inst),
1752	.array_to_slice => cg.airArrayToSlice(inst),
1753	.array_to_vector => unreachable, // legalize .expand_array_to_vector
1754	.alloc => cg.airAlloc(inst),
1755	.arg => cg.airArg(inst),
1756	.block => cg.airBlock(inst),
1757	.trap => cg.airTrap(inst),
1758	.unreach => cg.airUnreachable(inst),
1759	.breakpoint => cg.airBreakpoint(inst),
1760	.br => cg.airBr(inst),
1761	.repeat => cg.airRepeat(inst),
1762	.switch_dispatch => cg.airSwitchDispatch(inst),
1763	.cond_br => cg.airCondBr(inst),
1764
1765	.@"try" => cg.airTry(inst),
1766	.try_cold => cg.airTry(inst),
1767	.try_ptr => cg.airTryPtr(inst),
1768	.try_ptr_cold => cg.airTryPtr(inst),
1769
1770	.dbg_stmt => cg.airDbgStmt(inst),
1771	.dbg_empty_stmt => try cg.finishAir(inst, .none, &.{}),
1772	.dbg_inline_block => cg.airDbgInlineBlock(inst),
1773	.dbg_var_ptr => cg.airDbgVar(inst, .local_var, true),
1774	.dbg_var_val => cg.airDbgVar(inst, .local_var, false),
1775	.dbg_arg_inline => cg.airDbgVar(inst, .arg, false),
1776
1777	.call => cg.airCall(inst, .auto),
1778	.call_always_tail => cg.airCall(inst, .always_tail),
1779	.call_never_tail => cg.airCall(inst, .never_tail),
1780	.call_never_inline => cg.airCall(inst, .never_inline),
1781
1782	.is_err => cg.airIsErr(inst, .i32_ne, .value),
1783	.is_non_err => cg.airIsErr(inst, .i32_eq, .value),
1784	.is_err_ptr => cg.airIsErr(inst, .i32_ne, .ptr),
1785	.is_non_err_ptr => cg.airIsErr(inst, .i32_eq, .ptr),
1786
1787	.is_null => cg.airIsNull(inst, .i32_eq, .value),
1788	.is_non_null => cg.airIsNull(inst, .i32_ne, .value),
1789	.is_null_ptr => cg.airIsNull(inst, .i32_eq, .ptr),
1790	.is_non_null_ptr => cg.airIsNull(inst, .i32_ne, .ptr),
1791
1792	.load => cg.airLoad(inst),
1793	.loop => cg.airLoop(inst),
1794	.memset => cg.airMemset(inst, false),
1795	.memset_safe => cg.airMemset(inst, true),
1796	.optional_payload => cg.airOptionalPayload(inst),
1797	.optional_payload_ptr => cg.airOptionalPayloadPtr(inst),
1798	.optional_payload_ptr_set => cg.airOptionalPayloadPtrSet(inst),
1799	.ptr_add => cg.airPtrBinOp(inst, .add),
1800	.ptr_sub => cg.airPtrBinOp(inst, .sub),
1801	.ptr_elem_ptr => cg.airPtrElemPtr(inst),
1802	.ptr_elem_val => cg.airPtrElemVal(inst),
1803	.ret => cg.airRet(inst),
1804	.ret_safe => cg.airRet(inst), // TODO
1805	.ret_ptr => cg.airRetPtr(inst),
1806	.ret_load => cg.airRetLoad(inst),
1807	.splat => cg.airSplat(inst),
1808	.select => cg.airSelect(inst),
1809	.shuffle_one => cg.airShuffleOne(inst),
1810	.shuffle_two => cg.airShuffleTwo(inst),
1811	.reduce => cg.airReduce(inst),
1812	.aggregate_init => cg.airAggregateInit(inst),
1813	.union_init => cg.airUnionInit(inst),
1814	.prefetch => cg.airPrefetch(inst),
1815
1816	.slice => cg.airSlice(inst),
1817	.slice_len => cg.airSliceLen(inst),
1818	.slice_elem_val => cg.airSliceElemVal(inst),
1819	.slice_elem_ptr => cg.airSliceElemPtr(inst),
1820	.slice_ptr => cg.airSlicePtr(inst),
1821	.ptr_slice_len_ptr => cg.airPtrSliceFieldPtr(inst, cg.ptrSize()),
1822	.ptr_slice_ptr_ptr => cg.airPtrSliceFieldPtr(inst, 0),
1823	.store => cg.airStore(inst, false),
1824	.store_safe => cg.airStore(inst, true),
1825
1826	.set_union_tag => cg.airSetUnionTag(inst),
1827	.get_union_tag => cg.airGetUnionTag(inst),
1828	.struct_field_ptr => cg.airStructFieldPtr(inst),
1829	.struct_field_ptr_index_0 => cg.airStructFieldPtrIndex(inst, 0),
1830	.struct_field_ptr_index_1 => cg.airStructFieldPtrIndex(inst, 1),
1831	.struct_field_ptr_index_2 => cg.airStructFieldPtrIndex(inst, 2),
1832	.struct_field_ptr_index_3 => cg.airStructFieldPtrIndex(inst, 3),
1833	.agg_field_val => cg.airAggFieldVal(inst),
1834	.field_parent_ptr => cg.airFieldParentPtr(inst),
1835
1836	.switch_br => cg.airSwitchBr(inst, false),
1837	.loop_switch_br => cg.airSwitchBr(inst, true),
1838
1839	.wrap_optional => cg.airWrapOptional(inst),
1840	.unwrap_errunion_payload => cg.airUnwrapErrUnionPayload(inst, false),
1841	.unwrap_errunion_payload_ptr => cg.airUnwrapErrUnionPayload(inst, true),
1842	.unwrap_errunion_err => cg.airUnwrapErrUnionError(inst, false),
1843	.unwrap_errunion_err_ptr => cg.airUnwrapErrUnionError(inst, true),
1844	.wrap_errunion_payload => cg.airWrapErrUnionPayload(inst),
1845	.wrap_errunion_err => cg.airWrapErrUnionErr(inst),
1846	.errunion_payload_ptr_set => cg.airErrUnionPayloadPtrSet(inst),
1847	.error_name => cg.airErrorName(inst),
1848
1849	.wasm_memory_size => cg.airWasmMemorySize(inst),
1850	.wasm_memory_grow => cg.airWasmMemoryGrow(inst),
1851
1852	.memcpy => cg.airMemcpy(inst),
1853	.memmove => cg.airMemmove(inst),
1854
1855	.ret_addr => cg.airRetAddr(inst),
1856	.tag_name => cg.airTagName(inst),
1857
1858	.error_set_has_value => cg.airErrorSetHasValue(inst),
1859	.frame_addr => cg.airFrameAddress(inst),
1860
1861	.runtime_nav_ptr => cg.airRuntimeNavPtr(inst),
1862
1863	.assembly => cg.airAsm(inst),
1864
1865	.c_va_arg => try cg.airVaArg(inst),
1866	.c_va_copy => try cg.airVaCopy(inst),
1867	.c_va_end => try cg.airVaEnd(inst),
1868	.c_va_start => try cg.airVaStart(inst),
1869
1870	.is_named_enum_value => try cg.airIsNamedEnumValue(inst),
1871
1872	.err_return_trace,
1873	.set_err_return_trace,
1874	.save_err_return_trace_index,
1875	.addrspace_cast,
1876	=> |tag| return cg.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),
1877
1878	.atomic_load => cg.airAtomicLoad(inst),
1879	.atomic_store_unordered,
1880	.atomic_store_monotonic,
1881	.atomic_store_release,
1882	.atomic_store_seq_cst,
1883	// in WebAssembly, all atomic instructions are sequentially ordered.
1884	=> cg.airAtomicStore(inst),
1885	.atomic_rmw => cg.airAtomicRmw(inst),
1886	.cmpxchg_weak => cg.airCmpxchg(inst),
1887	.cmpxchg_strong => cg.airCmpxchg(inst),
1888
1889	.add_optimized,
1890	.sub_optimized,
1891	.mul_optimized,
1892	.div_float_optimized,
1893	.div_trunc_optimized,
1894	.div_floor_optimized,
1895	.div_ceil_optimized,
1896	.div_exact_optimized,
1897	.rem_optimized,
1898	.mod_optimized,
1899	.neg_optimized,
1900	.cmp_lt_optimized,
1901	.cmp_lte_optimized,
1902	.cmp_eq_optimized,
1903	.cmp_gte_optimized,
1904	.cmp_gt_optimized,
1905	.cmp_neq_optimized,
1906	.cmp_vector_optimized,
1907	.reduce_optimized,
1908	.int_from_float_optimized,
1909	=> return cg.fail("TODO implement optimized float mode", .{}),
1910
1911	.add_safe,
1912	.sub_safe,
1913	.mul_safe,
1914	.bit_cast_safe,
1915	.int_cast_safe,
1916	.int_from_float_safe,
1917	.int_from_float_optimized_safe,
1918	=> return cg.fail("TODO implement safety_checked_instructions", .{}),
1919
1920	.work_item_id,
1921	.work_group_size,
1922	.work_group_id,
1923	.spirv_runtime_array_len,
1924	=> unreachable,
1925	};
1926	}
1927
1928	fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1929	const zcu = cg.pt.zcu;
1930	const ip = &zcu.intern_pool;
1931
1932	for (body) |inst| {
1933	if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) {
1934	continue;
1935	}
1936	const old_bookkeeping_value = cg.air_bookkeeping;
1937	try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, 1);
1938	try cg.genInst(inst);
1939
1940	if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) {
1941	std.debug.panic("Missing call to `finishAir` in AIR instruction %{d} ('{t}')", .{
1942	inst,
1943	cg.air.instructions.items(.tag)[@backingInt(inst)],
1944	});
1945	}
1946	}
1947	}
1948
1949	fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1950	const zcu = cg.pt.zcu;
1951	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
1952	const operand = try cg.resolveInst(un_op);
1953	const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
1954	const ret_ty = Type.fromInterned(fn_info.return_type);
1955
1956	// result must be stored in the stack and we return a pointer
1957	// to the stack instead
1958	if (cg.return_value != .none) {
1959	try cg.store(cg.return_value, operand, ret_ty, 0);
1960	} else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBits(zcu)) {
1961	switch (abi.classifyType(ret_ty, zcu, cg.target)) {
1962	.direct => |scalar_type| {
1963	if (!isByRef(ret_ty, zcu, cg.target)) {
1964	try cg.emitWValue(operand);
1965	} else {
1966	_ = try cg.load(operand, scalar_type, 0);
1967	}
1968	},
1969	.double_i64, .indirect => unreachable,
1970	.unrolled => |vector| {
1971	assert(vector.len == 1);
1972	_ = try cg.load(operand, vector.elem_type, 0);
1973	},
1974	}
1975	} else {
1976	if (!ret_ty.hasRuntimeBits(zcu) and ret_ty.isError(zcu)) {
1977	try cg.addImm32(0);
1978	} else {
1979	try cg.emitWValue(operand);
1980	}
1981	}
1982	try cg.restoreStackPointer();
1983	try cg.addTag(.@"return");
1984
1985	return cg.finishAir(inst, .none, &.{un_op});
1986	}
1987
1988	fn airRetPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1989	const zcu = cg.pt.zcu;
1990	const child_type = cg.typeOfIndex(inst).childType(zcu);
1991
1992	const result = result: {
1993	if (!child_type.hasRuntimeBits(zcu)) {
1994	break :result try cg.allocStack(Type.usize); // create pointer to void
1995	}
1996
1997	const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
1998	if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {
1999	break :result cg.return_value;
2000	}
2001
2002	break :result try cg.allocStackPtr(inst);
2003	};
2004
2005	return cg.finishAir(inst, result, &.{});
2006	}
2007
2008	fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2009	const zcu = cg.pt.zcu;
2010	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
2011	const operand = try cg.resolveInst(un_op);
2012	const ret_ty = cg.typeOf(un_op).childType(zcu);
2013
2014	const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
2015	if (!ret_ty.hasRuntimeBits(zcu)) {
2016	if (ret_ty.isError(zcu)) {
2017	try cg.addImm32(0);
2018	}
2019	} else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {
2020	if (fn_info.cc == .wasm_mvp) {
2021	switch (abi.classifyType(ret_ty, zcu, cg.target)) {
2022	.direct => |scalar_type| _ = try cg.load(operand, scalar_type, 0),
2023	.double_i64, .indirect => unreachable,
2024	.unrolled => |vector| {
2025	assert(vector.len == 1);
2026	_ = try cg.load(operand, vector.elem_type, 0);
2027	},
2028	}
2029	} else {
2030	_ = try cg.load(operand, ret_ty, 0);
2031	}
2032	}
2033
2034	try cg.restoreStackPointer();
2035	try cg.addTag(.@"return");
2036	return cg.finishAir(inst, .none, &.{un_op});
2037	}
2038
2039	fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) InnerError!void {
2040	if (modifier == .always_tail) return cg.fail("TODO implement tail calls for wasm", .{});
2041	const call = cg.air.unwrapCall(inst);
2042	const args = call.args;
2043	const ty = cg.typeOf(call.callee);
2044
2045	const pt = cg.pt;
2046	const zcu = pt.zcu;
2047	const ip = &zcu.intern_pool;
2048	const fn_ty = switch (ty.zigTypeTag(zcu)) {
2049	.@"fn" => ty,
2050	.pointer => ty.childType(zcu),
2051	else => unreachable,
2052	};
2053	const ret_ty = fn_ty.fnReturnType(zcu);
2054	const fn_info = zcu.typeToFunc(fn_ty).?;
2055	const first_param_sret = firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target);
2056
2057	const callee: ?InternPool.Nav.Index = blk: {
2058	const func_val: Value = .fromInterned(call.callee.toInterned() orelse break :blk null);
2059
2060	switch (ip.indexToKey(func_val.toIntern())) {
2061	inline .func, .@"extern" => |x| break :blk x.owner_nav,
2062	.ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) {
2063	.nav => |nav| break :blk nav,
2064	else => {},
2065	},
2066	else => {},
2067	}
2068	return cg.fail("unable to lower callee to a function index", .{});
2069	};
2070
2071	const sret: WValue = if (first_param_sret)
2072	try cg.allocStack(ret_ty)
2073	else
2074	.none;
2075
2076	const fixed_arg_count = fn_info.param_types.len;
2077
2078	const varargs_buf: WValue = if (fn_info.is_var_args) buf: {
2079	var varargs_size: u32 = 0;
2080	var varargs_align: Alignment = .fromByteUnits(1);
2081
2082	for (args[fixed_arg_count..]) |arg| {
2083	const arg_ty = cg.typeOf(arg);
2084	if (!arg_ty.hasRuntimeBits(zcu)) continue;
2085
2086	const arg_size = std.math.cast(u32, arg_ty.abiSize(zcu)) orelse {
2087	return cg.fail("argument type {f} too large for wasm varargs buffer", .{arg_ty.fmt(pt)});
2088	};
2089	const arg_align = arg_ty.abiAlignment(zcu);
2090
2091	varargs_align = varargs_align.max(arg_align);
2092	varargs_size = @intCast(arg_align.forward(varargs_size));
2093	varargs_size += arg_size;
2094	}
2095
2096	if (varargs_size == 0) varargs_size = 1;
2097
2098	const buffer = try cg.allocStackBytes(varargs_size, varargs_align);
2099
2100	var offset: u32 = 0;
2101	for (args[fixed_arg_count..]) |arg| {
2102	const arg_ty = cg.typeOf(arg);
2103	if (!arg_ty.hasRuntimeBits(zcu)) continue;
2104
2105	const arg_val = try cg.resolveInst(arg);
2106	const arg_size = std.math.cast(u32, arg_ty.abiSize(zcu)) orelse {
2107	return cg.fail("argument type {f} too large for wasm varargs buffer", .{arg_ty.fmt(pt)});
2108	};
2109	const arg_align = arg_ty.abiAlignment(zcu);
2110
2111	offset = @intCast(arg_align.forward(offset));
2112	try cg.store(buffer, arg_val, arg_ty, offset);
2113	offset += arg_size;
2114	}
2115
2116	break :buf buffer;
2117	} else .none;
2118
2119	if (first_param_sret) {
2120	try cg.lowerToStack(sret);
2121	}
2122
2123	for (args, 0..) |arg, arg_i| {
2124	if (fn_info.is_var_args and arg_i >= fixed_arg_count) break;
2125
2126	const arg_ty = cg.typeOf(arg);
2127	if (!arg_ty.hasRuntimeBits(zcu)) continue;
2128
2129	const arg_val = try cg.resolveInst(arg);
2130	try cg.lowerArg(fn_info.cc, arg_ty, arg_val);
2131	}
2132
2133	if (fn_info.is_var_args) {
2134	try cg.lowerToStack(varargs_buf);
2135	}
2136
2137	if (callee) |nav_index| {
2138	try cg.addInst(.{ .tag = .call_nav, .data = .{ .nav_index = nav_index } });
2139	} else {
2140	// in this case we call a function pointer
2141	// so load its value onto the stack
2142	assert(ty.zigTypeTag(zcu) == .pointer);
2143	const operand = try cg.resolveInst(call.callee);
2144	try cg.emitWValue(operand);
2145
2146	try cg.mir_func_tys.put(cg.gpa, fn_ty.toIntern(), {});
2147	try cg.addInst(.{
2148	.tag = .call_indirect,
2149	.data = .{ .ip_index = fn_ty.toIntern() },
2150	});
2151	}
2152
2153	const result_value = result_value: {
2154	if (!ret_ty.hasRuntimeBits(zcu) and !ret_ty.isError(zcu)) {
2155	break :result_value .none;
2156	} else if (first_param_sret) {
2157	break :result_value sret;
2158	} else if (zcu.typeToFunc(fn_ty).?.cc == .wasm_mvp) {
2159	switch (abi.classifyType(ret_ty, zcu, cg.target)) {
2160	.direct => |scalar_type| {
2161	if (!isByRef(ret_ty, zcu, cg.target)) {
2162	const result_local = try cg.allocLocal(ret_ty);
2163	try cg.addLocal(.local_set, result_local.local.value);
2164	break :result_value result_local;
2165	} else {
2166	const result_local = try cg.allocLocal(scalar_type);
2167	try cg.addLocal(.local_set, result_local.local.value);
2168	const result = try cg.allocStack(ret_ty);
2169	try cg.store(result, result_local, scalar_type, 0);
2170	break :result_value result;
2171	}
2172	},
2173	.double_i64, .indirect => unreachable,
2174	.unrolled => |vector| {
2175	assert(vector.len == 1);
2176	const result_local = try cg.allocLocal(vector.elem_type);
2177	// save call result from operand stack
2178	try cg.addLocal(.local_set, result_local.local.value);
2179	const result = try cg.allocStack(ret_ty);
2180	try cg.store(result, result_local, vector.elem_type, 0);
2181	break :result_value result;
2182	},
2183	}
2184	} else {
2185	const result_local = try cg.allocLocal(ret_ty);
2186	try cg.addLocal(.local_set, result_local.local.value);
2187	break :result_value result_local;
2188	}
2189	};
2190
2191	var bt = cg.liveness.iterateBigTomb(inst);
2192	cg.feed(&bt, call.callee);
2193	for (args) |arg| cg.feed(&bt, arg);
2194	return cg.finishAirResult(inst, result_value);
2195	}
2196
2197	fn airVaStart(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2198	try cg.emitWValue(cg.varargs);
2199	return cg.finishAir(inst, .stack, &.{});
2200	}
2201
2202	fn airVaEnd(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2203	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
2204	return cg.finishAir(inst, .none, &.{un_op});
2205	}
2206
2207	fn airVaCopy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2208	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2209	const operand = try cg.resolveInst(ty_op.operand);
2210
2211	const result = try cg.load(operand, .usize, 0);
2212
2213	return cg.finishAir(inst, result, &.{ty_op.operand});
2214	}
2215
2216	fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2217	const zcu = cg.pt.zcu;
2218	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2219	const operand = try cg.resolveInst(ty_op.operand);
2220
2221	const ty = cg.typeOfIndex(inst);
2222
2223	if (!ty.hasRuntimeBits(zcu)) {
2224	return cg.finishAir(inst, .none, &.{ty_op.operand});
2225	}
2226
2227	const is_f32_va_arg = ty.toIntern() == .f32_type;
2228	const load_ty: Type = if (is_f32_va_arg) Type.f64 else ty;
2229
2230	const abi_size: u32 = @intCast(load_ty.abiSize(zcu));
2231	const abi_align: u32 = @intCast(load_ty.abiAlignment(zcu).toByteUnits().?);
2232
2233	const arg_ptr = try cg.allocLocal(.usize);
2234	_ = try cg.load(operand, .usize, 0);
2235
2236	if (abi_align > 1) {
2237	switch (cg.ptr_size) {
2238	.wasm32 => {
2239	try cg.addImm32(abi_align - 1);
2240	try cg.addTag(.i32_add);
2241	try cg.addImm32(~(abi_align - 1));
2242	try cg.addTag(.i32_and);
2243	},
2244	.wasm64 => {
2245	try cg.addImm64(abi_align - 1);
2246	try cg.addTag(.i64_add);
2247	try cg.addImm64(~@as(u64, abi_align - 1));
2248	try cg.addTag(.i64_and);
2249	},
2250	}
2251	}
2252
2253	try cg.addLocal(.local_set, arg_ptr.local.value);
2254
2255	try cg.lowerToStack(operand);
2256	try cg.lowerToStack(arg_ptr);
2257	switch (cg.ptr_size) {
2258	.wasm32 => {
2259	try cg.addImm32(abi_size);
2260	try cg.addTag(.i32_add);
2261	},
2262	.wasm64 => {
2263	try cg.addImm64(abi_size);
2264	try cg.addTag(.i64_add);
2265	},
2266	}
2267	try cg.store(.stack, .stack, .usize, 0);
2268
2269	const result = if (is_f32_va_arg) result: {
2270	const promoted = try cg.load(arg_ptr, Type.f64, 0);
2271	try cg.emitWValue(promoted);
2272	try cg.addTag(.f32_demote_f64);
2273
2274	const result_local = try cg.allocLocal(Type.f32);
2275	try cg.addLocal(.local_set, result_local.local.value);
2276	break :result result_local;
2277	} else try cg.load(arg_ptr, ty, 0);
2278
2279	return cg.finishAir(inst, result, &.{ty_op.operand});
2280	}
2281
2282	fn airAlloc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2283	const value = try cg.allocStackPtr(inst);
2284	return cg.finishAir(inst, value, &.{});
2285	}
2286
2287	fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2288	const pt = cg.pt;
2289	const zcu = pt.zcu;
2290	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2291
2292	const lhs = try cg.resolveInst(bin_op.lhs);
2293	const rhs = try cg.resolveInst(bin_op.rhs);
2294	const ptr_ty = cg.typeOf(bin_op.lhs);
2295	const ptr_info = ptr_ty.ptrInfo(zcu);
2296	const elem_ty = ptr_ty.childType(zcu);
2297
2298	if (!safety and bin_op.rhs == .undef) {
2299	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2300	}
2301
2302	const offset: u32 = switch (ptr_info.flags.vector_index) {
2303	.none => offset: {
2304	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
2305	break :offset 0;
2306	},
2307	else => |index| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),
2308	};
2309	try cg.store(lhs, rhs, elem_ty, offset);
2310	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2311	}
2312
2313	fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void {
2314	assert(!(lhs != .stack and rhs == .stack));
2315	const pt = cg.pt;
2316	const zcu = pt.zcu;
2317	const abi_size = ty.abiSize(zcu);
2318
2319	if (!ty.hasRuntimeBits(zcu)) return;
2320
2321	if (isByRef(ty, zcu, cg.target)) {
2322	const offset_ptr: WValue = switch (offset + lhs.offset()) {
2323	0 => lhs,
2324	else => |total_offset| ptr: {
2325	try cg.emitWValue(lhs);
2326	try cg.addImm32(total_offset);
2327	try cg.addTag(.i32_add);
2328	break :ptr .stack;
2329	},
2330	};
2331	return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
2332	}
2333
2334	if (ty.zigTypeTag(zcu) == .vector) {
2335	try cg.emitWValue(lhs);
2336	try cg.lowerToStack(rhs);
2337	// TODO: Add helper functions for simd opcodes
2338	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2339	// stores as := opcode, offset, alignment (opcode::memarg)
2340	try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2341	@backingInt(std.wasm.SimdOpcode.v128_store),
2342	offset + lhs.offset(),
2343	@intCast(ty.abiAlignment(zcu).toByteUnits().?),
2344	});
2345	return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2346	}
2347
2348	const store_opcode: Mir.Inst.Tag = opcode: {
2349	if (ty.isAnyFloat()) {
2350	break :opcode switch (abi_size) {
2351	2 => .i32_store16,
2352	4 => .f32_store,
2353	8 => .f64_store,
2354	else => unreachable,
2355	};
2356	} else {
2357	break :opcode switch (abi_size) {
2358	1 => .i32_store8,
2359	2 => .i32_store16,
2360	4 => .i32_store,
2361	8 => .i64_store,
2362	else => unreachable,
2363	};
2364	}
2365	};
2366
2367	try cg.emitWValue(lhs);
2368	try cg.lowerToStack(rhs);
2369
2370	try cg.addMemArg(
2371	store_opcode,
2372	.{
2373	.offset = offset + lhs.offset(),
2374	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
2375	},
2376	);
2377	}
2378
2379	fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2380	const pt = cg.pt;
2381	const zcu = pt.zcu;
2382	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2383	const operand = try cg.resolveInst(ty_op.operand);
2384	const elem_ty = ty_op.ty;
2385	const ptr_ty = cg.typeOf(ty_op.operand);
2386	const ptr_info = ptr_ty.ptrInfo(zcu);
2387
2388	assert(elem_ty.hasRuntimeBits(zcu));
2389
2390	const offset: u32 = switch (ptr_info.flags.vector_index) {
2391	.none => offset: {
2392	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
2393	break :offset 0;
2394	},
2395	else => |index| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),
2396	};
2397	const result = try cg.load(operand, elem_ty, offset);
2398	return cg.finishAir(inst, result, &.{ty_op.operand});
2399	}
2400
2401	/// Loads an operand from the linear memory section.
2402	/// NOTE: Leaves the value on the stack, if isByRef == false.
2403	fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
2404	const zcu = cg.pt.zcu;
2405	if (isByRef(ty, zcu, cg.target)) {
2406	const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {
2407	0 => operand,
2408	else => |total_offset| ptr: {
2409	try cg.emitWValue(operand);
2410	try cg.addImm32(total_offset);
2411	try cg.addTag(.i32_add);
2412	break :ptr .stack;
2413	},
2414	};
2415	const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
2416	const new_ptr = try cg.allocStack(ty);
2417	try cg.store(new_ptr, src_ptr, ty, 0);
2418	return new_ptr;
2419	}
2420
2421	// load local's value from memory by its stack position
2422	try cg.emitWValue(operand);
2423
2424	if (ty.zigTypeTag(zcu) == .vector) {
2425	// TODO: Add helper functions for simd opcodes
2426	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2427	// stores as := opcode, offset, alignment (opcode::memarg)
2428	try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2429	@backingInt(std.wasm.SimdOpcode.v128_load),
2430	offset + operand.offset(),
2431	@intCast(ty.abiAlignment(zcu).toByteUnits().?),
2432	});
2433	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2434	return .stack;
2435	}
2436
2437	const abi_size = ty.abiSize(zcu);
2438	const load_opcode: Mir.Inst.Tag = opcode: {
2439	if (ty.isAnyFloat()) {
2440	break :opcode switch (abi_size) {
2441	2 => .i32_load16_u,
2442	4 => .f32_load,
2443	8 => .f64_load,
2444	else => unreachable,
2445	};
2446	} else {
2447	const is_signed = if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness == .signed else false;
2448	break :opcode switch (abi_size) {
2449	1 => if (is_signed) .i32_load8_s else .i32_load8_u,
2450	2 => if (is_signed) .i32_load16_s else .i32_load16_u,
2451	4 => .i32_load,
2452	8 => .i64_load,
2453	else => unreachable,
2454	};
2455	}
2456	};
2457
2458	try cg.addMemArg(
2459	load_opcode,
2460	.{
2461	.offset = offset + operand.offset(),
2462	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
2463	},
2464	);
2465
2466	if (ty.isAbiInt(zcu)) {
2467	const int_info: IntType = .fromType(cg, ty);
2468	switch (int_info.bits) {
2469	8, 16, 32, 64 => {},
2470	else => _ = try cg.intWrap(int_info, .stack),
2471	}
2472	}
2473
2474	return .stack;
2475	}
2476
2477	fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2478	const pt = cg.pt;
2479	const zcu = pt.zcu;
2480	const arg_index = cg.arg_index;
2481	const arg = cg.args[arg_index];
2482	const cc = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?.cc;
2483	const arg_ty = cg.typeOfIndex(inst);
2484	if (cc == .wasm_mvp) {
2485	switch (abi.classifyType(arg_ty, zcu, cg.target)) {
2486	.direct => |scalar_type| {
2487	cg.arg_index += 1;
2488	if (isByRef(arg_ty, zcu, cg.target)) {
2489	const result = try cg.allocStack(arg_ty);
2490	try cg.store(result, arg, scalar_type, 0);
2491	return cg.finishAir(inst, result, &.{});
2492	}
2493	},
2494	.indirect => cg.arg_index += 1,
2495	.double_i64 => {
2496	cg.arg_index += 2;
2497	const result = try cg.allocStack(arg_ty);
2498	try cg.store(result, arg, Type.u64, 0);
2499	try cg.store(result, cg.args[arg_index + 1], Type.u64, 8);
2500	return cg.finishAir(inst, result, &.{});
2501	},
2502	.unrolled => |vector| {
2503	const result = try cg.allocStack(arg_ty);
2504	const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));
2505	for (0..vector.len) |index| {
2506	try cg.store(result, cg.args[cg.arg_index], vector.elem_type, @intCast(index * elem_size));
2507	cg.arg_index += 1;
2508	}
2509	return cg.finishAir(inst, result, &.{});
2510	},
2511	}
2512	} else {
2513	cg.arg_index += 1;
2514	}
2515
2516	return cg.finishAir(inst, arg, &.{});
2517	}
2518
2519	const IntType = struct {
2520	is_signed: bool,
2521	bits: u16,
2522
2523	const @"i32": IntType = .{ .is_signed = true, .bits = 32 };
2524	const @"i64": IntType = .{ .is_signed = true, .bits = 64 };
2525	const @"u32": IntType = .{ .is_signed = false, .bits = 32 };
2526	const @"u64": IntType = .{ .is_signed = false, .bits = 64 };
2527
2528	// Adapted from x86_64 backend
2529	// Differ from Type.intInfo as it treats pointers/booleans/packed/enums/errors as integer
2530	fn fromType(cg: *CodeGen, ty: Type) IntType {
2531	const zcu = cg.pt.zcu;
2532	const ip = &zcu.intern_pool;
2533	var ty_index = ty.ip_index;
2534	while (true) switch (ip.indexToKey(ty_index)) {
2535	.int_type => |int_type| return .{ .is_signed = int_type.signedness == .signed, .bits = int_type.bits },
2536	.ptr_type => |ptr_type| return switch (ptr_type.flags.size) {
2537	.one, .many, .c => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
2538	.slice => unreachable,
2539	},
2540	.opt_type => |opt_child| return if (!Type.fromInterned(opt_child).hasRuntimeBits(zcu))
2541	.{ .is_signed = false, .bits = 1 }
2542	else switch (ip.indexToKey(opt_child)) {
2543	.ptr_type => |ptr_type| switch (ptr_type.flags.size) {
2544	.one, .many => switch (ptr_type.flags.is_allowzero) {
2545	false => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
2546	true => unreachable,
2547	},
2548	.slice, .c => unreachable,
2549	},
2550	else => unreachable,
2551	},
2552	.error_union_type => |error_union_type| return if (!Type.fromInterned(error_union_type.payload_type)
2553	.hasRuntimeBits(zcu)) .{ .is_signed = false, .bits = zcu.errorSetBits() } else unreachable,
2554	.simple_type => |simple_type| return switch (simple_type) {
2555	.bool => .{ .is_signed = false, .bits = 1 },
2556	.anyerror, .adhoc_inferred_error_set => .{ .is_signed = false, .bits = zcu.errorSetBits() },
2557	.isize => .{ .is_signed = true, .bits = cg.target.ptrBitWidth() },
2558	.usize => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
2559	.c_char => .{ .is_signed = cg.target.cCharSignedness().? == .signed, .bits = cg.target.cTypeBitSize(.char).? },
2560	.c_short => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.short).? },
2561	.c_ushort => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.short).? },
2562	.c_int => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.int).? },
2563	.c_uint => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.int).? },
2564	.c_long => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.long).? },
2565	.c_ulong => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.long).? },
2566	.c_longlong => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.longlong).? },
2567	.c_ulonglong => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.longlong).? },
2568	.f16, .f32, .f64, .f80, .f128, .c_longdouble => unreachable,
2569	.anyopaque, .void, .type, .comptime_int, .comptime_float, .noreturn, .null, .undefined, .enum_literal, .generic_poison => unreachable,
2570	},
2571	.enum_type,
2572	.struct_type,
2573	.union_type,
2574	=> ty_index = Type.fromInterned(ty_index).backingIntType(zcu).toIntern(),
2575	.error_set_type, .inferred_error_set_type => return .{ .is_signed = false, .bits = zcu.errorSetBits() },
2576	else => unreachable,
2577	};
2578	}
2579	};
2580
2581	fn intBackingBits(cg: *CodeGen, bits: u16) u16 {
2582	return switch (bits) {
2583	0 => unreachable,
2584	1...32 => 32,
2585	33...64 => 64,
2586	else => std.zig.target.intByteSize(cg.target, bits) * 8,
2587	};
2588	}
2589
2590	fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2591	switch (ty.bits) {
2592	0 => unreachable,
2593	1...32 => {
2594	try cg.emitWValue(lhs);
2595	try cg.emitWValue(rhs);
2596	try cg.addTag(.i32_add);
2597	return .stack;
2598	},
2599	33...64 => {
2600	try cg.emitWValue(lhs);
2601	try cg.emitWValue(rhs);
2602	try cg.addTag(.i64_add);
2603	return .stack;
2604	},
2605	65...128 => {
2606	const result = try cg.allocStack(Type.u128);
2607
2608	var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
2609	defer lhs_lsb.free(cg);
2610	var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
2611	defer rhs_lsb.free(cg);
2612	var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2613	defer op_lsb.free(cg);
2614
2615	const lhs_msb = try cg.load(lhs, Type.u64, 8);
2616	const rhs_msb = try cg.load(rhs, Type.u64, 8);
2617	const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb);
2618
2619	const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb);
2620	const tmp = try cg.intCast(.u64, .u32, lt);
2621	var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2622	defer tmp_op.free(cg);
2623
2624	try cg.store(result, op_lsb, Type.u64, 0);
2625	try cg.store(result, tmp_op, Type.u64, 8);
2626	return result;
2627	},
2628	else => {
2629	const result = try cg.allocInt(ty);
2630
2631	try cg.lowerToStack(result);
2632	try cg.lowerToStack(lhs);
2633	try cg.lowerToStack(rhs);
2634	try cg.addImm32(@intFromBool(ty.is_signed));
2635	try cg.addImm32(ty.bits);
2636	try cg.addCallIntrinsic(.__addo_limb64);
2637	try cg.addTag(.drop);
2638	return result;
2639	},
2640	}
2641	}
2642
2643	fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2644	switch (ty.bits) {
2645	0 => unreachable,
2646	1...32 => {
2647	try cg.emitWValue(lhs);
2648	try cg.emitWValue(rhs);
2649	try cg.addTag(.i32_sub);
2650	return .stack;
2651	},
2652	33...64 => {
2653	try cg.emitWValue(lhs);
2654	try cg.emitWValue(rhs);
2655	try cg.addTag(.i64_sub);
2656	return .stack;
2657	},
2658	65...128 => {
2659	const result = try cg.allocStack(Type.u128);
2660
2661	var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
2662	defer lhs_lsb.free(cg);
2663	var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
2664	defer rhs_lsb.free(cg);
2665	var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2666	defer op_lsb.free(cg);
2667
2668	const lhs_msb = try cg.load(lhs, Type.u64, 8);
2669	const rhs_msb = try cg.load(rhs, Type.u64, 8);
2670	const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb);
2671
2672	const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb);
2673	const tmp = try cg.intCast(.u64, .u32, lt);
2674	var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2675	defer tmp_op.free(cg);
2676
2677	try cg.store(result, op_lsb, Type.u64, 0);
2678	try cg.store(result, tmp_op, Type.u64, 8);
2679	return result;
2680	},
2681	else => {
2682	const result = try cg.allocInt(ty);
2683
2684	try cg.lowerToStack(result);
2685	try cg.lowerToStack(lhs);
2686	try cg.lowerToStack(rhs);
2687	try cg.addImm32(@intFromBool(ty.is_signed));
2688	try cg.addImm32(ty.bits);
2689	try cg.addCallIntrinsic(.__subo_limb64);
2690	try cg.addTag(.drop);
2691
2692	return result;
2693	},
2694	}
2695	}
2696
2697	fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2698	switch (ty.bits) {
2699	0 => unreachable,
2700	1...32 => {
2701	try cg.emitWValue(lhs);
2702	try cg.emitWValue(rhs);
2703	try cg.addTag(.i32_mul);
2704	return .stack;
2705	},
2706	33...64 => {
2707	try cg.emitWValue(lhs);
2708	try cg.emitWValue(rhs);
2709	try cg.addTag(.i64_mul);
2710	return .stack;
2711	},
2712	65...128 => return cg.callIntrinsic(.__multi3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs }),
2713	else => {
2714	const result = try cg.allocInt(ty);
2715
2716	try cg.lowerToStack(result);
2717	try cg.lowerToStack(lhs);
2718	try cg.lowerToStack(rhs);
2719	try cg.addImm32(@intFromBool(ty.is_signed));
2720	try cg.addImm32(ty.bits);
2721	try cg.addCallIntrinsic(.__mulo_limb64);
2722	try cg.addTag(.drop);
2723
2724	return result;
2725	},
2726	}
2727	}
2728
2729	fn intDiv(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2730	switch (ty.bits) {
2731	0 => unreachable,
2732	1...32 => {
2733	try cg.emitWValue(lhs);
2734	try cg.emitWValue(rhs);
2735	try cg.addTag(if (ty.is_signed) .i32_div_s else .i32_div_u);
2736	return .stack;
2737	},
2738	33...64 => {
2739	try cg.emitWValue(lhs);
2740	try cg.emitWValue(rhs);
2741	try cg.addTag(if (ty.is_signed) .i64_div_s else .i64_div_u);
2742	return .stack;
2743	},
2744	65...128 => {
2745	if (ty.is_signed) {
2746	return cg.callIntrinsic(.__divti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2747	} else {
2748	return cg.callIntrinsic(.__udivti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2749	}
2750	},
2751	else => {
2752	const result = try cg.allocInt(ty);
2753	const bits = cg.intBackingBits(ty.bits);
2754	var tmp = try cg.allocInt(.{ .is_signed = false, .bits = bits * 2 });
2755	if (ty.is_signed) {
2756	_ = try cg.callIntrinsic(
2757	.__divei5,
2758	&.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
2759	.void,
2760	&.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
2761	);
2762	} else {
2763	_ = try cg.callIntrinsic(
2764	.__udivei5,
2765	&.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
2766	.void,
2767	&.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
2768	);
2769	}
2770	tmp.free(cg);
2771	return result;
2772	},
2773	}
2774	}
2775
2776	fn intDivFloor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2777	if (!ty.is_signed) {
2778	return cg.intDiv(ty, lhs, rhs);
2779	}
2780
2781	switch (ty.bits) {
2782	0 => unreachable,
2783	1...32 => {
2784	var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i32);
2785	defer q.free(cg);
2786
2787	const zero: WValue = .{ .imm32 = 0 };
2788
2789	const r = try cg.intRem(ty, lhs, rhs);
2790	var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
2791	defer r_nonzero.free(cg);
2792
2793	const sign_xor = try cg.intXor(ty, lhs, rhs);
2794	var sign_diff = try (try cg.intCmp(ty, .lt, sign_xor, zero)).toLocal(cg, Type.i32);
2795	defer sign_diff.free(cg);
2796
2797	try cg.emitWValue(q);
2798	const need_adjust = try cg.intAnd(.u32, r_nonzero, sign_diff);
2799	try cg.emitWValue(need_adjust);
2800	try cg.addTag(.i32_sub);
2801	return .stack;
2802	},
2803	33...64 => {
2804	var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i64);
2805	defer q.free(cg);
2806
2807	const zero: WValue = .{ .imm64 = 0 };
2808
2809	const r = try cg.intRem(ty, lhs, rhs);
2810	var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
2811	defer r_nonzero.free(cg);
2812
2813	const sign_xor = try cg.intXor(ty, lhs, rhs);
2814	var sign_diff = try (try cg.intCmp(ty, .lt, sign_xor, zero)).toLocal(cg, Type.i32);
2815	defer sign_diff.free(cg);
2816
2817	try cg.emitWValue(q);
2818	const need_adjust = try cg.intAnd(.u32, r_nonzero, sign_diff);
2819	try cg.emitWValue(need_adjust);
2820	try cg.addTag(.i64_extend_i32_u);
2821	try cg.addTag(.i64_sub);
2822	return .stack;
2823	},
2824	else => {
2825	const q = try cg.intDiv(ty, lhs, rhs);
2826
2827	const zero = try cg.intZeroValue(ty);
2828
2829	const r = try cg.intRem(ty, lhs, rhs);
2830	_ = try cg.intCmp(ty, .neq, r, zero);
2831
2832	const sign_xor = try cg.intXor(ty, lhs, rhs);
2833	_ = try cg.intCmp(ty, .lt, sign_xor, zero);
2834	var adjust = try (try cg.intAnd(.u32, .stack, .stack)).toLocal(cg, Type.u32);
2835
2836	const adjust_bigint = try cg.intCast(ty, .u32, adjust);
2837	adjust.free(cg);
2838	return try cg.intSub(ty, q, adjust_bigint);
2839	},
2840	}
2841	}
2842
2843	fn intDivCeil(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2844	switch (ty.bits) {
2845	0 => unreachable,
2846	1...32 => {
2847	var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i32);
2848	defer q.free(cg);
2849
2850	const zero: WValue = .{ .imm32 = 0 };
2851
2852	const r = try cg.intRem(ty, lhs, rhs);
2853	var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
2854	defer r_nonzero.free(cg);
2855
2856	if (!ty.is_signed) {
2857	try cg.emitWValue(q);
2858	try cg.emitWValue(r_nonzero);
2859	try cg.addTag(.i32_add);
2860	return .stack;
2861	}
2862
2863	const sign_xor = try cg.intXor(ty, lhs, rhs);
2864	var same_sign = try (try cg.intCmp(ty, .gte, sign_xor, zero)).toLocal(cg, Type.i32);
2865	defer same_sign.free(cg);
2866
2867	try cg.emitWValue(q);
2868	const need_adjust = try cg.intAnd(.u32, r_nonzero, same_sign);
2869	try cg.emitWValue(need_adjust);
2870	try cg.addTag(.i32_add);
2871	return .stack;
2872	},
2873	33...64 => {
2874	var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i64);
2875	defer q.free(cg);
2876
2877	const zero: WValue = .{ .imm64 = 0 };
2878
2879	const r = try cg.intRem(ty, lhs, rhs);
2880	var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
2881	defer r_nonzero.free(cg);
2882
2883	if (!ty.is_signed) {
2884	try cg.emitWValue(q);
2885	try cg.emitWValue(r_nonzero);
2886	try cg.addTag(.i64_extend_i32_u);
2887	try cg.addTag(.i64_add);
2888	return .stack;
2889	}
2890
2891	const sign_xor = try cg.intXor(ty, lhs, rhs);
2892	var same_sign = try (try cg.intCmp(ty, .gte, sign_xor, zero)).toLocal(cg, Type.i32);
2893	defer same_sign.free(cg);
2894
2895	try cg.emitWValue(q);
2896	const need_adjust = try cg.intAnd(.u32, r_nonzero, same_sign);
2897	try cg.emitWValue(need_adjust);
2898	try cg.addTag(.i64_extend_i32_u);
2899	try cg.addTag(.i64_add);
2900	return .stack;
2901	},
2902	else => {
2903	var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.usize);
2904	defer q.free(cg);
2905
2906	const zero = try cg.intZeroValue(ty);
2907
2908	const r = try cg.intRem(ty, lhs, rhs);
2909	var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.u32);
2910	defer r_nonzero.free(cg);
2911
2912	if (!ty.is_signed) {
2913	var adjust_bigint = try (try cg.intCast(ty, .u32, r_nonzero)).toLocal(cg, Type.usize);
2914	defer adjust_bigint.free(cg);
2915
2916	return try cg.intAdd(ty, q, adjust_bigint);
2917	}
2918
2919	const sign_xor = try cg.intXor(ty, lhs, rhs);
2920	var same_sign = try (try cg.intCmp(ty, .gte, sign_xor, zero)).toLocal(cg, Type.u32);
2921	defer same_sign.free(cg);
2922
2923	var adjust = try (try cg.intAnd(.u32, r_nonzero, same_sign)).toLocal(cg, Type.u32);
2924	defer adjust.free(cg);
2925
2926	var adjust_bigint = try (try cg.intCast(ty, .u32, adjust)).toLocal(cg, Type.usize);
2927	defer adjust_bigint.free(cg);
2928
2929	return try cg.intAdd(ty, q, adjust_bigint);
2930	},
2931	}
2932	}
2933
2934	fn intRem(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2935	switch (ty.bits) {
2936	0 => unreachable,
2937	1...32 => {
2938	try cg.emitWValue(lhs);
2939	try cg.emitWValue(rhs);
2940	try cg.addTag(if (ty.is_signed) .i32_rem_s else .i32_rem_u);
2941	return .stack;
2942	},
2943	33...64 => {
2944	try cg.emitWValue(lhs);
2945	try cg.emitWValue(rhs);
2946	try cg.addTag(if (ty.is_signed) .i64_rem_s else .i64_rem_u);
2947	return .stack;
2948	},
2949	65...128 => {
2950	if (ty.is_signed) {
2951	return cg.callIntrinsic(.__modti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2952	} else {
2953	return cg.callIntrinsic(.__umodti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2954	}
2955	},
2956	else => {
2957	const result = try cg.allocInt(ty);
2958	const bits = cg.intBackingBits(ty.bits);
2959	var tmp = try cg.allocInt(.{ .is_signed = false, .bits = bits * 2 });
2960	if (ty.is_signed) {
2961	_ = try cg.callIntrinsic(
2962	.__modei5,
2963	&.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
2964	.void,
2965	&.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
2966	);
2967	} else {
2968	_ = try cg.callIntrinsic(
2969	.__umodei5,
2970	&.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
2971	.void,
2972	&.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
2973	);
2974	}
2975	tmp.free(cg);
2976	return result;
2977	},
2978	}
2979	}
2980
2981	fn intMod(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2982	if (!ty.is_signed) {
2983	return cg.intRem(ty, lhs, rhs);
2984	}
2985
2986	// mod_s(a, b) = rem_s(rem_s(a, b) + b, b)
2987	const rem = try cg.intRem(ty, lhs, rhs);
2988	const sum = try cg.intAdd(ty, rem, rhs);
2989	return cg.intRem(ty, sum, rhs);
2990	}
2991
2992	fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2993	switch (ty.bits) {
2994	0 => unreachable,
2995	1...32 => {
2996	try cg.emitWValue(lhs);
2997	try cg.emitWValue(rhs);
2998	try cg.addTag(.i32_and);
2999	return .stack;
3000	},
3001	33...64 => {
3002	try cg.emitWValue(lhs);
3003	try cg.emitWValue(rhs);
3004	try cg.addTag(.i64_and);
3005	return .stack;
3006	},
3007	65...128 => {
3008	const result = try cg.allocStack(Type.u128);
3009
3010	const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3011	const rhs_lsb = try cg.load(rhs, Type.u64, 0);
3012	const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3013	try cg.store(result, and_lsb, Type.u64, 0);
3014
3015	const lhs_msb = try cg.load(lhs, Type.u64, 8);
3016	const rhs_msb = try cg.load(rhs, Type.u64, 8);
3017	const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3018	try cg.store(result, and_msb, Type.u64, 8);
3019
3020	return result;
3021	},
3022	else => {
3023	const result = try cg.allocInt(ty);
3024
3025	try cg.lowerToStack(result);
3026	try cg.lowerToStack(lhs);
3027	try cg.lowerToStack(rhs);
3028	try cg.addImm32(ty.bits);
3029	try cg.addCallIntrinsic(.__and_limb64);
3030
3031	return result;
3032	},
3033	}
3034	}
3035
3036	fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3037	switch (ty.bits) {
3038	0 => unreachable,
3039	1...32 => {
3040	try cg.emitWValue(lhs);
3041	try cg.emitWValue(rhs);
3042	try cg.addTag(.i32_or);
3043	return .stack;
3044	},
3045	33...64 => {
3046	try cg.emitWValue(lhs);
3047	try cg.emitWValue(rhs);
3048	try cg.addTag(.i64_or);
3049	return .stack;
3050	},
3051	65...128 => {
3052	const result = try cg.allocStack(Type.u128);
3053
3054	const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3055	const rhs_lsb = try cg.load(rhs, Type.u64, 0);
3056	const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3057	try cg.store(result, or_lsb, Type.u64, 0);
3058
3059	const lhs_msb = try cg.load(lhs, Type.u64, 8);
3060	const rhs_msb = try cg.load(rhs, Type.u64, 8);
3061	const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3062	try cg.store(result, or_msb, Type.u64, 8);
3063
3064	return result;
3065	},
3066	else => {
3067	const result = try cg.allocInt(ty);
3068
3069	try cg.lowerToStack(result);
3070	try cg.lowerToStack(lhs);
3071	try cg.lowerToStack(rhs);
3072	try cg.addImm32(ty.bits);
3073	try cg.addCallIntrinsic(.__or_limb64);
3074
3075	return result;
3076	},
3077	}
3078	}
3079
3080	fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3081	switch (ty.bits) {
3082	0 => unreachable,
3083	1...32 => {
3084	try cg.emitWValue(lhs);
3085	try cg.emitWValue(rhs);
3086	try cg.addTag(.i32_xor);
3087	return .stack;
3088	},
3089	33...64 => {
3090	try cg.emitWValue(lhs);
3091	try cg.emitWValue(rhs);
3092	try cg.addTag(.i64_xor);
3093	return .stack;
3094	},
3095	65...128 => {
3096	const result = try cg.allocStack(Type.u128);
3097
3098	const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3099	const rhs_lsb = try cg.load(rhs, Type.u64, 0);
3100	const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3101	try cg.store(result, xor_lsb, Type.u64, 0);
3102
3103	const lhs_msb = try cg.load(lhs, Type.u64, 8);
3104	const rhs_msb = try cg.load(rhs, Type.u64, 8);
3105	const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3106	try cg.store(result, xor_msb, Type.u64, 8);
3107
3108	return result;
3109	},
3110	else => {
3111	const result = try cg.allocInt(ty);
3112
3113	try cg.lowerToStack(result);
3114	try cg.lowerToStack(lhs);
3115	try cg.lowerToStack(rhs);
3116	try cg.addImm32(ty.bits);
3117	try cg.addCallIntrinsic(.__xor_limb64);
3118
3119	return result;
3120	},
3121	}
3122	}
3123
3124	fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3125	switch (ty.bits) {
3126	0 => unreachable,
3127	1 => {
3128	try cg.emitWValue(operand);
3129	if (ty.is_signed) {
3130	try cg.addImm32(~@as(u32, 0));
3131	try cg.addTag(.i32_xor);
3132	} else {
3133	try cg.addTag(.i32_eqz);
3134	}
3135	return .stack;
3136	},
3137	2...32 => {
3138	const mask: u32 = if (ty.is_signed)
3139	~@as(u32, 0)
3140	else
3141	~@as(u32, 0) >> @intCast(32 - ty.bits);
3142	try cg.emitWValue(operand);
3143	try cg.addImm32(mask);
3144	try cg.addTag(.i32_xor);
3145	return .stack;
3146	},
3147	33...64 => {
3148	const mask: u64 = if (ty.is_signed)
3149	~@as(u64, 0)
3150	else
3151	~@as(u64, 0) >> @intCast(64 - ty.bits);
3152	try cg.emitWValue(operand);
3153	try cg.addImm64(mask);
3154	try cg.addTag(.i64_xor);
3155	return .stack;
3156	},
3157	65...128 => {
3158	const result = try cg.allocStack(Type.u128);
3159
3160	try cg.emitWValue(result);
3161	_ = try cg.load(operand, Type.u64, 0);
3162	try cg.addImm64(~@as(u64, 0));
3163	try cg.addTag(.i64_xor);
3164	try cg.store(.stack, .stack, Type.u64, result.offset());
3165
3166	try cg.emitWValue(result);
3167	_ = try cg.load(operand, Type.u64, 8);
3168	const high_mask: u64 = if (ty.is_signed)
3169	~@as(u64, 0)
3170	else
3171	~@as(u64, 0) >> @intCast(128 - ty.bits);
3172	try cg.addImm64(high_mask);
3173	try cg.addTag(.i64_xor);
3174	try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
3175
3176	return result;
3177	},
3178	else => {
3179	const result = try cg.allocInt(ty);
3180
3181	try cg.lowerToStack(result);
3182	try cg.lowerToStack(operand);
3183	try cg.addImm32(@intFromBool(ty.is_signed));
3184	try cg.addImm32(ty.bits);
3185	try cg.addCallIntrinsic(.__not_limb64);
3186
3187	return result;
3188	},
3189	}
3190	}
3191
3192	// rhs is a shift count, pointing to i32 value
3193	// does not perform wrapping, padding bits does not satisfy invariant
3194	fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3195	switch (ty.bits) {
3196	0 => unreachable,
3197	1...32 => {
3198	try cg.emitWValue(lhs);
3199	try cg.emitWValue(rhs);
3200	try cg.addTag(.i32_shl);
3201	return .stack;
3202	},
3203	33...64 => {
3204	try cg.emitWValue(lhs);
3205	try cg.emitWValue(rhs);
3206	try cg.addTag(.i64_extend_i32_u);
3207	try cg.addTag(.i64_shl);
3208	return .stack;
3209	},
3210	65...128 => return cg.callIntrinsic(.__ashlti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs }),
3211	else => {
3212	const result = try cg.allocInt(ty);
3213
3214	try cg.lowerToStack(result);
3215	try cg.lowerToStack(lhs);
3216	try cg.lowerToStack(rhs);
3217	try cg.addImm32(@intFromBool(ty.is_signed));
3218	try cg.addImm32(ty.bits);
3219	try cg.addCallIntrinsic(.__shlo_limb64);
3220	try cg.addTag(.drop);
3221
3222	return result;
3223	},
3224	}
3225	}
3226
3227	// rhs is a shift count, pointing to i32 value
3228	fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3229	switch (ty.bits) {
3230	0 => unreachable,
3231	1...32 => {
3232	try cg.emitWValue(lhs);
3233	try cg.emitWValue(rhs);
3234	try cg.addTag(if (ty.is_signed) .i32_shr_s else .i32_shr_u);
3235	return .stack;
3236	},
3237	33...64 => {
3238	try cg.emitWValue(lhs);
3239	try cg.emitWValue(rhs);
3240	try cg.addTag(.i64_extend_i32_u);
3241	try cg.addTag(if (ty.is_signed) .i64_shr_s else .i64_shr_u);
3242	return .stack;
3243	},
3244	65...128 => {
3245	if (ty.is_signed) {
3246	return cg.callIntrinsic(.__ashrti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs });
3247	} else {
3248	return cg.callIntrinsic(.__lshrti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs });
3249	}
3250	},
3251	else => {
3252	const result = try cg.allocInt(ty);
3253
3254	try cg.lowerToStack(result);
3255	try cg.lowerToStack(lhs);
3256	try cg.lowerToStack(rhs);
3257	try cg.addImm32(@intFromBool(ty.is_signed));
3258	try cg.addImm32(ty.bits);
3259	try cg.addCallIntrinsic(.__shr_limb64);
3260
3261	return result;
3262	},
3263	}
3264	}
3265
3266	fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3267	if (!ty.is_signed) return operand;
3268	switch (ty.bits) {
3269	0 => unreachable,
3270	1...32 => {
3271	try cg.emitWValue(operand);
3272	try cg.addImm32(31);
3273	try cg.addTag(.i32_shr_s);
3274
3275	var mask = try cg.allocLocal(Type.i32);
3276	defer mask.free(cg);
3277	try cg.addLocal(.local_tee, mask.local.value);
3278
3279	try cg.emitWValue(operand);
3280	try cg.addTag(.i32_xor);
3281	try cg.emitWValue(mask);
3282	try cg.addTag(.i32_sub);
3283	return .stack;
3284	},
3285	33...64 => {
3286	try cg.emitWValue(operand);
3287	try cg.addImm64(63);
3288	try cg.addTag(.i64_shr_s);
3289
3290	var mask = try cg.allocLocal(Type.i64);
3291	defer mask.free(cg);
3292	try cg.addLocal(.local_tee, mask.local.value);
3293
3294	try cg.emitWValue(operand);
3295	try cg.addTag(.i64_xor);
3296	try cg.emitWValue(mask);
3297	try cg.addTag(.i64_sub);
3298	return .stack;
3299	},
3300	65...128 => {
3301	const u128_ty: IntType = .{ .is_signed = false, .bits = 128 };
3302
3303	const mask = try cg.allocStack(Type.u128);
3304	try cg.emitWValue(mask);
3305	try cg.emitWValue(mask);
3306
3307	_ = try cg.load(operand, Type.u64, 8);
3308	try cg.addImm64(63);
3309	try cg.addTag(.i64_shr_s);
3310
3311	var tmp = try cg.allocLocal(Type.u64);
3312	defer tmp.free(cg);
3313	try cg.addLocal(.local_tee, tmp.local.value);
3314	try cg.store(.stack, .stack, Type.u64, mask.offset() + 0);
3315	try cg.emitWValue(tmp);
3316	try cg.store(.stack, .stack, Type.u64, mask.offset() + 8);
3317
3318	const a = try cg.intXor(u128_ty, operand, mask);
3319	const b = try cg.intSub(u128_ty, a, mask);
3320	return b;
3321	},
3322	else => {
3323	const result = try cg.allocInt(ty);
3324
3325	try cg.lowerToStack(result);
3326	try cg.lowerToStack(operand);
3327	try cg.addImm32(ty.bits);
3328	try cg.addCallIntrinsic(.__abs_limb64);
3329
3330	return result;
3331	},
3332	}
3333	}
3334
3335	fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3336	try cg.lowerToStack(lhs);
3337	try cg.lowerToStack(rhs);
3338	_ = try cg.intCmp(ty, .gt, lhs, rhs);
3339	try cg.addTag(.select);
3340	return .stack;
3341	}
3342
3343	fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3344	try cg.lowerToStack(lhs);
3345	try cg.lowerToStack(rhs);
3346	_ = try cg.intCmp(ty, .lt, lhs, rhs);
3347	try cg.addTag(.select);
3348	return .stack;
3349	}
3350
3351	fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3352	switch (ty.bits) {
3353	0 => unreachable,
3354	1...32 => {
3355	if (ty.is_signed and ty.bits < 32) {
3356	const mask: u32 = ~@as(u32, 0) >> @intCast(32 - ty.bits);
3357	_ = try cg.intAnd(.u32, operand, .{ .imm32 = mask });
3358	} else {
3359	try cg.emitWValue(operand);
3360	}
3361	try cg.addTag(.i32_clz);
3362	if (ty.bits < 32) {
3363	try cg.addImm32(32 - ty.bits);
3364	try cg.addTag(.i32_sub);
3365	}
3366	return .stack;
3367	},
3368	33...64 => {
3369	if (ty.is_signed and ty.bits < 64) {
3370	const mask: u64 = ~@as(u64, 0) >> @intCast(64 - ty.bits);
3371	_ = try cg.intAnd(.u64, operand, .{ .imm64 = mask });
3372	} else {
3373	try cg.emitWValue(operand);
3374	}
3375	try cg.addTag(.i64_clz);
3376	try cg.addTag(.i32_wrap_i64);
3377	if (ty.bits < 64) {
3378	try cg.addImm32(64 - ty.bits);
3379	try cg.addTag(.i32_sub);
3380	}
3381	return .stack;
3382	},
3383	65...128 => {
3384	var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64);
3385	defer msb.free(cg);
3386
3387	if (ty.is_signed and ty.bits < 128) {
3388	const mask: u64 = ~@as(u64, 0) >> @intCast(128 - ty.bits);
3389	_ = try cg.intAnd(.u64, msb, .{ .imm64 = mask });
3390	} else {
3391	try cg.emitWValue(msb);
3392	}
3393
3394	try cg.addTag(.i64_clz);
3395	_ = try cg.load(operand, Type.u64, 0);
3396	try cg.addTag(.i64_clz);
3397	try cg.emitWValue(.{ .imm64 = 64 });
3398	try cg.addTag(.i64_add);
3399	_ = try cg.intCmp(.u64, .neq, msb, .{ .imm64 = 0 });
3400	try cg.addTag(.select);
3401	try cg.addTag(.i32_wrap_i64);
3402
3403	if (ty.bits < 128) {
3404	try cg.addImm32(128 - ty.bits);
3405	try cg.addTag(.i32_sub);
3406	}
3407
3408	return .stack;
3409	},
3410	else => {
3411	try cg.lowerToStack(operand);
3412	try cg.addImm32(ty.bits);
3413	try cg.addCallIntrinsic(.__clz_limb64);
3414
3415	return .stack;
3416	},
3417	}
3418	}
3419
3420	fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3421	switch (ty.bits) {
3422	0 => unreachable,
3423	1...32 => {
3424	if (ty.bits < 32) {
3425	_ = try cg.intOr(.u32, operand, .{ .imm32 = @as(u32, 1) << @intCast(ty.bits) });
3426	} else {
3427	try cg.emitWValue(operand);
3428	}
3429	try cg.addTag(.i32_ctz);
3430	return .stack;
3431	},
3432	33...64 => {
3433	if (ty.bits < 64) {
3434	_ = try cg.intOr(.u64, operand, .{ .imm64 = @as(u64, 1) << @intCast(ty.bits) });
3435	} else {
3436	try cg.emitWValue(operand);
3437	}
3438	try cg.addTag(.i64_ctz);
3439	try cg.addTag(.i32_wrap_i64);
3440	return .stack;
3441	},
3442	65...128 => {
3443	var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64);
3444	defer lsb.free(cg);
3445
3446	try cg.emitWValue(lsb);
3447	try cg.addTag(.i64_ctz);
3448
3449	_ = try cg.load(operand, Type.u64, 8);
3450	if (ty.bits < 128) {
3451	try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64));
3452	try cg.addTag(.i64_or);
3453	}
3454	try cg.addTag(.i64_ctz);
3455	try cg.addImm64(64);
3456	try cg.addTag(.i64_add);
3457	_ = try cg.intCmp(.u64, .neq, lsb, .{ .imm64 = 0 });
3458	try cg.addTag(.select);
3459	try cg.addTag(.i32_wrap_i64);
3460	return .stack;
3461	},
3462	else => {
3463	try cg.lowerToStack(operand);
3464	try cg.addImm32(ty.bits);
3465	try cg.addCallIntrinsic(.__ctz_limb64);
3466
3467	return .stack;
3468	},
3469	}
3470	}
3471
3472	fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3473	switch (ty.bits) {
3474	0 => unreachable,
3475	1...32 => {
3476	try cg.emitWValue(operand);
3477	if (ty.is_signed and ty.bits < 32) {
3478	try cg.addImm32(32 - ty.bits);
3479	try cg.addTag(.i32_shl);
3480	}
3481	try cg.addTag(.i32_popcnt);
3482	return .stack;
3483	},
3484	33...64 => {
3485	try cg.emitWValue(operand);
3486	if (ty.is_signed and ty.bits < 64) {
3487	try cg.addImm64(64 - ty.bits);
3488	try cg.addTag(.i64_shl);
3489	}
3490	try cg.addTag(.i64_popcnt);
3491	try cg.addTag(.i32_wrap_i64);
3492	return .stack;
3493	},
3494	65...128 => {
3495	_ = try cg.load(operand, Type.u64, 0);
3496	try cg.addTag(.i64_popcnt);
3497	_ = try cg.load(operand, Type.u64, 8);
3498	if (ty.is_signed and ty.bits < 128) {
3499	try cg.addImm64(128 - ty.bits);
3500	try cg.addTag(.i64_shl);
3501	}
3502	try cg.addTag(.i64_popcnt);
3503
3504	try cg.addTag(.i64_add);
3505	try cg.addTag(.i32_wrap_i64);
3506	return .stack;
3507	},
3508	else => {
3509	try cg.lowerToStack(operand);
3510	try cg.addImm32(ty.bits);
3511	try cg.addCallIntrinsic(.__popcount_limb64);
3512
3513	return .stack;
3514	},
3515	}
3516	}
3517
3518	fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3519	switch (ty.bits) {
3520	0 => unreachable,
3521	1...32 => {
3522	const intrin_ret = try cg.callIntrinsic(
3523	.__bitreversesi2,
3524	&.{.u32_type},
3525	Type.u32,
3526	&.{operand},
3527	);
3528	if (ty.bits == 32) return intrin_ret;
3529	return cg.intShr(ty, intrin_ret, .{ .imm32 = 32 - ty.bits });
3530	},
3531	33...64 => {
3532	const intrin_ret = try cg.callIntrinsic(
3533	.__bitreversedi2,
3534	&.{.u64_type},
3535	Type.u64,
3536	&.{operand},
3537	);
3538	if (ty.bits == 64) return intrin_ret;
3539	return cg.intShr(ty, intrin_ret, .{ .imm32 = 64 - ty.bits });
3540	},
3541	65...128 => {
3542	const tmp = try cg.allocStack(Type.u128);
3543
3544	try cg.emitWValue(tmp);
3545	const hi = try cg.load(operand, Type.u64, 8);
3546	const hi_rev = try cg.callIntrinsic(
3547	.__bitreversedi2,
3548	&.{.u64_type},
3549	Type.u64,
3550	&.{hi},
3551	);
3552	try cg.emitWValue(hi_rev);
3553	try cg.store(.stack, .stack, Type.u64, tmp.offset());
3554
3555	try cg.emitWValue(tmp);
3556	const lo = try cg.load(operand, Type.u64, 0);
3557	const lo_rev = try cg.callIntrinsic(
3558	.__bitreversedi2,
3559	&.{.u64_type},
3560	Type.u64,
3561	&.{lo},
3562	);
3563	try cg.emitWValue(lo_rev);
3564	try cg.store(.stack, .stack, Type.u64, tmp.offset() + 8);
3565
3566	if (ty.bits < 128) {
3567	const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
3568	return cg.intShr(shift_ty, tmp, .{ .imm32 = 128 - ty.bits });
3569	} else {
3570	return tmp;
3571	}
3572	},
3573	else => {
3574	const result = try cg.allocInt(ty);
3575
3576	try cg.lowerToStack(result);
3577	try cg.lowerToStack(operand);
3578	try cg.addImm32(@intFromBool(ty.is_signed));
3579	try cg.addImm32(ty.bits);
3580	try cg.addCallIntrinsic(.__bitreverse_limb64);
3581
3582	return result;
3583	},
3584	}
3585	}
3586
3587	fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3588	switch (ty.bits) {
3589	0 => unreachable,
3590	1...32 => {
3591	const intrin_ret = try cg.callIntrinsic(
3592	.__bswapsi2,
3593	&.{.u32_type},
3594	Type.u32,
3595	&.{operand},
3596	);
3597	if (ty.bits == 32) return intrin_ret;
3598	return cg.intShr(ty, intrin_ret, .{ .imm32 = 32 - ty.bits });
3599	},
3600	33...64 => {
3601	const intrin_ret = try cg.callIntrinsic(
3602	.__bswapdi2,
3603	&.{.u64_type},
3604	Type.u64,
3605	&.{operand},
3606	);
3607	if (ty.bits == 64) return intrin_ret;
3608	return cg.intShr(ty, intrin_ret, .{ .imm32 = 64 - ty.bits });
3609	},
3610	65...128 => {
3611	const result = try cg.allocStack(Type.u128);
3612
3613	try cg.emitWValue(result);
3614
3615	const low = try cg.load(operand, Type.u64, 0);
3616	const swap_low = try cg.callIntrinsic(
3617	.__bswapdi2,
3618	&.{.u64_type},
3619	Type.u64,
3620	&.{low},
3621	);
3622	try cg.store(.stack, swap_low, Type.u64, result.offset() + 8);
3623
3624	try cg.emitWValue(result);
3625
3626	const high = try cg.load(operand, Type.u64, 8);
3627	const swap_high = try cg.callIntrinsic(
3628	.__bswapdi2,
3629	&.{.u64_type},
3630	Type.u64,
3631	&.{high},
3632	);
3633	try cg.store(.stack, swap_high, Type.u64, result.offset());
3634
3635	if (ty.bits < 128) {
3636	const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
3637	return cg.intShr(shift_ty, result, .{ .imm32 = 128 - ty.bits });
3638	} else {
3639	return result;
3640	}
3641	},
3642	else => {
3643	const result = try cg.allocInt(ty);
3644
3645	try cg.lowerToStack(result);
3646	try cg.lowerToStack(operand);
3647	try cg.addImm32(@intFromBool(ty.is_signed));
3648	try cg.addImm32(ty.bits);
3649	try cg.addCallIntrinsic(.__byteswap_limb64);
3650
3651	return result;
3652	},
3653	}
3654	}
3655
3656	fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3657	switch (ty.bits) {
3658	0 => unreachable,
3659	1...31 => {
3660	try cg.emitWValue(operand);
3661	if (ty.is_signed) {
3662	try cg.addImm32(32 - ty.bits);
3663	try cg.addTag(.i32_shl);
3664	try cg.addImm32(32 - ty.bits);
3665	try cg.addTag(.i32_shr_s);
3666	} else {
3667	try cg.addImm32(~@as(u32, 0) >> @intCast(32 - ty.bits));
3668	try cg.addTag(.i32_and);
3669	}
3670	return .stack;
3671	},
3672	32 => return operand,
3673	33...63 => {
3674	try cg.emitWValue(operand);
3675	if (ty.is_signed) {
3676	try cg.addImm64(64 - ty.bits);
3677	try cg.addTag(.i64_shl);
3678	try cg.addImm64(64 - ty.bits);
3679	try cg.addTag(.i64_shr_s);
3680	} else {
3681	try cg.addImm64(~@as(u64, 0) >> @intCast(64 - ty.bits));
3682	try cg.addTag(.i64_and);
3683	}
3684	return .stack;
3685	},
3686	64 => return operand,
3687	65...127 => {
3688	const result = try cg.allocStack(Type.u128);
3689
3690	try cg.emitWValue(result);
3691	_ = try cg.load(operand, Type.u64, 0);
3692	try cg.store(.stack, .stack, Type.u64, result.offset());
3693
3694	try cg.emitWValue(result);
3695	_ = try cg.load(operand, Type.u64, 8);
3696	if (ty.is_signed) {
3697	try cg.addImm64(128 - ty.bits);
3698	try cg.addTag(.i64_shl);
3699	try cg.addImm64(128 - ty.bits);
3700	try cg.addTag(.i64_shr_s);
3701	} else {
3702	try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));
3703	try cg.addTag(.i64_and);
3704	}
3705	try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
3706
3707	return result;
3708	},
3709	128 => return operand,
3710	else => {
3711	const bits = cg.intBackingBits(ty.bits);
3712	if (ty.bits == bits) return operand;
3713
3714	const result = try cg.allocInt(ty);
3715
3716	const used_len = @divCeil(ty.bits, 64) * 8;
3717
3718	if (ty.bits % 64 != 0) {
3719	try cg.memcpy(result, operand, .{ .imm32 = used_len - 8 });
3720	const pad = 64 - ty.bits % 64;
3721
3722	try cg.emitWValue(result);
3723	_ = try cg.load(operand, Type.u64, used_len - 8);
3724	if (ty.is_signed) {
3725	try cg.addImm64(pad);
3726	try cg.addTag(.i64_shl);
3727	try cg.addImm64(pad);
3728	try cg.addTag(.i64_shr_s);
3729	} else {
3730	try cg.addImm64(~@as(u64, 0) >> @intCast(pad));
3731	try cg.addTag(.i64_and);
3732	}
3733	try cg.store(.stack, .stack, Type.u64, result.offset() + used_len - 8);
3734	} else {
3735	try cg.memcpy(result, operand, .{ .imm32 = used_len });
3736	}
3737
3738	const full_len = @divExact(bits, 8);
3739	if (used_len + 8 == full_len) { // last limb needs sign extended
3740	try cg.emitWValue(result);
3741	if (ty.is_signed) {
3742	_ = try cg.load(result, Type.u64, used_len - 8);
3743	try cg.addImm64(63);
3744	try cg.addTag(.i64_shr_s);
3745	} else {
3746	try cg.addImm64(0);
3747	}
3748	try cg.store(.stack, .stack, Type.u64, result.offset() + used_len);
3749	}
3750
3751	return result;
3752	},
3753	}
3754	}
3755
3756	fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3757	if (int_ty.bits <= 32) {
3758	if (int_ty.is_signed) {
3759	return .{ .imm32 = (~@as(u32, 0) >> @intCast(32 - int_ty.bits)) >> 1 };
3760	} else {
3761	return .{ .imm32 = ~@as(u32, 0) >> @intCast(32 - int_ty.bits) };
3762	}
3763	} else if (int_ty.bits <= 64) {
3764	if (int_ty.is_signed) {
3765	return .{ .imm64 = (~@as(u64, 0) >> @intCast(64 - int_ty.bits)) >> 1 };
3766	} else {
3767	return .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - int_ty.bits) };
3768	}
3769	} else if (int_ty.bits <= 128) {
3770	const result = try cg.allocInt(int_ty);
3771	try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, 0);
3772
3773	if (int_ty.is_signed) {
3774	try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, Type.u64, 8);
3775	} else {
3776	try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, Type.u64, 8);
3777	}
3778	return result;
3779	} else {
3780	const result = try cg.allocInt(int_ty);
3781	const full_len = @divExact(cg.intBackingBits(int_ty.bits), 8);
3782	const used_len = @divCeil(int_ty.bits, 64) * 8;
3783
3784	try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0xFF });
3785
3786	if (int_ty.is_signed) {
3787	try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, Type.u64, used_len - 8);
3788	} else {
3789	try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, Type.u64, used_len - 8);
3790	}
3791
3792	if (used_len + 8 == full_len) {
3793	try cg.store(result, .{ .imm64 = 0 }, Type.u64, full_len - 8);
3794	}
3795
3796	return result;
3797	}
3798	}
3799
3800	fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3801	if (!int_ty.is_signed) {
3802	return cg.intZeroValue(int_ty);
3803	}
3804	if (int_ty.bits <= 32) {
3805	return .{ .imm32 = ~@as(u32, 0) << @intCast(int_ty.bits - 1) };
3806	} else if (int_ty.bits <= 64) {
3807	return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) };
3808	} else if (int_ty.bits <= 128) {
3809	const result = try cg.allocInt(int_ty);
3810	try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
3811	try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, Type.u64, 8);
3812	return result;
3813	} else {
3814	const result = try cg.allocInt(int_ty);
3815	const full_len = @divExact(cg.intBackingBits(int_ty.bits), 8);
3816	const used_len = @divCeil(int_ty.bits, 64) * 8;
3817
3818	try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0 });
3819	try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, Type.u64, used_len - 8);
3820
3821	if (used_len + 8 == full_len) {
3822	try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, full_len - 8);
3823	}
3824
3825	return result;
3826	}
3827	}
3828
3829	fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3830	const raw_val = try cg.intAdd(int_ty, lhs, rhs);
3831	var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
3832	defer op_val.free(cg);
3833
3834	const max_val = try cg.intMaxValue(int_ty);
3835
3836	if (int_ty.is_signed) {
3837	const zero = try cg.intZeroValue(int_ty);
3838	var rhs_is_neg = try cg.toLocalInt(try cg.intCmp(int_ty, .lt, rhs, zero), .u32);
3839	defer rhs_is_neg.free(cg);
3840	const min_val = try cg.intMinValue(int_ty);
3841
3842	try cg.lowerToStack(min_val);
3843	try cg.lowerToStack(max_val);
3844	try cg.emitWValue(rhs_is_neg);
3845	try cg.addTag(.select);
3846
3847	try cg.lowerToStack(op_val);
3848	const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs);
3849	const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3850	try cg.emitWValue(is_overflow);
3851	try cg.addTag(.select);
3852	return .stack;
3853	} else {
3854	try cg.lowerToStack(max_val);
3855	try cg.lowerToStack(op_val);
3856
3857	const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs);
3858	try cg.emitWValue(is_overflow);
3859	try cg.addTag(.select);
3860	return .stack;
3861	}
3862	}
3863
3864	fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3865	const raw_val = try cg.intSub(int_ty, lhs, rhs);
3866	var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
3867	defer op_val.free(cg);
3868
3869	if (int_ty.is_signed) {
3870	const zero = try cg.intZeroValue(int_ty);
3871	var rhs_is_neg = try cg.toLocalInt(try cg.intCmp(int_ty, .lt, rhs, zero), .u32);
3872	defer rhs_is_neg.free(cg);
3873	const max_val = try cg.intMaxValue(int_ty);
3874	const min_val = try cg.intMinValue(int_ty);
3875
3876	try cg.lowerToStack(max_val);
3877	try cg.lowerToStack(min_val);
3878	try cg.emitWValue(rhs_is_neg);
3879	try cg.addTag(.select);
3880
3881	try cg.lowerToStack(op_val);
3882	const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs);
3883	const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3884	try cg.emitWValue(is_overflow);
3885	try cg.addTag(.select);
3886	return .stack;
3887	} else {
3888	const zero = try cg.intZeroValue(int_ty);
3889
3890	try cg.lowerToStack(zero);
3891	try cg.lowerToStack(op_val);
3892	const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs);
3893	try cg.emitWValue(is_overflow);
3894	try cg.addTag(.select);
3895	return .stack;
3896	}
3897	}
3898
3899	fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3900	const ext_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = int_ty.bits * 2 };
3901
3902	const lhs_ext = try cg.intCast(ext_ty, int_ty, lhs);
3903	const rhs_ext = try cg.intCast(ext_ty, int_ty, rhs);
3904
3905	var mul_ext = try cg.toLocalInt(try cg.intMul(ext_ty, lhs_ext, rhs_ext), ext_ty);
3906	defer mul_ext.free(cg);
3907
3908	var op_val = try cg.toLocalInt(try cg.intTrunc(int_ty, ext_ty, mul_ext), int_ty);
3909	defer op_val.free(cg);
3910	const max_val = try cg.intMaxValue(int_ty);
3911
3912	if (int_ty.is_signed) {
3913	const min_val = try cg.intMinValue(int_ty);
3914
3915	try cg.lowerToStack(min_val);
3916
3917	try cg.lowerToStack(max_val);
3918	try cg.lowerToStack(op_val);
3919	const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3920	const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3921	try cg.emitWValue(ov_pos);
3922	try cg.addTag(.select);
3923
3924	const min_ext = try cg.intCast(ext_ty, int_ty, min_val);
3925	const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext);
3926	try cg.lowerToStack(ov_neg);
3927	try cg.addTag(.select);
3928	return .stack;
3929	} else {
3930	try cg.lowerToStack(max_val);
3931	try cg.lowerToStack(op_val);
3932	const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3933	const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3934	try cg.emitWValue(is_overflow);
3935	try cg.addTag(.select);
3936	return .stack;
3937	}
3938	}
3939
3940	fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3941	const raw_val = try cg.intShl(int_ty, lhs, rhs);
3942	var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
3943	defer op_val.free(cg);
3944
3945	var check_val = try cg.toLocalInt(try cg.intShr(int_ty, op_val, rhs), int_ty);
3946	defer check_val.free(cg);
3947
3948	const max_val = try cg.intMaxValue(int_ty);
3949
3950	if (int_ty.is_signed) {
3951	const zero = try cg.intZeroValue(int_ty);
3952	const min_val = try cg.intMinValue(int_ty);
3953
3954	try cg.lowerToStack(min_val);
3955	try cg.lowerToStack(max_val);
3956	const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero);
3957	try cg.emitWValue(lhs_is_neg);
3958	try cg.addTag(.select);
3959
3960	try cg.lowerToStack(op_val);
3961	const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3962	try cg.emitWValue(is_overflow);
3963	try cg.addTag(.select);
3964	return .stack;
3965	} else {
3966	try cg.lowerToStack(max_val);
3967	try cg.lowerToStack(op_val);
3968	const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3969	try cg.emitWValue(is_overflow);
3970	try cg.addTag(.select);
3971	return .stack;
3972	}
3973	}
3974
3975	fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3976	switch (int_ty.bits) {
3977	0 => unreachable,
3978	1...32 => return .{ .imm32 = 0 },
3979	33...64 => return .{ .imm64 = 0 },
3980	65...128 => {
3981	const result = try cg.allocInt(int_ty);
3982	try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
3983	try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
3984	return result;
3985	},
3986	else => {
3987	const result = try cg.allocInt(int_ty);
3988	const full_len = @divExact(cg.intBackingBits(int_ty.bits), 8);
3989	try cg.memset(Type.u8, result, .{ .imm32 = full_len }, .{ .imm32 = 0 });
3990	return result;
3991	},
3992	}
3993	}
3994
3995	fn toLocalInt(cg: *CodeGen, value: WValue, int_ty: IntType) InnerError!WValue {
3996	switch (value) {
3997	.stack => {
3998	const ty: Type = switch (int_ty.bits) {
3999	0 => unreachable,
4000	1...32 => .u32,
4001	33...64 => .u64,
4002	65...128 => .u128,
4003	else => return cg.fail("TODO: Support toLocalInt for integer bitsize: {d}", .{int_ty.bits}),
4004	};
4005	const new_local = try cg.allocLocal(ty);
4006	try cg.addLocal(.local_set, new_local.local.value);
4007	return new_local;
4008	},
4009	.local, .stack_offset => return value,
4010	else => unreachable,
4011	}
4012	}
4013
4014	const OverflowResult = struct {
4015	result: WValue,
4016	ov: WValue,
4017	};
4018
4019	fn intAddOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
4020	switch (ty.bits) {
4021	0 => unreachable,
4022	1...128 => {
4023	const raw_result = try cg.intAdd(ty, lhs, rhs);
4024	const op_result = try cg.intWrap(ty, raw_result);
4025	const op_tmp = try cg.toLocalInt(op_result, ty);
4026
4027	const overflow_bit = if (ty.is_signed) blk: {
4028	const zero = try cg.intZeroValue(ty);
4029	const rhs_is_neg = try cg.intCmp(ty, .lt, rhs, zero);
4030	const overflow_cmp = try cg.intCmp(ty, .lt, op_tmp, lhs);
4031	break :blk try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
4032	} else try cg.intCmp(ty, .lt, op_tmp, lhs);
4033
4034	return .{ .result = op_tmp, .ov = overflow_bit };
4035	},
4036	else => {
4037	const result = try cg.allocInt(ty);
4038
4039	try cg.lowerToStack(result);
4040	try cg.lowerToStack(lhs);
4041	try cg.lowerToStack(rhs);
4042	try cg.addImm32(@intFromBool(ty.is_signed));
4043	try cg.addImm32(ty.bits);
4044	try cg.addCallIntrinsic(.__addo_limb64);
4045
4046	return .{ .result = result, .ov = .stack };
4047	},
4048	}
4049	}
4050
4051	fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
4052	switch (ty.bits) {
4053	0 => unreachable,
4054	1...128 => {
4055	const raw_result = try cg.intSub(ty, lhs, rhs);
4056	const op_result = try cg.intWrap(ty, raw_result);
4057	const op_tmp = try cg.toLocalInt(op_result, ty);
4058
4059	const overflow_bit = if (ty.is_signed) blk: {
4060	const zero = try cg.intZeroValue(ty);
4061	const rhs_is_neg = try cg.intCmp(ty, .lt, rhs, zero);
4062	const overflow_cmp = try cg.intCmp(ty, .gt, op_tmp, lhs);
4063	break :blk try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
4064	} else try cg.intCmp(ty, .gt, op_tmp, lhs);
4065
4066	return .{ .result = op_tmp, .ov = overflow_bit };
4067	},
4068	else => {
4069	const result = try cg.allocInt(ty);
4070
4071	try cg.lowerToStack(result);
4072	try cg.lowerToStack(lhs);
4073	try cg.lowerToStack(rhs);
4074	try cg.addImm32(@intFromBool(ty.is_signed));
4075	try cg.addImm32(ty.bits);
4076	try cg.addCallIntrinsic(.__subo_limb64);
4077	return .{ .result = result, .ov = .stack };
4078	},
4079	}
4080	}
4081
4082	fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
4083	const overflow_bit = try cg.allocLocal(Type.u32);
4084	try cg.addImm32(0);
4085	try cg.addLocal(.local_set, overflow_bit.local.value);
4086
4087	const result_val = if (int_ty.bits <= 32) blk: {
4088	const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 64 };
4089	const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
4090	const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
4091	const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
4092	const bin_op = try cg.toLocalInt(mul_raw, new_ty);
4093
4094	const res = try cg.intTrunc(int_ty, new_ty, bin_op);
4095	const res_tmp = try cg.toLocalInt(res, int_ty);
4096
4097	const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
4098	_ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
4099	try cg.addLocal(.local_set, overflow_bit.local.value);
4100	break :blk res_tmp;
4101	} else if (int_ty.bits <= 64) blk: {
4102	const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 128 };
4103	const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
4104	const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
4105	const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
4106	const bin_op = try cg.toLocalInt(mul_raw, new_ty);
4107
4108	const res = try cg.intTrunc(int_ty, new_ty, bin_op);
4109	const res_tmp = try cg.toLocalInt(res, int_ty);
4110
4111	const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
4112	_ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
4113	try cg.addLocal(.local_set, overflow_bit.local.value);
4114	break :blk res_tmp;
4115	} else if (int_ty.bits == 128 and int_ty.is_signed) blk: {
4116	const overflow_ret = try cg.allocStack(Type.i32);
4117	const res = try cg.callIntrinsic(
4118	.__muloti4,
4119	&[_]InternPool.Index{ .i128_type, .i128_type, .usize_type },
4120	Type.i128,
4121	&.{ lhs, rhs, overflow_ret },
4122	);
4123	_ = try cg.load(overflow_ret, Type.i32, 0);
4124	try cg.addLocal(.local_set, overflow_bit.local.value);
4125	break :blk res;
4126	} else {
4127	const result = try cg.allocInt(int_ty);
4128
4129	try cg.lowerToStack(result);
4130	try cg.lowerToStack(lhs);
4131	try cg.lowerToStack(rhs);
4132	try cg.addImm32(@intFromBool(int_ty.is_signed));
4133	try cg.addImm32(int_ty.bits);
4134	try cg.addCallIntrinsic(.__mulo_limb64);
4135
4136	return .{ .result = result, .ov = .stack };
4137	};
4138
4139	return .{ .result = result_val, .ov = .{ .local = overflow_bit.local } };
4140	}
4141
4142	fn intShlOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
4143	switch (ty.bits) {
4144	0 => unreachable,
4145	1...128 => {
4146	const raw_shl = try cg.intShl(ty, lhs, rhs);
4147	const wrapped_shl = try cg.intWrap(ty, raw_shl);
4148	const shl_tmp = try cg.toLocalInt(wrapped_shl, ty);
4149
4150	const shr = try cg.intShr(ty, shl_tmp, rhs);
4151	const overflow_bit = try cg.intCmp(ty, .neq, shr, lhs);
4152
4153	return .{ .result = shl_tmp, .ov = overflow_bit };
4154	},
4155	else => {
4156	const result = try cg.allocInt(ty);
4157
4158	try cg.lowerToStack(result);
4159	try cg.lowerToStack(lhs);
4160	try cg.lowerToStack(rhs);
4161	try cg.addImm32(@intFromBool(ty.is_signed));
4162	try cg.addImm32(ty.bits);
4163	try cg.addCallIntrinsic(.__shlo_limb64);
4164
4165	return .{ .result = result, .ov = .stack };
4166	},
4167	}
4168	}
4169
4170	fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {
4171	const src_bits: u16 = cg.intBackingBits(src_ty.bits);
4172	const dest_bits: u16 = cg.intBackingBits(dest_ty.bits);
4173
4174	if (src_bits == dest_bits) {
4175	return operand;
4176	}
4177
4178	if (src_bits == 64 and dest_bits == 32) {
4179	try cg.emitWValue(operand);
4180	try cg.addTag(.i32_wrap_i64);
4181	return .stack;
4182	} else if (src_bits == 32 and dest_bits == 64) {
4183	try cg.emitWValue(operand);
4184	try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4185	return .stack;
4186	} else if (dest_bits >= 128) {
4187	const result = try cg.allocInt(dest_ty);
4188
4189	const dest_len = dest_bits / 8;
4190
4191	if (dest_bits <= src_bits) {
4192	assert(src_bits >= 128);
4193	try cg.memcpy(result, operand, .{ .imm32 = dest_len });
4194	} else {
4195	var src_len: u32 = undefined;
4196	if (src_bits == 32) {
4197	try cg.emitWValue(result);
4198	try cg.emitWValue(operand);
4199	try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4200	try cg.store(.stack, .stack, Type.u64, result.offset());
4201	src_len = 8;
4202	} else if (src_bits == 64) {
4203	try cg.emitWValue(result);
4204	try cg.emitWValue(operand);
4205	try cg.store(.stack, .stack, Type.u64, result.offset());
4206	src_len = 8;
4207	} else {
4208	src_len = src_bits / 8;
4209	try cg.memcpy(result, operand, .{ .imm32 = src_len });
4210	}
4211
4212	if (dest_bits == 128) {
4213	if (src_ty.is_signed) {
4214	try cg.emitWValue(result);
4215	if (src_bits == 32) {
4216	try cg.emitWValue(operand);
4217	try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4218	} else if (src_bits == 64) {
4219	try cg.emitWValue(operand);
4220	} else unreachable;
4221	const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4222	try cg.store(.stack, shr, Type.u64, 8 + result.offset());
4223	} else {
4224	try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
4225	}
4226	} else {
4227	var pad = result;
4228	pad.stack_offset.value += src_len;
4229	const memset_len = dest_len - src_len;
4230	if (src_ty.is_signed) {
4231	if (src_bits == 32) {
4232	try cg.emitWValue(operand);
4233	_ = try cg.intShr(IntType.i32, .stack, .{ .imm32 = 31 });
4234	} else if (src_bits == 64) {
4235	try cg.emitWValue(operand);
4236	_ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4237	try cg.addTag(.i32_wrap_i64);
4238	} else {
4239	_ = try cg.load(operand, Type.u64, src_len - 8);
4240	_ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4241	try cg.addTag(.i32_wrap_i64);
4242	}
4243	var sign_byte = try @as(WValue, .stack).toLocal(cg, Type.u32);
4244	try cg.memset(Type.u8, pad, .{ .imm32 = memset_len }, sign_byte);
4245	sign_byte.free(cg);
4246	} else {
4247	try cg.memset(Type.u8, pad, .{ .imm32 = memset_len }, .{ .imm32 = 0 });
4248	}
4249	}
4250	}
4251
4252	return result;
4253	} else {
4254	assert(dest_bits <= 64);
4255	assert(src_bits >= 128);
4256	const load_ty = if (dest_bits == 32) Type.u32 else Type.u64;
4257	return cg.load(operand, load_ty, 0);
4258	}
4259	}
4260
4261	fn intTrunc(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {
4262	var result = try cg.intCast(dest_ty, src_ty, operand);
4263
4264	const dest_wasm_bits = cg.intBackingBits(dest_ty.bits);
4265
4266	if (dest_wasm_bits != dest_ty.bits) {
4267	result = try cg.intWrap(dest_ty, result);
4268	}
4269
4270	return result;
4271	}
4272
4273	const FloatType = enum {
4274	f16,
4275	f32,
4276	f64,
4277	f80,
4278	f128,
4279
4280	fn fromType(cg: *CodeGen, ty: Type) FloatType {
4281	assert(ty.isRuntimeFloat());
4282	return switch (ty.floatBits(cg.target)) {
4283	16 => .f16,
4284	32 => .f32,
4285	64 => .f64,
4286	80 => .f80,
4287	128 => .f128,
4288	else => unreachable,
4289	};
4290	}
4291	};
4292
4293	fn floatAdd(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4294	switch (ty) {
4295	.f16 => return cg.callIntrinsic(.__addhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4296	.f32 => {
4297	try cg.emitWValue(lhs);
4298	try cg.emitWValue(rhs);
4299	try cg.addTag(.f32_add);
4300	return .stack;
4301	},
4302	.f64 => {
4303	try cg.emitWValue(lhs);
4304	try cg.emitWValue(rhs);
4305	try cg.addTag(.f64_add);
4306	return .stack;
4307	},
4308	.f80 => return cg.callIntrinsic(.__addxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4309	.f128 => return cg.callIntrinsic(.__addtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4310	}
4311	}
4312
4313	fn floatSub(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4314	switch (ty) {
4315	.f16 => return cg.callIntrinsic(.__subhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4316	.f32 => {
4317	try cg.emitWValue(lhs);
4318	try cg.emitWValue(rhs);
4319	try cg.addTag(.f32_sub);
4320	return .stack;
4321	},
4322	.f64 => {
4323	try cg.emitWValue(lhs);
4324	try cg.emitWValue(rhs);
4325	try cg.addTag(.f64_sub);
4326	return .stack;
4327	},
4328	.f80 => return cg.callIntrinsic(.__subxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4329	.f128 => return cg.callIntrinsic(.__subtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4330	}
4331	}
4332
4333	fn floatMul(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4334	switch (ty) {
4335	.f16 => return cg.callIntrinsic(.__mulhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4336	.f32 => {
4337	try cg.emitWValue(lhs);
4338	try cg.emitWValue(rhs);
4339	try cg.addTag(.f32_mul);
4340	return .stack;
4341	},
4342	.f64 => {
4343	try cg.emitWValue(lhs);
4344	try cg.emitWValue(rhs);
4345	try cg.addTag(.f64_mul);
4346	return .stack;
4347	},
4348	.f80 => return cg.callIntrinsic(.__mulxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4349	.f128 => return cg.callIntrinsic(.__multf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4350	}
4351	}
4352
4353	fn floatMulAdd(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue, addend: WValue) InnerError!WValue {
4354	const mul_result = try cg.floatMul(ty, lhs, rhs);
4355	return cg.floatAdd(ty, mul_result, addend);
4356	}
4357
4358	fn floatDiv(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4359	switch (ty) {
4360	.f16 => return cg.callIntrinsic(.__divhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4361	.f32 => {
4362	try cg.emitWValue(lhs);
4363	try cg.emitWValue(rhs);
4364	try cg.addTag(.f32_div);
4365	return .stack;
4366	},
4367	.f64 => {
4368	try cg.emitWValue(lhs);
4369	try cg.emitWValue(rhs);
4370	try cg.addTag(.f64_div);
4371	return .stack;
4372	},
4373	.f80 => return cg.callIntrinsic(.__divxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4374	.f128 => return cg.callIntrinsic(.__divtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4375	}
4376	}
4377
4378	fn floatRem(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4379	switch (ty) {
4380	.f16 => return cg.callIntrinsic(.__fmodh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4381	.f32 => return cg.callIntrinsic(.fmodf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
4382	.f64 => return cg.callIntrinsic(.fmod, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
4383	.f80 => return cg.callIntrinsic(.__fmodx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4384	.f128 => return cg.callIntrinsic(.fmodf128, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4385	}
4386	}
4387
4388	// div_trunc(a, b) = trunc(a / b)
4389	fn floatDivTrunc(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4390	const div_result = try cg.floatDiv(ty, lhs, rhs);
4391	return cg.floatTrunc(ty, div_result);
4392	}
4393
4394	// div_floor(a, b) = floor(a / b)
4395	fn floatDivFloor(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4396	const div_result = try cg.floatDiv(ty, lhs, rhs);
4397	return cg.floatFloor(ty, div_result);
4398	}
4399
4400	// div_ceil(a, b) = ceil(a / b)
4401	fn floatDivCeil(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4402	const div_result = try cg.floatDiv(ty, lhs, rhs);
4403	return cg.floatCeil(ty, div_result);
4404	}
4405
4406	// mod(a, b) = fmod(fmod(a, b) + b, b)
4407	fn floatMod(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4408	const r = try cg.floatRem(ty, lhs, rhs);
4409	const s = try cg.floatAdd(ty, r, rhs);
4410	return cg.floatRem(ty, s, rhs);
4411	}
4412
4413	// wasm fN_max NaN semantics differ with Zig
4414	fn floatMax(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4415	switch (ty) {
4416	.f16 => return cg.callIntrinsic(.__fmaxh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4417	.f32 => return cg.callIntrinsic(.fmaxf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
4418	.f64 => return cg.callIntrinsic(.fmax, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
4419	.f80 => return cg.callIntrinsic(.__fmaxx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4420	.f128 => return cg.callIntrinsic(.fmaxf128, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4421	}
4422	}
4423
4424	// wasm fN_min NaN semantics differ with Zig
4425	fn floatMin(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4426	switch (ty) {
4427	.f16 => return cg.callIntrinsic(.__fminh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4428	.f32 => return cg.callIntrinsic(.fminf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
4429	.f64 => return cg.callIntrinsic(.fmin, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
4430	.f80 => return cg.callIntrinsic(.__fminx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4431	.f128 => return cg.callIntrinsic(.fminf128, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4432	}
4433	}
4434
4435	fn floatSqrt(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4436	switch (ty) {
4437	.f16 => return cg.callIntrinsic(.__sqrth, &.{.f16_type}, Type.f16, &.{arg}),
4438	.f32 => {
4439	try cg.emitWValue(arg);
4440	try cg.addTag(.f32_sqrt);
4441	return .stack;
4442	},
4443	.f64 => {
4444	try cg.emitWValue(arg);
4445	try cg.addTag(.f64_sqrt);
4446	return .stack;
4447	},
4448	.f80 => return cg.callIntrinsic(.__sqrtx, &.{.f80_type}, Type.f80, &.{arg}),
4449	.f128 => return cg.callIntrinsic(.sqrtf128, &.{.f128_type}, Type.f128, &.{arg}),
4450	}
4451	}
4452
4453	fn floatSin(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4454	switch (ty) {
4455	.f16 => return cg.callIntrinsic(.__sinh, &.{.f16_type}, Type.f16, &.{arg}),
4456	.f32 => return cg.callIntrinsic(.sinf, &.{.f32_type}, Type.f32, &.{arg}),
4457	.f64 => return cg.callIntrinsic(.sin, &.{.f64_type}, Type.f64, &.{arg}),
4458	.f80 => return cg.callIntrinsic(.__sinx, &.{.f80_type}, Type.f80, &.{arg}),
4459	.f128 => return cg.callIntrinsic(.sinf128, &.{.f128_type}, Type.f128, &.{arg}),
4460	}
4461	}
4462
4463	fn floatCos(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4464	switch (ty) {
4465	.f16 => return cg.callIntrinsic(.__cosh, &.{.f16_type}, Type.f16, &.{arg}),
4466	.f32 => return cg.callIntrinsic(.cosf, &.{.f32_type}, Type.f32, &.{arg}),
4467	.f64 => return cg.callIntrinsic(.cos, &.{.f64_type}, Type.f64, &.{arg}),
4468	.f80 => return cg.callIntrinsic(.__cosx, &.{.f80_type}, Type.f80, &.{arg}),
4469	.f128 => return cg.callIntrinsic(.cosf128, &.{.f128_type}, Type.f128, &.{arg}),
4470	}
4471	}
4472
4473	fn floatTan(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4474	switch (ty) {
4475	.f16 => return cg.callIntrinsic(.__tanh, &.{.f16_type}, Type.f16, &.{arg}),
4476	.f32 => return cg.callIntrinsic(.tanf, &.{.f32_type}, Type.f32, &.{arg}),
4477	.f64 => return cg.callIntrinsic(.tan, &.{.f64_type}, Type.f64, &.{arg}),
4478	.f80 => return cg.callIntrinsic(.__tanx, &.{.f80_type}, Type.f80, &.{arg}),
4479	.f128 => return cg.callIntrinsic(.tanf128, &.{.f128_type}, Type.f128, &.{arg}),
4480	}
4481	}
4482
4483	fn floatExp(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4484	switch (ty) {
4485	.f16 => return cg.callIntrinsic(.__exph, &.{.f16_type}, Type.f16, &.{arg}),
4486	.f32 => return cg.callIntrinsic(.expf, &.{.f32_type}, Type.f32, &.{arg}),
4487	.f64 => return cg.callIntrinsic(.exp, &.{.f64_type}, Type.f64, &.{arg}),
4488	.f80 => return cg.callIntrinsic(.__expx, &.{.f80_type}, Type.f80, &.{arg}),
4489	.f128 => return cg.callIntrinsic(.expf128, &.{.f128_type}, Type.f128, &.{arg}),
4490	}
4491	}
4492
4493	fn floatExp2(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4494	switch (ty) {
4495	.f16 => return cg.callIntrinsic(.__exp2h, &.{.f16_type}, Type.f16, &.{arg}),
4496	.f32 => return cg.callIntrinsic(.exp2f, &.{.f32_type}, Type.f32, &.{arg}),
4497	.f64 => return cg.callIntrinsic(.exp2, &.{.f64_type}, Type.f64, &.{arg}),
4498	.f80 => return cg.callIntrinsic(.__exp2x, &.{.f80_type}, Type.f80, &.{arg}),
4499	.f128 => return cg.callIntrinsic(.exp2f128, &.{.f128_type}, Type.f128, &.{arg}),
4500	}
4501	}
4502
4503	fn floatLog(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4504	switch (ty) {
4505	.f16 => return cg.callIntrinsic(.__logh, &.{.f16_type}, Type.f16, &.{arg}),
4506	.f32 => return cg.callIntrinsic(.logf, &.{.f32_type}, Type.f32, &.{arg}),
4507	.f64 => return cg.callIntrinsic(.log, &.{.f64_type}, Type.f64, &.{arg}),
4508	.f80 => return cg.callIntrinsic(.__logx, &.{.f80_type}, Type.f80, &.{arg}),
4509	.f128 => return cg.callIntrinsic(.logf128, &.{.f128_type}, Type.f128, &.{arg}),
4510	}
4511	}
4512
4513	fn floatLog2(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4514	switch (ty) {
4515	.f16 => return cg.callIntrinsic(.__log2h, &.{.f16_type}, Type.f16, &.{arg}),
4516	.f32 => return cg.callIntrinsic(.log2f, &.{.f32_type}, Type.f32, &.{arg}),
4517	.f64 => return cg.callIntrinsic(.log2, &.{.f64_type}, Type.f64, &.{arg}),
4518	.f80 => return cg.callIntrinsic(.__log2x, &.{.f80_type}, Type.f80, &.{arg}),
4519	.f128 => return cg.callIntrinsic(.log2f128, &.{.f128_type}, Type.f128, &.{arg}),
4520	}
4521	}
4522
4523	fn floatLog10(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4524	switch (ty) {
4525	.f16 => return cg.callIntrinsic(.__log10h, &.{.f16_type}, Type.f16, &.{arg}),
4526	.f32 => return cg.callIntrinsic(.log10f, &.{.f32_type}, Type.f32, &.{arg}),
4527	.f64 => return cg.callIntrinsic(.log10, &.{.f64_type}, Type.f64, &.{arg}),
4528	.f80 => return cg.callIntrinsic(.__log10x, &.{.f80_type}, Type.f80, &.{arg}),
4529	.f128 => return cg.callIntrinsic(.log10f128, &.{.f128_type}, Type.f128, &.{arg}),
4530	}
4531	}
4532
4533	fn floatFloor(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4534	switch (ty) {
4535	.f16 => return cg.callIntrinsic(.__floorh, &.{.f16_type}, Type.f16, &.{arg}),
4536	.f32 => {
4537	try cg.emitWValue(arg);
4538	try cg.addTag(.f32_floor);
4539	return .stack;
4540	},
4541	.f64 => {
4542	try cg.emitWValue(arg);
4543	try cg.addTag(.f64_floor);
4544	return .stack;
4545	},
4546	.f80 => return cg.callIntrinsic(.__floorx, &.{.f80_type}, Type.f80, &.{arg}),
4547	.f128 => return cg.callIntrinsic(.floorf128, &.{.f128_type}, Type.f128, &.{arg}),
4548	}
4549	}
4550
4551	fn floatCeil(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4552	switch (ty) {
4553	.f16 => return cg.callIntrinsic(.__ceilh, &.{.f16_type}, Type.f16, &.{arg}),
4554	.f32 => {
4555	try cg.emitWValue(arg);
4556	try cg.addTag(.f32_ceil);
4557	return .stack;
4558	},
4559	.f64 => {
4560	try cg.emitWValue(arg);
4561	try cg.addTag(.f64_ceil);
4562	return .stack;
4563	},
4564	.f80 => return cg.callIntrinsic(.__ceilx, &.{.f80_type}, Type.f80, &.{arg}),
4565	.f128 => return cg.callIntrinsic(.ceilf128, &.{.f128_type}, Type.f128, &.{arg}),
4566	}
4567	}
4568
4569	fn floatRound(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4570	switch (ty) {
4571	.f16 => return cg.callIntrinsic(.__roundh, &.{.f16_type}, Type.f16, &.{arg}),
4572	.f32 => {
4573	try cg.emitWValue(arg);
4574	try cg.addTag(.f32_nearest);
4575	return .stack;
4576	},
4577	.f64 => {
4578	try cg.emitWValue(arg);
4579	try cg.addTag(.f64_nearest);
4580	return .stack;
4581	},
4582	.f80 => return cg.callIntrinsic(.__roundx, &.{.f80_type}, Type.f80, &.{arg}),
4583	.f128 => return cg.callIntrinsic(.roundf128, &.{.f128_type}, Type.f128, &.{arg}),
4584	}
4585	}
4586
4587	fn floatTrunc(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4588	switch (ty) {
4589	.f16 => return cg.callIntrinsic(.__trunch, &.{.f16_type}, Type.f16, &.{arg}),
4590	.f32 => {
4591	try cg.emitWValue(arg);
4592	try cg.addTag(.f32_trunc);
4593	return .stack;
4594	},
4595	.f64 => {
4596	try cg.emitWValue(arg);
4597	try cg.addTag(.f64_trunc);
4598	return .stack;
4599	},
4600	.f80 => return cg.callIntrinsic(.__truncx, &.{.f80_type}, Type.f80, &.{arg}),
4601	.f128 => return cg.callIntrinsic(.truncf128, &.{.f128_type}, Type.f128, &.{arg}),
4602	}
4603	}
4604
4605	fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4606	switch (ty) {
4607	.f16 => {
4608	try cg.emitWValue(arg);
4609	try cg.addImm32(0x8000);
4610	try cg.addTag(.i32_xor);
4611	return .stack;
4612	},
4613	.f32 => {
4614	try cg.emitWValue(arg);
4615	try cg.addTag(.f32_neg);
4616	return .stack;
4617	},
4618	.f64 => {
4619	try cg.emitWValue(arg);
4620	try cg.addTag(.f64_neg);
4621	return .stack;
4622	},
4623	.f80, .f128 => {
4624	const result = try cg.allocStack(Type.f128);
4625	try cg.emitWValue(result);
4626	try cg.emitWValue(arg);
4627	try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
4628	try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
4629	try cg.emitWValue(result);
4630	try cg.emitWValue(arg);
4631	try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
4632	if (ty == .f80) {
4633	try cg.addImm64(0x8000);
4634	} else {
4635	try cg.addImm64(0x8000000000000000);
4636	}
4637	try cg.addTag(.i64_xor);
4638	try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
4639	return result;
4640	},
4641	}
4642	}
4643
4644	fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4645	switch (ty) {
4646	.f16 => {
4647	try cg.emitWValue(arg);
4648	try cg.addImm32(0x7FFF);
4649	try cg.addTag(.i32_and);
4650	return .stack;
4651	},
4652	.f32 => {
4653	try cg.emitWValue(arg);
4654	try cg.addTag(.f32_abs);
4655	return .stack;
4656	},
4657	.f64 => {
4658	try cg.emitWValue(arg);
4659	try cg.addTag(.f64_abs);
4660	return .stack;
4661	},
4662	.f80, .f128 => {
4663	const result = try cg.allocStack(Type.f128);
4664	try cg.emitWValue(result);
4665	try cg.emitWValue(arg);
4666	try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
4667	try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
4668	try cg.emitWValue(result);
4669	try cg.emitWValue(arg);
4670	try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
4671	if (ty == .f80) {
4672	try cg.addImm64(0x7FFF);
4673	} else {
4674	try cg.addImm64(0x7FFFFFFFFFFFFFFF);
4675	}
4676	try cg.addTag(.i64_and);
4677	try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
4678	return result;
4679	},
4680	}
4681	}
4682
4683	fn floatExtendCast(cg: *CodeGen, dest_ty: FloatType, src_ty: FloatType, operand: WValue) InnerError!WValue {
4684	switch (dest_ty) {
4685	.f16 => unreachable,
4686	.f32 => switch (src_ty) {
4687	.f16 => {
4688	_ = try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand});
4689	return .stack;
4690	},
4691	else => unreachable,
4692	},
4693	.f64 => switch (src_ty) {
4694	.f16 => {
4695	_ = try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand});
4696	try cg.addTag(.f64_promote_f32);
4697	return .stack;
4698	},
4699	.f32 => {
4700	try cg.emitWValue(operand);
4701	try cg.addTag(.f64_promote_f32);
4702	return .stack;
4703	},
4704	else => unreachable,
4705	},
4706	.f80 => switch (src_ty) {
4707	.f16 => return cg.callIntrinsic(.__extendhfxf2, &.{.f16_type}, Type.f80, &.{operand}),
4708	.f32 => return cg.callIntrinsic(.__extendsfxf2, &.{.f32_type}, Type.f80, &.{operand}),
4709	.f64 => return cg.callIntrinsic(.__extenddfxf2, &.{.f64_type}, Type.f80, &.{operand}),
4710	else => unreachable,
4711	},
4712	.f128 => switch (src_ty) {
4713	.f16 => return cg.callIntrinsic(.__extendhftf2, &.{.f16_type}, Type.f128, &.{operand}),
4714	.f32 => return cg.callIntrinsic(.__extendsftf2, &.{.f32_type}, Type.f128, &.{operand}),
4715	.f64 => return cg.callIntrinsic(.__extenddftf2, &.{.f64_type}, Type.f128, &.{operand}),
4716	.f80 => return cg.callIntrinsic(.__extendxftf2, &.{.f80_type}, Type.f128, &.{operand}),
4717	else => unreachable,
4718	},
4719	}
4720	}
4721
4722	fn floatTruncCast(cg: *CodeGen, dest_ty: FloatType, src_ty: FloatType, operand: WValue) InnerError!WValue {
4723	switch (dest_ty) {
4724	.f16 => switch (src_ty) {
4725	.f32 => return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{operand}),
4726	.f64 => {
4727	try cg.emitWValue(operand);
4728	try cg.addTag(.f32_demote_f64);
4729	return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{.stack});
4730	},
4731	.f80 => return cg.callIntrinsic(.__truncxfhf2, &.{.f80_type}, Type.f16, &.{operand}),
4732	.f128 => return cg.callIntrinsic(.__trunctfhf2, &.{.f128_type}, Type.f16, &.{operand}),
4733	else => unreachable,
4734	},
4735	.f32 => switch (src_ty) {
4736	.f64 => {
4737	try cg.emitWValue(operand);
4738	try cg.addTag(.f32_demote_f64);
4739	return .stack;
4740	},
4741	.f80 => return cg.callIntrinsic(.__truncxfsf2, &.{.f80_type}, Type.f32, &.{operand}),
4742	.f128 => return cg.callIntrinsic(.__trunctfsf2, &.{.f128_type}, Type.f32, &.{operand}),
4743	else => unreachable,
4744	},
4745	.f64 => switch (src_ty) {
4746	.f80 => return cg.callIntrinsic(.__truncxfdf2, &.{.f80_type}, Type.f64, &.{operand}),
4747	.f128 => return cg.callIntrinsic(.__trunctfdf2, &.{.f128_type}, Type.f64, &.{operand}),
4748	else => unreachable,
4749	},
4750	.f80 => switch (src_ty) {
4751	.f128 => return cg.callIntrinsic(.__trunctfxf2, &.{.f128_type}, Type.f80, &.{operand}),
4752	else => unreachable,
4753	},
4754	.f128 => unreachable,
4755	}
4756	}
4757
4758	fn intFromFloat(cg: *CodeGen, dest_ty: IntType, src_ty: FloatType, operand: WValue) InnerError!WValue {
4759	switch (dest_ty.bits) {
4760	0 => unreachable,
4761	1...32 => switch (src_ty) {
4762	.f16 => {
4763	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfsi else .__fixunshfsi;
4764	return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u32, &.{operand});
4765	},
4766	.f32 => {
4767	try cg.emitWValue(operand);
4768	try cg.addTag(if (dest_ty.is_signed) .i32_trunc_f32_s else .i32_trunc_f32_u);
4769	return .stack;
4770	},
4771	.f64 => {
4772	try cg.emitWValue(operand);
4773	try cg.addTag(if (dest_ty.is_signed) .i32_trunc_f64_s else .i32_trunc_f64_u);
4774	return .stack;
4775	},
4776	.f80 => {
4777	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfsi else .__fixunsxfsi;
4778	return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u32, &.{operand});
4779	},
4780	.f128 => {
4781	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfsi else .__fixunstfsi;
4782	return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u32, &.{operand});
4783	},
4784	},
4785	33...64 => switch (src_ty) {
4786	.f16 => {
4787	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfdi else .__fixunshfdi;
4788	return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u64, &.{operand});
4789	},
4790	.f32 => {
4791	try cg.emitWValue(operand);
4792	try cg.addTag(if (dest_ty.is_signed) .i64_trunc_f32_s else .i64_trunc_f32_u);
4793	return .stack;
4794	},
4795	.f64 => {
4796	try cg.emitWValue(operand);
4797	try cg.addTag(if (dest_ty.is_signed) .i64_trunc_f64_s else .i64_trunc_f64_u);
4798	return .stack;
4799	},
4800	.f80 => {
4801	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfdi else .__fixunsxfdi;
4802	return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u64, &.{operand});
4803	},
4804	.f128 => {
4805	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfdi else .__fixunstfdi;
4806	return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u64, &.{operand});
4807	},
4808	},
4809	65...128 => switch (src_ty) {
4810	.f16 => {
4811	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfti else .__fixunshfti;
4812	return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u128, &.{operand});
4813	},
4814	.f32 => {
4815	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixsfti else .__fixunssfti;
4816	return cg.callIntrinsic(intrinsic, &.{.f32_type}, Type.u128, &.{operand});
4817	},
4818	.f64 => {
4819	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixdfti else .__fixunsdfti;
4820	return cg.callIntrinsic(intrinsic, &.{.f64_type}, Type.u128, &.{operand});
4821	},
4822	.f80 => {
4823	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfti else .__fixunsxfti;
4824	return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u128, &.{operand});
4825	},
4826	.f128 => {
4827	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfti else .__fixunstfti;
4828	return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u128, &.{operand});
4829	},
4830	},
4831	else => {
4832	const result = try cg.allocInt(dest_ty);
4833
4834	switch (src_ty) {
4835	.f16 => {
4836	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfei else .__fixunshfei;
4837	_ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f16_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
4838	},
4839	.f32 => {
4840	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixsfei else .__fixunssfei;
4841	_ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f32_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
4842	},
4843	.f64 => {
4844	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixdfei else .__fixunsdfei;
4845	_ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f64_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
4846	},
4847	.f80 => {
4848	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfei else .__fixunsxfei;
4849	_ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f80_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
4850	},
4851	.f128 => {
4852	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfei else .__fixunstfei;
4853	_ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f128_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
4854	},
4855	}
4856
4857	return result;
4858	},
4859	}
4860	}
4861
4862	fn floatFromInt(cg: *CodeGen, dest_ty: FloatType, src_ty: IntType, operand: WValue) InnerError!WValue {
4863	switch (dest_ty) {
4864	.f16 => switch (src_ty.bits) {
4865	0 => unreachable,
4866	1...32 => {
4867	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsihf else .__floatunsihf;
4868	return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f16, &.{operand});
4869	},
4870	33...64 => {
4871	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatdihf else .__floatundihf;
4872	return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f16, &.{operand});
4873	},
4874	65...128 => {
4875	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattihf else .__floatuntihf;
4876	return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f16, &.{operand});
4877	},
4878	else => {
4879	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateihf else .__floatuneihf;
4880	return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f16, &.{ operand, .{ .imm32 = src_ty.bits } });
4881	},
4882	},
4883	.f32 => switch (src_ty.bits) {
4884	0 => unreachable,
4885	1...32 => {
4886	try cg.emitWValue(operand);
4887	try cg.addTag(if (src_ty.is_signed) .f32_convert_i32_s else .f32_convert_i32_u);
4888	return .stack;
4889	},
4890	33...64 => {
4891	try cg.emitWValue(operand);
4892	try cg.addTag(if (src_ty.is_signed) .f32_convert_i64_s else .f32_convert_i64_u);
4893	return .stack;
4894	},
4895	65...128 => {
4896	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattisf else .__floatuntisf;
4897	return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f32, &.{operand});
4898	},
4899	else => {
4900	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateisf else .__floatuneisf;
4901	return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f32, &.{ operand, .{ .imm32 = src_ty.bits } });
4902	},
4903	},
4904	.f64 => switch (src_ty.bits) {
4905	0 => unreachable,
4906	1...32 => {
4907	try cg.emitWValue(operand);
4908	try cg.addTag(if (src_ty.is_signed) .f64_convert_i32_s else .f64_convert_i32_u);
4909	return .stack;
4910	},
4911	33...64 => {
4912	try cg.emitWValue(operand);
4913	try cg.addTag(if (src_ty.is_signed) .f64_convert_i64_s else .f64_convert_i64_u);
4914	return .stack;
4915	},
4916	65...128 => {
4917	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattidf else .__floatuntidf;
4918	return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f64, &.{operand});
4919	},
4920	else => {
4921	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateidf else .__floatuneidf;
4922	return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f64, &.{ operand, .{ .imm32 = src_ty.bits } });
4923	},
4924	},
4925	.f80 => switch (src_ty.bits) {
4926	0 => unreachable,
4927	1...32 => {
4928	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsixf else .__floatunsixf;
4929	return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f80, &.{operand});
4930	},
4931	33...64 => {
4932	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatdixf else .__floatundixf;
4933	return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f80, &.{operand});
4934	},
4935	65...128 => {
4936	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattixf else .__floatuntixf;
4937	return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f80, &.{operand});
4938	},
4939	else => {
4940	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateixf else .__floatuneixf;
4941	return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f80, &.{ operand, .{ .imm32 = src_ty.bits } });
4942	},
4943	},
4944	.f128 => switch (src_ty.bits) {
4945	0 => unreachable,
4946	1...32 => {
4947	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsitf else .__floatunsitf;
4948	return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f128, &.{operand});
4949	},
4950	33...64 => {
4951	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatditf else .__floatunditf;
4952	return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f128, &.{operand});
4953	},
4954	65...128 => {
4955	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattitf else .__floatuntitf;
4956	return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f128, &.{operand});
4957	},
4958	else => {
4959	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateitf else .__floatuneitf;
4960	return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f128, &.{ operand, .{ .imm32 = src_ty.bits } });
4961	},
4962	},
4963	}
4964	}
4965
4966	fn lowerPtr(cg: *CodeGen, ptr_val: InternPool.Index, prev_offset: u64) InnerError!WValue {
4967	const pt = cg.pt;
4968	const zcu = pt.zcu;
4969	const ip = &zcu.intern_pool;
4970	const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr;
4971	const offset: u64 = prev_offset + ptr.byte_offset;
4972	return switch (ptr.base_addr) {
4973	.nav => |nav| return if (ip.getNav(nav).getExtern(ip) != null or
4974	Type.fromInterned(ip.getNav(nav).resolved.?.type).isRuntimeFnOrHasRuntimeBits(zcu))
4975	.{ .nav_ref = .{ .nav_index = nav, .offset = @intCast(offset) } }
4976	else
4977	.{ .imm32 = @intCast(zcu.navAlignment(nav).forward(@as(u32, 0xaaaaaaaa))) },
4978	.uav => |uav| return if (Type.fromInterned(ip.typeOf(uav.val)).isRuntimeFnOrHasRuntimeBits(zcu))
4979	.{ .uav_ref = .{ .ip_index = uav.val, .offset = @intCast(offset), .orig_ptr_ty = uav.orig_ty } }
4980	else
4981	.{ .imm32 = @intCast(Type.fromInterned(uav.orig_ty).ptrAlignment(zcu).forward(@as(u32, 0xaaaaaaaa))) },
4982	.int => return cg.lowerConstant(try pt.intValue(.usize, offset)),
4983	.eu_payload => |eu_ptr| try cg.lowerPtr(
4984	eu_ptr,
4985	offset + codegen.errUnionPayloadOffset(
4986	Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu),
4987	zcu,
4988	),
4989	),
4990	.opt_payload => |opt_ptr| return cg.lowerPtr(opt_ptr, offset),
4991	.field => |field| {
4992	const base_ptr = Value.fromInterned(field.base);
4993	const base_ty = base_ptr.typeOf(zcu).childType(zcu);
4994	const field_off: u64 = switch (base_ty.zigTypeTag(zcu)) {
4995	.pointer => off: {
4996	assert(base_ty.isSlice(zcu));
4997	break :off switch (field.index) {
4998	Value.slice_ptr_index => 0,
4999	Value.slice_len_index => @divExact(cg.target.ptrBitWidth(), 8),
5000	else => unreachable,
5001	};
5002	},
5003	.@"struct" => switch (base_ty.containerLayout(zcu)) {
5004	.auto => base_ty.structFieldOffset(@intCast(field.index), zcu),
5005	.@"extern", .@"packed" => unreachable,
5006	},
5007	.@"union" => switch (base_ty.containerLayout(zcu)) {
5008	.auto => base_ty.structFieldOffset(@intCast(field.index), zcu),
5009	.@"extern", .@"packed" => unreachable,
5010	},
5011	else => unreachable,
5012	};
5013	return cg.lowerPtr(field.base, offset + field_off);
5014	},
5015	.arr_elem, .comptime_field, .comptime_alloc => unreachable,
5016	};
5017	}
5018
5019	/// Asserts that `isByRef` returns `false` for `val.typeOf(zcu)`.
5020	fn lowerConstant(cg: *CodeGen, val: Value) InnerError!WValue {
5021	const pt = cg.pt;
5022	const zcu = pt.zcu;
5023	const ty = val.typeOf(zcu);
5024	assert(!isByRef(ty, zcu, cg.target));
5025	const ip = &zcu.intern_pool;
5026	if (val.isUndef(zcu)) return cg.emitUndefined(ty);
5027
5028	switch (ip.indexToKey(val.ip_index)) {
5029	.int_type,
5030	.ptr_type,
5031	.array_type,
5032	.vector_type,
5033	.opt_type,
5034	.anyframe_type,
5035	.error_union_type,
5036	.simple_type,
5037	.struct_type,
5038	.tuple_type,
5039	.union_type,
5040	.opaque_type,
5041	.spirv_type,
5042	.enum_type,
5043	.func_type,
5044	.error_set_type,
5045	.inferred_error_set_type,
5046	=> unreachable, // types, not values
5047
5048	.undef => unreachable, // handled above
5049	.simple_value => |simple_value| switch (simple_value) {
5050	.void,
5051	.null,
5052	.@"unreachable",
5053	=> unreachable, // non-runtime values
5054	.false, .true => return .{ .imm32 = switch (simple_value) {
5055	.false => 0,
5056	.true => 1,
5057	else => unreachable,
5058	} },
5059	},
5060	.@"extern",
5061	.func,
5062	.enum_literal,
5063	=> unreachable, // non-runtime values
5064	.int => {
5065	const int_info = ty.intInfo(zcu);
5066	switch (int_info.signedness) {
5067	.signed => switch (int_info.bits) {
5068	0...32 => return .{ .imm32 = @bitCast(@as(i32, @intCast(val.toSignedInt(zcu)))) },
5069	33...64 => return .{ .imm64 = @bitCast(val.toSignedInt(zcu)) },
5070	else => unreachable,
5071	},
5072	.unsigned => switch (int_info.bits) {
5073	0...32 => return .{ .imm32 = @intCast(val.toUnsignedInt(zcu)) },
5074	33...64 => return .{ .imm64 = val.toUnsignedInt(zcu) },
5075	else => unreachable,
5076	},
5077	}
5078	},
5079	.err => |err| {
5080	const int = try pt.getErrorValue(err.name);
5081	return .{ .imm32 = int };
5082	},
5083	.error_union => |error_union| {
5084	const err_int_ty = try pt.errorIntType();
5085	const err_val: Value = switch (error_union.val) {
5086	.err_name => |err_name| .fromInterned(try pt.intern(.{ .err = .{
5087	.ty = ty.errorUnionSet(zcu).toIntern(),
5088	.name = err_name,
5089	} })),
5090	.payload => try pt.intValue(err_int_ty, 0),
5091	};
5092	const payload_type = ty.errorUnionPayload(zcu);
5093	if (!payload_type.hasRuntimeBits(zcu)) {
5094	// We use the error type directly as the type.
5095	return cg.lowerConstant(err_val);
5096	}
5097
5098	return cg.fail("Wasm TODO: lowerConstant error union with non-zero-bit payload type", .{});
5099	},
5100	.enum_tag => |enum_tag| return cg.lowerConstant(.fromInterned(enum_tag.int)),
5101	.float => |float| switch (float.storage) {
5102	.f16 => |f16_val| return .{ .imm32 = @as(u16, @bitCast(f16_val)) },
5103	.f32 => |f32_val| return .{ .float32 = f32_val },
5104	.f64 => |f64_val| return .{ .float64 = f64_val },
5105	else => unreachable,
5106	},
5107	.slice => unreachable, // isByRef == true
5108	.ptr => return cg.lowerPtr(val.toIntern(), 0),
5109	.opt => if (ty.optionalReprIsPayload(zcu)) {
5110	if (val.optionalValue(zcu)) |payload| {
5111	return cg.lowerConstant(payload);
5112	} else {
5113	return .{ .imm32 = 0 };
5114	}
5115	} else {
5116	return .{ .imm32 = @intFromBool(!val.isNull(zcu)) };
5117	},
5118	.aggregate => switch (ip.indexToKey(ty.ip_index)) {
5119	.array_type => return cg.fail("Wasm TODO: LowerConstant for {f}", .{ty.fmt(pt)}),
5120	.vector_type => {
5121	assert(determineSimdStoreStrategy(ty, zcu, cg.target) == .direct);
5122	var buf: [16]u8 = undefined;
5123	val.writeToMemory(zcu, &buf) catch unreachable;
5124	return cg.storeSimdImmd(buf);
5125	},
5126	.struct_type => unreachable, // packed structs use `bitpack`
5127	else => unreachable,
5128	},
5129	.un => unreachable, // packed unions use `bitpack`
5130	.bitpack => |bitpack| return cg.lowerConstant(.fromInterned(bitpack.backing_int_val)),
5131	.memoized_call => unreachable,
5132	}
5133	}
5134
5135	/// Stores the value as a 128bit-immediate value by storing it inside
5136	/// the list and returning the index into this list as `WValue`.
5137	fn storeSimdImmd(cg: *CodeGen, value: [16]u8) !WValue {
5138	const index = @as(u32, @intCast(cg.simd_immediates.items.len));
5139	try cg.simd_immediates.append(cg.gpa, value);
5140	return .{ .imm128 = index };
5141	}
5142
5143	fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue {
5144	const zcu = cg.pt.zcu;
5145	switch (ty.zigTypeTag(zcu)) {
5146	.bool, .error_set => return .{ .imm32 = 0xaaaaaaaa },
5147	.int, .@"enum" => switch (ty.intInfo(zcu).bits) {
5148	0...32 => return .{ .imm32 = 0xaaaaaaaa },
5149	33...64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa },
5150	else => unreachable,
5151	},
5152	.float => switch (ty.floatBits(cg.target)) {
5153	16 => return .{ .imm32 = 0xaaaaaaaa },
5154	32 => return .{ .float32 = @as(f32, @bitCast(@as(u32, 0xaaaaaaaa))) },
5155	64 => return .{ .float64 = @as(f64, @bitCast(@as(u64, 0xaaaaaaaaaaaaaaaa))) },
5156	else => unreachable,
5157	},
5158	.pointer => switch (cg.ptr_size) {
5159	.wasm32 => return .{ .imm32 = 0xaaaaaaaa },
5160	.wasm64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa },
5161	},
5162	.optional => {
5163	const pl_ty = ty.optionalChild(zcu);
5164	if (ty.optionalReprIsPayload(zcu)) {
5165	return cg.emitUndefined(pl_ty);
5166	}
5167	return .{ .imm32 = 0xaaaaaaaa };
5168	},
5169	.error_union => {
5170	return .{ .imm32 = 0xaaaaaaaa };
5171	},
5172	.@"struct", .@"union" => {
5173	const backing_int_ty = ty.backingIntType(zcu);
5174	return cg.emitUndefined(backing_int_ty);
5175	},
5176	else => return cg.fail("Wasm TODO: emitUndefined for type: {t}\n", .{ty.zigTypeTag(zcu)}),
5177	}
5178	}
5179
5180	fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5181	const block = cg.air.unwrapBlock(inst);
5182	try cg.lowerBlock(inst, block.ty, block.body);
5183	}
5184
5185	fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {
5186	const zcu = cg.pt.zcu;
5187	// if wasm_block_ty is non-empty, we create a register to store the temporary value
5188	const block_result: WValue = if (block_ty.hasRuntimeBits(zcu))
5189	try cg.allocLocal(block_ty)
5190	else
5191	.none;
5192
5193	try cg.startBlock(.block, .empty);
5194	// Here we set the current block idx, so breaks know the depth to jump
5195	// to when breaking out.
5196	try cg.blocks.putNoClobber(cg.gpa, inst, .{
5197	.label = cg.block_depth,
5198	.value = block_result,
5199	});
5200
5201	{
5202	try cg.branches.append(cg.gpa, .{});
5203	defer {
5204	var branch = cg.branches.pop().?;
5205	branch.deinit(cg.gpa);
5206	}
5207	try cg.genBody(body);
5208	try cg.endBlock();
5209	}
5210
5211	return cg.finishAir(inst, block_result, &.{});
5212	}
5213
5214	/// appends a new wasm block to the code section and increases the `block_depth` by 1
5215	fn startBlock(cg: *CodeGen, block_tag: std.wasm.Opcode, block_type: std.wasm.BlockType) !void {
5216	cg.block_depth += 1;
5217	try cg.addInst(.{
5218	.tag = Mir.Inst.Tag.fromOpcode(block_tag),
5219	.data = .{ .block_type = block_type },
5220	});
5221	}
5222
5223	/// Ends the current wasm block and decreases the `block_depth` by 1
5224	fn endBlock(cg: *CodeGen) !void {
5225	try cg.addTag(.end);
5226	cg.block_depth -= 1;
5227	}
5228
5229	fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5230	const block = cg.air.unwrapBlock(inst);
5231
5232	// result type of loop is always 'noreturn', meaning we can always
5233	// emit the wasm type 'block_empty'.
5234	try cg.startBlock(.loop, .empty);
5235
5236	try cg.loops.putNoClobber(cg.gpa, inst, cg.block_depth);
5237	defer assert(cg.loops.remove(inst));
5238
5239	try cg.genBody(block.body);
5240	try cg.endBlock();
5241
5242	return cg.finishAir(inst, .none, &.{});
5243	}
5244
5245	fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5246	const cond_br = cg.air.unwrapCondBr(inst);
5247	const condition = try cg.resolveInst(cond_br.condition);
5248	const then_body = cond_br.then_body;
5249	const else_body = cond_br.else_body;
5250
5251	// result type is always noreturn, so use `block_empty` as type.
5252	try cg.startBlock(.block, .empty);
5253	// emit the conditional value
5254	try cg.emitWValue(condition);
5255
5256	// we inserted the block in front of the condition
5257	// so now check if condition matches. If not, break outside this block
5258	// and continue with the then codepath
5259	try cg.addLabel(.br_if, 0);
5260
5261	try cg.branches.ensureUnusedCapacity(cg.gpa, 2);
5262	{
5263	cg.branches.appendAssumeCapacity(.{});
5264	defer {
5265	var else_stack = cg.branches.pop().?;
5266	else_stack.deinit(cg.gpa);
5267	}
5268	try cg.genBody(else_body);
5269	try cg.endBlock();
5270	}
5271
5272	// Outer block that matches the condition
5273	{
5274	cg.branches.appendAssumeCapacity(.{});
5275	defer {
5276	var then_stack = cg.branches.pop().?;
5277	then_stack.deinit(cg.gpa);
5278	}
5279	try cg.genBody(then_body);
5280	}
5281
5282	return cg.finishAir(inst, .none, &.{});
5283	}
5284
5285	fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) InnerError!void {
5286	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5287	const lhs = try cg.resolveInst(bin_op.lhs);
5288	const rhs = try cg.resolveInst(bin_op.rhs);
5289	const operand_ty = cg.typeOf(bin_op.lhs);
5290	const zcu = cg.pt.zcu;
5291
5292	const type_tag = operand_ty.zigTypeTag(zcu);
5293
5294	if (type_tag == .vector) {
5295	return cg.fail("TODO: implement AIR op: cmp for vectors", .{});
5296	}
5297
5298	if (type_tag == .optional and !operand_ty.optionalReprIsPayload(zcu)) {
5299	const payload_ty = operand_ty.optionalChild(zcu);
5300
5301	if (payload_ty.hasRuntimeBits(zcu)) {
5302	assert(op == .eq or op == .neq);
5303	assert(!isByRef(payload_ty, zcu, cg.target));
5304
5305	var result = try cg.allocLocal(Type.i32);
5306	defer result.free(cg);
5307
5308	var lhs_null = try cg.allocLocal(Type.i32);
5309	defer lhs_null.free(cg);
5310
5311	try cg.startBlock(.block, .empty);
5312
5313	try cg.addImm32(if (op == .eq) 0 else 1);
5314	try cg.addLocal(.local_set, result.local.value);
5315
5316	_ = try cg.isNull(lhs, operand_ty, .i32_eq, .value);
5317	try cg.addLocal(.local_tee, lhs_null.local.value);
5318	_ = try cg.isNull(rhs, operand_ty, .i32_eq, .value);
5319	try cg.addTag(.i32_ne);
5320	try cg.addLabel(.br_if, 0);
5321
5322	try cg.addImm32(if (op == .eq) 1 else 0);
5323	try cg.addLocal(.local_set, result.local.value);
5324
5325	try cg.addLocal(.local_get, lhs_null.local.value);
5326	try cg.addLabel(.br_if, 0);
5327
5328	_ = try cg.load(lhs, payload_ty, 0);
5329	_ = try cg.load(rhs, payload_ty, 0);
5330
5331	if (payload_ty.isAnyFloat()) {
5332	_ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack);
5333	} else {
5334	_ = try cg.intCmp(.fromType(cg, payload_ty), op, .stack, .stack);
5335	}
5336
5337	try cg.addLocal(.local_set, result.local.value);
5338	try cg.endBlock();
5339
5340	try cg.addLocal(.local_get, result.local.value);
5341	try cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
5342	} else {
5343	const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);
5344	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
5345	}
5346	} else if (type_tag == .float) {
5347	const result = try cg.floatCmp(.fromType(cg, operand_ty), op, lhs, rhs);
5348	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
5349	} else {
5350	const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);
5351	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
5352	}
5353	}
5354
5355	fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue, rhs: WValue) InnerError!WValue {
5356	switch (ty.bits) {
5357	0 => unreachable,
5358	1...32 => {
5359	// lhs or rhs could be stack pointers
5360	try cg.lowerToStack(lhs);
5361	try cg.lowerToStack(rhs);
5362	const opcode: Mir.Inst.Tag = switch (op) {
5363	.eq => .i32_eq,
5364	.neq => .i32_ne,
5365	.lt => if (ty.is_signed) .i32_lt_s else .i32_lt_u,
5366	.lte => if (ty.is_signed) .i32_le_s else .i32_le_u,
5367	.gte => if (ty.is_signed) .i32_ge_s else .i32_ge_u,
5368	.gt => if (ty.is_signed) .i32_gt_s else .i32_gt_u,
5369	};
5370	try cg.addTag(opcode);
5371	return .stack;
5372	},
5373	33...64 => {
5374	// lhs or rhs could be stack pointers
5375	try cg.lowerToStack(lhs);
5376	try cg.lowerToStack(rhs);
5377	const opcode: Mir.Inst.Tag = switch (op) {
5378	.eq => .i64_eq,
5379	.neq => .i64_ne,
5380	.lt => if (ty.is_signed) .i64_lt_s else .i64_lt_u,
5381	.lte => if (ty.is_signed) .i64_le_s else .i64_le_u,
5382	.gte => if (ty.is_signed) .i64_ge_s else .i64_ge_u,
5383	.gt => if (ty.is_signed) .i64_gt_s else .i64_gt_u,
5384	};
5385	try cg.addTag(opcode);
5386	return .stack;
5387	},
5388	65...128 => {
5389	var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);
5390	defer lhs_msb.free(cg);
5391	var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);
5392	defer rhs_msb.free(cg);
5393
5394	switch (op) {
5395	.eq, .neq => {
5396	const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb);
5397	const lhs_lsb = try cg.load(lhs, Type.u64, 0);
5398	const rhs_lsb = try cg.load(rhs, Type.u64, 0);
5399	const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb);
5400	const or_result = try cg.intOr(.u64, xor_high, xor_low);
5401
5402	switch (op) {
5403	.eq => return cg.intCmp(.u64, .eq, or_result, .{ .imm64 = 0 }),
5404	.neq => return cg.intCmp(.u64, .neq, or_result, .{ .imm64 = 0 }),
5405	else => unreachable,
5406	}
5407	},
5408	else => {
5409	const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64;
5410
5411	const lhs_lsb = try cg.load(lhs, Type.u64, 0);
5412	const rhs_lsb = try cg.load(rhs, Type.u64, 0);
5413
5414	// leave values on stack for 'select'
5415	_ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb);
5416	_ = try cg.intCmp(word_int_ty, op, lhs_msb, rhs_msb);
5417	_ = try cg.intCmp(word_int_ty, .eq, lhs_msb, rhs_msb);
5418	try cg.addTag(.select);
5419	},
5420	}
5421
5422	return .stack;
5423	},
5424	else => {
5425	try cg.lowerToStack(lhs);
5426	try cg.lowerToStack(rhs);
5427	try cg.addImm32(@intFromBool(ty.is_signed));
5428	try cg.addImm32(ty.bits);
5429	try cg.addCallIntrinsic(.__cmp_limb64);
5430	try cg.addImm32(0);
5431	try cg.addTag(switch (op) {
5432	.eq => .i32_eq,
5433	.neq => .i32_ne,
5434	.lt => .i32_lt_s,
5435	.lte => .i32_le_s,
5436	.gte => .i32_ge_s,
5437	.gt => .i32_gt_s,
5438	});
5439	return .stack;
5440	},
5441	}
5442	}
5443
5444	fn floatCmp(cg: *CodeGen, ty: FloatType, op: std.math.CompareOperator, lhs: WValue, rhs: WValue) InnerError!WValue {
5445	switch (ty) {
5446	.f16 => {
5447	_ = try cg.floatExtendCast(.f32, .f16, lhs);
5448	_ = try cg.floatExtendCast(.f32, .f16, rhs);
5449	try cg.addTag(switch (op) {
5450	.eq => .f32_eq,
5451	.neq => .f32_ne,
5452	.lt => .f32_lt,
5453	.lte => .f32_le,
5454	.gte => .f32_ge,
5455	.gt => .f32_gt,
5456	});
5457	return .stack;
5458	},
5459	.f32 => {
5460	try cg.emitWValue(lhs);
5461	try cg.emitWValue(rhs);
5462	try cg.addTag(switch (op) {
5463	.eq => .f32_eq,
5464	.neq => .f32_ne,
5465	.lt => .f32_lt,
5466	.lte => .f32_le,
5467	.gte => .f32_ge,
5468	.gt => .f32_gt,
5469	});
5470	return .stack;
5471	},
5472	.f64 => {
5473	try cg.emitWValue(lhs);
5474	try cg.emitWValue(rhs);
5475	try cg.addTag(switch (op) {
5476	.eq => .f64_eq,
5477	.neq => .f64_ne,
5478	.lt => .f64_lt,
5479	.lte => .f64_le,
5480	.gte => .f64_ge,
5481	.gt => .f64_gt,
5482	});
5483	return .stack;
5484	},
5485	.f80 => {
5486	const intrinsic: Mir.Intrinsic = switch (op) {
5487	.lt => .__ltxf2,
5488	.lte => .__lexf2,
5489	.eq => .__eqxf2,
5490	.neq => .__nexf2,
5491	.gte => .__gexf2,
5492	.gt => .__gtxf2,
5493	};
5494	const result = try cg.callIntrinsic(intrinsic, &.{ .f80_type, .f80_type }, Type.bool, &.{ lhs, rhs });
5495	return cg.intCmp(.i32, op, result, .{ .imm32 = 0 });
5496	},
5497	.f128 => {
5498	const intrinsic: Mir.Intrinsic = switch (op) {
5499	.lt => .__lttf2,
5500	.lte => .__letf2,
5501	.eq => .__eqtf2,
5502	.neq => .__netf2,
5503	.gte => .__getf2,
5504	.gt => .__gttf2,
5505	};
5506	const result = try cg.callIntrinsic(intrinsic, &.{ .f128_type, .f128_type }, Type.bool, &.{ lhs, rhs });
5507	return cg.intCmp(.i32, op, result, .{ .imm32 = 0 });
5508	},
5509	}
5510	}
5511
5512	fn airCmpVector(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5513	_ = inst;
5514	return cg.fail("TODO implement airCmpVector for wasm", .{});
5515	}
5516
5517	fn airCmpLteErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5518	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
5519	const operand = try cg.resolveInst(un_op);
5520
5521	try cg.emitWValue(operand);
5522	const pt = cg.pt;
5523	const err_int_ty = try pt.errorIntType();
5524	try cg.addTag(.errors_len);
5525	const result = try cg.intCmp(.fromType(cg, err_int_ty), .lt, .stack, .stack);
5526
5527	return cg.finishAir(inst, result, &.{un_op});
5528	}
5529
5530	fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5531	const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;
5532	const block = cg.blocks.get(br.block_inst).?;
5533
5534	// if operand has codegen bits we should break with a value
5535	if (block.value != .none) {
5536	const operand = try cg.resolveInst(br.operand);
5537	try cg.lowerToStack(operand);
5538	try cg.addLocal(.local_set, block.value.local.value);
5539	}
5540
5541	// We map every block to its block index.
5542	// We then determine how far we have to jump to it by subtracting it from current block depth
5543	const idx: u32 = cg.block_depth - block.label;
5544	try cg.addLabel(.br, idx);
5545
5546	return cg.finishAir(inst, .none, &.{br.operand});
5547	}
5548
5549	fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5550	const repeat = cg.air.instructions.items(.data)[@backingInt(inst)].repeat;
5551	const loop_label = cg.loops.get(repeat.loop_inst).?;
5552
5553	const idx: u32 = cg.block_depth - loop_label;
5554	try cg.addLabel(.br, idx);
5555
5556	return cg.finishAir(inst, .none, &.{});
5557	}
5558
5559	fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5560	try cg.addTag(.@"unreachable");
5561	return cg.finishAir(inst, .none, &.{});
5562	}
5563
5564	fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5565	// unsupported by wasm itfunc. Can be implemented once we support DWARF
5566	// for wasm
5567	try cg.addTag(.@"unreachable");
5568	return cg.finishAir(inst, .none, &.{});
5569	}
5570
5571	fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5572	try cg.addTag(.@"unreachable");
5573	return cg.finishAir(inst, .none, &.{});
5574	}
5575
5576	fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5577	const zcu = cg.pt.zcu;
5578	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5579
5580	const operand_ty = cg.typeOf(ty_op.operand);
5581	const dest_ty = cg.typeOfIndex(inst);
5582	assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
5583	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
5584	assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
5585
5586	const operand = try cg.resolveInst(ty_op.operand);
5587	const result = cg.reuseOperand(ty_op.operand, operand);
5588	return cg.finishAir(inst, result, &.{ty_op.operand});
5589	}
5590
5591	fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5592	const zcu = cg.pt.zcu;
5593	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5594
5595	const operand_ty = cg.typeOf(ty_op.operand);
5596	const dest_ty = cg.typeOfIndex(inst);
5597	assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
5598	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
5599	assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
5600
5601	const operand = try cg.resolveInst(ty_op.operand);
5602	const result = switch (operand) {
5603	.stack_offset => try cg.buildPointerOffset(operand, 0, .new),
5604	else => cg.reuseOperand(ty_op.operand, operand),
5605	};
5606	return cg.finishAir(inst, result, &.{ty_op.operand});
5607	}
5608
5609	fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5610	const zcu = cg.pt.zcu;
5611	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5612
5613	const union_ty = cg.typeOfIndex(inst);
5614	const enum_ty = cg.typeOf(ty_op.operand);
5615	const layout = union_ty.unionGetLayout(zcu);
5616
5617	const enum_value = try cg.resolveInst(ty_op.operand);
5618	const result = try cg.allocStack(union_ty);
5619	try cg.store(result, enum_value, enum_ty, @intCast(layout.tagOffset()));
5620
5621	return cg.finishAir(inst, result, &.{ty_op.operand});
5622	}
5623
5624	fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5625	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5626	const operand = try cg.resolveInst(ty_op.operand);
5627	const dest_ty = cg.typeOfIndex(inst);
5628	const src_ty = cg.typeOf(ty_op.operand);
5629
5630	const result = (try cg.bitcast(dest_ty, src_ty, operand)) orelse cg.reuseOperand(ty_op.operand, operand);
5631
5632	return cg.finishAir(inst, result, &.{ty_op.operand});
5633	}
5634
5635	const BitcastClass = union(enum) {
5636	int: IntType,
5637	float: FloatType,
5638	aggregate, // arrays and vectors.
5639	};
5640
5641	fn bitcastClass(cg: *CodeGen, ty: Type) BitcastClass {
5642	const zcu = cg.pt.zcu;
5643	return switch (ty.zigTypeTag(zcu)) {
5644	.bool,
5645	.int,
5646	.@"enum",
5647	.error_set,
5648	.@"struct",
5649	.@"union",
5650	=> .{ .int = .fromType(cg, ty) },
5651	.float => .{ .float = .fromType(cg, ty) },
5652	.array, .vector => .aggregate,
5653	else => unreachable,
5654	};
5655	}
5656
5657	fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerError!?WValue {
5658	if (dest_ty.eql(src_ty)) return null;
5659
5660	const zcu = cg.pt.zcu;
5661	const src_class = cg.bitcastClass(src_ty);
5662	const dest_class = cg.bitcastClass(dest_ty);
5663	const src_by_ref = isByRef(src_ty, zcu, cg.target);
5664	const dest_by_ref = isByRef(dest_ty, zcu, cg.target);
5665
5666	const needs_wrapping = switch (dest_class) {
5667	.int => |dest_int| dest_int.bits != cg.intBackingBits(dest_int.bits) and
5668	switch (src_class) {
5669	.int => |src_int| src_int.is_signed != dest_int.is_signed,
5670	.float, .aggregate => true,
5671	},
5672	.float, .aggregate => false,
5673	};
5674
5675	if (src_by_ref and dest_by_ref) {
5676	if (needs_wrapping) return try cg.intWrap(dest_class.int, operand);
5677	return null;
5678	}
5679
5680	if (dest_by_ref) {
5681	const result = try cg.allocStack(src_ty);
5682	try cg.store(result, operand, src_ty, 0);
5683	return result;
5684	}
5685
5686	if (src_by_ref) {
5687	return try cg.load(operand, dest_ty, 0);
5688	}
5689
5690	switch (src_class) {
5691	.float => |float_ty| switch (float_ty) {
5692	.f16 => return try cg.intWrap(dest_class.int, operand),
5693	.f32 => {
5694	try cg.emitWValue(operand);
5695	try cg.addTag(.i32_reinterpret_f32);
5696	return .stack;
5697	},
5698	.f64 => {
5699	try cg.emitWValue(operand);
5700	try cg.addTag(.i64_reinterpret_f64);
5701	return .stack;
5702	},
5703	.f80, .f128 => unreachable,
5704	},
5705	.int => {},
5706	.aggregate => unreachable,
5707	}
5708
5709	switch (dest_class) {
5710	.float => |float_ty| switch (float_ty) {
5711	.f16 => return null,
5712	.f32 => {
5713	try cg.emitWValue(operand);
5714	try cg.addTag(.f32_reinterpret_i32);
5715	return .stack;
5716	},
5717	.f64 => {
5718	try cg.emitWValue(operand);
5719	try cg.addTag(.f64_reinterpret_i64);
5720	return .stack;
5721	},
5722	.f80, .f128 => unreachable,
5723	},
5724	.int => {},
5725	.aggregate => unreachable,
5726	}
5727
5728	if (needs_wrapping) {
5729	return try cg.intWrap(dest_class.int, operand);
5730	}
5731
5732	return null;
5733	}
5734
5735	fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5736	const zcu = cg.pt.zcu;
5737	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5738	const extra = cg.air.extraData(Air.StructField, ty_pl.payload);
5739
5740	const struct_ptr = try cg.resolveInst(extra.data.struct_operand);
5741	const struct_ptr_ty = cg.typeOf(extra.data.struct_operand);
5742	const struct_ty = struct_ptr_ty.childType(zcu);
5743	const result = try cg.structFieldPtr(inst, extra.data.struct_operand, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index);
5744	return cg.finishAir(inst, result, &.{extra.data.struct_operand});
5745	}
5746
5747	fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerError!void {
5748	const zcu = cg.pt.zcu;
5749	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5750	const struct_ptr = try cg.resolveInst(ty_op.operand);
5751	const struct_ptr_ty = cg.typeOf(ty_op.operand);
5752	const struct_ty = struct_ptr_ty.childType(zcu);
5753
5754	const result = try cg.structFieldPtr(inst, ty_op.operand, struct_ptr, struct_ptr_ty, struct_ty, index);
5755	return cg.finishAir(inst, result, &.{ty_op.operand});
5756	}
5757
5758	fn structFieldPtr(
5759	cg: *CodeGen,
5760	inst: Air.Inst.Index,
5761	ref: Air.Inst.Ref,
5762	struct_ptr: WValue,
5763	struct_ptr_ty: Type,
5764	struct_ty: Type,
5765	index: u32,
5766	) InnerError!WValue {
5767	const pt = cg.pt;
5768	const zcu = pt.zcu;
5769	const result_ty = cg.typeOfIndex(inst);
5770	const struct_ptr_ty_info = struct_ptr_ty.ptrInfo(zcu);
5771
5772	const offset = switch (struct_ty.containerLayout(zcu)) {
5773	.@"packed" => switch (struct_ty.zigTypeTag(zcu)) {
5774	.@"struct" => offset: {
5775	if (result_ty.ptrInfo(zcu).packed_offset.host_size != 0) {
5776	break :offset @as(u32, 0);
5777	}
5778	const struct_type = zcu.typeToStruct(struct_ty).?;
5779	break :offset @divExact(zcu.structPackedFieldBitOffset(struct_type, index) + struct_ptr_ty_info.packed_offset.bit_offset, 8);
5780	},
5781	.@"union" => 0,
5782	else => unreachable,
5783	},
5784	else => struct_ty.structFieldOffset(index, zcu),
5785	};
5786	// save a load and store when we can simply reuse the operand
5787	if (offset == 0) {
5788	return cg.reuseOperand(ref, struct_ptr);
5789	}
5790	switch (struct_ptr) {
5791	.stack_offset => |stack_offset| {
5792	return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } };
5793	},
5794	else => return cg.buildPointerOffset(struct_ptr, offset, .new),
5795	}
5796	}
5797
5798	fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5799	const pt = cg.pt;
5800	const zcu = pt.zcu;
5801	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5802	const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data;
5803
5804	const struct_ty = cg.typeOf(struct_field.struct_operand);
5805	const operand = try cg.resolveInst(struct_field.struct_operand);
5806	const field_index = struct_field.field_index;
5807	const field_ty = struct_ty.fieldType(field_index, zcu);
5808	if (!field_ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{struct_field.struct_operand});
5809
5810	const result: WValue = switch (struct_ty.containerLayout(zcu)) {
5811	.@"packed" => unreachable, // legalize .expand_packed_agg_field_val
5812	else => result: {
5813	const offset = std.math.cast(u32, struct_ty.structFieldOffset(field_index, zcu)) orelse {
5814	return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});
5815	};
5816	if (isByRef(field_ty, zcu, cg.target)) {
5817	switch (operand) {
5818	.stack_offset => |stack_offset| {
5819	break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } };
5820	},
5821	else => break :result try cg.buildPointerOffset(operand, offset, .new),
5822	}
5823	}
5824	break :result try cg.load(operand, field_ty, offset);
5825	},
5826	};
5827
5828	return cg.finishAir(inst, result, &.{struct_field.struct_operand});
5829	}
5830
5831	fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void {
5832	const pt = cg.pt;
5833	const zcu = pt.zcu;
5834
5835	const switch_br = cg.air.unwrapSwitch(inst);
5836	const target_ty = cg.typeOf(switch_br.operand);
5837
5838	assert(target_ty.hasRuntimeBits(zcu));
5839
5840	// swap target value with placeholder local, for dispatching
5841	const target = if (is_dispatch_loop) target: {
5842	const initial_target = try cg.resolveInst(switch_br.operand);
5843	const target: WValue = try cg.allocLocal(target_ty);
5844	try cg.lowerToStack(initial_target);
5845	try cg.addLocal(.local_set, target.local.value);
5846
5847	try cg.startBlock(.loop, .empty); // dispatch loop start
5848	try cg.blocks.putNoClobber(cg.gpa, inst, .{
5849	.label = cg.block_depth,
5850	.value = target,
5851	});
5852
5853	break :target target;
5854	} else try cg.resolveInst(switch_br.operand);
5855
5856	const has_else_body = switch_br.else_body_len != 0;
5857	const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions
5858	try cg.branches.ensureUnusedCapacity(cg.gpa, switch_br.cases_len + @intFromBool(has_else_body));
5859
5860	if (switch_br.cases_len == 0) {
5861	assert(has_else_body);
5862
5863	var it = switch_br.iterateCases();
5864	const else_body = it.elseBody();
5865
5866	cg.branches.appendAssumeCapacity(.{});
5867	defer {
5868	var else_branch = cg.branches.pop().?;
5869	else_branch.deinit(cg.gpa);
5870	}
5871	try cg.genBody(else_body);
5872
5873	if (is_dispatch_loop) {
5874	try cg.endBlock(); // dispatch loop end
5875	}
5876	return cg.finishAir(inst, .none, &.{});
5877	}
5878
5879	var min: ?Value = null;
5880	var max: ?Value = null;
5881	var branching_size: u32 = 0; // single item +1, range +2
5882
5883	{
5884	var cases_it = switch_br.iterateCases();
5885	while (cases_it.next()) |case| {
5886	for (case.items) |item| {
5887	const val = Value.fromInterned(item.toInterned().?);
5888	if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val;
5889	if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val;
5890	branching_size += 1;
5891	}
5892	for (case.ranges) |range| {
5893	const low = Value.fromInterned(range[0].toInterned().?);
5894	if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low;
5895	const high = Value.fromInterned(range[1].toInterned().?);
5896	if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high;
5897	branching_size += 2;
5898	}
5899	}
5900	}
5901
5902	var min_space: Value.BigIntSpace = undefined;
5903	const min_bigint = min.?.toBigInt(&min_space, zcu);
5904	var max_space: Value.BigIntSpace = undefined;
5905	const max_bigint = max.?.toBigInt(&max_space, zcu);
5906	const limbs = try cg.gpa.alloc(
5907	std.math.big.Limb,
5908	@max(min_bigint.limbs.len, max_bigint.limbs.len) + 1,
5909	);
5910	defer cg.gpa.free(limbs);
5911
5912	const width_maybe: ?u32 = width: {
5913	var width_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
5914	width_bigint.sub(max_bigint, min_bigint);
5915	width_bigint.addScalar(width_bigint.toConst(), 1);
5916	break :width width_bigint.toConst().toInt(u32) catch null;
5917	};
5918
5919	try cg.startBlock(.block, .empty); // whole switch block start
5920
5921	for (0..branch_count) |_| {
5922	try cg.startBlock(.block, .empty);
5923	}
5924
5925	// Heuristic on deciding when to use .br_table instead of .br_if jump table
5926	// 1. Differences between lowest and highest values should fit into u32
5927	// 2. .br_table should be applied for "dense" switch, we test it by checking .br_if jumps will need more instructions
5928	// 3. Do not use .br_table for tiny switches
5929	const use_br_table = cond: {
5930	const width = width_maybe orelse break :cond false;
5931	if (width > 2 * branching_size) break :cond false;
5932	if (width < 2 or branch_count < 2) break :cond false;
5933	break :cond true;
5934	};
5935
5936	const int_ty: IntType = .fromType(cg, target_ty);
5937
5938	if (use_br_table) {
5939	const width = width_maybe.?;
5940
5941	const br_value_original = try cg.intSub(int_ty, target, try cg.resolveValue(min.?));
5942	_ = try cg.intCast(.u32, int_ty, br_value_original);
5943
5944	const jump_table: Mir.JumpTable = .{ .length = width + 1 };
5945	const table_extra_index = try cg.addExtra(jump_table);
5946	try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } });
5947
5948	const branch_list = try cg.mir_extra.addManyAsSlice(cg.gpa, width + 1);
5949	@memset(branch_list, branch_count - 1);
5950
5951	var cases_it = switch_br.iterateCases();
5952	while (cases_it.next()) |case| {
5953	for (case.items) |item| {
5954	const val = Value.fromInterned(item.toInterned().?);
5955	var val_space: Value.BigIntSpace = undefined;
5956	const val_bigint = val.toBigInt(&val_space, zcu);
5957	var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
5958	index_bigint.sub(val_bigint, min_bigint);
5959	branch_list[index_bigint.toConst().toInt(u32) catch unreachable] = case.idx;
5960	}
5961	for (case.ranges) |range| {
5962	var low_space: Value.BigIntSpace = undefined;
5963	const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu);
5964	var high_space: Value.BigIntSpace = undefined;
5965	const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu);
5966	var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
5967	index_bigint.sub(low_bigint, min_bigint);
5968	const start = index_bigint.toConst().toInt(u32) catch unreachable;
5969	index_bigint.sub(high_bigint, min_bigint);
5970	const end = (index_bigint.toConst().toInt(u32) catch unreachable) + 1;
5971	@memset(branch_list[start..end], case.idx);
5972	}
5973	}
5974	} else {
5975	var cases_it = switch_br.iterateCases();
5976	while (cases_it.next()) |case| {
5977	for (case.items) |ref| {
5978	const val = try cg.resolveInst(ref);
5979	_ = try cg.intCmp(int_ty, .eq, target, val);
5980	try cg.addLabel(.br_if, case.idx); // item match found
5981	}
5982	for (case.ranges) |range| {
5983	const low = try cg.resolveInst(range[0]);
5984	const high = try cg.resolveInst(range[1]);
5985
5986	const gte = try cg.intCmp(int_ty, .gte, target, low);
5987	const lte = try cg.intCmp(int_ty, .lte, target, high);
5988	_ = try cg.intAnd(.u32, gte, lte);
5989	try cg.addLabel(.br_if, case.idx); // range match found
5990	}
5991	}
5992	try cg.addLabel(.br, branch_count - 1);
5993	}
5994
5995	var cases_it = switch_br.iterateCases();
5996	while (cases_it.next()) |case| {
5997	try cg.endBlock();
5998
5999	cg.branches.appendAssumeCapacity(.{});
6000	defer {
6001	var case_branch = cg.branches.pop().?;
6002	case_branch.deinit(cg.gpa);
6003	}
6004	try cg.genBody(case.body);
6005
6006	try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch
6007	}
6008
6009	try cg.endBlock();
6010	if (has_else_body) {
6011	const else_body = cases_it.elseBody();
6012
6013	cg.branches.appendAssumeCapacity(.{});
6014	defer {
6015	var else_branch = cg.branches.pop().?;
6016	else_branch.deinit(cg.gpa);
6017	}
6018	try cg.genBody(else_body);
6019	} else {
6020	try cg.addTag(.@"unreachable");
6021	}
6022
6023	try cg.endBlock(); // whole switch block end
6024
6025	if (is_dispatch_loop) {
6026	try cg.endBlock(); // dispatch loop end
6027	}
6028
6029	return cg.finishAir(inst, .none, &.{});
6030	}
6031
6032	fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6033	const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;
6034	const switch_loop = cg.blocks.get(br.block_inst).?;
6035
6036	const operand = try cg.resolveInst(br.operand);
6037	try cg.lowerToStack(operand);
6038	try cg.addLocal(.local_set, switch_loop.value.local.value);
6039
6040	const idx: u32 = cg.block_depth - switch_loop.label;
6041	try cg.addLabel(.br, idx);
6042
6043	return cg.finishAir(inst, .none, &.{br.operand});
6044	}
6045
6046	fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void {
6047	const zcu = cg.pt.zcu;
6048	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
6049	const operand = try cg.resolveInst(un_op);
6050	const err_union_ty = switch (op_kind) {
6051	.value => cg.typeOf(un_op),
6052	.ptr => cg.typeOf(un_op).childType(zcu),
6053	};
6054	const pl_ty = err_union_ty.errorUnionPayload(zcu);
6055
6056	const result: WValue = result: {
6057	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
6058	switch (opcode) {
6059	.i32_ne => break :result .{ .imm32 = 0 },
6060	.i32_eq => break :result .{ .imm32 = 1 },
6061	else => unreachable,
6062	}
6063	}
6064
6065	try cg.emitWValue(operand);
6066	if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
6067	try cg.addMemArg(.i32_load16_u, .{
6068	.offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),
6069	.alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
6070	});
6071	}
6072
6073	// Compare the error value with '0'
6074	try cg.addImm32(0);
6075	try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
6076	break :result .stack;
6077	};
6078	return cg.finishAir(inst, result, &.{un_op});
6079	}
6080
6081	/// E!T -> T op_is_ptr == false
6082	/// *(E!T) -> *T op_is_prt == true
6083	fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void {
6084	const zcu = cg.pt.zcu;
6085	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6086
6087	const operand = try cg.resolveInst(ty_op.operand);
6088	const op_ty = cg.typeOf(ty_op.operand);
6089	const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty;
6090	const payload_ty = eu_ty.errorUnionPayload(zcu);
6091
6092	const result: WValue = result: {
6093	if (!payload_ty.hasRuntimeBits(zcu)) {
6094	if (op_is_ptr) {
6095	break :result cg.reuseOperand(ty_op.operand, operand);
6096	} else {
6097	break :result .none;
6098	}
6099	}
6100
6101	const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
6102	if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {
6103	break :result try cg.buildPointerOffset(operand, pl_offset, .new);
6104	} else {
6105	assert(isByRef(eu_ty, zcu, cg.target));
6106	break :result try cg.load(operand, payload_ty, pl_offset);
6107	}
6108	};
6109	return cg.finishAir(inst, result, &.{ty_op.operand});
6110	}
6111
6112	/// E!T -> E op_is_ptr == false
6113	/// *(E!T) -> E op_is_ptr == true
6114	/// NOTE: op_is_ptr will not change return type
6115	fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void {
6116	const zcu = cg.pt.zcu;
6117	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6118
6119	const operand = try cg.resolveInst(ty_op.operand);
6120	const op_ty = cg.typeOf(ty_op.operand);
6121	const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty;
6122	const payload_ty = eu_ty.errorUnionPayload(zcu);
6123
6124	const result: WValue = result: {
6125	if (eu_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
6126	break :result .{ .imm32 = 0 };
6127	}
6128
6129	const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
6130	if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {
6131	break :result try cg.load(operand, Type.anyerror, err_offset);
6132	} else {
6133	assert(!payload_ty.hasRuntimeBits(zcu));
6134	break :result cg.reuseOperand(ty_op.operand, operand);
6135	}
6136	};
6137	return cg.finishAir(inst, result, &.{ty_op.operand});
6138	}
6139
6140	fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6141	const zcu = cg.pt.zcu;
6142	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6143
6144	const operand = try cg.resolveInst(ty_op.operand);
6145	const err_ty = cg.typeOfIndex(inst);
6146
6147	const pl_ty = cg.typeOf(ty_op.operand);
6148	const result = result: {
6149	if (!pl_ty.hasRuntimeBits(zcu)) {
6150	break :result cg.reuseOperand(ty_op.operand, operand);
6151	}
6152
6153	const err_union = try cg.allocStack(err_ty);
6154	const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
6155	try cg.store(payload_ptr, operand, pl_ty, 0);
6156
6157	// ensure we also write '0' to the error part, so any present stack value gets overwritten by it.
6158	try cg.emitWValue(err_union);
6159	try cg.addImm32(0);
6160	const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
6161	try cg.addMemArg(.i32_store16, .{
6162	.offset = err_union.offset() + err_val_offset,
6163	.alignment = 2,
6164	});
6165	break :result err_union;
6166	};
6167	return cg.finishAir(inst, result, &.{ty_op.operand});
6168	}
6169
6170	fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6171	const zcu = cg.pt.zcu;
6172	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6173
6174	const operand = try cg.resolveInst(ty_op.operand);
6175	const err_ty = ty_op.ty;
6176	const pl_ty = err_ty.errorUnionPayload(zcu);
6177
6178	const result = result: {
6179	if (!pl_ty.hasRuntimeBits(zcu)) {
6180	break :result cg.reuseOperand(ty_op.operand, operand);
6181	}
6182
6183	const err_union = try cg.allocStack(err_ty);
6184	// store error value
6185	try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu)));
6186
6187	// write 'undefined' to the payload
6188	const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
6189	const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));
6190	try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });
6191
6192	break :result err_union;
6193	};
6194	return cg.finishAir(inst, result, &.{ty_op.operand});
6195	}
6196
6197	const OpKind = enum { value, ptr };
6198
6199	fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!void {
6200	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
6201	const operand = try cg.resolveInst(un_op);
6202
6203	const op_ty = cg.typeOf(un_op);
6204	const result = try cg.isNull(operand, op_ty, opcode, op_kind);
6205	return cg.finishAir(inst, result, &.{un_op});
6206	}
6207
6208	/// For a given type and operand, checks if it's considered `null`.
6209	/// NOTE: Leaves the result on the stack
6210	fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!WValue {
6211	const pt = cg.pt;
6212	const zcu = pt.zcu;
6213	try cg.emitWValue(operand);
6214	const optional_ty = switch (op_kind) {
6215	.value => op_ty,
6216	.ptr => op_ty.childType(zcu),
6217	};
6218	const payload_ty = optional_ty.optionalChild(zcu);
6219	if (!optional_ty.optionalReprIsPayload(zcu)) {
6220	// When payload is zero-bits, we can treat operand as a value, rather than
6221	// a pointer to the stack value
6222	if (payload_ty.hasRuntimeBits(zcu)) {
6223	const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6224	return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});
6225	};
6226	try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });
6227	}
6228	} else if (payload_ty.isSlice(zcu)) {
6229	switch (cg.ptr_size) {
6230	.wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }),
6231	.wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }),
6232	}
6233	} else {
6234	if (op_kind == .ptr) {
6235	try cg.addMemArg(.i32_load, .{
6236	.offset = operand.offset(),
6237	.alignment = 4,
6238	});
6239	}
6240	}
6241
6242	// Compare the null value with '0'
6243	try cg.addImm32(0);
6244	try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
6245
6246	return .stack;
6247	}
6248
6249	fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6250	const zcu = cg.pt.zcu;
6251	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6252	const opt_ty = cg.typeOf(ty_op.operand);
6253	const payload_ty = cg.typeOfIndex(inst);
6254	if (!payload_ty.hasRuntimeBits(zcu)) {
6255	return cg.finishAir(inst, .none, &.{ty_op.operand});
6256	}
6257
6258	const result = result: {
6259	const operand = try cg.resolveInst(ty_op.operand);
6260	if (opt_ty.optionalReprIsPayload(zcu)) break :result cg.reuseOperand(ty_op.operand, operand);
6261
6262	if (isByRef(payload_ty, zcu, cg.target)) {
6263	break :result try cg.buildPointerOffset(operand, 0, .new);
6264	}
6265
6266	break :result try cg.load(operand, payload_ty, 0);
6267	};
6268	return cg.finishAir(inst, result, &.{ty_op.operand});
6269	}
6270
6271	fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6272	const zcu = cg.pt.zcu;
6273	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6274	const operand = try cg.resolveInst(ty_op.operand);
6275	const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);
6276
6277	const result = result: {
6278	const payload_ty = opt_ty.optionalChild(zcu);
6279	if (!payload_ty.hasRuntimeBits(zcu) or opt_ty.optionalReprIsPayload(zcu)) {
6280	break :result cg.reuseOperand(ty_op.operand, operand);
6281	}
6282
6283	break :result try cg.buildPointerOffset(operand, 0, .new);
6284	};
6285	return cg.finishAir(inst, result, &.{ty_op.operand});
6286	}
6287
6288	fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6289	const pt = cg.pt;
6290	const zcu = pt.zcu;
6291	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6292	const operand = try cg.resolveInst(ty_op.operand);
6293	const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);
6294	const payload_ty = opt_ty.optionalChild(zcu);
6295
6296	if (opt_ty.optionalReprIsPayload(zcu)) {
6297	return cg.finishAir(inst, operand, &.{ty_op.operand});
6298	}
6299
6300	const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6301	return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});
6302	};
6303
6304	try cg.emitWValue(operand);
6305	try cg.addImm32(1);
6306	try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });
6307
6308	const result = try cg.buildPointerOffset(operand, 0, .new);
6309	return cg.finishAir(inst, result, &.{ty_op.operand});
6310	}
6311
6312	fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6313	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6314	const payload_ty = cg.typeOf(ty_op.operand);
6315	const pt = cg.pt;
6316	const zcu = pt.zcu;
6317
6318	const result = result: {
6319	if (!payload_ty.hasRuntimeBits(zcu)) {
6320	const non_null_bit = try cg.allocStack(Type.u1);
6321	try cg.emitWValue(non_null_bit);
6322	try cg.addImm32(1);
6323	try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 });
6324	break :result non_null_bit;
6325	}
6326
6327	const operand = try cg.resolveInst(ty_op.operand);
6328	const op_ty = cg.typeOfIndex(inst);
6329	if (op_ty.optionalReprIsPayload(zcu)) {
6330	break :result cg.reuseOperand(ty_op.operand, operand);
6331	}
6332	const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6333	return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)});
6334	};
6335
6336	// Create optional type, set the non-null bit, and store the operand inside the optional type
6337	const result_ptr = try cg.allocStack(op_ty);
6338	try cg.emitWValue(result_ptr);
6339	try cg.addImm32(1);
6340	try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });
6341
6342	const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);
6343	try cg.store(payload_ptr, operand, payload_ty, 0);
6344	break :result result_ptr;
6345	};
6346
6347	return cg.finishAir(inst, result, &.{ty_op.operand});
6348	}
6349
6350	fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6351	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6352	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6353
6354	const lhs = try cg.resolveInst(bin_op.lhs);
6355	const rhs = try cg.resolveInst(bin_op.rhs);
6356	const slice_ty = cg.typeOfIndex(inst);
6357
6358	const slice = try cg.allocStack(slice_ty);
6359	try cg.store(slice, lhs, Type.usize, 0);
6360	try cg.store(slice, rhs, Type.usize, cg.ptrSize());
6361
6362	return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });
6363	}
6364
6365	fn airSliceLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6366	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6367
6368	const operand = try cg.resolveInst(ty_op.operand);
6369	return cg.finishAir(inst, try cg.sliceLen(operand), &.{ty_op.operand});
6370	}
6371
6372	fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6373	const zcu = cg.pt.zcu;
6374	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6375
6376	const slice_ty = cg.typeOf(bin_op.lhs);
6377	const slice = try cg.resolveInst(bin_op.lhs);
6378	const index = try cg.resolveInst(bin_op.rhs);
6379	const elem_ty = slice_ty.childType(zcu);
6380	const elem_size = elem_ty.abiSize(zcu);
6381
6382	// load pointer onto stack
6383	_ = try cg.load(slice, Type.usize, 0);
6384
6385	// calculate index into slice
6386	try cg.emitWValue(index);
6387	try cg.addImm32(@intCast(elem_size));
6388	try cg.addTag(.i32_mul);
6389	try cg.addTag(.i32_add);
6390
6391	const elem_result = try cg.load(.stack, elem_ty, 0);
6392
6393	return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6394	}
6395
6396	fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6397	const zcu = cg.pt.zcu;
6398	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6399	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6400
6401	const elem_ty = ty_pl.ty.childType(zcu);
6402	const elem_size = elem_ty.abiSize(zcu);
6403
6404	const slice = try cg.resolveInst(bin_op.lhs);
6405	const index = try cg.resolveInst(bin_op.rhs);
6406
6407	_ = try cg.load(slice, Type.usize, 0);
6408
6409	// calculate index into slice
6410	try cg.emitWValue(index);
6411	try cg.addImm32(@intCast(elem_size));
6412	try cg.addTag(.i32_mul);
6413	try cg.addTag(.i32_add);
6414
6415	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6416	}
6417
6418	fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6419	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6420	const operand = try cg.resolveInst(ty_op.operand);
6421	return cg.finishAir(inst, try cg.slicePtr(operand), &.{ty_op.operand});
6422	}
6423
6424	fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {
6425	const ptr = try cg.load(operand, Type.usize, 0);
6426	return ptr.toLocal(cg, Type.usize);
6427	}
6428
6429	fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {
6430	const len = try cg.load(operand, Type.usize, cg.ptrSize());
6431	return len.toLocal(cg, Type.usize);
6432	}
6433
6434	fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6435	const zcu = cg.pt.zcu;
6436	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6437
6438	const operand = try cg.resolveInst(ty_op.operand);
6439	const array_ty = cg.typeOf(ty_op.operand).childType(zcu);
6440	const slice_ty = ty_op.ty;
6441
6442	// create a slice on the stack
6443	const slice_local = try cg.allocStack(slice_ty);
6444
6445	try cg.store(slice_local, operand, Type.usize, 0);
6446
6447	// store the length of the array in the slice
6448	const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
6449	try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize());
6450
6451	return cg.finishAir(inst, slice_local, &.{ty_op.operand});
6452	}
6453
6454	fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6455	const zcu = cg.pt.zcu;
6456	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6457
6458	const ptr_ty = cg.typeOf(bin_op.lhs);
6459	const ptr = try cg.resolveInst(bin_op.lhs);
6460	const index = try cg.resolveInst(bin_op.rhs);
6461	const elem_ty = ptr_ty.childType(zcu);
6462	const elem_size = elem_ty.abiSize(zcu);
6463
6464	// load pointer onto the stack
6465	if (ptr_ty.isSlice(zcu)) {
6466	_ = try cg.load(ptr, Type.usize, 0);
6467	} else {
6468	try cg.lowerToStack(ptr);
6469	}
6470
6471	// calculate index into slice
6472	try cg.emitWValue(index);
6473	try cg.addImm32(@intCast(elem_size));
6474	try cg.addTag(.i32_mul);
6475	try cg.addTag(.i32_add);
6476
6477	const elem_result = try cg.load(.stack, elem_ty, 0);
6478
6479	return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6480	}
6481
6482	fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6483	const zcu = cg.pt.zcu;
6484	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6485	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6486
6487	const ptr_ty = cg.typeOf(bin_op.lhs);
6488	const elem_ty = ty_pl.ty.childType(zcu);
6489	const elem_size = elem_ty.abiSize(zcu);
6490
6491	const ptr = try cg.resolveInst(bin_op.lhs);
6492	const index = try cg.resolveInst(bin_op.rhs);
6493
6494	// load pointer onto the stack
6495	if (ptr_ty.isSlice(zcu)) {
6496	_ = try cg.load(ptr, Type.usize, 0);
6497	} else {
6498	try cg.lowerToStack(ptr);
6499	}
6500
6501	// calculate index into ptr
6502	try cg.emitWValue(index);
6503	try cg.addImm32(@intCast(elem_size));
6504	try cg.addTag(.i32_mul);
6505	try cg.addTag(.i32_add);
6506
6507	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6508	}
6509
6510	fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerError!void {
6511	const zcu = cg.pt.zcu;
6512	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6513	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6514
6515	const ptr = try cg.resolveInst(bin_op.lhs);
6516	const offset = try cg.resolveInst(bin_op.rhs);
6517	const ptr_ty = cg.typeOf(bin_op.lhs);
6518	const pointee_ty = switch (ptr_ty.ptrSize(zcu)) {
6519	.one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type
6520	else => ptr_ty.childType(zcu),
6521	};
6522
6523	try cg.lowerToStack(ptr);
6524	try cg.emitWValue(offset);
6525
6526	switch (cg.ptr_size) {
6527	.wasm32 => {
6528	try cg.addImm32(@intCast(pointee_ty.abiSize(zcu)));
6529	try cg.addTag(.i32_mul);
6530	try cg.addTag(switch (op) {
6531	.add => .i32_add,
6532	.sub => .i32_sub,
6533	});
6534	},
6535	.wasm64 => {
6536	try cg.addImm64(pointee_ty.abiSize(zcu));
6537	try cg.addTag(.i64_mul);
6538	try cg.addTag(switch (op) {
6539	.add => .i64_add,
6540	.sub => .i64_sub,
6541	});
6542	},
6543	}
6544
6545	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6546	}
6547
6548	fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
6549	const zcu = cg.pt.zcu;
6550	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6551
6552	const ptr = try cg.resolveInst(bin_op.lhs);
6553	const ptr_ty = cg.typeOf(bin_op.lhs);
6554	const value = try cg.resolveInst(bin_op.rhs);
6555	const len = switch (ptr_ty.ptrSize(zcu)) {
6556	.slice => try cg.sliceLen(ptr),
6557	.one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }),
6558	.c, .many => unreachable,
6559	};
6560
6561	const elem_ty = if (ptr_ty.ptrSize(zcu) == .one)
6562	ptr_ty.childType(zcu).childType(zcu)
6563	else
6564	ptr_ty.childType(zcu);
6565
6566	if (!safety and bin_op.rhs == .undef) {
6567	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6568	}
6569
6570	const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty);
6571	try cg.memset(elem_ty, dst_ptr, len, value);
6572
6573	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6574	}
6575
6576	/// Sets a region of memory at `ptr` to the value of `value`
6577	/// When the user has enabled the bulk_memory feature, we lower
6578	/// this to wasm's memset instruction. When the feature is not present,
6579	/// we implement it manually.
6580	fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {
6581	const zcu = cg.pt.zcu;
6582	const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6583
6584	// When bulk_memory is enabled, we lower it to wasm's memset instruction.
6585	// If not, we lower it ourselves.
6586	if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) {
6587	const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);
6588
6589	if (!len0_ok) {
6590	try cg.startBlock(.block, .empty);
6591
6592	// Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is
6593	// out of memory bounds. This can easily happen in Zig in a case such as:
6594	//
6595	// const ptr: [*]u8 = undefined;
6596	// var len: usize = runtime_zero();
6597	// @memset(ptr[0..len], 42);
6598	//
6599	// So explicitly avoid using `memory.fill` in the `len == 0` case. Lovely design.
6600	try cg.emitWValue(len);
6601	try cg.addTag(.i32_eqz);
6602	try cg.addLabel(.br_if, 0);
6603	}
6604
6605	try cg.lowerToStack(ptr);
6606	try cg.emitWValue(value);
6607	try cg.emitWValue(len);
6608	try cg.addExtended(.memory_fill);
6609
6610	if (!len0_ok) {
6611	try cg.endBlock();
6612	}
6613
6614	return;
6615	}
6616
6617	const final_len: WValue = switch (len) {
6618	.imm32 => |val| .{ .imm32 = val * abi_size },
6619	.imm64 => |val| .{ .imm64 = val * abi_size },
6620	else => if (abi_size != 1) blk: {
6621	const new_len = try cg.ensureAllocLocal(Type.usize);
6622	try cg.emitWValue(len);
6623	switch (cg.ptr_size) {
6624	.wasm32 => {
6625	try cg.emitWValue(.{ .imm32 = abi_size });
6626	try cg.addTag(.i32_mul);
6627	},
6628	.wasm64 => {
6629	try cg.emitWValue(.{ .imm64 = abi_size });
6630	try cg.addTag(.i64_mul);
6631	},
6632	}
6633	try cg.addLocal(.local_set, new_len.local.value);
6634	break :blk new_len;
6635	} else len,
6636	};
6637
6638	var end_ptr = try cg.allocLocal(Type.usize);
6639	defer end_ptr.free(cg);
6640	var new_ptr = try cg.buildPointerOffset(ptr, 0, .new);
6641	defer new_ptr.free(cg);
6642
6643	// get the loop conditional: if current pointer address equals final pointer's address
6644	try cg.lowerToStack(ptr);
6645	try cg.emitWValue(final_len);
6646	switch (cg.ptr_size) {
6647	.wasm32 => try cg.addTag(.i32_add),
6648	.wasm64 => try cg.addTag(.i64_add),
6649	}
6650	try cg.addLocal(.local_set, end_ptr.local.value);
6651
6652	// outer block to jump to when loop is done
6653	try cg.startBlock(.block, .empty);
6654	try cg.startBlock(.loop, .empty);
6655
6656	// check for condition for loop end
6657	try cg.emitWValue(new_ptr);
6658	try cg.emitWValue(end_ptr);
6659	switch (cg.ptr_size) {
6660	.wasm32 => try cg.addTag(.i32_eq),
6661	.wasm64 => try cg.addTag(.i64_eq),
6662	}
6663	try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
6664
6665	// store the value at the current position of the pointer
6666	try cg.store(new_ptr, value, elem_ty, 0);
6667
6668	// move the pointer to the next element
6669	try cg.emitWValue(new_ptr);
6670	switch (cg.ptr_size) {
6671	.wasm32 => {
6672	try cg.emitWValue(.{ .imm32 = abi_size });
6673	try cg.addTag(.i32_add);
6674	},
6675	.wasm64 => {
6676	try cg.emitWValue(.{ .imm64 = abi_size });
6677	try cg.addTag(.i64_add);
6678	},
6679	}
6680	try cg.addLocal(.local_set, new_ptr.local.value);
6681
6682	// end of loop
6683	try cg.addLabel(.br, 0); // jump to start of loop
6684	try cg.endBlock();
6685	try cg.endBlock();
6686	}
6687
6688	fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6689	const zcu = cg.pt.zcu;
6690	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6691
6692	const array_ty = cg.typeOf(bin_op.lhs);
6693	const array = try cg.resolveInst(bin_op.lhs);
6694	const index = try cg.resolveInst(bin_op.rhs);
6695	const elem_ty = array_ty.childType(zcu);
6696	const elem_size = elem_ty.abiSize(zcu);
6697
6698	if (isByRef(array_ty, zcu, cg.target)) {
6699	try cg.lowerToStack(array);
6700	try cg.emitWValue(index);
6701	try cg.addImm32(@intCast(elem_size));
6702	try cg.addTag(.i32_mul);
6703	try cg.addTag(.i32_add);
6704	} else {
6705	assert(array_ty.zigTypeTag(zcu) == .vector);
6706
6707	switch (index) {
6708	inline .imm32, .imm64 => |lane| {
6709	const opcode: std.wasm.SimdOpcode = switch (elem_ty.bitSize(zcu)) {
6710	8 => if (elem_ty.isSignedInt(zcu)) .i8x16_extract_lane_s else .i8x16_extract_lane_u,
6711	16 => if (elem_ty.isSignedInt(zcu)) .i16x8_extract_lane_s else .i16x8_extract_lane_u,
6712	32 => if (elem_ty.isInt(zcu)) .i32x4_extract_lane else .f32x4_extract_lane,
6713	64 => if (elem_ty.isInt(zcu)) .i64x2_extract_lane else .f64x2_extract_lane,
6714	else => unreachable,
6715	};
6716
6717	var operands = [_]u32{ @backingInt(opcode), @as(u8, @intCast(lane)) };
6718
6719	try cg.emitWValue(array);
6720
6721	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6722	try cg.mir_extra.appendSlice(cg.gpa, &operands);
6723	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6724
6725	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6726	},
6727	else => {
6728	const stack_vec = try cg.allocStack(array_ty);
6729	try cg.store(stack_vec, array, array_ty, 0);
6730
6731	// Is a non-unrolled vector (v128)
6732	try cg.lowerToStack(stack_vec);
6733	try cg.emitWValue(index);
6734	try cg.addImm32(@intCast(elem_size));
6735	try cg.addTag(.i32_mul);
6736	try cg.addTag(.i32_add);
6737	},
6738	}
6739	}
6740
6741	const result = if (isByRef(elem_ty, zcu, cg.target))
6742	.stack
6743	else
6744	try cg.load(.stack, elem_ty, 0);
6745	return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6746	}
6747
6748	fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6749	const zcu = cg.pt.zcu;
6750	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6751	const operand = try cg.resolveInst(ty_op.operand);
6752	const ty = cg.typeOfIndex(inst);
6753	const elem_ty = ty.childType(zcu);
6754
6755	if (determineSimdStoreStrategy(ty, zcu, cg.target) == .direct) blk: {
6756	switch (operand) {
6757	// when the operand lives in the linear memory section, we can directly
6758	// load and splat the value at once. Meaning we do not first have to load
6759	// the scalar value onto the stack.
6760	.stack_offset, .nav_ref, .uav_ref => {
6761	const opcode = switch (elem_ty.bitSize(zcu)) {
6762	8 => @backingInt(std.wasm.SimdOpcode.v128_load8_splat),
6763	16 => @backingInt(std.wasm.SimdOpcode.v128_load16_splat),
6764	32 => @backingInt(std.wasm.SimdOpcode.v128_load32_splat),
6765	64 => @backingInt(std.wasm.SimdOpcode.v128_load64_splat),
6766	else => break :blk, // Cannot make use of simd-instructions
6767	};
6768	try cg.emitWValue(operand);
6769	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6770	// stores as := opcode, offset, alignment (opcode::memarg)
6771	try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
6772	opcode,
6773	operand.offset(),
6774	@intCast(elem_ty.abiAlignment(zcu).toByteUnits().?),
6775	});
6776	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6777	return cg.finishAir(inst, .stack, &.{ty_op.operand});
6778	},
6779	.local => {
6780	const opcode = switch (elem_ty.bitSize(zcu)) {
6781	8 => @backingInt(std.wasm.SimdOpcode.i8x16_splat),
6782	16 => @backingInt(std.wasm.SimdOpcode.i16x8_splat),
6783	32 => if (elem_ty.isInt(zcu)) @backingInt(std.wasm.SimdOpcode.i32x4_splat) else @backingInt(std.wasm.SimdOpcode.f32x4_splat),
6784	64 => if (elem_ty.isInt(zcu)) @backingInt(std.wasm.SimdOpcode.i64x2_splat) else @backingInt(std.wasm.SimdOpcode.f64x2_splat),
6785	else => break :blk, // Cannot make use of simd-instructions
6786	};
6787	try cg.emitWValue(operand);
6788	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6789	try cg.mir_extra.append(cg.gpa, opcode);
6790	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6791	return cg.finishAir(inst, .stack, &.{ty_op.operand});
6792	},
6793	else => unreachable,
6794	}
6795	}
6796
6797	const vector_len = @as(usize, @intCast(ty.vectorLen(zcu)));
6798	const result = try cg.allocStack(ty);
6799	const elem_byte_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6800	var index: usize = 0;
6801	var offset: u32 = 0;
6802	while (index < vector_len) : (index += 1) {
6803	try cg.store(result, operand, elem_ty, offset);
6804	offset += elem_byte_size;
6805	}
6806
6807	return cg.finishAir(inst, result, &.{ty_op.operand});
6808	}
6809
6810	fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6811	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
6812	const operand = try cg.resolveInst(pl_op.operand);
6813
6814	_ = operand;
6815	return cg.fail("TODO: Implement wasm airSelect", .{});
6816	}
6817
6818	fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6819	const pt = cg.pt;
6820	const zcu = pt.zcu;
6821
6822	const unwrapped = cg.air.unwrapShuffleOne(zcu, inst);
6823	const result_ty = unwrapped.result_ty;
6824	const mask = unwrapped.mask;
6825	const operand = try cg.resolveInst(unwrapped.operand);
6826
6827	const elem_ty = result_ty.childType(zcu);
6828	const elem_size = elem_ty.abiSize(zcu);
6829
6830	// TODO: this function could have an `i8x16_shuffle` fast path like `airShuffleTwo` if we were
6831	// to lower the comptime-known operands to a non-by-ref vector value.
6832
6833	// TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
6834	// I tried to fix it, but I couldn't make much sense of how this backend handles memory.
6835	if (!isByRef(result_ty, zcu, cg.target) or
6836	!isByRef(cg.typeOf(unwrapped.operand), zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{});
6837
6838	const dest_alloc = try cg.allocStack(result_ty);
6839	for (mask, 0..) |mask_elem, out_idx| {
6840	try cg.emitWValue(dest_alloc);
6841	const elem_val = switch (mask_elem.unwrap()) {
6842	.elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)),
6843	.value => |val| try cg.lowerConstant(.fromInterned(val)),
6844	};
6845	try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
6846	}
6847	return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
6848	}
6849
6850	fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6851	const pt = cg.pt;
6852	const zcu = pt.zcu;
6853
6854	const unwrapped = cg.air.unwrapShuffleTwo(zcu, inst);
6855	const result_ty = unwrapped.result_ty;
6856	const mask = unwrapped.mask;
6857	const operand_a = try cg.resolveInst(unwrapped.operand_a);
6858	const operand_b = try cg.resolveInst(unwrapped.operand_b);
6859
6860	const a_ty = cg.typeOf(unwrapped.operand_a);
6861	const b_ty = cg.typeOf(unwrapped.operand_b);
6862	const elem_ty = result_ty.childType(zcu);
6863	const elem_size = elem_ty.abiSize(zcu);
6864
6865	// WASM has `i8x16_shuffle`, which we can apply if the element type bit size is a multiple of 8
6866	// and the input and output vectors have a bit size of 128 (and are hence not by-ref). Otherwise,
6867	// we fall back to a naive loop lowering.
6868	if (!isByRef(a_ty, zcu, cg.target) and
6869	!isByRef(b_ty, zcu, cg.target) and
6870	!isByRef(result_ty, zcu, cg.target) and
6871	elem_ty.bitSize(zcu) % 8 == 0)
6872	{
6873	var lane_map: [16]u8 align(4) = undefined;
6874	const lanes_per_elem: usize = @intCast(elem_ty.bitSize(zcu) / 8);
6875	for (mask, 0..) |mask_elem, out_idx| {
6876	const out_first_lane = out_idx * lanes_per_elem;
6877	const in_first_lane = switch (mask_elem.unwrap()) {
6878	.a_elem => |i| i * lanes_per_elem,
6879	.b_elem => |i| i * lanes_per_elem + 16,
6880	.undef => 0, // doesn't matter
6881	};
6882	for (lane_map[out_first_lane..][0..lanes_per_elem], in_first_lane..) |*out, in| {
6883	out.* = @intCast(in);
6884	}
6885	}
6886	try cg.emitWValue(operand_a);
6887	try cg.emitWValue(operand_b);
6888	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6889	try cg.mir_extra.appendSlice(cg.gpa, &.{
6890	@backingInt(std.wasm.SimdOpcode.i8x16_shuffle),
6891	@bitCast(lane_map[0..4].*),
6892	@bitCast(lane_map[4..8].*),
6893	@bitCast(lane_map[8..12].*),
6894	@bitCast(lane_map[12..].*),
6895	});
6896	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6897	return cg.finishAir(inst, .stack, &.{ unwrapped.operand_a, unwrapped.operand_b });
6898	}
6899
6900	// TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
6901	// I tried to fix it, but I couldn't make much sense of how this backend handles memory.
6902	if (!isByRef(result_ty, zcu, cg.target) or
6903	!isByRef(a_ty, zcu, cg.target) or
6904	!isByRef(b_ty, zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{});
6905
6906	const dest_alloc = try cg.allocStack(result_ty);
6907	for (mask, 0..) |mask_elem, out_idx| {
6908	try cg.emitWValue(dest_alloc);
6909	const elem_val = switch (mask_elem.unwrap()) {
6910	.a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)),
6911	.b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)),
6912	.undef => try cg.emitUndefined(elem_ty),
6913	};
6914	try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
6915	}
6916	return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
6917	}
6918
6919	fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6920	const reduce = cg.air.instructions.items(.data)[@backingInt(inst)].reduce;
6921	const operand = try cg.resolveInst(reduce.operand);
6922
6923	_ = operand;
6924	return cg.fail("TODO: Implement wasm airReduce", .{});
6925	}
6926
6927	fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6928	const pt = cg.pt;
6929	const zcu = pt.zcu;
6930	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6931	const result_ty = cg.typeOfIndex(inst);
6932	const len = @as(usize, @intCast(result_ty.arrayLen(zcu)));
6933	const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]);
6934
6935	const result: WValue = result_value: {
6936	switch (result_ty.zigTypeTag(zcu)) {
6937	.array, .vector => {
6938	const result = try cg.allocStack(result_ty);
6939	const elem_ty = result_ty.childType(zcu);
6940	const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6941	const sentinel = result_ty.sentinel(zcu);
6942
6943	// When the element type is by reference, we must copy the entire
6944	// value. It is therefore safer to move the offset pointer and store
6945	// each value individually, instead of using store offsets.
6946	if (isByRef(elem_ty, zcu, cg.target)) {
6947	// copy stack pointer into a temporary local, which is
6948	// moved for each element to store each value in the right position.
6949	const offset = try cg.buildPointerOffset(result, 0, .new);
6950	for (elements, 0..) |elem, elem_index| {
6951	const elem_val = try cg.resolveInst(elem);
6952	try cg.store(offset, elem_val, elem_ty, 0);
6953
6954	if (elem_index < elements.len - 1 or sentinel != null) {
6955	_ = try cg.buildPointerOffset(offset, elem_size, .modify);
6956	}
6957	}
6958	if (sentinel) |s| {
6959	const val = try cg.resolveValue(s);
6960	try cg.store(offset, val, elem_ty, 0);
6961	}
6962	} else {
6963	var offset: u32 = 0;
6964	for (elements) |elem| {
6965	const elem_val = try cg.resolveInst(elem);
6966	try cg.store(result, elem_val, elem_ty, offset);
6967	offset += elem_size;
6968	}
6969	if (sentinel) |s| {
6970	const val = try cg.resolveValue(s);
6971	try cg.store(result, val, elem_ty, offset);
6972	}
6973	}
6974	break :result_value result;
6975	},
6976	.@"struct" => switch (result_ty.containerLayout(zcu)) {
6977	.@"packed" => unreachable, // legalize .expand_packed_aggregate_init
6978	else => {
6979	const result = try cg.allocStack(result_ty);
6980	for (elements, 0..) |elem, elem_index| {
6981	if (try result_ty.structFieldValueComptime(pt, elem_index) != null) continue;
6982
6983	const elem_ty = result_ty.fieldType(elem_index, zcu);
6984	const field_offset = result_ty.structFieldOffset(elem_index, zcu);
6985	const offset = try cg.buildPointerOffset(result, field_offset, .new);
6986
6987	const value = try cg.resolveInst(elem);
6988	try cg.store(offset, value, elem_ty, 0);
6989	}
6990
6991	break :result_value result;
6992	},
6993	},
6994	else => unreachable,
6995	}
6996	};
6997
6998	var bt = cg.liveness.iterateBigTomb(inst);
6999	for (elements) |arg| cg.feed(&bt, arg);
7000	return cg.finishAirResult(inst, result);
7001	}
7002
7003	fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7004	const pt = cg.pt;
7005	const zcu = pt.zcu;
7006	const ip = &zcu.intern_pool;
7007	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7008	const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;
7009
7010	const result = result: {
7011	const union_ty = cg.typeOfIndex(inst);
7012	const layout = union_ty.unionGetLayout(zcu);
7013	const union_obj = zcu.typeToUnion(union_ty).?;
7014	const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
7015	const field_name = ip.loadEnumType(union_obj.enum_tag_type).field_names.get(ip)[extra.field_index];
7016
7017	const tag_int = blk: {
7018	const tag_ty = union_ty.unionTagTypeHypothetical(zcu);
7019	const enum_field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
7020	const tag_val = try pt.enumValueFieldIndex(tag_ty, enum_field_index);
7021	break :blk try cg.lowerConstant(tag_val);
7022	};
7023	if (layout.payload_size == 0) {
7024	if (layout.tag_size == 0) {
7025	break :result .none;
7026	}
7027	assert(!isByRef(union_ty, zcu, cg.target));
7028	break :result tag_int;
7029	}
7030
7031	if (isByRef(union_ty, zcu, cg.target)) {
7032	const result_ptr = try cg.allocStack(union_ty);
7033	const payload = try cg.resolveInst(extra.init);
7034	if (layout.tag_align.compare(.gte, layout.payload_align)) {
7035	if (isByRef(field_ty, zcu, cg.target)) {
7036	const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);
7037	try cg.store(payload_ptr, payload, field_ty, 0);
7038	} else {
7039	try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size));
7040	}
7041
7042	if (layout.tag_size > 0) {
7043	try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), 0);
7044	}
7045	} else {
7046	try cg.store(result_ptr, payload, field_ty, 0);
7047	if (layout.tag_size > 0) {
7048	try cg.store(
7049	result_ptr,
7050	tag_int,
7051	.fromInterned(union_obj.enum_tag_type),
7052	@intCast(layout.payload_size),
7053	);
7054	}
7055	}
7056	break :result result_ptr;
7057	} else {
7058	unreachable;
7059	}
7060	};
7061
7062	return cg.finishAir(inst, result, &.{extra.init});
7063	}
7064
7065	fn airPrefetch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7066	const prefetch = cg.air.instructions.items(.data)[@backingInt(inst)].prefetch;
7067	return cg.finishAir(inst, .none, &.{prefetch.ptr});
7068	}
7069
7070	fn airWasmMemorySize(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7071	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
7072
7073	try cg.addLabel(.memory_size, pl_op.payload);
7074	return cg.finishAir(inst, .stack, &.{pl_op.operand});
7075	}
7076
7077	fn airWasmMemoryGrow(cg: *CodeGen, inst: Air.Inst.Index) !void {
7078	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
7079
7080	const operand = try cg.resolveInst(pl_op.operand);
7081	try cg.emitWValue(operand);
7082	try cg.addLabel(.memory_grow, pl_op.payload);
7083	return cg.finishAir(inst, .stack, &.{pl_op.operand});
7084	}
7085
7086	fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7087	const pt = cg.pt;
7088	const zcu = pt.zcu;
7089	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7090	const un_ty = cg.typeOf(bin_op.lhs).childType(zcu);
7091	const tag_ty = cg.typeOf(bin_op.rhs);
7092	const layout = un_ty.unionGetLayout(zcu);
7093	if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7094
7095	const union_ptr = try cg.resolveInst(bin_op.lhs);
7096	const new_tag = try cg.resolveInst(bin_op.rhs);
7097	if (layout.payload_size == 0) {
7098	try cg.store(union_ptr, new_tag, tag_ty, 0);
7099	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7100	}
7101
7102	// when the tag alignment is smaller than the payload, the field will be stored
7103	// after the payload.
7104	const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {
7105	break :blk @intCast(layout.payload_size);
7106	} else 0;
7107	try cg.store(union_ptr, new_tag, tag_ty, offset);
7108	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7109	}
7110
7111	fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7112	const zcu = cg.pt.zcu;
7113	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7114
7115	const un_ty = cg.typeOf(ty_op.operand);
7116	const tag_ty = cg.typeOfIndex(inst);
7117	const layout = un_ty.unionGetLayout(zcu);
7118	if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ty_op.operand});
7119
7120	const operand = try cg.resolveInst(ty_op.operand);
7121	// when the tag alignment is smaller than the payload, the field will be stored
7122	// after the payload.
7123	const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align))
7124	@intCast(layout.payload_size)
7125	else
7126	0;
7127	const result = try cg.load(operand, tag_ty, offset);
7128	return cg.finishAir(inst, result, &.{ty_op.operand});
7129	}
7130
7131	fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7132	const zcu = cg.pt.zcu;
7133	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7134
7135	const err_set_ty = cg.typeOf(ty_op.operand).childType(zcu);
7136	const payload_ty = err_set_ty.errorUnionPayload(zcu);
7137	const operand = try cg.resolveInst(ty_op.operand);
7138
7139	// set error-tag to '0' to annotate error union is non-error
7140	try cg.store(
7141	operand,
7142	.{ .imm32 = 0 },
7143	Type.anyerror,
7144	@intCast(errUnionErrorOffset(payload_ty, zcu)),
7145	);
7146
7147	const result = result: {
7148	if (!payload_ty.hasRuntimeBits(zcu)) {
7149	break :result cg.reuseOperand(ty_op.operand, operand);
7150	}
7151
7152	break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new);
7153	};
7154	return cg.finishAir(inst, result, &.{ty_op.operand});
7155	}
7156
7157	fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7158	const pt = cg.pt;
7159	const zcu = pt.zcu;
7160	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7161	const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
7162
7163	const field_ptr = try cg.resolveInst(extra.field_ptr);
7164	const parent_ptr_ty = cg.typeOfIndex(inst);
7165	const parent_ty = parent_ptr_ty.childType(zcu);
7166	const field_ptr_ty = cg.typeOf(extra.field_ptr);
7167	const field_index = extra.field_index;
7168	const field_offset = switch (parent_ty.containerLayout(zcu)) {
7169	.auto, .@"extern" => parent_ty.structFieldOffset(field_index, zcu),
7170	.@"packed" => offset: {
7171	const parent_ptr_offset = parent_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset;
7172	const field_offset = if (zcu.typeToStruct(parent_ty)) |loaded_struct| zcu.structPackedFieldBitOffset(loaded_struct, field_index) else 0;
7173	const field_ptr_offset = field_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset;
7174	break :offset @divExact(parent_ptr_offset + field_offset - field_ptr_offset, 8);
7175	},
7176	};
7177
7178	const result = if (field_offset != 0) result: {
7179	const base = try cg.buildPointerOffset(field_ptr, 0, .new);
7180	try cg.addLocal(.local_get, base.local.value);
7181	try cg.addImm32(@intCast(field_offset));
7182	try cg.addTag(.i32_sub);
7183	try cg.addLocal(.local_set, base.local.value);
7184	break :result base;
7185	} else cg.reuseOperand(extra.field_ptr, field_ptr);
7186
7187	return cg.finishAir(inst, result, &.{extra.field_ptr});
7188	}
7189
7190	fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {
7191	const zcu = cg.pt.zcu;
7192	if (ptr_ty.isSlice(zcu)) {
7193	return cg.slicePtr(ptr);
7194	} else {
7195	return ptr;
7196	}
7197	}
7198
7199	fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7200	const zcu = cg.pt.zcu;
7201	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7202	const dst = try cg.resolveInst(bin_op.lhs);
7203	const dst_ty = cg.typeOf(bin_op.lhs);
7204	const ptr_elem_ty = dst_ty.childType(zcu);
7205	const src = try cg.resolveInst(bin_op.rhs);
7206	const src_ty = cg.typeOf(bin_op.rhs);
7207	const len = switch (dst_ty.ptrSize(zcu)) {
7208	.slice => blk: {
7209	const slice_len = try cg.sliceLen(dst);
7210	if (ptr_elem_ty.abiSize(zcu) != 1) {
7211	try cg.emitWValue(slice_len);
7212	try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
7213	try cg.addTag(.i32_mul);
7214	try cg.addLocal(.local_set, slice_len.local.value);
7215	}
7216	break :blk slice_len;
7217	},
7218	.one => @as(WValue, .{
7219	.imm32 = @as(u32, @intCast(ptr_elem_ty.arrayLen(zcu) * ptr_elem_ty.childType(zcu).abiSize(zcu))),
7220	}),
7221	.c, .many => unreachable,
7222	};
7223	const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
7224	const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
7225	try cg.memcpy(dst_ptr, src_ptr, len);
7226
7227	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7228	}
7229
7230	fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7231	const zcu = cg.pt.zcu;
7232	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7233	const dst = try cg.resolveInst(bin_op.lhs);
7234	const dst_ty = cg.typeOf(bin_op.lhs);
7235	const ptr_elem_ty = dst_ty.childType(zcu);
7236	const src = try cg.resolveInst(bin_op.rhs);
7237	const src_ty = cg.typeOf(bin_op.rhs);
7238	const len = switch (dst_ty.ptrSize(zcu)) {
7239	.slice => blk: {
7240	const slice_len = try cg.sliceLen(dst);
7241	if (ptr_elem_ty.abiSize(zcu) != 1) {
7242	try cg.emitWValue(slice_len);
7243	try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
7244	try cg.addTag(.i32_mul);
7245	try cg.addLocal(.local_set, slice_len.local.value);
7246	}
7247	break :blk slice_len;
7248	},
7249	.one => @as(WValue, .{
7250	.imm32 = @as(u32, @intCast(ptr_elem_ty.arrayLen(zcu) * ptr_elem_ty.childType(zcu).abiSize(zcu))),
7251	}),
7252	.c, .many => unreachable,
7253	};
7254	const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
7255	const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
7256	try cg.memmove(dst_ptr, src_ptr, len);
7257
7258	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7259	}
7260
7261	fn airRetAddr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7262	// TODO: Implement this properly once stack serialization is solved
7263	return cg.finishAir(inst, switch (cg.ptr_size) {
7264	.wasm32 => .{ .imm32 = 0 },
7265	.wasm64 => .{ .imm64 = 0 },
7266	}, &.{});
7267	}
7268
7269	fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7270	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7271	const operand = try cg.resolveInst(un_op);
7272	// Each entry to this table is a slice (ptr+len).
7273	// The operand in this instruction represents the index within this table.
7274	// This means to get the final name, we emit the base pointer and then perform
7275	// pointer arithmetic to find the pointer to this slice and return that.
7276	//
7277	// As the names are global and the slice elements are constant, we do not have
7278	// to make a copy of the ptr+value but can point towards them directly.
7279	const pt = cg.pt;
7280	const name_ty = Type.slice_const_u8_sentinel_0;
7281	const abi_size = name_ty.abiSize(pt.zcu);
7282
7283	// Lowers to a i32.const or i64.const with the error table memory address.
7284	cg.error_name_table_ref_count += 1;
7285	try cg.addTag(.error_name_table_ref);
7286	try cg.emitWValue(operand);
7287	switch (cg.ptr_size) {
7288	.wasm32 => {
7289	try cg.addImm32(@intCast(abi_size));
7290	try cg.addTag(.i32_mul);
7291	try cg.addTag(.i32_add);
7292	},
7293	.wasm64 => {
7294	try cg.addImm64(abi_size);
7295	try cg.addTag(.i64_mul);
7296	try cg.addTag(.i64_add);
7297	},
7298	}
7299
7300	return cg.finishAir(inst, .stack, &.{un_op});
7301	}
7302
7303	fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {
7304	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7305	const slice_ptr = try cg.resolveInst(ty_op.operand);
7306	const result = try cg.buildPointerOffset(slice_ptr, offset, .new);
7307	return cg.finishAir(inst, result, &.{ty_op.operand});
7308	}
7309
7310	fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7311	const dbg_stmt = cg.air.instructions.items(.data)[@backingInt(inst)].dbg_stmt;
7312	try cg.addInst(.{ .tag = .dbg_line, .data = .{
7313	.payload = try cg.addExtra(Mir.DbgLineColumn{
7314	.line = dbg_stmt.line,
7315	.column = dbg_stmt.column,
7316	}),
7317	} });
7318	return cg.finishAir(inst, .none, &.{});
7319	}
7320
7321	fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7322	const block = cg.air.unwrapDbgBlock(inst);
7323	// TODO
7324	try cg.lowerBlock(inst, block.ty, block.body);
7325	}
7326
7327	fn airDbgVar(
7328	cg: *CodeGen,
7329	inst: Air.Inst.Index,
7330	local_tag: link.File.Dwarf.WipNav.LocalVarTag,
7331	is_ptr: bool,
7332	) InnerError!void {
7333	_ = is_ptr;
7334	_ = local_tag;
7335	return cg.finishAir(inst, .none, &.{});
7336	}
7337
7338	fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7339	const unwrapped_try = cg.air.unwrapTry(inst);
7340	const body = unwrapped_try.else_body;
7341	const err_union = try cg.resolveInst(unwrapped_try.error_union);
7342	const err_union_ty = cg.typeOf(unwrapped_try.error_union);
7343	const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);
7344	return cg.finishAir(inst, result, &.{unwrapped_try.error_union});
7345	}
7346
7347	fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7348	const zcu = cg.pt.zcu;
7349	const unwrapped_try = cg.air.unwrapTryPtr(inst);
7350	const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr);
7351	const body = unwrapped_try.else_body;
7352	const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu);
7353	const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);
7354	return cg.finishAir(inst, result, &.{unwrapped_try.error_union_ptr});
7355	}
7356
7357	fn lowerTry(
7358	cg: *CodeGen,
7359	inst: Air.Inst.Index,
7360	err_union: WValue,
7361	body: []const Air.Inst.Index,
7362	err_union_ty: Type,
7363	operand_is_ptr: bool,
7364	) InnerError!WValue {
7365	_ = inst;
7366	const zcu = cg.pt.zcu;
7367
7368	const pl_ty = err_union_ty.errorUnionPayload(zcu);
7369	const pl_has_bits = pl_ty.hasRuntimeBits(zcu);
7370
7371	if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7372	// Block we can jump out of when error is not set
7373	try cg.startBlock(.block, .empty);
7374
7375	// check if the error tag is set for the error union.
7376	try cg.emitWValue(err_union);
7377	if (pl_has_bits or operand_is_ptr) {
7378	const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
7379	try cg.addMemArg(.i32_load16_u, .{
7380	.offset = err_union.offset() + err_offset,
7381	.alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
7382	});
7383	}
7384	try cg.addTag(.i32_eqz);
7385	try cg.addLabel(.br_if, 0); // jump out of block when error is '0'
7386
7387	try cg.branches.append(cg.gpa, .{});
7388	defer {
7389	var branch = cg.branches.pop().?;
7390	branch.deinit(cg.gpa);
7391	}
7392	try cg.genBody(body);
7393	try cg.endBlock();
7394	}
7395
7396	// if we reach here it means error was not set, and we want the payload
7397	if (!pl_has_bits and !operand_is_ptr) {
7398	return .none;
7399	}
7400
7401	const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
7402	if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {
7403	return buildPointerOffset(cg, err_union, pl_offset, .new);
7404	}
7405	const payload = try cg.load(err_union, pl_ty, pl_offset);
7406	return payload.toLocal(cg, pl_ty);
7407	}
7408
7409	/// Calls a compiler-rt intrinsic by creating an undefined symbol,
7410	/// then lowering the arguments and calling the symbol as a function call.
7411	/// This function call assumes the C-ABI.
7412	/// Asserts arguments are not stack values when the return value is
7413	/// passed as the first parameter.
7414	/// May leave the return value on the stack.
7415	fn callIntrinsic(
7416	cg: *CodeGen,
7417	intrinsic: Mir.Intrinsic,
7418	param_types: []const InternPool.Index,
7419	return_type: Type,
7420	args: []const WValue,
7421	) InnerError!WValue {
7422	assert(param_types.len == args.len);
7423	const zcu = cg.pt.zcu;
7424
7425	// Always pass over C-ABI
7426
7427	const want_sret_param = firstParamSRet(.{ .wasm_mvp = .{} }, return_type, zcu, cg.target);
7428	// if we want return as first param, we allocate a pointer to stack,
7429	// and emit it as our first argument
7430	const sret = if (want_sret_param) blk: {
7431	const sret_local = try cg.allocStack(return_type);
7432	try cg.lowerToStack(sret_local);
7433	break :blk sret_local;
7434	} else .none;
7435
7436	// Lower all arguments to the stack before we call our function
7437	for (args, 0..) |arg, arg_i| {
7438	assert(!(want_sret_param and arg == .stack));
7439	assert(Type.fromInterned(param_types[arg_i]).hasRuntimeBits(zcu));
7440	try cg.lowerArg(.{ .wasm_mvp = .{} }, Type.fromInterned(param_types[arg_i]), arg);
7441	}
7442
7443	try cg.addInst(.{ .tag = .call_intrinsic, .data = .{ .intrinsic = intrinsic } });
7444
7445	if (!return_type.hasRuntimeBits(zcu)) {
7446	return .none;
7447	} else if (want_sret_param) {
7448	return sret;
7449	} else {
7450	return .stack;
7451	}
7452	}
7453
7454	fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7455	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7456	const operand = try cg.resolveInst(un_op);
7457	const enum_ty = cg.typeOf(un_op);
7458
7459	try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });
7460	try cg.lowerToStack(operand);
7461	try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
7462
7463	switch (cg.ptr_size) {
7464	.wasm32 => {
7465	try cg.addImm32(@intCast(8));
7466	try cg.addTag(.i32_mul);
7467	try cg.addTag(.i32_add);
7468	},
7469	.wasm64 => {
7470	try cg.addImm64(8);
7471	try cg.addTag(.i64_mul);
7472	try cg.addTag(.i64_add);
7473	},
7474	}
7475
7476	return cg.finishAir(inst, .stack, &.{un_op});
7477	}
7478
7479	fn airIsNamedEnumValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7480	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7481	const operand = try cg.resolveInst(un_op);
7482	const enum_ty = cg.typeOf(un_op);
7483
7484	try cg.lowerToStack(operand);
7485	try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
7486	try cg.addImm32(~@as(u32, 0));
7487	try cg.addTag(.i32_ne);
7488
7489	return cg.finishAir(inst, .stack, &.{un_op});
7490	}
7491
7492	fn airErrorSetHasValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7493	const zcu = cg.pt.zcu;
7494	const ip = &zcu.intern_pool;
7495	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7496
7497	const operand = try cg.resolveInst(ty_op.operand);
7498	const error_set_ty = ty_op.ty;
7499	const result = try cg.allocLocal(Type.bool);
7500
7501	const names = error_set_ty.errorSetNames(zcu);
7502	var values = try std.array_list.Managed(u32).initCapacity(cg.gpa, names.len);
7503	defer values.deinit();
7504
7505	var lowest: ?u32 = null;
7506	var highest: ?u32 = null;
7507	for (0..names.len) |name_index| {
7508	const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
7509	if (lowest) |*l| {
7510	if (err_int < l.*) {
7511	l.* = err_int;
7512	}
7513	} else {
7514	lowest = err_int;
7515	}
7516	if (highest) |*h| {
7517	if (err_int > h.*) {
7518	h.* = err_int;
7519	}
7520	} else {
7521	highest = err_int;
7522	}
7523
7524	values.appendAssumeCapacity(err_int);
7525	}
7526
7527	// start block for 'true' branch
7528	try cg.startBlock(.block, .empty);
7529	// start block for 'false' branch
7530	try cg.startBlock(.block, .empty);
7531	// block for the jump table itself
7532	try cg.startBlock(.block, .empty);
7533
7534	// lower operand to determine jump table target
7535	try cg.emitWValue(operand);
7536	try cg.addImm32(lowest.?);
7537	try cg.addTag(.i32_sub);
7538
7539	// Account for default branch so always add '1'
7540	const depth = @as(u32, @intCast(highest.? - lowest.? + 1));
7541	const jump_table: Mir.JumpTable = .{ .length = depth + 1 };
7542	const table_extra_index = try cg.addExtra(jump_table);
7543	try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } });
7544	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, depth + 1);
7545
7546	var value: u32 = lowest.?;
7547	while (value <= highest.?) : (value += 1) {
7548	const idx: u32 = blk: {
7549	for (values.items) |val| {
7550	if (val == value) break :blk 1;
7551	}
7552	break :blk 0;
7553	};
7554	cg.mir_extra.appendAssumeCapacity(idx);
7555	}
7556	cg.mir_extra.appendAssumeCapacity(0); // outside lowest...highest
7557	try cg.endBlock();
7558
7559	// 'false' branch (i.e. error set does not have value
7560	// ensure we set local to 0 in case the local was re-used.
7561	try cg.addImm32(0);
7562	try cg.addLocal(.local_set, result.local.value);
7563	try cg.addLabel(.br, 1);
7564	try cg.endBlock();
7565
7566	// 'true' branch
7567	try cg.addImm32(1);
7568	try cg.addLocal(.local_set, result.local.value);
7569	try cg.addLabel(.br, 0);
7570	try cg.endBlock();
7571
7572	return cg.finishAir(inst, result, &.{ty_op.operand});
7573	}
7574
7575	inline fn useAtomicFeature(cg: *const CodeGen) bool {
7576	return cg.target.cpu.has(.wasm, .atomics);
7577	}
7578
7579	fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7580	const zcu = cg.pt.zcu;
7581	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7582	const extra = cg.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
7583
7584	const ptr_ty = cg.typeOf(extra.ptr);
7585	const ty = ptr_ty.childType(zcu);
7586	const result_ty = cg.typeOfIndex(inst);
7587
7588	const int_ty: IntType = .fromType(cg, ty);
7589
7590	const ptr_operand = try cg.resolveInst(extra.ptr);
7591	const expected_val = try cg.resolveInst(extra.expected_value);
7592	const new_val = try cg.resolveInst(extra.new_value);
7593
7594	const cmp_result = try cg.allocLocal(Type.bool);
7595
7596	const ptr_val = if (cg.useAtomicFeature()) val: {
7597	const val_local = try cg.allocLocal(ty);
7598	try cg.emitWValue(ptr_operand);
7599	try cg.lowerToStack(expected_val);
7600	try cg.lowerToStack(new_val);
7601	try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) {
7602	1 => .i32_atomic_rmw8_cmpxchg_u,
7603	2 => .i32_atomic_rmw16_cmpxchg_u,
7604	4 => .i32_atomic_rmw_cmpxchg,
7605	8 => .i32_atomic_rmw_cmpxchg,
7606	else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),
7607	}, .{
7608	.offset = ptr_operand.offset(),
7609	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7610	});
7611	try cg.addLocal(.local_tee, val_local.local.value);
7612	_ = try cg.intCmp(int_ty, .eq, .stack, expected_val);
7613	try cg.addLocal(.local_set, cmp_result.local.value);
7614	break :val val_local;
7615	} else val: {
7616	if (ty.abiSize(zcu) > 8) {
7617	return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{});
7618	}
7619	const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, 0), cg, ty);
7620
7621	try cg.lowerToStack(ptr_operand);
7622	try cg.lowerToStack(new_val);
7623	try cg.emitWValue(ptr_val);
7624	_ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);
7625	try cg.addLocal(.local_tee, cmp_result.local.value);
7626	try cg.addTag(.select);
7627	try cg.store(.stack, .stack, ty, 0);
7628
7629	break :val ptr_val;
7630	};
7631
7632	const result = if (isByRef(result_ty, zcu, cg.target)) val: {
7633	try cg.emitWValue(cmp_result);
7634	try cg.addImm32(~@as(u32, 0));
7635	try cg.addTag(.i32_xor);
7636	try cg.addImm32(1);
7637	try cg.addTag(.i32_and);
7638	const and_result = try WValue.toLocal(.stack, cg, Type.bool);
7639	const result_ptr = try cg.allocStack(result_ty);
7640	try cg.store(result_ptr, and_result, Type.bool, @as(u32, @intCast(ty.abiSize(zcu))));
7641	try cg.store(result_ptr, ptr_val, ty, 0);
7642	break :val result_ptr;
7643	} else val: {
7644	try cg.addImm32(0);
7645	try cg.emitWValue(ptr_val);
7646	try cg.emitWValue(cmp_result);
7647	try cg.addTag(.select);
7648	break :val .stack;
7649	};
7650
7651	return cg.finishAir(inst, result, &.{ extra.ptr, extra.expected_value, extra.new_value });
7652	}
7653
7654	fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7655	const zcu = cg.pt.zcu;
7656	const atomic_load = cg.air.instructions.items(.data)[@backingInt(inst)].atomic_load;
7657	const ptr = try cg.resolveInst(atomic_load.ptr);
7658	const ty = cg.typeOfIndex(inst);
7659
7660	if (cg.useAtomicFeature()) {
7661	const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
7662	1 => .i32_atomic_load8_u,
7663	2 => .i32_atomic_load16_u,
7664	4 => .i32_atomic_load,
7665	8 => .i64_atomic_load,
7666	else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
7667	};
7668	try cg.emitWValue(ptr);
7669	try cg.addAtomicMemArg(tag, .{
7670	.offset = ptr.offset(),
7671	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7672	});
7673	} else {
7674	_ = try cg.load(ptr, ty, 0);
7675	}
7676
7677	return cg.finishAir(inst, .stack, &.{atomic_load.ptr});
7678	}
7679
7680	fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7681	const zcu = cg.pt.zcu;
7682	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
7683	const extra = cg.air.extraData(Air.AtomicRmw, pl_op.payload).data;
7684
7685	const ptr = try cg.resolveInst(pl_op.operand);
7686	const operand = try cg.resolveInst(extra.operand);
7687	const ty = cg.typeOfIndex(inst);
7688	const op: std.lang.AtomicRmwOp = extra.op();
7689
7690	if (cg.useAtomicFeature()) {
7691	const int_ty: IntType = .fromType(cg, ty);
7692	switch (op) {
7693	.Max,
7694	.Min,
7695	.Nand,
7696	=> {
7697	const tmp = try cg.load(ptr, ty, 0);
7698	const value = try tmp.toLocal(cg, ty);
7699
7700	// create a loop to cmpxchg the new value
7701	try cg.startBlock(.loop, .empty);
7702
7703	try cg.emitWValue(ptr);
7704	try cg.emitWValue(value);
7705	if (op == .Nand) {
7706	const and_res = try cg.intAnd(int_ty, value, operand);
7707	if (int_ty.bits <= 32) {
7708	try cg.addImm32(~@as(u32, 0));
7709	} else if (int_ty.bits <= 64) {
7710	try cg.addImm64(~@as(u64, 0));
7711	} else {
7712	return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7713	}
7714	_ = try cg.intXor(int_ty, and_res, .stack);
7715	} else {
7716	try cg.emitWValue(value);
7717	try cg.emitWValue(operand);
7718	_ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, value, operand);
7719	try cg.addTag(.select);
7720	}
7721	try cg.addAtomicMemArg(
7722	switch (ty.abiSize(zcu)) {
7723	1 => .i32_atomic_rmw8_cmpxchg_u,
7724	2 => .i32_atomic_rmw16_cmpxchg_u,
7725	4 => .i32_atomic_rmw_cmpxchg,
7726	8 => .i64_atomic_rmw_cmpxchg,
7727	else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}),
7728	},
7729	.{
7730	.offset = ptr.offset(),
7731	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7732	},
7733	);
7734	const select_res = try cg.allocLocal(ty);
7735	try cg.addLocal(.local_tee, select_res.local.value);
7736	_ = try cg.intCmp(int_ty, .neq, .stack, value); // leave on stack so we can use it for br_if
7737
7738	try cg.emitWValue(select_res);
7739	try cg.addLocal(.local_set, value.local.value);
7740
7741	try cg.addLabel(.br_if, 0);
7742	try cg.endBlock();
7743	return cg.finishAir(inst, value, &.{ pl_op.operand, extra.operand });
7744	},
7745
7746	// the other operations have their own instructions for Wasm.
7747	else => {
7748	try cg.emitWValue(ptr);
7749	try cg.emitWValue(operand);
7750	const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
7751	1 => switch (op) {
7752	.Xchg => .i32_atomic_rmw8_xchg_u,
7753	.Add => .i32_atomic_rmw8_add_u,
7754	.Sub => .i32_atomic_rmw8_sub_u,
7755	.And => .i32_atomic_rmw8_and_u,
7756	.Or => .i32_atomic_rmw8_or_u,
7757	.Xor => .i32_atomic_rmw8_xor_u,
7758	else => unreachable,
7759	},
7760	2 => switch (op) {
7761	.Xchg => .i32_atomic_rmw16_xchg_u,
7762	.Add => .i32_atomic_rmw16_add_u,
7763	.Sub => .i32_atomic_rmw16_sub_u,
7764	.And => .i32_atomic_rmw16_and_u,
7765	.Or => .i32_atomic_rmw16_or_u,
7766	.Xor => .i32_atomic_rmw16_xor_u,
7767	else => unreachable,
7768	},
7769	4 => switch (op) {
7770	.Xchg => .i32_atomic_rmw_xchg,
7771	.Add => .i32_atomic_rmw_add,
7772	.Sub => .i32_atomic_rmw_sub,
7773	.And => .i32_atomic_rmw_and,
7774	.Or => .i32_atomic_rmw_or,
7775	.Xor => .i32_atomic_rmw_xor,
7776	else => unreachable,
7777	},
7778	8 => switch (op) {
7779	.Xchg => .i64_atomic_rmw_xchg,
7780	.Add => .i64_atomic_rmw_add,
7781	.Sub => .i64_atomic_rmw_sub,
7782	.And => .i64_atomic_rmw_and,
7783	.Or => .i64_atomic_rmw_or,
7784	.Xor => .i64_atomic_rmw_xor,
7785	else => unreachable,
7786	},
7787	else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}),
7788	};
7789	try cg.addAtomicMemArg(tag, .{
7790	.offset = ptr.offset(),
7791	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7792	});
7793	return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });
7794	},
7795	}
7796	} else {
7797	const loaded = try cg.load(ptr, ty, 0);
7798	const result = try loaded.toLocal(cg, ty);
7799
7800	switch (op) {
7801	.Xchg => {
7802	try cg.store(ptr, operand, ty, 0);
7803	},
7804	.Add,
7805	.Sub,
7806	=> {
7807	if (ty.isAnyFloat()) {
7808	const float_ty: FloatType = .fromType(cg, ty);
7809	try cg.emitWValue(ptr);
7810	_ = switch (op) {
7811	.Add => try cg.floatAdd(float_ty, result, operand),
7812	.Sub => try cg.floatSub(float_ty, result, operand),
7813	else => unreachable,
7814	};
7815	try cg.store(.stack, .stack, ty, ptr.offset());
7816	} else {
7817	const int_ty: IntType = .fromType(cg, ty);
7818	try cg.emitWValue(ptr);
7819	_ = switch (op) {
7820	.Add => try cg.intAdd(int_ty, result, operand),
7821	.Sub => try cg.intSub(int_ty, result, operand),
7822	else => unreachable,
7823	};
7824	_ = try cg.intWrap(int_ty, .stack);
7825	try cg.store(.stack, .stack, ty, ptr.offset());
7826	}
7827	},
7828	.And,
7829	.Or,
7830	.Xor,
7831	=> {
7832	const int_ty: IntType = .fromType(cg, ty);
7833	try cg.emitWValue(ptr);
7834	_ = switch (op) {
7835	.And => try cg.intAnd(int_ty, result, operand),
7836	.Or => try cg.intOr(int_ty, result, operand),
7837	.Xor => try cg.intXor(int_ty, result, operand),
7838	else => unreachable,
7839	};
7840	try cg.store(.stack, .stack, ty, ptr.offset());
7841	},
7842	.Max,
7843	.Min,
7844	=> {
7845	if (ty.isAnyFloat()) {
7846	const float_ty: FloatType = .fromType(cg, ty);
7847	try cg.emitWValue(ptr);
7848	try cg.emitWValue(result);
7849	try cg.emitWValue(operand);
7850	_ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);
7851	try cg.addTag(.select);
7852	try cg.store(.stack, .stack, ty, ptr.offset());
7853	} else {
7854	const int_ty: IntType = .fromType(cg, ty);
7855	try cg.emitWValue(ptr);
7856	try cg.emitWValue(result);
7857	try cg.emitWValue(operand);
7858	_ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);
7859	try cg.addTag(.select);
7860	try cg.store(.stack, .stack, ty, ptr.offset());
7861	}
7862	},
7863	.Nand => {
7864	const int_ty: IntType = .fromType(cg, ty);
7865	try cg.emitWValue(ptr);
7866	const and_res = try cg.intAnd(int_ty, result, operand);
7867	if (int_ty.bits <= 32) {
7868	try cg.addImm32(~@as(u32, 0));
7869	} else if (int_ty.bits <= 64) {
7870	try cg.addImm64(~@as(u64, 0));
7871	} else {
7872	return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7873	}
7874	_ = try cg.intXor(int_ty, and_res, .stack);
7875	try cg.store(.stack, .stack, ty, ptr.offset());
7876	},
7877	}
7878
7879	return cg.finishAir(inst, result, &.{ pl_op.operand, extra.operand });
7880	}
7881	}
7882
7883	fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7884	const zcu = cg.pt.zcu;
7885	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7886
7887	const ptr = try cg.resolveInst(bin_op.lhs);
7888	const operand = try cg.resolveInst(bin_op.rhs);
7889	const ptr_ty = cg.typeOf(bin_op.lhs);
7890	const ty = ptr_ty.childType(zcu);
7891
7892	if (cg.useAtomicFeature()) {
7893	const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
7894	1 => .i32_atomic_store8,
7895	2 => .i32_atomic_store16,
7896	4 => .i32_atomic_store,
7897	8 => .i64_atomic_store,
7898	else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
7899	};
7900	try cg.emitWValue(ptr);
7901	try cg.lowerToStack(operand);
7902	try cg.addAtomicMemArg(tag, .{
7903	.offset = ptr.offset(),
7904	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7905	});
7906	} else {
7907	try cg.store(ptr, operand, ty, 0);
7908	}
7909
7910	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7911	}
7912
7913	fn airFrameAddress(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7914	if (cg.initial_stack_value == .none) {
7915	try cg.initializeStack();
7916	}
7917	try cg.emitWValue(cg.bottom_stack_value);
7918	return cg.finishAir(inst, .stack, &.{});
7919	}
7920
7921	fn airRuntimeNavPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7922	const ty_nav = cg.air.instructions.items(.data)[@backingInt(inst)].ty_nav;
7923	const mod = cg.pt.zcu.navFileScope(cg.owner_nav).mod.?;
7924	if (mod.single_threaded) {
7925	const result: WValue = .{ .nav_ref = .{
7926	.nav_index = ty_nav.nav,
7927	.offset = 0,
7928	} };
7929	return cg.finishAir(inst, result, &.{});
7930	}
7931	return cg.fail("TODO: thread-local variables", .{});
7932	}
7933
7934	fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7935	const unwrapped_asm = cg.air.unwrapAsm(inst);
7936	const outputs = unwrapped_asm.outputs;
7937	const inputs = unwrapped_asm.inputs;
7938
7939	const zcu = cg.pt.zcu;
7940	const output_ty = cg.typeOfIndex(inst);
7941
7942	const result: WValue = if (output_ty.hasRuntimeBits(zcu))
7943	try cg.allocLocal(output_ty)
7944	else
7945	.none;
7946
7947	if (unwrapped_asm.source.len != 0) {
7948	var local_map: assembly.LocalMap = .empty;
7949	defer local_map.deinit(cg.gpa);
7950
7951	{
7952	var it = unwrapped_asm.iterateOutputs();
7953	if (it.next()) |output| {
7954	const constraint = output.constraint;
7955	assert(output.operand == .none);
7956	const name = output.name;
7957
7958	if (!mem.eql(u8, constraint, "=r")) {
7959	return cg.fail("Self-hosted wasm backend requires output constraint to be equal \"=r\"", .{});
7960	}
7961
7962	const gop = try local_map.getOrPutValue(cg.gpa, name, result.local.value);
7963	assert(!gop.found_existing); // first value
7964
7965	assert(it.next() == null);
7966	}
7967	}
7968
7969	{
7970	var it = unwrapped_asm.iterateInputs();
7971	while (it.next()) |input| {
7972	const constraint = input.constraint;
7973	const operand = try cg.resolveInst(input.operand);
7974	const name = input.name;
7975
7976	if (!mem.eql(u8, constraint, "r")) {
7977	return cg.fail("Self-hosted wasm backend requires input constraint to be equal \"r\"", .{});
7978	}
7979
7980	try cg.lowerToStack(operand);
7981	const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand));
7982
7983	const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value);
7984	if (gop.found_existing) {
7985	return cg.fail("Duplicate asm variable name \"{s}\"", .{name});
7986	}
7987	}
7988	}
7989
7990	try assembly.assemble(cg, unwrapped_asm.source, &local_map);
7991	}
7992
7993	var bt = cg.liveness.iterateBigTomb(inst);
7994	for (outputs) |output| if (output != .none) cg.feed(&bt, output);
7995	for (inputs) |input| cg.feed(&bt, input);
7996	return cg.finishAirResult(inst, result);
7997	}
7998
7999	fn typeOf(cg: *CodeGen, inst: Air.Inst.Ref) Type {
8000	const zcu = cg.pt.zcu;
8001	return cg.air.typeOf(inst, &zcu.intern_pool);
8002	}
8003
8004	fn typeOfIndex(cg: *CodeGen, inst: Air.Inst.Index) Type {
8005	const zcu = cg.pt.zcu;
8006	return cg.air.typeOfIndex(inst, &zcu.intern_pool);
8007	}