Meg's Mirrors

zig

1	// Compilation
2	pt: Zcu.PerThread,
3	zcu: *Zcu,
4	gpa: Allocator,
5	arena: Allocator,
6	air: Air,
7	liveness: Air.Liveness,
8	owner_nav: InternPool.Nav.Index,
9	base_line: u32,
10
11	// Module-level output (accumulated across the nav's codegen)
12	next_result_id: Word = 1,
13	decls: std.ArrayList(Decl) = .empty,
14	decl_deps: std.ArrayList(Decl.Index) = .empty,
15	nav_link: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Decl.Index) = .empty,
16	uav_link: std.AutoHashMapUnmanaged(struct { InternPool.Index, spec.StorageClass }, Decl.Index) = .empty,
17	entry_points: std.array_hash_map.Auto(Id, EntryPoint) = .empty,
18	error_buffer: ?Decl.Index = null,
19	struct_types: std.array_hash_map.Custom(StructType, Id, StructType.HashContext, true) = .empty,
20	/// SPIR-V ids of OpVariables whose pointee is a Block struct
21	block_var_ids: std.AutoHashMapUnmanaged(Id, void) = .empty,
22	builtins: std.AutoHashMapUnmanaged(struct { spec.BuiltIn, spec.StorageClass }, Decl.Index) = .empty,
23	sections: struct {
24	// Module layout, according to SPIR-V Spec section 2.4, "Logical Layout of a Module".
25	extended_instruction_set: Section = .{},
26	memory_model: Section = .{},
27	execution_modes: Section = .{},
28	debug_strings: Section = .{},
29	debug_names: Section = .{},
30	annotations: Section = .{},
31	globals: Section = .{},
32	functions: Section = .{},
33	} = .{},
34
35	// Per-function state (reset between top-level genNav calls)
36	prologue: Section = .{},
37	body: Section = .{},
38	args: std.ArrayList(Id) = .empty,
39	next_arg_index: u32 = 0,
40	/// Caches the limb extractions for composite integer values so repeated
41	/// arithmetic on the same operand doesn't re-emit `OpCompositeExtract` per
42	/// limb per use. Slices are owned by `cg.arena`.
43	composite_limbs: std.AutoHashMapUnmanaged(Id, []const Id) = .empty,
44	block_stack: std.ArrayList(*Block) = .empty,
45	block_label: Id = .none,
46	/// Whether the current block has been terminated by a terminator
47	/// instruction (e.g. OpKill from inline assembly). When true, no further
48	/// branch instructions should be emitted for the current block.
49	block_terminated: bool = false,
50	block_results: std.AutoHashMapUnmanaged(Air.Inst.Index, Id) = .empty,
51	inst_results: std.AutoHashMapUnmanaged(Air.Inst.Index, Id) = .empty,
52	tracked_allocas: std.AutoHashMapUnmanaged(Id, ?Id) = .empty,
53	loop_switches: std.AutoHashMapUnmanaged(Air.Inst.Index, LoopSwitch) = .empty,
54	id_scratch: std.ArrayList(Id) = .empty,
55
56	fn hasInt64(target: *const std.Target) bool {
57	return target.cpu.arch == .spirv64 or target.cpu.has(.spirv, .int64);
58	}
59
60	fn bigIntBits(cg: *const CodeGen) u16 {
61	return if (hasInt64(cg.zcu.getTarget())) 64 else 32;
62	}
63
64	fn limbType(cg: *const CodeGen) Type {
65	return if (cg.bigIntBits() == 64) .u64 else .u32;
66	}
67
68	fn limbTypeId(cg: *CodeGen) !Id {
69	return cg.resolveType(cg.limbType(), .direct);
70	}
71
72	/// Data can be lowered into in two basic representations: indirect, which is when
73	/// a type is stored in memory, and direct, which is how a type is stored when its
74	/// a direct SPIR-V value.
75	pub const Repr = enum {
76	/// A SPIR-V value as it would be used in operations.
77	direct,
78	/// A SPIR-V value as it is stored in memory.
79	indirect,
80	};
81
82	/// A function or global, tracked here so the linker can order globals and build
83	/// per-entry-point interface lists.
84	pub const Decl = struct {
85	pub const Index = enum(u32) { _ };
86	pub const Kind = enum { func, global, invocation_global };
87
88	kind: Kind,
89	/// Result-id of the associated OpFunction / OpVariable / InvocationGlobal.
90	result_id: Id,
91	/// Range into `decl_deps` for this decl's dependencies.
92	begin_dep: usize = 0,
93	end_dep: usize = 0,
94	/// Whether an extern-function stub has been emitted.
95	has_extern_stub: bool = false,
96	};
97
98	pub const EntryPoint = struct {
99	decl_index: Decl.Index,
100	name: []const u8,
101	cc: std.builtin.CallingConvention,
102	};
103
104	const StructType = struct {
105	fields: []const Id,
106	ip_index: InternPool.Index,
107
108	const HashContext = struct {
109	pub fn hash(_: @This(), ty: StructType) u32 {
110	var hasher = std.hash.Wyhash.init(0);
111	hasher.update(std.mem.sliceAsBytes(ty.fields));
112	hasher.update(std.mem.asBytes(&ty.ip_index));
113	return @truncate(hasher.final());
114	}
115
116	pub fn eql(_: @This(), a: StructType, b: StructType, _: usize) bool {
117	return a.ip_index == b.ip_index and std.mem.eql(Id, a.fields, b.fields);
118	}
119	};
120	};
121
122	pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
123	return comptime &.initMany(&.{
124	.expand_bit_cast_safe,
125	.expand_int_cast_safe,
126	.expand_int_from_float_safe,
127	.expand_int_from_float_optimized_safe,
128	.expand_add_safe,
129	.expand_sub_safe,
130	.expand_mul_safe,
131
132	.expand_array_splat,
133	.expand_array_to_vector,
134	});
135	}
136
137	const LoopSwitch = struct { cond_var: Id, continue_label: Id };
138
139	/// Pointer-typed AIR refs should resolve through `resolvePtr` to handle the
140	/// `tracked_allocas` case explicitly at every use site.
141	const Ptr = union(enum) {
142	id: Id,
143	/// Function-local pointer whose value lives in `tracked_allocas` rather
144	/// than a real OpVariable. `slot` is the current pointee value.
145	tracked: struct { id: Id, slot: *?Id },
146	};
147
148	/// Tracks how control flow leaves a Zig `block` under SPIR-V's structured
149	/// control flow rules.
150	const Block = union(enum) {
151	const Incoming = struct {
152	src_label: Id,
153	/// Block index (u32) that control flow should jump to next.
154	next_block: Id,
155	};
156
157	const SelectionMerge = struct {
158	incoming: Incoming,
159	/// Label of the cond_br's merge block (undefined for top-of-stack).
160	merge_block: Id,
161	};
162
163	/// Selection blocks can't use early exits. Closing requires a "merge ladder"
164	/// of nested OpSelectionMerge instructions, one per pending merge.
165	selection: struct {
166	merge_stack: std.ArrayList(SelectionMerge) = .empty,
167	},
168	/// Loop blocks early-exit by jumping to the loop merge label.
169	loop: struct {
170	merges: std.ArrayList(Incoming) = .empty,
171	merge_block: Id,
172	},
173
174	fn deinit(block: *Block, gpa: Allocator) void {
175	switch (block.*) {
176	.selection => |*merge| merge.merge_stack.deinit(gpa),
177	.loop => |*merge| merge.merges.deinit(gpa),
178	}
179	block.* = undefined;
180	}
181	};
182
183	pub fn deinit(cg: *CodeGen) void {
184	const gpa = cg.gpa;
185	cg.block_stack.deinit(gpa);
186	cg.block_results.deinit(gpa);
187	cg.args.deinit(gpa);
188	cg.composite_limbs.deinit(gpa);
189	cg.tracked_allocas.deinit(gpa);
190	cg.inst_results.deinit(gpa);
191	cg.loop_switches.deinit(gpa);
192	cg.id_scratch.deinit(gpa);
193	cg.prologue.deinit(gpa);
194	cg.body.deinit(gpa);
195
196	cg.nav_link.deinit(gpa);
197	cg.uav_link.deinit(gpa);
198
199	cg.sections.extended_instruction_set.deinit(gpa);
200	cg.sections.memory_model.deinit(gpa);
201	cg.sections.execution_modes.deinit(gpa);
202	cg.sections.debug_strings.deinit(gpa);
203	cg.sections.debug_names.deinit(gpa);
204	cg.sections.annotations.deinit(gpa);
205	cg.sections.globals.deinit(gpa);
206	cg.sections.functions.deinit(gpa);
207
208	cg.struct_types.deinit(gpa);
209	cg.block_var_ids.deinit(gpa);
210	cg.builtins.deinit(gpa);
211
212	cg.decls.deinit(gpa);
213	cg.decl_deps.deinit(gpa);
214	cg.entry_points.deinit(gpa);
215	}
216
217	pub fn generate(
218	_: *link.File,
219	pt: Zcu.PerThread,
220	func_index: InternPool.Index,
221	air: *const Air,
222	liveness: *const ?Air.Liveness,
223	) codegen.Error!Mir {
224	const zcu = pt.zcu;
225	const gpa = zcu.gpa;
226	const nav = zcu.funcInfo(func_index).owner_nav;
227
228	var arena = std.heap.ArenaAllocator.init(gpa);
229	defer arena.deinit();
230
231	var cg: CodeGen = .{
232	.pt = pt,
233	.gpa = gpa,
234	.arena = arena.allocator(),
235	.zcu = zcu,
236	.air = air.*,
237	.liveness = liveness.*.?,
238	.owner_nav = nav,
239	.base_line = zcu.navSrcLine(nav),
240	};
241	defer cg.deinit();
242
243	try cg.genNav(true);
244
245	return cg.serializeToMir(gpa);
246	}
247
248	pub fn generateNav(
249	pt: Zcu.PerThread,
250	nav_index: InternPool.Nav.Index,
251	) codegen.Error!Mir {
252	const zcu = pt.zcu;
253	const gpa = zcu.gpa;
254
255	var arena = std.heap.ArenaAllocator.init(gpa);
256	defer arena.deinit();
257
258	var cg: CodeGen = .{
259	.pt = pt,
260	.gpa = gpa,
261	.arena = arena.allocator(),
262	.zcu = zcu,
263	.air = undefined,
264	.liveness = undefined,
265	.owner_nav = nav_index,
266	.base_line = zcu.navSrcLine(nav_index),
267	};
268	defer cg.deinit();
269
270	try cg.genNav(false);
271
272	return cg.serializeToMir(gpa);
273	}
274
275	fn serializeToMir(cg: *CodeGen, gpa: Allocator) codegen.Error!Mir {
276	const owner_entry = cg.nav_link.get(cg.owner_nav);
277	const owner_decl_index = owner_entry orelse return .{
278	.id_bound = cg.next_result_id,
279	.owner_nav = cg.owner_nav,
280	.kind = .func,
281	.decl_result_id = .none,
282	.extended_instruction_set = &.{},
283	.globals = &.{},
284	.functions = &.{},
285	.annotations = &.{},
286	.debug_names = &.{},
287	.debug_strings = &.{},
288	.execution_modes = &.{},
289	.nav_refs = &.{},
290	.uav_refs = &.{},
291	.decl_deps = &.{},
292	.internal_globals = &.{},
293	.entry_points = &.{},
294	};
295
296	const owner_decl = cg.declPtr(owner_decl_index);
297
298	var nav_refs: std.ArrayList(Mir.NavRef) = .empty;
299	defer nav_refs.deinit(gpa);
300	var nav_it = cg.nav_link.iterator();
301	while (nav_it.next()) |entry| {
302	if (entry.key_ptr.* == cg.owner_nav) continue;
303	const decl = cg.declPtr(entry.value_ptr.*);
304	try nav_refs.append(gpa, .{
305	.local_id = decl.result_id,
306	.nav = entry.key_ptr.*,
307	.kind = decl.kind,
308	});
309	}
310
311	var uav_refs: std.ArrayList(Mir.UavRef) = .empty;
312	defer uav_refs.deinit(gpa);
313	var uav_it = cg.uav_link.iterator();
314	while (uav_it.next()) |entry| {
315	const decl = cg.declPtr(entry.value_ptr.*);
316	try uav_refs.append(gpa, .{
317	.local_id = decl.result_id,
318	.val = entry.key_ptr.*[0],
319	.storage_class = entry.key_ptr.*[1],
320	.kind = decl.kind,
321	});
322	}
323
324	var decl_deps: std.ArrayList(Mir.DeclDep) = .empty;
325	defer decl_deps.deinit(gpa);
326	var internal_globals: std.ArrayList(Id) = .empty;
327	defer internal_globals.deinit(gpa);
328
329	const deps = cg.decl_deps.items[owner_decl.begin_dep..owner_decl.end_dep];
330	for (deps) |dep_index| {
331	const dep_decl = cg.declPtr(dep_index);
332	var found = false;
333	nav_it.index = 0;
334	while (nav_it.next()) |entry| {
335	if (entry.value_ptr.* == dep_index) {
336	try decl_deps.append(gpa, .{
337	.kind = dep_decl.kind,
338	.nav = entry.key_ptr.*,
339	});
340	found = true;
341	break;
342	}
343	}
344	if (!found and dep_decl.kind == .global) {
345	try internal_globals.append(gpa, dep_decl.result_id);
346	}
347	}
348
349	var ep_list: std.ArrayList(Mir.EntryPoint) = .empty;
350	defer ep_list.deinit(gpa);
351	var ep_it = cg.entry_points.iterator();
352	while (ep_it.next()) |entry| {
353	const ep = entry.value_ptr;
354	const ep_decl = cg.declPtr(ep.decl_index);
355	try ep_list.append(gpa, .{
356	.local_id = ep_decl.result_id,
357	.name = try gpa.dupe(u8, ep.name),
358	.cc = ep.cc,
359	});
360	}
361
362	return .{
363	.id_bound = cg.next_result_id,
364	.owner_nav = cg.owner_nav,
365	.kind = owner_decl.kind,
366	.decl_result_id = owner_decl.result_id,
367	.extended_instruction_set = try cg.sections.extended_instruction_set.instructions.toOwnedSlice(gpa),
368	.globals = try cg.sections.globals.instructions.toOwnedSlice(gpa),
369	.functions = try cg.sections.functions.instructions.toOwnedSlice(gpa),
370	.annotations = try cg.sections.annotations.instructions.toOwnedSlice(gpa),
371	.debug_names = try cg.sections.debug_names.instructions.toOwnedSlice(gpa),
372	.debug_strings = try cg.sections.debug_strings.instructions.toOwnedSlice(gpa),
373	.execution_modes = try cg.sections.execution_modes.instructions.toOwnedSlice(gpa),
374	.nav_refs = try nav_refs.toOwnedSlice(gpa),
375	.uav_refs = try uav_refs.toOwnedSlice(gpa),
376	.decl_deps = try decl_deps.toOwnedSlice(gpa),
377	.internal_globals = try internal_globals.toOwnedSlice(gpa),
378	.entry_points = try ep_list.toOwnedSlice(gpa),
379	};
380	}
381
382	fn typeOf(cg: *CodeGen, inst: Air.Inst.Ref) Type {
383	const zcu = cg.zcu;
384	return cg.air.typeOf(inst, &zcu.intern_pool);
385	}
386
387	fn typeOfIndex(cg: *CodeGen, inst: Air.Inst.Index) Type {
388	const zcu = cg.zcu;
389	return cg.air.typeOfIndex(inst, &zcu.intern_pool);
390	}
391
392	/// Does not generate the nav.
393	pub fn resolveNav(cg: *CodeGen, ip: *InternPool, nav_index: InternPool.Nav.Index) !Decl.Index {
394	const entry = try cg.nav_link.getOrPut(cg.gpa, nav_index);
395	if (!entry.found_existing) {
396	const nav = ip.getNav(nav_index);
397	// TODO: Extern fn?
398	const kind: Decl.Kind = if (ip.isFunctionType(nav.resolved.?.type))
399	.func
400	else switch (nav.resolved.?.@"addrspace") {
401	.generic => .invocation_global,
402	else => .global,
403	};
404	entry.value_ptr.* = try cg.allocDecl(kind);
405	}
406
407	return entry.value_ptr.*;
408	}
409
410	pub fn allocIds(cg: *CodeGen, n: u32) spec.IdRange {
411	defer cg.next_result_id += n;
412	return .{ .base = cg.next_result_id, .len = n };
413	}
414
415	pub fn allocId(cg: *CodeGen) Id {
416	return cg.allocIds(1).at(0);
417	}
418
419	pub fn idBound(cg: *const CodeGen) Word {
420	return cg.next_result_id;
421	}
422
423	pub fn addEntryPointDeps(
424	cg: *CodeGen,
425	decl_index: Decl.Index,
426	seen: *std.bit_set.Dynamic,
427	interface: *std.ArrayList(Id),
428	) !void {
429	const decl = cg.declPtr(decl_index);
430	const deps = cg.decl_deps.items[decl.begin_dep..decl.end_dep];
431
432	if (seen.isSet(@backingInt(decl_index))) {
433	return;
434	}
435
436	seen.set(@backingInt(decl_index));
437
438	if (decl.kind == .global) {
439	try interface.append(cg.gpa, decl.result_id);
440	}
441
442	for (deps) |dep| {
443	try cg.addEntryPointDeps(dep, seen, interface);
444	}
445	}
446
447	pub fn importInstructionSet(cg: *CodeGen, set: spec.InstructionSet) !Id {
448	assert(set != .core);
449	const result_id = cg.allocId();
450	try cg.sections.extended_instruction_set.emit(cg.gpa, .OpExtInstImport, .{
451	.id_result = result_id,
452	.name = @tagName(set),
453	});
454	return result_id;
455	}
456
457	pub fn boolType(cg: *CodeGen) !Id {
458	const result_id = cg.allocId();
459	try cg.sections.globals.emit(cg.gpa, .OpTypeBool, .{
460	.id_result = result_id,
461	});
462	return result_id;
463	}
464
465	pub fn voidType(cg: *CodeGen) !Id {
466	const result_id = cg.allocId();
467	try cg.sections.globals.emit(cg.gpa, .OpTypeVoid, .{
468	.id_result = result_id,
469	});
470	try cg.debugName(result_id, "void");
471	return result_id;
472	}
473
474	pub fn opaqueType(cg: *CodeGen, name: []const u8) !Id {
475	const result_id = cg.allocId();
476	try cg.sections.globals.emit(cg.gpa, .OpTypeOpaque, .{
477	.id_result = result_id,
478	.literal_string = name,
479	});
480	try cg.debugName(result_id, name);
481	return result_id;
482	}
483
484	pub fn backingIntBits(cg: *const CodeGen, bits: u16) struct { u16, bool } {
485	assert(bits != 0);
486	const target = cg.zcu.getTarget();
487	const ints = [_]struct { bits: u16, enabled: bool }{
488	.{ .bits = 8, .enabled = target.cpu.has(.spirv, .int8) },
489	.{ .bits = 16, .enabled = target.cpu.has(.spirv, .int16) },
490	.{ .bits = 32, .enabled = true },
491	.{ .bits = 64, .enabled = hasInt64(target) },
492	};
493
494	for (ints) |int| {
495	if (bits <= int.bits and int.enabled) return .{ int.bits, false };
496	}
497
498	return .{ std.mem.alignForward(u16, bits, cg.bigIntBits()), true };
499	}
500
501	pub fn intType(cg: *CodeGen, signedness: std.lang.Signedness, bits: u16) !Id {
502	assert(bits > 0);
503
504	const target = cg.zcu.getTarget();
505	const actual_signedness = switch (target.os.tag) {
506	// Kernel only supports unsigned ints.
507	.opencl, .amdhsa => .unsigned,
508	else => signedness,
509	};
510	const backing_bits, const big_int = cg.backingIntBits(bits);
511	if (big_int) {
512	const limb_bits = cg.bigIntBits();
513	const limb_ty = try cg.intType(.unsigned, limb_bits);
514	const len_ty = try cg.intType(.unsigned, 32);
515	const len_id = cg.allocId();
516	try cg.sections.globals.emit(cg.gpa, .OpConstant, .{
517	.id_result_type = len_ty,
518	.id_result = len_id,
519	.value = .{ .uint32 = backing_bits / limb_bits },
520	});
521	return cg.arrayType(len_id, limb_ty);
522	}
523
524	const result_id = cg.allocId();
525	try cg.sections.globals.emit(cg.gpa, .OpTypeInt, .{
526	.id_result = result_id,
527	.width = backing_bits,
528	.signedness = switch (actual_signedness) {
529	.signed => 1,
530	.unsigned => 0,
531	},
532	});
533	switch (actual_signedness) {
534	.signed => try cg.debugNameFmt(result_id, "i{}", .{backing_bits}),
535	.unsigned => try cg.debugNameFmt(result_id, "u{}", .{backing_bits}),
536	}
537	return result_id;
538	}
539
540	pub fn floatType(cg: *CodeGen, bits: u16) !Id {
541	assert(bits > 0);
542	const result_id = cg.allocId();
543	try cg.sections.globals.emit(cg.gpa, .OpTypeFloat, .{
544	.id_result = result_id,
545	.width = bits,
546	});
547	try cg.debugNameFmt(result_id, "f{}", .{bits});
548	return result_id;
549	}
550
551	pub fn vectorType(cg: *CodeGen, len: u32, child_ty_id: Id) !Id {
552	const result_id = cg.allocId();
553	try cg.sections.globals.emit(cg.gpa, .OpTypeVector, .{
554	.id_result = result_id,
555	.component_type = child_ty_id,
556	.component_count = len,
557	});
558	return result_id;
559	}
560
561	pub fn arrayType(cg: *CodeGen, len_id: Id, child_ty_id: Id) !Id {
562	const result_id = cg.allocId();
563	try cg.sections.globals.emit(cg.gpa, .OpTypeArray, .{
564	.id_result = result_id,
565	.element_type = child_ty_id,
566	.length = len_id,
567	});
568	return result_id;
569	}
570
571	pub fn ptrType(cg: *CodeGen, child_ty_id: Id, storage_class: spec.StorageClass) !Id {
572	const result_id = cg.allocId();
573	try cg.sections.globals.emit(cg.gpa, .OpTypePointer, .{
574	.id_result = result_id,
575	.storage_class = storage_class,
576	.type = child_ty_id,
577	});
578	return result_id;
579	}
580
581	pub fn structType(
582	cg: *CodeGen,
583	types: []const Id,
584	maybe_names: ?[]const []const u8,
585	ip_index: InternPool.Index,
586	) !Id {
587	const actual_ip_index = if (cg.zcu.comp.config.root_strip) .none else ip_index;
588
589	if (cg.struct_types.get(.{ .fields = types, .ip_index = actual_ip_index })) |id| return id;
590	const result_id = cg.allocId();
591	const types_dup = try cg.arena.dupe(Id, types);
592	try cg.sections.globals.emit(cg.gpa, .OpTypeStruct, .{
593	.id_result = result_id,
594	.id_ref = types_dup,
595	});
596
597	if (maybe_names) |names| {
598	assert(names.len == types.len);
599	for (names, 0..) |name, i| {
600	try cg.memberDebugName(result_id, @intCast(i), name);
601	}
602	}
603
604	try cg.struct_types.put(
605	cg.gpa,
606	.{ .fields = types_dup, .ip_index = actual_ip_index },
607	result_id,
608	);
609	return result_id;
610	}
611
612	/// Returns the layout-decorated variant of `ty` for use inside a Vulkan/OpenGL
613	/// interface block. Vulkan forbids nested Block decorations, so recursive calls
614	/// pass `false`, except through an array, whose elements are each a
615	/// block of their own.
616	///
617	/// This is distinct from `resolveType` because SPIR-V forbids such decorations
618	/// on the pointee of a Function-scope variable.
619	pub fn layoutType(cg: *CodeGen, ty: Type, is_block_root: bool) Error!Id {
620	const gpa = cg.gpa;
621	const zcu = cg.zcu;
622	const ip = &zcu.intern_pool;
623
624	const result_id: Id = switch (ty.zigTypeTag(zcu)) {
625	.@"struct" => id: {
626	const struct_type = ip.loadStructType(ty.toIntern());
627	if (struct_type.layout == .@"packed") return cg.resolveType(ty, .indirect);
628
629	var member_types: std.ArrayList(Id) = .empty;
630	defer member_types.deinit(gpa);
631	const id = cg.allocId();
632	if (is_block_root) try cg.decorate(id, .block);
633	var it = struct_type.iterateRuntimeOrder(ip);
634	while (it.next()) |field_index| {
635	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
636	if (!field_ty.hasRuntimeBits(zcu)) continue;
637	try cg.decorateMember(id, @intCast(member_types.items.len), .{ .offset = .{
638	.byte_offset = @intCast(ty.structFieldOffset(field_index, zcu)),
639	} });
640	try member_types.append(gpa, try cg.layoutType(field_ty, false));
641	}
642	try cg.sections.globals.emit(gpa, .OpTypeStruct, .{
643	.id_result = id,
644	.id_ref = member_types.items,
645	});
646	break :id id;
647	},
648	.@"union" => id: {
649	const union_obj = zcu.typeToUnion(ty).?;
650	if (union_obj.layout == .@"packed") return cg.resolveType(ty, .indirect);
651
652	const layout = cg.unionLayout(ty);
653	if (!layout.has_payload) return cg.resolveType(ty, .indirect);
654
655	const id = cg.allocId();
656	if (is_block_root) try cg.decorate(id, .block);
657
658	var member_types: [4]Id = undefined;
659	const u8_id = try cg.resolveType(.u8, .direct);
660	if (layout.tag_size != 0) {
661	const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
662	try cg.decorateMember(id, layout.tag_index, .{ .offset = .{
663	.byte_offset = @intCast(ty.unionGetLayout(zcu).tagOffset()),
664	} });
665	member_types[layout.tag_index] = try cg.layoutType(tag_ty, false);
666	}
667	if (layout.payload_size != 0) {
668	try cg.decorateMember(id, layout.payload_index, .{ .offset = .{
669	.byte_offset = @intCast(ty.unionGetLayout(zcu).payloadOffset()),
670	} });
671	member_types[layout.payload_index] = try cg.layoutType(layout.payload_ty, false);
672	}
673	if (layout.payload_padding_size != 0) {
674	const len_id = try cg.constInt(.u32, layout.payload_padding_size);
675	const arr_id = try cg.arrayType(len_id, u8_id);
676	try cg.decorate(arr_id, .{ .array_stride = .{ .array_stride = 1 } });
677	member_types[layout.payload_padding_index] = arr_id;
678	}
679	if (layout.padding_size != 0) {
680	const len_id = try cg.constInt(.u32, layout.padding_size);
681	const arr_id = try cg.arrayType(len_id, u8_id);
682	try cg.decorate(arr_id, .{ .array_stride = .{ .array_stride = 1 } });
683	member_types[layout.padding_index] = arr_id;
684	}
685	try cg.sections.globals.emit(gpa, .OpTypeStruct, .{
686	.id_result = id,
687	.id_ref = member_types[0..layout.total_fields],
688	});
689	break :id id;
690	},
691	.array => id: {
692	const elem_ty = ty.childType(zcu);
693	const elem_ty_id = try cg.layoutType(elem_ty, is_block_root);
694	const total_len = std.math.cast(u32, ty.arrayLenIncludingSentinel(zcu)) orelse
695	return cg.fail("array type of {} elements is too large", .{ty.arrayLenIncludingSentinel(zcu)});
696	const id = try cg.arrayType(try cg.constInt(.u32, total_len), elem_ty_id);
697	if (!is_block_root and elem_ty.hasRuntimeBits(zcu)) {
698	try cg.decorate(id, .{
699	.array_stride = .{ .array_stride = @intCast(elem_ty.abiSize(zcu)) },
700	});
701	}
702	break :id id;
703	},
704	.spirv => if (ty.isSpirvRuntimeArray(zcu)) id: {
705	const elem_ty = ty.childType(zcu);
706	const elem_ty_id = try cg.layoutType(elem_ty, is_block_root);
707	const id = cg.allocId();
708	try cg.sections.globals.emit(gpa, .OpTypeRuntimeArray, .{
709	.id_result = id,
710	.element_type = elem_ty_id,
711	});
712	if (!is_block_root and elem_ty.hasRuntimeBits(zcu)) {
713	try cg.decorate(id, .{
714	.array_stride = .{ .array_stride = @intCast(elem_ty.abiSize(zcu)) },
715	});
716	}
717	break :id id;
718	} else return cg.resolveType(ty, .indirect),
719	else => return cg.resolveType(ty, .indirect),
720	};
721
722	return result_id;
723	}
724
725	pub fn functionType(cg: *CodeGen, return_ty_id: Id, param_type_ids: []const Id) !Id {
726	const result_id = cg.allocId();
727	try cg.sections.globals.emit(cg.gpa, .OpTypeFunction, .{
728	.id_result = result_id,
729	.return_type = return_ty_id,
730	.id_ref_2 = param_type_ids,
731	});
732	return result_id;
733	}
734
735	pub fn constUndef(cg: *CodeGen, ty_id: Id) !Id {
736	const result_id = cg.allocId();
737	try cg.sections.globals.emit(cg.gpa, .OpUndef, .{
738	.id_result_type = ty_id,
739	.id_result = result_id,
740	});
741	return result_id;
742	}
743
744	pub fn constNull(cg: *CodeGen, ty_id: Id) !Id {
745	const result_id = cg.allocId();
746	try cg.sections.globals.emit(cg.gpa, .OpConstantNull, .{
747	.id_result_type = ty_id,
748	.id_result = result_id,
749	});
750	return result_id;
751	}
752
753	pub fn decorate(
754	cg: *CodeGen,
755	target: Id,
756	decoration: spec.Decoration.Extended,
757	) !void {
758	try cg.sections.annotations.emit(cg.gpa, .OpDecorate, .{
759	.target = target,
760	.decoration = decoration,
761	});
762	}
763
764	pub fn decorateMember(
765	cg: *CodeGen,
766	structure_type: Id,
767	member: u32,
768	decoration: spec.Decoration.Extended,
769	) !void {
770	try cg.sections.annotations.emit(cg.gpa, .OpMemberDecorate, .{
771	.structure_type = structure_type,
772	.member = member,
773	.decoration = decoration,
774	});
775	}
776
777	pub fn allocDecl(cg: *CodeGen, kind: Decl.Kind) !Decl.Index {
778	try cg.decls.append(cg.gpa, .{
779	.kind = kind,
780	.result_id = cg.allocId(),
781	});
782
783	return @as(Decl.Index, @fromBackingInt(@intCast(@as(u32, @intCast(cg.decls.items.len - 1)))));
784	}
785
786	pub fn declPtr(cg: *CodeGen, index: Decl.Index) *Decl {
787	return &cg.decls.items[@backingInt(index)];
788	}
789
790	pub fn debugName(cg: *CodeGen, target: Id, name: []const u8) !void {
791	if (cg.zcu.comp.config.root_strip) return;
792	try cg.sections.debug_names.emit(cg.gpa, .OpName, .{
793	.target = target,
794	.name = name,
795	});
796	}
797
798	pub fn debugNameFmt(cg: *CodeGen, target: Id, comptime fmt: []const u8, args: anytype) !void {
799	if (cg.zcu.comp.config.root_strip) return;
800	const name = try std.fmt.allocPrint(cg.gpa, fmt, args);
801	defer cg.gpa.free(name);
802	try cg.debugName(target, name);
803	}
804
805	pub fn memberDebugName(cg: *CodeGen, target: Id, member: u32, name: []const u8) !void {
806	if (cg.zcu.comp.config.root_strip) return;
807	try cg.sections.debug_names.emit(cg.gpa, .OpMemberName, .{
808	.type = target,
809	.member = member,
810	.name = name,
811	});
812	}
813
814	pub fn storageClass(cg: *const CodeGen, as: std.lang.AddressSpace) spec.StorageClass {
815	const target = cg.zcu.getTarget();
816	return switch (as) {
817	.generic => .function,
818	.global => switch (target.os.tag) {
819	.opencl, .amdhsa => .cross_workgroup,
820	else => .storage_buffer,
821	},
822	.push_constant => .push_constant,
823	.output => .output,
824	.uniform => .uniform,
825	.storage_buffer => .storage_buffer,
826	.physical_storage_buffer => .physical_storage_buffer,
827	.constant => .uniform_constant,
828	.shared => .workgroup,
829	.local => .function,
830	.input => .input,
831	.gs,
832	.fs,
833	.ss,
834	.far,
835	.param,
836	.flash,
837	.flash1,
838	.flash2,
839	.flash3,
840	.flash4,
841	.flash5,
842	.cog,
843	.lut,
844	.hub,
845	.externref,
846	.funcref,
847	=> unreachable,
848	};
849	}
850
851	const Error = codegen.Error;
852
853	pub fn genNav(cg: *CodeGen, do_codegen: bool) Error!void {
854	const gpa = cg.gpa;
855	const zcu = cg.zcu;
856	const ip = &zcu.intern_pool;
857	const target = zcu.getTarget();
858
859	const nav = ip.getNav(cg.owner_nav);
860	const val = zcu.navValue(cg.owner_nav);
861	const ty = val.typeOf(zcu);
862
863	if (!do_codegen and !ty.hasRuntimeBits(zcu)) {
864	const child_ty = if (ty.zigTypeTag(zcu) == .pointer) ty.childType(zcu) else ty;
865	if (child_ty.zigTypeTag(zcu) != .spirv) return;
866	}
867
868	const spv_decl_index = try cg.resolveNav(ip, cg.owner_nav);
869	const decl = cg.declPtr(spv_decl_index);
870	const result_id = decl.result_id;
871	decl.begin_dep = cg.decl_deps.items.len;
872
873	switch (decl.kind) {
874	.func => {
875	if (nav.resolved.?.is_extern_decl) {
876	_ = try cg.resolveType(ty, .direct);
877	try emitExternFnStub(cg, nav, decl, ty);
878	decl.end_dep = cg.decl_deps.items.len;
879	return;
880	}
881
882	const fn_info = zcu.typeToFunc(ty).?;
883	const return_ty_id = try cg.resolveFnReturnType(.fromInterned(fn_info.return_type));
884	const is_test = zcu.test_functions.contains(cg.owner_nav);
885
886	const func_result_id = if (is_test) cg.allocId() else result_id;
887	const prototype_ty_id = try cg.resolveType(ty, .direct);
888	try cg.prologue.emit(gpa, .OpFunction, .{
889	.id_result_type = return_ty_id,
890	.id_result = func_result_id,
891	.function_type = prototype_ty_id,
892	// Note: the backend will never be asked to generate an inline function
893	// (this is handled in sema), so we don't need to set function_control here.
894	.function_control = .{},
895	});
896
897	try cg.args.ensureUnusedCapacity(gpa, fn_info.param_types.len);
898	for (fn_info.param_types.get(ip)) |param_ty_index| {
899	const param_ty: Type = .fromInterned(param_ty_index);
900	if (!param_ty.hasRuntimeBits(zcu)) continue;
901
902	const param_type_id = try cg.resolveType(param_ty, .direct);
903	const arg_result_id = cg.allocId();
904	try cg.prologue.emit(gpa, .OpFunctionParameter, .{
905	.id_result_type = param_type_id,
906	.id_result = arg_result_id,
907	});
908	cg.args.appendAssumeCapacity(arg_result_id);
909	}
910
911	// TODO: This could probably be done in a better way...
912	const root_block_id = cg.allocId();
913
914	// The root block of a function declaration should appear before OpVariable instructions,
915	// so it is generated into the function's prologue.
916	try cg.prologue.emit(gpa, .OpLabel, .{
917	.id_result = root_block_id,
918	});
919	cg.block_label = root_block_id;
920
921	const main_body = cg.air.getMainBody();
922	_ = try cg.genStructuredBody(.selection, main_body);
923	// We always expect paths to here to end, but we still need the block
924	// to act as a dummy merge block.
925	try cg.body.emit(gpa, .OpUnreachable, {});
926	try cg.body.emit(gpa, .OpFunctionEnd, {});
927	// Append the actual code into the functions section.
928	try cg.sections.functions.append(gpa, cg.prologue);
929	try cg.sections.functions.append(gpa, cg.body);
930
931	// Temporarily generate a test kernel declaration if this is a test function.
932	if (is_test) {
933	try cg.generateTestEntryPoint(nav.fqn.toSlice(ip), spv_decl_index, func_result_id);
934	}
935
936	try cg.debugName(func_result_id, nav.fqn.toSlice(ip));
937	},
938	.global => {
939	const key = ip.indexToKey(val.toIntern()).@"extern";
940
941	const storage_class = cg.storageClass(nav.resolved.?.@"addrspace");
942	assert(storage_class != .generic); // These should be instance globals
943
944	const as = nav.resolved.?.@"addrspace";
945	const ty_id = try cg.pointeeType(as, ty, true);
946	const ptr_ty_id = try cg.ptrType(ty_id, storage_class);
947
948	try cg.sections.globals.emit(gpa, .OpVariable, .{
949	.id_result_type = ptr_ty_id,
950	.id_result = result_id,
951	.storage_class = storage_class,
952	});
953
954	switch (target.os.tag) {
955	.vulkan, .opengl => {
956	switch (storage_class) {
957	.uniform,
958	.push_constant,
959	.storage_buffer,
960	.physical_storage_buffer,
961	=> {
962	if (ty.hasRuntimeBits(zcu)) {
963	if (!ty.isSpirvRuntimeArray(zcu)) {
964	try cg.decorate(
965	ptr_ty_id,
966	.{ .array_stride = .{ .array_stride = @intCast(ty.abiSize(zcu)) } },
967	);
968	}
969	if (!cg.needsLayout(as, ty)) try cg.decorateLayout(ty, ty_id);
970	}
971	if (key.is_const and storage_class == .storage_buffer) {
972	try cg.decorate(result_id, .non_writable);
973	}
974	},
975	else => {},
976	}
977
978	if (key.decoration) |decoration| switch (decoration) {
979	.location => |location| {
980	if (storage_class != .output and storage_class != .input and storage_class != .uniform_constant) {
981	return cg.fail("storage class must be one of (output, input, uniform_constant) but is {s}", .{@tagName(storage_class)});
982	}
983	try cg.decorate(result_id, .{
984	.location = .{ .location = location },
985	});
986	},
987	.flat => |location| {
988	try cg.decorate(result_id, .{ .location = .{ .location = location } });
989	try cg.decorate(result_id, .flat);
990	},
991	.descriptor => |descriptor| {
992	if (storage_class != .storage_buffer and storage_class != .uniform and storage_class != .uniform_constant) {
993	return cg.fail("storage class must be one of (storage_buffer, uniform, uniform_constant) but is {s}", .{@tagName(storage_class)});
994	}
995	try cg.decorate(result_id, .{
996	.binding = .{ .binding_point = descriptor.binding },
997	});
998
999	try cg.decorate(result_id, .{
1000	.descriptor_set = .{ .descriptor_set = descriptor.set },
1001	});
1002	},
1003	};
1004	},
1005	else => {},
1006	}
1007
1008	if (std.meta.stringToEnum(spec.BuiltIn, nav.fqn.toSlice(ip))) |built_in| {
1009	try cg.decorate(result_id, .{ .built_in = .{ .built_in = built_in } });
1010	}
1011
1012	try cg.debugName(result_id, nav.fqn.toSlice(ip));
1013	},
1014	.invocation_global => {
1015	// `@extern()` produces an invocation_global whose value is a
1016	// comptime-known pointer to an underlying extern symbol's Nav.
1017	// The pointer is inlined at use sites so we don't need a Function-scope wrapper here.
1018	if (ip.indexToKey(val.toIntern()) == .ptr) alias: {
1019	const ptr_key = ip.indexToKey(val.toIntern()).ptr;
1020	if (ptr_key.base_addr != .nav or ptr_key.byte_offset != 0) break :alias;
1021	const underlying_nav = ip.getNav(ptr_key.base_addr.nav);
1022	if (!underlying_nav.resolved.?.is_extern_decl) break :alias;
1023	cg.declPtr(spv_decl_index).end_dep = cg.decl_deps.items.len;
1024	return;
1025	}
1026
1027	const ty_id = try cg.resolveType(ty, .indirect);
1028	const ptr_ty_id = try cg.ptrType(ty_id, .function);
1029
1030	// TODO: Combine with resolveAnonDecl?
1031	const void_ty_id = try cg.resolveType(.void, .direct);
1032	const initializer_proto_ty_id = try cg.functionType(void_ty_id, &.{});
1033
1034	const initializer_id = cg.allocId();
1035	try cg.prologue.emit(gpa, .OpFunction, .{
1036	.id_result_type = try cg.resolveType(.void, .direct),
1037	.id_result = initializer_id,
1038	.function_control = .{},
1039	.function_type = initializer_proto_ty_id,
1040	});
1041
1042	const root_block_id = cg.allocId();
1043	try cg.prologue.emit(gpa, .OpLabel, .{
1044	.id_result = root_block_id,
1045	});
1046	cg.block_label = root_block_id;
1047
1048	const val_id = try cg.constant(ty, val, .indirect);
1049	try cg.body.emit(gpa, .OpStore, .{
1050	.pointer = result_id,
1051	.object = val_id,
1052	});
1053
1054	try cg.body.emit(gpa, .OpReturn, {});
1055	try cg.body.emit(gpa, .OpFunctionEnd, {});
1056	try cg.sections.functions.append(gpa, cg.prologue);
1057	try cg.sections.functions.append(gpa, cg.body);
1058
1059	try cg.debugNameFmt(initializer_id, "initializer of {f}", .{nav.fqn.fmt(ip)});
1060	try cg.debugName(result_id, nav.fqn.toSlice(ip));
1061
1062	try cg.sections.globals.emit(gpa, .OpExtInst, .{
1063	.id_result_type = ptr_ty_id,
1064	.id_result = result_id,
1065	.set = try cg.importInstructionSet(.zig),
1066	.instruction = .{ .inst = @backingInt(spec.Zig.InvocationGlobal) },
1067	.id_ref_4 = &.{initializer_id},
1068	});
1069	},
1070	}
1071
1072	cg.declPtr(spv_decl_index).end_dep = cg.decl_deps.items.len;
1073	}
1074
1075	fn decorateLayout(cg: *CodeGen, ty: Type, ty_id: spec.Id) Error!void {
1076	const zcu = cg.zcu;
1077	const ip = &zcu.intern_pool;
1078	switch (ty.zigTypeTag(zcu)) {
1079	.array => {
1080	const elem_ty = ty.childType(zcu);
1081	if (!elem_ty.hasRuntimeBits(zcu)) return;
1082	try cg.decorate(ty_id, .{
1083	.array_stride = .{ .array_stride = @intCast(elem_ty.abiSize(zcu)) },
1084	});
1085	try cg.decorateLayout(elem_ty, try cg.resolveType(elem_ty, .indirect));
1086	},
1087	.vector => {
1088	const elem_ty = ty.childType(zcu);
1089	try cg.decorateLayout(elem_ty, try cg.resolveType(elem_ty, .indirect));
1090	if (cg.isSpvVector(ty)) return;
1091	try cg.decorate(ty_id, .{
1092	.array_stride = .{ .array_stride = @intCast(elem_ty.abiSize(zcu)) },
1093	});
1094	},
1095	.@"struct" => switch (ip.indexToKey(ty.toIntern())) {
1096	.struct_type => {
1097	const struct_type = ip.loadStructType(ty.toIntern());
1098	if (struct_type.layout == .@"packed") return;
1099	var it = struct_type.iterateRuntimeOrder(ip);
1100	var member: u32 = 0;
1101	while (it.next()) |field_index| {
1102	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
1103	if (!field_ty.hasRuntimeBits(zcu)) continue;
1104	const offset: u32 = @intCast(ty.structFieldOffset(field_index, zcu));
1105	try cg.decorateMember(ty_id, member, .{ .offset = .{ .byte_offset = offset } });
1106	try cg.decorateLayout(field_ty, try cg.resolveType(field_ty, .indirect));
1107	member += 1;
1108	}
1109	},
1110	.tuple_type => |tuple| {
1111	for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| {
1112	if (field_val != .none) continue;
1113	const ft: Type = .fromInterned(field_ty);
1114	if (ft.hasRuntimeBits(zcu)) try cg.decorateLayout(ft, try cg.resolveType(ft, .indirect));
1115	}
1116	},
1117	else => {},
1118	},
1119	.@"union" => {
1120	const union_obj = zcu.typeToUnion(ty).?;
1121	if (union_obj.layout == .@"packed") return;
1122	const layout = cg.unionLayout(ty);
1123	if (layout.tag_size != 0) {
1124	const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
1125	try cg.decorateLayout(tag_ty, try cg.resolveType(tag_ty, .indirect));
1126	}
1127	if (layout.has_payload) {
1128	try cg.decorateLayout(layout.payload_ty, try cg.resolveType(layout.payload_ty, .indirect));
1129	}
1130	const u8_id = try cg.resolveType(.u8, .direct);
1131	if (layout.payload_padding_size != 0) {
1132	const len_id = try cg.constInt(.u32, layout.payload_padding_size);
1133	const arr_id = try cg.arrayType(len_id, u8_id);
1134	try cg.decorate(arr_id, .{ .array_stride = .{ .array_stride = 1 } });
1135	}
1136	if (layout.padding_size != 0) {
1137	const len_id = try cg.constInt(.u32, layout.padding_size);
1138	const arr_id = try cg.arrayType(len_id, u8_id);
1139	try cg.decorate(arr_id, .{ .array_stride = .{ .array_stride = 1 } });
1140	}
1141	},
1142	.optional => {
1143	const payload_ty = ty.optionalChild(zcu);
1144	if (payload_ty.hasRuntimeBits(zcu)) try cg.decorateLayout(payload_ty, try cg.resolveType(payload_ty, .indirect));
1145	},
1146	.error_union => {
1147	const payload_ty = ty.errorUnionPayload(zcu);
1148	if (payload_ty.hasRuntimeBits(zcu)) try cg.decorateLayout(payload_ty, try cg.resolveType(payload_ty, .indirect));
1149	},
1150	else => {},
1151	}
1152	}
1153
1154	pub fn fail(cg: *CodeGen, comptime format: []const u8, args: anytype) Error {
1155	@branchHint(.cold);
1156	return cg.zcu.codegenFail(cg.owner_nav, format, args);
1157	}
1158
1159	pub fn todo(cg: *CodeGen, comptime format: []const u8, args: anytype) Error {
1160	return cg.fail("TODO (SPIR-V): " ++ format, args);
1161	}
1162
1163	/// This imports the "default" extended instruction set for the target
1164	/// For OpenCL, OpenCL.std.100. For Vulkan and OpenGL, GLSL.std.450.
1165	fn importExtendedSet(cg: *CodeGen) !Id {
1166	const target = cg.zcu.getTarget();
1167	return switch (target.os.tag) {
1168	.opencl, .amdhsa => try cg.importInstructionSet(.@"OpenCL.std"),
1169	.vulkan, .opengl => try cg.importInstructionSet(.@"GLSL.std.450"),
1170	else => unreachable,
1171	};
1172	}
1173
1174	/// Fetch the result-id for a previously generated instruction or constant.
1175	fn resolve(cg: *CodeGen, inst: Air.Inst.Ref) !Id {
1176	const zcu = cg.zcu;
1177	const ip = &zcu.intern_pool;
1178	if (inst.toInterned()) |val_ip_index| {
1179	const ty = cg.typeOf(inst);
1180	if (ty.zigTypeTag(zcu) == .@"fn") {
1181	const val_key = zcu.intern_pool.indexToKey(val_ip_index);
1182	const fn_nav = switch (val_key) {
1183	.@"extern" => |@"extern"| @"extern".owner_nav,
1184	.func => |func| func.owner_nav,
1185	else => unreachable,
1186	};
1187	const spv_decl_index = try cg.resolveNav(ip, fn_nav);
1188	try cg.decl_deps.append(cg.gpa, spv_decl_index);
1189	const decl = cg.declPtr(spv_decl_index);
1190	if (val_key == .@"extern") {
1191	const nav = ip.getNav(fn_nav);
1192	const nav_ty: Type = .fromInterned(nav.resolved.?.type);
1193	try emitExternFnStub(cg, nav, decl, nav_ty);
1194	}
1195	return decl.result_id;
1196	}
1197
1198	return try cg.constant(ty, .fromInterned(val_ip_index), .direct);
1199	}
1200	const index = inst.toIndex().?;
1201	return cg.inst_results.get(index).?; // Assertion means instruction does not dominate usage.
1202	}
1203
1204	fn resolveUav(cg: *CodeGen, val: InternPool.Index) !Id {
1205	const gpa = cg.gpa;
1206
1207	// TODO: This cannot be a function at this point, but it should probably be handled anyway.
1208
1209	const zcu = cg.zcu;
1210	const ty: Type = .fromInterned(zcu.intern_pool.typeOf(val));
1211	const ty_id = try cg.resolveType(ty, .indirect);
1212
1213	const spv_decl_index = blk: {
1214	const entry = try cg.uav_link.getOrPut(gpa, .{ val, .function });
1215	if (entry.found_existing) {
1216	try cg.addFunctionDep(entry.value_ptr.*, .function);
1217	return cg.declPtr(entry.value_ptr.*).result_id;
1218	}
1219
1220	const spv_decl_index = try cg.allocDecl(.invocation_global);
1221	try cg.addFunctionDep(spv_decl_index, .function);
1222	entry.value_ptr.* = spv_decl_index;
1223	break :blk spv_decl_index;
1224	};
1225
1226	// TODO: At some point we will be able to generate this all constant here, but then all of
1227	// constant() will need to be implemented such that it doesn't generate any at-runtime code.
1228	// NOTE: Because this is a global, we really only want to initialize it once. Therefore the
1229	// constant lowering of this value will need to be deferred to an initializer similar to
1230	// other globals.
1231
1232	const result_id = cg.declPtr(spv_decl_index).result_id;
1233
1234	{
1235	// Save the current state so that we can temporarily generate into a different function.
1236	// TODO: This should probably be made a little more robust.
1237	const func_prologue = cg.prologue;
1238	const func_body = cg.body;
1239	const block_label = cg.block_label;
1240	defer {
1241	cg.prologue = func_prologue;
1242	cg.body = func_body;
1243	cg.block_label = block_label;
1244	}
1245
1246	cg.prologue = .{};
1247	cg.body = .{};
1248	defer {
1249	cg.prologue.deinit(gpa);
1250	cg.body.deinit(gpa);
1251	}
1252
1253	const void_ty_id = try cg.resolveType(.void, .direct);
1254	const initializer_proto_ty_id = try cg.functionType(void_ty_id, &.{});
1255
1256	const initializer_id = cg.allocId();
1257	try cg.prologue.emit(gpa, .OpFunction, .{
1258	.id_result_type = try cg.resolveType(.void, .direct),
1259	.id_result = initializer_id,
1260	.function_control = .{},
1261	.function_type = initializer_proto_ty_id,
1262	});
1263	const root_block_id = cg.allocId();
1264	try cg.prologue.emit(gpa, .OpLabel, .{
1265	.id_result = root_block_id,
1266	});
1267	cg.block_label = root_block_id;
1268
1269	const val_id = try cg.constant(ty, .fromInterned(val), .indirect);
1270	try cg.body.emit(gpa, .OpStore, .{
1271	.pointer = result_id,
1272	.object = val_id,
1273	});
1274
1275	try cg.body.emit(gpa, .OpReturn, {});
1276	try cg.body.emit(gpa, .OpFunctionEnd, {});
1277
1278	try cg.sections.functions.append(gpa, cg.prologue);
1279	try cg.sections.functions.append(gpa, cg.body);
1280
1281	try cg.debugNameFmt(initializer_id, "initializer of __anon_{d}", .{@backingInt(val)});
1282
1283	const fn_decl_ptr_ty_id = try cg.ptrType(ty_id, .function);
1284	try cg.sections.globals.emit(gpa, .OpExtInst, .{
1285	.id_result_type = fn_decl_ptr_ty_id,
1286	.id_result = result_id,
1287	.set = try cg.importInstructionSet(.zig),
1288	.instruction = .{ .inst = @backingInt(spec.Zig.InvocationGlobal) },
1289	.id_ref_4 = &.{initializer_id},
1290	});
1291	}
1292
1293	return result_id;
1294	}
1295
1296	fn resolvePtr(cg: *CodeGen, ref: Air.Inst.Ref) !Ptr {
1297	const id = try cg.resolve(ref);
1298	if (cg.tracked_allocas.getPtr(id)) |slot| return .{ .tracked = .{ .id = id, .slot = slot } };
1299	return .{ .id = id };
1300	}
1301
1302	fn addFunctionDep(cg: *CodeGen, decl_index: Decl.Index, storage_class: StorageClass) !void {
1303	const gpa = cg.gpa;
1304	const target = cg.zcu.getTarget();
1305	if (target.cpu.has(.spirv, .v1_4)) {
1306	try cg.decl_deps.append(gpa, decl_index);
1307	} else {
1308	// Before version 1.4, the interface’s storage classes are limited to the Input and Output
1309	if (storage_class == .input or storage_class == .output) {
1310	try cg.decl_deps.append(gpa, decl_index);
1311	}
1312	}
1313	}
1314
1315	/// Start a new SPIR-V block, Emits the label of the new block, and stores which
1316	/// block we are currently generating.
1317	/// Note that there is no such thing as nested blocks like in ZIR or AIR, so we don't need to
1318	/// keep track of the previous block.
1319	fn beginSpvBlock(cg: *CodeGen, label: Id) !void {
1320	try cg.body.emit(cg.gpa, .OpLabel, .{ .id_result = label });
1321	cg.block_label = label;
1322	cg.block_terminated = false;
1323	}
1324
1325	const ArithmeticTypeInfo = struct {
1326	const Class = enum {
1327	bool,
1328	/// A regular, **native**, integer.
1329	/// This is only returned when the backend supports this int as a native type (when
1330	/// the relevant capability is enabled).
1331	integer,
1332	/// A regular float. These are all required to be natively supported. Floating points
1333	/// for which the relevant capability is not enabled are not emulated.
1334	float,
1335	/// An integer of a 'strange' size (which' bit size is not the same as its backing
1336	/// type. **Note**: this may **also** include power-of-2 integers for which the
1337	/// relevant capability is not enabled), but still within the limits of the largest
1338	/// natively supported integer type.
1339	strange_integer,
1340	/// An integer with more bits than the largest natively supported integer type.
1341	composite_integer,
1342	};
1343
1344	/// A classification of the inner type.
1345	/// These scenarios will all have to be handled slightly different.
1346	class: Class,
1347	/// The number of bits in the inner type.
1348	/// This is the actual number of bits of the type, not the size of the backing integer.
1349	bits: u16,
1350	/// The number of bits required to store the type.
1351	/// For `integer` and `float`, this is equal to `bits`.
1352	/// For `strange_integer` and `bool` this is the size of the backing integer.
1353	/// For `composite_integer` this is the elements count.
1354	backing_bits: u16,
1355	/// Null if this type is a scalar, or the length of the vector otherwise.
1356	vector_len: ?u32,
1357	/// Whether the inner type is signed. Only relevant for integers.
1358	signedness: std.lang.Signedness,
1359	};
1360
1361	fn arithmeticTypeInfo(cg: *CodeGen, ty: Type) ArithmeticTypeInfo {
1362	const zcu = cg.zcu;
1363	const target = cg.zcu.getTarget();
1364	var scalar_ty = ty.scalarType(zcu);
1365	if (scalar_ty.zigTypeTag(zcu) == .@"enum") {
1366	scalar_ty = scalar_ty.backingIntType(zcu);
1367	}
1368	const vector_len = if (ty.isVector(zcu)) ty.vectorLen(zcu) else null;
1369	return switch (scalar_ty.zigTypeTag(zcu)) {
1370	.bool => .{
1371	.bits = 1, // Doesn't matter for this class.
1372	.backing_bits = cg.backingIntBits(1).@"0",
1373	.vector_len = vector_len,
1374	.signedness = .unsigned, // Technically, but doesn't matter for this class.
1375	.class = .bool,
1376	},
1377	.float => .{
1378	.bits = scalar_ty.floatBits(target),
1379	.backing_bits = scalar_ty.floatBits(target), // TODO: F80?
1380	.vector_len = vector_len,
1381	.signedness = .signed, // Technically, but doesn't matter for this class.
1382	.class = .float,
1383	},
1384	.int => blk: {
1385	const int_info = scalar_ty.intInfo(zcu);
1386	// TODO: Maybe it's useful to also return this value.
1387	const backing_bits, const big_int = cg.backingIntBits(int_info.bits);
1388	break :blk .{
1389	.bits = int_info.bits,
1390	.backing_bits = backing_bits,
1391	.vector_len = vector_len,
1392	.signedness = int_info.signedness,
1393	.class = class: {
1394	if (big_int) break :class .composite_integer;
1395	break :class if (backing_bits == int_info.bits) .integer else .strange_integer;
1396	},
1397	};
1398	},
1399	.@"enum" => unreachable,
1400	.vector => unreachable,
1401	else => unreachable, // Unhandled arithmetic type
1402	};
1403	}
1404
1405	/// Checks whether the type can be directly translated to SPIR-V vectors
1406	fn isSpvVector(cg: *CodeGen, ty: Type) bool {
1407	const zcu = cg.zcu;
1408	const target = cg.zcu.getTarget();
1409	if (ty.zigTypeTag(zcu) != .vector) return false;
1410
1411	// TODO: This check must be expanded for types that can be represented
1412	// as integers (enums / packed structs?) and types that are represented
1413	// by multiple SPIR-V values.
1414	const scalar_ty = ty.scalarType(zcu);
1415	switch (scalar_ty.zigTypeTag(zcu)) {
1416	.bool,
1417	.int,
1418	.float,
1419	=> {},
1420	else => return false,
1421	}
1422
1423	const elem_ty = ty.childType(zcu);
1424	const len = ty.vectorLen(zcu);
1425
1426	if (elem_ty.isNumeric(zcu) or elem_ty.toIntern() == .bool_type) {
1427	if (len > 1 and len <= 4) return true;
1428	if (target.cpu.has(.spirv, .vector16)) return (len == 8 or len == 16);
1429	}
1430
1431	return false;
1432	}
1433
1434	/// Emits a bool constant in a particular representation.
1435	fn constBool(cg: *CodeGen, value: bool, repr: Repr) !Id {
1436	switch (repr) {
1437	.indirect => return cg.constInt(.u1, @intFromBool(value)),
1438	.direct => {
1439	const result_ty_id = try cg.boolType();
1440	const result_id = cg.allocId();
1441	switch (value) {
1442	inline else => |value_ct| try cg.sections.globals.emit(
1443	cg.gpa,
1444	if (value_ct) .OpConstantTrue else .OpConstantFalse,
1445	.{ .id_result_type = result_ty_id, .id_result = result_id },
1446	),
1447	}
1448	return result_id;
1449	},
1450	}
1451	}
1452
1453	/// Emits an integer constant.
1454	/// This function, unlike cg.constInt, takes care to bitcast
1455	/// the value to an unsigned int first for Kernels.
1456	fn constInt(cg: *CodeGen, ty: Type, value: anytype) !Id {
1457	const gpa = cg.gpa;
1458	const zcu = cg.zcu;
1459	const target = cg.zcu.getTarget();
1460	const scalar_ty = ty.scalarType(zcu);
1461	const int_info = scalar_ty.intInfo(zcu);
1462	// Use backing bits so that negatives are sign extended
1463	const backing_bits, const big_int = cg.backingIntBits(int_info.bits);
1464	assert(backing_bits != 0); // u0 is comptime
1465
1466	const result_ty_id = try cg.resolveType(scalar_ty, .indirect);
1467	const signedness: Signedness = switch (@typeInfo(@TypeOf(value))) {
1468	.int => |int| int.signedness,
1469	.comptime_int => if (value < 0) .signed else .unsigned,
1470	else => unreachable,
1471	};
1472	if (@TypeOf(value) != comptime_int and @sizeOf(@TypeOf(value)) >= 4 and big_int) {
1473	const value64: u64 = switch (signedness) {
1474	.signed => @bitCast(@as(i64, @intCast(value))),
1475	.unsigned => @as(u64, @intCast(value)),
1476	};
1477	const n_limbs = backing_bits / cg.bigIntBits();
1478	const fill: u32 = if (signedness == .signed and value < 0) 0xFFFFFFFF else 0;
1479	const scratch_top = cg.id_scratch.items.len;
1480	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
1481	const constituents = try cg.id_scratch.addManyAsSlice(gpa, n_limbs);
1482	for (constituents, 0..) |*c, i| {
1483	c.* = try cg.constInt(
1484	.u32,
1485	if (i < 2) @as(u32, @truncate(value64 >> @intCast(i * 32))) else fill,
1486	);
1487	}
1488	return cg.constructComposite(result_ty_id, constituents);
1489	}
1490
1491	const final_value: spec.LiteralContextDependentNumber = switch (target.os.tag) {
1492	.opencl, .amdhsa => blk: {
1493	const value64: u64 = switch (signedness) {
1494	.signed => @bitCast(@as(i64, @intCast(value))),
1495	.unsigned => @as(u64, @intCast(value)),
1496	};
1497
1498	// Manually truncate the value to the right amount of bits.
1499	const truncated_value = if (backing_bits == 64)
1500	value64
1501	else
1502	value64 & (@as(u64, 1) << @intCast(backing_bits)) - 1;
1503
1504	break :blk switch (backing_bits) {
1505	1...32 => .{ .uint32 = @truncate(truncated_value) },
1506	33...64 => .{ .uint64 = truncated_value },
1507	else => unreachable,
1508	};
1509	},
1510	else => switch (backing_bits) {
1511	1...32 => if (signedness == .signed) .{ .int32 = @intCast(value) } else .{ .uint32 = @intCast(value) },
1512	33...64 => if (signedness == .signed) .{ .int64 = value } else .{ .uint64 = value },
1513	else => unreachable,
1514	},
1515	};
1516
1517	const result_id = cg.allocId();
1518	try cg.sections.globals.emit(cg.gpa, .OpConstant, .{
1519	.id_result_type = result_ty_id,
1520	.id_result = result_id,
1521	.value = final_value,
1522	});
1523
1524	if (!ty.isVector(zcu)) return result_id;
1525	return cg.constructCompositeSplat(ty, result_id);
1526	}
1527
1528	/// Construct a composite value from its constituents.
1529	/// In logical addressing mode (Vulkan/OpenGL), OpCompositeConstruct cannot accept
1530	/// pointer operands, so for struct types we use alloc, store for each field and load instead.
1531	pub fn constructComposite(cg: *CodeGen, result_ty_id: Id, constituents: []const Id) !Id {
1532	const gpa = cg.gpa;
1533
1534	const maybe_fields: ?[]const Id = for (cg.struct_types.keys(), cg.struct_types.values()) |key, val| {
1535	if (val == result_ty_id) break key.fields;
1536	} else null;
1537	if (maybe_fields) |fields| {
1538	assert(fields.len == constituents.len);
1539	const u32_ty_id = try cg.intType(.unsigned, 32);
1540	const var_id = try cg.alloc(result_ty_id, null);
1541	for (fields, constituents, 0..) |field_ty_id, constituent, i| {
1542	const field_ptr_ty_id = try cg.ptrType(field_ty_id, .function);
1543	const index_id = cg.allocId();
1544	try cg.sections.globals.emit(gpa, .OpConstant, .{
1545	.id_result_type = u32_ty_id,
1546	.id_result = index_id,
1547	.value = .{ .uint32 = @intCast(i) },
1548	});
1549	const field_ptr = try cg.accessChainId(field_ptr_ty_id, var_id, &.{index_id});
1550	try cg.body.emit(gpa, .OpStore, .{
1551	.pointer = field_ptr,
1552	.object = constituent,
1553	});
1554	}
1555	const result_id = cg.allocId();
1556	try cg.body.emit(gpa, .OpLoad, .{
1557	.id_result_type = result_ty_id,
1558	.id_result = result_id,
1559	.pointer = var_id,
1560	});
1561	return result_id;
1562	}
1563
1564	const result_id = cg.allocId();
1565	try cg.body.emit(gpa, .OpCompositeConstruct, .{
1566	.id_result_type = result_ty_id,
1567	.id_result = result_id,
1568	.constituents = constituents,
1569	});
1570	return result_id;
1571	}
1572
1573	/// Construct a composite at runtime with all lanes set to the same value.
1574	/// ty must be an aggregate type.
1575	fn constructCompositeSplat(cg: *CodeGen, ty: Type, constituent: Id) !Id {
1576	const gpa = cg.gpa;
1577	const zcu = cg.zcu;
1578	const n: usize = @intCast(ty.arrayLen(zcu));
1579
1580	const scratch_top = cg.id_scratch.items.len;
1581	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
1582
1583	const constituents = try cg.id_scratch.addManyAsSlice(gpa, n);
1584	@memset(constituents, constituent);
1585
1586	const result_ty_id = try cg.resolveType(ty, .direct);
1587	return cg.constructComposite(result_ty_id, constituents);
1588	}
1589
1590	/// This function generates a load for a constant in direct (ie, non-memory) representation.
1591	/// When the constant is simple, it can be generated directly using OpConstant instructions.
1592	/// When the constant is more complicated however, it needs to be constructed using multiple values. This
1593	/// is done by emitting a sequence of instructions that initialize the value.
1594	//
1595	/// This function should only be called during function code generation.
1596	fn constant(cg: *CodeGen, ty: Type, val: Value, repr: Repr) Error!Id {
1597	const gpa = cg.gpa;
1598
1599	const pt = cg.pt;
1600	const zcu = cg.zcu;
1601	const target = cg.zcu.getTarget();
1602	const result_ty_id = try cg.resolveType(ty, repr);
1603	const ip = &zcu.intern_pool;
1604
1605	log.debug("lowering constant: ty = {f}, val = {f}, key = {s}", .{ ty.fmt(pt), val.fmtValue(pt), @tagName(ip.indexToKey(val.toIntern())) });
1606	if (val.isUndef(zcu)) {
1607	return cg.constUndef(result_ty_id);
1608	}
1609
1610	const cacheable_id = cache: {
1611	switch (ip.indexToKey(val.toIntern())) {
1612	.int_type,
1613	.ptr_type,
1614	.array_type,
1615	.vector_type,
1616	.opt_type,
1617	.anyframe_type,
1618	.error_union_type,
1619	.simple_type,
1620	.struct_type,
1621	.tuple_type,
1622	.union_type,
1623	.opaque_type,
1624	.spirv_type,
1625	.enum_type,
1626	.func_type,
1627	.error_set_type,
1628	.inferred_error_set_type,
1629	=> unreachable, // types, not values
1630
1631	.undef => unreachable, // handled above
1632
1633	.@"extern",
1634	.func,
1635	.enum_literal,
1636	=> unreachable, // non-runtime values
1637
1638	.simple_value => |simple_value| switch (simple_value) {
1639	.void,
1640	.null,
1641	.@"unreachable",
1642	=> unreachable, // non-runtime values
1643
1644	.false, .true => break :cache try cg.constBool(val.toBool(), repr),
1645	},
1646	.int => {
1647	const int_info = ty.intInfo(zcu);
1648	const backing_bits, const is_big_int = cg.backingIntBits(int_info.bits);
1649	if (is_big_int) {
1650	const limb_bits = cg.bigIntBits();
1651	const n_limbs = backing_bits / limb_bits;
1652	const big_result_ty_id = try cg.resolveType(ty, .indirect);
1653	var bigint_space: Value.BigIntSpace = undefined;
1654	const bigint = val.toBigInt(&bigint_space, zcu);
1655	const limb_bytes = try gpa.alloc(u8, backing_bits / 8);
1656	defer gpa.free(limb_bytes);
1657	bigint.writeTwosComplement(limb_bytes, .little);
1658	const scratch_top = cg.id_scratch.items.len;
1659	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
1660	const constituents = try cg.id_scratch.addManyAsSlice(gpa, n_limbs);
1661	switch (limb_bits) {
1662	32 => {
1663	const limbs_u32: []u32 = @ptrCast(@alignCast(limb_bytes));
1664	for (constituents, limbs_u32) |*c, v| {
1665	const host_v = if (builtin.cpu.arch.endian() == .big) @byteSwap(v) else v;
1666	c.* = try cg.constInt(.u32, host_v);
1667	}
1668	},
1669	64 => {
1670	const limbs_u64: []u64 = @ptrCast(@alignCast(limb_bytes));
1671	for (constituents, limbs_u64) |*c, v| {
1672	const host_v = if (builtin.cpu.arch.endian() == .big) @byteSwap(v) else v;
1673	c.* = try cg.constInt(.u64, host_v);
1674	}
1675	},
1676	else => unreachable,
1677	}
1678	break :cache try cg.constructComposite(big_result_ty_id, constituents);
1679	}
1680	if (ty.isSignedInt(zcu)) {
1681	break :cache try cg.constInt(ty, val.toSignedInt(zcu));
1682	} else {
1683	break :cache try cg.constInt(ty, val.toUnsignedInt(zcu));
1684	}
1685	},
1686	.float => {
1687	const lit: spec.LiteralContextDependentNumber = switch (ty.floatBits(target)) {
1688	16 => .{ .uint32 = @as(u16, @bitCast(val.toFloat(f16, zcu))) },
1689	32 => .{ .float32 = val.toFloat(f32, zcu) },
1690	64 => .{ .float64 = val.toFloat(f64, zcu) },
1691	80, 128 => unreachable, // TODO
1692	else => unreachable,
1693	};
1694	const lit_id = cg.allocId();
1695	try cg.sections.globals.emit(gpa, .OpConstant, .{
1696	.id_result_type = result_ty_id,
1697	.id_result = lit_id,
1698	.value = lit,
1699	});
1700	break :cache lit_id;
1701	},
1702	.err => |err| {
1703	const value = try pt.getErrorValue(err.name);
1704	break :cache try cg.constInt(ty, value);
1705	},
1706	.error_union => |error_union| {
1707	// TODO: Error unions may be constructed with constant instructions if the payload type
1708	// allows it. For now, just generate it here regardless.
1709	const err_ty = ty.errorUnionSet(zcu);
1710	const payload_ty = ty.errorUnionPayload(zcu);
1711	const err_val_id = switch (error_union.val) {
1712	.err_name => |err_name| try cg.constInt(
1713	err_ty,
1714	try pt.getErrorValue(err_name),
1715	),
1716	.payload => try cg.constInt(err_ty, 0),
1717	};
1718	const eu_layout = cg.errorUnionLayout(payload_ty);
1719	if (!eu_layout.payload_has_bits) {
1720	// We use the error type directly as the type.
1721	break :cache err_val_id;
1722	}
1723
1724	const payload_val_id = switch (error_union.val) {
1725	.err_name => try cg.constant(payload_ty, .undef, .indirect),
1726	.payload => |p| try cg.constant(payload_ty, .fromInterned(p), .indirect),
1727	};
1728
1729	var constituents: [2]Id = undefined;
1730	var types: [2]Type = undefined;
1731	if (eu_layout.error_first) {
1732	constituents[0] = err_val_id;
1733	constituents[1] = payload_val_id;
1734	types = .{ err_ty, payload_ty };
1735	} else {
1736	constituents[0] = payload_val_id;
1737	constituents[1] = err_val_id;
1738	types = .{ payload_ty, err_ty };
1739	}
1740
1741	const comp_ty_id = try cg.resolveType(ty, .direct);
1742	return try cg.constructComposite(comp_ty_id, &constituents);
1743	},
1744	.enum_tag => {
1745	const int_val = val.backingInt(zcu);
1746	const int_ty = ty.backingIntType(zcu);
1747	break :cache try cg.constant(int_ty, int_val, repr);
1748	},
1749	.ptr => return cg.constantPtr(val),
1750	.slice => |slice| {
1751	const ptr_id = try cg.constantPtr(.fromInterned(slice.ptr));
1752	const len_id = try cg.constant(.usize, .fromInterned(slice.len), .indirect);
1753	const comp_ty_id = try cg.resolveType(ty, .direct);
1754	return try cg.constructComposite(comp_ty_id, &.{ ptr_id, len_id });
1755	},
1756	.opt => {
1757	const payload_ty = ty.optionalChild(zcu);
1758	const maybe_payload_val = val.optionalValue(zcu);
1759
1760	if (!payload_ty.hasRuntimeBits(zcu)) {
1761	break :cache try cg.constBool(maybe_payload_val != null, .indirect);
1762	} else if (ty.optionalReprIsPayload(zcu)) {
1763	// Optional representation is a nullable pointer or slice.
1764	if (maybe_payload_val) |payload_val| {
1765	return try cg.constant(payload_ty, payload_val, .indirect);
1766	} else {
1767	break :cache try cg.constNull(result_ty_id);
1768	}
1769	}
1770
1771	// Optional representation is a structure.
1772	// { Payload, Bool }
1773
1774	const has_pl_id = try cg.constBool(maybe_payload_val != null, .indirect);
1775	const payload_id = if (maybe_payload_val) |payload_val|
1776	try cg.constant(payload_ty, payload_val, .indirect)
1777	else
1778	try cg.constUndef(try cg.resolveType(payload_ty, .indirect));
1779
1780	const comp_ty_id = try cg.resolveType(ty, .direct);
1781	return try cg.constructComposite(comp_ty_id, &.{ payload_id, has_pl_id });
1782	},
1783	.aggregate => |aggregate| switch (ip.indexToKey(ty.ip_index)) {
1784	inline .array_type, .vector_type => |array_type, tag| {
1785	const elem_ty: Type = .fromInterned(array_type.child);
1786
1787	const scratch_top = cg.id_scratch.items.len;
1788	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
1789	const constituents = try cg.id_scratch.addManyAsSlice(gpa, @intCast(ty.arrayLenIncludingSentinel(zcu)));
1790
1791	const child_repr: Repr = switch (tag) {
1792	.array_type => .indirect,
1793	.vector_type => .direct,
1794	else => unreachable,
1795	};
1796
1797	switch (aggregate.storage) {
1798	.bytes => |bytes| {
1799	// TODO: This is really space inefficient, perhaps there is a better
1800	// way to do it?
1801	for (constituents, bytes.toSlice(constituents.len, ip)) |*constituent, byte| {
1802	constituent.* = try cg.constInt(elem_ty, byte);
1803	}
1804	},
1805	.elems => |elems| {
1806	for (constituents, elems) |*constituent, elem| {
1807	constituent.* = try cg.constant(elem_ty, .fromInterned(elem), child_repr);
1808	}
1809	},
1810	.repeated_elem => |elem| {
1811	@memset(constituents, try cg.constant(elem_ty, .fromInterned(elem), child_repr));
1812	},
1813	}
1814
1815	const comp_ty_id = try cg.resolveType(ty, .direct);
1816	return cg.constructComposite(comp_ty_id, constituents);
1817	},
1818	.struct_type => {
1819	const struct_type = zcu.typeToStruct(ty).?;
1820	assert(struct_type.layout != .@"packed"); // packed structs use `bitpack`
1821
1822	var types: std.ArrayList(Type) = .empty;
1823	defer types.deinit(gpa);
1824
1825	var constituents: std.ArrayList(Id) = .empty;
1826	defer constituents.deinit(gpa);
1827
1828	var it = struct_type.iterateRuntimeOrder(ip);
1829	while (it.next()) |field_index| {
1830	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
1831	if (!field_ty.hasRuntimeBits(zcu)) {
1832	// This is a zero-bit field - we only needed it for the alignment.
1833	continue;
1834	}
1835
1836	// TODO: Padding?
1837	const field_val = try val.fieldValue(pt, field_index);
1838	const field_id = try cg.constant(field_ty, field_val, .indirect);
1839
1840	try types.append(gpa, field_ty);
1841	try constituents.append(gpa, field_id);
1842	}
1843
1844	const comp_ty_id = try cg.resolveType(ty, .direct);
1845	return try cg.constructComposite(comp_ty_id, constituents.items);
1846	},
1847	.tuple_type => |tuple| {
1848	var constituents: std.ArrayList(Id) = .empty;
1849	defer constituents.deinit(gpa);
1850
1851	for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty, field_val, i| {
1852	if (field_val != .none) continue;
1853	const ft: Type = .fromInterned(field_ty);
1854	if (!ft.hasRuntimeBits(zcu)) continue;
1855
1856	const fv = try val.fieldValue(pt, i);
1857	const field_id = try cg.constant(ft, fv, .indirect);
1858	try constituents.append(gpa, field_id);
1859	}
1860
1861	const comp_ty_id = try cg.resolveType(ty, .direct);
1862	return try cg.constructComposite(comp_ty_id, constituents.items);
1863	},
1864	else => unreachable,
1865	},
1866	.un => |un| {
1867	assert(ty.containerLayout(zcu) != .@"packed"); // packed unions use `bitpack`
1868	if (un.tag == .none) {
1869	@panic("TODO");
1870	}
1871	const active_field = ty.unionTagFieldIndex(.fromInterned(un.tag), zcu).?;
1872	const union_obj = zcu.typeToUnion(ty).?;
1873	const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[active_field]);
1874	const payload = if (field_ty.hasRuntimeBits(zcu))
1875	try cg.constant(field_ty, .fromInterned(un.val), .direct)
1876	else
1877	null;
1878	return try cg.unionInit(ty, active_field, payload);
1879	},
1880	.bitpack => |bitpack| {
1881	const int_val: Value = .fromInterned(bitpack.backing_int_val);
1882	break :cache try cg.constant(int_val.typeOf(zcu), int_val, repr);
1883	},
1884
1885	.memoized_call => unreachable,
1886	}
1887	};
1888	return cacheable_id;
1889	}
1890
1891	fn constantPtr(cg: *CodeGen, ptr_val: Value) !Id {
1892	const pt = cg.pt;
1893	const zcu = cg.zcu;
1894	const gpa = cg.gpa;
1895
1896	if (ptr_val.isUndef(zcu)) {
1897	const result_ty = ptr_val.typeOf(zcu);
1898	const result_ty_id = try cg.resolveType(result_ty, .direct);
1899	return cg.constUndef(result_ty_id);
1900	}
1901
1902	var arena = std.heap.ArenaAllocator.init(gpa);
1903	defer arena.deinit();
1904
1905	const derivation = try ptr_val.pointerDerivation(arena.allocator(), pt, null);
1906	return cg.derivePtr(derivation);
1907	}
1908
1909	fn derivePtr(cg: *CodeGen, derivation: Value.PointerDeriveStep) !Id {
1910	const gpa = cg.gpa;
1911	const pt = cg.pt;
1912	const zcu = cg.zcu;
1913	const target = zcu.getTarget();
1914	switch (derivation) {
1915	.comptime_alloc_ptr, .comptime_field_ptr => unreachable,
1916	.int => |int| {
1917	if (target.os.tag != .opencl) {
1918	if (int.ptr_ty.ptrAddressSpace(zcu) != .physical_storage_buffer) {
1919	return cg.fail(
1920	"cannot cast integer to pointer with address space '{s}'",
1921	.{@tagName(int.ptr_ty.ptrAddressSpace(zcu))},
1922	);
1923	}
1924	}
1925	const result_ty_id = try cg.resolveType(int.ptr_ty, .direct);
1926	// TODO: This can probably be an OpSpecConstantOp Bitcast, but
1927	// that is not implemented by Mesa yet. Therefore, just generate it
1928	// as a runtime operation.
1929	const result_ptr_id = cg.allocId();
1930	const value_id = try cg.constInt(.usize, int.addr);
1931	try cg.body.emit(gpa, .OpConvertUToPtr, .{
1932	.id_result_type = result_ty_id,
1933	.id_result = result_ptr_id,
1934	.integer_value = value_id,
1935	});
1936	return result_ptr_id;
1937	},
1938	.nav_ptr => |nav_index| {
1939	const ip = &zcu.intern_pool;
1940	const result_ptr_ty = try pt.navPtrType(nav_index);
1941	const ty_id = try cg.resolveType(result_ptr_ty, .direct);
1942	const nav = ip.getNav(nav_index);
1943	const nav_ty: Type = .fromInterned(nav.resolved.?.type);
1944
1945	switch (nav.resolved.?.value) {
1946	.none => {},
1947	else => |value| switch (ip.indexToKey(value)) {
1948	// TODO: Properly lower function pointers; for now substitute undef.
1949	.func => return try cg.constUndef(ty_id),
1950	.@"extern" => if (ip.isFunctionType(nav_ty.toIntern())) {
1951	const spv_decl_index = try cg.resolveNav(ip, nav_index);
1952	const decl = cg.declPtr(spv_decl_index);
1953	try emitExternFnStub(cg, nav, decl, nav_ty);
1954	return decl.result_id;
1955	},
1956	else => {},
1957	},
1958	}
1959
1960	if (!nav_ty.hasRuntimeBits(zcu) and nav_ty.zigTypeTag(zcu) != .spirv) {
1961	return cg.constUndef(ty_id);
1962	}
1963
1964	const spv_decl_index = try cg.resolveNav(ip, nav_index);
1965	const spv_decl = cg.declPtr(spv_decl_index);
1966	assert(spv_decl.kind != .func);
1967	const storage_class = cg.storageClass(nav.resolved.?.@"addrspace");
1968	try cg.addFunctionDep(spv_decl_index, storage_class);
1969
1970	const nav_ty_id = try cg.resolveType(nav_ty, .indirect);
1971	const decl_ptr_ty_id = try cg.ptrType(nav_ty_id, storage_class);
1972	if (cg.needsLayout(nav.resolved.?.@"addrspace", nav_ty)) {
1973	try cg.block_var_ids.put(gpa, spv_decl.result_id, {});
1974	}
1975	if (decl_ptr_ty_id == ty_id) return spv_decl.result_id;
1976	switch (target.os.tag) {
1977	.vulkan, .opengl => return spv_decl.result_id,
1978	else => {},
1979	}
1980	const casted_ptr_id = cg.allocId();
1981	try cg.body.emit(gpa, .OpBitcast, .{
1982	.id_result_type = ty_id,
1983	.id_result = casted_ptr_id,
1984	.operand = spv_decl.result_id,
1985	});
1986	return casted_ptr_id;
1987	},
1988	.uav_ptr => |uav| {
1989	const ip = &zcu.intern_pool;
1990	const result_ptr_ty: Type = .fromInterned(uav.orig_ty);
1991	const ty_id = try cg.resolveType(result_ptr_ty, .direct);
1992	const uav_ty: Type = .fromInterned(ip.typeOf(uav.val));
1993
1994	switch (ip.indexToKey(uav.val)) {
1995	.func => unreachable, // TODO
1996	.@"extern" => assert(!ip.isFunctionType(uav_ty.toIntern())),
1997	else => {},
1998	}
1999
2000	if (!uav_ty.hasRuntimeBits(zcu) and uav_ty.zigTypeTag(zcu) != .spirv) {
2001	return cg.constUndef(ty_id);
2002	}
2003
2004	// Uav refs are always generic.
2005	assert(result_ptr_ty.ptrAddressSpace(zcu) == .generic);
2006	const uav_ty_id = try cg.resolveType(uav_ty, .indirect);
2007	const decl_ptr_ty_id = try cg.ptrType(uav_ty_id, .function);
2008	const ptr_id = try cg.resolveUav(uav.val);
2009
2010	if (decl_ptr_ty_id == ty_id) return ptr_id;
2011	switch (target.os.tag) {
2012	.vulkan, .opengl => return ptr_id,
2013	else => {},
2014	}
2015	const casted_ptr_id = cg.allocId();
2016	try cg.body.emit(gpa, .OpBitcast, .{
2017	.id_result_type = ty_id,
2018	.id_result = casted_ptr_id,
2019	.operand = ptr_id,
2020	});
2021	return casted_ptr_id;
2022	},
2023	.eu_payload_ptr => @panic("TODO"),
2024	.opt_payload_ptr => @panic("TODO"),
2025	.field_ptr => |field| {
2026	const parent_ptr_id = try cg.derivePtr(field.parent.*);
2027	const parent_ptr_ty = try field.parent.ptrType(pt);
2028	return cg.structFieldPtr(field.result_ptr_ty, parent_ptr_ty, parent_ptr_id, field.field_idx);
2029	},
2030	.elem_ptr => |elem| {
2031	const parent_ptr_id = try cg.derivePtr(elem.parent.*);
2032	const parent_ptr_ty = try elem.parent.ptrType(pt);
2033	const index_id = try cg.constInt(.usize, elem.elem_idx);
2034	return cg.ptrElemPtr(parent_ptr_ty, parent_ptr_id, index_id);
2035	},
2036	.offset_and_cast => |oac| {
2037	const parent_ptr_id = try cg.derivePtr(oac.parent.*);
2038	const parent_ptr_ty = try oac.parent.ptrType(pt);
2039
2040	if (oac.new_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {
2041	return parent_ptr_id;
2042	}
2043
2044	if (oac.byte_offset == 0) {
2045	var depth: u32 = 0;
2046	var cur = parent_ptr_ty.childType(zcu);
2047	const dst_child = oac.new_ptr_ty.childType(zcu);
2048	while (cur.toIntern() != dst_child.toIntern()) {
2049	switch (cur.zigTypeTag(zcu)) {
2050	.array => {
2051	if (dst_child.zigTypeTag(zcu) == .array and
2052	dst_child.childType(zcu).toIntern() == cur.childType(zcu).toIntern() and
2053	dst_child.arrayLenIncludingSentinel(zcu) <= cur.arrayLenIncludingSentinel(zcu))
2054	{
2055	cur = dst_child;
2056	break;
2057	}
2058	cur = cur.childType(zcu);
2059	depth += 1;
2060	},
2061	.@"struct" => {
2062	if (cur.structFieldCount(zcu) == 0) break;
2063	if (cur.structFieldOffset(0, zcu) != 0) break;
2064	cur = cur.fieldType(0, zcu);
2065	depth += 1;
2066	},
2067	else => break,
2068	}
2069	}
2070	if (cur.toIntern() == dst_child.toIntern()) {
2071	if (depth != 0) {
2072	const as = oac.new_ptr_ty.ptrAddressSpace(zcu);
2073	const child_ty_id = try cg.pointeeType(as, dst_child, false);
2074	const result_ty_id = try cg.ptrType(child_ty_id, cg.storageClass(as));
2075	const scratch_top = cg.id_scratch.items.len;
2076	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
2077	const zero = try cg.constInt(.u32, 0);
2078	const ids = try cg.id_scratch.addManyAsSlice(gpa, depth);
2079	@memset(ids, zero);
2080	return cg.accessChainId(result_ty_id, parent_ptr_id, ids);
2081	} else {
2082	return parent_ptr_id;
2083	}
2084	}
2085	const result_ty_id = try cg.resolveType(oac.new_ptr_ty, .direct);
2086	if (target.os.tag == .opencl) {
2087	const result_ptr_id = cg.allocId();
2088	try cg.body.emit(gpa, .OpBitcast, .{
2089	.id_result_type = result_ty_id,
2090	.id_result = result_ptr_id,
2091	.operand = parent_ptr_id,
2092	});
2093	return result_ptr_id;
2094	}
2095	}
2096
2097	return cg.fail("cannot cast pointer '{f}' to '{f}'", .{
2098	parent_ptr_ty.fmt(pt),
2099	oac.new_ptr_ty.fmt(pt),
2100	});
2101	},
2102	}
2103	}
2104
2105	/// Emit a stub OpFunction/OpFunctionEnd + Import linkage decoration for an
2106	/// extern function so the module is structurally valid. The stub will be
2107	/// replaced by the real definition at link time.
2108	fn emitExternFnStub(cg: *CodeGen, nav: InternPool.Nav, decl: *Decl, fn_ty: Type) !void {
2109	if (decl.has_extern_stub) return;
2110	decl.has_extern_stub = true;
2111
2112	const gpa = cg.gpa;
2113	const zcu = cg.zcu;
2114	const ip = &zcu.intern_pool;
2115	const fn_info = zcu.typeToFunc(fn_ty).?;
2116	const return_ty_id = try cg.resolveFnReturnType(.fromInterned(fn_info.return_type));
2117	const prototype_ty_id = try cg.resolveType(fn_ty, .direct);
2118
2119	var stub: Section = .{};
2120	defer stub.deinit(gpa);
2121	try stub.emit(gpa, .OpFunction, .{
2122	.id_result_type = return_ty_id,
2123	.id_result = decl.result_id,
2124	.function_type = prototype_ty_id,
2125	.function_control = .{},
2126	});
2127	for (fn_info.param_types.get(ip)) |param_ty_index| {
2128	const param_ty: Type = .fromInterned(param_ty_index);
2129	if (!param_ty.hasRuntimeBits(zcu)) continue;
2130	const param_type_id = try cg.resolveType(param_ty, .direct);
2131	try stub.emit(gpa, .OpFunctionParameter, .{
2132	.id_result_type = param_type_id,
2133	.id_result = cg.allocId(),
2134	});
2135	}
2136	try stub.emit(gpa, .OpFunctionEnd, {});
2137	try cg.sections.functions.append(gpa, stub);
2138
2139	const extern_name = nav.getExtern(ip).?.name.toSlice(ip);
2140	try cg.sections.annotations.emit(gpa, .OpDecorate, .{
2141	.target = decl.result_id,
2142	.decoration = .{ .linkage_attributes = .{
2143	.name = extern_name,
2144	.linkage_type = .import,
2145	} },
2146	});
2147	try cg.debugName(decl.result_id, extern_name);
2148	}
2149
2150	fn resolveTypeName(cg: *CodeGen, ty: Type) ![]const u8 {
2151	const gpa = cg.gpa;
2152	var aw: std.Io.Writer.Allocating = .init(gpa);
2153	defer aw.deinit();
2154	ty.print(&aw.writer, cg.pt, null) catch |err| switch (err) {
2155	error.WriteFailed => return error.OutOfMemory,
2156	};
2157	return try aw.toOwnedSlice();
2158	}
2159
2160	/// Generate a union type. Union types are always generated with the
2161	/// most aligned field active. If the tag alignment is greater
2162	/// than that of the payload, a regular union (non-packed, with both tag and
2163	/// payload), will be generated as follows:
2164	/// struct {
2165	/// tag: TagType,
2166	/// payload: MostAlignedFieldType,
2167	/// payload_padding: [payload_size - @sizeOf(MostAlignedFieldType)]u8,
2168	/// padding: [padding_size]u8,
2169	/// }
2170	/// If the payload alignment is greater than that of the tag:
2171	/// struct {
2172	/// payload: MostAlignedFieldType,
2173	/// payload_padding: [payload_size - @sizeOf(MostAlignedFieldType)]u8,
2174	/// tag: TagType,
2175	/// padding: [padding_size]u8,
2176	/// }
2177	/// If any of the fields' size is 0, it will be omitted.
2178	fn resolveFnReturnType(cg: *CodeGen, ret_ty: Type) !Id {
2179	const zcu = cg.zcu;
2180	if (!ret_ty.hasRuntimeBits(zcu)) {
2181	// If the return type is an error set or an error union, then we make this
2182	// anyerror return type instead, so that it can be coerced into a function
2183	// pointer type which has anyerror as the return type.
2184	if (ret_ty.isError(zcu)) {
2185	return cg.resolveType(.anyerror, .direct);
2186	} else {
2187	return cg.resolveType(.void, .direct);
2188	}
2189	}
2190
2191	return try cg.resolveType(ret_ty, .direct);
2192	}
2193
2194	fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {
2195	const gpa = cg.gpa;
2196	const pt = cg.pt;
2197	const zcu = cg.zcu;
2198	const ip = &zcu.intern_pool;
2199	const target = cg.zcu.getTarget();
2200
2201	log.debug("resolveType: ty = {f}", .{ty.fmt(pt)});
2202
2203	switch (ty.zigTypeTag(zcu)) {
2204	.noreturn => {
2205	assert(repr == .direct);
2206	return try cg.voidType();
2207	},
2208	.void => switch (repr) {
2209	.direct => return try cg.voidType(),
2210	.indirect => {
2211	if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{});
2212	return try cg.opaqueType("void");
2213	},
2214	},
2215	.bool => switch (repr) {
2216	.direct => return try cg.boolType(),
2217	.indirect => return try cg.resolveType(.u1, .indirect),
2218	},
2219	.int => {
2220	if (ty.toIntern() == .u0_type) {
2221	assert(repr == .indirect);
2222	if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{});
2223	return try cg.opaqueType("u0");
2224	}
2225	const int_info = ty.intInfo(zcu);
2226	return try cg.intType(int_info.signedness, int_info.bits);
2227	},
2228	.@"enum" => return try cg.resolveType(ty.backingIntType(zcu), repr),
2229	.float => {
2230	const bits = ty.floatBits(target);
2231	const supported = switch (bits) {
2232	16 => target.cpu.has(.spirv, .float16),
2233	32 => true,
2234	64 => target.cpu.has(.spirv, .float64),
2235	else => false,
2236	};
2237	if (!supported) return cg.fail(
2238	"'{f}' is not supported on the current SPIR-V feature set",
2239	.{ty.fmt(cg.pt)},
2240	);
2241	return try cg.floatType(bits);
2242	},
2243	.array => {
2244	const elem_ty = ty.childType(zcu);
2245	const elem_ty_id = try cg.resolveType(elem_ty, .indirect);
2246	const total_len = std.math.cast(u32, ty.arrayLenIncludingSentinel(zcu)) orelse {
2247	return cg.fail("array type of {} elements is too large", .{ty.arrayLenIncludingSentinel(zcu)});
2248	};
2249
2250	if (!elem_ty.hasRuntimeBits(zcu)) {
2251	assert(repr == .indirect);
2252	if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{});
2253	return try cg.opaqueType("zero-sized-array");
2254	} else if (total_len == 0) {
2255	// The size of the array would be 0, but that is not allowed in SPIR-V.
2256	// This path can be reached for example when there is a slicing of a pointer
2257	// that produces a zero-length array. In all cases where this type can be generated,
2258	// this should be an indirect path.
2259	assert(repr == .indirect);
2260	// In this case, we have an array of a non-zero sized type. In this case,
2261	// generate an array of 1 element instead, so that ptr_elem_ptr instructions
2262	// can be lowered to ptrAccessChain instead of manually performing the math.
2263	const len_id = try cg.constInt(.u32, 1);
2264	return try cg.arrayType(len_id, elem_ty_id);
2265	} else {
2266	const total_len_id = try cg.constInt(.u32, total_len);
2267	return try cg.arrayType(total_len_id, elem_ty_id);
2268	}
2269	},
2270	.vector => {
2271	const elem_ty = ty.childType(zcu);
2272	const elem_ty_id = try cg.resolveType(elem_ty, repr);
2273	const len = ty.vectorLen(zcu);
2274	if (cg.isSpvVector(ty)) return try cg.vectorType(len, elem_ty_id);
2275	const len_id = try cg.constInt(.u32, len);
2276	return try cg.arrayType(len_id, elem_ty_id);
2277	},
2278	.@"fn" => switch (repr) {
2279	.direct => {
2280	const fn_info = zcu.typeToFunc(ty).?;
2281
2282	assert(!fn_info.is_var_args);
2283	switch (fn_info.cc) {
2284	.auto,
2285	.spirv_kernel,
2286	.spirv_fragment,
2287	.spirv_vertex,
2288	.spirv_device,
2289	.spirv_task,
2290	.spirv_mesh,
2291	=> {},
2292	else => unreachable,
2293	}
2294
2295	const return_ty_id = try cg.resolveFnReturnType(.fromInterned(fn_info.return_type));
2296
2297	const scratch_top = cg.id_scratch.items.len;
2298	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
2299	const param_ty_ids = try cg.id_scratch.addManyAsSlice(gpa, fn_info.param_types.len);
2300
2301	var param_index: usize = 0;
2302	for (fn_info.param_types.get(ip)) |param_ty_index| {
2303	const param_ty: Type = .fromInterned(param_ty_index);
2304	if (!param_ty.hasRuntimeBits(zcu)) continue;
2305
2306	param_ty_ids[param_index] = try cg.resolveType(param_ty, .direct);
2307	param_index += 1;
2308	}
2309
2310	return try cg.functionType(return_ty_id, param_ty_ids[0..param_index]);
2311	},
2312	.indirect => {
2313	// TODO: Represent function pointers properly.
2314	// For now, just use an usize type.
2315	return try cg.resolveType(.usize, .indirect);
2316	},
2317	},
2318	.pointer => {
2319	const ptr_info = ty.ptrInfo(zcu);
2320
2321	const child_ty: Type = switch (ptr_info.packed_offset.host_size) {
2322	0 => .fromInterned(ptr_info.child),
2323	else => switch (ptr_info.flags.vector_index) {
2324	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate these usages of `pt`.
2325	.none => try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8),
2326	else => try pt.vectorType(.{
2327	.child = ptr_info.child,
2328	.len = ptr_info.packed_offset.host_size,
2329	}),
2330	},
2331	};
2332	const child_ty_id = try cg.pointeeType(ptr_info.flags.address_space, child_ty, false);
2333	const storage_class = cg.storageClass(ptr_info.flags.address_space);
2334	const ptr_ty_id = try cg.ptrType(child_ty_id, storage_class);
2335
2336	if (ptr_info.flags.size != .slice) {
2337	return ptr_ty_id;
2338	}
2339
2340	const size_ty_id = try cg.resolveType(.usize, .direct);
2341	return try cg.structType(
2342	&.{ ptr_ty_id, size_ty_id },
2343	&.{ "ptr", "len" },
2344	.none,
2345	);
2346	},
2347	.@"struct" => {
2348	const struct_type = switch (ip.indexToKey(ty.toIntern())) {
2349	.tuple_type => |tuple| {
2350	const scratch_top = cg.id_scratch.items.len;
2351	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
2352	const member_types = try cg.id_scratch.addManyAsSlice(gpa, tuple.values.len);
2353
2354	var member_index: usize = 0;
2355	for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| {
2356	if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
2357
2358	member_types[member_index] = try cg.resolveType(.fromInterned(field_ty), .indirect);
2359	member_index += 1;
2360	}
2361
2362	const result_id = try cg.structType(
2363	member_types[0..member_index],
2364	null,
2365	.none,
2366	);
2367	const type_name = try cg.resolveTypeName(ty);
2368	defer gpa.free(type_name);
2369	try cg.debugName(result_id, type_name);
2370	return result_id;
2371	},
2372	.struct_type => ip.loadStructType(ty.toIntern()),
2373	else => unreachable,
2374	};
2375
2376	if (struct_type.layout == .@"packed") {
2377	return try cg.resolveType(.fromInterned(struct_type.packed_backing_int_type), .direct);
2378	}
2379
2380	var member_types: std.ArrayList(Id) = .empty;
2381	defer member_types.deinit(gpa);
2382
2383	var member_names: std.ArrayList([]const u8) = .empty;
2384	defer member_names.deinit(gpa);
2385
2386	var it = struct_type.iterateRuntimeOrder(ip);
2387	while (it.next()) |field_index| {
2388	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
2389	if (!field_ty.hasRuntimeBits(zcu)) continue;
2390
2391	const field_name = struct_type.field_names.get(ip)[field_index];
2392	try member_types.append(gpa, try cg.resolveType(field_ty, .indirect));
2393	try member_names.append(gpa, field_name.toSlice(ip));
2394	}
2395
2396	const result_id = try cg.structType(
2397	member_types.items,
2398	member_names.items,
2399	ty.toIntern(),
2400	);
2401
2402	const type_name = try cg.resolveTypeName(ty);
2403	defer gpa.free(type_name);
2404	try cg.debugName(result_id, type_name);
2405
2406	return result_id;
2407	},
2408	.optional => {
2409	const payload_ty = ty.optionalChild(zcu);
2410	if (!payload_ty.hasRuntimeBits(zcu)) {
2411	// Just use a bool.
2412	// Note: Always generate the bool with indirect format, to save on some sanity
2413	// Perform the conversion to a direct bool when the field is extracted.
2414	return try cg.resolveType(.bool, .indirect);
2415	}
2416
2417	const payload_ty_id = try cg.resolveType(payload_ty, .indirect);
2418	if (ty.optionalReprIsPayload(zcu)) {
2419	// Optional is actually a pointer or a slice.
2420	return payload_ty_id;
2421	}
2422
2423	const bool_ty_id = try cg.resolveType(.bool, .indirect);
2424
2425	return try cg.structType(
2426	&.{ payload_ty_id, bool_ty_id },
2427	&.{ "payload", "valid" },
2428	.none,
2429	);
2430	},
2431	.@"union" => {
2432	const union_obj = zcu.typeToUnion(ty).?;
2433	if (union_obj.layout == .@"packed") {
2434	return try cg.intType(.unsigned, @intCast(ty.bitSize(zcu)));
2435	}
2436	const layout = cg.unionLayout(ty);
2437	if (!layout.has_payload) {
2438	return try cg.resolveType(.fromInterned(union_obj.enum_tag_type), .indirect);
2439	}
2440	var member_types: [4]Id = undefined;
2441	var member_names: [4][]const u8 = undefined;
2442	const u8_ty_id = try cg.resolveType(.u8, .direct);
2443	if (layout.tag_size != 0) {
2444	member_types[layout.tag_index] = try cg.resolveType(.fromInterned(union_obj.enum_tag_type), .indirect);
2445	member_names[layout.tag_index] = "(tag)";
2446	}
2447	if (layout.payload_size != 0) {
2448	member_types[layout.payload_index] = try cg.resolveType(layout.payload_ty, .indirect);
2449	member_names[layout.payload_index] = "(payload)";
2450	}
2451	if (layout.payload_padding_size != 0) {
2452	const len_id = try cg.constInt(.u32, layout.payload_padding_size);
2453	member_types[layout.payload_padding_index] = try cg.arrayType(len_id, u8_ty_id);
2454	member_names[layout.payload_padding_index] = "(payload padding)";
2455	}
2456	if (layout.padding_size != 0) {
2457	const len_id = try cg.constInt(.u32, layout.padding_size);
2458	member_types[layout.padding_index] = try cg.arrayType(len_id, u8_ty_id);
2459	member_names[layout.padding_index] = "(padding)";
2460	}
2461	const result_id = try cg.structType(
2462	member_types[0..layout.total_fields],
2463	member_names[0..layout.total_fields],
2464	.none,
2465	);
2466	const type_name = try cg.resolveTypeName(ty);
2467	defer gpa.free(type_name);
2468	try cg.debugName(result_id, type_name);
2469	return result_id;
2470	},
2471	.error_set => {
2472	const err_int_ty = try pt.errorIntType();
2473	return try cg.resolveType(err_int_ty, repr);
2474	},
2475	.error_union => {
2476	const payload_ty = ty.errorUnionPayload(zcu);
2477	const err_ty = ty.errorUnionSet(zcu);
2478	const error_ty_id = try cg.resolveType(err_ty, .indirect);
2479
2480	const eu_layout = cg.errorUnionLayout(payload_ty);
2481	if (!eu_layout.payload_has_bits) {
2482	return error_ty_id;
2483	}
2484
2485	const payload_ty_id = try cg.resolveType(payload_ty, .indirect);
2486
2487	var member_types: [2]Id = undefined;
2488	var member_names: [2][]const u8 = undefined;
2489	if (eu_layout.error_first) {
2490	// Put the error first
2491	member_types = .{ error_ty_id, payload_ty_id };
2492	member_names = .{ "error", "payload" };
2493	// TODO: ABI padding?
2494	} else {
2495	// Put the payload first.
2496	member_types = .{ payload_ty_id, error_ty_id };
2497	member_names = .{ "payload", "error" };
2498	// TODO: ABI padding?
2499	}
2500
2501	return try cg.structType(&member_types, &member_names, .none);
2502	},
2503	.@"opaque" => {
2504	if (target.os.tag != .opencl) return cg.fail("cannot generate opaque type", .{});
2505	const type_name = try cg.resolveTypeName(ty);
2506	defer gpa.free(type_name);
2507	return try cg.opaqueType(type_name);
2508	},
2509	.spirv => {
2510	const spirv_type = ip.loadSpirvType(ty.toIntern());
2511	const result_id = cg.allocId();
2512	switch (spirv_type.flags.tag) {
2513	.sampler => try cg.sections.globals.emit(gpa, .OpTypeSampler, .{ .id_result = result_id }),
2514	.image => {
2515	const sampled_type_id = try cg.resolveType(.fromInterned(spirv_type.ty), .direct);
2516	try cg.sections.globals.emit(gpa, .OpTypeImage, .{
2517	.id_result = result_id,
2518	.sampled_type = sampled_type_id,
2519	.dim = switch (spirv_type.flags.dim) {
2520	.@"1d" => .@"1d",
2521	.@"2d" => .@"2d",
2522	.@"3d" => .@"3d",
2523	.cube => .cube,
2524	},
2525	.depth = switch (spirv_type.flags.depth) {
2526	.not_depth => 0,
2527	.depth => 1,
2528	.unknown => 2,
2529	},
2530	.arrayed = @intFromBool(spirv_type.flags.is_arrayed),
2531	.ms = @intFromBool(spirv_type.flags.is_multisampled),
2532	.sampled = switch (spirv_type.flags.usage) {
2533	.unknown => 0,
2534	.sampled => 1,
2535	.storage => 2,
2536	},
2537	.image_format = switch (spirv_type.flags.format) {
2538	.unknown => .unknown,
2539	.rgba32f => .rgba32f,
2540	.rgba32i => .rgba32i,
2541	.rgba32u => .rgba32ui,
2542	.rgba16f => .rgba16f,
2543	.rgba16i => .rgba16i,
2544	.rgba16u => .rgba16ui,
2545	.rgba8unorm => .rgba8,
2546	.rgba8snorm => .rgba8snorm,
2547	.rgba8i => .rgba8i,
2548	.rgba8u => .rgba8ui,
2549	.r32f => .r32f,
2550	.r32i => .r32i,
2551	.r32u => .r32ui,
2552	},
2553	.access_qualifier = switch (spirv_type.flags.access) {
2554	.unknown => null,
2555	.read_only => .read_only,
2556	.write_only => .write_only,
2557	.read_write => .read_write,
2558	},
2559	});
2560	},
2561	.sampled_image => {
2562	const image_ty_id = try cg.resolveType(.fromInterned(spirv_type.ty), .indirect);
2563	try cg.sections.globals.emit(gpa, .OpTypeSampledImage, .{
2564	.id_result = result_id,
2565	.image_type = image_ty_id,
2566	});
2567	},
2568	.runtime_array => {
2569	const elem_ty: Type = .fromInterned(spirv_type.ty);
2570	const elem_ty_id = try cg.resolveType(elem_ty, .indirect);
2571	try cg.sections.globals.emit(gpa, .OpTypeRuntimeArray, .{
2572	.id_result = result_id,
2573	.element_type = elem_ty_id,
2574	});
2575	if (elem_ty.hasRuntimeBits(zcu)) {
2576	try cg.decorate(result_id, .{ .array_stride = .{
2577	.array_stride = @intCast(elem_ty.abiSize(zcu)),
2578	} });
2579	}
2580	},
2581	}
2582	return result_id;
2583	},
2584
2585	.null,
2586	.undefined,
2587	.enum_literal,
2588	.comptime_float,
2589	.comptime_int,
2590	.type,
2591	=> unreachable, // Must be comptime.
2592
2593	.frame, .@"anyframe" => unreachable, // TODO
2594	}
2595	}
2596
2597	const ErrorUnionLayout = struct {
2598	payload_has_bits: bool,
2599	error_first: bool,
2600
2601	fn errorFieldIndex(cg: @This()) u32 {
2602	assert(cg.payload_has_bits);
2603	return if (cg.error_first) 0 else 1;
2604	}
2605
2606	fn payloadFieldIndex(cg: @This()) u32 {
2607	assert(cg.payload_has_bits);
2608	return if (cg.error_first) 1 else 0;
2609	}
2610	};
2611
2612	fn errorUnionLayout(cg: *CodeGen, payload_ty: Type) ErrorUnionLayout {
2613	const zcu = cg.zcu;
2614
2615	const error_align = Type.abiAlignment(.anyerror, zcu);
2616	const payload_align = payload_ty.abiAlignment(zcu);
2617
2618	const error_first = error_align.compare(.gt, payload_align);
2619	return .{
2620	.payload_has_bits = payload_ty.hasRuntimeBits(zcu),
2621	.error_first = error_first,
2622	};
2623	}
2624
2625	const UnionLayout = struct {
2626	/// If false, this union is represented
2627	/// by only an integer of the tag type.
2628	has_payload: bool,
2629	tag_size: u32,
2630	tag_index: u32,
2631	/// Note: This is the size of the payload type itcg, NOT the size of the ENTIRE payload.
2632	/// Use `has_payload` instead!!
2633	payload_ty: Type,
2634	payload_size: u32,
2635	payload_index: u32,
2636	payload_padding_size: u32,
2637	payload_padding_index: u32,
2638	padding_size: u32,
2639	padding_index: u32,
2640	total_fields: u32,
2641	};
2642
2643	fn unionLayout(cg: *CodeGen, ty: Type) UnionLayout {
2644	const zcu = cg.zcu;
2645	const ip = &zcu.intern_pool;
2646	const layout = ty.unionGetLayout(zcu);
2647	const union_obj = zcu.typeToUnion(ty).?;
2648
2649	var union_layout: UnionLayout = .{
2650	.has_payload = layout.payload_size != 0,
2651	.tag_size = @intCast(layout.tag_size),
2652	.tag_index = undefined,
2653	.payload_ty = undefined,
2654	.payload_size = undefined,
2655	.payload_index = undefined,
2656	.payload_padding_size = undefined,
2657	.payload_padding_index = undefined,
2658	.padding_size = @intCast(layout.padding),
2659	.padding_index = undefined,
2660	.total_fields = undefined,
2661	};
2662
2663	if (union_layout.has_payload) {
2664	const most_aligned_field = layout.most_aligned_field;
2665	const most_aligned_field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[most_aligned_field]);
2666	union_layout.payload_ty = most_aligned_field_ty;
2667	union_layout.payload_size = @intCast(most_aligned_field_ty.abiSize(zcu));
2668	} else {
2669	union_layout.payload_size = 0;
2670	}
2671
2672	union_layout.payload_padding_size = @intCast(layout.payload_size - union_layout.payload_size);
2673
2674	const tag_first = layout.tag_align.compare(.gte, layout.payload_align);
2675	var field_index: u32 = 0;
2676
2677	if (union_layout.tag_size != 0 and tag_first) {
2678	union_layout.tag_index = field_index;
2679	field_index += 1;
2680	}
2681
2682	if (union_layout.payload_size != 0) {
2683	union_layout.payload_index = field_index;
2684	field_index += 1;
2685	}
2686
2687	if (union_layout.payload_padding_size != 0) {
2688	union_layout.payload_padding_index = field_index;
2689	field_index += 1;
2690	}
2691
2692	if (union_layout.tag_size != 0 and !tag_first) {
2693	union_layout.tag_index = field_index;
2694	field_index += 1;
2695	}
2696
2697	if (union_layout.padding_size != 0) {
2698	union_layout.padding_index = field_index;
2699	field_index += 1;
2700	}
2701
2702	union_layout.total_fields = field_index;
2703
2704	return union_layout;
2705	}
2706
2707	/// This structure represents a "temporary" value: Something we are currently
2708	/// operating on. It typically lives no longer than the function that
2709	/// implements a particular AIR operation. These are used to easier
2710	/// implement vectorizable operations (see Vectorization and the build*
2711	/// functions), and typically are only used for vectors of primitive types.
2712	const Temporary = struct {
2713	/// The type of the temporary. This is here mainly
2714	/// for easier bookkeeping. Because we will never really
2715	/// store Temporaries, they only cause extra stack space,
2716	/// therefore no real storage is wasted.
2717	ty: Type,
2718	/// The value that this temporary holds. This is not necessarily
2719	/// a value that is actually usable, or a single value: It is virtual
2720	/// until materialize() is called, at which point is turned into
2721	/// the usual SPIR-V representation of `cg.ty`.
2722	value: Temporary.Value,
2723
2724	const Value = union(enum) {
2725	singleton: Id,
2726	exploded_vector: IdRange,
2727	};
2728
2729	fn init(ty: Type, singleton: Id) Temporary {
2730	return .{ .ty = ty, .value = .{ .singleton = singleton } };
2731	}
2732
2733	fn materialize(temp: Temporary, cg: *CodeGen) !Id {
2734	const gpa = cg.gpa;
2735	const zcu = cg.zcu;
2736	switch (temp.value) {
2737	.singleton => |id| return id,
2738	.exploded_vector => |range| {
2739	assert(temp.ty.isVector(zcu));
2740	assert(temp.ty.vectorLen(zcu) == range.len);
2741
2742	const scratch_top = cg.id_scratch.items.len;
2743	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
2744	const constituents = try cg.id_scratch.addManyAsSlice(gpa, range.len);
2745	for (constituents, 0..range.len) |*id, i| {
2746	id.* = range.at(i);
2747	}
2748
2749	const result_ty_id = try cg.resolveType(temp.ty, .direct);
2750	return cg.constructComposite(result_ty_id, constituents);
2751	},
2752	}
2753	}
2754
2755	fn vectorization(temp: Temporary, cg: *CodeGen) Vectorization {
2756	return .fromType(temp.ty, cg);
2757	}
2758
2759	fn pun(temp: Temporary, new_ty: Type) Temporary {
2760	return .{
2761	.ty = new_ty,
2762	.value = temp.value,
2763	};
2764	}
2765
2766	/// 'Explode' a temporary into separate elements. This turns a vector
2767	/// into a bag of elements.
2768	fn explode(temp: Temporary, cg: *CodeGen) !IdRange {
2769	const zcu = cg.zcu;
2770
2771	// If the value is a scalar, then this is a no-op.
2772	if (!temp.ty.isVector(zcu)) {
2773	return switch (temp.value) {
2774	.singleton => |id| .{ .base = @backingInt(id), .len = 1 },
2775	.exploded_vector => |range| range,
2776	};
2777	}
2778
2779	const ty_id = try cg.resolveType(temp.ty.scalarType(zcu), .direct);
2780	const n = temp.ty.vectorLen(zcu);
2781	const results = cg.allocIds(n);
2782
2783	const id = switch (temp.value) {
2784	.singleton => |id| id,
2785	.exploded_vector => |range| return range,
2786	};
2787
2788	for (0..n) |i| {
2789	const indexes = [_]u32{@intCast(i)};
2790	try cg.body.emit(cg.gpa, .OpCompositeExtract, .{
2791	.id_result_type = ty_id,
2792	.id_result = results.at(i),
2793	.composite = id,
2794	.indexes = &indexes,
2795	});
2796	}
2797
2798	return results;
2799	}
2800	};
2801
2802	/// composite integers are represented as [N]u32 arrays
2803	const CompositeInt = struct {
2804	cg: *CodeGen,
2805	limbs: []Id,
2806	n_limbs: u16,
2807	info: ArithmeticTypeInfo,
2808
2809	fn init(cg: *CodeGen, composite_id: Id, info: ArithmeticTypeInfo) !CompositeInt {
2810	const n_limbs: u16 = info.backing_bits / cg.bigIntBits();
2811	const gpa = cg.gpa;
2812	if (cg.composite_limbs.get(composite_id)) |cached| {
2813	assert(cached.len == n_limbs);
2814	const limbs = try cg.id_scratch.addManyAsSlice(gpa, n_limbs);
2815	@memcpy(limbs, cached);
2816	return .{ .cg = cg, .limbs = limbs, .n_limbs = n_limbs, .info = info };
2817	}
2818	const limb_ty_id = try cg.limbTypeId();
2819	const limbs = try cg.id_scratch.addManyAsSlice(gpa, n_limbs);
2820	for (limbs, 0..) |*limb, i| {
2821	const result_id = cg.allocId();
2822	try cg.body.emit(gpa, .OpCompositeExtract, .{
2823	.id_result_type = limb_ty_id,
2824	.id_result = result_id,
2825	.composite = composite_id,
2826	.indexes = &.{@as(u32, @intCast(i))},
2827	});
2828	limb.* = result_id;
2829	}
2830	const cached = try cg.arena.dupe(Id, limbs);
2831	try cg.composite_limbs.put(gpa, composite_id, cached);
2832	return .{ .cg = cg, .limbs = limbs, .n_limbs = n_limbs, .info = info };
2833	}
2834
2835	fn fromLimbs(cg: *CodeGen, limbs: []Id, info: ArithmeticTypeInfo) CompositeInt {
2836	return .{
2837	.cg = cg,
2838	.limbs = limbs,
2839	.n_limbs = @intCast(limbs.len),
2840	.info = info,
2841	};
2842	}
2843
2844	fn zero(cg: *CodeGen, info: ArithmeticTypeInfo) !CompositeInt {
2845	const n_limbs: u16 = info.backing_bits / cg.bigIntBits();
2846	const limbs = try cg.id_scratch.addManyAsSlice(cg.gpa, n_limbs);
2847	const zero_id = try cg.constInt(cg.limbType(), @as(u64, 0));
2848	for (limbs) |*limb| limb.* = zero_id;
2849	return .{ .cg = cg, .limbs = limbs, .n_limbs = n_limbs, .info = info };
2850	}
2851
2852	fn materialize(ci: CompositeInt, ty: Type) !Id {
2853	const result_ty_id = try ci.cg.resolveType(ty, .indirect);
2854	return ci.cg.constructComposite(result_ty_id, ci.limbs);
2855	}
2856
2857	fn limbBinOp(ci: CompositeInt, opcode: Opcode, lhs: Id, rhs: Id) !Id {
2858	const cg = ci.cg;
2859	const gpa = cg.gpa;
2860	const limb_ty_id = try cg.limbTypeId();
2861	const result_id = cg.allocId();
2862	try cg.body.emitRaw(gpa, opcode, 4);
2863	cg.body.writeOperand(Id, limb_ty_id);
2864	cg.body.writeOperand(Id, result_id);
2865	cg.body.writeOperand(Id, lhs);
2866	cg.body.writeOperand(Id, rhs);
2867	return result_id;
2868	}
2869
2870	fn limbUnOp(ci: CompositeInt, opcode: Opcode, operand: Id) !Id {
2871	const cg = ci.cg;
2872	const gpa = cg.gpa;
2873	const limb_ty_id = try cg.limbTypeId();
2874	const result_id = cg.allocId();
2875	try cg.body.emitRaw(gpa, opcode, 3);
2876	cg.body.writeOperand(Id, limb_ty_id);
2877	cg.body.writeOperand(Id, result_id);
2878	cg.body.writeOperand(Id, operand);
2879	return result_id;
2880	}
2881
2882	fn bitwiseOp(ci: CompositeInt, other: CompositeInt, opcode: Opcode) !CompositeInt {
2883	const cg = ci.cg;
2884	const gpa = cg.gpa;
2885	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
2886	for (result_limbs, 0..) |*r, i| {
2887	r.* = try ci.limbBinOp(opcode, ci.limbs[i], other.limbs[i]);
2888	}
2889	return .fromLimbs(cg, result_limbs, ci.info);
2890	}
2891
2892	fn bitwiseNot(ci: CompositeInt) !CompositeInt {
2893	const cg = ci.cg;
2894	const gpa = cg.gpa;
2895	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
2896	for (result_limbs, 0..) |*r, i| {
2897	r.* = try ci.limbUnOp(.OpNot, ci.limbs[i]);
2898	}
2899	return .fromLimbs(cg, result_limbs, ci.info);
2900	}
2901
2902	fn cmp(ci: CompositeInt, other: CompositeInt, op: std.math.CompareOperator) !Id {
2903	const cg = ci.cg;
2904	const gpa = cg.gpa;
2905	const bool_ty_id = try cg.resolveType(.bool, .direct);
2906
2907	switch (op) {
2908	.eq, .neq => {
2909	var result = blk: {
2910	const r = cg.allocId();
2911	try cg.body.emit(gpa, .OpIEqual, .{
2912	.id_result_type = bool_ty_id,
2913	.id_result = r,
2914	.operand_1 = ci.limbs[0],
2915	.operand_2 = other.limbs[0],
2916	});
2917	break :blk r;
2918	};
2919	for (1..ci.n_limbs) |i| {
2920	const limb_eq = cg.allocId();
2921	try cg.body.emit(gpa, .OpIEqual, .{
2922	.id_result_type = bool_ty_id,
2923	.id_result = limb_eq,
2924	.operand_1 = ci.limbs[i],
2925	.operand_2 = other.limbs[i],
2926	});
2927	const combined = cg.allocId();
2928	try cg.body.emit(gpa, .OpLogicalAnd, .{
2929	.id_result_type = bool_ty_id,
2930	.id_result = combined,
2931	.operand_1 = result,
2932	.operand_2 = limb_eq,
2933	});
2934	result = combined;
2935	}
2936	if (op == .neq) {
2937	const negated = cg.allocId();
2938	try cg.body.emit(gpa, .OpLogicalNot, .{
2939	.id_result_type = bool_ty_id,
2940	.id_result = negated,
2941	.operand = result,
2942	});
2943	result = negated;
2944	}
2945	return result;
2946	},
2947	.lt, .lte, .gt, .gte => {
2948	const is_lt = (op == .lt or op == .lte);
2949	const is_strict = (op == .lt or op == .gt);
2950	var result = try cg.constBool(!is_strict, .direct);
2951
2952	for (0..ci.n_limbs) |i| {
2953	const l = ci.limbs[i];
2954	const r = other.limbs[i];
2955	const limb_ne = cg.allocId();
2956	try cg.body.emit(gpa, .OpINotEqual, .{
2957	.id_result_type = bool_ty_id,
2958	.id_result = limb_ne,
2959	.operand_1 = l,
2960	.operand_2 = r,
2961	});
2962
2963	const is_top = (i == ci.n_limbs - 1);
2964	const use_signed = is_top and ci.info.signedness == .signed;
2965	var cmp_l = l;
2966	var cmp_r = r;
2967	if (use_signed) {
2968	const signed_limb_ty: Type = if (cg.bigIntBits() == 64) .i64 else .i32;
2969	const signed_limb_ty_id = try cg.resolveType(signed_limb_ty, .direct);
2970	const sl = cg.allocId();
2971	try cg.body.emit(gpa, .OpBitcast, .{
2972	.id_result_type = signed_limb_ty_id,
2973	.id_result = sl,
2974	.operand = l,
2975	});
2976	const sr = cg.allocId();
2977	try cg.body.emit(gpa, .OpBitcast, .{
2978	.id_result_type = signed_limb_ty_id,
2979	.id_result = sr,
2980	.operand = r,
2981	});
2982	cmp_l = sl;
2983	cmp_r = sr;
2984	}
2985
2986	const cmp_opcode: Opcode = if (is_lt)
2987	(if (use_signed) .OpSLessThan else .OpULessThan)
2988	else
2989	(if (use_signed) .OpSGreaterThan else .OpUGreaterThan);
2990
2991	const limb_cmp = cg.allocId();
2992	try cg.body.emitRaw(gpa, cmp_opcode, 4);
2993	cg.body.writeOperand(Id, bool_ty_id);
2994	cg.body.writeOperand(Id, limb_cmp);
2995	cg.body.writeOperand(Id, cmp_l);
2996	cg.body.writeOperand(Id, cmp_r);
2997
2998	const selected = cg.allocId();
2999	try cg.body.emit(gpa, .OpSelect, .{
3000	.id_result_type = bool_ty_id,
3001	.id_result = selected,
3002	.condition = limb_ne,
3003	.object_1 = limb_cmp,
3004	.object_2 = result,
3005	});
3006	result = selected;
3007	}
3008	return result;
3009	},
3010	}
3011	}
3012
3013	fn addSub(ci: CompositeInt, other: CompositeInt, comptime is_add: bool) !CompositeInt {
3014	const cg = ci.cg;
3015	const gpa = cg.gpa;
3016	const pt = cg.pt;
3017	const zcu = cg.zcu;
3018	const ip = &zcu.intern_pool;
3019	const comp = zcu.comp;
3020	const io = comp.io;
3021
3022	const limb_bits = cg.bigIntBits();
3023	const limb_zig = try pt.intType(.unsigned, limb_bits);
3024	const limb_ty_id = try cg.limbTypeId();
3025	const carry_struct_ty: Type = .fromInterned(try ip.getTupleType(gpa, io, pt.tid, .{
3026	.types = &.{ limb_zig.toIntern(), limb_zig.toIntern() },
3027	.values = &.{ .none, .none },
3028	}));
3029	const carry_struct_ty_id = try cg.resolveType(carry_struct_ty, .direct);
3030
3031	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
3032	var carry_id = try cg.constInt(cg.limbType(), @as(u64, 0));
3033
3034	const opcode: Opcode = if (is_add) .OpIAddCarry else .OpISubBorrow;
3035
3036	for (0..ci.n_limbs) |i| {
3037	const op1 = cg.allocId();
3038	try cg.body.emitRaw(gpa, opcode, 4);
3039	cg.body.writeOperand(Id, carry_struct_ty_id);
3040	cg.body.writeOperand(Id, op1);
3041	cg.body.writeOperand(Id, ci.limbs[i]);
3042	cg.body.writeOperand(Id, other.limbs[i]);
3043
3044	const sum1 = cg.allocId();
3045	try cg.body.emit(gpa, .OpCompositeExtract, .{
3046	.id_result_type = limb_ty_id,
3047	.id_result = sum1,
3048	.composite = op1,
3049	.indexes = &.{0},
3050	});
3051	const carry1 = cg.allocId();
3052	try cg.body.emit(gpa, .OpCompositeExtract, .{
3053	.id_result_type = limb_ty_id,
3054	.id_result = carry1,
3055	.composite = op1,
3056	.indexes = &.{1},
3057	});
3058
3059	const op2 = cg.allocId();
3060	try cg.body.emitRaw(gpa, opcode, 4);
3061	cg.body.writeOperand(Id, carry_struct_ty_id);
3062	cg.body.writeOperand(Id, op2);
3063	cg.body.writeOperand(Id, sum1);
3064	cg.body.writeOperand(Id, carry_id);
3065
3066	result_limbs[i] = cg.allocId();
3067	try cg.body.emit(gpa, .OpCompositeExtract, .{
3068	.id_result_type = limb_ty_id,
3069	.id_result = result_limbs[i],
3070	.composite = op2,
3071	.indexes = &.{0},
3072	});
3073	const carry2 = cg.allocId();
3074	try cg.body.emit(gpa, .OpCompositeExtract, .{
3075	.id_result_type = limb_ty_id,
3076	.id_result = carry2,
3077	.composite = op2,
3078	.indexes = &.{1},
3079	});
3080
3081	carry_id = try ci.limbBinOp(.OpBitwiseOr, carry1, carry2);
3082	}
3083
3084	return .fromLimbs(cg, result_limbs, ci.info);
3085	}
3086
3087	fn shl(ci: CompositeInt, shift_amt_id: Id) !CompositeInt {
3088	const cg = ci.cg;
3089	const gpa = cg.gpa;
3090	const limb_bits = cg.bigIntBits();
3091	const limb_ty = cg.limbType();
3092	const limb_ty_id = try cg.limbTypeId();
3093	const bool_ty_id = try cg.resolveType(.bool, .direct);
3094	const zero_id = try cg.constInt(limb_ty, @as(u64, 0));
3095	const log2_bits_id = try cg.constInt(limb_ty, @as(u64, std.math.log2_int(u16, limb_bits)));
3096	const bits_minus_1_id = try cg.constInt(limb_ty, @as(u64, limb_bits - 1));
3097	const bits_id = try cg.constInt(limb_ty, @as(u64, limb_bits));
3098
3099	const whole = try ci.limbBinOp(.OpShiftRightLogical, shift_amt_id, log2_bits_id);
3100	const frac = try ci.limbBinOp(.OpBitwiseAnd, shift_amt_id, bits_minus_1_id);
3101	const comp_frac = try ci.limbBinOp(.OpISub, bits_id, frac);
3102	const frac_is_zero = blk: {
3103	const r = cg.allocId();
3104	try cg.body.emit(gpa, .OpIEqual, .{
3105	.id_result_type = bool_ty_id,
3106	.id_result = r,
3107	.operand_1 = frac,
3108	.operand_2 = zero_id,
3109	});
3110	break :blk r;
3111	};
3112
3113	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
3114
3115	for (0..ci.n_limbs) |i| {
3116	const i_id = try cg.constInt(limb_ty, @as(u64, @intCast(i)));
3117	var main_val = zero_id;
3118	var carry_val = zero_id;
3119
3120	for (0..ci.n_limbs) |j| {
3121	const j_id = try cg.constInt(limb_ty, @as(u64, @intCast(j)));
3122	const j_plus_whole = try ci.limbBinOp(.OpIAdd, j_id, whole);
3123
3124	const is_main = blk: {
3125	const r = cg.allocId();
3126	try cg.body.emit(gpa, .OpIEqual, .{
3127	.id_result_type = bool_ty_id,
3128	.id_result = r,
3129	.operand_1 = j_plus_whole,
3130	.operand_2 = i_id,
3131	});
3132	break :blk r;
3133	};
3134	const shifted = try ci.limbBinOp(.OpShiftLeftLogical, ci.limbs[j], frac);
3135	main_val = blk: {
3136	const r = cg.allocId();
3137	try cg.body.emit(gpa, .OpSelect, .{
3138	.id_result_type = limb_ty_id,
3139	.id_result = r,
3140	.condition = is_main,
3141	.object_1 = shifted,
3142	.object_2 = main_val,
3143	});
3144	break :blk r;
3145	};
3146
3147	const one_id = try cg.constInt(limb_ty, @as(u64, 1));
3148	const j_plus_whole_plus_1 = try ci.limbBinOp(.OpIAdd, j_plus_whole, one_id);
3149	const is_carry = blk: {
3150	const r = cg.allocId();
3151	try cg.body.emit(gpa, .OpIEqual, .{
3152	.id_result_type = bool_ty_id,
3153	.id_result = r,
3154	.operand_1 = j_plus_whole_plus_1,
3155	.operand_2 = i_id,
3156	});
3157	break :blk r;
3158	};
3159	const carry_shifted = try ci.limbBinOp(.OpShiftRightLogical, ci.limbs[j], comp_frac);
3160	const guarded_carry = blk: {
3161	const r = cg.allocId();
3162	try cg.body.emit(gpa, .OpSelect, .{
3163	.id_result_type = limb_ty_id,
3164	.id_result = r,
3165	.condition = frac_is_zero,
3166	.object_1 = zero_id,
3167	.object_2 = carry_shifted,
3168	});
3169	break :blk r;
3170	};
3171	carry_val = blk: {
3172	const r = cg.allocId();
3173	try cg.body.emit(gpa, .OpSelect, .{
3174	.id_result_type = limb_ty_id,
3175	.id_result = r,
3176	.condition = is_carry,
3177	.object_1 = guarded_carry,
3178	.object_2 = carry_val,
3179	});
3180	break :blk r;
3181	};
3182	}
3183
3184	result_limbs[i] = try ci.limbBinOp(.OpBitwiseOr, main_val, carry_val);
3185	}
3186
3187	return .fromLimbs(cg, result_limbs, ci.info);
3188	}
3189
3190	fn shr(ci: CompositeInt, shift_amt_id: Id, comptime is_arithmetic: bool) !CompositeInt {
3191	const cg = ci.cg;
3192	const gpa = cg.gpa;
3193	const limb_bits = cg.bigIntBits();
3194	const limb_ty = cg.limbType();
3195	const limb_ty_id = try cg.limbTypeId();
3196	const bool_ty_id = try cg.resolveType(.bool, .direct);
3197	const zero_id = try cg.constInt(limb_ty, @as(u64, 0));
3198	const log2_bits_id = try cg.constInt(limb_ty, @as(u64, std.math.log2_int(u16, limb_bits)));
3199	const bits_minus_1_id = try cg.constInt(limb_ty, @as(u64, limb_bits - 1));
3200	const bits_id = try cg.constInt(limb_ty, @as(u64, limb_bits));
3201
3202	const whole = try ci.limbBinOp(.OpShiftRightLogical, shift_amt_id, log2_bits_id);
3203	const frac = try ci.limbBinOp(.OpBitwiseAnd, shift_amt_id, bits_minus_1_id);
3204	const comp_frac = try ci.limbBinOp(.OpISub, bits_id, frac);
3205	const frac_is_zero = blk: {
3206	const r = cg.allocId();
3207	try cg.body.emit(gpa, .OpIEqual, .{
3208	.id_result_type = bool_ty_id,
3209	.id_result = r,
3210	.operand_1 = frac,
3211	.operand_2 = zero_id,
3212	});
3213	break :blk r;
3214	};
3215
3216	const fill_id = if (is_arithmetic) blk: {
3217	const signed_limb_ty: Type = if (limb_bits == 64) .i64 else .i32;
3218	const signed_limb_ty_id = try cg.resolveType(signed_limb_ty, .direct);
3219	const msb_signed = cg.allocId();
3220	try cg.body.emit(gpa, .OpBitcast, .{
3221	.id_result_type = signed_limb_ty_id,
3222	.id_result = msb_signed,
3223	.operand = ci.limbs[ci.n_limbs - 1],
3224	});
3225	const shift_amt = try cg.constInt(signed_limb_ty, @as(u64, limb_bits - 1));
3226	const sign_ext = cg.allocId();
3227	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
3228	.id_result_type = signed_limb_ty_id,
3229	.id_result = sign_ext,
3230	.base = msb_signed,
3231	.shift = shift_amt,
3232	});
3233	const back = cg.allocId();
3234	try cg.body.emit(gpa, .OpBitcast, .{
3235	.id_result_type = limb_ty_id,
3236	.id_result = back,
3237	.operand = sign_ext,
3238	});
3239	break :blk back;
3240	} else zero_id;
3241
3242	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
3243
3244	const arith_carry_init = if (is_arithmetic) blk: {
3245	const shifted_fill = try ci.limbBinOp(.OpShiftLeftLogical, fill_id, comp_frac);
3246	const guarded = cg.allocId();
3247	try cg.body.emit(gpa, .OpSelect, .{
3248	.id_result_type = limb_ty_id,
3249	.id_result = guarded,
3250	.condition = frac_is_zero,
3251	.object_1 = zero_id,
3252	.object_2 = shifted_fill,
3253	});
3254	break :blk guarded;
3255	} else zero_id;
3256
3257	for (0..ci.n_limbs) |i| {
3258	const i_id = try cg.constInt(limb_ty, @as(u64, @intCast(i)));
3259	var main_val = fill_id;
3260	var carry_val = arith_carry_init;
3261
3262	for (0..ci.n_limbs) |j| {
3263	const j_id = try cg.constInt(limb_ty, @as(u64, @intCast(j)));
3264	const i_plus_whole = try ci.limbBinOp(.OpIAdd, i_id, whole);
3265	const is_main = blk: {
3266	const r = cg.allocId();
3267	try cg.body.emit(gpa, .OpIEqual, .{
3268	.id_result_type = bool_ty_id,
3269	.id_result = r,
3270	.operand_1 = j_id,
3271	.operand_2 = i_plus_whole,
3272	});
3273	break :blk r;
3274	};
3275	const shifted = try ci.limbBinOp(.OpShiftRightLogical, ci.limbs[j], frac);
3276	main_val = blk: {
3277	const r = cg.allocId();
3278	try cg.body.emit(gpa, .OpSelect, .{
3279	.id_result_type = limb_ty_id,
3280	.id_result = r,
3281	.condition = is_main,
3282	.object_1 = shifted,
3283	.object_2 = main_val,
3284	});
3285	break :blk r;
3286	};
3287
3288	const one_id = try cg.constInt(limb_ty, @as(u64, 1));
3289	const i_plus_whole_plus_1 = try ci.limbBinOp(.OpIAdd, i_plus_whole, one_id);
3290	const is_carry = blk: {
3291	const r = cg.allocId();
3292	try cg.body.emit(gpa, .OpIEqual, .{
3293	.id_result_type = bool_ty_id,
3294	.id_result = r,
3295	.operand_1 = j_id,
3296	.operand_2 = i_plus_whole_plus_1,
3297	});
3298	break :blk r;
3299	};
3300	const carry_shifted = try ci.limbBinOp(.OpShiftLeftLogical, ci.limbs[j], comp_frac);
3301	const guarded_carry = blk: {
3302	const r = cg.allocId();
3303	try cg.body.emit(gpa, .OpSelect, .{
3304	.id_result_type = limb_ty_id,
3305	.id_result = r,
3306	.condition = frac_is_zero,
3307	.object_1 = zero_id,
3308	.object_2 = carry_shifted,
3309	});
3310	break :blk r;
3311	};
3312	carry_val = blk: {
3313	const r = cg.allocId();
3314	try cg.body.emit(gpa, .OpSelect, .{
3315	.id_result_type = limb_ty_id,
3316	.id_result = r,
3317	.condition = is_carry,
3318	.object_1 = guarded_carry,
3319	.object_2 = carry_val,
3320	});
3321	break :blk r;
3322	};
3323	}
3324
3325	result_limbs[i] = try ci.limbBinOp(.OpBitwiseOr, main_val, carry_val);
3326	}
3327
3328	return .fromLimbs(cg, result_limbs, ci.info);
3329	}
3330
3331	fn mul(ci: CompositeInt, other: CompositeInt, comptime wide: bool) ![]Id {
3332	const cg = ci.cg;
3333	const gpa = cg.gpa;
3334	const pt = cg.pt;
3335	const zcu = cg.zcu;
3336	const ip = &zcu.intern_pool;
3337	const comp = zcu.comp;
3338	const io = comp.io;
3339	const target = zcu.getTarget();
3340
3341	const n: usize = ci.n_limbs;
3342	const total: usize = if (wide) 2 * n else n;
3343	const limb_bits = cg.bigIntBits();
3344	const limb_zig = try pt.intType(.unsigned, limb_bits);
3345	const limb_ty_id = try cg.limbTypeId();
3346
3347	const pair_struct_ty: Type = .fromInterned(try ip.getTupleType(gpa, io, pt.tid, .{
3348	.types = &.{ limb_zig.toIntern(), limb_zig.toIntern() },
3349	.values = &.{ .none, .none },
3350	}));
3351	const pair_struct_ty_id = try cg.resolveType(pair_struct_ty, .direct);
3352
3353	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, total);
3354	const zero_id = try cg.constInt(cg.limbType(), @as(u64, 0));
3355	for (result_limbs) |*r| r.* = zero_id;
3356
3357	for (0..n) |i| {
3358	var carry_id = zero_id;
3359	for (0..n) |j| {
3360	const k = i + j;
3361	if (k >= total) break;
3362
3363	var lo: Id = undefined;
3364	var hi: Id = undefined;
3365	switch (target.os.tag) {
3366	.opencl => {
3367	lo = cg.allocId();
3368	try cg.body.emit(gpa, .OpIMul, .{
3369	.id_result_type = limb_ty_id,
3370	.id_result = lo,
3371	.operand_1 = ci.limbs[i],
3372	.operand_2 = other.limbs[j],
3373	});
3374
3375	const set = try cg.importExtendedSet();
3376	hi = cg.allocId();
3377	try cg.body.emit(gpa, .OpExtInst, .{
3378	.id_result_type = limb_ty_id,
3379	.id_result = hi,
3380	.set = set,
3381	.instruction = .{ .inst = @backingInt(spec.OpenClOpcode.u_mul_hi) },
3382	.id_ref_4 = &.{ ci.limbs[i], other.limbs[j] },
3383	});
3384	},
3385	else => {
3386	const mul_result = cg.allocId();
3387	try cg.body.emit(gpa, .OpUMulExtended, .{
3388	.id_result_type = pair_struct_ty_id,
3389	.id_result = mul_result,
3390	.operand_1 = ci.limbs[i],
3391	.operand_2 = other.limbs[j],
3392	});
3393
3394	lo = cg.allocId();
3395	try cg.body.emit(gpa, .OpCompositeExtract, .{
3396	.id_result_type = limb_ty_id,
3397	.id_result = lo,
3398	.composite = mul_result,
3399	.indexes = &.{0},
3400	});
3401	hi = cg.allocId();
3402	try cg.body.emit(gpa, .OpCompositeExtract, .{
3403	.id_result_type = limb_ty_id,
3404	.id_result = hi,
3405	.composite = mul_result,
3406	.indexes = &.{1},
3407	});
3408	},
3409	}
3410
3411	const add1 = cg.allocId();
3412	try cg.body.emit(gpa, .OpIAddCarry, .{
3413	.id_result_type = pair_struct_ty_id,
3414	.id_result = add1,
3415	.operand_1 = result_limbs[k],
3416	.operand_2 = lo,
3417	});
3418
3419	const sum1 = cg.allocId();
3420	try cg.body.emit(gpa, .OpCompositeExtract, .{
3421	.id_result_type = limb_ty_id,
3422	.id_result = sum1,
3423	.composite = add1,
3424	.indexes = &.{0},
3425	});
3426	const c1 = cg.allocId();
3427	try cg.body.emit(gpa, .OpCompositeExtract, .{
3428	.id_result_type = limb_ty_id,
3429	.id_result = c1,
3430	.composite = add1,
3431	.indexes = &.{1},
3432	});
3433
3434	const add2 = cg.allocId();
3435	try cg.body.emit(gpa, .OpIAddCarry, .{
3436	.id_result_type = pair_struct_ty_id,
3437	.id_result = add2,
3438	.operand_1 = sum1,
3439	.operand_2 = carry_id,
3440	});
3441
3442	result_limbs[k] = cg.allocId();
3443	try cg.body.emit(gpa, .OpCompositeExtract, .{
3444	.id_result_type = limb_ty_id,
3445	.id_result = result_limbs[k],
3446	.composite = add2,
3447	.indexes = &.{0},
3448	});
3449	const c2 = cg.allocId();
3450	try cg.body.emit(gpa, .OpCompositeExtract, .{
3451	.id_result_type = limb_ty_id,
3452	.id_result = c2,
3453	.composite = add2,
3454	.indexes = &.{1},
3455	});
3456
3457	const hi_plus_c1 = try ci.limbBinOp(.OpIAdd, hi, c1);
3458	carry_id = try ci.limbBinOp(.OpIAdd, hi_plus_c1, c2);
3459	}
3460	if (wide and i + n < 2 * n) {
3461	result_limbs[i + n] = try ci.limbBinOp(.OpIAdd, result_limbs[i + n], carry_id);
3462	}
3463	}
3464
3465	return result_limbs;
3466	}
3467
3468	fn normalize(ci: CompositeInt) !CompositeInt {
3469	if (ci.info.bits == ci.info.backing_bits) return ci;
3470	const cg = ci.cg;
3471	const gpa = cg.gpa;
3472	const limb_bits = cg.bigIntBits();
3473	const top_bits: u16 = ci.info.bits % limb_bits;
3474	assert(top_bits != 0);
3475
3476	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, ci.n_limbs);
3477	for (0..ci.n_limbs - 1) |i| {
3478	result_limbs[i] = ci.limbs[i];
3479	}
3480
3481	const top_limb = ci.limbs[ci.n_limbs - 1];
3482	const limb_ty = cg.limbType();
3483	const limb_signed_ty: Type = if (limb_bits == 64) .i64 else .i32;
3484	switch (ci.info.signedness) {
3485	.unsigned => {
3486	const mask_val: u64 = (@as(u64, 1) << @as(u6, @intCast(top_bits))) - 1;
3487	const mask_id = try cg.constInt(limb_ty, mask_val);
3488	result_limbs[ci.n_limbs - 1] = try ci.limbBinOp(.OpBitwiseAnd, top_limb, mask_id);
3489	},
3490	.signed => {
3491	const limb_ty_id = try cg.limbTypeId();
3492	const signed_ty_id = try cg.resolveType(limb_signed_ty, .direct);
3493	const shift_amt: u32 = @intCast(limb_bits - top_bits);
3494	const shift_id = try cg.constInt(limb_ty, shift_amt);
3495
3496	const as_signed = cg.allocId();
3497	try cg.body.emit(gpa, .OpBitcast, .{
3498	.id_result_type = signed_ty_id,
3499	.id_result = as_signed,
3500	.operand = top_limb,
3501	});
3502	const shifted_left = cg.allocId();
3503	try cg.body.emit(gpa, .OpShiftLeftLogical, .{
3504	.id_result_type = signed_ty_id,
3505	.id_result = shifted_left,
3506	.base = as_signed,
3507	.shift = shift_id,
3508	});
3509	const shifted_right = cg.allocId();
3510	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
3511	.id_result_type = signed_ty_id,
3512	.id_result = shifted_right,
3513	.base = shifted_left,
3514	.shift = shift_id,
3515	});
3516	const back = cg.allocId();
3517	try cg.body.emit(gpa, .OpBitcast, .{
3518	.id_result_type = limb_ty_id,
3519	.id_result = back,
3520	.operand = shifted_right,
3521	});
3522	result_limbs[ci.n_limbs - 1] = back;
3523	},
3524	}
3525
3526	return .fromLimbs(cg, result_limbs, ci.info);
3527	}
3528	};
3529
3530	/// Initialize a `Temporary` from an AIR value.
3531	fn temporary(cg: *CodeGen, inst: Air.Inst.Ref) !Temporary {
3532	return .{
3533	.ty = cg.typeOf(inst),
3534	.value = .{ .singleton = try cg.resolve(inst) },
3535	};
3536	}
3537
3538	/// This union describes how a particular operation should be vectorized.
3539	/// That depends on the operation and number of components of the inputs.
3540	const Vectorization = union(enum) {
3541	/// This is an operation between scalars.
3542	scalar,
3543	/// This operation is unrolled into separate operations.
3544	/// Inputs may still be SPIR-V vectors, for example,
3545	/// when the operation can't be vectorized in SPIR-V.
3546	/// Value is number of components.
3547	unrolled: u32,
3548
3549	/// Derive a vectorization from a particular type
3550	fn fromType(ty: Type, cg: *CodeGen) Vectorization {
3551	const zcu = cg.zcu;
3552	if (!ty.isVector(zcu)) return .scalar;
3553	return .{ .unrolled = ty.vectorLen(zcu) };
3554	}
3555
3556	/// Given two vectorization methods, compute a "unification": a fallback
3557	/// that works for both, according to the following rules:
3558	/// - Scalars may broadcast
3559	/// - SPIR-V vectorized operations will unroll
3560	/// - Prefer scalar > unrolled
3561	fn unify(a: Vectorization, b: Vectorization) Vectorization {
3562	if (a == .scalar and b == .scalar) return .scalar;
3563	if (a == .unrolled or b == .unrolled) {
3564	if (a == .unrolled and b == .unrolled) assert(a.components() == b.components());
3565	if (a == .unrolled) return .{ .unrolled = a.components() };
3566	return .{ .unrolled = b.components() };
3567	}
3568	unreachable;
3569	}
3570
3571	/// Query the number of components that inputs of this operation have.
3572	/// Note: for broadcasting scalars, this returns the number of elements
3573	/// that the broadcasted vector would have.
3574	fn components(vec: Vectorization) u32 {
3575	return switch (vec) {
3576	.scalar => 1,
3577	.unrolled => |n| n,
3578	};
3579	}
3580
3581	/// Turns `ty` into the result-type of the entire operation.
3582	/// `ty` may be a scalar or vector, it doesn't matter.
3583	fn resultType(vec: Vectorization, cg: *CodeGen, ty: Type) !Type {
3584	const pt = cg.pt;
3585	const zcu = cg.zcu;
3586	const scalar_ty = ty.scalarType(zcu);
3587	return switch (vec) {
3588	.scalar => scalar_ty,
3589	.unrolled => |n| try pt.vectorType(.{ .len = n, .child = scalar_ty.toIntern() }),
3590	};
3591	}
3592
3593	/// Before a temporary can be used, some setup may need to be one. This function implements
3594	/// this setup, and returns a new type that holds the relevant information on how to access
3595	/// elements of the input.
3596	fn prepare(vec: Vectorization, cg: *CodeGen, tmp: Temporary) !PreparedOperand {
3597	const zcu = cg.zcu;
3598	const is_vector = tmp.ty.isVector(zcu);
3599	const value: PreparedOperand.Value = switch (tmp.value) {
3600	.singleton => |id| switch (vec) {
3601	.scalar => blk: {
3602	assert(!is_vector);
3603	break :blk .{ .scalar = id };
3604	},
3605	.unrolled => blk: {
3606	if (is_vector) break :blk .{ .vector_exploded = try tmp.explode(cg) };
3607	break :blk .{ .scalar_broadcast = id };
3608	},
3609	},
3610	.exploded_vector => |range| switch (vec) {
3611	.scalar => unreachable,
3612	.unrolled => |n| blk: {
3613	assert(range.len == n);
3614	break :blk .{ .vector_exploded = range };
3615	},
3616	},
3617	};
3618
3619	return .{
3620	.ty = tmp.ty,
3621	.value = value,
3622	};
3623	}
3624
3625	/// Finalize the results of an operation back into a temporary. `results` is
3626	/// a list of result-ids of the operation.
3627	fn finalize(vec: Vectorization, ty: Type, results: IdRange) Temporary {
3628	assert(vec.components() == results.len);
3629	return .{
3630	.ty = ty,
3631	.value = switch (vec) {
3632	.scalar => .{ .singleton = results.at(0) },
3633	.unrolled => .{ .exploded_vector = results },
3634	},
3635	};
3636	}
3637
3638	/// This struct represents an operand that has gone through some setup, and is
3639	/// ready to be used as part of an operation.
3640	const PreparedOperand = struct {
3641	ty: Type,
3642	value: PreparedOperand.Value,
3643
3644	/// The types of value that a prepared operand can hold internally. Depends
3645	/// on the operation and input value.
3646	const Value = union(enum) {
3647	/// A single scalar value that is used by a scalar operation.
3648	scalar: Id,
3649	/// A single scalar that is broadcasted in an unrolled operation.
3650	scalar_broadcast: Id,
3651	/// A vector represented by a consecutive list of IDs that is used in an unrolled operation.
3652	vector_exploded: IdRange,
3653	};
3654
3655	/// Query the value at a particular index of the operation. Note that
3656	/// the index is *not* the component/lane, but the index of the *operation*.
3657	fn at(op: PreparedOperand, i: usize) Id {
3658	switch (op.value) {
3659	.scalar => |id| {
3660	assert(i == 0);
3661	return id;
3662	},
3663	.scalar_broadcast => |id| return id,
3664	.vector_exploded => |range| return range.at(i),
3665	}
3666	}
3667	};
3668	};
3669
3670	/// A utility function to compute the vectorization style of
3671	/// a list of values. These values may be any of the following:
3672	/// - A `Vectorization` instance
3673	/// - A Type, in which case the vectorization is computed via `Vectorization.fromType`.
3674	/// - A Temporary, in which case the vectorization is computed via `Temporary.vectorization`.
3675	fn vectorization(cg: *CodeGen, args: anytype) Vectorization {
3676	var v: Vectorization = undefined;
3677	assert(args.len >= 1);
3678	inline for (args, 0..) |arg, i| {
3679	const iv: Vectorization = switch (@TypeOf(arg)) {
3680	Vectorization => arg,
3681	Type => Vectorization.fromType(arg, cg),
3682	Temporary => arg.vectorization(cg),
3683	else => @compileError("invalid type"),
3684	};
3685	if (i == 0) {
3686	v = iv;
3687	} else {
3688	v = v.unify(iv);
3689	}
3690	}
3691	return v;
3692	}
3693
3694	/// This function builds an OpSConvert of OpUConvert depending on the
3695	/// signedness of the types.
3696	fn buildConvert(cg: *CodeGen, dst_ty: Type, src: Temporary) !Temporary {
3697	const zcu = cg.zcu;
3698
3699	const v = cg.vectorization(.{ dst_ty, src });
3700	const result_ty = try v.resultType(cg, dst_ty);
3701
3702	const dst_scalar = dst_ty.scalarType(zcu);
3703	const src_scalar = src.ty.scalarType(zcu);
3704	if (dst_scalar.toIntern() == src_scalar.toIntern()) {
3705	return src.pun(result_ty);
3706	}
3707	if (dst_scalar.isInt(zcu) and src_scalar.isInt(zcu)) {
3708	const dst_info = dst_scalar.intInfo(zcu);
3709	const src_info = src_scalar.intInfo(zcu);
3710	if (cg.backingIntBits(dst_info.bits).@"0" == cg.backingIntBits(src_info.bits).@"0" and
3711	dst_info.signedness == src_info.signedness)
3712	{
3713	return src.pun(result_ty);
3714	}
3715	}
3716
3717	const ops = v.components();
3718	const results = cg.allocIds(ops);
3719
3720	const op_result_ty = dst_ty.scalarType(zcu);
3721	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3722
3723	const opcode: Opcode = blk: {
3724	if (dst_ty.scalarType(zcu).isAnyFloat()) break :blk .OpFConvert;
3725	if (dst_ty.scalarType(zcu).isSignedInt(zcu)) break :blk .OpSConvert;
3726	break :blk .OpUConvert;
3727	};
3728
3729	const op_src = try v.prepare(cg, src);
3730
3731	for (0..ops) |i| {
3732	try cg.body.emitRaw(cg.gpa, opcode, 3);
3733	cg.body.writeOperand(Id, op_result_ty_id);
3734	cg.body.writeOperand(Id, results.at(i));
3735	cg.body.writeOperand(Id, op_src.at(i));
3736	}
3737
3738	return v.finalize(result_ty, results);
3739	}
3740
3741	fn buildSelect(cg: *CodeGen, condition: Temporary, lhs: Temporary, rhs: Temporary) !Temporary {
3742	const zcu = cg.zcu;
3743
3744	const v = cg.vectorization(.{ condition, lhs, rhs });
3745	const ops = v.components();
3746	const results = cg.allocIds(ops);
3747
3748	const op_result_ty = lhs.ty.scalarType(zcu);
3749	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3750	const result_ty = try v.resultType(cg, lhs.ty);
3751
3752	assert(condition.ty.scalarType(zcu).zigTypeTag(zcu) == .bool);
3753
3754	const cond = try v.prepare(cg, condition);
3755	const object_1 = try v.prepare(cg, lhs);
3756	const object_2 = try v.prepare(cg, rhs);
3757
3758	for (0..ops) |i| {
3759	try cg.body.emit(cg.gpa, .OpSelect, .{
3760	.id_result_type = op_result_ty_id,
3761	.id_result = results.at(i),
3762	.condition = cond.at(i),
3763	.object_1 = object_1.at(i),
3764	.object_2 = object_2.at(i),
3765	});
3766	}
3767
3768	return v.finalize(result_ty, results);
3769	}
3770
3771	fn buildCmp(cg: *CodeGen, opcode: Opcode, lhs: Temporary, rhs: Temporary) !Temporary {
3772	const v = cg.vectorization(.{ lhs, rhs });
3773	const ops = v.components();
3774	const results = cg.allocIds(ops);
3775
3776	const op_result_ty: Type = .bool;
3777	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3778	const result_ty = try v.resultType(cg, Type.bool);
3779
3780	const op_lhs = try v.prepare(cg, lhs);
3781	const op_rhs = try v.prepare(cg, rhs);
3782
3783	for (0..ops) |i| {
3784	try cg.body.emitRaw(cg.gpa, opcode, 4);
3785	cg.body.writeOperand(Id, op_result_ty_id);
3786	cg.body.writeOperand(Id, results.at(i));
3787	cg.body.writeOperand(Id, op_lhs.at(i));
3788	cg.body.writeOperand(Id, op_rhs.at(i));
3789	}
3790
3791	return v.finalize(result_ty, results);
3792	}
3793
3794	const UnaryOp = enum {
3795	l_not,
3796	bit_not,
3797	i_neg,
3798	f_neg,
3799	i_abs,
3800	f_abs,
3801	clz,
3802	ctz,
3803	floor,
3804	ceil,
3805	trunc,
3806	round,
3807	sqrt,
3808	sin,
3809	cos,
3810	tan,
3811	exp,
3812	exp2,
3813	log,
3814	log2,
3815	log10,
3816
3817	pub fn extInstOpcode(op: UnaryOp, target: *const std.Target) ?u32 {
3818	return switch (target.os.tag) {
3819	.opencl => @backingInt(@as(spec.OpenClOpcode, switch (op) {
3820	.i_abs => .s_abs,
3821	.f_abs => .fabs,
3822	.clz => .clz,
3823	.ctz => .ctz,
3824	.floor => .floor,
3825	.ceil => .ceil,
3826	.trunc => .trunc,
3827	.round => .round,
3828	.sqrt => .sqrt,
3829	.sin => .sin,
3830	.cos => .cos,
3831	.tan => .tan,
3832	.exp => .exp,
3833	.exp2 => .exp2,
3834	.log => .log,
3835	.log2 => .log2,
3836	.log10 => .log10,
3837	else => return null,
3838	})),
3839	// Note: We'll need to check these for floating point accuracy
3840	// Vulkan does not put tight requirements on these, for correction
3841	// we might want to emulate them at some point.
3842	.vulkan, .opengl => @backingInt(@as(spec.GlslOpcode, switch (op) {
3843	.i_abs => .SAbs,
3844	.f_abs => .FAbs,
3845	.floor => .Floor,
3846	.ceil => .Ceil,
3847	.trunc => .Trunc,
3848	.round => .Round,
3849	.sin => .Sin,
3850	.cos => .Cos,
3851	.tan => .Tan,
3852	.sqrt => .Sqrt,
3853	.exp => .Exp,
3854	.exp2 => .Exp2,
3855	.log => .Log,
3856	.log2 => .Log2,
3857	else => return null,
3858	})),
3859	else => unreachable,
3860	};
3861	}
3862	};
3863
3864	fn buildUnary(cg: *CodeGen, op: UnaryOp, operand: Temporary) !Temporary {
3865	const zcu = cg.zcu;
3866	const target = cg.zcu.getTarget();
3867	const v = cg.vectorization(.{operand});
3868	const ops = v.components();
3869	const results = cg.allocIds(ops);
3870	const op_result_ty = operand.ty.scalarType(zcu);
3871	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3872	const result_ty = try v.resultType(cg, operand.ty);
3873	const op_operand = try v.prepare(cg, operand);
3874
3875	if (op.extInstOpcode(target)) |opcode| {
3876	const set = try cg.importExtendedSet();
3877	for (0..ops) |i| {
3878	try cg.body.emit(cg.gpa, .OpExtInst, .{
3879	.id_result_type = op_result_ty_id,
3880	.id_result = results.at(i),
3881	.set = set,
3882	.instruction = .{ .inst = opcode },
3883	.id_ref_4 = &.{op_operand.at(i)},
3884	});
3885	}
3886	} else {
3887	const opcode: Opcode = switch (op) {
3888	.l_not => .OpLogicalNot,
3889	.bit_not => .OpNot,
3890	.i_neg => .OpSNegate,
3891	.f_neg => .OpFNegate,
3892	else => return cg.todo(
3893	"implement unary operation '{s}' for {s} os",
3894	.{ @tagName(op), @tagName(target.os.tag) },
3895	),
3896	};
3897	for (0..ops) |i| {
3898	try cg.body.emitRaw(cg.gpa, opcode, 3);
3899	cg.body.writeOperand(Id, op_result_ty_id);
3900	cg.body.writeOperand(Id, results.at(i));
3901	cg.body.writeOperand(Id, op_operand.at(i));
3902	}
3903	}
3904
3905	return v.finalize(result_ty, results);
3906	}
3907
3908	fn buildBinary(cg: *CodeGen, opcode: Opcode, lhs: Temporary, rhs: Temporary) !Temporary {
3909	const zcu = cg.zcu;
3910
3911	const v = cg.vectorization(.{ lhs, rhs });
3912	const ops = v.components();
3913	const results = cg.allocIds(ops);
3914
3915	const op_result_ty = lhs.ty.scalarType(zcu);
3916	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3917	const result_ty = try v.resultType(cg, lhs.ty);
3918
3919	const op_lhs = try v.prepare(cg, lhs);
3920	const op_rhs = try v.prepare(cg, rhs);
3921
3922	for (0..ops) |i| {
3923	try cg.body.emitRaw(cg.gpa, opcode, 4);
3924	cg.body.writeOperand(Id, op_result_ty_id);
3925	cg.body.writeOperand(Id, results.at(i));
3926	cg.body.writeOperand(Id, op_lhs.at(i));
3927	cg.body.writeOperand(Id, op_rhs.at(i));
3928	}
3929
3930	return v.finalize(result_ty, results);
3931	}
3932
3933	/// This function builds an extended multiplication, either OpSMulExtended or OpUMulExtended on Vulkan,
3934	/// or OpIMul and s_mul_hi or u_mul_hi on OpenCL.
3935	fn buildWideMul(
3936	cg: *CodeGen,
3937	signedness: std.lang.Signedness,
3938	lhs: Temporary,
3939	rhs: Temporary,
3940	) !struct { Temporary, Temporary } {
3941	const pt = cg.pt;
3942	const zcu = cg.zcu;
3943	const comp = zcu.comp;
3944	const gpa = comp.gpa;
3945	const io = comp.io;
3946	const target = cg.zcu.getTarget();
3947	const ip = &zcu.intern_pool;
3948
3949	const v = lhs.vectorization(cg).unify(rhs.vectorization(cg));
3950	const ops = v.components();
3951
3952	const arith_op_ty = lhs.ty.scalarType(zcu);
3953	const arith_op_ty_id = try cg.resolveType(arith_op_ty, .direct);
3954
3955	const lhs_op = try v.prepare(cg, lhs);
3956	const rhs_op = try v.prepare(cg, rhs);
3957
3958	const value_results = cg.allocIds(ops);
3959	const overflow_results = cg.allocIds(ops);
3960
3961	switch (target.os.tag) {
3962	.opencl => {
3963	// Currently, SPIRV-LLVM-Translator based backends cannot deal with OpSMulExtended and
3964	// OpUMulExtended. For these we will use the OpenCL s_mul_hi to compute the high-order bits
3965	// instead.
3966	const set = try cg.importExtendedSet();
3967	const overflow_inst: spec.OpenClOpcode = switch (signedness) {
3968	.signed => .s_mul_hi,
3969	.unsigned => .u_mul_hi,
3970	};
3971
3972	for (0..ops) |i| {
3973	try cg.body.emit(gpa, .OpIMul, .{
3974	.id_result_type = arith_op_ty_id,
3975	.id_result = value_results.at(i),
3976	.operand_1 = lhs_op.at(i),
3977	.operand_2 = rhs_op.at(i),
3978	});
3979
3980	try cg.body.emit(gpa, .OpExtInst, .{
3981	.id_result_type = arith_op_ty_id,
3982	.id_result = overflow_results.at(i),
3983	.set = set,
3984	.instruction = .{ .inst = @backingInt(overflow_inst) },
3985	.id_ref_4 = &.{ lhs_op.at(i), rhs_op.at(i) },
3986	});
3987	}
3988	},
3989	.vulkan, .opengl => {
3990	// Operations return a struct{T, T}
3991	// where T is maybe vectorized.
3992	const op_result_ty: Type = .fromInterned(try ip.getTupleType(gpa, io, pt.tid, .{
3993	.types = &.{ arith_op_ty.toIntern(), arith_op_ty.toIntern() },
3994	.values = &.{ .none, .none },
3995	}));
3996	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
3997
3998	const opcode: Opcode = switch (signedness) {
3999	.signed => .OpSMulExtended,
4000	.unsigned => .OpUMulExtended,
4001	};
4002
4003	for (0..ops) |i| {
4004	const op_result = cg.allocId();
4005
4006	try cg.body.emitRaw(gpa, opcode, 4);
4007	cg.body.writeOperand(Id, op_result_ty_id);
4008	cg.body.writeOperand(Id, op_result);
4009	cg.body.writeOperand(Id, lhs_op.at(i));
4010	cg.body.writeOperand(Id, rhs_op.at(i));
4011
4012	// The above operation returns a struct. We might want to expand
4013	// Temporary to deal with the fact that these are structs eventually,
4014	// but for now, take the struct apart and return two separate vectors.
4015
4016	try cg.body.emit(gpa, .OpCompositeExtract, .{
4017	.id_result_type = arith_op_ty_id,
4018	.id_result = value_results.at(i),
4019	.composite = op_result,
4020	.indexes = &.{0},
4021	});
4022
4023	try cg.body.emit(gpa, .OpCompositeExtract, .{
4024	.id_result_type = arith_op_ty_id,
4025	.id_result = overflow_results.at(i),
4026	.composite = op_result,
4027	.indexes = &.{1},
4028	});
4029	}
4030	},
4031	else => unreachable,
4032	}
4033
4034	const result_ty = try v.resultType(cg, lhs.ty);
4035	return .{
4036	v.finalize(result_ty, value_results),
4037	v.finalize(result_ty, overflow_results),
4038	};
4039	}
4040
4041	/// The SPIR-V backend is not yet advanced enough to support the std testing infrastructure.
4042	/// In order to be able to run tests, we "temporarily" lower test kernels into separate entry-
4043	/// points. The test executor will then be able to invoke these to run the tests.
4044	/// Note that tests are lowered according to std.lang.TestFn, which is `fn () anyerror!void`.
4045	/// (anyerror!void has the same layout as anyerror).
4046	/// Each test declaration generates a function like.
4047	/// %anyerror = OpTypeInt 0 16
4048	/// %p_invocation_globals_struct_ty = ...
4049	/// %p_anyerror = OpTypePointer CrossWorkgroup %anyerror
4050	/// %K = OpTypeFunction %void %p_invocation_globals_struct_ty %p_anyerror
4051	///
4052	/// %test = OpFunction %void %K
4053	/// %p_invocation_globals = OpFunctionParameter p_invocation_globals_struct_ty
4054	/// %p_err = OpFunctionParameter %p_anyerror
4055	/// %lbl = OpLabel
4056	/// %result = OpFunctionCall %anyerror %func %p_invocation_globals
4057	/// OpStore %p_err %result
4058	/// OpFunctionEnd
4059	/// TODO is to also write out the error as a function call parameter, and to somehow fetch
4060	/// the name of an error in the text executor.
4061	fn generateTestEntryPoint(
4062	cg: *CodeGen,
4063	name: []const u8,
4064	spv_decl_index: Decl.Index,
4065	test_id: Id,
4066	) !void {
4067	const gpa = cg.gpa;
4068	const zcu = cg.zcu;
4069	const target = cg.zcu.getTarget();
4070
4071	const anyerror_ty_id = try cg.resolveType(.anyerror, .direct);
4072	const ptr_anyerror_ty = try cg.pt.ptrType(.{
4073	.child = .anyerror_type,
4074	.flags = .{ .address_space = .global },
4075	});
4076	const ptr_anyerror_ty_id = try cg.resolveType(ptr_anyerror_ty, .direct);
4077
4078	const kernel_id = cg.declPtr(spv_decl_index).result_id;
4079
4080	const section = &cg.sections.functions;
4081
4082	const p_error_id = cg.allocId();
4083	switch (target.os.tag) {
4084	.opencl, .amdhsa => {
4085	const void_ty_id = try cg.resolveType(.void, .direct);
4086	const kernel_proto_ty_id = try cg.functionType(void_ty_id, &.{ptr_anyerror_ty_id});
4087
4088	try section.emit(gpa, .OpFunction, .{
4089	.id_result_type = try cg.resolveType(.void, .direct),
4090	.id_result = kernel_id,
4091	.function_control = .{},
4092	.function_type = kernel_proto_ty_id,
4093	});
4094
4095	try section.emit(gpa, .OpFunctionParameter, .{
4096	.id_result_type = ptr_anyerror_ty_id,
4097	.id_result = p_error_id,
4098	});
4099
4100	try section.emit(gpa, .OpLabel, .{
4101	.id_result = cg.allocId(),
4102	});
4103	},
4104	.vulkan, .opengl => {
4105	if (cg.error_buffer == null) {
4106	const spv_err_decl_index = try cg.allocDecl(.global);
4107	const err_buf_result_id = cg.declPtr(spv_err_decl_index).result_id;
4108
4109	const buffer_struct_ty_id = cg.allocId();
4110	try cg.sections.globals.emit(gpa, .OpTypeStruct, .{
4111	.id_result = buffer_struct_ty_id,
4112	.id_ref = &.{anyerror_ty_id},
4113	});
4114	try cg.memberDebugName(buffer_struct_ty_id, 0, "error_out");
4115	try cg.decorate(buffer_struct_ty_id, .block);
4116	try cg.decorateMember(buffer_struct_ty_id, 0, .{ .offset = .{ .byte_offset = 0 } });
4117
4118	const ptr_buffer_struct_ty_id = cg.allocId();
4119	try cg.sections.globals.emit(gpa, .OpTypePointer, .{
4120	.id_result = ptr_buffer_struct_ty_id,
4121	.storage_class = cg.storageClass(.global),
4122	.type = buffer_struct_ty_id,
4123	});
4124
4125	try cg.sections.globals.emit(gpa, .OpVariable, .{
4126	.id_result_type = ptr_buffer_struct_ty_id,
4127	.id_result = err_buf_result_id,
4128	.storage_class = cg.storageClass(.global),
4129	});
4130	try cg.decorate(err_buf_result_id, .{ .descriptor_set = .{ .descriptor_set = 0 } });
4131	try cg.decorate(err_buf_result_id, .{ .binding = .{ .binding_point = 0 } });
4132
4133	cg.error_buffer = spv_err_decl_index;
4134	}
4135
4136	const void_ty_id = try cg.resolveType(.void, .direct);
4137	const kernel_proto_ty_id = try cg.functionType(void_ty_id, &.{});
4138	try section.emit(gpa, .OpFunction, .{
4139	.id_result_type = try cg.resolveType(.void, .direct),
4140	.id_result = kernel_id,
4141	.function_control = .{},
4142	.function_type = kernel_proto_ty_id,
4143	});
4144	try section.emit(gpa, .OpLabel, .{
4145	.id_result = cg.allocId(),
4146	});
4147
4148	const spv_err_decl_index = cg.error_buffer.?;
4149	const buffer_id = cg.declPtr(spv_err_decl_index).result_id;
4150	try cg.decl_deps.append(gpa, spv_err_decl_index);
4151
4152	const zero_id = try cg.constInt(.u32, 0);
4153	try section.emit(gpa, .OpInBoundsAccessChain, .{
4154	.id_result_type = ptr_anyerror_ty_id,
4155	.id_result = p_error_id,
4156	.base = buffer_id,
4157	.indexes = &.{zero_id},
4158	});
4159	},
4160	else => unreachable,
4161	}
4162
4163	const error_id = cg.allocId();
4164	try section.emit(gpa, .OpFunctionCall, .{
4165	.id_result_type = anyerror_ty_id,
4166	.id_result = error_id,
4167	.function = test_id,
4168	});
4169	// Note: Convert to direct not required.
4170	try section.emit(gpa, .OpStore, .{
4171	.pointer = p_error_id,
4172	.object = error_id,
4173	.memory_access = .{
4174	.aligned = .{ .literal_integer = @intCast(Type.abiAlignment(.anyerror, zcu).toByteUnits().?) },
4175	},
4176	});
4177	try section.emit(gpa, .OpReturn, {});
4178	try section.emit(gpa, .OpFunctionEnd, {});
4179
4180	// Just generate a quick other name because the intel runtime crashes when the entry-
4181	// point name is the same as a different OpName.
4182	const test_name = try std.fmt.allocPrint(cg.arena, "test {s}", .{name});
4183
4184	const ep_gop = try cg.entry_points.getOrPut(cg.gpa, cg.declPtr(spv_decl_index).result_id);
4185	ep_gop.value_ptr.* = .{
4186	.decl_index = spv_decl_index,
4187	.name = test_name,
4188	.cc = .{ .spirv_kernel = .{ .x = 1, .y = 1, .z = 1 } },
4189	};
4190	}
4191
4192	fn intFromBool(cg: *CodeGen, value: Temporary, result_ty: Type) !Temporary {
4193	const zero_id = try cg.constInt(result_ty, 0);
4194	const one_id = try cg.constInt(result_ty, 1);
4195
4196	return try cg.buildSelect(
4197	value,
4198	Temporary.init(result_ty, one_id),
4199	Temporary.init(result_ty, zero_id),
4200	);
4201	}
4202
4203	/// Convert representation from indirect (in memory) to direct (in 'register')
4204	/// This converts the argument type from resolveType(ty, .indirect) to resolveType(ty, .direct).
4205	fn convertToDirect(cg: *CodeGen, ty: Type, operand_id: Id) !Id {
4206	const pt = cg.pt;
4207	const zcu = cg.zcu;
4208	switch (ty.scalarType(zcu).zigTypeTag(zcu)) {
4209	.bool => {
4210	const false_id = try cg.constBool(false, .indirect);
4211	const operand_ty = blk: {
4212	if (!ty.isVector(zcu)) break :blk Type.u1;
4213	break :blk try pt.vectorType(.{
4214	.len = ty.vectorLen(zcu),
4215	.child = .u1_type,
4216	});
4217	};
4218
4219	const result = try cg.buildCmp(
4220	.OpINotEqual,
4221	Temporary.init(operand_ty, operand_id),
4222	Temporary.init(.u1, false_id),
4223	);
4224	return try result.materialize(cg);
4225	},
4226	else => return operand_id,
4227	}
4228	}
4229
4230	/// Convert representation from direct (in 'register) to direct (in memory)
4231	/// This converts the argument type from resolveType(ty, .direct) to resolveType(ty, .indirect).
4232	fn convertToIndirect(cg: *CodeGen, ty: Type, operand_id: Id) !Id {
4233	const zcu = cg.zcu;
4234	switch (ty.scalarType(zcu).zigTypeTag(zcu)) {
4235	.bool => {
4236	const result = try cg.intFromBool(.init(ty, operand_id), .u1);
4237	return try result.materialize(cg);
4238	},
4239	else => return operand_id,
4240	}
4241	}
4242
4243	fn extractField(cg: *CodeGen, result_ty: Type, object: Id, field: u32) !Id {
4244	const result_ty_id = try cg.resolveType(result_ty, .indirect);
4245	const result_id = cg.allocId();
4246	const indexes = [_]u32{field};
4247	try cg.body.emit(cg.gpa, .OpCompositeExtract, .{
4248	.id_result_type = result_ty_id,
4249	.id_result = result_id,
4250	.composite = object,
4251	.indexes = &indexes,
4252	});
4253	// Convert bools; direct structs have their field types as indirect values.
4254	return try cg.convertToDirect(result_ty, result_id);
4255	}
4256
4257	fn extractVectorComponent(cg: *CodeGen, result_ty: Type, vector_id: Id, field: u32) !Id {
4258	const result_ty_id = try cg.resolveType(result_ty, .direct);
4259	const result_id = cg.allocId();
4260	const indexes = [_]u32{field};
4261	try cg.body.emit(cg.gpa, .OpCompositeExtract, .{
4262	.id_result_type = result_ty_id,
4263	.id_result = result_id,
4264	.composite = vector_id,
4265	.indexes = &indexes,
4266	});
4267	// Vector components are already stored in direct representation.
4268	return result_id;
4269	}
4270
4271	const MemoryOptions = struct {
4272	is_volatile: bool = false,
4273	ptr_address_space: std.lang.AddressSpace = .generic,
4274	};
4275
4276	/// Returns true if a pointee at address space must use the
4277	/// layout-decorated variant rather than the bare type.
4278	fn needsLayout(cg: *CodeGen, as: std.lang.AddressSpace, pointee_ty: Type) bool {
4279	const target = cg.zcu.getTarget();
4280	if (target.os.tag != .vulkan and target.os.tag != .opengl) return false;
4281	return switch (as) {
4282	.uniform,
4283	.push_constant,
4284	.storage_buffer,
4285	.physical_storage_buffer,
4286	=> switch (pointee_ty.zigTypeTag(cg.zcu)) {
4287	.@"struct", .@"union", .array => true,
4288	.spirv => pointee_ty.isSpirvRuntimeArray(cg.zcu),
4289	else => false,
4290	},
4291	else => false,
4292	};
4293	}
4294
4295	fn pointeeType(cg: *CodeGen, as: std.lang.AddressSpace, ty: Type, is_block_root: bool) !Id {
4296	return if (cg.needsLayout(as, ty))
4297	cg.layoutType(ty, is_block_root)
4298	else
4299	cg.resolveType(ty, .indirect);
4300	}
4301
4302	fn convertLayout(cg: *CodeGen, dst_ty_id: Id, src_id: Id, src_ty_id: Id) !Id {
4303	if (dst_ty_id == src_ty_id) return src_id;
4304	const id = cg.allocId();
4305	try cg.body.emit(cg.gpa, .OpCopyLogical, .{
4306	.id_result_type = dst_ty_id,
4307	.id_result = id,
4308	.operand = src_id,
4309	});
4310	return id;
4311	}
4312
4313	fn load(cg: *CodeGen, value_ty: Type, ptr_id: Id, options: MemoryOptions) !Id {
4314	const zcu = cg.zcu;
4315	const alignment: u32 = @intCast(value_ty.abiAlignment(zcu).toByteUnits().?);
4316	const bare_ty_id = try cg.resolveType(value_ty, .indirect);
4317	const load_ty_id = if (cg.needsLayout(options.ptr_address_space, value_ty))
4318	try cg.layoutType(value_ty, cg.block_var_ids.contains(ptr_id))
4319	else
4320	bare_ty_id;
4321	const loaded_id = cg.allocId();
4322	try cg.body.emit(cg.gpa, .OpLoad, .{
4323	.id_result_type = load_ty_id,
4324	.id_result = loaded_id,
4325	.pointer = ptr_id,
4326	.memory_access = .{
4327	.@"volatile" = options.is_volatile,
4328	.aligned = .{ .literal_integer = alignment },
4329	},
4330	});
4331	const result_id = try cg.convertLayout(bare_ty_id, loaded_id, load_ty_id);
4332	return try cg.convertToDirect(value_ty, result_id);
4333	}
4334
4335	fn store(cg: *CodeGen, value_ty: Type, ptr_id: Id, value_id: Id, options: MemoryOptions) !void {
4336	const zcu = cg.zcu;
4337	const alignment: u32 = @intCast(value_ty.abiAlignment(zcu).toByteUnits().?);
4338	const bare_value_id = try cg.convertToIndirect(value_ty, value_id);
4339	const bare_ty_id = try cg.resolveType(value_ty, .indirect);
4340	const store_ty_id = if (cg.needsLayout(options.ptr_address_space, value_ty))
4341	try cg.layoutType(value_ty, cg.block_var_ids.contains(ptr_id))
4342	else
4343	bare_ty_id;
4344	const object_id = try cg.convertLayout(store_ty_id, bare_value_id, bare_ty_id);
4345	try cg.body.emit(cg.gpa, .OpStore, .{
4346	.pointer = ptr_id,
4347	.object = object_id,
4348	.memory_access = .{
4349	.@"volatile" = options.is_volatile,
4350	.aligned = .{ .literal_integer = alignment },
4351	},
4352	});
4353	}
4354
4355	fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) !void {
4356	for (body) |inst| {
4357	try cg.genInst(inst);
4358	}
4359	}
4360
4361	fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {
4362	const gpa = cg.gpa;
4363	const zcu = cg.zcu;
4364	const ip = &zcu.intern_pool;
4365	if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip))
4366	return;
4367
4368	const air_tags = cg.air.instructions.items(.tag);
4369	const maybe_result_id: ?Id = switch (air_tags[@backingInt(inst)]) {
4370	// zig fmt: off
4371	.add, .add_wrap, .add_optimized => try cg.airArithOp(inst, .OpFAdd, .OpIAdd, .OpIAdd),
4372	.sub, .sub_wrap, .sub_optimized => try cg.airArithOp(inst, .OpFSub, .OpISub, .OpISub),
4373	.mul, .mul_wrap, .mul_optimized => try cg.airArithOp(inst, .OpFMul, .OpIMul, .OpIMul),
4374
4375	.sqrt => try cg.airUnOpSimple(inst, .sqrt),
4376	.sin => try cg.airUnOpSimple(inst, .sin),
4377	.cos => try cg.airUnOpSimple(inst, .cos),
4378	.tan => try cg.airUnOpSimple(inst, .tan),
4379	.exp => try cg.airUnOpSimple(inst, .exp),
4380	.exp2 => try cg.airUnOpSimple(inst, .exp2),
4381	.log => try cg.airUnOpSimple(inst, .log),
4382	.log2 => try cg.airUnOpSimple(inst, .log2),
4383	.log10 => try cg.airUnOpSimple(inst, .log10),
4384	.abs => try cg.airAbs(inst),
4385	.floor => try cg.airUnOpSimple(inst, .floor),
4386	.ceil => try cg.airUnOpSimple(inst, .ceil),
4387	.round => try cg.airUnOpSimple(inst, .round),
4388	.trunc_float => try cg.airUnOpSimple(inst, .trunc),
4389	.neg, .neg_optimized => try cg.airUnOpSimple(inst, .f_neg),
4390
4391	.div_float, .div_float_optimized => try cg.airArithOp(inst, .OpFDiv, .OpSDiv, .OpUDiv),
4392	.div_floor, .div_floor_optimized => try cg.airDivFloor(inst),
4393	.div_trunc, .div_trunc_optimized => try cg.airDivTrunc(inst),
4394
4395	.rem, .rem_optimized => try cg.airArithOp(inst, .OpFRem, .OpSRem, .OpUMod),
4396	.mod, .mod_optimized => try cg.airArithOp(inst, .OpFMod, .OpSMod, .OpUMod),
4397
4398	.add_with_overflow => try cg.airAddSubOverflow(inst, .OpIAdd, .OpULessThan, .OpSLessThan),
4399	.sub_with_overflow => try cg.airAddSubOverflow(inst, .OpISub, .OpUGreaterThan, .OpSGreaterThan),
4400	.mul_with_overflow => try cg.airMulOverflow(inst),
4401	.shl_with_overflow => try cg.airShlOverflow(inst),
4402
4403	.mul_add => try cg.airMulAdd(inst),
4404
4405	.ctz => try cg.airClzCtz(inst, .ctz),
4406	.clz => try cg.airClzCtz(inst, .clz),
4407
4408	.select => try cg.airSelect(inst),
4409
4410	.splat => try cg.airSplat(inst),
4411	.reduce, .reduce_optimized => try cg.airReduce(inst),
4412	.shuffle_one => try cg.airShuffleOne(inst),
4413	.shuffle_two => try cg.airShuffleTwo(inst),
4414
4415	.ptr_add => try cg.airPtrAdd(inst),
4416	.ptr_sub => try cg.airPtrSub(inst),
4417
4418	.bit_and => try cg.airBitwiseOp(inst, .bit_and),
4419	.bit_or => try cg.airBitwiseOp(inst, .bit_or),
4420	.xor => try cg.airBitwiseOp(inst, .xor),
4421
4422	.shl, .shl_exact => try cg.airShift(inst, .OpShiftLeftLogical, .OpShiftLeftLogical),
4423	.shr, .shr_exact => try cg.airShift(inst, .OpShiftRightLogical, .OpShiftRightArithmetic),
4424
4425	.min => try cg.airMinMax(inst, .min),
4426	.max => try cg.airMinMax(inst, .max),
4427
4428	.bit_cast => try cg.airBitCast(inst),
4429	.ptr_cast => try cg.airBitCast(inst),
4430	.ptr_from_int => try cg.airBitCast(inst),
4431	.int_from_ptr => try cg.airBitCast(inst),
4432	.error_cast => try cg.airBitCast(inst),
4433	.error_from_int => try cg.airBitCast(inst),
4434	.int_from_error => try cg.airBitCast(inst),
4435	.union_from_enum => try cg.airBitCast(inst),
4436	.int_cast, .trunc => try cg.airIntCast(inst),
4437	.float_from_int => try cg.airFloatFromInt(inst),
4438	.int_from_float => try cg.airIntFromFloat(inst),
4439	.fpext, .fptrunc => try cg.airFloatCast(inst),
4440	.not => try cg.airNot(inst),
4441
4442	.array_to_slice => try cg.airArrayToSlice(inst),
4443	.array_to_vector => unreachable, // legalize .expand_array_to_vector
4444	.slice => try cg.airSlice(inst),
4445	.aggregate_init => try cg.airAggregateInit(inst),
4446	.memcpy => return cg.airMemcpy(inst),
4447	.memmove => return cg.airMemmove(inst),
4448
4449	.slice_ptr => try cg.airSliceField(inst, 0),
4450	.slice_len => try cg.airSliceField(inst, 1),
4451	.ptr_slice_ptr_ptr => try cg.airStructFieldPtrIndex(inst, 0),
4452	.ptr_slice_len_ptr => try cg.airStructFieldPtrIndex(inst, 1),
4453	.spirv_runtime_array_len => try cg.airSpirvRuntimeArrayLen(inst),
4454	.slice_elem_ptr => try cg.airSliceElemPtr(inst),
4455	.slice_elem_val => try cg.airSliceElemVal(inst),
4456	.ptr_elem_ptr => try cg.airPtrElemPtr(inst),
4457	.ptr_elem_val => try cg.airPtrElemVal(inst),
4458	.array_elem_val => try cg.airArrayElemVal(inst),
4459
4460	.set_union_tag => return cg.airSetUnionTag(inst),
4461	.get_union_tag => try cg.airGetUnionTag(inst),
4462	.union_init => try cg.airUnionInit(inst),
4463
4464	.agg_field_val => try cg.airAggFieldVal(inst),
4465	.field_parent_ptr => try cg.airFieldParentPtr(inst),
4466
4467	.struct_field_ptr => try cg.airStructFieldPtr(inst),
4468
4469	.struct_field_ptr_index_0 => try cg.airStructFieldPtrIndex(inst, 0),
4470	.struct_field_ptr_index_1 => try cg.airStructFieldPtrIndex(inst, 1),
4471	.struct_field_ptr_index_2 => try cg.airStructFieldPtrIndex(inst, 2),
4472	.struct_field_ptr_index_3 => try cg.airStructFieldPtrIndex(inst, 3),
4473
4474	.cmp_eq => try cg.airCmp(inst, .eq),
4475	.cmp_neq => try cg.airCmp(inst, .neq),
4476	.cmp_gt => try cg.airCmp(inst, .gt),
4477	.cmp_gte => try cg.airCmp(inst, .gte),
4478	.cmp_lt => try cg.airCmp(inst, .lt),
4479	.cmp_lte => try cg.airCmp(inst, .lte),
4480	.cmp_vector => try cg.airVectorCmp(inst),
4481
4482	.arg => cg.airArg(),
4483	.alloc => try cg.airAlloc(inst),
4484	// TODO: We probably need to have a special implementation of this for the C abi.
4485	.ret_ptr => try cg.airAlloc(inst),
4486	.block => try cg.airBlock(inst),
4487
4488	.load => try cg.airLoad(inst),
4489	.store, .store_safe => return cg.airStore(inst),
4490
4491	.br => return cg.airBr(inst),
4492	// For now just ignore this instruction. This effectively falls back on the old implementation,
4493	// this doesn't change anything for us.
4494	.repeat => return,
4495	.breakpoint => return,
4496	.cond_br => return cg.airCondBr(inst),
4497	.loop => return cg.airLoop(inst),
4498	.ret => return cg.airRet(inst),
4499	.ret_safe => return cg.airRet(inst), // TODO
4500	.ret_load => return cg.airRetLoad(inst),
4501	.@"try" => try cg.airTry(inst),
4502	.switch_br => return cg.airSwitchBr(inst),
4503	.loop_switch_br => return cg.airLoopSwitchBr(inst),
4504	.switch_dispatch => return cg.airSwitchDispatch(inst),
4505	.unreach, .trap => return cg.airUnreach(),
4506
4507	.dbg_empty_stmt => return,
4508	.dbg_stmt => return cg.airDbgStmt(inst),
4509	.dbg_inline_block => try cg.airDbgInlineBlock(inst),
4510	.dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => return cg.airDbgVar(inst),
4511
4512	.unwrap_errunion_err => try cg.airErrUnionErr(inst),
4513	.unwrap_errunion_payload => try cg.airErrUnionPayload(inst),
4514	.wrap_errunion_err => try cg.airWrapErrUnionErr(inst),
4515	.wrap_errunion_payload => try cg.airWrapErrUnionPayload(inst),
4516
4517	.is_null => try cg.airIsNull(inst, false, .is_null),
4518	.is_non_null => try cg.airIsNull(inst, false, .is_non_null),
4519	.is_null_ptr => try cg.airIsNull(inst, true, .is_null),
4520	.is_non_null_ptr => try cg.airIsNull(inst, true, .is_non_null),
4521	.is_err => try cg.airIsErr(inst, .is_err),
4522	.is_non_err => try cg.airIsErr(inst, .is_non_err),
4523
4524	.optional_payload => try cg.airUnwrapOptional(inst),
4525	.optional_payload_ptr => try cg.airUnwrapOptionalPtr(inst),
4526	.optional_payload_ptr_set => try cg.airSetOptionalPtr(inst),
4527	.wrap_optional => try cg.airWrapOptional(inst),
4528
4529	.assembly => try cg.airAssembly(inst),
4530
4531	.call => try cg.airCall(inst, .auto),
4532	.call_always_tail => try cg.airCall(inst, .always_tail),
4533	.call_never_tail => try cg.airCall(inst, .never_tail),
4534	.call_never_inline => try cg.airCall(inst, .never_inline),
4535
4536	.work_item_id => try cg.airWorkItemId(inst),
4537	.work_group_size => try cg.airWorkGroupSize(inst),
4538	.work_group_id => try cg.airWorkGroupId(inst),
4539
4540	// zig fmt: on
4541
4542	else => |tag| return cg.todo("implement AIR tag {s}", .{@tagName(tag)}),
4543	};
4544
4545	const result_id = maybe_result_id orelse return;
4546	try cg.inst_results.putNoClobber(gpa, inst, result_id);
4547	}
4548
4549	fn airBinOpSimple(cg: *CodeGen, inst: Air.Inst.Index, op: Opcode) !?Id {
4550	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4551	const lhs = try cg.temporary(bin_op.lhs);
4552	const rhs = try cg.temporary(bin_op.rhs);
4553
4554	const result = try cg.buildBinary(op, lhs, rhs);
4555	return try result.materialize(cg);
4556	}
4557
4558	const BitwiseOp = enum { bit_and, bit_or, xor };
4559
4560	fn airBitwiseOp(cg: *CodeGen, inst: Air.Inst.Index, op: BitwiseOp) !?Id {
4561	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4562	const lhs = try cg.temporary(bin_op.lhs);
4563	const rhs = try cg.temporary(bin_op.rhs);
4564	const info = cg.arithmeticTypeInfo(lhs.ty);
4565
4566	// SPIR-V requires logical opcodes for booleans, bitwise opcodes for integers.
4567	const opcode: Opcode = switch (info.class) {
4568	.bool => switch (op) {
4569	.bit_and => .OpLogicalAnd,
4570	.bit_or => .OpLogicalOr,
4571	.xor => .OpLogicalNotEqual,
4572	},
4573	.integer, .strange_integer => switch (op) {
4574	.bit_and => .OpBitwiseAnd,
4575	.bit_or => .OpBitwiseOr,
4576	.xor => .OpBitwiseXor,
4577	},
4578	.float => unreachable,
4579	.composite_integer => {
4580	const spv_opcode: Opcode = switch (op) {
4581	.bit_and => .OpBitwiseAnd,
4582	.bit_or => .OpBitwiseOr,
4583	.xor => .OpBitwiseXor,
4584	};
4585	const lhs_id = try lhs.materialize(cg);
4586	const rhs_id = try rhs.materialize(cg);
4587	const scratch_top = cg.id_scratch.items.len;
4588	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4589	const ci_lhs = try CompositeInt.init(cg, lhs_id, info);
4590	const ci_rhs = try CompositeInt.init(cg, rhs_id, info);
4591	const ci_result = try ci_lhs.bitwiseOp(ci_rhs, spv_opcode);
4592	return try ci_result.materialize(lhs.ty);
4593	},
4594	};
4595
4596	const result = try cg.buildBinary(opcode, lhs, rhs);
4597	return try result.materialize(cg);
4598	}
4599
4600	fn airShift(cg: *CodeGen, inst: Air.Inst.Index, unsigned: Opcode, signed: Opcode) !?Id {
4601	const zcu = cg.zcu;
4602	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4603
4604	const base = try cg.temporary(bin_op.lhs);
4605	const shift = try cg.temporary(bin_op.rhs);
4606
4607	const result_ty = cg.typeOfIndex(inst);
4608
4609	const info = cg.arithmeticTypeInfo(result_ty);
4610	switch (info.class) {
4611	.composite_integer => {
4612	const shift_info = cg.arithmeticTypeInfo(shift.ty);
4613	const limb_ty = cg.limbType();
4614	const shift_amt_id = switch (shift_info.class) {
4615	.composite_integer => blk: {
4616	const shift_id = try shift.materialize(cg);
4617	const limb_ty_id = try cg.limbTypeId();
4618	const result_id = cg.allocId();
4619	try cg.body.emit(cg.gpa, .OpCompositeExtract, .{
4620	.id_result_type = limb_ty_id,
4621	.id_result = result_id,
4622	.composite = shift_id,
4623	.indexes = &.{@as(u32, 0)},
4624	});
4625	break :blk result_id;
4626	},
4627	else => blk: {
4628	const converted = try cg.buildConvert(limb_ty, shift);
4629	break :blk try converted.materialize(cg);
4630	},
4631	};
4632	const base_id = try base.materialize(cg);
4633	const scratch_top = cg.id_scratch.items.len;
4634	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4635	const ci = try CompositeInt.init(cg, base_id, info);
4636	const ci_result = if (unsigned == .OpShiftLeftLogical)
4637	try ci.shl(shift_amt_id)
4638	else switch (info.signedness) {
4639	.unsigned => try ci.shr(shift_amt_id, false),
4640	.signed => try ci.shr(shift_amt_id, true),
4641	};
4642	const normalized = try ci_result.normalize();
4643	return try normalized.materialize(result_ty);
4644	},
4645	.integer, .strange_integer => {},
4646	.float, .bool => unreachable,
4647	}
4648
4649	// Sometimes Zig doesn't make both of the arguments the same types here. SPIR-V expects that,
4650	// so just manually upcast it if required.
4651
4652	// Note: The sign may differ here between the shift and the base type, in case
4653	// of an arithmetic right shift. SPIR-V still expects the same type,
4654	// so in that case we have to cast convert to signed.
4655	const casted_shift = try cg.buildConvert(base.ty.scalarType(zcu), shift);
4656
4657	const shifted = switch (info.signedness) {
4658	.unsigned => try cg.buildBinary(unsigned, base, casted_shift),
4659	.signed => try cg.buildBinary(signed, base, casted_shift),
4660	};
4661
4662	const result = try cg.normalize(shifted, info);
4663	return try result.materialize(cg);
4664	}
4665
4666	const MinMax = enum {
4667	min,
4668	max,
4669
4670	pub fn extInstOpcode(
4671	op: MinMax,
4672	target: *const std.Target,
4673	info: ArithmeticTypeInfo,
4674	) u32 {
4675	return switch (target.os.tag) {
4676	.opencl => @backingInt(@as(spec.OpenClOpcode, switch (info.class) {
4677	.float => switch (op) {
4678	.min => .fmin,
4679	.max => .fmax,
4680	},
4681	.integer, .strange_integer, .composite_integer => switch (info.signedness) {
4682	.signed => switch (op) {
4683	.min => .s_min,
4684	.max => .s_max,
4685	},
4686	.unsigned => switch (op) {
4687	.min => .u_min,
4688	.max => .u_max,
4689	},
4690	},
4691	.bool => unreachable,
4692	})),
4693	.vulkan, .opengl => @backingInt(@as(spec.GlslOpcode, switch (info.class) {
4694	.float => switch (op) {
4695	.min => .FMin,
4696	.max => .FMax,
4697	},
4698	.integer, .strange_integer, .composite_integer => switch (info.signedness) {
4699	.signed => switch (op) {
4700	.min => .SMin,
4701	.max => .SMax,
4702	},
4703	.unsigned => switch (op) {
4704	.min => .UMin,
4705	.max => .UMax,
4706	},
4707	},
4708	.bool => unreachable,
4709	})),
4710	else => unreachable,
4711	};
4712	}
4713	};
4714
4715	fn airMinMax(cg: *CodeGen, inst: Air.Inst.Index, op: MinMax) !?Id {
4716	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4717
4718	const lhs = try cg.temporary(bin_op.lhs);
4719	const rhs = try cg.temporary(bin_op.rhs);
4720
4721	const result = try cg.minMax(lhs, rhs, op);
4722	return try result.materialize(cg);
4723	}
4724
4725	fn minMax(cg: *CodeGen, lhs: Temporary, rhs: Temporary, op: MinMax) !Temporary {
4726	const zcu = cg.zcu;
4727	const target = zcu.getTarget();
4728	const info = cg.arithmeticTypeInfo(lhs.ty);
4729
4730	const v = cg.vectorization(.{ lhs, rhs });
4731	const ops = v.components();
4732	const results = cg.allocIds(ops);
4733
4734	const op_result_ty = lhs.ty.scalarType(zcu);
4735	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
4736	const result_ty = try v.resultType(cg, lhs.ty);
4737
4738	const op_lhs = try v.prepare(cg, lhs);
4739	const op_rhs = try v.prepare(cg, rhs);
4740
4741	const set = try cg.importExtendedSet();
4742	const opcode = op.extInstOpcode(target, info);
4743	for (0..ops) |i| {
4744	try cg.body.emit(cg.gpa, .OpExtInst, .{
4745	.id_result_type = op_result_ty_id,
4746	.id_result = results.at(i),
4747	.set = set,
4748	.instruction = .{ .inst = opcode },
4749	.id_ref_4 = &.{ op_lhs.at(i), op_rhs.at(i) },
4750	});
4751	}
4752
4753	return v.finalize(result_ty, results);
4754	}
4755
4756	/// This function normalizes values to a canonical representation
4757	/// after some arithmetic operation. This mostly consists of wrapping
4758	/// behavior for strange integers:
4759	/// - Unsigned integers are bitwise masked with a mask that only passes
4760	/// the valid bits through.
4761	/// - Signed integers are also sign extended if they are negative.
4762	/// All other values are returned unmodified (this makes strange integer
4763	/// wrapping easier to use in generic operations).
4764	fn normalize(cg: *CodeGen, value: Temporary, info: ArithmeticTypeInfo) !Temporary {
4765	const zcu = cg.zcu;
4766	const ty = value.ty;
4767	switch (info.class) {
4768	.integer, .bool, .float => return value,
4769	.composite_integer => {
4770	if (info.bits == info.backing_bits) return value;
4771	const val_id = try value.materialize(cg);
4772	const scratch_top = cg.id_scratch.items.len;
4773	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4774	const ci = try CompositeInt.init(cg, val_id, info);
4775	const normalized = try ci.normalize();
4776	return .init(ty, try normalized.materialize(ty));
4777	},
4778	.strange_integer => switch (info.signedness) {
4779	.unsigned => {
4780	const mask_value = @as(u64, std.math.maxInt(u64)) >> @as(u6, @intCast(64 - info.bits));
4781	const mask_id = try cg.constInt(ty.scalarType(zcu), mask_value);
4782	return try cg.buildBinary(.OpBitwiseAnd, value, Temporary.init(ty.scalarType(zcu), mask_id));
4783	},
4784	.signed => {
4785	// Shift left and right so that we can copy the sight bit that way.
4786	const shift_amt_id = try cg.constInt(ty.scalarType(zcu), info.backing_bits - info.bits);
4787	const shift_amt: Temporary = .init(ty.scalarType(zcu), shift_amt_id);
4788	const left = try cg.buildBinary(.OpShiftLeftLogical, value, shift_amt);
4789	return try cg.buildBinary(.OpShiftRightArithmetic, left, shift_amt);
4790	},
4791	},
4792	}
4793	}
4794
4795	fn airDivFloor(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
4796	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4797
4798	const lhs = try cg.temporary(bin_op.lhs);
4799	const rhs = try cg.temporary(bin_op.rhs);
4800
4801	const info = cg.arithmeticTypeInfo(lhs.ty);
4802	switch (info.class) {
4803	.composite_integer => return cg.todo("div_floor for composite integers", .{}),
4804	.integer, .strange_integer => {
4805	switch (info.signedness) {
4806	.unsigned => {
4807	const result = try cg.buildBinary(.OpUDiv, lhs, rhs);
4808	return try result.materialize(cg);
4809	},
4810	.signed => {},
4811	}
4812
4813	// For signed integers:
4814	// (a / b) - (a % b != 0 && a < 0 != b < 0);
4815	// There shouldn't be any overflow issues.
4816
4817	const div = try cg.buildBinary(.OpSDiv, lhs, rhs);
4818	const rem = try cg.buildBinary(.OpSRem, lhs, rhs);
4819	const zero: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, 0));
4820	const rem_non_zero = try cg.buildCmp(.OpINotEqual, rem, zero);
4821	const lhs_rhs_xor = try cg.buildBinary(.OpBitwiseXor, lhs, rhs);
4822	const signs_differ = try cg.buildCmp(.OpSLessThan, lhs_rhs_xor, zero);
4823	const adjust = try cg.buildBinary(.OpLogicalAnd, rem_non_zero, signs_differ);
4824	const result = try cg.buildBinary(.OpISub, div, try cg.intFromBool(adjust, div.ty));
4825	return try result.materialize(cg);
4826	},
4827	.float => {
4828	const div = try cg.buildBinary(.OpFDiv, lhs, rhs);
4829	const result = try cg.buildUnary(.floor, div);
4830	return try result.materialize(cg);
4831	},
4832	.bool => unreachable,
4833	}
4834	}
4835
4836	fn airDivTrunc(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
4837	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4838	const lhs = try cg.temporary(bin_op.lhs);
4839	const rhs = try cg.temporary(bin_op.rhs);
4840	const info = cg.arithmeticTypeInfo(lhs.ty);
4841	switch (info.class) {
4842	.composite_integer => return cg.todo("div_trunc for composite integers", .{}),
4843	.integer, .strange_integer => switch (info.signedness) {
4844	.unsigned => {
4845	const result = try cg.buildBinary(.OpUDiv, lhs, rhs);
4846	return try result.materialize(cg);
4847	},
4848	.signed => {
4849	const result = try cg.buildBinary(.OpSDiv, lhs, rhs);
4850	return try result.materialize(cg);
4851	},
4852	},
4853	.float => {
4854	const div = try cg.buildBinary(.OpFDiv, lhs, rhs);
4855	const result = try cg.buildUnary(.trunc, div);
4856	return try result.materialize(cg);
4857	},
4858	.bool => unreachable,
4859	}
4860	}
4861
4862	fn airUnOpSimple(cg: *CodeGen, inst: Air.Inst.Index, op: UnaryOp) !?Id {
4863	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
4864	const operand = try cg.temporary(un_op);
4865	const result = try cg.buildUnary(op, operand);
4866	return try result.materialize(cg);
4867	}
4868
4869	fn airArithOp(
4870	cg: *CodeGen,
4871	inst: Air.Inst.Index,
4872	comptime fop: Opcode,
4873	comptime sop: Opcode,
4874	comptime uop: Opcode,
4875	) !?Id {
4876	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
4877	const lhs = try cg.temporary(bin_op.lhs);
4878	const rhs = try cg.temporary(bin_op.rhs);
4879	const info = cg.arithmeticTypeInfo(lhs.ty);
4880	const result = switch (info.class) {
4881	.composite_integer => res: {
4882	const lhs_id = try lhs.materialize(cg);
4883	const rhs_id = try rhs.materialize(cg);
4884	const scratch_top = cg.id_scratch.items.len;
4885	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4886	const ci_lhs = try CompositeInt.init(cg, lhs_id, info);
4887	const ci_rhs = try CompositeInt.init(cg, rhs_id, info);
4888	const ci_result = switch (uop) {
4889	.OpIAdd => try ci_lhs.addSub(ci_rhs, true),
4890	.OpISub => try ci_lhs.addSub(ci_rhs, false),
4891	.OpIMul => CompositeInt.fromLimbs(cg, try ci_lhs.mul(ci_rhs, false), info),
4892	else => return cg.todo("arith op for composite integers", .{}),
4893	};
4894	const normalized = try ci_result.normalize();
4895	break :res Temporary.init(lhs.ty, try normalized.materialize(lhs.ty));
4896	},
4897	.integer, .strange_integer => res: {
4898	const raw = switch (info.signedness) {
4899	.signed => try cg.buildBinary(sop, lhs, rhs),
4900	.unsigned => try cg.buildBinary(uop, lhs, rhs),
4901	};
4902	break :res try cg.normalize(raw, info);
4903	},
4904	.float => try cg.buildBinary(fop, lhs, rhs),
4905	.bool => unreachable,
4906	};
4907	return try result.materialize(cg);
4908	}
4909
4910	fn airAbs(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
4911	const zcu = cg.zcu;
4912	const target = zcu.getTarget();
4913	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
4914	const value = try cg.temporary(ty_op.operand);
4915	// Note: operand_ty may be signed, while ty is always unsigned.
4916	const result_ty = cg.typeOfIndex(inst);
4917	const operand_info = cg.arithmeticTypeInfo(value.ty);
4918	const result: Temporary = switch (operand_info.class) {
4919	.float => try cg.buildUnary(.f_abs, value),
4920	.integer, .strange_integer => abs: {
4921	var abs_value = try cg.buildUnary(.i_abs, value);
4922	switch (target.os.tag) {
4923	.vulkan, .opengl => {
4924	if (value.ty.intInfo(zcu).signedness == .signed) {
4925	const abs_id = try abs_value.materialize(cg);
4926	const dst_ty_id = try cg.resolveType(result_ty, .direct);
4927	const cast_id = cg.allocId();
4928	try cg.body.emit(cg.gpa, .OpBitcast, .{
4929	.id_result_type = dst_ty_id,
4930	.id_result = cast_id,
4931	.operand = abs_id,
4932	});
4933	abs_value = .init(result_ty, cast_id);
4934	}
4935	},
4936	else => {},
4937	}
4938	break :abs try cg.normalize(abs_value, cg.arithmeticTypeInfo(result_ty));
4939	},
4940	.composite_integer => abs: {
4941	const val_id = try value.materialize(cg);
4942	const scratch_top = cg.id_scratch.items.len;
4943	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4944	const ci = try CompositeInt.init(cg, val_id, operand_info);
4945	const ci_z = try CompositeInt.zero(cg, operand_info);
4946	const is_neg = try ci.cmp(ci_z, .lt);
4947	const ci_neg = try ci_z.addSub(ci, false);
4948	const result_info = cg.arithmeticTypeInfo(result_ty);
4949	const limb_ty_id = try cg.limbTypeId();
4950	const result_limbs = try cg.id_scratch.addManyAsSlice(cg.gpa, ci.n_limbs);
4951	for (0..ci.n_limbs) |i| {
4952	result_limbs[i] = cg.allocId();
4953	try cg.body.emit(cg.gpa, .OpSelect, .{
4954	.id_result_type = limb_ty_id,
4955	.id_result = result_limbs[i],
4956	.condition = is_neg,
4957	.object_1 = ci_neg.limbs[i],
4958	.object_2 = ci.limbs[i],
4959	});
4960	}
4961	const ci_result = CompositeInt.fromLimbs(cg, result_limbs, result_info);
4962	const normalized = try ci_result.normalize();
4963	break :abs .init(result_ty, try normalized.materialize(result_ty));
4964	},
4965	.bool => unreachable,
4966	};
4967	return try result.materialize(cg);
4968	}
4969
4970	fn airAddSubOverflow(
4971	cg: *CodeGen,
4972	inst: Air.Inst.Index,
4973	comptime add: Opcode,
4974	u_opcode: Opcode,
4975	s_opcode: Opcode,
4976	) !?Id {
4977	// Note: OpIAddCarry and OpISubBorrow are not really useful here: For unsigned numbers,
4978	// there is in both cases only one extra operation required. For signed operations,
4979	// the overflow bit is set then going from 0x80.. to 0x00.., but this doesn't actually
4980	// normally set a carry bit. So the SPIR-V overflow operations are not particularly
4981	// useful here.
4982
4983	_ = s_opcode;
4984
4985	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
4986	const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
4987	const lhs = try cg.temporary(extra.lhs);
4988	const rhs = try cg.temporary(extra.rhs);
4989	const result_ty = cg.typeOfIndex(inst);
4990
4991	const info = cg.arithmeticTypeInfo(lhs.ty);
4992	switch (info.class) {
4993	.composite_integer => {
4994	const lhs_id = try lhs.materialize(cg);
4995	const rhs_id = try rhs.materialize(cg);
4996	const scratch_top = cg.id_scratch.items.len;
4997	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
4998	const ci_lhs = try CompositeInt.init(cg, lhs_id, info);
4999	const ci_rhs = try CompositeInt.init(cg, rhs_id, info);
5000	const ci_sum = if (add == .OpIAdd) try ci_lhs.addSub(ci_rhs, true) else try ci_lhs.addSub(ci_rhs, false);
5001	const ci_result = try ci_sum.normalize();
5002	const result_val_id = try ci_result.materialize(lhs.ty);
5003
5004	const ov_bool = switch (info.signedness) {
5005	.unsigned => blk: {
5006	const ci_res2 = try CompositeInt.init(cg, result_val_id, info);
5007	const ci_lhs2 = try CompositeInt.init(cg, lhs_id, info);
5008	break :blk if (add == .OpIAdd)
5009	try ci_res2.cmp(ci_lhs2, .lt)
5010	else
5011	try ci_res2.cmp(ci_lhs2, .gt);
5012	},
5013	.signed => blk: {
5014	const ci_res2 = try CompositeInt.init(cg, result_val_id, info);
5015	const ci_lhs2 = try CompositeInt.init(cg, lhs_id, info);
5016	const ci_rhs2 = try CompositeInt.init(cg, rhs_id, info);
5017	const ci_z = try CompositeInt.zero(cg, info);
5018	const lhs_neg = try ci_lhs2.cmp(ci_z, .lt);
5019	const rhs_neg = try ci_rhs2.cmp(ci_z, .lt);
5020	const res_neg = try ci_res2.cmp(ci_z, .lt);
5021
5022	const bool_ty_id = try cg.resolveType(.bool, .direct);
5023	const signs_match = cg.allocId();
5024	try cg.body.emit(cg.gpa, .OpLogicalEqual, .{
5025	.id_result_type = bool_ty_id,
5026	.id_result = signs_match,
5027	.operand_1 = lhs_neg,
5028	.operand_2 = rhs_neg,
5029	});
5030	const res_sign_diff = cg.allocId();
5031	try cg.body.emit(cg.gpa, .OpLogicalNotEqual, .{
5032	.id_result_type = bool_ty_id,
5033	.id_result = res_sign_diff,
5034	.operand_1 = lhs_neg,
5035	.operand_2 = res_neg,
5036	});
5037	const ov_cond = if (add == .OpIAdd) signs_match else blk2: {
5038	const not_match = cg.allocId();
5039	try cg.body.emit(cg.gpa, .OpLogicalNot, .{
5040	.id_result_type = bool_ty_id,
5041	.id_result = not_match,
5042	.operand = signs_match,
5043	});
5044	break :blk2 not_match;
5045	};
5046	const ov_result = cg.allocId();
5047	try cg.body.emit(cg.gpa, .OpLogicalAnd, .{
5048	.id_result_type = bool_ty_id,
5049	.id_result = ov_result,
5050	.operand_1 = ov_cond,
5051	.operand_2 = res_sign_diff,
5052	});
5053	break :blk ov_result;
5054	},
5055	};
5056	const ov = try cg.intFromBool(.init(.bool, ov_bool), .u1);
5057	const result_ty_id = try cg.resolveType(result_ty, .direct);
5058	return try cg.constructComposite(result_ty_id, &.{ result_val_id, try ov.materialize(cg) });
5059	},
5060	.strange_integer, .integer => {},
5061	.float, .bool => unreachable,
5062	}
5063
5064	const sum = try cg.buildBinary(add, lhs, rhs);
5065	const result = try cg.normalize(sum, info);
5066	const overflowed = switch (info.signedness) {
5067	// Overflow happened if the result is smaller than either of the operands. It doesn't matter which.
5068	// For subtraction the conditions need to be swapped.
5069	.unsigned => try cg.buildCmp(u_opcode, result, lhs),
5070	// For signed operations, we check the signs of the operands and the result.
5071	.signed => blk: {
5072	// Signed overflow detection using the sign bits of the operands and the result.
5073	// For addition (a + b), overflow occurs if the operands have the same sign
5074	// and the result's sign is different from the operands' sign.
5075	// (sign(a) == sign(b)) && (sign(a) != sign(result))
5076	// For subtraction (a - b), overflow occurs if the operands have different signs
5077	// and the result's sign is different from the minuend's (a's) sign.
5078	// (sign(a) != sign(b)) && (sign(a) != sign(result))
5079	const zero: Temporary = .init(rhs.ty, try cg.constInt(rhs.ty, 0));
5080	const lhs_is_neg = try cg.buildCmp(.OpSLessThan, lhs, zero);
5081	const rhs_is_neg = try cg.buildCmp(.OpSLessThan, rhs, zero);
5082	const result_is_neg = try cg.buildCmp(.OpSLessThan, result, zero);
5083	const signs_match = try cg.buildCmp(.OpLogicalEqual, lhs_is_neg, rhs_is_neg);
5084	const result_sign_differs = try cg.buildCmp(.OpLogicalNotEqual, lhs_is_neg, result_is_neg);
5085	const overflow_condition = switch (add) {
5086	.OpIAdd => signs_match,
5087	.OpISub => try cg.buildUnary(.l_not, signs_match),
5088	else => unreachable,
5089	};
5090	break :blk try cg.buildCmp(.OpLogicalAnd, overflow_condition, result_sign_differs);
5091	},
5092	};
5093
5094	const ov = try cg.intFromBool(overflowed, .u1);
5095	const result_ty_id = try cg.resolveType(result_ty, .direct);
5096	return try cg.constructComposite(result_ty_id, &.{ try result.materialize(cg), try ov.materialize(cg) });
5097	}
5098
5099	fn airMulOverflow(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5100	const pt = cg.pt;
5101	const gpa = cg.gpa;
5102	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5103	const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5104	const lhs = try cg.temporary(extra.lhs);
5105	const rhs = try cg.temporary(extra.rhs);
5106	const result_ty = cg.typeOfIndex(inst);
5107
5108	const info = cg.arithmeticTypeInfo(lhs.ty);
5109	switch (info.class) {
5110	.composite_integer => {
5111	const lhs_id = try lhs.materialize(cg);
5112	const rhs_id = try rhs.materialize(cg);
5113	const scratch_top = cg.id_scratch.items.len;
5114	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5115	const ci_lhs = try CompositeInt.init(cg, lhs_id, info);
5116	const ci_rhs = try CompositeInt.init(cg, rhs_id, info);
5117
5118	const low_limbs = try ci_lhs.mul(ci_rhs, false);
5119	const ci_result = try CompositeInt.fromLimbs(cg, low_limbs, info).normalize();
5120	const result_val_id = try ci_result.materialize(lhs.ty);
5121
5122	const ci_lhs2 = try CompositeInt.init(cg, lhs_id, info);
5123	const ci_rhs2 = try CompositeInt.init(cg, rhs_id, info);
5124	const wide_limbs = try ci_lhs2.mul(ci_rhs2, true);
5125	const high_limbs = wide_limbs[ci_lhs2.n_limbs..];
5126
5127	const bool_ty_id = try cg.resolveType(.bool, .direct);
5128	const limb_ty_id = try cg.limbTypeId();
5129	const limb_ty = cg.limbType();
5130	const n: usize = info.backing_bits / cg.bigIntBits();
5131
5132	const ov_bool = switch (info.signedness) {
5133	.unsigned => blk: {
5134	const zero_id = try cg.constInt(limb_ty, @as(u64, 0));
5135	var any_nonzero = cg.allocId();
5136	try cg.body.emit(gpa, .OpINotEqual, .{
5137	.id_result_type = bool_ty_id,
5138	.id_result = any_nonzero,
5139	.operand_1 = high_limbs[0],
5140	.operand_2 = zero_id,
5141	});
5142
5143	for (1..n) |i| {
5144	const limb_nz = cg.allocId();
5145	try cg.body.emit(gpa, .OpINotEqual, .{
5146	.id_result_type = bool_ty_id,
5147	.id_result = limb_nz,
5148	.operand_1 = high_limbs[i],
5149	.operand_2 = zero_id,
5150	});
5151
5152	const combined = cg.allocId();
5153	try cg.body.emit(gpa, .OpLogicalOr, .{
5154	.id_result_type = bool_ty_id,
5155	.id_result = combined,
5156	.operand_1 = any_nonzero,
5157	.operand_2 = limb_nz,
5158	});
5159	any_nonzero = combined;
5160	}
5161
5162	break :blk any_nonzero;
5163	},
5164	.signed => blk: {
5165	const ci_res = try CompositeInt.init(cg, result_val_id, info);
5166	const top_limb = ci_res.limbs[n - 1];
5167	const signed_limb_ty: Type = if (cg.bigIntBits() == 64) .i64 else .i32;
5168	const signed_limb_ty_id = try cg.resolveType(signed_limb_ty, .direct);
5169
5170	const top_bits: u16 = if (info.bits % cg.bigIntBits() == 0)
5171	cg.bigIntBits()
5172	else
5173	info.bits % cg.bigIntBits();
5174
5175	const shift_amt: u64 = top_bits - 1;
5176	const shift_id = try cg.constInt(limb_ty, shift_amt);
5177
5178	const as_signed = cg.allocId();
5179	try cg.body.emit(gpa, .OpBitcast, .{
5180	.id_result_type = signed_limb_ty_id,
5181	.id_result = as_signed,
5182	.operand = top_limb,
5183	});
5184	const sign_ext = cg.allocId();
5185	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
5186	.id_result_type = signed_limb_ty_id,
5187	.id_result = sign_ext,
5188	.base = as_signed,
5189	.shift = shift_id,
5190	});
5191	const expected = cg.allocId();
5192	try cg.body.emit(gpa, .OpBitcast, .{
5193	.id_result_type = limb_ty_id,
5194	.id_result = expected,
5195	.operand = sign_ext,
5196	});
5197
5198	var any_mismatch = cg.allocId();
5199	try cg.body.emit(gpa, .OpINotEqual, .{
5200	.id_result_type = bool_ty_id,
5201	.id_result = any_mismatch,
5202	.operand_1 = high_limbs[0],
5203	.operand_2 = expected,
5204	});
5205
5206	for (1..n) |i| {
5207	const limb_ne = cg.allocId();
5208	try cg.body.emit(gpa, .OpINotEqual, .{
5209	.id_result_type = bool_ty_id,
5210	.id_result = limb_ne,
5211	.operand_1 = high_limbs[i],
5212	.operand_2 = expected,
5213	});
5214
5215	const combined = cg.allocId();
5216	try cg.body.emit(gpa, .OpLogicalOr, .{
5217	.id_result_type = bool_ty_id,
5218	.id_result = combined,
5219	.operand_1 = any_mismatch,
5220	.operand_2 = limb_ne,
5221	});
5222	any_mismatch = combined;
5223	}
5224
5225	if (info.bits != info.backing_bits) {
5226	const top_bits_s: u16 = info.bits % cg.bigIntBits();
5227	const s_shift_id = try cg.constInt(limb_ty, @as(u64, top_bits_s - 1));
5228
5229	const top_as_signed = cg.allocId();
5230	try cg.body.emit(gpa, .OpBitcast, .{
5231	.id_result_type = signed_limb_ty_id,
5232	.id_result = top_as_signed,
5233	.operand = top_limb,
5234	});
5235	const top_sign_ext = cg.allocId();
5236	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
5237	.id_result_type = signed_limb_ty_id,
5238	.id_result = top_sign_ext,
5239	.base = top_as_signed,
5240	.shift = s_shift_id,
5241	});
5242	const top_expected = cg.allocId();
5243	try cg.body.emit(gpa, .OpBitcast, .{
5244	.id_result_type = limb_ty_id,
5245	.id_result = top_expected,
5246	.operand = top_sign_ext,
5247	});
5248	const top_mismatch = cg.allocId();
5249	try cg.body.emit(gpa, .OpINotEqual, .{
5250	.id_result_type = bool_ty_id,
5251	.id_result = top_mismatch,
5252	.operand_1 = top_limb,
5253	.operand_2 = top_expected,
5254	});
5255
5256	const combined = cg.allocId();
5257	try cg.body.emit(gpa, .OpLogicalOr, .{
5258	.id_result_type = bool_ty_id,
5259	.id_result = combined,
5260	.operand_1 = any_mismatch,
5261	.operand_2 = top_mismatch,
5262	});
5263	any_mismatch = combined;
5264	}
5265
5266	break :blk any_mismatch;
5267	},
5268	};
5269
5270	const ov = try cg.intFromBool(.init(.bool, ov_bool), .u1);
5271	const result_ty_id = try cg.resolveType(result_ty, .direct);
5272	return try cg.constructComposite(result_ty_id, &.{ result_val_id, try ov.materialize(cg) });
5273	},
5274	.strange_integer, .integer => {},
5275	.float, .bool => unreachable,
5276	}
5277
5278	// There are 3 cases which we have to deal with:
5279	// - If info.bits < 32 / 2, we will upcast to 32 and check the higher bits
5280	// - If info.bits > 32 / 2, we have to use extended multiplication
5281	// - Additionally, if info.bits != 32, we'll have to check the high bits
5282	// of the result too.
5283
5284	const target = cg.zcu.getTarget();
5285	const largest_int_bits: u16 = if (hasInt64(target)) 64 else 32;
5286	// If non-null, the number of bits that the multiplication should be performed in. If
5287	// null, we have to use wide multiplication.
5288	const maybe_op_ty_bits: ?u16 = switch (info.bits) {
5289	0 => unreachable,
5290	1...16 => 32,
5291	17...32 => if (largest_int_bits > 32) 64 else null, // Upcast if we can.
5292	33...64 => null, // Always use wide multiplication.
5293	else => unreachable,
5294	};
5295
5296	const result, const overflowed = switch (info.signedness) {
5297	.unsigned => blk: {
5298	if (maybe_op_ty_bits) |op_ty_bits| {
5299	const op_ty = try pt.intType(.unsigned, op_ty_bits);
5300	const casted_lhs = try cg.buildConvert(op_ty, lhs);
5301	const casted_rhs = try cg.buildConvert(op_ty, rhs);
5302	const full_result = try cg.buildBinary(.OpIMul, casted_lhs, casted_rhs);
5303	const low_bits = try cg.buildConvert(lhs.ty, full_result);
5304	const result = try cg.normalize(low_bits, info);
5305	// Shift the result bits away to get the overflow bits.
5306	const shift: Temporary = .init(full_result.ty, try cg.constInt(full_result.ty, info.bits));
5307	const overflow = try cg.buildBinary(.OpShiftRightLogical, full_result, shift);
5308	// Directly check if its zero in the op_ty without converting first.
5309	const zero: Temporary = .init(full_result.ty, try cg.constInt(full_result.ty, 0));
5310	const overflowed = try cg.buildCmp(.OpINotEqual, zero, overflow);
5311	break :blk .{ result, overflowed };
5312	}
5313
5314	const low_bits, const high_bits = try cg.buildWideMul(.unsigned, lhs, rhs);
5315
5316	// Truncate the result, if required.
5317	const result = try cg.normalize(low_bits, info);
5318
5319	// Overflow happened if the high-bits of the result are non-zero OR if the
5320	// high bits of the low word of the result (those outside the range of the
5321	// int) are nonzero.
5322	const zero: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, 0));
5323	const high_overflowed = try cg.buildCmp(.OpINotEqual, zero, high_bits);
5324
5325	// If no overflow bits in low_bits, no extra work needs to be done.
5326	if (info.backing_bits == info.bits) break :blk .{ result, high_overflowed };
5327
5328	// Shift the result bits away to get the overflow bits.
5329	const shift: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, info.bits));
5330	const low_overflow = try cg.buildBinary(.OpShiftRightLogical, low_bits, shift);
5331	const low_overflowed = try cg.buildCmp(.OpINotEqual, zero, low_overflow);
5332
5333	const overflowed = try cg.buildCmp(.OpLogicalOr, low_overflowed, high_overflowed);
5334
5335	break :blk .{ result, overflowed };
5336	},
5337	.signed => blk: {
5338	// - lhs >= 0, rhxs >= 0: expect positive; overflow should be 0
5339	// - lhs == 0 : expect positive; overflow should be 0
5340	// - rhs == 0: expect positive; overflow should be 0
5341	// - lhs > 0, rhs < 0: expect negative; overflow should be -1
5342	// - lhs < 0, rhs > 0: expect negative; overflow should be -1
5343	// - lhs <= 0, rhs <= 0: expect positive; overflow should be 0
5344	// ------
5345	// overflow should be -1 when
5346	// (lhs > 0 && rhs < 0) || (lhs < 0 && rhs > 0)
5347
5348	const zero: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, 0));
5349	const lhs_negative = try cg.buildCmp(.OpSLessThan, lhs, zero);
5350	const rhs_negative = try cg.buildCmp(.OpSLessThan, rhs, zero);
5351	const lhs_positive = try cg.buildCmp(.OpSGreaterThan, lhs, zero);
5352	const rhs_positive = try cg.buildCmp(.OpSGreaterThan, rhs, zero);
5353
5354	// Set to `true` if we expect -1.
5355	const expected_overflow_bit = try cg.buildBinary(
5356	.OpLogicalOr,
5357	try cg.buildCmp(.OpLogicalAnd, lhs_positive, rhs_negative),
5358	try cg.buildCmp(.OpLogicalAnd, lhs_negative, rhs_positive),
5359	);
5360
5361	if (maybe_op_ty_bits) |op_ty_bits| {
5362	const op_ty = try pt.intType(.signed, op_ty_bits);
5363	// Assume normalized; sign bit is set. We want a sign extend.
5364	const casted_lhs = try cg.buildConvert(op_ty, lhs);
5365	const casted_rhs = try cg.buildConvert(op_ty, rhs);
5366
5367	const full_result = try cg.buildBinary(.OpIMul, casted_lhs, casted_rhs);
5368
5369	// Truncate to the result type.
5370	const low_bits = try cg.buildConvert(lhs.ty, full_result);
5371	const result = try cg.normalize(low_bits, info);
5372
5373	// Now, we need to check the overflow bits AND the sign
5374	// bit for the expected overflow bits.
5375	// To do that, shift out everything bit the sign bit and
5376	// then check what remains.
5377	const shift: Temporary = .init(full_result.ty, try cg.constInt(full_result.ty, info.bits - 1));
5378	// Use SRA so that any sign bits are duplicated. Now we can just check if ALL bits are set
5379	// for negative cases.
5380	const overflow = try cg.buildBinary(.OpShiftRightArithmetic, full_result, shift);
5381
5382	const long_all_set: Temporary = .init(full_result.ty, try cg.constInt(full_result.ty, -1));
5383	const long_zero: Temporary = .init(full_result.ty, try cg.constInt(full_result.ty, 0));
5384	const mask = try cg.buildSelect(expected_overflow_bit, long_all_set, long_zero);
5385
5386	const overflowed = try cg.buildCmp(.OpINotEqual, mask, overflow);
5387
5388	break :blk .{ result, overflowed };
5389	}
5390
5391	const low_bits, const high_bits = try cg.buildWideMul(.signed, lhs, rhs);
5392
5393	// Truncate result if required.
5394	const result = try cg.normalize(low_bits, info);
5395
5396	const all_set: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, -1));
5397	const mask = try cg.buildSelect(expected_overflow_bit, all_set, zero);
5398
5399	// Like with unsigned, overflow happened if high_bits are not the ones we expect,
5400	// and we also need to check some ones from the low bits.
5401
5402	const high_overflowed = try cg.buildCmp(.OpINotEqual, mask, high_bits);
5403
5404	// If no overflow bits in low_bits, no extra work needs to be done.
5405	// Careful, we still have to check the sign bit, so this branch
5406	// only goes for i33 and such.
5407	if (info.backing_bits == info.bits + 1) break :blk .{ result, high_overflowed };
5408
5409	// Shift the result bits away to get the overflow bits.
5410	const shift: Temporary = .init(lhs.ty, try cg.constInt(lhs.ty, info.bits - 1));
5411	// Use SRA so that any sign bits are duplicated. Now we can just check if ALL bits are set
5412	// for negative cases.
5413	const low_overflow = try cg.buildBinary(.OpShiftRightArithmetic, low_bits, shift);
5414	const low_overflowed = try cg.buildCmp(.OpINotEqual, mask, low_overflow);
5415
5416	const overflowed = try cg.buildCmp(.OpLogicalOr, low_overflowed, high_overflowed);
5417
5418	break :blk .{ result, overflowed };
5419	},
5420	};
5421
5422	const ov = try cg.intFromBool(overflowed, .u1);
5423
5424	const result_ty_id = try cg.resolveType(result_ty, .direct);
5425	return try cg.constructComposite(result_ty_id, &.{ try result.materialize(cg), try ov.materialize(cg) });
5426	}
5427
5428	fn airShlOverflow(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5429	const zcu = cg.zcu;
5430
5431	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5432	const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5433
5434	const base = try cg.temporary(extra.lhs);
5435	const shift = try cg.temporary(extra.rhs);
5436
5437	const result_ty = cg.typeOfIndex(inst);
5438
5439	const info = cg.arithmeticTypeInfo(base.ty);
5440	switch (info.class) {
5441	.composite_integer => return cg.todo("shl-with-overflow for composite integers", .{}),
5442	.integer, .strange_integer => {},
5443	.float, .bool => unreachable,
5444	}
5445
5446	// Sometimes Zig doesn't make both of the arguments the same types here. SPIR-V expects that,
5447	// so just manually upcast it if required.
5448	const casted_shift = try cg.buildConvert(base.ty.scalarType(zcu), shift);
5449
5450	const left = try cg.buildBinary(.OpShiftLeftLogical, base, casted_shift);
5451	const result = try cg.normalize(left, info);
5452
5453	const right = switch (info.signedness) {
5454	.unsigned => try cg.buildBinary(.OpShiftRightLogical, result, casted_shift),
5455	.signed => try cg.buildBinary(.OpShiftRightArithmetic, result, casted_shift),
5456	};
5457
5458	const overflowed = try cg.buildCmp(.OpINotEqual, base, right);
5459	const ov = try cg.intFromBool(overflowed, .u1);
5460
5461	const result_ty_id = try cg.resolveType(result_ty, .direct);
5462	return try cg.constructComposite(result_ty_id, &.{ try result.materialize(cg), try ov.materialize(cg) });
5463	}
5464
5465	fn airMulAdd(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5466	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
5467	const extra = cg.air.extraData(Air.Bin, pl_op.payload).data;
5468
5469	const a = try cg.temporary(extra.lhs);
5470	const b = try cg.temporary(extra.rhs);
5471	const c = try cg.temporary(pl_op.operand);
5472
5473	const result_ty = cg.typeOfIndex(inst);
5474	const info = cg.arithmeticTypeInfo(result_ty);
5475	assert(info.class == .float); // .mul_add is only emitted for floats
5476
5477	const zcu = cg.zcu;
5478	const target = zcu.getTarget();
5479
5480	const v = cg.vectorization(.{ a, b, c });
5481	const ops = v.components();
5482	const results = cg.allocIds(ops);
5483
5484	const op_result_ty = a.ty.scalarType(zcu);
5485	const op_result_ty_id = try cg.resolveType(op_result_ty, .direct);
5486	const result_temp_ty = try v.resultType(cg, a.ty);
5487
5488	const op_a = try v.prepare(cg, a);
5489	const op_b = try v.prepare(cg, b);
5490	const op_c = try v.prepare(cg, c);
5491
5492	const set = try cg.importExtendedSet();
5493	const opcode: u32 = switch (target.os.tag) {
5494	.opencl => @backingInt(spec.OpenClOpcode.fma),
5495	// NOTE: Vulkan's FMA does not meet Zig's nor OpenCL's precision guarantees and needs
5496	// to be emulated.
5497	.vulkan, .opengl => @backingInt(spec.GlslOpcode.Fma),
5498	else => unreachable,
5499	};
5500
5501	for (0..ops) |i| {
5502	try cg.body.emit(cg.gpa, .OpExtInst, .{
5503	.id_result_type = op_result_ty_id,
5504	.id_result = results.at(i),
5505	.set = set,
5506	.instruction = .{ .inst = opcode },
5507	.id_ref_4 = &.{ op_a.at(i), op_b.at(i), op_c.at(i) },
5508	});
5509	}
5510
5511	const result = v.finalize(result_temp_ty, results);
5512	return try result.materialize(cg);
5513	}
5514
5515	fn airClzCtz(cg: *CodeGen, inst: Air.Inst.Index, op: UnaryOp) !?Id {
5516	if (cg.liveness.isUnused(inst)) return null;
5517
5518	const zcu = cg.zcu;
5519	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5520	const operand = try cg.temporary(ty_op.operand);
5521
5522	const scalar_result_ty = cg.typeOfIndex(inst).scalarType(zcu);
5523
5524	const info = cg.arithmeticTypeInfo(operand.ty);
5525	switch (info.class) {
5526	.composite_integer => return cg.todo("@clz/@ctz for composite integers", .{}),
5527	.integer, .strange_integer => {},
5528	.float, .bool => unreachable,
5529	}
5530
5531	const count = try cg.buildUnary(op, operand);
5532
5533	// Result of OpenCL ctz/clz returns operand.ty, and we want result_ty.
5534	// result_ty is always large enough to hold the result, so we might have to down
5535	// cast it.
5536	const result = try cg.buildConvert(scalar_result_ty, count);
5537	return try result.materialize(cg);
5538	}
5539
5540	fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5541	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
5542	const extra = cg.air.extraData(Air.Bin, pl_op.payload).data;
5543	const pred = try cg.temporary(pl_op.operand);
5544	const a = try cg.temporary(extra.lhs);
5545	const b = try cg.temporary(extra.rhs);
5546
5547	const result = try cg.buildSelect(pred, a, b);
5548	return try result.materialize(cg);
5549	}
5550
5551	fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5552	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5553
5554	const operand_id = try cg.resolve(ty_op.operand);
5555	const result_ty = cg.typeOfIndex(inst);
5556
5557	return try cg.constructCompositeSplat(result_ty, operand_id);
5558	}
5559
5560	fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5561	const zcu = cg.zcu;
5562	const reduce = cg.air.instructions.items(.data)[@backingInt(inst)].reduce;
5563	const operand = try cg.resolve(reduce.operand);
5564	const operand_ty = cg.typeOf(reduce.operand);
5565	const scalar_ty = operand_ty.scalarType(zcu);
5566	const info = cg.arithmeticTypeInfo(operand_ty);
5567	const len = operand_ty.vectorLen(zcu);
5568	const first = try cg.extractVectorComponent(scalar_ty, operand, 0);
5569
5570	switch (reduce.operation) {
5571	.Min, .Max => |op| {
5572	var result: Temporary = .init(scalar_ty, first);
5573	const cmp_op: MinMax = switch (op) {
5574	.Max => .max,
5575	.Min => .min,
5576	else => unreachable,
5577	};
5578	for (1..len) |i| {
5579	const lhs = result;
5580	const rhs_id = try cg.extractVectorComponent(scalar_ty, operand, @intCast(i));
5581	const rhs: Temporary = .init(scalar_ty, rhs_id);
5582
5583	result = try cg.minMax(lhs, rhs, cmp_op);
5584	}
5585
5586	return try result.materialize(cg);
5587	},
5588	else => {},
5589	}
5590
5591	const opcode: Opcode = switch (info.class) {
5592	.bool => switch (reduce.operation) {
5593	.And => .OpLogicalAnd,
5594	.Or => .OpLogicalOr,
5595	.Xor => .OpLogicalNotEqual,
5596	else => unreachable,
5597	},
5598	.strange_integer, .integer => switch (reduce.operation) {
5599	.And => .OpBitwiseAnd,
5600	.Or => .OpBitwiseOr,
5601	.Xor => .OpBitwiseXor,
5602	.Add => .OpIAdd,
5603	.Mul => .OpIMul,
5604	else => unreachable,
5605	},
5606	.float => switch (reduce.operation) {
5607	.Add => .OpFAdd,
5608	.Mul => .OpFMul,
5609	else => unreachable,
5610	},
5611	.composite_integer => return cg.todo("@reduce for composite integers", .{}),
5612	};
5613
5614	const needs_normalize = info.class == .strange_integer and
5615	(reduce.operation == .Add or reduce.operation == .Mul);
5616
5617	var result: Temporary = .init(scalar_ty, first);
5618	for (1..len) |i| {
5619	const rhs_id = try cg.extractVectorComponent(scalar_ty, operand, @intCast(i));
5620	const rhs: Temporary = .init(scalar_ty, rhs_id);
5621	const stepped = try cg.buildBinary(opcode, result, rhs);
5622	result = if (needs_normalize) try cg.normalize(stepped, info) else stepped;
5623	}
5624
5625	return try result.materialize(cg);
5626	}
5627
5628	fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5629	const zcu = cg.zcu;
5630	const gpa = zcu.gpa;
5631
5632	const unwrapped = cg.air.unwrapShuffleOne(zcu, inst);
5633	const mask = unwrapped.mask;
5634	const result_ty = unwrapped.result_ty;
5635	const elem_ty = result_ty.childType(zcu);
5636	const operand = try cg.resolve(unwrapped.operand);
5637
5638	const scratch_top = cg.id_scratch.items.len;
5639	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5640	const constituents = try cg.id_scratch.addManyAsSlice(gpa, mask.len);
5641
5642	for (constituents, mask) |*id, mask_elem| {
5643	id.* = switch (mask_elem.unwrap()) {
5644	.elem => |idx| try cg.extractVectorComponent(elem_ty, operand, idx),
5645	.value => |val| try cg.constant(elem_ty, .fromInterned(val), .direct),
5646	};
5647	}
5648
5649	const result_ty_id = try cg.resolveType(result_ty, .direct);
5650	return try cg.constructComposite(result_ty_id, constituents);
5651	}
5652
5653	fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5654	const zcu = cg.zcu;
5655	const gpa = zcu.gpa;
5656
5657	const unwrapped = cg.air.unwrapShuffleTwo(zcu, inst);
5658	const mask = unwrapped.mask;
5659	const result_ty = unwrapped.result_ty;
5660	const elem_ty = result_ty.childType(zcu);
5661	const elem_ty_id = try cg.resolveType(elem_ty, .direct);
5662	const operand_a = try cg.resolve(unwrapped.operand_a);
5663	const operand_b = try cg.resolve(unwrapped.operand_b);
5664
5665	const scratch_top = cg.id_scratch.items.len;
5666	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5667	const constituents = try cg.id_scratch.addManyAsSlice(gpa, mask.len);
5668
5669	for (constituents, mask) |*id, mask_elem| {
5670	id.* = switch (mask_elem.unwrap()) {
5671	.a_elem => |idx| try cg.extractVectorComponent(elem_ty, operand_a, idx),
5672	.b_elem => |idx| try cg.extractVectorComponent(elem_ty, operand_b, idx),
5673	.undef => try cg.constUndef(elem_ty_id),
5674	};
5675	}
5676
5677	const result_ty_id = try cg.resolveType(result_ty, .direct);
5678	return try cg.constructComposite(result_ty_id, constituents);
5679	}
5680
5681	fn accessChainId(
5682	cg: *CodeGen,
5683	result_ty_id: Id,
5684	base: Id,
5685	indices: []const Id,
5686	) !Id {
5687	const result_id = cg.allocId();
5688	try cg.body.emit(cg.gpa, .OpInBoundsAccessChain, .{
5689	.id_result_type = result_ty_id,
5690	.id_result = result_id,
5691	.base = base,
5692	.indexes = indices,
5693	});
5694	return result_id;
5695	}
5696
5697	/// AccessChain is essentially PtrAccessChain with 0 as initial argument. The effective
5698	/// difference lies in whether the resulting type of the first dereference will be the
5699	/// same as that of the base pointer, or that of a dereferenced base pointer. AccessChain
5700	/// is the latter and PtrAccessChain is the former.
5701	fn accessChain(
5702	cg: *CodeGen,
5703	result_ty_id: Id,
5704	base: Id,
5705	indices: []const u32,
5706	) !Id {
5707	const gpa = cg.gpa;
5708	const scratch_top = cg.id_scratch.items.len;
5709	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5710	const ids = try cg.id_scratch.addManyAsSlice(gpa, indices.len);
5711	for (indices, ids) |index, *id| {
5712	id.* = try cg.constInt(.u32, index);
5713	}
5714	return try cg.accessChainId(result_ty_id, base, ids);
5715	}
5716
5717	fn ptrAccessChain(
5718	cg: *CodeGen,
5719	result_ty_id: Id,
5720	base: Id,
5721	element: Id,
5722	indices: []const u32,
5723	) !Id {
5724	const gpa = cg.gpa;
5725	const target = cg.zcu.getTarget();
5726
5727	const scratch_top = cg.id_scratch.items.len;
5728	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5729	const ids = try cg.id_scratch.addManyAsSlice(gpa, indices.len);
5730	for (indices, ids) |index, *id| {
5731	id.* = try cg.constInt(.u32, index);
5732	}
5733
5734	const result_id = cg.allocId();
5735	switch (target.os.tag) {
5736	.opencl, .amdhsa => {
5737	try cg.body.emit(gpa, .OpInBoundsPtrAccessChain, .{
5738	.id_result_type = result_ty_id,
5739	.id_result = result_id,
5740	.base = base,
5741	.element = element,
5742	.indexes = ids,
5743	});
5744	},
5745	.vulkan, .opengl => {
5746	assert(target.cpu.has(.spirv, .variable_pointers) or
5747	target.cpu.has(.spirv, .variable_pointers_storage_buffer));
5748	try cg.body.emit(gpa, .OpPtrAccessChain, .{
5749	.id_result_type = result_ty_id,
5750	.id_result = result_id,
5751	.base = base,
5752	.element = element,
5753	.indexes = ids,
5754	});
5755	},
5756	else => unreachable,
5757	}
5758	return result_id;
5759	}
5760
5761	fn ptrAdd(cg: *CodeGen, result_ty: Type, ptr_ty: Type, ptr_id: Id, offset_id: Id) !Id {
5762	const zcu = cg.zcu;
5763	const as = result_ty.ptrAddressSpace(zcu);
5764	const child_ty_id = try cg.pointeeType(as, result_ty.childType(zcu), false);
5765	const result_ty_id = try cg.ptrType(child_ty_id, cg.storageClass(as));
5766	return switch (ptr_ty.ptrSize(zcu)) {
5767	.one => cg.accessChainId(result_ty_id, ptr_id, &.{offset_id}),
5768	.c, .many => cg.ptrAccessChain(result_ty_id, ptr_id, offset_id, &.{}),
5769	.slice => blk: {
5770	// TODO: This is probably incorrect. A slice should be returned here, though this is what llvm does.
5771	const slice_ptr_id = try cg.extractField(result_ty, ptr_id, 0);
5772	break :blk cg.ptrAccessChain(result_ty_id, slice_ptr_id, offset_id, &.{});
5773	},
5774	};
5775	}
5776
5777	fn airPtrAdd(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5778	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5779	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5780	const ptr_id = try cg.resolve(bin_op.lhs);
5781	const offset_id = try cg.resolve(bin_op.rhs);
5782	const ptr_ty = cg.typeOf(bin_op.lhs);
5783	const result_ty = cg.typeOfIndex(inst);
5784
5785	return try cg.ptrAdd(result_ty, ptr_ty, ptr_id, offset_id);
5786	}
5787
5788	fn airPtrSub(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5789	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5790	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
5791	const ptr_id = try cg.resolve(bin_op.lhs);
5792	const ptr_ty = cg.typeOf(bin_op.lhs);
5793	const offset_id = try cg.resolve(bin_op.rhs);
5794	const offset_ty = cg.typeOf(bin_op.rhs);
5795	const offset_ty_id = try cg.resolveType(offset_ty, .direct);
5796	const result_ty = cg.typeOfIndex(inst);
5797
5798	const negative_offset_id = cg.allocId();
5799	try cg.body.emit(cg.gpa, .OpSNegate, .{
5800	.id_result_type = offset_ty_id,
5801	.id_result = negative_offset_id,
5802	.operand = offset_id,
5803	});
5804	return try cg.ptrAdd(result_ty, ptr_ty, ptr_id, negative_offset_id);
5805	}
5806
5807	fn cmp(
5808	cg: *CodeGen,
5809	op: std.math.CompareOperator,
5810	lhs: Temporary,
5811	rhs: Temporary,
5812	) !Temporary {
5813	const gpa = cg.gpa;
5814	const pt = cg.pt;
5815	const zcu = cg.zcu;
5816	const scalar_ty = lhs.ty.scalarType(zcu);
5817	const is_vector = lhs.ty.isVector(zcu);
5818
5819	switch (scalar_ty.zigTypeTag(zcu)) {
5820	.int, .bool, .float => {},
5821	.@"enum" => {
5822	assert(!is_vector);
5823	const ty = lhs.ty.backingIntType(zcu);
5824	return try cg.cmp(op, lhs.pun(ty), rhs.pun(ty));
5825	},
5826	.@"struct" => {
5827	const struct_ty = zcu.typeToPackedStruct(scalar_ty).?;
5828	const ty: Type = .fromInterned(struct_ty.packed_backing_int_type);
5829	return try cg.cmp(op, lhs.pun(ty), rhs.pun(ty));
5830	},
5831	.error_set => {
5832	assert(!is_vector);
5833	const err_int_ty = try pt.errorIntType();
5834	return try cg.cmp(op, lhs.pun(err_int_ty), rhs.pun(err_int_ty));
5835	},
5836	.pointer => {
5837	assert(!is_vector);
5838	// Note that while SPIR-V offers OpPtrEqual and OpPtrNotEqual, they are
5839	// currently not implemented in the SPIR-V LLVM translator. Thus, we emit these using
5840	// OpConvertPtrToU...
5841
5842	const usize_ty_id = try cg.resolveType(.usize, .direct);
5843
5844	const lhs_int_id = cg.allocId();
5845	try cg.body.emit(gpa, .OpConvertPtrToU, .{
5846	.id_result_type = usize_ty_id,
5847	.id_result = lhs_int_id,
5848	.pointer = try lhs.materialize(cg),
5849	});
5850
5851	const rhs_int_id = cg.allocId();
5852	try cg.body.emit(gpa, .OpConvertPtrToU, .{
5853	.id_result_type = usize_ty_id,
5854	.id_result = rhs_int_id,
5855	.pointer = try rhs.materialize(cg),
5856	});
5857
5858	const lhs_int: Temporary = .init(.usize, lhs_int_id);
5859	const rhs_int: Temporary = .init(.usize, rhs_int_id);
5860	return try cg.cmp(op, lhs_int, rhs_int);
5861	},
5862	.optional => {
5863	assert(!is_vector);
5864
5865	const ty = lhs.ty;
5866
5867	const payload_ty = ty.optionalChild(zcu);
5868	if (ty.optionalReprIsPayload(zcu)) {
5869	assert(payload_ty.hasRuntimeBits(zcu));
5870	assert(!payload_ty.isSlice(zcu));
5871
5872	return try cg.cmp(op, lhs.pun(payload_ty), rhs.pun(payload_ty));
5873	}
5874
5875	const lhs_id = try lhs.materialize(cg);
5876	const rhs_id = try rhs.materialize(cg);
5877
5878	const lhs_valid_id = if (payload_ty.hasRuntimeBits(zcu))
5879	try cg.extractField(.bool, lhs_id, 1)
5880	else
5881	try cg.convertToDirect(.bool, lhs_id);
5882
5883	const rhs_valid_id = if (payload_ty.hasRuntimeBits(zcu))
5884	try cg.extractField(.bool, rhs_id, 1)
5885	else
5886	try cg.convertToDirect(.bool, rhs_id);
5887
5888	const lhs_valid: Temporary = .init(.bool, lhs_valid_id);
5889	const rhs_valid: Temporary = .init(.bool, rhs_valid_id);
5890
5891	if (!payload_ty.hasRuntimeBits(zcu)) {
5892	return try cg.cmp(op, lhs_valid, rhs_valid);
5893	}
5894
5895	// a = lhs_valid
5896	// b = rhs_valid
5897	// c = lhs_pl == rhs_pl
5898	//
5899	// For op == .eq we have:
5900	// a == b && a -> c
5901	// = a == b && (!a || c)
5902	//
5903	// For op == .neq we have
5904	// a == b && a -> c
5905	// = !(a == b && a -> c)
5906	// = a != b || !(a -> c
5907	// = a != b || !(!a || c)
5908	// = a != b || a && !c
5909
5910	const lhs_pl_id = try cg.extractField(payload_ty, lhs_id, 0);
5911	const rhs_pl_id = try cg.extractField(payload_ty, rhs_id, 0);
5912
5913	const lhs_pl: Temporary = .init(payload_ty, lhs_pl_id);
5914	const rhs_pl: Temporary = .init(payload_ty, rhs_pl_id);
5915
5916	return switch (op) {
5917	.eq => try cg.buildBinary(
5918	.OpLogicalAnd,
5919	try cg.cmp(.eq, lhs_valid, rhs_valid),
5920	try cg.buildBinary(
5921	.OpLogicalOr,
5922	try cg.buildUnary(.l_not, lhs_valid),
5923	try cg.cmp(.eq, lhs_pl, rhs_pl),
5924	),
5925	),
5926	.neq => try cg.buildBinary(
5927	.OpLogicalOr,
5928	try cg.cmp(.neq, lhs_valid, rhs_valid),
5929	try cg.buildBinary(
5930	.OpLogicalAnd,
5931	lhs_valid,
5932	try cg.cmp(.neq, lhs_pl, rhs_pl),
5933	),
5934	),
5935	else => unreachable,
5936	};
5937	},
5938	else => |ty| return cg.todo("implement cmp operation for '{s}' type", .{@tagName(ty)}),
5939	}
5940
5941	const info = cg.arithmeticTypeInfo(scalar_ty);
5942	const pred: Opcode = switch (info.class) {
5943	.composite_integer => {
5944	const lhs_id = try lhs.materialize(cg);
5945	const rhs_id = try rhs.materialize(cg);
5946	const scratch_top = cg.id_scratch.items.len;
5947	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
5948	const ci_lhs = try CompositeInt.init(cg, lhs_id, info);
5949	const ci_rhs = try CompositeInt.init(cg, rhs_id, info);
5950	const result_id = try ci_lhs.cmp(ci_rhs, op);
5951	return .init(.bool, result_id);
5952	},
5953	.float => switch (op) {
5954	.eq => .OpFOrdEqual,
5955	.neq => .OpFUnordNotEqual,
5956	.lt => .OpFOrdLessThan,
5957	.lte => .OpFOrdLessThanEqual,
5958	.gt => .OpFOrdGreaterThan,
5959	.gte => .OpFOrdGreaterThanEqual,
5960	},
5961	.bool => switch (op) {
5962	.eq => .OpLogicalEqual,
5963	.neq => .OpLogicalNotEqual,
5964	else => unreachable,
5965	},
5966	.integer, .strange_integer => switch (info.signedness) {
5967	.signed => switch (op) {
5968	.eq => .OpIEqual,
5969	.neq => .OpINotEqual,
5970	.lt => .OpSLessThan,
5971	.lte => .OpSLessThanEqual,
5972	.gt => .OpSGreaterThan,
5973	.gte => .OpSGreaterThanEqual,
5974	},
5975	.unsigned => switch (op) {
5976	.eq => .OpIEqual,
5977	.neq => .OpINotEqual,
5978	.lt => .OpULessThan,
5979	.lte => .OpULessThanEqual,
5980	.gt => .OpUGreaterThan,
5981	.gte => .OpUGreaterThanEqual,
5982	},
5983	},
5984	};
5985
5986	return try cg.buildCmp(pred, lhs, rhs);
5987	}
5988
5989	fn airCmp(
5990	cg: *CodeGen,
5991	inst: Air.Inst.Index,
5992	comptime op: std.math.CompareOperator,
5993	) !?Id {
5994	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5995	const lhs = try cg.temporary(bin_op.lhs);
5996	const rhs = try cg.temporary(bin_op.rhs);
5997
5998	const result = try cg.cmp(op, lhs, rhs);
5999	return try result.materialize(cg);
6000	}
6001
6002	fn airVectorCmp(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6003	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6004	const vec_cmp = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data;
6005	const lhs = try cg.temporary(vec_cmp.lhs);
6006	const rhs = try cg.temporary(vec_cmp.rhs);
6007	const op = vec_cmp.compareOperator();
6008
6009	const result = try cg.cmp(op, lhs, rhs);
6010	return try result.materialize(cg);
6011	}
6012
6013	/// Bitcast one type to another. Note: both types, input, output are expected in **direct** representation.
6014	fn bitCast(
6015	cg: *CodeGen,
6016	dst_ty: Type,
6017	src_ty: Type,
6018	src_id: Id,
6019	) !Id {
6020	const gpa = cg.gpa;
6021	const zcu = cg.zcu;
6022	const target = zcu.getTarget();
6023
6024	if (src_ty.toIntern() == dst_ty.toIntern()) return src_id;
6025	if (src_ty.isPtrAtRuntime(zcu) and dst_ty.isPtrAtRuntime(zcu)) switch (target.os.tag) {
6026	.vulkan, .opengl => if (src_ty.ptrAddressSpace(zcu) != .physical_storage_buffer) {
6027	const src_child = src_ty.childType(zcu);
6028	const dst_child = dst_ty.childType(zcu);
6029	if (!dst_child.hasRuntimeBits(zcu)) return src_id;
6030	if (src_child.toIntern() == dst_child.toIntern()) return src_id;
6031	if (src_ty.ptrInfo(zcu).packed_offset.host_size != 0 or
6032	dst_ty.ptrInfo(zcu).packed_offset.host_size != 0) return src_id;
6033
6034	var indices: std.ArrayList(u32) = .empty;
6035	defer indices.deinit(gpa);
6036	var cur = src_child;
6037	while (cur.toIntern() != dst_child.toIntern()) : (try indices.append(gpa, 0)) {
6038	cur = switch (cur.zigTypeTag(zcu)) {
6039	.array, .vector => cur.childType(zcu),
6040	.@"struct" => field: {
6041	for (0..cur.structFieldCount(zcu)) |i| {
6042	const field_ty = cur.fieldType(i, zcu);
6043	if (field_ty.hasRuntimeBits(zcu) and cur.structFieldOffset(i, zcu) == 0) break :field field_ty;
6044	}
6045	unreachable;
6046	},
6047	else => unreachable,
6048	};
6049	}
6050	const dst_ty_id = try cg.resolveType(dst_ty, .direct);
6051	return try cg.accessChain(dst_ty_id, src_id, indices.items);
6052	},
6053	else => {},
6054	};
6055
6056	const dst_ty_id = try cg.resolveType(dst_ty, .direct);
6057	const result_id = blk: {
6058	// Big-int ↔ big-int bitcast: the indirect representation is an array,
6059	// which OpBitcast cannot operate on. The arrays are bitwise identical
6060	// apart from the top limb's padding; the normalize pass below fixes
6061	// the padding.
6062	if (src_ty.isInt(zcu) and dst_ty.isInt(zcu)) {
6063	const src_info = src_ty.intInfo(zcu);
6064	const dst_info = dst_ty.intInfo(zcu);
6065	const src_backing, const src_big = cg.backingIntBits(src_info.bits);
6066	const dst_backing, const dst_big = cg.backingIntBits(dst_info.bits);
6067	if (src_backing == dst_backing and src_big and dst_big) break :blk src_id;
6068	}
6069
6070	// TODO: Some more cases are missing here
6071	// See fn bitCast in llvm.zig
6072
6073	if (src_ty.zigTypeTag(zcu) == .int and dst_ty.isPtrAtRuntime(zcu)) {
6074	if (target.os.tag != .opencl) {
6075	if (dst_ty.ptrAddressSpace(zcu) != .physical_storage_buffer) {
6076	return cg.fail(
6077	"cannot cast integer to pointer with address space '{s}'",
6078	.{@tagName(dst_ty.ptrAddressSpace(zcu))},
6079	);
6080	}
6081	}
6082
6083	const result_id = cg.allocId();
6084	try cg.body.emit(gpa, .OpConvertUToPtr, .{
6085	.id_result_type = dst_ty_id,
6086	.id_result = result_id,
6087	.integer_value = src_id,
6088	});
6089	break :blk result_id;
6090	}
6091
6092	// We can only use OpBitcast for specific conversions: between numerical types, and
6093	// between pointers. If the resolved spir-v types fall into this category then emit OpBitcast,
6094	// otherwise use a temporary and perform a pointer cast.
6095	const can_bitcast = (src_ty.isNumeric(zcu) and dst_ty.isNumeric(zcu)) or (src_ty.isPtrAtRuntime(zcu) and dst_ty.isPtrAtRuntime(zcu));
6096	if (can_bitcast) {
6097	const result_id = cg.allocId();
6098	try cg.body.emit(gpa, .OpBitcast, .{
6099	.id_result_type = dst_ty_id,
6100	.id_result = result_id,
6101	.operand = src_id,
6102	});
6103
6104	break :blk result_id;
6105	}
6106
6107	switch (target.os.tag) {
6108	.vulkan, .opengl => {
6109	// Logical addressing forbids OpBitcast on pointers. Allocate
6110	// the temp with dst_ty so the load reads through a slot of the right type.
6111	const dst_ty_indirect_id = try cg.resolveType(dst_ty, .indirect);
6112	const tmp_id = try cg.alloc(dst_ty_indirect_id, null);
6113	try cg.store(dst_ty, tmp_id, src_id, .{});
6114	break :blk try cg.load(dst_ty, tmp_id, .{});
6115	},
6116	else => {},
6117	}
6118
6119	const dst_ptr_ty_id = try cg.ptrType(dst_ty_id, .function);
6120
6121	const src_ty_indirect_id = try cg.resolveType(src_ty, .indirect);
6122	const tmp_id = try cg.alloc(src_ty_indirect_id, null);
6123	try cg.store(src_ty, tmp_id, src_id, .{});
6124	const casted_ptr_id = cg.allocId();
6125	try cg.body.emit(gpa, .OpBitcast, .{
6126	.id_result_type = dst_ptr_ty_id,
6127	.id_result = casted_ptr_id,
6128	.operand = tmp_id,
6129	});
6130	break :blk try cg.load(dst_ty, casted_ptr_id, .{});
6131	};
6132
6133	// Because strange integers use sign-extended representation, we may need to normalize
6134	// the result here.
6135	// TODO: This detail could cause stuff like @as(*const i1, @ptrCast(&@as(u1, 1))) to break
6136	// should we change the representation of strange integers?
6137	if (dst_ty.zigTypeTag(zcu) == .int) {
6138	const info = cg.arithmeticTypeInfo(dst_ty);
6139	const result = try cg.normalize(Temporary.init(dst_ty, result_id), info);
6140	return try result.materialize(cg);
6141	}
6142
6143	return result_id;
6144	}
6145
6146	fn airBitCast(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6147	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6148	const operand_ty = cg.typeOf(ty_op.operand);
6149	const result_ty = cg.typeOfIndex(inst);
6150	if (operand_ty.toIntern() == .bool_type) {
6151	const operand = try cg.temporary(ty_op.operand);
6152	const result = try cg.intFromBool(operand, .u1);
6153	return try result.materialize(cg);
6154	}
6155	if (operand_ty.zigTypeTag(cg.zcu) == .pointer) {
6156	switch (try cg.resolvePtr(ty_op.operand)) {
6157	.tracked => |t| return t.id, // TODO
6158	.id => |operand_id| return try cg.bitCast(result_ty, operand_ty, operand_id),
6159	}
6160	}
6161	const operand_id = try cg.resolve(ty_op.operand);
6162	return try cg.bitCast(result_ty, operand_ty, operand_id);
6163	}
6164
6165	fn airIntCast(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6166	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6167	const src = try cg.temporary(ty_op.operand);
6168	const dst_ty = cg.typeOfIndex(inst);
6169
6170	const src_info = cg.arithmeticTypeInfo(src.ty);
6171	const dst_info = cg.arithmeticTypeInfo(dst_ty);
6172
6173	const src_composite = src_info.class == .composite_integer;
6174	const dst_composite = dst_info.class == .composite_integer;
6175
6176	if (src_composite or dst_composite) {
6177	const gpa = cg.gpa;
6178	const scratch_top = cg.id_scratch.items.len;
6179	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
6180
6181	if (src_composite and dst_composite) {
6182	const src_id = try src.materialize(cg);
6183	const limb_bits = cg.bigIntBits();
6184	const limb_ty = cg.limbType();
6185	const limb_ty_id = try cg.limbTypeId();
6186	const src_n: u16 = src_info.backing_bits / limb_bits;
6187	const dst_n: u16 = dst_info.backing_bits / limb_bits;
6188	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, dst_n);
6189	const min_n = @min(src_n, dst_n);
6190	for (0..min_n) |i| {
6191	result_limbs[i] = cg.allocId();
6192	try cg.body.emit(gpa, .OpCompositeExtract, .{
6193	.id_result_type = limb_ty_id,
6194	.id_result = result_limbs[i],
6195	.composite = src_id,
6196	.indexes = &.{@as(u32, @intCast(i))},
6197	});
6198	}
6199	if (dst_n > src_n) {
6200	const fill = if (src_info.signedness == .signed) blk: {
6201	const signed_limb_ty: Type = if (limb_bits == 64) .i64 else .i32;
6202	const signed_limb_ty_id = try cg.resolveType(signed_limb_ty, .direct);
6203	const msb = result_limbs[src_n - 1];
6204	const msb_signed = cg.allocId();
6205	try cg.body.emit(gpa, .OpBitcast, .{
6206	.id_result_type = signed_limb_ty_id,
6207	.id_result = msb_signed,
6208	.operand = msb,
6209	});
6210	const shift_amt = try cg.constInt(signed_limb_ty, @as(u64, limb_bits - 1));
6211	const sign_ext = cg.allocId();
6212	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
6213	.id_result_type = signed_limb_ty_id,
6214	.id_result = sign_ext,
6215	.base = msb_signed,
6216	.shift = shift_amt,
6217	});
6218	const back = cg.allocId();
6219	try cg.body.emit(gpa, .OpBitcast, .{
6220	.id_result_type = limb_ty_id,
6221	.id_result = back,
6222	.operand = sign_ext,
6223	});
6224	break :blk back;
6225	} else try cg.constInt(limb_ty, @as(u64, 0));
6226	for (min_n..dst_n) |i| {
6227	result_limbs[i] = fill;
6228	}
6229	}
6230	const ci = CompositeInt.fromLimbs(cg, result_limbs, dst_info);
6231	const normalized = try ci.normalize();
6232	return try normalized.materialize(dst_ty);
6233	} else if (src_composite and !dst_composite) {
6234	const src_id = try src.materialize(cg);
6235	const limb_bits = cg.bigIntBits();
6236	const limb_ty = cg.limbType();
6237	const limb_ty_id = try cg.limbTypeId();
6238	if (dst_info.backing_bits <= limb_bits) {
6239	const limb0 = cg.allocId();
6240	try cg.body.emit(gpa, .OpCompositeExtract, .{
6241	.id_result_type = limb_ty_id,
6242	.id_result = limb0,
6243	.composite = src_id,
6244	.indexes = &.{@as(u32, 0)},
6245	});
6246	const tmp: Temporary = .init(limb_ty, limb0);
6247	const converted = try cg.buildConvert(dst_ty, tmp);
6248	const result = if (dst_info.bits < src_info.bits)
6249	try cg.normalize(converted, dst_info)
6250	else
6251	converted;
6252	return try result.materialize(cg);
6253	} else {
6254	assert(limb_bits == 32); // dst > 64 while limbs are 64 shouldn't happen — dst fits in one 64-bit limb.
6255	const limb0 = cg.allocId();
6256	try cg.body.emit(gpa, .OpCompositeExtract, .{
6257	.id_result_type = limb_ty_id,
6258	.id_result = limb0,
6259	.composite = src_id,
6260	.indexes = &.{@as(u32, 0)},
6261	});
6262	const limb1 = cg.allocId();
6263	try cg.body.emit(gpa, .OpCompositeExtract, .{
6264	.id_result_type = limb_ty_id,
6265	.id_result = limb1,
6266	.composite = src_id,
6267	.indexes = &.{@as(u32, 1)},
6268	});
6269	const u64_ty_id = try cg.resolveType(.u64, .direct);
6270	const lo = cg.allocId();
6271	try cg.body.emit(gpa, .OpUConvert, .{
6272	.id_result_type = u64_ty_id,
6273	.id_result = lo,
6274	.unsigned_value = limb0,
6275	});
6276	const hi = cg.allocId();
6277	try cg.body.emit(gpa, .OpUConvert, .{
6278	.id_result_type = u64_ty_id,
6279	.id_result = hi,
6280	.unsigned_value = limb1,
6281	});
6282	const shift32 = try cg.constInt(.u64, @as(u64, 32));
6283	const hi_shifted = cg.allocId();
6284	try cg.body.emit(gpa, .OpShiftLeftLogical, .{
6285	.id_result_type = u64_ty_id,
6286	.id_result = hi_shifted,
6287	.base = hi,
6288	.shift = shift32,
6289	});
6290	const combined = cg.allocId();
6291	try cg.body.emit(gpa, .OpBitwiseOr, .{
6292	.id_result_type = u64_ty_id,
6293	.id_result = combined,
6294	.operand_1 = lo,
6295	.operand_2 = hi_shifted,
6296	});
6297	const tmp: Temporary = .init(.u64, combined);
6298	const converted = try cg.buildConvert(dst_ty, tmp);
6299	const result = if (dst_info.bits < src_info.bits)
6300	try cg.normalize(converted, dst_info)
6301	else
6302	converted;
6303	return try result.materialize(cg);
6304	}
6305	} else {
6306	const limb_bits = cg.bigIntBits();
6307	const limb_ty = cg.limbType();
6308	const limb_ty_id = try cg.limbTypeId();
6309	const dst_n: u16 = dst_info.backing_bits / limb_bits;
6310	const result_limbs = try cg.id_scratch.addManyAsSlice(gpa, dst_n);
6311
6312	if (src_info.backing_bits <= limb_bits) {
6313	const converted = try cg.buildConvert(limb_ty, src);
6314	result_limbs[0] = try converted.materialize(cg);
6315	} else {
6316	const src_as_u64 = try cg.buildConvert(.u64, src);
6317	const src_id = try src_as_u64.materialize(cg);
6318	result_limbs[0] = cg.allocId();
6319	try cg.body.emit(gpa, .OpUConvert, .{
6320	.id_result_type = limb_ty_id,
6321	.id_result = result_limbs[0],
6322	.unsigned_value = src_id,
6323	});
6324	const u64_ty_id = try cg.resolveType(.u64, .direct);
6325	const shift32 = try cg.constInt(.u64, @as(u64, 32));
6326	const hi = cg.allocId();
6327	try cg.body.emit(gpa, .OpShiftRightLogical, .{
6328	.id_result_type = u64_ty_id,
6329	.id_result = hi,
6330	.base = src_id,
6331	.shift = shift32,
6332	});
6333	result_limbs[1] = cg.allocId();
6334	try cg.body.emit(gpa, .OpUConvert, .{
6335	.id_result_type = limb_ty_id,
6336	.id_result = result_limbs[1],
6337	.unsigned_value = hi,
6338	});
6339	}
6340	// Sign/zero-extend remaining limbs.
6341	const fill_start: u16 = if (src_info.backing_bits <= limb_bits) 1 else 2;
6342	const fill = if (src_info.signedness == .signed) blk: {
6343	const signed_limb_ty: Type = if (limb_bits == 64) .i64 else .i32;
6344	const signed_limb_ty_id = try cg.resolveType(signed_limb_ty, .direct);
6345	const msb = result_limbs[fill_start - 1];
6346	const msb_signed = cg.allocId();
6347	try cg.body.emit(gpa, .OpBitcast, .{
6348	.id_result_type = signed_limb_ty_id,
6349	.id_result = msb_signed,
6350	.operand = msb,
6351	});
6352	const shift_amt = try cg.constInt(signed_limb_ty, @as(u64, limb_bits - 1));
6353	const sign_ext = cg.allocId();
6354	try cg.body.emit(gpa, .OpShiftRightArithmetic, .{
6355	.id_result_type = signed_limb_ty_id,
6356	.id_result = sign_ext,
6357	.base = msb_signed,
6358	.shift = shift_amt,
6359	});
6360	const back = cg.allocId();
6361	try cg.body.emit(gpa, .OpBitcast, .{
6362	.id_result_type = limb_ty_id,
6363	.id_result = back,
6364	.operand = sign_ext,
6365	});
6366	break :blk back;
6367	} else try cg.constInt(limb_ty, @as(u64, 0));
6368	for (fill_start..dst_n) |i| {
6369	result_limbs[i] = fill;
6370	}
6371	const ci = CompositeInt.fromLimbs(cg, result_limbs, dst_info);
6372	const normalized = try ci.normalize();
6373	return try normalized.materialize(dst_ty);
6374	}
6375	}
6376
6377	if (src_info.backing_bits == dst_info.backing_bits) {
6378	const result = if (dst_info.bits < src_info.bits)
6379	try cg.normalize(src.pun(dst_ty), dst_info)
6380	else
6381	src.pun(dst_ty);
6382	return try result.materialize(cg);
6383	}
6384
6385	const converted = try cg.buildConvert(dst_ty, src);
6386
6387	// Make sure to normalize the result if shrinking.
6388	// Because strange ints are sign extended in their backing
6389	// type, we don't need to normalize when growing the type. The
6390	// representation is already the same.
6391	const result = if (dst_info.bits < src_info.bits)
6392	try cg.normalize(converted, dst_info)
6393	else
6394	converted;
6395
6396	return try result.materialize(cg);
6397	}
6398
6399	fn intFromPtr(cg: *CodeGen, operand_id: Id) !Id {
6400	const result_type_id = try cg.resolveType(.usize, .direct);
6401	const result_id = cg.allocId();
6402	try cg.body.emit(cg.gpa, .OpConvertPtrToU, .{
6403	.id_result_type = result_type_id,
6404	.id_result = result_id,
6405	.pointer = operand_id,
6406	});
6407	return result_id;
6408	}
6409
6410	fn airFloatFromInt(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6411	const gpa = cg.gpa;
6412	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6413	const operand_ty = cg.typeOf(ty_op.operand);
6414	const operand_id = try cg.resolve(ty_op.operand);
6415	const result_ty = cg.typeOfIndex(inst);
6416	const operand_info = cg.arithmeticTypeInfo(operand_ty);
6417	const result_id = cg.allocId();
6418	const result_ty_id = try cg.resolveType(result_ty, .direct);
6419	switch (operand_info.signedness) {
6420	.signed => try cg.body.emit(gpa, .OpConvertSToF, .{
6421	.id_result_type = result_ty_id,
6422	.id_result = result_id,
6423	.signed_value = operand_id,
6424	}),
6425	.unsigned => try cg.body.emit(gpa, .OpConvertUToF, .{
6426	.id_result_type = result_ty_id,
6427	.id_result = result_id,
6428	.unsigned_value = operand_id,
6429	}),
6430	}
6431	return result_id;
6432	}
6433
6434	fn airIntFromFloat(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6435	const gpa = cg.gpa;
6436	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6437	const operand_id = try cg.resolve(ty_op.operand);
6438	const result_ty = cg.typeOfIndex(inst);
6439	const result_info = cg.arithmeticTypeInfo(result_ty);
6440	const result_ty_id = try cg.resolveType(result_ty, .direct);
6441	const result_id = cg.allocId();
6442	switch (result_info.signedness) {
6443	.signed => try cg.body.emit(gpa, .OpConvertFToS, .{
6444	.id_result_type = result_ty_id,
6445	.id_result = result_id,
6446	.float_value = operand_id,
6447	}),
6448	.unsigned => try cg.body.emit(gpa, .OpConvertFToU, .{
6449	.id_result_type = result_ty_id,
6450	.id_result = result_id,
6451	.float_value = operand_id,
6452	}),
6453	}
6454	return result_id;
6455	}
6456
6457	fn airFloatCast(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6458	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6459	const operand = try cg.temporary(ty_op.operand);
6460	const dest_ty = cg.typeOfIndex(inst);
6461	const result = try cg.buildConvert(dest_ty, operand);
6462	return try result.materialize(cg);
6463	}
6464
6465	fn airNot(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6466	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6467	const operand = try cg.temporary(ty_op.operand);
6468	const result_ty = cg.typeOfIndex(inst);
6469	const info = cg.arithmeticTypeInfo(result_ty);
6470
6471	const result = switch (info.class) {
6472	.bool => try cg.buildUnary(.l_not, operand),
6473	.float => unreachable,
6474	.composite_integer => blk: {
6475	const op_id = try operand.materialize(cg);
6476	const scratch_top = cg.id_scratch.items.len;
6477	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
6478	const ci = try CompositeInt.init(cg, op_id, info);
6479	const notted = try ci.bitwiseNot();
6480	const normalized = try notted.normalize();
6481	break :blk Temporary.init(result_ty, try normalized.materialize(result_ty));
6482	},
6483	.strange_integer, .integer => blk: {
6484	const complement = try cg.buildUnary(.bit_not, operand);
6485	break :blk try cg.normalize(complement, info);
6486	},
6487	};
6488
6489	return try result.materialize(cg);
6490	}
6491
6492	fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6493	const zcu = cg.zcu;
6494	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6495	const array_ptr_ty = cg.typeOf(ty_op.operand);
6496	const array_ty = array_ptr_ty.childType(zcu);
6497	const slice_ty = cg.typeOfIndex(inst);
6498	const elem_ptr_ty = slice_ty.slicePtrFieldType(zcu);
6499
6500	const elem_ptr_ty_id = try cg.resolveType(elem_ptr_ty, .direct);
6501
6502	const array_ptr_id = try cg.resolve(ty_op.operand);
6503	const len_id = try cg.constInt(.usize, array_ty.arrayLen(zcu));
6504
6505	const elem_ptr_id = if (!array_ty.hasRuntimeBits(zcu))
6506	// Note: The pointer is something like *opaque{}, so we need to bitcast it to the element type.
6507	try cg.bitCast(elem_ptr_ty, array_ptr_ty, array_ptr_id)
6508	else
6509	// Convert the pointer-to-array to a pointer to the first element.
6510	try cg.accessChain(elem_ptr_ty_id, array_ptr_id, &.{0});
6511
6512	const slice_ty_id = try cg.resolveType(slice_ty, .direct);
6513	return try cg.constructComposite(slice_ty_id, &.{ elem_ptr_id, len_id });
6514	}
6515
6516	fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6517	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6518	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6519	const ptr_id = try cg.resolve(bin_op.lhs);
6520	const len_id = try cg.resolve(bin_op.rhs);
6521	const slice_ty = cg.typeOfIndex(inst);
6522	const slice_ty_id = try cg.resolveType(slice_ty, .direct);
6523	return try cg.constructComposite(slice_ty_id, &.{ ptr_id, len_id });
6524	}
6525
6526	fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6527	const gpa = cg.gpa;
6528	const pt = cg.pt;
6529	const zcu = cg.zcu;
6530	const ip = &zcu.intern_pool;
6531	const target = cg.zcu.getTarget();
6532	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6533	const result_ty = cg.typeOfIndex(inst);
6534	const len: usize = @intCast(result_ty.arrayLen(zcu));
6535	const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]);
6536
6537	switch (result_ty.zigTypeTag(zcu)) {
6538	.@"struct" => {
6539	if (zcu.typeToPackedStruct(result_ty)) |struct_type| {
6540	comptime assert(Type.packed_struct_layout_version == 2);
6541	const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type);
6542	var running_int_id = try cg.constInt(backing_int_ty, 0);
6543	var running_bits: u16 = 0;
6544	for (struct_type.field_types.get(ip), elements) |field_ty_ip, element| {
6545	const field_ty: Type = .fromInterned(field_ty_ip);
6546	if (!field_ty.hasRuntimeBits(zcu)) continue;
6547	const field_id = try cg.resolve(element);
6548	const ty_bit_size: u16 = @intCast(field_ty.bitSize(zcu));
6549	const field_int_ty = try cg.pt.intType(.unsigned, ty_bit_size);
6550	const field_int_id = blk: {
6551	if (field_ty.isPtrAtRuntime(zcu)) {
6552	assert(target.cpu.arch == .spirv64 and
6553	field_ty.ptrAddressSpace(zcu) == .storage_buffer);
6554	break :blk try cg.intFromPtr(field_id);
6555	}
6556	break :blk try cg.bitCast(field_int_ty, field_ty, field_id);
6557	};
6558	const shift_rhs = try cg.constInt(backing_int_ty, running_bits);
6559	const extended_int_conv = try cg.buildConvert(backing_int_ty, .{
6560	.ty = field_int_ty,
6561	.value = .{ .singleton = field_int_id },
6562	});
6563	const shifted = try cg.buildBinary(.OpShiftLeftLogical, extended_int_conv, .{
6564	.ty = backing_int_ty,
6565	.value = .{ .singleton = shift_rhs },
6566	});
6567	const running_int_tmp = try cg.buildBinary(
6568	.OpBitwiseOr,
6569	.{ .ty = backing_int_ty, .value = .{ .singleton = running_int_id } },
6570	shifted,
6571	);
6572	running_int_id = try running_int_tmp.materialize(cg);
6573	running_bits += ty_bit_size;
6574	}
6575	return running_int_id;
6576	}
6577
6578	const scratch_top = cg.id_scratch.items.len;
6579	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
6580	const constituents = try cg.id_scratch.addManyAsSlice(gpa, elements.len);
6581
6582	const types = try gpa.alloc(Type, elements.len);
6583	defer gpa.free(types);
6584
6585	var index: usize = 0;
6586
6587	switch (ip.indexToKey(result_ty.toIntern())) {
6588	.tuple_type => |tuple| {
6589	for (tuple.types.get(ip), elements, 0..) |field_ty, element, i| {
6590	if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
6591	assert(Type.fromInterned(field_ty).hasRuntimeBits(zcu));
6592
6593	const id = try cg.resolve(element);
6594	types[index] = .fromInterned(field_ty);
6595	constituents[index] = try cg.convertToIndirect(.fromInterned(field_ty), id);
6596	index += 1;
6597	}
6598	},
6599	.struct_type => {
6600	const struct_type = ip.loadStructType(result_ty.toIntern());
6601	var it = struct_type.iterateRuntimeOrder(ip);
6602	for (elements, 0..) |element, i| {
6603	const field_index = it.next().?;
6604	if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
6605	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
6606	assert(field_ty.hasRuntimeBits(zcu));
6607
6608	const id = try cg.resolve(element);
6609	types[index] = field_ty;
6610	constituents[index] = try cg.convertToIndirect(field_ty, id);
6611	index += 1;
6612	}
6613	},
6614	else => unreachable,
6615	}
6616
6617	const result_ty_id = try cg.resolveType(result_ty, .direct);
6618	return try cg.constructComposite(result_ty_id, constituents[0..index]);
6619	},
6620	.vector => {
6621	const n_elems = result_ty.vectorLen(zcu);
6622	const scratch_top = cg.id_scratch.items.len;
6623	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
6624	const elem_ids = try cg.id_scratch.addManyAsSlice(gpa, n_elems);
6625
6626	for (elements, 0..) |element, i| {
6627	elem_ids[i] = try cg.resolve(element);
6628	}
6629
6630	const result_ty_id = try cg.resolveType(result_ty, .direct);
6631	return try cg.constructComposite(result_ty_id, elem_ids);
6632	},
6633	.array => {
6634	const array_info = result_ty.arrayInfo(zcu);
6635	const n_elems: usize = @intCast(result_ty.arrayLenIncludingSentinel(zcu));
6636	const scratch_top = cg.id_scratch.items.len;
6637	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
6638	const elem_ids = try cg.id_scratch.addManyAsSlice(gpa, n_elems);
6639
6640	for (elements, 0..) |element, i| {
6641	const id = try cg.resolve(element);
6642	elem_ids[i] = try cg.convertToIndirect(array_info.elem_type, id);
6643	}
6644
6645	if (array_info.sentinel) |sentinel_val| {
6646	elem_ids[n_elems - 1] = try cg.constant(array_info.elem_type, sentinel_val, .indirect);
6647	}
6648
6649	const result_ty_id = try cg.resolveType(result_ty, .direct);
6650	return try cg.constructComposite(result_ty_id, elem_ids);
6651	},
6652	else => unreachable,
6653	}
6654	}
6655
6656	fn sliceOrArrayPtr(cg: *CodeGen, operand_id: Id, ty: Type) !Id {
6657	const zcu = cg.zcu;
6658	if (ty.isSlice(zcu)) {
6659	const ptr_ty = ty.slicePtrFieldType(zcu);
6660	return cg.extractField(ptr_ty, operand_id, 0);
6661	}
6662	return operand_id;
6663	}
6664
6665	fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) !void {
6666	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6667	const dest_slice = try cg.resolve(bin_op.lhs);
6668	const src_slice = try cg.resolve(bin_op.rhs);
6669	const dest_ty = cg.typeOf(bin_op.lhs);
6670	const src_ty = cg.typeOf(bin_op.rhs);
6671	const dest_ptr = try cg.sliceOrArrayPtr(dest_slice, dest_ty);
6672	const src_ptr = try cg.sliceOrArrayPtr(src_slice, src_ty);
6673	const len = switch (dest_ty.ptrSize(cg.zcu)) {
6674	.slice => try cg.extractField(.usize, dest_slice, 1),
6675	.one => len: {
6676	const array_ty = dest_ty.childType(cg.zcu);
6677	const elem_ty = array_ty.childType(cg.zcu);
6678	const size = array_ty.arrayLenIncludingSentinel(cg.zcu) * elem_ty.abiSize(cg.zcu);
6679	break :len try cg.constInt(.usize, size);
6680	},
6681	.many, .c => unreachable,
6682	};
6683	try cg.body.emit(cg.gpa, .OpCopyMemorySized, .{
6684	.target = dest_ptr,
6685	.source = src_ptr,
6686	.size = len,
6687	});
6688	}
6689
6690	fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) !void {
6691	_ = inst;
6692	return cg.fail("TODO implement airMemcpy for spirv", .{});
6693	}
6694
6695	fn airSliceField(cg: *CodeGen, inst: Air.Inst.Index, field: u32) !?Id {
6696	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6697	const field_ty = cg.typeOfIndex(inst);
6698	const operand_id = try cg.resolve(ty_op.operand);
6699	return try cg.extractField(field_ty, operand_id, field);
6700	}
6701
6702	fn airSpirvRuntimeArrayLen(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6703	const gpa = cg.gpa;
6704	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6705	const extra = cg.air.extraData(Air.StructField, ty_pl.payload).data;
6706	const struct_ptr_id = try cg.resolve(extra.struct_operand);
6707	const u32_ty_id = try cg.intType(.unsigned, 32);
6708	const result_id = cg.allocId();
6709	try cg.body.emit(gpa, .OpArrayLength, .{
6710	.id_result_type = u32_ty_id,
6711	.id_result = result_id,
6712	.structure = struct_ptr_id,
6713	.array_member = extra.field_index,
6714	});
6715	return result_id;
6716	}
6717
6718	fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6719	const zcu = cg.zcu;
6720	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6721	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6722	const slice_ty = cg.typeOf(bin_op.lhs);
6723	if (!slice_ty.isVolatilePtr(zcu) and cg.liveness.isUnused(inst)) return null;
6724
6725	const slice_id = try cg.resolve(bin_op.lhs);
6726	const index_id = try cg.resolve(bin_op.rhs);
6727
6728	const ptr_ty = cg.typeOfIndex(inst);
6729	const ptr_ty_id = try cg.resolveType(ptr_ty, .direct);
6730
6731	const slice_ptr = try cg.extractField(ptr_ty, slice_id, 0);
6732	return try cg.ptrAccessChain(ptr_ty_id, slice_ptr, index_id, &.{});
6733	}
6734
6735	fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6736	const zcu = cg.zcu;
6737	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6738	const slice_ty = cg.typeOf(bin_op.lhs);
6739	if (!slice_ty.isVolatilePtr(zcu) and cg.liveness.isUnused(inst)) return null;
6740
6741	const slice_id = try cg.resolve(bin_op.lhs);
6742	const index_id = try cg.resolve(bin_op.rhs);
6743
6744	const ptr_ty = slice_ty.slicePtrFieldType(zcu);
6745	const ptr_ty_id = try cg.resolveType(ptr_ty, .direct);
6746
6747	const slice_ptr = try cg.extractField(ptr_ty, slice_id, 0);
6748	const elem_ptr = try cg.ptrAccessChain(ptr_ty_id, slice_ptr, index_id, &.{});
6749	return try cg.load(slice_ty.childType(zcu), elem_ptr, .{ .is_volatile = slice_ty.isVolatilePtr(zcu) });
6750	}
6751
6752	fn ptrElemPtr(cg: *CodeGen, ptr_ty: Type, ptr_id: Id, index_id: Id) !Id {
6753	const zcu = cg.zcu;
6754	// Construct new pointer type for the resulting pointer
6755	const as = ptr_ty.ptrAddressSpace(zcu);
6756	const is_single_ptr = ptr_ty.isSinglePointer(zcu);
6757	const elem_is_block = cg.block_var_ids.contains(ptr_id);
6758	const elem_ty_id = try cg.pointeeType(as, ptr_ty.indexableElem(zcu), elem_is_block);
6759	const elem_ptr_ty_id = try cg.ptrType(elem_ty_id, cg.storageClass(as));
6760	if (is_single_ptr) {
6761	// Pointer-to-array. In this case, the resulting pointer is not of the same type
6762	// as the ptr_ty (we want a *T, not a *[N]T), and hence we need to use accessChain.
6763	return cg.accessChainId(elem_ptr_ty_id, ptr_id, &.{index_id});
6764	} else {
6765	// Resulting pointer type is the same as the ptr_ty, so use ptrAccessChain
6766	return cg.ptrAccessChain(elem_ptr_ty_id, ptr_id, index_id, &.{});
6767	}
6768	}
6769
6770	fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6771	const zcu = cg.zcu;
6772	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6773	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6774	const src_ptr_ty = cg.typeOf(bin_op.lhs);
6775	const elem_ty = src_ptr_ty.childType(zcu);
6776	const ptr_id = try cg.resolve(bin_op.lhs);
6777
6778	assert(elem_ty.hasRuntimeBits(zcu));
6779
6780	const index_id = try cg.resolve(bin_op.rhs);
6781	return try cg.ptrElemPtr(src_ptr_ty, ptr_id, index_id);
6782	}
6783
6784	fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6785	const gpa = cg.gpa;
6786	const zcu = cg.zcu;
6787	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6788	const array_ty = cg.typeOf(bin_op.lhs);
6789	const elem_ty = array_ty.childType(zcu);
6790	const array_id = try cg.resolve(bin_op.lhs);
6791	const index_id = try cg.resolve(bin_op.rhs);
6792
6793	// SPIR-V doesn't have an array indexing function for some damn reason.
6794	// For now, just generate a temporary and use that.
6795	// TODO: This backend probably also should use isByRef from llvm...
6796
6797	const is_vector = array_ty.isVector(zcu);
6798	const elem_repr: Repr = if (is_vector) .direct else .indirect;
6799	const array_ty_id = try cg.resolveType(array_ty, .direct);
6800	const elem_ty_id = try cg.resolveType(elem_ty, elem_repr);
6801	const ptr_array_ty_id = try cg.ptrType(array_ty_id, .function);
6802	const ptr_elem_ty_id = try cg.ptrType(elem_ty_id, .function);
6803
6804	const tmp_id = cg.allocId();
6805	try cg.prologue.emit(gpa, .OpVariable, .{
6806	.id_result_type = ptr_array_ty_id,
6807	.id_result = tmp_id,
6808	.storage_class = .function,
6809	});
6810
6811	try cg.body.emit(gpa, .OpStore, .{
6812	.pointer = tmp_id,
6813	.object = array_id,
6814	});
6815
6816	const elem_ptr_id = try cg.accessChainId(ptr_elem_ty_id, tmp_id, &.{index_id});
6817
6818	const result_id = cg.allocId();
6819	try cg.body.emit(gpa, .OpLoad, .{
6820	.id_result_type = try cg.resolveType(elem_ty, elem_repr),
6821	.id_result = result_id,
6822	.pointer = elem_ptr_id,
6823	});
6824
6825	if (is_vector) {
6826	// Result is already in direct representation
6827	return result_id;
6828	}
6829
6830	// This is an array type; the elements are stored in indirect representation.
6831	// We have to convert the type to direct.
6832
6833	return try cg.convertToDirect(elem_ty, result_id);
6834	}
6835
6836	fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6837	const zcu = cg.zcu;
6838	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6839	const ptr_ty = cg.typeOf(bin_op.lhs);
6840	const elem_ty = cg.typeOfIndex(inst);
6841	const ptr_id = try cg.resolve(bin_op.lhs);
6842	const index_id = try cg.resolve(bin_op.rhs);
6843	const elem_ptr_id = try cg.ptrElemPtr(ptr_ty, ptr_id, index_id);
6844	return try cg.load(elem_ty, elem_ptr_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
6845	}
6846
6847	fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !void {
6848	const zcu = cg.zcu;
6849	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6850	const un_ptr_ty = cg.typeOf(bin_op.lhs);
6851	const un_ty = un_ptr_ty.childType(zcu);
6852	const layout = cg.unionLayout(un_ty);
6853
6854	if (layout.tag_size == 0) return;
6855
6856	const tag_ty = un_ty.unionTagTypeRuntime(zcu).?;
6857	const tag_ty_id = try cg.resolveType(tag_ty, .indirect);
6858	const tag_ptr_ty_id = try cg.ptrType(tag_ty_id, cg.storageClass(un_ptr_ty.ptrAddressSpace(zcu)));
6859
6860	const union_ptr_id = try cg.resolve(bin_op.lhs);
6861	const new_tag_id = try cg.resolve(bin_op.rhs);
6862
6863	if (!layout.has_payload) {
6864	try cg.store(tag_ty, union_ptr_id, new_tag_id, .{ .is_volatile = un_ptr_ty.isVolatilePtr(zcu) });
6865	} else {
6866	const ptr_id = try cg.accessChain(tag_ptr_ty_id, union_ptr_id, &.{layout.tag_index});
6867	try cg.store(tag_ty, ptr_id, new_tag_id, .{ .is_volatile = un_ptr_ty.isVolatilePtr(zcu) });
6868	}
6869	}
6870
6871	fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6872	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6873	const un_ty = cg.typeOf(ty_op.operand);
6874
6875	const zcu = cg.zcu;
6876	const layout = cg.unionLayout(un_ty);
6877	if (layout.tag_size == 0) return null;
6878
6879	const union_handle = try cg.resolve(ty_op.operand);
6880	if (!layout.has_payload) return union_handle;
6881
6882	const tag_ty = un_ty.unionTagTypeRuntime(zcu).?;
6883	return try cg.extractField(tag_ty, union_handle, layout.tag_index);
6884	}
6885
6886	fn unionInit(
6887	cg: *CodeGen,
6888	ty: Type,
6889	active_field: u32,
6890	payload: ?Id,
6891	) !Id {
6892	// To initialize a union, generate a temporary variable with the
6893	// union type, then get the field pointer and pointer-cast it to the
6894	// right type to store it. Finally load the entire union.
6895
6896	// Note: The result here is not cached, because it generates runtime code.
6897
6898	const pt = cg.pt;
6899	const zcu = cg.zcu;
6900	const ip = &zcu.intern_pool;
6901	const union_ty = zcu.typeToUnion(ty).?;
6902	const tag_ty: Type = .fromInterned(union_ty.enum_tag_type);
6903
6904	const layout = cg.unionLayout(ty);
6905	const payload_ty: Type = .fromInterned(union_ty.field_types.get(ip)[active_field]);
6906
6907	assert(union_ty.layout != .@"packed");
6908
6909	const tag_int = if (layout.tag_size != 0) blk: {
6910	const tag_val = try pt.enumValueFieldIndex(tag_ty, active_field);
6911	const tag_int_val = tag_val.backingInt(zcu);
6912	break :blk tag_int_val.toUnsignedInt(zcu);
6913	} else 0;
6914
6915	if (!layout.has_payload) {
6916	return try cg.constInt(tag_ty, tag_int);
6917	}
6918
6919	const ty_id = try cg.resolveType(ty, .indirect);
6920	const tmp_id = try cg.alloc(ty_id, null);
6921
6922	if (layout.tag_size != 0) {
6923	const tag_ty_id = try cg.resolveType(tag_ty, .indirect);
6924	const tag_ptr_ty_id = try cg.ptrType(tag_ty_id, .function);
6925	const ptr_id = try cg.accessChain(tag_ptr_ty_id, tmp_id, &.{@as(u32, @intCast(layout.tag_index))});
6926	const tag_id = try cg.constInt(tag_ty, tag_int);
6927	try cg.store(tag_ty, ptr_id, tag_id, .{});
6928	}
6929
6930	if (payload_ty.hasRuntimeBits(zcu)) {
6931	const layout_payload_ty_id = try cg.resolveType(layout.payload_ty, .indirect);
6932	const pl_ptr_ty_id = try cg.ptrType(layout_payload_ty_id, .function);
6933	const pl_ptr_id = try cg.accessChain(pl_ptr_ty_id, tmp_id, &.{layout.payload_index});
6934	const active_pl_ptr_id = if (!layout.payload_ty.eql(payload_ty)) blk: {
6935	const payload_ty_id = try cg.resolveType(payload_ty, .indirect);
6936	const active_pl_ptr_ty_id = try cg.ptrType(payload_ty_id, .function);
6937	const active_pl_ptr_id = cg.allocId();
6938	try cg.body.emit(cg.gpa, .OpBitcast, .{
6939	.id_result_type = active_pl_ptr_ty_id,
6940	.id_result = active_pl_ptr_id,
6941	.operand = pl_ptr_id,
6942	});
6943	break :blk active_pl_ptr_id;
6944	} else pl_ptr_id;
6945
6946	try cg.store(payload_ty, active_pl_ptr_id, payload.?, .{});
6947	} else {
6948	assert(payload == null);
6949	}
6950
6951	// Just leave the padding fields uninitialized...
6952	// TODO: Or should we initialize them with undef explicitly?
6953
6954	return try cg.load(ty, tmp_id, .{});
6955	}
6956
6957	fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6958	const zcu = cg.zcu;
6959	const ip = &zcu.intern_pool;
6960	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6961	const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;
6962	const ty = cg.typeOfIndex(inst);
6963
6964	const union_obj = zcu.typeToUnion(ty).?;
6965	const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6966	const payload = if (field_ty.hasRuntimeBits(zcu))
6967	try cg.resolve(extra.init)
6968	else
6969	null;
6970	return try cg.unionInit(ty, extra.field_index, payload);
6971	}
6972
6973	fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6974	const pt = cg.pt;
6975	const zcu = cg.zcu;
6976	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6977	const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data;
6978
6979	const object_ty = cg.typeOf(struct_field.struct_operand);
6980	const object_id = try cg.resolve(struct_field.struct_operand);
6981	const field_index = struct_field.field_index;
6982	const field_ty = object_ty.fieldType(field_index, zcu);
6983
6984	assert(field_ty.hasRuntimeBits(zcu));
6985
6986	switch (object_ty.zigTypeTag(zcu)) {
6987	.@"struct" => switch (object_ty.containerLayout(zcu)) {
6988	.@"packed" => {
6989	const struct_ty = zcu.typeToPackedStruct(object_ty).?;
6990	const struct_backing_int_bits = cg.backingIntBits(@intCast(object_ty.bitSize(zcu))).@"0";
6991	const bit_offset = zcu.structPackedFieldBitOffset(struct_ty, field_index);
6992	// We use the same int type the packed struct is backed by, because even though it would
6993	// be valid SPIR-V to use an smaller type like u16, some implementations like PoCL will complain.
6994	const bit_offset_id = try cg.constInt(object_ty, bit_offset);
6995	const signedness = if (field_ty.isInt(zcu)) field_ty.intInfo(zcu).signedness else .unsigned;
6996	const field_bit_size: u16 = @intCast(field_ty.bitSize(zcu));
6997	const field_int_ty = try pt.intType(signedness, field_bit_size);
6998	const shift_lhs: Temporary = .{ .ty = object_ty, .value = .{ .singleton = object_id } };
6999	const shift = try cg.buildBinary(.OpShiftRightLogical, shift_lhs, .{ .ty = object_ty, .value = .{ .singleton = bit_offset_id } });
7000	const mask_id = try cg.constInt(object_ty, (@as(u64, 1) << @as(u6, @intCast(field_bit_size))) - 1);
7001	const masked = try cg.buildBinary(.OpBitwiseAnd, shift, .{ .ty = object_ty, .value = .{ .singleton = mask_id } });
7002	const result_id = blk: {
7003	if (cg.backingIntBits(field_bit_size).@"0" == struct_backing_int_bits)
7004	break :blk try cg.bitCast(field_int_ty, object_ty, try masked.materialize(cg));
7005	const trunc = try cg.buildConvert(field_int_ty, masked);
7006	break :blk try trunc.materialize(cg);
7007	};
7008	if (field_ty.ip_index == .bool_type) return try cg.convertToDirect(.bool, result_id);
7009	if (field_ty.isInt(zcu)) return result_id;
7010	return try cg.bitCast(field_ty, field_int_ty, result_id);
7011	},
7012	else => return try cg.extractField(field_ty, object_id, field_index),
7013	},
7014	.@"union" => switch (object_ty.containerLayout(zcu)) {
7015	.@"packed" => {
7016	const backing_int_ty = try pt.intType(.unsigned, @intCast(object_ty.bitSize(zcu)));
7017	const signedness = if (field_ty.isInt(zcu)) field_ty.intInfo(zcu).signedness else .unsigned;
7018	const field_bit_size: u16 = @intCast(field_ty.bitSize(zcu));
7019	const int_ty = try pt.intType(signedness, field_bit_size);
7020	const mask_id = try cg.constInt(backing_int_ty, (@as(u64, 1) << @as(u6, @intCast(field_bit_size))) - 1);
7021	const masked = try cg.buildBinary(
7022	.OpBitwiseAnd,
7023	.{ .ty = backing_int_ty, .value = .{ .singleton = object_id } },
7024	.{ .ty = backing_int_ty, .value = .{ .singleton = mask_id } },
7025	);
7026	const result_id = blk: {
7027	if (cg.backingIntBits(field_bit_size).@"0" == cg.backingIntBits(@intCast(backing_int_ty.bitSize(zcu))).@"0")
7028	break :blk try cg.bitCast(int_ty, backing_int_ty, try masked.materialize(cg));
7029	const trunc = try cg.buildConvert(int_ty, masked);
7030	break :blk try trunc.materialize(cg);
7031	};
7032	if (field_ty.ip_index == .bool_type) return try cg.convertToDirect(.bool, result_id);
7033	if (field_ty.isInt(zcu)) return result_id;
7034	return try cg.bitCast(field_ty, int_ty, result_id);
7035	},
7036	else => {
7037	// Store, ptr-elem-ptr, pointer-cast, load
7038	const layout = cg.unionLayout(object_ty);
7039	assert(layout.has_payload);
7040
7041	const object_ty_id = try cg.resolveType(object_ty, .indirect);
7042	const tmp_id = try cg.alloc(object_ty_id, null);
7043	try cg.store(object_ty, tmp_id, object_id, .{});
7044
7045	const layout_payload_ty_id = try cg.resolveType(layout.payload_ty, .indirect);
7046	const pl_ptr_ty_id = try cg.ptrType(layout_payload_ty_id, .function);
7047	const pl_ptr_id = try cg.accessChain(pl_ptr_ty_id, tmp_id, &.{layout.payload_index});
7048
7049	if (field_ty.toIntern() == layout.payload_ty.toIntern()) {
7050	return try cg.load(field_ty, pl_ptr_id, .{});
7051	}
7052
7053	switch (zcu.getTarget().os.tag) {
7054	.vulkan, .opengl => {
7055	// Logical addressing forbids OpBitcast on pointers. Load the
7056	// payload as its type and bitcast the value instead.
7057	const payload_id = try cg.load(layout.payload_ty, pl_ptr_id, .{});
7058	return try cg.bitCast(field_ty, layout.payload_ty, payload_id);
7059	},
7060	else => {},
7061	}
7062
7063	const field_ty_id = try cg.resolveType(field_ty, .indirect);
7064	const active_pl_ptr_ty_id = try cg.ptrType(field_ty_id, .function);
7065	const active_pl_ptr_id = cg.allocId();
7066	try cg.body.emit(cg.gpa, .OpBitcast, .{
7067	.id_result_type = active_pl_ptr_ty_id,
7068	.id_result = active_pl_ptr_id,
7069	.operand = pl_ptr_id,
7070	});
7071	return try cg.load(field_ty, active_pl_ptr_id, .{});
7072	},
7073	},
7074	else => unreachable,
7075	}
7076	}
7077
7078	fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7079	const zcu = cg.zcu;
7080	const target = zcu.getTarget();
7081	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7082	const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
7083
7084	const parent_ptr_ty = ty_pl.ty;
7085	const parent_ty = parent_ptr_ty.childType(zcu);
7086	const result_ty_id = try cg.resolveType(parent_ptr_ty, .indirect);
7087
7088	const field_ptr = try cg.resolve(extra.field_ptr);
7089	const field_ptr_ty = cg.typeOf(extra.field_ptr);
7090	const field_ptr_int = try cg.intFromPtr(field_ptr);
7091	const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
7092
7093	const base_ptr_int = base_ptr_int: {
7094	if (field_offset == 0) break :base_ptr_int field_ptr_int;
7095
7096	const field_offset_id = try cg.constInt(.usize, field_offset);
7097	const field_ptr_tmp: Temporary = .init(.usize, field_ptr_int);
7098	const field_offset_tmp: Temporary = .init(.usize, field_offset_id);
7099	const result = try cg.buildBinary(.OpISub, field_ptr_tmp, field_offset_tmp);
7100	break :base_ptr_int try result.materialize(cg);
7101	};
7102
7103	if (target.os.tag != .opencl) {
7104	if (field_ptr_ty.ptrAddressSpace(zcu) != .physical_storage_buffer) {
7105	return cg.fail(
7106	"cannot cast integer to pointer with address space '{s}'",
7107	.{@tagName(field_ptr_ty.ptrAddressSpace(zcu))},
7108	);
7109	}
7110	}
7111
7112	const base_ptr = cg.allocId();
7113	try cg.body.emit(cg.gpa, .OpConvertUToPtr, .{
7114	.id_result_type = result_ty_id,
7115	.id_result = base_ptr,
7116	.integer_value = base_ptr_int,
7117	});
7118
7119	return base_ptr;
7120	}
7121
7122	fn structFieldPtr(
7123	cg: *CodeGen,
7124	result_ptr_ty: Type,
7125	object_ptr_ty: Type,
7126	object_ptr: Id,
7127	field_index: u32,
7128	) !Id {
7129	const result_ty_id = try cg.resolveType(result_ptr_ty, .direct);
7130
7131	const zcu = cg.zcu;
7132	const object_ty = object_ptr_ty.childType(zcu);
7133	switch (object_ty.zigTypeTag(zcu)) {
7134	.pointer => {
7135	assert(object_ty.isSlice(zcu));
7136	return cg.accessChain(result_ty_id, object_ptr, &.{field_index});
7137	},
7138	.@"struct" => switch (object_ty.containerLayout(zcu)) {
7139	.@"packed" => {
7140	const byte_offset = codegen.fieldOffset(object_ptr_ty, result_ptr_ty, field_index, zcu);
7141	if (byte_offset == 0) return object_ptr;
7142	const usize_ty_id = try cg.resolveType(.usize, .direct);
7143	const base_int = cg.allocId();
7144	try cg.body.emit(cg.gpa, .OpConvertPtrToU, .{
7145	.id_result_type = usize_ty_id,
7146	.id_result = base_int,
7147	.pointer = object_ptr,
7148	});
7149	const offset_id = try cg.constInt(.usize, byte_offset);
7150	const adjusted = try cg.buildBinary(.OpIAdd, .{ .ty = .usize, .value = .{ .singleton = base_int } }, .{ .ty = .usize, .value = .{ .singleton = offset_id } });
7151	const adjusted_id = try adjusted.materialize(cg);
7152	const result_id = cg.allocId();
7153	try cg.body.emit(cg.gpa, .OpConvertUToPtr, .{
7154	.id_result_type = result_ty_id,
7155	.id_result = result_id,
7156	.integer_value = adjusted_id,
7157	});
7158	return result_id;
7159	},
7160	.auto, .@"extern" => {
7161	return try cg.accessChain(result_ty_id, object_ptr, &.{field_index});
7162	},
7163	},
7164	.@"union" => switch (object_ty.containerLayout(zcu)) {
7165	.@"packed" => return cg.todo("implement field access for packed unions", .{}),
7166	.auto, .@"extern" => {
7167	const layout = cg.unionLayout(object_ty);
7168	if (!layout.has_payload) {
7169	// Asked to get a pointer to a zero-sized field. Just lower this
7170	// to undefined, there is no reason to make it be a valid pointer.
7171	return try cg.constUndef(result_ty_id);
7172	}
7173
7174	const storage_class = cg.storageClass(object_ptr_ty.ptrAddressSpace(zcu));
7175	const field_ty = result_ptr_ty.childType(zcu);
7176	if (field_ty.toIntern() == layout.payload_ty.toIntern()) {
7177	if (object_ty.containerLayout(zcu) == .@"packed") return object_ptr;
7178	return try cg.accessChain(result_ty_id, object_ptr, &.{layout.payload_index});
7179	}
7180
7181	switch (zcu.getTarget().os.tag) {
7182	.vulkan, .opengl => {
7183	// Logical addressing forbids OpBitcast on pointers. If the field
7184	// type is structurally identical to the payload type (dedup will
7185	// unify them) the access chain typed as the field type is valid.
7186	if (object_ty.containerLayout(zcu) == .@"packed") return object_ptr;
7187	return try cg.accessChain(result_ty_id, object_ptr, &.{layout.payload_index});
7188	},
7189	else => {},
7190	}
7191
7192	const layout_payload_ty_id = try cg.resolveType(layout.payload_ty, .indirect);
7193	const pl_ptr_ty_id = try cg.ptrType(layout_payload_ty_id, storage_class);
7194	const pl_ptr_id = blk: {
7195	if (object_ty.containerLayout(zcu) == .@"packed") break :blk object_ptr;
7196	break :blk try cg.accessChain(pl_ptr_ty_id, object_ptr, &.{layout.payload_index});
7197	};
7198
7199	const active_pl_ptr_id = cg.allocId();
7200	try cg.body.emit(cg.gpa, .OpBitcast, .{
7201	.id_result_type = result_ty_id,
7202	.id_result = active_pl_ptr_id,
7203	.operand = pl_ptr_id,
7204	});
7205	return active_pl_ptr_id;
7206	},
7207	},
7208	else => unreachable,
7209	}
7210	}
7211
7212	fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7213	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7214	const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data;
7215	const struct_ptr = try cg.resolve(struct_field.struct_operand);
7216	const struct_ptr_ty = cg.typeOf(struct_field.struct_operand);
7217	const result_ptr_ty = cg.typeOfIndex(inst);
7218	return try cg.structFieldPtr(result_ptr_ty, struct_ptr_ty, struct_ptr, struct_field.field_index);
7219	}
7220
7221	fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, field_index: u32) !?Id {
7222	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7223	const struct_ptr = try cg.resolve(ty_op.operand);
7224	const struct_ptr_ty = cg.typeOf(ty_op.operand);
7225	const result_ptr_ty = cg.typeOfIndex(inst);
7226	return try cg.structFieldPtr(result_ptr_ty, struct_ptr_ty, struct_ptr, field_index);
7227	}
7228
7229	fn alloc(cg: *CodeGen, ty_id: Id, initializer: ?Id) !Id {
7230	const ptr_ty_id = try cg.ptrType(ty_id, .function);
7231	const result_id = cg.allocId();
7232	try cg.prologue.emit(cg.gpa, .OpVariable, .{
7233	.id_result_type = ptr_ty_id,
7234	.id_result = result_id,
7235	.storage_class = .function,
7236	.initializer = initializer,
7237	});
7238	return result_id;
7239	}
7240
7241	fn airAlloc(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7242	const zcu = cg.zcu;
7243	const target = zcu.getTarget();
7244	const ptr_ty = cg.typeOfIndex(inst);
7245	const child_ty = ptr_ty.childType(zcu);
7246
7247	switch (target.os.tag) {
7248	.vulkan, .opengl => {
7249	if (child_ty.zigTypeTag(zcu) == .pointer and !child_ty.isSlice(zcu)) {
7250	const as = child_ty.ptrAddressSpace(zcu);
7251	if (cg.storageClass(as) == .function) {
7252	const result_id = cg.allocId();
7253	try cg.tracked_allocas.put(cg.gpa, result_id, null);
7254	return result_id;
7255	}
7256	}
7257	},
7258	else => {},
7259	}
7260
7261	const child_ty_id = try cg.resolveType(child_ty, .indirect);
7262	const ptr_align = ptr_ty.ptrAlignment(zcu);
7263	const result_id = try cg.alloc(child_ty_id, null);
7264	if (ptr_align != child_ty.abiAlignment(zcu)) {
7265	if (target.os.tag != .opencl) return cg.fail("cannot apply alignment to variables", .{});
7266	try cg.decorate(result_id, .{
7267	.alignment = .{ .alignment = @intCast(ptr_align.toByteUnits().?) },
7268	});
7269	}
7270	return result_id;
7271	}
7272
7273	fn airArg(cg: *CodeGen) Id {
7274	defer cg.next_arg_index += 1;
7275	return cg.args.items[cg.next_arg_index];
7276	}
7277
7278	/// Given a slice of incoming block connections, returns the block-id of the next
7279	/// block to jump to. This function emits instructions, so it should be emitted
7280	/// inside the merge block of the block.
7281	/// This function should only be called with structured control flow generation.
7282	fn structuredNextBlock(cg: *CodeGen, incoming: []const Block.Incoming) !Id {
7283	const result_id = cg.allocId();
7284	const block_id_ty_id = try cg.resolveType(.u32, .direct);
7285	try cg.body.emitRaw(cg.gpa, .OpPhi, @intCast(2 + incoming.len * 2)); // result type + result + variable/parent...
7286	cg.body.writeOperand(Id, block_id_ty_id);
7287	cg.body.writeOperand(Id, result_id);
7288
7289	for (incoming) |incoming_block| {
7290	cg.body.writeOperand(spec.PairIdRefIdRef, .{ incoming_block.next_block, incoming_block.src_label });
7291	}
7292
7293	return result_id;
7294	}
7295
7296	/// Jumps to the block with the target block-id. This function must only be called when
7297	/// terminating a body, there should be no instructions after it.
7298	/// This function should only be called with structured control flow generation.
7299	fn structuredBreak(cg: *CodeGen, target_block: Id) !void {
7300	if (cg.block_terminated) return;
7301
7302	const gpa = cg.gpa;
7303	const sblock = cg.block_stack.last().?;
7304	const merge_block = switch (sblock.*) {
7305	.selection => |*merge| blk: {
7306	const merge_label = cg.allocId();
7307	try merge.merge_stack.append(gpa, .{
7308	.incoming = .{
7309	.src_label = cg.block_label,
7310	.next_block = target_block,
7311	},
7312	.merge_block = merge_label,
7313	});
7314	break :blk merge_label;
7315	},
7316	// Loop blocks do not end in a break. Not through a direct break,
7317	// and also not through another instruction like cond_br or unreachable (these
7318	// situations are replaced by `cond_br` in sema, or there is a `block` instruction
7319	// placed around them).
7320	.loop => unreachable,
7321	};
7322
7323	try cg.body.emit(gpa, .OpBranch, .{ .target_label = merge_block });
7324	}
7325
7326	/// Generate a body in a way that exits the body using only structured constructs.
7327	/// Returns the block-id of the next block to jump to. After this function, a jump
7328	/// should still be emitted to the block that should follow this structured body.
7329	/// This function should only be called with structured control flow generation.
7330	fn genStructuredBody(
7331	cg: *CodeGen,
7332	/// This parameter defines the method that this structured body is exited with.
7333	block_merge_type: union(enum) {
7334	/// Using selection; early exits from this body are surrounded with
7335	/// if() statements.
7336	selection,
7337	/// Using loops; loops can be early exited by jumping to the merge block at
7338	/// any time.
7339	loop: struct {
7340	merge_label: Id,
7341	continue_label: Id,
7342	},
7343	},
7344	body: []const Air.Inst.Index,
7345	) !Id {
7346	const gpa = cg.gpa;
7347
7348	var sblock: Block = switch (block_merge_type) {
7349	.loop => |merge| .{ .loop = .{
7350	.merge_block = merge.merge_label,
7351	} },
7352	.selection => .{ .selection = .{} },
7353	};
7354	defer sblock.deinit(gpa);
7355
7356	{
7357	try cg.block_stack.append(gpa, &sblock);
7358	defer _ = cg.block_stack.pop();
7359
7360	try cg.genBody(body);
7361	}
7362
7363	switch (sblock) {
7364	.selection => |merge| {
7365	// Now generate the merge block for all merges that
7366	// still need to be performed.
7367	const merge_stack = merge.merge_stack.items;
7368
7369	// If no merges on the stack, this block didn't generate any jumps (all paths
7370	// ended with a return or an unreachable). In that case, we don't need to do
7371	// any merging.
7372	if (merge_stack.len == 0) {
7373	// We still need to return a value of a next block to jump to.
7374	// For example, if we have code like
7375	// if (x) {
7376	// if (y) return else return;
7377	// } else {}
7378	// then we still need the outer to have an OpSelectionMerge and consequently
7379	// a phi node. In that case we can just return bogus, since we know that its
7380	// path will never be taken.
7381
7382	// Make sure that we are still in a block when exiting the function.
7383	// TODO: Can we get rid of that?
7384	try cg.beginSpvBlock(cg.allocId());
7385	const block_id_ty_id = try cg.resolveType(.u32, .direct);
7386	return try cg.constUndef(block_id_ty_id);
7387	}
7388
7389	// The top-most merge actually only has a single source, the
7390	// final jump of the block, or the merge block of a sub-block, cond_br,
7391	// or loop. Therefore we just need to generate a block with a jump to the
7392	// next merge block.
7393	try cg.beginSpvBlock(merge_stack[merge_stack.len - 1].merge_block);
7394
7395	// Now generate a merge ladder for the remaining merges in the stack.
7396	var incoming: Block.Incoming = .{
7397	.src_label = cg.block_label,
7398	.next_block = merge_stack[merge_stack.len - 1].incoming.next_block,
7399	};
7400	var i = merge_stack.len - 1;
7401	while (i > 0) {
7402	i -= 1;
7403	const step = merge_stack[i];
7404
7405	try cg.body.emit(gpa, .OpBranch, .{ .target_label = step.merge_block });
7406	try cg.beginSpvBlock(step.merge_block);
7407	const next_block = try cg.structuredNextBlock(&.{ incoming, step.incoming });
7408	incoming = .{
7409	.src_label = step.merge_block,
7410	.next_block = next_block,
7411	};
7412	}
7413
7414	return incoming.next_block;
7415	},
7416	.loop => |merge| {
7417	// Close the loop by jumping to the continue label
7418
7419	try cg.body.emit(gpa, .OpBranch, .{ .target_label = block_merge_type.loop.continue_label });
7420	// For blocks we must simple merge all the incoming blocks to get the next block.
7421	try cg.beginSpvBlock(merge.merge_block);
7422	return try cg.structuredNextBlock(merge.merges.items);
7423	},
7424	}
7425	}
7426
7427	fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7428	const block = cg.air.unwrapBlock(inst);
7429	return cg.lowerBlock(inst, block.body);
7430	}
7431
7432	fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index) !?Id {
7433	// In AIR, a block doesn't really define an entry point like a block, but
7434	// more like a scope that breaks can jump out of and "return" a value from.
7435	// This cannot be directly modelled in SPIR-V, so in a block instruction,
7436	// we're going to split up the current block by first generating the code
7437	// of the block, then a label, and then generate the rest of the current
7438	// ir.Block in a different SPIR-V block.
7439
7440	const gpa = cg.gpa;
7441	const zcu = cg.zcu;
7442	const ty = cg.typeOfIndex(inst);
7443	const have_block_result = ty.hasRuntimeBits(zcu);
7444
7445	const maybe_block_result_var_id = if (have_block_result) blk: {
7446	const ty_id = try cg.resolveType(ty, .indirect);
7447	const block_result_var_id = try cg.alloc(ty_id, null);
7448	try cg.block_results.putNoClobber(gpa, inst, block_result_var_id);
7449	break :blk block_result_var_id;
7450	} else null;
7451	defer if (have_block_result) assert(cg.block_results.remove(inst));
7452
7453	const next_block = try cg.genStructuredBody(.selection, body);
7454
7455	// When encountering a block instruction, we are always at least in the function's scope,
7456	// so there always has to be another entry.
7457	assert(cg.block_stack.items.len > 0);
7458
7459	// Check if the target of the branch was this current block.
7460	const this_block = try cg.constInt(.u32, @backingInt(inst));
7461	const jump_to_this_block_id = cg.allocId();
7462	const bool_ty_id = try cg.resolveType(.bool, .direct);
7463	try cg.body.emit(gpa, .OpIEqual, .{
7464	.id_result_type = bool_ty_id,
7465	.id_result = jump_to_this_block_id,
7466	.operand_1 = next_block,
7467	.operand_2 = this_block,
7468	});
7469
7470	const sblock = cg.block_stack.last().?;
7471
7472	if (ty.isNoReturn(zcu)) {
7473	// If this block is noreturn, this instruction is the last of a block,
7474	// and we must simply jump to the block's merge unconditionally.
7475	try cg.structuredBreak(next_block);
7476	} else {
7477	switch (sblock.*) {
7478	.selection => |*merge| {
7479	// To jump out of a selection block, push a new entry onto its merge stack and
7480	// generate a conditional branch to there and to the instructions following this block.
7481	const merge_label = cg.allocId();
7482	const then_label = cg.allocId();
7483	try cg.body.emit(gpa, .OpSelectionMerge, .{
7484	.merge_block = merge_label,
7485	.selection_control = .{},
7486	});
7487	try cg.body.emit(gpa, .OpBranchConditional, .{
7488	.condition = jump_to_this_block_id,
7489	.true_label = then_label,
7490	.false_label = merge_label,
7491	});
7492	try merge.merge_stack.append(gpa, .{
7493	.incoming = .{
7494	.src_label = cg.block_label,
7495	.next_block = next_block,
7496	},
7497	.merge_block = merge_label,
7498	});
7499
7500	try cg.beginSpvBlock(then_label);
7501	},
7502	.loop => |*merge| {
7503	// To jump out of a loop block, generate a conditional that exits the block
7504	// to the loop merge if the target ID is not the one of this block.
7505	const continue_label = cg.allocId();
7506	try cg.body.emit(gpa, .OpBranchConditional, .{
7507	.condition = jump_to_this_block_id,
7508	.true_label = continue_label,
7509	.false_label = merge.merge_block,
7510	});
7511	try merge.merges.append(gpa, .{
7512	.src_label = cg.block_label,
7513	.next_block = next_block,
7514	});
7515	try cg.beginSpvBlock(continue_label);
7516	},
7517	}
7518	}
7519
7520	if (maybe_block_result_var_id) |block_result_var_id| {
7521	return try cg.load(ty, block_result_var_id, .{});
7522	}
7523
7524	return null;
7525	}
7526
7527	fn airBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
7528	const zcu = cg.zcu;
7529	const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;
7530	const operand_ty = cg.typeOf(br.operand);
7531
7532	if (operand_ty.hasRuntimeBits(zcu)) {
7533	const operand_id = try cg.resolve(br.operand);
7534	const block_result_var_id = cg.block_results.get(br.block_inst).?;
7535	try cg.store(operand_ty, block_result_var_id, operand_id, .{});
7536	}
7537
7538	const next_block = try cg.constInt(.u32, @backingInt(br.block_inst));
7539	try cg.structuredBreak(next_block);
7540	}
7541
7542	fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
7543	const gpa = cg.gpa;
7544	const cond_br = cg.air.unwrapCondBr(inst);
7545	const then_body = cond_br.then_body;
7546	const else_body = cond_br.else_body;
7547	const condition_id = try cg.resolve(cond_br.condition);
7548
7549	const then_label = cg.allocId();
7550	const else_label = cg.allocId();
7551
7552	const merge_label = cg.allocId();
7553
7554	try cg.body.emit(gpa, .OpSelectionMerge, .{
7555	.merge_block = merge_label,
7556	.selection_control = .{},
7557	});
7558	try cg.body.emit(gpa, .OpBranchConditional, .{
7559	.condition = condition_id,
7560	.true_label = then_label,
7561	.false_label = else_label,
7562	});
7563
7564	try cg.beginSpvBlock(then_label);
7565	const then_next = try cg.genStructuredBody(.selection, then_body);
7566	const then_incoming: Block.Incoming = .{
7567	.src_label = cg.block_label,
7568	.next_block = then_next,
7569	};
7570
7571	if (!cg.block_terminated) {
7572	try cg.body.emit(gpa, .OpBranch, .{ .target_label = merge_label });
7573	}
7574
7575	try cg.beginSpvBlock(else_label);
7576	const else_next = try cg.genStructuredBody(.selection, else_body);
7577	const else_incoming: Block.Incoming = .{
7578	.src_label = cg.block_label,
7579	.next_block = else_next,
7580	};
7581
7582	if (!cg.block_terminated) {
7583	try cg.body.emit(gpa, .OpBranch, .{ .target_label = merge_label });
7584	}
7585
7586	try cg.beginSpvBlock(merge_label);
7587	const next_block = try cg.structuredNextBlock(&.{ then_incoming, else_incoming });
7588
7589	try cg.structuredBreak(next_block);
7590	}
7591
7592	fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) !void {
7593	const gpa = cg.gpa;
7594	const block = cg.air.unwrapBlock(inst);
7595
7596	const body_label = cg.allocId();
7597
7598	const header_label = cg.allocId();
7599	const merge_label = cg.allocId();
7600	const continue_label = cg.allocId();
7601
7602	// The back-edge must point to the loop header, so generate a separate block for the
7603	// loop header so that we don't accidentally include some instructions from there
7604	// in the loop.
7605
7606	try cg.body.emit(gpa, .OpBranch, .{ .target_label = header_label });
7607	try cg.beginSpvBlock(header_label);
7608
7609	// Emit loop header and jump to loop body
7610	try cg.body.emit(gpa, .OpLoopMerge, .{
7611	.merge_block = merge_label,
7612	.continue_target = continue_label,
7613	.loop_control = .{},
7614	});
7615
7616	try cg.body.emit(gpa, .OpBranch, .{ .target_label = body_label });
7617
7618	try cg.beginSpvBlock(body_label);
7619
7620	const next_block = try cg.genStructuredBody(.{ .loop = .{
7621	.merge_label = merge_label,
7622	.continue_label = continue_label,
7623	} }, block.body);
7624	try cg.structuredBreak(next_block);
7625
7626	try cg.beginSpvBlock(continue_label);
7627
7628	try cg.body.emit(gpa, .OpBranch, .{ .target_label = header_label });
7629	}
7630
7631	fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7632	const zcu = cg.zcu;
7633	const pt = cg.pt;
7634	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7635	const ptr_ty = cg.typeOf(ty_op.operand);
7636	const ptr_info = ptr_ty.ptrInfo(zcu);
7637	const elem_ty = cg.typeOfIndex(inst);
7638	const ptr = try cg.resolvePtr(ty_op.operand);
7639	assert(ptr_info.child == elem_ty.toIntern());
7640
7641	const operand_ptr_id = switch (ptr) {
7642	.tracked => |t| return t.slot.*.?,
7643	.id => |id| id,
7644	};
7645
7646	if (ptr_info.packed_offset.host_size != 0 and
7647	ptr_info.flags.vector_index == .none)
7648	{
7649	const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
7650	const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
7651	const host_int_ty = try pt.intType(.unsigned, host_bits);
7652	const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7653	const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
7654	const field_int_ty = try pt.intType(signedness, elem_bit_size);
7655	const narrowed = if (ptr_info.packed_offset.bit_offset > 0) blk: {
7656	const bit_offset_id = try cg.constInt(host_int_ty, ptr_info.packed_offset.bit_offset);
7657	const shifted = try cg.buildBinary(.OpShiftRightLogical, .{ .ty = host_int_ty, .value = .{ .singleton = host_val } }, .{ .ty = host_int_ty, .value = .{ .singleton = bit_offset_id } });
7658	break :blk try shifted.materialize(cg);
7659	} else host_val;
7660	const result_id = blk: {
7661	if (cg.backingIntBits(elem_bit_size).@"0" == cg.backingIntBits(host_bits).@"0")
7662	break :blk try cg.bitCast(field_int_ty, host_int_ty, narrowed);
7663	const trunc = try cg.buildConvert(field_int_ty, .{ .ty = host_int_ty, .value = .{ .singleton = narrowed } });
7664	break :blk try trunc.materialize(cg);
7665	};
7666	if (elem_ty.ip_index == .bool_type) return try cg.convertToDirect(.bool, result_id);
7667	if (elem_ty.isInt(zcu)) return result_id;
7668	return try cg.bitCast(elem_ty, field_int_ty, result_id);
7669	}
7670
7671	const ptr_id = switch (ptr_info.flags.vector_index) {
7672	.none => operand_ptr_id,
7673	else => |index| ptr_id: {
7674	const elem_ptr_ty_id = try cg.ptrType(
7675	try cg.resolveType(elem_ty, .indirect),
7676	cg.storageClass(ptr_info.flags.address_space),
7677	);
7678	break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@backingInt(index)});
7679	},
7680	};
7681	return try cg.load(elem_ty, ptr_id, .{
7682	.is_volatile = ptr_info.flags.is_volatile,
7683	.ptr_address_space = ptr_info.flags.address_space,
7684	});
7685	}
7686
7687	fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
7688	const zcu = cg.zcu;
7689	const pt = cg.pt;
7690	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7691	const ptr_ty = cg.typeOf(bin_op.lhs);
7692	const ptr_info = ptr_ty.ptrInfo(zcu);
7693	const elem_ty: Type = .fromInterned(ptr_info.child);
7694	const value_id = try cg.resolve(bin_op.rhs);
7695	const operand_ptr_id = switch (try cg.resolvePtr(bin_op.lhs)) {
7696	.tracked => |t| {
7697	t.slot.* = value_id;
7698	return;
7699	},
7700	.id => |id| id,
7701	};
7702
7703	if (ptr_info.packed_offset.host_size != 0 and
7704	ptr_info.flags.vector_index == .none)
7705	{
7706	const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
7707	const host_int_ty = try pt.intType(.unsigned, host_bits);
7708	const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7709	const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
7710	const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
7711	const field_int_ty = try pt.intType(signedness, elem_bit_size);
7712
7713	var value_as_int: Id = undefined;
7714	if (elem_ty.ip_index == .bool_type) {
7715	value_as_int = try cg.convertToIndirect(.bool, value_id);
7716	value_as_int = try cg.bitCast(field_int_ty, .u1, value_as_int);
7717	} else if (elem_ty.isInt(zcu)) {
7718	value_as_int = value_id;
7719	} else {
7720	value_as_int = try cg.bitCast(field_int_ty, elem_ty, value_id);
7721	}
7722
7723	const extended = blk: {
7724	if (cg.backingIntBits(elem_bit_size).@"0" == cg.backingIntBits(host_bits).@"0")
7725	break :blk try cg.bitCast(host_int_ty, field_int_ty, value_as_int);
7726	const conv = try cg.buildConvert(host_int_ty, .{ .ty = field_int_ty, .value = .{ .singleton = value_as_int } });
7727	break :blk try conv.materialize(cg);
7728	};
7729
7730	const bit_offset = ptr_info.packed_offset.bit_offset;
7731	const field_mask = (@as(u64, 1) << @as(u6, @intCast(elem_bit_size))) - 1;
7732	const host_mask = if (host_bits == 64) @as(u64, std.math.maxInt(u64)) else (@as(u64, 1) << @as(u6, @intCast(host_bits))) - 1;
7733	const clear_mask = ~(field_mask << @as(u6, @intCast(bit_offset))) & host_mask;
7734	const clear_mask_id = try cg.constInt(host_int_ty, clear_mask);
7735	const cleared = try cg.buildBinary(.OpBitwiseAnd, .{ .ty = host_int_ty, .value = .{ .singleton = host_val } }, .{ .ty = host_int_ty, .value = .{ .singleton = clear_mask_id } });
7736	const bit_offset_id = try cg.constInt(host_int_ty, bit_offset);
7737	const shifted_val = try cg.buildBinary(.OpShiftLeftLogical, .{ .ty = host_int_ty, .value = .{ .singleton = extended } }, .{ .ty = host_int_ty, .value = .{ .singleton = bit_offset_id } });
7738	const combined = try cg.buildBinary(.OpBitwiseOr, cleared, shifted_val);
7739	const combined_id = try combined.materialize(cg);
7740
7741	try cg.store(host_int_ty, operand_ptr_id, combined_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7742	return;
7743	}
7744
7745	const ptr_id = switch (ptr_info.flags.vector_index) {
7746	.none => operand_ptr_id,
7747	else => |index| ptr_id: {
7748	const elem_ptr_ty_id = try cg.ptrType(
7749	try cg.resolveType(elem_ty, .indirect),
7750	cg.storageClass(ptr_info.flags.address_space),
7751	);
7752	break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@backingInt(index)});
7753	},
7754	};
7755
7756	try cg.store(elem_ty, ptr_id, value_id, .{
7757	.is_volatile = ptr_info.flags.is_volatile,
7758	.ptr_address_space = ptr_info.flags.address_space,
7759	});
7760	}
7761
7762	fn airRet(cg: *CodeGen, inst: Air.Inst.Index) !void {
7763	const gpa = cg.gpa;
7764	const zcu = cg.zcu;
7765	const operand = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7766	const ret_ty = cg.typeOf(operand);
7767	if (!ret_ty.hasRuntimeBits(zcu)) {
7768	const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
7769	if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
7770	// Functions with an empty error set are emitted with an error code
7771	// return type and return zero so they can be function pointers coerced
7772	// to functions that return anyerror.
7773	const no_err_id = try cg.constInt(.anyerror, 0);
7774	return try cg.body.emit(gpa, .OpReturnValue, .{ .value = no_err_id });
7775	} else {
7776	return try cg.body.emit(gpa, .OpReturn, {});
7777	}
7778	}
7779
7780	const operand_id = try cg.resolve(operand);
7781	try cg.body.emit(gpa, .OpReturnValue, .{ .value = operand_id });
7782	}
7783
7784	fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) !void {
7785	const gpa = cg.gpa;
7786	const zcu = cg.zcu;
7787	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7788	const ptr_ty = cg.typeOf(un_op);
7789	const ret_ty = ptr_ty.childType(zcu);
7790
7791	if (!ret_ty.hasRuntimeBits(zcu)) {
7792	const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
7793	if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
7794	// Functions with an empty error set are emitted with an error code
7795	// return type and return zero so they can be function pointers coerced
7796	// to functions that return anyerror.
7797	const no_err_id = try cg.constInt(.anyerror, 0);
7798	return try cg.body.emit(gpa, .OpReturnValue, .{ .value = no_err_id });
7799	} else {
7800	return try cg.body.emit(gpa, .OpReturn, {});
7801	}
7802	}
7803
7804	const value = switch (try cg.resolvePtr(un_op)) {
7805	.tracked => |t| t.slot.*.?,
7806	.id => |ptr| try cg.load(ret_ty, ptr, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) }),
7807	};
7808	try cg.body.emit(gpa, .OpReturnValue, .{
7809	.value = value,
7810	});
7811	}
7812
7813	fn airTry(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7814	const gpa = cg.gpa;
7815	const zcu = cg.zcu;
7816	const unwrapped_try = cg.air.unwrapTry(inst);
7817	const body = unwrapped_try.else_body;
7818
7819	const err_union_id = try cg.resolve(unwrapped_try.error_union);
7820	const err_union_ty = cg.air.typeOf(unwrapped_try.error_union, &zcu.intern_pool);
7821	const payload_ty = cg.typeOfIndex(inst);
7822
7823	const bool_ty_id = try cg.resolveType(.bool, .direct);
7824
7825	const eu_layout = cg.errorUnionLayout(payload_ty);
7826
7827	if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7828	const err_id = if (eu_layout.payload_has_bits)
7829	try cg.extractField(.anyerror, err_union_id, eu_layout.errorFieldIndex())
7830	else
7831	err_union_id;
7832
7833	const zero_id = try cg.constInt(.anyerror, 0);
7834	const is_err_id = cg.allocId();
7835	try cg.body.emit(gpa, .OpINotEqual, .{
7836	.id_result_type = bool_ty_id,
7837	.id_result = is_err_id,
7838	.operand_1 = err_id,
7839	.operand_2 = zero_id,
7840	});
7841
7842	// When there is an error, we must evaluate `body`. Otherwise we must continue
7843	// with the current body.
7844	// Just generate a new block here, then generate a new block inline for the remainder of the body.
7845
7846	const err_block = cg.allocId();
7847	const ok_block = cg.allocId();
7848
7849	// According to AIR documentation, this block is guaranteed
7850	// to not break and end in a return instruction. Thus,
7851	// we can just naively use the ok block as the merge block here.
7852	try cg.body.emit(gpa, .OpSelectionMerge, .{
7853	.merge_block = ok_block,
7854	.selection_control = .{},
7855	});
7856
7857	try cg.body.emit(gpa, .OpBranchConditional, .{
7858	.condition = is_err_id,
7859	.true_label = err_block,
7860	.false_label = ok_block,
7861	});
7862
7863	try cg.beginSpvBlock(err_block);
7864	try cg.genBody(body);
7865
7866	try cg.beginSpvBlock(ok_block);
7867	}
7868
7869	if (!eu_layout.payload_has_bits) {
7870	return null;
7871	}
7872
7873	// Now just extract the payload, if required.
7874	return try cg.extractField(payload_ty, err_union_id, eu_layout.payloadFieldIndex());
7875	}
7876
7877	fn airErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7878	const zcu = cg.zcu;
7879	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7880	const operand_id = try cg.resolve(ty_op.operand);
7881	const err_union_ty = cg.typeOf(ty_op.operand);
7882	const err_ty_id = try cg.resolveType(.anyerror, .direct);
7883
7884	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7885	// No error possible, so just return undefined.
7886	return try cg.constUndef(err_ty_id);
7887	}
7888
7889	const payload_ty = err_union_ty.errorUnionPayload(zcu);
7890	const eu_layout = cg.errorUnionLayout(payload_ty);
7891
7892	if (!eu_layout.payload_has_bits) {
7893	// If no payload, error union is represented by error set.
7894	return operand_id;
7895	}
7896
7897	return try cg.extractField(.anyerror, operand_id, eu_layout.errorFieldIndex());
7898	}
7899
7900	fn airErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7901	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7902	const operand_id = try cg.resolve(ty_op.operand);
7903	const payload_ty = cg.typeOfIndex(inst);
7904	const eu_layout = cg.errorUnionLayout(payload_ty);
7905
7906	if (!eu_layout.payload_has_bits) {
7907	return null; // No error possible.
7908	}
7909
7910	return try cg.extractField(payload_ty, operand_id, eu_layout.payloadFieldIndex());
7911	}
7912
7913	fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7914	const zcu = cg.zcu;
7915	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7916	const err_union_ty = cg.typeOfIndex(inst);
7917	const payload_ty = err_union_ty.errorUnionPayload(zcu);
7918	const operand_id = try cg.resolve(ty_op.operand);
7919	const eu_layout = cg.errorUnionLayout(payload_ty);
7920
7921	if (!eu_layout.payload_has_bits) {
7922	return operand_id;
7923	}
7924
7925	const payload_ty_id = try cg.resolveType(payload_ty, .indirect);
7926
7927	var members: [2]Id = undefined;
7928	members[eu_layout.errorFieldIndex()] = operand_id;
7929	members[eu_layout.payloadFieldIndex()] = try cg.constUndef(payload_ty_id);
7930
7931	var types: [2]Type = undefined;
7932	types[eu_layout.errorFieldIndex()] = .anyerror;
7933	types[eu_layout.payloadFieldIndex()] = payload_ty;
7934
7935	const err_union_ty_id = try cg.resolveType(err_union_ty, .direct);
7936	return try cg.constructComposite(err_union_ty_id, &members);
7937	}
7938
7939	fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
7940	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7941	const err_union_ty = cg.typeOfIndex(inst);
7942	const operand_id = try cg.resolve(ty_op.operand);
7943	const payload_ty = cg.typeOf(ty_op.operand);
7944	const eu_layout = cg.errorUnionLayout(payload_ty);
7945
7946	if (!eu_layout.payload_has_bits) {
7947	return try cg.constInt(.anyerror, 0);
7948	}
7949
7950	var members: [2]Id = undefined;
7951	members[eu_layout.errorFieldIndex()] = try cg.constInt(.anyerror, 0);
7952	members[eu_layout.payloadFieldIndex()] = try cg.convertToIndirect(payload_ty, operand_id);
7953
7954	var types: [2]Type = undefined;
7955	types[eu_layout.errorFieldIndex()] = .anyerror;
7956	types[eu_layout.payloadFieldIndex()] = payload_ty;
7957
7958	const err_union_ty_id = try cg.resolveType(err_union_ty, .direct);
7959	return try cg.constructComposite(err_union_ty_id, &members);
7960	}
7961
7962	fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, is_pointer: bool, pred: enum { is_null, is_non_null }) !?Id {
7963	const zcu = cg.zcu;
7964	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7965	const operand_id = try cg.resolve(un_op);
7966	const operand_ty = cg.typeOf(un_op);
7967	const optional_ty = if (is_pointer) operand_ty.childType(zcu) else operand_ty;
7968	const payload_ty = optional_ty.optionalChild(zcu);
7969
7970	const bool_ty_id = try cg.resolveType(.bool, .direct);
7971
7972	if (optional_ty.optionalReprIsPayload(zcu)) {
7973	// Pointer payload represents nullability: pointer or slice.
7974	const loaded_id = if (is_pointer)
7975	try cg.load(optional_ty, operand_id, .{})
7976	else
7977	operand_id;
7978
7979	const ptr_ty = if (payload_ty.isSlice(zcu))
7980	payload_ty.slicePtrFieldType(zcu)
7981	else
7982	payload_ty;
7983
7984	const ptr_id = if (payload_ty.isSlice(zcu))
7985	try cg.extractField(ptr_ty, loaded_id, 0)
7986	else
7987	loaded_id;
7988
7989	const ptr_ty_id = try cg.resolveType(ptr_ty, .direct);
7990	const null_id = try cg.constNull(ptr_ty_id);
7991	const null_tmp: Temporary = .init(ptr_ty, null_id);
7992	const ptr: Temporary = .init(ptr_ty, ptr_id);
7993
7994	const op: std.math.CompareOperator = switch (pred) {
7995	.is_null => .eq,
7996	.is_non_null => .neq,
7997	};
7998	const result = try cg.cmp(op, ptr, null_tmp);
7999	return try result.materialize(cg);
8000	}
8001
8002	const is_non_null_id = blk: {
8003	if (is_pointer) {
8004	if (payload_ty.hasRuntimeBits(zcu)) {
8005	const storage_class = cg.storageClass(operand_ty.ptrAddressSpace(zcu));
8006	const bool_indirect_ty_id = try cg.resolveType(.bool, .indirect);
8007	const bool_ptr_ty_id = try cg.ptrType(bool_indirect_ty_id, storage_class);
8008	const tag_ptr_id = try cg.accessChain(bool_ptr_ty_id, operand_id, &.{1});
8009	break :blk try cg.load(.bool, tag_ptr_id, .{});
8010	}
8011
8012	break :blk try cg.load(.bool, operand_id, .{});
8013	}
8014
8015	break :blk if (payload_ty.hasRuntimeBits(zcu))
8016	try cg.extractField(.bool, operand_id, 1)
8017	else
8018	// Optional representation is bool indicating whether the optional is set
8019	// Optionals with no payload are represented as an (indirect) bool, so convert
8020	// it back to the direct bool here.
8021	try cg.convertToDirect(.bool, operand_id);
8022	};
8023
8024	return switch (pred) {
8025	.is_null => blk: {
8026	// Invert condition
8027	const result_id = cg.allocId();
8028	try cg.body.emit(cg.gpa, .OpLogicalNot, .{
8029	.id_result_type = bool_ty_id,
8030	.id_result = result_id,
8031	.operand = is_non_null_id,
8032	});
8033	break :blk result_id;
8034	},
8035	.is_non_null => is_non_null_id,
8036	};
8037	}
8038
8039	fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, pred: enum { is_err, is_non_err }) !?Id {
8040	const zcu = cg.zcu;
8041	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
8042	const operand_id = try cg.resolve(un_op);
8043	const err_union_ty = cg.typeOf(un_op);
8044
8045	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
8046	return try cg.constBool(pred == .is_non_err, .direct);
8047	}
8048
8049	const payload_ty = err_union_ty.errorUnionPayload(zcu);
8050	const eu_layout = cg.errorUnionLayout(payload_ty);
8051	const bool_ty_id = try cg.resolveType(.bool, .direct);
8052
8053	const error_id = if (!eu_layout.payload_has_bits)
8054	operand_id
8055	else
8056	try cg.extractField(.anyerror, operand_id, eu_layout.errorFieldIndex());
8057
8058	const result_id = cg.allocId();
8059	switch (pred) {
8060	inline else => |pred_ct| try cg.body.emit(
8061	cg.gpa,
8062	switch (pred_ct) {
8063	.is_err => .OpINotEqual,
8064	.is_non_err => .OpIEqual,
8065	},
8066	.{
8067	.id_result_type = bool_ty_id,
8068	.id_result = result_id,
8069	.operand_1 = error_id,
8070	.operand_2 = try cg.constInt(.anyerror, 0),
8071	},
8072	),
8073	}
8074	return result_id;
8075	}
8076
8077	fn airUnwrapOptional(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8078	const zcu = cg.zcu;
8079	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
8080	const operand_id = try cg.resolve(ty_op.operand);
8081	const optional_ty = cg.typeOf(ty_op.operand);
8082	const payload_ty = cg.typeOfIndex(inst);
8083
8084	if (!payload_ty.hasRuntimeBits(zcu)) return null;
8085
8086	if (optional_ty.optionalReprIsPayload(zcu)) {
8087	return operand_id;
8088	}
8089
8090	return try cg.extractField(payload_ty, operand_id, 0);
8091	}
8092
8093	fn airUnwrapOptionalPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8094	const zcu = cg.zcu;
8095	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
8096	const operand_id = try cg.resolve(ty_op.operand);
8097	const operand_ty = cg.typeOf(ty_op.operand);
8098	const optional_ty = operand_ty.childType(zcu);
8099	const payload_ty = optional_ty.optionalChild(zcu);
8100	const result_ty = cg.typeOfIndex(inst);
8101	const result_ty_id = try cg.resolveType(result_ty, .direct);
8102
8103	if (!payload_ty.hasRuntimeBits(zcu)) {
8104	// There is no payload, but we still need to return a valid pointer.
8105	// We can just return anything here, so just return a pointer to the operand.
8106	return try cg.bitCast(result_ty, operand_ty, operand_id);
8107	}
8108
8109	if (optional_ty.optionalReprIsPayload(zcu)) {
8110	// They are the same value.
8111	return try cg.bitCast(result_ty, operand_ty, operand_id);
8112	}
8113
8114	return try cg.accessChain(result_ty_id, operand_id, &.{0});
8115	}
8116
8117	fn airSetOptionalPtr(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8118	const zcu = cg.zcu;
8119	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
8120
8121	const ptr_ty = cg.typeOf(ty_op.operand);
8122	const ptr_id = try cg.resolve(ty_op.operand);
8123
8124	const optional_ty = ptr_ty.childType(zcu);
8125	const payload_ty = optional_ty.optionalChild(zcu);
8126	const result_ty = cg.typeOfIndex(inst);
8127
8128	if (optional_ty.optionalReprIsPayload(zcu)) {
8129	return try cg.bitCast(result_ty, ptr_ty, ptr_id);
8130	}
8131
8132	const storage_class = cg.storageClass(ptr_ty.ptrAddressSpace(zcu));
8133	const bool_indirect_ty_id = try cg.resolveType(.bool, .indirect);
8134	const bool_ptr_ty_id = try cg.ptrType(bool_indirect_ty_id, storage_class);
8135	const result_ty_id = try cg.resolveType(result_ty, .direct);
8136
8137	const bool_ptr_id, const ret = switch (payload_ty.hasRuntimeBits(zcu)) {
8138	true => .{
8139	try cg.accessChain(bool_ptr_ty_id, ptr_id, &.{1}),
8140	try cg.accessChain(result_ty_id, ptr_id, &.{0}),
8141	},
8142	false => .{ ptr_id, try cg.bitCast(result_ty, ptr_ty, ptr_id) },
8143	};
8144
8145	try cg.store(.bool, bool_ptr_id, try cg.constBool(true, .direct), .{});
8146
8147	return ret;
8148	}
8149
8150	fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8151	const zcu = cg.zcu;
8152	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
8153	const payload_ty = cg.typeOf(ty_op.operand);
8154
8155	assert(payload_ty.hasRuntimeBits(zcu));
8156
8157	const operand_id = try cg.resolve(ty_op.operand);
8158
8159	const optional_ty = cg.typeOfIndex(inst);
8160	if (optional_ty.optionalReprIsPayload(zcu)) {
8161	return operand_id;
8162	}
8163
8164	const payload_id = try cg.convertToIndirect(payload_ty, operand_id);
8165	const members = [_]Id{ payload_id, try cg.constBool(true, .indirect) };
8166	const optional_ty_id = try cg.resolveType(optional_ty, .direct);
8167	return try cg.constructComposite(optional_ty_id, &members);
8168	}
8169
8170	fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
8171	const gpa = cg.gpa;
8172	const zcu = cg.zcu;
8173	const target = cg.zcu.getTarget();
8174	const switch_br = cg.air.unwrapSwitch(inst);
8175	const cond_ty = cg.typeOf(switch_br.operand);
8176	const cond = try cg.resolve(switch_br.operand);
8177	var cond_indirect = try cg.convertToIndirect(cond_ty, cond);
8178
8179	const cond_words: u32 = switch (cond_ty.zigTypeTag(zcu)) {
8180	.bool, .error_set => 1,
8181	.int => blk: {
8182	const bits = cond_ty.intInfo(zcu).bits;
8183	const backing_bits, const big_int = cg.backingIntBits(bits);
8184	if (big_int) return cg.todo("implement composite int switch", .{});
8185	break :blk if (backing_bits <= 32) 1 else 2;
8186	},
8187	.@"enum" => blk: {
8188	const int_ty = cond_ty.backingIntType(zcu);
8189	const int_info = int_ty.intInfo(zcu);
8190	const backing_bits, const big_int = cg.backingIntBits(int_info.bits);
8191	if (big_int) return cg.todo("implement composite int switch", .{});
8192	break :blk if (backing_bits <= 32) 1 else 2;
8193	},
8194	.pointer => blk: {
8195	cond_indirect = try cg.intFromPtr(cond_indirect);
8196	break :blk target.ptrBitWidth() / 32;
8197	},
8198	// TODO: Figure out which types apply here, and work around them as we can only do integers.
8199	else => return cg.todo("implement switch for type {s}", .{@tagName(cond_ty.zigTypeTag(zcu))}),
8200	};
8201
8202	const num_cases = switch_br.cases_len;
8203
8204	// compute the total number of scalar arms and find the last range case
8205	var num_conditions: u32 = 0;
8206	var last_range_case: ?u32 = null;
8207	{
8208	var it = switch_br.iterateCases();
8209	while (it.next()) |case| {
8210	if (case.ranges.len > 0) {
8211	last_range_case = case.idx;
8212	} else {
8213	num_conditions += @intCast(case.items.len);
8214	}
8215	}
8216	}
8217
8218	// First, pre-allocate the labels for the cases.
8219	const case_labels = cg.allocIds(num_cases);
8220	// We always need the default case - if zig has none, we will generate unreachable there.
8221	const default_label = cg.allocId();
8222	const switch_default = if (last_range_case != null) cg.allocId() else default_label;
8223
8224	const merge_label = cg.allocId();
8225
8226	try cg.body.emit(gpa, .OpSelectionMerge, .{
8227	.merge_block = merge_label,
8228	.selection_control = .{},
8229	});
8230
8231	// Emit the instruction before generating the blocks.
8232	try cg.body.emitRaw(gpa, .OpSwitch, 2 + (cond_words + 1) * num_conditions);
8233	cg.body.writeOperand(Id, cond_indirect);
8234	cg.body.writeOperand(Id, switch_default);
8235
8236	// Emit the non-range cases into the OpSwitch.
8237	// Cases with ranges are handled by the conditional chain below.
8238	{
8239	var it = switch_br.iterateCases();
8240	while (it.next()) |case| {
8241	if (case.ranges.len > 0) continue;
8242	const label = case_labels.at(case.idx);
8243
8244	for (case.items) |item| {
8245	const value: Value = .fromInterned(item.toInterned().?);
8246	const int_val: u64 = switch (cond_ty.zigTypeTag(zcu)) {
8247	.bool, .int => if (cond_ty.isSignedInt(zcu)) @bitCast(value.toSignedInt(zcu)) else value.toUnsignedInt(zcu),
8248	.@"enum" => value.backingInt(zcu).toUnsignedInt(zcu),
8249	.error_set => value.getErrorInt(zcu),
8250	.pointer => value.toUnsignedInt(zcu),
8251	else => unreachable,
8252	};
8253	const int_lit: spec.LiteralContextDependentNumber = switch (cond_words) {
8254	1 => .{ .uint32 = @intCast(int_val) },
8255	2 => .{ .uint64 = int_val },
8256	else => unreachable,
8257	};
8258	cg.body.writeOperand(spec.LiteralContextDependentNumber, int_lit);
8259	cg.body.writeOperand(Id, label);
8260	}
8261	}
8262	}
8263
8264	var incoming_structured_blocks: std.ArrayList(Block.Incoming) = .empty;
8265	defer incoming_structured_blocks.deinit(gpa);
8266	try incoming_structured_blocks.ensureUnusedCapacity(gpa, num_cases + 1);
8267
8268	// emit the range-checking chain as nested if-else inside the switch's default branch.
8269	// each range case becomes:
8270	// - check condition,
8271	// - if true emit case body and branch to merge,
8272	// - else continue to next check or default
8273	if (last_range_case != null) {
8274	const cond_tmp: Temporary = .init(cond_ty, cond);
8275	const bool_ty_id = try cg.resolveType(.bool, .direct);
8276
8277	try cg.beginSpvBlock(switch_default);
8278
8279	var it_range = switch_br.iterateCases();
8280	while (it_range.next()) |case| {
8281	if (case.ranges.len == 0) continue;
8282
8283	var case_cond: ?Id = null;
8284
8285	for (case.items) |item| {
8286	const item_tmp: Temporary = try cg.temporary(item);
8287	const eq = try (try cg.cmp(.eq, cond_tmp, item_tmp)).materialize(cg);
8288	case_cond = if (case_cond) |prev| blk: {
8289	const combined = cg.allocId();
8290	try cg.body.emit(gpa, .OpLogicalOr, .{
8291	.id_result_type = bool_ty_id,
8292	.id_result = combined,
8293	.operand_1 = prev,
8294	.operand_2 = eq,
8295	});
8296	break :blk combined;
8297	} else eq;
8298	}
8299
8300	for (case.ranges) |range| {
8301	const lo_tmp: Temporary = try cg.temporary(range[0]);
8302	const hi_tmp: Temporary = try cg.temporary(range[1]);
8303	const ge = try (try cg.cmp(.gte, cond_tmp, lo_tmp)).materialize(cg);
8304	const le = try (try cg.cmp(.lte, cond_tmp, hi_tmp)).materialize(cg);
8305	const in_range = cg.allocId();
8306	try cg.body.emit(gpa, .OpLogicalAnd, .{
8307	.id_result_type = bool_ty_id,
8308	.id_result = in_range,
8309	.operand_1 = ge,
8310	.operand_2 = le,
8311	});
8312	case_cond = if (case_cond) |prev| blk: {
8313	const combined = cg.allocId();
8314	try cg.body.emit(gpa, .OpLogicalOr, .{
8315	.id_result_type = bool_ty_id,
8316	.id_result = combined,
8317	.operand_1 = prev,
8318	.operand_2 = in_range,
8319	});
8320	break :blk combined;
8321	} else in_range;
8322	}
8323
8324	const case_label = case_labels.at(case.idx);
8325	const is_last = case.idx == last_range_case.?;
8326	const next_check = if (is_last) default_label else cg.allocId();
8327
8328	try cg.body.emit(gpa, .OpSelectionMerge, .{
8329	.merge_block = next_check,
8330	.selection_control = .{},
8331	});
8332
8333	try cg.body.emit(gpa, .OpBranchConditional, .{
8334	.condition = case_cond.?,
8335	.true_label = case_label,
8336	.false_label = next_check,
8337	});
8338
8339	if (!is_last) {
8340	try cg.beginSpvBlock(next_check);
8341	}
8342	}
8343	}
8344
8345	// emit bodies
8346	var it = switch_br.iterateCases();
8347	while (it.next()) |case| {
8348	const label = case_labels.at(case.idx);
8349
8350	try cg.beginSpvBlock(label);
8351
8352	const next_block = try cg.genStructuredBody(.selection, case.body);
8353	incoming_structured_blocks.appendAssumeCapacity(.{
8354	.src_label = cg.block_label,
8355	.next_block = next_block,
8356	});
8357
8358	try cg.body.emit(gpa, .OpBranch, .{ .target_label = merge_label });
8359	}
8360
8361	const else_body = blk: {
8362	var it_else = switch_br.iterateCases();
8363	while (it_else.next()) |_| {}
8364	break :blk it_else.elseBody();
8365	};
8366	try cg.beginSpvBlock(default_label);
8367	if (else_body.len != 0) {
8368	const next_block = try cg.genStructuredBody(.selection, else_body);
8369	incoming_structured_blocks.appendAssumeCapacity(.{
8370	.src_label = cg.block_label,
8371	.next_block = next_block,
8372	});
8373
8374	try cg.body.emit(gpa, .OpBranch, .{ .target_label = merge_label });
8375	} else {
8376	try cg.body.emit(gpa, .OpUnreachable, {});
8377	}
8378
8379	try cg.beginSpvBlock(merge_label);
8380	const next_block = try cg.structuredNextBlock(incoming_structured_blocks.items);
8381	try cg.structuredBreak(next_block);
8382	}
8383
8384	fn airLoopSwitchBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
8385	const gpa = cg.gpa;
8386	const zcu = cg.zcu;
8387	const target = cg.zcu.getTarget();
8388	const switch_br = cg.air.unwrapSwitch(inst);
8389	const cond_ty = cg.typeOf(switch_br.operand);
8390	const initial_cond = try cg.resolve(switch_br.operand);
8391	var initial_cond_indirect = try cg.convertToIndirect(cond_ty, initial_cond);
8392
8393	const cond_words: u32 = switch (cond_ty.zigTypeTag(zcu)) {
8394	.bool, .error_set => 1,
8395	.int => blk: {
8396	const bits = cond_ty.intInfo(zcu).bits;
8397	const backing_bits, const big_int = cg.backingIntBits(bits);
8398	if (big_int) return cg.todo("implement composite int loop switch", .{});
8399	break :blk if (backing_bits <= 32) 1 else 2;
8400	},
8401	.@"enum" => blk: {
8402	const int_ty = cond_ty.backingIntType(zcu);
8403	const int_info = int_ty.intInfo(zcu);
8404	const backing_bits, const big_int = cg.backingIntBits(int_info.bits);
8405	if (big_int) return cg.todo("implement composite int loop switch", .{});
8406	break :blk if (backing_bits <= 32) 1 else 2;
8407	},
8408	.pointer => blk: {
8409	initial_cond_indirect = try cg.intFromPtr(initial_cond_indirect);
8410	break :blk target.ptrBitWidth() / 32;
8411	},
8412	else => return cg.todo("implement loop switch for type {s}", .{@tagName(cond_ty.zigTypeTag(zcu))}),
8413	};
8414
8415	const cond_ty_id = try cg.resolveType(cond_ty, .indirect);
8416	const cond_var = try cg.alloc(cond_ty_id, null);
8417	try cg.store(cond_ty, cond_var, initial_cond_indirect, .{});
8418
8419	const num_cases = switch_br.cases_len;
8420
8421	var num_conditions: u32 = 0;
8422	var last_range_case: ?u32 = null;
8423	{
8424	var it = switch_br.iterateCases();
8425	while (it.next()) |case| {
8426	if (case.ranges.len > 0) {
8427	last_range_case = case.idx;
8428	} else {
8429	num_conditions += @intCast(case.items.len);
8430	}
8431	}
8432	}
8433
8434	const case_labels = cg.allocIds(num_cases);
8435	const default_label = cg.allocId();
8436	const switch_default = if (last_range_case != null) cg.allocId() else default_label;
8437
8438	const header_label = cg.allocId();
8439	const loop_merge = cg.allocId();
8440	const continue_label = cg.allocId();
8441	const switch_merge = cg.allocId();
8442	const body_label = cg.allocId();
8443
8444	// switch_dispatch signals "continue the loop" by using this sentinel as the
8445	// next_block in structuredBreak. at switch_merge, a phi + comparison distinguishes
8446	// dispatch (continue) from break (exit)
8447	const dispatch_sentinel = try cg.constInt(.u32, @backingInt(inst));
8448
8449	try cg.loop_switches.putNoClobber(gpa, inst, .{
8450	.cond_var = cond_var,
8451	.continue_label = dispatch_sentinel,
8452	});
8453	defer assert(cg.loop_switches.remove(inst));
8454
8455	try cg.body.emit(gpa, .OpBranch, .{ .target_label = header_label });
8456	try cg.beginSpvBlock(header_label);
8457
8458	try cg.body.emit(gpa, .OpLoopMerge, .{
8459	.merge_block = loop_merge,
8460	.continue_target = continue_label,
8461	.loop_control = .{},
8462	});
8463
8464	try cg.body.emit(gpa, .OpBranch, .{ .target_label = body_label });
8465	try cg.beginSpvBlock(body_label);
8466
8467	const cond = try cg.load(cond_ty, cond_var, .{});
8468	const cond_indirect = try cg.convertToIndirect(cond_ty, cond);
8469
8470	try cg.body.emit(gpa, .OpSelectionMerge, .{
8471	.merge_block = switch_merge,
8472	.selection_control = .{},
8473	});
8474
8475	try cg.body.emitRaw(gpa, .OpSwitch, 2 + (cond_words + 1) * num_conditions);
8476	cg.body.writeOperand(Id, cond_indirect);
8477	cg.body.writeOperand(Id, switch_default);
8478
8479	{
8480	var it = switch_br.iterateCases();
8481	while (it.next()) |case| {
8482	if (case.ranges.len > 0) continue;
8483	const label = case_labels.at(case.idx);
8484	for (case.items) |item| {
8485	const value: Value = .fromInterned(item.toInterned().?);
8486	const int_val: u64 = switch (cond_ty.zigTypeTag(zcu)) {
8487	.bool, .int => if (cond_ty.isSignedInt(zcu)) @bitCast(value.toSignedInt(zcu)) else value.toUnsignedInt(zcu),
8488	.@"enum" => value.backingInt(zcu).toUnsignedInt(zcu),
8489	.error_set => value.getErrorInt(zcu),
8490	.pointer => value.toUnsignedInt(zcu),
8491	else => unreachable,
8492	};
8493	const int_lit: spec.LiteralContextDependentNumber = switch (cond_words) {
8494	1 => .{ .uint32 = @intCast(int_val) },
8495	2 => .{ .uint64 = int_val },
8496	else => unreachable,
8497	};
8498	cg.body.writeOperand(spec.LiteralContextDependentNumber, int_lit);
8499	cg.body.writeOperand(Id, label);
8500	}
8501	}
8502	}
8503
8504	var incoming_structured_blocks: std.ArrayList(Block.Incoming) = .empty;
8505	defer incoming_structured_blocks.deinit(gpa);
8506	try incoming_structured_blocks.ensureUnusedCapacity(gpa, num_cases + 1);
8507
8508	if (last_range_case != null) {
8509	const cond_tmp: Temporary = .init(cond_ty, cond);
8510	const bool_ty_id = try cg.resolveType(.bool, .direct);
8511
8512	try cg.beginSpvBlock(switch_default);
8513
8514	var it_range = switch_br.iterateCases();
8515	while (it_range.next()) |case| {
8516	if (case.ranges.len == 0) continue;
8517
8518	var case_cond: ?Id = null;
8519
8520	for (case.items) |item| {
8521	const item_tmp: Temporary = try cg.temporary(item);
8522	const eq = try (try cg.cmp(.eq, cond_tmp, item_tmp)).materialize(cg);
8523	case_cond = if (case_cond) |prev| blk: {
8524	const combined = cg.allocId();
8525	try cg.body.emit(gpa, .OpLogicalOr, .{
8526	.id_result_type = bool_ty_id,
8527	.id_result = combined,
8528	.operand_1 = prev,
8529	.operand_2 = eq,
8530	});
8531	break :blk combined;
8532	} else eq;
8533	}
8534
8535	for (case.ranges) |range| {
8536	const lo_tmp: Temporary = try cg.temporary(range[0]);
8537	const hi_tmp: Temporary = try cg.temporary(range[1]);
8538	const ge = try (try cg.cmp(.gte, cond_tmp, lo_tmp)).materialize(cg);
8539	const le = try (try cg.cmp(.lte, cond_tmp, hi_tmp)).materialize(cg);
8540	const in_range = cg.allocId();
8541	try cg.body.emit(gpa, .OpLogicalAnd, .{
8542	.id_result_type = bool_ty_id,
8543	.id_result = in_range,
8544	.operand_1 = ge,
8545	.operand_2 = le,
8546	});
8547	case_cond = if (case_cond) |prev| blk: {
8548	const combined = cg.allocId();
8549	try cg.body.emit(gpa, .OpLogicalOr, .{
8550	.id_result_type = bool_ty_id,
8551	.id_result = combined,
8552	.operand_1 = prev,
8553	.operand_2 = in_range,
8554	});
8555	break :blk combined;
8556	} else in_range;
8557	}
8558
8559	const case_label = case_labels.at(case.idx);
8560	const is_last = case.idx == last_range_case.?;
8561	const next_check = if (is_last) default_label else cg.allocId();
8562
8563	try cg.body.emit(gpa, .OpSelectionMerge, .{
8564	.merge_block = next_check,
8565	.selection_control = .{},
8566	});
8567
8568	try cg.body.emit(gpa, .OpBranchConditional, .{
8569	.condition = case_cond.?,
8570	.true_label = case_label,
8571	.false_label = next_check,
8572	});
8573
8574	if (!is_last) {
8575	try cg.beginSpvBlock(next_check);
8576	}
8577	}
8578	}
8579
8580	{
8581	var it = switch_br.iterateCases();
8582	while (it.next()) |case| {
8583	const label = case_labels.at(case.idx);
8584	try cg.beginSpvBlock(label);
8585
8586	const next_block = try cg.genStructuredBody(.selection, case.body);
8587	incoming_structured_blocks.appendAssumeCapacity(.{
8588	.src_label = cg.block_label,
8589	.next_block = next_block,
8590	});
8591	try cg.body.emit(gpa, .OpBranch, .{ .target_label = switch_merge });
8592	}
8593	}
8594
8595	const else_body = blk: {
8596	var it_else = switch_br.iterateCases();
8597	while (it_else.next()) |_| {}
8598	break :blk it_else.elseBody();
8599	};
8600	try cg.beginSpvBlock(default_label);
8601	if (else_body.len != 0) {
8602	const next_block = try cg.genStructuredBody(.selection, else_body);
8603	incoming_structured_blocks.appendAssumeCapacity(.{
8604	.src_label = cg.block_label,
8605	.next_block = next_block,
8606	});
8607	try cg.body.emit(gpa, .OpBranch, .{ .target_label = switch_merge });
8608	} else {
8609	try cg.body.emit(gpa, .OpUnreachable, {});
8610	}
8611
8612	try cg.beginSpvBlock(switch_merge);
8613	const next_block = try cg.structuredNextBlock(incoming_structured_blocks.items);
8614
8615	const is_dispatch = cg.allocId();
8616	const bool_ty_id = try cg.resolveType(.bool, .direct);
8617	try cg.body.emit(gpa, .OpIEqual, .{
8618	.id_result_type = bool_ty_id,
8619	.id_result = is_dispatch,
8620	.operand_1 = next_block,
8621	.operand_2 = dispatch_sentinel,
8622	});
8623
8624	const dispatch_check_merge = cg.allocId();
8625	try cg.body.emit(gpa, .OpSelectionMerge, .{
8626	.merge_block = dispatch_check_merge,
8627	.selection_control = .{},
8628	});
8629	const exit_block = cg.allocId();
8630	try cg.body.emit(gpa, .OpBranchConditional, .{
8631	.condition = is_dispatch,
8632	.true_label = dispatch_check_merge,
8633	.false_label = exit_block,
8634	});
8635
8636	try cg.beginSpvBlock(exit_block);
8637	try cg.body.emit(gpa, .OpBranch, .{ .target_label = loop_merge });
8638
8639	try cg.beginSpvBlock(dispatch_check_merge);
8640	try cg.body.emit(gpa, .OpBranch, .{ .target_label = continue_label });
8641
8642	try cg.beginSpvBlock(continue_label);
8643	try cg.body.emit(gpa, .OpBranch, .{ .target_label = header_label });
8644
8645	try cg.beginSpvBlock(loop_merge);
8646	try cg.structuredBreak(next_block);
8647	}
8648
8649	fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) !void {
8650	const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;
8651	const loop_switch = cg.loop_switches.get(br.block_inst).?;
8652	const cond_ty = cg.typeOf(br.operand);
8653	const operand = try cg.resolve(br.operand);
8654	const operand_indirect = try cg.convertToIndirect(cond_ty, operand);
8655
8656	try cg.store(cond_ty, loop_switch.cond_var, operand_indirect, .{});
8657	try cg.structuredBreak(loop_switch.continue_label);
8658	}
8659
8660	fn airUnreach(cg: *CodeGen) !void {
8661	try cg.body.emit(cg.gpa, .OpUnreachable, {});
8662	}
8663
8664	fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) !void {
8665	const zcu = cg.zcu;
8666	const dbg_stmt = cg.air.instructions.items(.data)[@backingInt(inst)].dbg_stmt;
8667	const path = zcu.navFileScope(cg.owner_nav).sub_file_path;
8668
8669	if (zcu.comp.config.root_strip) return;
8670
8671	const path_id = cg.allocId();
8672	try cg.sections.debug_strings.emit(cg.gpa, .OpString, .{
8673	.id_result = path_id,
8674	.string = path,
8675	});
8676	try cg.body.emit(cg.gpa, .OpLine, .{
8677	.file = path_id,
8678	.line = cg.base_line + dbg_stmt.line + 1,
8679	.column = dbg_stmt.column + 1,
8680	});
8681	}
8682
8683	fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8684	const zcu = cg.zcu;
8685	const block = cg.air.unwrapDbgBlock(inst);
8686	const old_base_line = cg.base_line;
8687	defer cg.base_line = old_base_line;
8688	cg.base_line = zcu.navSrcLine(zcu.funcInfo(block.func).owner_nav);
8689	return cg.lowerBlock(inst, block.body);
8690	}
8691
8692	fn airDbgVar(cg: *CodeGen, inst: Air.Inst.Index) !void {
8693	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
8694	const target_id = switch (try cg.resolvePtr(pl_op.operand)) {
8695	.tracked => return,
8696	.id => |id| id,
8697	};
8698	const name: Air.NullTerminatedString = @fromBackingInt(@intCast(pl_op.payload));
8699	try cg.debugName(target_id, name.toSlice(cg.air));
8700	}
8701
8702	fn airAssembly(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8703	const gpa = cg.gpa;
8704	const zcu = cg.zcu;
8705	const unwrapped_asm = cg.air.unwrapAsm(inst);
8706
8707	const is_volatile = unwrapped_asm.is_volatile;
8708	const outputs_len = unwrapped_asm.outputs.len;
8709
8710	if (!is_volatile and cg.liveness.isUnused(inst)) return null;
8711
8712	if (outputs_len > 1) {
8713	return cg.todo("implement inline asm with more than 1 output", .{});
8714	}
8715
8716	var ass: Assembler = .{ .cg = cg };
8717	defer ass.deinit();
8718
8719	var it = unwrapped_asm.iterateOutputs();
8720	while (it.next()) |out| {
8721	if (out.operand != .none) {
8722	return cg.todo("implement inline asm with non-returned output", .{});
8723	}
8724	}
8725
8726	it = unwrapped_asm.iterateInputs();
8727	while (it.next()) |in| {
8728	const input_ty = cg.typeOf(in.operand);
8729
8730	if (std.mem.eql(u8, in.constraint, "c")) {
8731	const val: Value = .fromInterned(in.operand.toInterned().?);
8732	const ip = &zcu.intern_pool;
8733	const target = cg.pt.zcu.getTarget();
8734	switch (input_ty.zigTypeTag(zcu)) {
8735	.int => {
8736	const bits: u64 = switch (input_ty.intInfo(zcu).signedness) {
8737	.unsigned => val.toUnsignedInt(zcu),
8738	.signed => @bitCast(val.toSignedInt(zcu)),
8739	};
8740	try ass.value_map.put(gpa, in.name, .{ .constant = bits });
8741	},
8742	.float => {
8743	const bits: u64 = switch (input_ty.floatBits(target)) {
8744	16 => @as(u16, @bitCast(val.toFloat(f16, zcu))),
8745	32 => @as(u32, @bitCast(val.toFloat(f32, zcu))),
8746	64 => @bitCast(val.toFloat(f64, zcu)),
8747	else => unreachable, // Sema rejects unsupported float widths.
8748	};
8749	try ass.value_map.put(gpa, in.name, .{ .constant = bits });
8750	},
8751	.vector => {
8752	const child_ty = input_ty.childType(zcu);
8753	const child_kind = child_ty.zigTypeTag(zcu);
8754	const child_bit_width: u16 = switch (child_kind) {
8755	.bool => 0,
8756	.int => @intCast(child_ty.intInfo(zcu).bits),
8757	.float => child_ty.floatBits(target),
8758	else => unreachable, // Sema rejects unsupported vector element types.
8759	};
8760	const vec_len: usize = @intCast(input_ty.vectorLen(zcu));
8761	const values = try gpa.alloc(u64, vec_len);
8762	errdefer gpa.free(values);
8763	for (values, 0..) |*out, i| {
8764	const elem: Value = try val.elemValue(cg.pt, i);
8765	out.* = switch (child_kind) {
8766	.bool => @intFromBool(elem.toBool()),
8767	.int => switch (child_ty.intInfo(zcu).signedness) {
8768	.unsigned => elem.toUnsignedInt(zcu),
8769	.signed => @bitCast(elem.toSignedInt(zcu)),
8770	},
8771	.float => switch (child_bit_width) {
8772	16 => @as(u16, @bitCast(elem.toFloat(f16, zcu))),
8773	32 => @as(u32, @bitCast(elem.toFloat(f32, zcu))),
8774	64 => @bitCast(elem.toFloat(f64, zcu)),
8775	else => unreachable,
8776	},
8777	else => unreachable,
8778	};
8779	}
8780	const child_ty_id = try cg.resolveType(child_ty, .direct);
8781	try ass.value_map.put(gpa, in.name, .{ .constant_composite = .{
8782	.child = child_ty_id,
8783	.child_kind = child_kind,
8784	.child_bit_width = child_bit_width,
8785	.values = values,
8786	} });
8787	},
8788	.@"enum" => switch (ip.indexToKey(val.toIntern())) {
8789	.enum_literal => |str| try ass.value_map.put(gpa, in.name, .{ .string = str.toSlice(ip) }),
8790	else => unreachable,
8791	},
8792	else => unreachable, // Sema rejects unsupported types.
8793	}
8794	} else if (std.mem.eql(u8, in.constraint, "t")) {
8795	// type
8796	if (input_ty.zigTypeTag(zcu) == .type) {
8797	// This assembly input is a type instead of a value.
8798	// That's fine for now, just make sure to resolve it as such.
8799	const ty_id = try cg.resolveType(in.operand.toType(), .direct);
8800	try ass.value_map.put(gpa, in.name, .{ .ty = ty_id });
8801	} else {
8802	const ty_id = try cg.resolveType(input_ty, .direct);
8803	try ass.value_map.put(gpa, in.name, .{ .ty = ty_id });
8804	}
8805	} else {
8806	if (input_ty.zigTypeTag(zcu) == .type) {
8807	return cg.fail("use the 't' constraint to supply types to SPIR-V inline assembly", .{});
8808	}
8809
8810	const val_id = try cg.resolve(in.operand);
8811	try ass.value_map.put(gpa, in.name, .{ .value = val_id });
8812	}
8813	}
8814	// TODO: do something with clobbers
8815	_ = unwrapped_asm.clobbers;
8816
8817	const asm_source = unwrapped_asm.source;
8818
8819	ass.assemble(asm_source) catch |err| switch (err) {
8820	error.AssembleFail => {
8821	// TODO: For now the compiler only supports a single error message per decl,
8822	// so to translate the possible multiple errors from the assembler, emit
8823	// them as notes here.
8824	// TODO: Translate proper error locations.
8825	assert(ass.errors.items.len != 0);
8826	const msg: *Zcu.ErrorMsg = msg: {
8827	const src_loc = zcu.navSrcLoc(cg.owner_nav);
8828	var msg: *Zcu.ErrorMsg = try .create(zcu.gpa, src_loc, "failed to assemble SPIR-V inline assembly", .{});
8829	errdefer msg.destroy(zcu.gpa);
8830
8831	const notes = try zcu.gpa.alloc(Zcu.ErrorMsg, ass.errors.items.len);
8832	errdefer zcu.gpa.free(notes);
8833
8834	var i: usize = 0;
8835	errdefer for (notes[0..i]) |*note| {
8836	note.deinit(zcu.gpa);
8837	};
8838
8839	while (i < ass.errors.items.len) : (i += 1) {
8840	notes[i] = try Zcu.ErrorMsg.init(zcu.gpa, src_loc, "{s}", .{ass.errors.items[i].msg});
8841	}
8842
8843	msg.notes = notes;
8844	break :msg msg;
8845	};
8846	return zcu.codegenFailMsg(cg.owner_nav, msg);
8847	},
8848	else => |others| return others,
8849	};
8850
8851	it = unwrapped_asm.iterateOutputs();
8852	while (it.next()) |out| {
8853	const result = ass.value_map.get(out.name) orelse return {
8854	return cg.fail("invalid asm output '{s}'", .{out.name});
8855	};
8856	switch (result) {
8857	.just_declared, .unresolved_forward_reference => unreachable,
8858	.ty => return cg.fail("cannot return spir-v type as value from assembly", .{}),
8859	.value => |ref| return ref,
8860	.constant, .constant_composite, .string => return cg.fail("cannot return constant from assembly", .{}),
8861	}
8862	// TODO: Multiple results
8863	// TODO: Check that the output type from assembly is the same as the type actually expected by Zig.
8864
8865	}
8866
8867	return null;
8868	}
8869
8870	fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) !?Id {
8871	_ = modifier;
8872
8873	const gpa = cg.gpa;
8874	const zcu = cg.zcu;
8875	const air_call = cg.air.unwrapCall(inst);
8876	const args = air_call.args;
8877	const callee_ty = cg.typeOf(air_call.callee);
8878	const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
8879	.@"fn" => callee_ty,
8880	else => unreachable, // rejected by Sema for SPIR-V
8881	};
8882	const fn_info = zcu.typeToFunc(zig_fn_ty).?;
8883	const return_type = fn_info.return_type;
8884
8885	const result_type_id = try cg.resolveFnReturnType(.fromInterned(return_type));
8886	const result_id = cg.allocId();
8887	const callee_id = try cg.resolve(air_call.callee);
8888
8889	const scratch_top = cg.id_scratch.items.len;
8890	defer cg.id_scratch.shrinkRetainingCapacity(scratch_top);
8891	const params = try cg.id_scratch.addManyAsSlice(gpa, args.len);
8892
8893	var n_params: usize = 0;
8894	for (args) |arg| {
8895	// Note: resolve() might emit instructions, so we need to call it
8896	// before starting to emit OpFunctionCall instructions. Hence the
8897	// temporary params buffer.
8898	const arg_ty = cg.typeOf(arg);
8899	if (!arg_ty.hasRuntimeBits(zcu)) continue;
8900
8901	if (arg_ty.zigTypeTag(zcu) == .pointer and !arg_ty.isSlice(zcu) and
8902	!arg_ty.childType(zcu).hasRuntimeBits(zcu) and
8903	cg.storageClass(arg_ty.ptrAddressSpace(zcu)) == .function)
8904	{
8905	// in logical addressing, pointer arguments to function calls
8906	// must be memory object declarations (OpVariable). for pointers to
8907	// zero-sized types, the source value may not be a variable, so just
8908	// allocate a dummy one.
8909	const child_ty_id = try cg.resolveType(arg_ty.childType(zcu), .indirect);
8910	params[n_params] = try cg.alloc(child_ty_id, null);
8911	} else {
8912	params[n_params] = try cg.resolve(arg);
8913	}
8914	n_params += 1;
8915	}
8916
8917	try cg.body.emit(gpa, .OpFunctionCall, .{
8918	.id_result_type = result_type_id,
8919	.id_result = result_id,
8920	.function = callee_id,
8921	.id_ref_3 = params[0..n_params],
8922	});
8923
8924	if (cg.liveness.isUnused(inst) or !Type.fromInterned(return_type).hasRuntimeBits(zcu)) {
8925	return null;
8926	}
8927
8928	return result_id;
8929	}
8930
8931	fn builtin3D(
8932	cg: *CodeGen,
8933	result_ty: Type,
8934	built_in: spec.BuiltIn,
8935	dimension: u32,
8936	out_of_range_value: anytype,
8937	) !Id {
8938	const gpa = cg.gpa;
8939	if (dimension >= 3) return try cg.constInt(result_ty, out_of_range_value);
8940	const u32_ty_id = try cg.intType(.unsigned, 32);
8941	const vec_ty_id = try cg.vectorType(3, u32_ty_id);
8942	const ptr_ty_id = try cg.ptrType(vec_ty_id, .input);
8943	const builtins_gop = try cg.builtins.getOrPut(gpa, .{ built_in, .input });
8944	if (!builtins_gop.found_existing) {
8945	builtins_gop.value_ptr.* = try cg.allocDecl(.global);
8946	const decl = cg.declPtr(builtins_gop.value_ptr.*);
8947	try cg.sections.globals.emit(gpa, .OpVariable, .{
8948	.id_result_type = ptr_ty_id,
8949	.id_result = decl.result_id,
8950	.storage_class = .input,
8951	});
8952	try cg.decorate(decl.result_id, .{ .built_in = .{ .built_in = built_in } });
8953	}
8954	const spv_decl_index = builtins_gop.value_ptr.*;
8955	try cg.decl_deps.append(gpa, spv_decl_index);
8956	const ptr_id = cg.declPtr(spv_decl_index).result_id;
8957	const vec_id = cg.allocId();
8958	try cg.body.emit(gpa, .OpLoad, .{
8959	.id_result_type = vec_ty_id,
8960	.id_result = vec_id,
8961	.pointer = ptr_id,
8962	});
8963	return try cg.extractVectorComponent(result_ty, vec_id, dimension);
8964	}
8965
8966	fn airWorkItemId(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8967	if (cg.liveness.isUnused(inst)) return null;
8968	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
8969	const dimension = pl_op.payload;
8970	return try cg.builtin3D(.u32, .local_invocation_id, dimension, 0);
8971	}
8972
8973	// TODO: this must be an OpConstant/OpSpec but even then the driver crashes.
8974	fn airWorkGroupSize(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8975	if (cg.liveness.isUnused(inst)) return null;
8976	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
8977	const dimension = pl_op.payload;
8978	return try cg.builtin3D(.u32, .workgroup_size, dimension, 0);
8979	}
8980
8981	fn airWorkGroupId(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
8982	if (cg.liveness.isUnused(inst)) return null;
8983	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
8984	const dimension = pl_op.payload;
8985	return try cg.builtin3D(.u32, .workgroup_id, dimension, 0);
8986	}
8987
8988	const std = @import("std");
8989	const Allocator = std.mem.Allocator;
8990	const Target = std.Target;
8991	const Signedness = std.lang.Signedness;
8992	const assert = std.debug.assert;
8993	const log = std.log.scoped(.codegen);
8994
8995	const builtin = @import("builtin");
8996	const link = @import("../../link.zig");
8997	const codegen = @import("../../codegen.zig");
8998	const Zcu = @import("../../Zcu.zig");
8999	const Type = @import("../../Type.zig");
9000	const Value = @import("../../Value.zig");
9001	const Air = @import("../../Air.zig");
9002	const InternPool = @import("../../InternPool.zig");
9003	const Section = @import("Section.zig");
9004	const Assembler = @import("Assembler.zig");
9005	const Mir = @import("Mir.zig");
9006
9007	const spec = @import("spec.zig");
9008	const Opcode = spec.Opcode;
9009	const Word = spec.Word;
9010	const Id = spec.Id;
9011	const IdRange = spec.IdRange;
9012	const StorageClass = spec.StorageClass;
9013
9014	const CodeGen = @This();