From e3ee37f983ffe655b5e9516c12a1a0d69a967e12 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Sat, 21 Mar 2026 14:26:21 +0000 Subject: [PATCH 01/15] llvm: start separating out function codegen This mostly just moves `FuncGen` into its own file, but there are also a few more cleanups, such as removing `NavGen` (it no longer served any purpose) and slightly simplifying the logic for emitting codegen errors. --- src/codegen/llvm.zig | 8812 ++-------------------------------- src/codegen/llvm/FuncGen.zig | 8145 +++++++++++++++++++++++++++++++ 2 files changed, 8473 insertions(+), 8484 deletions(-) create mode 100644 src/codegen/llvm/FuncGen.zig diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index aba2e5a12989acbf8ea48d71c8f96912153a12e9..f5fae777a2691b31839e90fcc309801be7251f49 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -1,16 +1,24 @@ const builtin = @import("builtin"); +const FuncGen = @import("llvm/FuncGen.zig"); +const buildAllocaInner = FuncGen.buildAllocaInner; +const isByRef = FuncGen.isByRef; +const firstParamSRet = FuncGen.firstParamSRet; +const lowerFnRetTy = FuncGen.lowerFnRetTy; +const iterateParamTypes = FuncGen.iterateParamTypes; +const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt; +const aarch64_c_abi = @import("aarch64/abi.zig"); + const std = @import("std"); const Io = std.Io; const assert = std.debug.assert; const Allocator = std.mem.Allocator; const log = std.log.scoped(.codegen); -const math = std.math; const DW = std.dwarf; const Builder = std.zig.llvm.Builder; const build_options = @import("build_options"); -const llvm = if (build_options.have_llvm) +const bindings = if (build_options.have_llvm) @import("llvm/bindings.zig") else @compileError("LLVM unavailable"); @@ -24,21 +32,9 @@ const Air = @import("../Air.zig"); const Value = @import("../Value.zig"); const Type = @import("../Type.zig"); const codegen = @import("../codegen.zig"); -const x86_64_abi = @import("x86_64/abi.zig"); -const wasm_c_abi = @import("wasm/abi.zig"); -const aarch64_c_abi = @import("aarch64/abi.zig"); -const arm_c_abi = @import("arm/abi.zig"); -const riscv_c_abi = @import("riscv64/abi.zig"); -const mips_c_abi = @import("mips/abi.zig"); const dev = @import("../dev.zig"); const target_util = @import("../target.zig"); -const libcFloatPrefix = target_util.libcFloatPrefix; -const libcFloatSuffix = target_util.libcFloatSuffix; -const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev; -const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev; - -const Error = error{ OutOfMemory, CodegenFail }; pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features { return comptime &.initMany(&.{ @@ -493,7 +489,7 @@ pub fn dataLayout(target: *const std.Target) []const u8 { }; } -// Avoid depending on `llvm.CodeModel` in the bitcode-only case. +// Avoid depending on `bindings.CodeModel` in the bitcode-only case. const CodeModel = enum { default, tiny, @@ -602,7 +598,7 @@ pub const Object = struct { pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn; - pub const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type); + const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type); pub fn create(arena: Allocator, comp: *Compilation) !Ptr { dev.check(.llvm_backend); @@ -1026,20 +1022,20 @@ pub const Object = struct { initializeLLVMTarget(comp.root_mod.resolved_target.result.cpu.arch); - const context: *llvm.Context = llvm.Context.create(); + const context: *bindings.Context = .create(); errdefer context.dispose(); - const bitcode_memory_buffer = llvm.MemoryBuffer.createMemoryBufferWithMemoryRange( + const bitcode_memory_buffer = bindings.MemoryBuffer.createMemoryBufferWithMemoryRange( @ptrCast(bitcode.ptr), bitcode.len * 4, "BitcodeBuffer", - llvm.Bool.False, + bindings.Bool.False, ); defer bitcode_memory_buffer.dispose(); context.enableBrokenDebugInfoCheck(); - var module: *llvm.Module = undefined; + var module: *bindings.Module = undefined; if (context.parseBitcodeInContext2(bitcode_memory_buffer, &module).toBool() or context.getBrokenDebugInfo()) { return diags.fail("Failed to parse bitcode", .{}); } @@ -1047,28 +1043,28 @@ pub const Object = struct { }; defer context.dispose(); - var target: *llvm.Target = undefined; + var target: *bindings.Target = undefined; var error_message: [*:0]const u8 = undefined; - if (llvm.Target.getFromTriple(target_triple_sentinel, &target, &error_message).toBool()) { - defer llvm.disposeMessage(error_message); + if (bindings.Target.getFromTriple(target_triple_sentinel, &target, &error_message).toBool()) { + defer bindings.disposeMessage(error_message); return diags.fail("LLVM failed to parse '{s}': {s}", .{ target_triple_sentinel, error_message }); } const optimize_mode = comp.root_mod.optimize_mode; - const opt_level: llvm.CodeGenOptLevel = if (optimize_mode == .Debug) + const opt_level: bindings.CodeGenOptLevel = if (optimize_mode == .Debug) .None else .Aggressive; - const reloc_mode: llvm.RelocMode = if (comp.root_mod.pic) + const reloc_mode: bindings.RelocMode = if (comp.root_mod.pic) .PIC else if (comp.config.link_mode == .dynamic) - llvm.RelocMode.DynamicNoPIC + bindings.RelocMode.DynamicNoPIC else .Static; - const code_model: llvm.CodeModel = switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) { + const code_model: bindings.CodeModel = switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) { .default => .Default, .tiny => .Tiny, .small => .Small, @@ -1077,12 +1073,12 @@ pub const Object = struct { .large => .Large, }; - const float_abi: llvm.TargetMachine.FloatABI = if (comp.root_mod.resolved_target.result.abi.float() == .hard) + const float_abi: bindings.TargetMachine.FloatABI = if (comp.root_mod.resolved_target.result.abi.float() == .hard) .Hard else .Soft; - var target_machine = llvm.TargetMachine.create( + var target_machine = bindings.TargetMachine.create( target, target_triple_sentinel, if (comp.root_mod.resolved_target.result.cpu.model.llvm_name) |s| s.ptr else null, @@ -1105,7 +1101,7 @@ pub const Object = struct { // Unfortunately, LLVM shits the bed when we ask for both binary and assembly. // So we call the entire pipeline multiple times if this is requested. // var error_message: [*:0]const u8 = undefined; - var lowered_options: llvm.TargetMachine.EmitOptions = .{ + var lowered_options: bindings.TargetMachine.EmitOptions = .{ .is_debug = options.is_debug, .is_small = options.is_small, .time_report_out = null, // set below to make sure it's only set for a single `emitToFile` @@ -1154,7 +1150,7 @@ pub const Object = struct { }; if (options.asm_path != null and options.bin_path != null) { if (target_machine.emitToFile(module, &error_message, &lowered_options)) { - defer llvm.disposeMessage(error_message); + defer bindings.disposeMessage(error_message); return diags.fail("LLVM failed to emit bin={s} ir={s}: {s}", .{ emit_bin_msg, post_llvm_ir_msg, error_message, }); @@ -1170,7 +1166,7 @@ pub const Object = struct { lowered_options.asm_filename = if (options.asm_path) |x| x.ptr else null; if (target_machine.emitToFile(module, &error_message, &lowered_options)) { - defer llvm.disposeMessage(error_message); + defer bindings.disposeMessage(error_message); return diags.fail("LLVM failed to emit asm={s} bin={s} ir={s} bc={s}: {s}", .{ emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg, error_message, }); @@ -1201,13 +1197,6 @@ pub const Object = struct { const fn_info = zcu.typeToFunc(fn_ty).?; const target = &owner_mod.resolved_target.result; - var ng: NavGen = .{ - .object = o, - .nav_index = func.owner_nav, - .pt = pt, - .err_msg = null, - }; - const function_index = try o.resolveLlvmFunction(pt, func.owner_nav); var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); @@ -1293,9 +1282,7 @@ pub const Object = struct { var llvm_arg_i: u32 = 0; - // This gets the LLVM values from the function and stores them in `ng.args`. - const sret = firstParamSRet(fn_info, zcu, target); - const ret_ptr: Builder.Value = if (sret) param: { + const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: { const param = wip.arg(llvm_arg_i); llvm_arg_i += 1; break :param param; @@ -1398,7 +1385,7 @@ pub const Object = struct { const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); const ptr_info = param_ty.ptrInfo(zcu); - if (math.cast(u5, it.zig_index - 1)) |i| { + if (std.math.cast(u5, it.zig_index - 1)) |i| { if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); } @@ -1543,10 +1530,12 @@ pub const Object = struct { }; var fg: FuncGen = .{ + .object = o, + .nav_index = func.owner_nav, + .pt = pt, .gpa = gpa, .air = air.*, .liveness = liveness.*.?, - .ng = &ng, .wip = wip, .is_naked = fn_info.cc == .naked, .fuzz = fuzz, @@ -1561,20 +1550,19 @@ pub const Object = struct { .sync_scope = if (owner_mod.single_threaded) .singlethread else .system, .file = file, .scope = subprogram, + .inlined_at = .none, .base_line = zcu.navSrcLine(func.owner_nav), .prev_dbg_line = 0, .prev_dbg_column = 0, .err_ret_trace = err_ret_trace, .disable_intrinsics = disable_intrinsics, + .allowzero_access = false, }; defer fg.deinit(); deinit_wip = false; fg.genBody(air.getMainBody(), .poi) catch |err| switch (err) { - error.CodegenFail => switch (zcu.codegenFailMsg(func.owner_nav, ng.err_msg.?)) { - error.CodegenFail => return, - error.OutOfMemory => |e| return e, - }, + error.CodegenFail => return, // MLUGG TODO else => |e| return e, }; @@ -1618,15 +1606,142 @@ pub const Object = struct { try o.flushTypePool(pt); } - pub fn updateNav(self: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void { - var ng: NavGen = .{ - .object = self, - .nav_index = nav_index, - .pt = pt, - .err_msg = null, + pub fn updateNav(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void { + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + + const nav = ip.getNav(nav_index); + const resolved = nav.resolved.?; + + const lib_name, const linkage, const visibility: Builder.Visibility, const is_dll_import, const init_val, const owner_nav = switch (ip.indexToKey(resolved.value)) { + else => .{ .none, .internal, .default, false, resolved.value, nav_index }, + .@"extern" => |e| .{ e.lib_name, e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import, .none, e.owner_nav }, }; - try ng.genDecl(); - try self.flushTypePool(pt); + const ty: Type = .fromInterned(nav.resolved.?.type); + + if (linkage != .internal and ip.isFunctionType(ty.toIntern())) { + const function_index = try o.resolveLlvmFunction(pt, owner_nav); + // Add parameter attributes which weren't set by `resolveLlvmFunction` + const fn_info = zcu.typeToFunc(ty).?; + var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); + defer attributes.deinit(&o.builder); + var it = iterateParamTypes(o, pt, fn_info); + if (firstParamSRet(fn_info, zcu, zcu.getTarget())) it.llvm_index += 1; + if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) it.llvm_index += 1; + while (try it.next()) |lowering| switch (lowering) { + .byval => { + const param_index = it.zig_index - 1; + const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); + if (!isByRef(param_ty, zcu)) { + try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); + } + }, + .byref => { + const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param_llvm_ty = try o.lowerType(pt, param_ty); + const alignment = param_ty.abiAlignment(zcu); + try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty); + }, + .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), + // No attributes needed for these. + .no_bits, + .abi_sized_int, + .multiple_llvm_types, + .float_array, + .i32_array, + .i64_array, + => continue, + + .slice => unreachable, // extern functions do not support slice types. + }; + function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); + } else { + const variable_index = try o.resolveGlobalNav(pt, nav_index); + variable_index.setAlignment(zcu.navAlignment(nav_index).toLlvm(), &o.builder); + if (resolved.@"linksection".toSlice(ip)) |section| + variable_index.setSection(try o.builder.string(section), &o.builder); + if (resolved.@"const") variable_index.setMutability(.constant, &o.builder); + try variable_index.setInitializer(switch (init_val) { + .none => .no_init, + else => try o.lowerValue(pt, init_val), + }, &o.builder); + variable_index.setVisibility(visibility, &o.builder); + + const file_scope = zcu.navFileScopeIndex(nav_index); + const mod = zcu.fileByIndex(file_scope).mod.?; + if (resolved.@"threadlocal" and !mod.single_threaded) + variable_index.setThreadLocal(.generaldynamic, &o.builder); + + const line_number = zcu.navSrcLine(nav_index) + 1; + + if (!mod.strip) { + const debug_file = try o.getDebugFile(pt, file_scope); + + const debug_global_var = try o.builder.debugGlobalVar( + try o.builder.metadataString(nav.name.toSlice(ip)), // Name + try o.builder.metadataStringFromStrtabString(variable_index.name(&o.builder)), // Linkage name + debug_file, // File + debug_file, // Scope + line_number, + try o.getDebugType(pt, ty), + variable_index, + .{ .local = linkage == .internal }, + ); + + const debug_expression = try o.builder.debugExpression(&.{}); + + const debug_global_var_expression = try o.builder.debugGlobalVarExpression( + debug_global_var, + debug_expression, + ); + + variable_index.setGlobalVariableExpression(debug_global_var_expression, &o.builder); + try o.debug_globals.append(o.gpa, debug_global_var_expression); + } + } + + switch (linkage) { + .internal => {}, + .strong, .weak => { + const global_index = o.nav_map.get(nav_index).?; + + const decl_name = decl_name: { + if (zcu.getTarget().cpu.arch.isWasm() and ty.zigTypeTag(zcu) == .@"fn") { + if (lib_name.toSlice(ip)) |lib_name_slice| { + if (!std.mem.eql(u8, lib_name_slice, "c")) { + break :decl_name try o.builder.strtabStringFmt("{f}|{s}", .{ nav.name.fmt(ip), lib_name_slice }); + } + } + } + break :decl_name try o.builder.strtabString(nav.name.toSlice(ip)); + }; + + if (o.builder.getGlobal(decl_name)) |other_global| { + if (other_global != global_index) { + // Another global already has this name; just use it in place of this global. + try global_index.replace(other_global, &o.builder); + return; + } + } + + try global_index.rename(decl_name, &o.builder); + global_index.setUnnamedAddr(.default, &o.builder); + if (is_dll_import) { + global_index.setDllStorageClass(.dllimport, &o.builder); + } else if (zcu.comp.config.dll_export_fns) { + global_index.setDllStorageClass(.default, &o.builder); + } + + global_index.setLinkage(switch (linkage) { + .internal => unreachable, + .strong => .external, + .weak => .extern_weak, + .link_once => unreachable, + }, &o.builder); + global_index.setVisibility(visibility, &o.builder); + }, + .link_once => unreachable, + } } fn flushTypePool(o: *Object, pt: Zcu.PerThread) Allocator.Error!void { @@ -1904,7 +2019,7 @@ pub const Object = struct { } } - fn getDebugFile(o: *Object, pt: Zcu.PerThread, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata { + pub fn getDebugFile(o: *Object, pt: Zcu.PerThread, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata { const gpa = o.gpa; const gop = try o.debug_file_map.getOrPut(gpa, file_index); errdefer assert(o.debug_file_map.remove(file_index)); @@ -1920,7 +2035,7 @@ pub const Object = struct { return gop.value_ptr.*; } - fn getDebugType(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Metadata { + pub fn getDebugType(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Metadata { assert(!o.builder.strip); const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern()); return o.debug_types.items[@intFromEnum(index)]; @@ -2212,13 +2327,14 @@ pub const Object = struct { defer debug_param_types.deinit(gpa); // Return type goes first. - const sret = firstParamSRet(fn_info, zcu, target); - const ret_ty: Type = if (sret) .void else .fromInterned(fn_info.return_type); - debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ret_ty)); - - if (sret) { - const ptr_ty = try pt.singleMutPtrType(Type.fromInterned(fn_info.return_type)); + if (firstParamSRet(fn_info, zcu, target)) { + // Actual return type is void, then first arg is the sret pointer. + const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type)); + debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void)); debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ptr_ty)); + } else { + const ret_ty: Type = .fromInterned(fn_info.return_type); + debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ret_ty)); } if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) { @@ -2661,7 +2777,7 @@ pub const Object = struct { /// If the llvm function does not exist, create it. /// Note that this can be called before the function's semantic analysis has /// completed, so if any attributes rely on that, they must be done in updateFunc, not here. - fn resolveLlvmFunction( + pub fn resolveLlvmFunction( o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index, @@ -2677,7 +2793,6 @@ pub const Object = struct { const fn_info = zcu.typeToFunc(ty).?; const target = &owner_mod.resolved_target.result; - const sret = firstParamSRet(fn_info, zcu, target); const is_extern, const lib_name = if (nav.getExtern(ip)) |@"extern"| .{ true, @"extern".lib_name } @@ -2712,7 +2827,7 @@ pub const Object = struct { } var llvm_arg_i: u32 = 0; - if (sret) { + if (firstParamSRet(fn_info, zcu, target)) { // Sret pointers must not be address 0 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); @@ -2988,11 +3103,11 @@ pub const Object = struct { return variable_index; } - fn errorIntType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Type { + pub fn errorIntType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Type { return o.builder.intType(pt.zcu.errorSetBits()); } - fn lowerType(o: *Object, pt: Zcu.PerThread, t: Type) Allocator.Error!Builder.Type { + pub fn lowerType(o: *Object, pt: Zcu.PerThread, t: Type) Allocator.Error!Builder.Type { const zcu = pt.zcu; const target = zcu.getTarget(); const ip = &zcu.intern_pool; @@ -3408,7 +3523,7 @@ pub const Object = struct { return gop.value_ptr.*; }, .enum_type => try o.lowerType(pt, t.intTagType(zcu)), - .func_type => |func_type| try o.lowerTypeFn(pt, func_type), + .func_type => |func_type| try o.lowerFnType(pt, func_type), .error_set_type, .inferred_error_set_type => try o.errorIntType(pt), // values, not types .undef, @@ -3434,7 +3549,7 @@ pub const Object = struct { }; } - fn lowerTypeFn(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { + fn lowerFnType(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { const zcu = pt.zcu; const ip = &zcu.intern_pool; const target = zcu.getTarget(); @@ -3500,7 +3615,7 @@ pub const Object = struct { ); } - fn lowerValue(o: *Object, pt: Zcu.PerThread, arg_val: InternPool.Index) Allocator.Error!Builder.Constant { + pub fn lowerValue(o: *Object, pt: Zcu.PerThread, arg_val: InternPool.Index) Allocator.Error!Builder.Constant { const zcu = pt.zcu; const ip = &zcu.intern_pool; const target = zcu.getTarget(); @@ -4076,7 +4191,7 @@ pub const Object = struct { /// This logic is very similar to `lowerNavRefValue` but for anonymous declarations. /// Maybe the logic could be unified. - fn lowerUavRef( + pub fn lowerUavRef( o: *Object, pt: Zcu.PerThread, uav: InternPool.Key.Ptr.BaseAddr.Uav, @@ -4113,7 +4228,7 @@ pub const Object = struct { return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty)); } - fn lowerNavRefValue(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant { + pub fn lowerNavRefValue(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant { const zcu = pt.zcu; const ip = &zcu.intern_pool; @@ -4139,7 +4254,7 @@ pub const Object = struct { return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty)); } - fn lowerPtrToVoid(o: *Object, pt: Zcu.PerThread, ptr_ty: Type) Allocator.Error!Builder.Constant { + pub fn lowerPtrToVoid(o: *Object, pt: Zcu.PerThread, ptr_ty: Type) Allocator.Error!Builder.Constant { const zcu = pt.zcu; // Even though we are pointing at something which has zero bits (e.g. `void`), // Pointers are defined to have bits. So we must return something here. @@ -4163,30 +4278,7 @@ pub const Object = struct { return o.builder.castConst(.inttoptr, try o.builder.intConst(llvm_usize, int), llvm_ptr_ty); } - /// If the operand type of an atomic operation is not byte sized we need to - /// widen it before using it and then truncate the result. - /// RMW exchange of floating-point values is bitcasted to same-sized integer - /// types to work around a LLVM deficiency when targeting ARM/AArch64. - fn getAtomicAbiType(o: *Object, pt: Zcu.PerThread, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type { - const zcu = pt.zcu; - switch (ty.zigTypeTag(zcu)) { - .int, .@"enum", .@"struct", .@"union" => {}, - .float => { - if (!is_rmw_xchg) return .none; - return o.builder.intType(@intCast(ty.abiSize(zcu) * 8)); - }, - .bool => return .i8, - else => return .none, - } - const bit_count = ty.bitSize(zcu); - if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) { - return o.builder.intType(@intCast(ty.abiSize(zcu) * 8)); - } else { - return .none; - } - } - - fn addByValParamAttrs( + pub fn addByValParamAttrs( o: *Object, pt: Zcu.PerThread, attributes: *Builder.FunctionAttributes.Wip, @@ -4198,7 +4290,7 @@ pub const Object = struct { const zcu = pt.zcu; if (param_ty.isPtrAtRuntime(zcu)) { const ptr_info = param_ty.ptrInfo(zcu); - if (math.cast(u5, param_index)) |i| { + if (std.math.cast(u5, param_index)) |i| { if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); } @@ -4232,7 +4324,7 @@ pub const Object = struct { }; } - fn addByRefParamAttrs( + pub fn addByRefParamAttrs( o: *Object, attributes: *Builder.FunctionAttributes.Wip, llvm_arg_i: u32, @@ -4246,14 +4338,15 @@ pub const Object = struct { if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder); } - fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint { + pub fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint { return o.struct_field_map.get(.{ .struct_ty = struct_ty.toIntern(), .field_index = @intCast(field_index), }); } - fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index { + /// MLUGG TODO: this is super dumb + pub fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index { const name = try o.builder.strtabString(lt_errors_fn_name); if (o.builder.getGlobal(name)) |llvm_fn| return llvm_fn.ptrConst(&o.builder).kind.function; @@ -4275,7 +4368,7 @@ pub const Object = struct { return function_index; } - fn getEnumTagNameFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index { + pub fn getEnumTagNameFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index { const zcu = pt.zcu; const ip = &zcu.intern_pool; const enum_type = ip.loadEnumType(enum_ty.toIntern()); @@ -4353,12 +4446,12 @@ pub const Object = struct { return function_index; } - fn lazyAbiAlignment(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Alignment.Lazy { + pub fn lazyAbiAlignment(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Alignment.Lazy { const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern()); return o.lazy_abi_aligns.items[@intFromEnum(index)]; } - fn updateIsNamedEnumValueFunction( + pub fn updateIsNamedEnumValueFunction( o: *Object, pt: Zcu.PerThread, enum_ty: Type, @@ -4411,7150 +4504,6 @@ pub const Object = struct { } }; -pub const NavGen = struct { - object: *Object, - nav_index: InternPool.Nav.Index, - pt: Zcu.PerThread, - err_msg: ?*Zcu.ErrorMsg, - - fn ownerModule(ng: NavGen) *Package.Module { - return ng.pt.zcu.navFileScope(ng.nav_index).mod.?; - } - - fn todo(ng: *NavGen, comptime format: []const u8, args: anytype) Error { - @branchHint(.cold); - assert(ng.err_msg == null); - const o = ng.object; - const gpa = o.gpa; - const src_loc = ng.pt.zcu.navSrcLoc(ng.nav_index); - ng.err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "TODO (LLVM): " ++ format, args); - return error.CodegenFail; - } - - fn genDecl(ng: *NavGen) !void { - const o = ng.object; - const pt = ng.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const nav_index = ng.nav_index; - const nav = ip.getNav(nav_index); - const resolved = nav.resolved.?; - - const lib_name, const linkage, const visibility: Builder.Visibility, const is_dll_import, const init_val, const owner_nav = switch (ip.indexToKey(resolved.value)) { - else => .{ .none, .internal, .default, false, resolved.value, nav_index }, - .@"extern" => |e| .{ e.lib_name, e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import, .none, e.owner_nav }, - }; - const ty: Type = .fromInterned(nav.resolved.?.type); - - if (linkage != .internal and ip.isFunctionType(ty.toIntern())) { - const function_index = try o.resolveLlvmFunction(pt, owner_nav); - // Add parameter attributes which weren't set by `resolveLlvmFunction` - const fn_info = zcu.typeToFunc(ty).?; - var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); - defer attributes.deinit(&o.builder); - var it = iterateParamTypes(o, pt, fn_info); - if (firstParamSRet(fn_info, zcu, zcu.getTarget())) it.llvm_index += 1; - if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) it.llvm_index += 1; - while (try it.next()) |lowering| switch (lowering) { - .byval => { - const param_index = it.zig_index - 1; - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); - if (!isByRef(param_ty, zcu)) { - try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); - } - }, - .byref => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(pt, param_ty); - const alignment = param_ty.abiAlignment(zcu); - try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty); - }, - .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), - // No attributes needed for these. - .no_bits, - .abi_sized_int, - .multiple_llvm_types, - .float_array, - .i32_array, - .i64_array, - => continue, - - .slice => unreachable, // extern functions do not support slice types. - }; - function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); - } else { - const variable_index = try o.resolveGlobalNav(pt, nav_index); - variable_index.setAlignment(zcu.navAlignment(nav_index).toLlvm(), &o.builder); - if (resolved.@"linksection".toSlice(ip)) |section| - variable_index.setSection(try o.builder.string(section), &o.builder); - if (resolved.@"const") variable_index.setMutability(.constant, &o.builder); - try variable_index.setInitializer(switch (init_val) { - .none => .no_init, - else => try o.lowerValue(pt, init_val), - }, &o.builder); - variable_index.setVisibility(visibility, &o.builder); - - const file_scope = zcu.navFileScopeIndex(nav_index); - const mod = zcu.fileByIndex(file_scope).mod.?; - if (resolved.@"threadlocal" and !mod.single_threaded) - variable_index.setThreadLocal(.generaldynamic, &o.builder); - - const line_number = zcu.navSrcLine(nav_index) + 1; - - if (!mod.strip) { - const debug_file = try o.getDebugFile(pt, file_scope); - - const debug_global_var = try o.builder.debugGlobalVar( - try o.builder.metadataString(nav.name.toSlice(ip)), // Name - try o.builder.metadataStringFromStrtabString(variable_index.name(&o.builder)), // Linkage name - debug_file, // File - debug_file, // Scope - line_number, - try o.getDebugType(pt, ty), - variable_index, - .{ .local = linkage == .internal }, - ); - - const debug_expression = try o.builder.debugExpression(&.{}); - - const debug_global_var_expression = try o.builder.debugGlobalVarExpression( - debug_global_var, - debug_expression, - ); - - variable_index.setGlobalVariableExpression(debug_global_var_expression, &o.builder); - try o.debug_globals.append(o.gpa, debug_global_var_expression); - } - } - - switch (linkage) { - .internal => {}, - .strong, .weak => { - const global_index = o.nav_map.get(nav_index).?; - - const decl_name = decl_name: { - if (zcu.getTarget().cpu.arch.isWasm() and ty.zigTypeTag(zcu) == .@"fn") { - if (lib_name.toSlice(ip)) |lib_name_slice| { - if (!std.mem.eql(u8, lib_name_slice, "c")) { - break :decl_name try o.builder.strtabStringFmt("{f}|{s}", .{ nav.name.fmt(ip), lib_name_slice }); - } - } - } - break :decl_name try o.builder.strtabString(nav.name.toSlice(ip)); - }; - - if (o.builder.getGlobal(decl_name)) |other_global| { - if (other_global != global_index) { - // Another global already has this name; just use it in place of this global. - try global_index.replace(other_global, &o.builder); - return; - } - } - - try global_index.rename(decl_name, &o.builder); - global_index.setUnnamedAddr(.default, &o.builder); - if (is_dll_import) { - global_index.setDllStorageClass(.dllimport, &o.builder); - } else if (zcu.comp.config.dll_export_fns) { - global_index.setDllStorageClass(.default, &o.builder); - } - - global_index.setLinkage(switch (linkage) { - .internal => unreachable, - .strong => .external, - .weak => .extern_weak, - .link_once => unreachable, - }, &o.builder); - global_index.setVisibility(visibility, &o.builder); - }, - .link_once => unreachable, - } - } -}; - -pub const FuncGen = struct { - gpa: Allocator, - ng: *NavGen, - air: Air, - liveness: Air.Liveness, - wip: Builder.WipFunction, - is_naked: bool, - fuzz: ?Fuzz, - - file: Builder.Metadata, - scope: Builder.Metadata, - - inlined_at: Builder.Metadata.Optional = .none, - - base_line: u32, - prev_dbg_line: c_uint, - prev_dbg_column: c_uint, - - /// This stores the LLVM values used in a function, such that they can be referred to - /// in other instructions. This table is cleared before every function is generated. - func_inst_table: std.AutoHashMapUnmanaged(Air.Inst.Ref, Builder.Value), - - /// If the return type is sret, this is the result pointer. Otherwise null. - /// Note that this can disagree with isByRef for the return type in the case - /// of C ABI functions. - ret_ptr: Builder.Value, - /// Any function that needs to perform Valgrind client requests needs an array alloca - /// instruction, however a maximum of one per function is needed. - valgrind_client_request_array: Builder.Value = .none, - /// These fields are used to refer to the LLVM value of the function parameters - /// in an Arg instruction. - /// This list may be shorter than the list according to the zig type system; - /// it omits 0-bit types. If the function uses sret as the first parameter, - /// this slice does not include it. - args: []const Builder.Value, - arg_index: u32, - arg_inline_index: u32, - - err_ret_trace: Builder.Value = .none, - - /// This data structure is used to implement breaking to blocks. - blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { - parent_bb: Builder.Function.Block.Index, - breaks: *BreakList, - }), - - /// Maps `loop` instructions to the bb to branch to to repeat the loop. - loops: std.AutoHashMapUnmanaged(Air.Inst.Index, Builder.Function.Block.Index), - - /// Maps `loop_switch_br` instructions to the information required to lower - /// dispatches (`switch_dispatch` instructions). - switch_dispatch_info: std.AutoHashMapUnmanaged(Air.Inst.Index, SwitchDispatchInfo), - - sync_scope: Builder.SyncScope, - - disable_intrinsics: bool, - - /// Have we seen loads or stores involving `allowzero` pointers? - allowzero_access: bool = false, - - fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void { - // LLVM already considers null pointers to be valid in non-generic address spaces, so avoid - // pessimizing optimization for functions with accesses to such pointers. - if (info.flags.address_space == .generic and info.flags.is_allowzero) self.allowzero_access = true; - } - - const Fuzz = struct { - counters_variable: Builder.Variable.Index, - pcs: std.ArrayList(Builder.Constant), - - fn deinit(f: *Fuzz, gpa: Allocator) void { - f.pcs.deinit(gpa); - f.* = undefined; - } - }; - - const SwitchDispatchInfo = struct { - /// These are the blocks corresponding to each switch case. - /// The final element corresponds to the `else` case. - /// Slices allocated into `gpa`. - case_blocks: []Builder.Function.Block.Index, - /// This is `.none` if `jmp_table` is set, since we won't use a `switch` instruction to dispatch. - switch_weights: Builder.Function.Instruction.BrCond.Weights, - /// If not `null`, we have manually constructed a jump table to reach the desired block. - /// `table` can be used if the value is between `min` and `max` inclusive. - /// We perform this lowering manually to avoid some questionable behavior from LLVM. - /// See `airSwitchBr` for details. - jmp_table: ?JmpTable, - - const JmpTable = struct { - min: Builder.Constant, - max: Builder.Constant, - in_bounds_hint: enum { none, unpredictable, likely, unlikely }, - /// Pointer to the jump table itself, to be used with `indirectbr`. - /// The index into the jump table is the dispatch condition minus `min`. - /// The table values are `blockaddress` constants corresponding to blocks in `case_blocks`. - table: Builder.Constant, - /// `true` if `table` conatins a reference to the `else` block. - /// In this case, the `indirectbr` must include the `else` block in its target list. - table_includes_else: bool, - }; - }; - - const BreakList = union { - list: std.MultiArrayList(struct { - bb: Builder.Function.Block.Index, - val: Builder.Value, - }), - len: usize, - }; - - fn deinit(self: *FuncGen) void { - const gpa = self.gpa; - if (self.fuzz) |*f| f.deinit(self.gpa); - self.wip.deinit(); - self.func_inst_table.deinit(gpa); - self.blocks.deinit(gpa); - self.loops.deinit(gpa); - var it = self.switch_dispatch_info.valueIterator(); - while (it.next()) |info| { - self.gpa.free(info.case_blocks); - } - self.switch_dispatch_info.deinit(gpa); - } - - fn todo(self: *FuncGen, comptime format: []const u8, args: anytype) Error { - @branchHint(.cold); - return self.ng.todo(format, args); - } - - fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value { - const gpa = self.gpa; - const gop = try self.func_inst_table.getOrPut(gpa, inst); - if (gop.found_existing) return gop.value_ptr.*; - - const llvm_val = try self.resolveValue((try self.air.value(inst, self.ng.pt)).?); - gop.value_ptr.* = llvm_val.toValue(); - return llvm_val.toValue(); - } - - fn resolveValue(self: *FuncGen, val: Value) Error!Builder.Constant { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty = val.typeOf(zcu); - const llvm_val = try o.lowerValue(pt, val.toIntern()); - if (!isByRef(ty, zcu)) return llvm_val; - - // We have an LLVM value but we need to create a global constant and - // set the value as its initializer, and then return a pointer to the global. - const target = zcu.getTarget(); - const variable_index = try o.builder.addVariable( - .empty, - llvm_val.typeOf(&o.builder), - toLlvmGlobalAddressSpace(.generic, target), - ); - try variable_index.setInitializer(llvm_val, &o.builder); - variable_index.setLinkage(.private, &o.builder); - variable_index.setMutability(.constant, &o.builder); - variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); - variable_index.setAlignment(ty.abiAlignment(zcu).toLlvm(), &o.builder); - return o.builder.convConst( - variable_index.toConst(&o.builder), - try o.builder.ptrType(toLlvmAddressSpace(.generic, target)), - ); - } - - fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) Error!void { - const o = self.ng.object; - const zcu = self.ng.pt.zcu; - const ip = &zcu.intern_pool; - const air_tags = self.air.instructions.items(.tag); - switch (coverage_point) { - .none => {}, - .poi => if (self.fuzz) |*fuzz| { - const poi_index = fuzz.pcs.items.len; - const base_ptr = fuzz.counters_variable.toValue(&o.builder); - const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{ - try o.builder.intValue(.i32, poi_index), - }, ""); - const one = try o.builder.intValue(.i8, 1); - _ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, ""); - - // LLVM does not allow blockaddress on the entry block. - const pc = if (self.wip.cursor.block == .entry) - self.wip.function.toConst(&o.builder) - else - try o.builder.blockAddrConst(self.wip.function, self.wip.cursor.block); - const gpa = self.gpa; - try fuzz.pcs.append(gpa, pc); - }, - } - for (body, 0..) |inst, i| { - if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue; - - const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) { - // zig fmt: off - - // No "scalarize" legalizations are enabled, so these instructions never appear. - .legalize_vec_elem_val => unreachable, - .legalize_vec_store_elem => unreachable, - // No soft float legalizations are enabled. - .legalize_compiler_rt_call => unreachable, - - .add => try self.airAdd(inst, .normal), - .add_optimized => try self.airAdd(inst, .fast), - .add_wrap => try self.airAddWrap(inst), - .add_sat => try self.airAddSat(inst), - - .sub => try self.airSub(inst, .normal), - .sub_optimized => try self.airSub(inst, .fast), - .sub_wrap => try self.airSubWrap(inst), - .sub_sat => try self.airSubSat(inst), - - .mul => try self.airMul(inst, .normal), - .mul_optimized => try self.airMul(inst, .fast), - .mul_wrap => try self.airMulWrap(inst), - .mul_sat => try self.airMulSat(inst), - - .add_safe => try self.airSafeArithmetic(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"), - .sub_safe => try self.airSafeArithmetic(inst, .@"ssub.with.overflow", .@"usub.with.overflow"), - .mul_safe => try self.airSafeArithmetic(inst, .@"smul.with.overflow", .@"umul.with.overflow"), - - .div_float => try self.airDivFloat(inst, .normal), - .div_trunc => try self.airDivTrunc(inst, .normal), - .div_floor => try self.airDivFloor(inst, .normal), - .div_exact => try self.airDivExact(inst, .normal), - .rem => try self.airRem(inst, .normal), - .mod => try self.airMod(inst, .normal), - .abs => try self.airAbs(inst), - .ptr_add => try self.airPtrAdd(inst), - .ptr_sub => try self.airPtrSub(inst), - .shl => try self.airShl(inst), - .shl_sat => try self.airShlSat(inst), - .shl_exact => try self.airShlExact(inst), - .min => try self.airMin(inst), - .max => try self.airMax(inst), - .slice => try self.airSlice(inst), - .mul_add => try self.airMulAdd(inst), - - .div_float_optimized => try self.airDivFloat(inst, .fast), - .div_trunc_optimized => try self.airDivTrunc(inst, .fast), - .div_floor_optimized => try self.airDivFloor(inst, .fast), - .div_exact_optimized => try self.airDivExact(inst, .fast), - .rem_optimized => try self.airRem(inst, .fast), - .mod_optimized => try self.airMod(inst, .fast), - - .add_with_overflow => try self.airOverflow(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"), - .sub_with_overflow => try self.airOverflow(inst, .@"ssub.with.overflow", .@"usub.with.overflow"), - .mul_with_overflow => try self.airOverflow(inst, .@"smul.with.overflow", .@"umul.with.overflow"), - .shl_with_overflow => try self.airShlWithOverflow(inst), - - .bit_and, .bool_and => try self.airAnd(inst), - .bit_or, .bool_or => try self.airOr(inst), - .xor => try self.airXor(inst), - .shr => try self.airShr(inst, false), - .shr_exact => try self.airShr(inst, true), - - .sqrt => try self.airUnaryOp(inst, .sqrt), - .sin => try self.airUnaryOp(inst, .sin), - .cos => try self.airUnaryOp(inst, .cos), - .tan => try self.airUnaryOp(inst, .tan), - .exp => try self.airUnaryOp(inst, .exp), - .exp2 => try self.airUnaryOp(inst, .exp2), - .log => try self.airUnaryOp(inst, .log), - .log2 => try self.airUnaryOp(inst, .log2), - .log10 => try self.airUnaryOp(inst, .log10), - .floor => try self.airUnaryOp(inst, .floor), - .ceil => try self.airUnaryOp(inst, .ceil), - .round => try self.airUnaryOp(inst, .round), - .trunc_float => try self.airUnaryOp(inst, .trunc), - - .neg => try self.airNeg(inst, .normal), - .neg_optimized => try self.airNeg(inst, .fast), - - .cmp_eq => try self.airCmp(inst, .eq, .normal), - .cmp_gt => try self.airCmp(inst, .gt, .normal), - .cmp_gte => try self.airCmp(inst, .gte, .normal), - .cmp_lt => try self.airCmp(inst, .lt, .normal), - .cmp_lte => try self.airCmp(inst, .lte, .normal), - .cmp_neq => try self.airCmp(inst, .neq, .normal), - - .cmp_eq_optimized => try self.airCmp(inst, .eq, .fast), - .cmp_gt_optimized => try self.airCmp(inst, .gt, .fast), - .cmp_gte_optimized => try self.airCmp(inst, .gte, .fast), - .cmp_lt_optimized => try self.airCmp(inst, .lt, .fast), - .cmp_lte_optimized => try self.airCmp(inst, .lte, .fast), - .cmp_neq_optimized => try self.airCmp(inst, .neq, .fast), - - .cmp_vector => try self.airCmpVector(inst, .normal), - .cmp_vector_optimized => try self.airCmpVector(inst, .fast), - .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), - - .is_non_null => try self.airIsNonNull(inst, false, .ne), - .is_non_null_ptr => try self.airIsNonNull(inst, true , .ne), - .is_null => try self.airIsNonNull(inst, false, .eq), - .is_null_ptr => try self.airIsNonNull(inst, true , .eq), - - .is_non_err => try self.airIsErr(inst, .eq, false), - .is_non_err_ptr => try self.airIsErr(inst, .eq, true), - .is_err => try self.airIsErr(inst, .ne, false), - .is_err_ptr => try self.airIsErr(inst, .ne, true), - - .alloc => try self.airAlloc(inst), - .ret_ptr => try self.airRetPtr(inst), - .arg => try self.airArg(inst), - .bitcast => try self.airBitCast(inst), - .breakpoint => try self.airBreakpoint(inst), - .ret_addr => try self.airRetAddr(inst), - .frame_addr => try self.airFrameAddress(inst), - .@"try" => try self.airTry(inst, false), - .try_cold => try self.airTry(inst, true), - .try_ptr => try self.airTryPtr(inst, false), - .try_ptr_cold => try self.airTryPtr(inst, true), - .intcast => try self.airIntCast(inst, false), - .intcast_safe => try self.airIntCast(inst, true), - .trunc => try self.airTrunc(inst), - .fptrunc => try self.airFptrunc(inst), - .fpext => try self.airFpext(inst), - .load => try self.airLoad(inst), - .not => try self.airNot(inst), - .store => try self.airStore(inst, false), - .store_safe => try self.airStore(inst, true), - .assembly => try self.airAssembly(inst), - .slice_ptr => try self.airSliceField(inst, 0), - .slice_len => try self.airSliceField(inst, 1), - - .ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0), - .ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1), - - .int_from_float => try self.airIntFromFloat(inst, .normal), - .int_from_float_optimized => try self.airIntFromFloat(inst, .fast), - .int_from_float_safe => unreachable, // handled by `legalizeFeatures` - .int_from_float_optimized_safe => unreachable, // handled by `legalizeFeatures` - - .array_to_slice => try self.airArrayToSlice(inst), - .float_from_int => try self.airFloatFromInt(inst), - .cmpxchg_weak => try self.airCmpxchg(inst, .weak), - .cmpxchg_strong => try self.airCmpxchg(inst, .strong), - .atomic_rmw => try self.airAtomicRmw(inst), - .atomic_load => try self.airAtomicLoad(inst), - .memset => try self.airMemset(inst, false), - .memset_safe => try self.airMemset(inst, true), - .memcpy => try self.airMemcpy(inst), - .memmove => try self.airMemmove(inst), - .set_union_tag => try self.airSetUnionTag(inst), - .get_union_tag => try self.airGetUnionTag(inst), - .clz => try self.airClzCtz(inst, .ctlz), - .ctz => try self.airClzCtz(inst, .cttz), - .popcount => try self.airBitOp(inst, .ctpop), - .byte_swap => try self.airByteSwap(inst), - .bit_reverse => try self.airBitOp(inst, .bitreverse), - .tag_name => try self.airTagName(inst), - .error_name => try self.airErrorName(inst), - .splat => try self.airSplat(inst), - .select => try self.airSelect(inst), - .shuffle_one => try self.airShuffleOne(inst), - .shuffle_two => try self.airShuffleTwo(inst), - .aggregate_init => try self.airAggregateInit(inst), - .union_init => try self.airUnionInit(inst), - .prefetch => try self.airPrefetch(inst), - .addrspace_cast => try self.airAddrSpaceCast(inst), - - .is_named_enum_value => try self.airIsNamedEnumValue(inst), - .error_set_has_value => try self.airErrorSetHasValue(inst), - - .reduce => try self.airReduce(inst, .normal), - .reduce_optimized => try self.airReduce(inst, .fast), - - .atomic_store_unordered => try self.airAtomicStore(inst, .unordered), - .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic), - .atomic_store_release => try self.airAtomicStore(inst, .release), - .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst), - - .struct_field_ptr => try self.airStructFieldPtr(inst), - .struct_field_val => try self.airStructFieldVal(inst), - - .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0), - .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1), - .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2), - .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3), - - .field_parent_ptr => try self.airFieldParentPtr(inst), - - .array_elem_val => try self.airArrayElemVal(inst), - .slice_elem_val => try self.airSliceElemVal(inst), - .slice_elem_ptr => try self.airSliceElemPtr(inst), - .ptr_elem_val => try self.airPtrElemVal(inst), - .ptr_elem_ptr => try self.airPtrElemPtr(inst), - - .optional_payload => try self.airOptionalPayload(inst), - .optional_payload_ptr => try self.airOptionalPayloadPtr(inst), - .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst), - - .unwrap_errunion_payload => try self.airErrUnionPayload(inst, false), - .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true), - .unwrap_errunion_err => try self.airErrUnionErr(inst, false), - .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true), - .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst), - .err_return_trace => try self.airErrReturnTrace(inst), - .set_err_return_trace => try self.airSetErrReturnTrace(inst), - .save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst), - - .wrap_optional => try self.airWrapOptional(body[i..]), - .wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]), - .wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]), - - .wasm_memory_size => try self.airWasmMemorySize(inst), - .wasm_memory_grow => try self.airWasmMemoryGrow(inst), - - .runtime_nav_ptr => try self.airRuntimeNavPtr(inst), - - .inferred_alloc, .inferred_alloc_comptime => unreachable, - - .dbg_stmt => try self.airDbgStmt(inst), - .dbg_empty_stmt => try self.airDbgEmptyStmt(inst), - .dbg_var_ptr => try self.airDbgVarPtr(inst), - .dbg_var_val => try self.airDbgVarVal(inst, false), - .dbg_arg_inline => try self.airDbgVarVal(inst, true), - - .c_va_arg => try self.airCVaArg(inst), - .c_va_copy => try self.airCVaCopy(inst), - .c_va_end => try self.airCVaEnd(inst), - .c_va_start => try self.airCVaStart(inst), - - .work_item_id => try self.airWorkItemId(inst), - .work_group_size => try self.airWorkGroupSize(inst), - .work_group_id => try self.airWorkGroupId(inst), - - // Instructions that are known to always be `noreturn` based on their tag. - .br => return self.airBr(inst), - .repeat => return self.airRepeat(inst), - .switch_dispatch => return self.airSwitchDispatch(inst), - .cond_br => return self.airCondBr(inst), - .switch_br => return self.airSwitchBr(inst, false), - .loop_switch_br => return self.airSwitchBr(inst, true), - .loop => return self.airLoop(inst), - .ret => return self.airRet(inst, false), - .ret_safe => return self.airRet(inst, true), - .ret_load => return self.airRetLoad(inst), - .trap => return self.airTrap(inst), - .unreach => return self.airUnreach(inst), - - // Instructions which may be `noreturn`. - .block => res: { - const res = try self.airBlock(inst); - if (self.typeOfIndex(inst).isNoReturn(zcu)) return; - break :res res; - }, - .dbg_inline_block => res: { - const res = try self.airDbgInlineBlock(inst); - if (self.typeOfIndex(inst).isNoReturn(zcu)) return; - break :res res; - }, - .call, .call_always_tail, .call_never_tail, .call_never_inline => |tag| res: { - const res = try self.airCall(inst, switch (tag) { - .call => .auto, - .call_always_tail => .always_tail, - .call_never_tail => .never_tail, - .call_never_inline => .never_inline, - else => unreachable, - }); - // TODO: the AIR we emit for calls is a bit weird - the instruction has - // type `noreturn`, but there are instructions (and maybe a safety check) following - // nonetheless. The `unreachable` or safety check should be emitted by backends instead. - //if (self.typeOfIndex(inst).isNoReturn(mod)) return; - break :res res; - }, - - // zig fmt: on - }; - if (val != .none) try self.func_inst_table.putNoClobber(self.gpa, inst.toRef(), val); - } - unreachable; - } - - fn genBodyDebugScope( - self: *FuncGen, - maybe_inline_func: ?InternPool.Index, - body: []const Air.Inst.Index, - coverage_point: Air.CoveragePoint, - ) Error!void { - if (self.wip.strip) return self.genBody(body, coverage_point); - - const old_debug_location = self.wip.debug_location; - const old_file = self.file; - const old_inlined_at = self.inlined_at; - const old_base_line = self.base_line; - defer if (maybe_inline_func) |_| { - self.wip.debug_location = old_debug_location; - self.file = old_file; - self.inlined_at = old_inlined_at; - self.base_line = old_base_line; - }; - - const old_scope = self.scope; - defer self.scope = old_scope; - - if (maybe_inline_func) |inline_func| { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - - const func = zcu.funcInfo(inline_func); - const nav = ip.getNav(func.owner_nav); - const file_scope = zcu.navFileScopeIndex(func.owner_nav); - const mod = zcu.fileByIndex(file_scope).mod.?; - - self.file = try o.getDebugFile(pt, file_scope); - - self.base_line = zcu.navSrcLine(func.owner_nav); - const line_number = self.base_line + 1; - self.inlined_at = try self.wip.debug_location.toMetadata(&o.builder); - - const fn_ty = try pt.funcType(.{ - .param_types = &.{}, - .return_type = .void_type, - }); - - self.scope = try o.builder.debugSubprogram( - self.file, - try o.builder.metadataString(nav.name.toSlice(&zcu.intern_pool)), - try o.builder.metadataString(nav.fqn.toSlice(&zcu.intern_pool)), - line_number, - line_number + func.lbrace_line, - try o.getDebugType(pt, fn_ty), - .{ - .di_flags = .{ .StaticMember = true }, - .sp_flags = .{ - .Optimized = mod.optimize_mode != .Debug, - .Definition = true, - .LocalToUnit = true, // inline functions cannot be exported - }, - }, - o.debug_compile_unit.unwrap().?, - ); - } - - self.scope = try self.ng.object.builder.debugLexicalBlock( - self.scope, - self.file, - self.prev_dbg_line, - self.prev_dbg_column, - ); - self.wip.debug_location = .{ .location = .{ - .line = self.prev_dbg_line, - .column = self.prev_dbg_column, - .scope = self.scope.toOptional(), - .inlined_at = self.inlined_at, - } }; - - try self.genBody(body, coverage_point); - } - - pub const CallAttr = enum { - Auto, - NeverTail, - NeverInline, - AlwaysTail, - AlwaysInline, - }; - - fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value { - const air_call = self.air.unwrapCall(inst); - const args = air_call.args; - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const callee_ty = self.typeOf(air_call.callee); - const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) { - .@"fn" => callee_ty, - .pointer => callee_ty.childType(zcu), - else => unreachable, - }; - const fn_info = zcu.typeToFunc(zig_fn_ty).?; - const return_type = Type.fromInterned(fn_info.return_type); - const llvm_fn = try self.resolveInst(air_call.callee); - const target = zcu.getTarget(); - const sret = firstParamSRet(fn_info, zcu, target); - - var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa); - defer llvm_args.deinit(); - - var attributes: Builder.FunctionAttributes.Wip = .{}; - defer attributes.deinit(&o.builder); - - if (self.disable_intrinsics) { - try attributes.addFnAttr(.nobuiltin, &o.builder); - } - - switch (modifier) { - .auto, .always_tail => {}, - .never_tail, .never_inline => try attributes.addFnAttr(.@"noinline", &o.builder), - .no_suspend, .always_inline, .compile_time => unreachable, - } - - const ret_ptr = if (!sret) null else blk: { - const llvm_ret_ty = try o.lowerType(pt, return_type); - try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder); - - const alignment = return_type.abiAlignment(zcu).toLlvm(); - const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment); - try llvm_args.append(ret_ptr); - break :blk ret_ptr; - }; - - const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; - if (err_return_tracing) { - assert(self.err_ret_trace != .none); - try llvm_args.append(self.err_ret_trace); - } - - var it = iterateParamTypes(o, pt, fn_info); - while (try it.nextCall(self, args)) |lowering| switch (lowering) { - .no_bits => continue, - .byval => { - const arg = args[it.zig_index - 1]; - const param_ty = self.typeOf(arg); - const llvm_arg = try self.resolveInst(arg); - const llvm_param_ty = try o.lowerType(pt, param_ty); - if (isByRef(param_ty, zcu)) { - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, ""); - try llvm_args.append(loaded); - } else { - try llvm_args.append(llvm_arg); - } - }, - .byref => { - const arg = args[it.zig_index - 1]; - const param_ty = self.typeOf(arg); - const llvm_arg = try self.resolveInst(arg); - if (isByRef(param_ty, zcu)) { - try llvm_args.append(llvm_arg); - } else { - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - const param_llvm_ty = llvm_arg.typeOfWip(&self.wip); - const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment); - _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment); - try llvm_args.append(arg_ptr); - } - }, - .byref_mut => { - const arg = args[it.zig_index - 1]; - const param_ty = self.typeOf(arg); - const llvm_arg = try self.resolveInst(arg); - - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - const param_llvm_ty = try o.lowerType(pt, param_ty); - const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment); - if (isByRef(param_ty, zcu)) { - const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, ""); - _ = try self.wip.store(.normal, loaded, arg_ptr, alignment); - } else { - _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment); - } - try llvm_args.append(arg_ptr); - }, - .abi_sized_int => { - const arg = args[it.zig_index - 1]; - const param_ty = self.typeOf(arg); - const llvm_arg = try self.resolveInst(arg); - const int_llvm_ty = try o.builder.intType(@intCast(param_ty.abiSize(zcu) * 8)); - - if (isByRef(param_ty, zcu)) { - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - const loaded = try self.wip.load(.normal, int_llvm_ty, llvm_arg, alignment, ""); - try llvm_args.append(loaded); - } else { - // LLVM does not allow bitcasting structs so we must allocate - // a local, store as one type, and then load as another type. - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - const int_ptr = try self.buildAlloca(int_llvm_ty, alignment); - _ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment); - const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, ""); - try llvm_args.append(loaded); - } - }, - .slice => { - const arg = args[it.zig_index - 1]; - const llvm_arg = try self.resolveInst(arg); - const ptr = try self.wip.extractValue(llvm_arg, &.{0}, ""); - const len = try self.wip.extractValue(llvm_arg, &.{1}, ""); - try llvm_args.appendSlice(&.{ ptr, len }); - }, - .multiple_llvm_types => { - const arg = args[it.zig_index - 1]; - const param_ty = self.typeOf(arg); - const llvm_types = it.types_buffer[0..it.types_len]; - const llvm_arg = try self.resolveInst(arg); - const is_by_ref = isByRef(param_ty, zcu); - const arg_ptr = if (is_by_ref) llvm_arg else ptr: { - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment); - _ = try self.wip.store(.normal, llvm_arg, ptr, alignment); - break :ptr ptr; - }; - - const llvm_ty = try o.builder.structType(.normal, llvm_types); - try llvm_args.ensureUnusedCapacity(it.types_len); - for (llvm_types, 0..) |field_ty, i| { - const alignment = - Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8)); - const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, ""); - const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, ""); - llvm_args.appendAssumeCapacity(loaded); - } - }, - .float_array => |count| { - const arg = args[it.zig_index - 1]; - const arg_ty = self.typeOf(arg); - var llvm_arg = try self.resolveInst(arg); - const alignment = arg_ty.abiAlignment(zcu).toLlvm(); - if (!isByRef(arg_ty, zcu)) { - const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment); - _ = try self.wip.store(.normal, llvm_arg, ptr, alignment); - llvm_arg = ptr; - } - - const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?); - const array_ty = try o.builder.arrayType(count, float_ty); - - const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, ""); - try llvm_args.append(loaded); - }, - .i32_array, .i64_array => |arr_len| { - const elem_size: u8 = if (lowering == .i32_array) 32 else 64; - const arg = args[it.zig_index - 1]; - const arg_ty = self.typeOf(arg); - var llvm_arg = try self.resolveInst(arg); - const alignment = arg_ty.abiAlignment(zcu).toLlvm(); - if (!isByRef(arg_ty, zcu)) { - const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment); - _ = try self.wip.store(.normal, llvm_arg, ptr, alignment); - llvm_arg = ptr; - } - - const array_ty = - try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size))); - const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, ""); - try llvm_args.append(loaded); - }, - }; - - { - // Add argument attributes. - it = iterateParamTypes(o, pt, fn_info); - it.llvm_index += @intFromBool(sret); - it.llvm_index += @intFromBool(err_return_tracing); - while (try it.next()) |lowering| switch (lowering) { - .byval => { - const param_index = it.zig_index - 1; - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); - if (!isByRef(param_ty, zcu)) { - try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); - } - }, - .byref => { - const param_index = it.zig_index - 1; - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); - const param_llvm_ty = try o.lowerType(pt, param_ty); - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty); - }, - .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), - // No attributes needed for these. - .no_bits, - .abi_sized_int, - .multiple_llvm_types, - .float_array, - .i32_array, - .i64_array, - => continue, - - .slice => { - assert(!it.byval_attr); - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const ptr_info = param_ty.ptrInfo(zcu); - const llvm_arg_i = it.llvm_index - 2; - - if (math.cast(u5, it.zig_index - 1)) |i| { - if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { - try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); - } - } - if (param_ty.zigTypeTag(zcu) != .optional and - !ptr_info.flags.is_allowzero and - ptr_info.flags.address_space == .generic) - { - try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); - } - if (ptr_info.flags.is_const) { - try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder); - } - const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) { - else => |a| .wrap(a.toLlvm()), - .none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)), - }; - try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder); - }, - }; - } - - const call = try self.wip.call( - switch (modifier) { - .auto, .never_inline => .normal, - .never_tail => .notail, - .always_tail => .musttail, - .no_suspend, .always_inline, .compile_time => unreachable, - }, - toLlvmCallConvTag(fn_info.cc, target).?, - try attributes.finish(&o.builder), - try o.lowerType(pt, zig_fn_ty), - llvm_fn, - llvm_args.items, - "", - ); - - if (fn_info.return_type == .noreturn_type and modifier != .always_tail) { - return .none; - } - - if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) { - return .none; - } - - const llvm_ret_ty = try o.lowerType(pt, return_type); - if (ret_ptr) |rp| { - if (isByRef(return_type, zcu)) { - return rp; - } else { - // our by-ref status disagrees with sret so we must load. - const return_alignment = return_type.abiAlignment(zcu).toLlvm(); - return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, ""); - } - } - - const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); - - if (abi_ret_ty != llvm_ret_ty) { - // In this case the function return type is honoring the calling convention by having - // a different LLVM type than the usual one. We solve this here at the callsite - // by using our canonical type, then loading it if necessary. - const alignment = return_type.abiAlignment(zcu).toLlvm(); - const rp = try self.buildAlloca(abi_ret_ty, alignment); - _ = try self.wip.store(.normal, call, rp, alignment); - return if (isByRef(return_type, zcu)) - rp - else - try self.wip.load(.normal, llvm_ret_ty, rp, alignment, ""); - } - - if (isByRef(return_type, zcu)) { - // our by-ref status disagrees with sret so we must allocate, store, - // and return the allocation pointer. - const alignment = return_type.abiAlignment(zcu).toLlvm(); - const rp = try self.buildAlloca(llvm_ret_ty, alignment); - _ = try self.wip.store(.normal, call, rp, alignment); - return rp; - } else { - return call; - } - } - - fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) !void { - const o = fg.ng.object; - const pt = fg.ng.pt; - const zcu = pt.zcu; - const target = zcu.getTarget(); - const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin())); - const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?; - const panic_global = try o.resolveLlvmFunction(pt, panic_func.owner_nav); - - const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto; - if (has_err_trace) assert(fg.err_ret_trace != .none); - _ = try fg.wip.callIntrinsicAssumeCold(); - _ = try fg.wip.call( - .normal, - toLlvmCallConvTag(fn_info.cc, target).?, - .none, - panic_global.typeOf(&o.builder), - panic_global.toValue(&o.builder), - if (has_err_trace) &.{fg.err_ret_trace} else &.{}, - "", - ); - _ = try fg.wip.@"unreachable"(); - } - - fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const ret_ty = self.typeOf(un_op); - - if (self.ret_ptr != .none) { - const ptr_ty = try pt.singleMutPtrType(ret_ty); - - const operand = try self.resolveInst(un_op); - const val_is_undef = if (try self.air.value(un_op, pt)) |val| val.isUndef(zcu) else false; - if (val_is_undef and safety) undef: { - const ptr_info = ptr_ty.ptrInfo(zcu); - const needs_bitmask = (ptr_info.packed_offset.host_size != 0); - if (needs_bitmask) { - // TODO: only some bits are to be undef, we cannot write with a simple memset. - // meanwhile, ignore the write rather than stomping over valid bits. - // https://github.com/ziglang/zig/issues/15337 - break :undef; - } - const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu)); - _ = try self.wip.callMemSet( - self.ret_ptr, - ptr_ty.ptrAlignment(zcu).toLlvm(), - try o.builder.intValue(.i8, 0xaa), - len, - .normal, - self.disable_intrinsics, - ); - const owner_mod = self.ng.ownerModule(); - if (owner_mod.valgrind) { - try self.valgrindMarkUndef(self.ret_ptr, len); - } - _ = try self.wip.retVoid(); - return; - } - - const unwrapped_operand = operand.unwrap(); - const unwrapped_ret = self.ret_ptr.unwrap(); - - // Return value was stored previously - if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) { - _ = try self.wip.retVoid(); - return; - } - - try self.store(self.ret_ptr, ptr_ty, operand, .none); - _ = try self.wip.retVoid(); - return; - } - const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.ng.nav_index).resolved.?.type)).?; - if (!ret_ty.hasRuntimeBits(zcu)) { - if (Type.fromInterned(fn_info.return_type).isError(zcu)) { - // Functions with an empty error set are emitted with an error code - // return type and return zero so they can be function pointers coerced - // to functions that return anyerror. - _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0)); - } else { - _ = try self.wip.retVoid(); - } - return; - } - - const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); - const operand = try self.resolveInst(un_op); - const val_is_undef = if (try self.air.value(un_op, pt)) |val| val.isUndef(zcu) else false; - const alignment = ret_ty.abiAlignment(zcu).toLlvm(); - - if (val_is_undef and safety) { - const llvm_ret_ty = operand.typeOfWip(&self.wip); - const rp = try self.buildAlloca(llvm_ret_ty, alignment); - const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu)); - _ = try self.wip.callMemSet( - rp, - alignment, - try o.builder.intValue(.i8, 0xaa), - len, - .normal, - self.disable_intrinsics, - ); - const owner_mod = self.ng.ownerModule(); - if (owner_mod.valgrind) { - try self.valgrindMarkUndef(rp, len); - } - _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, "")); - return; - } - - if (isByRef(ret_ty, zcu)) { - // operand is a pointer however self.ret_ptr is null so that means - // we need to return a value. - _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, "")); - return; - } - - const llvm_ret_ty = operand.typeOfWip(&self.wip); - if (abi_ret_ty == llvm_ret_ty) { - _ = try self.wip.ret(operand); - return; - } - - const rp = try self.buildAlloca(llvm_ret_ty, alignment); - _ = try self.wip.store(.normal, operand, rp, alignment); - _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, "")); - return; - } - - fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const ptr_ty = self.typeOf(un_op); - const ret_ty = ptr_ty.childType(zcu); - const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.ng.nav_index).resolved.?.type)).?; - if (!ret_ty.hasRuntimeBits(zcu)) { - if (Type.fromInterned(fn_info.return_type).isError(zcu)) { - // Functions with an empty error set are emitted with an error code - // return type and return zero so they can be function pointers coerced - // to functions that return anyerror. - _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0)); - } else { - _ = try self.wip.retVoid(); - } - return; - } - if (self.ret_ptr != .none) { - _ = try self.wip.retVoid(); - return; - } - const ptr = try self.resolveInst(un_op); - const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); - const alignment = ret_ty.abiAlignment(zcu).toLlvm(); - _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, "")); - return; - } - - fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const list = try self.resolveInst(ty_op.operand); - const arg_ty = ty_op.ty.toType(); - const llvm_arg_ty = try o.lowerType(pt, arg_ty); - - return self.wip.vaArg(list, llvm_arg_ty, ""); - } - - fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const src_list = try self.resolveInst(ty_op.operand); - const va_list_ty = ty_op.ty.toType(); - const llvm_va_list_ty = try o.lowerType(pt, va_list_ty); - - const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); - const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); - - _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, ""); - return if (isByRef(va_list_ty, zcu)) - dest_list - else - try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, ""); - } - - fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const src_list = try self.resolveInst(un_op); - - _ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, ""); - return .none; - } - - fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const va_list_ty = self.typeOfIndex(inst); - const llvm_va_list_ty = try o.lowerType(pt, va_list_ty); - - const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); - const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); - - _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, ""); - return if (isByRef(va_list_ty, zcu)) - dest_list - else - try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, ""); - } - - fn airCmp( - self: *FuncGen, - inst: Air.Inst.Index, - op: math.CompareOperator, - fast: Builder.FastMathKind, - ) !Builder.Value { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const operand_ty = self.typeOf(bin_op.lhs); - - return self.cmp(fast, op, operand_ty, lhs, rhs); - } - - fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data; - - const lhs = try self.resolveInst(extra.lhs); - const rhs = try self.resolveInst(extra.rhs); - const vec_ty = self.typeOf(extra.lhs); - const cmp_op = extra.compareOperator(); - - return self.cmp(fast, cmp_op, vec_ty, lhs, rhs); - } - - fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - const llvm_fn = try o.getCmpLtErrorsLenFunction(pt); - return self.wip.call( - .normal, - .fastcc, - .none, - llvm_fn.typeOf(&o.builder), - llvm_fn.toValue(&o.builder), - &.{operand}, - "", - ); - } - - fn cmp( - self: *FuncGen, - fast: Builder.FastMathKind, - op: math.CompareOperator, - operand_ty: Type, - lhs: Builder.Value, - rhs: Builder.Value, - ) Allocator.Error!Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const scalar_ty = operand_ty.scalarType(zcu); - const int_ty = switch (scalar_ty.zigTypeTag(zcu)) { - .@"enum" => scalar_ty.intTagType(zcu), - .int, .bool, .pointer, .error_set => scalar_ty, - .optional => blk: { - const payload_ty = operand_ty.optionalChild(zcu); - if (!payload_ty.hasRuntimeBits(zcu) or - operand_ty.optionalReprIsPayload(zcu)) - { - break :blk operand_ty; - } - // We need to emit instructions to check for equality/inequality - // of optionals that are not pointers. - const is_by_ref = isByRef(scalar_ty, zcu); - const opt_llvm_ty = try o.lowerType(pt, scalar_ty); - const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal); - const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal); - const llvm_i2 = try o.builder.intType(2); - const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, ""); - const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, ""); - const lhs_shifted = try self.wip.bin(.shl, lhs_non_null_i2, try o.builder.intValue(llvm_i2, 1), ""); - const lhs_rhs_ored = try self.wip.bin(.@"or", lhs_shifted, rhs_non_null_i2, ""); - const both_null_block = try self.wip.block(1, "BothNull"); - const mixed_block = try self.wip.block(1, "Mixed"); - const both_pl_block = try self.wip.block(1, "BothNonNull"); - const end_block = try self.wip.block(3, "End"); - var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none); - defer wip_switch.finish(&self.wip); - try wip_switch.addCase( - try o.builder.intConst(llvm_i2, 0b00), - both_null_block, - &self.wip, - ); - try wip_switch.addCase( - try o.builder.intConst(llvm_i2, 0b11), - both_pl_block, - &self.wip, - ); - - self.wip.cursor = .{ .block = both_null_block }; - _ = try self.wip.br(end_block); - - self.wip.cursor = .{ .block = mixed_block }; - _ = try self.wip.br(end_block); - - self.wip.cursor = .{ .block = both_pl_block }; - const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true); - const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true); - const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload); - _ = try self.wip.br(end_block); - const both_pl_block_end = self.wip.cursor.block; - - self.wip.cursor = .{ .block = end_block }; - const llvm_i1_0 = Builder.Value.false; - const llvm_i1_1 = Builder.Value.true; - const incoming_values: [3]Builder.Value = .{ - switch (op) { - .eq => llvm_i1_1, - .neq => llvm_i1_0, - else => unreachable, - }, - switch (op) { - .eq => llvm_i1_0, - .neq => llvm_i1_1, - else => unreachable, - }, - payload_cmp, - }; - - const phi = try self.wip.phi(.i1, ""); - phi.finish( - &incoming_values, - &.{ both_null_block, mixed_block, both_pl_block_end }, - &self.wip, - ); - return phi.toValue(); - }, - .float => return self.buildFloatCmp(fast, op, operand_ty, .{ lhs, rhs }), - .@"struct", .@"union" => scalar_ty.bitpackBackingInt(zcu), - else => unreachable, - }; - const is_signed = int_ty.isSignedInt(zcu); - const cond: Builder.IntegerCondition = switch (op) { - .eq => .eq, - .neq => .ne, - .lt => if (is_signed) .slt else .ult, - .lte => if (is_signed) .sle else .ule, - .gt => if (is_signed) .sgt else .ugt, - .gte => if (is_signed) .sge else .uge, - }; - return self.wip.icmp(cond, lhs, rhs, ""); - } - - fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const block = self.air.unwrapBlock(inst); - return self.lowerBlock(inst, null, block.body); - } - - fn lowerBlock( - self: *FuncGen, - inst: Air.Inst.Index, - maybe_inline_func: ?InternPool.Index, - body: []const Air.Inst.Index, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const inst_ty = self.typeOfIndex(inst); - - if (inst_ty.isNoReturn(zcu)) { - try self.genBodyDebugScope(maybe_inline_func, body, .none); - return .none; - } - - const have_block_result = inst_ty.hasRuntimeBits(zcu); - - var breaks: BreakList = if (have_block_result) .{ .list = .{} } else .{ .len = 0 }; - defer if (have_block_result) breaks.list.deinit(self.gpa); - - const parent_bb = try self.wip.block(0, "Block"); - try self.blocks.putNoClobber(self.gpa, inst, .{ - .parent_bb = parent_bb, - .breaks = &breaks, - }); - defer assert(self.blocks.remove(inst)); - - try self.genBodyDebugScope(maybe_inline_func, body, .none); - - self.wip.cursor = .{ .block = parent_bb }; - - // Create a phi node only if the block returns a value. - if (have_block_result) { - const raw_llvm_ty = try o.lowerType(pt, inst_ty); - const llvm_ty: Builder.Type = ty: { - // If the zig tag type is a function, this represents an actual function body; not - // a pointer to it. LLVM IR allows the call instruction to use function bodies instead - // of function pointers, however the phi makes it a runtime value and therefore - // the LLVM type has to be wrapped in a pointer. - if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) { - break :ty .ptr; - } - break :ty raw_llvm_ty; - }; - - parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len); - const phi = try self.wip.phi(llvm_ty, ""); - phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip); - return phi.toValue(); - } else { - parent_bb.ptr(&self.wip).incoming = @intCast(breaks.len); - return .none; - } - } - - fn airBr(self: *FuncGen, inst: Air.Inst.Index) !void { - const zcu = self.ng.pt.zcu; - const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br; - const block = self.blocks.get(branch.block_inst).?; - - // Add the values to the lists only if the break provides a value. - const operand_ty = self.typeOf(branch.operand); - if (operand_ty.hasRuntimeBits(zcu)) { - const val = try self.resolveInst(branch.operand); - - // For the phi node, we need the basic blocks and the values of the - // break instructions. - try block.breaks.list.append(self.gpa, .{ .bb = self.wip.cursor.block, .val = val }); - } else block.breaks.len += 1; - _ = try self.wip.br(block.parent_bb); - } - - fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) !void { - const repeat = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat; - const loop_bb = self.loops.get(repeat.loop_inst).?; - loop_bb.ptr(&self.wip).incoming += 1; - _ = try self.wip.br(loop_bb); - } - - fn lowerSwitchDispatch( - self: *FuncGen, - switch_inst: Air.Inst.Index, - cond_ref: Air.Inst.Ref, - dispatch_info: SwitchDispatchInfo, - ) !void { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const cond_ty = self.typeOf(cond_ref); - const switch_br = self.air.unwrapSwitch(switch_inst); - - if (try self.air.value(cond_ref, pt)) |cond_val| { - // Comptime-known dispatch. Iterate the cases to find the correct - // one, and branch to the corresponding element of `case_blocks`. - var it = switch_br.iterateCases(); - const target_case_idx = target: while (it.next()) |case| { - for (case.items) |item| { - const val = Value.fromInterned(item.toInterned().?); - if (cond_val.compareHetero(.eq, val, zcu)) break :target case.idx; - } - for (case.ranges) |range| { - const low = Value.fromInterned(range[0].toInterned().?); - const high = Value.fromInterned(range[1].toInterned().?); - if (cond_val.compareHetero(.gte, low, zcu) and - cond_val.compareHetero(.lte, high, zcu)) - { - break :target case.idx; - } - } - } else dispatch_info.case_blocks.len - 1; - const target_block = dispatch_info.case_blocks[target_case_idx]; - target_block.ptr(&self.wip).incoming += 1; - _ = try self.wip.br(target_block); - return; - } - - // Runtime-known dispatch. - const cond = try self.resolveInst(cond_ref); - - if (dispatch_info.jmp_table) |jmp_table| { - // We should use the constructed jump table. - // First, check the bounds to branch to the `else` case if needed. - const inbounds = try self.wip.bin( - .@"and", - try self.cmp(.normal, .gte, cond_ty, cond, jmp_table.min.toValue()), - try self.cmp(.normal, .lte, cond_ty, cond, jmp_table.max.toValue()), - "", - ); - const jmp_table_block = try self.wip.block(1, "Then"); - const else_block = dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]; - else_block.ptr(&self.wip).incoming += 1; - _ = try self.wip.brCond(inbounds, jmp_table_block, else_block, switch (jmp_table.in_bounds_hint) { - .none => .none, - .unpredictable => .unpredictable, - .likely => .then_likely, - .unlikely => .else_likely, - }); - - self.wip.cursor = .{ .block = jmp_table_block }; - - // Figure out the list of blocks we might branch to. - // This includes all case blocks, but it might not include the `else` block if - // the table is dense. - const target_blocks_len = dispatch_info.case_blocks.len - @intFromBool(!jmp_table.table_includes_else); - const target_blocks = dispatch_info.case_blocks[0..target_blocks_len]; - - // Make sure to cast the index to a usize so it's not treated as negative! - const table_index = try self.wip.conv( - .unsigned, - try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""), - try o.lowerType(pt, .usize), - "", - ); - const target_ptr_ptr = try self.wip.gep( - .inbounds, - .ptr, - jmp_table.table.toValue(), - &.{table_index}, - "", - ); - const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, ""); - - // Do the branch! - _ = try self.wip.indirectbr(target_ptr, target_blocks); - - // Mark all target blocks as having one more incoming branch. - for (target_blocks) |case_block| { - case_block.ptr(&self.wip).incoming += 1; - } - - return; - } - - // We must lower to an actual LLVM `switch` instruction. - // The switch prongs will correspond to our scalar cases. Ranges will - // be handled by conditional branches in the `else` prong. - - const llvm_usize = try o.lowerType(pt, Type.usize); - const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder)) - try self.wip.cast(.ptrtoint, cond, llvm_usize, "") - else - cond; - - const llvm_cases_len, const last_range_case = info: { - var llvm_cases_len: u32 = 0; - var last_range_case: ?u32 = null; - var it = switch_br.iterateCases(); - while (it.next()) |case| { - if (case.ranges.len > 0) last_range_case = case.idx; - llvm_cases_len += @intCast(case.items.len); - } - break :info .{ llvm_cases_len, last_range_case }; - }; - - // The `else` of the LLVM `switch` is the actual `else` prong only - // if there are no ranges. Otherwise, the `else` will have a - // conditional chain before the "true" `else` prong. - const llvm_else_block = if (last_range_case == null) - dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1] - else - try self.wip.block(0, "RangeTest"); - - llvm_else_block.ptr(&self.wip).incoming += 1; - - var wip_switch = try self.wip.@"switch"(cond_int, llvm_else_block, llvm_cases_len, dispatch_info.switch_weights); - defer wip_switch.finish(&self.wip); - - // Construct the actual cases. Set the cursor to the `else` block so - // we can construct ranges at the same time as scalar cases. - self.wip.cursor = .{ .block = llvm_else_block }; - - var it = switch_br.iterateCases(); - while (it.next()) |case| { - const case_block = dispatch_info.case_blocks[case.idx]; - - for (case.items) |item| { - const llvm_item = (try self.resolveInst(item)).toConst().?; - const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder)) - try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize) - else - llvm_item; - try wip_switch.addCase(llvm_int_item, case_block, &self.wip); - } - case_block.ptr(&self.wip).incoming += @intCast(case.items.len); - - if (case.ranges.len == 0) continue; - - // Add a conditional for the ranges, directing to the relevant bb. - // We don't need to consider `cold` branch hints since that information is stored - // in the target bb body, but we do care about likely/unlikely/unpredictable. - - const hint = switch_br.getHint(case.idx); - - var range_cond: ?Builder.Value = null; - for (case.ranges) |range| { - const llvm_min = try self.resolveInst(range[0]); - const llvm_max = try self.resolveInst(range[1]); - const cond_part = try self.wip.bin( - .@"and", - try self.cmp(.normal, .gte, cond_ty, cond, llvm_min), - try self.cmp(.normal, .lte, cond_ty, cond, llvm_max), - "", - ); - if (range_cond) |prev| { - range_cond = try self.wip.bin(.@"or", prev, cond_part, ""); - } else range_cond = cond_part; - } - - // If the check fails, we either branch to the "true" `else` case, - // or to the next range condition. - const range_else_block = if (case.idx == last_range_case.?) - dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1] - else - try self.wip.block(0, "RangeTest"); - - _ = try self.wip.brCond(range_cond.?, case_block, range_else_block, switch (hint) { - .none, .cold => .none, - .unpredictable => .unpredictable, - .likely => .then_likely, - .unlikely => .else_likely, - }); - case_block.ptr(&self.wip).incoming += 1; - range_else_block.ptr(&self.wip).incoming += 1; - - // Construct the next range conditional (if any) in the false branch. - self.wip.cursor = .{ .block = range_else_block }; - } - } - - fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) !void { - const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br; - const dispatch_info = self.switch_dispatch_info.get(br.block_inst).?; - return self.lowerSwitchDispatch(br.block_inst, br.operand, dispatch_info); - } - - fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void { - const cond_br = self.air.unwrapCondBr(inst); - const cond = try self.resolveInst(cond_br.condition); - const then_body = cond_br.then_body; - const else_body = cond_br.else_body; - - const Hint = enum { - none, - unpredictable, - then_likely, - else_likely, - then_cold, - else_cold, - }; - const hint: Hint = switch (cond_br.branch_hints.true) { - .none => switch (cond_br.branch_hints.false) { - .none => .none, - .likely => .else_likely, - .unlikely => .then_likely, - .cold => .else_cold, - .unpredictable => .unpredictable, - }, - .likely => switch (cond_br.branch_hints.false) { - .none => .then_likely, - .likely => .unpredictable, - .unlikely => .then_likely, - .cold => .else_cold, - .unpredictable => .unpredictable, - }, - .unlikely => switch (cond_br.branch_hints.false) { - .none => .else_likely, - .likely => .else_likely, - .unlikely => .unpredictable, - .cold => .else_cold, - .unpredictable => .unpredictable, - }, - .cold => .then_cold, - .unpredictable => .unpredictable, - }; - - const then_block = try self.wip.block(1, "Then"); - const else_block = try self.wip.block(1, "Else"); - _ = try self.wip.brCond(cond, then_block, else_block, switch (hint) { - .none, .then_cold, .else_cold => .none, - .unpredictable => .unpredictable, - .then_likely => .then_likely, - .else_likely => .else_likely, - }); - - self.wip.cursor = .{ .block = then_block }; - if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold(); - try self.genBodyDebugScope(null, then_body, cond_br.branch_hints.then_cov); - - self.wip.cursor = .{ .block = else_block }; - if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold(); - try self.genBodyDebugScope(null, else_body, cond_br.branch_hints.else_cov); - - // No need to reset the insert cursor since this instruction is noreturn. - } - - fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { - const unwrapped_try = self.air.unwrapTry(inst); - const err_union = try self.resolveInst(unwrapped_try.error_union); - const body = unwrapped_try.else_body; - const err_union_ty = self.typeOf(unwrapped_try.error_union); - const is_unused = self.liveness.isUnused(inst); - return lowerTry(self, err_union, body, err_union_ty, false, .none, false, is_unused, err_cold); - } - - fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { - const zcu = self.ng.pt.zcu; - const unwrapped_try = self.air.unwrapTryPtr(inst); - const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr); - const body = unwrapped_try.else_body; - const err_union_ptr_ty = self.typeOf(unwrapped_try.error_union_ptr); - const err_union_ty = err_union_ptr_ty.childType(zcu); - const is_unused = self.liveness.isUnused(inst); - - self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu)); - - return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), true, is_unused, err_cold); - } - - fn lowerTry( - fg: *FuncGen, - err_union: Builder.Value, - body: []const Air.Inst.Index, - err_union_ty: Type, - operand_is_ptr: bool, - operand_ptr_align: InternPool.Alignment, - can_elide_load: bool, - is_unused: bool, - err_cold: bool, - ) !Builder.Value { - const o = fg.ng.object; - const pt = fg.ng.pt; - const zcu = pt.zcu; - const payload_ty = err_union_ty.errorUnionPayload(zcu); - const payload_has_bits = payload_ty.hasRuntimeBits(zcu); - const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); - const error_type = try o.errorIntType(pt); - - const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{ - operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)), - operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)), - } else .{ .none, .none }; - - if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { - const loaded = loaded: { - const access_kind: Builder.MemoryAccessKind = - if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - - if (!payload_has_bits) { - break :loaded if (operand_is_ptr) - try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "") - else - err_union; - } - const err_field_index = try errUnionErrorOffset(payload_ty, pt); - if (operand_is_ptr or isByRef(err_union_ty, zcu)) { - const err_field_ptr = - try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, ""); - break :loaded try fg.wip.load( - if (operand_is_ptr) access_kind else .normal, - error_type, - err_field_ptr, - err_set_align.toLlvm(), - "", - ); - } - break :loaded try fg.wip.extractValue(err_union, &.{err_field_index}, ""); - }; - const zero = try o.builder.intValue(error_type, 0); - const is_err = try fg.wip.icmp(.ne, loaded, zero, ""); - - const return_block = try fg.wip.block(1, "TryRet"); - const continue_block = try fg.wip.block(1, "TryCont"); - _ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely); - - fg.wip.cursor = .{ .block = return_block }; - if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold(); - try fg.genBodyDebugScope(null, body, .poi); - - fg.wip.cursor = .{ .block = continue_block }; - } - if (is_unused) return .none; - if (!payload_has_bits) return if (operand_is_ptr) err_union else .none; - const offset = try errUnionPayloadOffset(payload_ty, pt); - if (operand_is_ptr) { - return fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, ""); - } else if (isByRef(err_union_ty, zcu)) { - const payload_ptr = try fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, ""); - if (isByRef(payload_ty, zcu)) { - if (can_elide_load) - return payload_ptr; - - return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal); - } - const load_ty = err_union_llvm_ty.structFields(&o.builder)[offset]; - return fg.wip.load(.normal, load_ty, payload_ptr, payload_align.toLlvm(), ""); - } - return fg.wip.extractValue(err_union, &.{offset}, ""); - } - - fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - - const switch_br = self.air.unwrapSwitch(inst); - - // For `loop_switch_br`, we need these BBs prepared ahead of time to generate dispatches. - // For `switch_br`, they allow us to sometimes generate better IR by sharing a BB between - // scalar and range cases in the same prong. - // +1 for `else` case. This is not the same as the LLVM `else` prong, as that may first contain - // conditionals to handle ranges. - const case_blocks = try self.gpa.alloc(Builder.Function.Block.Index, switch_br.cases_len + 1); - defer self.gpa.free(case_blocks); - // We set incoming as 0 for now, and increment it as we construct dispatches. - for (case_blocks[0 .. case_blocks.len - 1]) |*b| b.* = try self.wip.block(0, "Case"); - case_blocks[case_blocks.len - 1] = try self.wip.block(0, "Default"); - - // There's a special case here to manually generate a jump table in some cases. - // - // Labeled switch in Zig is intended to follow the "direct threading" pattern. We would ideally use a jump - // table, and each `continue` has its own indirect `jmp`, to allow the branch predictor to more accurately - // use data patterns to predict future dispatches. The problem, however, is that LLVM emits fascinatingly - // bad asm for this. Not only does it not share the jump table -- which we really need it to do to prevent - // destroying the cache -- but it also actually generates slightly different jump tables for each case, - // and *a separate conditional branch beforehand* to handle dispatching back to the case we're currently - // within(!!). - // - // This asm is really, really, not what we want. As such, we will construct the jump table manually where - // appropriate (the values are dense and relatively few), and use it when lowering dispatches. - - const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: { - if (!is_dispatch_loop) break :jmp_table null; - - // Workaround for: - // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/lib/MC/WasmObjectWriter.cpp#L560 - // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll - if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null; - - // On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just - // about acceptable - it won't fill L1d cache on most CPUs. - const max_table_len = 1024; - - const cond_ty = self.typeOf(switch_br.operand); - switch (cond_ty.zigTypeTag(zcu)) { - .bool, .pointer => break :jmp_table null, - .@"enum", .int, .error_set, .@"struct", .@"union" => {}, - else => unreachable, - } - - if (cond_ty.intInfo(zcu).signedness == .signed) break :jmp_table null; - - // Don't worry about the size of the type -- it's irrelevant, because the prong values could be fairly dense. - // If they are, then we will construct a jump table. - const min, const max = self.switchCaseItemRange(switch_br) orelse break :jmp_table null; - const min_int = min.getUnsignedInt(zcu) orelse break :jmp_table null; - const max_int = max.getUnsignedInt(zcu) orelse break :jmp_table null; - const table_len = max_int - min_int + 1; - if (table_len > max_table_len) break :jmp_table null; - - const table_elems = try self.gpa.alloc(Builder.Constant, @intCast(table_len)); - defer self.gpa.free(table_elems); - - // Set them all to the `else` branch, then iterate over the AIR switch - // and replace all values which correspond to other prongs. - @memset(table_elems, try o.builder.blockAddrConst( - self.wip.function, - case_blocks[case_blocks.len - 1], - )); - var item_count: u32 = 0; - var it = switch_br.iterateCases(); - while (it.next()) |case| { - const case_block = case_blocks[case.idx]; - const case_block_addr = try o.builder.blockAddrConst( - self.wip.function, - case_block, - ); - for (case.items) |item| { - const val = Value.fromInterned(item.toInterned().?); - const table_idx = val.toUnsignedInt(zcu) - min_int; - table_elems[@intCast(table_idx)] = case_block_addr; - item_count += 1; - } - for (case.ranges) |range| { - const low = Value.fromInterned(range[0].toInterned().?); - const high = Value.fromInterned(range[1].toInterned().?); - const low_idx = low.toUnsignedInt(zcu) - min_int; - const high_idx = high.toUnsignedInt(zcu) - min_int; - @memset(table_elems[@intCast(low_idx)..@intCast(high_idx + 1)], case_block_addr); - item_count += @intCast(high_idx + 1 - low_idx); - } - } - - const table_llvm_ty = try o.builder.arrayType(table_elems.len, .ptr); - const table_val = try o.builder.arrayConst(table_llvm_ty, table_elems); - - const table_variable = try o.builder.addVariable( - try o.builder.strtabStringFmt("__jmptab_{d}", .{@intFromEnum(inst)}), - table_llvm_ty, - .default, - ); - try table_variable.setInitializer(table_val, &o.builder); - table_variable.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); - table_variable.setUnnamedAddr(.unnamed_addr, &o.builder); - - const table_includes_else = item_count != table_len; - - break :jmp_table .{ - .min = try o.lowerValue(pt, min.toIntern()), - .max = try o.lowerValue(pt, max.toIntern()), - .in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) { - .none, .cold => .none, - .unpredictable => .unpredictable, - .likely => .likely, - .unlikely => .unlikely, - }, - .table = table_variable.toConst(&o.builder), - .table_includes_else = table_includes_else, - }; - }; - - const weights: Builder.Function.Instruction.BrCond.Weights = weights: { - if (jmp_table != null) break :weights .none; // not used - - // First pass. If any weights are `.unpredictable`, unpredictable. - // If all are `.none` or `.cold`, none. - var any_likely = false; - for (0..switch_br.cases_len) |case_idx| { - switch (switch_br.getHint(@intCast(case_idx))) { - .none, .cold => {}, - .likely, .unlikely => any_likely = true, - .unpredictable => break :weights .unpredictable, - } - } - switch (switch_br.getElseHint()) { - .none, .cold => {}, - .likely, .unlikely => any_likely = true, - .unpredictable => break :weights .unpredictable, - } - if (!any_likely) break :weights .none; - - const llvm_cases_len = llvm_cases_len: { - var len: u32 = 0; - var it = switch_br.iterateCases(); - while (it.next()) |case| len += @intCast(case.items.len); - break :llvm_cases_len len; - }; - - var weights = try self.gpa.alloc(Builder.Metadata, 1 + llvm_cases_len + 1); - defer self.gpa.free(weights); - var weight_idx: usize = 0; - - const branch_weights_str = try o.builder.metadataString("branch_weights"); - weights[weight_idx] = branch_weights_str.toMetadata(); - weight_idx += 1; - - const else_weight: u32 = switch (switch_br.getElseHint()) { - .unpredictable => unreachable, - .none, .cold => 1000, - .likely => 2000, - .unlikely => 1, - }; - weights[weight_idx] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight)); - weight_idx += 1; - - var it = switch_br.iterateCases(); - while (it.next()) |case| { - const weight_val: u32 = switch (switch_br.getHint(case.idx)) { - .unpredictable => unreachable, - .none, .cold => 1000, - .likely => 2000, - .unlikely => 1, - }; - const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val)); - @memset(weights[weight_idx..][0..case.items.len], weight_meta); - weight_idx += case.items.len; - } - - assert(weight_idx == weights.len); - break :weights .fromMetadata(try o.builder.metadataTuple(weights)); - }; - - const dispatch_info: SwitchDispatchInfo = .{ - .case_blocks = case_blocks, - .switch_weights = weights, - .jmp_table = jmp_table, - }; - - if (is_dispatch_loop) { - try self.switch_dispatch_info.putNoClobber(self.gpa, inst, dispatch_info); - } - defer if (is_dispatch_loop) { - assert(self.switch_dispatch_info.remove(inst)); - }; - - // Generate the initial dispatch. - // If this is a simple `switch_br`, this is the only dispatch. - try self.lowerSwitchDispatch(inst, switch_br.operand, dispatch_info); - - // Iterate the cases and generate their bodies. - var it = switch_br.iterateCases(); - while (it.next()) |case| { - const case_block = case_blocks[case.idx]; - self.wip.cursor = .{ .block = case_block }; - if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold(); - try self.genBodyDebugScope(null, case.body, .none); - } - self.wip.cursor = .{ .block = case_blocks[case_blocks.len - 1] }; - const else_body = it.elseBody(); - if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold(); - if (else_body.len > 0) { - try self.genBodyDebugScope(null, it.elseBody(), .none); - } else { - _ = try self.wip.@"unreachable"(); - } - } - - fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) ?[2]Value { - const zcu = self.ng.pt.zcu; - var it = switch_br.iterateCases(); - var min: ?Value = null; - var max: ?Value = null; - while (it.next()) |case| { - for (case.items) |item| { - const val = Value.fromInterned(item.toInterned().?); - const low = if (min) |m| val.compareHetero(.lt, m, zcu) else true; - const high = if (max) |m| val.compareHetero(.gt, m, zcu) else true; - if (low) min = val; - if (high) max = val; - } - for (case.ranges) |range| { - const vals: [2]Value = .{ - Value.fromInterned(range[0].toInterned().?), - Value.fromInterned(range[1].toInterned().?), - }; - const low = if (min) |m| vals[0].compareHetero(.lt, m, zcu) else true; - const high = if (max) |m| vals[1].compareHetero(.gt, m, zcu) else true; - if (low) min = vals[0]; - if (high) max = vals[1]; - } - } - if (min == null) { - assert(max == null); - return null; - } - return .{ min.?, max.? }; - } - - fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void { - const block = self.air.unwrapBlock(inst); - const body = block.body; - const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time - _ = try self.wip.br(loop_block); - - try self.loops.putNoClobber(self.gpa, inst, loop_block); - defer assert(self.loops.remove(inst)); - - self.wip.cursor = .{ .block = loop_block }; - try self.genBodyDebugScope(null, body, .none); - } - - fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand_ty = self.typeOf(ty_op.operand); - const array_ty = operand_ty.childType(zcu); - const llvm_usize = try o.lowerType(pt, Type.usize); - const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu)); - const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst)); - const operand = try self.resolveInst(ty_op.operand); - if (!array_ty.hasRuntimeBits(zcu)) - return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, ""); - const ptr = try self.wip.gep(.inbounds, try o.lowerType(pt, array_ty), operand, &.{ - try o.builder.intValue(llvm_usize, 0), try o.builder.intValue(llvm_usize, 0), - }, ""); - return self.wip.buildAggregate(slice_llvm_ty, &.{ ptr, len }, ""); - } - - fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try self.resolveInst(ty_op.operand); - const operand_ty = self.typeOf(ty_op.operand); - const operand_scalar_ty = operand_ty.scalarType(zcu); - const is_signed_int = operand_scalar_ty.isSignedInt(zcu); - - const dest_ty = self.typeOfIndex(inst); - const dest_scalar_ty = dest_ty.scalarType(zcu); - const dest_llvm_ty = try o.lowerType(pt, dest_ty); - const target = zcu.getTarget(); - - if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv( - if (is_signed_int) .signed else .unsigned, - operand, - dest_llvm_ty, - "", - ); - - const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu))) orelse { - return self.todo("float_from_int from '{f}' without intrinsics", .{operand_scalar_ty.fmt(pt)}); - }; - const rt_int_ty = try o.builder.intType(rt_int_bits); - var extended = try self.wip.conv( - if (is_signed_int) .signed else .unsigned, - operand, - rt_int_ty, - "", - ); - const dest_bits = dest_scalar_ty.floatBits(target); - const compiler_rt_operand_abbrev = compilerRtIntAbbrev(rt_int_bits); - const compiler_rt_dest_abbrev = compilerRtFloatAbbrev(dest_bits); - const sign_prefix = if (is_signed_int) "" else "un"; - const fn_name = try o.builder.strtabStringFmt("__float{s}{s}i{s}f", .{ - sign_prefix, - compiler_rt_operand_abbrev, - compiler_rt_dest_abbrev, - }); - - var param_type = rt_int_ty; - if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) { - // On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard - // i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have. - param_type = try o.builder.vectorType(.normal, 2, .i64); - extended = try self.wip.cast(.bitcast, extended, param_type, ""); - } - - const libc_fn = try self.getLibcFunction(fn_name, &.{param_type}, dest_llvm_ty); - return self.wip.call( - .normal, - .ccc, - .none, - libc_fn.typeOf(&o.builder), - libc_fn.toValue(&o.builder), - &.{extended}, - "", - ); - } - - fn airIntFromFloat( - self: *FuncGen, - inst: Air.Inst.Index, - fast: Builder.FastMathKind, - ) !Builder.Value { - _ = fast; - - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const target = zcu.getTarget(); - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const operand = try self.resolveInst(ty_op.operand); - const operand_ty = self.typeOf(ty_op.operand); - const operand_scalar_ty = operand_ty.scalarType(zcu); - - const dest_ty = self.typeOfIndex(inst); - const dest_scalar_ty = dest_ty.scalarType(zcu); - const dest_llvm_ty = try o.lowerType(pt, dest_ty); - - if (intrinsicsAllowed(operand_scalar_ty, target)) { - // TODO set fast math flag - return self.wip.conv( - if (dest_scalar_ty.isSignedInt(zcu)) .signed else .unsigned, - operand, - dest_llvm_ty, - "", - ); - } - - const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu))) orelse { - return self.todo("int_from_float to '{f}' without intrinsics", .{dest_scalar_ty.fmt(pt)}); - }; - const ret_ty = try o.builder.intType(rt_int_bits); - const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: { - // On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard - // i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have. - break :b try o.builder.vectorType(.normal, 2, .i64); - } else ret_ty; - - const operand_bits = operand_scalar_ty.floatBits(target); - const compiler_rt_operand_abbrev = compilerRtFloatAbbrev(operand_bits); - - const compiler_rt_dest_abbrev = compilerRtIntAbbrev(rt_int_bits); - const sign_prefix = if (dest_scalar_ty.isSignedInt(zcu)) "" else "uns"; - - const fn_name = try o.builder.strtabStringFmt("__fix{s}{s}f{s}i", .{ - sign_prefix, - compiler_rt_operand_abbrev, - compiler_rt_dest_abbrev, - }); - - const operand_llvm_ty = try o.lowerType(pt, operand_ty); - const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty); - var result = try self.wip.call( - .normal, - .ccc, - .none, - libc_fn.typeOf(&o.builder), - libc_fn.toValue(&o.builder), - &.{operand}, - "", - ); - - if (libc_ret_ty != ret_ty) result = try self.wip.cast(.bitcast, result, ret_ty, ""); - if (ret_ty != dest_llvm_ty) result = try self.wip.cast(.trunc, result, dest_llvm_ty, ""); - return result; - } - - fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value { - const zcu = fg.ng.pt.zcu; - return if (ty.isSlice(zcu)) fg.wip.extractValue(ptr, &.{0}, "") else ptr; - } - - fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value { - const o = fg.ng.object; - const pt = fg.ng.pt; - const zcu = pt.zcu; - const llvm_usize = try o.lowerType(pt, Type.usize); - switch (ty.ptrSize(zcu)) { - .slice => { - const len = try fg.wip.extractValue(ptr, &.{1}, ""); - const elem_ty = ty.childType(zcu); - const abi_size = elem_ty.abiSize(zcu); - if (abi_size == 1) return len; - const abi_size_llvm_val = try o.builder.intValue(llvm_usize, abi_size); - return fg.wip.bin(.@"mul nuw", len, abi_size_llvm_val, ""); - }, - .one => { - const array_ty = ty.childType(zcu); - const elem_ty = array_ty.childType(zcu); - const abi_size = elem_ty.abiSize(zcu); - return o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu) * abi_size); - }, - .many, .c => unreachable, - } - } - - fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Value { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - return self.wip.extractValue(operand, &.{index}, ""); - } - - fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: c_uint) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const slice_ptr = try self.resolveInst(ty_op.operand); - const slice_ptr_ty = self.typeOf(ty_op.operand); - const slice_llvm_ty = try o.lowerType(pt, slice_ptr_ty.childType(zcu)); - - return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, ""); - } - - fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const slice_ty = self.typeOf(bin_op.lhs); - const slice = try self.resolveInst(bin_op.lhs); - const index = try self.resolveInst(bin_op.rhs); - const slice_info = slice_ty.ptrInfo(zcu); - assert(slice_info.flags.size == .slice); - const elem_ty: Type = .fromInterned(slice_info.child); - const llvm_elem_ty = try o.lowerType(pt, elem_ty); - const base_ptr = try self.wip.extractValue(slice, &.{0}, ""); - const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, ""); - const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)); - const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal; - self.maybeMarkAllowZeroAccess(slice_info); - if (isByRef(elem_ty, zcu)) { - return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind); - } else { - return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm()); - } - } - - fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const slice_ty = self.typeOf(bin_op.lhs); - - const slice = try self.resolveInst(bin_op.lhs); - const index = try self.resolveInst(bin_op.rhs); - const llvm_elem_ty = try o.lowerType(pt, slice_ty.childType(zcu)); - const base_ptr = try self.wip.extractValue(slice, &.{0}, ""); - return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, ""); - } - - fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const array_ty = self.typeOf(bin_op.lhs); - const array_llvm_val = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const array_llvm_ty = try o.lowerType(pt, array_ty); - const elem_ty = array_ty.childType(zcu); - if (isByRef(array_ty, zcu)) { - const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{ - try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), - rhs, - }, ""); - if (isByRef(elem_ty, zcu)) { - const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm(); - return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal); - } else { - return self.loadTruncate(.normal, elem_ty, elem_ptr, .default); - } - } - - // This branch can be reached for vectors, which are always by-value. - return self.wip.extractElement(array_llvm_val, rhs, ""); - } - - fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const ptr_ty = self.typeOf(bin_op.lhs); - const elem_ty = ptr_ty.indexableElem(zcu); - const llvm_elem_ty = try o.lowerType(pt, elem_ty); - const base_ptr = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, ""); - if (isByRef(elem_ty, zcu)) { - self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm(); - return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal); - } - - self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - - return self.load(ptr, ptr_ty); - } - - fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const ptr_ty = self.typeOf(bin_op.lhs); - const elem_ty = ptr_ty.indexableElem(zcu); - assert(elem_ty.hasRuntimeBits(zcu)); - - const base_ptr = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const elem_ptr = ty_pl.ty.toType(); - if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr; - - const llvm_elem_ty = try o.lowerType(pt, elem_ty); - return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, ""); - } - - fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data; - const struct_ptr = try self.resolveInst(struct_field.struct_operand); - const struct_ptr_ty = self.typeOf(struct_field.struct_operand); - return self.fieldPtr(struct_ptr, struct_ptr_ty, struct_field.field_index); - } - - fn airStructFieldPtrIndex( - self: *FuncGen, - inst: Air.Inst.Index, - field_index: u32, - ) !Builder.Value { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const struct_ptr = try self.resolveInst(ty_op.operand); - const struct_ptr_ty = self.typeOf(ty_op.operand); - return self.fieldPtr(struct_ptr, struct_ptr_ty, field_index); - } - - fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data; - const struct_ty = self.typeOf(struct_field.struct_operand); - const struct_llvm_val = try self.resolveInst(struct_field.struct_operand); - const field_index = struct_field.field_index; - const field_ty = struct_ty.fieldType(field_index, zcu); - if (!field_ty.hasRuntimeBits(zcu)) return .none; - - if (!isByRef(struct_ty, zcu)) { - assert(!isByRef(field_ty, zcu)); - switch (struct_ty.zigTypeTag(zcu)) { - .@"struct" => switch (struct_ty.containerLayout(zcu)) { - .@"packed" => { - const struct_type = zcu.typeToStruct(struct_ty).?; - const bit_offset = zcu.structPackedFieldBitOffset(struct_type, field_index); - const containing_int = struct_llvm_val; - const shift_amt = - try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset); - const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, ""); - const elem_llvm_ty = try o.lowerType(pt, field_ty); - if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) { - const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu))); - const truncated_int = - try self.wip.cast(.trunc, shifted_value, same_size_int, ""); - return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); - } - return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, ""); - }, - else => { - const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; - return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, ""); - }, - }, - .@"union" => { - assert(struct_ty.containerLayout(zcu) == .@"packed"); - const containing_int = struct_llvm_val; - const elem_llvm_ty = try o.lowerType(pt, field_ty); - if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) { - const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu))); - const truncated_int = - try self.wip.cast(.trunc, containing_int, same_size_int, ""); - return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); - } - return self.wip.cast(.trunc, containing_int, elem_llvm_ty, ""); - }, - else => unreachable, - } - } - - switch (struct_ty.zigTypeTag(zcu)) { - .@"struct" => { - const layout = struct_ty.containerLayout(zcu); - assert(layout != .@"packed"); - const struct_llvm_ty = try o.lowerType(pt, struct_ty); - const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; - const field_ptr = - try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, ""); - const explicit_alignment = struct_ty.explicitFieldAlignment(field_index, zcu); - const field_ptr_ty = try pt.ptrType(.{ - .child = field_ty.toIntern(), - .flags = .{ .alignment = explicit_alignment }, - }); - if (isByRef(field_ty, zcu)) { - const alignment = switch (explicit_alignment) { - .none => field_ty.abiAlignment(zcu), - else => |a| a, - }; - return self.loadByRef(field_ptr, field_ty, alignment.toLlvm(), .normal); - } else { - return self.load(field_ptr, field_ptr_ty); - } - }, - .@"union" => { - const union_llvm_ty = try o.lowerType(pt, struct_ty); - const layout = struct_ty.unionGetLayout(zcu); - const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)); - const field_ptr = - try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, ""); - const payload_alignment = layout.payload_align.toLlvm(); - if (isByRef(field_ty, zcu)) { - return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal); - } else { - return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment); - } - }, - else => unreachable, - } - } - - fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; - - const field_ptr = try self.resolveInst(extra.field_ptr); - - const parent_ty = ty_pl.ty.toType().childType(zcu); - const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu); - if (field_offset == 0) return field_ptr; - - const res_ty = try o.lowerType(pt, ty_pl.ty.toType()); - const llvm_usize = try o.lowerType(pt, Type.usize); - - const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, ""); - const base_ptr_int = try self.wip.bin( - .@"sub nuw", - field_ptr_int, - try o.builder.intValue(llvm_usize, field_offset), - "", - ); - return self.wip.cast(.inttoptr, base_ptr_int, res_ty, ""); - } - - fn airNot(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - - return self.wip.not(operand, ""); - } - - fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) !void { - _ = inst; - _ = try self.wip.@"unreachable"(); - } - - fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt; - self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1); - self.prev_dbg_column = @intCast(dbg_stmt.column + 1); - - self.wip.debug_location = .{ .location = .{ - .line = self.prev_dbg_line, - .column = self.prev_dbg_column, - .scope = self.scope.toOptional(), - .inlined_at = self.inlined_at, - } }; - - return .none; - } - - fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - _ = self; - _ = inst; - return .none; - } - - fn airDbgInlineBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const block = self.air.unwrapDbgBlock(inst); - self.arg_inline_index = 0; - return self.lowerBlock(inst, block.func, block.body); - } - - fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const operand = try self.resolveInst(pl_op.operand); - const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); - const ptr_ty = self.typeOf(pl_op.operand); - - const debug_local_var = try o.builder.debugLocalVar( - try o.builder.metadataString(name.toSlice(self.air)), - self.file, - self.scope, - self.prev_dbg_line, - try o.getDebugType(pt, ptr_ty.childType(zcu)), - ); - - _ = try self.wip.callIntrinsic( - .normal, - .none, - .@"dbg.declare", - &.{}, - &.{ - (try self.wip.debugValue(operand)).toValue(), - debug_local_var.toValue(), - (try o.builder.debugExpression(&.{})).toValue(), - }, - "", - ); - - return .none; - } - - fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const operand = try self.resolveInst(pl_op.operand); - const operand_ty = self.typeOf(pl_op.operand); - const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); - const name_slice = name.toSlice(self.air); - const metadata_name = if (name_slice.len > 0) try o.builder.metadataString(name_slice) else null; - const debug_local_var = if (is_arg) try o.builder.debugParameter( - metadata_name, - self.file, - self.scope, - self.prev_dbg_line, - try o.getDebugType(pt, operand_ty), - arg_no: { - self.arg_inline_index += 1; - break :arg_no self.arg_inline_index; - }, - ) else try o.builder.debugLocalVar( - metadata_name, - self.file, - self.scope, - self.prev_dbg_line, - try o.getDebugType(pt, operand_ty), - ); - - const zcu = pt.zcu; - const owner_mod = self.ng.ownerModule(); - if (isByRef(operand_ty, zcu)) { - _ = try self.wip.callIntrinsic( - .normal, - .none, - .@"dbg.declare", - &.{}, - &.{ - (try self.wip.debugValue(operand)).toValue(), - debug_local_var.toValue(), - (try o.builder.debugExpression(&.{})).toValue(), - }, - "", - ); - } else if (owner_mod.optimize_mode == .Debug and !self.is_naked) { - // We avoid taking this path for naked functions because there's no guarantee that such - // functions even have a valid stack pointer, making the `alloca` + `store` unsafe. - - const alignment = operand_ty.abiAlignment(zcu).toLlvm(); - const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment); - _ = try self.wip.store(.normal, operand, alloca, alignment); - _ = try self.wip.callIntrinsic( - .normal, - .none, - .@"dbg.declare", - &.{}, - &.{ - (try self.wip.debugValue(alloca)).toValue(), - debug_local_var.toValue(), - (try o.builder.debugExpression(&.{})).toValue(), - }, - "", - ); - } else { - _ = try self.wip.callIntrinsic( - .normal, - .none, - .@"dbg.value", - &.{}, - &.{ - (try self.wip.debugValue(operand)).toValue(), - debug_local_var.toValue(), - (try o.builder.debugExpression(&.{})).toValue(), - }, - "", - ); - } - return .none; - } - - fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - // Eventually, the Zig compiler needs to be reworked to have inline - // assembly go through the same parsing code regardless of backend, and - // have LLVM-flavored inline assembly be *output* from that assembler. - // We don't have such an assembler implemented yet though. For now, - // this implementation feeds the inline assembly code directly to LLVM. - - const o = self.ng.object; - const unwrapped_asm = self.air.unwrapAsm(inst); - const is_volatile = unwrapped_asm.is_volatile; - const gpa = self.gpa; - - const outputs = unwrapped_asm.outputs; - const inputs = unwrapped_asm.inputs; - - var llvm_constraints: std.ArrayList(u8) = .empty; - defer llvm_constraints.deinit(gpa); - - var arena_allocator = std.heap.ArenaAllocator.init(gpa); - defer arena_allocator.deinit(); - const arena = arena_allocator.allocator(); - - // The exact number of return / parameter values depends on which output values - // are passed by reference as indirect outputs (determined below). - const max_return_count = outputs.len; - const llvm_ret_types = try arena.alloc(Builder.Type, max_return_count); - const llvm_ret_indirect = try arena.alloc(bool, max_return_count); - const llvm_rw_vals = try arena.alloc(Builder.Value, max_return_count); - - const max_param_count = max_return_count + inputs.len + outputs.len; - const llvm_param_types = try arena.alloc(Builder.Type, max_param_count); - const llvm_param_values = try arena.alloc(Builder.Value, max_param_count); - // This stores whether we need to add an elementtype attribute and - // if so, the element type itself. - const llvm_param_attrs = try arena.alloc(Builder.Type, max_param_count); - const pt = self.ng.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const target = zcu.getTarget(); - - var llvm_ret_i: usize = 0; - var llvm_param_i: usize = 0; - var total_i: usize = 0; - - var name_map: std.StringArrayHashMapUnmanaged(u16) = .empty; - try name_map.ensureUnusedCapacity(arena, max_param_count); - - var it = unwrapped_asm.iterateOutputs(); - while (it.next()) |output| { - const constraint = output.constraint; - const name = output.name; - - try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 3); - if (total_i != 0) { - llvm_constraints.appendAssumeCapacity(','); - } - llvm_constraints.appendAssumeCapacity('='); - - if (output.operand != .none) { - const output_inst = try self.resolveInst(output.operand); - const output_ty = self.typeOf(output.operand); - assert(output_ty.zigTypeTag(zcu) == .pointer); - const elem_llvm_ty = try o.lowerType(pt, output_ty.childType(zcu)); - - switch (constraint[0]) { - '=' => {}, - '+' => llvm_rw_vals[output.index] = output_inst, - else => return self.todo("unsupported output constraint on output type '{c}'", .{ - constraint[0], - }), - } - - self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu)); - - // Pass any non-return outputs indirectly, if the constraint accepts a memory location - llvm_ret_indirect[output.index] = constraintAllowsMemory(constraint); - if (llvm_ret_indirect[output.index]) { - // Pass the result by reference as an indirect output (e.g. "=*m") - llvm_constraints.appendAssumeCapacity('*'); - - llvm_param_values[llvm_param_i] = output_inst; - llvm_param_types[llvm_param_i] = output_inst.typeOfWip(&self.wip); - llvm_param_attrs[llvm_param_i] = elem_llvm_ty; - llvm_param_i += 1; - } else { - // Pass the result directly (e.g. "=r") - llvm_ret_types[llvm_ret_i] = elem_llvm_ty; - llvm_ret_i += 1; - } - } else { - switch (constraint[0]) { - '=' => {}, - else => return self.todo("unsupported output constraint on result type '{s}'", .{ - constraint, - }), - } - - llvm_ret_indirect[output.index] = false; - - const ret_ty = self.typeOfIndex(inst); - llvm_ret_types[llvm_ret_i] = try o.lowerType(pt, ret_ty); - llvm_ret_i += 1; - } - - // LLVM uses commas internally to separate different constraints, - // alternative constraints are achieved with pipes. - // We still allow the user to use commas in a way that is similar - // to GCC's inline assembly. - // http://llvm.org/docs/LangRef.html#constraint-codes - for (constraint[1..]) |byte| { - switch (byte) { - ',' => llvm_constraints.appendAssumeCapacity('|'), - '*' => {}, // Indirect outputs are handled above - else => llvm_constraints.appendAssumeCapacity(byte), - } - } - - if (!std.mem.eql(u8, name, "_")) { - const gop = name_map.getOrPutAssumeCapacity(name); - if (gop.found_existing) return self.todo("duplicate asm output name '{s}'", .{name}); - gop.value_ptr.* = @intCast(total_i); - } - total_i += 1; - } - - it = unwrapped_asm.iterateInputs(); - while (it.next()) |input| { - const constraint = input.constraint; - const name = input.name; - - const arg_llvm_value = try self.resolveInst(input.operand); - const arg_ty = self.typeOf(input.operand); - const is_by_ref = isByRef(arg_ty, zcu); - if (is_by_ref) { - if (constraintAllowsMemory(constraint)) { - llvm_param_values[llvm_param_i] = arg_llvm_value; - llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip); - } else { - const alignment = arg_ty.abiAlignment(zcu).toLlvm(); - const arg_llvm_ty = try o.lowerType(pt, arg_ty); - const load_inst = - try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, ""); - llvm_param_values[llvm_param_i] = load_inst; - llvm_param_types[llvm_param_i] = arg_llvm_ty; - } - } else { - if (constraintAllowsRegister(constraint)) { - llvm_param_values[llvm_param_i] = arg_llvm_value; - llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip); - } else { - const alignment = arg_ty.abiAlignment(zcu).toLlvm(); - const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment); - _ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment); - llvm_param_values[llvm_param_i] = arg_ptr; - llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip); - } - } - - try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 1); - if (total_i != 0) { - llvm_constraints.appendAssumeCapacity(','); - } - for (constraint) |byte| { - llvm_constraints.appendAssumeCapacity(switch (byte) { - ',' => '|', - else => byte, - }); - } - - if (!std.mem.eql(u8, name, "_")) { - const gop = name_map.getOrPutAssumeCapacity(name); - if (gop.found_existing) return self.todo("duplicate asm input name '{s}'", .{name}); - gop.value_ptr.* = @intCast(total_i); - } - - // In the case of indirect inputs, LLVM requires the callsite to have - // an elementtype() attribute. - llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: { - if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu)); - - break :blk try o.lowerType(pt, if (is_by_ref) arg_ty else arg_ty.childType(zcu)); - } else .none; - - llvm_param_i += 1; - total_i += 1; - } - - it = unwrapped_asm.iterateOutputs(); - while (it.next()) |output| { - const constraint = output.constraint; - - if (constraint[0] != '+') continue; - - const rw_ty = self.typeOf(output.operand); - const llvm_elem_ty = try o.lowerType(pt, rw_ty.childType(zcu)); - if (llvm_ret_indirect[output.index]) { - llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index]; - llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip); - } else { - const alignment = rw_ty.abiAlignment(zcu).toLlvm(); - const loaded = try self.wip.load( - if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, - llvm_elem_ty, - llvm_rw_vals[output.index], - alignment, - "", - ); - llvm_param_values[llvm_param_i] = loaded; - llvm_param_types[llvm_param_i] = llvm_elem_ty; - } - - try llvm_constraints.print(gpa, ",{d}", .{output.index}); - - // In the case of indirect inputs, LLVM requires the callsite to have - // an elementtype() attribute. - llvm_param_attrs[llvm_param_i] = if (llvm_ret_indirect[output.index]) llvm_elem_ty else .none; - - llvm_param_i += 1; - total_i += 1; - } - - if (total_i != 0) try llvm_constraints.append(gpa, ','); - const clobbers_val: Value = .fromInterned(unwrapped_asm.clobbers); - const clobbers_ty = clobbers_val.typeOf(zcu); - var clobbers_bigint_buf: Value.BigIntSpace = undefined; - const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu); - for (0..clobbers_ty.structFieldCount(zcu)) |field_index| { - assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type); - const limb_bits = @bitSizeOf(std.math.big.Limb); - if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false - switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) { - 0 => continue, // field is false - 1 => {}, // field is true - } - const name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?; - total_i += try appendConstraints(gpa, &llvm_constraints, name, target); - } - - // We have finished scanning through all inputs/outputs, so the number of - // parameters and return values is known. - const param_count = llvm_param_i; - const return_count = llvm_ret_i; - - // For some targets, Clang unconditionally adds some clobbers to all inline assembly. - // While this is probably not strictly necessary, if we don't follow Clang's lead - // here then we may risk tripping LLVM bugs since anything not used by Clang tends - // to be buggy and regress often. - switch (target.cpu.arch) { - .x86_64, .x86 => { - try llvm_constraints.appendSlice(gpa, "~{dirflag},~{fpsr},~{flags},"); - total_i += 3; - }, - .mips, .mipsel, .mips64, .mips64el => { - try llvm_constraints.appendSlice(gpa, "~{$1},"); - total_i += 1; - }, - else => {}, - } - - if (std.mem.endsWith(u8, llvm_constraints.items, ",")) llvm_constraints.items.len -= 1; - - const asm_source = unwrapped_asm.source; - - // hackety hacks until stage2 has proper inline asm in the frontend. - var rendered_template = std.array_list.Managed(u8).init(gpa); - defer rendered_template.deinit(); - - const State = enum { start, percent, input, modifier }; - - var state: State = .start; - - var name_start: usize = undefined; - var modifier_start: usize = undefined; - for (asm_source, 0..) |byte, i| { - switch (state) { - .start => switch (byte) { - '%' => state = .percent, - '$' => try rendered_template.appendSlice("$$"), - else => try rendered_template.append(byte), - }, - .percent => switch (byte) { - '%' => { - try rendered_template.append('%'); - state = .start; - }, - '[' => { - try rendered_template.append('$'); - try rendered_template.append('{'); - name_start = i + 1; - state = .input; - }, - '=' => { - try rendered_template.appendSlice("${:uid}"); - state = .start; - }, - else => { - try rendered_template.append('%'); - try rendered_template.append(byte); - state = .start; - }, - }, - .input => switch (byte) { - ']', ':' => { - const name = asm_source[name_start..i]; - - const index = name_map.get(name) orelse { - // we should validate the assembly in Sema; by now it is too late - return self.todo("unknown input or output name: '{s}'", .{name}); - }; - try rendered_template.print("{d}", .{index}); - if (byte == ':') { - try rendered_template.append(':'); - modifier_start = i + 1; - state = .modifier; - } else { - try rendered_template.append('}'); - state = .start; - } - }, - else => {}, - }, - .modifier => switch (byte) { - ']' => { - try rendered_template.appendSlice(asm_source[modifier_start..i]); - try rendered_template.append('}'); - state = .start; - }, - else => {}, - }, - } - } - - var attributes: Builder.FunctionAttributes.Wip = .{}; - defer attributes.deinit(&o.builder); - for (llvm_param_attrs[0..param_count], 0..) |llvm_elem_ty, i| if (llvm_elem_ty != .none) - try attributes.addParamAttr(i, .{ .elementtype = llvm_elem_ty }, &o.builder); - - const ret_llvm_ty = switch (return_count) { - 0 => .void, - 1 => llvm_ret_types[0], - else => try o.builder.structType(.normal, llvm_ret_types), - }; - const llvm_fn_ty = try o.builder.fnType(ret_llvm_ty, llvm_param_types[0..param_count], .normal); - const call = try self.wip.callAsm( - try attributes.finish(&o.builder), - llvm_fn_ty, - .{ .sideeffect = is_volatile }, - try o.builder.string(rendered_template.items), - try o.builder.string(llvm_constraints.items), - llvm_param_values[0..param_count], - "", - ); - - var ret_val = call; - llvm_ret_i = 0; - for (outputs, 0..) |output, i| { - if (llvm_ret_indirect[i]) continue; - - const output_value = if (return_count > 1) - try self.wip.extractValue(call, &[_]u32{@intCast(llvm_ret_i)}, "") - else - call; - - if (output != .none) { - const output_ptr = try self.resolveInst(output); - const output_ptr_ty = self.typeOf(output); - const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm(); - _ = try self.wip.store( - if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, - output_value, - output_ptr, - alignment, - ); - } else { - ret_val = output_value; - } - llvm_ret_i += 1; - } - - return ret_val; - } - - fn airIsNonNull( - self: *FuncGen, - inst: Air.Inst.Index, - operand_is_ptr: bool, - cond: Builder.IntegerCondition, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - const operand_ty = self.typeOf(un_op); - const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; - const optional_llvm_ty = try o.lowerType(pt, optional_ty); - const payload_ty = optional_ty.optionalChild(zcu); - - const access_kind: Builder.MemoryAccessKind = - if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - - if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); - - if (optional_ty.optionalReprIsPayload(zcu)) { - const loaded = if (operand_is_ptr) - try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") - else - operand; - if (payload_ty.isSlice(zcu)) { - const slice_ptr = try self.wip.extractValue(loaded, &.{0}, ""); - const ptr_ty = try o.builder.ptrType(toLlvmAddressSpace( - payload_ty.ptrAddressSpace(zcu), - zcu.getTarget(), - )); - return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), ""); - } - return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), ""); - } - - comptime assert(optional_layout_version == 3); - - if (!payload_ty.hasRuntimeBits(zcu)) { - const loaded = if (operand_is_ptr) - try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") - else - operand; - return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), ""); - } - - const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu); - return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind); - } - - fn airIsErr( - self: *FuncGen, - inst: Air.Inst.Index, - cond: Builder.IntegerCondition, - operand_is_ptr: bool, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - const operand_ty = self.typeOf(un_op); - const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; - const payload_ty = err_union_ty.errorUnionPayload(zcu); - const error_type = try o.errorIntType(pt); - const zero = try o.builder.intValue(error_type, 0); - - const access_kind: Builder.MemoryAccessKind = - if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - - if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { - const val: Builder.Constant = switch (cond) { - .eq => .true, // 0 == 0 - .ne => .false, // 0 != 0 - else => unreachable, - }; - return val.toValue(); - } - - if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); - - if (!payload_ty.hasRuntimeBits(zcu)) { - const loaded = if (operand_is_ptr) - try self.wip.load(access_kind, try o.lowerType(pt, err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") - else - operand; - return self.wip.icmp(cond, loaded, zero, ""); - } - - const err_field_index = try errUnionErrorOffset(payload_ty, pt); - - const loaded = if (operand_is_ptr or isByRef(err_union_ty, zcu)) loaded: { - const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); - const err_alignment = if (operand_is_ptr) - operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu)) - else - .none; - const err_field_ptr = - try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, ""); - break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, err_alignment.toLlvm(), ""); - } else try self.wip.extractValue(operand, &.{err_field_index}, ""); - return self.wip.icmp(cond, loaded, zero, ""); - } - - fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const optional_ty = self.typeOf(ty_op.operand).childType(zcu); - const payload_ty = optional_ty.optionalChild(zcu); - if (!payload_ty.hasRuntimeBits(zcu)) { - // We have a pointer to a zero-bit value and we need to return - // a pointer to a zero-bit value. - return operand; - } - if (optional_ty.optionalReprIsPayload(zcu)) { - // The payload and the optional are the same value. - return operand; - } - return self.wip.gepStruct(try o.lowerType(pt, optional_ty), operand, 0, ""); - } - - fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - comptime assert(optional_layout_version == 3); - - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const optional_ptr_ty = self.typeOf(ty_op.operand); - const optional_ty = optional_ptr_ty.childType(zcu); - const payload_ty = optional_ty.optionalChild(zcu); - const non_null_bit = try o.builder.intValue(.i8, 1); - - const access_kind: Builder.MemoryAccessKind = - if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - - if (!payload_ty.hasRuntimeBits(zcu)) { - self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu)); - - // We have a pointer to a i8. We need to set it to 1 and then return the same pointer. - // Default alignment store because align of the non null bit is 1 anyway. - _ = try self.wip.store(access_kind, non_null_bit, operand, .default); - return operand; - } - if (optional_ty.optionalReprIsPayload(zcu)) { - // The payload and the optional are the same value. - // Setting to non-null will be done when the payload is set. - return operand; - } - - // First set the non-null bit. - const optional_llvm_ty = try o.lowerType(pt, optional_ty); - const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, ""); - - self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu)); - - // Default alignment store because align of the non null bit is 1 anyway. - _ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default); - - // Then return the payload pointer (only if it's used). - if (self.liveness.isUnused(inst)) return .none; - - return self.wip.gepStruct(optional_llvm_ty, operand, 0, ""); - } - - fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const optional_ty = self.typeOf(ty_op.operand); - const payload_ty = self.typeOfIndex(inst); - if (!payload_ty.hasRuntimeBits(zcu)) return .none; - - if (optional_ty.optionalReprIsPayload(zcu)) { - // Payload value is the same as the optional value. - return operand; - } - - const opt_llvm_ty = try o.lowerType(pt, optional_ty); - return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false); - } - - fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const operand_ty = self.typeOf(ty_op.operand); - const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; - const result_ty = self.typeOfIndex(inst); - const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty; - - if (!payload_ty.hasRuntimeBits(zcu)) { - return if (operand_is_ptr) operand else .none; - } - const offset = try errUnionPayloadOffset(payload_ty, pt); - const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); - if (operand_is_ptr) { - return self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); - } else if (isByRef(err_union_ty, zcu)) { - const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm(); - const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); - if (isByRef(payload_ty, zcu)) { - return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); - } - const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset]; - return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, ""); - } - return self.wip.extractValue(operand, &.{offset}, ""); - } - - fn airErrUnionErr( - self: *FuncGen, - inst: Air.Inst.Index, - operand_is_ptr: bool, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const operand_ty = self.typeOf(ty_op.operand); - const error_type = try o.errorIntType(pt); - const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; - if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { - if (operand_is_ptr) { - return operand; - } else { - return o.builder.intValue(error_type, 0); - } - } - - const access_kind: Builder.MemoryAccessKind = - if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - - const payload_ty = err_union_ty.errorUnionPayload(zcu); - if (!payload_ty.hasRuntimeBits(zcu)) { - if (!operand_is_ptr) return operand; - - self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); - - return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), ""); - } - - const offset = try errUnionErrorOffset(payload_ty, pt); - - if (operand_is_ptr or isByRef(err_union_ty, zcu)) { - if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); - - const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); - const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); - return self.wip.load(access_kind, error_type, err_field_ptr, .default, ""); - } - - return self.wip.extractValue(operand, &.{offset}, ""); - } - - fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const err_union_ptr_ty = self.typeOf(ty_op.operand); - const err_union_ty = err_union_ptr_ty.childType(zcu); - const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu); - - const payload_ty = err_union_ty.errorUnionPayload(zcu); - const non_error_val = try o.builder.intValue(try o.errorIntType(pt), 0); - - const access_kind: Builder.MemoryAccessKind = - if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - - if (!payload_ty.hasRuntimeBits(zcu)) { - self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); - _ = try self.wip.store(access_kind, non_error_val, operand, err_union_ptr_align.toLlvm()); - return operand; - } - const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); - { - self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); - - const err_int_ty = try pt.errorIntType(); - const error_alignment = err_int_ty.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm(); - const error_offset = try errUnionErrorOffset(payload_ty, pt); - // First set the non-error value. - const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, ""); - _ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment); - } - // Then return the payload pointer (only if it is used). - if (self.liveness.isUnused(inst)) return .none; - - const payload_offset = try errUnionPayloadOffset(payload_ty, pt); - return self.wip.gepStruct(err_union_llvm_ty, operand, payload_offset, ""); - } - - fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value { - assert(self.err_ret_trace != .none); - return self.err_ret_trace; - } - - fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - self.err_ret_trace = try self.resolveInst(un_op); - return .none; - } - - fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const struct_ty = ty_pl.ty.toType(); - const field_index = ty_pl.payload; - - const struct_llvm_ty = try o.lowerType(pt, struct_ty); - const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; - assert(self.err_ret_trace != .none); - const field_ptr = try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, ""); - const field_alignment = struct_ty.explicitFieldAlignment(field_index, zcu); - const field_ty = struct_ty.fieldType(field_index, zcu); - const field_ptr_ty = try pt.ptrType(.{ - .child = field_ty.toIntern(), - .flags = .{ .alignment = field_alignment }, - }); - return self.load(field_ptr, field_ptr_ty); - } - - /// As an optimization, we want to avoid unnecessary copies of - /// error union/optional types when returning from a function. - /// Here, we scan forward in the current block, looking to see - /// if the next instruction is a return (ignoring debug instructions). - /// - /// The first instruction of `body_tail` is a wrap instruction. - fn isNextRet( - self: *FuncGen, - body_tail: []const Air.Inst.Index, - ) bool { - const air_tags = self.air.instructions.items(.tag); - for (body_tail[1..]) |body_inst| { - switch (air_tags[@intFromEnum(body_inst)]) { - .ret => return true, - .dbg_stmt => continue, - else => return false, - } - } - // The only way to get here is to hit the end of a loop instruction - // (implicit repeat). - return false; - } - - fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const inst = body_tail[0]; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const payload_ty = self.typeOf(ty_op.operand); - const non_null_bit = try o.builder.intValue(.i8, 1); - comptime assert(optional_layout_version == 3); - assert(payload_ty.hasRuntimeBits(zcu)); - const operand = try self.resolveInst(ty_op.operand); - const optional_ty = self.typeOfIndex(inst); - if (optional_ty.optionalReprIsPayload(zcu)) return operand; - const llvm_optional_ty = try o.lowerType(pt, optional_ty); - if (isByRef(optional_ty, zcu)) { - const directReturn = self.isNextRet(body_tail); - const optional_ptr = if (directReturn) - self.ret_ptr - else brk: { - const alignment = optional_ty.abiAlignment(zcu).toLlvm(); - const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment); - break :brk optional_ptr; - }; - - const payload_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 0, ""); - const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); - try self.store(payload_ptr, payload_ptr_ty, operand, .none); - const non_null_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 1, ""); - _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default); - return optional_ptr; - } - return self.wip.buildAggregate(llvm_optional_ty, &.{ operand, non_null_bit }, ""); - } - - fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const inst = body_tail[0]; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const err_un_ty = self.typeOfIndex(inst); - const operand = try self.resolveInst(ty_op.operand); - const payload_ty = self.typeOf(ty_op.operand); - assert(payload_ty.hasRuntimeBits(zcu)); - const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0); - const err_un_llvm_ty = try o.lowerType(pt, err_un_ty); - - const payload_offset = try errUnionPayloadOffset(payload_ty, pt); - const error_offset = try errUnionErrorOffset(payload_ty, pt); - if (isByRef(err_un_ty, zcu)) { - const directReturn = self.isNextRet(body_tail); - const result_ptr = if (directReturn) - self.ret_ptr - else brk: { - const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm(); - const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); - break :brk result_ptr; - }; - - const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, ""); - const err_int_ty = try pt.errorIntType(); - const error_alignment = err_int_ty.abiAlignment(pt.zcu).toLlvm(); - _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment); - const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, ""); - const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); - try self.store(payload_ptr, payload_ptr_ty, operand, .none); - return result_ptr; - } - var fields: [2]Builder.Value = undefined; - fields[payload_offset] = operand; - fields[error_offset] = ok_err_code; - return self.wip.buildAggregate(err_un_llvm_ty, &fields, ""); - } - - fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const inst = body_tail[0]; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const err_un_ty = self.typeOfIndex(inst); - const payload_ty = err_un_ty.errorUnionPayload(zcu); - const operand = try self.resolveInst(ty_op.operand); - if (!payload_ty.hasRuntimeBits(zcu)) return operand; - const err_un_llvm_ty = try o.lowerType(pt, err_un_ty); - - const payload_offset = try errUnionPayloadOffset(payload_ty, pt); - const error_offset = try errUnionErrorOffset(payload_ty, pt); - if (isByRef(err_un_ty, zcu)) { - const directReturn = self.isNextRet(body_tail); - const result_ptr = if (directReturn) - self.ret_ptr - else brk: { - const alignment = err_un_ty.abiAlignment(zcu).toLlvm(); - const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); - break :brk result_ptr; - }; - - const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, ""); - const err_int_ty = try pt.errorIntType(); - const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm(); - _ = try self.wip.store(.normal, operand, err_ptr, error_alignment); - const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, ""); - const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); - // TODO store undef to payload_ptr - _ = payload_ptr; - _ = payload_ptr_ty; - return result_ptr; - } - - // TODO set payload bytes to undef - const undef = try o.builder.undefValue(err_un_llvm_ty); - return self.wip.insertValue(undef, operand, &.{error_offset}, ""); - } - - fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const index = pl_op.payload; - const llvm_usize = try o.lowerType(pt, Type.usize); - return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{ - try o.builder.intValue(.i32, index), - }, ""); - } - - fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const index = pl_op.payload; - const llvm_isize = try o.lowerType(pt, Type.isize); - return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{ - try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand), - }, ""); - } - - fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = fg.ng.object; - const pt = fg.ng.pt; - const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; - const llvm_ptr_const = try o.lowerNavRefValue(pt, ty_nav.nav); - return llvm_ptr_const.toValue(); - } - - fn airMin(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, .normal, inst_ty, 2, .{ lhs, rhs }); - return self.wip.callIntrinsic( - .normal, - .none, - if (scalar_ty.isSignedInt(zcu)) .smin else .umin, - &.{try o.lowerType(pt, inst_ty)}, - &.{ lhs, rhs }, - "", - ); - } - - fn airMax(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, .normal, inst_ty, 2, .{ lhs, rhs }); - return self.wip.callIntrinsic( - .normal, - .none, - if (scalar_ty.isSignedInt(zcu)) .smax else .umax, - &.{try o.lowerType(pt, inst_ty)}, - &.{ lhs, rhs }, - "", - ); - } - - fn airSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const ptr = try self.resolveInst(bin_op.lhs); - const len = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - return self.wip.buildAggregate(try o.lowerType(pt, inst_ty), &.{ ptr, len }, ""); - } - - fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const zcu = self.ng.pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, fast, inst_ty, 2, .{ lhs, rhs }); - return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"add nsw" else .@"add nuw", lhs, rhs, ""); - } - - fn airSafeArithmetic( - fg: *FuncGen, - inst: Air.Inst.Index, - signed_intrinsic: Builder.Intrinsic, - unsigned_intrinsic: Builder.Intrinsic, - ) !Builder.Value { - const o = fg.ng.object; - const pt = fg.ng.pt; - const zcu = pt.zcu; - - const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try fg.resolveInst(bin_op.lhs); - const rhs = try fg.resolveInst(bin_op.rhs); - const inst_ty = fg.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - - const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; - const llvm_inst_ty = try o.lowerType(pt, inst_ty); - const results = - try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, ""); - - const overflow_bits = try fg.wip.extractValue(results, &.{1}, ""); - const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip); - const overflow_bit = if (overflow_bits_ty.isVector(&o.builder)) - try fg.wip.callIntrinsic( - .normal, - .none, - .@"vector.reduce.or", - &.{overflow_bits_ty}, - &.{overflow_bits}, - "", - ) - else - overflow_bits; - - const fail_block = try fg.wip.block(1, "OverflowFail"); - const ok_block = try fg.wip.block(1, "OverflowOk"); - _ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none); - - fg.wip.cursor = .{ .block = fail_block }; - try fg.buildSimplePanic(.integer_overflow); - - fg.wip.cursor = .{ .block = ok_block }; - return fg.wip.extractValue(results, &.{0}, ""); - } - - fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - return self.wip.bin(.add, lhs, rhs, ""); - } - - fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - assert(scalar_ty.zigTypeTag(zcu) == .int); - return self.wip.callIntrinsic( - .normal, - .none, - if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat", - &.{try o.lowerType(pt, inst_ty)}, - &.{ lhs, rhs }, - "", - ); - } - - fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const zcu = self.ng.pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, fast, inst_ty, 2, .{ lhs, rhs }); - return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"sub nsw" else .@"sub nuw", lhs, rhs, ""); - } - - fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - return self.wip.bin(.sub, lhs, rhs, ""); - } - - fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - assert(scalar_ty.zigTypeTag(zcu) == .int); - return self.wip.callIntrinsic( - .normal, - .none, - if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat", - &.{try o.lowerType(pt, inst_ty)}, - &.{ lhs, rhs }, - "", - ); - } - - fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const zcu = self.ng.pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, fast, inst_ty, 2, .{ lhs, rhs }); - return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"mul nsw" else .@"mul nuw", lhs, rhs, ""); - } - - fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - return self.wip.bin(.mul, lhs, rhs, ""); - } - - fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - assert(scalar_ty.zigTypeTag(zcu) == .int); - return self.wip.callIntrinsic( - .normal, - .none, - if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat", - &.{try o.lowerType(pt, inst_ty)}, - &.{ lhs, rhs, .@"0" }, - "", - ); - } - - fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - - return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); - } - - fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const zcu = self.ng.pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isRuntimeFloat()) { - const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); - return self.buildFloatOp(.trunc, fast, inst_ty, 1, .{result}); - } - return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .sdiv else .udiv, lhs, rhs, ""); - } - - fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isRuntimeFloat()) { - const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); - return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result}); - } - if (scalar_ty.isSignedInt(zcu)) { - const inst_llvm_ty = try o.lowerType(pt, inst_ty); - - const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; - var stack align(@max( - @alignOf(std.heap.StackFallbackAllocator(0)), - @alignOf(ExpectedContents), - )) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); - const allocator = stack.get(); - - const scalar_bits = inst_llvm_ty.scalarBits(&o.builder); - var smin_big_int: std.math.big.int.Mutable = .{ - .limbs = try allocator.alloc( - std.math.big.Limb, - std.math.big.int.calcTwosCompLimbCount(scalar_bits), - ), - .len = undefined, - .positive = undefined, - }; - defer allocator.free(smin_big_int.limbs); - smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits); - const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst( - inst_llvm_ty.scalarType(&o.builder), - smin_big_int.toConst(), - )); - - const div = try self.wip.bin(.sdiv, lhs, rhs, "divFloor.div"); - const rem = try self.wip.bin(.srem, lhs, rhs, "divFloor.rem"); - const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "divFloor.rhs_sign"); - const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "divFloor.rem_xor_rhs_sign"); - const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "divFloor.need_correction"); - const correction = try self.wip.cast(.sext, need_correction, inst_llvm_ty, "divFloor.correction"); - return self.wip.bin(.@"add nsw", div, correction, "divFloor"); - } - return self.wip.bin(.udiv, lhs, rhs, ""); - } - - fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const zcu = self.ng.pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); - return self.wip.bin( - if (scalar_ty.isSignedInt(zcu)) .@"sdiv exact" else .@"udiv exact", - lhs, - rhs, - "", - ); - } - - fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const zcu = self.ng.pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isRuntimeFloat()) - return self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs }); - return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) - .srem - else - .urem, lhs, rhs, ""); - } - - fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const inst_ty = self.typeOfIndex(inst); - const inst_llvm_ty = try o.lowerType(pt, inst_ty); - const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isRuntimeFloat()) { - const a = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs }); - const b = try self.buildFloatOp(.add, fast, inst_ty, 2, .{ a, rhs }); - const c = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ b, rhs }); - const zero = try o.builder.zeroInitValue(inst_llvm_ty); - const ltz = try self.buildFloatCmp(fast, .lt, inst_ty, .{ lhs, zero }); - return self.wip.select(fast, ltz, c, a, ""); - } - if (scalar_ty.isSignedInt(zcu)) { - const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; - var stack align(@max( - @alignOf(std.heap.StackFallbackAllocator(0)), - @alignOf(ExpectedContents), - )) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); - const allocator = stack.get(); - - const scalar_bits = inst_llvm_ty.scalarBits(&o.builder); - var smin_big_int: std.math.big.int.Mutable = .{ - .limbs = try allocator.alloc( - std.math.big.Limb, - std.math.big.int.calcTwosCompLimbCount(scalar_bits), - ), - .len = undefined, - .positive = undefined, - }; - defer allocator.free(smin_big_int.limbs); - smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits); - const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst( - inst_llvm_ty.scalarType(&o.builder), - smin_big_int.toConst(), - )); - - const rem = try self.wip.bin(.srem, lhs, rhs, "mod.rem"); - const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "mod.rhs_sign"); - const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "mod.rem_xor_rhs_sign"); - const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "mod.need_correction"); - const zero = try o.builder.zeroInitValue(inst_llvm_ty); - const correction = try self.wip.select(.normal, need_correction, rhs, zero, "mod.correction"); - return self.wip.bin(.@"add nsw", correction, rem, "mod"); - } - return self.wip.bin(.urem, lhs, rhs, ""); - } - - fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const ptr = try self.resolveInst(bin_op.lhs); - const offset = try self.resolveInst(bin_op.rhs); - const ptr_ty = self.typeOf(bin_op.lhs); - const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu)); - switch (ptr_ty.ptrSize(zcu)) { - // It's a pointer to an array, so according to LLVM we need an extra GEP index. - .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{ - try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), offset, - }, ""), - .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{offset}, ""), - .slice => { - const base = try self.wip.extractValue(ptr, &.{0}, ""); - return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{offset}, ""); - }, - } - } - - fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const ptr = try self.resolveInst(bin_op.lhs); - const offset = try self.resolveInst(bin_op.rhs); - const negative_offset = try self.wip.neg(offset, ""); - const ptr_ty = self.typeOf(bin_op.lhs); - const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu)); - switch (ptr_ty.ptrSize(zcu)) { - // It's a pointer to an array, so according to LLVM we need an extra GEP index. - .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{ - try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), negative_offset, - }, ""), - .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{negative_offset}, ""), - .slice => { - const base = try self.wip.extractValue(ptr, &.{0}, ""); - return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{negative_offset}, ""); - }, - } - } - - fn airOverflow( - self: *FuncGen, - inst: Air.Inst.Index, - signed_intrinsic: Builder.Intrinsic, - unsigned_intrinsic: Builder.Intrinsic, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; - - const lhs = try self.resolveInst(extra.lhs); - const rhs = try self.resolveInst(extra.rhs); - - const lhs_ty = self.typeOf(extra.lhs); - const scalar_ty = lhs_ty.scalarType(zcu); - const inst_ty = self.typeOfIndex(inst); - - const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; - const llvm_inst_ty = try o.lowerType(pt, inst_ty); - const llvm_lhs_ty = try o.lowerType(pt, lhs_ty); - const results = - try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, ""); - - const result_val = try self.wip.extractValue(results, &.{0}, ""); - const overflow_bit = try self.wip.extractValue(results, &.{1}, ""); - - const result_index = o.llvmFieldIndex(inst_ty, 0).?; - const overflow_index = o.llvmFieldIndex(inst_ty, 1).?; - - if (isByRef(inst_ty, zcu)) { - const result_alignment = inst_ty.abiAlignment(zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment); - { - const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, ""); - _ = try self.wip.store(.normal, result_val, field_ptr, result_alignment); - } - { - const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, overflow_index, ""); - _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1)); - } - - return alloca_inst; - } - - var fields: [2]Builder.Value = undefined; - fields[result_index] = result_val; - fields[overflow_index] = overflow_bit; - return self.wip.buildAggregate(llvm_inst_ty, &fields, ""); - } - - fn buildElementwiseCall( - self: *FuncGen, - llvm_fn: Builder.Function.Index, - args_vectors: []const Builder.Value, - result_vector: Builder.Value, - vector_len: usize, - ) !Builder.Value { - const o = self.ng.object; - assert(args_vectors.len <= 3); - - var i: usize = 0; - var result = result_vector; - while (i < vector_len) : (i += 1) { - const index_i32 = try o.builder.intValue(.i32, i); - - var args: [3]Builder.Value = undefined; - for (args[0..args_vectors.len], args_vectors) |*arg_elem, arg_vector| { - arg_elem.* = try self.wip.extractElement(arg_vector, index_i32, ""); - } - const result_elem = try self.wip.call( - .normal, - .ccc, - .none, - llvm_fn.typeOf(&o.builder), - llvm_fn.toValue(&o.builder), - args[0..args_vectors.len], - "", - ); - result = try self.wip.insertElement(result, result_elem, index_i32, ""); - } - return result; - } - - fn getLibcFunction( - self: *FuncGen, - fn_name: Builder.StrtabString, - param_types: []const Builder.Type, - return_type: Builder.Type, - ) Allocator.Error!Builder.Function.Index { - const o = self.ng.object; - if (o.builder.getGlobal(fn_name)) |global| return switch (global.ptrConst(&o.builder).kind) { - .alias => |alias| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function, - .function => |function| function, - .variable, .replaced => unreachable, - }; - return o.builder.addFunction( - try o.builder.fnType(return_type, param_types, .normal), - fn_name, - toLlvmAddressSpace(.generic, self.ng.pt.zcu.getTarget()), - ); - } - - /// Creates a floating point comparison by lowering to the appropriate - /// hardware instruction or softfloat routine for the target - fn buildFloatCmp( - self: *FuncGen, - fast: Builder.FastMathKind, - pred: math.CompareOperator, - ty: Type, - params: [2]Builder.Value, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const target = zcu.getTarget(); - const scalar_ty = ty.scalarType(zcu); - const scalar_llvm_ty = try o.lowerType(pt, scalar_ty); - - if (intrinsicsAllowed(scalar_ty, target)) { - const cond: Builder.FloatCondition = switch (pred) { - .eq => .oeq, - .neq => .une, - .lt => .olt, - .lte => .ole, - .gt => .ogt, - .gte => .oge, - }; - return self.wip.fcmp(fast, cond, params[0], params[1], ""); - } - - const float_bits = scalar_ty.floatBits(target); - const compiler_rt_float_abbrev = compilerRtFloatAbbrev(float_bits); - const fn_base_name = switch (pred) { - .neq => "ne", - .eq => "eq", - .lt => "lt", - .lte => "le", - .gt => "gt", - .gte => "ge", - }; - const fn_name = try o.builder.strtabStringFmt("__{s}{s}f2", .{ fn_base_name, compiler_rt_float_abbrev }); - - const libc_fn = try self.getLibcFunction(fn_name, &.{ scalar_llvm_ty, scalar_llvm_ty }, .i32); - - const int_cond: Builder.IntegerCondition = switch (pred) { - .eq => .eq, - .neq => .ne, - .lt => .slt, - .lte => .sle, - .gt => .sgt, - .gte => .sge, - }; - - if (ty.zigTypeTag(zcu) == .vector) { - const vec_len = ty.vectorLen(zcu); - const vector_result_ty = try o.builder.vectorType(.normal, vec_len, .i32); - - const init = try o.builder.poisonValue(vector_result_ty); - const result = try self.buildElementwiseCall(libc_fn, ¶ms, init, vec_len); - - const zero_vector = try o.builder.splatValue(vector_result_ty, .@"0"); - return self.wip.icmp(int_cond, result, zero_vector, ""); - } - - const result = try self.wip.call( - .normal, - .ccc, - .none, - libc_fn.typeOf(&o.builder), - libc_fn.toValue(&o.builder), - ¶ms, - "", - ); - return self.wip.icmp(int_cond, result, .@"0", ""); - } - - const FloatOp = enum { - add, - ceil, - cos, - div, - exp, - exp2, - fabs, - floor, - fma, - fmax, - fmin, - fmod, - log, - log10, - log2, - mul, - neg, - round, - sin, - sqrt, - sub, - tan, - trunc, - }; - - const FloatOpStrat = union(enum) { - intrinsic: []const u8, - libc: Builder.String, - }; - - /// Creates a floating point operation (add, sub, fma, sqrt, exp, etc.) - /// by lowering to the appropriate hardware instruction or softfloat - /// routine for the target - fn buildFloatOp( - self: *FuncGen, - comptime op: FloatOp, - fast: Builder.FastMathKind, - ty: Type, - comptime params_len: usize, - params: [params_len]Builder.Value, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const target = zcu.getTarget(); - const scalar_ty = ty.scalarType(zcu); - const llvm_ty = try o.lowerType(pt, ty); - - if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) { - // Some operations are dedicated LLVM instructions, not available as intrinsics - .neg => return self.wip.un(.fneg, params[0], ""), - .add, .sub, .mul, .div, .fmod => return self.wip.bin(switch (fast) { - .normal => switch (op) { - .add => .fadd, - .sub => .fsub, - .mul => .fmul, - .div => .fdiv, - .fmod => .frem, - else => unreachable, - }, - .fast => switch (op) { - .add => .@"fadd fast", - .sub => .@"fsub fast", - .mul => .@"fmul fast", - .div => .@"fdiv fast", - .fmod => .@"frem fast", - else => unreachable, - }, - }, params[0], params[1], ""), - .fmax, - .fmin, - .ceil, - .cos, - .exp, - .exp2, - .fabs, - .floor, - .log, - .log10, - .log2, - .round, - .sin, - .sqrt, - .trunc, - .fma, - => return self.wip.callIntrinsic(fast, .none, switch (op) { - .fmax => .maxnum, - .fmin => .minnum, - .ceil => .ceil, - .cos => .cos, - .exp => .exp, - .exp2 => .exp2, - .fabs => .fabs, - .floor => .floor, - .log => .log, - .log10 => .log10, - .log2 => .log2, - .round => .round, - .sin => .sin, - .sqrt => .sqrt, - .trunc => .trunc, - .fma => .fma, - else => unreachable, - }, &.{llvm_ty}, ¶ms, ""), - .tan => unreachable, - }; - - const float_bits = scalar_ty.floatBits(target); - const fn_name = switch (op) { - .neg => { - // In this case we can generate a softfloat negation by XORing the - // bits with a constant. - const int_ty = try o.builder.intType(@intCast(float_bits)); - const cast_ty = try llvm_ty.changeScalar(int_ty, &o.builder); - const sign_mask = try o.builder.splatValue( - cast_ty, - try o.builder.intConst(int_ty, @as(u128, 1) << @intCast(float_bits - 1)), - ); - const bitcasted_operand = try self.wip.cast(.bitcast, params[0], cast_ty, ""); - const result = try self.wip.bin(.xor, bitcasted_operand, sign_mask, ""); - return self.wip.cast(.bitcast, result, llvm_ty, ""); - }, - .add, .sub, .div, .mul => try o.builder.strtabStringFmt("__{s}{s}f3", .{ - @tagName(op), compilerRtFloatAbbrev(float_bits), - }), - .ceil, - .cos, - .exp, - .exp2, - .fabs, - .floor, - .fma, - .fmax, - .fmin, - .fmod, - .log, - .log10, - .log2, - .round, - .sin, - .sqrt, - .tan, - .trunc, - => try o.builder.strtabStringFmt("{s}{s}{s}", .{ - libcFloatPrefix(float_bits), @tagName(op), libcFloatSuffix(float_bits), - }), - }; - - const scalar_llvm_ty = llvm_ty.scalarType(&o.builder); - const libc_fn = try self.getLibcFunction( - fn_name, - ([1]Builder.Type{scalar_llvm_ty} ** 3)[0..params.len], - scalar_llvm_ty, - ); - if (ty.zigTypeTag(zcu) == .vector) { - const result = try o.builder.poisonValue(llvm_ty); - return self.buildElementwiseCall(libc_fn, ¶ms, result, ty.vectorLen(zcu)); - } - - return self.wip.call( - fast.toCallKind(), - .ccc, - .none, - libc_fn.typeOf(&o.builder), - libc_fn.toValue(&o.builder), - ¶ms, - "", - ); - } - - fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Bin, pl_op.payload).data; - - const mulend1 = try self.resolveInst(extra.lhs); - const mulend2 = try self.resolveInst(extra.rhs); - const addend = try self.resolveInst(pl_op.operand); - - const ty = self.typeOfIndex(inst); - return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend }); - } - - fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; - - const lhs = try self.resolveInst(extra.lhs); - const rhs = try self.resolveInst(extra.rhs); - - const lhs_ty = self.typeOf(extra.lhs); - if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu)) - return self.ng.todo("implement vector shifts with scalar rhs", .{}); - const lhs_scalar_ty = lhs_ty.scalarType(zcu); - - const dest_ty = self.typeOfIndex(inst); - const llvm_dest_ty = try o.lowerType(pt, dest_ty); - - const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); - - const result = try self.wip.bin(.shl, lhs, casted_rhs, ""); - const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu)) - .ashr - else - .lshr, result, casted_rhs, ""); - - const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, ""); - - const result_index = o.llvmFieldIndex(dest_ty, 0).?; - const overflow_index = o.llvmFieldIndex(dest_ty, 1).?; - - if (isByRef(dest_ty, zcu)) { - const result_alignment = dest_ty.abiAlignment(zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment); - { - const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, ""); - _ = try self.wip.store(.normal, result, field_ptr, result_alignment); - } - { - const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, overflow_index, ""); - _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1)); - } - return alloca_inst; - } - - var fields: [2]Builder.Value = undefined; - fields[result_index] = result; - fields[overflow_index] = overflow_bit; - return self.wip.buildAggregate(llvm_dest_ty, &fields, ""); - } - - fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - return self.wip.bin(.@"and", lhs, rhs, ""); - } - - fn airOr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - return self.wip.bin(.@"or", lhs, rhs, ""); - } - - fn airXor(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - return self.wip.bin(.xor, lhs, rhs, ""); - } - - fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const lhs_ty = self.typeOf(bin_op.lhs); - if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) - return self.ng.todo("implement vector shifts with scalar rhs", .{}); - const lhs_scalar_ty = lhs_ty.scalarType(zcu); - - const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); - return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu)) - .@"shl nsw" - else - .@"shl nuw", lhs, casted_rhs, ""); - } - - fn airShl(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const lhs_ty = self.typeOf(bin_op.lhs); - if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) - return self.ng.todo("implement vector shifts with scalar rhs", .{}); - - const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); - return self.wip.bin(.shl, lhs, casted_rhs, ""); - } - - fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const lhs_ty = self.typeOf(bin_op.lhs); - const lhs_info = lhs_ty.intInfo(zcu); - const llvm_lhs_ty = try o.lowerType(pt, lhs_ty); - const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder); - - const rhs_ty = self.typeOf(bin_op.rhs); - if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) - return self.ng.todo("implement vector shifts with scalar rhs", .{}); - const rhs_info = rhs_ty.intInfo(zcu); - assert(rhs_info.signedness == .unsigned); - const llvm_rhs_ty = try o.lowerType(pt, rhs_ty); - const llvm_rhs_scalar_ty = llvm_rhs_ty.scalarType(&o.builder); - - const result = try self.wip.callIntrinsic( - .normal, - .none, - switch (lhs_info.signedness) { - .signed => .@"sshl.sat", - .unsigned => .@"ushl.sat", - }, - &.{llvm_lhs_ty}, - &.{ lhs, try self.wip.conv(.unsigned, rhs, llvm_lhs_ty, "") }, - "", - ); - - // LLVM langref says "If b is (statically or dynamically) equal to or - // larger than the integer bit width of the arguments, the result is a - // poison value." - // However Zig semantics says that saturating shift left can never produce - // undefined; instead it saturates. - if (rhs_info.bits <= math.log2_int(u16, lhs_info.bits)) return result; - const bits = try o.builder.splatValue( - llvm_rhs_ty, - try o.builder.intConst(llvm_rhs_scalar_ty, lhs_info.bits), - ); - const in_range = try self.wip.icmp(.ult, rhs, bits, ""); - const lhs_sat = lhs_sat: switch (lhs_info.signedness) { - .signed => { - const zero = try o.builder.splatValue( - llvm_lhs_ty, - try o.builder.intConst(llvm_lhs_scalar_ty, 0), - ); - const smin = try o.builder.splatValue( - llvm_lhs_ty, - try minIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu), - ); - const smax = try o.builder.splatValue( - llvm_lhs_ty, - try maxIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu), - ); - const lhs_lt_zero = try self.wip.icmp(.slt, lhs, zero, ""); - const slimit = try self.wip.select(.normal, lhs_lt_zero, smin, smax, ""); - const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, ""); - break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, slimit, ""); - }, - .unsigned => { - const zero = try o.builder.splatValue( - llvm_lhs_ty, - try o.builder.intConst(llvm_lhs_scalar_ty, 0), - ); - const umax = try o.builder.splatValue( - llvm_lhs_ty, - try o.builder.intConst(llvm_lhs_scalar_ty, -1), - ); - const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, ""); - break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, umax, ""); - }, - }; - return self.wip.select(.normal, in_range, result, lhs_sat, ""); - } - - fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const lhs_ty = self.typeOf(bin_op.lhs); - if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) - return self.ng.todo("implement vector shifts with scalar rhs", .{}); - const lhs_scalar_ty = lhs_ty.scalarType(zcu); - - const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); - const is_signed_int = lhs_scalar_ty.isSignedInt(zcu); - - return self.wip.bin(if (is_exact) - if (is_signed_int) .@"ashr exact" else .@"lshr exact" - else if (is_signed_int) .ashr else .lshr, lhs, casted_rhs, ""); - } - - fn airAbs(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const operand_ty = self.typeOf(ty_op.operand); - const scalar_ty = operand_ty.scalarType(zcu); - - switch (scalar_ty.zigTypeTag(zcu)) { - .int => return self.wip.callIntrinsic( - .normal, - .none, - .abs, - &.{try o.lowerType(pt, operand_ty)}, - &.{ operand, try o.builder.intValue(.i1, 0) }, - "", - ), - .float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}), - else => unreachable, - } - } - - fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { - const o = fg.ng.object; - const pt = fg.ng.pt; - const zcu = pt.zcu; - const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const dest_ty = fg.typeOfIndex(inst); - const dest_llvm_ty = try o.lowerType(pt, dest_ty); - const operand = try fg.resolveInst(ty_op.operand); - const operand_ty = fg.typeOf(ty_op.operand); - const operand_info = operand_ty.intInfo(zcu); - - const dest_is_enum = dest_ty.zigTypeTag(zcu) == .@"enum"; - - bounds_check: { - const dest_scalar = dest_ty.scalarType(zcu); - const operand_scalar = operand_ty.scalarType(zcu); - - const dest_info = dest_ty.intInfo(zcu); - - const have_min_check, const have_max_check = c: { - const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed); - const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed); - - const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0; - const operand_maybe_neg = operand_info.signedness == .signed and operand_info.bits > 0; - - break :c .{ - operand_maybe_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits), - dest_pos_bits < operand_pos_bits, - }; - }; - - if (!have_min_check and !have_max_check) break :bounds_check; - - const operand_llvm_ty = try o.lowerType(pt, operand_ty); - const operand_scalar_llvm_ty = try o.lowerType(pt, operand_scalar); - - const is_vector = operand_ty.zigTypeTag(zcu) == .vector; - assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector)); - - const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .integer_out_of_bounds; - - if (have_min_check) { - const min_const_scalar = try minIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu); - const min_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, min_const_scalar) else min_const_scalar.toValue(); - const ok_maybe_vec = try fg.cmp(.normal, .gte, operand_ty, operand, min_val); - const ok = if (is_vector) ok: { - const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip); - break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, ""); - } else ok_maybe_vec; - if (safety) { - const fail_block = try fg.wip.block(1, "IntMinFail"); - const ok_block = try fg.wip.block(1, "IntMinOk"); - _ = try fg.wip.brCond(ok, ok_block, fail_block, .none); - fg.wip.cursor = .{ .block = fail_block }; - try fg.buildSimplePanic(panic_id); - fg.wip.cursor = .{ .block = ok_block }; - } else { - _ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, ""); - } - } - - if (have_max_check) { - const max_const_scalar = try maxIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu); - const max_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, max_const_scalar) else max_const_scalar.toValue(); - const ok_maybe_vec = try fg.cmp(.normal, .lte, operand_ty, operand, max_val); - const ok = if (is_vector) ok: { - const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip); - break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, ""); - } else ok_maybe_vec; - if (safety) { - const fail_block = try fg.wip.block(1, "IntMaxFail"); - const ok_block = try fg.wip.block(1, "IntMaxOk"); - _ = try fg.wip.brCond(ok, ok_block, fail_block, .none); - fg.wip.cursor = .{ .block = fail_block }; - try fg.buildSimplePanic(panic_id); - fg.wip.cursor = .{ .block = ok_block }; - } else { - _ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, ""); - } - } - } - - const result = try fg.wip.conv(switch (operand_info.signedness) { - .signed => .signed, - .unsigned => .unsigned, - }, operand, dest_llvm_ty, ""); - - if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) { - const llvm_fn = try fg.getIsNamedEnumValueFunction(dest_ty); - const is_valid_enum_val = try fg.wip.call( - .normal, - .fastcc, - .none, - llvm_fn.typeOf(&o.builder), - llvm_fn.toValue(&o.builder), - &.{result}, - "", - ); - const fail_block = try fg.wip.block(1, "ValidEnumFail"); - const ok_block = try fg.wip.block(1, "ValidEnumOk"); - _ = try fg.wip.brCond(is_valid_enum_val, ok_block, fail_block, .none); - fg.wip.cursor = .{ .block = fail_block }; - try fg.buildSimplePanic(.invalid_enum_value); - fg.wip.cursor = .{ .block = ok_block }; - } - - return result; - } - - fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const dest_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst)); - return self.wip.cast(.trunc, operand, dest_llvm_ty, ""); - } - - fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const operand_ty = self.typeOf(ty_op.operand); - const dest_ty = self.typeOfIndex(inst); - const target = zcu.getTarget(); - - if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) { - return self.wip.cast(.fptrunc, operand, try o.lowerType(pt, dest_ty), ""); - } else { - const operand_llvm_ty = try o.lowerType(pt, operand_ty); - const dest_llvm_ty = try o.lowerType(pt, dest_ty); - - const dest_bits = dest_ty.floatBits(target); - const src_bits = operand_ty.floatBits(target); - const fn_name = try o.builder.strtabStringFmt("__trunc{s}f{s}f2", .{ - compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits), - }); - - const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty); - return self.wip.call( - .normal, - .ccc, - .none, - libc_fn.typeOf(&o.builder), - libc_fn.toValue(&o.builder), - &.{operand}, - "", - ); - } - } - - fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const operand_ty = self.typeOf(ty_op.operand); - const dest_ty = self.typeOfIndex(inst); - const target = zcu.getTarget(); - - if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) { - return self.wip.cast(.fpext, operand, try o.lowerType(pt, dest_ty), ""); - } else { - const operand_llvm_ty = try o.lowerType(pt, operand_ty); - const dest_llvm_ty = try o.lowerType(pt, dest_ty); - - const dest_bits = dest_ty.scalarType(zcu).floatBits(target); - const src_bits = operand_ty.scalarType(zcu).floatBits(target); - const fn_name = try o.builder.strtabStringFmt("__extend{s}f{s}f2", .{ - compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits), - }); - - const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty); - if (dest_ty.isVector(zcu)) return self.buildElementwiseCall( - libc_fn, - &.{operand}, - try o.builder.poisonValue(dest_llvm_ty), - dest_ty.vectorLen(zcu), - ); - return self.wip.call( - .normal, - .ccc, - .none, - libc_fn.typeOf(&o.builder), - libc_fn.toValue(&o.builder), - &.{operand}, - "", - ); - } - } - - fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand_ty = self.typeOf(ty_op.operand); - const inst_ty = self.typeOfIndex(inst); - const operand = try self.resolveInst(ty_op.operand); - return self.bitCast(operand, operand_ty, inst_ty); - } - - fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const operand_is_ref = isByRef(operand_ty, zcu); - const result_is_ref = isByRef(inst_ty, zcu); - const llvm_dest_ty = try o.lowerType(pt, inst_ty); - - if (operand_is_ref and result_is_ref) { - // They are both pointers, so just return the same opaque pointer :) - return operand; - } - - if (llvm_dest_ty.isInteger(&o.builder) and - operand.typeOfWip(&self.wip).isInteger(&o.builder)) - { - return self.wip.conv(.unsigned, operand, llvm_dest_ty, ""); - } - - const operand_scalar_ty = operand_ty.scalarType(zcu); - const inst_scalar_ty = inst_ty.scalarType(zcu); - if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) { - return self.wip.cast(.inttoptr, operand, llvm_dest_ty, ""); - } - if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) { - return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, ""); - } - - if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) { - const elem_ty = operand_ty.childType(zcu); - if (!result_is_ref) { - return self.ng.todo("implement bitcast vector to non-ref array", .{}); - } - const alignment = inst_ty.abiAlignment(zcu).toLlvm(); - const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment); - const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; - if (bitcast_ok) { - _ = try self.wip.store(.normal, operand, array_ptr, alignment); - } else { - // If the ABI size of the element type is not evenly divisible by size in bits; - // a simple bitcast will not work, and we fall back to extractelement. - const llvm_usize = try o.lowerType(pt, Type.usize); - const usize_zero = try o.builder.intValue(llvm_usize, 0); - const vector_len = operand_ty.arrayLen(zcu); - var i: u64 = 0; - while (i < vector_len) : (i += 1) { - const elem_ptr = try self.wip.gep(.inbounds, llvm_dest_ty, array_ptr, &.{ - usize_zero, try o.builder.intValue(llvm_usize, i), - }, ""); - const elem = - try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), ""); - _ = try self.wip.store(.normal, elem, elem_ptr, .default); - } - } - return array_ptr; - } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) { - const elem_ty = operand_ty.childType(zcu); - const llvm_vector_ty = try o.lowerType(pt, inst_ty); - if (!operand_is_ref) return self.ng.todo("implement bitcast non-ref array to vector", .{}); - - const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; - if (bitcast_ok) { - // The array is aligned to the element's alignment, while the vector might have a completely - // different alignment. This means we need to enforce the alignment of this load. - const alignment = elem_ty.abiAlignment(zcu).toLlvm(); - return self.wip.load(.normal, llvm_vector_ty, operand, alignment, ""); - } else { - // If the ABI size of the element type is not evenly divisible by size in bits; - // a simple bitcast will not work, and we fall back to extractelement. - const array_llvm_ty = try o.lowerType(pt, operand_ty); - const elem_llvm_ty = try o.lowerType(pt, elem_ty); - const llvm_usize = try o.lowerType(pt, Type.usize); - const usize_zero = try o.builder.intValue(llvm_usize, 0); - const vector_len = operand_ty.arrayLen(zcu); - var vector = try o.builder.poisonValue(llvm_vector_ty); - var i: u64 = 0; - while (i < vector_len) : (i += 1) { - const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, operand, &.{ - usize_zero, try o.builder.intValue(llvm_usize, i), - }, ""); - const elem = try self.wip.load(.normal, elem_llvm_ty, elem_ptr, .default, ""); - vector = - try self.wip.insertElement(vector, elem, try o.builder.intValue(.i32, i), ""); - } - return vector; - } - } - - if (operand_is_ref) { - const alignment = operand_ty.abiAlignment(zcu).toLlvm(); - return self.wip.load(.normal, llvm_dest_ty, operand, alignment, ""); - } - - if (result_is_ref) { - const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm(); - const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment); - _ = try self.wip.store(.normal, operand, result_ptr, alignment); - return result_ptr; - } - - if (llvm_dest_ty.isStruct(&o.builder) or - ((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and - operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu))) - { - // Both our operand and our result are values, not pointers, - // but LLVM won't let us bitcast struct values or vectors with padding bits. - // Therefore, we store operand to alloca, then load for result. - const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm(); - const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment); - _ = try self.wip.store(.normal, operand, result_ptr, alignment); - return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, ""); - } - - return self.wip.cast(.bitcast, operand, llvm_dest_ty, ""); - } - - fn airArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const arg_val = self.args[self.arg_index]; - self.arg_index += 1; - - // llvm does not support debug info for naked function arguments - if (self.is_naked) return arg_val; - - const inst_ty = self.typeOfIndex(inst); - - const func = zcu.funcInfo(zcu.navValue(self.ng.nav_index).toIntern()); - const func_zir = func.zir_body_inst.resolveFull(&zcu.intern_pool).?; - const file = zcu.fileByIndex(func_zir.file); - - const mod = file.mod.?; - if (mod.strip) return arg_val; - const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg; - const zir = &file.zir.?; - const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?); - - const lbrace_line = zcu.navSrcLine(func.owner_nav) + func.lbrace_line + 1; - const lbrace_col = func.lbrace_column + 1; - - const debug_parameter = try o.builder.debugParameter( - if (name.len > 0) try o.builder.metadataString(name) else null, - self.file, - self.scope, - lbrace_line, - try o.getDebugType(pt, inst_ty), - self.arg_index, - ); - - const old_location = self.wip.debug_location; - self.wip.debug_location = .{ .location = .{ - .line = lbrace_line, - .column = lbrace_col, - .scope = self.scope.toOptional(), - .inlined_at = .none, - } }; - - if (isByRef(inst_ty, zcu)) { - _ = try self.wip.callIntrinsic( - .normal, - .none, - .@"dbg.declare", - &.{}, - &.{ - (try self.wip.debugValue(arg_val)).toValue(), - debug_parameter.toValue(), - (try o.builder.debugExpression(&.{})).toValue(), - }, - "", - ); - } else if (mod.optimize_mode == .Debug) { - const alignment = inst_ty.abiAlignment(zcu).toLlvm(); - const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment); - _ = try self.wip.store(.normal, arg_val, alloca, alignment); - _ = try self.wip.callIntrinsic( - .normal, - .none, - .@"dbg.declare", - &.{}, - &.{ - (try self.wip.debugValue(alloca)).toValue(), - debug_parameter.toValue(), - (try o.builder.debugExpression(&.{})).toValue(), - }, - "", - ); - } else { - _ = try self.wip.callIntrinsic( - .normal, - .none, - .@"dbg.value", - &.{}, - &.{ - (try self.wip.debugValue(arg_val)).toValue(), - debug_parameter.toValue(), - (try o.builder.debugExpression(&.{})).toValue(), - }, - "", - ); - } - - self.wip.debug_location = old_location; - return arg_val; - } - - fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ptr_ty = self.typeOfIndex(inst); - const pointee_type = ptr_ty.childType(zcu); - if (!pointee_type.hasRuntimeBits(zcu)) - return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue(); - - const pointee_llvm_ty = try o.lowerType(pt, pointee_type); - const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); - return self.buildAlloca(pointee_llvm_ty, alignment); - } - - fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ptr_ty = self.typeOfIndex(inst); - const ret_ty = ptr_ty.childType(zcu); - if (!ret_ty.hasRuntimeBits(zcu)) - return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue(); - if (self.ret_ptr != .none) return self.ret_ptr; - const ret_llvm_ty = try o.lowerType(pt, ret_ty); - const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); - return self.buildAlloca(ret_llvm_ty, alignment); - } - - /// Use this instead of builder.buildAlloca, because this function makes sure to - /// put the alloca instruction at the top of the function! - fn buildAlloca( - self: *FuncGen, - llvm_ty: Builder.Type, - alignment: Builder.Alignment, - ) Allocator.Error!Builder.Value { - const target = self.ng.pt.zcu.getTarget(); - return buildAllocaInner(&self.wip, llvm_ty, alignment, target); - } - - fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const dest_ptr = try self.resolveInst(bin_op.lhs); - const ptr_ty = self.typeOf(bin_op.lhs); - const operand_ty = ptr_ty.childType(zcu); - - const val_is_undef = if (try self.air.value(bin_op.rhs, pt)) |val| val.isUndef(zcu) else false; - if (val_is_undef) { - const owner_mod = self.ng.ownerModule(); - - // Even if safety is disabled, we still emit a memset to undefined since it conveys - // extra information to LLVM, and LLVM will optimize it out. Safety makes the difference - // between using 0xaa or actual undefined for the fill byte. - // - // However, for Debug builds specifically, we avoid emitting the memset because LLVM - // will neither use the information nor get rid of the memset, thus leaving an - // unexpected call in the user's code. This is problematic if the code in question is - // not ready to correctly make calls yet, such as in our early PIE startup code, or in - // the early stages of a dynamic linker, etc. - if (!safety and owner_mod.optimize_mode == .Debug) { - return .none; - } - - const ptr_info = ptr_ty.ptrInfo(zcu); - const needs_bitmask = (ptr_info.packed_offset.host_size != 0); - if (needs_bitmask) { - // TODO: only some bits are to be undef, we cannot write with a simple memset. - // meanwhile, ignore the write rather than stomping over valid bits. - // https://github.com/ziglang/zig/issues/15337 - return .none; - } - - self.maybeMarkAllowZeroAccess(ptr_info); - - const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), operand_ty.abiSize(zcu)); - _ = try self.wip.callMemSet( - dest_ptr, - ptr_ty.ptrAlignment(zcu).toLlvm(), - if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8), - len, - if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, - self.disable_intrinsics, - ); - if (safety and owner_mod.valgrind) { - try self.valgrindMarkUndef(dest_ptr, len); - } - return .none; - } - - self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - - const src_operand = try self.resolveInst(bin_op.rhs); - try self.store(dest_ptr, ptr_ty, src_operand, .none); - return .none; - } - - fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const pt = fg.ng.pt; - const zcu = pt.zcu; - const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const ptr_ty = fg.typeOf(ty_op.operand); - const ptr_info = ptr_ty.ptrInfo(zcu); - const ptr = try fg.resolveInst(ty_op.operand); - fg.maybeMarkAllowZeroAccess(ptr_info); - return fg.load(ptr, ptr_ty); - } - - fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !void { - _ = inst; - const target = self.ng.object.target; - if ((target.cpu.arch == .mips or target.cpu.arch == .mipsel) and - target.cpu.has(.mips, .notraps)) - { - // Emit a MIPS `break` instruction followed by an infinite loop (to fulfill the noreturn) - // since this CPU does not support trap instructions. - const o = self.ng.object; - _ = try self.wip.callAsm( - .none, - try o.builder.fnType(.void, &.{}, .normal), - .{ .sideeffect = true }, - try o.builder.string("break\n0:\nj 0b\nnop"), - try o.builder.string("~{memory}"), - &.{}, - "", - ); - } else { - _ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, ""); - } - _ = try self.wip.@"unreachable"(); - } - - fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - _ = inst; - _ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, ""); - return .none; - } - - fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - _ = inst; - const o = self.ng.object; - const pt = self.ng.pt; - const llvm_usize = try o.lowerType(pt, Type.usize); - if (!target_util.supportsReturnAddress(self.ng.pt.zcu.getTarget(), self.ng.ownerModule().optimize_mode)) { - // https://github.com/ziglang/zig/issues/11946 - return o.builder.intValue(llvm_usize, 0); - } - const result = try self.wip.callIntrinsic(.normal, .none, .returnaddress, &.{}, &.{.@"0"}, ""); - return self.wip.cast(.ptrtoint, result, llvm_usize, ""); - } - - fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - _ = inst; - const o = self.ng.object; - const pt = self.ng.pt; - const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, ""); - return self.wip.cast(.ptrtoint, result, try o.lowerType(pt, Type.usize), ""); - } - - fn airCmpxchg( - self: *FuncGen, - inst: Air.Inst.Index, - kind: Builder.Function.Instruction.CmpXchg.Kind, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data; - const ptr = try self.resolveInst(extra.ptr); - const ptr_ty = self.typeOf(extra.ptr); - var expected_value = try self.resolveInst(extra.expected_value); - var new_value = try self.resolveInst(extra.new_value); - const operand_ty = ptr_ty.childType(zcu); - const llvm_operand_ty = try o.lowerType(pt, operand_ty); - const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, false); - if (llvm_abi_ty != .none) { - // operand needs widening and truncating - const signedness: Builder.Function.Instruction.Cast.Signedness = - if (operand_ty.isSignedInt(zcu)) .signed else .unsigned; - expected_value = try self.wip.conv(signedness, expected_value, llvm_abi_ty, ""); - new_value = try self.wip.conv(signedness, new_value, llvm_abi_ty, ""); - } - - self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - - const result = try self.wip.cmpxchg( - kind, - if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, - ptr, - expected_value, - new_value, - self.sync_scope, - toLlvmAtomicOrdering(extra.successOrder()), - toLlvmAtomicOrdering(extra.failureOrder()), - ptr_ty.ptrAlignment(zcu).toLlvm(), - "", - ); - - const optional_ty = self.typeOfIndex(inst); - - var payload = try self.wip.extractValue(result, &.{0}, ""); - if (llvm_abi_ty != .none) payload = try self.wip.cast(.trunc, payload, llvm_operand_ty, ""); - const success_bit = try self.wip.extractValue(result, &.{1}, ""); - - if (optional_ty.optionalReprIsPayload(zcu)) { - const zero = try o.builder.zeroInitValue(payload.typeOfWip(&self.wip)); - return self.wip.select(.normal, success_bit, zero, payload, ""); - } - - comptime assert(optional_layout_version == 3); - - const non_null_bit = try self.wip.not(success_bit, ""); - return buildOptional(self, optional_ty, payload, non_null_bit); - } - - fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; - const ptr = try self.resolveInst(pl_op.operand); - const ptr_ty = self.typeOf(pl_op.operand); - const operand_ty = ptr_ty.childType(zcu); - const operand = try self.resolveInst(extra.operand); - const is_signed_int = operand_ty.isSignedInt(zcu); - const is_float = operand_ty.isRuntimeFloat(); - const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float); - const ordering = toLlvmAtomicOrdering(extra.ordering()); - const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, op == .xchg); - const llvm_operand_ty = try o.lowerType(pt, operand_ty); - - const access_kind: Builder.MemoryAccessKind = - if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); - - self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - - if (llvm_abi_ty != .none) { - // operand needs widening and truncating or bitcasting. - return self.wip.cast(if (is_float) .bitcast else .trunc, try self.wip.atomicrmw( - access_kind, - op, - ptr, - try self.wip.cast( - if (is_float) .bitcast else if (is_signed_int) .sext else .zext, - operand, - llvm_abi_ty, - "", - ), - self.sync_scope, - ordering, - ptr_alignment, - "", - ), llvm_operand_ty, ""); - } - - if (!llvm_operand_ty.isPointer(&o.builder)) return self.wip.atomicrmw( - access_kind, - op, - ptr, - operand, - self.sync_scope, - ordering, - ptr_alignment, - "", - ); - - // It's a pointer but we need to treat it as an int. - return self.wip.cast(.inttoptr, try self.wip.atomicrmw( - access_kind, - op, - ptr, - try self.wip.cast(.ptrtoint, operand, try o.lowerType(pt, Type.usize), ""), - self.sync_scope, - ordering, - ptr_alignment, - "", - ), llvm_operand_ty, ""); - } - - fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; - const ptr = try self.resolveInst(atomic_load.ptr); - const ptr_ty = self.typeOf(atomic_load.ptr); - const info = ptr_ty.ptrInfo(zcu); - const elem_ty = Type.fromInterned(info.child); - if (!elem_ty.hasRuntimeBits(zcu)) return .none; - const ordering = toLlvmAtomicOrdering(atomic_load.order); - const llvm_abi_ty = try o.getAtomicAbiType(pt, elem_ty, false); - const ptr_alignment = (if (info.flags.alignment != .none) - @as(InternPool.Alignment, info.flags.alignment) - else - Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm(); - const access_kind: Builder.MemoryAccessKind = - if (info.flags.is_volatile) .@"volatile" else .normal; - const elem_llvm_ty = try o.lowerType(pt, elem_ty); - - self.maybeMarkAllowZeroAccess(info); - - if (llvm_abi_ty != .none) { - // operand needs widening and truncating - const loaded = try self.wip.loadAtomic( - access_kind, - llvm_abi_ty, - ptr, - self.sync_scope, - ordering, - ptr_alignment, - "", - ); - return self.wip.cast(.trunc, loaded, elem_llvm_ty, ""); - } - return self.wip.loadAtomic( - access_kind, - elem_llvm_ty, - ptr, - self.sync_scope, - ordering, - ptr_alignment, - "", - ); - } - - fn airAtomicStore( - self: *FuncGen, - inst: Air.Inst.Index, - ordering: Builder.AtomicOrdering, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const ptr_ty = self.typeOf(bin_op.lhs); - const operand_ty = ptr_ty.childType(zcu); - if (!operand_ty.hasRuntimeBits(zcu)) return .none; - const ptr = try self.resolveInst(bin_op.lhs); - var element = try self.resolveInst(bin_op.rhs); - const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, false); - - if (llvm_abi_ty != .none) { - // operand needs widening - element = try self.wip.conv( - if (operand_ty.isSignedInt(zcu)) .signed else .unsigned, - element, - llvm_abi_ty, - "", - ); - } - - self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - - try self.store(ptr, ptr_ty, element, ordering); - return .none; - } - - fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const dest_slice = try self.resolveInst(bin_op.lhs); - const ptr_ty = self.typeOf(bin_op.lhs); - const elem_ty = self.typeOf(bin_op.rhs); - const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm(); - const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty); - const access_kind: Builder.MemoryAccessKind = - if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - - self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - - if (try self.air.value(bin_op.rhs, pt)) |elem_val| { - if (elem_val.isUndef(zcu)) { - // Even if safety is disabled, we still emit a memset to undefined since it conveys - // extra information to LLVM. However, safety makes the difference between using - // 0xaa or actual undefined for the fill byte. - const fill_byte = if (safety) - try o.builder.intValue(.i8, 0xaa) - else - try o.builder.undefValue(.i8); - const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); - _ = try self.wip.callMemSet( - dest_ptr, - dest_ptr_align, - fill_byte, - len, - access_kind, - self.disable_intrinsics, - ); - const owner_mod = self.ng.ownerModule(); - if (safety and owner_mod.valgrind) { - try self.valgrindMarkUndef(dest_ptr, len); - } - return .none; - } - - // Test if the element value is compile-time known to be a - // repeating byte pattern, for example, `@as(u64, 0)` has a - // repeating byte pattern of 0 bytes. In such case, the memset - // intrinsic can be used. - if (try elem_val.hasRepeatedByteRepr(pt)) |byte_val| { - const fill_byte = try o.builder.intValue(.i8, byte_val); - const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); - _ = try self.wip.callMemSet( - dest_ptr, - dest_ptr_align, - fill_byte, - len, - access_kind, - self.disable_intrinsics, - ); - return .none; - } - } - - const value = try self.resolveInst(bin_op.rhs); - const elem_abi_size = elem_ty.abiSize(zcu); - - if (elem_abi_size == 1) { - // In this case we can take advantage of LLVM's intrinsic. - const fill_byte = try self.bitCast(value, elem_ty, Type.u8); - const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); - - _ = try self.wip.callMemSet( - dest_ptr, - dest_ptr_align, - fill_byte, - len, - access_kind, - self.disable_intrinsics, - ); - return .none; - } - - // non-byte-sized element. lower with a loop. something like this: - - // entry: - // ... - // %end_ptr = getelementptr %ptr, %len - // br %loop - // loop: - // %it_ptr = phi body %next_ptr, entry %ptr - // %end = cmp eq %it_ptr, %end_ptr - // br %end, %body, %end - // body: - // store %it_ptr, %value - // %next_ptr = getelementptr %it_ptr, 1 - // br %loop - // end: - // ... - const entry_block = self.wip.cursor.block; - const loop_block = try self.wip.block(2, "InlineMemsetLoop"); - const body_block = try self.wip.block(1, "InlineMemsetBody"); - const end_block = try self.wip.block(1, "InlineMemsetEnd"); - - const llvm_usize_ty = try o.lowerType(pt, Type.usize); - const len = switch (ptr_ty.ptrSize(zcu)) { - .slice => try self.wip.extractValue(dest_slice, &.{1}, ""), - .one => try o.builder.intValue(llvm_usize_ty, ptr_ty.childType(zcu).arrayLen(zcu)), - .many, .c => unreachable, - }; - const elem_llvm_ty = try o.lowerType(pt, elem_ty); - const end_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, dest_ptr, &.{len}, ""); - _ = try self.wip.br(loop_block); - - self.wip.cursor = .{ .block = loop_block }; - const it_ptr = try self.wip.phi(.ptr, ""); - const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, ""); - _ = try self.wip.brCond(end, body_block, end_block, .none); - - self.wip.cursor = .{ .block = body_block }; - const elem_abi_align = elem_ty.abiAlignment(zcu); - const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm(); - if (isByRef(elem_ty, zcu)) { - _ = try self.wip.callMemCpy( - it_ptr.toValue(), - it_ptr_align, - value, - elem_abi_align.toLlvm(), - try o.builder.intValue(llvm_usize_ty, elem_abi_size), - access_kind, - self.disable_intrinsics, - ); - } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align); - const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{ - try o.builder.intValue(llvm_usize_ty, 1), - }, ""); - _ = try self.wip.br(loop_block); - - self.wip.cursor = .{ .block = end_block }; - it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip); - return .none; - } - - fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const dest_slice = try self.resolveInst(bin_op.lhs); - const dest_ptr_ty = self.typeOf(bin_op.lhs); - const src_slice = try self.resolveInst(bin_op.rhs); - const src_ptr_ty = self.typeOf(bin_op.rhs); - const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty); - const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty); - const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty); - const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or - dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - - self.maybeMarkAllowZeroAccess(dest_ptr_ty.ptrInfo(zcu)); - self.maybeMarkAllowZeroAccess(src_ptr_ty.ptrInfo(zcu)); - - _ = try self.wip.callMemCpy( - dest_ptr, - dest_ptr_ty.ptrAlignment(zcu).toLlvm(), - src_ptr, - src_ptr_ty.ptrAlignment(zcu).toLlvm(), - len, - access_kind, - self.disable_intrinsics, - ); - return .none; - } - - fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const dest_slice = try self.resolveInst(bin_op.lhs); - const dest_ptr_ty = self.typeOf(bin_op.lhs); - const src_slice = try self.resolveInst(bin_op.rhs); - const src_ptr_ty = self.typeOf(bin_op.rhs); - const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty); - const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty); - const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty); - const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or - dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - - _ = try self.wip.callMemMove( - dest_ptr, - dest_ptr_ty.ptrAlignment(zcu).toLlvm(), - src_ptr, - src_ptr_ty.ptrAlignment(zcu).toLlvm(), - len, - access_kind, - ); - return .none; - } - - fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; - const un_ptr_ty = self.typeOf(bin_op.lhs); - const un_ty = un_ptr_ty.childType(zcu); - const layout = un_ty.unionGetLayout(zcu); - if (layout.tag_size == 0) return .none; - - const access_kind: Builder.MemoryAccessKind = - if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - - self.maybeMarkAllowZeroAccess(un_ptr_ty.ptrInfo(zcu)); - - const union_ptr = try self.resolveInst(bin_op.lhs); - const new_tag = try self.resolveInst(bin_op.rhs); - const union_ptr_align = un_ptr_ty.ptrAlignment(zcu); - if (layout.payload_size == 0) { - _ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm()); - return .none; - } - const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); - const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(pt, un_ty), union_ptr, tag_index, ""); - const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) { - 0 => union_ptr_align, - else => |off| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))), - }; - _ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm()); - return .none; - } - - fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const un_ty = self.typeOf(ty_op.operand); - const layout = un_ty.unionGetLayout(zcu); - if (layout.tag_size == 0) return .none; - const union_handle = try self.resolveInst(ty_op.operand); - if (isByRef(un_ty, zcu)) { - const llvm_un_ty = try o.lowerType(pt, un_ty); - if (layout.payload_size == 0) - return self.wip.load(.normal, llvm_un_ty, union_handle, .default, ""); - const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); - const tag_field_ptr = try self.wip.gepStruct(llvm_un_ty, union_handle, tag_index, ""); - const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index]; - return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, ""); - } else { - if (layout.payload_size == 0) return union_handle; - const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); - return self.wip.extractValue(union_handle, &.{tag_index}, ""); - } - } - - fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) !Builder.Value { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - const operand_ty = self.typeOf(un_op); - - return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand}); - } - - fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - const operand_ty = self.typeOf(un_op); - - return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand}); - } - - fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const inst_ty = self.typeOfIndex(inst); - const operand_ty = self.typeOf(ty_op.operand); - const operand = try self.resolveInst(ty_op.operand); - - const result = try self.wip.callIntrinsic( - .normal, - .none, - intrinsic, - &.{try o.lowerType(pt, operand_ty)}, - &.{ operand, .false }, - "", - ); - return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); - } - - fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const inst_ty = self.typeOfIndex(inst); - const operand_ty = self.typeOf(ty_op.operand); - const operand = try self.resolveInst(ty_op.operand); - - const result = try self.wip.callIntrinsic( - .normal, - .none, - intrinsic, - &.{try o.lowerType(pt, operand_ty)}, - &.{operand}, - "", - ); - return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); - } - - fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand_ty = self.typeOf(ty_op.operand); - var bits = operand_ty.intInfo(zcu).bits; - assert(bits % 8 == 0); - - const inst_ty = self.typeOfIndex(inst); - var operand = try self.resolveInst(ty_op.operand); - var llvm_operand_ty = try o.lowerType(pt, operand_ty); - - if (bits % 16 == 8) { - // If not an even byte-multiple, we need zero-extend + shift-left 1 byte - // The truncated result at the end will be the correct bswap - const scalar_ty = try o.builder.intType(@intCast(bits + 8)); - if (operand_ty.zigTypeTag(zcu) == .vector) { - const vec_len = operand_ty.vectorLen(zcu); - llvm_operand_ty = try o.builder.vectorType(.normal, vec_len, scalar_ty); - } else llvm_operand_ty = scalar_ty; - - const shift_amt = - try o.builder.splatValue(llvm_operand_ty, try o.builder.intConst(scalar_ty, 8)); - const extended = try self.wip.cast(.zext, operand, llvm_operand_ty, ""); - operand = try self.wip.bin(.shl, extended, shift_amt, ""); - - bits = bits + 8; - } - - const result = - try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, ""); - return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); - } - - fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const operand = try self.resolveInst(ty_op.operand); - const error_set_ty = ty_op.ty.toType(); - - const names = error_set_ty.errorSetNames(zcu); - const valid_block = try self.wip.block(@intCast(names.len), "Valid"); - const invalid_block = try self.wip.block(1, "Invalid"); - const end_block = try self.wip.block(2, "End"); - var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none); - defer wip_switch.finish(&self.wip); - - for (0..names.len) |name_index| { - const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?; - const this_tag_int_value = try o.builder.intConst(try o.errorIntType(pt), err_int); - try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip); - } - self.wip.cursor = .{ .block = valid_block }; - _ = try self.wip.br(end_block); - - self.wip.cursor = .{ .block = invalid_block }; - _ = try self.wip.br(end_block); - - self.wip.cursor = .{ .block = end_block }; - const phi = try self.wip.phi(.i1, ""); - phi.finish(&.{ .true, .false }, &.{ valid_block, invalid_block }, &self.wip); - return phi.toValue(); - } - - fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - const enum_ty = self.typeOf(un_op); - - const llvm_fn = try self.getIsNamedEnumValueFunction(enum_ty); - return self.wip.call( - .normal, - .fastcc, - .none, - llvm_fn.typeOf(&o.builder), - llvm_fn.toValue(&o.builder), - &.{operand}, - "", - ); - } - - fn getIsNamedEnumValueFunction(self: *FuncGen, enum_ty: Type) !Builder.Function.Index { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - - const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern()); - if (gop.found_existing) return gop.value_ptr.*; - errdefer assert(o.named_enum_map.remove(enum_ty.toIntern())); - const function_index = try o.builder.addFunction( - // Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type. - // TODO: change the builder API so we don't need to do this. - try o.builder.fnType(.void, &.{}, .normal), - try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}), - toLlvmAddressSpace(.generic, zcu.getTarget()), - ); - gop.value_ptr.* = function_index; - try o.updateIsNamedEnumValueFunction(pt, enum_ty, function_index); - return function_index; - } - - fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - const enum_ty = self.typeOf(un_op); - - const llvm_fn = try o.getEnumTagNameFunction(pt, enum_ty); - return self.wip.call( - .normal, - .fastcc, - .none, - llvm_fn.typeOf(&o.builder), - llvm_fn.toValue(&o.builder), - &.{operand}, - "", - ); - } - - fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; - const operand = try self.resolveInst(un_op); - const slice_ty = self.typeOfIndex(inst); - const slice_llvm_ty = try o.lowerType(pt, slice_ty); - - // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed. - const extended_operand = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), ""); - - const error_name_table_ptr = try self.getErrorNameTable(); - const error_name_table = - try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, ""); - const error_name_ptr = - try self.wip.gep(.inbounds, slice_llvm_ty, error_name_table, &.{extended_operand}, ""); - return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, ""); - } - - fn airSplat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const scalar = try self.resolveInst(ty_op.operand); - const vector_ty = self.typeOfIndex(inst); - return self.wip.splatVector(try o.lowerType(pt, vector_ty), scalar, ""); - } - - fn airSelect(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const extra = self.air.extraData(Air.Bin, pl_op.payload).data; - const pred = try self.resolveInst(pl_op.operand); - const a = try self.resolveInst(extra.lhs); - const b = try self.resolveInst(extra.rhs); - - return self.wip.select(.normal, pred, a, b, ""); - } - - fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = fg.ng.object; - const pt = fg.ng.pt; - const zcu = pt.zcu; - const gpa = zcu.gpa; - - const unwrapped = fg.air.unwrapShuffleOne(zcu, inst); - - const operand = try fg.resolveInst(unwrapped.operand); - const mask = unwrapped.mask; - const operand_ty = fg.typeOf(unwrapped.operand); - const llvm_operand_ty = try o.lowerType(pt, operand_ty); - const llvm_result_ty = try o.lowerType(pt, unwrapped.result_ty); - const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu)); - const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty); - const llvm_poison_mask_elem = try o.builder.poisonConst(.i32); - const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32); - - // LLVM requires that the two input vectors have the same length, so lowering isn't trivial. - // And, in the words of jacobly0: "llvm sucks at shuffles so we do have to hold its hand at - // least a bit". So, there are two cases here. - // - // If the operand length equals the mask length, we do just the one `shufflevector`, where - // the second operand is a constant vector with comptime-known elements at the right indices - // and poison values elsewhere (in the indices which won't be selected). - // - // Otherwise, we lower to *two* `shufflevector` instructions. The first shuffles the runtime - // operand with an all-poison vector to extract and correctly position all of the runtime - // elements. We also make a constant vector with all of the comptime elements correctly - // positioned. Then, our second instruction selects elements from those "runtime-or-poison" - // and "comptime-or-poison" vectors to compute the result. - - // This buffer is used primarily for the mask constants. - const llvm_elem_buf = try gpa.alloc(Builder.Constant, mask.len); - defer gpa.free(llvm_elem_buf); - - // ...but first, we'll collect all of the comptime-known values. - var any_defined_comptime_value = false; - for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| { - llvm_elem.* = switch (mask_elem.unwrap()) { - .elem => llvm_poison_elem, - .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: { - any_defined_comptime_value = true; - break :elem try o.lowerValue(pt, val); - } else llvm_poison_elem, - }; - } - // This vector is like the result, but runtime elements are replaced with poison. - const comptime_and_poison: Builder.Value = if (any_defined_comptime_value) vec: { - break :vec try o.builder.vectorValue(llvm_result_ty, llvm_elem_buf); - } else try o.builder.poisonValue(llvm_result_ty); - - if (operand_ty.vectorLen(zcu) == mask.len) { - // input length equals mask/output length, so we lower to one instruction - for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| { - llvm_elem.* = switch (mask_elem.unwrap()) { - .elem => |idx| try o.builder.intConst(.i32, idx), - .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: { - break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx); - } else llvm_poison_mask_elem, - }; - } - return fg.wip.shuffleVector( - operand, - comptime_and_poison, - try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf), - "", - ); - } - - for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| { - llvm_elem.* = switch (mask_elem.unwrap()) { - .elem => |idx| try o.builder.intConst(.i32, idx), - .value => llvm_poison_mask_elem, - }; - } - // This vector is like our result, but all comptime-known elements are poison. - const runtime_and_poison = try fg.wip.shuffleVector( - operand, - try o.builder.poisonValue(llvm_operand_ty), - try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf), - "", - ); - - if (!any_defined_comptime_value) { - // `comptime_and_poison` is just poison; a second shuffle would be a nop. - return runtime_and_poison; - } - - // In this second shuffle, the inputs, the mask, and the output all have the same length. - for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| { - llvm_elem.* = switch (mask_elem.unwrap()) { - .elem => try o.builder.intConst(.i32, elem_idx), - .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: { - break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx); - } else llvm_poison_mask_elem, - }; - } - // Merge the runtime and comptime elements with the mask we just built. - return fg.wip.shuffleVector( - runtime_and_poison, - comptime_and_poison, - try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf), - "", - ); - } - - fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = fg.ng.object; - const pt = fg.ng.pt; - const zcu = pt.zcu; - const gpa = zcu.gpa; - - const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst); - - const mask = unwrapped.mask; - const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu)); - const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32); - const llvm_poison_mask_elem = try o.builder.poisonConst(.i32); - - // This is kind of simpler than in `airShuffleOne`. We extend the shorter vector to the - // length of the longer one with an initial `shufflevector` if necessary, and then do the - // actual computation with a second `shufflevector`. - - const operand_a_len = fg.typeOf(unwrapped.operand_a).vectorLen(zcu); - const operand_b_len = fg.typeOf(unwrapped.operand_b).vectorLen(zcu); - const operand_len: u32 = @max(operand_a_len, operand_b_len); - - // If we need to extend an operand, this is the type that mask will have. - const llvm_operand_mask_ty = try o.builder.vectorType(.normal, operand_len, .i32); - - const llvm_elem_buf = try gpa.alloc(Builder.Constant, @max(mask.len, operand_len)); - defer gpa.free(llvm_elem_buf); - - const operand_a: Builder.Value = extend: { - const raw = try fg.resolveInst(unwrapped.operand_a); - if (operand_a_len == operand_len) break :extend raw; - // Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>` - const mask_elems = llvm_elem_buf[0..operand_len]; - for (mask_elems[0..operand_a_len], 0..) |*llvm_elem, elem_idx| { - llvm_elem.* = try o.builder.intConst(.i32, elem_idx); - } - @memset(mask_elems[operand_a_len..], llvm_poison_mask_elem); - const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_a_len, llvm_elem_ty); - break :extend try fg.wip.shuffleVector( - raw, - try o.builder.poisonValue(llvm_this_operand_ty), - try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems), - "", - ); - }; - const operand_b: Builder.Value = extend: { - const raw = try fg.resolveInst(unwrapped.operand_b); - if (operand_b_len == operand_len) break :extend raw; - // Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>` - const mask_elems = llvm_elem_buf[0..operand_len]; - for (mask_elems[0..operand_b_len], 0..) |*llvm_elem, elem_idx| { - llvm_elem.* = try o.builder.intConst(.i32, elem_idx); - } - @memset(mask_elems[operand_b_len..], llvm_poison_mask_elem); - const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_b_len, llvm_elem_ty); - break :extend try fg.wip.shuffleVector( - raw, - try o.builder.poisonValue(llvm_this_operand_ty), - try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems), - "", - ); - }; - - // `operand_a` and `operand_b` now have the same length (we've extended the shorter one with - // an initial shuffle if necessary). Now for the easy bit. - - const mask_elems = llvm_elem_buf[0..mask.len]; - for (mask, mask_elems) |mask_elem, *llvm_mask_elem| { - llvm_mask_elem.* = switch (mask_elem.unwrap()) { - .a_elem => |idx| try o.builder.intConst(.i32, idx), - .b_elem => |idx| try o.builder.intConst(.i32, operand_len + idx), - .undef => llvm_poison_mask_elem, - }; - } - return fg.wip.shuffleVector( - operand_a, - operand_b, - try o.builder.vectorValue(llvm_mask_ty, mask_elems), - "", - ); - } - - /// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result. - /// - /// Equivalent to: - /// reduce: { - /// var i: usize = 0; - /// var accum: T = init; - /// while (i < vec.len) : (i += 1) { - /// accum = llvm_fn(accum, vec[i]); - /// } - /// break :reduce accum; - /// } - /// - fn buildReducedCall( - self: *FuncGen, - llvm_fn: Builder.Function.Index, - operand_vector: Builder.Value, - vector_len: usize, - accum_init: Builder.Value, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const usize_ty = try o.lowerType(pt, Type.usize); - const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len); - const llvm_result_ty = accum_init.typeOfWip(&self.wip); - - // Allocate and initialize our mutable variables - const i_ptr = try self.buildAlloca(usize_ty, .default); - _ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default); - const accum_ptr = try self.buildAlloca(llvm_result_ty, .default); - _ = try self.wip.store(.normal, accum_init, accum_ptr, .default); - - // Setup the loop - const loop = try self.wip.block(2, "ReduceLoop"); - const loop_exit = try self.wip.block(1, "AfterReduce"); - _ = try self.wip.br(loop); - { - self.wip.cursor = .{ .block = loop }; - - // while (i < vec.len) - const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, ""); - const cond = try self.wip.icmp(.ult, i, llvm_vector_len, ""); - const loop_then = try self.wip.block(1, "ReduceLoopThen"); - - _ = try self.wip.brCond(cond, loop_then, loop_exit, .none); - - { - self.wip.cursor = .{ .block = loop_then }; - - // accum = f(accum, vec[i]); - const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, ""); - const element = try self.wip.extractElement(operand_vector, i, ""); - const new_accum = try self.wip.call( - .normal, - .ccc, - .none, - llvm_fn.typeOf(&o.builder), - llvm_fn.toValue(&o.builder), - &.{ accum, element }, - "", - ); - _ = try self.wip.store(.normal, new_accum, accum_ptr, .default); - - // i += 1 - const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), ""); - _ = try self.wip.store(.normal, new_i, i_ptr, .default); - _ = try self.wip.br(loop); - } - } - - self.wip.cursor = .{ .block = loop_exit }; - return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, ""); - } - - fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const target = zcu.getTarget(); - - const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce; - const operand = try self.resolveInst(reduce.operand); - const operand_ty = self.typeOf(reduce.operand); - const llvm_operand_ty = try o.lowerType(pt, operand_ty); - const scalar_ty = self.typeOfIndex(inst); - const llvm_scalar_ty = try o.lowerType(pt, scalar_ty); - - switch (reduce.operation) { - .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { - .And => .@"vector.reduce.and", - .Or => .@"vector.reduce.or", - .Xor => .@"vector.reduce.xor", - else => unreachable, - }, &.{llvm_operand_ty}, &.{operand}, ""), - .Min, .Max => switch (scalar_ty.zigTypeTag(zcu)) { - .int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { - .Min => if (scalar_ty.isSignedInt(zcu)) - .@"vector.reduce.smin" - else - .@"vector.reduce.umin", - .Max => if (scalar_ty.isSignedInt(zcu)) - .@"vector.reduce.smax" - else - .@"vector.reduce.umax", - else => unreachable, - }, &.{llvm_operand_ty}, &.{operand}, ""), - .float => if (intrinsicsAllowed(scalar_ty, target)) - return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) { - .Min => .@"vector.reduce.fmin", - .Max => .@"vector.reduce.fmax", - else => unreachable, - }, &.{llvm_operand_ty}, &.{operand}, ""), - else => unreachable, - }, - .Add, .Mul => switch (scalar_ty.zigTypeTag(zcu)) { - .int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { - .Add => .@"vector.reduce.add", - .Mul => .@"vector.reduce.mul", - else => unreachable, - }, &.{llvm_operand_ty}, &.{operand}, ""), - .float => if (intrinsicsAllowed(scalar_ty, target)) - return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) { - .Add => .@"vector.reduce.fadd", - .Mul => .@"vector.reduce.fmul", - else => unreachable, - }, &.{llvm_operand_ty}, &.{ switch (reduce.operation) { - .Add => try o.builder.fpValue(llvm_scalar_ty, -0.0), - .Mul => try o.builder.fpValue(llvm_scalar_ty, 1.0), - else => unreachable, - }, operand }, ""), - else => unreachable, - }, - } - - // Reduction could not be performed with intrinsics. - // Use a manual loop over a softfloat call instead. - const float_bits = scalar_ty.floatBits(target); - const fn_name = switch (reduce.operation) { - .Min => try o.builder.strtabStringFmt("{s}fmin{s}", .{ - libcFloatPrefix(float_bits), libcFloatSuffix(float_bits), - }), - .Max => try o.builder.strtabStringFmt("{s}fmax{s}", .{ - libcFloatPrefix(float_bits), libcFloatSuffix(float_bits), - }), - .Add => try o.builder.strtabStringFmt("__add{s}f3", .{ - compilerRtFloatAbbrev(float_bits), - }), - .Mul => try o.builder.strtabStringFmt("__mul{s}f3", .{ - compilerRtFloatAbbrev(float_bits), - }), - else => unreachable, - }; - - const libc_fn = - try self.getLibcFunction(fn_name, &.{ llvm_scalar_ty, llvm_scalar_ty }, llvm_scalar_ty); - const init_val = switch (llvm_scalar_ty) { - .i16 => try o.builder.intValue(.i16, @as(i16, @bitCast( - @as(f16, switch (reduce.operation) { - .Min, .Max => std.math.nan(f16), - .Add => -0.0, - .Mul => 1.0, - else => unreachable, - }), - ))), - .i80 => try o.builder.intValue(.i80, @as(i80, @bitCast( - @as(f80, switch (reduce.operation) { - .Min, .Max => std.math.nan(f80), - .Add => -0.0, - .Mul => 1.0, - else => unreachable, - }), - ))), - .i128 => try o.builder.intValue(.i128, @as(i128, @bitCast( - @as(f128, switch (reduce.operation) { - .Min, .Max => std.math.nan(f128), - .Add => -0.0, - .Mul => 1.0, - else => unreachable, - }), - ))), - else => unreachable, - }; - return self.buildReducedCall(libc_fn, operand, operand_ty.vectorLen(zcu), init_val); - } - - fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const result_ty = self.typeOfIndex(inst); - const len: usize = @intCast(result_ty.arrayLen(zcu)); - const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); - const llvm_result_ty = try o.lowerType(pt, result_ty); - - switch (result_ty.zigTypeTag(zcu)) { - .vector => { - var vector = try o.builder.poisonValue(llvm_result_ty); - for (elements, 0..) |elem, i| { - const index_u32 = try o.builder.intValue(.i32, i); - const llvm_elem = try self.resolveInst(elem); - vector = try self.wip.insertElement(vector, llvm_elem, index_u32, ""); - } - return vector; - }, - .@"struct" => { - if (zcu.typeToPackedStruct(result_ty)) |struct_type| { - const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type); - const big_bits = backing_int_ty.bitSize(zcu); - const int_ty = try o.builder.intType(@intCast(big_bits)); - comptime assert(Type.packed_struct_layout_version == 2); - var running_int = try o.builder.intValue(int_ty, 0); - var running_bits: u16 = 0; - for (elements, struct_type.field_types.get(ip)) |elem, field_ty| { - if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue; - - const non_int_val = try self.resolveInst(elem); - const ty_bit_size: u16 = @intCast(Type.fromInterned(field_ty).bitSize(zcu)); - const small_int_ty = try o.builder.intType(ty_bit_size); - const small_int_val = if (Type.fromInterned(field_ty).isPtrAtRuntime(zcu)) - try self.wip.cast(.ptrtoint, non_int_val, small_int_ty, "") - else - try self.wip.cast(.bitcast, non_int_val, small_int_ty, ""); - const shift_rhs = try o.builder.intValue(int_ty, running_bits); - const extended_int_val = - try self.wip.conv(.unsigned, small_int_val, int_ty, ""); - const shifted = try self.wip.bin(.shl, extended_int_val, shift_rhs, ""); - running_int = try self.wip.bin(.@"or", running_int, shifted, ""); - running_bits += ty_bit_size; - } - return running_int; - } - - assert(result_ty.containerLayout(zcu) != .@"packed"); - - if (isByRef(result_ty, zcu)) { - // TODO in debug builds init to undef so that the padding will be 0xaa - // even if we fully populate the fields. - const alignment = result_ty.abiAlignment(zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment); - - for (elements, 0..) |elem, i| { - if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue; - - const llvm_elem = try self.resolveInst(elem); - const llvm_i = o.llvmFieldIndex(result_ty, i).?; - const field_ptr = try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, ""); - - const field_ptr_ty = try pt.ptrType(.{ - .child = self.typeOf(elem).toIntern(), - .flags = .{ - .alignment = result_ty.explicitFieldAlignment(i, zcu), - }, - }); - try self.store(field_ptr, field_ptr_ty, llvm_elem, .none); - } - - return alloca_inst; - } else { - var result = try o.builder.poisonValue(llvm_result_ty); - for (elements, 0..) |elem, i| { - if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue; - - const llvm_elem = try self.resolveInst(elem); - const llvm_i = o.llvmFieldIndex(result_ty, i).?; - result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, ""); - } - return result; - } - }, - .array => { - assert(isByRef(result_ty, zcu)); - - const llvm_usize = try o.lowerType(pt, Type.usize); - const usize_zero = try o.builder.intValue(llvm_usize, 0); - const alignment = result_ty.abiAlignment(zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment); - - const array_info = result_ty.arrayInfo(zcu); - const elem_ptr_ty = try pt.ptrType(.{ - .child = array_info.elem_type.toIntern(), - }); - - for (elements, 0..) |elem, i| { - const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{ - usize_zero, try o.builder.intValue(llvm_usize, i), - }, ""); - const llvm_elem = try self.resolveInst(elem); - try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none); - } - if (array_info.sentinel) |sent_val| { - const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{ - usize_zero, try o.builder.intValue(llvm_usize, array_info.len), - }, ""); - const llvm_elem = try self.resolveValue(sent_val); - try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none); - } - - return alloca_inst; - }, - else => unreachable, - } - } - - fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; - const union_ty = self.typeOfIndex(inst); - const union_llvm_ty = try o.lowerType(pt, union_ty); - const union_obj = zcu.typeToUnion(union_ty).?; - - assert(union_obj.layout != .@"packed"); - - const layout = Type.getUnionLayout(union_obj, zcu); - - const tag_int_val = blk: { - const tag_ty = union_ty.unionTagTypeHypothetical(zcu); - const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index); - break :blk tag_val.intFromEnum(zcu); - }; - if (layout.payload_size == 0) { - if (layout.tag_size == 0) { - return .none; - } - assert(!isByRef(union_ty, zcu)); - var big_int_space: Value.BigIntSpace = undefined; - const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu); - return try o.builder.bigIntValue(union_llvm_ty, tag_big_int); - } - assert(isByRef(union_ty, zcu)); - // The llvm type of the alloca will be the named LLVM union type, and will not - // necessarily match the format that we need, depending on which tag is active. - // We must construct the correct unnamed struct type here, in order to then set - // the fields appropriately. - const alignment = layout.abi_align.toLlvm(); - const result_ptr = try self.buildAlloca(union_llvm_ty, alignment); - const llvm_payload = try self.resolveInst(extra.init); - const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]); - const field_llvm_ty = try o.lowerType(pt, field_ty); - const field_size = field_ty.abiSize(zcu); - const field_align = union_ty.explicitFieldAlignment(extra.field_index, zcu); - const llvm_usize = try o.lowerType(pt, Type.usize); - const usize_zero = try o.builder.intValue(llvm_usize, 0); - - assert(field_ty.hasRuntimeBits(zcu)); - - const llvm_union_ty = t: { - const payload_ty = p: { - if (field_size == layout.payload_size) { - break :p field_llvm_ty; - } - const padding_len = layout.payload_size - field_size; - break :p try o.builder.structType(.@"packed", &.{ - field_llvm_ty, try o.builder.arrayType(padding_len, .i8), - }); - }; - if (layout.tag_size == 0) break :t try o.builder.structType(.normal, &.{payload_ty}); - const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); - var fields: [3]Builder.Type = undefined; - var fields_len: usize = 2; - if (layout.tag_align.compare(.gte, layout.payload_align)) { - fields = .{ tag_ty, payload_ty, undefined }; - } else { - fields = .{ payload_ty, tag_ty, undefined }; - } - if (layout.padding != 0) { - fields[fields_len] = try o.builder.arrayType(layout.padding, .i8); - fields_len += 1; - } - break :t try o.builder.structType(.normal, fields[0..fields_len]); - }; - - // Now we follow the layout as expressed above with GEP instructions to set the - // tag and the payload. - const field_ptr_ty = try pt.ptrType(.{ - .child = field_ty.toIntern(), - .flags = .{ .alignment = field_align }, - }); - if (layout.tag_size == 0) { - const indices = [3]Builder.Value{ usize_zero, .@"0", .@"0" }; - const len: usize = if (field_size == layout.payload_size) 2 else 3; - const field_ptr = - try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], ""); - try self.store(field_ptr, field_ptr_ty, llvm_payload, .none); - return result_ptr; - } - - { - const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align)); - const indices: [3]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, payload_index), .@"0" }; - const len: usize = if (field_size == layout.payload_size) 2 else 3; - const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], ""); - try self.store(field_ptr, field_ptr_ty, llvm_payload, .none); - } - { - const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); - const indices: [2]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, tag_index) }; - const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, &indices, ""); - const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); - var big_int_space: Value.BigIntSpace = undefined; - const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu); - const llvm_tag = try o.builder.bigIntValue(tag_ty, tag_big_int); - const tag_alignment = Type.fromInterned(union_obj.enum_tag_type).abiAlignment(zcu).toLlvm(); - _ = try self.wip.store(.normal, llvm_tag, field_ptr, tag_alignment); - } - - return result_ptr; - } - - fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch; - - comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.read) == 0); - comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.write) == 1); - - comptime assert(prefetch.locality >= 0); - comptime assert(prefetch.locality <= 3); - - comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.instruction) == 0); - comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.data) == 1); - - // LLVM fails during codegen of instruction cache prefetchs for these architectures. - // This is an LLVM bug as the prefetch intrinsic should be a noop if not supported - // by the target. - // To work around this, don't emit llvm.prefetch in this case. - // See https://bugs.llvm.org/show_bug.cgi?id=21037 - const zcu = self.ng.pt.zcu; - const target = zcu.getTarget(); - switch (prefetch.cache) { - .instruction => switch (target.cpu.arch) { - .x86_64, - .x86, - .powerpc, - .powerpcle, - .powerpc64, - .powerpc64le, - => return .none, - .arm, .armeb, .thumb, .thumbeb => { - switch (prefetch.rw) { - .write => return .none, - else => {}, - } - }, - else => {}, - }, - .data => {}, - } - - _ = try self.wip.callIntrinsic(.normal, .none, .prefetch, &.{.ptr}, &.{ - try self.sliceOrArrayPtr(try self.resolveInst(prefetch.ptr), self.typeOf(prefetch.ptr)), - try o.builder.intValue(.i32, prefetch.rw), - try o.builder.intValue(.i32, prefetch.locality), - try o.builder.intValue(.i32, prefetch.cache), - }, ""); - return .none; - } - - fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const inst_ty = self.typeOfIndex(inst); - const operand = try self.resolveInst(ty_op.operand); - - return self.wip.cast(.addrspacecast, operand, try o.lowerType(pt, inst_ty), ""); - } - - fn workIntrinsic( - self: *FuncGen, - dimension: u32, - default: u32, - comptime basename: []const u8, - ) !Builder.Value { - return self.wip.callIntrinsic(.normal, .none, switch (dimension) { - 0 => @field(Builder.Intrinsic, basename ++ ".x"), - 1 => @field(Builder.Intrinsic, basename ++ ".y"), - 2 => @field(Builder.Intrinsic, basename ++ ".z"), - else => return self.ng.object.builder.intValue(.i32, default), - }, &.{}, &.{}, ""); - } - - fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const target = self.ng.pt.zcu.getTarget(); - - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const dimension = pl_op.payload; - - return switch (target.cpu.arch) { - .amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workitem.id"), - .nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.tid"), - else => unreachable, - }; - } - - fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const target = pt.zcu.getTarget(); - - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const dimension = pl_op.payload; - - switch (target.cpu.arch) { - .amdgcn => { - if (dimension >= 3) return .@"1"; - - // Fetch the dispatch pointer, which points to this structure: - // https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/adae6c61e10d371f7cbc3d0e94ae2c070cab18a4/src/inc/hsa.h#L2913 - const dispatch_ptr = - try self.wip.callIntrinsic(.normal, .none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, ""); - - // Load the work_group_* member from the struct as u16. - // Just treat the dispatch pointer as an array of u16 to keep things simple. - const workgroup_size_ptr = try self.wip.gep(.inbounds, .i16, dispatch_ptr, &.{ - try o.builder.intValue(try o.lowerType(pt, Type.usize), 2 + dimension), - }, ""); - const workgroup_size_alignment = comptime Builder.Alignment.fromByteUnits(2); - return self.wip.load(.normal, .i16, workgroup_size_ptr, workgroup_size_alignment, ""); - }, - .nvptx, .nvptx64 => { - return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid"); - }, - else => unreachable, - } - } - - fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const target = self.ng.pt.zcu.getTarget(); - - const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; - const dimension = pl_op.payload; - - return switch (target.cpu.arch) { - .amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workgroup.id"), - .nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.ctaid"), - else => unreachable, - }; - } - - fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index { - const o = self.ng.object; - const pt = self.ng.pt; - - const table = o.error_name_table; - if (table != .none) return table; - - // TODO: Address space - const variable_index = - try o.builder.addVariable(try o.builder.strtabString("__zig_err_name_table"), .ptr, .default); - variable_index.setLinkage(.private, &o.builder); - variable_index.setMutability(.constant, &o.builder); - variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); - variable_index.setAlignment( - Type.slice_const_u8_sentinel_0.abiAlignment(pt.zcu).toLlvm(), - &o.builder, - ); - - o.error_name_table = variable_index; - return variable_index; - } - - /// Assumes the optional is not pointer-like and payload has bits. - fn optCmpNull( - self: *FuncGen, - cond: Builder.IntegerCondition, - opt_llvm_ty: Builder.Type, - opt_handle: Builder.Value, - is_by_ref: bool, - access_kind: Builder.MemoryAccessKind, - ) Allocator.Error!Builder.Value { - const o = self.ng.object; - const field = b: { - if (is_by_ref) { - const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, ""); - break :b try self.wip.load(access_kind, .i8, field_ptr, .default, ""); - } - break :b try self.wip.extractValue(opt_handle, &.{1}, ""); - }; - comptime assert(optional_layout_version == 3); - - return self.wip.icmp(cond, field, try o.builder.intValue(.i8, 0), ""); - } - - /// Assumes the optional is not pointer-like and payload has bits. - fn optPayloadHandle( - fg: *FuncGen, - opt_llvm_ty: Builder.Type, - opt_handle: Builder.Value, - opt_ty: Type, - can_elide_load: bool, - ) !Builder.Value { - const pt = fg.ng.pt; - const zcu = pt.zcu; - const payload_ty = opt_ty.optionalChild(zcu); - - if (isByRef(opt_ty, zcu)) { - // We have a pointer and we need to return a pointer to the first field. - const payload_ptr = try fg.wip.gepStruct(opt_llvm_ty, opt_handle, 0, ""); - - const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm(); - if (isByRef(payload_ty, zcu)) { - if (can_elide_load) - return payload_ptr; - - return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); - } - return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_alignment); - } - - assert(!isByRef(payload_ty, zcu)); - return fg.wip.extractValue(opt_handle, &.{0}, ""); - } - - fn buildOptional( - self: *FuncGen, - optional_ty: Type, - payload: Builder.Value, - non_null_bit: Builder.Value, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const optional_llvm_ty = try o.lowerType(pt, optional_ty); - const non_null_field = try self.wip.cast(.zext, non_null_bit, .i8, ""); - - if (isByRef(optional_ty, zcu)) { - const payload_alignment = optional_ty.abiAlignment(pt.zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment); - - { - const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, ""); - _ = try self.wip.store(.normal, payload, field_ptr, payload_alignment); - } - { - const non_null_alignment = comptime Builder.Alignment.fromByteUnits(1); - const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 1, ""); - _ = try self.wip.store(.normal, non_null_field, field_ptr, non_null_alignment); - } - - return alloca_inst; - } - - return self.wip.buildAggregate(optional_llvm_ty, &.{ payload, non_null_field }, ""); - } - - fn fieldPtr( - self: *FuncGen, - aggregate_ptr: Builder.Value, - aggregate_ptr_ty: Type, - field_index: u32, - ) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const aggregate_ty = aggregate_ptr_ty.childType(zcu); - if (aggregate_ty.containerLayout(zcu) == .@"packed") { - // A pointer to a bitpack field is equivalent to a pointer to the whole bitpack; the - // bit offset is represented in the pointer *type*. - return aggregate_ptr; - } - switch (aggregate_ty.zigTypeTag(zcu)) { - .@"struct" => { - if (!aggregate_ty.hasRuntimeBits(zcu)) { - return aggregate_ptr; - } - const struct_llvm_ty = try o.lowerType(pt, aggregate_ty); - if (o.llvmFieldIndex(aggregate_ty, field_index)) |llvm_field_index| { - return self.wip.gepStruct(struct_llvm_ty, aggregate_ptr, llvm_field_index, ""); - } else { - // If we found no index then this means this is a zero sized field at the - // end of the struct. Treat our struct pointer as an array of two and get - // the index to the element at index `1` to get a pointer to the end of - // the struct. - const llvm_index = try o.builder.intValue( - try o.lowerType(pt, Type.usize), - @intFromBool(aggregate_ty.hasRuntimeBits(zcu)), - ); - return self.wip.gep(.inbounds, struct_llvm_ty, aggregate_ptr, &.{llvm_index}, ""); - } - }, - .@"union" => { - const layout = aggregate_ty.unionGetLayout(zcu); - if (layout.payload_size == 0) return aggregate_ptr; - const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)); - const union_llvm_ty = try o.lowerType(pt, aggregate_ty); - return self.wip.gepStruct(union_llvm_ty, aggregate_ptr, payload_index, ""); - }, - else => unreachable, - } - } - - /// Load a value and, if needed, mask out padding bits for non byte-sized integer values. - fn loadTruncate( - fg: *FuncGen, - access_kind: Builder.MemoryAccessKind, - payload_ty: Type, - payload_ptr: Builder.Value, - payload_alignment: Builder.Alignment, - ) !Builder.Value { - // from https://llvm.org/docs/LangRef.html#load-instruction : - // "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. " - // => so load the byte aligned value and trunc the unwanted bits. - - const o = fg.ng.object; - const pt = fg.ng.pt; - const zcu = pt.zcu; - const payload_llvm_ty = try o.lowerType(pt, payload_ty); - const abi_size = payload_ty.abiSize(zcu); - - const load_llvm_ty = if (payload_ty.isAbiInt(zcu)) - try o.builder.intType(@intCast(abi_size * 8)) - else - payload_llvm_ty; - const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, ""); - const shifted = if (payload_llvm_ty != load_llvm_ty and o.target.cpu.arch.endian() == .big) - try fg.wip.bin(.lshr, loaded, try o.builder.intValue( - load_llvm_ty, - (payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8, - ), "") - else - loaded; - - return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, ""); - } - - /// Load a by-ref type by constructing a new alloca and performing a memcpy. - fn loadByRef( - fg: *FuncGen, - ptr: Builder.Value, - pointee_type: Type, - ptr_alignment: Builder.Alignment, - access_kind: Builder.MemoryAccessKind, - ) !Builder.Value { - const o = fg.ng.object; - const pt = fg.ng.pt; - const pointee_llvm_ty = try o.lowerType(pt, pointee_type); - const result_align = InternPool.Alignment.fromLlvm(ptr_alignment) - .max(pointee_type.abiAlignment(pt.zcu)).toLlvm(); - const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align); - const size_bytes = pointee_type.abiSize(pt.zcu); - _ = try fg.wip.callMemCpy( - result_ptr, - result_align, - ptr, - ptr_alignment, - try o.builder.intValue(try o.lowerType(pt, Type.usize), size_bytes), - access_kind, - fg.disable_intrinsics, - ); - return result_ptr; - } - - /// This function always performs a copy. For isByRef=true types, it creates a new - /// alloca and copies the value into it, then returns the alloca instruction. - /// For isByRef=false types, it creates a load instruction and returns it. - fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const info = ptr_ty.ptrInfo(zcu); - const elem_ty = Type.fromInterned(info.child); - if (!elem_ty.hasRuntimeBits(zcu)) return .none; - - const ptr_alignment = (if (info.flags.alignment != .none) - @as(InternPool.Alignment, info.flags.alignment) - else - elem_ty.abiAlignment(zcu)).toLlvm(); - - const access_kind: Builder.MemoryAccessKind = - if (info.flags.is_volatile) .@"volatile" else .normal; - - if (info.flags.vector_index != .none) { - const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); - const vec_elem_ty = try o.lowerType(pt, elem_ty); - const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); - - const loaded_vector = try self.wip.load(access_kind, vec_ty, ptr, ptr_alignment, ""); - return self.wip.extractElement(loaded_vector, index_u32, ""); - } - - if (info.packed_offset.host_size == 0) { - if (isByRef(elem_ty, zcu)) { - return self.loadByRef(ptr, elem_ty, ptr_alignment, access_kind); - } - return self.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment); - } - - const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8)); - const containing_int = - try self.wip.load(access_kind, containing_int_ty, ptr, ptr_alignment, ""); - - const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); - const shift_amt = try o.builder.intValue(containing_int_ty, info.packed_offset.bit_offset); - const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, ""); - const elem_llvm_ty = try o.lowerType(pt, elem_ty); - - if (isByRef(elem_ty, zcu)) { - const result_align = elem_ty.abiAlignment(zcu).toLlvm(); - const result_ptr = try self.buildAlloca(elem_llvm_ty, result_align); - - const same_size_int = try o.builder.intType(@intCast(elem_bits)); - const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); - _ = try self.wip.store(.normal, truncated_int, result_ptr, result_align); - return result_ptr; - } - - if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) { - const same_size_int = try o.builder.intType(@intCast(elem_bits)); - const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); - return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); - } - - if (elem_ty.isPtrAtRuntime(zcu)) { - const same_size_int = try o.builder.intType(@intCast(elem_bits)); - const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); - return self.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, ""); - } - - return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, ""); - } - - fn store( - self: *FuncGen, - ptr: Builder.Value, - ptr_ty: Type, - elem: Builder.Value, - ordering: Builder.AtomicOrdering, - ) !void { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const info = ptr_ty.ptrInfo(zcu); - const elem_ty = Type.fromInterned(info.child); - if (!elem_ty.hasRuntimeBits(zcu)) { - return; - } - const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); - const access_kind: Builder.MemoryAccessKind = - if (info.flags.is_volatile) .@"volatile" else .normal; - - if (info.flags.vector_index != .none) { - const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); - const vec_elem_ty = try o.lowerType(pt, elem_ty); - const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); - - const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, ""); - - const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, ""); - - assert(ordering == .none); - _ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment); - return; - } - - if (info.packed_offset.host_size != 0) { - const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8)); - assert(ordering == .none); - const containing_int = - try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, ""); - const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); - const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset); - // Convert to equally-sized integer type in order to perform the bit - // operations on the value to store - const value_bits_type = try o.builder.intType(@intCast(elem_bits)); - const value_bits = if (elem_ty.isPtrAtRuntime(zcu)) - try self.wip.cast(.ptrtoint, elem, value_bits_type, "") - else - try self.wip.cast(.bitcast, elem, value_bits_type, ""); - - const mask_val = blk: { - const zext = try self.wip.cast( - .zext, - try o.builder.intValue(value_bits_type, -1), - containing_int_ty, - "", - ); - const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), ""); - break :blk try self.wip.bin( - .xor, - shl, - try o.builder.intValue(containing_int_ty, -1), - "", - ); - }; - - const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, ""); - const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, ""); - const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), ""); - const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, ""); - - assert(ordering == .none); - _ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment); - return; - } - if (!isByRef(elem_ty, zcu)) { - _ = try self.wip.storeAtomic( - access_kind, - elem, - ptr, - self.sync_scope, - ordering, - ptr_alignment, - ); - return; - } - assert(ordering == .none); - _ = try self.wip.callMemCpy( - ptr, - ptr_alignment, - elem, - elem_ty.abiAlignment(zcu).toLlvm(), - try o.builder.intValue(try o.lowerType(pt, Type.usize), elem_ty.abiSize(zcu)), - access_kind, - self.disable_intrinsics, - ); - } - - fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void { - const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545; - const o = fg.ng.object; - const pt = fg.ng.pt; - const usize_ty = try o.lowerType(pt, Type.usize); - const zero = try o.builder.intValue(usize_ty, 0); - const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED); - const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, ""); - _ = try valgrindClientRequest(fg, zero, req, ptr_as_usize, len, zero, zero, zero); - } - - fn valgrindClientRequest( - fg: *FuncGen, - default_value: Builder.Value, - request: Builder.Value, - a1: Builder.Value, - a2: Builder.Value, - a3: Builder.Value, - a4: Builder.Value, - a5: Builder.Value, - ) Allocator.Error!Builder.Value { - const o = fg.ng.object; - const pt = fg.ng.pt; - const zcu = pt.zcu; - const target = zcu.getTarget(); - if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value; - - const llvm_usize = try o.lowerType(pt, Type.usize); - const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm(); - - const array_llvm_ty = try o.builder.arrayType(6, llvm_usize); - const array_ptr = if (fg.valgrind_client_request_array == .none) a: { - const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment); - fg.valgrind_client_request_array = array_ptr; - break :a array_ptr; - } else fg.valgrind_client_request_array; - const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 }; - const zero = try o.builder.intValue(llvm_usize, 0); - for (array_elements, 0..) |elem, i| { - const elem_ptr = try fg.wip.gep(.inbounds, array_llvm_ty, array_ptr, &.{ - zero, try o.builder.intValue(llvm_usize, i), - }, ""); - _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment); - } - - const arch_specific: struct { - template: [:0]const u8, - constraints: [:0]const u8, - } = switch (target.cpu.arch) { - .arm, .armeb, .thumb, .thumbeb => .{ - .template = - \\ mov r12, r12, ror #3 ; mov r12, r12, ror #13 - \\ mov r12, r12, ror #29 ; mov r12, r12, ror #19 - \\ orr r10, r10, r10 - , - .constraints = "={r3},{r4},{r3},~{cc},~{memory}", - }, - .aarch64, .aarch64_be => .{ - .template = - \\ ror x12, x12, #3 ; ror x12, x12, #13 - \\ ror x12, x12, #51 ; ror x12, x12, #61 - \\ orr x10, x10, x10 - , - .constraints = "={x3},{x4},{x3},~{cc},~{memory}", - }, - .mips, .mipsel => .{ - .template = - \\ srl $$0, $$0, 13 - \\ srl $$0, $$0, 29 - \\ srl $$0, $$0, 3 - \\ srl $$0, $$0, 19 - \\ or $$13, $$13, $$13 - , - .constraints = "={$11},{$12},{$11},~{memory},~{$1}", - }, - .mips64, .mips64el => .{ - .template = - \\ dsll $$0, $$0, 3 ; dsll $$0, $$0, 13 - \\ dsll $$0, $$0, 29 ; dsll $$0, $$0, 19 - \\ or $$13, $$13, $$13 - , - .constraints = "={$11},{$12},{$11},~{memory},~{$1}", - }, - .powerpc, .powerpcle => .{ - .template = - \\ rlwinm 0, 0, 3, 0, 31 ; rlwinm 0, 0, 13, 0, 31 - \\ rlwinm 0, 0, 29, 0, 31 ; rlwinm 0, 0, 19, 0, 31 - \\ or 1, 1, 1 - , - .constraints = "={r3},{r4},{r3},~{cc},~{memory}", - }, - .powerpc64, .powerpc64le => .{ - .template = - \\ rotldi 0, 0, 3 ; rotldi 0, 0, 13 - \\ rotldi 0, 0, 61 ; rotldi 0, 0, 51 - \\ or 1, 1, 1 - , - .constraints = "={r3},{r4},{r3},~{cc},~{memory}", - }, - .riscv64 => .{ - .template = - \\ .option push - \\ .option norvc - \\ srli zero, zero, 3 - \\ srli zero, zero, 13 - \\ srli zero, zero, 51 - \\ srli zero, zero, 61 - \\ or a0, a0, a0 - \\ .option pop - , - .constraints = "={a3},{a4},{a3},~{cc},~{memory}", - }, - .s390x => .{ - .template = - \\ lr %r15, %r15 - \\ lr %r1, %r1 - \\ lr %r2, %r2 - \\ lr %r3, %r3 - \\ lr %r2, %r2 - , - .constraints = "={r3},{r2},{r3},~{cc},~{memory}", - }, - .x86 => .{ - .template = - \\ roll $$3, %edi ; roll $$13, %edi - \\ roll $$61, %edi ; roll $$51, %edi - \\ xchgl %ebx, %ebx - , - .constraints = "={edx},{eax},{edx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}", - }, - .x86_64 => .{ - .template = - \\ rolq $$3, %rdi ; rolq $$13, %rdi - \\ rolq $$61, %rdi ; rolq $$51, %rdi - \\ xchgq %rbx, %rbx - , - .constraints = "={rdx},{rax},{rdx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}", - }, - else => unreachable, - }; - - return fg.wip.callAsm( - .none, - try o.builder.fnType(llvm_usize, &.{ llvm_usize, llvm_usize }, .normal), - .{ .sideeffect = true }, - try o.builder.string(arch_specific.template), - try o.builder.string(arch_specific.constraints), - &.{ try fg.wip.cast(.ptrtoint, array_ptr, llvm_usize, ""), default_value }, - "", - ); - } - - fn typeOf(fg: *FuncGen, inst: Air.Inst.Ref) Type { - const zcu = fg.ng.pt.zcu; - return fg.air.typeOf(inst, &zcu.intern_pool); - } - - fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type { - const zcu = fg.ng.pt.zcu; - return fg.air.typeOfIndex(inst, &zcu.intern_pool); - } -}; - -fn toLlvmAtomicOrdering(atomic_order: std.builtin.AtomicOrder) Builder.AtomicOrdering { - return switch (atomic_order) { - .unordered => .unordered, - .monotonic => .monotonic, - .acquire => .acquire, - .release => .release, - .acq_rel => .acq_rel, - .seq_cst => .seq_cst, - }; -} - -fn toLlvmAtomicRmwBinOp( - op: std.builtin.AtomicRmwOp, - is_signed: bool, - is_float: bool, -) Builder.Function.Instruction.AtomicRmw.Operation { - return switch (op) { - .Xchg => .xchg, - .Add => if (is_float) .fadd else return .add, - .Sub => if (is_float) .fsub else return .sub, - .And => .@"and", - .Nand => .nand, - .Or => .@"or", - .Xor => .xor, - .Max => if (is_float) .fmax else if (is_signed) .max else return .umax, - .Min => if (is_float) .fmin else if (is_signed) .min else return .umin, - }; -} - const CallingConventionInfo = struct { /// The LLVM calling convention to use. llvm_cc: Builder.CallConv, @@ -11593,7 +4542,7 @@ pub fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: *const std.Targ .inreg_param_count = register_params, }; } -fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv { +pub fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv { if (target.cCallingConvention()) |default_c| { if (cc_tag == default_c) { return .ccc; @@ -11724,7 +4673,7 @@ fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const s } /// Convert a zig-address space to an llvm address space. -fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace { +pub fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace { for (llvmAddrSpaceInfo(target)) |info| if (info.zig == address_space) return info.llvm; unreachable; } @@ -11811,20 +4760,6 @@ fn llvmAddrSpaceInfo(target: *const std.Target) []const AddrSpaceInfo { }; } -/// On some targets, local values that are in the generic address space must be generated into a -/// different address, space and then cast back to the generic address space. -/// For example, on GPUs local variable declarations must be generated into the local address space. -/// This function returns the address space local values should be generated into. -fn llvmAllocaAddressSpace(target: *const std.Target) Builder.AddrSpace { - return switch (target.cpu.arch) { - // On amdgcn, locals should be generated into the private address space. - // To make Zig not impossible to use, these are then converted to addresses in the - // generic address space and treates as regular pointers. This is the way that HIP also does it. - .amdgcn => Builder.AddrSpace.amdgpu.private, - else => .default, - }; -} - /// On some targets, global values that are in the generic address space must be generated into a /// different address space, and then cast back to the generic address space. fn llvmDefaultGlobalAddressSpace(target: *const std.Target) Builder.AddrSpace { @@ -11845,731 +4780,10 @@ fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, targ }; } -fn returnTypeByRef(zcu: *Zcu, target: *const std.Target, ty: Type) bool { - if (isByRef(ty, zcu)) { - return true; - } else if (target.cpu.arch.isX86() and - !target.cpu.has(.x86, .evex512) and - ty.totalVectorBits(zcu) >= 512) - { - // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns - // "512-bit vector arguments require 'evex512' for AVX512" - return true; - } else { - return false; - } -} - -fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: *Zcu, target: *const std.Target) bool { - const return_type = Type.fromInterned(fn_info.return_type); - if (!return_type.hasRuntimeBits(zcu)) return false; - - return switch (fn_info.cc) { - .auto => returnTypeByRef(zcu, target, return_type), - .x86_64_sysv => firstParamSRetSystemV(return_type, zcu, target), - .x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory, - .x86_sysv, .x86_win => isByRef(return_type, zcu), - .x86_stdcall => !isScalar(zcu, return_type), - .wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect, - .aarch64_aapcs, - .aarch64_aapcs_darwin, - .aarch64_aapcs_win, - => aarch64_c_abi.classifyType(return_type, zcu) == .memory, - .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) { - .memory, .i64_array => true, - .i32_array => |size| size != 1, - .byval => false, - }, - .riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory, - .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) { - .memory, .i32_array => true, - .byval => false, - }, - else => false, // TODO: investigate other targets/callconvs - }; -} - -fn firstParamSRetSystemV(ty: Type, zcu: *Zcu, target: *const std.Target) bool { - const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret); - if (class[0] == .memory) return true; - if (class[0] == .x87 and class[2] != .none) return true; - return false; -} - -/// In order to support the C calling convention, some return types need to be lowered -/// completely differently in the function prototype to honor the C ABI, and then -/// be effectively bitcasted to the actual return type. -fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { - const zcu = pt.zcu; - const return_type = Type.fromInterned(fn_info.return_type); - if (!return_type.hasRuntimeBits(zcu)) { - assert(!return_type.isError(zcu)); - return .void; - } - const target = zcu.getTarget(); - switch (fn_info.cc) { - .@"inline" => unreachable, - .auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(pt, return_type), - - .x86_64_sysv => return lowerSystemVFnRetTy(o, pt, fn_info), - .x86_64_win => return lowerWin64FnRetTy(o, pt, fn_info), - .x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(pt, return_type) else .void, - .x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(pt, return_type), - .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) { - .memory => return .void, - .float_array => return o.lowerType(pt, return_type), - .byval => return o.lowerType(pt, return_type), - .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))), - .double_integer => return o.builder.arrayType(2, .i64), - }, - .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) { - .memory, .i64_array => return .void, - .i32_array => |len| return if (len == 1) .i32 else .void, - .byval => return o.lowerType(pt, return_type), - }, - .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) { - .memory, .i32_array => return .void, - .byval => return o.lowerType(pt, return_type), - }, - .riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) { - .memory => return .void, - .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))), - .double_integer => { - const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) { - .riscv64, .riscv64be => .i64, - .riscv32, .riscv32be => .i32, - else => unreachable, - }; - return o.builder.structType(.normal, &.{ integer, integer }); - }, - .byval => return o.lowerType(pt, return_type), - .fields => { - var types_len: usize = 0; - var types: [8]Builder.Type = undefined; - for (0..return_type.structFieldCount(zcu)) |field_index| { - const field_ty = return_type.fieldType(field_index, zcu); - if (!field_ty.hasRuntimeBits(zcu)) continue; - types[types_len] = try o.lowerType(pt, field_ty); - types_len += 1; - } - return o.builder.structType(.normal, types[0..types_len]); - }, - }, - .wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) { - .direct => |scalar_ty| return o.lowerType(pt, scalar_ty), - .indirect => return .void, - }, - // TODO investigate other callconvs - else => return o.lowerType(pt, return_type), - } -} - -fn lowerWin64FnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { - const zcu = pt.zcu; - const return_type = Type.fromInterned(fn_info.return_type); - switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) { - .integer => { - if (isScalar(zcu, return_type)) { - return o.lowerType(pt, return_type); - } else { - return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8)); - } - }, - .win_i128 => return o.builder.vectorType(.normal, 2, .i64), - .memory => return .void, - .sse => return o.lowerType(pt, return_type), - else => unreachable, - } -} - -fn lowerSystemVFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const return_type = Type.fromInterned(fn_info.return_type); - return_type.assertHasLayout(zcu); - if (isScalar(zcu, return_type)) { - return o.lowerType(pt, return_type); - } - const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret); - var types_index: u32 = 0; - var types_buffer: [8]Builder.Type = undefined; - for (classes) |class| { - switch (class) { - .integer => { - types_buffer[types_index] = .i64; - types_index += 1; - }, - .sse => { - types_buffer[types_index] = .double; - types_index += 1; - }, - .sseup => { - if (types_buffer[types_index - 1] == .double) { - types_buffer[types_index - 1] = .fp128; - } else { - types_buffer[types_index] = .double; - types_index += 1; - } - }, - .float => { - types_buffer[types_index] = .float; - types_index += 1; - }, - .float_combine => { - types_buffer[types_index] = try o.builder.vectorType(.normal, 2, .float); - types_index += 1; - }, - .x87 => { - if (types_index != 0 or classes[2] != .none) return .void; - types_buffer[types_index] = .x86_fp80; - types_index += 1; - }, - .x87up => continue, - .none => break, - .memory, .integer_per_element => return .void, - .win_i128 => unreachable, // windows only - } - } - const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer}); - if (first_non_integer == null or classes[first_non_integer.?] == .none) { - assert(first_non_integer orelse classes.len == types_index); - switch (ip.indexToKey(return_type.toIntern())) { - .struct_type => { - const size = return_type.abiSize(zcu); - assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index); - if (size % 8 > 0) { - types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8)); - } - }, - else => {}, - } - if (types_index == 1) return types_buffer[0]; - } - return o.builder.structType(.normal, types_buffer[0..types_index]); -} - -const ParamTypeIterator = struct { - object: *Object, - pt: Zcu.PerThread, - fn_info: InternPool.Key.FuncType, - zig_index: u32, - llvm_index: u32, - types_len: u32, - types_buffer: [8]Builder.Type, - byval_attr: bool, - - const Lowering = union(enum) { - no_bits, - byval, - byref, - byref_mut, - abi_sized_int, - multiple_llvm_types, - slice, - float_array: u8, - i32_array: u8, - i64_array: u8, - }; - - fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering { - if (it.zig_index >= it.fn_info.param_types.len) return null; - const ip = &it.pt.zcu.intern_pool; - const ty = it.fn_info.param_types.get(ip)[it.zig_index]; - it.byval_attr = false; - return nextInner(it, Type.fromInterned(ty)); - } - - /// `airCall` uses this instead of `next` so that it can take into account variadic functions. - fn nextCall(it: *ParamTypeIterator, fg: *FuncGen, args: []const Air.Inst.Ref) Allocator.Error!?Lowering { - assert(std.meta.eql(it.pt, fg.ng.pt)); - const ip = &it.pt.zcu.intern_pool; - if (it.zig_index >= it.fn_info.param_types.len) { - if (it.zig_index >= args.len) { - return null; - } else { - return nextInner(it, fg.typeOf(args[it.zig_index])); - } - } else { - return nextInner(it, Type.fromInterned(it.fn_info.param_types.get(ip)[it.zig_index])); - } - } - - fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering { - const pt = it.pt; - const zcu = pt.zcu; - const target = zcu.getTarget(); - - if (!ty.hasRuntimeBits(zcu)) { - it.zig_index += 1; - return .no_bits; - } - switch (it.fn_info.cc) { - .@"inline" => unreachable, - .auto => { - it.zig_index += 1; - it.llvm_index += 1; - if (ty.isSlice(zcu) or - (ty.zigTypeTag(zcu) == .optional and ty.optionalChild(zcu).isSlice(zcu) and !ty.ptrAllowsZero(zcu))) - { - it.llvm_index += 1; - return .slice; - } else if (isByRef(ty, zcu)) { - return .byref; - } else if (target.cpu.arch.isX86() and - !target.cpu.has(.x86, .evex512) and - ty.totalVectorBits(zcu) >= 512) - { - // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns - // "512-bit vector arguments require 'evex512' for AVX512" - return .byref; - } else { - return .byval; - } - }, - .async => { - @panic("TODO implement async function lowering in the LLVM backend"); - }, - .x86_64_sysv => return it.nextSystemV(ty), - .x86_64_win => return it.nextWin64(ty), - .x86_stdcall => { - it.zig_index += 1; - it.llvm_index += 1; - - if (isScalar(zcu, ty)) { - return .byval; - } else { - it.byval_attr = true; - return .byref; - } - }, - .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => { - it.zig_index += 1; - it.llvm_index += 1; - switch (aarch64_c_abi.classifyType(ty, zcu)) { - .memory => return .byref_mut, - .float_array => |len| return Lowering{ .float_array = len }, - .byval => return .byval, - .integer => { - it.types_len = 1; - it.types_buffer[0] = .i64; - return .multiple_llvm_types; - }, - .double_integer => return Lowering{ .i64_array = 2 }, - } - }, - .arm_aapcs, .arm_aapcs_vfp => { - it.zig_index += 1; - it.llvm_index += 1; - switch (arm_c_abi.classifyType(ty, zcu, .arg)) { - .memory => { - it.byval_attr = true; - return .byref; - }, - .byval => return .byval, - .i32_array => |size| return Lowering{ .i32_array = size }, - .i64_array => |size| return Lowering{ .i64_array = size }, - } - }, - .mips_o32 => { - it.zig_index += 1; - it.llvm_index += 1; - switch (mips_c_abi.classifyType(ty, zcu, .arg)) { - .memory => { - it.byval_attr = true; - return .byref; - }, - .byval => return .byval, - .i32_array => |size| return Lowering{ .i32_array = size }, - } - }, - .riscv64_lp64, .riscv32_ilp32 => { - it.zig_index += 1; - it.llvm_index += 1; - switch (riscv_c_abi.classifyType(ty, zcu)) { - .memory => return .byref_mut, - .byval => return .byval, - .integer => return .abi_sized_int, - .double_integer => return Lowering{ .i64_array = 2 }, - .fields => { - it.types_len = 0; - for (0..ty.structFieldCount(zcu)) |field_index| { - const field_ty = ty.fieldType(field_index, zcu); - if (!field_ty.hasRuntimeBits(zcu)) continue; - it.types_buffer[it.types_len] = try it.object.lowerType(pt, field_ty); - it.types_len += 1; - } - it.llvm_index += it.types_len - 1; - return .multiple_llvm_types; - }, - } - }, - .wasm_mvp => switch (wasm_c_abi.classifyType(ty, zcu)) { - .direct => |scalar_ty| { - if (isScalar(zcu, ty)) { - it.zig_index += 1; - it.llvm_index += 1; - return .byval; - } else { - var types_buffer: [8]Builder.Type = undefined; - types_buffer[0] = try it.object.lowerType(pt, scalar_ty); - it.types_buffer = types_buffer; - it.types_len = 1; - it.llvm_index += 1; - it.zig_index += 1; - return .multiple_llvm_types; - } - }, - .indirect => { - it.zig_index += 1; - it.llvm_index += 1; - it.byval_attr = true; - return .byref; - }, - }, - // TODO investigate other callconvs - else => { - it.zig_index += 1; - it.llvm_index += 1; - return .byval; - }, - } - } - - fn nextWin64(it: *ParamTypeIterator, ty: Type) ?Lowering { - const zcu = it.pt.zcu; - switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) { - .integer => { - if (isScalar(zcu, ty)) { - it.zig_index += 1; - it.llvm_index += 1; - return .byval; - } else { - it.zig_index += 1; - it.llvm_index += 1; - return .abi_sized_int; - } - }, - .win_i128 => { - it.zig_index += 1; - it.llvm_index += 1; - return .byref; - }, - .memory => { - it.zig_index += 1; - it.llvm_index += 1; - return .byref_mut; - }, - .sse => { - it.zig_index += 1; - it.llvm_index += 1; - return .byval; - }, - else => unreachable, - } - } - - fn nextSystemV(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering { - const zcu = it.pt.zcu; - const ip = &zcu.intern_pool; - ty.assertHasLayout(zcu); - const classes = x86_64_abi.classifySystemV(ty, zcu, zcu.getTarget(), .arg); - if (classes[0] == .memory) { - it.zig_index += 1; - it.llvm_index += 1; - it.byval_attr = true; - return .byref; - } - if (isScalar(zcu, ty)) { - it.zig_index += 1; - it.llvm_index += 1; - return .byval; - } - var types_index: u32 = 0; - var types_buffer: [8]Builder.Type = undefined; - for (classes) |class| { - switch (class) { - .integer => { - types_buffer[types_index] = .i64; - types_index += 1; - }, - .sse => { - types_buffer[types_index] = .double; - types_index += 1; - }, - .sseup => { - if (types_buffer[types_index - 1] == .double) { - types_buffer[types_index - 1] = .fp128; - } else { - types_buffer[types_index] = .double; - types_index += 1; - } - }, - .float => { - types_buffer[types_index] = .float; - types_index += 1; - }, - .float_combine => { - types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float); - types_index += 1; - }, - .x87 => { - it.zig_index += 1; - it.llvm_index += 1; - it.byval_attr = true; - return .byref; - }, - .x87up => unreachable, - .none => break, - .memory => unreachable, // handled above - .win_i128 => unreachable, // windows only - .integer_per_element => { - @panic("TODO"); - }, - } - } - const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer}); - if (first_non_integer == null or classes[first_non_integer.?] == .none) { - assert(first_non_integer orelse classes.len == types_index); - if (types_index == 1) { - it.zig_index += 1; - it.llvm_index += 1; - return .abi_sized_int; - } - if (it.llvm_index + types_index > 6) { - it.zig_index += 1; - it.llvm_index += 1; - it.byval_attr = true; - return .byref; - } - switch (ip.indexToKey(ty.toIntern())) { - .struct_type => { - const size = ty.abiSize(zcu); - assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index); - if (size % 8 > 0) { - types_buffer[types_index - 1] = - try it.object.builder.intType(@intCast(size % 8 * 8)); - } - }, - else => {}, - } - } - it.types_len = types_index; - it.types_buffer = types_buffer; - it.llvm_index += types_index; - it.zig_index += 1; - return .multiple_llvm_types; - } -}; - -fn iterateParamTypes(object: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) ParamTypeIterator { - return .{ - .object = object, - .pt = pt, - .fn_info = fn_info, - .zig_index = 0, - .llvm_index = 0, - .types_len = 0, - .types_buffer = undefined, - .byval_attr = false, - }; -} - -/// This function deliberately does not handle `_BitInt` because it typically -/// has different ABI than regular integer types, and there is no currently no -/// way to determine whether a Zig integer type is meant to represent e.g. `int` -/// or `_BitInt(32)`. -fn ccAbiPromoteInt(cc: std.builtin.CallingConvention, zcu: *Zcu, ty: Type) ?std.builtin.Signedness { - switch (cc) { - .auto, .@"inline", .async => return null, - else => {}, - } - - const int_info = switch (ty.zigTypeTag(zcu)) { - .bool => Type.u1.intInfo(zcu), - else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return null, - }; - assert(int_info.bits >= 0); - - const target = zcu.getTarget(); - return switch (target.cpu.arch) { - .aarch64, - .aarch64_be, - => switch (target.os.tag) { - .driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => switch (int_info.bits) { - 8, 16 => int_info.signedness, - else => null, - }, - else => null, - }, - - .avr, - => switch (int_info.bits) { - 8 => int_info.signedness, - else => null, - }, - - .lanai, - => null, - - .loongarch64, - .riscv64, - .riscv64be, - => switch (int_info.bits) { - 8, 16 => int_info.signedness, - 32 => .signed, - else => null, - }, - - .mips, - .mipsel, - .mips64, - .mips64el, - => switch (int_info.bits) { - 8, 16, 64 => int_info.signedness, - // https://github.com/llvm/llvm-project/issues/179088 - // 32 => .signed, - else => null, - }, - - .powerpc64, - .powerpc64le, - .s390x, - .sparc64, - .ve, - => switch (int_info.bits) { - 8, 16, 32 => int_info.signedness, - else => null, - }, - - else => switch (int_info.bits) { - 8, 16 => int_info.signedness, - else => null, - }, - }; -} - -/// This is the one source of truth for whether a type is passed around as an LLVM pointer, -/// or as an LLVM value. -fn isByRef(ty: Type, zcu: *Zcu) bool { - // For tuples and structs, if there are more than this many non-void - // fields, then we make it byref, otherwise byval. - const max_fields_byval = 0; - const ip = &zcu.intern_pool; - - switch (ty.zigTypeTag(zcu)) { - .type, - .comptime_int, - .comptime_float, - .enum_literal, - .undefined, - .null, - .@"opaque", - => unreachable, - - .noreturn, - .void, - .bool, - .int, - .float, - .pointer, - .error_set, - .@"fn", - .@"enum", - .vector, - .@"anyframe", - => return false, - - .array, .frame => return ty.hasRuntimeBits(zcu), - .@"struct" => { - const struct_type = switch (ip.indexToKey(ty.toIntern())) { - .tuple_type => |tuple| { - var count: usize = 0; - for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| { - if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue; - - count += 1; - if (count > max_fields_byval) return true; - if (isByRef(Type.fromInterned(field_ty), zcu)) return true; - } - return false; - }, - .struct_type => ip.loadStructType(ty.toIntern()), - else => unreachable, - }; - - // Packed structs are represented to LLVM as integers. - if (struct_type.layout == .@"packed") return false; - - const field_types = struct_type.field_types.get(ip); - var it = struct_type.iterateRuntimeOrder(ip); - var count: usize = 0; - while (it.next()) |field_index| { - count += 1; - if (count > max_fields_byval) return true; - const field_ty = Type.fromInterned(field_types[field_index]); - if (isByRef(field_ty, zcu)) return true; - } - return false; - }, - .@"union" => switch (ty.containerLayout(zcu)) { - .@"packed" => return false, - else => return ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu), - }, - .error_union => { - const payload_ty = ty.errorUnionPayload(zcu); - if (!payload_ty.hasRuntimeBits(zcu)) { - return false; - } - return true; - }, - .optional => { - const payload_ty = ty.optionalChild(zcu); - if (!payload_ty.hasRuntimeBits(zcu)) { - return false; - } - if (ty.optionalReprIsPayload(zcu)) { - return false; - } - return true; - }, - } -} - -fn isScalar(zcu: *Zcu, ty: Type) bool { - return switch (ty.zigTypeTag(zcu)) { - .void, - .bool, - .noreturn, - .int, - .float, - .pointer, - .optional, - .error_set, - .@"enum", - .@"anyframe", - .vector, - => true, - - .@"struct" => ty.containerLayout(zcu) == .@"packed", - .@"union" => ty.containerLayout(zcu) == .@"packed", - else => false, - }; -} - -/// This function returns true if we expect LLVM to lower x86_fp80 correctly -/// and false if we expect LLVM to crash if it encounters an x86_fp80 type, -/// or if it produces miscompilations. -fn backendSupportsF80(target: *const std.Target) bool { - return switch (target.cpu.arch) { - .x86, .x86_64 => !target.cpu.has(.x86, .soft_float), - else => false, - }; -} - /// This function returns true if we expect LLVM to lower f16 correctly /// and false if we expect LLVM to crash if it encounters an f16 type, /// or if it produces miscompilations. -fn backendSupportsF16(target: *const std.Target) bool { +pub fn backendSupportsF16(target: *const std.Target) bool { return switch (target.cpu.arch) { // https://github.com/llvm/llvm-project/issues/97981 .csky, @@ -12594,10 +4808,20 @@ fn backendSupportsF16(target: *const std.Target) bool { }; } +/// This function returns true if we expect LLVM to lower x86_fp80 correctly +/// and false if we expect LLVM to crash if it encounters an x86_fp80 type, +/// or if it produces miscompilations. +pub fn backendSupportsF80(target: *const std.Target) bool { + return switch (target.cpu.arch) { + .x86, .x86_64 => !target.cpu.has(.x86, .soft_float), + else => false, + }; +} + /// This function returns true if we expect LLVM to lower f128 correctly, /// and false if we expect LLVM to crash if it encounters an f128 type, /// or if it produces miscompilations. -fn backendSupportsF128(target: *const std.Target) bool { +pub fn backendSupportsF128(target: *const std.Target) bool { return switch (target.cpu.arch) { // https://github.com/llvm/llvm-project/issues/121122 .amdgcn, @@ -12616,17 +4840,6 @@ fn backendSupportsF128(target: *const std.Target) bool { }; } -/// LLVM does not support all relevant intrinsics for all targets, so we -/// may need to manually generate a compiler-rt call. -fn intrinsicsAllowed(scalar_ty: Type, target: *const std.Target) bool { - return switch (scalar_ty.toIntern()) { - .f16_type => backendSupportsF16(target), - .f80_type => (target.cTypeBitSize(.longdouble) == 80) and backendSupportsF80(target), - .f128_type => (target.cTypeBitSize(.longdouble) == 128) and backendSupportsF128(target), - else => true, - }; -} - /// We need to insert extra padding if LLVM's isn't enough. /// However we don't want to ever call LLVMABIAlignmentOfType or /// LLVMABISizeOfType because these functions will trip assertions @@ -12638,264 +4851,188 @@ const struct_layout_version = 2; // TODO: Restore the non_null field to i1 once // https://github.com/llvm/llvm-project/issues/56585/ is fixed -const optional_layout_version = 3; +pub const optional_layout_version = 3; const lt_errors_fn_name = "__zig_lt_errors_len"; -fn compilerRtIntBits(bits: u16) ?u16 { - inline for (.{ 32, 64, 128 }) |b| { - if (bits <= b) { - return b; - } - } - return null; -} - -fn buildAllocaInner( - wip: *Builder.WipFunction, - llvm_ty: Builder.Type, - alignment: Builder.Alignment, - target: *const std.Target, -) Allocator.Error!Builder.Value { - const address_space = llvmAllocaAddressSpace(target); - - const alloca = blk: { - const prev_cursor = wip.cursor; - const prev_debug_location = wip.debug_location; - defer { - wip.cursor = prev_cursor; - if (wip.cursor.block == .entry) wip.cursor.instruction += 1; - wip.debug_location = prev_debug_location; - } - - wip.cursor = .{ .block = .entry }; - wip.debug_location = .no_location; - break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, ""); - }; - - // The pointer returned from this function should have the generic address space, - // if this isn't the case then cast it to the generic address space. - return wip.conv(.unneeded, alloca, .ptr, ""); -} - -fn errUnionPayloadOffset(payload_ty: Type, pt: Zcu.PerThread) !u1 { - const zcu = pt.zcu; - const err_int_ty = try pt.errorIntType(); - return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.gt, payload_ty.abiAlignment(zcu))); -} - -fn errUnionErrorOffset(payload_ty: Type, pt: Zcu.PerThread) !u1 { - const zcu = pt.zcu; - const err_int_ty = try pt.errorIntType(); - return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.lte, payload_ty.abiAlignment(zcu))); -} - -/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location -/// -/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang -fn constraintAllowsMemory(constraint: []const u8) bool { - // TODO: This implementation is woefully incomplete. - for (constraint) |byte| { - switch (byte) { - '=', '*', ',', '&' => {}, - 'm', 'o', 'X', 'g' => return true, - else => {}, - } - } else return false; -} - -/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register -/// -/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang -fn constraintAllowsRegister(constraint: []const u8) bool { - // TODO: This implementation is woefully incomplete. - for (constraint) |byte| { - switch (byte) { - '=', '*', ',', '&' => {}, - 'm', 'o' => {}, - else => return true, - } - } else return false; -} - pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void { switch (arch) { .aarch64, .aarch64_be => { - llvm.LLVMInitializeAArch64Target(); - llvm.LLVMInitializeAArch64TargetInfo(); - llvm.LLVMInitializeAArch64TargetMC(); - llvm.LLVMInitializeAArch64AsmPrinter(); - llvm.LLVMInitializeAArch64AsmParser(); + bindings.LLVMInitializeAArch64Target(); + bindings.LLVMInitializeAArch64TargetInfo(); + bindings.LLVMInitializeAArch64TargetMC(); + bindings.LLVMInitializeAArch64AsmPrinter(); + bindings.LLVMInitializeAArch64AsmParser(); }, .amdgcn => { - llvm.LLVMInitializeAMDGPUTarget(); - llvm.LLVMInitializeAMDGPUTargetInfo(); - llvm.LLVMInitializeAMDGPUTargetMC(); - llvm.LLVMInitializeAMDGPUAsmPrinter(); - llvm.LLVMInitializeAMDGPUAsmParser(); + bindings.LLVMInitializeAMDGPUTarget(); + bindings.LLVMInitializeAMDGPUTargetInfo(); + bindings.LLVMInitializeAMDGPUTargetMC(); + bindings.LLVMInitializeAMDGPUAsmPrinter(); + bindings.LLVMInitializeAMDGPUAsmParser(); }, .thumb, .thumbeb, .arm, .armeb => { - llvm.LLVMInitializeARMTarget(); - llvm.LLVMInitializeARMTargetInfo(); - llvm.LLVMInitializeARMTargetMC(); - llvm.LLVMInitializeARMAsmPrinter(); - llvm.LLVMInitializeARMAsmParser(); + bindings.LLVMInitializeARMTarget(); + bindings.LLVMInitializeARMTargetInfo(); + bindings.LLVMInitializeARMTargetMC(); + bindings.LLVMInitializeARMAsmPrinter(); + bindings.LLVMInitializeARMAsmParser(); }, .avr => { - llvm.LLVMInitializeAVRTarget(); - llvm.LLVMInitializeAVRTargetInfo(); - llvm.LLVMInitializeAVRTargetMC(); - llvm.LLVMInitializeAVRAsmPrinter(); - llvm.LLVMInitializeAVRAsmParser(); + bindings.LLVMInitializeAVRTarget(); + bindings.LLVMInitializeAVRTargetInfo(); + bindings.LLVMInitializeAVRTargetMC(); + bindings.LLVMInitializeAVRAsmPrinter(); + bindings.LLVMInitializeAVRAsmParser(); }, .bpfel, .bpfeb => { - llvm.LLVMInitializeBPFTarget(); - llvm.LLVMInitializeBPFTargetInfo(); - llvm.LLVMInitializeBPFTargetMC(); - llvm.LLVMInitializeBPFAsmPrinter(); - llvm.LLVMInitializeBPFAsmParser(); + bindings.LLVMInitializeBPFTarget(); + bindings.LLVMInitializeBPFTargetInfo(); + bindings.LLVMInitializeBPFTargetMC(); + bindings.LLVMInitializeBPFAsmPrinter(); + bindings.LLVMInitializeBPFAsmParser(); }, .hexagon => { - llvm.LLVMInitializeHexagonTarget(); - llvm.LLVMInitializeHexagonTargetInfo(); - llvm.LLVMInitializeHexagonTargetMC(); - llvm.LLVMInitializeHexagonAsmPrinter(); - llvm.LLVMInitializeHexagonAsmParser(); + bindings.LLVMInitializeHexagonTarget(); + bindings.LLVMInitializeHexagonTargetInfo(); + bindings.LLVMInitializeHexagonTargetMC(); + bindings.LLVMInitializeHexagonAsmPrinter(); + bindings.LLVMInitializeHexagonAsmParser(); }, .lanai => { - llvm.LLVMInitializeLanaiTarget(); - llvm.LLVMInitializeLanaiTargetInfo(); - llvm.LLVMInitializeLanaiTargetMC(); - llvm.LLVMInitializeLanaiAsmPrinter(); - llvm.LLVMInitializeLanaiAsmParser(); + bindings.LLVMInitializeLanaiTarget(); + bindings.LLVMInitializeLanaiTargetInfo(); + bindings.LLVMInitializeLanaiTargetMC(); + bindings.LLVMInitializeLanaiAsmPrinter(); + bindings.LLVMInitializeLanaiAsmParser(); }, .mips, .mipsel, .mips64, .mips64el => { - llvm.LLVMInitializeMipsTarget(); - llvm.LLVMInitializeMipsTargetInfo(); - llvm.LLVMInitializeMipsTargetMC(); - llvm.LLVMInitializeMipsAsmPrinter(); - llvm.LLVMInitializeMipsAsmParser(); + bindings.LLVMInitializeMipsTarget(); + bindings.LLVMInitializeMipsTargetInfo(); + bindings.LLVMInitializeMipsTargetMC(); + bindings.LLVMInitializeMipsAsmPrinter(); + bindings.LLVMInitializeMipsAsmParser(); }, .msp430 => { - llvm.LLVMInitializeMSP430Target(); - llvm.LLVMInitializeMSP430TargetInfo(); - llvm.LLVMInitializeMSP430TargetMC(); - llvm.LLVMInitializeMSP430AsmPrinter(); - llvm.LLVMInitializeMSP430AsmParser(); + bindings.LLVMInitializeMSP430Target(); + bindings.LLVMInitializeMSP430TargetInfo(); + bindings.LLVMInitializeMSP430TargetMC(); + bindings.LLVMInitializeMSP430AsmPrinter(); + bindings.LLVMInitializeMSP430AsmParser(); }, .nvptx, .nvptx64 => { - llvm.LLVMInitializeNVPTXTarget(); - llvm.LLVMInitializeNVPTXTargetInfo(); - llvm.LLVMInitializeNVPTXTargetMC(); - llvm.LLVMInitializeNVPTXAsmPrinter(); + bindings.LLVMInitializeNVPTXTarget(); + bindings.LLVMInitializeNVPTXTargetInfo(); + bindings.LLVMInitializeNVPTXTargetMC(); + bindings.LLVMInitializeNVPTXAsmPrinter(); // There is no LLVMInitializeNVPTXAsmParser function available. }, .powerpc, .powerpcle, .powerpc64, .powerpc64le => { - llvm.LLVMInitializePowerPCTarget(); - llvm.LLVMInitializePowerPCTargetInfo(); - llvm.LLVMInitializePowerPCTargetMC(); - llvm.LLVMInitializePowerPCAsmPrinter(); - llvm.LLVMInitializePowerPCAsmParser(); + bindings.LLVMInitializePowerPCTarget(); + bindings.LLVMInitializePowerPCTargetInfo(); + bindings.LLVMInitializePowerPCTargetMC(); + bindings.LLVMInitializePowerPCAsmPrinter(); + bindings.LLVMInitializePowerPCAsmParser(); }, .riscv32, .riscv32be, .riscv64, .riscv64be => { - llvm.LLVMInitializeRISCVTarget(); - llvm.LLVMInitializeRISCVTargetInfo(); - llvm.LLVMInitializeRISCVTargetMC(); - llvm.LLVMInitializeRISCVAsmPrinter(); - llvm.LLVMInitializeRISCVAsmParser(); + bindings.LLVMInitializeRISCVTarget(); + bindings.LLVMInitializeRISCVTargetInfo(); + bindings.LLVMInitializeRISCVTargetMC(); + bindings.LLVMInitializeRISCVAsmPrinter(); + bindings.LLVMInitializeRISCVAsmParser(); }, .sparc, .sparc64 => { - llvm.LLVMInitializeSparcTarget(); - llvm.LLVMInitializeSparcTargetInfo(); - llvm.LLVMInitializeSparcTargetMC(); - llvm.LLVMInitializeSparcAsmPrinter(); - llvm.LLVMInitializeSparcAsmParser(); + bindings.LLVMInitializeSparcTarget(); + bindings.LLVMInitializeSparcTargetInfo(); + bindings.LLVMInitializeSparcTargetMC(); + bindings.LLVMInitializeSparcAsmPrinter(); + bindings.LLVMInitializeSparcAsmParser(); }, .s390x => { - llvm.LLVMInitializeSystemZTarget(); - llvm.LLVMInitializeSystemZTargetInfo(); - llvm.LLVMInitializeSystemZTargetMC(); - llvm.LLVMInitializeSystemZAsmPrinter(); - llvm.LLVMInitializeSystemZAsmParser(); + bindings.LLVMInitializeSystemZTarget(); + bindings.LLVMInitializeSystemZTargetInfo(); + bindings.LLVMInitializeSystemZTargetMC(); + bindings.LLVMInitializeSystemZAsmPrinter(); + bindings.LLVMInitializeSystemZAsmParser(); }, .wasm32, .wasm64 => { - llvm.LLVMInitializeWebAssemblyTarget(); - llvm.LLVMInitializeWebAssemblyTargetInfo(); - llvm.LLVMInitializeWebAssemblyTargetMC(); - llvm.LLVMInitializeWebAssemblyAsmPrinter(); - llvm.LLVMInitializeWebAssemblyAsmParser(); + bindings.LLVMInitializeWebAssemblyTarget(); + bindings.LLVMInitializeWebAssemblyTargetInfo(); + bindings.LLVMInitializeWebAssemblyTargetMC(); + bindings.LLVMInitializeWebAssemblyAsmPrinter(); + bindings.LLVMInitializeWebAssemblyAsmParser(); }, .x86, .x86_64 => { - llvm.LLVMInitializeX86Target(); - llvm.LLVMInitializeX86TargetInfo(); - llvm.LLVMInitializeX86TargetMC(); - llvm.LLVMInitializeX86AsmPrinter(); - llvm.LLVMInitializeX86AsmParser(); + bindings.LLVMInitializeX86Target(); + bindings.LLVMInitializeX86TargetInfo(); + bindings.LLVMInitializeX86TargetMC(); + bindings.LLVMInitializeX86AsmPrinter(); + bindings.LLVMInitializeX86AsmParser(); }, .xtensa => { if (build_options.llvm_has_xtensa) { - llvm.LLVMInitializeXtensaTarget(); - llvm.LLVMInitializeXtensaTargetInfo(); - llvm.LLVMInitializeXtensaTargetMC(); + bindings.LLVMInitializeXtensaTarget(); + bindings.LLVMInitializeXtensaTargetInfo(); + bindings.LLVMInitializeXtensaTargetMC(); // There is no LLVMInitializeXtensaAsmPrinter function. - llvm.LLVMInitializeXtensaAsmParser(); + bindings.LLVMInitializeXtensaAsmParser(); } }, .xcore => { - llvm.LLVMInitializeXCoreTarget(); - llvm.LLVMInitializeXCoreTargetInfo(); - llvm.LLVMInitializeXCoreTargetMC(); - llvm.LLVMInitializeXCoreAsmPrinter(); + bindings.LLVMInitializeXCoreTarget(); + bindings.LLVMInitializeXCoreTargetInfo(); + bindings.LLVMInitializeXCoreTargetMC(); + bindings.LLVMInitializeXCoreAsmPrinter(); // There is no LLVMInitializeXCoreAsmParser function. }, .m68k => { if (build_options.llvm_has_m68k) { - llvm.LLVMInitializeM68kTarget(); - llvm.LLVMInitializeM68kTargetInfo(); - llvm.LLVMInitializeM68kTargetMC(); - llvm.LLVMInitializeM68kAsmPrinter(); - llvm.LLVMInitializeM68kAsmParser(); + bindings.LLVMInitializeM68kTarget(); + bindings.LLVMInitializeM68kTargetInfo(); + bindings.LLVMInitializeM68kTargetMC(); + bindings.LLVMInitializeM68kAsmPrinter(); + bindings.LLVMInitializeM68kAsmParser(); } }, .csky => { if (build_options.llvm_has_csky) { - llvm.LLVMInitializeCSKYTarget(); - llvm.LLVMInitializeCSKYTargetInfo(); - llvm.LLVMInitializeCSKYTargetMC(); + bindings.LLVMInitializeCSKYTarget(); + bindings.LLVMInitializeCSKYTargetInfo(); + bindings.LLVMInitializeCSKYTargetMC(); // There is no LLVMInitializeCSKYAsmPrinter function. - llvm.LLVMInitializeCSKYAsmParser(); + bindings.LLVMInitializeCSKYAsmParser(); } }, .ve => { - llvm.LLVMInitializeVETarget(); - llvm.LLVMInitializeVETargetInfo(); - llvm.LLVMInitializeVETargetMC(); - llvm.LLVMInitializeVEAsmPrinter(); - llvm.LLVMInitializeVEAsmParser(); + bindings.LLVMInitializeVETarget(); + bindings.LLVMInitializeVETargetInfo(); + bindings.LLVMInitializeVETargetMC(); + bindings.LLVMInitializeVEAsmPrinter(); + bindings.LLVMInitializeVEAsmParser(); }, .arc => { if (build_options.llvm_has_arc) { - llvm.LLVMInitializeARCTarget(); - llvm.LLVMInitializeARCTargetInfo(); - llvm.LLVMInitializeARCTargetMC(); - llvm.LLVMInitializeARCAsmPrinter(); + bindings.LLVMInitializeARCTarget(); + bindings.LLVMInitializeARCTargetInfo(); + bindings.LLVMInitializeARCTargetMC(); + bindings.LLVMInitializeARCAsmPrinter(); // There is no LLVMInitializeARCAsmParser function. } }, .loongarch32, .loongarch64 => { - llvm.LLVMInitializeLoongArchTarget(); - llvm.LLVMInitializeLoongArchTargetInfo(); - llvm.LLVMInitializeLoongArchTargetMC(); - llvm.LLVMInitializeLoongArchAsmPrinter(); - llvm.LLVMInitializeLoongArchAsmParser(); + bindings.LLVMInitializeLoongArchTarget(); + bindings.LLVMInitializeLoongArchTargetInfo(); + bindings.LLVMInitializeLoongArchTargetMC(); + bindings.LLVMInitializeLoongArchAsmPrinter(); + bindings.LLVMInitializeLoongArchAsmParser(); }, .spirv32, .spirv64, => { - llvm.LLVMInitializeSPIRVTarget(); - llvm.LLVMInitializeSPIRVTargetInfo(); - llvm.LLVMInitializeSPIRVTargetMC(); - llvm.LLVMInitializeSPIRVAsmPrinter(); + bindings.LLVMInitializeSPIRVTarget(); + bindings.LLVMInitializeSPIRVTargetInfo(); + bindings.LLVMInitializeSPIRVTargetMC(); + bindings.LLVMInitializeSPIRVAsmPrinter(); }, // LLVM does does not have a backend for these. @@ -12916,296 +5053,3 @@ pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void { => unreachable, } } - -fn minIntConst(b: *Builder, min_ty: Type, as_ty: Builder.Type, zcu: *const Zcu) Allocator.Error!Builder.Constant { - const info = min_ty.intInfo(zcu); - if (info.signedness == .unsigned or info.bits == 0) { - return b.intConst(as_ty, 0); - } - if (std.math.cast(u6, info.bits - 1)) |shift| { - const min_val: i64 = @as(i64, std.math.minInt(i64)) >> (63 - shift); - return b.intConst(as_ty, min_val); - } - var res: std.math.big.int.Managed = try .init(zcu.gpa); - defer res.deinit(); - try res.setTwosCompIntLimit(.min, info.signedness, info.bits); - return b.bigIntConst(as_ty, res.toConst()); -} - -fn maxIntConst(b: *Builder, max_ty: Type, as_ty: Builder.Type, zcu: *const Zcu) Allocator.Error!Builder.Constant { - const info = max_ty.intInfo(zcu); - switch (info.bits) { - 0 => return b.intConst(as_ty, 0), - 1 => switch (info.signedness) { - .signed => return b.intConst(as_ty, 0), - .unsigned => return b.intConst(as_ty, 1), - }, - else => {}, - } - const unsigned_bits = switch (info.signedness) { - .unsigned => info.bits, - .signed => info.bits - 1, - }; - if (std.math.cast(u6, unsigned_bits)) |shift| { - const max_val: u64 = (@as(u64, 1) << shift) - 1; - return b.intConst(as_ty, max_val); - } - var res: std.math.big.int.Managed = try .init(zcu.gpa); - defer res.deinit(); - try res.setTwosCompIntLimit(.max, info.signedness, info.bits); - return b.bigIntConst(as_ty, res.toConst()); -} - -/// Appends zero or more LLVM constraints to `llvm_constraints`, returning how many were added. -fn appendConstraints( - gpa: Allocator, - llvm_constraints: *std.ArrayList(u8), - zig_name: []const u8, - target: *const std.Target, -) error{OutOfMemory}!usize { - switch (target.cpu.arch) { - .mips, .mipsel, .mips64, .mips64el => if (mips_clobber_overrides.get(zig_name)) |llvm_tag| { - const llvm_name = @tagName(llvm_tag); - try llvm_constraints.ensureUnusedCapacity(gpa, llvm_name.len + 4); - llvm_constraints.appendSliceAssumeCapacity("~{"); - llvm_constraints.appendSliceAssumeCapacity(llvm_name); - llvm_constraints.appendSliceAssumeCapacity("},"); - return 1; - }, - else => {}, - } - - try llvm_constraints.ensureUnusedCapacity(gpa, zig_name.len + 4); - llvm_constraints.appendSliceAssumeCapacity("~{"); - llvm_constraints.appendSliceAssumeCapacity(zig_name); - llvm_constraints.appendSliceAssumeCapacity("},"); - return 1; -} - -const mips_clobber_overrides = std.StaticStringMap(enum { - @"$msair", - @"$msacsr", - @"$msaaccess", - @"$msasave", - @"$msamodify", - @"$msarequest", - @"$msamap", - @"$msaunmap", - @"$f0", - @"$f1", - @"$f2", - @"$f3", - @"$f4", - @"$f5", - @"$f6", - @"$f7", - @"$f8", - @"$f9", - @"$f10", - @"$f11", - @"$f12", - @"$f13", - @"$f14", - @"$f15", - @"$f16", - @"$f17", - @"$f18", - @"$f19", - @"$f20", - @"$f21", - @"$f22", - @"$f23", - @"$f24", - @"$f25", - @"$f26", - @"$f27", - @"$f28", - @"$f29", - @"$f30", - @"$f31", - @"$fcc0", - @"$fcc1", - @"$fcc2", - @"$fcc3", - @"$fcc4", - @"$fcc5", - @"$fcc6", - @"$fcc7", - @"$w0", - @"$w1", - @"$w2", - @"$w3", - @"$w4", - @"$w5", - @"$w6", - @"$w7", - @"$w8", - @"$w9", - @"$w10", - @"$w11", - @"$w12", - @"$w13", - @"$w14", - @"$w15", - @"$w16", - @"$w17", - @"$w18", - @"$w19", - @"$w20", - @"$w21", - @"$w22", - @"$w23", - @"$w24", - @"$w25", - @"$w26", - @"$w27", - @"$w28", - @"$w29", - @"$w30", - @"$w31", - @"$0", - @"$1", - @"$2", - @"$3", - @"$4", - @"$5", - @"$6", - @"$7", - @"$8", - @"$9", - @"$10", - @"$11", - @"$12", - @"$13", - @"$14", - @"$15", - @"$16", - @"$17", - @"$18", - @"$19", - @"$20", - @"$21", - @"$22", - @"$23", - @"$24", - @"$25", - @"$26", - @"$27", - @"$28", - @"$29", - @"$30", - @"$31", -}).initComptime(.{ - .{ "msa_ir", .@"$msair" }, - .{ "msa_csr", .@"$msacsr" }, - .{ "msa_access", .@"$msaaccess" }, - .{ "msa_save", .@"$msasave" }, - .{ "msa_modify", .@"$msamodify" }, - .{ "msa_request", .@"$msarequest" }, - .{ "msa_map", .@"$msamap" }, - .{ "msa_unmap", .@"$msaunmap" }, - .{ "f0", .@"$f0" }, - .{ "f1", .@"$f1" }, - .{ "f2", .@"$f2" }, - .{ "f3", .@"$f3" }, - .{ "f4", .@"$f4" }, - .{ "f5", .@"$f5" }, - .{ "f6", .@"$f6" }, - .{ "f7", .@"$f7" }, - .{ "f8", .@"$f8" }, - .{ "f9", .@"$f9" }, - .{ "f10", .@"$f10" }, - .{ "f11", .@"$f11" }, - .{ "f12", .@"$f12" }, - .{ "f13", .@"$f13" }, - .{ "f14", .@"$f14" }, - .{ "f15", .@"$f15" }, - .{ "f16", .@"$f16" }, - .{ "f17", .@"$f17" }, - .{ "f18", .@"$f18" }, - .{ "f19", .@"$f19" }, - .{ "f20", .@"$f20" }, - .{ "f21", .@"$f21" }, - .{ "f22", .@"$f22" }, - .{ "f23", .@"$f23" }, - .{ "f24", .@"$f24" }, - .{ "f25", .@"$f25" }, - .{ "f26", .@"$f26" }, - .{ "f27", .@"$f27" }, - .{ "f28", .@"$f28" }, - .{ "f29", .@"$f29" }, - .{ "f30", .@"$f30" }, - .{ "f31", .@"$f31" }, - .{ "fcc0", .@"$fcc0" }, - .{ "fcc1", .@"$fcc1" }, - .{ "fcc2", .@"$fcc2" }, - .{ "fcc3", .@"$fcc3" }, - .{ "fcc4", .@"$fcc4" }, - .{ "fcc5", .@"$fcc5" }, - .{ "fcc6", .@"$fcc6" }, - .{ "fcc7", .@"$fcc7" }, - .{ "w0", .@"$w0" }, - .{ "w1", .@"$w1" }, - .{ "w2", .@"$w2" }, - .{ "w3", .@"$w3" }, - .{ "w4", .@"$w4" }, - .{ "w5", .@"$w5" }, - .{ "w6", .@"$w6" }, - .{ "w7", .@"$w7" }, - .{ "w8", .@"$w8" }, - .{ "w9", .@"$w9" }, - .{ "w10", .@"$w10" }, - .{ "w11", .@"$w11" }, - .{ "w12", .@"$w12" }, - .{ "w13", .@"$w13" }, - .{ "w14", .@"$w14" }, - .{ "w15", .@"$w15" }, - .{ "w16", .@"$w16" }, - .{ "w17", .@"$w17" }, - .{ "w18", .@"$w18" }, - .{ "w19", .@"$w19" }, - .{ "w20", .@"$w20" }, - .{ "w21", .@"$w21" }, - .{ "w22", .@"$w22" }, - .{ "w23", .@"$w23" }, - .{ "w24", .@"$w24" }, - .{ "w25", .@"$w25" }, - .{ "w26", .@"$w26" }, - .{ "w27", .@"$w27" }, - .{ "w28", .@"$w28" }, - .{ "w29", .@"$w29" }, - .{ "w30", .@"$w30" }, - .{ "w31", .@"$w31" }, - .{ "r0", .@"$0" }, - .{ "r1", .@"$1" }, - .{ "r2", .@"$2" }, - .{ "r3", .@"$3" }, - .{ "r4", .@"$4" }, - .{ "r5", .@"$5" }, - .{ "r6", .@"$6" }, - .{ "r7", .@"$7" }, - .{ "r8", .@"$8" }, - .{ "r9", .@"$9" }, - .{ "r10", .@"$10" }, - .{ "r11", .@"$11" }, - .{ "r12", .@"$12" }, - .{ "r13", .@"$13" }, - .{ "r14", .@"$14" }, - .{ "r15", .@"$15" }, - .{ "r16", .@"$16" }, - .{ "r17", .@"$17" }, - .{ "r18", .@"$18" }, - .{ "r19", .@"$19" }, - .{ "r20", .@"$20" }, - .{ "r21", .@"$21" }, - .{ "r22", .@"$22" }, - .{ "r23", .@"$23" }, - .{ "r24", .@"$24" }, - .{ "r25", .@"$25" }, - .{ "r26", .@"$26" }, - .{ "r27", .@"$27" }, - .{ "r28", .@"$28" }, - .{ "r29", .@"$29" }, - .{ "r30", .@"$30" }, - .{ "r31", .@"$31" }, -}); diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig new file mode 100644 index 0000000000000000000000000000000000000000..52408a023a840a87965d0e31725f6e6e4d8a6887 --- /dev/null +++ b/src/codegen/llvm/FuncGen.zig @@ -0,0 +1,8145 @@ +const FuncGen = @This(); + +pub const Error = Zcu.CodegenFailError; + +object: *Object, +nav_index: InternPool.Nav.Index, +pt: Zcu.PerThread, +gpa: Allocator, +air: Air, +liveness: Air.Liveness, +wip: Builder.WipFunction, +is_naked: bool, +fuzz: ?Fuzz, + +file: Builder.Metadata, +scope: Builder.Metadata, + +inlined_at: Builder.Metadata.Optional, + +base_line: u32, +prev_dbg_line: c_uint, +prev_dbg_column: c_uint, + +/// This stores the LLVM values used in a function, such that they can be referred to +/// in other instructions. This table is cleared before every function is generated. +func_inst_table: std.AutoHashMapUnmanaged(Air.Inst.Ref, Builder.Value), + +/// If the return type is sret, this is the result pointer. Otherwise null. +/// Note that this can disagree with isByRef for the return type in the case +/// of C ABI functions. +ret_ptr: Builder.Value, +/// Any function that needs to perform Valgrind client requests needs an array alloca +/// instruction, however a maximum of one per function is needed. +valgrind_client_request_array: Builder.Value = .none, +/// These fields are used to refer to the LLVM value of the function parameters +/// in an Arg instruction. +/// This list may be shorter than the list according to the zig type system; +/// it omits 0-bit types. If the function uses sret as the first parameter, +/// this slice does not include it. +args: []const Builder.Value, +arg_index: u32, +arg_inline_index: u32, + +err_ret_trace: Builder.Value, + +/// This data structure is used to implement breaking to blocks. +blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { + parent_bb: Builder.Function.Block.Index, + breaks: *BreakList, +}), + +/// Maps `loop` instructions to the bb to branch to to repeat the loop. +loops: std.AutoHashMapUnmanaged(Air.Inst.Index, Builder.Function.Block.Index), + +/// Maps `loop_switch_br` instructions to the information required to lower +/// dispatches (`switch_dispatch` instructions). +switch_dispatch_info: std.AutoHashMapUnmanaged(Air.Inst.Index, SwitchDispatchInfo), + +sync_scope: Builder.SyncScope, + +disable_intrinsics: bool, + +/// Have we seen loads or stores involving `allowzero` pointers? +allowzero_access: bool, + +fn todo(fg: *FuncGen, comptime format: []const u8, args: anytype) Error { + @branchHint(.cold); + return fg.pt.zcu.codegenFail( + fg.nav_index, + "TODO (LLVM): " ++ format, + args, + ); +} + +fn ownerModule(fg: *const FuncGen) *Package.Module { + return fg.pt.zcu.navFileScope(fg.nav_index).mod.?; +} + +fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void { + // LLVM already considers null pointers to be valid in non-generic address spaces, so avoid + // pessimizing optimization for functions with accesses to such pointers. + if (info.flags.address_space == .generic and info.flags.is_allowzero) self.allowzero_access = true; +} + +pub const Fuzz = struct { + counters_variable: Builder.Variable.Index, + pcs: std.ArrayList(Builder.Constant), + + fn deinit(f: *Fuzz, gpa: Allocator) void { + f.pcs.deinit(gpa); + f.* = undefined; + } +}; + +const SwitchDispatchInfo = struct { + /// These are the blocks corresponding to each switch case. + /// The final element corresponds to the `else` case. + /// Slices allocated into `gpa`. + case_blocks: []Builder.Function.Block.Index, + /// This is `.none` if `jmp_table` is set, since we won't use a `switch` instruction to dispatch. + switch_weights: Builder.Function.Instruction.BrCond.Weights, + /// If not `null`, we have manually constructed a jump table to reach the desired block. + /// `table` can be used if the value is between `min` and `max` inclusive. + /// We perform this lowering manually to avoid some questionable behavior from LLVM. + /// See `airSwitchBr` for details. + jmp_table: ?JmpTable, + + const JmpTable = struct { + min: Builder.Constant, + max: Builder.Constant, + in_bounds_hint: enum { none, unpredictable, likely, unlikely }, + /// Pointer to the jump table itself, to be used with `indirectbr`. + /// The index into the jump table is the dispatch condition minus `min`. + /// The table values are `blockaddress` constants corresponding to blocks in `case_blocks`. + table: Builder.Constant, + /// `true` if `table` conatins a reference to the `else` block. + /// In this case, the `indirectbr` must include the `else` block in its target list. + table_includes_else: bool, + }; +}; + +const BreakList = union { + list: std.MultiArrayList(struct { + bb: Builder.Function.Block.Index, + val: Builder.Value, + }), + len: usize, +}; + +pub fn deinit(self: *FuncGen) void { + const gpa = self.gpa; + if (self.fuzz) |*f| f.deinit(self.gpa); + self.wip.deinit(); + self.func_inst_table.deinit(gpa); + self.blocks.deinit(gpa); + self.loops.deinit(gpa); + var it = self.switch_dispatch_info.valueIterator(); + while (it.next()) |info| { + self.gpa.free(info.case_blocks); + } + self.switch_dispatch_info.deinit(gpa); +} + +fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value { + const gpa = self.gpa; + const gop = try self.func_inst_table.getOrPut(gpa, inst); + if (gop.found_existing) return gop.value_ptr.*; + + const llvm_val = try self.resolveValue((try self.air.value(inst, self.pt)).?); + gop.value_ptr.* = llvm_val.toValue(); + return llvm_val.toValue(); +} + +fn resolveValue(self: *FuncGen, val: Value) Error!Builder.Constant { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty = val.typeOf(zcu); + if (!isByRef(ty, zcu)) { + return o.lowerValue(pt, val.toIntern()); + } else { + // We need a pointer to a global constant, i.e. a UAV. + return o.lowerUavRef(pt, .{ + .val = val.toIntern(), + .orig_ty = (try pt.singleConstPtrType(ty)).toIntern(), + }); + } +} + +pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) Error!void { + const o = self.object; + const zcu = self.pt.zcu; + const ip = &zcu.intern_pool; + const air_tags = self.air.instructions.items(.tag); + switch (coverage_point) { + .none => {}, + .poi => if (self.fuzz) |*fuzz| { + const poi_index = fuzz.pcs.items.len; + const base_ptr = fuzz.counters_variable.toValue(&o.builder); + const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{ + try o.builder.intValue(.i32, poi_index), + }, ""); + const one = try o.builder.intValue(.i8, 1); + _ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, ""); + + // LLVM does not allow blockaddress on the entry block. + const pc = if (self.wip.cursor.block == .entry) + self.wip.function.toConst(&o.builder) + else + try o.builder.blockAddrConst(self.wip.function, self.wip.cursor.block); + const gpa = self.gpa; + try fuzz.pcs.append(gpa, pc); + }, + } + for (body, 0..) |inst, i| { + if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue; + + const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) { + // zig fmt: off + + // No "scalarize" legalizations are enabled, so these instructions never appear. + .legalize_vec_elem_val => unreachable, + .legalize_vec_store_elem => unreachable, + // No soft float legalizations are enabled. + .legalize_compiler_rt_call => unreachable, + + .add => try self.airAdd(inst, .normal), + .add_optimized => try self.airAdd(inst, .fast), + .add_wrap => try self.airAddWrap(inst), + .add_sat => try self.airAddSat(inst), + + .sub => try self.airSub(inst, .normal), + .sub_optimized => try self.airSub(inst, .fast), + .sub_wrap => try self.airSubWrap(inst), + .sub_sat => try self.airSubSat(inst), + + .mul => try self.airMul(inst, .normal), + .mul_optimized => try self.airMul(inst, .fast), + .mul_wrap => try self.airMulWrap(inst), + .mul_sat => try self.airMulSat(inst), + + .add_safe => try self.airSafeArithmetic(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"), + .sub_safe => try self.airSafeArithmetic(inst, .@"ssub.with.overflow", .@"usub.with.overflow"), + .mul_safe => try self.airSafeArithmetic(inst, .@"smul.with.overflow", .@"umul.with.overflow"), + + .div_float => try self.airDivFloat(inst, .normal), + .div_trunc => try self.airDivTrunc(inst, .normal), + .div_floor => try self.airDivFloor(inst, .normal), + .div_exact => try self.airDivExact(inst, .normal), + .rem => try self.airRem(inst, .normal), + .mod => try self.airMod(inst, .normal), + .abs => try self.airAbs(inst), + .ptr_add => try self.airPtrAdd(inst), + .ptr_sub => try self.airPtrSub(inst), + .shl => try self.airShl(inst), + .shl_sat => try self.airShlSat(inst), + .shl_exact => try self.airShlExact(inst), + .min => try self.airMin(inst), + .max => try self.airMax(inst), + .slice => try self.airSlice(inst), + .mul_add => try self.airMulAdd(inst), + + .div_float_optimized => try self.airDivFloat(inst, .fast), + .div_trunc_optimized => try self.airDivTrunc(inst, .fast), + .div_floor_optimized => try self.airDivFloor(inst, .fast), + .div_exact_optimized => try self.airDivExact(inst, .fast), + .rem_optimized => try self.airRem(inst, .fast), + .mod_optimized => try self.airMod(inst, .fast), + + .add_with_overflow => try self.airOverflow(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"), + .sub_with_overflow => try self.airOverflow(inst, .@"ssub.with.overflow", .@"usub.with.overflow"), + .mul_with_overflow => try self.airOverflow(inst, .@"smul.with.overflow", .@"umul.with.overflow"), + .shl_with_overflow => try self.airShlWithOverflow(inst), + + .bit_and, .bool_and => try self.airAnd(inst), + .bit_or, .bool_or => try self.airOr(inst), + .xor => try self.airXor(inst), + .shr => try self.airShr(inst, false), + .shr_exact => try self.airShr(inst, true), + + .sqrt => try self.airUnaryOp(inst, .sqrt), + .sin => try self.airUnaryOp(inst, .sin), + .cos => try self.airUnaryOp(inst, .cos), + .tan => try self.airUnaryOp(inst, .tan), + .exp => try self.airUnaryOp(inst, .exp), + .exp2 => try self.airUnaryOp(inst, .exp2), + .log => try self.airUnaryOp(inst, .log), + .log2 => try self.airUnaryOp(inst, .log2), + .log10 => try self.airUnaryOp(inst, .log10), + .floor => try self.airUnaryOp(inst, .floor), + .ceil => try self.airUnaryOp(inst, .ceil), + .round => try self.airUnaryOp(inst, .round), + .trunc_float => try self.airUnaryOp(inst, .trunc), + + .neg => try self.airNeg(inst, .normal), + .neg_optimized => try self.airNeg(inst, .fast), + + .cmp_eq => try self.airCmp(inst, .eq, .normal), + .cmp_gt => try self.airCmp(inst, .gt, .normal), + .cmp_gte => try self.airCmp(inst, .gte, .normal), + .cmp_lt => try self.airCmp(inst, .lt, .normal), + .cmp_lte => try self.airCmp(inst, .lte, .normal), + .cmp_neq => try self.airCmp(inst, .neq, .normal), + + .cmp_eq_optimized => try self.airCmp(inst, .eq, .fast), + .cmp_gt_optimized => try self.airCmp(inst, .gt, .fast), + .cmp_gte_optimized => try self.airCmp(inst, .gte, .fast), + .cmp_lt_optimized => try self.airCmp(inst, .lt, .fast), + .cmp_lte_optimized => try self.airCmp(inst, .lte, .fast), + .cmp_neq_optimized => try self.airCmp(inst, .neq, .fast), + + .cmp_vector => try self.airCmpVector(inst, .normal), + .cmp_vector_optimized => try self.airCmpVector(inst, .fast), + .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), + + .is_non_null => try self.airIsNonNull(inst, false, .ne), + .is_non_null_ptr => try self.airIsNonNull(inst, true , .ne), + .is_null => try self.airIsNonNull(inst, false, .eq), + .is_null_ptr => try self.airIsNonNull(inst, true , .eq), + + .is_non_err => try self.airIsErr(inst, .eq, false), + .is_non_err_ptr => try self.airIsErr(inst, .eq, true), + .is_err => try self.airIsErr(inst, .ne, false), + .is_err_ptr => try self.airIsErr(inst, .ne, true), + + .alloc => try self.airAlloc(inst), + .ret_ptr => try self.airRetPtr(inst), + .arg => try self.airArg(inst), + .bitcast => try self.airBitCast(inst), + .breakpoint => try self.airBreakpoint(inst), + .ret_addr => try self.airRetAddr(inst), + .frame_addr => try self.airFrameAddress(inst), + .@"try" => try self.airTry(inst, false), + .try_cold => try self.airTry(inst, true), + .try_ptr => try self.airTryPtr(inst, false), + .try_ptr_cold => try self.airTryPtr(inst, true), + .intcast => try self.airIntCast(inst, false), + .intcast_safe => try self.airIntCast(inst, true), + .trunc => try self.airTrunc(inst), + .fptrunc => try self.airFptrunc(inst), + .fpext => try self.airFpext(inst), + .load => try self.airLoad(inst), + .not => try self.airNot(inst), + .store => try self.airStore(inst, false), + .store_safe => try self.airStore(inst, true), + .assembly => try self.airAssembly(inst), + .slice_ptr => try self.airSliceField(inst, 0), + .slice_len => try self.airSliceField(inst, 1), + + .ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0), + .ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1), + + .int_from_float => try self.airIntFromFloat(inst, .normal), + .int_from_float_optimized => try self.airIntFromFloat(inst, .fast), + .int_from_float_safe => unreachable, // handled by `legalizeFeatures` + .int_from_float_optimized_safe => unreachable, // handled by `legalizeFeatures` + + .array_to_slice => try self.airArrayToSlice(inst), + .float_from_int => try self.airFloatFromInt(inst), + .cmpxchg_weak => try self.airCmpxchg(inst, .weak), + .cmpxchg_strong => try self.airCmpxchg(inst, .strong), + .atomic_rmw => try self.airAtomicRmw(inst), + .atomic_load => try self.airAtomicLoad(inst), + .memset => try self.airMemset(inst, false), + .memset_safe => try self.airMemset(inst, true), + .memcpy => try self.airMemcpy(inst), + .memmove => try self.airMemmove(inst), + .set_union_tag => try self.airSetUnionTag(inst), + .get_union_tag => try self.airGetUnionTag(inst), + .clz => try self.airClzCtz(inst, .ctlz), + .ctz => try self.airClzCtz(inst, .cttz), + .popcount => try self.airBitOp(inst, .ctpop), + .byte_swap => try self.airByteSwap(inst), + .bit_reverse => try self.airBitOp(inst, .bitreverse), + .tag_name => try self.airTagName(inst), + .error_name => try self.airErrorName(inst), + .splat => try self.airSplat(inst), + .select => try self.airSelect(inst), + .shuffle_one => try self.airShuffleOne(inst), + .shuffle_two => try self.airShuffleTwo(inst), + .aggregate_init => try self.airAggregateInit(inst), + .union_init => try self.airUnionInit(inst), + .prefetch => try self.airPrefetch(inst), + .addrspace_cast => try self.airAddrSpaceCast(inst), + + .is_named_enum_value => try self.airIsNamedEnumValue(inst), + .error_set_has_value => try self.airErrorSetHasValue(inst), + + .reduce => try self.airReduce(inst, .normal), + .reduce_optimized => try self.airReduce(inst, .fast), + + .atomic_store_unordered => try self.airAtomicStore(inst, .unordered), + .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic), + .atomic_store_release => try self.airAtomicStore(inst, .release), + .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst), + + .struct_field_ptr => try self.airStructFieldPtr(inst), + .struct_field_val => try self.airStructFieldVal(inst), + + .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0), + .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1), + .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2), + .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3), + + .field_parent_ptr => try self.airFieldParentPtr(inst), + + .array_elem_val => try self.airArrayElemVal(inst), + .slice_elem_val => try self.airSliceElemVal(inst), + .slice_elem_ptr => try self.airSliceElemPtr(inst), + .ptr_elem_val => try self.airPtrElemVal(inst), + .ptr_elem_ptr => try self.airPtrElemPtr(inst), + + .optional_payload => try self.airOptionalPayload(inst), + .optional_payload_ptr => try self.airOptionalPayloadPtr(inst), + .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst), + + .unwrap_errunion_payload => try self.airErrUnionPayload(inst, false), + .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true), + .unwrap_errunion_err => try self.airErrUnionErr(inst, false), + .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true), + .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst), + .err_return_trace => try self.airErrReturnTrace(inst), + .set_err_return_trace => try self.airSetErrReturnTrace(inst), + .save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst), + + .wrap_optional => try self.airWrapOptional(body[i..]), + .wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]), + .wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]), + + .wasm_memory_size => try self.airWasmMemorySize(inst), + .wasm_memory_grow => try self.airWasmMemoryGrow(inst), + + .runtime_nav_ptr => try self.airRuntimeNavPtr(inst), + + .inferred_alloc, .inferred_alloc_comptime => unreachable, + + .dbg_stmt => try self.airDbgStmt(inst), + .dbg_empty_stmt => try self.airDbgEmptyStmt(inst), + .dbg_var_ptr => try self.airDbgVarPtr(inst), + .dbg_var_val => try self.airDbgVarVal(inst, false), + .dbg_arg_inline => try self.airDbgVarVal(inst, true), + + .c_va_arg => try self.airCVaArg(inst), + .c_va_copy => try self.airCVaCopy(inst), + .c_va_end => try self.airCVaEnd(inst), + .c_va_start => try self.airCVaStart(inst), + + .work_item_id => try self.airWorkItemId(inst), + .work_group_size => try self.airWorkGroupSize(inst), + .work_group_id => try self.airWorkGroupId(inst), + + // Instructions that are known to always be `noreturn` based on their tag. + .br => return self.airBr(inst), + .repeat => return self.airRepeat(inst), + .switch_dispatch => return self.airSwitchDispatch(inst), + .cond_br => return self.airCondBr(inst), + .switch_br => return self.airSwitchBr(inst, false), + .loop_switch_br => return self.airSwitchBr(inst, true), + .loop => return self.airLoop(inst), + .ret => return self.airRet(inst, false), + .ret_safe => return self.airRet(inst, true), + .ret_load => return self.airRetLoad(inst), + .trap => return self.airTrap(inst), + .unreach => return self.airUnreach(inst), + + // Instructions which may be `noreturn`. + .block => res: { + const res = try self.airBlock(inst); + if (self.typeOfIndex(inst).isNoReturn(zcu)) return; + break :res res; + }, + .dbg_inline_block => res: { + const res = try self.airDbgInlineBlock(inst); + if (self.typeOfIndex(inst).isNoReturn(zcu)) return; + break :res res; + }, + .call, .call_always_tail, .call_never_tail, .call_never_inline => |tag| res: { + const res = try self.airCall(inst, switch (tag) { + .call => .auto, + .call_always_tail => .always_tail, + .call_never_tail => .never_tail, + .call_never_inline => .never_inline, + else => unreachable, + }); + // TODO: the AIR we emit for calls is a bit weird - the instruction has + // type `noreturn`, but there are instructions (and maybe a safety check) following + // nonetheless. The `unreachable` or safety check should be emitted by backends instead. + //if (self.typeOfIndex(inst).isNoReturn(mod)) return; + break :res res; + }, + + // zig fmt: on + }; + if (val != .none) try self.func_inst_table.putNoClobber(self.gpa, inst.toRef(), val); + } + unreachable; +} + +fn genBodyDebugScope( + self: *FuncGen, + maybe_inline_func: ?InternPool.Index, + body: []const Air.Inst.Index, + coverage_point: Air.CoveragePoint, +) Error!void { + const o = self.object; + + if (self.wip.strip) return self.genBody(body, coverage_point); + + const old_debug_location = self.wip.debug_location; + const old_file = self.file; + const old_inlined_at = self.inlined_at; + const old_base_line = self.base_line; + defer if (maybe_inline_func) |_| { + self.wip.debug_location = old_debug_location; + self.file = old_file; + self.inlined_at = old_inlined_at; + self.base_line = old_base_line; + }; + + const old_scope = self.scope; + defer self.scope = old_scope; + + if (maybe_inline_func) |inline_func| { + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + + const func = zcu.funcInfo(inline_func); + const nav = ip.getNav(func.owner_nav); + const file_scope = zcu.navFileScopeIndex(func.owner_nav); + const mod = zcu.fileByIndex(file_scope).mod.?; + + self.file = try o.getDebugFile(pt, file_scope); + + self.base_line = zcu.navSrcLine(func.owner_nav); + const line_number = self.base_line + 1; + self.inlined_at = try self.wip.debug_location.toMetadata(&o.builder); + + const fn_ty = try pt.funcType(.{ + .param_types = &.{}, + .return_type = .void_type, + }); + + self.scope = try o.builder.debugSubprogram( + self.file, + try o.builder.metadataString(nav.name.toSlice(&zcu.intern_pool)), + try o.builder.metadataString(nav.fqn.toSlice(&zcu.intern_pool)), + line_number, + line_number + func.lbrace_line, + try o.getDebugType(pt, fn_ty), + .{ + .di_flags = .{ .StaticMember = true }, + .sp_flags = .{ + .Optimized = mod.optimize_mode != .Debug, + .Definition = true, + .LocalToUnit = true, // inline functions cannot be exported + }, + }, + o.debug_compile_unit.unwrap().?, + ); + } + + self.scope = try o.builder.debugLexicalBlock( + self.scope, + self.file, + self.prev_dbg_line, + self.prev_dbg_column, + ); + self.wip.debug_location = .{ .location = .{ + .line = self.prev_dbg_line, + .column = self.prev_dbg_column, + .scope = self.scope.toOptional(), + .inlined_at = self.inlined_at, + } }; + + try self.genBody(body, coverage_point); +} + +const CallAttr = enum { + Auto, + NeverTail, + NeverInline, + AlwaysTail, + AlwaysInline, +}; + +fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value { + const air_call = self.air.unwrapCall(inst); + const args = air_call.args; + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const callee_ty = self.typeOf(air_call.callee); + const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) { + .@"fn" => callee_ty, + .pointer => callee_ty.childType(zcu), + else => unreachable, + }; + const fn_info = zcu.typeToFunc(zig_fn_ty).?; + const return_type = Type.fromInterned(fn_info.return_type); + const llvm_fn = try self.resolveInst(air_call.callee); + const target = zcu.getTarget(); + const sret = firstParamSRet(fn_info, zcu, target); + + var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa); + defer llvm_args.deinit(); + + var attributes: Builder.FunctionAttributes.Wip = .{}; + defer attributes.deinit(&o.builder); + + if (self.disable_intrinsics) { + try attributes.addFnAttr(.nobuiltin, &o.builder); + } + + switch (modifier) { + .auto, .always_tail => {}, + .never_tail, .never_inline => try attributes.addFnAttr(.@"noinline", &o.builder), + .no_suspend, .always_inline, .compile_time => unreachable, + } + + const ret_ptr = if (!sret) null else blk: { + const llvm_ret_ty = try o.lowerType(pt, return_type); + try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder); + + const alignment = return_type.abiAlignment(zcu).toLlvm(); + const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment); + try llvm_args.append(ret_ptr); + break :blk ret_ptr; + }; + + const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; + if (err_return_tracing) { + assert(self.err_ret_trace != .none); + try llvm_args.append(self.err_ret_trace); + } + + var it = iterateParamTypes(o, pt, fn_info); + while (try it.nextCall(self, args)) |lowering| switch (lowering) { + .no_bits => continue, + .byval => { + const arg = args[it.zig_index - 1]; + const param_ty = self.typeOf(arg); + const llvm_arg = try self.resolveInst(arg); + const llvm_param_ty = try o.lowerType(pt, param_ty); + if (isByRef(param_ty, zcu)) { + const alignment = param_ty.abiAlignment(zcu).toLlvm(); + const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, ""); + try llvm_args.append(loaded); + } else { + try llvm_args.append(llvm_arg); + } + }, + .byref => { + const arg = args[it.zig_index - 1]; + const param_ty = self.typeOf(arg); + const llvm_arg = try self.resolveInst(arg); + if (isByRef(param_ty, zcu)) { + try llvm_args.append(llvm_arg); + } else { + const alignment = param_ty.abiAlignment(zcu).toLlvm(); + const param_llvm_ty = llvm_arg.typeOfWip(&self.wip); + const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment); + _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment); + try llvm_args.append(arg_ptr); + } + }, + .byref_mut => { + const arg = args[it.zig_index - 1]; + const param_ty = self.typeOf(arg); + const llvm_arg = try self.resolveInst(arg); + + const alignment = param_ty.abiAlignment(zcu).toLlvm(); + const param_llvm_ty = try o.lowerType(pt, param_ty); + const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment); + if (isByRef(param_ty, zcu)) { + const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, ""); + _ = try self.wip.store(.normal, loaded, arg_ptr, alignment); + } else { + _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment); + } + try llvm_args.append(arg_ptr); + }, + .abi_sized_int => { + const arg = args[it.zig_index - 1]; + const param_ty = self.typeOf(arg); + const llvm_arg = try self.resolveInst(arg); + const int_llvm_ty = try o.builder.intType(@intCast(param_ty.abiSize(zcu) * 8)); + + if (isByRef(param_ty, zcu)) { + const alignment = param_ty.abiAlignment(zcu).toLlvm(); + const loaded = try self.wip.load(.normal, int_llvm_ty, llvm_arg, alignment, ""); + try llvm_args.append(loaded); + } else { + // LLVM does not allow bitcasting structs so we must allocate + // a local, store as one type, and then load as another type. + const alignment = param_ty.abiAlignment(zcu).toLlvm(); + const int_ptr = try self.buildAlloca(int_llvm_ty, alignment); + _ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment); + const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, ""); + try llvm_args.append(loaded); + } + }, + .slice => { + const arg = args[it.zig_index - 1]; + const llvm_arg = try self.resolveInst(arg); + const ptr = try self.wip.extractValue(llvm_arg, &.{0}, ""); + const len = try self.wip.extractValue(llvm_arg, &.{1}, ""); + try llvm_args.appendSlice(&.{ ptr, len }); + }, + .multiple_llvm_types => { + const arg = args[it.zig_index - 1]; + const param_ty = self.typeOf(arg); + const llvm_types = it.types_buffer[0..it.types_len]; + const llvm_arg = try self.resolveInst(arg); + const is_by_ref = isByRef(param_ty, zcu); + const arg_ptr = if (is_by_ref) llvm_arg else ptr: { + const alignment = param_ty.abiAlignment(zcu).toLlvm(); + const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment); + _ = try self.wip.store(.normal, llvm_arg, ptr, alignment); + break :ptr ptr; + }; + + const llvm_ty = try o.builder.structType(.normal, llvm_types); + try llvm_args.ensureUnusedCapacity(it.types_len); + for (llvm_types, 0..) |field_ty, i| { + const alignment = + Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8)); + const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, ""); + const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, ""); + llvm_args.appendAssumeCapacity(loaded); + } + }, + .float_array => |count| { + const arg = args[it.zig_index - 1]; + const arg_ty = self.typeOf(arg); + var llvm_arg = try self.resolveInst(arg); + const alignment = arg_ty.abiAlignment(zcu).toLlvm(); + if (!isByRef(arg_ty, zcu)) { + const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment); + _ = try self.wip.store(.normal, llvm_arg, ptr, alignment); + llvm_arg = ptr; + } + + const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?); + const array_ty = try o.builder.arrayType(count, float_ty); + + const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, ""); + try llvm_args.append(loaded); + }, + .i32_array, .i64_array => |arr_len| { + const elem_size: u8 = if (lowering == .i32_array) 32 else 64; + const arg = args[it.zig_index - 1]; + const arg_ty = self.typeOf(arg); + var llvm_arg = try self.resolveInst(arg); + const alignment = arg_ty.abiAlignment(zcu).toLlvm(); + if (!isByRef(arg_ty, zcu)) { + const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment); + _ = try self.wip.store(.normal, llvm_arg, ptr, alignment); + llvm_arg = ptr; + } + + const array_ty = + try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size))); + const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, ""); + try llvm_args.append(loaded); + }, + }; + + { + // Add argument attributes. + it = iterateParamTypes(o, pt, fn_info); + it.llvm_index += @intFromBool(sret); + it.llvm_index += @intFromBool(err_return_tracing); + while (try it.next()) |lowering| switch (lowering) { + .byval => { + const param_index = it.zig_index - 1; + const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); + if (!isByRef(param_ty, zcu)) { + try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); + } + }, + .byref => { + const param_index = it.zig_index - 1; + const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); + const param_llvm_ty = try o.lowerType(pt, param_ty); + const alignment = param_ty.abiAlignment(zcu).toLlvm(); + try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty); + }, + .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), + // No attributes needed for these. + .no_bits, + .abi_sized_int, + .multiple_llvm_types, + .float_array, + .i32_array, + .i64_array, + => continue, + + .slice => { + assert(!it.byval_attr); + const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const ptr_info = param_ty.ptrInfo(zcu); + const llvm_arg_i = it.llvm_index - 2; + + if (math.cast(u5, it.zig_index - 1)) |i| { + if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { + try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); + } + } + if (param_ty.zigTypeTag(zcu) != .optional and + !ptr_info.flags.is_allowzero and + ptr_info.flags.address_space == .generic) + { + try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); + } + if (ptr_info.flags.is_const) { + try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder); + } + const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) { + else => |a| .wrap(a.toLlvm()), + .none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)), + }; + try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder); + }, + }; + } + + const call = try self.wip.call( + switch (modifier) { + .auto, .never_inline => .normal, + .never_tail => .notail, + .always_tail => .musttail, + .no_suspend, .always_inline, .compile_time => unreachable, + }, + toLlvmCallConvTag(fn_info.cc, target).?, + try attributes.finish(&o.builder), + try o.lowerType(pt, zig_fn_ty), + llvm_fn, + llvm_args.items, + "", + ); + + if (fn_info.return_type == .noreturn_type and modifier != .always_tail) { + return .none; + } + + if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) { + return .none; + } + + const llvm_ret_ty = try o.lowerType(pt, return_type); + if (ret_ptr) |rp| { + if (isByRef(return_type, zcu)) { + return rp; + } else { + // our by-ref status disagrees with sret so we must load. + const return_alignment = return_type.abiAlignment(zcu).toLlvm(); + return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, ""); + } + } + + const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); + + if (abi_ret_ty != llvm_ret_ty) { + // In this case the function return type is honoring the calling convention by having + // a different LLVM type than the usual one. We solve this here at the callsite + // by using our canonical type, then loading it if necessary. + const alignment = return_type.abiAlignment(zcu).toLlvm(); + const rp = try self.buildAlloca(abi_ret_ty, alignment); + _ = try self.wip.store(.normal, call, rp, alignment); + return if (isByRef(return_type, zcu)) + rp + else + try self.wip.load(.normal, llvm_ret_ty, rp, alignment, ""); + } + + if (isByRef(return_type, zcu)) { + // our by-ref status disagrees with sret so we must allocate, store, + // and return the allocation pointer. + const alignment = return_type.abiAlignment(zcu).toLlvm(); + const rp = try self.buildAlloca(llvm_ret_ty, alignment); + _ = try self.wip.store(.normal, call, rp, alignment); + return rp; + } else { + return call; + } +} + +fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) !void { + const o = fg.object; + const pt = fg.pt; + const zcu = pt.zcu; + const target = zcu.getTarget(); + const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin())); + const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?; + const panic_global = try o.resolveLlvmFunction(pt, panic_func.owner_nav); + + const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto; + if (has_err_trace) assert(fg.err_ret_trace != .none); + _ = try fg.wip.callIntrinsicAssumeCold(); + _ = try fg.wip.call( + .normal, + toLlvmCallConvTag(fn_info.cc, target).?, + .none, + panic_global.typeOf(&o.builder), + panic_global.toValue(&o.builder), + if (has_err_trace) &.{fg.err_ret_trace} else &.{}, + "", + ); + _ = try fg.wip.@"unreachable"(); +} + +fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const ret_ty = self.typeOf(un_op); + + if (self.ret_ptr != .none) { + const ptr_ty = try pt.singleMutPtrType(ret_ty); + + const operand = try self.resolveInst(un_op); + const val_is_undef = if (try self.air.value(un_op, pt)) |val| val.isUndef(zcu) else false; + if (val_is_undef and safety) undef: { + const ptr_info = ptr_ty.ptrInfo(zcu); + const needs_bitmask = (ptr_info.packed_offset.host_size != 0); + if (needs_bitmask) { + // TODO: only some bits are to be undef, we cannot write with a simple memset. + // meanwhile, ignore the write rather than stomping over valid bits. + // https://github.com/ziglang/zig/issues/15337 + break :undef; + } + const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu)); + _ = try self.wip.callMemSet( + self.ret_ptr, + ptr_ty.ptrAlignment(zcu).toLlvm(), + try o.builder.intValue(.i8, 0xaa), + len, + .normal, + self.disable_intrinsics, + ); + const owner_mod = self.ownerModule(); + if (owner_mod.valgrind) { + try self.valgrindMarkUndef(self.ret_ptr, len); + } + _ = try self.wip.retVoid(); + return; + } + + const unwrapped_operand = operand.unwrap(); + const unwrapped_ret = self.ret_ptr.unwrap(); + + // Return value was stored previously + if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) { + _ = try self.wip.retVoid(); + return; + } + + try self.store(self.ret_ptr, ptr_ty, operand, .none); + _ = try self.wip.retVoid(); + return; + } + const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?; + if (!ret_ty.hasRuntimeBits(zcu)) { + if (Type.fromInterned(fn_info.return_type).isError(zcu)) { + // Functions with an empty error set are emitted with an error code + // return type and return zero so they can be function pointers coerced + // to functions that return anyerror. + _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0)); + } else { + _ = try self.wip.retVoid(); + } + return; + } + + const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); + const operand = try self.resolveInst(un_op); + const val_is_undef = if (try self.air.value(un_op, pt)) |val| val.isUndef(zcu) else false; + const alignment = ret_ty.abiAlignment(zcu).toLlvm(); + + if (val_is_undef and safety) { + const llvm_ret_ty = operand.typeOfWip(&self.wip); + const rp = try self.buildAlloca(llvm_ret_ty, alignment); + const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu)); + _ = try self.wip.callMemSet( + rp, + alignment, + try o.builder.intValue(.i8, 0xaa), + len, + .normal, + self.disable_intrinsics, + ); + const owner_mod = self.ownerModule(); + if (owner_mod.valgrind) { + try self.valgrindMarkUndef(rp, len); + } + _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, "")); + return; + } + + if (isByRef(ret_ty, zcu)) { + // operand is a pointer however self.ret_ptr is null so that means + // we need to return a value. + _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, "")); + return; + } + + const llvm_ret_ty = operand.typeOfWip(&self.wip); + if (abi_ret_ty == llvm_ret_ty) { + _ = try self.wip.ret(operand); + return; + } + + const rp = try self.buildAlloca(llvm_ret_ty, alignment); + _ = try self.wip.store(.normal, operand, rp, alignment); + _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, "")); + return; +} + +fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const ptr_ty = self.typeOf(un_op); + const ret_ty = ptr_ty.childType(zcu); + const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?; + if (!ret_ty.hasRuntimeBits(zcu)) { + if (Type.fromInterned(fn_info.return_type).isError(zcu)) { + // Functions with an empty error set are emitted with an error code + // return type and return zero so they can be function pointers coerced + // to functions that return anyerror. + _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0)); + } else { + _ = try self.wip.retVoid(); + } + return; + } + if (self.ret_ptr != .none) { + _ = try self.wip.retVoid(); + return; + } + const ptr = try self.resolveInst(un_op); + const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); + const alignment = ret_ty.abiAlignment(zcu).toLlvm(); + _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, "")); + return; +} + +fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const list = try self.resolveInst(ty_op.operand); + const arg_ty = ty_op.ty.toType(); + const llvm_arg_ty = try o.lowerType(pt, arg_ty); + + return self.wip.vaArg(list, llvm_arg_ty, ""); +} + +fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const src_list = try self.resolveInst(ty_op.operand); + const va_list_ty = ty_op.ty.toType(); + const llvm_va_list_ty = try o.lowerType(pt, va_list_ty); + + const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); + const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); + + _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, ""); + return if (isByRef(va_list_ty, zcu)) + dest_list + else + try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, ""); +} + +fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const src_list = try self.resolveInst(un_op); + + _ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, ""); + return .none; +} + +fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const va_list_ty = self.typeOfIndex(inst); + const llvm_va_list_ty = try o.lowerType(pt, va_list_ty); + + const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); + const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); + + _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, ""); + return if (isByRef(va_list_ty, zcu)) + dest_list + else + try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, ""); +} + +fn airCmp( + self: *FuncGen, + inst: Air.Inst.Index, + op: math.CompareOperator, + fast: Builder.FastMathKind, +) !Builder.Value { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const operand_ty = self.typeOf(bin_op.lhs); + + return self.cmp(fast, op, operand_ty, lhs, rhs); +} + +fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data; + + const lhs = try self.resolveInst(extra.lhs); + const rhs = try self.resolveInst(extra.rhs); + const vec_ty = self.typeOf(extra.lhs); + const cmp_op = extra.compareOperator(); + + return self.cmp(fast, cmp_op, vec_ty, lhs, rhs); +} + +fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + const llvm_fn = try o.getCmpLtErrorsLenFunction(pt); + return self.wip.call( + .normal, + .fastcc, + .none, + llvm_fn.typeOf(&o.builder), + llvm_fn.toValue(&o.builder), + &.{operand}, + "", + ); +} + +fn cmp( + self: *FuncGen, + fast: Builder.FastMathKind, + op: math.CompareOperator, + operand_ty: Type, + lhs: Builder.Value, + rhs: Builder.Value, +) Allocator.Error!Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const scalar_ty = operand_ty.scalarType(zcu); + const int_ty = switch (scalar_ty.zigTypeTag(zcu)) { + .@"enum" => scalar_ty.intTagType(zcu), + .int, .bool, .pointer, .error_set => scalar_ty, + .optional => blk: { + const payload_ty = operand_ty.optionalChild(zcu); + if (!payload_ty.hasRuntimeBits(zcu) or + operand_ty.optionalReprIsPayload(zcu)) + { + break :blk operand_ty; + } + // We need to emit instructions to check for equality/inequality + // of optionals that are not pointers. + const is_by_ref = isByRef(scalar_ty, zcu); + const opt_llvm_ty = try o.lowerType(pt, scalar_ty); + const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal); + const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal); + const llvm_i2 = try o.builder.intType(2); + const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, ""); + const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, ""); + const lhs_shifted = try self.wip.bin(.shl, lhs_non_null_i2, try o.builder.intValue(llvm_i2, 1), ""); + const lhs_rhs_ored = try self.wip.bin(.@"or", lhs_shifted, rhs_non_null_i2, ""); + const both_null_block = try self.wip.block(1, "BothNull"); + const mixed_block = try self.wip.block(1, "Mixed"); + const both_pl_block = try self.wip.block(1, "BothNonNull"); + const end_block = try self.wip.block(3, "End"); + var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none); + defer wip_switch.finish(&self.wip); + try wip_switch.addCase( + try o.builder.intConst(llvm_i2, 0b00), + both_null_block, + &self.wip, + ); + try wip_switch.addCase( + try o.builder.intConst(llvm_i2, 0b11), + both_pl_block, + &self.wip, + ); + + self.wip.cursor = .{ .block = both_null_block }; + _ = try self.wip.br(end_block); + + self.wip.cursor = .{ .block = mixed_block }; + _ = try self.wip.br(end_block); + + self.wip.cursor = .{ .block = both_pl_block }; + const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true); + const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true); + const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload); + _ = try self.wip.br(end_block); + const both_pl_block_end = self.wip.cursor.block; + + self.wip.cursor = .{ .block = end_block }; + const llvm_i1_0 = Builder.Value.false; + const llvm_i1_1 = Builder.Value.true; + const incoming_values: [3]Builder.Value = .{ + switch (op) { + .eq => llvm_i1_1, + .neq => llvm_i1_0, + else => unreachable, + }, + switch (op) { + .eq => llvm_i1_0, + .neq => llvm_i1_1, + else => unreachable, + }, + payload_cmp, + }; + + const phi = try self.wip.phi(.i1, ""); + phi.finish( + &incoming_values, + &.{ both_null_block, mixed_block, both_pl_block_end }, + &self.wip, + ); + return phi.toValue(); + }, + .float => return self.buildFloatCmp(fast, op, operand_ty, .{ lhs, rhs }), + .@"struct", .@"union" => scalar_ty.bitpackBackingInt(zcu), + else => unreachable, + }; + const is_signed = int_ty.isSignedInt(zcu); + const cond: Builder.IntegerCondition = switch (op) { + .eq => .eq, + .neq => .ne, + .lt => if (is_signed) .slt else .ult, + .lte => if (is_signed) .sle else .ule, + .gt => if (is_signed) .sgt else .ugt, + .gte => if (is_signed) .sge else .uge, + }; + return self.wip.icmp(cond, lhs, rhs, ""); +} + +fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const block = self.air.unwrapBlock(inst); + return self.lowerBlock(inst, null, block.body); +} + +fn lowerBlock( + self: *FuncGen, + inst: Air.Inst.Index, + maybe_inline_func: ?InternPool.Index, + body: []const Air.Inst.Index, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const inst_ty = self.typeOfIndex(inst); + + if (inst_ty.isNoReturn(zcu)) { + try self.genBodyDebugScope(maybe_inline_func, body, .none); + return .none; + } + + const have_block_result = inst_ty.hasRuntimeBits(zcu); + + var breaks: BreakList = if (have_block_result) .{ .list = .{} } else .{ .len = 0 }; + defer if (have_block_result) breaks.list.deinit(self.gpa); + + const parent_bb = try self.wip.block(0, "Block"); + try self.blocks.putNoClobber(self.gpa, inst, .{ + .parent_bb = parent_bb, + .breaks = &breaks, + }); + defer assert(self.blocks.remove(inst)); + + try self.genBodyDebugScope(maybe_inline_func, body, .none); + + self.wip.cursor = .{ .block = parent_bb }; + + // Create a phi node only if the block returns a value. + if (have_block_result) { + const raw_llvm_ty = try o.lowerType(pt, inst_ty); + const llvm_ty: Builder.Type = ty: { + // If the zig tag type is a function, this represents an actual function body; not + // a pointer to it. LLVM IR allows the call instruction to use function bodies instead + // of function pointers, however the phi makes it a runtime value and therefore + // the LLVM type has to be wrapped in a pointer. + if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) { + break :ty .ptr; + } + break :ty raw_llvm_ty; + }; + + parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len); + const phi = try self.wip.phi(llvm_ty, ""); + phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip); + return phi.toValue(); + } else { + parent_bb.ptr(&self.wip).incoming = @intCast(breaks.len); + return .none; + } +} + +fn airBr(self: *FuncGen, inst: Air.Inst.Index) !void { + const zcu = self.pt.zcu; + const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br; + const block = self.blocks.get(branch.block_inst).?; + + // Add the values to the lists only if the break provides a value. + const operand_ty = self.typeOf(branch.operand); + if (operand_ty.hasRuntimeBits(zcu)) { + const val = try self.resolveInst(branch.operand); + + // For the phi node, we need the basic blocks and the values of the + // break instructions. + try block.breaks.list.append(self.gpa, .{ .bb = self.wip.cursor.block, .val = val }); + } else block.breaks.len += 1; + _ = try self.wip.br(block.parent_bb); +} + +fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) !void { + const repeat = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat; + const loop_bb = self.loops.get(repeat.loop_inst).?; + loop_bb.ptr(&self.wip).incoming += 1; + _ = try self.wip.br(loop_bb); +} + +fn lowerSwitchDispatch( + self: *FuncGen, + switch_inst: Air.Inst.Index, + cond_ref: Air.Inst.Ref, + dispatch_info: SwitchDispatchInfo, +) !void { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const cond_ty = self.typeOf(cond_ref); + const switch_br = self.air.unwrapSwitch(switch_inst); + + if (try self.air.value(cond_ref, pt)) |cond_val| { + // Comptime-known dispatch. Iterate the cases to find the correct + // one, and branch to the corresponding element of `case_blocks`. + var it = switch_br.iterateCases(); + const target_case_idx = target: while (it.next()) |case| { + for (case.items) |item| { + const val = Value.fromInterned(item.toInterned().?); + if (cond_val.compareHetero(.eq, val, zcu)) break :target case.idx; + } + for (case.ranges) |range| { + const low = Value.fromInterned(range[0].toInterned().?); + const high = Value.fromInterned(range[1].toInterned().?); + if (cond_val.compareHetero(.gte, low, zcu) and + cond_val.compareHetero(.lte, high, zcu)) + { + break :target case.idx; + } + } + } else dispatch_info.case_blocks.len - 1; + const target_block = dispatch_info.case_blocks[target_case_idx]; + target_block.ptr(&self.wip).incoming += 1; + _ = try self.wip.br(target_block); + return; + } + + // Runtime-known dispatch. + const cond = try self.resolveInst(cond_ref); + + if (dispatch_info.jmp_table) |jmp_table| { + // We should use the constructed jump table. + // First, check the bounds to branch to the `else` case if needed. + const inbounds = try self.wip.bin( + .@"and", + try self.cmp(.normal, .gte, cond_ty, cond, jmp_table.min.toValue()), + try self.cmp(.normal, .lte, cond_ty, cond, jmp_table.max.toValue()), + "", + ); + const jmp_table_block = try self.wip.block(1, "Then"); + const else_block = dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]; + else_block.ptr(&self.wip).incoming += 1; + _ = try self.wip.brCond(inbounds, jmp_table_block, else_block, switch (jmp_table.in_bounds_hint) { + .none => .none, + .unpredictable => .unpredictable, + .likely => .then_likely, + .unlikely => .else_likely, + }); + + self.wip.cursor = .{ .block = jmp_table_block }; + + // Figure out the list of blocks we might branch to. + // This includes all case blocks, but it might not include the `else` block if + // the table is dense. + const target_blocks_len = dispatch_info.case_blocks.len - @intFromBool(!jmp_table.table_includes_else); + const target_blocks = dispatch_info.case_blocks[0..target_blocks_len]; + + // Make sure to cast the index to a usize so it's not treated as negative! + const table_index = try self.wip.conv( + .unsigned, + try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""), + try o.lowerType(pt, .usize), + "", + ); + const target_ptr_ptr = try self.wip.gep( + .inbounds, + .ptr, + jmp_table.table.toValue(), + &.{table_index}, + "", + ); + const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, ""); + + // Do the branch! + _ = try self.wip.indirectbr(target_ptr, target_blocks); + + // Mark all target blocks as having one more incoming branch. + for (target_blocks) |case_block| { + case_block.ptr(&self.wip).incoming += 1; + } + + return; + } + + // We must lower to an actual LLVM `switch` instruction. + // The switch prongs will correspond to our scalar cases. Ranges will + // be handled by conditional branches in the `else` prong. + + const llvm_usize = try o.lowerType(pt, Type.usize); + const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder)) + try self.wip.cast(.ptrtoint, cond, llvm_usize, "") + else + cond; + + const llvm_cases_len, const last_range_case = info: { + var llvm_cases_len: u32 = 0; + var last_range_case: ?u32 = null; + var it = switch_br.iterateCases(); + while (it.next()) |case| { + if (case.ranges.len > 0) last_range_case = case.idx; + llvm_cases_len += @intCast(case.items.len); + } + break :info .{ llvm_cases_len, last_range_case }; + }; + + // The `else` of the LLVM `switch` is the actual `else` prong only + // if there are no ranges. Otherwise, the `else` will have a + // conditional chain before the "true" `else` prong. + const llvm_else_block = if (last_range_case == null) + dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1] + else + try self.wip.block(0, "RangeTest"); + + llvm_else_block.ptr(&self.wip).incoming += 1; + + var wip_switch = try self.wip.@"switch"(cond_int, llvm_else_block, llvm_cases_len, dispatch_info.switch_weights); + defer wip_switch.finish(&self.wip); + + // Construct the actual cases. Set the cursor to the `else` block so + // we can construct ranges at the same time as scalar cases. + self.wip.cursor = .{ .block = llvm_else_block }; + + var it = switch_br.iterateCases(); + while (it.next()) |case| { + const case_block = dispatch_info.case_blocks[case.idx]; + + for (case.items) |item| { + const llvm_item = (try self.resolveInst(item)).toConst().?; + const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder)) + try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize) + else + llvm_item; + try wip_switch.addCase(llvm_int_item, case_block, &self.wip); + } + case_block.ptr(&self.wip).incoming += @intCast(case.items.len); + + if (case.ranges.len == 0) continue; + + // Add a conditional for the ranges, directing to the relevant bb. + // We don't need to consider `cold` branch hints since that information is stored + // in the target bb body, but we do care about likely/unlikely/unpredictable. + + const hint = switch_br.getHint(case.idx); + + var range_cond: ?Builder.Value = null; + for (case.ranges) |range| { + const llvm_min = try self.resolveInst(range[0]); + const llvm_max = try self.resolveInst(range[1]); + const cond_part = try self.wip.bin( + .@"and", + try self.cmp(.normal, .gte, cond_ty, cond, llvm_min), + try self.cmp(.normal, .lte, cond_ty, cond, llvm_max), + "", + ); + if (range_cond) |prev| { + range_cond = try self.wip.bin(.@"or", prev, cond_part, ""); + } else range_cond = cond_part; + } + + // If the check fails, we either branch to the "true" `else` case, + // or to the next range condition. + const range_else_block = if (case.idx == last_range_case.?) + dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1] + else + try self.wip.block(0, "RangeTest"); + + _ = try self.wip.brCond(range_cond.?, case_block, range_else_block, switch (hint) { + .none, .cold => .none, + .unpredictable => .unpredictable, + .likely => .then_likely, + .unlikely => .else_likely, + }); + case_block.ptr(&self.wip).incoming += 1; + range_else_block.ptr(&self.wip).incoming += 1; + + // Construct the next range conditional (if any) in the false branch. + self.wip.cursor = .{ .block = range_else_block }; + } +} + +fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) !void { + const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br; + const dispatch_info = self.switch_dispatch_info.get(br.block_inst).?; + return self.lowerSwitchDispatch(br.block_inst, br.operand, dispatch_info); +} + +fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void { + const cond_br = self.air.unwrapCondBr(inst); + const cond = try self.resolveInst(cond_br.condition); + const then_body = cond_br.then_body; + const else_body = cond_br.else_body; + + const Hint = enum { + none, + unpredictable, + then_likely, + else_likely, + then_cold, + else_cold, + }; + const hint: Hint = switch (cond_br.branch_hints.true) { + .none => switch (cond_br.branch_hints.false) { + .none => .none, + .likely => .else_likely, + .unlikely => .then_likely, + .cold => .else_cold, + .unpredictable => .unpredictable, + }, + .likely => switch (cond_br.branch_hints.false) { + .none => .then_likely, + .likely => .unpredictable, + .unlikely => .then_likely, + .cold => .else_cold, + .unpredictable => .unpredictable, + }, + .unlikely => switch (cond_br.branch_hints.false) { + .none => .else_likely, + .likely => .else_likely, + .unlikely => .unpredictable, + .cold => .else_cold, + .unpredictable => .unpredictable, + }, + .cold => .then_cold, + .unpredictable => .unpredictable, + }; + + const then_block = try self.wip.block(1, "Then"); + const else_block = try self.wip.block(1, "Else"); + _ = try self.wip.brCond(cond, then_block, else_block, switch (hint) { + .none, .then_cold, .else_cold => .none, + .unpredictable => .unpredictable, + .then_likely => .then_likely, + .else_likely => .else_likely, + }); + + self.wip.cursor = .{ .block = then_block }; + if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold(); + try self.genBodyDebugScope(null, then_body, cond_br.branch_hints.then_cov); + + self.wip.cursor = .{ .block = else_block }; + if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold(); + try self.genBodyDebugScope(null, else_body, cond_br.branch_hints.else_cov); + + // No need to reset the insert cursor since this instruction is noreturn. +} + +fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { + const unwrapped_try = self.air.unwrapTry(inst); + const err_union = try self.resolveInst(unwrapped_try.error_union); + const body = unwrapped_try.else_body; + const err_union_ty = self.typeOf(unwrapped_try.error_union); + const is_unused = self.liveness.isUnused(inst); + return lowerTry(self, err_union, body, err_union_ty, false, .none, false, is_unused, err_cold); +} + +fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { + const zcu = self.pt.zcu; + const unwrapped_try = self.air.unwrapTryPtr(inst); + const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr); + const body = unwrapped_try.else_body; + const err_union_ptr_ty = self.typeOf(unwrapped_try.error_union_ptr); + const err_union_ty = err_union_ptr_ty.childType(zcu); + const is_unused = self.liveness.isUnused(inst); + + self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu)); + + return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), true, is_unused, err_cold); +} + +fn lowerTry( + fg: *FuncGen, + err_union: Builder.Value, + body: []const Air.Inst.Index, + err_union_ty: Type, + operand_is_ptr: bool, + operand_ptr_align: InternPool.Alignment, + can_elide_load: bool, + is_unused: bool, + err_cold: bool, +) !Builder.Value { + const o = fg.object; + const pt = fg.pt; + const zcu = pt.zcu; + const payload_ty = err_union_ty.errorUnionPayload(zcu); + const payload_has_bits = payload_ty.hasRuntimeBits(zcu); + const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); + const error_type = try o.errorIntType(pt); + + const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{ + operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)), + operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)), + } else .{ .none, .none }; + + if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { + const loaded = loaded: { + const access_kind: Builder.MemoryAccessKind = + if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + + if (!payload_has_bits) { + break :loaded if (operand_is_ptr) + try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "") + else + err_union; + } + const err_field_index = try errUnionErrorFieldIndex(payload_ty, pt); + if (operand_is_ptr or isByRef(err_union_ty, zcu)) { + const err_field_ptr = + try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, ""); + break :loaded try fg.wip.load( + if (operand_is_ptr) access_kind else .normal, + error_type, + err_field_ptr, + err_set_align.toLlvm(), + "", + ); + } + break :loaded try fg.wip.extractValue(err_union, &.{err_field_index}, ""); + }; + const zero = try o.builder.intValue(error_type, 0); + const is_err = try fg.wip.icmp(.ne, loaded, zero, ""); + + const return_block = try fg.wip.block(1, "TryRet"); + const continue_block = try fg.wip.block(1, "TryCont"); + _ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely); + + fg.wip.cursor = .{ .block = return_block }; + if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold(); + try fg.genBodyDebugScope(null, body, .poi); + + fg.wip.cursor = .{ .block = continue_block }; + } + if (is_unused) return .none; + if (!payload_has_bits) return if (operand_is_ptr) err_union else .none; + const offset = try errUnionPayloadFieldIndex(payload_ty, pt); + if (operand_is_ptr) { + return fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, ""); + } else if (isByRef(err_union_ty, zcu)) { + const payload_ptr = try fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, ""); + if (isByRef(payload_ty, zcu)) { + if (can_elide_load) + return payload_ptr; + + return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal); + } + const load_ty = err_union_llvm_ty.structFields(&o.builder)[offset]; + return fg.wip.load(.normal, load_ty, payload_ptr, payload_align.toLlvm(), ""); + } + return fg.wip.extractValue(err_union, &.{offset}, ""); +} + +fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + + const switch_br = self.air.unwrapSwitch(inst); + + // For `loop_switch_br`, we need these BBs prepared ahead of time to generate dispatches. + // For `switch_br`, they allow us to sometimes generate better IR by sharing a BB between + // scalar and range cases in the same prong. + // +1 for `else` case. This is not the same as the LLVM `else` prong, as that may first contain + // conditionals to handle ranges. + const case_blocks = try self.gpa.alloc(Builder.Function.Block.Index, switch_br.cases_len + 1); + defer self.gpa.free(case_blocks); + // We set incoming as 0 for now, and increment it as we construct dispatches. + for (case_blocks[0 .. case_blocks.len - 1]) |*b| b.* = try self.wip.block(0, "Case"); + case_blocks[case_blocks.len - 1] = try self.wip.block(0, "Default"); + + // There's a special case here to manually generate a jump table in some cases. + // + // Labeled switch in Zig is intended to follow the "direct threading" pattern. We would ideally use a jump + // table, and each `continue` has its own indirect `jmp`, to allow the branch predictor to more accurately + // use data patterns to predict future dispatches. The problem, however, is that LLVM emits fascinatingly + // bad asm for this. Not only does it not share the jump table -- which we really need it to do to prevent + // destroying the cache -- but it also actually generates slightly different jump tables for each case, + // and *a separate conditional branch beforehand* to handle dispatching back to the case we're currently + // within(!!). + // + // This asm is really, really, not what we want. As such, we will construct the jump table manually where + // appropriate (the values are dense and relatively few), and use it when lowering dispatches. + + const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: { + if (!is_dispatch_loop) break :jmp_table null; + + // Workaround for: + // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/lib/MC/WasmObjectWriter.cpp#L560 + // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll + if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null; + + // On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just + // about acceptable - it won't fill L1d cache on most CPUs. + const max_table_len = 1024; + + const cond_ty = self.typeOf(switch_br.operand); + switch (cond_ty.zigTypeTag(zcu)) { + .bool, .pointer => break :jmp_table null, + .@"enum", .int, .error_set, .@"struct", .@"union" => {}, + else => unreachable, + } + + if (cond_ty.intInfo(zcu).signedness == .signed) break :jmp_table null; + + // Don't worry about the size of the type -- it's irrelevant, because the prong values could be fairly dense. + // If they are, then we will construct a jump table. + const min, const max = self.switchCaseItemRange(switch_br) orelse break :jmp_table null; + const min_int = min.getUnsignedInt(zcu) orelse break :jmp_table null; + const max_int = max.getUnsignedInt(zcu) orelse break :jmp_table null; + const table_len = max_int - min_int + 1; + if (table_len > max_table_len) break :jmp_table null; + + const table_elems = try self.gpa.alloc(Builder.Constant, @intCast(table_len)); + defer self.gpa.free(table_elems); + + // Set them all to the `else` branch, then iterate over the AIR switch + // and replace all values which correspond to other prongs. + @memset(table_elems, try o.builder.blockAddrConst( + self.wip.function, + case_blocks[case_blocks.len - 1], + )); + var item_count: u32 = 0; + var it = switch_br.iterateCases(); + while (it.next()) |case| { + const case_block = case_blocks[case.idx]; + const case_block_addr = try o.builder.blockAddrConst( + self.wip.function, + case_block, + ); + for (case.items) |item| { + const val = Value.fromInterned(item.toInterned().?); + const table_idx = val.toUnsignedInt(zcu) - min_int; + table_elems[@intCast(table_idx)] = case_block_addr; + item_count += 1; + } + for (case.ranges) |range| { + const low = Value.fromInterned(range[0].toInterned().?); + const high = Value.fromInterned(range[1].toInterned().?); + const low_idx = low.toUnsignedInt(zcu) - min_int; + const high_idx = high.toUnsignedInt(zcu) - min_int; + @memset(table_elems[@intCast(low_idx)..@intCast(high_idx + 1)], case_block_addr); + item_count += @intCast(high_idx + 1 - low_idx); + } + } + + const table_llvm_ty = try o.builder.arrayType(table_elems.len, .ptr); + const table_val = try o.builder.arrayConst(table_llvm_ty, table_elems); + + const table_variable = try o.builder.addVariable( + try o.builder.strtabStringFmt("__jmptab_{d}", .{@intFromEnum(inst)}), + table_llvm_ty, + .default, + ); + try table_variable.setInitializer(table_val, &o.builder); + table_variable.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); + table_variable.setUnnamedAddr(.unnamed_addr, &o.builder); + + const table_includes_else = item_count != table_len; + + break :jmp_table .{ + .min = try o.lowerValue(pt, min.toIntern()), + .max = try o.lowerValue(pt, max.toIntern()), + .in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) { + .none, .cold => .none, + .unpredictable => .unpredictable, + .likely => .likely, + .unlikely => .unlikely, + }, + .table = table_variable.toConst(&o.builder), + .table_includes_else = table_includes_else, + }; + }; + + const weights: Builder.Function.Instruction.BrCond.Weights = weights: { + if (jmp_table != null) break :weights .none; // not used + + // First pass. If any weights are `.unpredictable`, unpredictable. + // If all are `.none` or `.cold`, none. + var any_likely = false; + for (0..switch_br.cases_len) |case_idx| { + switch (switch_br.getHint(@intCast(case_idx))) { + .none, .cold => {}, + .likely, .unlikely => any_likely = true, + .unpredictable => break :weights .unpredictable, + } + } + switch (switch_br.getElseHint()) { + .none, .cold => {}, + .likely, .unlikely => any_likely = true, + .unpredictable => break :weights .unpredictable, + } + if (!any_likely) break :weights .none; + + const llvm_cases_len = llvm_cases_len: { + var len: u32 = 0; + var it = switch_br.iterateCases(); + while (it.next()) |case| len += @intCast(case.items.len); + break :llvm_cases_len len; + }; + + var weights = try self.gpa.alloc(Builder.Metadata, 1 + llvm_cases_len + 1); + defer self.gpa.free(weights); + var weight_idx: usize = 0; + + const branch_weights_str = try o.builder.metadataString("branch_weights"); + weights[weight_idx] = branch_weights_str.toMetadata(); + weight_idx += 1; + + const else_weight: u32 = switch (switch_br.getElseHint()) { + .unpredictable => unreachable, + .none, .cold => 1000, + .likely => 2000, + .unlikely => 1, + }; + weights[weight_idx] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight)); + weight_idx += 1; + + var it = switch_br.iterateCases(); + while (it.next()) |case| { + const weight_val: u32 = switch (switch_br.getHint(case.idx)) { + .unpredictable => unreachable, + .none, .cold => 1000, + .likely => 2000, + .unlikely => 1, + }; + const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val)); + @memset(weights[weight_idx..][0..case.items.len], weight_meta); + weight_idx += case.items.len; + } + + assert(weight_idx == weights.len); + break :weights .fromMetadata(try o.builder.metadataTuple(weights)); + }; + + const dispatch_info: SwitchDispatchInfo = .{ + .case_blocks = case_blocks, + .switch_weights = weights, + .jmp_table = jmp_table, + }; + + if (is_dispatch_loop) { + try self.switch_dispatch_info.putNoClobber(self.gpa, inst, dispatch_info); + } + defer if (is_dispatch_loop) { + assert(self.switch_dispatch_info.remove(inst)); + }; + + // Generate the initial dispatch. + // If this is a simple `switch_br`, this is the only dispatch. + try self.lowerSwitchDispatch(inst, switch_br.operand, dispatch_info); + + // Iterate the cases and generate their bodies. + var it = switch_br.iterateCases(); + while (it.next()) |case| { + const case_block = case_blocks[case.idx]; + self.wip.cursor = .{ .block = case_block }; + if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold(); + try self.genBodyDebugScope(null, case.body, .none); + } + self.wip.cursor = .{ .block = case_blocks[case_blocks.len - 1] }; + const else_body = it.elseBody(); + if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold(); + if (else_body.len > 0) { + try self.genBodyDebugScope(null, it.elseBody(), .none); + } else { + _ = try self.wip.@"unreachable"(); + } +} + +fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) ?[2]Value { + const zcu = self.pt.zcu; + var it = switch_br.iterateCases(); + var min: ?Value = null; + var max: ?Value = null; + while (it.next()) |case| { + for (case.items) |item| { + const val = Value.fromInterned(item.toInterned().?); + const low = if (min) |m| val.compareHetero(.lt, m, zcu) else true; + const high = if (max) |m| val.compareHetero(.gt, m, zcu) else true; + if (low) min = val; + if (high) max = val; + } + for (case.ranges) |range| { + const vals: [2]Value = .{ + Value.fromInterned(range[0].toInterned().?), + Value.fromInterned(range[1].toInterned().?), + }; + const low = if (min) |m| vals[0].compareHetero(.lt, m, zcu) else true; + const high = if (max) |m| vals[1].compareHetero(.gt, m, zcu) else true; + if (low) min = vals[0]; + if (high) max = vals[1]; + } + } + if (min == null) { + assert(max == null); + return null; + } + return .{ min.?, max.? }; +} + +fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void { + const block = self.air.unwrapBlock(inst); + const body = block.body; + const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time + _ = try self.wip.br(loop_block); + + try self.loops.putNoClobber(self.gpa, inst, loop_block); + defer assert(self.loops.remove(inst)); + + self.wip.cursor = .{ .block = loop_block }; + try self.genBodyDebugScope(null, body, .none); +} + +fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand_ty = self.typeOf(ty_op.operand); + const array_ty = operand_ty.childType(zcu); + const llvm_usize = try o.lowerType(pt, Type.usize); + const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu)); + const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst)); + const operand = try self.resolveInst(ty_op.operand); + if (!array_ty.hasRuntimeBits(zcu)) + return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, ""); + const ptr = try self.wip.gep(.inbounds, try o.lowerType(pt, array_ty), operand, &.{ + try o.builder.intValue(llvm_usize, 0), try o.builder.intValue(llvm_usize, 0), + }, ""); + return self.wip.buildAggregate(slice_llvm_ty, &.{ ptr, len }, ""); +} + +fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try self.resolveInst(ty_op.operand); + const operand_ty = self.typeOf(ty_op.operand); + const operand_scalar_ty = operand_ty.scalarType(zcu); + const is_signed_int = operand_scalar_ty.isSignedInt(zcu); + + const dest_ty = self.typeOfIndex(inst); + const dest_scalar_ty = dest_ty.scalarType(zcu); + const dest_llvm_ty = try o.lowerType(pt, dest_ty); + const target = zcu.getTarget(); + + if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv( + if (is_signed_int) .signed else .unsigned, + operand, + dest_llvm_ty, + "", + ); + + const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu))) orelse { + return self.todo("float_from_int from '{f}' without intrinsics", .{operand_scalar_ty.fmt(pt)}); + }; + const rt_int_ty = try o.builder.intType(rt_int_bits); + var extended = try self.wip.conv( + if (is_signed_int) .signed else .unsigned, + operand, + rt_int_ty, + "", + ); + const dest_bits = dest_scalar_ty.floatBits(target); + const compiler_rt_operand_abbrev = compilerRtIntAbbrev(rt_int_bits); + const compiler_rt_dest_abbrev = compilerRtFloatAbbrev(dest_bits); + const sign_prefix = if (is_signed_int) "" else "un"; + const fn_name = try o.builder.strtabStringFmt("__float{s}{s}i{s}f", .{ + sign_prefix, + compiler_rt_operand_abbrev, + compiler_rt_dest_abbrev, + }); + + var param_type = rt_int_ty; + if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) { + // On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard + // i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have. + param_type = try o.builder.vectorType(.normal, 2, .i64); + extended = try self.wip.cast(.bitcast, extended, param_type, ""); + } + + const libc_fn = try self.getLibcFunction(fn_name, &.{param_type}, dest_llvm_ty); + return self.wip.call( + .normal, + .ccc, + .none, + libc_fn.typeOf(&o.builder), + libc_fn.toValue(&o.builder), + &.{extended}, + "", + ); +} + +fn airIntFromFloat( + self: *FuncGen, + inst: Air.Inst.Index, + fast: Builder.FastMathKind, +) !Builder.Value { + _ = fast; + + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const target = zcu.getTarget(); + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + + const operand = try self.resolveInst(ty_op.operand); + const operand_ty = self.typeOf(ty_op.operand); + const operand_scalar_ty = operand_ty.scalarType(zcu); + + const dest_ty = self.typeOfIndex(inst); + const dest_scalar_ty = dest_ty.scalarType(zcu); + const dest_llvm_ty = try o.lowerType(pt, dest_ty); + + if (intrinsicsAllowed(operand_scalar_ty, target)) { + // TODO set fast math flag + return self.wip.conv( + if (dest_scalar_ty.isSignedInt(zcu)) .signed else .unsigned, + operand, + dest_llvm_ty, + "", + ); + } + + const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu))) orelse { + return self.todo("int_from_float to '{f}' without intrinsics", .{dest_scalar_ty.fmt(pt)}); + }; + const ret_ty = try o.builder.intType(rt_int_bits); + const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: { + // On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard + // i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have. + break :b try o.builder.vectorType(.normal, 2, .i64); + } else ret_ty; + + const operand_bits = operand_scalar_ty.floatBits(target); + const compiler_rt_operand_abbrev = compilerRtFloatAbbrev(operand_bits); + + const compiler_rt_dest_abbrev = compilerRtIntAbbrev(rt_int_bits); + const sign_prefix = if (dest_scalar_ty.isSignedInt(zcu)) "" else "uns"; + + const fn_name = try o.builder.strtabStringFmt("__fix{s}{s}f{s}i", .{ + sign_prefix, + compiler_rt_operand_abbrev, + compiler_rt_dest_abbrev, + }); + + const operand_llvm_ty = try o.lowerType(pt, operand_ty); + const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty); + var result = try self.wip.call( + .normal, + .ccc, + .none, + libc_fn.typeOf(&o.builder), + libc_fn.toValue(&o.builder), + &.{operand}, + "", + ); + + if (libc_ret_ty != ret_ty) result = try self.wip.cast(.bitcast, result, ret_ty, ""); + if (ret_ty != dest_llvm_ty) result = try self.wip.cast(.trunc, result, dest_llvm_ty, ""); + return result; +} + +fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value { + const zcu = fg.pt.zcu; + return if (ty.isSlice(zcu)) fg.wip.extractValue(ptr, &.{0}, "") else ptr; +} + +fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value { + const o = fg.object; + const pt = fg.pt; + const zcu = pt.zcu; + const llvm_usize = try o.lowerType(pt, Type.usize); + switch (ty.ptrSize(zcu)) { + .slice => { + const len = try fg.wip.extractValue(ptr, &.{1}, ""); + const elem_ty = ty.childType(zcu); + const abi_size = elem_ty.abiSize(zcu); + if (abi_size == 1) return len; + const abi_size_llvm_val = try o.builder.intValue(llvm_usize, abi_size); + return fg.wip.bin(.@"mul nuw", len, abi_size_llvm_val, ""); + }, + .one => { + const array_ty = ty.childType(zcu); + const elem_ty = array_ty.childType(zcu); + const abi_size = elem_ty.abiSize(zcu); + return o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu) * abi_size); + }, + .many, .c => unreachable, + } +} + +fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Value { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + return self.wip.extractValue(operand, &.{index}, ""); +} + +fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: c_uint) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const slice_ptr = try self.resolveInst(ty_op.operand); + const slice_ptr_ty = self.typeOf(ty_op.operand); + const slice_llvm_ty = try o.lowerType(pt, slice_ptr_ty.childType(zcu)); + + return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, ""); +} + +fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const slice_ty = self.typeOf(bin_op.lhs); + const slice = try self.resolveInst(bin_op.lhs); + const index = try self.resolveInst(bin_op.rhs); + const slice_info = slice_ty.ptrInfo(zcu); + assert(slice_info.flags.size == .slice); + const elem_ty: Type = .fromInterned(slice_info.child); + const llvm_elem_ty = try o.lowerType(pt, elem_ty); + const base_ptr = try self.wip.extractValue(slice, &.{0}, ""); + const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, ""); + const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)); + const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal; + self.maybeMarkAllowZeroAccess(slice_info); + if (isByRef(elem_ty, zcu)) { + return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind); + } else { + return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm()); + } +} + +fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const slice_ty = self.typeOf(bin_op.lhs); + + const slice = try self.resolveInst(bin_op.lhs); + const index = try self.resolveInst(bin_op.rhs); + const llvm_elem_ty = try o.lowerType(pt, slice_ty.childType(zcu)); + const base_ptr = try self.wip.extractValue(slice, &.{0}, ""); + return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, ""); +} + +fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const array_ty = self.typeOf(bin_op.lhs); + const array_llvm_val = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const array_llvm_ty = try o.lowerType(pt, array_ty); + const elem_ty = array_ty.childType(zcu); + if (isByRef(array_ty, zcu)) { + const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{ + try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), + rhs, + }, ""); + if (isByRef(elem_ty, zcu)) { + const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm(); + return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal); + } else { + return self.loadTruncate(.normal, elem_ty, elem_ptr, .default); + } + } + + // This branch can be reached for vectors, which are always by-value. + return self.wip.extractElement(array_llvm_val, rhs, ""); +} + +fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const ptr_ty = self.typeOf(bin_op.lhs); + const elem_ty = ptr_ty.indexableElem(zcu); + const llvm_elem_ty = try o.lowerType(pt, elem_ty); + const base_ptr = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, ""); + if (isByRef(elem_ty, zcu)) { + self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); + const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm(); + return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal); + } + + self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); + + return self.load(ptr, ptr_ty); +} + +fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const ptr_ty = self.typeOf(bin_op.lhs); + const elem_ty = ptr_ty.indexableElem(zcu); + assert(elem_ty.hasRuntimeBits(zcu)); + + const base_ptr = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const elem_ptr = ty_pl.ty.toType(); + if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr; + + const llvm_elem_ty = try o.lowerType(pt, elem_ty); + return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, ""); +} + +fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data; + const struct_ptr = try self.resolveInst(struct_field.struct_operand); + const struct_ptr_ty = self.typeOf(struct_field.struct_operand); + return self.fieldPtr(struct_ptr, struct_ptr_ty, struct_field.field_index); +} + +fn airStructFieldPtrIndex( + self: *FuncGen, + inst: Air.Inst.Index, + field_index: u32, +) !Builder.Value { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const struct_ptr = try self.resolveInst(ty_op.operand); + const struct_ptr_ty = self.typeOf(ty_op.operand); + return self.fieldPtr(struct_ptr, struct_ptr_ty, field_index); +} + +fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data; + const struct_ty = self.typeOf(struct_field.struct_operand); + const struct_llvm_val = try self.resolveInst(struct_field.struct_operand); + const field_index = struct_field.field_index; + const field_ty = struct_ty.fieldType(field_index, zcu); + if (!field_ty.hasRuntimeBits(zcu)) return .none; + + if (!isByRef(struct_ty, zcu)) { + assert(!isByRef(field_ty, zcu)); + switch (struct_ty.zigTypeTag(zcu)) { + .@"struct" => switch (struct_ty.containerLayout(zcu)) { + .@"packed" => { + const struct_type = zcu.typeToStruct(struct_ty).?; + const bit_offset = zcu.structPackedFieldBitOffset(struct_type, field_index); + const containing_int = struct_llvm_val; + const shift_amt = + try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset); + const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, ""); + const elem_llvm_ty = try o.lowerType(pt, field_ty); + if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) { + const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu))); + const truncated_int = + try self.wip.cast(.trunc, shifted_value, same_size_int, ""); + return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); + } + return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, ""); + }, + else => { + const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; + return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, ""); + }, + }, + .@"union" => { + assert(struct_ty.containerLayout(zcu) == .@"packed"); + const containing_int = struct_llvm_val; + const elem_llvm_ty = try o.lowerType(pt, field_ty); + if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) { + const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu))); + const truncated_int = + try self.wip.cast(.trunc, containing_int, same_size_int, ""); + return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); + } + return self.wip.cast(.trunc, containing_int, elem_llvm_ty, ""); + }, + else => unreachable, + } + } + + switch (struct_ty.zigTypeTag(zcu)) { + .@"struct" => { + const layout = struct_ty.containerLayout(zcu); + assert(layout != .@"packed"); + const struct_llvm_ty = try o.lowerType(pt, struct_ty); + const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; + const field_ptr = + try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, ""); + const explicit_alignment = struct_ty.explicitFieldAlignment(field_index, zcu); + const field_ptr_ty = try pt.ptrType(.{ + .child = field_ty.toIntern(), + .flags = .{ .alignment = explicit_alignment }, + }); + if (isByRef(field_ty, zcu)) { + const alignment = switch (explicit_alignment) { + .none => field_ty.abiAlignment(zcu), + else => |a| a, + }; + return self.loadByRef(field_ptr, field_ty, alignment.toLlvm(), .normal); + } else { + return self.load(field_ptr, field_ptr_ty); + } + }, + .@"union" => { + const union_llvm_ty = try o.lowerType(pt, struct_ty); + const layout = struct_ty.unionGetLayout(zcu); + const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)); + const field_ptr = + try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, ""); + const payload_alignment = layout.payload_align.toLlvm(); + if (isByRef(field_ty, zcu)) { + return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal); + } else { + return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment); + } + }, + else => unreachable, + } +} + +fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; + + const field_ptr = try self.resolveInst(extra.field_ptr); + + const parent_ty = ty_pl.ty.toType().childType(zcu); + const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu); + if (field_offset == 0) return field_ptr; + + const res_ty = try o.lowerType(pt, ty_pl.ty.toType()); + const llvm_usize = try o.lowerType(pt, Type.usize); + + const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, ""); + const base_ptr_int = try self.wip.bin( + .@"sub nuw", + field_ptr_int, + try o.builder.intValue(llvm_usize, field_offset), + "", + ); + return self.wip.cast(.inttoptr, base_ptr_int, res_ty, ""); +} + +fn airNot(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + + return self.wip.not(operand, ""); +} + +fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) !void { + _ = inst; + _ = try self.wip.@"unreachable"(); +} + +fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt; + self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1); + self.prev_dbg_column = @intCast(dbg_stmt.column + 1); + + self.wip.debug_location = .{ .location = .{ + .line = self.prev_dbg_line, + .column = self.prev_dbg_column, + .scope = self.scope.toOptional(), + .inlined_at = self.inlined_at, + } }; + + return .none; +} + +fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + _ = self; + _ = inst; + return .none; +} + +fn airDbgInlineBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const block = self.air.unwrapDbgBlock(inst); + self.arg_inline_index = 0; + return self.lowerBlock(inst, block.func, block.body); +} + +fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const operand = try self.resolveInst(pl_op.operand); + const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); + const ptr_ty = self.typeOf(pl_op.operand); + + const debug_local_var = try o.builder.debugLocalVar( + try o.builder.metadataString(name.toSlice(self.air)), + self.file, + self.scope, + self.prev_dbg_line, + try o.getDebugType(pt, ptr_ty.childType(zcu)), + ); + + _ = try self.wip.callIntrinsic( + .normal, + .none, + .@"dbg.declare", + &.{}, + &.{ + (try self.wip.debugValue(operand)).toValue(), + debug_local_var.toValue(), + (try o.builder.debugExpression(&.{})).toValue(), + }, + "", + ); + + return .none; +} + +fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) !Builder.Value { + const o = self.object; + const pt = self.pt; + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const operand = try self.resolveInst(pl_op.operand); + const operand_ty = self.typeOf(pl_op.operand); + const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); + const name_slice = name.toSlice(self.air); + const metadata_name = if (name_slice.len > 0) try o.builder.metadataString(name_slice) else null; + const debug_local_var = if (is_arg) try o.builder.debugParameter( + metadata_name, + self.file, + self.scope, + self.prev_dbg_line, + try o.getDebugType(pt, operand_ty), + arg_no: { + self.arg_inline_index += 1; + break :arg_no self.arg_inline_index; + }, + ) else try o.builder.debugLocalVar( + metadata_name, + self.file, + self.scope, + self.prev_dbg_line, + try o.getDebugType(pt, operand_ty), + ); + + const zcu = pt.zcu; + const owner_mod = self.ownerModule(); + if (isByRef(operand_ty, zcu)) { + _ = try self.wip.callIntrinsic( + .normal, + .none, + .@"dbg.declare", + &.{}, + &.{ + (try self.wip.debugValue(operand)).toValue(), + debug_local_var.toValue(), + (try o.builder.debugExpression(&.{})).toValue(), + }, + "", + ); + } else if (owner_mod.optimize_mode == .Debug and !self.is_naked) { + // We avoid taking this path for naked functions because there's no guarantee that such + // functions even have a valid stack pointer, making the `alloca` + `store` unsafe. + + const alignment = operand_ty.abiAlignment(zcu).toLlvm(); + const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment); + _ = try self.wip.store(.normal, operand, alloca, alignment); + _ = try self.wip.callIntrinsic( + .normal, + .none, + .@"dbg.declare", + &.{}, + &.{ + (try self.wip.debugValue(alloca)).toValue(), + debug_local_var.toValue(), + (try o.builder.debugExpression(&.{})).toValue(), + }, + "", + ); + } else { + _ = try self.wip.callIntrinsic( + .normal, + .none, + .@"dbg.value", + &.{}, + &.{ + (try self.wip.debugValue(operand)).toValue(), + debug_local_var.toValue(), + (try o.builder.debugExpression(&.{})).toValue(), + }, + "", + ); + } + return .none; +} + +fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + // Eventually, the Zig compiler needs to be reworked to have inline + // assembly go through the same parsing code regardless of backend, and + // have LLVM-flavored inline assembly be *output* from that assembler. + // We don't have such an assembler implemented yet though. For now, + // this implementation feeds the inline assembly code directly to LLVM. + + const o = self.object; + const unwrapped_asm = self.air.unwrapAsm(inst); + const is_volatile = unwrapped_asm.is_volatile; + const gpa = self.gpa; + + const outputs = unwrapped_asm.outputs; + const inputs = unwrapped_asm.inputs; + + var llvm_constraints: std.ArrayList(u8) = .empty; + defer llvm_constraints.deinit(gpa); + + var arena_allocator = std.heap.ArenaAllocator.init(gpa); + defer arena_allocator.deinit(); + const arena = arena_allocator.allocator(); + + // The exact number of return / parameter values depends on which output values + // are passed by reference as indirect outputs (determined below). + const max_return_count = outputs.len; + const llvm_ret_types = try arena.alloc(Builder.Type, max_return_count); + const llvm_ret_indirect = try arena.alloc(bool, max_return_count); + const llvm_rw_vals = try arena.alloc(Builder.Value, max_return_count); + + const max_param_count = max_return_count + inputs.len + outputs.len; + const llvm_param_types = try arena.alloc(Builder.Type, max_param_count); + const llvm_param_values = try arena.alloc(Builder.Value, max_param_count); + // This stores whether we need to add an elementtype attribute and + // if so, the element type itself. + const llvm_param_attrs = try arena.alloc(Builder.Type, max_param_count); + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const target = zcu.getTarget(); + + var llvm_ret_i: usize = 0; + var llvm_param_i: usize = 0; + var total_i: usize = 0; + + var name_map: std.StringArrayHashMapUnmanaged(u16) = .empty; + try name_map.ensureUnusedCapacity(arena, max_param_count); + + var it = unwrapped_asm.iterateOutputs(); + while (it.next()) |output| { + const constraint = output.constraint; + const name = output.name; + + try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 3); + if (total_i != 0) { + llvm_constraints.appendAssumeCapacity(','); + } + llvm_constraints.appendAssumeCapacity('='); + + if (output.operand != .none) { + const output_inst = try self.resolveInst(output.operand); + const output_ty = self.typeOf(output.operand); + assert(output_ty.zigTypeTag(zcu) == .pointer); + const elem_llvm_ty = try o.lowerType(pt, output_ty.childType(zcu)); + + switch (constraint[0]) { + '=' => {}, + '+' => llvm_rw_vals[output.index] = output_inst, + else => return self.todo("unsupported output constraint on output type '{c}'", .{ + constraint[0], + }), + } + + self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu)); + + // Pass any non-return outputs indirectly, if the constraint accepts a memory location + llvm_ret_indirect[output.index] = constraintAllowsMemory(constraint); + if (llvm_ret_indirect[output.index]) { + // Pass the result by reference as an indirect output (e.g. "=*m") + llvm_constraints.appendAssumeCapacity('*'); + + llvm_param_values[llvm_param_i] = output_inst; + llvm_param_types[llvm_param_i] = output_inst.typeOfWip(&self.wip); + llvm_param_attrs[llvm_param_i] = elem_llvm_ty; + llvm_param_i += 1; + } else { + // Pass the result directly (e.g. "=r") + llvm_ret_types[llvm_ret_i] = elem_llvm_ty; + llvm_ret_i += 1; + } + } else { + switch (constraint[0]) { + '=' => {}, + else => return self.todo("unsupported output constraint on result type '{s}'", .{ + constraint, + }), + } + + llvm_ret_indirect[output.index] = false; + + const ret_ty = self.typeOfIndex(inst); + llvm_ret_types[llvm_ret_i] = try o.lowerType(pt, ret_ty); + llvm_ret_i += 1; + } + + // LLVM uses commas internally to separate different constraints, + // alternative constraints are achieved with pipes. + // We still allow the user to use commas in a way that is similar + // to GCC's inline assembly. + // http://llvm.org/docs/LangRef.html#constraint-codes + for (constraint[1..]) |byte| { + switch (byte) { + ',' => llvm_constraints.appendAssumeCapacity('|'), + '*' => {}, // Indirect outputs are handled above + else => llvm_constraints.appendAssumeCapacity(byte), + } + } + + if (!std.mem.eql(u8, name, "_")) { + const gop = name_map.getOrPutAssumeCapacity(name); + if (gop.found_existing) return self.todo("duplicate asm output name '{s}'", .{name}); + gop.value_ptr.* = @intCast(total_i); + } + total_i += 1; + } + + it = unwrapped_asm.iterateInputs(); + while (it.next()) |input| { + const constraint = input.constraint; + const name = input.name; + + const arg_llvm_value = try self.resolveInst(input.operand); + const arg_ty = self.typeOf(input.operand); + const is_by_ref = isByRef(arg_ty, zcu); + if (is_by_ref) { + if (constraintAllowsMemory(constraint)) { + llvm_param_values[llvm_param_i] = arg_llvm_value; + llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip); + } else { + const alignment = arg_ty.abiAlignment(zcu).toLlvm(); + const arg_llvm_ty = try o.lowerType(pt, arg_ty); + const load_inst = + try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, ""); + llvm_param_values[llvm_param_i] = load_inst; + llvm_param_types[llvm_param_i] = arg_llvm_ty; + } + } else { + if (constraintAllowsRegister(constraint)) { + llvm_param_values[llvm_param_i] = arg_llvm_value; + llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip); + } else { + const alignment = arg_ty.abiAlignment(zcu).toLlvm(); + const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment); + _ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment); + llvm_param_values[llvm_param_i] = arg_ptr; + llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip); + } + } + + try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 1); + if (total_i != 0) { + llvm_constraints.appendAssumeCapacity(','); + } + for (constraint) |byte| { + llvm_constraints.appendAssumeCapacity(switch (byte) { + ',' => '|', + else => byte, + }); + } + + if (!std.mem.eql(u8, name, "_")) { + const gop = name_map.getOrPutAssumeCapacity(name); + if (gop.found_existing) return self.todo("duplicate asm input name '{s}'", .{name}); + gop.value_ptr.* = @intCast(total_i); + } + + // In the case of indirect inputs, LLVM requires the callsite to have + // an elementtype() attribute. + llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: { + if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu)); + + break :blk try o.lowerType(pt, if (is_by_ref) arg_ty else arg_ty.childType(zcu)); + } else .none; + + llvm_param_i += 1; + total_i += 1; + } + + it = unwrapped_asm.iterateOutputs(); + while (it.next()) |output| { + const constraint = output.constraint; + + if (constraint[0] != '+') continue; + + const rw_ty = self.typeOf(output.operand); + const llvm_elem_ty = try o.lowerType(pt, rw_ty.childType(zcu)); + if (llvm_ret_indirect[output.index]) { + llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index]; + llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip); + } else { + const alignment = rw_ty.abiAlignment(zcu).toLlvm(); + const loaded = try self.wip.load( + if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, + llvm_elem_ty, + llvm_rw_vals[output.index], + alignment, + "", + ); + llvm_param_values[llvm_param_i] = loaded; + llvm_param_types[llvm_param_i] = llvm_elem_ty; + } + + try llvm_constraints.print(gpa, ",{d}", .{output.index}); + + // In the case of indirect inputs, LLVM requires the callsite to have + // an elementtype() attribute. + llvm_param_attrs[llvm_param_i] = if (llvm_ret_indirect[output.index]) llvm_elem_ty else .none; + + llvm_param_i += 1; + total_i += 1; + } + + if (total_i != 0) try llvm_constraints.append(gpa, ','); + const clobbers_val: Value = .fromInterned(unwrapped_asm.clobbers); + const clobbers_ty = clobbers_val.typeOf(zcu); + var clobbers_bigint_buf: Value.BigIntSpace = undefined; + const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu); + for (0..clobbers_ty.structFieldCount(zcu)) |field_index| { + assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type); + const limb_bits = @bitSizeOf(std.math.big.Limb); + if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false + switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) { + 0 => continue, // field is false + 1 => {}, // field is true + } + const name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?; + total_i += try appendConstraints(gpa, &llvm_constraints, name, target); + } + + // We have finished scanning through all inputs/outputs, so the number of + // parameters and return values is known. + const param_count = llvm_param_i; + const return_count = llvm_ret_i; + + // For some targets, Clang unconditionally adds some clobbers to all inline assembly. + // While this is probably not strictly necessary, if we don't follow Clang's lead + // here then we may risk tripping LLVM bugs since anything not used by Clang tends + // to be buggy and regress often. + switch (target.cpu.arch) { + .x86_64, .x86 => { + try llvm_constraints.appendSlice(gpa, "~{dirflag},~{fpsr},~{flags},"); + total_i += 3; + }, + .mips, .mipsel, .mips64, .mips64el => { + try llvm_constraints.appendSlice(gpa, "~{$1},"); + total_i += 1; + }, + else => {}, + } + + if (std.mem.endsWith(u8, llvm_constraints.items, ",")) llvm_constraints.items.len -= 1; + + const asm_source = unwrapped_asm.source; + + // hackety hacks until stage2 has proper inline asm in the frontend. + var rendered_template = std.array_list.Managed(u8).init(gpa); + defer rendered_template.deinit(); + + const State = enum { start, percent, input, modifier }; + + var state: State = .start; + + var name_start: usize = undefined; + var modifier_start: usize = undefined; + for (asm_source, 0..) |byte, i| { + switch (state) { + .start => switch (byte) { + '%' => state = .percent, + '$' => try rendered_template.appendSlice("$$"), + else => try rendered_template.append(byte), + }, + .percent => switch (byte) { + '%' => { + try rendered_template.append('%'); + state = .start; + }, + '[' => { + try rendered_template.append('$'); + try rendered_template.append('{'); + name_start = i + 1; + state = .input; + }, + '=' => { + try rendered_template.appendSlice("${:uid}"); + state = .start; + }, + else => { + try rendered_template.append('%'); + try rendered_template.append(byte); + state = .start; + }, + }, + .input => switch (byte) { + ']', ':' => { + const name = asm_source[name_start..i]; + + const index = name_map.get(name) orelse { + // we should validate the assembly in Sema; by now it is too late + return self.todo("unknown input or output name: '{s}'", .{name}); + }; + try rendered_template.print("{d}", .{index}); + if (byte == ':') { + try rendered_template.append(':'); + modifier_start = i + 1; + state = .modifier; + } else { + try rendered_template.append('}'); + state = .start; + } + }, + else => {}, + }, + .modifier => switch (byte) { + ']' => { + try rendered_template.appendSlice(asm_source[modifier_start..i]); + try rendered_template.append('}'); + state = .start; + }, + else => {}, + }, + } + } + + var attributes: Builder.FunctionAttributes.Wip = .{}; + defer attributes.deinit(&o.builder); + for (llvm_param_attrs[0..param_count], 0..) |llvm_elem_ty, i| if (llvm_elem_ty != .none) + try attributes.addParamAttr(i, .{ .elementtype = llvm_elem_ty }, &o.builder); + + const ret_llvm_ty = switch (return_count) { + 0 => .void, + 1 => llvm_ret_types[0], + else => try o.builder.structType(.normal, llvm_ret_types), + }; + const llvm_fn_ty = try o.builder.fnType(ret_llvm_ty, llvm_param_types[0..param_count], .normal); + const call = try self.wip.callAsm( + try attributes.finish(&o.builder), + llvm_fn_ty, + .{ .sideeffect = is_volatile }, + try o.builder.string(rendered_template.items), + try o.builder.string(llvm_constraints.items), + llvm_param_values[0..param_count], + "", + ); + + var ret_val = call; + llvm_ret_i = 0; + for (outputs, 0..) |output, i| { + if (llvm_ret_indirect[i]) continue; + + const output_value = if (return_count > 1) + try self.wip.extractValue(call, &[_]u32{@intCast(llvm_ret_i)}, "") + else + call; + + if (output != .none) { + const output_ptr = try self.resolveInst(output); + const output_ptr_ty = self.typeOf(output); + const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm(); + _ = try self.wip.store( + if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, + output_value, + output_ptr, + alignment, + ); + } else { + ret_val = output_value; + } + llvm_ret_i += 1; + } + + return ret_val; +} + +fn airIsNonNull( + self: *FuncGen, + inst: Air.Inst.Index, + operand_is_ptr: bool, + cond: Builder.IntegerCondition, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + const operand_ty = self.typeOf(un_op); + const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; + const optional_llvm_ty = try o.lowerType(pt, optional_ty); + const payload_ty = optional_ty.optionalChild(zcu); + + const access_kind: Builder.MemoryAccessKind = + if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + + if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); + + if (optional_ty.optionalReprIsPayload(zcu)) { + const loaded = if (operand_is_ptr) + try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") + else + operand; + if (payload_ty.isSlice(zcu)) { + const slice_ptr = try self.wip.extractValue(loaded, &.{0}, ""); + const ptr_ty = try o.builder.ptrType(toLlvmAddressSpace( + payload_ty.ptrAddressSpace(zcu), + zcu.getTarget(), + )); + return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), ""); + } + return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), ""); + } + + comptime assert(optional_layout_version == 3); + + if (!payload_ty.hasRuntimeBits(zcu)) { + const loaded = if (operand_is_ptr) + try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") + else + operand; + return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), ""); + } + + const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu); + return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind); +} + +fn airIsErr( + self: *FuncGen, + inst: Air.Inst.Index, + cond: Builder.IntegerCondition, + operand_is_ptr: bool, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + const operand_ty = self.typeOf(un_op); + const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; + const payload_ty = err_union_ty.errorUnionPayload(zcu); + const error_type = try o.errorIntType(pt); + const zero = try o.builder.intValue(error_type, 0); + + const access_kind: Builder.MemoryAccessKind = + if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + + if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { + const val: Builder.Constant = switch (cond) { + .eq => .true, // 0 == 0 + .ne => .false, // 0 != 0 + else => unreachable, + }; + return val.toValue(); + } + + if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); + + if (!payload_ty.hasRuntimeBits(zcu)) { + const loaded = if (operand_is_ptr) + try self.wip.load(access_kind, try o.lowerType(pt, err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") + else + operand; + return self.wip.icmp(cond, loaded, zero, ""); + } + + const err_field_index = try errUnionErrorFieldIndex(payload_ty, pt); + + const loaded = if (operand_is_ptr or isByRef(err_union_ty, zcu)) loaded: { + const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); + const err_alignment = if (operand_is_ptr) + operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu)) + else + .none; + const err_field_ptr = + try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, ""); + break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, err_alignment.toLlvm(), ""); + } else try self.wip.extractValue(operand, &.{err_field_index}, ""); + return self.wip.icmp(cond, loaded, zero, ""); +} + +fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const optional_ty = self.typeOf(ty_op.operand).childType(zcu); + const payload_ty = optional_ty.optionalChild(zcu); + if (!payload_ty.hasRuntimeBits(zcu)) { + // We have a pointer to a zero-bit value and we need to return + // a pointer to a zero-bit value. + return operand; + } + if (optional_ty.optionalReprIsPayload(zcu)) { + // The payload and the optional are the same value. + return operand; + } + return self.wip.gepStruct(try o.lowerType(pt, optional_ty), operand, 0, ""); +} + +fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + comptime assert(optional_layout_version == 3); + + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const optional_ptr_ty = self.typeOf(ty_op.operand); + const optional_ty = optional_ptr_ty.childType(zcu); + const payload_ty = optional_ty.optionalChild(zcu); + const non_null_bit = try o.builder.intValue(.i8, 1); + + const access_kind: Builder.MemoryAccessKind = + if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + + if (!payload_ty.hasRuntimeBits(zcu)) { + self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu)); + + // We have a pointer to a i8. We need to set it to 1 and then return the same pointer. + // Default alignment store because align of the non null bit is 1 anyway. + _ = try self.wip.store(access_kind, non_null_bit, operand, .default); + return operand; + } + if (optional_ty.optionalReprIsPayload(zcu)) { + // The payload and the optional are the same value. + // Setting to non-null will be done when the payload is set. + return operand; + } + + // First set the non-null bit. + const optional_llvm_ty = try o.lowerType(pt, optional_ty); + const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, ""); + + self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu)); + + // Default alignment store because align of the non null bit is 1 anyway. + _ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default); + + // Then return the payload pointer (only if it's used). + if (self.liveness.isUnused(inst)) return .none; + + return self.wip.gepStruct(optional_llvm_ty, operand, 0, ""); +} + +fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const optional_ty = self.typeOf(ty_op.operand); + const payload_ty = self.typeOfIndex(inst); + if (!payload_ty.hasRuntimeBits(zcu)) return .none; + + if (optional_ty.optionalReprIsPayload(zcu)) { + // Payload value is the same as the optional value. + return operand; + } + + const opt_llvm_ty = try o.lowerType(pt, optional_ty); + return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false); +} + +fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const operand_ty = self.typeOf(ty_op.operand); + const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; + const result_ty = self.typeOfIndex(inst); + const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty; + + if (!payload_ty.hasRuntimeBits(zcu)) { + return if (operand_is_ptr) operand else .none; + } + const offset = try errUnionPayloadFieldIndex(payload_ty, pt); + const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); + if (operand_is_ptr) { + return self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); + } else if (isByRef(err_union_ty, zcu)) { + const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm(); + const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); + if (isByRef(payload_ty, zcu)) { + return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); + } + const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset]; + return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, ""); + } + return self.wip.extractValue(operand, &.{offset}, ""); +} + +fn airErrUnionErr( + self: *FuncGen, + inst: Air.Inst.Index, + operand_is_ptr: bool, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const operand_ty = self.typeOf(ty_op.operand); + const error_type = try o.errorIntType(pt); + const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; + if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { + if (operand_is_ptr) { + return operand; + } else { + return o.builder.intValue(error_type, 0); + } + } + + const access_kind: Builder.MemoryAccessKind = + if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + + const payload_ty = err_union_ty.errorUnionPayload(zcu); + if (!payload_ty.hasRuntimeBits(zcu)) { + if (!operand_is_ptr) return operand; + + self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); + + return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), ""); + } + + const offset = try errUnionErrorFieldIndex(payload_ty, pt); + + if (operand_is_ptr or isByRef(err_union_ty, zcu)) { + if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); + + const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); + const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); + return self.wip.load(access_kind, error_type, err_field_ptr, .default, ""); + } + + return self.wip.extractValue(operand, &.{offset}, ""); +} + +fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const err_union_ptr_ty = self.typeOf(ty_op.operand); + const err_union_ty = err_union_ptr_ty.childType(zcu); + const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu); + + const payload_ty = err_union_ty.errorUnionPayload(zcu); + const non_error_val = try o.builder.intValue(try o.errorIntType(pt), 0); + + const access_kind: Builder.MemoryAccessKind = + if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + + if (!payload_ty.hasRuntimeBits(zcu)) { + self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); + _ = try self.wip.store(access_kind, non_error_val, operand, err_union_ptr_align.toLlvm()); + return operand; + } + const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); + { + self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); + + const err_int_ty = try pt.errorIntType(); + const error_alignment = err_int_ty.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm(); + const error_offset = try errUnionErrorFieldIndex(payload_ty, pt); + // First set the non-error value. + const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, ""); + _ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment); + } + // Then return the payload pointer (only if it is used). + if (self.liveness.isUnused(inst)) return .none; + + const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt); + return self.wip.gepStruct(err_union_llvm_ty, operand, payload_offset, ""); +} + +fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value { + assert(self.err_ret_trace != .none); + return self.err_ret_trace; +} + +fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + self.err_ret_trace = try self.resolveInst(un_op); + return .none; +} + +fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const struct_ty = ty_pl.ty.toType(); + const field_index = ty_pl.payload; + + const struct_llvm_ty = try o.lowerType(pt, struct_ty); + const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; + assert(self.err_ret_trace != .none); + const field_ptr = try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, ""); + const field_alignment = struct_ty.explicitFieldAlignment(field_index, zcu); + const field_ty = struct_ty.fieldType(field_index, zcu); + const field_ptr_ty = try pt.ptrType(.{ + .child = field_ty.toIntern(), + .flags = .{ .alignment = field_alignment }, + }); + return self.load(field_ptr, field_ptr_ty); +} + +/// As an optimization, we want to avoid unnecessary copies of +/// error union/optional types when returning from a function. +/// Here, we scan forward in the current block, looking to see +/// if the next instruction is a return (ignoring debug instructions). +/// +/// The first instruction of `body_tail` is a wrap instruction. +fn isNextRet( + self: *FuncGen, + body_tail: []const Air.Inst.Index, +) bool { + const air_tags = self.air.instructions.items(.tag); + for (body_tail[1..]) |body_inst| { + switch (air_tags[@intFromEnum(body_inst)]) { + .ret => return true, + .dbg_stmt => continue, + else => return false, + } + } + // The only way to get here is to hit the end of a loop instruction + // (implicit repeat). + return false; +} + +fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const inst = body_tail[0]; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const payload_ty = self.typeOf(ty_op.operand); + const non_null_bit = try o.builder.intValue(.i8, 1); + comptime assert(optional_layout_version == 3); + assert(payload_ty.hasRuntimeBits(zcu)); + const operand = try self.resolveInst(ty_op.operand); + const optional_ty = self.typeOfIndex(inst); + if (optional_ty.optionalReprIsPayload(zcu)) return operand; + const llvm_optional_ty = try o.lowerType(pt, optional_ty); + if (isByRef(optional_ty, zcu)) { + const directReturn = self.isNextRet(body_tail); + const optional_ptr = if (directReturn) + self.ret_ptr + else brk: { + const alignment = optional_ty.abiAlignment(zcu).toLlvm(); + const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment); + break :brk optional_ptr; + }; + + const payload_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 0, ""); + const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); + try self.store(payload_ptr, payload_ptr_ty, operand, .none); + const non_null_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 1, ""); + _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default); + return optional_ptr; + } + return self.wip.buildAggregate(llvm_optional_ty, &.{ operand, non_null_bit }, ""); +} + +fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const inst = body_tail[0]; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const err_un_ty = self.typeOfIndex(inst); + const operand = try self.resolveInst(ty_op.operand); + const payload_ty = self.typeOf(ty_op.operand); + assert(payload_ty.hasRuntimeBits(zcu)); + const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0); + const err_un_llvm_ty = try o.lowerType(pt, err_un_ty); + + const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt); + const error_offset = try errUnionErrorFieldIndex(payload_ty, pt); + if (isByRef(err_un_ty, zcu)) { + const directReturn = self.isNextRet(body_tail); + const result_ptr = if (directReturn) + self.ret_ptr + else brk: { + const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm(); + const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); + break :brk result_ptr; + }; + + const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, ""); + const err_int_ty = try pt.errorIntType(); + const error_alignment = err_int_ty.abiAlignment(pt.zcu).toLlvm(); + _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment); + const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, ""); + const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); + try self.store(payload_ptr, payload_ptr_ty, operand, .none); + return result_ptr; + } + var fields: [2]Builder.Value = undefined; + fields[payload_offset] = operand; + fields[error_offset] = ok_err_code; + return self.wip.buildAggregate(err_un_llvm_ty, &fields, ""); +} + +fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const inst = body_tail[0]; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const err_un_ty = self.typeOfIndex(inst); + const payload_ty = err_un_ty.errorUnionPayload(zcu); + const operand = try self.resolveInst(ty_op.operand); + if (!payload_ty.hasRuntimeBits(zcu)) return operand; + const err_un_llvm_ty = try o.lowerType(pt, err_un_ty); + + const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt); + const error_offset = try errUnionErrorFieldIndex(payload_ty, pt); + if (isByRef(err_un_ty, zcu)) { + const directReturn = self.isNextRet(body_tail); + const result_ptr = if (directReturn) + self.ret_ptr + else brk: { + const alignment = err_un_ty.abiAlignment(zcu).toLlvm(); + const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); + break :brk result_ptr; + }; + + const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, ""); + const err_int_ty = try pt.errorIntType(); + const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm(); + _ = try self.wip.store(.normal, operand, err_ptr, error_alignment); + const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, ""); + const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); + // TODO store undef to payload_ptr + _ = payload_ptr; + _ = payload_ptr_ty; + return result_ptr; + } + + // TODO set payload bytes to undef + const undef = try o.builder.undefValue(err_un_llvm_ty); + return self.wip.insertValue(undef, operand, &.{error_offset}, ""); +} + +fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const index = pl_op.payload; + const llvm_usize = try o.lowerType(pt, Type.usize); + return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{ + try o.builder.intValue(.i32, index), + }, ""); +} + +fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const index = pl_op.payload; + const llvm_isize = try o.lowerType(pt, Type.isize); + return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{ + try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand), + }, ""); +} + +fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = fg.object; + const pt = fg.pt; + const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; + const llvm_ptr_const = try o.lowerNavRefValue(pt, ty_nav.nav); + return llvm_ptr_const.toValue(); +} + +fn airMin(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + + if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, .normal, inst_ty, 2, .{ lhs, rhs }); + return self.wip.callIntrinsic( + .normal, + .none, + if (scalar_ty.isSignedInt(zcu)) .smin else .umin, + &.{try o.lowerType(pt, inst_ty)}, + &.{ lhs, rhs }, + "", + ); +} + +fn airMax(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + + if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, .normal, inst_ty, 2, .{ lhs, rhs }); + return self.wip.callIntrinsic( + .normal, + .none, + if (scalar_ty.isSignedInt(zcu)) .smax else .umax, + &.{try o.lowerType(pt, inst_ty)}, + &.{ lhs, rhs }, + "", + ); +} + +fn airSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const ptr = try self.resolveInst(bin_op.lhs); + const len = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + return self.wip.buildAggregate(try o.lowerType(pt, inst_ty), &.{ ptr, len }, ""); +} + +fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const zcu = self.pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + + if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, fast, inst_ty, 2, .{ lhs, rhs }); + return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"add nsw" else .@"add nuw", lhs, rhs, ""); +} + +fn airSafeArithmetic( + fg: *FuncGen, + inst: Air.Inst.Index, + signed_intrinsic: Builder.Intrinsic, + unsigned_intrinsic: Builder.Intrinsic, +) !Builder.Value { + const o = fg.object; + const pt = fg.pt; + const zcu = pt.zcu; + + const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try fg.resolveInst(bin_op.lhs); + const rhs = try fg.resolveInst(bin_op.rhs); + const inst_ty = fg.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + + const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; + const llvm_inst_ty = try o.lowerType(pt, inst_ty); + const results = + try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, ""); + + const overflow_bits = try fg.wip.extractValue(results, &.{1}, ""); + const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip); + const overflow_bit = if (overflow_bits_ty.isVector(&o.builder)) + try fg.wip.callIntrinsic( + .normal, + .none, + .@"vector.reduce.or", + &.{overflow_bits_ty}, + &.{overflow_bits}, + "", + ) + else + overflow_bits; + + const fail_block = try fg.wip.block(1, "OverflowFail"); + const ok_block = try fg.wip.block(1, "OverflowOk"); + _ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none); + + fg.wip.cursor = .{ .block = fail_block }; + try fg.buildSimplePanic(.integer_overflow); + + fg.wip.cursor = .{ .block = ok_block }; + return fg.wip.extractValue(results, &.{0}, ""); +} + +fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + return self.wip.bin(.add, lhs, rhs, ""); +} + +fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + assert(scalar_ty.zigTypeTag(zcu) == .int); + return self.wip.callIntrinsic( + .normal, + .none, + if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat", + &.{try o.lowerType(pt, inst_ty)}, + &.{ lhs, rhs }, + "", + ); +} + +fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const zcu = self.pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + + if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, fast, inst_ty, 2, .{ lhs, rhs }); + return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"sub nsw" else .@"sub nuw", lhs, rhs, ""); +} + +fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + return self.wip.bin(.sub, lhs, rhs, ""); +} + +fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + assert(scalar_ty.zigTypeTag(zcu) == .int); + return self.wip.callIntrinsic( + .normal, + .none, + if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat", + &.{try o.lowerType(pt, inst_ty)}, + &.{ lhs, rhs }, + "", + ); +} + +fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const zcu = self.pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + + if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, fast, inst_ty, 2, .{ lhs, rhs }); + return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"mul nsw" else .@"mul nuw", lhs, rhs, ""); +} + +fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + return self.wip.bin(.mul, lhs, rhs, ""); +} + +fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + assert(scalar_ty.zigTypeTag(zcu) == .int); + return self.wip.callIntrinsic( + .normal, + .none, + if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat", + &.{try o.lowerType(pt, inst_ty)}, + &.{ lhs, rhs, .@"0" }, + "", + ); +} + +fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + + return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); +} + +fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const zcu = self.pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + + if (scalar_ty.isRuntimeFloat()) { + const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); + return self.buildFloatOp(.trunc, fast, inst_ty, 1, .{result}); + } + return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .sdiv else .udiv, lhs, rhs, ""); +} + +fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + + if (scalar_ty.isRuntimeFloat()) { + const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); + return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result}); + } + if (scalar_ty.isSignedInt(zcu)) { + const inst_llvm_ty = try o.lowerType(pt, inst_ty); + + const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; + var stack align(@max( + @alignOf(std.heap.StackFallbackAllocator(0)), + @alignOf(ExpectedContents), + )) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); + const allocator = stack.get(); + + const scalar_bits = inst_llvm_ty.scalarBits(&o.builder); + var smin_big_int: std.math.big.int.Mutable = .{ + .limbs = try allocator.alloc( + std.math.big.Limb, + std.math.big.int.calcTwosCompLimbCount(scalar_bits), + ), + .len = undefined, + .positive = undefined, + }; + defer allocator.free(smin_big_int.limbs); + smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits); + const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst( + inst_llvm_ty.scalarType(&o.builder), + smin_big_int.toConst(), + )); + + const div = try self.wip.bin(.sdiv, lhs, rhs, "divFloor.div"); + const rem = try self.wip.bin(.srem, lhs, rhs, "divFloor.rem"); + const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "divFloor.rhs_sign"); + const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "divFloor.rem_xor_rhs_sign"); + const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "divFloor.need_correction"); + const correction = try self.wip.cast(.sext, need_correction, inst_llvm_ty, "divFloor.correction"); + return self.wip.bin(.@"add nsw", div, correction, "divFloor"); + } + return self.wip.bin(.udiv, lhs, rhs, ""); +} + +fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const zcu = self.pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + + if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); + return self.wip.bin( + if (scalar_ty.isSignedInt(zcu)) .@"sdiv exact" else .@"udiv exact", + lhs, + rhs, + "", + ); +} + +fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const zcu = self.pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const scalar_ty = inst_ty.scalarType(zcu); + + if (scalar_ty.isRuntimeFloat()) + return self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs }); + return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) + .srem + else + .urem, lhs, rhs, ""); +} + +fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const inst_ty = self.typeOfIndex(inst); + const inst_llvm_ty = try o.lowerType(pt, inst_ty); + const scalar_ty = inst_ty.scalarType(zcu); + + if (scalar_ty.isRuntimeFloat()) { + const a = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs }); + const b = try self.buildFloatOp(.add, fast, inst_ty, 2, .{ a, rhs }); + const c = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ b, rhs }); + const zero = try o.builder.zeroInitValue(inst_llvm_ty); + const ltz = try self.buildFloatCmp(fast, .lt, inst_ty, .{ lhs, zero }); + return self.wip.select(fast, ltz, c, a, ""); + } + if (scalar_ty.isSignedInt(zcu)) { + const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; + var stack align(@max( + @alignOf(std.heap.StackFallbackAllocator(0)), + @alignOf(ExpectedContents), + )) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); + const allocator = stack.get(); + + const scalar_bits = inst_llvm_ty.scalarBits(&o.builder); + var smin_big_int: std.math.big.int.Mutable = .{ + .limbs = try allocator.alloc( + std.math.big.Limb, + std.math.big.int.calcTwosCompLimbCount(scalar_bits), + ), + .len = undefined, + .positive = undefined, + }; + defer allocator.free(smin_big_int.limbs); + smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits); + const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst( + inst_llvm_ty.scalarType(&o.builder), + smin_big_int.toConst(), + )); + + const rem = try self.wip.bin(.srem, lhs, rhs, "mod.rem"); + const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "mod.rhs_sign"); + const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "mod.rem_xor_rhs_sign"); + const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "mod.need_correction"); + const zero = try o.builder.zeroInitValue(inst_llvm_ty); + const correction = try self.wip.select(.normal, need_correction, rhs, zero, "mod.correction"); + return self.wip.bin(.@"add nsw", correction, rem, "mod"); + } + return self.wip.bin(.urem, lhs, rhs, ""); +} + +fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const ptr = try self.resolveInst(bin_op.lhs); + const offset = try self.resolveInst(bin_op.rhs); + const ptr_ty = self.typeOf(bin_op.lhs); + const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu)); + switch (ptr_ty.ptrSize(zcu)) { + // It's a pointer to an array, so according to LLVM we need an extra GEP index. + .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{ + try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), offset, + }, ""), + .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{offset}, ""), + .slice => { + const base = try self.wip.extractValue(ptr, &.{0}, ""); + return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{offset}, ""); + }, + } +} + +fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + const ptr = try self.resolveInst(bin_op.lhs); + const offset = try self.resolveInst(bin_op.rhs); + const negative_offset = try self.wip.neg(offset, ""); + const ptr_ty = self.typeOf(bin_op.lhs); + const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu)); + switch (ptr_ty.ptrSize(zcu)) { + // It's a pointer to an array, so according to LLVM we need an extra GEP index. + .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{ + try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), negative_offset, + }, ""), + .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{negative_offset}, ""), + .slice => { + const base = try self.wip.extractValue(ptr, &.{0}, ""); + return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{negative_offset}, ""); + }, + } +} + +fn airOverflow( + self: *FuncGen, + inst: Air.Inst.Index, + signed_intrinsic: Builder.Intrinsic, + unsigned_intrinsic: Builder.Intrinsic, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; + + const lhs = try self.resolveInst(extra.lhs); + const rhs = try self.resolveInst(extra.rhs); + + const lhs_ty = self.typeOf(extra.lhs); + const scalar_ty = lhs_ty.scalarType(zcu); + const inst_ty = self.typeOfIndex(inst); + + const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; + const llvm_inst_ty = try o.lowerType(pt, inst_ty); + const llvm_lhs_ty = try o.lowerType(pt, lhs_ty); + const results = + try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, ""); + + const result_val = try self.wip.extractValue(results, &.{0}, ""); + const overflow_bit = try self.wip.extractValue(results, &.{1}, ""); + + const result_index = o.llvmFieldIndex(inst_ty, 0).?; + const overflow_index = o.llvmFieldIndex(inst_ty, 1).?; + + if (isByRef(inst_ty, zcu)) { + const result_alignment = inst_ty.abiAlignment(zcu).toLlvm(); + const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment); + { + const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, ""); + _ = try self.wip.store(.normal, result_val, field_ptr, result_alignment); + } + { + const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, overflow_index, ""); + _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1)); + } + + return alloca_inst; + } + + var fields: [2]Builder.Value = undefined; + fields[result_index] = result_val; + fields[overflow_index] = overflow_bit; + return self.wip.buildAggregate(llvm_inst_ty, &fields, ""); +} + +fn buildElementwiseCall( + self: *FuncGen, + llvm_fn: Builder.Function.Index, + args_vectors: []const Builder.Value, + result_vector: Builder.Value, + vector_len: usize, +) !Builder.Value { + const o = self.object; + assert(args_vectors.len <= 3); + + var i: usize = 0; + var result = result_vector; + while (i < vector_len) : (i += 1) { + const index_i32 = try o.builder.intValue(.i32, i); + + var args: [3]Builder.Value = undefined; + for (args[0..args_vectors.len], args_vectors) |*arg_elem, arg_vector| { + arg_elem.* = try self.wip.extractElement(arg_vector, index_i32, ""); + } + const result_elem = try self.wip.call( + .normal, + .ccc, + .none, + llvm_fn.typeOf(&o.builder), + llvm_fn.toValue(&o.builder), + args[0..args_vectors.len], + "", + ); + result = try self.wip.insertElement(result, result_elem, index_i32, ""); + } + return result; +} + +fn getLibcFunction( + self: *FuncGen, + fn_name: Builder.StrtabString, + param_types: []const Builder.Type, + return_type: Builder.Type, +) Allocator.Error!Builder.Function.Index { + const o = self.object; + if (o.builder.getGlobal(fn_name)) |global| return switch (global.ptrConst(&o.builder).kind) { + .alias => |alias| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function, + .function => |function| function, + .variable, .replaced => unreachable, + }; + return o.builder.addFunction( + try o.builder.fnType(return_type, param_types, .normal), + fn_name, + toLlvmAddressSpace(.generic, self.pt.zcu.getTarget()), + ); +} + +/// Creates a floating point comparison by lowering to the appropriate +/// hardware instruction or softfloat routine for the target +fn buildFloatCmp( + self: *FuncGen, + fast: Builder.FastMathKind, + pred: math.CompareOperator, + ty: Type, + params: [2]Builder.Value, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const target = zcu.getTarget(); + const scalar_ty = ty.scalarType(zcu); + const scalar_llvm_ty = try o.lowerType(pt, scalar_ty); + + if (intrinsicsAllowed(scalar_ty, target)) { + const cond: Builder.FloatCondition = switch (pred) { + .eq => .oeq, + .neq => .une, + .lt => .olt, + .lte => .ole, + .gt => .ogt, + .gte => .oge, + }; + return self.wip.fcmp(fast, cond, params[0], params[1], ""); + } + + const float_bits = scalar_ty.floatBits(target); + const compiler_rt_float_abbrev = compilerRtFloatAbbrev(float_bits); + const fn_base_name = switch (pred) { + .neq => "ne", + .eq => "eq", + .lt => "lt", + .lte => "le", + .gt => "gt", + .gte => "ge", + }; + const fn_name = try o.builder.strtabStringFmt("__{s}{s}f2", .{ fn_base_name, compiler_rt_float_abbrev }); + + const libc_fn = try self.getLibcFunction(fn_name, &.{ scalar_llvm_ty, scalar_llvm_ty }, .i32); + + const int_cond: Builder.IntegerCondition = switch (pred) { + .eq => .eq, + .neq => .ne, + .lt => .slt, + .lte => .sle, + .gt => .sgt, + .gte => .sge, + }; + + if (ty.zigTypeTag(zcu) == .vector) { + const vec_len = ty.vectorLen(zcu); + const vector_result_ty = try o.builder.vectorType(.normal, vec_len, .i32); + + const init = try o.builder.poisonValue(vector_result_ty); + const result = try self.buildElementwiseCall(libc_fn, ¶ms, init, vec_len); + + const zero_vector = try o.builder.splatValue(vector_result_ty, .@"0"); + return self.wip.icmp(int_cond, result, zero_vector, ""); + } + + const result = try self.wip.call( + .normal, + .ccc, + .none, + libc_fn.typeOf(&o.builder), + libc_fn.toValue(&o.builder), + ¶ms, + "", + ); + return self.wip.icmp(int_cond, result, .@"0", ""); +} + +const FloatOp = enum { + add, + ceil, + cos, + div, + exp, + exp2, + fabs, + floor, + fma, + fmax, + fmin, + fmod, + log, + log10, + log2, + mul, + neg, + round, + sin, + sqrt, + sub, + tan, + trunc, +}; + +const FloatOpStrat = union(enum) { + intrinsic: []const u8, + libc: Builder.String, +}; + +/// Creates a floating point operation (add, sub, fma, sqrt, exp, etc.) +/// by lowering to the appropriate hardware instruction or softfloat +/// routine for the target +fn buildFloatOp( + self: *FuncGen, + comptime op: FloatOp, + fast: Builder.FastMathKind, + ty: Type, + comptime params_len: usize, + params: [params_len]Builder.Value, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const target = zcu.getTarget(); + const scalar_ty = ty.scalarType(zcu); + const llvm_ty = try o.lowerType(pt, ty); + + if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) { + // Some operations are dedicated LLVM instructions, not available as intrinsics + .neg => return self.wip.un(.fneg, params[0], ""), + .add, .sub, .mul, .div, .fmod => return self.wip.bin(switch (fast) { + .normal => switch (op) { + .add => .fadd, + .sub => .fsub, + .mul => .fmul, + .div => .fdiv, + .fmod => .frem, + else => unreachable, + }, + .fast => switch (op) { + .add => .@"fadd fast", + .sub => .@"fsub fast", + .mul => .@"fmul fast", + .div => .@"fdiv fast", + .fmod => .@"frem fast", + else => unreachable, + }, + }, params[0], params[1], ""), + .fmax, + .fmin, + .ceil, + .cos, + .exp, + .exp2, + .fabs, + .floor, + .log, + .log10, + .log2, + .round, + .sin, + .sqrt, + .trunc, + .fma, + => return self.wip.callIntrinsic(fast, .none, switch (op) { + .fmax => .maxnum, + .fmin => .minnum, + .ceil => .ceil, + .cos => .cos, + .exp => .exp, + .exp2 => .exp2, + .fabs => .fabs, + .floor => .floor, + .log => .log, + .log10 => .log10, + .log2 => .log2, + .round => .round, + .sin => .sin, + .sqrt => .sqrt, + .trunc => .trunc, + .fma => .fma, + else => unreachable, + }, &.{llvm_ty}, ¶ms, ""), + .tan => unreachable, + }; + + const float_bits = scalar_ty.floatBits(target); + const fn_name = switch (op) { + .neg => { + // In this case we can generate a softfloat negation by XORing the + // bits with a constant. + const int_ty = try o.builder.intType(@intCast(float_bits)); + const cast_ty = try llvm_ty.changeScalar(int_ty, &o.builder); + const sign_mask = try o.builder.splatValue( + cast_ty, + try o.builder.intConst(int_ty, @as(u128, 1) << @intCast(float_bits - 1)), + ); + const bitcasted_operand = try self.wip.cast(.bitcast, params[0], cast_ty, ""); + const result = try self.wip.bin(.xor, bitcasted_operand, sign_mask, ""); + return self.wip.cast(.bitcast, result, llvm_ty, ""); + }, + .add, .sub, .div, .mul => try o.builder.strtabStringFmt("__{s}{s}f3", .{ + @tagName(op), compilerRtFloatAbbrev(float_bits), + }), + .ceil, + .cos, + .exp, + .exp2, + .fabs, + .floor, + .fma, + .fmax, + .fmin, + .fmod, + .log, + .log10, + .log2, + .round, + .sin, + .sqrt, + .tan, + .trunc, + => try o.builder.strtabStringFmt("{s}{s}{s}", .{ + libcFloatPrefix(float_bits), @tagName(op), libcFloatSuffix(float_bits), + }), + }; + + const scalar_llvm_ty = llvm_ty.scalarType(&o.builder); + const libc_fn = try self.getLibcFunction( + fn_name, + ([1]Builder.Type{scalar_llvm_ty} ** 3)[0..params.len], + scalar_llvm_ty, + ); + if (ty.zigTypeTag(zcu) == .vector) { + const result = try o.builder.poisonValue(llvm_ty); + return self.buildElementwiseCall(libc_fn, ¶ms, result, ty.vectorLen(zcu)); + } + + return self.wip.call( + fast.toCallKind(), + .ccc, + .none, + libc_fn.typeOf(&o.builder), + libc_fn.toValue(&o.builder), + ¶ms, + "", + ); +} + +fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Bin, pl_op.payload).data; + + const mulend1 = try self.resolveInst(extra.lhs); + const mulend2 = try self.resolveInst(extra.rhs); + const addend = try self.resolveInst(pl_op.operand); + + const ty = self.typeOfIndex(inst); + return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend }); +} + +fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; + + const lhs = try self.resolveInst(extra.lhs); + const rhs = try self.resolveInst(extra.rhs); + + const lhs_ty = self.typeOf(extra.lhs); + if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu)) + return self.todo("implement vector shifts with scalar rhs", .{}); + const lhs_scalar_ty = lhs_ty.scalarType(zcu); + + const dest_ty = self.typeOfIndex(inst); + const llvm_dest_ty = try o.lowerType(pt, dest_ty); + + const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); + + const result = try self.wip.bin(.shl, lhs, casted_rhs, ""); + const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu)) + .ashr + else + .lshr, result, casted_rhs, ""); + + const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, ""); + + const result_index = o.llvmFieldIndex(dest_ty, 0).?; + const overflow_index = o.llvmFieldIndex(dest_ty, 1).?; + + if (isByRef(dest_ty, zcu)) { + const result_alignment = dest_ty.abiAlignment(zcu).toLlvm(); + const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment); + { + const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, ""); + _ = try self.wip.store(.normal, result, field_ptr, result_alignment); + } + { + const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, overflow_index, ""); + _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1)); + } + return alloca_inst; + } + + var fields: [2]Builder.Value = undefined; + fields[result_index] = result; + fields[overflow_index] = overflow_bit; + return self.wip.buildAggregate(llvm_dest_ty, &fields, ""); +} + +fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + return self.wip.bin(.@"and", lhs, rhs, ""); +} + +fn airOr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + return self.wip.bin(.@"or", lhs, rhs, ""); +} + +fn airXor(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + return self.wip.bin(.xor, lhs, rhs, ""); +} + +fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const lhs_ty = self.typeOf(bin_op.lhs); + if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) + return self.todo("implement vector shifts with scalar rhs", .{}); + const lhs_scalar_ty = lhs_ty.scalarType(zcu); + + const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); + return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu)) + .@"shl nsw" + else + .@"shl nuw", lhs, casted_rhs, ""); +} + +fn airShl(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const lhs_ty = self.typeOf(bin_op.lhs); + if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) + return self.todo("implement vector shifts with scalar rhs", .{}); + + const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); + return self.wip.bin(.shl, lhs, casted_rhs, ""); +} + +fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const lhs_ty = self.typeOf(bin_op.lhs); + const lhs_info = lhs_ty.intInfo(zcu); + const llvm_lhs_ty = try o.lowerType(pt, lhs_ty); + const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder); + + const rhs_ty = self.typeOf(bin_op.rhs); + if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) + return self.todo("implement vector shifts with scalar rhs", .{}); + const rhs_info = rhs_ty.intInfo(zcu); + assert(rhs_info.signedness == .unsigned); + const llvm_rhs_ty = try o.lowerType(pt, rhs_ty); + const llvm_rhs_scalar_ty = llvm_rhs_ty.scalarType(&o.builder); + + const result = try self.wip.callIntrinsic( + .normal, + .none, + switch (lhs_info.signedness) { + .signed => .@"sshl.sat", + .unsigned => .@"ushl.sat", + }, + &.{llvm_lhs_ty}, + &.{ lhs, try self.wip.conv(.unsigned, rhs, llvm_lhs_ty, "") }, + "", + ); + + // LLVM langref says "If b is (statically or dynamically) equal to or + // larger than the integer bit width of the arguments, the result is a + // poison value." + // However Zig semantics says that saturating shift left can never produce + // undefined; instead it saturates. + if (rhs_info.bits <= math.log2_int(u16, lhs_info.bits)) return result; + const bits = try o.builder.splatValue( + llvm_rhs_ty, + try o.builder.intConst(llvm_rhs_scalar_ty, lhs_info.bits), + ); + const in_range = try self.wip.icmp(.ult, rhs, bits, ""); + const lhs_sat = lhs_sat: switch (lhs_info.signedness) { + .signed => { + const zero = try o.builder.splatValue( + llvm_lhs_ty, + try o.builder.intConst(llvm_lhs_scalar_ty, 0), + ); + const smin = try o.builder.splatValue( + llvm_lhs_ty, + try minIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu), + ); + const smax = try o.builder.splatValue( + llvm_lhs_ty, + try maxIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu), + ); + const lhs_lt_zero = try self.wip.icmp(.slt, lhs, zero, ""); + const slimit = try self.wip.select(.normal, lhs_lt_zero, smin, smax, ""); + const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, ""); + break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, slimit, ""); + }, + .unsigned => { + const zero = try o.builder.splatValue( + llvm_lhs_ty, + try o.builder.intConst(llvm_lhs_scalar_ty, 0), + ); + const umax = try o.builder.splatValue( + llvm_lhs_ty, + try o.builder.intConst(llvm_lhs_scalar_ty, -1), + ); + const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, ""); + break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, umax, ""); + }, + }; + return self.wip.select(.normal, in_range, result, lhs_sat, ""); +} + +fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const lhs_ty = self.typeOf(bin_op.lhs); + if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) + return self.todo("implement vector shifts with scalar rhs", .{}); + const lhs_scalar_ty = lhs_ty.scalarType(zcu); + + const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); + const is_signed_int = lhs_scalar_ty.isSignedInt(zcu); + + return self.wip.bin(if (is_exact) + if (is_signed_int) .@"ashr exact" else .@"lshr exact" + else if (is_signed_int) .ashr else .lshr, lhs, casted_rhs, ""); +} + +fn airAbs(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const operand_ty = self.typeOf(ty_op.operand); + const scalar_ty = operand_ty.scalarType(zcu); + + switch (scalar_ty.zigTypeTag(zcu)) { + .int => return self.wip.callIntrinsic( + .normal, + .none, + .abs, + &.{try o.lowerType(pt, operand_ty)}, + &.{ operand, try o.builder.intValue(.i1, 0) }, + "", + ), + .float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}), + else => unreachable, + } +} + +fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { + const o = fg.object; + const pt = fg.pt; + const zcu = pt.zcu; + const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const dest_ty = fg.typeOfIndex(inst); + const dest_llvm_ty = try o.lowerType(pt, dest_ty); + const operand = try fg.resolveInst(ty_op.operand); + const operand_ty = fg.typeOf(ty_op.operand); + const operand_info = operand_ty.intInfo(zcu); + + const dest_is_enum = dest_ty.zigTypeTag(zcu) == .@"enum"; + + bounds_check: { + const dest_scalar = dest_ty.scalarType(zcu); + const operand_scalar = operand_ty.scalarType(zcu); + + const dest_info = dest_ty.intInfo(zcu); + + const have_min_check, const have_max_check = c: { + const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed); + const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed); + + const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0; + const operand_maybe_neg = operand_info.signedness == .signed and operand_info.bits > 0; + + break :c .{ + operand_maybe_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits), + dest_pos_bits < operand_pos_bits, + }; + }; + + if (!have_min_check and !have_max_check) break :bounds_check; + + const operand_llvm_ty = try o.lowerType(pt, operand_ty); + const operand_scalar_llvm_ty = try o.lowerType(pt, operand_scalar); + + const is_vector = operand_ty.zigTypeTag(zcu) == .vector; + assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector)); + + const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .integer_out_of_bounds; + + if (have_min_check) { + const min_const_scalar = try minIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu); + const min_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, min_const_scalar) else min_const_scalar.toValue(); + const ok_maybe_vec = try fg.cmp(.normal, .gte, operand_ty, operand, min_val); + const ok = if (is_vector) ok: { + const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip); + break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, ""); + } else ok_maybe_vec; + if (safety) { + const fail_block = try fg.wip.block(1, "IntMinFail"); + const ok_block = try fg.wip.block(1, "IntMinOk"); + _ = try fg.wip.brCond(ok, ok_block, fail_block, .none); + fg.wip.cursor = .{ .block = fail_block }; + try fg.buildSimplePanic(panic_id); + fg.wip.cursor = .{ .block = ok_block }; + } else { + _ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, ""); + } + } + + if (have_max_check) { + const max_const_scalar = try maxIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu); + const max_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, max_const_scalar) else max_const_scalar.toValue(); + const ok_maybe_vec = try fg.cmp(.normal, .lte, operand_ty, operand, max_val); + const ok = if (is_vector) ok: { + const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip); + break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, ""); + } else ok_maybe_vec; + if (safety) { + const fail_block = try fg.wip.block(1, "IntMaxFail"); + const ok_block = try fg.wip.block(1, "IntMaxOk"); + _ = try fg.wip.brCond(ok, ok_block, fail_block, .none); + fg.wip.cursor = .{ .block = fail_block }; + try fg.buildSimplePanic(panic_id); + fg.wip.cursor = .{ .block = ok_block }; + } else { + _ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, ""); + } + } + } + + const result = try fg.wip.conv(switch (operand_info.signedness) { + .signed => .signed, + .unsigned => .unsigned, + }, operand, dest_llvm_ty, ""); + + if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) { + const llvm_fn = try fg.getIsNamedEnumValueFunction(dest_ty); + const is_valid_enum_val = try fg.wip.call( + .normal, + .fastcc, + .none, + llvm_fn.typeOf(&o.builder), + llvm_fn.toValue(&o.builder), + &.{result}, + "", + ); + const fail_block = try fg.wip.block(1, "ValidEnumFail"); + const ok_block = try fg.wip.block(1, "ValidEnumOk"); + _ = try fg.wip.brCond(is_valid_enum_val, ok_block, fail_block, .none); + fg.wip.cursor = .{ .block = fail_block }; + try fg.buildSimplePanic(.invalid_enum_value); + fg.wip.cursor = .{ .block = ok_block }; + } + + return result; +} + +fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const dest_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst)); + return self.wip.cast(.trunc, operand, dest_llvm_ty, ""); +} + +fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const operand_ty = self.typeOf(ty_op.operand); + const dest_ty = self.typeOfIndex(inst); + const target = zcu.getTarget(); + + if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) { + return self.wip.cast(.fptrunc, operand, try o.lowerType(pt, dest_ty), ""); + } else { + const operand_llvm_ty = try o.lowerType(pt, operand_ty); + const dest_llvm_ty = try o.lowerType(pt, dest_ty); + + const dest_bits = dest_ty.floatBits(target); + const src_bits = operand_ty.floatBits(target); + const fn_name = try o.builder.strtabStringFmt("__trunc{s}f{s}f2", .{ + compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits), + }); + + const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty); + return self.wip.call( + .normal, + .ccc, + .none, + libc_fn.typeOf(&o.builder), + libc_fn.toValue(&o.builder), + &.{operand}, + "", + ); + } +} + +fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const operand_ty = self.typeOf(ty_op.operand); + const dest_ty = self.typeOfIndex(inst); + const target = zcu.getTarget(); + + if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) { + return self.wip.cast(.fpext, operand, try o.lowerType(pt, dest_ty), ""); + } else { + const operand_llvm_ty = try o.lowerType(pt, operand_ty); + const dest_llvm_ty = try o.lowerType(pt, dest_ty); + + const dest_bits = dest_ty.scalarType(zcu).floatBits(target); + const src_bits = operand_ty.scalarType(zcu).floatBits(target); + const fn_name = try o.builder.strtabStringFmt("__extend{s}f{s}f2", .{ + compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits), + }); + + const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty); + if (dest_ty.isVector(zcu)) return self.buildElementwiseCall( + libc_fn, + &.{operand}, + try o.builder.poisonValue(dest_llvm_ty), + dest_ty.vectorLen(zcu), + ); + return self.wip.call( + .normal, + .ccc, + .none, + libc_fn.typeOf(&o.builder), + libc_fn.toValue(&o.builder), + &.{operand}, + "", + ); + } +} + +fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand_ty = self.typeOf(ty_op.operand); + const inst_ty = self.typeOfIndex(inst); + const operand = try self.resolveInst(ty_op.operand); + return self.bitCast(operand, operand_ty, inst_ty); +} + +fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const operand_is_ref = isByRef(operand_ty, zcu); + const result_is_ref = isByRef(inst_ty, zcu); + const llvm_dest_ty = try o.lowerType(pt, inst_ty); + + if (operand_is_ref and result_is_ref) { + // They are both pointers, so just return the same opaque pointer :) + return operand; + } + + if (llvm_dest_ty.isInteger(&o.builder) and + operand.typeOfWip(&self.wip).isInteger(&o.builder)) + { + return self.wip.conv(.unsigned, operand, llvm_dest_ty, ""); + } + + const operand_scalar_ty = operand_ty.scalarType(zcu); + const inst_scalar_ty = inst_ty.scalarType(zcu); + if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) { + return self.wip.cast(.inttoptr, operand, llvm_dest_ty, ""); + } + if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) { + return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, ""); + } + + if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) { + const elem_ty = operand_ty.childType(zcu); + if (!result_is_ref) { + return self.todo("implement bitcast vector to non-ref array", .{}); + } + const alignment = inst_ty.abiAlignment(zcu).toLlvm(); + const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment); + const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; + if (bitcast_ok) { + _ = try self.wip.store(.normal, operand, array_ptr, alignment); + } else { + // If the ABI size of the element type is not evenly divisible by size in bits; + // a simple bitcast will not work, and we fall back to extractelement. + const llvm_usize = try o.lowerType(pt, Type.usize); + const usize_zero = try o.builder.intValue(llvm_usize, 0); + const vector_len = operand_ty.arrayLen(zcu); + var i: u64 = 0; + while (i < vector_len) : (i += 1) { + const elem_ptr = try self.wip.gep(.inbounds, llvm_dest_ty, array_ptr, &.{ + usize_zero, try o.builder.intValue(llvm_usize, i), + }, ""); + const elem = + try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), ""); + _ = try self.wip.store(.normal, elem, elem_ptr, .default); + } + } + return array_ptr; + } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) { + const elem_ty = operand_ty.childType(zcu); + const llvm_vector_ty = try o.lowerType(pt, inst_ty); + if (!operand_is_ref) return self.todo("implement bitcast non-ref array to vector", .{}); + + const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; + if (bitcast_ok) { + // The array is aligned to the element's alignment, while the vector might have a completely + // different alignment. This means we need to enforce the alignment of this load. + const alignment = elem_ty.abiAlignment(zcu).toLlvm(); + return self.wip.load(.normal, llvm_vector_ty, operand, alignment, ""); + } else { + // If the ABI size of the element type is not evenly divisible by size in bits; + // a simple bitcast will not work, and we fall back to extractelement. + const array_llvm_ty = try o.lowerType(pt, operand_ty); + const elem_llvm_ty = try o.lowerType(pt, elem_ty); + const llvm_usize = try o.lowerType(pt, Type.usize); + const usize_zero = try o.builder.intValue(llvm_usize, 0); + const vector_len = operand_ty.arrayLen(zcu); + var vector = try o.builder.poisonValue(llvm_vector_ty); + var i: u64 = 0; + while (i < vector_len) : (i += 1) { + const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, operand, &.{ + usize_zero, try o.builder.intValue(llvm_usize, i), + }, ""); + const elem = try self.wip.load(.normal, elem_llvm_ty, elem_ptr, .default, ""); + vector = + try self.wip.insertElement(vector, elem, try o.builder.intValue(.i32, i), ""); + } + return vector; + } + } + + if (operand_is_ref) { + const alignment = operand_ty.abiAlignment(zcu).toLlvm(); + return self.wip.load(.normal, llvm_dest_ty, operand, alignment, ""); + } + + if (result_is_ref) { + const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm(); + const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment); + _ = try self.wip.store(.normal, operand, result_ptr, alignment); + return result_ptr; + } + + if (llvm_dest_ty.isStruct(&o.builder) or + ((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and + operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu))) + { + // Both our operand and our result are values, not pointers, + // but LLVM won't let us bitcast struct values or vectors with padding bits. + // Therefore, we store operand to alloca, then load for result. + const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm(); + const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment); + _ = try self.wip.store(.normal, operand, result_ptr, alignment); + return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, ""); + } + + return self.wip.cast(.bitcast, operand, llvm_dest_ty, ""); +} + +fn airArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const arg_val = self.args[self.arg_index]; + self.arg_index += 1; + + // llvm does not support debug info for naked function arguments + if (self.is_naked) return arg_val; + + const inst_ty = self.typeOfIndex(inst); + + const func = zcu.funcInfo(zcu.navValue(self.nav_index).toIntern()); + const func_zir = func.zir_body_inst.resolveFull(&zcu.intern_pool).?; + const file = zcu.fileByIndex(func_zir.file); + + const mod = file.mod.?; + if (mod.strip) return arg_val; + const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg; + const zir = &file.zir.?; + const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?); + + const lbrace_line = zcu.navSrcLine(func.owner_nav) + func.lbrace_line + 1; + const lbrace_col = func.lbrace_column + 1; + + const debug_parameter = try o.builder.debugParameter( + if (name.len > 0) try o.builder.metadataString(name) else null, + self.file, + self.scope, + lbrace_line, + try o.getDebugType(pt, inst_ty), + self.arg_index, + ); + + const old_location = self.wip.debug_location; + self.wip.debug_location = .{ .location = .{ + .line = lbrace_line, + .column = lbrace_col, + .scope = self.scope.toOptional(), + .inlined_at = .none, + } }; + + if (isByRef(inst_ty, zcu)) { + _ = try self.wip.callIntrinsic( + .normal, + .none, + .@"dbg.declare", + &.{}, + &.{ + (try self.wip.debugValue(arg_val)).toValue(), + debug_parameter.toValue(), + (try o.builder.debugExpression(&.{})).toValue(), + }, + "", + ); + } else if (mod.optimize_mode == .Debug) { + const alignment = inst_ty.abiAlignment(zcu).toLlvm(); + const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment); + _ = try self.wip.store(.normal, arg_val, alloca, alignment); + _ = try self.wip.callIntrinsic( + .normal, + .none, + .@"dbg.declare", + &.{}, + &.{ + (try self.wip.debugValue(alloca)).toValue(), + debug_parameter.toValue(), + (try o.builder.debugExpression(&.{})).toValue(), + }, + "", + ); + } else { + _ = try self.wip.callIntrinsic( + .normal, + .none, + .@"dbg.value", + &.{}, + &.{ + (try self.wip.debugValue(arg_val)).toValue(), + debug_parameter.toValue(), + (try o.builder.debugExpression(&.{})).toValue(), + }, + "", + ); + } + + self.wip.debug_location = old_location; + return arg_val; +} + +fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ptr_ty = self.typeOfIndex(inst); + const pointee_type = ptr_ty.childType(zcu); + if (!pointee_type.hasRuntimeBits(zcu)) + return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue(); + + const pointee_llvm_ty = try o.lowerType(pt, pointee_type); + const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); + return self.buildAlloca(pointee_llvm_ty, alignment); +} + +fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ptr_ty = self.typeOfIndex(inst); + const ret_ty = ptr_ty.childType(zcu); + if (!ret_ty.hasRuntimeBits(zcu)) + return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue(); + if (self.ret_ptr != .none) return self.ret_ptr; + const ret_llvm_ty = try o.lowerType(pt, ret_ty); + const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); + return self.buildAlloca(ret_llvm_ty, alignment); +} + +/// Use this instead of builder.buildAlloca, because this function makes sure to +/// put the alloca instruction at the top of the function! +fn buildAlloca( + self: *FuncGen, + llvm_ty: Builder.Type, + alignment: Builder.Alignment, +) Allocator.Error!Builder.Value { + const target = self.pt.zcu.getTarget(); + return buildAllocaInner(&self.wip, llvm_ty, alignment, target); +} + +fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const dest_ptr = try self.resolveInst(bin_op.lhs); + const ptr_ty = self.typeOf(bin_op.lhs); + const operand_ty = ptr_ty.childType(zcu); + + const val_is_undef = if (try self.air.value(bin_op.rhs, pt)) |val| val.isUndef(zcu) else false; + if (val_is_undef) { + const owner_mod = self.ownerModule(); + + // Even if safety is disabled, we still emit a memset to undefined since it conveys + // extra information to LLVM, and LLVM will optimize it out. Safety makes the difference + // between using 0xaa or actual undefined for the fill byte. + // + // However, for Debug builds specifically, we avoid emitting the memset because LLVM + // will neither use the information nor get rid of the memset, thus leaving an + // unexpected call in the user's code. This is problematic if the code in question is + // not ready to correctly make calls yet, such as in our early PIE startup code, or in + // the early stages of a dynamic linker, etc. + if (!safety and owner_mod.optimize_mode == .Debug) { + return .none; + } + + const ptr_info = ptr_ty.ptrInfo(zcu); + const needs_bitmask = (ptr_info.packed_offset.host_size != 0); + if (needs_bitmask) { + // TODO: only some bits are to be undef, we cannot write with a simple memset. + // meanwhile, ignore the write rather than stomping over valid bits. + // https://github.com/ziglang/zig/issues/15337 + return .none; + } + + self.maybeMarkAllowZeroAccess(ptr_info); + + const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), operand_ty.abiSize(zcu)); + _ = try self.wip.callMemSet( + dest_ptr, + ptr_ty.ptrAlignment(zcu).toLlvm(), + if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8), + len, + if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, + self.disable_intrinsics, + ); + if (safety and owner_mod.valgrind) { + try self.valgrindMarkUndef(dest_ptr, len); + } + return .none; + } + + self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); + + const src_operand = try self.resolveInst(bin_op.rhs); + try self.store(dest_ptr, ptr_ty, src_operand, .none); + return .none; +} + +fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const pt = fg.pt; + const zcu = pt.zcu; + const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const ptr_ty = fg.typeOf(ty_op.operand); + const ptr_info = ptr_ty.ptrInfo(zcu); + const ptr = try fg.resolveInst(ty_op.operand); + fg.maybeMarkAllowZeroAccess(ptr_info); + return fg.load(ptr, ptr_ty); +} + +fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !void { + _ = inst; + const target = self.pt.zcu.getTarget(); + if ((target.cpu.arch == .mips or target.cpu.arch == .mipsel) and + target.cpu.has(.mips, .notraps)) + { + // Emit a MIPS `break` instruction followed by an infinite loop (to fulfil the noreturn) + // since this CPU does not support trap instructions. + const o = self.object; + _ = try self.wip.callAsm( + .none, + try o.builder.fnType(.void, &.{}, .normal), + .{ .sideeffect = true }, + try o.builder.string("break\n0:\nj 0b\nnop"), + try o.builder.string("~{memory}"), + &.{}, + "", + ); + } else { + _ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, ""); + } + _ = try self.wip.@"unreachable"(); +} + +fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + _ = inst; + _ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, ""); + return .none; +} + +fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + _ = inst; + const o = self.object; + const pt = self.pt; + const llvm_usize = try o.lowerType(pt, Type.usize); + if (!target_util.supportsReturnAddress(self.pt.zcu.getTarget(), self.ownerModule().optimize_mode)) { + // https://github.com/ziglang/zig/issues/11946 + return o.builder.intValue(llvm_usize, 0); + } + const result = try self.wip.callIntrinsic(.normal, .none, .returnaddress, &.{}, &.{.@"0"}, ""); + return self.wip.cast(.ptrtoint, result, llvm_usize, ""); +} + +fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + _ = inst; + const o = self.object; + const pt = self.pt; + const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, ""); + return self.wip.cast(.ptrtoint, result, try o.lowerType(pt, Type.usize), ""); +} + +fn airCmpxchg( + self: *FuncGen, + inst: Air.Inst.Index, + kind: Builder.Function.Instruction.CmpXchg.Kind, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data; + const ptr = try self.resolveInst(extra.ptr); + const ptr_ty = self.typeOf(extra.ptr); + var expected_value = try self.resolveInst(extra.expected_value); + var new_value = try self.resolveInst(extra.new_value); + const operand_ty = ptr_ty.childType(zcu); + const llvm_operand_ty = try o.lowerType(pt, operand_ty); + const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false); + if (llvm_abi_ty != .none) { + // operand needs widening and truncating + const signedness: Builder.Function.Instruction.Cast.Signedness = + if (operand_ty.isSignedInt(zcu)) .signed else .unsigned; + expected_value = try self.wip.conv(signedness, expected_value, llvm_abi_ty, ""); + new_value = try self.wip.conv(signedness, new_value, llvm_abi_ty, ""); + } + + self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); + + const result = try self.wip.cmpxchg( + kind, + if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, + ptr, + expected_value, + new_value, + self.sync_scope, + toLlvmAtomicOrdering(extra.successOrder()), + toLlvmAtomicOrdering(extra.failureOrder()), + ptr_ty.ptrAlignment(zcu).toLlvm(), + "", + ); + + const optional_ty = self.typeOfIndex(inst); + + var payload = try self.wip.extractValue(result, &.{0}, ""); + if (llvm_abi_ty != .none) payload = try self.wip.cast(.trunc, payload, llvm_operand_ty, ""); + const success_bit = try self.wip.extractValue(result, &.{1}, ""); + + if (optional_ty.optionalReprIsPayload(zcu)) { + const zero = try o.builder.zeroInitValue(payload.typeOfWip(&self.wip)); + return self.wip.select(.normal, success_bit, zero, payload, ""); + } + + comptime assert(optional_layout_version == 3); + + const non_null_bit = try self.wip.not(success_bit, ""); + return buildOptional(self, optional_ty, payload, non_null_bit); +} + +fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; + const ptr = try self.resolveInst(pl_op.operand); + const ptr_ty = self.typeOf(pl_op.operand); + const operand_ty = ptr_ty.childType(zcu); + const operand = try self.resolveInst(extra.operand); + const is_signed_int = operand_ty.isSignedInt(zcu); + const is_float = operand_ty.isRuntimeFloat(); + const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float); + const ordering = toLlvmAtomicOrdering(extra.ordering()); + const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg); + const llvm_operand_ty = try o.lowerType(pt, operand_ty); + + const access_kind: Builder.MemoryAccessKind = + if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); + + self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); + + if (llvm_abi_ty != .none) { + // operand needs widening and truncating or bitcasting. + return self.wip.cast(if (is_float) .bitcast else .trunc, try self.wip.atomicrmw( + access_kind, + op, + ptr, + try self.wip.cast( + if (is_float) .bitcast else if (is_signed_int) .sext else .zext, + operand, + llvm_abi_ty, + "", + ), + self.sync_scope, + ordering, + ptr_alignment, + "", + ), llvm_operand_ty, ""); + } + + if (!llvm_operand_ty.isPointer(&o.builder)) return self.wip.atomicrmw( + access_kind, + op, + ptr, + operand, + self.sync_scope, + ordering, + ptr_alignment, + "", + ); + + // It's a pointer but we need to treat it as an int. + return self.wip.cast(.inttoptr, try self.wip.atomicrmw( + access_kind, + op, + ptr, + try self.wip.cast(.ptrtoint, operand, try o.lowerType(pt, Type.usize), ""), + self.sync_scope, + ordering, + ptr_alignment, + "", + ), llvm_operand_ty, ""); +} + +fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; + const ptr = try self.resolveInst(atomic_load.ptr); + const ptr_ty = self.typeOf(atomic_load.ptr); + const info = ptr_ty.ptrInfo(zcu); + const elem_ty = Type.fromInterned(info.child); + if (!elem_ty.hasRuntimeBits(zcu)) return .none; + const ordering = toLlvmAtomicOrdering(atomic_load.order); + const llvm_abi_ty = try self.getAtomicAbiType(elem_ty, false); + const ptr_alignment = (if (info.flags.alignment != .none) + @as(InternPool.Alignment, info.flags.alignment) + else + Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm(); + const access_kind: Builder.MemoryAccessKind = + if (info.flags.is_volatile) .@"volatile" else .normal; + const elem_llvm_ty = try o.lowerType(pt, elem_ty); + + self.maybeMarkAllowZeroAccess(info); + + if (llvm_abi_ty != .none) { + // operand needs widening and truncating + const loaded = try self.wip.loadAtomic( + access_kind, + llvm_abi_ty, + ptr, + self.sync_scope, + ordering, + ptr_alignment, + "", + ); + return self.wip.cast(.trunc, loaded, elem_llvm_ty, ""); + } + return self.wip.loadAtomic( + access_kind, + elem_llvm_ty, + ptr, + self.sync_scope, + ordering, + ptr_alignment, + "", + ); +} + +fn airAtomicStore( + self: *FuncGen, + inst: Air.Inst.Index, + ordering: Builder.AtomicOrdering, +) !Builder.Value { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const ptr_ty = self.typeOf(bin_op.lhs); + const operand_ty = ptr_ty.childType(zcu); + if (!operand_ty.hasRuntimeBits(zcu)) return .none; + const ptr = try self.resolveInst(bin_op.lhs); + var element = try self.resolveInst(bin_op.rhs); + const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false); + + if (llvm_abi_ty != .none) { + // operand needs widening + element = try self.wip.conv( + if (operand_ty.isSignedInt(zcu)) .signed else .unsigned, + element, + llvm_abi_ty, + "", + ); + } + + self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); + + try self.store(ptr, ptr_ty, element, ordering); + return .none; +} + +fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const dest_slice = try self.resolveInst(bin_op.lhs); + const ptr_ty = self.typeOf(bin_op.lhs); + const elem_ty = self.typeOf(bin_op.rhs); + const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm(); + const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty); + const access_kind: Builder.MemoryAccessKind = + if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + + self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); + + if (try self.air.value(bin_op.rhs, pt)) |elem_val| { + if (elem_val.isUndef(zcu)) { + // Even if safety is disabled, we still emit a memset to undefined since it conveys + // extra information to LLVM. However, safety makes the difference between using + // 0xaa or actual undefined for the fill byte. + const fill_byte = if (safety) + try o.builder.intValue(.i8, 0xaa) + else + try o.builder.undefValue(.i8); + const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); + _ = try self.wip.callMemSet( + dest_ptr, + dest_ptr_align, + fill_byte, + len, + access_kind, + self.disable_intrinsics, + ); + const owner_mod = self.ownerModule(); + if (safety and owner_mod.valgrind) { + try self.valgrindMarkUndef(dest_ptr, len); + } + return .none; + } + + // Test if the element value is compile-time known to be a + // repeating byte pattern, for example, `@as(u64, 0)` has a + // repeating byte pattern of 0 bytes. In such case, the memset + // intrinsic can be used. + if (try elem_val.hasRepeatedByteRepr(pt)) |byte_val| { + const fill_byte = try o.builder.intValue(.i8, byte_val); + const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); + _ = try self.wip.callMemSet( + dest_ptr, + dest_ptr_align, + fill_byte, + len, + access_kind, + self.disable_intrinsics, + ); + return .none; + } + } + + const value = try self.resolveInst(bin_op.rhs); + const elem_abi_size = elem_ty.abiSize(zcu); + + if (elem_abi_size == 1) { + // In this case we can take advantage of LLVM's intrinsic. + const fill_byte = try self.bitCast(value, elem_ty, Type.u8); + const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); + + _ = try self.wip.callMemSet( + dest_ptr, + dest_ptr_align, + fill_byte, + len, + access_kind, + self.disable_intrinsics, + ); + return .none; + } + + // non-byte-sized element. lower with a loop. something like this: + + // entry: + // ... + // %end_ptr = getelementptr %ptr, %len + // br %loop + // loop: + // %it_ptr = phi body %next_ptr, entry %ptr + // %end = cmp eq %it_ptr, %end_ptr + // br %end, %body, %end + // body: + // store %it_ptr, %value + // %next_ptr = getelementptr %it_ptr, 1 + // br %loop + // end: + // ... + const entry_block = self.wip.cursor.block; + const loop_block = try self.wip.block(2, "InlineMemsetLoop"); + const body_block = try self.wip.block(1, "InlineMemsetBody"); + const end_block = try self.wip.block(1, "InlineMemsetEnd"); + + const llvm_usize_ty = try o.lowerType(pt, Type.usize); + const len = switch (ptr_ty.ptrSize(zcu)) { + .slice => try self.wip.extractValue(dest_slice, &.{1}, ""), + .one => try o.builder.intValue(llvm_usize_ty, ptr_ty.childType(zcu).arrayLen(zcu)), + .many, .c => unreachable, + }; + const elem_llvm_ty = try o.lowerType(pt, elem_ty); + const end_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, dest_ptr, &.{len}, ""); + _ = try self.wip.br(loop_block); + + self.wip.cursor = .{ .block = loop_block }; + const it_ptr = try self.wip.phi(.ptr, ""); + const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, ""); + _ = try self.wip.brCond(end, body_block, end_block, .none); + + self.wip.cursor = .{ .block = body_block }; + const elem_abi_align = elem_ty.abiAlignment(zcu); + const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm(); + if (isByRef(elem_ty, zcu)) { + _ = try self.wip.callMemCpy( + it_ptr.toValue(), + it_ptr_align, + value, + elem_abi_align.toLlvm(), + try o.builder.intValue(llvm_usize_ty, elem_abi_size), + access_kind, + self.disable_intrinsics, + ); + } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align); + const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{ + try o.builder.intValue(llvm_usize_ty, 1), + }, ""); + _ = try self.wip.br(loop_block); + + self.wip.cursor = .{ .block = end_block }; + it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip); + return .none; +} + +fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const dest_slice = try self.resolveInst(bin_op.lhs); + const dest_ptr_ty = self.typeOf(bin_op.lhs); + const src_slice = try self.resolveInst(bin_op.rhs); + const src_ptr_ty = self.typeOf(bin_op.rhs); + const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty); + const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty); + const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty); + const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or + dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + + self.maybeMarkAllowZeroAccess(dest_ptr_ty.ptrInfo(zcu)); + self.maybeMarkAllowZeroAccess(src_ptr_ty.ptrInfo(zcu)); + + _ = try self.wip.callMemCpy( + dest_ptr, + dest_ptr_ty.ptrAlignment(zcu).toLlvm(), + src_ptr, + src_ptr_ty.ptrAlignment(zcu).toLlvm(), + len, + access_kind, + self.disable_intrinsics, + ); + return .none; +} + +fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const dest_slice = try self.resolveInst(bin_op.lhs); + const dest_ptr_ty = self.typeOf(bin_op.lhs); + const src_slice = try self.resolveInst(bin_op.rhs); + const src_ptr_ty = self.typeOf(bin_op.rhs); + const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty); + const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty); + const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty); + const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or + dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + + _ = try self.wip.callMemMove( + dest_ptr, + dest_ptr_ty.ptrAlignment(zcu).toLlvm(), + src_ptr, + src_ptr_ty.ptrAlignment(zcu).toLlvm(), + len, + access_kind, + ); + return .none; +} + +fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; + const un_ptr_ty = self.typeOf(bin_op.lhs); + const un_ty = un_ptr_ty.childType(zcu); + const layout = un_ty.unionGetLayout(zcu); + if (layout.tag_size == 0) return .none; + + const access_kind: Builder.MemoryAccessKind = + if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; + + self.maybeMarkAllowZeroAccess(un_ptr_ty.ptrInfo(zcu)); + + const union_ptr = try self.resolveInst(bin_op.lhs); + const new_tag = try self.resolveInst(bin_op.rhs); + const union_ptr_align = un_ptr_ty.ptrAlignment(zcu); + if (layout.payload_size == 0) { + _ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm()); + return .none; + } + const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); + const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(pt, un_ty), union_ptr, tag_index, ""); + const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) { + 0 => union_ptr_align, + else => |off| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))), + }; + _ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm()); + return .none; +} + +fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const un_ty = self.typeOf(ty_op.operand); + const layout = un_ty.unionGetLayout(zcu); + if (layout.tag_size == 0) return .none; + const union_handle = try self.resolveInst(ty_op.operand); + if (isByRef(un_ty, zcu)) { + const llvm_un_ty = try o.lowerType(pt, un_ty); + if (layout.payload_size == 0) + return self.wip.load(.normal, llvm_un_ty, union_handle, .default, ""); + const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); + const tag_field_ptr = try self.wip.gepStruct(llvm_un_ty, union_handle, tag_index, ""); + const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index]; + return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, ""); + } else { + if (layout.payload_size == 0) return union_handle; + const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); + return self.wip.extractValue(union_handle, &.{tag_index}, ""); + } +} + +fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) !Builder.Value { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + const operand_ty = self.typeOf(un_op); + + return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand}); +} + +fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + const operand_ty = self.typeOf(un_op); + + return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand}); +} + +fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value { + const o = self.object; + const pt = self.pt; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const inst_ty = self.typeOfIndex(inst); + const operand_ty = self.typeOf(ty_op.operand); + const operand = try self.resolveInst(ty_op.operand); + + const result = try self.wip.callIntrinsic( + .normal, + .none, + intrinsic, + &.{try o.lowerType(pt, operand_ty)}, + &.{ operand, .false }, + "", + ); + return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); +} + +fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value { + const o = self.object; + const pt = self.pt; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const inst_ty = self.typeOfIndex(inst); + const operand_ty = self.typeOf(ty_op.operand); + const operand = try self.resolveInst(ty_op.operand); + + const result = try self.wip.callIntrinsic( + .normal, + .none, + intrinsic, + &.{try o.lowerType(pt, operand_ty)}, + &.{operand}, + "", + ); + return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); +} + +fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand_ty = self.typeOf(ty_op.operand); + var bits = operand_ty.intInfo(zcu).bits; + assert(bits % 8 == 0); + + const inst_ty = self.typeOfIndex(inst); + var operand = try self.resolveInst(ty_op.operand); + var llvm_operand_ty = try o.lowerType(pt, operand_ty); + + if (bits % 16 == 8) { + // If not an even byte-multiple, we need zero-extend + shift-left 1 byte + // The truncated result at the end will be the correct bswap + const scalar_ty = try o.builder.intType(@intCast(bits + 8)); + if (operand_ty.zigTypeTag(zcu) == .vector) { + const vec_len = operand_ty.vectorLen(zcu); + llvm_operand_ty = try o.builder.vectorType(.normal, vec_len, scalar_ty); + } else llvm_operand_ty = scalar_ty; + + const shift_amt = + try o.builder.splatValue(llvm_operand_ty, try o.builder.intConst(scalar_ty, 8)); + const extended = try self.wip.cast(.zext, operand, llvm_operand_ty, ""); + operand = try self.wip.bin(.shl, extended, shift_amt, ""); + + bits = bits + 8; + } + + const result = + try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, ""); + return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); +} + +fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const operand = try self.resolveInst(ty_op.operand); + const error_set_ty = ty_op.ty.toType(); + + const names = error_set_ty.errorSetNames(zcu); + const valid_block = try self.wip.block(@intCast(names.len), "Valid"); + const invalid_block = try self.wip.block(1, "Invalid"); + const end_block = try self.wip.block(2, "End"); + var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none); + defer wip_switch.finish(&self.wip); + + for (0..names.len) |name_index| { + const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?; + const this_tag_int_value = try o.builder.intConst(try o.errorIntType(pt), err_int); + try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip); + } + self.wip.cursor = .{ .block = valid_block }; + _ = try self.wip.br(end_block); + + self.wip.cursor = .{ .block = invalid_block }; + _ = try self.wip.br(end_block); + + self.wip.cursor = .{ .block = end_block }; + const phi = try self.wip.phi(.i1, ""); + phi.finish(&.{ .true, .false }, &.{ valid_block, invalid_block }, &self.wip); + return phi.toValue(); +} + +fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + const enum_ty = self.typeOf(un_op); + + const llvm_fn = try self.getIsNamedEnumValueFunction(enum_ty); + return self.wip.call( + .normal, + .fastcc, + .none, + llvm_fn.typeOf(&o.builder), + llvm_fn.toValue(&o.builder), + &.{operand}, + "", + ); +} + +fn getIsNamedEnumValueFunction(self: *FuncGen, enum_ty: Type) !Builder.Function.Index { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + + const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern()); + if (gop.found_existing) return gop.value_ptr.*; + errdefer assert(o.named_enum_map.remove(enum_ty.toIntern())); + const function_index = try o.builder.addFunction( + // Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type. + // TODO: change the builder API so we don't need to do this. + try o.builder.fnType(.void, &.{}, .normal), + try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}), + toLlvmAddressSpace(.generic, zcu.getTarget()), + ); + gop.value_ptr.* = function_index; + try o.updateIsNamedEnumValueFunction(pt, enum_ty, function_index); + return function_index; +} + +fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + const enum_ty = self.typeOf(un_op); + + const llvm_fn = try o.getEnumTagNameFunction(pt, enum_ty); + return self.wip.call( + .normal, + .fastcc, + .none, + llvm_fn.typeOf(&o.builder), + llvm_fn.toValue(&o.builder), + &.{operand}, + "", + ); +} + +fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; + const operand = try self.resolveInst(un_op); + const slice_ty = self.typeOfIndex(inst); + const slice_llvm_ty = try o.lowerType(pt, slice_ty); + + // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed. + const extended_operand = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), ""); + + const error_name_table_ptr = try self.getErrorNameTable(); + const error_name_table = + try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, ""); + const error_name_ptr = + try self.wip.gep(.inbounds, slice_llvm_ty, error_name_table, &.{extended_operand}, ""); + return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, ""); +} + +fn airSplat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const scalar = try self.resolveInst(ty_op.operand); + const vector_ty = self.typeOfIndex(inst); + return self.wip.splatVector(try o.lowerType(pt, vector_ty), scalar, ""); +} + +fn airSelect(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const extra = self.air.extraData(Air.Bin, pl_op.payload).data; + const pred = try self.resolveInst(pl_op.operand); + const a = try self.resolveInst(extra.lhs); + const b = try self.resolveInst(extra.rhs); + + return self.wip.select(.normal, pred, a, b, ""); +} + +fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = fg.object; + const pt = fg.pt; + const zcu = pt.zcu; + const gpa = zcu.gpa; + + const unwrapped = fg.air.unwrapShuffleOne(zcu, inst); + + const operand = try fg.resolveInst(unwrapped.operand); + const mask = unwrapped.mask; + const operand_ty = fg.typeOf(unwrapped.operand); + const llvm_operand_ty = try o.lowerType(pt, operand_ty); + const llvm_result_ty = try o.lowerType(pt, unwrapped.result_ty); + const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu)); + const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty); + const llvm_poison_mask_elem = try o.builder.poisonConst(.i32); + const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32); + + // LLVM requires that the two input vectors have the same length, so lowering isn't trivial. + // And, in the words of jacobly0: "llvm sucks at shuffles so we do have to hold its hand at + // least a bit". So, there are two cases here. + // + // If the operand length equals the mask length, we do just the one `shufflevector`, where + // the second operand is a constant vector with comptime-known elements at the right indices + // and poison values elsewhere (in the indices which won't be selected). + // + // Otherwise, we lower to *two* `shufflevector` instructions. The first shuffles the runtime + // operand with an all-poison vector to extract and correctly position all of the runtime + // elements. We also make a constant vector with all of the comptime elements correctly + // positioned. Then, our second instruction selects elements from those "runtime-or-poison" + // and "comptime-or-poison" vectors to compute the result. + + // This buffer is used primarily for the mask constants. + const llvm_elem_buf = try gpa.alloc(Builder.Constant, mask.len); + defer gpa.free(llvm_elem_buf); + + // ...but first, we'll collect all of the comptime-known values. + var any_defined_comptime_value = false; + for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| { + llvm_elem.* = switch (mask_elem.unwrap()) { + .elem => llvm_poison_elem, + .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: { + any_defined_comptime_value = true; + break :elem try o.lowerValue(pt, val); + } else llvm_poison_elem, + }; + } + // This vector is like the result, but runtime elements are replaced with poison. + const comptime_and_poison: Builder.Value = if (any_defined_comptime_value) vec: { + break :vec try o.builder.vectorValue(llvm_result_ty, llvm_elem_buf); + } else try o.builder.poisonValue(llvm_result_ty); + + if (operand_ty.vectorLen(zcu) == mask.len) { + // input length equals mask/output length, so we lower to one instruction + for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| { + llvm_elem.* = switch (mask_elem.unwrap()) { + .elem => |idx| try o.builder.intConst(.i32, idx), + .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: { + break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx); + } else llvm_poison_mask_elem, + }; + } + return fg.wip.shuffleVector( + operand, + comptime_and_poison, + try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf), + "", + ); + } + + for (mask, llvm_elem_buf) |mask_elem, *llvm_elem| { + llvm_elem.* = switch (mask_elem.unwrap()) { + .elem => |idx| try o.builder.intConst(.i32, idx), + .value => llvm_poison_mask_elem, + }; + } + // This vector is like our result, but all comptime-known elements are poison. + const runtime_and_poison = try fg.wip.shuffleVector( + operand, + try o.builder.poisonValue(llvm_operand_ty), + try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf), + "", + ); + + if (!any_defined_comptime_value) { + // `comptime_and_poison` is just poison; a second shuffle would be a nop. + return runtime_and_poison; + } + + // In this second shuffle, the inputs, the mask, and the output all have the same length. + for (mask, llvm_elem_buf, 0..) |mask_elem, *llvm_elem, elem_idx| { + llvm_elem.* = switch (mask_elem.unwrap()) { + .elem => try o.builder.intConst(.i32, elem_idx), + .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: { + break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx); + } else llvm_poison_mask_elem, + }; + } + // Merge the runtime and comptime elements with the mask we just built. + return fg.wip.shuffleVector( + runtime_and_poison, + comptime_and_poison, + try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf), + "", + ); +} + +fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = fg.object; + const pt = fg.pt; + const zcu = pt.zcu; + const gpa = zcu.gpa; + + const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst); + + const mask = unwrapped.mask; + const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu)); + const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32); + const llvm_poison_mask_elem = try o.builder.poisonConst(.i32); + + // This is kind of simpler than in `airShuffleOne`. We extend the shorter vector to the + // length of the longer one with an initial `shufflevector` if necessary, and then do the + // actual computation with a second `shufflevector`. + + const operand_a_len = fg.typeOf(unwrapped.operand_a).vectorLen(zcu); + const operand_b_len = fg.typeOf(unwrapped.operand_b).vectorLen(zcu); + const operand_len: u32 = @max(operand_a_len, operand_b_len); + + // If we need to extend an operand, this is the type that mask will have. + const llvm_operand_mask_ty = try o.builder.vectorType(.normal, operand_len, .i32); + + const llvm_elem_buf = try gpa.alloc(Builder.Constant, @max(mask.len, operand_len)); + defer gpa.free(llvm_elem_buf); + + const operand_a: Builder.Value = extend: { + const raw = try fg.resolveInst(unwrapped.operand_a); + if (operand_a_len == operand_len) break :extend raw; + // Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>` + const mask_elems = llvm_elem_buf[0..operand_len]; + for (mask_elems[0..operand_a_len], 0..) |*llvm_elem, elem_idx| { + llvm_elem.* = try o.builder.intConst(.i32, elem_idx); + } + @memset(mask_elems[operand_a_len..], llvm_poison_mask_elem); + const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_a_len, llvm_elem_ty); + break :extend try fg.wip.shuffleVector( + raw, + try o.builder.poisonValue(llvm_this_operand_ty), + try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems), + "", + ); + }; + const operand_b: Builder.Value = extend: { + const raw = try fg.resolveInst(unwrapped.operand_b); + if (operand_b_len == operand_len) break :extend raw; + // Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>` + const mask_elems = llvm_elem_buf[0..operand_len]; + for (mask_elems[0..operand_b_len], 0..) |*llvm_elem, elem_idx| { + llvm_elem.* = try o.builder.intConst(.i32, elem_idx); + } + @memset(mask_elems[operand_b_len..], llvm_poison_mask_elem); + const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_b_len, llvm_elem_ty); + break :extend try fg.wip.shuffleVector( + raw, + try o.builder.poisonValue(llvm_this_operand_ty), + try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems), + "", + ); + }; + + // `operand_a` and `operand_b` now have the same length (we've extended the shorter one with + // an initial shuffle if necessary). Now for the easy bit. + + const mask_elems = llvm_elem_buf[0..mask.len]; + for (mask, mask_elems) |mask_elem, *llvm_mask_elem| { + llvm_mask_elem.* = switch (mask_elem.unwrap()) { + .a_elem => |idx| try o.builder.intConst(.i32, idx), + .b_elem => |idx| try o.builder.intConst(.i32, operand_len + idx), + .undef => llvm_poison_mask_elem, + }; + } + return fg.wip.shuffleVector( + operand_a, + operand_b, + try o.builder.vectorValue(llvm_mask_ty, mask_elems), + "", + ); +} + +/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result. +/// +/// Equivalent to: +/// reduce: { +/// var i: usize = 0; +/// var accum: T = init; +/// while (i < vec.len) : (i += 1) { +/// accum = llvm_fn(accum, vec[i]); +/// } +/// break :reduce accum; +/// } +/// +fn buildReducedCall( + self: *FuncGen, + llvm_fn: Builder.Function.Index, + operand_vector: Builder.Value, + vector_len: usize, + accum_init: Builder.Value, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const usize_ty = try o.lowerType(pt, Type.usize); + const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len); + const llvm_result_ty = accum_init.typeOfWip(&self.wip); + + // Allocate and initialize our mutable variables + const i_ptr = try self.buildAlloca(usize_ty, .default); + _ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default); + const accum_ptr = try self.buildAlloca(llvm_result_ty, .default); + _ = try self.wip.store(.normal, accum_init, accum_ptr, .default); + + // Setup the loop + const loop = try self.wip.block(2, "ReduceLoop"); + const loop_exit = try self.wip.block(1, "AfterReduce"); + _ = try self.wip.br(loop); + { + self.wip.cursor = .{ .block = loop }; + + // while (i < vec.len) + const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, ""); + const cond = try self.wip.icmp(.ult, i, llvm_vector_len, ""); + const loop_then = try self.wip.block(1, "ReduceLoopThen"); + + _ = try self.wip.brCond(cond, loop_then, loop_exit, .none); + + { + self.wip.cursor = .{ .block = loop_then }; + + // accum = f(accum, vec[i]); + const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, ""); + const element = try self.wip.extractElement(operand_vector, i, ""); + const new_accum = try self.wip.call( + .normal, + .ccc, + .none, + llvm_fn.typeOf(&o.builder), + llvm_fn.toValue(&o.builder), + &.{ accum, element }, + "", + ); + _ = try self.wip.store(.normal, new_accum, accum_ptr, .default); + + // i += 1 + const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), ""); + _ = try self.wip.store(.normal, new_i, i_ptr, .default); + _ = try self.wip.br(loop); + } + } + + self.wip.cursor = .{ .block = loop_exit }; + return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, ""); +} + +fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const target = zcu.getTarget(); + + const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce; + const operand = try self.resolveInst(reduce.operand); + const operand_ty = self.typeOf(reduce.operand); + const llvm_operand_ty = try o.lowerType(pt, operand_ty); + const scalar_ty = self.typeOfIndex(inst); + const llvm_scalar_ty = try o.lowerType(pt, scalar_ty); + + switch (reduce.operation) { + .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { + .And => .@"vector.reduce.and", + .Or => .@"vector.reduce.or", + .Xor => .@"vector.reduce.xor", + else => unreachable, + }, &.{llvm_operand_ty}, &.{operand}, ""), + .Min, .Max => switch (scalar_ty.zigTypeTag(zcu)) { + .int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { + .Min => if (scalar_ty.isSignedInt(zcu)) + .@"vector.reduce.smin" + else + .@"vector.reduce.umin", + .Max => if (scalar_ty.isSignedInt(zcu)) + .@"vector.reduce.smax" + else + .@"vector.reduce.umax", + else => unreachable, + }, &.{llvm_operand_ty}, &.{operand}, ""), + .float => if (intrinsicsAllowed(scalar_ty, target)) + return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) { + .Min => .@"vector.reduce.fmin", + .Max => .@"vector.reduce.fmax", + else => unreachable, + }, &.{llvm_operand_ty}, &.{operand}, ""), + else => unreachable, + }, + .Add, .Mul => switch (scalar_ty.zigTypeTag(zcu)) { + .int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { + .Add => .@"vector.reduce.add", + .Mul => .@"vector.reduce.mul", + else => unreachable, + }, &.{llvm_operand_ty}, &.{operand}, ""), + .float => if (intrinsicsAllowed(scalar_ty, target)) + return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) { + .Add => .@"vector.reduce.fadd", + .Mul => .@"vector.reduce.fmul", + else => unreachable, + }, &.{llvm_operand_ty}, &.{ switch (reduce.operation) { + .Add => try o.builder.fpValue(llvm_scalar_ty, -0.0), + .Mul => try o.builder.fpValue(llvm_scalar_ty, 1.0), + else => unreachable, + }, operand }, ""), + else => unreachable, + }, + } + + // Reduction could not be performed with intrinsics. + // Use a manual loop over a softfloat call instead. + const float_bits = scalar_ty.floatBits(target); + const fn_name = switch (reduce.operation) { + .Min => try o.builder.strtabStringFmt("{s}fmin{s}", .{ + libcFloatPrefix(float_bits), libcFloatSuffix(float_bits), + }), + .Max => try o.builder.strtabStringFmt("{s}fmax{s}", .{ + libcFloatPrefix(float_bits), libcFloatSuffix(float_bits), + }), + .Add => try o.builder.strtabStringFmt("__add{s}f3", .{ + compilerRtFloatAbbrev(float_bits), + }), + .Mul => try o.builder.strtabStringFmt("__mul{s}f3", .{ + compilerRtFloatAbbrev(float_bits), + }), + else => unreachable, + }; + + const libc_fn = + try self.getLibcFunction(fn_name, &.{ llvm_scalar_ty, llvm_scalar_ty }, llvm_scalar_ty); + const init_val = switch (llvm_scalar_ty) { + .i16 => try o.builder.intValue(.i16, @as(i16, @bitCast( + @as(f16, switch (reduce.operation) { + .Min, .Max => std.math.nan(f16), + .Add => -0.0, + .Mul => 1.0, + else => unreachable, + }), + ))), + .i80 => try o.builder.intValue(.i80, @as(i80, @bitCast( + @as(f80, switch (reduce.operation) { + .Min, .Max => std.math.nan(f80), + .Add => -0.0, + .Mul => 1.0, + else => unreachable, + }), + ))), + .i128 => try o.builder.intValue(.i128, @as(i128, @bitCast( + @as(f128, switch (reduce.operation) { + .Min, .Max => std.math.nan(f128), + .Add => -0.0, + .Mul => 1.0, + else => unreachable, + }), + ))), + else => unreachable, + }; + return self.buildReducedCall(libc_fn, operand, operand_ty.vectorLen(zcu), init_val); +} + +fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const result_ty = self.typeOfIndex(inst); + const len: usize = @intCast(result_ty.arrayLen(zcu)); + const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); + const llvm_result_ty = try o.lowerType(pt, result_ty); + + switch (result_ty.zigTypeTag(zcu)) { + .vector => { + var vector = try o.builder.poisonValue(llvm_result_ty); + for (elements, 0..) |elem, i| { + const index_u32 = try o.builder.intValue(.i32, i); + const llvm_elem = try self.resolveInst(elem); + vector = try self.wip.insertElement(vector, llvm_elem, index_u32, ""); + } + return vector; + }, + .@"struct" => { + if (zcu.typeToPackedStruct(result_ty)) |struct_type| { + const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type); + const big_bits = backing_int_ty.bitSize(zcu); + const int_ty = try o.builder.intType(@intCast(big_bits)); + comptime assert(Type.packed_struct_layout_version == 2); + var running_int = try o.builder.intValue(int_ty, 0); + var running_bits: u16 = 0; + for (elements, struct_type.field_types.get(ip)) |elem, field_ty| { + if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue; + + const non_int_val = try self.resolveInst(elem); + const ty_bit_size: u16 = @intCast(Type.fromInterned(field_ty).bitSize(zcu)); + const small_int_ty = try o.builder.intType(ty_bit_size); + const small_int_val = if (Type.fromInterned(field_ty).isPtrAtRuntime(zcu)) + try self.wip.cast(.ptrtoint, non_int_val, small_int_ty, "") + else + try self.wip.cast(.bitcast, non_int_val, small_int_ty, ""); + const shift_rhs = try o.builder.intValue(int_ty, running_bits); + const extended_int_val = + try self.wip.conv(.unsigned, small_int_val, int_ty, ""); + const shifted = try self.wip.bin(.shl, extended_int_val, shift_rhs, ""); + running_int = try self.wip.bin(.@"or", running_int, shifted, ""); + running_bits += ty_bit_size; + } + return running_int; + } + + assert(result_ty.containerLayout(zcu) != .@"packed"); + + if (isByRef(result_ty, zcu)) { + // TODO in debug builds init to undef so that the padding will be 0xaa + // even if we fully populate the fields. + const alignment = result_ty.abiAlignment(zcu).toLlvm(); + const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment); + + for (elements, 0..) |elem, i| { + if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue; + + const llvm_elem = try self.resolveInst(elem); + const llvm_i = o.llvmFieldIndex(result_ty, i).?; + const field_ptr = try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, ""); + + const field_ptr_ty = try pt.ptrType(.{ + .child = self.typeOf(elem).toIntern(), + .flags = .{ + .alignment = result_ty.explicitFieldAlignment(i, zcu), + }, + }); + try self.store(field_ptr, field_ptr_ty, llvm_elem, .none); + } + + return alloca_inst; + } else { + var result = try o.builder.poisonValue(llvm_result_ty); + for (elements, 0..) |elem, i| { + if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue; + + const llvm_elem = try self.resolveInst(elem); + const llvm_i = o.llvmFieldIndex(result_ty, i).?; + result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, ""); + } + return result; + } + }, + .array => { + assert(isByRef(result_ty, zcu)); + + const llvm_usize = try o.lowerType(pt, Type.usize); + const usize_zero = try o.builder.intValue(llvm_usize, 0); + const alignment = result_ty.abiAlignment(zcu).toLlvm(); + const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment); + + const array_info = result_ty.arrayInfo(zcu); + const elem_ptr_ty = try pt.ptrType(.{ + .child = array_info.elem_type.toIntern(), + }); + + for (elements, 0..) |elem, i| { + const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{ + usize_zero, try o.builder.intValue(llvm_usize, i), + }, ""); + const llvm_elem = try self.resolveInst(elem); + try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none); + } + if (array_info.sentinel) |sent_val| { + const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{ + usize_zero, try o.builder.intValue(llvm_usize, array_info.len), + }, ""); + const llvm_elem = try self.resolveValue(sent_val); + try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none); + } + + return alloca_inst; + }, + else => unreachable, + } +} + +fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; + const union_ty = self.typeOfIndex(inst); + const union_llvm_ty = try o.lowerType(pt, union_ty); + const union_obj = zcu.typeToUnion(union_ty).?; + + assert(union_obj.layout != .@"packed"); + + const layout = Type.getUnionLayout(union_obj, zcu); + + const tag_int_val = blk: { + const tag_ty = union_ty.unionTagTypeHypothetical(zcu); + const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index); + break :blk tag_val.intFromEnum(zcu); + }; + if (layout.payload_size == 0) { + if (layout.tag_size == 0) { + return .none; + } + assert(!isByRef(union_ty, zcu)); + var big_int_space: Value.BigIntSpace = undefined; + const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu); + return try o.builder.bigIntValue(union_llvm_ty, tag_big_int); + } + assert(isByRef(union_ty, zcu)); + // The llvm type of the alloca will be the named LLVM union type, and will not + // necessarily match the format that we need, depending on which tag is active. + // We must construct the correct unnamed struct type here, in order to then set + // the fields appropriately. + const alignment = layout.abi_align.toLlvm(); + const result_ptr = try self.buildAlloca(union_llvm_ty, alignment); + const llvm_payload = try self.resolveInst(extra.init); + const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]); + const field_llvm_ty = try o.lowerType(pt, field_ty); + const field_size = field_ty.abiSize(zcu); + const field_align = union_ty.explicitFieldAlignment(extra.field_index, zcu); + const llvm_usize = try o.lowerType(pt, Type.usize); + const usize_zero = try o.builder.intValue(llvm_usize, 0); + + assert(field_ty.hasRuntimeBits(zcu)); + + const llvm_union_ty = t: { + const payload_ty = p: { + if (field_size == layout.payload_size) { + break :p field_llvm_ty; + } + const padding_len = layout.payload_size - field_size; + break :p try o.builder.structType(.@"packed", &.{ + field_llvm_ty, try o.builder.arrayType(padding_len, .i8), + }); + }; + if (layout.tag_size == 0) break :t try o.builder.structType(.normal, &.{payload_ty}); + const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); + var fields: [3]Builder.Type = undefined; + var fields_len: usize = 2; + if (layout.tag_align.compare(.gte, layout.payload_align)) { + fields = .{ tag_ty, payload_ty, undefined }; + } else { + fields = .{ payload_ty, tag_ty, undefined }; + } + if (layout.padding != 0) { + fields[fields_len] = try o.builder.arrayType(layout.padding, .i8); + fields_len += 1; + } + break :t try o.builder.structType(.normal, fields[0..fields_len]); + }; + + // Now we follow the layout as expressed above with GEP instructions to set the + // tag and the payload. + const field_ptr_ty = try pt.ptrType(.{ + .child = field_ty.toIntern(), + .flags = .{ .alignment = field_align }, + }); + if (layout.tag_size == 0) { + const indices = [3]Builder.Value{ usize_zero, .@"0", .@"0" }; + const len: usize = if (field_size == layout.payload_size) 2 else 3; + const field_ptr = + try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], ""); + try self.store(field_ptr, field_ptr_ty, llvm_payload, .none); + return result_ptr; + } + + { + const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align)); + const indices: [3]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, payload_index), .@"0" }; + const len: usize = if (field_size == layout.payload_size) 2 else 3; + const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], ""); + try self.store(field_ptr, field_ptr_ty, llvm_payload, .none); + } + { + const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); + const indices: [2]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, tag_index) }; + const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, &indices, ""); + const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); + var big_int_space: Value.BigIntSpace = undefined; + const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu); + const llvm_tag = try o.builder.bigIntValue(tag_ty, tag_big_int); + const tag_alignment = Type.fromInterned(union_obj.enum_tag_type).abiAlignment(zcu).toLlvm(); + _ = try self.wip.store(.normal, llvm_tag, field_ptr, tag_alignment); + } + + return result_ptr; +} + +fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch; + + comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.read) == 0); + comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.write) == 1); + + comptime assert(prefetch.locality >= 0); + comptime assert(prefetch.locality <= 3); + + comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.instruction) == 0); + comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.data) == 1); + + // LLVM fails during codegen of instruction cache prefetchs for these architectures. + // This is an LLVM bug as the prefetch intrinsic should be a noop if not supported + // by the target. + // To work around this, don't emit llvm.prefetch in this case. + // See https://bugs.llvm.org/show_bug.cgi?id=21037 + const zcu = self.pt.zcu; + const target = zcu.getTarget(); + switch (prefetch.cache) { + .instruction => switch (target.cpu.arch) { + .x86_64, + .x86, + .powerpc, + .powerpcle, + .powerpc64, + .powerpc64le, + => return .none, + .arm, .armeb, .thumb, .thumbeb => { + switch (prefetch.rw) { + .write => return .none, + else => {}, + } + }, + else => {}, + }, + .data => {}, + } + + _ = try self.wip.callIntrinsic(.normal, .none, .prefetch, &.{.ptr}, &.{ + try self.sliceOrArrayPtr(try self.resolveInst(prefetch.ptr), self.typeOf(prefetch.ptr)), + try o.builder.intValue(.i32, prefetch.rw), + try o.builder.intValue(.i32, prefetch.locality), + try o.builder.intValue(.i32, prefetch.cache), + }, ""); + return .none; +} + +fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; + const inst_ty = self.typeOfIndex(inst); + const operand = try self.resolveInst(ty_op.operand); + + return self.wip.cast(.addrspacecast, operand, try o.lowerType(pt, inst_ty), ""); +} + +fn workIntrinsic( + self: *FuncGen, + dimension: u32, + default: u32, + comptime basename: []const u8, +) !Builder.Value { + return self.wip.callIntrinsic(.normal, .none, switch (dimension) { + 0 => @field(Builder.Intrinsic, basename ++ ".x"), + 1 => @field(Builder.Intrinsic, basename ++ ".y"), + 2 => @field(Builder.Intrinsic, basename ++ ".z"), + else => return self.object.builder.intValue(.i32, default), + }, &.{}, &.{}, ""); +} + +fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const target = self.pt.zcu.getTarget(); + + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const dimension = pl_op.payload; + + return switch (target.cpu.arch) { + .amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workitem.id"), + .nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.tid"), + else => unreachable, + }; +} + +fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const o = self.object; + const pt = self.pt; + const target = pt.zcu.getTarget(); + + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const dimension = pl_op.payload; + + switch (target.cpu.arch) { + .amdgcn => { + if (dimension >= 3) return .@"1"; + + // Fetch the dispatch pointer, which points to this structure: + // https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/adae6c61e10d371f7cbc3d0e94ae2c070cab18a4/src/inc/hsa.h#L2913 + const dispatch_ptr = + try self.wip.callIntrinsic(.normal, .none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, ""); + + // Load the work_group_* member from the struct as u16. + // Just treat the dispatch pointer as an array of u16 to keep things simple. + const workgroup_size_ptr = try self.wip.gep(.inbounds, .i16, dispatch_ptr, &.{ + try o.builder.intValue(try o.lowerType(pt, Type.usize), 2 + dimension), + }, ""); + const workgroup_size_alignment = comptime Builder.Alignment.fromByteUnits(2); + return self.wip.load(.normal, .i16, workgroup_size_ptr, workgroup_size_alignment, ""); + }, + .nvptx, .nvptx64 => { + return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid"); + }, + else => unreachable, + } +} + +fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { + const target = self.pt.zcu.getTarget(); + + const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; + const dimension = pl_op.payload; + + return switch (target.cpu.arch) { + .amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workgroup.id"), + .nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.ctaid"), + else => unreachable, + }; +} + +fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index { + const o = self.object; + const pt = self.pt; + + const table = o.error_name_table; + if (table != .none) return table; + + // TODO: Address space + const variable_index = + try o.builder.addVariable(try o.builder.strtabString("__zig_err_name_table"), .ptr, .default); + variable_index.setLinkage(.private, &o.builder); + variable_index.setMutability(.constant, &o.builder); + variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); + variable_index.setAlignment( + Type.slice_const_u8_sentinel_0.abiAlignment(pt.zcu).toLlvm(), + &o.builder, + ); + + o.error_name_table = variable_index; + return variable_index; +} + +/// Assumes the optional is not pointer-like and payload has bits. +fn optCmpNull( + self: *FuncGen, + cond: Builder.IntegerCondition, + opt_llvm_ty: Builder.Type, + opt_handle: Builder.Value, + is_by_ref: bool, + access_kind: Builder.MemoryAccessKind, +) Allocator.Error!Builder.Value { + const o = self.object; + const field = b: { + if (is_by_ref) { + const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, ""); + break :b try self.wip.load(access_kind, .i8, field_ptr, .default, ""); + } + break :b try self.wip.extractValue(opt_handle, &.{1}, ""); + }; + comptime assert(optional_layout_version == 3); + + return self.wip.icmp(cond, field, try o.builder.intValue(.i8, 0), ""); +} + +/// Assumes the optional is not pointer-like and payload has bits. +fn optPayloadHandle( + fg: *FuncGen, + opt_llvm_ty: Builder.Type, + opt_handle: Builder.Value, + opt_ty: Type, + can_elide_load: bool, +) !Builder.Value { + const pt = fg.pt; + const zcu = pt.zcu; + const payload_ty = opt_ty.optionalChild(zcu); + + if (isByRef(opt_ty, zcu)) { + // We have a pointer and we need to return a pointer to the first field. + const payload_ptr = try fg.wip.gepStruct(opt_llvm_ty, opt_handle, 0, ""); + + const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm(); + if (isByRef(payload_ty, zcu)) { + if (can_elide_load) + return payload_ptr; + + return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); + } + return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_alignment); + } + + assert(!isByRef(payload_ty, zcu)); + return fg.wip.extractValue(opt_handle, &.{0}, ""); +} + +fn buildOptional( + self: *FuncGen, + optional_ty: Type, + payload: Builder.Value, + non_null_bit: Builder.Value, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const optional_llvm_ty = try o.lowerType(pt, optional_ty); + const non_null_field = try self.wip.cast(.zext, non_null_bit, .i8, ""); + + if (isByRef(optional_ty, zcu)) { + const payload_alignment = optional_ty.abiAlignment(pt.zcu).toLlvm(); + const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment); + + { + const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, ""); + _ = try self.wip.store(.normal, payload, field_ptr, payload_alignment); + } + { + const non_null_alignment = comptime Builder.Alignment.fromByteUnits(1); + const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 1, ""); + _ = try self.wip.store(.normal, non_null_field, field_ptr, non_null_alignment); + } + + return alloca_inst; + } + + return self.wip.buildAggregate(optional_llvm_ty, &.{ payload, non_null_field }, ""); +} + +fn fieldPtr( + self: *FuncGen, + aggregate_ptr: Builder.Value, + aggregate_ptr_ty: Type, + field_index: u32, +) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const aggregate_ty = aggregate_ptr_ty.childType(zcu); + if (aggregate_ty.containerLayout(zcu) == .@"packed") { + // A pointer to a bitpack field is equivalent to a pointer to the whole bitpack; the + // bit offset is represented in the pointer *type*. + return aggregate_ptr; + } + switch (aggregate_ty.zigTypeTag(zcu)) { + .@"struct" => { + if (!aggregate_ty.hasRuntimeBits(zcu)) { + return aggregate_ptr; + } + const struct_llvm_ty = try o.lowerType(pt, aggregate_ty); + if (o.llvmFieldIndex(aggregate_ty, field_index)) |llvm_field_index| { + return self.wip.gepStruct(struct_llvm_ty, aggregate_ptr, llvm_field_index, ""); + } else { + // If we found no index then this means this is a zero sized field at the + // end of the struct. Treat our struct pointer as an array of two and get + // the index to the element at index `1` to get a pointer to the end of + // the struct. + const llvm_index = try o.builder.intValue( + try o.lowerType(pt, Type.usize), + @intFromBool(aggregate_ty.hasRuntimeBits(zcu)), + ); + return self.wip.gep(.inbounds, struct_llvm_ty, aggregate_ptr, &.{llvm_index}, ""); + } + }, + .@"union" => { + const layout = aggregate_ty.unionGetLayout(zcu); + if (layout.payload_size == 0) return aggregate_ptr; + const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)); + const union_llvm_ty = try o.lowerType(pt, aggregate_ty); + return self.wip.gepStruct(union_llvm_ty, aggregate_ptr, payload_index, ""); + }, + else => unreachable, + } +} + +/// Load a value and, if needed, mask out padding bits for non byte-sized integer values. +fn loadTruncate( + fg: *FuncGen, + access_kind: Builder.MemoryAccessKind, + payload_ty: Type, + payload_ptr: Builder.Value, + payload_alignment: Builder.Alignment, +) !Builder.Value { + // from https://llvm.org/docs/LangRef.html#load-instruction : + // "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. " + // => so load the byte aligned value and trunc the unwanted bits. + + const o = fg.object; + const pt = fg.pt; + const zcu = pt.zcu; + const payload_llvm_ty = try o.lowerType(pt, payload_ty); + const abi_size = payload_ty.abiSize(zcu); + + const load_llvm_ty = if (payload_ty.isAbiInt(zcu)) + try o.builder.intType(@intCast(abi_size * 8)) + else + payload_llvm_ty; + const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, ""); + const shifted = if (payload_llvm_ty != load_llvm_ty and o.target.cpu.arch.endian() == .big) + try fg.wip.bin(.lshr, loaded, try o.builder.intValue( + load_llvm_ty, + (payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8, + ), "") + else + loaded; + + return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, ""); +} + +/// Load a by-ref type by constructing a new alloca and performing a memcpy. +fn loadByRef( + fg: *FuncGen, + ptr: Builder.Value, + pointee_type: Type, + ptr_alignment: Builder.Alignment, + access_kind: Builder.MemoryAccessKind, +) !Builder.Value { + const o = fg.object; + const pt = fg.pt; + const pointee_llvm_ty = try o.lowerType(pt, pointee_type); + const result_align = InternPool.Alignment.fromLlvm(ptr_alignment) + .max(pointee_type.abiAlignment(pt.zcu)).toLlvm(); + const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align); + const size_bytes = pointee_type.abiSize(pt.zcu); + _ = try fg.wip.callMemCpy( + result_ptr, + result_align, + ptr, + ptr_alignment, + try o.builder.intValue(try o.lowerType(pt, Type.usize), size_bytes), + access_kind, + fg.disable_intrinsics, + ); + return result_ptr; +} + +/// This function always performs a copy. For isByRef=true types, it creates a new +/// alloca and copies the value into it, then returns the alloca instruction. +/// For isByRef=false types, it creates a load instruction and returns it. +fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) !Builder.Value { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const info = ptr_ty.ptrInfo(zcu); + const elem_ty = Type.fromInterned(info.child); + if (!elem_ty.hasRuntimeBits(zcu)) return .none; + + const ptr_alignment = (if (info.flags.alignment != .none) + @as(InternPool.Alignment, info.flags.alignment) + else + elem_ty.abiAlignment(zcu)).toLlvm(); + + const access_kind: Builder.MemoryAccessKind = + if (info.flags.is_volatile) .@"volatile" else .normal; + + if (info.flags.vector_index != .none) { + const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); + const vec_elem_ty = try o.lowerType(pt, elem_ty); + const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); + + const loaded_vector = try self.wip.load(access_kind, vec_ty, ptr, ptr_alignment, ""); + return self.wip.extractElement(loaded_vector, index_u32, ""); + } + + if (info.packed_offset.host_size == 0) { + if (isByRef(elem_ty, zcu)) { + return self.loadByRef(ptr, elem_ty, ptr_alignment, access_kind); + } + return self.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment); + } + + const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8)); + const containing_int = + try self.wip.load(access_kind, containing_int_ty, ptr, ptr_alignment, ""); + + const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); + const shift_amt = try o.builder.intValue(containing_int_ty, info.packed_offset.bit_offset); + const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, ""); + const elem_llvm_ty = try o.lowerType(pt, elem_ty); + + if (isByRef(elem_ty, zcu)) { + const result_align = elem_ty.abiAlignment(zcu).toLlvm(); + const result_ptr = try self.buildAlloca(elem_llvm_ty, result_align); + + const same_size_int = try o.builder.intType(@intCast(elem_bits)); + const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); + _ = try self.wip.store(.normal, truncated_int, result_ptr, result_align); + return result_ptr; + } + + if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) { + const same_size_int = try o.builder.intType(@intCast(elem_bits)); + const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); + return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); + } + + if (elem_ty.isPtrAtRuntime(zcu)) { + const same_size_int = try o.builder.intType(@intCast(elem_bits)); + const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); + return self.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, ""); + } + + return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, ""); +} + +fn store( + self: *FuncGen, + ptr: Builder.Value, + ptr_ty: Type, + elem: Builder.Value, + ordering: Builder.AtomicOrdering, +) !void { + const o = self.object; + const pt = self.pt; + const zcu = pt.zcu; + const info = ptr_ty.ptrInfo(zcu); + const elem_ty = Type.fromInterned(info.child); + if (!elem_ty.hasRuntimeBits(zcu)) { + return; + } + const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); + const access_kind: Builder.MemoryAccessKind = + if (info.flags.is_volatile) .@"volatile" else .normal; + + if (info.flags.vector_index != .none) { + const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); + const vec_elem_ty = try o.lowerType(pt, elem_ty); + const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); + + const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, ""); + + const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, ""); + + assert(ordering == .none); + _ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment); + return; + } + + if (info.packed_offset.host_size != 0) { + const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8)); + assert(ordering == .none); + const containing_int = + try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, ""); + const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); + const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset); + // Convert to equally-sized integer type in order to perform the bit + // operations on the value to store + const value_bits_type = try o.builder.intType(@intCast(elem_bits)); + const value_bits = if (elem_ty.isPtrAtRuntime(zcu)) + try self.wip.cast(.ptrtoint, elem, value_bits_type, "") + else + try self.wip.cast(.bitcast, elem, value_bits_type, ""); + + const mask_val = blk: { + const zext = try self.wip.cast( + .zext, + try o.builder.intValue(value_bits_type, -1), + containing_int_ty, + "", + ); + const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), ""); + break :blk try self.wip.bin( + .xor, + shl, + try o.builder.intValue(containing_int_ty, -1), + "", + ); + }; + + const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, ""); + const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, ""); + const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), ""); + const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, ""); + + assert(ordering == .none); + _ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment); + return; + } + if (!isByRef(elem_ty, zcu)) { + _ = try self.wip.storeAtomic( + access_kind, + elem, + ptr, + self.sync_scope, + ordering, + ptr_alignment, + ); + return; + } + assert(ordering == .none); + _ = try self.wip.callMemCpy( + ptr, + ptr_alignment, + elem, + elem_ty.abiAlignment(zcu).toLlvm(), + try o.builder.intValue(try o.lowerType(pt, Type.usize), elem_ty.abiSize(zcu)), + access_kind, + self.disable_intrinsics, + ); +} + +fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void { + const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545; + const o = fg.object; + const pt = fg.pt; + const usize_ty = try o.lowerType(pt, Type.usize); + const zero = try o.builder.intValue(usize_ty, 0); + const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED); + const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, ""); + _ = try valgrindClientRequest(fg, zero, req, ptr_as_usize, len, zero, zero, zero); +} + +fn valgrindClientRequest( + fg: *FuncGen, + default_value: Builder.Value, + request: Builder.Value, + a1: Builder.Value, + a2: Builder.Value, + a3: Builder.Value, + a4: Builder.Value, + a5: Builder.Value, +) Allocator.Error!Builder.Value { + const o = fg.object; + const pt = fg.pt; + const zcu = pt.zcu; + const target = zcu.getTarget(); + if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value; + + const llvm_usize = try o.lowerType(pt, Type.usize); + const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm(); + + const array_llvm_ty = try o.builder.arrayType(6, llvm_usize); + const array_ptr = if (fg.valgrind_client_request_array == .none) a: { + const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment); + fg.valgrind_client_request_array = array_ptr; + break :a array_ptr; + } else fg.valgrind_client_request_array; + const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 }; + const zero = try o.builder.intValue(llvm_usize, 0); + for (array_elements, 0..) |elem, i| { + const elem_ptr = try fg.wip.gep(.inbounds, array_llvm_ty, array_ptr, &.{ + zero, try o.builder.intValue(llvm_usize, i), + }, ""); + _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment); + } + + const arch_specific: struct { + template: [:0]const u8, + constraints: [:0]const u8, + } = switch (target.cpu.arch) { + .arm, .armeb, .thumb, .thumbeb => .{ + .template = + \\ mov r12, r12, ror #3 ; mov r12, r12, ror #13 + \\ mov r12, r12, ror #29 ; mov r12, r12, ror #19 + \\ orr r10, r10, r10 + , + .constraints = "={r3},{r4},{r3},~{cc},~{memory}", + }, + .aarch64, .aarch64_be => .{ + .template = + \\ ror x12, x12, #3 ; ror x12, x12, #13 + \\ ror x12, x12, #51 ; ror x12, x12, #61 + \\ orr x10, x10, x10 + , + .constraints = "={x3},{x4},{x3},~{cc},~{memory}", + }, + .mips, .mipsel => .{ + .template = + \\ srl $$0, $$0, 13 + \\ srl $$0, $$0, 29 + \\ srl $$0, $$0, 3 + \\ srl $$0, $$0, 19 + \\ or $$13, $$13, $$13 + , + .constraints = "={$11},{$12},{$11},~{memory},~{$1}", + }, + .mips64, .mips64el => .{ + .template = + \\ dsll $$0, $$0, 3 ; dsll $$0, $$0, 13 + \\ dsll $$0, $$0, 29 ; dsll $$0, $$0, 19 + \\ or $$13, $$13, $$13 + , + .constraints = "={$11},{$12},{$11},~{memory},~{$1}", + }, + .powerpc, .powerpcle => .{ + .template = + \\ rlwinm 0, 0, 3, 0, 31 ; rlwinm 0, 0, 13, 0, 31 + \\ rlwinm 0, 0, 29, 0, 31 ; rlwinm 0, 0, 19, 0, 31 + \\ or 1, 1, 1 + , + .constraints = "={r3},{r4},{r3},~{cc},~{memory}", + }, + .powerpc64, .powerpc64le => .{ + .template = + \\ rotldi 0, 0, 3 ; rotldi 0, 0, 13 + \\ rotldi 0, 0, 61 ; rotldi 0, 0, 51 + \\ or 1, 1, 1 + , + .constraints = "={r3},{r4},{r3},~{cc},~{memory}", + }, + .riscv64 => .{ + .template = + \\ .option push + \\ .option norvc + \\ srli zero, zero, 3 + \\ srli zero, zero, 13 + \\ srli zero, zero, 51 + \\ srli zero, zero, 61 + \\ or a0, a0, a0 + \\ .option pop + , + .constraints = "={a3},{a4},{a3},~{cc},~{memory}", + }, + .s390x => .{ + .template = + \\ lr %r15, %r15 + \\ lr %r1, %r1 + \\ lr %r2, %r2 + \\ lr %r3, %r3 + \\ lr %r2, %r2 + , + .constraints = "={r3},{r2},{r3},~{cc},~{memory}", + }, + .x86 => .{ + .template = + \\ roll $$3, %edi ; roll $$13, %edi + \\ roll $$61, %edi ; roll $$51, %edi + \\ xchgl %ebx, %ebx + , + .constraints = "={edx},{eax},{edx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}", + }, + .x86_64 => .{ + .template = + \\ rolq $$3, %rdi ; rolq $$13, %rdi + \\ rolq $$61, %rdi ; rolq $$51, %rdi + \\ xchgq %rbx, %rbx + , + .constraints = "={rdx},{rax},{rdx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}", + }, + else => unreachable, + }; + + return fg.wip.callAsm( + .none, + try o.builder.fnType(llvm_usize, &.{ llvm_usize, llvm_usize }, .normal), + .{ .sideeffect = true }, + try o.builder.string(arch_specific.template), + try o.builder.string(arch_specific.constraints), + &.{ try fg.wip.cast(.ptrtoint, array_ptr, llvm_usize, ""), default_value }, + "", + ); +} + +fn typeOf(fg: *FuncGen, inst: Air.Inst.Ref) Type { + const zcu = fg.pt.zcu; + return fg.air.typeOf(inst, &zcu.intern_pool); +} + +fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type { + const zcu = fg.pt.zcu; + return fg.air.typeOfIndex(inst, &zcu.intern_pool); +} + +const ParamTypeIterator = struct { + object: *Object, + pt: Zcu.PerThread, + fn_info: InternPool.Key.FuncType, + zig_index: u32, + llvm_index: u32, + types_len: u32, + types_buffer: [8]Builder.Type, + byval_attr: bool, + + const Lowering = union(enum) { + no_bits, + byval, + byref, + byref_mut, + abi_sized_int, + multiple_llvm_types, + slice, + float_array: u8, + i32_array: u8, + i64_array: u8, + }; + + pub fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering { + if (it.zig_index >= it.fn_info.param_types.len) return null; + const ip = &it.pt.zcu.intern_pool; + const ty = it.fn_info.param_types.get(ip)[it.zig_index]; + it.byval_attr = false; + return nextInner(it, Type.fromInterned(ty)); + } + + /// `airCall` uses this instead of `next` so that it can take into account variadic functions. + fn nextCall(it: *ParamTypeIterator, fg: *FuncGen, args: []const Air.Inst.Ref) Allocator.Error!?Lowering { + assert(std.meta.eql(it.pt, fg.pt)); + const ip = &it.pt.zcu.intern_pool; + if (it.zig_index >= it.fn_info.param_types.len) { + if (it.zig_index >= args.len) { + return null; + } else { + return nextInner(it, fg.typeOf(args[it.zig_index])); + } + } else { + return nextInner(it, Type.fromInterned(it.fn_info.param_types.get(ip)[it.zig_index])); + } + } + + fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering { + const pt = it.pt; + const zcu = pt.zcu; + const target = zcu.getTarget(); + + if (!ty.hasRuntimeBits(zcu)) { + it.zig_index += 1; + return .no_bits; + } + switch (it.fn_info.cc) { + .@"inline" => unreachable, + .auto => { + it.zig_index += 1; + it.llvm_index += 1; + if (ty.isSlice(zcu) or + (ty.zigTypeTag(zcu) == .optional and ty.optionalChild(zcu).isSlice(zcu) and !ty.ptrAllowsZero(zcu))) + { + it.llvm_index += 1; + return .slice; + } else if (isByRef(ty, zcu)) { + return .byref; + } else if (target.cpu.arch.isX86() and + !target.cpu.has(.x86, .evex512) and + ty.totalVectorBits(zcu) >= 512) + { + // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns + // "512-bit vector arguments require 'evex512' for AVX512" + return .byref; + } else { + return .byval; + } + }, + .async => { + @panic("TODO implement async function lowering in the LLVM backend"); + }, + .x86_64_sysv => return it.nextSystemV(ty), + .x86_64_win => return it.nextWin64(ty), + .x86_stdcall => { + it.zig_index += 1; + it.llvm_index += 1; + + if (isScalar(zcu, ty)) { + return .byval; + } else { + it.byval_attr = true; + return .byref; + } + }, + .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => { + it.zig_index += 1; + it.llvm_index += 1; + switch (aarch64_c_abi.classifyType(ty, zcu)) { + .memory => return .byref_mut, + .float_array => |len| return Lowering{ .float_array = len }, + .byval => return .byval, + .integer => { + it.types_len = 1; + it.types_buffer[0] = .i64; + return .multiple_llvm_types; + }, + .double_integer => return Lowering{ .i64_array = 2 }, + } + }, + .arm_aapcs, .arm_aapcs_vfp => { + it.zig_index += 1; + it.llvm_index += 1; + switch (arm_c_abi.classifyType(ty, zcu, .arg)) { + .memory => { + it.byval_attr = true; + return .byref; + }, + .byval => return .byval, + .i32_array => |size| return Lowering{ .i32_array = size }, + .i64_array => |size| return Lowering{ .i64_array = size }, + } + }, + .mips_o32 => { + it.zig_index += 1; + it.llvm_index += 1; + switch (mips_c_abi.classifyType(ty, zcu, .arg)) { + .memory => { + it.byval_attr = true; + return .byref; + }, + .byval => return .byval, + .i32_array => |size| return Lowering{ .i32_array = size }, + } + }, + .riscv64_lp64, .riscv32_ilp32 => { + it.zig_index += 1; + it.llvm_index += 1; + switch (riscv_c_abi.classifyType(ty, zcu)) { + .memory => return .byref_mut, + .byval => return .byval, + .integer => return .abi_sized_int, + .double_integer => return Lowering{ .i64_array = 2 }, + .fields => { + it.types_len = 0; + for (0..ty.structFieldCount(zcu)) |field_index| { + const field_ty = ty.fieldType(field_index, zcu); + if (!field_ty.hasRuntimeBits(zcu)) continue; + it.types_buffer[it.types_len] = try it.object.lowerType(pt, field_ty); + it.types_len += 1; + } + it.llvm_index += it.types_len - 1; + return .multiple_llvm_types; + }, + } + }, + .wasm_mvp => switch (wasm_c_abi.classifyType(ty, zcu)) { + .direct => |scalar_ty| { + if (isScalar(zcu, ty)) { + it.zig_index += 1; + it.llvm_index += 1; + return .byval; + } else { + var types_buffer: [8]Builder.Type = undefined; + types_buffer[0] = try it.object.lowerType(pt, scalar_ty); + it.types_buffer = types_buffer; + it.types_len = 1; + it.llvm_index += 1; + it.zig_index += 1; + return .multiple_llvm_types; + } + }, + .indirect => { + it.zig_index += 1; + it.llvm_index += 1; + it.byval_attr = true; + return .byref; + }, + }, + // TODO investigate other callconvs + else => { + it.zig_index += 1; + it.llvm_index += 1; + return .byval; + }, + } + } + + fn nextWin64(it: *ParamTypeIterator, ty: Type) ?Lowering { + const zcu = it.pt.zcu; + switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) { + .integer => { + if (isScalar(zcu, ty)) { + it.zig_index += 1; + it.llvm_index += 1; + return .byval; + } else { + it.zig_index += 1; + it.llvm_index += 1; + return .abi_sized_int; + } + }, + .win_i128 => { + it.zig_index += 1; + it.llvm_index += 1; + return .byref; + }, + .memory => { + it.zig_index += 1; + it.llvm_index += 1; + return .byref_mut; + }, + .sse => { + it.zig_index += 1; + it.llvm_index += 1; + return .byval; + }, + else => unreachable, + } + } + + fn nextSystemV(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering { + const zcu = it.pt.zcu; + const ip = &zcu.intern_pool; + ty.assertHasLayout(zcu); + const classes = x86_64_abi.classifySystemV(ty, zcu, zcu.getTarget(), .arg); + if (classes[0] == .memory) { + it.zig_index += 1; + it.llvm_index += 1; + it.byval_attr = true; + return .byref; + } + if (isScalar(zcu, ty)) { + it.zig_index += 1; + it.llvm_index += 1; + return .byval; + } + var types_index: u32 = 0; + var types_buffer: [8]Builder.Type = undefined; + for (classes) |class| { + switch (class) { + .integer => { + types_buffer[types_index] = .i64; + types_index += 1; + }, + .sse => { + types_buffer[types_index] = .double; + types_index += 1; + }, + .sseup => { + if (types_buffer[types_index - 1] == .double) { + types_buffer[types_index - 1] = .fp128; + } else { + types_buffer[types_index] = .double; + types_index += 1; + } + }, + .float => { + types_buffer[types_index] = .float; + types_index += 1; + }, + .float_combine => { + types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float); + types_index += 1; + }, + .x87 => { + it.zig_index += 1; + it.llvm_index += 1; + it.byval_attr = true; + return .byref; + }, + .x87up => unreachable, + .none => break, + .memory => unreachable, // handled above + .win_i128 => unreachable, // windows only + .integer_per_element => { + @panic("TODO"); + }, + } + } + const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer}); + if (first_non_integer == null or classes[first_non_integer.?] == .none) { + assert(first_non_integer orelse classes.len == types_index); + if (types_index == 1) { + it.zig_index += 1; + it.llvm_index += 1; + return .abi_sized_int; + } + if (it.llvm_index + types_index > 6) { + it.zig_index += 1; + it.llvm_index += 1; + it.byval_attr = true; + return .byref; + } + switch (ip.indexToKey(ty.toIntern())) { + .struct_type => { + const size = ty.abiSize(zcu); + assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index); + if (size % 8 > 0) { + types_buffer[types_index - 1] = + try it.object.builder.intType(@intCast(size % 8 * 8)); + } + }, + else => {}, + } + } + it.types_len = types_index; + it.types_buffer = types_buffer; + it.llvm_index += types_index; + it.zig_index += 1; + return .multiple_llvm_types; + } +}; +pub fn iterateParamTypes(object: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) ParamTypeIterator { + return .{ + .object = object, + .pt = pt, + .fn_info = fn_info, + .zig_index = 0, + .llvm_index = 0, + .types_len = 0, + .types_buffer = undefined, + .byval_attr = false, + }; +} + +fn returnTypeByRef(zcu: *Zcu, target: *const std.Target, ty: Type) bool { + if (isByRef(ty, zcu)) { + return true; + } else if (target.cpu.arch.isX86() and + !target.cpu.has(.x86, .evex512) and + ty.totalVectorBits(zcu) >= 512) + { + // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns + // "512-bit vector arguments require 'evex512' for AVX512" + return true; + } else { + return false; + } +} + +pub fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: *Zcu, target: *const std.Target) bool { + const return_type = Type.fromInterned(fn_info.return_type); + if (!return_type.hasRuntimeBits(zcu)) return false; + + return switch (fn_info.cc) { + .auto => returnTypeByRef(zcu, target, return_type), + .x86_64_sysv => firstParamSRetSystemV(return_type, zcu, target), + .x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory, + .x86_sysv, .x86_win => isByRef(return_type, zcu), + .x86_stdcall => !isScalar(zcu, return_type), + .wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect, + .aarch64_aapcs, + .aarch64_aapcs_darwin, + .aarch64_aapcs_win, + => aarch64_c_abi.classifyType(return_type, zcu) == .memory, + .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) { + .memory, .i64_array => true, + .i32_array => |size| size != 1, + .byval => false, + }, + .riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory, + .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) { + .memory, .i32_array => true, + .byval => false, + }, + else => false, // TODO: investigate other targets/callconvs + }; +} + +fn firstParamSRetSystemV(ty: Type, zcu: *Zcu, target: *const std.Target) bool { + const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret); + if (class[0] == .memory) return true; + if (class[0] == .x87 and class[2] != .none) return true; + return false; +} + +/// In order to support the C calling convention, some return types need to be lowered +/// completely differently in the function prototype to honor the C ABI, and then +/// be effectively bitcasted to the actual return type. +pub fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { + const zcu = pt.zcu; + const return_type = Type.fromInterned(fn_info.return_type); + if (!return_type.hasRuntimeBits(zcu)) { + assert(!return_type.isError(zcu)); + return .void; + } + const target = zcu.getTarget(); + switch (fn_info.cc) { + .@"inline" => unreachable, + .auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(pt, return_type), + + .x86_64_sysv => return lowerSystemVFnRetTy(o, pt, fn_info), + .x86_64_win => return lowerWin64FnRetTy(o, pt, fn_info), + .x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(pt, return_type) else .void, + .x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(pt, return_type), + .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) { + .memory => return .void, + .float_array => return o.lowerType(pt, return_type), + .byval => return o.lowerType(pt, return_type), + .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))), + .double_integer => return o.builder.arrayType(2, .i64), + }, + .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) { + .memory, .i64_array => return .void, + .i32_array => |len| return if (len == 1) .i32 else .void, + .byval => return o.lowerType(pt, return_type), + }, + .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) { + .memory, .i32_array => return .void, + .byval => return o.lowerType(pt, return_type), + }, + .riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) { + .memory => return .void, + .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))), + .double_integer => { + const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) { + .riscv64, .riscv64be => .i64, + .riscv32, .riscv32be => .i32, + else => unreachable, + }; + return o.builder.structType(.normal, &.{ integer, integer }); + }, + .byval => return o.lowerType(pt, return_type), + .fields => { + var types_len: usize = 0; + var types: [8]Builder.Type = undefined; + for (0..return_type.structFieldCount(zcu)) |field_index| { + const field_ty = return_type.fieldType(field_index, zcu); + if (!field_ty.hasRuntimeBits(zcu)) continue; + types[types_len] = try o.lowerType(pt, field_ty); + types_len += 1; + } + return o.builder.structType(.normal, types[0..types_len]); + }, + }, + .wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) { + .direct => |scalar_ty| return o.lowerType(pt, scalar_ty), + .indirect => return .void, + }, + // TODO investigate other callconvs + else => return o.lowerType(pt, return_type), + } +} + +fn lowerWin64FnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { + const zcu = pt.zcu; + const return_type = Type.fromInterned(fn_info.return_type); + switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) { + .integer => { + if (isScalar(zcu, return_type)) { + return o.lowerType(pt, return_type); + } else { + return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8)); + } + }, + .win_i128 => return o.builder.vectorType(.normal, 2, .i64), + .memory => return .void, + .sse => return o.lowerType(pt, return_type), + else => unreachable, + } +} + +fn lowerSystemVFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const return_type = Type.fromInterned(fn_info.return_type); + return_type.assertHasLayout(zcu); + if (isScalar(zcu, return_type)) { + return o.lowerType(pt, return_type); + } + const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret); + var types_index: u32 = 0; + var types_buffer: [8]Builder.Type = undefined; + for (classes) |class| { + switch (class) { + .integer => { + types_buffer[types_index] = .i64; + types_index += 1; + }, + .sse => { + types_buffer[types_index] = .double; + types_index += 1; + }, + .sseup => { + if (types_buffer[types_index - 1] == .double) { + types_buffer[types_index - 1] = .fp128; + } else { + types_buffer[types_index] = .double; + types_index += 1; + } + }, + .float => { + types_buffer[types_index] = .float; + types_index += 1; + }, + .float_combine => { + types_buffer[types_index] = try o.builder.vectorType(.normal, 2, .float); + types_index += 1; + }, + .x87 => { + if (types_index != 0 or classes[2] != .none) return .void; + types_buffer[types_index] = .x86_fp80; + types_index += 1; + }, + .x87up => continue, + .none => break, + .memory, .integer_per_element => return .void, + .win_i128 => unreachable, // windows only + } + } + const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer}); + if (first_non_integer == null or classes[first_non_integer.?] == .none) { + assert(first_non_integer orelse classes.len == types_index); + switch (ip.indexToKey(return_type.toIntern())) { + .struct_type => { + const size = return_type.abiSize(zcu); + assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index); + if (size % 8 > 0) { + types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8)); + } + }, + else => {}, + } + if (types_index == 1) return types_buffer[0]; + } + return o.builder.structType(.normal, types_buffer[0..types_index]); +} + +/// This function deliberately does not handle `_BitInt` because it typically +/// has different ABI than regular integer types, and there is no currently no +/// way to determine whether a Zig integer type is meant to represent e.g. `int` +/// or `_BitInt(32)`. +pub fn ccAbiPromoteInt(cc: std.builtin.CallingConvention, zcu: *Zcu, ty: Type) ?std.builtin.Signedness { + switch (cc) { + .auto, .@"inline", .async => return null, + else => {}, + } + + const int_info = switch (ty.zigTypeTag(zcu)) { + .bool => Type.u1.intInfo(zcu), + else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return null, + }; + assert(int_info.bits >= 0); + + const target = zcu.getTarget(); + return switch (target.cpu.arch) { + .aarch64, + .aarch64_be, + => switch (target.os.tag) { + .driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => switch (int_info.bits) { + 8, 16 => int_info.signedness, + else => null, + }, + else => null, + }, + + .avr, + => switch (int_info.bits) { + 8 => int_info.signedness, + else => null, + }, + + .lanai, + => null, + + .loongarch64, + .riscv64, + .riscv64be, + => switch (int_info.bits) { + 8, 16 => int_info.signedness, + 32 => .signed, + else => null, + }, + + .mips, + .mipsel, + .mips64, + .mips64el, + => switch (int_info.bits) { + 8, 16, 64 => int_info.signedness, + // https://github.com/llvm/llvm-project/issues/179088 + // 32 => .signed, + else => null, + }, + + .powerpc64, + .powerpc64le, + .s390x, + .sparc64, + .ve, + => switch (int_info.bits) { + 8, 16, 32 => int_info.signedness, + else => null, + }, + + else => switch (int_info.bits) { + 8, 16 => int_info.signedness, + else => null, + }, + }; +} + +fn isScalar(zcu: *Zcu, ty: Type) bool { + return switch (ty.zigTypeTag(zcu)) { + .void, + .bool, + .noreturn, + .int, + .float, + .pointer, + .optional, + .error_set, + .@"enum", + .@"anyframe", + .vector, + => true, + + .@"struct" => ty.containerLayout(zcu) == .@"packed", + .@"union" => ty.containerLayout(zcu) == .@"packed", + else => false, + }; +} + +pub fn buildAllocaInner( + wip: *Builder.WipFunction, + llvm_ty: Builder.Type, + alignment: Builder.Alignment, + target: *const std.Target, +) Allocator.Error!Builder.Value { + const address_space = llvmAllocaAddressSpace(target); + + const alloca = blk: { + const prev_cursor = wip.cursor; + const prev_debug_location = wip.debug_location; + defer { + wip.cursor = prev_cursor; + if (wip.cursor.block == .entry) wip.cursor.instruction += 1; + wip.debug_location = prev_debug_location; + } + + wip.cursor = .{ .block = .entry }; + wip.debug_location = .no_location; + break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, ""); + }; + + // The pointer returned from this function should have the generic address space, + // if this isn't the case then cast it to the generic address space. + return wip.conv(.unneeded, alloca, .ptr, ""); +} + +/// This is the one source of truth for whether a type is passed around as an LLVM pointer, +/// or as an LLVM value. +pub fn isByRef(ty: Type, zcu: *Zcu) bool { + // For tuples and structs, if there are more than this many non-void + // fields, then we make it byref, otherwise byval. + const max_fields_byval = 0; + const ip = &zcu.intern_pool; + + switch (ty.zigTypeTag(zcu)) { + .type, + .comptime_int, + .comptime_float, + .enum_literal, + .undefined, + .null, + .@"opaque", + => unreachable, + + .noreturn, + .void, + .bool, + .int, + .float, + .pointer, + .error_set, + .@"fn", + .@"enum", + .vector, + .@"anyframe", + => return false, + + .array, .frame => return ty.hasRuntimeBits(zcu), + .@"struct" => { + const struct_type = switch (ip.indexToKey(ty.toIntern())) { + .tuple_type => |tuple| { + var count: usize = 0; + for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| { + if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue; + + count += 1; + if (count > max_fields_byval) return true; + if (isByRef(Type.fromInterned(field_ty), zcu)) return true; + } + return false; + }, + .struct_type => ip.loadStructType(ty.toIntern()), + else => unreachable, + }; + + // Packed structs are represented to LLVM as integers. + if (struct_type.layout == .@"packed") return false; + + const field_types = struct_type.field_types.get(ip); + var it = struct_type.iterateRuntimeOrder(ip); + var count: usize = 0; + while (it.next()) |field_index| { + count += 1; + if (count > max_fields_byval) return true; + const field_ty = Type.fromInterned(field_types[field_index]); + if (isByRef(field_ty, zcu)) return true; + } + return false; + }, + .@"union" => switch (ty.containerLayout(zcu)) { + .@"packed" => return false, + else => return ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu), + }, + .error_union => { + const payload_ty = ty.errorUnionPayload(zcu); + if (!payload_ty.hasRuntimeBits(zcu)) { + return false; + } + return true; + }, + .optional => { + const payload_ty = ty.optionalChild(zcu); + if (!payload_ty.hasRuntimeBits(zcu)) { + return false; + } + if (ty.optionalReprIsPayload(zcu)) { + return false; + } + return true; + }, + } +} + +/// If the operand type of an atomic operation is not byte sized we need to +/// widen it before using it and then truncate the result. +/// RMW exchange of floating-point values is bitcasted to same-sized integer +/// types to work around a LLVM deficiency when targeting ARM/AArch64. +fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type { + const zcu = fg.pt.zcu; + switch (ty.zigTypeTag(zcu)) { + .int, .@"enum", .@"struct", .@"union" => {}, + .float => { + if (!is_rmw_xchg) return .none; + return fg.object.builder.intType(@intCast(ty.abiSize(zcu) * 8)); + }, + .bool => return .i8, + else => return .none, + } + const bit_count = ty.bitSize(zcu); + if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) { + return fg.object.builder.intType(@intCast(ty.abiSize(zcu) * 8)); + } else { + return .none; + } +} + +fn errUnionPayloadFieldIndex(payload_ty: Type, pt: Zcu.PerThread) !u1 { + const zcu = pt.zcu; + const err_int_ty = try pt.errorIntType(); + return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.gt, payload_ty.abiAlignment(zcu))); +} + +fn errUnionErrorFieldIndex(payload_ty: Type, pt: Zcu.PerThread) !u1 { + const zcu = pt.zcu; + const err_int_ty = try pt.errorIntType(); + return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.lte, payload_ty.abiAlignment(zcu))); +} + +fn compilerRtIntBits(bits: u16) ?u16 { + inline for (.{ 32, 64, 128 }) |b| { + if (bits <= b) { + return b; + } + } + return null; +} + +/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location +/// +/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang +fn constraintAllowsMemory(constraint: []const u8) bool { + // TODO: This implementation is woefully incomplete. + for (constraint) |byte| { + switch (byte) { + '=', '*', ',', '&' => {}, + 'm', 'o', 'X', 'g' => return true, + else => {}, + } + } else return false; +} + +/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register +/// +/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang +fn constraintAllowsRegister(constraint: []const u8) bool { + // TODO: This implementation is woefully incomplete. + for (constraint) |byte| { + switch (byte) { + '=', '*', ',', '&' => {}, + 'm', 'o' => {}, + else => return true, + } + } else return false; +} + +/// Appends zero or more LLVM constraints to `llvm_constraints`, returning how many were added. +fn appendConstraints( + gpa: Allocator, + llvm_constraints: *std.ArrayList(u8), + zig_name: []const u8, + target: *const std.Target, +) error{OutOfMemory}!usize { + switch (target.cpu.arch) { + .mips, .mipsel, .mips64, .mips64el => if (mips_clobber_overrides.get(zig_name)) |llvm_tag| { + const llvm_name = @tagName(llvm_tag); + try llvm_constraints.ensureUnusedCapacity(gpa, llvm_name.len + 4); + llvm_constraints.appendSliceAssumeCapacity("~{"); + llvm_constraints.appendSliceAssumeCapacity(llvm_name); + llvm_constraints.appendSliceAssumeCapacity("},"); + return 1; + }, + else => {}, + } + + try llvm_constraints.ensureUnusedCapacity(gpa, zig_name.len + 4); + llvm_constraints.appendSliceAssumeCapacity("~{"); + llvm_constraints.appendSliceAssumeCapacity(zig_name); + llvm_constraints.appendSliceAssumeCapacity("},"); + return 1; +} + +/// LLVM does not support all relevant intrinsics for all targets, so we +/// may need to manually generate a compiler-rt call. +fn intrinsicsAllowed(scalar_ty: Type, target: *const std.Target) bool { + return switch (scalar_ty.toIntern()) { + .f16_type => backendSupportsF16(target), + .f80_type => (target.cTypeBitSize(.longdouble) == 80) and backendSupportsF80(target), + .f128_type => (target.cTypeBitSize(.longdouble) == 128) and backendSupportsF128(target), + else => true, + }; +} + +fn toLlvmAtomicOrdering(atomic_order: std.builtin.AtomicOrder) Builder.AtomicOrdering { + return switch (atomic_order) { + .unordered => .unordered, + .monotonic => .monotonic, + .acquire => .acquire, + .release => .release, + .acq_rel => .acq_rel, + .seq_cst => .seq_cst, + }; +} + +fn toLlvmAtomicRmwBinOp( + op: std.builtin.AtomicRmwOp, + is_signed: bool, + is_float: bool, +) Builder.Function.Instruction.AtomicRmw.Operation { + return switch (op) { + .Xchg => .xchg, + .Add => if (is_float) .fadd else return .add, + .Sub => if (is_float) .fsub else return .sub, + .And => .@"and", + .Nand => .nand, + .Or => .@"or", + .Xor => .xor, + .Max => if (is_float) .fmax else if (is_signed) .max else return .umax, + .Min => if (is_float) .fmin else if (is_signed) .min else return .umin, + }; +} + +fn minIntConst(b: *Builder, min_ty: Type, as_ty: Builder.Type, zcu: *const Zcu) Allocator.Error!Builder.Constant { + const info = min_ty.intInfo(zcu); + if (info.signedness == .unsigned or info.bits == 0) { + return b.intConst(as_ty, 0); + } + if (std.math.cast(u6, info.bits - 1)) |shift| { + const min_val: i64 = @as(i64, std.math.minInt(i64)) >> (63 - shift); + return b.intConst(as_ty, min_val); + } + var res: std.math.big.int.Managed = try .init(zcu.gpa); + defer res.deinit(); + try res.setTwosCompIntLimit(.min, info.signedness, info.bits); + return b.bigIntConst(as_ty, res.toConst()); +} + +fn maxIntConst(b: *Builder, max_ty: Type, as_ty: Builder.Type, zcu: *const Zcu) Allocator.Error!Builder.Constant { + const info = max_ty.intInfo(zcu); + switch (info.bits) { + 0 => return b.intConst(as_ty, 0), + 1 => switch (info.signedness) { + .signed => return b.intConst(as_ty, 0), + .unsigned => return b.intConst(as_ty, 1), + }, + else => {}, + } + const unsigned_bits = switch (info.signedness) { + .unsigned => info.bits, + .signed => info.bits - 1, + }; + if (std.math.cast(u6, unsigned_bits)) |shift| { + const max_val: u64 = (@as(u64, 1) << shift) - 1; + return b.intConst(as_ty, max_val); + } + var res: std.math.big.int.Managed = try .init(zcu.gpa); + defer res.deinit(); + try res.setTwosCompIntLimit(.max, info.signedness, info.bits); + return b.bigIntConst(as_ty, res.toConst()); +} + +/// On some targets, local values that are in the generic address space must be generated into a +/// different address, space and then cast back to the generic address space. +/// For example, on GPUs local variable declarations must be generated into the local address space. +/// This function returns the address space local values should be generated into. +fn llvmAllocaAddressSpace(target: *const std.Target) Builder.AddrSpace { + return switch (target.cpu.arch) { + // On amdgcn, locals should be generated into the private address space. + // To make Zig not impossible to use, these are then converted to addresses in the + // generic address space and treates as regular pointers. This is the way that HIP also does it. + .amdgcn => Builder.AddrSpace.amdgpu.private, + else => .default, + }; +} + +const mips_clobber_overrides = std.StaticStringMap(enum { + @"$msair", + @"$msacsr", + @"$msaaccess", + @"$msasave", + @"$msamodify", + @"$msarequest", + @"$msamap", + @"$msaunmap", + @"$f0", + @"$f1", + @"$f2", + @"$f3", + @"$f4", + @"$f5", + @"$f6", + @"$f7", + @"$f8", + @"$f9", + @"$f10", + @"$f11", + @"$f12", + @"$f13", + @"$f14", + @"$f15", + @"$f16", + @"$f17", + @"$f18", + @"$f19", + @"$f20", + @"$f21", + @"$f22", + @"$f23", + @"$f24", + @"$f25", + @"$f26", + @"$f27", + @"$f28", + @"$f29", + @"$f30", + @"$f31", + @"$fcc0", + @"$fcc1", + @"$fcc2", + @"$fcc3", + @"$fcc4", + @"$fcc5", + @"$fcc6", + @"$fcc7", + @"$w0", + @"$w1", + @"$w2", + @"$w3", + @"$w4", + @"$w5", + @"$w6", + @"$w7", + @"$w8", + @"$w9", + @"$w10", + @"$w11", + @"$w12", + @"$w13", + @"$w14", + @"$w15", + @"$w16", + @"$w17", + @"$w18", + @"$w19", + @"$w20", + @"$w21", + @"$w22", + @"$w23", + @"$w24", + @"$w25", + @"$w26", + @"$w27", + @"$w28", + @"$w29", + @"$w30", + @"$w31", + @"$0", + @"$1", + @"$2", + @"$3", + @"$4", + @"$5", + @"$6", + @"$7", + @"$8", + @"$9", + @"$10", + @"$11", + @"$12", + @"$13", + @"$14", + @"$15", + @"$16", + @"$17", + @"$18", + @"$19", + @"$20", + @"$21", + @"$22", + @"$23", + @"$24", + @"$25", + @"$26", + @"$27", + @"$28", + @"$29", + @"$30", + @"$31", +}).initComptime(.{ + .{ "msa_ir", .@"$msair" }, + .{ "msa_csr", .@"$msacsr" }, + .{ "msa_access", .@"$msaaccess" }, + .{ "msa_save", .@"$msasave" }, + .{ "msa_modify", .@"$msamodify" }, + .{ "msa_request", .@"$msarequest" }, + .{ "msa_map", .@"$msamap" }, + .{ "msa_unmap", .@"$msaunmap" }, + .{ "f0", .@"$f0" }, + .{ "f1", .@"$f1" }, + .{ "f2", .@"$f2" }, + .{ "f3", .@"$f3" }, + .{ "f4", .@"$f4" }, + .{ "f5", .@"$f5" }, + .{ "f6", .@"$f6" }, + .{ "f7", .@"$f7" }, + .{ "f8", .@"$f8" }, + .{ "f9", .@"$f9" }, + .{ "f10", .@"$f10" }, + .{ "f11", .@"$f11" }, + .{ "f12", .@"$f12" }, + .{ "f13", .@"$f13" }, + .{ "f14", .@"$f14" }, + .{ "f15", .@"$f15" }, + .{ "f16", .@"$f16" }, + .{ "f17", .@"$f17" }, + .{ "f18", .@"$f18" }, + .{ "f19", .@"$f19" }, + .{ "f20", .@"$f20" }, + .{ "f21", .@"$f21" }, + .{ "f22", .@"$f22" }, + .{ "f23", .@"$f23" }, + .{ "f24", .@"$f24" }, + .{ "f25", .@"$f25" }, + .{ "f26", .@"$f26" }, + .{ "f27", .@"$f27" }, + .{ "f28", .@"$f28" }, + .{ "f29", .@"$f29" }, + .{ "f30", .@"$f30" }, + .{ "f31", .@"$f31" }, + .{ "fcc0", .@"$fcc0" }, + .{ "fcc1", .@"$fcc1" }, + .{ "fcc2", .@"$fcc2" }, + .{ "fcc3", .@"$fcc3" }, + .{ "fcc4", .@"$fcc4" }, + .{ "fcc5", .@"$fcc5" }, + .{ "fcc6", .@"$fcc6" }, + .{ "fcc7", .@"$fcc7" }, + .{ "w0", .@"$w0" }, + .{ "w1", .@"$w1" }, + .{ "w2", .@"$w2" }, + .{ "w3", .@"$w3" }, + .{ "w4", .@"$w4" }, + .{ "w5", .@"$w5" }, + .{ "w6", .@"$w6" }, + .{ "w7", .@"$w7" }, + .{ "w8", .@"$w8" }, + .{ "w9", .@"$w9" }, + .{ "w10", .@"$w10" }, + .{ "w11", .@"$w11" }, + .{ "w12", .@"$w12" }, + .{ "w13", .@"$w13" }, + .{ "w14", .@"$w14" }, + .{ "w15", .@"$w15" }, + .{ "w16", .@"$w16" }, + .{ "w17", .@"$w17" }, + .{ "w18", .@"$w18" }, + .{ "w19", .@"$w19" }, + .{ "w20", .@"$w20" }, + .{ "w21", .@"$w21" }, + .{ "w22", .@"$w22" }, + .{ "w23", .@"$w23" }, + .{ "w24", .@"$w24" }, + .{ "w25", .@"$w25" }, + .{ "w26", .@"$w26" }, + .{ "w27", .@"$w27" }, + .{ "w28", .@"$w28" }, + .{ "w29", .@"$w29" }, + .{ "w30", .@"$w30" }, + .{ "w31", .@"$w31" }, + .{ "r0", .@"$0" }, + .{ "r1", .@"$1" }, + .{ "r2", .@"$2" }, + .{ "r3", .@"$3" }, + .{ "r4", .@"$4" }, + .{ "r5", .@"$5" }, + .{ "r6", .@"$6" }, + .{ "r7", .@"$7" }, + .{ "r8", .@"$8" }, + .{ "r9", .@"$9" }, + .{ "r10", .@"$10" }, + .{ "r11", .@"$11" }, + .{ "r12", .@"$12" }, + .{ "r13", .@"$13" }, + .{ "r14", .@"$14" }, + .{ "r15", .@"$15" }, + .{ "r16", .@"$16" }, + .{ "r17", .@"$17" }, + .{ "r18", .@"$18" }, + .{ "r19", .@"$19" }, + .{ "r20", .@"$20" }, + .{ "r21", .@"$21" }, + .{ "r22", .@"$22" }, + .{ "r23", .@"$23" }, + .{ "r24", .@"$24" }, + .{ "r25", .@"$25" }, + .{ "r26", .@"$26" }, + .{ "r27", .@"$27" }, + .{ "r28", .@"$28" }, + .{ "r29", .@"$29" }, + .{ "r30", .@"$30" }, + .{ "r31", .@"$31" }, +}); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Builder = std.zig.llvm.Builder; +const assert = std.debug.assert; +const math = std.math; + +const x86_64_abi = @import("../x86_64/abi.zig"); +const wasm_c_abi = @import("../wasm/abi.zig"); +const aarch64_c_abi = @import("../aarch64/abi.zig"); +const arm_c_abi = @import("../arm/abi.zig"); +const riscv_c_abi = @import("../riscv64/abi.zig"); +const mips_c_abi = @import("../mips/abi.zig"); + +const Zcu = @import("../../Zcu.zig"); +const Air = @import("../../Air.zig"); +const Package = @import("../../Package.zig"); +const InternPool = @import("../../InternPool.zig"); +const Value = @import("../../Value.zig"); +const Type = @import("../../Type.zig"); + +const target_util = @import("../../target.zig"); +const libcFloatPrefix = target_util.libcFloatPrefix; +const libcFloatSuffix = target_util.libcFloatSuffix; +const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev; +const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev; + +const llvm = @import("../llvm.zig"); +const Object = llvm.Object; +const toLlvmCallConvTag = llvm.toLlvmCallConvTag; +const toLlvmAddressSpace = llvm.toLlvmAddressSpace; +const optional_layout_version = llvm.optional_layout_version; +const backendSupportsF16 = llvm.backendSupportsF16; +const backendSupportsF80 = llvm.backendSupportsF80; +const backendSupportsF128 = llvm.backendSupportsF128; -- 2.54.0 From 5941c9da08ae0d8ba4add37f0baeafdcd160dbd4 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Sun, 22 Mar 2026 20:11:35 +0000 Subject: [PATCH 02/15] llvm: remove almost all GEPs LLVM is gradually transitioning from the `getelementptr` instruction to a new `ptradd` instruction. The latter instruction doesn't actually exist yet, but for now, LLVM is considering `getelementptr i8` to be equivalent. LLVM is already internally canonicalizing `getelementptr` usages to this pattern in many cases, and it's far easier for us to emit that, so... let's do so! For runtime indexing this does sometimes require an explicit multiplication to scale an index to a byte offset. The helper function `llvm.FuncGen.ptraddScaled` makes this common pattern more convenient. A particularly nice side effect from this is that after removing some dead code (left over from before we made all `struct`s etc by-ref), it has eliminated the need to maintain that nasty mapping between Zig field indices and LLVM field indices. `FuncGen` no longer cares at all how aggregate types are lowered! Slices are still by-val at least for now, but they never lived in that mapping because their structure is simple and consistent (they always have a pointer at field index 0 and a usize at field index 1, with no explicit padding necessary). --- src/Type.zig | 2 +- src/codegen/llvm.zig | 61 +- src/codegen/llvm/FuncGen.zig | 1099 ++++++++++++---------------------- 3 files changed, 402 insertions(+), 760 deletions(-) diff --git a/src/Type.zig b/src/Type.zig index 2aa2332cf2bf8461254334e83143d13ffac88bd7..7902788d1cbfcb1101485629012f369aa913cead 100644 --- a/src/Type.zig +++ b/src/Type.zig @@ -1594,7 +1594,7 @@ pub fn unionTagFieldIndex(ty: Type, enum_tag: Value, zcu: *const Zcu) ?u32 { return zcu.unionTagFieldIndex(union_obj, enum_tag); } -pub fn unionHasAllZeroBitFieldTypes(ty: Type, zcu: *Zcu) bool { +pub fn unionHasAllZeroBitFieldTypes(ty: Type, zcu: *const Zcu) bool { assertHasLayout(ty, zcu); const ip = &zcu.intern_pool; const union_obj = zcu.typeToUnion(ty).?; diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index f5fae777a2691b31839e90fcc309801be7251f49..d9f7b4b54fbc0e76910e65b62c2636a3e7004ade 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -578,24 +578,9 @@ pub const Object = struct { /// Memoizes a null `?usize` value. null_opt_usize: Builder.Constant, - /// When an LLVM struct type is created, an entry is inserted into this - /// table for every zig source field of the struct that has a corresponding - /// LLVM struct field. comptime fields are not included. Zero-bit fields are - /// mapped to a field at the correct byte, which may be a padding field, or - /// are not mapped, in which case they are semantically at the end of the - /// struct. - /// The value is the LLVM struct field index. - /// This is denormalized data. - struct_field_map: std.AutoHashMapUnmanaged(ZigStructField, c_uint), - /// Values for `@llvm.used`. used: std.ArrayList(Builder.Constant), - const ZigStructField = struct { - struct_ty: InternPool.Index, - field_index: u32, - }; - pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn; const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type); @@ -688,7 +673,6 @@ pub const Object = struct { .type_map = .empty, .error_name_table = .none, .null_opt_usize = .no_init, - .struct_field_map = .empty, .used = .empty, }; return obj; @@ -708,7 +692,6 @@ pub const Object = struct { self.named_enum_map.deinit(gpa); self.type_map.deinit(gpa); self.builder.deinit(); - self.struct_field_map.deinit(gpa); self.* = undefined; } @@ -3311,7 +3294,6 @@ pub const Object = struct { // Although we can estimate how much capacity to add, these cannot be // relied upon because of the recursive calls to lowerType below. try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_type.field_types.len); - try o.struct_field_map.ensureUnusedCapacity(o.gpa, struct_type.field_types.len); comptime assert(struct_layout_version == 2); var offset: u64 = 0; @@ -3336,23 +3318,8 @@ pub const Object = struct { try o.builder.arrayType(padding_len, .i8), ); - if (!field_ty.hasRuntimeBits(zcu)) { - // This is a zero-bit field. If there are runtime bits after this field, - // map to the next LLVM field (which we know exists): otherwise, don't - // map the field, indicating it's at the end of the struct. - if (offset != struct_type.size) { - try o.struct_field_map.put(o.gpa, .{ - .struct_ty = t.toIntern(), - .field_index = field_index, - }, @intCast(llvm_field_types.items.len)); - } - continue; - } + if (!field_ty.hasRuntimeBits(zcu)) continue; - try o.struct_field_map.put(o.gpa, .{ - .struct_ty = t.toIntern(), - .field_index = field_index, - }, @intCast(llvm_field_types.items.len)); try llvm_field_types.append(o.gpa, try o.lowerType(pt, field_ty)); offset += field_ty.abiSize(zcu); @@ -3385,19 +3352,15 @@ pub const Object = struct { // Although we can estimate how much capacity to add, these cannot be // relied upon because of the recursive calls to lowerType below. try llvm_field_types.ensureUnusedCapacity(o.gpa, tuple_type.types.len); - try o.struct_field_map.ensureUnusedCapacity(o.gpa, tuple_type.types.len); comptime assert(struct_layout_version == 2); var offset: u64 = 0; var big_align: InternPool.Alignment = .none; - const struct_size = t.abiSize(zcu); - for ( tuple_type.types.get(ip), tuple_type.values.get(ip), - 0.., - ) |field_ty, field_val, field_index| { + ) |field_ty, field_val| { if (field_val != .none) continue; const field_align = Type.fromInterned(field_ty).abiAlignment(zcu); @@ -3411,21 +3374,8 @@ pub const Object = struct { try o.builder.arrayType(padding_len, .i8), ); if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) { - // This is a zero-bit field. If there are runtime bits after this field, - // map to the next LLVM field (which we know exists): otherwise, don't - // map the field, indicating it's at the end of the struct. - if (offset != struct_size) { - try o.struct_field_map.put(o.gpa, .{ - .struct_ty = t.toIntern(), - .field_index = @intCast(field_index), - }, @intCast(llvm_field_types.items.len)); - } continue; } - try o.struct_field_map.put(o.gpa, .{ - .struct_ty = t.toIntern(), - .field_index = @intCast(field_index), - }, @intCast(llvm_field_types.items.len)); try llvm_field_types.append(o.gpa, try o.lowerType(pt, Type.fromInterned(field_ty))); offset += Type.fromInterned(field_ty).abiSize(zcu); @@ -4338,13 +4288,6 @@ pub const Object = struct { if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder); } - pub fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint { - return o.struct_field_map.get(.{ - .struct_ty = struct_ty.toIntern(), - .field_index = @intCast(field_index), - }); - } - /// MLUGG TODO: this is super dumb pub fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index { const name = try o.builder.strtabString(lt_errors_fn_name); diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index 52408a023a840a87965d0e31725f6e6e4d8a6887..30cbbab76d5efff5747f4b383c54ea056055e20b 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -141,7 +141,7 @@ pub fn deinit(self: *FuncGen) void { self.switch_dispatch_info.deinit(gpa); } -fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value { +fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) Allocator.Error!Builder.Value { const gpa = self.gpa; const gop = try self.func_inst_table.getOrPut(gpa, inst); if (gop.found_existing) return gop.value_ptr.*; @@ -151,7 +151,7 @@ fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value { return llvm_val.toValue(); } -fn resolveValue(self: *FuncGen, val: Value) Error!Builder.Constant { +fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -177,9 +177,7 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air .poi => if (self.fuzz) |*fuzz| { const poi_index = fuzz.pcs.items.len; const base_ptr = fuzz.counters_variable.toValue(&o.builder); - const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{ - try o.builder.intValue(.i32, poi_index), - }, ""); + const ptr = try self.ptraddConst(base_ptr, poi_index); const one = try o.builder.intValue(.i8, 1); _ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, ""); @@ -704,8 +702,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif const llvm_ty = try o.builder.structType(.normal, llvm_types); try llvm_args.ensureUnusedCapacity(it.types_len); for (llvm_types, 0..) |field_ty, i| { - const alignment = - Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8)); + const alignment: Builder.Alignment = .fromByteUnits(@divExact(target.ptrBitWidth(), 8)); const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, ""); const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, ""); llvm_args.appendAssumeCapacity(loaded); @@ -1153,10 +1150,8 @@ fn cmp( } // We need to emit instructions to check for equality/inequality // of optionals that are not pointers. - const is_by_ref = isByRef(scalar_ty, zcu); - const opt_llvm_ty = try o.lowerType(pt, scalar_ty); - const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal); - const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal); + const lhs_non_null = try self.optCmpNull(.ne, scalar_ty, lhs, .normal); + const rhs_non_null = try self.optCmpNull(.ne, scalar_ty, rhs, .normal); const llvm_i2 = try o.builder.intType(2); const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, ""); const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, ""); @@ -1186,8 +1181,8 @@ fn cmp( _ = try self.wip.br(end_block); self.wip.cursor = .{ .block = both_pl_block }; - const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true); - const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true); + const lhs_payload = try self.optPayloadHandle(lhs, scalar_ty, true); + const rhs_payload = try self.optPayloadHandle(rhs, scalar_ty, true); const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload); _ = try self.wip.br(end_block); const both_pl_block_end = self.wip.cursor.block; @@ -1392,12 +1387,10 @@ fn lowerSwitchDispatch( try o.lowerType(pt, .usize), "", ); - const target_ptr_ptr = try self.wip.gep( - .inbounds, - .ptr, + const target_ptr_ptr = try self.ptraddScaled( jmp_table.table.toValue(), - &.{table_index}, - "", + table_index, + Type.usize.abiSize(zcu), ); const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, ""); @@ -1580,7 +1573,7 @@ fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { const body = unwrapped_try.else_body; const err_union_ty = self.typeOf(unwrapped_try.error_union); const is_unused = self.liveness.isUnused(inst); - return lowerTry(self, err_union, body, err_union_ty, false, .none, false, is_unused, err_cold); + return lowerTry(self, err_union, body, err_union_ty, false, .none, is_unused, err_cold); } fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { @@ -1594,7 +1587,7 @@ fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Valu self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu)); - return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), true, is_unused, err_cold); + return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), is_unused, err_cold); } fn lowerTry( @@ -1604,7 +1597,6 @@ fn lowerTry( err_union_ty: Type, operand_is_ptr: bool, operand_ptr_align: InternPool.Alignment, - can_elide_load: bool, is_unused: bool, err_cold: bool, ) !Builder.Value { @@ -1613,7 +1605,6 @@ fn lowerTry( const zcu = pt.zcu; const payload_ty = err_union_ty.errorUnionPayload(zcu); const payload_has_bits = payload_ty.hasRuntimeBits(zcu); - const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); const error_type = try o.errorIntType(pt); const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{ @@ -1632,19 +1623,17 @@ fn lowerTry( else err_union; } - const err_field_index = try errUnionErrorFieldIndex(payload_ty, pt); - if (operand_is_ptr or isByRef(err_union_ty, zcu)) { - const err_field_ptr = - try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, ""); - break :loaded try fg.wip.load( - if (operand_is_ptr) access_kind else .normal, - error_type, - err_field_ptr, - err_set_align.toLlvm(), - "", - ); - } - break :loaded try fg.wip.extractValue(err_union, &.{err_field_index}, ""); + + assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits + const offset = codegen.errUnionErrorOffset(payload_ty, zcu); + const err_field_ptr = try fg.ptraddConst(err_union, offset); + break :loaded try fg.wip.load( + if (operand_is_ptr) access_kind else .normal, + error_type, + err_field_ptr, + err_set_align.toLlvm(), + "", + ); }; const zero = try o.builder.intValue(error_type, 0); const is_err = try fg.wip.icmp(.ne, loaded, zero, ""); @@ -1661,21 +1650,15 @@ fn lowerTry( } if (is_unused) return .none; if (!payload_has_bits) return if (operand_is_ptr) err_union else .none; - const offset = try errUnionPayloadFieldIndex(payload_ty, pt); + assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits + const payload_ptr = try fg.ptraddConst(err_union, codegen.errUnionPayloadOffset(payload_ty, zcu)); if (operand_is_ptr) { - return fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, ""); - } else if (isByRef(err_union_ty, zcu)) { - const payload_ptr = try fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, ""); - if (isByRef(payload_ty, zcu)) { - if (can_elide_load) - return payload_ptr; - - return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal); - } - const load_ty = err_union_llvm_ty.structFields(&o.builder)[offset]; - return fg.wip.load(.normal, load_ty, payload_ptr, payload_align.toLlvm(), ""); + return payload_ptr; + } else if (isByRef(payload_ty, zcu)) { + return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal); + } else { + return fg.wip.load(.normal, try o.lowerType(pt, payload_ty), payload_ptr, payload_align.toLlvm(), ""); } - return fg.wip.extractValue(err_union, &.{offset}, ""); } fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void { @@ -1950,12 +1933,7 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu)); const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst)); const operand = try self.resolveInst(ty_op.operand); - if (!array_ty.hasRuntimeBits(zcu)) - return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, ""); - const ptr = try self.wip.gep(.inbounds, try o.lowerType(pt, array_ty), operand, &.{ - try o.builder.intValue(llvm_usize, 0), try o.builder.intValue(llvm_usize, 0), - }, ""); - return self.wip.buildAggregate(slice_llvm_ty, &.{ ptr, len }, ""); + return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, ""); } fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { @@ -2126,20 +2104,14 @@ fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Valu return self.wip.extractValue(operand, &.{index}, ""); } -fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: c_uint) !Builder.Value { - const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; +fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: u1) !Builder.Value { + const zcu = self.pt.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const slice_ptr = try self.resolveInst(ty_op.operand); - const slice_ptr_ty = self.typeOf(ty_op.operand); - const slice_llvm_ty = try o.lowerType(pt, slice_ptr_ty.childType(zcu)); - - return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, ""); + return self.ptraddConst(slice_ptr, index * Type.usize.abiSize(zcu)); } fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -2149,9 +2121,8 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const slice_info = slice_ty.ptrInfo(zcu); assert(slice_info.flags.size == .slice); const elem_ty: Type = .fromInterned(slice_info.child); - const llvm_elem_ty = try o.lowerType(pt, elem_ty); const base_ptr = try self.wip.extractValue(slice, &.{0}, ""); - const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, ""); + const ptr = try self.ptraddScaled(base_ptr, index, elem_ty.abiSize(zcu)); const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)); const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal; self.maybeMarkAllowZeroAccess(slice_info); @@ -2163,7 +2134,6 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; const pt = self.pt; const zcu = pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; @@ -2172,13 +2142,11 @@ fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const slice = try self.resolveInst(bin_op.lhs); const index = try self.resolveInst(bin_op.rhs); - const llvm_elem_ty = try o.lowerType(pt, slice_ty.childType(zcu)); const base_ptr = try self.wip.extractValue(slice, &.{0}, ""); - return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, ""); + return self.ptraddScaled(base_ptr, index, slice_ty.childType(zcu).abiSize(zcu)); } fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -2186,16 +2154,12 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const array_ty = self.typeOf(bin_op.lhs); const array_llvm_val = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); - const array_llvm_ty = try o.lowerType(pt, array_ty); const elem_ty = array_ty.childType(zcu); if (isByRef(array_ty, zcu)) { - const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{ - try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), - rhs, - }, ""); + const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu)); if (isByRef(elem_ty, zcu)) { - const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm(); - return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal); + const elem_align = elem_ty.abiAlignment(zcu).toLlvm(); + return self.loadByRef(elem_ptr, elem_ty, elem_align, .normal); } else { return self.loadTruncate(.normal, elem_ty, elem_ptr, .default); } @@ -2206,16 +2170,14 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const ptr_ty = self.typeOf(bin_op.lhs); const elem_ty = ptr_ty.indexableElem(zcu); - const llvm_elem_ty = try o.lowerType(pt, elem_ty); const base_ptr = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); - const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, ""); + const ptr = try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)); if (isByRef(elem_ty, zcu)) { self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm(); @@ -2228,7 +2190,6 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; const pt = self.pt; const zcu = pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; @@ -2243,8 +2204,7 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const elem_ptr = ty_pl.ty.toType(); if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr; - const llvm_elem_ty = try o.lowerType(pt, elem_ty); - return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, ""); + return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)); } fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { @@ -2276,86 +2236,59 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const struct_llvm_val = try self.resolveInst(struct_field.struct_operand); const field_index = struct_field.field_index; const field_ty = struct_ty.fieldType(field_index, zcu); - if (!field_ty.hasRuntimeBits(zcu)) return .none; + assert(field_ty.hasRuntimeBits(zcu)); if (!isByRef(struct_ty, zcu)) { + // All auto/extern struct/union types are by-ref, unless they have no runtime bits, in which + // case we shouldn't be seeing this instruction to begin with. Therefore we must be dealing + // with a `packed struct` or `packed union`. + assert(struct_ty.containerLayout(zcu) == .@"packed"); assert(!isByRef(field_ty, zcu)); - switch (struct_ty.zigTypeTag(zcu)) { - .@"struct" => switch (struct_ty.containerLayout(zcu)) { - .@"packed" => { - const struct_type = zcu.typeToStruct(struct_ty).?; - const bit_offset = zcu.structPackedFieldBitOffset(struct_type, field_index); - const containing_int = struct_llvm_val; - const shift_amt = - try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset); - const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, ""); - const elem_llvm_ty = try o.lowerType(pt, field_ty); - if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) { - const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu))); - const truncated_int = - try self.wip.cast(.trunc, shifted_value, same_size_int, ""); - return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); - } - return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, ""); - }, - else => { - const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; - return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, ""); - }, - }, - .@"union" => { - assert(struct_ty.containerLayout(zcu) == .@"packed"); - const containing_int = struct_llvm_val; - const elem_llvm_ty = try o.lowerType(pt, field_ty); - if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) { - const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu))); - const truncated_int = - try self.wip.cast(.trunc, containing_int, same_size_int, ""); - return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); - } - return self.wip.cast(.trunc, containing_int, elem_llvm_ty, ""); + const field_int_val: Builder.Value = switch (struct_ty.zigTypeTag(zcu)) { + .@"struct" => field_int_val: { + const llvm_field_int_ty = try o.builder.intType(@intCast(field_ty.bitSize(zcu))); + const bit_offset = zcu.structPackedFieldBitOffset( + zcu.intern_pool.loadStructType(struct_ty.toIntern()), + field_index, + ); + const shift_bits = try o.builder.intValue(struct_llvm_val.typeOfWip(&self.wip), bit_offset); + const shifted = try self.wip.bin(.lshr, struct_llvm_val, shift_bits, ""); + break :field_int_val try self.wip.cast(.trunc, shifted, llvm_field_int_ty, ""); }, + .@"union" => struct_llvm_val, else => unreachable, + }; + switch (field_ty.zigTypeTag(zcu)) { + else => unreachable, // not packable + .void => unreachable, // opv bug in sema + .int, .bool, .@"enum", .@"struct", .@"union" => { + // Represented as integers, so already done + return field_int_val; + }, + .float => { + // bitcast int->float + return self.wip.cast(.bitcast, field_int_val, try o.lowerType(pt, field_ty), ""); + }, } } - switch (struct_ty.zigTypeTag(zcu)) { - .@"struct" => { - const layout = struct_ty.containerLayout(zcu); - assert(layout != .@"packed"); - const struct_llvm_ty = try o.lowerType(pt, struct_ty); - const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; - const field_ptr = - try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, ""); - const explicit_alignment = struct_ty.explicitFieldAlignment(field_index, zcu); - const field_ptr_ty = try pt.ptrType(.{ - .child = field_ty.toIntern(), - .flags = .{ .alignment = explicit_alignment }, - }); - if (isByRef(field_ty, zcu)) { - const alignment = switch (explicit_alignment) { - .none => field_ty.abiAlignment(zcu), - else => |a| a, - }; - return self.loadByRef(field_ptr, field_ty, alignment.toLlvm(), .normal); - } else { - return self.load(field_ptr, field_ptr_ty); - } - }, - .@"union" => { - const union_llvm_ty = try o.lowerType(pt, struct_ty); - const layout = struct_ty.unionGetLayout(zcu); - const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)); - const field_ptr = - try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, ""); - const payload_alignment = layout.payload_align.toLlvm(); - if (isByRef(field_ty, zcu)) { - return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal); - } else { - return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment); - } - }, + const offset: u64 = switch (struct_ty.zigTypeTag(zcu)) { + .@"struct" => struct_ty.structFieldOffset(field_index, zcu), + .@"union" => struct_ty.unionGetLayout(zcu).payloadOffset(), else => unreachable, + }; + + const struct_ptr_align = struct_ty.abiAlignment(zcu); + const field_ptr = try self.ptraddConst(struct_llvm_val, offset); + const field_ptr_align: InternPool.Alignment = switch (offset) { + 0 => struct_ptr_align, + else => struct_ptr_align.minStrict(.fromLog2Units(@ctz(offset))), + }; + + if (isByRef(field_ty, zcu)) { + return self.loadByRef(field_ptr, field_ty, field_ptr_align.toLlvm(), .normal); + } else { + return self.loadTruncate(.normal, field_ty, field_ptr, field_ptr_align.toLlvm()); } } @@ -2964,8 +2897,7 @@ fn airIsNonNull( return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), ""); } - const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu); - return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind); + return self.optCmpNull(cond, optional_ty, operand, access_kind); } fn airIsErr( @@ -3006,40 +2938,23 @@ fn airIsErr( operand; return self.wip.icmp(cond, loaded, zero, ""); } + assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref - const err_field_index = try errUnionErrorFieldIndex(payload_ty, pt); - - const loaded = if (operand_is_ptr or isByRef(err_union_ty, zcu)) loaded: { - const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); - const err_alignment = if (operand_is_ptr) - operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu)) - else - .none; - const err_field_ptr = - try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, ""); - break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, err_alignment.toLlvm(), ""); - } else try self.wip.extractValue(operand, &.{err_field_index}, ""); + const err_align = if (operand_is_ptr) + operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu)) + else + .none; + const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu)); + const loaded = try self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), ""); return self.wip.icmp(cond, loaded, zero, ""); } -fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; +fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); - const optional_ty = self.typeOf(ty_op.operand).childType(zcu); - const payload_ty = optional_ty.optionalChild(zcu); - if (!payload_ty.hasRuntimeBits(zcu)) { - // We have a pointer to a zero-bit value and we need to return - // a pointer to a zero-bit value. - return operand; - } - if (optional_ty.optionalReprIsPayload(zcu)) { - // The payload and the optional are the same value. - return operand; - } - return self.wip.gepStruct(try o.lowerType(pt, optional_ty), operand, 0, ""); + // If `Type.optionalReprIsPayload`, then the address should be the same. Otherwise, optional + // layouts always put the payload at offset 0, so... the address should still be the same. + return operand; } fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { @@ -3072,9 +2987,9 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value return operand; } - // First set the non-null bit. - const optional_llvm_ty = try o.lowerType(pt, optional_ty); - const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, ""); + // First set the non-null bit. It's always immediately after the payload (no padding) because it + // has alignment 1. + const non_null_ptr = try self.ptraddConst(operand, payload_ty.abiSize(zcu)); self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu)); @@ -3084,11 +2999,10 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value // Then return the payload pointer (only if it's used). if (self.liveness.isUnused(inst)) return .none; - return self.wip.gepStruct(optional_llvm_ty, operand, 0, ""); + return operand; // payload is at offset 0 } fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; const pt = self.pt; const zcu = pt.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; @@ -3102,8 +3016,7 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return operand; } - const opt_llvm_ty = try o.lowerType(pt, optional_ty); - return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false); + return self.optPayloadHandle(operand, optional_ty, false); } fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value { @@ -3120,20 +3033,18 @@ fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool if (!payload_ty.hasRuntimeBits(zcu)) { return if (operand_is_ptr) operand else .none; } - const offset = try errUnionPayloadFieldIndex(payload_ty, pt); - const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); + const payload_ptr = try self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu)); if (operand_is_ptr) { - return self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); - } else if (isByRef(err_union_ty, zcu)) { - const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm(); - const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); - if (isByRef(payload_ty, zcu)) { - return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); - } - const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset]; + return payload_ptr; + } + assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits + const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm(); + if (isByRef(payload_ty, zcu)) { + return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); + } else { + const payload_llvm_ty = try o.lowerType(pt, payload_ty); return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, ""); } - return self.wip.extractValue(operand, &.{offset}, ""); } fn airErrUnionErr( @@ -3169,17 +3080,18 @@ fn airErrUnionErr( return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), ""); } - const offset = try errUnionErrorFieldIndex(payload_ty, pt); + assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits - if (operand_is_ptr or isByRef(err_union_ty, zcu)) { - if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); + if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu)); - const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); - const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, ""); - return self.wip.load(access_kind, error_type, err_field_ptr, .default, ""); - } + const err_align: InternPool.Alignment = a: { + const err_abi_align = Type.anyerror.abiAlignment(zcu); + if (!operand_is_ptr) break :a err_abi_align; + break :a err_abi_align.minStrict(operand_ty.ptrAlignment(zcu)); + }; - return self.wip.extractValue(operand, &.{offset}, ""); + const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu)); + return self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), ""); } fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { @@ -3198,27 +3110,18 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value const access_kind: Builder.MemoryAccessKind = if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - if (!payload_ty.hasRuntimeBits(zcu)) { - self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); - _ = try self.wip.store(access_kind, non_error_val, operand, err_union_ptr_align.toLlvm()); - return operand; - } - const err_union_llvm_ty = try o.lowerType(pt, err_union_ty); - { - self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); + self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu)); - const err_int_ty = try pt.errorIntType(); - const error_alignment = err_int_ty.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm(); - const error_offset = try errUnionErrorFieldIndex(payload_ty, pt); + { + const error_align = Type.anyerror.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm(); // First set the non-error value. - const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, ""); - _ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment); + const error_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu)); + _ = try self.wip.store(access_kind, non_error_val, error_ptr, error_align); } + // Then return the payload pointer (only if it is used). if (self.liveness.isUnused(inst)) return .none; - - const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt); - return self.wip.gepStruct(err_union_llvm_ty, operand, payload_offset, ""); + return self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu)); } fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value { @@ -3233,7 +3136,6 @@ fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -3241,15 +3143,20 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Val const struct_ty = ty_pl.ty.toType(); const field_index = ty_pl.payload; - const struct_llvm_ty = try o.lowerType(pt, struct_ty); - const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?; assert(self.err_ret_trace != .none); - const field_ptr = try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, ""); - const field_alignment = struct_ty.explicitFieldAlignment(field_index, zcu); + const field_ty = struct_ty.fieldType(field_index, zcu); + const field_offset = struct_ty.structFieldOffset(field_index, zcu); + const field_align = switch (field_offset) { + 0 => struct_ty.abiAlignment(zcu), + else => struct_ty.abiAlignment(zcu).minStrict(.fromLog2Units(@ctz(field_offset))), + }; + + const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset); + const field_ptr_ty = try pt.ptrType(.{ .child = field_ty.toIntern(), - .flags = .{ .alignment = field_alignment }, + .flags = .{ .alignment = field_align }, }); return self.load(field_ptr, field_ptr_ty); } @@ -3290,25 +3197,23 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.V const operand = try self.resolveInst(ty_op.operand); const optional_ty = self.typeOfIndex(inst); if (optional_ty.optionalReprIsPayload(zcu)) return operand; + assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload` const llvm_optional_ty = try o.lowerType(pt, optional_ty); - if (isByRef(optional_ty, zcu)) { - const directReturn = self.isNextRet(body_tail); - const optional_ptr = if (directReturn) - self.ret_ptr - else brk: { - const alignment = optional_ty.abiAlignment(zcu).toLlvm(); - const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment); - break :brk optional_ptr; - }; + const optional_ptr = if (self.isNextRet(body_tail)) + self.ret_ptr + else brk: { + const alignment = optional_ty.abiAlignment(zcu).toLlvm(); + const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment); + break :brk optional_ptr; + }; - const payload_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 0, ""); - const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); - try self.store(payload_ptr, payload_ptr_ty, operand, .none); - const non_null_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 1, ""); - _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default); - return optional_ptr; - } - return self.wip.buildAggregate(llvm_optional_ty, &.{ operand, non_null_bit }, ""); + const payload_ptr = optional_ptr; // payload always at offset 0 + const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); + try self.store(payload_ptr, payload_ptr_ty, operand, .none); + // Non-null bit immediately after payload (no padding because the bit has alignment 1). + const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu)); + _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default); + return optional_ptr; } fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { @@ -3321,34 +3226,25 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Bu const operand = try self.resolveInst(ty_op.operand); const payload_ty = self.typeOf(ty_op.operand); assert(payload_ty.hasRuntimeBits(zcu)); + assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0); const err_un_llvm_ty = try o.lowerType(pt, err_un_ty); - const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt); - const error_offset = try errUnionErrorFieldIndex(payload_ty, pt); - if (isByRef(err_un_ty, zcu)) { - const directReturn = self.isNextRet(body_tail); - const result_ptr = if (directReturn) - self.ret_ptr - else brk: { - const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm(); - const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); - break :brk result_ptr; - }; + const result_ptr = if (self.isNextRet(body_tail)) + self.ret_ptr + else brk: { + const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm(); + const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); + break :brk result_ptr; + }; - const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, ""); - const err_int_ty = try pt.errorIntType(); - const error_alignment = err_int_ty.abiAlignment(pt.zcu).toLlvm(); - _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment); - const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, ""); - const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); - try self.store(payload_ptr, payload_ptr_ty, operand, .none); - return result_ptr; - } - var fields: [2]Builder.Value = undefined; - fields[payload_offset] = operand; - fields[error_offset] = ok_err_code; - return self.wip.buildAggregate(err_un_llvm_ty, &fields, ""); + const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu)); + const error_alignment = Type.anyerror.abiAlignment(pt.zcu).toLlvm(); + _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment); + const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu)); + const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); + try self.store(payload_ptr, payload_ptr_ty, operand, .none); + return result_ptr; } fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { @@ -3361,35 +3257,26 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builde const payload_ty = err_un_ty.errorUnionPayload(zcu); const operand = try self.resolveInst(ty_op.operand); if (!payload_ty.hasRuntimeBits(zcu)) return operand; + assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref const err_un_llvm_ty = try o.lowerType(pt, err_un_ty); - const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt); - const error_offset = try errUnionErrorFieldIndex(payload_ty, pt); - if (isByRef(err_un_ty, zcu)) { - const directReturn = self.isNextRet(body_tail); - const result_ptr = if (directReturn) - self.ret_ptr - else brk: { - const alignment = err_un_ty.abiAlignment(zcu).toLlvm(); - const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); - break :brk result_ptr; - }; + const result_ptr = if (self.isNextRet(body_tail)) + self.ret_ptr + else brk: { + const alignment = err_un_ty.abiAlignment(zcu).toLlvm(); + const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); + break :brk result_ptr; + }; - const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, ""); - const err_int_ty = try pt.errorIntType(); - const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm(); - _ = try self.wip.store(.normal, operand, err_ptr, error_alignment); - const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, ""); - const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); - // TODO store undef to payload_ptr - _ = payload_ptr; - _ = payload_ptr_ty; - return result_ptr; - } - - // TODO set payload bytes to undef - const undef = try o.builder.undefValue(err_un_llvm_ty); - return self.wip.insertValue(undef, operand, &.{error_offset}, ""); + const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu)); + const error_alignment = Type.anyerror.abiAlignment(zcu).toLlvm(); + _ = try self.wip.store(.normal, operand, err_ptr, error_alignment); + const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu)); + const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); + // TODO store undef to payload_ptr + _ = payload_ptr; + _ = payload_ptr_ty; + return result_ptr; } fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { @@ -3804,26 +3691,18 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Bui } fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const ptr = try self.resolveInst(bin_op.lhs); - const offset = try self.resolveInst(bin_op.rhs); + const ptr_or_slice = try self.resolveInst(bin_op.lhs); + const index = try self.resolveInst(bin_op.rhs); const ptr_ty = self.typeOf(bin_op.lhs); - const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu)); - switch (ptr_ty.ptrSize(zcu)) { - // It's a pointer to an array, so according to LLVM we need an extra GEP index. - .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{ - try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), offset, - }, ""), - .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{offset}, ""), - .slice => { - const base = try self.wip.extractValue(ptr, &.{0}, ""); - return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{offset}, ""); - }, - } + const elem_ty = ptr_ty.indexableElem(zcu); + const ptr = switch (ptr_ty.ptrSize(zcu)) { + .one, .many, .c => ptr_or_slice, + .slice => try self.wip.extractValue(ptr_or_slice, &.{0}, ""), + }; + return self.ptraddScaled(ptr, index, elem_ty.abiSize(zcu)); } fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { @@ -3832,22 +3711,18 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const zcu = pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const ptr = try self.resolveInst(bin_op.lhs); - const offset = try self.resolveInst(bin_op.rhs); - const negative_offset = try self.wip.neg(offset, ""); + const ptr_or_slice = try self.resolveInst(bin_op.lhs); + const llvm_usize_ty = try o.lowerType(pt, .usize); const ptr_ty = self.typeOf(bin_op.lhs); - const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu)); - switch (ptr_ty.ptrSize(zcu)) { - // It's a pointer to an array, so according to LLVM we need an extra GEP index. - .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{ - try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), negative_offset, - }, ""), - .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{negative_offset}, ""), - .slice => { - const base = try self.wip.extractValue(ptr, &.{0}, ""); - return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{negative_offset}, ""); - }, - } + const elem_ty = ptr_ty.indexableElem(zcu); + const ptr = switch (ptr_ty.ptrSize(zcu)) { + .one, .many, .c => ptr_or_slice, + .slice => try self.wip.extractValue(ptr_or_slice, &.{0}, ""), + }; + const scale_val = try o.builder.intValue(llvm_usize_ty, -@as(i65, elem_ty.abiSize(zcu))); + const positive_index = try self.resolveInst(bin_op.rhs); + const negative_offset = try self.wip.bin(.@"mul nsw", positive_index, scale_val, ""); + return self.ptradd(ptr, negative_offset); } fn airOverflow( @@ -3868,6 +3743,7 @@ fn airOverflow( const lhs_ty = self.typeOf(extra.lhs); const scalar_ty = lhs_ty.scalarType(zcu); const inst_ty = self.typeOfIndex(inst); + assert(isByRef(inst_ty, zcu)); // auto structs are by-ref const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; const llvm_inst_ty = try o.lowerType(pt, inst_ty); @@ -3878,28 +3754,22 @@ fn airOverflow( const result_val = try self.wip.extractValue(results, &.{0}, ""); const overflow_bit = try self.wip.extractValue(results, &.{1}, ""); - const result_index = o.llvmFieldIndex(inst_ty, 0).?; - const overflow_index = o.llvmFieldIndex(inst_ty, 1).?; + const result_alignment = inst_ty.abiAlignment(zcu).toLlvm(); + const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment); - if (isByRef(inst_ty, zcu)) { - const result_alignment = inst_ty.abiAlignment(zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment); - { - const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, ""); - _ = try self.wip.store(.normal, result_val, field_ptr, result_alignment); - } - { - const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, overflow_index, ""); - _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1)); - } + { + // Store to 'result: IntType' field + const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(0, zcu)); + _ = try self.wip.store(.normal, result_val, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm()); + } - return alloca_inst; + { + // Store to 'overflow: u1' field + const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(1, zcu)); + _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1)); } - var fields: [2]Builder.Value = undefined; - fields[result_index] = result_val; - fields[overflow_index] = overflow_bit; - return self.wip.buildAggregate(llvm_inst_ty, &fields, ""); + return alloca_inst; } fn buildElementwiseCall( @@ -4227,6 +4097,7 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const lhs_scalar_ty = lhs_ty.scalarType(zcu); const dest_ty = self.typeOfIndex(inst); + assert(isByRef(dest_ty, zcu)); // auto structs are by-ref const llvm_dest_ty = try o.lowerType(pt, dest_ty); const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); @@ -4239,27 +4110,22 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, ""); - const result_index = o.llvmFieldIndex(dest_ty, 0).?; - const overflow_index = o.llvmFieldIndex(dest_ty, 1).?; + const result_alignment = dest_ty.abiAlignment(zcu).toLlvm(); + const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment); - if (isByRef(dest_ty, zcu)) { - const result_alignment = dest_ty.abiAlignment(zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment); - { - const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, ""); - _ = try self.wip.store(.normal, result, field_ptr, result_alignment); - } - { - const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, overflow_index, ""); - _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1)); - } - return alloca_inst; + { + // Store to 'result: IntType' field + const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(0, zcu)); + _ = try self.wip.store(.normal, result, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm()); } - var fields: [2]Builder.Value = undefined; - fields[result_index] = result; - fields[overflow_index] = overflow_bit; - return self.wip.buildAggregate(llvm_dest_ty, &fields, ""); + { + // Store to 'overflow: u1' field + const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(1, zcu)); + _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1)); + } + + return alloca_inst; } fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { @@ -4678,7 +4544,8 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty } if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) { - const elem_ty = operand_ty.childType(zcu); + const elem_ty = inst_ty.childType(zcu); + assert(elem_ty.toIntern() == operand_scalar_ty.toIntern()); if (!result_is_ref) { return self.todo("implement bitcast vector to non-ref array", .{}); } @@ -4690,22 +4557,19 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty } else { // If the ABI size of the element type is not evenly divisible by size in bits; // a simple bitcast will not work, and we fall back to extractelement. - const llvm_usize = try o.lowerType(pt, Type.usize); - const usize_zero = try o.builder.intValue(llvm_usize, 0); + const elem_size = elem_ty.abiSize(zcu); const vector_len = operand_ty.arrayLen(zcu); var i: u64 = 0; while (i < vector_len) : (i += 1) { - const elem_ptr = try self.wip.gep(.inbounds, llvm_dest_ty, array_ptr, &.{ - usize_zero, try o.builder.intValue(llvm_usize, i), - }, ""); - const elem = - try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), ""); - _ = try self.wip.store(.normal, elem, elem_ptr, .default); + const arr_elem_ptr = try self.ptraddConst(array_ptr, i * elem_size); + const vec_elem = try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), ""); + _ = try self.wip.store(.normal, vec_elem, arr_elem_ptr, .default); } } return array_ptr; } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) { const elem_ty = operand_ty.childType(zcu); + assert(elem_ty.toIntern() == inst_scalar_ty.toIntern()); const llvm_vector_ty = try o.lowerType(pt, inst_ty); if (!operand_is_ref) return self.todo("implement bitcast non-ref array to vector", .{}); @@ -4718,20 +4582,15 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty } else { // If the ABI size of the element type is not evenly divisible by size in bits; // a simple bitcast will not work, and we fall back to extractelement. - const array_llvm_ty = try o.lowerType(pt, operand_ty); const elem_llvm_ty = try o.lowerType(pt, elem_ty); - const llvm_usize = try o.lowerType(pt, Type.usize); - const usize_zero = try o.builder.intValue(llvm_usize, 0); + const elem_size = elem_ty.abiSize(zcu); const vector_len = operand_ty.arrayLen(zcu); var vector = try o.builder.poisonValue(llvm_vector_ty); var i: u64 = 0; while (i < vector_len) : (i += 1) { - const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, operand, &.{ - usize_zero, try o.builder.intValue(llvm_usize, i), - }, ""); - const elem = try self.wip.load(.normal, elem_llvm_ty, elem_ptr, .default, ""); - vector = - try self.wip.insertElement(vector, elem, try o.builder.intValue(.i32, i), ""); + const arr_elem_ptr = try self.ptraddConst(operand, i * elem_size); + const arr_elem = try self.wip.load(.normal, elem_llvm_ty, arr_elem_ptr, .default, ""); + vector = try self.wip.insertElement(vector, arr_elem, try o.builder.intValue(.i32, i), ""); } return vector; } @@ -5066,10 +4925,23 @@ fn airCmpxchg( return self.wip.select(.normal, success_bit, zero, payload, ""); } + assert(isByRef(optional_ty, zcu)); + comptime assert(optional_layout_version == 3); const non_null_bit = try self.wip.not(success_bit, ""); - return buildOptional(self, optional_ty, payload, non_null_bit); + + const payload_align = operand_ty.abiAlignment(zcu).toLlvm(); + const alloca_inst = try self.buildAlloca(try o.lowerType(pt, optional_ty), payload_align); + + // Payload is always the first field at offset 0, so address is `alloca_inst` + _ = try self.wip.store(.normal, payload, alloca_inst, payload_align); + + // Non-null bit is after payload with no padding because it has alignment 1 + const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu)); + _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, comptime .fromByteUnits(1)); + + return alloca_inst; } fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { @@ -5315,13 +5187,15 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value const end_block = try self.wip.block(1, "InlineMemsetEnd"); const llvm_usize_ty = try o.lowerType(pt, Type.usize); - const len = switch (ptr_ty.ptrSize(zcu)) { - .slice => try self.wip.extractValue(dest_slice, &.{1}, ""), - .one => try o.builder.intValue(llvm_usize_ty, ptr_ty.childType(zcu).arrayLen(zcu)), + const end_ptr = switch (ptr_ty.ptrSize(zcu)) { + .slice => try self.ptraddScaled( + dest_ptr, + try self.wip.extractValue(dest_slice, &.{1}, ""), + elem_abi_size, + ), + .one => try self.ptraddConst(dest_ptr, ptr_ty.childType(zcu).abiSize(zcu)), .many, .c => unreachable, }; - const elem_llvm_ty = try o.lowerType(pt, elem_ty); - const end_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, dest_ptr, &.{len}, ""); _ = try self.wip.br(loop_block); self.wip.cursor = .{ .block = loop_block }; @@ -5343,9 +5217,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value self.disable_intrinsics, ); } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align); - const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{ - try o.builder.intValue(llvm_usize_ty, 1), - }, ""); + const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size); _ = try self.wip.br(loop_block); self.wip.cursor = .{ .block = end_block }; @@ -5408,14 +5280,13 @@ fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const un_ptr_ty = self.typeOf(bin_op.lhs); const un_ty = un_ptr_ty.childType(zcu); const layout = un_ty.unionGetLayout(zcu); - if (layout.tag_size == 0) return .none; + assert(layout.tag_size != 0); const access_kind: Builder.MemoryAccessKind = if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; @@ -5429,8 +5300,7 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { _ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm()); return .none; } - const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); - const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(pt, un_ty), union_ptr, tag_index, ""); + const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset()); const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) { 0 => union_ptr_align, else => |off| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))), @@ -5446,20 +5316,20 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const un_ty = self.typeOf(ty_op.operand); const layout = un_ty.unionGetLayout(zcu); - if (layout.tag_size == 0) return .none; - const union_handle = try self.resolveInst(ty_op.operand); + assert(layout.tag_size != 0); + const union_ptr = try self.resolveInst(ty_op.operand); if (isByRef(un_ty, zcu)) { const llvm_un_ty = try o.lowerType(pt, un_ty); if (layout.payload_size == 0) - return self.wip.load(.normal, llvm_un_ty, union_handle, .default, ""); + return self.wip.load(.normal, llvm_un_ty, union_ptr, .default, ""); const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); - const tag_field_ptr = try self.wip.gepStruct(llvm_un_ty, union_handle, tag_index, ""); + const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset()); const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index]; return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, ""); } else { - if (layout.payload_size == 0) return union_handle; + if (layout.payload_size == 0) return union_ptr; const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); - return self.wip.extractValue(union_handle, &.{tag_index}, ""); + return self.wip.extractValue(union_ptr, &.{tag_index}, ""); } } @@ -5646,19 +5516,18 @@ fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const o = self.object; const pt = self.pt; + const zcu = pt.zcu; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); const slice_ty = self.typeOfIndex(inst); const slice_llvm_ty = try o.lowerType(pt, slice_ty); // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed. - const extended_operand = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), ""); + const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), ""); const error_name_table_ptr = try self.getErrorNameTable(); - const error_name_table = - try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, ""); - const error_name_ptr = - try self.wip.gep(.inbounds, slice_llvm_ty, error_name_table, &.{extended_operand}, ""); + const error_name_table = try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, ""); + const error_name_ptr = try self.ptraddScaled(error_name_table, operand_usize, slice_ty.abiSize(zcu)); return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, ""); } @@ -6074,8 +5943,9 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } return vector; }, - .@"struct" => { - if (zcu.typeToPackedStruct(result_ty)) |struct_type| { + .@"struct" => switch (result_ty.containerLayout(zcu)) { + .@"packed" => { + const struct_type = ip.loadStructType(result_ty.toIntern()); const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type); const big_bits = backing_int_ty.bitSize(zcu); const int_ty = try o.builder.intType(@intCast(big_bits)); @@ -6100,69 +5970,68 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { running_bits += ty_bit_size; } return running_int; - } - - assert(result_ty.containerLayout(zcu) != .@"packed"); - - if (isByRef(result_ty, zcu)) { + }, + .auto, .@"extern" => { + assert(isByRef(result_ty, zcu)); // TODO in debug builds init to undef so that the padding will be 0xaa // even if we fully populate the fields. - const alignment = result_ty.abiAlignment(zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment); + const struct_align = result_ty.abiAlignment(zcu); + const alloca_inst = try self.buildAlloca(llvm_result_ty, struct_align.toLlvm()); - for (elements, 0..) |elem, i| { - if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue; + for (elements, 0..) |elem, field_index| { + if (result_ty.structFieldIsComptime(field_index, zcu)) continue; + const field_ty = result_ty.fieldType(field_index, zcu); + if (!field_ty.hasRuntimeBits(zcu)) continue; + const offset = result_ty.structFieldOffset(field_index, zcu); + const field_ptr = try self.ptraddConst(alloca_inst, offset); + const field_ptr_align: InternPool.Alignment = switch (offset) { + 0 => struct_align, + else => struct_align.minStrict(.fromLog2Units(@ctz(offset))), + }; - const llvm_elem = try self.resolveInst(elem); - const llvm_i = o.llvmFieldIndex(result_ty, i).?; - const field_ptr = try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, ""); + const llvm_field_val = try self.resolveInst(elem); - const field_ptr_ty = try pt.ptrType(.{ - .child = self.typeOf(elem).toIntern(), - .flags = .{ - .alignment = result_ty.explicitFieldAlignment(i, zcu), - }, - }); - try self.store(field_ptr, field_ptr_ty, llvm_elem, .none); + if (isByRef(field_ty, zcu)) { + _ = try self.wip.callMemCpy( + field_ptr, + field_ptr_align.toLlvm(), + llvm_field_val, + field_ty.abiAlignment(zcu).toLlvm(), + try o.builder.intValue(try o.lowerType(pt, .usize), field_ty.abiSize(zcu)), + .normal, + self.disable_intrinsics, + ); + } else { + _ = try self.wip.store( + .normal, + llvm_field_val, + field_ptr, + field_ptr_align.toLlvm(), + ); + } } return alloca_inst; - } else { - var result = try o.builder.poisonValue(llvm_result_ty); - for (elements, 0..) |elem, i| { - if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue; - - const llvm_elem = try self.resolveInst(elem); - const llvm_i = o.llvmFieldIndex(result_ty, i).?; - result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, ""); - } - return result; - } + }, }, .array => { assert(isByRef(result_ty, zcu)); - const llvm_usize = try o.lowerType(pt, Type.usize); - const usize_zero = try o.builder.intValue(llvm_usize, 0); const alignment = result_ty.abiAlignment(zcu).toLlvm(); const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment); const array_info = result_ty.arrayInfo(zcu); - const elem_ptr_ty = try pt.ptrType(.{ - .child = array_info.elem_type.toIntern(), - }); + const elem_ptr_ty = try pt.singleConstPtrType(array_info.elem_type); + + const elem_size = array_info.elem_type.abiSize(zcu); for (elements, 0..) |elem, i| { - const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{ - usize_zero, try o.builder.intValue(llvm_usize, i), - }, ""); + const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i); const llvm_elem = try self.resolveInst(elem); try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none); } if (array_info.sentinel) |sent_val| { - const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{ - usize_zero, try o.builder.intValue(llvm_usize, array_info.len), - }, ""); + const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len); const llvm_elem = try self.resolveValue(sent_val); try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none); } @@ -6188,95 +6057,30 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const layout = Type.getUnionLayout(union_obj, zcu); - const tag_int_val = blk: { - const tag_ty = union_ty.unionTagTypeHypothetical(zcu); - const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index); - break :blk tag_val.intFromEnum(zcu); - }; - if (layout.payload_size == 0) { - if (layout.tag_size == 0) { - return .none; - } - assert(!isByRef(union_ty, zcu)); - var big_int_space: Value.BigIntSpace = undefined; - const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu); - return try o.builder.bigIntValue(union_llvm_ty, tag_big_int); - } + assert(layout.payload_size != 0); // otherwise the value would be comptime-known assert(isByRef(union_ty, zcu)); - // The llvm type of the alloca will be the named LLVM union type, and will not - // necessarily match the format that we need, depending on which tag is active. - // We must construct the correct unnamed struct type here, in order to then set - // the fields appropriately. + const alignment = layout.abi_align.toLlvm(); const result_ptr = try self.buildAlloca(union_llvm_ty, alignment); const llvm_payload = try self.resolveInst(extra.init); const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]); - const field_llvm_ty = try o.lowerType(pt, field_ty); - const field_size = field_ty.abiSize(zcu); - const field_align = union_ty.explicitFieldAlignment(extra.field_index, zcu); - const llvm_usize = try o.lowerType(pt, Type.usize); - const usize_zero = try o.builder.intValue(llvm_usize, 0); - assert(field_ty.hasRuntimeBits(zcu)); - const llvm_union_ty = t: { - const payload_ty = p: { - if (field_size == layout.payload_size) { - break :p field_llvm_ty; - } - const padding_len = layout.payload_size - field_size; - break :p try o.builder.structType(.@"packed", &.{ - field_llvm_ty, try o.builder.arrayType(padding_len, .i8), - }); - }; - if (layout.tag_size == 0) break :t try o.builder.structType(.normal, &.{payload_ty}); - const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); - var fields: [3]Builder.Type = undefined; - var fields_len: usize = 2; - if (layout.tag_align.compare(.gte, layout.payload_align)) { - fields = .{ tag_ty, payload_ty, undefined }; - } else { - fields = .{ payload_ty, tag_ty, undefined }; - } - if (layout.padding != 0) { - fields[fields_len] = try o.builder.arrayType(layout.padding, .i8); - fields_len += 1; - } - break :t try o.builder.structType(.normal, fields[0..fields_len]); - }; - - // Now we follow the layout as expressed above with GEP instructions to set the - // tag and the payload. - const field_ptr_ty = try pt.ptrType(.{ - .child = field_ty.toIntern(), - .flags = .{ .alignment = field_align }, - }); - if (layout.tag_size == 0) { - const indices = [3]Builder.Value{ usize_zero, .@"0", .@"0" }; - const len: usize = if (field_size == layout.payload_size) 2 else 3; - const field_ptr = - try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], ""); - try self.store(field_ptr, field_ptr_ty, llvm_payload, .none); - return result_ptr; - } - { - const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align)); - const indices: [3]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, payload_index), .@"0" }; - const len: usize = if (field_size == layout.payload_size) 2 else 3; - const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], ""); - try self.store(field_ptr, field_ptr_ty, llvm_payload, .none); + const payload_ptr_ty = try pt.ptrType(.{ + .child = field_ty.toIntern(), + .flags = .{ .alignment = layout.payload_align }, + }); + const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset()); + try self.store(payload_ptr, payload_ptr_ty, llvm_payload, .none); } - { - const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); - const indices: [2]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, tag_index) }; - const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, &indices, ""); - const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); - var big_int_space: Value.BigIntSpace = undefined; - const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu); - const llvm_tag = try o.builder.bigIntValue(tag_ty, tag_big_int); - const tag_alignment = Type.fromInterned(union_obj.enum_tag_type).abiAlignment(zcu).toLlvm(); - _ = try self.wip.store(.normal, llvm_tag, field_ptr, tag_alignment); + + if (layout.tag_size != 0) { + const tag_ty: Type = .fromInterned(union_obj.enum_tag_type); + const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index); + const llvm_tag_val = try o.lowerValue(pt, tag_val.toIntern()); + const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset()); + _ = try self.wip.store(.normal, llvm_tag_val.toValue(), tag_ptr, layout.tag_align.toLlvm()); } return result_ptr; @@ -6369,7 +6173,6 @@ fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; const pt = self.pt; const target = pt.zcu.getTarget(); @@ -6387,11 +6190,8 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { // Load the work_group_* member from the struct as u16. // Just treat the dispatch pointer as an array of u16 to keep things simple. - const workgroup_size_ptr = try self.wip.gep(.inbounds, .i16, dispatch_ptr, &.{ - try o.builder.intValue(try o.lowerType(pt, Type.usize), 2 + dimension), - }, ""); - const workgroup_size_alignment = comptime Builder.Alignment.fromByteUnits(2); - return self.wip.load(.normal, .i16, workgroup_size_ptr, workgroup_size_alignment, ""); + const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2); + return self.wip.load(.normal, .i16, workgroup_size_ptr, comptime .fromByteUnits(2), ""); }, .nvptx, .nvptx64 => { return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid"); @@ -6435,88 +6235,45 @@ fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index { return variable_index; } -/// Assumes the optional is not pointer-like and payload has bits. +/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits. fn optCmpNull( self: *FuncGen, cond: Builder.IntegerCondition, - opt_llvm_ty: Builder.Type, - opt_handle: Builder.Value, - is_by_ref: bool, + opt_ty: Type, + opt_ptr: Builder.Value, access_kind: Builder.MemoryAccessKind, ) Allocator.Error!Builder.Value { - const o = self.object; - const field = b: { - if (is_by_ref) { - const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, ""); - break :b try self.wip.load(access_kind, .i8, field_ptr, .default, ""); - } - break :b try self.wip.extractValue(opt_handle, &.{1}, ""); - }; + const zcu = self.pt.zcu; + assert(isByRef(opt_ty, zcu)); comptime assert(optional_layout_version == 3); - - return self.wip.icmp(cond, field, try o.builder.intValue(.i8, 0), ""); + // Non-null bit is always after the payload, with no padding because it has alignment 1. + const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu)); + const non_null = try self.wip.load(access_kind, .i8, non_null_ptr, .default, ""); + return self.wip.icmp(cond, non_null, try self.object.builder.intValue(.i8, 0), ""); } -/// Assumes the optional is not pointer-like and payload has bits. +/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits. fn optPayloadHandle( fg: *FuncGen, - opt_llvm_ty: Builder.Type, - opt_handle: Builder.Value, + opt_ptr: Builder.Value, opt_ty: Type, can_elide_load: bool, ) !Builder.Value { const pt = fg.pt; const zcu = pt.zcu; + assert(isByRef(opt_ty, zcu)); const payload_ty = opt_ty.optionalChild(zcu); - if (isByRef(opt_ty, zcu)) { - // We have a pointer and we need to return a pointer to the first field. - const payload_ptr = try fg.wip.gepStruct(opt_llvm_ty, opt_handle, 0, ""); - - const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm(); - if (isByRef(payload_ty, zcu)) { - if (can_elide_load) - return payload_ptr; - - return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); - } - return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_alignment); + // Payload is first field so always at the same address as the optional itself. + const payload_ptr = opt_ptr; + + const payload_align = payload_ty.abiAlignment(zcu).toLlvm(); + if (isByRef(payload_ty, zcu)) { + if (can_elide_load) return payload_ptr; + return fg.loadByRef(payload_ptr, payload_ty, payload_align, .normal); + } else { + return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_align); } - - assert(!isByRef(payload_ty, zcu)); - return fg.wip.extractValue(opt_handle, &.{0}, ""); -} - -fn buildOptional( - self: *FuncGen, - optional_ty: Type, - payload: Builder.Value, - non_null_bit: Builder.Value, -) !Builder.Value { - const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; - const optional_llvm_ty = try o.lowerType(pt, optional_ty); - const non_null_field = try self.wip.cast(.zext, non_null_bit, .i8, ""); - - if (isByRef(optional_ty, zcu)) { - const payload_alignment = optional_ty.abiAlignment(pt.zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment); - - { - const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, ""); - _ = try self.wip.store(.normal, payload, field_ptr, payload_alignment); - } - { - const non_null_alignment = comptime Builder.Alignment.fromByteUnits(1); - const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 1, ""); - _ = try self.wip.store(.normal, non_null_field, field_ptr, non_null_alignment); - } - - return alloca_inst; - } - - return self.wip.buildAggregate(optional_llvm_ty, &.{ payload, non_null_field }, ""); } fn fieldPtr( @@ -6525,7 +6282,6 @@ fn fieldPtr( aggregate_ptr_ty: Type, field_index: u32, ) !Builder.Value { - const o = self.object; const pt = self.pt; const zcu = pt.zcu; const aggregate_ty = aggregate_ptr_ty.childType(zcu); @@ -6534,35 +6290,12 @@ fn fieldPtr( // bit offset is represented in the pointer *type*. return aggregate_ptr; } - switch (aggregate_ty.zigTypeTag(zcu)) { - .@"struct" => { - if (!aggregate_ty.hasRuntimeBits(zcu)) { - return aggregate_ptr; - } - const struct_llvm_ty = try o.lowerType(pt, aggregate_ty); - if (o.llvmFieldIndex(aggregate_ty, field_index)) |llvm_field_index| { - return self.wip.gepStruct(struct_llvm_ty, aggregate_ptr, llvm_field_index, ""); - } else { - // If we found no index then this means this is a zero sized field at the - // end of the struct. Treat our struct pointer as an array of two and get - // the index to the element at index `1` to get a pointer to the end of - // the struct. - const llvm_index = try o.builder.intValue( - try o.lowerType(pt, Type.usize), - @intFromBool(aggregate_ty.hasRuntimeBits(zcu)), - ); - return self.wip.gep(.inbounds, struct_llvm_ty, aggregate_ptr, &.{llvm_index}, ""); - } - }, - .@"union" => { - const layout = aggregate_ty.unionGetLayout(zcu); - if (layout.payload_size == 0) return aggregate_ptr; - const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)); - const union_llvm_ty = try o.lowerType(pt, aggregate_ty); - return self.wip.gepStruct(union_llvm_ty, aggregate_ptr, payload_index, ""); - }, + const offset: u64 = switch (aggregate_ty.zigTypeTag(zcu)) { + .@"struct" => aggregate_ty.structFieldOffset(field_index, zcu), + .@"union" => aggregate_ty.unionGetLayout(zcu).payloadOffset(), else => unreachable, - } + }; + return self.ptraddConst(aggregate_ptr, offset); } /// Load a value and, if needed, mask out padding bits for non byte-sized integer values. @@ -6828,11 +6561,8 @@ fn valgrindClientRequest( break :a array_ptr; } else fg.valgrind_client_request_array; const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 }; - const zero = try o.builder.intValue(llvm_usize, 0); for (array_elements, 0..) |elem, i| { - const elem_ptr = try fg.wip.gep(.inbounds, array_llvm_ty, array_ptr, &.{ - zero, try o.builder.intValue(llvm_usize, i), - }, ""); + const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu)); _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment); } @@ -7605,13 +7335,8 @@ pub fn buildAllocaInner( /// This is the one source of truth for whether a type is passed around as an LLVM pointer, /// or as an LLVM value. -pub fn isByRef(ty: Type, zcu: *Zcu) bool { - // For tuples and structs, if there are more than this many non-void - // fields, then we make it byref, otherwise byval. - const max_fields_byval = 0; - const ip = &zcu.intern_pool; - - switch (ty.zigTypeTag(zcu)) { +pub fn isByRef(ty: Type, zcu: *const Zcu) bool { + return switch (ty.zigTypeTag(zcu)) { .type, .comptime_int, .comptime_float, @@ -7632,62 +7357,24 @@ pub fn isByRef(ty: Type, zcu: *Zcu) bool { .@"enum", .vector, .@"anyframe", - => return false, + => false, - .array, .frame => return ty.hasRuntimeBits(zcu), - .@"struct" => { - const struct_type = switch (ip.indexToKey(ty.toIntern())) { - .tuple_type => |tuple| { - var count: usize = 0; - for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| { - if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue; + .array, + .error_union, + .frame, + => ty.hasRuntimeBits(zcu), - count += 1; - if (count > max_fields_byval) return true; - if (isByRef(Type.fromInterned(field_ty), zcu)) return true; - } - return false; - }, - .struct_type => ip.loadStructType(ty.toIntern()), - else => unreachable, - }; + .optional => ty.hasRuntimeBits(zcu) and !ty.optionalReprIsPayload(zcu), - // Packed structs are represented to LLVM as integers. - if (struct_type.layout == .@"packed") return false; - - const field_types = struct_type.field_types.get(ip); - var it = struct_type.iterateRuntimeOrder(ip); - var count: usize = 0; - while (it.next()) |field_index| { - count += 1; - if (count > max_fields_byval) return true; - const field_ty = Type.fromInterned(field_types[field_index]); - if (isByRef(field_ty, zcu)) return true; - } - return false; + .@"struct" => switch (ty.containerLayout(zcu)) { + .@"packed" => false, + .auto, .@"extern" => ty.hasRuntimeBits(zcu), }, .@"union" => switch (ty.containerLayout(zcu)) { - .@"packed" => return false, - else => return ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu), + .@"packed" => false, + else => ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu), }, - .error_union => { - const payload_ty = ty.errorUnionPayload(zcu); - if (!payload_ty.hasRuntimeBits(zcu)) { - return false; - } - return true; - }, - .optional => { - const payload_ty = ty.optionalChild(zcu); - if (!payload_ty.hasRuntimeBits(zcu)) { - return false; - } - if (ty.optionalReprIsPayload(zcu)) { - return false; - } - return true; - }, - } + }; } /// If the operand type of an atomic operation is not byte sized we need to @@ -7713,16 +7400,27 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E } } -fn errUnionPayloadFieldIndex(payload_ty: Type, pt: Zcu.PerThread) !u1 { - const zcu = pt.zcu; - const err_int_ty = try pt.errorIntType(); - return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.gt, payload_ty.abiAlignment(zcu))); +fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value { + if (offset == 0) return ptr; + const llvm_usize_ty = try fg.object.lowerType(fg.pt, .usize); + const offset_val = try fg.object.builder.intValue(llvm_usize_ty, offset); + return fg.ptradd(ptr, offset_val); } - -fn errUnionErrorFieldIndex(payload_ty: Type, pt: Zcu.PerThread) !u1 { - const zcu = pt.zcu; - const err_int_ty = try pt.errorIntType(); - return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.lte, payload_ty.abiAlignment(zcu))); +fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value { + switch (scale) { + 0 => return ptr, + 1 => return fg.ptradd(ptr, index), + else => { + const o = fg.object; + const llvm_usize_ty = try o.lowerType(fg.pt, .usize); + const scale_val = try o.builder.intValue(llvm_usize_ty, scale); + const offset = try fg.wip.bin(.@"mul nuw", index, scale_val, ""); + return fg.ptradd(ptr, offset); + }, + } +} +fn ptradd(fg: *FuncGen, ptr: Builder.Value, offset: Builder.Value) Allocator.Error!Builder.Value { + return fg.wip.gep(.inbounds, .i8, ptr, &.{offset}, ""); } fn compilerRtIntBits(bits: u16) ?u16 { @@ -8128,6 +7826,7 @@ const Package = @import("../../Package.zig"); const InternPool = @import("../../InternPool.zig"); const Value = @import("../../Value.zig"); const Type = @import("../../Type.zig"); +const codegen = @import("../../codegen.zig"); const target_util = @import("../../target.zig"); const libcFloatPrefix = target_util.libcFloatPrefix; -- 2.54.0 From bd8088bb9826b40be281d5cca89eac7ff7b76241 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Mon, 23 Mar 2026 11:26:23 +0000 Subject: [PATCH 03/15] llvm: random enhancements and cleanups --- src/Air.zig | 17 +- src/Compilation.zig | 2 +- src/codegen/llvm.zig | 207 ++++----- src/codegen/llvm/FuncGen.zig | 807 ++++++++++++++++------------------- 4 files changed, 482 insertions(+), 551 deletions(-) diff --git a/src/Air.zig b/src/Air.zig index 3f7314e11674a00620f6cf9e0da5d6fbb8671d28..ff88990bfa52c8dfe7ed5a3a108f70ba9c11e598 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -870,13 +870,20 @@ pub const Inst = struct { /// Uses the `pl_op` field, payload represents the index of the target memory. wasm_memory_grow, - /// Returns `true` if and only if the operand, an integer with - /// the same size as the error integer type, is less than the - /// total number of errors in the Module. + /// Returns `true` if and only if the operand, an integer with the same + /// size as the error integer type, is less than *or equal to* the total + /// number of errors in the Zcu. The "or equal to" is a consequence of + /// value 0 being reserved for the "non-error" status in error unions. + /// MLUGG TODO: rename this instruction to `cmp_lte_errors_len` + /// + /// This instruction exists (as opposed to just using `cmp_lte` against + /// a constant) because the number of errors in the Zcu is not known + /// until `Compilation.flush`. Before then, semantic analysis could + /// discover new errors at any time. + /// /// Result type is always `bool`. + /// /// Uses the `un_op` field. - /// Note that the number of errors in the Module cannot be considered stable until - /// flush(). cmp_lt_errors_len, /// Returns pointer to current error return trace. diff --git a/src/Compilation.zig b/src/Compilation.zig index 3eb971f96b2ce0f46b4dbb65df8593c736ae4008..fb0e5cc83f02513600cd0fc0fe92032c20fa3337 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2485,7 +2485,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic, if (use_llvm) { if (opt_zcu) |zcu| { - zcu.llvm_object = try LlvmObject.create(arena, comp); + zcu.llvm_object = try LlvmObject.create(arena, zcu); } } diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index d9f7b4b54fbc0e76910e65b62c2636a3e7004ade..c5ea440405d1ceca9f3f9a1ff2266764bbdb76ad 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -549,7 +549,7 @@ pub const Object = struct { /// type from the global error set. debug_anyerror_fwd_ref: Builder.Metadata.Optional, - target: *const std.Target, + zcu: *Zcu, /// Ideally we would use `llvm_module.getNamedFunction` to go from *Decl to LLVM function, /// but that has some downsides: /// * we have to compute the fully qualified name every time we want to do the lookup @@ -574,9 +574,12 @@ pub const Object = struct { /// Note that the values are not added until `emit`, when all errors in /// the compilation are known. error_name_table: Builder.Variable.Index, - - /// Memoizes a null `?usize` value. - null_opt_usize: Builder.Constant, + /// Constant variable whose value is the number of errors in the Zcu. + /// + /// Initially `.none`---populated lazily by `getErrorsLen`. + /// + /// If this is not `.none`, the variable's initializer is set in `emit`. + errors_len_variable: Builder.Variable.Index, /// Values for `@llvm.used`. used: std.ArrayList(Builder.Constant), @@ -585,10 +588,11 @@ pub const Object = struct { const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type); - pub fn create(arena: Allocator, comp: *Compilation) !Ptr { + pub fn create(arena: Allocator, zcu: *Zcu) !Ptr { dev.check(.llvm_backend); + const comp = zcu.comp; const gpa = comp.gpa; - const target = &comp.root_mod.resolved_target.result; + const target = zcu.getTarget(); const llvm_target_triple = try targetTriple(arena, target); var builder = try Builder.init(.{ @@ -616,10 +620,7 @@ pub const Object = struct { // way already, but here we throw all that sweet information // into the garbage can by converting into absolute paths. What // a terrible tragedy. - const compile_unit_dir = blk: { - const zcu = comp.zcu orelse break :blk comp.dirs.cwd; - break :blk try zcu.main_mod.root.toAbsolute(comp.dirs, arena); - }; + const compile_unit_dir = try zcu.main_mod.root.toAbsolute(comp.dirs, arena); const debug_file = try builder.debugFile( try builder.metadataString(comp.root_name), @@ -665,14 +666,14 @@ pub const Object = struct { .debug_file_map = .empty, .debug_types = .empty, .debug_anyerror_fwd_ref = .none, - .target = target, + .zcu = zcu, .nav_map = .empty, .uav_map = .empty, .enum_tag_name_map = .empty, .named_enum_map = .empty, .type_map = .empty, .error_name_table = .none, - .null_opt_usize = .no_init, + .errors_len_variable = .none, .used = .empty, }; return obj; @@ -722,7 +723,7 @@ pub const Object = struct { name_variable_index.setLinkage(.private, &o.builder); name_variable_index.setMutability(.constant, &o.builder); name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); - name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); + name_variable_index.setAlignment(comptime .fromByteUnits(1), &o.builder); llvm_error.* = try o.builder.structConst(llvm_slice_ty, &.{ name_variable_index.toConst(&o.builder), @@ -730,52 +731,17 @@ pub const Object = struct { }); } - const table_variable_index = try o.builder.addVariable(.empty, llvm_table_ty, .default); - try table_variable_index.setInitializer( + try o.error_name_table.setInitializer( try o.builder.arrayConst(llvm_table_ty, llvm_errors), &o.builder, ); - table_variable_index.setLinkage(.private, &o.builder); - table_variable_index.setMutability(.constant, &o.builder); - table_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); - table_variable_index.setAlignment( - slice_ty.abiAlignment(zcu).toLlvm(), - &o.builder, - ); - - try o.error_name_table.setInitializer(table_variable_index.toConst(&o.builder), &o.builder); - } - - fn genCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !void { - // If there is no such function in the module, it means the source code does not need it. - const name = o.builder.strtabStringIfExists(lt_errors_fn_name) orelse return; - const llvm_fn = o.builder.getGlobal(name) orelse return; - const errors_len = pt.zcu.intern_pool.global_error_set.getNamesFromMainThread().len; - - var wip = try Builder.WipFunction.init(&o.builder, .{ - .function = llvm_fn.ptrConst(&o.builder).kind.function, - .strip = true, - }); - defer wip.deinit(); - wip.cursor = .{ .block = try wip.block(0, "Entry") }; - - // Example source of the following LLVM IR: - // fn __zig_lt_errors_len(index: u16) bool { - // return index <= total_errors_len; - // } - - const lhs = wip.arg(0); - const rhs = try o.builder.intValue(try o.errorIntType(pt), errors_len); - const is_lt = try wip.icmp(.ule, lhs, rhs, ""); - _ = try wip.ret(is_lt); - try wip.finish(); } - fn genModuleLevelAssembly(object: *Object, pt: Zcu.PerThread) Allocator.Error!void { + fn genModuleLevelAssembly(object: *Object) Allocator.Error!void { const b = &object.builder; const gpa = b.gpa; b.module_asm.clearRetainingCapacity(); - for (pt.zcu.global_assembly.values()) |assembly| { + for (object.zcu.global_assembly.values()) |assembly| { try b.module_asm.ensureUnusedCapacity(gpa, assembly.len + 1); b.module_asm.appendSliceAssumeCapacity(assembly); b.module_asm.appendAssumeCapacity('\n'); @@ -808,9 +774,13 @@ pub const Object = struct { const diags = &comp.link_diags; { + if (o.errors_len_variable != .none) { + const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len; + const init_val = try o.builder.intConst(try o.errorIntType(), errors_len); + try o.errors_len_variable.setInitializer(init_val, &o.builder); + } try o.genErrorNameTable(pt); - try o.genCmpLtErrorsLenFunction(pt); - try o.genModuleLevelAssembly(pt); + try o.genModuleLevelAssembly(); if (o.used.items.len > 0) { const array_llvm_ty = try o.builder.arrayType(o.used.items.len, .ptr); @@ -827,7 +797,7 @@ pub const Object = struct { if (!o.builder.strip) { if (o.debug_anyerror_fwd_ref.unwrap()) |fwd_ref| { - const debug_anyerror_type = try o.lowerDebugAnyerrorType(pt); + const debug_anyerror_type = try o.lowerDebugAnyerrorType(); o.builder.resolveDebugForwardReference(fwd_ref, debug_anyerror_type); } @@ -1454,7 +1424,7 @@ pub const Object = struct { } const file, const subprogram = if (!wip.strip) debug_info: { - const file = try o.getDebugFile(pt, file_scope); + const file = try o.getDebugFile(file_scope); const line_number = zcu.navSrcLine(func.owner_nav) + 1; const is_internal_linkage = ip.indexToKey(nav.resolved.?.value) != .@"extern"; @@ -1658,7 +1628,7 @@ pub const Object = struct { const line_number = zcu.navSrcLine(nav_index) + 1; if (!mod.strip) { - const debug_file = try o.getDebugFile(pt, file_scope); + const debug_file = try o.getDebugFile(file_scope); const debug_global_var = try o.builder.debugGlobalVar( try o.builder.metadataString(nav.name.toSlice(ip)), // Name @@ -1796,7 +1766,7 @@ pub const Object = struct { try global_index.rename(main_exp_name, &o.builder); break :i global_index; } - const llvm_addr_space = toLlvmAddressSpace(.generic, o.target); + const llvm_addr_space = toLlvmAddressSpace(.generic, zcu.getTarget()); const variable_index = try o.builder.addVariable( main_exp_name, try o.lowerType(pt, Type.fromInterned(ip.typeOf(exported_value))), @@ -2002,13 +1972,13 @@ pub const Object = struct { } } - pub fn getDebugFile(o: *Object, pt: Zcu.PerThread, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata { + pub fn getDebugFile(o: *Object, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata { const gpa = o.gpa; const gop = try o.debug_file_map.getOrPut(gpa, file_index); errdefer assert(o.debug_file_map.remove(file_index)); if (gop.found_existing) return gop.value_ptr.*; - const path = pt.zcu.fileByIndex(file_index).path; - const abs_path = try path.toAbsolute(pt.zcu.comp.dirs, gpa); + const path = o.zcu.fileByIndex(file_index).path; + const abs_path = try path.toAbsolute(o.zcu.comp.dirs, gpa); defer gpa.free(abs_path); gop.value_ptr.* = try o.builder.debugFile( @@ -2035,8 +2005,8 @@ pub const Object = struct { assert(!o.builder.strip); const gpa = o.gpa; - const target = o.target; const zcu = pt.zcu; + const target = zcu.getTarget(); const ip = &zcu.intern_pool; const name = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)}); @@ -2386,7 +2356,7 @@ pub const Object = struct { const struct_type = zcu.typeToStruct(ty).?; - const file = try o.getDebugFile(pt, struct_type.zir_index.resolveFile(ip)); + const file = try o.getDebugFile(struct_type.zir_index.resolveFile(ip)); const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| try o.namespaceToDebugScope(pt, parent_namespace) else @@ -2453,7 +2423,7 @@ pub const Object = struct { .@"union" => { const union_type = ip.loadUnionType(ty.toIntern()); - const file = try o.getDebugFile(pt, union_type.zir_index.resolveFile(ip)); + const file = try o.getDebugFile(union_type.zir_index.resolveFile(ip)); const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| try o.namespaceToDebugScope(pt, parent_namespace) else @@ -2611,7 +2581,7 @@ pub const Object = struct { ); }, .@"enum" => { - const file = try o.getDebugFile(pt, ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip)); + const file = try o.getDebugFile(ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip)); const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| try o.namespaceToDebugScope(pt, parent_namespace) else @@ -2672,7 +2642,7 @@ pub const Object = struct { return o.builder.debugSignedType(name, 0); } - const file = try o.getDebugFile(pt, ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip)); + const file = try o.getDebugFile(ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip)); const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace| try o.namespaceToDebugScope(pt, parent_namespace) else @@ -2697,8 +2667,8 @@ pub const Object = struct { } /// Called in `emit` so that the global error set is fully populated. - fn lowerDebugAnyerrorType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Metadata { - const zcu = pt.zcu; + fn lowerDebugAnyerrorType(o: *Object) Allocator.Error!Builder.Metadata { + const zcu = o.zcu; const ip = &zcu.intern_pool; const gpa = zcu.comp.gpa; @@ -2732,7 +2702,7 @@ pub const Object = struct { null, // file o.debug_compile_unit.unwrap().?, // scope 0, // line - try o.getDebugType(pt, try pt.intType(.unsigned, error_set_bits)), + try o.builder.debugUnsignedType(null, error_set_bits), Type.anyerror.abiSize(zcu) * 8, Type.anyerror.abiAlignment(zcu).toByteUnits().? * 8, try o.builder.metadataTuple(enumerators), @@ -2744,7 +2714,7 @@ pub const Object = struct { fn namespaceToDebugScope(o: *Object, pt: Zcu.PerThread, namespace_index: InternPool.NamespaceIndex) !Builder.Metadata { const zcu = pt.zcu; const namespace = zcu.namespacePtr(namespace_index); - if (namespace.parent == .none) return try o.getDebugFile(pt, namespace.file_scope); + if (namespace.parent == .none) return try o.getDebugFile(namespace.file_scope); return o.getDebugType(pt, .fromInterned(namespace.owner_type)); } @@ -3086,8 +3056,8 @@ pub const Object = struct { return variable_index; } - pub fn errorIntType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Type { - return o.builder.intType(pt.zcu.errorSetBits()); + pub fn errorIntType(o: *Object) Allocator.Error!Builder.Type { + return o.builder.intType(o.zcu.errorSetBits()); } pub fn lowerType(o: *Object, pt: Zcu.PerThread, t: Type) Allocator.Error!Builder.Type { @@ -3146,7 +3116,7 @@ pub const Object = struct { .bool_type => .i1, .void_type => .void, .type_type => unreachable, - .anyerror_type => try o.errorIntType(pt), + .anyerror_type => try o.errorIntType(), .comptime_int_type, .comptime_float_type, .noreturn_type, @@ -3168,7 +3138,7 @@ pub const Object = struct { .optional_noreturn_type => unreachable, .anyerror_void_error_union_type, .adhoc_inferred_error_set_type, - => try o.errorIntType(pt), + => try o.errorIntType(), .generic_poison_type, .empty_tuple_type, => unreachable, @@ -3241,7 +3211,7 @@ pub const Object = struct { .error_union_type => |error_union_type| { // Must stay in sync with `codegen.errUnionPayloadOffset`. // See logic in `lowerPtr`. - const error_type = try o.errorIntType(pt); + const error_type = try o.errorIntType(); if (!Type.fromInterned(error_union_type.payload_type).hasRuntimeBits(zcu)) return error_type; const payload_type = try o.lowerType(pt, Type.fromInterned(error_union_type.payload_type)); @@ -3474,7 +3444,7 @@ pub const Object = struct { }, .enum_type => try o.lowerType(pt, t.intTagType(zcu)), .func_type => |func_type| try o.lowerFnType(pt, func_type), - .error_set_type, .inferred_error_set_type => try o.errorIntType(pt), + .error_set_type, .inferred_error_set_type => try o.errorIntType(), // values, not types .undef, .simple_value, @@ -3624,7 +3594,7 @@ pub const Object = struct { }, .err => |err| { const int = try pt.getErrorValue(err.name); - const llvm_int = try o.builder.intConst(try o.errorIntType(pt), int); + const llvm_int = try o.builder.intConst(try o.errorIntType(), int); return llvm_int; }, .error_union => |error_union| { @@ -4288,29 +4258,40 @@ pub const Object = struct { if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder); } - /// MLUGG TODO: this is super dumb - pub fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index { - const name = try o.builder.strtabString(lt_errors_fn_name); - if (o.builder.getGlobal(name)) |llvm_fn| return llvm_fn.ptrConst(&o.builder).kind.function; + pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index { + if (o.error_name_table != .none) return o.error_name_table; - const zcu = pt.zcu; - const target = &zcu.root_mod.resolved_target.result; - const function_index = try o.builder.addFunction( - try o.builder.fnType(.i1, &.{try o.errorIntType(pt)}, .normal), - name, - toLlvmAddressSpace(.generic, target), + const name = try o.builder.strtabString("__zig_error_name_table"); + // TODO: Address space + const variable_index = try o.builder.addVariable(name, .ptr, .default); + variable_index.setLinkage(.private, &o.builder); + variable_index.setMutability(.constant, &o.builder); + variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); + variable_index.setAlignment( + Type.slice_const_u8_sentinel_0.abiAlignment(o.zcu).toLlvm(), + &o.builder, ); - var attributes: Builder.FunctionAttributes.Wip = .{}; - defer attributes.deinit(&o.builder); - try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer); + o.error_name_table = variable_index; + return variable_index; + } - function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); - function_index.setCallConv(.fastcc, &o.builder); - function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); - return function_index; + pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index { + const builder = &o.builder; + if (o.errors_len_variable == .none) { + const llvm_err_int_ty = try o.errorIntType(); + const name = try builder.strtabString("__zig_errors_len"); + const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default); + variable_index.setLinkage(.private, builder); + variable_index.setMutability(.constant, builder); + variable_index.setUnnamedAddr(.unnamed_addr, builder); + variable_index.setAlignment(Type.errorAbiAlignment(o.zcu).toLlvm(), builder); + o.errors_len_variable = variable_index; + } + return o.errors_len_variable; } + /// MLUGG TODO: this also needs incremental updates dumbass pub fn getEnumTagNameFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index { const zcu = pt.zcu; const ip = &zcu.intern_pool; @@ -4394,7 +4375,25 @@ pub const Object = struct { return o.lazy_abi_aligns.items[@intFromEnum(index)]; } - pub fn updateIsNamedEnumValueFunction( + pub fn getIsNamedEnumValueFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index { + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + + const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern()); + if (gop.found_existing) return gop.value_ptr.*; + errdefer assert(o.named_enum_map.remove(enum_ty.toIntern())); + const function_index = try o.builder.addFunction( + // Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type. + // TODO: change the builder API so we don't need to do this. + try o.builder.fnType(.void, &.{}, .normal), + try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}), + toLlvmAddressSpace(.generic, zcu.getTarget()), + ); + gop.value_ptr.* = function_index; + try o.updateIsNamedEnumValueFunction(pt, enum_ty, function_index); + return function_index; + } + fn updateIsNamedEnumValueFunction( o: *Object, pt: Zcu.PerThread, enum_ty: Type, @@ -4445,6 +4444,24 @@ pub const Object = struct { try wip.finish(); } + + pub fn getLibcFunction( + o: *Object, + fn_name: Builder.StrtabString, + param_types: []const Builder.Type, + return_type: Builder.Type, + ) Allocator.Error!Builder.Function.Index { + if (o.builder.getGlobal(fn_name)) |global| return switch (global.ptrConst(&o.builder).kind) { + .alias => |alias| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function, + .function => |function| function, + .variable, .replaced => unreachable, + }; + return o.builder.addFunction( + try o.builder.fnType(return_type, param_types, .normal), + fn_name, + toLlvmAddressSpace(.generic, o.zcu.getTarget()), + ); + } }; const CallingConventionInfo = struct { @@ -4796,8 +4813,6 @@ const struct_layout_version = 2; // https://github.com/llvm/llvm-project/issues/56585/ is fixed pub const optional_layout_version = 3; -const lt_errors_fn_name = "__zig_lt_errors_len"; - pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void { switch (arch) { .aarch64, .aarch64_be => { diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index 30cbbab76d5efff5747f4b383c54ea056055e20b..4b1edf9ff9c1d8076c9c736f71ed73f1b69de636 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -1,7 +1,5 @@ const FuncGen = @This(); -pub const Error = Zcu.CodegenFailError; - object: *Object, nav_index: InternPool.Nav.Index, pt: Zcu.PerThread, @@ -63,7 +61,17 @@ disable_intrinsics: bool, /// Have we seen loads or stores involving `allowzero` pointers? allowzero_access: bool, -fn todo(fg: *FuncGen, comptime format: []const u8, args: anytype) Error { +/// In general, codegen should never emit errors; we cannot report useful source locations for them +/// and they don't really play nicely with incremental compilation. The LLVM backend mostly obeys +/// this rule. Where it does not, it calls `todo` to emit an error, and results in this error set +/// being used for the function +/// +/// Please avoid using this error set in new code. Ideally, every fallible function in this file +/// should have the error set `Allocator.Error`. +const TodoError = Zcu.CodegenFailError; + +/// Avoid introducing new calls to this function---see documentation comment on `TodoError`. +fn todo(fg: *FuncGen, comptime format: []const u8, args: anytype) TodoError { @branchHint(.cold); return fg.pt.zcu.codegenFail( fg.nav_index, @@ -167,7 +175,11 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant { } } -pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) Error!void { +fn lowerType(fg: *const FuncGen, ty: Type) Allocator.Error!Builder.Type { + return fg.object.lowerType(fg.pt, ty); +} + +pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void { const o = self.object; const zcu = self.pt.zcu; const ip = &zcu.intern_pool; @@ -443,13 +455,16 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air // Instructions which may be `noreturn`. .block => res: { - const res = try self.airBlock(inst); - if (self.typeOfIndex(inst).isNoReturn(zcu)) return; + const block = self.air.unwrapBlock(inst); + const res = try self.lowerBlock(inst, null, block.body); + if (block.ty.isNoReturn(zcu)) return; break :res res; }, .dbg_inline_block => res: { - const res = try self.airDbgInlineBlock(inst); - if (self.typeOfIndex(inst).isNoReturn(zcu)) return; + const block = self.air.unwrapDbgBlock(inst); + self.arg_inline_index = 0; + const res = try self.lowerBlock(inst, block.func, block.body); + if (block.ty.isNoReturn(zcu)) return; break :res res; }, .call, .call_always_tail, .call_never_tail, .call_never_inline => |tag| res: { @@ -479,7 +494,7 @@ fn genBodyDebugScope( maybe_inline_func: ?InternPool.Index, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint, -) Error!void { +) TodoError!void { const o = self.object; if (self.wip.strip) return self.genBody(body, coverage_point); @@ -508,7 +523,7 @@ fn genBodyDebugScope( const file_scope = zcu.navFileScopeIndex(func.owner_nav); const mod = zcu.fileByIndex(file_scope).mod.?; - self.file = try o.getDebugFile(pt, file_scope); + self.file = try o.getDebugFile(file_scope); self.base_line = zcu.navSrcLine(func.owner_nav); const line_number = self.base_line + 1; @@ -562,7 +577,7 @@ const CallAttr = enum { AlwaysInline, }; -fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value { +fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) Allocator.Error!Builder.Value { const air_call = self.air.unwrapCall(inst); const args = air_call.args; const o = self.object; @@ -598,7 +613,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif } const ret_ptr = if (!sret) null else blk: { - const llvm_ret_ty = try o.lowerType(pt, return_type); + const llvm_ret_ty = try self.lowerType(return_type); try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder); const alignment = return_type.abiAlignment(zcu).toLlvm(); @@ -620,7 +635,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif const arg = args[it.zig_index - 1]; const param_ty = self.typeOf(arg); const llvm_arg = try self.resolveInst(arg); - const llvm_param_ty = try o.lowerType(pt, param_ty); + const llvm_param_ty = try self.lowerType(param_ty); if (isByRef(param_ty, zcu)) { const alignment = param_ty.abiAlignment(zcu).toLlvm(); const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, ""); @@ -649,7 +664,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif const llvm_arg = try self.resolveInst(arg); const alignment = param_ty.abiAlignment(zcu).toLlvm(); - const param_llvm_ty = try o.lowerType(pt, param_ty); + const param_llvm_ty = try self.lowerType(param_ty); const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment); if (isByRef(param_ty, zcu)) { const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, ""); @@ -719,7 +734,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif llvm_arg = ptr; } - const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?); + const float_ty = try self.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?); const array_ty = try o.builder.arrayType(count, float_ty); const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, ""); @@ -760,7 +775,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif .byref => { const param_index = it.zig_index - 1; const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); - const param_llvm_ty = try o.lowerType(pt, param_ty); + const param_llvm_ty = try self.lowerType(param_ty); const alignment = param_ty.abiAlignment(zcu).toLlvm(); try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty); }, @@ -812,7 +827,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif }, toLlvmCallConvTag(fn_info.cc, target).?, try attributes.finish(&o.builder), - try o.lowerType(pt, zig_fn_ty), + try self.lowerType(zig_fn_ty), llvm_fn, llvm_args.items, "", @@ -826,7 +841,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif return .none; } - const llvm_ret_ty = try o.lowerType(pt, return_type); + const llvm_ret_ty = try self.lowerType(return_type); if (ret_ptr) |rp| { if (isByRef(return_type, zcu)) { return rp; @@ -864,7 +879,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif } } -fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) !void { +fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!void { const o = fg.object; const pt = fg.pt; const zcu = pt.zcu; @@ -888,7 +903,7 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) !void { _ = try fg.wip.@"unreachable"(); } -fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void { +fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!void { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -910,7 +925,7 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void { // https://github.com/ziglang/zig/issues/15337 break :undef; } - const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu)); + const len = try o.builder.intValue(try self.lowerType(.usize), ret_ty.abiSize(zcu)); _ = try self.wip.callMemSet( self.ret_ptr, ptr_ty.ptrAlignment(zcu).toLlvm(), @@ -946,7 +961,7 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void { // Functions with an empty error set are emitted with an error code // return type and return zero so they can be function pointers coerced // to functions that return anyerror. - _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0)); + _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0)); } else { _ = try self.wip.retVoid(); } @@ -961,7 +976,7 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void { if (val_is_undef and safety) { const llvm_ret_ty = operand.typeOfWip(&self.wip); const rp = try self.buildAlloca(llvm_ret_ty, alignment); - const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu)); + const len = try o.builder.intValue(try self.lowerType(Type.usize), ret_ty.abiSize(zcu)); _ = try self.wip.callMemSet( rp, alignment, @@ -997,7 +1012,7 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void { return; } -fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void { +fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -1011,7 +1026,7 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void { // Functions with an empty error set are emitted with an error code // return type and return zero so they can be function pointers coerced // to functions that return anyerror. - _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0)); + _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0)); } else { _ = try self.wip.retVoid(); } @@ -1028,25 +1043,22 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void { return; } -fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; - const pt = self.pt; +fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const list = try self.resolveInst(ty_op.operand); const arg_ty = ty_op.ty.toType(); - const llvm_arg_ty = try o.lowerType(pt, arg_ty); + const llvm_arg_ty = try self.lowerType(arg_ty); return self.wip.vaArg(list, llvm_arg_ty, ""); } -fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const src_list = try self.resolveInst(ty_op.operand); const va_list_ty = ty_op.ty.toType(); - const llvm_va_list_ty = try o.lowerType(pt, va_list_ty); + const llvm_va_list_ty = try self.lowerType(va_list_ty); const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); @@ -1058,7 +1070,7 @@ fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, ""); } -fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const src_list = try self.resolveInst(un_op); @@ -1066,12 +1078,11 @@ fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return .none; } -fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const va_list_ty = self.typeOfIndex(inst); - const llvm_va_list_ty = try o.lowerType(pt, va_list_ty); + const llvm_va_list_ty = try self.lowerType(va_list_ty); const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); @@ -1088,7 +1099,7 @@ fn airCmp( inst: Air.Inst.Index, op: math.CompareOperator, fast: Builder.FastMathKind, -) !Builder.Value { +) Allocator.Error!Builder.Value { const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -1097,7 +1108,7 @@ fn airCmp( return self.cmp(fast, op, operand_ty, lhs, rhs); } -fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data; @@ -1109,21 +1120,20 @@ fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind return self.cmp(fast, cmp_op, vec_ty, lhs, rhs); } -fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); - const llvm_fn = try o.getCmpLtErrorsLenFunction(pt); - return self.wip.call( + const errors_len_ptr = try o.getErrorsLen(); + const errors_len_val = try self.wip.load( .normal, - .fastcc, - .none, - llvm_fn.typeOf(&o.builder), - llvm_fn.toValue(&o.builder), - &.{operand}, + try o.errorIntType(), + errors_len_ptr.toValue(&o.builder), + Type.errorAbiAlignment(o.zcu).toLlvm(), "", ); + // Despite the name, this instruction is actually lte. MLUGG TODO RENAME + return self.wip.icmp(.ule, operand, errors_len_val, ""); } fn cmp( @@ -1228,18 +1238,12 @@ fn cmp( return self.wip.icmp(cond, lhs, rhs, ""); } -fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const block = self.air.unwrapBlock(inst); - return self.lowerBlock(inst, null, block.body); -} - fn lowerBlock( self: *FuncGen, inst: Air.Inst.Index, maybe_inline_func: ?InternPool.Index, body: []const Air.Inst.Index, -) !Builder.Value { - const o = self.object; +) TodoError!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const inst_ty = self.typeOfIndex(inst); @@ -1267,7 +1271,7 @@ fn lowerBlock( // Create a phi node only if the block returns a value. if (have_block_result) { - const raw_llvm_ty = try o.lowerType(pt, inst_ty); + const raw_llvm_ty = try self.lowerType(inst_ty); const llvm_ty: Builder.Type = ty: { // If the zig tag type is a function, this represents an actual function body; not // a pointer to it. LLVM IR allows the call instruction to use function bodies instead @@ -1289,7 +1293,7 @@ fn lowerBlock( } } -fn airBr(self: *FuncGen, inst: Air.Inst.Index) !void { +fn airBr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void { const zcu = self.pt.zcu; const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br; const block = self.blocks.get(branch.block_inst).?; @@ -1306,7 +1310,7 @@ fn airBr(self: *FuncGen, inst: Air.Inst.Index) !void { _ = try self.wip.br(block.parent_bb); } -fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) !void { +fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void { const repeat = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat; const loop_bb = self.loops.get(repeat.loop_inst).?; loop_bb.ptr(&self.wip).incoming += 1; @@ -1318,7 +1322,7 @@ fn lowerSwitchDispatch( switch_inst: Air.Inst.Index, cond_ref: Air.Inst.Ref, dispatch_info: SwitchDispatchInfo, -) !void { +) Allocator.Error!void { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -1384,7 +1388,7 @@ fn lowerSwitchDispatch( const table_index = try self.wip.conv( .unsigned, try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""), - try o.lowerType(pt, .usize), + try self.lowerType(.usize), "", ); const target_ptr_ptr = try self.ptraddScaled( @@ -1409,7 +1413,7 @@ fn lowerSwitchDispatch( // The switch prongs will correspond to our scalar cases. Ranges will // be handled by conditional branches in the `else` prong. - const llvm_usize = try o.lowerType(pt, Type.usize); + const llvm_usize = try self.lowerType(Type.usize); const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder)) try self.wip.cast(.ptrtoint, cond, llvm_usize, "") else @@ -1501,13 +1505,13 @@ fn lowerSwitchDispatch( } } -fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) !void { +fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void { const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br; const dispatch_info = self.switch_dispatch_info.get(br.block_inst).?; return self.lowerSwitchDispatch(br.block_inst, br.operand, dispatch_info); } -fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void { +fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) TodoError!void { const cond_br = self.air.unwrapCondBr(inst); const cond = try self.resolveInst(cond_br.condition); const then_body = cond_br.then_body; @@ -1567,7 +1571,7 @@ fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void { // No need to reset the insert cursor since this instruction is noreturn. } -fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { +fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) TodoError!Builder.Value { const unwrapped_try = self.air.unwrapTry(inst); const err_union = try self.resolveInst(unwrapped_try.error_union); const body = unwrapped_try.else_body; @@ -1576,7 +1580,7 @@ fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { return lowerTry(self, err_union, body, err_union_ty, false, .none, is_unused, err_cold); } -fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value { +fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) TodoError!Builder.Value { const zcu = self.pt.zcu; const unwrapped_try = self.air.unwrapTryPtr(inst); const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr); @@ -1599,13 +1603,13 @@ fn lowerTry( operand_ptr_align: InternPool.Alignment, is_unused: bool, err_cold: bool, -) !Builder.Value { +) TodoError!Builder.Value { const o = fg.object; const pt = fg.pt; const zcu = pt.zcu; const payload_ty = err_union_ty.errorUnionPayload(zcu); const payload_has_bits = payload_ty.hasRuntimeBits(zcu); - const error_type = try o.errorIntType(pt); + const error_type = try o.errorIntType(); const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{ operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)), @@ -1657,11 +1661,11 @@ fn lowerTry( } else if (isByRef(payload_ty, zcu)) { return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal); } else { - return fg.wip.load(.normal, try o.lowerType(pt, payload_ty), payload_ptr, payload_align.toLlvm(), ""); + return fg.wip.load(.normal, try fg.lowerType(payload_ty), payload_ptr, payload_align.toLlvm(), ""); } } -fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void { +fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) TodoError!void { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -1909,7 +1913,7 @@ fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) ?[2]Value return .{ min.?, max.? }; } -fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void { +fn airLoop(self: *FuncGen, inst: Air.Inst.Index) TodoError!void { const block = self.air.unwrapBlock(inst); const body = block.body; const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time @@ -1922,21 +1926,21 @@ fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void { try self.genBodyDebugScope(null, body, .none); } -fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand_ty = self.typeOf(ty_op.operand); const array_ty = operand_ty.childType(zcu); - const llvm_usize = try o.lowerType(pt, Type.usize); + const llvm_usize = try self.lowerType(.usize); const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu)); - const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst)); + const slice_llvm_ty = try self.lowerType(self.typeOfIndex(inst)); const operand = try self.resolveInst(ty_op.operand); return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, ""); } -fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -1949,7 +1953,7 @@ fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const dest_ty = self.typeOfIndex(inst); const dest_scalar_ty = dest_ty.scalarType(zcu); - const dest_llvm_ty = try o.lowerType(pt, dest_ty); + const dest_llvm_ty = try self.lowerType(dest_ty); const target = zcu.getTarget(); if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv( @@ -1987,7 +1991,7 @@ fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { extended = try self.wip.cast(.bitcast, extended, param_type, ""); } - const libc_fn = try self.getLibcFunction(fn_name, &.{param_type}, dest_llvm_ty); + const libc_fn = try o.getLibcFunction(fn_name, &.{param_type}, dest_llvm_ty); return self.wip.call( .normal, .ccc, @@ -2003,7 +2007,7 @@ fn airIntFromFloat( self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind, -) !Builder.Value { +) TodoError!Builder.Value { _ = fast; const o = self.object; @@ -2018,7 +2022,7 @@ fn airIntFromFloat( const dest_ty = self.typeOfIndex(inst); const dest_scalar_ty = dest_ty.scalarType(zcu); - const dest_llvm_ty = try o.lowerType(pt, dest_ty); + const dest_llvm_ty = try self.lowerType(dest_ty); if (intrinsicsAllowed(operand_scalar_ty, target)) { // TODO set fast math flag @@ -2052,8 +2056,8 @@ fn airIntFromFloat( compiler_rt_dest_abbrev, }); - const operand_llvm_ty = try o.lowerType(pt, operand_ty); - const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty); + const operand_llvm_ty = try self.lowerType(operand_ty); + const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty); var result = try self.wip.call( .normal, .ccc, @@ -2078,7 +2082,7 @@ fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator. const o = fg.object; const pt = fg.pt; const zcu = pt.zcu; - const llvm_usize = try o.lowerType(pt, Type.usize); + const llvm_usize = try fg.lowerType(.usize); switch (ty.ptrSize(zcu)) { .slice => { const len = try fg.wip.extractValue(ptr, &.{1}, ""); @@ -2098,20 +2102,20 @@ fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator. } } -fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Value { +fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); return self.wip.extractValue(operand, &.{index}, ""); } -fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: u1) !Builder.Value { +fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: u1) Allocator.Error!Builder.Value { const zcu = self.pt.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const slice_ptr = try self.resolveInst(ty_op.operand); return self.ptraddConst(slice_ptr, index * Type.usize.abiSize(zcu)); } -fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -2133,7 +2137,7 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } } -fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; @@ -2146,7 +2150,7 @@ fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.ptraddScaled(base_ptr, index, slice_ty.childType(zcu).abiSize(zcu)); } -fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; @@ -2169,7 +2173,7 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.wip.extractElement(array_llvm_val, rhs, ""); } -fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -2189,7 +2193,7 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.load(ptr, ptr_ty); } -fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; @@ -2207,7 +2211,7 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)); } -fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data; const struct_ptr = try self.resolveInst(struct_field.struct_operand); @@ -2219,14 +2223,14 @@ fn airStructFieldPtrIndex( self: *FuncGen, inst: Air.Inst.Index, field_index: u32, -) !Builder.Value { +) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const struct_ptr = try self.resolveInst(ty_op.operand); const struct_ptr_ty = self.typeOf(ty_op.operand); return self.fieldPtr(struct_ptr, struct_ptr_ty, field_index); } -fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -2267,7 +2271,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { }, .float => { // bitcast int->float - return self.wip.cast(.bitcast, field_int_val, try o.lowerType(pt, field_ty), ""); + return self.wip.cast(.bitcast, field_int_val, try self.lowerType(field_ty), ""); }, } } @@ -2292,7 +2296,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } } -fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -2305,8 +2309,8 @@ fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu); if (field_offset == 0) return field_ptr; - const res_ty = try o.lowerType(pt, ty_pl.ty.toType()); - const llvm_usize = try o.lowerType(pt, Type.usize); + const res_ty = try self.lowerType(ty_pl.ty.toType()); + const llvm_usize = try self.lowerType(Type.usize); const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, ""); const base_ptr_int = try self.wip.bin( @@ -2318,19 +2322,19 @@ fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.wip.cast(.inttoptr, base_ptr_int, res_ty, ""); } -fn airNot(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airNot(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); return self.wip.not(operand, ""); } -fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) !void { +fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void { _ = inst; _ = try self.wip.@"unreachable"(); } -fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt; self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1); self.prev_dbg_column = @intCast(dbg_stmt.column + 1); @@ -2345,19 +2349,13 @@ fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return .none; } -fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { _ = self; _ = inst; return .none; } -fn airDbgInlineBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const block = self.air.unwrapDbgBlock(inst); - self.arg_inline_index = 0; - return self.lowerBlock(inst, block.func, block.body); -} - -fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -2390,7 +2388,7 @@ fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return .none; } -fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) !Builder.Value { +fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; @@ -2468,7 +2466,7 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) !Builder.Val return .none; } -fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value { // Eventually, the Zig compiler needs to be reworked to have inline // assembly go through the same parsing code regardless of backend, and // have LLVM-flavored inline assembly be *output* from that assembler. @@ -2530,7 +2528,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const output_inst = try self.resolveInst(output.operand); const output_ty = self.typeOf(output.operand); assert(output_ty.zigTypeTag(zcu) == .pointer); - const elem_llvm_ty = try o.lowerType(pt, output_ty.childType(zcu)); + const elem_llvm_ty = try self.lowerType(output_ty.childType(zcu)); switch (constraint[0]) { '=' => {}, @@ -2568,7 +2566,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { llvm_ret_indirect[output.index] = false; const ret_ty = self.typeOfIndex(inst); - llvm_ret_types[llvm_ret_i] = try o.lowerType(pt, ret_ty); + llvm_ret_types[llvm_ret_i] = try self.lowerType(ret_ty); llvm_ret_i += 1; } @@ -2607,7 +2605,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip); } else { const alignment = arg_ty.abiAlignment(zcu).toLlvm(); - const arg_llvm_ty = try o.lowerType(pt, arg_ty); + const arg_llvm_ty = try self.lowerType(arg_ty); const load_inst = try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, ""); llvm_param_values[llvm_param_i] = load_inst; @@ -2648,7 +2646,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: { if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu)); - break :blk try o.lowerType(pt, if (is_by_ref) arg_ty else arg_ty.childType(zcu)); + break :blk try self.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu)); } else .none; llvm_param_i += 1; @@ -2662,7 +2660,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { if (constraint[0] != '+') continue; const rw_ty = self.typeOf(output.operand); - const llvm_elem_ty = try o.lowerType(pt, rw_ty.childType(zcu)); + const llvm_elem_ty = try self.lowerType(rw_ty.childType(zcu)); if (llvm_ret_indirect[output.index]) { llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index]; llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip); @@ -2855,7 +2853,7 @@ fn airIsNonNull( inst: Air.Inst.Index, operand_is_ptr: bool, cond: Builder.IntegerCondition, -) !Builder.Value { +) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -2863,7 +2861,7 @@ fn airIsNonNull( const operand = try self.resolveInst(un_op); const operand_ty = self.typeOf(un_op); const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; - const optional_llvm_ty = try o.lowerType(pt, optional_ty); + const optional_llvm_ty = try self.lowerType(optional_ty); const payload_ty = optional_ty.optionalChild(zcu); const access_kind: Builder.MemoryAccessKind = @@ -2905,7 +2903,7 @@ fn airIsErr( inst: Air.Inst.Index, cond: Builder.IntegerCondition, operand_is_ptr: bool, -) !Builder.Value { +) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -2914,7 +2912,7 @@ fn airIsErr( const operand_ty = self.typeOf(un_op); const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; const payload_ty = err_union_ty.errorUnionPayload(zcu); - const error_type = try o.errorIntType(pt); + const error_type = try o.errorIntType(); const zero = try o.builder.intValue(error_type, 0); const access_kind: Builder.MemoryAccessKind = @@ -2933,7 +2931,7 @@ fn airIsErr( if (!payload_ty.hasRuntimeBits(zcu)) { const loaded = if (operand_is_ptr) - try self.wip.load(access_kind, try o.lowerType(pt, err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") + try self.wip.load(access_kind, try self.lowerType(err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") else operand; return self.wip.icmp(cond, loaded, zero, ""); @@ -2957,7 +2955,7 @@ fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!B return operand; } -fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { comptime assert(optional_layout_version == 3); const o = self.object; @@ -3002,7 +3000,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value return operand; // payload is at offset 0 } -fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; @@ -3019,8 +3017,7 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.optPayloadHandle(operand, optional_ty, false); } -fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value { - const o = self.object; +fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; @@ -3042,7 +3039,7 @@ fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool if (isByRef(payload_ty, zcu)) { return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); } else { - const payload_llvm_ty = try o.lowerType(pt, payload_ty); + const payload_llvm_ty = try self.lowerType(payload_ty); return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, ""); } } @@ -3051,14 +3048,14 @@ fn airErrUnionErr( self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool, -) !Builder.Value { +) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); const operand_ty = self.typeOf(ty_op.operand); - const error_type = try o.errorIntType(pt); + const error_type = try o.errorIntType(); const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) { if (operand_is_ptr) { @@ -3094,7 +3091,7 @@ fn airErrUnionErr( return self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), ""); } -fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -3105,7 +3102,7 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu); const payload_ty = err_union_ty.errorUnionPayload(zcu); - const non_error_val = try o.builder.intValue(try o.errorIntType(pt), 0); + const non_error_val = try o.builder.intValue(try o.errorIntType(), 0); const access_kind: Builder.MemoryAccessKind = if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; @@ -3124,18 +3121,18 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value return self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu)); } -fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value { +fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) Allocator.Error!Builder.Value { assert(self.err_ret_trace != .none); return self.err_ret_trace; } -fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; self.err_ret_trace = try self.resolveInst(un_op); return .none; } -fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; @@ -3184,7 +3181,7 @@ fn isNextRet( return false; } -fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { +fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -3198,7 +3195,7 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.V const optional_ty = self.typeOfIndex(inst); if (optional_ty.optionalReprIsPayload(zcu)) return operand; assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload` - const llvm_optional_ty = try o.lowerType(pt, optional_ty); + const llvm_optional_ty = try self.lowerType(optional_ty); const optional_ptr = if (self.isNextRet(body_tail)) self.ret_ptr else brk: { @@ -3216,7 +3213,7 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.V return optional_ptr; } -fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { +fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -3227,8 +3224,8 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Bu const payload_ty = self.typeOf(ty_op.operand); assert(payload_ty.hasRuntimeBits(zcu)); assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref - const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0); - const err_un_llvm_ty = try o.lowerType(pt, err_un_ty); + const ok_err_code = try o.builder.intValue(try o.errorIntType(), 0); + const err_un_llvm_ty = try self.lowerType(err_un_ty); const result_ptr = if (self.isNextRet(body_tail)) self.ret_ptr @@ -3247,8 +3244,7 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Bu return result_ptr; } -fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const inst = body_tail[0]; @@ -3258,7 +3254,7 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builde const operand = try self.resolveInst(ty_op.operand); if (!payload_ty.hasRuntimeBits(zcu)) return operand; assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref - const err_un_llvm_ty = try o.lowerType(pt, err_un_ty); + const err_un_llvm_ty = try self.lowerType(err_un_ty); const result_ptr = if (self.isNextRet(body_tail)) self.ret_ptr @@ -3279,29 +3275,27 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builde return result_ptr; } -fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const index = pl_op.payload; - const llvm_usize = try o.lowerType(pt, Type.usize); + const llvm_usize = try self.lowerType(Type.usize); return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{ try o.builder.intValue(.i32, index), }, ""); } -fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const index = pl_op.payload; - const llvm_isize = try o.lowerType(pt, Type.isize); + const llvm_isize = try self.lowerType(Type.isize); return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{ try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand), }, ""); } -fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = fg.object; const pt = fg.pt; const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; @@ -3309,8 +3303,7 @@ fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return llvm_ptr_const.toValue(); } -fn airMin(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -3324,14 +3317,13 @@ fn airMin(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { .normal, .none, if (scalar_ty.isSignedInt(zcu)) .smin else .umin, - &.{try o.lowerType(pt, inst_ty)}, + &.{try self.lowerType(inst_ty)}, &.{ lhs, rhs }, "", ); } -fn airMax(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -3345,24 +3337,22 @@ fn airMax(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { .normal, .none, if (scalar_ty.isSignedInt(zcu)) .smax else .umax, - &.{try o.lowerType(pt, inst_ty)}, + &.{try self.lowerType(inst_ty)}, &.{ lhs, rhs }, "", ); } -fn airSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; - const pt = self.pt; +fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; const ptr = try self.resolveInst(bin_op.lhs); const len = try self.resolveInst(bin_op.rhs); const inst_ty = self.typeOfIndex(inst); - return self.wip.buildAggregate(try o.lowerType(pt, inst_ty), &.{ ptr, len }, ""); + return self.wip.buildAggregate(try self.lowerType(inst_ty), &.{ ptr, len }, ""); } -fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const zcu = self.pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -3379,7 +3369,7 @@ fn airSafeArithmetic( inst: Air.Inst.Index, signed_intrinsic: Builder.Intrinsic, unsigned_intrinsic: Builder.Intrinsic, -) !Builder.Value { +) Allocator.Error!Builder.Value { const o = fg.object; const pt = fg.pt; const zcu = pt.zcu; @@ -3391,7 +3381,7 @@ fn airSafeArithmetic( const scalar_ty = inst_ty.scalarType(zcu); const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; - const llvm_inst_ty = try o.lowerType(pt, inst_ty); + const llvm_inst_ty = try fg.lowerType(inst_ty); const results = try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, ""); @@ -3420,7 +3410,7 @@ fn airSafeArithmetic( return fg.wip.extractValue(results, &.{0}, ""); } -fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3428,8 +3418,7 @@ fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.wip.bin(.add, lhs, rhs, ""); } -fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -3442,13 +3431,13 @@ fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { .normal, .none, if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat", - &.{try o.lowerType(pt, inst_ty)}, + &.{try self.lowerType(inst_ty)}, &.{ lhs, rhs }, "", ); } -fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const zcu = self.pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -3460,7 +3449,7 @@ fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Bui return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"sub nsw" else .@"sub nuw", lhs, rhs, ""); } -fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3468,8 +3457,7 @@ fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.wip.bin(.sub, lhs, rhs, ""); } -fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -3482,13 +3470,13 @@ fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { .normal, .none, if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat", - &.{try o.lowerType(pt, inst_ty)}, + &.{try self.lowerType(inst_ty)}, &.{ lhs, rhs }, "", ); } -fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const zcu = self.pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -3500,7 +3488,7 @@ fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Bui return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"mul nsw" else .@"mul nuw", lhs, rhs, ""); } -fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3508,8 +3496,7 @@ fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.wip.bin(.mul, lhs, rhs, ""); } -fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -3522,13 +3509,13 @@ fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { .normal, .none, if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat", - &.{try o.lowerType(pt, inst_ty)}, + &.{try self.lowerType(inst_ty)}, &.{ lhs, rhs, .@"0" }, "", ); } -fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3537,7 +3524,7 @@ fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); } -fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const zcu = self.pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -3552,7 +3539,7 @@ fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .sdiv else .udiv, lhs, rhs, ""); } -fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -3567,7 +3554,7 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result}); } if (scalar_ty.isSignedInt(zcu)) { - const inst_llvm_ty = try o.lowerType(pt, inst_ty); + const inst_llvm_ty = try self.lowerType(inst_ty); const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; var stack align(@max( @@ -3603,7 +3590,7 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) return self.wip.bin(.udiv, lhs, rhs, ""); } -fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const zcu = self.pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -3620,7 +3607,7 @@ fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) ); } -fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const zcu = self.pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -3636,7 +3623,7 @@ fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Bui .urem, lhs, rhs, ""); } -fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -3644,7 +3631,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Bui const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); const inst_ty = self.typeOfIndex(inst); - const inst_llvm_ty = try o.lowerType(pt, inst_ty); + const inst_llvm_ty = try self.lowerType(inst_ty); const scalar_ty = inst_ty.scalarType(zcu); if (scalar_ty.isRuntimeFloat()) { @@ -3690,7 +3677,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Bui return self.wip.bin(.urem, lhs, rhs, ""); } -fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const zcu = self.pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; @@ -3705,14 +3692,14 @@ fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.ptraddScaled(ptr, index, elem_ty.abiSize(zcu)); } -fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; const ptr_or_slice = try self.resolveInst(bin_op.lhs); - const llvm_usize_ty = try o.lowerType(pt, .usize); + const llvm_usize_ty = try self.lowerType(.usize); const ptr_ty = self.typeOf(bin_op.lhs); const elem_ty = ptr_ty.indexableElem(zcu); const ptr = switch (ptr_ty.ptrSize(zcu)) { @@ -3722,7 +3709,7 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const scale_val = try o.builder.intValue(llvm_usize_ty, -@as(i65, elem_ty.abiSize(zcu))); const positive_index = try self.resolveInst(bin_op.rhs); const negative_offset = try self.wip.bin(.@"mul nsw", positive_index, scale_val, ""); - return self.ptradd(ptr, negative_offset); + return self.wip.gep(.inbounds, .i8, ptr, &.{negative_offset}, ""); } fn airOverflow( @@ -3730,8 +3717,7 @@ fn airOverflow( inst: Air.Inst.Index, signed_intrinsic: Builder.Intrinsic, unsigned_intrinsic: Builder.Intrinsic, -) !Builder.Value { - const o = self.object; +) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; @@ -3746,8 +3732,8 @@ fn airOverflow( assert(isByRef(inst_ty, zcu)); // auto structs are by-ref const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; - const llvm_inst_ty = try o.lowerType(pt, inst_ty); - const llvm_lhs_ty = try o.lowerType(pt, lhs_ty); + const llvm_inst_ty = try self.lowerType(inst_ty); + const llvm_lhs_ty = try self.lowerType(lhs_ty); const results = try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, ""); @@ -3778,7 +3764,7 @@ fn buildElementwiseCall( args_vectors: []const Builder.Value, result_vector: Builder.Value, vector_len: usize, -) !Builder.Value { +) Allocator.Error!Builder.Value { const o = self.object; assert(args_vectors.len <= 3); @@ -3805,25 +3791,6 @@ fn buildElementwiseCall( return result; } -fn getLibcFunction( - self: *FuncGen, - fn_name: Builder.StrtabString, - param_types: []const Builder.Type, - return_type: Builder.Type, -) Allocator.Error!Builder.Function.Index { - const o = self.object; - if (o.builder.getGlobal(fn_name)) |global| return switch (global.ptrConst(&o.builder).kind) { - .alias => |alias| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function, - .function => |function| function, - .variable, .replaced => unreachable, - }; - return o.builder.addFunction( - try o.builder.fnType(return_type, param_types, .normal), - fn_name, - toLlvmAddressSpace(.generic, self.pt.zcu.getTarget()), - ); -} - /// Creates a floating point comparison by lowering to the appropriate /// hardware instruction or softfloat routine for the target fn buildFloatCmp( @@ -3832,13 +3799,13 @@ fn buildFloatCmp( pred: math.CompareOperator, ty: Type, params: [2]Builder.Value, -) !Builder.Value { +) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; const target = zcu.getTarget(); const scalar_ty = ty.scalarType(zcu); - const scalar_llvm_ty = try o.lowerType(pt, scalar_ty); + const scalar_llvm_ty = try self.lowerType(scalar_ty); if (intrinsicsAllowed(scalar_ty, target)) { const cond: Builder.FloatCondition = switch (pred) { @@ -3864,7 +3831,7 @@ fn buildFloatCmp( }; const fn_name = try o.builder.strtabStringFmt("__{s}{s}f2", .{ fn_base_name, compiler_rt_float_abbrev }); - const libc_fn = try self.getLibcFunction(fn_name, &.{ scalar_llvm_ty, scalar_llvm_ty }, .i32); + const libc_fn = try o.getLibcFunction(fn_name, &.{ scalar_llvm_ty, scalar_llvm_ty }, .i32); const int_cond: Builder.IntegerCondition = switch (pred) { .eq => .eq, @@ -3939,13 +3906,13 @@ fn buildFloatOp( ty: Type, comptime params_len: usize, params: [params_len]Builder.Value, -) !Builder.Value { +) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; const target = zcu.getTarget(); const scalar_ty = ty.scalarType(zcu); - const llvm_ty = try o.lowerType(pt, ty); + const llvm_ty = try self.lowerType(ty); if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) { // Some operations are dedicated LLVM instructions, not available as intrinsics @@ -4048,7 +4015,7 @@ fn buildFloatOp( }; const scalar_llvm_ty = llvm_ty.scalarType(&o.builder); - const libc_fn = try self.getLibcFunction( + const libc_fn = try o.getLibcFunction( fn_name, ([1]Builder.Type{scalar_llvm_ty} ** 3)[0..params.len], scalar_llvm_ty, @@ -4069,7 +4036,7 @@ fn buildFloatOp( ); } -fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Bin, pl_op.payload).data; @@ -4081,8 +4048,7 @@ fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend }); } -fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; @@ -4092,15 +4058,19 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const rhs = try self.resolveInst(extra.rhs); const lhs_ty = self.typeOf(extra.lhs); - if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu)) - return self.todo("implement vector shifts with scalar rhs", .{}); + if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu)) { + // `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize` + // features which we do not use. Therefore this branch is currently impossible. + unreachable; + } + const lhs_scalar_ty = lhs_ty.scalarType(zcu); const dest_ty = self.typeOfIndex(inst); assert(isByRef(dest_ty, zcu)); // auto structs are by-ref - const llvm_dest_ty = try o.lowerType(pt, dest_ty); + const llvm_dest_ty = try self.lowerType(dest_ty); - const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); + const casted_rhs = try self.wip.conv(.unsigned, rhs, try self.lowerType(lhs_ty), ""); const result = try self.wip.bin(.shl, lhs, casted_rhs, ""); const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu)) @@ -4128,29 +4098,28 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return alloca_inst; } -fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airAnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); return self.wip.bin(.@"and", lhs, rhs, ""); } -fn airOr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airOr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); return self.wip.bin(.@"or", lhs, rhs, ""); } -fn airXor(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airXor(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); return self.wip.bin(.xor, lhs, rhs, ""); } -fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -4159,19 +4128,21 @@ fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const rhs = try self.resolveInst(bin_op.rhs); const lhs_ty = self.typeOf(bin_op.lhs); - if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) - return self.todo("implement vector shifts with scalar rhs", .{}); + if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) { + // `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize` + // features which we do not use. Therefore this branch is currently impossible. + unreachable; + } const lhs_scalar_ty = lhs_ty.scalarType(zcu); - const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); + const casted_rhs = try self.wip.conv(.unsigned, rhs, try self.lowerType(lhs_ty), ""); return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu)) .@"shl nsw" else .@"shl nuw", lhs, casted_rhs, ""); } -fn airShl(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -4180,14 +4151,16 @@ fn airShl(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const rhs = try self.resolveInst(bin_op.rhs); const lhs_ty = self.typeOf(bin_op.lhs); - if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) - return self.todo("implement vector shifts with scalar rhs", .{}); - - const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); + if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) { + // `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize` + // features which we do not use. Therefore this branch is currently impossible. + unreachable; + } + const casted_rhs = try self.wip.conv(.unsigned, rhs, try self.lowerType(lhs_ty), ""); return self.wip.bin(.shl, lhs, casted_rhs, ""); } -fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -4198,15 +4171,18 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const lhs_ty = self.typeOf(bin_op.lhs); const lhs_info = lhs_ty.intInfo(zcu); - const llvm_lhs_ty = try o.lowerType(pt, lhs_ty); + const llvm_lhs_ty = try self.lowerType(lhs_ty); const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder); const rhs_ty = self.typeOf(bin_op.rhs); - if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) - return self.todo("implement vector shifts with scalar rhs", .{}); + if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) { + // `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize` + // features which we do not use. Therefore this branch is currently impossible. + unreachable; + } const rhs_info = rhs_ty.intInfo(zcu); assert(rhs_info.signedness == .unsigned); - const llvm_rhs_ty = try o.lowerType(pt, rhs_ty); + const llvm_rhs_ty = try self.lowerType(rhs_ty); const llvm_rhs_scalar_ty = llvm_rhs_ty.scalarType(&o.builder); const result = try self.wip.callIntrinsic( @@ -4267,8 +4243,7 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.wip.select(.normal, in_range, result, lhs_sat, ""); } -fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value { - const o = self.object; +fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -4277,11 +4252,14 @@ fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value { const rhs = try self.resolveInst(bin_op.rhs); const lhs_ty = self.typeOf(bin_op.lhs); - if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) - return self.todo("implement vector shifts with scalar rhs", .{}); + if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) { + // `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize` + // features which we do not use. Therefore this branch is currently impossible. + unreachable; + } const lhs_scalar_ty = lhs_ty.scalarType(zcu); - const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), ""); + const casted_rhs = try self.wip.conv(.unsigned, rhs, try self.lowerType(lhs_ty), ""); const is_signed_int = lhs_scalar_ty.isSignedInt(zcu); return self.wip.bin(if (is_exact) @@ -4289,7 +4267,7 @@ fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value { else if (is_signed_int) .ashr else .lshr, lhs, casted_rhs, ""); } -fn airAbs(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -4303,7 +4281,7 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { .normal, .none, .abs, - &.{try o.lowerType(pt, operand_ty)}, + &.{try self.lowerType(operand_ty)}, &.{ operand, try o.builder.intValue(.i1, 0) }, "", ), @@ -4312,13 +4290,13 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } } -fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { +fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value { const o = fg.object; const pt = fg.pt; const zcu = pt.zcu; const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const dest_ty = fg.typeOfIndex(inst); - const dest_llvm_ty = try o.lowerType(pt, dest_ty); + const dest_llvm_ty = try fg.lowerType(dest_ty); const operand = try fg.resolveInst(ty_op.operand); const operand_ty = fg.typeOf(ty_op.operand); const operand_info = operand_ty.intInfo(zcu); @@ -4346,8 +4324,8 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { if (!have_min_check and !have_max_check) break :bounds_check; - const operand_llvm_ty = try o.lowerType(pt, operand_ty); - const operand_scalar_llvm_ty = try o.lowerType(pt, operand_scalar); + const operand_llvm_ty = try fg.lowerType(operand_ty); + const operand_scalar_llvm_ty = try fg.lowerType(operand_scalar); const is_vector = operand_ty.zigTypeTag(zcu) == .vector; assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector)); @@ -4401,7 +4379,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { }, operand, dest_llvm_ty, ""); if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) { - const llvm_fn = try fg.getIsNamedEnumValueFunction(dest_ty); + const llvm_fn = try o.getIsNamedEnumValueFunction(pt, dest_ty); const is_valid_enum_val = try fg.wip.call( .normal, .fastcc, @@ -4422,16 +4400,14 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { return result; } -fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; - const pt = self.pt; +fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); - const dest_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst)); + const dest_llvm_ty = try self.lowerType(self.typeOfIndex(inst)); return self.wip.cast(.trunc, operand, dest_llvm_ty, ""); } -fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -4442,10 +4418,10 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const target = zcu.getTarget(); if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) { - return self.wip.cast(.fptrunc, operand, try o.lowerType(pt, dest_ty), ""); + return self.wip.cast(.fptrunc, operand, try self.lowerType(dest_ty), ""); } else { - const operand_llvm_ty = try o.lowerType(pt, operand_ty); - const dest_llvm_ty = try o.lowerType(pt, dest_ty); + const operand_llvm_ty = try self.lowerType(operand_ty); + const dest_llvm_ty = try self.lowerType(dest_ty); const dest_bits = dest_ty.floatBits(target); const src_bits = operand_ty.floatBits(target); @@ -4453,7 +4429,7 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits), }); - const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty); + const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty); return self.wip.call( .normal, .ccc, @@ -4466,7 +4442,7 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } } -fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -4477,10 +4453,10 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const target = zcu.getTarget(); if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) { - return self.wip.cast(.fpext, operand, try o.lowerType(pt, dest_ty), ""); + return self.wip.cast(.fpext, operand, try self.lowerType(dest_ty), ""); } else { - const operand_llvm_ty = try o.lowerType(pt, operand_ty); - const dest_llvm_ty = try o.lowerType(pt, dest_ty); + const operand_llvm_ty = try self.lowerType(operand_ty); + const dest_llvm_ty = try self.lowerType(dest_ty); const dest_bits = dest_ty.scalarType(zcu).floatBits(target); const src_bits = operand_ty.scalarType(zcu).floatBits(target); @@ -4488,7 +4464,7 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits), }); - const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty); + const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty); if (dest_ty.isVector(zcu)) return self.buildElementwiseCall( libc_fn, &.{operand}, @@ -4507,7 +4483,7 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } } -fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand_ty = self.typeOf(ty_op.operand); const inst_ty = self.typeOfIndex(inst); @@ -4515,13 +4491,13 @@ fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.bitCast(operand, operand_ty, inst_ty); } -fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) !Builder.Value { +fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; const operand_is_ref = isByRef(operand_ty, zcu); const result_is_ref = isByRef(inst_ty, zcu); - const llvm_dest_ty = try o.lowerType(pt, inst_ty); + const llvm_dest_ty = try self.lowerType(inst_ty); if (operand_is_ref and result_is_ref) { // They are both pointers, so just return the same opaque pointer :) @@ -4544,11 +4520,8 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty } if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) { - const elem_ty = inst_ty.childType(zcu); - assert(elem_ty.toIntern() == operand_scalar_ty.toIntern()); - if (!result_is_ref) { - return self.todo("implement bitcast vector to non-ref array", .{}); - } + const elem_ty = operand_scalar_ty; + assert(result_is_ref); // arrays are always by-ref provided they have runtime bits const alignment = inst_ty.abiAlignment(zcu).toLlvm(); const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment); const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; @@ -4569,9 +4542,8 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty return array_ptr; } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) { const elem_ty = operand_ty.childType(zcu); - assert(elem_ty.toIntern() == inst_scalar_ty.toIntern()); - const llvm_vector_ty = try o.lowerType(pt, inst_ty); - if (!operand_is_ref) return self.todo("implement bitcast non-ref array to vector", .{}); + assert(operand_is_ref); // arrays are always by-ref provided they have runtime bits + const llvm_vector_ty = try self.lowerType(inst_ty); const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; if (bitcast_ok) { @@ -4582,7 +4554,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty } else { // If the ABI size of the element type is not evenly divisible by size in bits; // a simple bitcast will not work, and we fall back to extractelement. - const elem_llvm_ty = try o.lowerType(pt, elem_ty); + const elem_llvm_ty = try self.lowerType(elem_ty); const elem_size = elem_ty.abiSize(zcu); const vector_len = operand_ty.arrayLen(zcu); var vector = try o.builder.poisonValue(llvm_vector_ty); @@ -4624,7 +4596,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty return self.wip.cast(.bitcast, operand, llvm_dest_ty, ""); } -fn airArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -4714,7 +4686,7 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return arg_val; } -fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -4723,12 +4695,12 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { if (!pointee_type.hasRuntimeBits(zcu)) return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue(); - const pointee_llvm_ty = try o.lowerType(pt, pointee_type); + const pointee_llvm_ty = try self.lowerType(pointee_type); const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); return self.buildAlloca(pointee_llvm_ty, alignment); } -fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -4737,7 +4709,7 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { if (!ret_ty.hasRuntimeBits(zcu)) return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue(); if (self.ret_ptr != .none) return self.ret_ptr; - const ret_llvm_ty = try o.lowerType(pt, ret_ty); + const ret_llvm_ty = try self.lowerType(ret_ty); const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); return self.buildAlloca(ret_llvm_ty, alignment); } @@ -4753,7 +4725,7 @@ fn buildAlloca( return buildAllocaInner(&self.wip, llvm_ty, alignment, target); } -fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { +fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -4790,7 +4762,7 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { self.maybeMarkAllowZeroAccess(ptr_info); - const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), operand_ty.abiSize(zcu)); + const len = try o.builder.intValue(try self.lowerType(Type.usize), operand_ty.abiSize(zcu)); _ = try self.wip.callMemSet( dest_ptr, ptr_ty.ptrAlignment(zcu).toLlvm(), @@ -4812,7 +4784,7 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { return .none; } -fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = fg.pt; const zcu = pt.zcu; const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; @@ -4823,7 +4795,7 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return fg.load(ptr, ptr_ty); } -fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !void { +fn airTrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void { _ = inst; const target = self.pt.zcu.getTarget(); if ((target.cpu.arch == .mips or target.cpu.arch == .mipsel) and @@ -4847,17 +4819,16 @@ fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !void { _ = try self.wip.@"unreachable"(); } -fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { _ = inst; _ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, ""); return .none; } -fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { _ = inst; const o = self.object; - const pt = self.pt; - const llvm_usize = try o.lowerType(pt, Type.usize); + const llvm_usize = try self.lowerType(Type.usize); if (!target_util.supportsReturnAddress(self.pt.zcu.getTarget(), self.ownerModule().optimize_mode)) { // https://github.com/ziglang/zig/issues/11946 return o.builder.intValue(llvm_usize, 0); @@ -4866,19 +4837,17 @@ fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.wip.cast(.ptrtoint, result, llvm_usize, ""); } -fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { _ = inst; - const o = self.object; - const pt = self.pt; const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, ""); - return self.wip.cast(.ptrtoint, result, try o.lowerType(pt, Type.usize), ""); + return self.wip.cast(.ptrtoint, result, try self.lowerType(Type.usize), ""); } fn airCmpxchg( self: *FuncGen, inst: Air.Inst.Index, kind: Builder.Function.Instruction.CmpXchg.Kind, -) !Builder.Value { +) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -4889,7 +4858,7 @@ fn airCmpxchg( var expected_value = try self.resolveInst(extra.expected_value); var new_value = try self.resolveInst(extra.new_value); const operand_ty = ptr_ty.childType(zcu); - const llvm_operand_ty = try o.lowerType(pt, operand_ty); + const llvm_operand_ty = try self.lowerType(operand_ty); const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false); if (llvm_abi_ty != .none) { // operand needs widening and truncating @@ -4932,7 +4901,7 @@ fn airCmpxchg( const non_null_bit = try self.wip.not(success_bit, ""); const payload_align = operand_ty.abiAlignment(zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(try o.lowerType(pt, optional_ty), payload_align); + const alloca_inst = try self.buildAlloca(try self.lowerType(optional_ty), payload_align); // Payload is always the first field at offset 0, so address is `alloca_inst` _ = try self.wip.store(.normal, payload, alloca_inst, payload_align); @@ -4944,7 +4913,7 @@ fn airCmpxchg( return alloca_inst; } -fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -4959,7 +4928,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float); const ordering = toLlvmAtomicOrdering(extra.ordering()); const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg); - const llvm_operand_ty = try o.lowerType(pt, operand_ty); + const llvm_operand_ty = try self.lowerType(operand_ty); const access_kind: Builder.MemoryAccessKind = if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; @@ -5002,7 +4971,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { access_kind, op, ptr, - try self.wip.cast(.ptrtoint, operand, try o.lowerType(pt, Type.usize), ""), + try self.wip.cast(.ptrtoint, operand, try self.lowerType(Type.usize), ""), self.sync_scope, ordering, ptr_alignment, @@ -5010,8 +4979,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { ), llvm_operand_ty, ""); } -fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; +fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; @@ -5028,7 +4996,7 @@ fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm(); const access_kind: Builder.MemoryAccessKind = if (info.flags.is_volatile) .@"volatile" else .normal; - const elem_llvm_ty = try o.lowerType(pt, elem_ty); + const elem_llvm_ty = try self.lowerType(elem_ty); self.maybeMarkAllowZeroAccess(info); @@ -5060,7 +5028,7 @@ fn airAtomicStore( self: *FuncGen, inst: Air.Inst.Index, ordering: Builder.AtomicOrdering, -) !Builder.Value { +) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -5087,7 +5055,7 @@ fn airAtomicStore( return .none; } -fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value { +fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -5186,7 +5154,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value const body_block = try self.wip.block(1, "InlineMemsetBody"); const end_block = try self.wip.block(1, "InlineMemsetEnd"); - const llvm_usize_ty = try o.lowerType(pt, Type.usize); + const llvm_usize_ty = try self.lowerType(Type.usize); const end_ptr = switch (ptr_ty.ptrSize(zcu)) { .slice => try self.ptraddScaled( dest_ptr, @@ -5225,7 +5193,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value return .none; } -fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -5254,7 +5222,7 @@ fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return .none; } -fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -5279,14 +5247,15 @@ fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return .none; } -fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const un_ptr_ty = self.typeOf(bin_op.lhs); const un_ty = un_ptr_ty.childType(zcu); const layout = un_ty.unionGetLayout(zcu); - assert(layout.tag_size != 0); + + if (layout.tag_size == 0) return .none; // TODO: stop Sema emitting this const access_kind: Builder.MemoryAccessKind = if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; @@ -5309,7 +5278,7 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return .none; } -fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -5319,7 +5288,7 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { assert(layout.tag_size != 0); const union_ptr = try self.resolveInst(ty_op.operand); if (isByRef(un_ty, zcu)) { - const llvm_un_ty = try o.lowerType(pt, un_ty); + const llvm_un_ty = try self.lowerType(un_ty); if (layout.payload_size == 0) return self.wip.load(.normal, llvm_un_ty, union_ptr, .default, ""); const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); @@ -5333,7 +5302,7 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } } -fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) !Builder.Value { +fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) Allocator.Error!Builder.Value { const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); const operand_ty = self.typeOf(un_op); @@ -5341,7 +5310,7 @@ fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) !Build return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand}); } -fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); const operand_ty = self.typeOf(un_op); @@ -5349,9 +5318,7 @@ fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Bui return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand}); } -fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value { - const o = self.object; - const pt = self.pt; +fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const inst_ty = self.typeOfIndex(inst); const operand_ty = self.typeOf(ty_op.operand); @@ -5361,16 +5328,14 @@ fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) .normal, .none, intrinsic, - &.{try o.lowerType(pt, operand_ty)}, + &.{try self.lowerType(operand_ty)}, &.{ operand, .false }, "", ); - return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); + return self.wip.conv(.unsigned, result, try self.lowerType(inst_ty), ""); } -fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value { - const o = self.object; - const pt = self.pt; +fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const inst_ty = self.typeOfIndex(inst); const operand_ty = self.typeOf(ty_op.operand); @@ -5380,14 +5345,14 @@ fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) .normal, .none, intrinsic, - &.{try o.lowerType(pt, operand_ty)}, + &.{try self.lowerType(operand_ty)}, &.{operand}, "", ); - return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); + return self.wip.conv(.unsigned, result, try self.lowerType(inst_ty), ""); } -fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -5398,7 +5363,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const inst_ty = self.typeOfIndex(inst); var operand = try self.resolveInst(ty_op.operand); - var llvm_operand_ty = try o.lowerType(pt, operand_ty); + var llvm_operand_ty = try self.lowerType(operand_ty); if (bits % 16 == 8) { // If not an even byte-multiple, we need zero-extend + shift-left 1 byte @@ -5419,10 +5384,10 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const result = try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, ""); - return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), ""); + return self.wip.conv(.unsigned, result, try self.lowerType(inst_ty), ""); } -fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -5440,7 +5405,7 @@ fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { for (0..names.len) |name_index| { const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?; - const this_tag_int_value = try o.builder.intConst(try o.errorIntType(pt), err_int); + const this_tag_int_value = try o.builder.intConst(try o.errorIntType(), err_int); try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip); } self.wip.cursor = .{ .block = valid_block }; @@ -5455,13 +5420,13 @@ fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return phi.toValue(); } -fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); const enum_ty = self.typeOf(un_op); - const llvm_fn = try self.getIsNamedEnumValueFunction(enum_ty); + const llvm_fn = try o.getIsNamedEnumValueFunction(self.pt, enum_ty); return self.wip.call( .normal, .fastcc, @@ -5473,28 +5438,7 @@ fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { ); } -fn getIsNamedEnumValueFunction(self: *FuncGen, enum_ty: Type) !Builder.Function.Index { - const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; - const ip = &zcu.intern_pool; - - const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern()); - if (gop.found_existing) return gop.value_ptr.*; - errdefer assert(o.named_enum_map.remove(enum_ty.toIntern())); - const function_index = try o.builder.addFunction( - // Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type. - // TODO: change the builder API so we don't need to do this. - try o.builder.fnType(.void, &.{}, .normal), - try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}), - toLlvmAddressSpace(.generic, zcu.getTarget()), - ); - gop.value_ptr.* = function_index; - try o.updateIsNamedEnumValueFunction(pt, enum_ty, function_index); - return function_index; -} - -fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airTagName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; @@ -5513,34 +5457,31 @@ fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { ); } -fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); const slice_ty = self.typeOfIndex(inst); - const slice_llvm_ty = try o.lowerType(pt, slice_ty); + const slice_llvm_ty = try self.lowerType(slice_ty); // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed. - const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), ""); + const operand_usize = try self.wip.conv(.unsigned, operand, try self.lowerType(.usize), ""); - const error_name_table_ptr = try self.getErrorNameTable(); - const error_name_table = try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, ""); - const error_name_ptr = try self.ptraddScaled(error_name_table, operand_usize, slice_ty.abiSize(zcu)); + const error_name_table_ptr = try o.getErrorNameTable(); + const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu)); return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, ""); } -fn airSplat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; - const pt = self.pt; +fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const scalar = try self.resolveInst(ty_op.operand); const vector_ty = self.typeOfIndex(inst); - return self.wip.splatVector(try o.lowerType(pt, vector_ty), scalar, ""); + return self.wip.splatVector(try self.lowerType(vector_ty), scalar, ""); } -fn airSelect(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Bin, pl_op.payload).data; const pred = try self.resolveInst(pl_op.operand); @@ -5550,7 +5491,7 @@ fn airSelect(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return self.wip.select(.normal, pred, a, b, ""); } -fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = fg.object; const pt = fg.pt; const zcu = pt.zcu; @@ -5561,9 +5502,9 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const operand = try fg.resolveInst(unwrapped.operand); const mask = unwrapped.mask; const operand_ty = fg.typeOf(unwrapped.operand); - const llvm_operand_ty = try o.lowerType(pt, operand_ty); - const llvm_result_ty = try o.lowerType(pt, unwrapped.result_ty); - const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu)); + const llvm_operand_ty = try fg.lowerType(operand_ty); + const llvm_result_ty = try fg.lowerType(unwrapped.result_ty); + const llvm_elem_ty = try fg.lowerType(unwrapped.result_ty.childType(zcu)); const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty); const llvm_poison_mask_elem = try o.builder.poisonConst(.i32); const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32); @@ -5657,7 +5598,7 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { ); } -fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = fg.object; const pt = fg.pt; const zcu = pt.zcu; @@ -5666,7 +5607,7 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst); const mask = unwrapped.mask; - const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu)); + const llvm_elem_ty = try fg.lowerType(unwrapped.result_ty.childType(zcu)); const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32); const llvm_poison_mask_elem = try o.builder.poisonConst(.i32); @@ -5756,10 +5697,9 @@ fn buildReducedCall( operand_vector: Builder.Value, vector_len: usize, accum_init: Builder.Value, -) !Builder.Value { +) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const usize_ty = try o.lowerType(pt, Type.usize); + const usize_ty = try self.lowerType(Type.usize); const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len); const llvm_result_ty = accum_init.typeOfWip(&self.wip); @@ -5811,7 +5751,7 @@ fn buildReducedCall( return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, ""); } -fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { +fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -5820,9 +5760,9 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) ! const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce; const operand = try self.resolveInst(reduce.operand); const operand_ty = self.typeOf(reduce.operand); - const llvm_operand_ty = try o.lowerType(pt, operand_ty); + const llvm_operand_ty = try self.lowerType(operand_ty); const scalar_ty = self.typeOfIndex(inst); - const llvm_scalar_ty = try o.lowerType(pt, scalar_ty); + const llvm_scalar_ty = try self.lowerType(scalar_ty); switch (reduce.operation) { .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { @@ -5890,8 +5830,7 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) ! else => unreachable, }; - const libc_fn = - try self.getLibcFunction(fn_name, &.{ llvm_scalar_ty, llvm_scalar_ty }, llvm_scalar_ty); + const libc_fn = try o.getLibcFunction(fn_name, &.{ llvm_scalar_ty, llvm_scalar_ty }, llvm_scalar_ty); const init_val = switch (llvm_scalar_ty) { .i16 => try o.builder.intValue(.i16, @as(i16, @bitCast( @as(f16, switch (reduce.operation) { @@ -5922,7 +5861,7 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) ! return self.buildReducedCall(libc_fn, operand, operand_ty.vectorLen(zcu), init_val); } -fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -5931,7 +5870,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const result_ty = self.typeOfIndex(inst); const len: usize = @intCast(result_ty.arrayLen(zcu)); const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); - const llvm_result_ty = try o.lowerType(pt, result_ty); + const llvm_result_ty = try self.lowerType(result_ty); switch (result_ty.zigTypeTag(zcu)) { .vector => { @@ -5997,7 +5936,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { field_ptr_align.toLlvm(), llvm_field_val, field_ty.abiAlignment(zcu).toLlvm(), - try o.builder.intValue(try o.lowerType(pt, .usize), field_ty.abiSize(zcu)), + try o.builder.intValue(try self.lowerType(.usize), field_ty.abiSize(zcu)), .normal, self.disable_intrinsics, ); @@ -6042,7 +5981,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } } -fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -6050,7 +5989,7 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; const union_ty = self.typeOfIndex(inst); - const union_llvm_ty = try o.lowerType(pt, union_ty); + const union_llvm_ty = try self.lowerType(union_ty); const union_obj = zcu.typeToUnion(union_ty).?; assert(union_obj.layout != .@"packed"); @@ -6086,7 +6025,7 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return result_ptr; } -fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch; @@ -6135,14 +6074,12 @@ fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { return .none; } -fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.object; - const pt = self.pt; +fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const inst_ty = self.typeOfIndex(inst); const operand = try self.resolveInst(ty_op.operand); - return self.wip.cast(.addrspacecast, operand, try o.lowerType(pt, inst_ty), ""); + return self.wip.cast(.addrspacecast, operand, try self.lowerType(inst_ty), ""); } fn workIntrinsic( @@ -6150,7 +6087,7 @@ fn workIntrinsic( dimension: u32, default: u32, comptime basename: []const u8, -) !Builder.Value { +) Allocator.Error!Builder.Value { return self.wip.callIntrinsic(.normal, .none, switch (dimension) { 0 => @field(Builder.Intrinsic, basename ++ ".x"), 1 => @field(Builder.Intrinsic, basename ++ ".y"), @@ -6159,7 +6096,7 @@ fn workIntrinsic( }, &.{}, &.{}, ""); } -fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const target = self.pt.zcu.getTarget(); const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; @@ -6172,7 +6109,7 @@ fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { }; } -fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const pt = self.pt; const target = pt.zcu.getTarget(); @@ -6200,7 +6137,7 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { } } -fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { +fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const target = self.pt.zcu.getTarget(); const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; @@ -6213,28 +6150,6 @@ fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { }; } -fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index { - const o = self.object; - const pt = self.pt; - - const table = o.error_name_table; - if (table != .none) return table; - - // TODO: Address space - const variable_index = - try o.builder.addVariable(try o.builder.strtabString("__zig_err_name_table"), .ptr, .default); - variable_index.setLinkage(.private, &o.builder); - variable_index.setMutability(.constant, &o.builder); - variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); - variable_index.setAlignment( - Type.slice_const_u8_sentinel_0.abiAlignment(pt.zcu).toLlvm(), - &o.builder, - ); - - o.error_name_table = variable_index; - return variable_index; -} - /// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits. fn optCmpNull( self: *FuncGen, @@ -6258,7 +6173,7 @@ fn optPayloadHandle( opt_ptr: Builder.Value, opt_ty: Type, can_elide_load: bool, -) !Builder.Value { +) Allocator.Error!Builder.Value { const pt = fg.pt; const zcu = pt.zcu; assert(isByRef(opt_ty, zcu)); @@ -6281,7 +6196,7 @@ fn fieldPtr( aggregate_ptr: Builder.Value, aggregate_ptr_ty: Type, field_index: u32, -) !Builder.Value { +) Allocator.Error!Builder.Value { const pt = self.pt; const zcu = pt.zcu; const aggregate_ty = aggregate_ptr_ty.childType(zcu); @@ -6305,7 +6220,7 @@ fn loadTruncate( payload_ty: Type, payload_ptr: Builder.Value, payload_alignment: Builder.Alignment, -) !Builder.Value { +) Allocator.Error!Builder.Value { // from https://llvm.org/docs/LangRef.html#load-instruction : // "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. " // => so load the byte aligned value and trunc the unwanted bits. @@ -6313,7 +6228,7 @@ fn loadTruncate( const o = fg.object; const pt = fg.pt; const zcu = pt.zcu; - const payload_llvm_ty = try o.lowerType(pt, payload_ty); + const payload_llvm_ty = try fg.lowerType(payload_ty); const abi_size = payload_ty.abiSize(zcu); const load_llvm_ty = if (payload_ty.isAbiInt(zcu)) @@ -6321,7 +6236,7 @@ fn loadTruncate( else payload_llvm_ty; const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, ""); - const shifted = if (payload_llvm_ty != load_llvm_ty and o.target.cpu.arch.endian() == .big) + const shifted = if (payload_llvm_ty != load_llvm_ty and zcu.getTarget().cpu.arch.endian() == .big) try fg.wip.bin(.lshr, loaded, try o.builder.intValue( load_llvm_ty, (payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8, @@ -6339,10 +6254,10 @@ fn loadByRef( pointee_type: Type, ptr_alignment: Builder.Alignment, access_kind: Builder.MemoryAccessKind, -) !Builder.Value { +) Allocator.Error!Builder.Value { const o = fg.object; const pt = fg.pt; - const pointee_llvm_ty = try o.lowerType(pt, pointee_type); + const pointee_llvm_ty = try fg.lowerType(pointee_type); const result_align = InternPool.Alignment.fromLlvm(ptr_alignment) .max(pointee_type.abiAlignment(pt.zcu)).toLlvm(); const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align); @@ -6352,7 +6267,7 @@ fn loadByRef( result_align, ptr, ptr_alignment, - try o.builder.intValue(try o.lowerType(pt, Type.usize), size_bytes), + try o.builder.intValue(try fg.lowerType(.usize), size_bytes), access_kind, fg.disable_intrinsics, ); @@ -6362,7 +6277,7 @@ fn loadByRef( /// This function always performs a copy. For isByRef=true types, it creates a new /// alloca and copies the value into it, then returns the alloca instruction. /// For isByRef=false types, it creates a load instruction and returns it. -fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) !Builder.Value { +fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -6380,7 +6295,7 @@ fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) !Builder.Value { if (info.flags.vector_index != .none) { const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); - const vec_elem_ty = try o.lowerType(pt, elem_ty); + const vec_elem_ty = try self.lowerType(elem_ty); const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); const loaded_vector = try self.wip.load(access_kind, vec_ty, ptr, ptr_alignment, ""); @@ -6401,7 +6316,7 @@ fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) !Builder.Value { const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); const shift_amt = try o.builder.intValue(containing_int_ty, info.packed_offset.bit_offset); const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, ""); - const elem_llvm_ty = try o.lowerType(pt, elem_ty); + const elem_llvm_ty = try self.lowerType(elem_ty); if (isByRef(elem_ty, zcu)) { const result_align = elem_ty.abiAlignment(zcu).toLlvm(); @@ -6434,7 +6349,7 @@ fn store( ptr_ty: Type, elem: Builder.Value, ordering: Builder.AtomicOrdering, -) !void { +) Allocator.Error!void { const o = self.object; const pt = self.pt; const zcu = pt.zcu; @@ -6449,7 +6364,7 @@ fn store( if (info.flags.vector_index != .none) { const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); - const vec_elem_ty = try o.lowerType(pt, elem_ty); + const vec_elem_ty = try self.lowerType(elem_ty); const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, ""); @@ -6518,7 +6433,7 @@ fn store( ptr_alignment, elem, elem_ty.abiAlignment(zcu).toLlvm(), - try o.builder.intValue(try o.lowerType(pt, Type.usize), elem_ty.abiSize(zcu)), + try o.builder.intValue(try self.lowerType(Type.usize), elem_ty.abiSize(zcu)), access_kind, self.disable_intrinsics, ); @@ -6527,8 +6442,7 @@ fn store( fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void { const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545; const o = fg.object; - const pt = fg.pt; - const usize_ty = try o.lowerType(pt, Type.usize); + const usize_ty = try fg.lowerType(.usize); const zero = try o.builder.intValue(usize_ty, 0); const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED); const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, ""); @@ -6551,7 +6465,7 @@ fn valgrindClientRequest( const target = zcu.getTarget(); if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value; - const llvm_usize = try o.lowerType(pt, Type.usize); + const llvm_usize = try fg.lowerType(.usize); const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm(); const array_llvm_ty = try o.builder.arrayType(6, llvm_usize); @@ -7360,11 +7274,12 @@ pub fn isByRef(ty: Type, zcu: *const Zcu) bool { => false, .array, - .error_union, .frame, => ty.hasRuntimeBits(zcu), - .optional => ty.hasRuntimeBits(zcu) and !ty.optionalReprIsPayload(zcu), + .error_union => ty.errorUnionPayload(zcu).hasRuntimeBits(zcu), + + .optional => !ty.optionalReprIsPayload(zcu) and ty.optionalChild(zcu).hasRuntimeBits(zcu), .@"struct" => switch (ty.containerLayout(zcu)) { .@"packed" => false, @@ -7402,25 +7317,19 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value { if (offset == 0) return ptr; - const llvm_usize_ty = try fg.object.lowerType(fg.pt, .usize); + const llvm_usize_ty = try fg.lowerType(.usize); const offset_val = try fg.object.builder.intValue(llvm_usize_ty, offset); - return fg.ptradd(ptr, offset_val); + return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, ""); } fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value { - switch (scale) { - 0 => return ptr, - 1 => return fg.ptradd(ptr, index), - else => { - const o = fg.object; - const llvm_usize_ty = try o.lowerType(fg.pt, .usize); - const scale_val = try o.builder.intValue(llvm_usize_ty, scale); - const offset = try fg.wip.bin(.@"mul nuw", index, scale_val, ""); - return fg.ptradd(ptr, offset); - }, - } -} -fn ptradd(fg: *FuncGen, ptr: Builder.Value, offset: Builder.Value) Allocator.Error!Builder.Value { - return fg.wip.gep(.inbounds, .i8, ptr, &.{offset}, ""); + if (scale == 0) return ptr; + // Right now LLVM seems to fare a bit worse with an explicit `mul nuw` instruction than it does + // if we use a bigger type for the GEP, so we'll do that. As I understand it, it has not yet + // been decided whether the planned `ptradd` instruction will accept a scale or not; if it does + // not then presumably upstream will improve their handling of explicit `mul nuw` computing the + // offset. + const llvm_scale_ty = try fg.object.builder.arrayType(scale, .i8); + return fg.wip.gep(.inbounds, llvm_scale_ty, ptr, &.{index}, ""); } fn compilerRtIntBits(bits: u16) ?u16 { -- 2.54.0 From c0f3a238314315a1e4a578b23cc66237eac90689 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Mon, 23 Mar 2026 14:49:11 +0000 Subject: [PATCH 04/15] llvm: get rid of a bunch of `PerThread` usages Also, notably, remove `Air.value`! The `onePossibleValue` check was actually dead code, because it is a bug if Sema ever emits code which considers a value of OPV type to be runtime-known---and at that point `Air.value` is just a thin wrapper around `Air.Ref.toInterned`. --- src/Air.zig | 9 - src/Sema.zig | 2 +- src/Sema/bitcast.zig | 4 +- src/Value.zig | 68 ++- src/Zcu/PerThread.zig | 2 +- src/codegen/aarch64/Select.zig | 2 +- src/codegen/c.zig | 24 +- src/codegen/llvm.zig | 429 +++++++++--------- src/codegen/llvm/FuncGen.zig | 751 +++++++++++++++----------------- src/codegen/riscv64/CodeGen.zig | 4 +- src/codegen/sparc64/CodeGen.zig | 4 +- src/codegen/spirv/CodeGen.zig | 17 +- src/codegen/wasm/CodeGen.zig | 6 +- src/codegen/x86_64/CodeGen.zig | 4 +- 14 files changed, 620 insertions(+), 706 deletions(-) diff --git a/src/Air.zig b/src/Air.zig index ff88990bfa52c8dfe7ed5a3a108f70ba9c11e598..a3de9c84c867cc0e4ca158087b0e60c7efc6e1c2 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -1843,15 +1843,6 @@ pub fn internedToRef(ip_index: InternPool.Index) Inst.Ref { return .fromIntern(ip_index); } -/// Returns `null` if runtime-known. -pub fn value(air: Air, inst: Inst.Ref, pt: Zcu.PerThread) !?Value { - if (inst.toInterned()) |ip_index| { - return .fromInterned(ip_index); - } - const index = inst.toIndex().?; - return air.typeOfIndex(index, &pt.zcu.intern_pool).onePossibleValue(pt); -} - pub const NullTerminatedString = enum(u32) { none = std.math.maxInt(u32), _, diff --git a/src/Sema.zig b/src/Sema.zig index 1876ec5dbea763a21673adc8ed0ad7f51d07b703..0629e827a97a3acd68dc811714020b600616e341 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -18896,7 +18896,7 @@ fn finishStructInit( var bit_offset: u16 = 0; for (field_inits) |field_init| { const field_val = sema.resolveValue(field_init).?; - field_val.writeToPackedMemory(pt, buf, bit_offset) catch |err| switch (err) { + field_val.writeToPackedMemory(zcu, buf, bit_offset) catch |err| switch (err) { error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers error.OutOfMemory => |e| return e, }; diff --git a/src/Sema/bitcast.zig b/src/Sema/bitcast.zig index 0e4c90027e959e02a9c023680a8c22489e5502f9..6d3da8daf4ec2a51b3f2c36b55812863e70f8d8a 100644 --- a/src/Sema/bitcast.zig +++ b/src/Sema/bitcast.zig @@ -443,7 +443,7 @@ const UnpackValueBits = struct { // This @intCast is okay because no primitive can exceed the size of a u16. const int_ty = try unpack.pt.intType(.unsigned, @intCast(bit_count)); const buf = try unpack.arena.alloc(u8, @intCast((val_bits + 7) / 8)); - try val.writeToPackedMemory(unpack.pt, buf, 0); + try val.writeToPackedMemory(zcu, buf, 0); const sub_val = try Value.readFromPackedMemory(int_ty, unpack.pt, buf, @intCast(bit_offset), unpack.arena); try unpack.primitive(sub_val); }, @@ -722,7 +722,7 @@ const PackValueBits = struct { const val = Value.fromInterned(ip_val); const ty = val.typeOf(zcu); if (!val.isUndef(zcu)) { - try val.writeToPackedMemory(pt, buf, cur_bit_off); + try val.writeToPackedMemory(zcu, buf, cur_bit_off); } cur_bit_off += @intCast(ty.bitSize(zcu)); } diff --git a/src/Value.zig b/src/Value.zig index 826fba11e6a214dd3248933d2111df58dce22626..cc0e577f84717beacf9398b0bdc7a98173d3276f 100644 --- a/src/Value.zig +++ b/src/Value.zig @@ -245,13 +245,12 @@ pub fn toBool(val: Value) bool { /// /// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past /// the end of the value in memory. -pub fn writeToMemory(val: Value, pt: Zcu.PerThread, buffer: []u8) error{ +pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{ ReinterpretDeclRef, IllDefinedMemoryLayout, Unimplemented, OutOfMemory, }!void { - const zcu = pt.zcu; const target = zcu.getTarget(); const endian = target.cpu.arch.endian(); const ip = &zcu.intern_pool; @@ -289,14 +288,18 @@ pub fn writeToMemory(val: Value, pt: Zcu.PerThread, buffer: []u8) error{ else => unreachable, }, .array => { + const aggregate = ip.indexToKey(val.toIntern()).aggregate; const len = ty.arrayLen(zcu); const elem_ty = ty.childType(zcu); const elem_size: usize = @intCast(elem_ty.abiSize(zcu)); var elem_i: usize = 0; var buf_off: usize = 0; while (elem_i < len) : (elem_i += 1) { - const elem_val = try val.elemValue(pt, elem_i); - try elem_val.writeToMemory(pt, buffer[buf_off..]); + switch (aggregate.storage) { + .bytes => |bytes| buffer[buf_off] = bytes.at(elem_i, ip), + .elems => |elems| try Value.fromInterned(elems[elem_i]).writeToMemory(zcu, buffer[buf_off..]), + .repeated_elem => |elem| try Value.fromInterned(elem).writeToMemory(zcu, buffer[buf_off..]), + } buf_off += elem_size; } }, @@ -304,7 +307,7 @@ pub fn writeToMemory(val: Value, pt: Zcu.PerThread, buffer: []u8) error{ // We use byte_count instead of abi_size here, so that any padding bytes // follow the data bytes, on both big- and little-endian systems. const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8; - return writeToPackedMemory(val, pt, buffer[0..byte_count], 0); + return writeToPackedMemory(val, zcu, buffer[0..byte_count], 0); }, .@"struct" => { const struct_type = zcu.typeToStruct(ty) orelse return error.IllDefinedMemoryLayout; @@ -320,42 +323,33 @@ pub fn writeToMemory(val: Value, pt: Zcu.PerThread, buffer: []u8) error{ .elems => |elems| elems[field_index], .repeated_elem => |elem| elem, }); - try writeToMemory(field_val, pt, buffer[off..]); + try writeToMemory(field_val, zcu, buffer[off..]); }, .@"packed" => { const int_index = ip.indexToKey(val.toIntern()).bitpack.backing_int_val; - return Value.fromInterned(int_index).writeToMemory(pt, buffer); + return Value.fromInterned(int_index).writeToMemory(zcu, buffer); }, } }, .@"union" => switch (ty.containerLayout(zcu)) { .auto => return error.IllDefinedMemoryLayout, // Sema is supposed to have emitted a compile error already .@"extern" => { - if (val.unionTag(zcu)) |union_tag| { - const union_obj = zcu.typeToUnion(ty).?; - const field_index = zcu.unionTagFieldIndex(union_obj, union_tag).?; - const field_type = Type.fromInterned(union_obj.field_types.get(ip)[field_index]); - const field_val = try val.fieldValue(pt, field_index); - const byte_count: usize = @intCast(field_type.abiSize(zcu)); - return writeToMemory(field_val, pt, buffer[0..byte_count]); - } else { - const backing_ty = try ty.externUnionBackingType(pt); - const byte_count: usize = @intCast(backing_ty.abiSize(zcu)); - return writeToMemory(val.unionPayload(zcu), pt, buffer[0..byte_count]); - } + const payload_val = val.unionPayload(zcu); + return writeToMemory(payload_val, zcu, buffer); }, .@"packed" => { const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val); - return writeToMemory(int_val, pt, buffer); + return writeToMemory(int_val, zcu, buffer); }, }, .optional => { if (!ty.isPtrLikeOptional(zcu)) return error.IllDefinedMemoryLayout; const opt_val = val.optionalValue(zcu); if (opt_val) |some| { - return some.writeToMemory(pt, buffer); + return some.writeToMemory(zcu, buffer); } else { - return writeToMemory(try pt.intValue(Type.usize, 0), pt, buffer); + const byte_count = Type.usize.abiSize(zcu); + @memset(buffer[0..@intCast(byte_count)], 0); // null pointer } }, else => return error.Unimplemented, @@ -368,11 +362,10 @@ pub fn writeToMemory(val: Value, pt: Zcu.PerThread, buffer: []u8) error{ /// big-endian packed memory layouts start at the end of the buffer. pub fn writeToPackedMemory( val: Value, - pt: Zcu.PerThread, + zcu: *const Zcu, buffer: []u8, bit_offset: usize, ) error{ ReinterpretDeclRef, OutOfMemory }!void { - const zcu = pt.zcu; const ip = &zcu.intern_pool; const target = zcu.getTarget(); const endian = target.cpu.arch.endian(); @@ -399,7 +392,7 @@ pub fn writeToPackedMemory( }, .@"enum" => { const int_val = val.intFromEnum(zcu); - return int_val.writeToPackedMemory(pt, buffer, bit_offset); + return int_val.writeToPackedMemory(zcu, buffer, bit_offset); }, .pointer => { assert(!ty.isSlice(zcu)); // No well defined layout. @@ -430,25 +423,29 @@ pub fn writeToPackedMemory( var bits: u16 = 0; var elem_i: usize = 0; + const aggregate = ip.indexToKey(val.toIntern()).aggregate; while (elem_i < len) : (elem_i += 1) { // On big-endian systems, LLVM reverses the element order of vectors by default const tgt_elem_i = if (endian == .big) len - elem_i - 1 else elem_i; - const elem_val = try val.elemValue(pt, tgt_elem_i); - try elem_val.writeToPackedMemory(pt, buffer, bit_offset + bits); + switch (aggregate.storage) { + .bytes => |bytes| std.mem.writePackedInt(u8, buffer, bit_offset + bits, bytes.at(tgt_elem_i, ip), endian), + .elems => |elems| try Value.fromInterned(elems[tgt_elem_i]).writeToPackedMemory(zcu, buffer, bit_offset + bits), + .repeated_elem => |elem| try Value.fromInterned(elem).writeToPackedMemory(zcu, buffer, bit_offset + bits), + } bits += elem_bit_size; } }, .@"struct", .@"union" => { assert(ty.containerLayout(zcu) == .@"packed"); const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val); - return int_val.writeToPackedMemory(pt, buffer, bit_offset); + return int_val.writeToPackedMemory(zcu, buffer, bit_offset); }, .optional => { assert(ty.isPtrLikeOptional(zcu)); if (val.optionalValue(zcu)) |ptr_val| { - return ptr_val.writeToPackedMemory(pt, buffer, bit_offset); + return ptr_val.writeToPackedMemory(zcu, buffer, bit_offset); } else { - return Value.zero_usize.writeToPackedMemory(pt, buffer, bit_offset); + return Value.zero_usize.writeToPackedMemory(zcu, buffer, bit_offset); } }, else => @panic("TODO implement writeToPackedMemory for more types"), @@ -889,7 +886,7 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value { const sfba = sfba_state.get(); const buf = try sfba.alloc(u8, @intCast((ty.bitSize(zcu) + 7) / 8)); defer sfba.free(buf); - int_val.writeToPackedMemory(pt, buf, 0) catch |err| switch (err) { + int_val.writeToPackedMemory(zcu, buf, 0) catch |err| switch (err) { error.ReinterpretDeclRef => unreachable, // it's an integer error.OutOfMemory => |e| return e, }; @@ -902,7 +899,7 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value { }; } -pub fn unionTag(val: Value, zcu: *Zcu) ?Value { +pub fn unionTag(val: Value, zcu: *const Zcu) ?Value { return switch (zcu.intern_pool.indexToKey(val.toIntern())) { .undef, .enum_tag => val, .un => |un| if (un.tag != .none) Value.fromInterned(un.tag) else return null, @@ -910,7 +907,7 @@ pub fn unionTag(val: Value, zcu: *Zcu) ?Value { }; } -pub fn unionPayload(val: Value, zcu: *Zcu) Value { +pub fn unionPayload(val: Value, zcu: *const Zcu) Value { return switch (zcu.intern_pool.indexToKey(val.toIntern())) { .un => |un| Value.fromInterned(un.val), else => unreachable, @@ -1605,15 +1602,14 @@ pub fn mulAddScalar( /// If the value is represented in-memory as a series of bytes that all /// have the same value, return that byte value, otherwise null. -pub fn hasRepeatedByteRepr(val: Value, pt: Zcu.PerThread) !?u8 { - const zcu = pt.zcu; +pub fn hasRepeatedByteRepr(val: Value, zcu: *const Zcu) !?u8 { const ty = val.typeOf(zcu); const abi_size = std.math.cast(usize, ty.abiSize(zcu)) orelse return null; assert(abi_size >= 1); const byte_buffer = try zcu.gpa.alloc(u8, abi_size); defer zcu.gpa.free(byte_buffer); - writeToMemory(val, pt, byte_buffer) catch |err| switch (err) { + writeToMemory(val, zcu, byte_buffer) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.ReinterpretDeclRef => return null, // TODO: The writeToMemory function was originally created for the purpose diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index 88dcbe2431fac65e7c3923253eaf2eacc9518771..c9972a300abea99261410ce5ea2de893040a3b8a 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -3751,7 +3751,7 @@ fn processExportsInner( if (skip_linker_work) return; if (zcu.llvm_object) |llvm_object| { - try zcu.handleUpdateExports(export_indices, llvm_object.updateExports(pt, exported, export_indices)); + try zcu.handleUpdateExports(export_indices, llvm_object.updateExports(exported, export_indices)); } else if (zcu.comp.bin_file) |lf| { try zcu.handleUpdateExports(export_indices, lf.updateExports(pt, exported, export_indices)); } diff --git a/src/codegen/aarch64/Select.zig b/src/codegen/aarch64/Select.zig index 779994980e8c6fee39bc3f262c2635d35dfdc135..c8a9f41cb3d6c2906e7011373ede8fca90a031a3 100644 --- a/src/codegen/aarch64/Select.zig +++ b/src/codegen/aarch64/Select.zig @@ -11364,7 +11364,7 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem const zcu = isel.pt.zcu; const ip = &zcu.intern_pool; if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true; - constant.writeToMemory(isel.pt, buffer) catch |err| switch (err) { + constant.writeToMemory(zcu, buffer) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.ReinterpretDeclRef, error.Unimplemented, error.IllDefinedMemoryLayout => return false, }; diff --git a/src/codegen/c.zig b/src/codegen/c.zig index e126cf535b08ee7fef804e051455e4f419c5a32f..1b72c8759907bb004e0c31850bc1e53ccde57dea 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -435,8 +435,7 @@ pub const Function = struct { fn resolveInst(f: *Function, ref: Air.Inst.Ref) !CValue { const gop = try f.value_map.getOrPut(ref); if (!gop.found_existing) { - const val = try f.air.value(ref, f.dg.pt); - gop.value_ptr.* = .{ .constant = val.? }; + gop.value_ptr.* = .{ .constant = .fromInterned(ref.toInterned().?) }; } return gop.value_ptr.*; } @@ -3389,7 +3388,7 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { const ptr_val = try f.resolveInst(bin_op.lhs); const src_ty = f.typeOf(bin_op.rhs); - const val_is_undef = if (try f.air.value(bin_op.rhs, pt)) |v| v.isUndef(zcu) else false; + const val_is_undef = if (bin_op.rhs.toInterned()) |ip_index| Value.fromInterned(ip_index).isUndef(zcu) else false; const w = &f.code.writer; if (val_is_undef) { @@ -3922,8 +3921,8 @@ fn airCall( callee: { known: { - const callee_val = (try f.air.value(call.callee, pt)) orelse break :known; - const fn_nav, const need_cast = switch (ip.indexToKey(callee_val.toIntern())) { + const callee_ip_index = call.callee.toInterned() orelse break :known; + const fn_nav, const need_cast = switch (ip.indexToKey(callee_ip_index)) { .@"extern" => |@"extern"| .{ @"extern".owner_nav, false }, .func => |func| .{ func.owner_nav, Type.fromInterned(func.ty).fnCallingConvention(zcu) != .naked and Type.fromInterned(func.uncoerced_ty).fnCallingConvention(zcu) == .naked }, @@ -4027,7 +4026,7 @@ fn airDbgVar(f: *Function, inst: Air.Inst.Index) !CValue { const tag = f.air.instructions.items(.tag)[@intFromEnum(inst)]; const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); - const operand_is_undef = if (try f.air.value(pl_op.operand, pt)) |v| v.isUndef(zcu) else false; + const operand_is_undef = if (pl_op.operand.toInterned()) |ip_index| Value.fromInterned(ip_index).isUndef(zcu) else false; if (!operand_is_undef) _ = try f.resolveInst(pl_op.operand); try reap(f, inst, &.{pl_op.operand}); @@ -4204,7 +4203,8 @@ fn airSwitchDispatch(f: *Function, inst: Air.Inst.Index) !void { const br = f.air.instructions.items(.data)[@intFromEnum(inst)].br; const w = &f.code.writer; - if (try f.air.value(br.operand, pt)) |cond_val| { + if (br.operand.toInterned()) |cond_ip_index| { + const cond_val: Value = .fromInterned(cond_ip_index); // Comptime-known dispatch. Iterate the cases to find the correct // one, and branch directly to the corresponding case. const switch_br = f.air.unwrapSwitch(br.block_inst); @@ -4539,12 +4539,12 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index, is_dispatch_loop: bool) !void try f.writeCValue(w, cond_val, .other); try w.writeAll(", "); } - const item_value = try f.air.value(item, pt); + const item_value: Value = .fromInterned(item.toInterned().?); // If `item_value` is a pointer with a known integer address, print the address // with no cast to avoid a warning. write_val: { if (cond_ty.zigTypeTag(zcu) == .pointer) { - if (item_value.?.getUnsignedInt(zcu)) |item_int| { + if (item_value.getUnsignedInt(zcu)) |item_int| { try w.print("{f}", .{try f.fmtIntLiteralDec(try pt.intValue(lowered_cond_ty, item_int))}); break :write_val; } @@ -4552,7 +4552,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index, is_dispatch_loop: bool) !void try f.renderType(w, .usize); try w.writeByte(')'); } - try f.dg.renderValue(w, (try f.air.value(item, pt)).?, .other); + try f.dg.renderValue(w, .fromInterned(item.toInterned().?), .other); } switch (cond_cint) { .zig_u128, .zig_i128 => try w.writeByte(')'), @@ -4710,7 +4710,7 @@ fn lowerSwitchCmp( try f.writeCValue(w, cond_val, .other); try w.writeAll(if (use_builtin) ", " else compareOperatorC(operator)); if (class == .big) try w.writeByte('&'); - try f.dg.renderValue(w, (try f.air.value(case_inst, pt)).?, .other); + try f.dg.renderValue(w, .fromInterned(case_inst.toInterned().?), .other); if (use_builtin) { try f.dg.renderBuiltinInfo(w, ty, if (class == .big) .bits else .none); try w.writeByte(')'); @@ -6100,7 +6100,7 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { const value = try f.resolveInst(bin_op.rhs); const elem_ty = f.typeOf(bin_op.rhs); const elem_abi_size = elem_ty.abiSize(zcu); - const val_is_undef = if (try f.air.value(bin_op.rhs, pt)) |val| val.isUndef(zcu) else false; + const val_is_undef = if (bin_op.rhs.toInterned()) |ip_index| Value.fromInterned(ip_index).isUndef(zcu) else false; const w = &f.code.writer; if (val_is_undef) { diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index c5ea440405d1ceca9f3f9a1ff2266764bbdb76ad..b34f630fc21e87c4e49d51dd9c9c27c991bbf0bf 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -696,11 +696,11 @@ pub const Object = struct { self.* = undefined; } - fn genErrorNameTable(o: *Object, pt: Zcu.PerThread) Allocator.Error!void { + fn genErrorNameTable(o: *Object) Allocator.Error!void { // If o.error_name_table is null, then it was not referenced by any instructions. if (o.error_name_table == .none) return; - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const error_name_list = ip.global_error_set.getNamesFromMainThread(); @@ -709,8 +709,8 @@ pub const Object = struct { // TODO: Address space const slice_ty = Type.slice_const_u8_sentinel_0; - const llvm_usize_ty = try o.lowerType(pt, Type.usize); - const llvm_slice_ty = try o.lowerType(pt, slice_ty); + const llvm_usize_ty = try o.lowerType(.usize); + const llvm_slice_ty = try o.lowerType(slice_ty); const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty); llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty); @@ -768,7 +768,7 @@ pub const Object = struct { }; pub fn emit(o: *Object, pt: Zcu.PerThread, options: EmitOptions) error{ LinkFailure, OutOfMemory }!void { - const zcu = pt.zcu; + const zcu = o.zcu; const comp = zcu.comp; const io = comp.io; const diags = &comp.link_diags; @@ -779,7 +779,7 @@ pub const Object = struct { const init_val = try o.builder.intConst(try o.errorIntType(), errors_len); try o.errors_len_variable.setInitializer(init_val, &o.builder); } - try o.genErrorNameTable(pt); + try o.genErrorNameTable(); try o.genModuleLevelAssembly(); if (o.used.items.len > 0) { @@ -1139,7 +1139,7 @@ pub const Object = struct { air: *const Air, liveness: *const ?Air.Liveness, ) !void { - const zcu = pt.zcu; + const zcu = o.zcu; const comp = zcu.comp; const ip = &zcu.intern_pool; const func = zcu.funcInfo(func_index); @@ -1150,7 +1150,7 @@ pub const Object = struct { const fn_info = zcu.typeToFunc(fn_ty).?; const target = &owner_mod.resolved_target.result; - const function_index = try o.resolveLlvmFunction(pt, func.owner_nav); + const function_index = try o.resolveLlvmFunction(func.owner_nav); var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); defer attributes.deinit(&o.builder); @@ -1262,7 +1262,7 @@ pub const Object = struct { defer args.deinit(gpa); { - var it = iterateParamTypes(o, pt, fn_info); + var it = iterateParamTypes(o, fn_info); while (try it.next()) |lowering| { try args.ensureUnusedCapacity(gpa, 1); @@ -1289,7 +1289,7 @@ pub const Object = struct { }, .byref => { const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(pt, param_ty); + const param_llvm_ty = try o.lowerType(param_ty); const param = wip.arg(llvm_arg_i); const alignment = param_ty.abiAlignment(zcu).toLlvm(); @@ -1304,7 +1304,7 @@ pub const Object = struct { }, .byref_mut => { const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(pt, param_ty); + const param_llvm_ty = try o.lowerType(param_ty); const param = wip.arg(llvm_arg_i); const alignment = param_ty.abiAlignment(zcu).toLlvm(); @@ -1323,7 +1323,7 @@ pub const Object = struct { const param = wip.arg(llvm_arg_i); llvm_arg_i += 1; - const param_llvm_ty = try o.lowerType(pt, param_ty); + const param_llvm_ty = try o.lowerType(param_ty); const alignment = param_ty.abiAlignment(zcu).toLlvm(); const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target); _ = try wip.store(.normal, param, arg_ptr, alignment); @@ -1362,7 +1362,7 @@ pub const Object = struct { const len_param = wip.arg(llvm_arg_i); llvm_arg_i += 1; - const slice_llvm_ty = try o.lowerType(pt, param_ty); + const slice_llvm_ty = try o.lowerType(param_ty); args.appendAssumeCapacity( try wip.buildAggregate(slice_llvm_ty, &.{ ptr_param, len_param }, ""), ); @@ -1371,7 +1371,7 @@ pub const Object = struct { assert(!it.byval_attr); const field_types = it.types_buffer[0..it.types_len]; const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(pt, param_ty); + const param_llvm_ty = try o.lowerType(param_ty); const param_alignment = param_ty.abiAlignment(zcu).toLlvm(); const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, param_alignment, target); const llvm_ty = try o.builder.structType(.normal, field_types); @@ -1391,7 +1391,7 @@ pub const Object = struct { }, .float_array => { const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(pt, param_ty); + const param_llvm_ty = try o.lowerType(param_ty); const param = wip.arg(llvm_arg_i); llvm_arg_i += 1; @@ -1406,7 +1406,7 @@ pub const Object = struct { }, .i32_array, .i64_array => { const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(pt, param_ty); + const param_llvm_ty = try o.lowerType(param_ty); const param = wip.arg(llvm_arg_i); llvm_arg_i += 1; @@ -1560,7 +1560,7 @@ pub const Object = struct { } pub fn updateNav(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void { - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const nav = ip.getNav(nav_index); @@ -1573,12 +1573,12 @@ pub const Object = struct { const ty: Type = .fromInterned(nav.resolved.?.type); if (linkage != .internal and ip.isFunctionType(ty.toIntern())) { - const function_index = try o.resolveLlvmFunction(pt, owner_nav); + const function_index = try o.resolveLlvmFunction(owner_nav); // Add parameter attributes which weren't set by `resolveLlvmFunction` const fn_info = zcu.typeToFunc(ty).?; var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); defer attributes.deinit(&o.builder); - var it = iterateParamTypes(o, pt, fn_info); + var it = iterateParamTypes(o, fn_info); if (firstParamSRet(fn_info, zcu, zcu.getTarget())) it.llvm_index += 1; if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) it.llvm_index += 1; while (try it.next()) |lowering| switch (lowering) { @@ -1591,7 +1591,7 @@ pub const Object = struct { }, .byref => { const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(pt, param_ty); + const param_llvm_ty = try o.lowerType(param_ty); const alignment = param_ty.abiAlignment(zcu); try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty); }, @@ -1609,14 +1609,14 @@ pub const Object = struct { }; function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); } else { - const variable_index = try o.resolveGlobalNav(pt, nav_index); + const variable_index = try o.resolveGlobalNav(nav_index); variable_index.setAlignment(zcu.navAlignment(nav_index).toLlvm(), &o.builder); if (resolved.@"linksection".toSlice(ip)) |section| variable_index.setSection(try o.builder.string(section), &o.builder); if (resolved.@"const") variable_index.setMutability(.constant, &o.builder); try variable_index.setInitializer(switch (init_val) { .none => .no_init, - else => try o.lowerValue(pt, init_val), + else => try o.lowerValue(init_val), }, &o.builder); variable_index.setVisibility(visibility, &o.builder); @@ -1703,14 +1703,13 @@ pub const Object = struct { pub fn updateExports( self: *Object, - pt: Zcu.PerThread, exported: Zcu.Exported, export_indices: []const Zcu.Export.Index, ) link.File.UpdateExportsError!void { - const zcu = pt.zcu; + const zcu = self.zcu; const nav_index = switch (exported) { .nav => |nav| nav, - .uav => |uav| return updateExportedValue(self, pt, uav, export_indices), + .uav => |uav| return updateExportedValue(self, uav, export_indices), }; const ip = &zcu.intern_pool; const global_index = self.nav_map.get(nav_index).?; @@ -1751,11 +1750,10 @@ pub const Object = struct { fn updateExportedValue( o: *Object, - pt: Zcu.PerThread, exported_value: InternPool.Index, export_indices: []const Zcu.Export.Index, ) link.File.UpdateExportsError!void { - const zcu = pt.zcu; + const zcu = o.zcu; const gpa = zcu.gpa; const ip = &zcu.intern_pool; const main_exp_name = try o.builder.strtabString(export_indices[0].ptr(zcu).opts.name.toSlice(ip)); @@ -1769,13 +1767,13 @@ pub const Object = struct { const llvm_addr_space = toLlvmAddressSpace(.generic, zcu.getTarget()); const variable_index = try o.builder.addVariable( main_exp_name, - try o.lowerType(pt, Type.fromInterned(ip.typeOf(exported_value))), + try o.lowerType(.fromInterned(ip.typeOf(exported_value))), llvm_addr_space, ); const global_index = variable_index.ptrConst(&o.builder).global; gop.value_ptr.* = global_index; // This line invalidates `gop`. - const init_val = try o.lowerValue(pt, exported_value); + const init_val = try o.lowerValue(exported_value); try variable_index.setInitializer(init_val, &o.builder); break :i global_index; }; @@ -1894,7 +1892,7 @@ pub const Object = struct { pub fn updateContainerType(o: *Object, pt: Zcu.PerThread, ty: InternPool.Index, success: bool) Allocator.Error!void { try o.type_pool.updateContainerType(pt, .{ .llvm = o }, ty, success); if (o.named_enum_map.get(ty)) |function_index| { - try o.updateIsNamedEnumValueFunction(pt, .fromInterned(ty), function_index); + try o.updateIsNamedEnumValueFunction(.fromInterned(ty), function_index); } } @@ -1902,7 +1900,8 @@ pub const Object = struct { /// /// `val` is always a type because `o.type_pool` only contains types. pub fn addConst(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void { - const zcu = pt.zcu; + _ = pt; + const zcu = o.zcu; const gpa = zcu.comp.gpa; assert(zcu.intern_pool.typeOf(val) == .type_type); @@ -1928,7 +1927,7 @@ pub const Object = struct { /// /// `val` is always a type because `o.type_pool` only contains types. pub fn updateConstIncomplete(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void { - const zcu = pt.zcu; + const zcu = o.zcu; assert(zcu.intern_pool.typeOf(val) == .type_type); const ty: Type = .fromInterned(val); @@ -1950,7 +1949,7 @@ pub const Object = struct { /// /// `val` is always a type because `o.type_pool` only contains types. pub fn updateConst(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void { - const zcu = pt.zcu; + const zcu = o.zcu; assert(zcu.intern_pool.typeOf(val) == .type_type); const ty: Type = .fromInterned(val); @@ -2005,7 +2004,7 @@ pub const Object = struct { assert(!o.builder.strip); const gpa = o.gpa; - const zcu = pt.zcu; + const zcu = o.zcu; const target = zcu.getTarget(); const ip = &zcu.intern_pool; @@ -2712,30 +2711,20 @@ pub const Object = struct { } fn namespaceToDebugScope(o: *Object, pt: Zcu.PerThread, namespace_index: InternPool.NamespaceIndex) !Builder.Metadata { - const zcu = pt.zcu; + const zcu = o.zcu; const namespace = zcu.namespacePtr(namespace_index); if (namespace.parent == .none) return try o.getDebugFile(namespace.file_scope); return o.getDebugType(pt, .fromInterned(namespace.owner_type)); } - fn allocTypeName(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error![:0]const u8 { - var aw: Io.Writer.Allocating = .init(o.gpa); - defer aw.deinit(); - ty.print(&aw.writer, pt, null) catch |err| switch (err) { - error.WriteFailed => return error.OutOfMemory, - }; - return aw.toOwnedSliceSentinel(0); - } - /// If the llvm function does not exist, create it. /// Note that this can be called before the function's semantic analysis has /// completed, so if any attributes rely on that, they must be done in updateFunc, not here. pub fn resolveLlvmFunction( o: *Object, - pt: Zcu.PerThread, nav_index: InternPool.Nav.Index, ) Allocator.Error!Builder.Function.Index { - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const gpa = o.gpa; const nav = ip.getNav(nav_index); @@ -2752,7 +2741,7 @@ pub const Object = struct { else .{ false, .none }; const function_index = try o.builder.addFunction( - try o.lowerType(pt, ty), + try o.lowerType(ty), try o.builder.strtabString((if (is_extern) nav.name else nav.fqn).toSlice(ip)), toLlvmAddressSpace(nav.resolved.?.@"addrspace", target), ); @@ -2785,7 +2774,7 @@ pub const Object = struct { try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); - const raw_llvm_ret_ty = try o.lowerType(pt, Type.fromInterned(fn_info.return_type)); + const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type)); try attributes.addParamAttr(llvm_arg_i, .{ .sret = raw_llvm_ret_ty }, &o.builder); llvm_arg_i += 1; @@ -2965,7 +2954,6 @@ pub const Object = struct { fn resolveGlobalUav( o: *Object, - pt: Zcu.PerThread, uav: InternPool.Index, llvm_addr_space: Builder.AddrSpace, alignment: InternPool.Alignment, @@ -2983,17 +2971,17 @@ pub const Object = struct { } errdefer assert(o.uav_map.remove(uav)); - const zcu = pt.zcu; + const zcu = o.zcu; const decl_ty = zcu.intern_pool.typeOf(uav); const variable_index = try o.builder.addVariable( try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav)}), - try o.lowerType(pt, Type.fromInterned(decl_ty)), + try o.lowerType(.fromInterned(decl_ty)), llvm_addr_space, ); gop.value_ptr.* = variable_index.ptrConst(&o.builder).global; - try variable_index.setInitializer(try o.lowerValue(pt, uav), &o.builder); + try variable_index.setInitializer(try o.lowerValue(uav), &o.builder); variable_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); variable_index.setMutability(.constant, &o.builder); variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); @@ -3003,14 +2991,13 @@ pub const Object = struct { fn resolveGlobalNav( o: *Object, - pt: Zcu.PerThread, nav_index: InternPool.Nav.Index, ) Allocator.Error!Builder.Variable.Index { const gop = try o.nav_map.getOrPut(o.gpa, nav_index); if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.variable; errdefer assert(o.nav_map.remove(nav_index)); - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const nav = ip.getNav(nav_index); const linkage: std.builtin.GlobalLinkage, const visibility: Builder.Visibility, const is_dll_import: bool = switch (nav.resolved.?.value) { @@ -3027,7 +3014,7 @@ pub const Object = struct { .strong, .weak => nav.name, .link_once => unreachable, }.toSlice(ip)), - try o.lowerType(pt, .fromInterned(nav.resolved.?.type)), + try o.lowerType(.fromInterned(nav.resolved.?.type)), toLlvmGlobalAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget()), ); gop.value_ptr.* = variable_index.ptrConst(&o.builder).global; @@ -3060,8 +3047,8 @@ pub const Object = struct { return o.builder.intType(o.zcu.errorSetBits()); } - pub fn lowerType(o: *Object, pt: Zcu.PerThread, t: Type) Allocator.Error!Builder.Type { - const zcu = pt.zcu; + pub fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type { + const zcu = o.zcu; const target = zcu.getTarget(); const ip = &zcu.intern_pool; return switch (t.toIntern()) { @@ -3134,7 +3121,7 @@ pub const Object = struct { => .ptr, .slice_const_u8_type, .slice_const_u8_sentinel_0_type, - => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(pt, Type.usize) }), + => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize) }), .optional_noreturn_type => unreachable, .anyerror_void_error_union_type, .adhoc_inferred_error_set_type, @@ -3175,24 +3162,24 @@ pub const Object = struct { .one, .many, .c => ptr_ty, .slice => try o.builder.structType(.normal, &.{ ptr_ty, - try o.lowerType(pt, Type.usize), + try o.lowerType(.usize), }), }; }, .array_type => |array_type| o.builder.arrayType( array_type.lenIncludingSentinel(), - try o.lowerType(pt, Type.fromInterned(array_type.child)), + try o.lowerType(.fromInterned(array_type.child)), ), .vector_type => |vector_type| o.builder.vectorType( .normal, vector_type.len, - try o.lowerType(pt, Type.fromInterned(vector_type.child)), + try o.lowerType(.fromInterned(vector_type.child)), ), .opt_type => |child_ty| { // Must stay in sync with `opt_payload` logic in `lowerPtr`. if (!Type.fromInterned(child_ty).hasRuntimeBits(zcu)) return .i8; - const payload_ty = try o.lowerType(pt, Type.fromInterned(child_ty)); + const payload_ty = try o.lowerType(.fromInterned(child_ty)); if (t.optionalReprIsPayload(zcu)) return payload_ty; comptime assert(optional_layout_version == 3); @@ -3214,7 +3201,7 @@ pub const Object = struct { const error_type = try o.errorIntType(); if (!Type.fromInterned(error_union_type.payload_type).hasRuntimeBits(zcu)) return error_type; - const payload_type = try o.lowerType(pt, Type.fromInterned(error_union_type.payload_type)); + const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type)); const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu); const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits())); @@ -3254,7 +3241,7 @@ pub const Object = struct { const struct_type = ip.loadStructType(t.toIntern()); if (struct_type.layout == .@"packed") { - const int_ty = try o.lowerType(pt, .fromInterned(struct_type.packed_backing_int_type)); + const int_ty = try o.lowerType(.fromInterned(struct_type.packed_backing_int_type)); try o.type_map.put(o.gpa, t.toIntern(), int_ty); return int_ty; } @@ -3290,7 +3277,7 @@ pub const Object = struct { if (!field_ty.hasRuntimeBits(zcu)) continue; - try llvm_field_types.append(o.gpa, try o.lowerType(pt, field_ty)); + try llvm_field_types.append(o.gpa, try o.lowerType(field_ty)); offset += field_ty.abiSize(zcu); } @@ -3346,7 +3333,7 @@ pub const Object = struct { if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) { continue; } - try llvm_field_types.append(o.gpa, try o.lowerType(pt, Type.fromInterned(field_ty))); + try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty))); offset += Type.fromInterned(field_ty).abiSize(zcu); } @@ -3367,7 +3354,7 @@ pub const Object = struct { const union_obj = ip.loadUnionType(t.toIntern()); if (union_obj.layout == .@"packed") { - const int_ty = try o.lowerType(pt, .fromInterned(union_obj.packed_backing_int_type)); + const int_ty = try o.lowerType(.fromInterned(union_obj.packed_backing_int_type)); try o.type_map.put(o.gpa, t.toIntern(), int_ty); return int_ty; } @@ -3375,13 +3362,13 @@ pub const Object = struct { const layout = Type.getUnionLayout(union_obj, zcu); if (layout.payload_size == 0) { - const enum_tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); + const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type)); try o.type_map.put(o.gpa, t.toIntern(), enum_tag_ty); return enum_tag_ty; } const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]); - const aligned_field_llvm_ty = try o.lowerType(pt, aligned_field_ty); + const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty); const payload_ty = ty: { if (layout.most_aligned_field_size == layout.payload_size) { @@ -3407,7 +3394,7 @@ pub const Object = struct { ); return ty; } - const enum_tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type)); + const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type)); // Put the tag before or after the payload depending on which one's // alignment is greater. @@ -3442,8 +3429,8 @@ pub const Object = struct { } return gop.value_ptr.*; }, - .enum_type => try o.lowerType(pt, t.intTagType(zcu)), - .func_type => |func_type| try o.lowerFnType(pt, func_type), + .enum_type => try o.lowerType(t.intTagType(zcu)), + .func_type => |func_type| try o.lowerFnType(func_type), .error_set_type, .inferred_error_set_type => try o.errorIntType(), // values, not types .undef, @@ -3469,11 +3456,11 @@ pub const Object = struct { }; } - fn lowerFnType(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { - const zcu = pt.zcu; + fn lowerFnType(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { + const zcu = o.zcu; const ip = &zcu.intern_pool; const target = zcu.getTarget(); - const ret_ty = try lowerFnRetTy(o, pt, fn_info); + const ret_ty = try lowerFnRetTy(o, fn_info); var llvm_params: std.ArrayList(Builder.Type) = .empty; defer llvm_params.deinit(o.gpa); @@ -3488,12 +3475,12 @@ pub const Object = struct { try llvm_params.append(o.gpa, llvm_ptr_ty); } - var it = iterateParamTypes(o, pt, fn_info); + var it = iterateParamTypes(o, fn_info); while (try it.next()) |lowering| switch (lowering) { .no_bits => continue, .byval => { const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - try llvm_params.append(o.gpa, try o.lowerType(pt, param_ty)); + try llvm_params.append(o.gpa, try o.lowerType(param_ty)); }, .byref, .byref_mut => { try llvm_params.append(o.gpa, .ptr); @@ -3508,7 +3495,7 @@ pub const Object = struct { const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); try llvm_params.appendSlice(o.gpa, &.{ try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)), - try o.lowerType(pt, Type.usize), + try o.lowerType(.usize), }); }, .multiple_llvm_types => { @@ -3516,7 +3503,7 @@ pub const Object = struct { }, .float_array => |count| { const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(param_ty, zcu).?); + const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?); try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty)); }, .i32_array, .i64_array => |arr_len| { @@ -3535,8 +3522,8 @@ pub const Object = struct { ); } - pub fn lowerValue(o: *Object, pt: Zcu.PerThread, arg_val: InternPool.Index) Allocator.Error!Builder.Constant { - const zcu = pt.zcu; + pub fn lowerValue(o: *Object, arg_val: InternPool.Index) Allocator.Error!Builder.Constant { + const zcu = o.zcu; const ip = &zcu.intern_pool; const target = zcu.getTarget(); @@ -3544,7 +3531,7 @@ pub const Object = struct { const val_key = ip.indexToKey(val.toIntern()); if (val.isUndef(zcu)) { - return o.builder.undefConst(try o.lowerType(pt, Type.fromInterned(val_key.typeOf()))); + return o.builder.undefConst(try o.lowerType(.fromInterned(val_key.typeOf()))); } const ty: Type = .fromInterned(val_key.typeOf()); @@ -3580,45 +3567,45 @@ pub const Object = struct { }, .enum_literal => unreachable, // non-runtime value .@"extern" => |@"extern"| { - const function_index = try o.resolveLlvmFunction(pt, @"extern".owner_nav); + const function_index = try o.resolveLlvmFunction(@"extern".owner_nav); return function_index.ptrConst(&o.builder).global.toConst(); }, .func => |func| { - const function_index = try o.resolveLlvmFunction(pt, func.owner_nav); + const function_index = try o.resolveLlvmFunction(func.owner_nav); return function_index.ptrConst(&o.builder).global.toConst(); }, .int => { var bigint_space: Value.BigIntSpace = undefined; const bigint = val.toBigInt(&bigint_space, zcu); - return lowerBigInt(o, pt, ty, bigint); + const llvm_int_ty = try o.builder.intType(ty.intInfo(zcu).bits); + return o.builder.bigIntConst(llvm_int_ty, bigint); }, .err => |err| { - const int = try pt.getErrorValue(err.name); - const llvm_int = try o.builder.intConst(try o.errorIntType(), int); - return llvm_int; + const int = zcu.intern_pool.getErrorValueIfExists(err.name).?; + return o.builder.intConst(try o.errorIntType(), int); }, .error_union => |error_union| { - const err_val = switch (error_union.val) { - .err_name => |err_name| try pt.intern(.{ .err = .{ - .ty = ty.errorUnionSet(zcu).toIntern(), - .name = err_name, - } }), - .payload => (try pt.intValue(try pt.errorIntType(), 0)).toIntern(), + const llvm_error_ty = try o.errorIntType(); + const llvm_error_value = switch (error_union.val) { + .err_name => |name| try o.builder.intConst( + llvm_error_ty, + zcu.intern_pool.getErrorValueIfExists(name).?, + ), + .payload => try o.builder.intConst(llvm_error_ty, 0), }; - const err_int_ty = try pt.errorIntType(); + const payload_type = ty.errorUnionPayload(zcu); if (!payload_type.hasRuntimeBits(zcu)) { // We use the error type directly as the type. - return o.lowerValue(pt, err_val); + return llvm_error_value; } const payload_align = payload_type.abiAlignment(zcu); - const error_align = err_int_ty.abiAlignment(zcu); - const llvm_error_value = try o.lowerValue(pt, err_val); - const llvm_payload_value = try o.lowerValue(pt, switch (error_union.val) { - .err_name => try pt.intern(.{ .undef = payload_type.toIntern() }), - .payload => |payload| payload, - }); + const error_align = Type.errorAbiAlignment(zcu); + const llvm_payload_value = switch (error_union.val) { + .err_name => try o.builder.undefConst(try o.lowerType(payload_type)), + .payload => |payload| try o.lowerValue(payload), + }; var fields: [3]Builder.Type = undefined; var vals: [3]Builder.Constant = undefined; @@ -3632,7 +3619,7 @@ pub const Object = struct { fields[0] = vals[0].typeOf(&o.builder); fields[1] = vals[1].typeOf(&o.builder); - const llvm_ty = try o.lowerType(pt, ty); + const llvm_ty = try o.lowerType(ty); const llvm_ty_fields = llvm_ty.structFields(&o.builder); if (llvm_ty_fields.len > 2) { assert(llvm_ty_fields.len == 3); @@ -3644,7 +3631,7 @@ pub const Object = struct { fields[0..llvm_ty_fields.len], ), vals[0..llvm_ty_fields.len]); }, - .enum_tag => |enum_tag| o.lowerValue(pt, enum_tag.int), + .enum_tag => |enum_tag| o.lowerValue(enum_tag.int), .float => switch (ty.floatBits(target)) { 16 => if (backendSupportsF16(target)) try o.builder.halfConst(val.toFloat(f16, zcu)) @@ -3659,10 +3646,10 @@ pub const Object = struct { 128 => try o.builder.fp128Const(val.toFloat(f128, zcu)), else => unreachable, }, - .ptr => try o.lowerPtr(pt, arg_val, 0), - .slice => |slice| return o.builder.structConst(try o.lowerType(pt, ty), &.{ - try o.lowerValue(pt, slice.ptr), - try o.lowerValue(pt, slice.len), + .ptr => try o.lowerPtr(arg_val, 0), + .slice => |slice| return o.builder.structConst(try o.lowerType(ty), &.{ + try o.lowerValue(slice.ptr), + try o.lowerValue(slice.len), }), .opt => |opt| { comptime assert(optional_layout_version == 3); @@ -3672,7 +3659,7 @@ pub const Object = struct { if (!payload_ty.hasRuntimeBits(zcu)) { return non_null_bit; } - const llvm_ty = try o.lowerType(pt, ty); + const llvm_ty = try o.lowerType(ty); if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) { .none => switch (llvm_ty.tag(&o.builder)) { .integer => try o.builder.intConst(llvm_ty, 0), @@ -3680,16 +3667,16 @@ pub const Object = struct { .structure => try o.builder.zeroInitConst(llvm_ty), else => unreachable, }, - else => |payload| try o.lowerValue(pt, payload), + else => |payload| try o.lowerValue(payload), }; assert(payload_ty.zigTypeTag(zcu) != .@"fn"); var fields: [3]Builder.Type = undefined; var vals: [3]Builder.Constant = undefined; - vals[0] = try o.lowerValue(pt, switch (opt.val) { - .none => try pt.intern(.{ .undef = payload_ty.toIntern() }), - else => |payload| payload, - }); + vals[0] = switch (opt.val) { + .none => try o.builder.undefConst(try o.lowerType(payload_ty)), + else => |payload| try o.lowerValue(payload), + }; vals[1] = non_null_bit; fields[0] = vals[0].typeOf(&o.builder); fields[1] = vals[1].typeOf(&o.builder); @@ -3705,14 +3692,14 @@ pub const Object = struct { fields[0..llvm_ty_fields.len], ), vals[0..llvm_ty_fields.len]); }, - .bitpack => |bitpack| return o.lowerValue(pt, bitpack.backing_int_val), + .bitpack => |bitpack| return o.lowerValue(bitpack.backing_int_val), .aggregate => |aggregate| switch (ip.indexToKey(ty.toIntern())) { .array_type => |array_type| switch (aggregate.storage) { .bytes => |bytes| try o.builder.stringConst(try o.builder.string( bytes.toSlice(array_type.lenIncludingSentinel(), ip), )), .elems => |elems| { - const array_ty = try o.lowerType(pt, ty); + const array_ty = try o.lowerType(ty); const elem_ty = array_ty.childType(&o.builder); assert(elems.len == array_ty.aggregateLen(&o.builder)); @@ -3732,7 +3719,7 @@ pub const Object = struct { var need_unnamed = false; for (vals, fields, elems) |*result_val, *result_field, elem| { - result_val.* = try o.lowerValue(pt, elem); + result_val.* = try o.lowerValue(elem); result_field.* = result_val.typeOf(&o.builder); if (result_field.* != elem_ty) need_unnamed = true; } @@ -3744,7 +3731,7 @@ pub const Object = struct { .repeated_elem => |elem| { const len: usize = @intCast(array_type.len); const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel()); - const array_ty = try o.lowerType(pt, ty); + const array_ty = try o.lowerType(ty); const elem_ty = array_ty.childType(&o.builder); const ExpectedContents = extern struct { @@ -3762,12 +3749,12 @@ pub const Object = struct { defer allocator.free(fields); var need_unnamed = false; - @memset(vals[0..len], try o.lowerValue(pt, elem)); + @memset(vals[0..len], try o.lowerValue(elem)); @memset(fields[0..len], vals[0].typeOf(&o.builder)); if (fields[0] != elem_ty) need_unnamed = true; if (array_type.sentinel != .none) { - vals[len] = try o.lowerValue(pt, array_type.sentinel); + vals[len] = try o.lowerValue(array_type.sentinel); fields[len] = vals[len].typeOf(&o.builder); if (fields[len] != elem_ty) need_unnamed = true; } @@ -3779,7 +3766,7 @@ pub const Object = struct { }, }, .vector_type => |vector_type| { - const vector_ty = try o.lowerType(pt, ty); + const vector_ty = try o.lowerType(ty); switch (aggregate.storage) { .bytes, .elems => { const ExpectedContents = [Builder.expected_fields_len]Builder.Constant; @@ -3796,7 +3783,7 @@ pub const Object = struct { result_val.* = try o.builder.intConst(.i8, byte); }, .elems => |elems| for (vals, elems) |*result_val, elem| { - result_val.* = try o.lowerValue(pt, elem); + result_val.* = try o.lowerValue(elem); }, .repeated_elem => unreachable, } @@ -3804,12 +3791,12 @@ pub const Object = struct { }, .repeated_elem => |elem| return o.builder.splatConst( vector_ty, - try o.lowerValue(pt, elem), + try o.lowerValue(elem), ), } }, .tuple_type => |tuple| { - const struct_ty = try o.lowerType(pt, ty); + const struct_ty = try o.lowerType(ty); const llvm_len = struct_ty.aggregateLen(&o.builder); const ExpectedContents = extern struct { @@ -3835,8 +3822,8 @@ pub const Object = struct { tuple.types.get(ip), tuple.values.get(ip), 0.., - ) |field_ty, field_val, field_index| { - if (field_val != .none) continue; + ) |field_ty, field_comptime_val, field_index| { + if (field_comptime_val != .none) continue; if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue; const field_align = Type.fromInterned(field_ty).abiAlignment(zcu); @@ -3854,8 +3841,11 @@ pub const Object = struct { llvm_index += 1; } - vals[llvm_index] = - try o.lowerValue(pt, (try val.fieldValue(pt, field_index)).toIntern()); + vals[llvm_index] = switch (aggregate.storage) { + .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)), + .elems => |elems| try o.lowerValue(elems[field_index]), + .repeated_elem => |elem| try o.lowerValue(elem), + }; fields[llvm_index] = vals[llvm_index].typeOf(&o.builder); if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index]) need_unnamed = true; @@ -3883,7 +3873,7 @@ pub const Object = struct { }, .struct_type => { const struct_type = ip.loadStructType(ty.toIntern()); - const struct_ty = try o.lowerType(pt, ty); + const struct_ty = try o.lowerType(ty); assert(struct_type.layout != .@"packed"); const llvm_len = struct_ty.aggregateLen(&o.builder); @@ -3927,10 +3917,11 @@ pub const Object = struct { continue; } - vals[llvm_index] = try o.lowerValue( - pt, - (try val.fieldValue(pt, field_index)).toIntern(), - ); + vals[llvm_index] = switch (aggregate.storage) { + .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)), + .elems => |elems| try o.lowerValue(elems[field_index]), + .repeated_elem => |elem| try o.lowerValue(elem), + }; fields[llvm_index] = vals[llvm_index].typeOf(&o.builder); if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index]) need_unnamed = true; @@ -3959,9 +3950,9 @@ pub const Object = struct { else => unreachable, }, .un => |un| { - const union_ty = try o.lowerType(pt, ty); + const union_ty = try o.lowerType(ty); const layout = ty.unionGetLayout(zcu); - if (layout.payload_size == 0) return o.lowerValue(pt, un.tag); + if (layout.payload_size == 0) return o.lowerValue(un.tag); const union_obj = zcu.typeToUnion(ty).?; const container_layout = union_obj.layout; @@ -3982,7 +3973,7 @@ pub const Object = struct { const padding_len = layout.payload_size; break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8)); } - const payload = try o.lowerValue(pt, un.val); + const payload = try o.lowerValue(un.val); const payload_ty = payload.typeOf(&o.builder); if (payload_ty != union_ty.structFields(&o.builder)[ @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align)) @@ -3997,7 +3988,7 @@ pub const Object = struct { ); } else p: { assert(layout.tag_size == 0); - const union_val = try o.lowerValue(pt, un.val); + const union_val = try o.lowerValue(un.val); need_unnamed = true; break :p union_val; }; @@ -4007,7 +3998,7 @@ pub const Object = struct { try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty}) else union_ty, &.{payload}); - const tag = try o.lowerValue(pt, un.tag); + const tag = try o.lowerValue(un.tag); const tag_ty = tag.typeOf(&o.builder); var fields: [3]Builder.Type = undefined; var vals: [3]Builder.Constant = undefined; @@ -4033,52 +4024,45 @@ pub const Object = struct { }; } - fn lowerBigInt( - o: *Object, - pt: Zcu.PerThread, - ty: Type, - bigint: std.math.big.int.Const, - ) Allocator.Error!Builder.Constant { - const zcu = pt.zcu; - return o.builder.bigIntConst(try o.builder.intType(ty.intInfo(zcu).bits), bigint); - } - fn lowerPtr( o: *Object, - pt: Zcu.PerThread, ptr_val: InternPool.Index, prev_offset: u64, ) Allocator.Error!Builder.Constant { - const zcu = pt.zcu; + const zcu = o.zcu; const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr; const offset: u64 = prev_offset + ptr.byte_offset; return switch (ptr.base_addr) { .nav => |nav| { - const base_ptr = try o.lowerNavRefValue(pt, nav); + const base_ptr = try o.lowerNavRefValue(nav); return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{ try o.builder.intConst(.i64, offset), }); }, .uav => |uav| { - const base_ptr = try o.lowerUavRef(pt, uav); + const orig_ptr_ty: Type = .fromInterned(uav.orig_ty); + const base_ptr = try o.lowerUavRef( + uav.val, + orig_ptr_ty.ptrAlignment(zcu), + orig_ptr_ty.ptrAddressSpace(zcu), + ); return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{ try o.builder.intConst(.i64, offset), }); }, .int => try o.builder.castConst( .inttoptr, - try o.builder.intConst(try o.lowerType(pt, Type.usize), offset), - try o.lowerType(pt, Type.fromInterned(ptr.ty)), + try o.builder.intConst(try o.lowerType(.usize), offset), + try o.lowerType(.fromInterned(ptr.ty)), ), .eu_payload => |eu_ptr| try o.lowerPtr( - pt, eu_ptr, offset + codegen.errUnionPayloadOffset( Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu), zcu, ), ), - .opt_payload => |opt_ptr| try o.lowerPtr(pt, opt_ptr, offset), + .opt_payload => |opt_ptr| try o.lowerPtr(opt_ptr, offset), .field => |field| { const agg_ty = Value.fromInterned(field.base).typeOf(zcu).childType(zcu); const field_off: u64 = switch (agg_ty.zigTypeTag(zcu)) { @@ -4096,13 +4080,13 @@ pub const Object = struct { }, else => unreachable, }; - return o.lowerPtr(pt, field.base, offset + field_off); + return o.lowerPtr(field.base, offset + field_off); }, .arr_elem => |arr_elem| { const base_ptr_ty = Value.fromInterned(arr_elem.base).typeOf(zcu); assert(base_ptr_ty.ptrSize(zcu) == .many); const elem_size = base_ptr_ty.childType(zcu).abiSize(zcu); - return o.lowerPtr(pt, arr_elem.base, offset + elem_size * arr_elem.index); + return o.lowerPtr(arr_elem.base, offset + elem_size * arr_elem.index); }, .comptime_field => unreachable, .comptime_alloc => unreachable, @@ -4113,88 +4097,84 @@ pub const Object = struct { /// Maybe the logic could be unified. pub fn lowerUavRef( o: *Object, - pt: Zcu.PerThread, - uav: InternPool.Key.Ptr.BaseAddr.Uav, + uav_val: InternPool.Index, + /// Must not be `.none`. + @"align": InternPool.Alignment, + @"addrspace": std.builtin.AddressSpace, ) Allocator.Error!Builder.Constant { - const zcu = pt.zcu; + assert(@"align" != .none); + + const zcu = o.zcu; const ip = &zcu.intern_pool; - const uav_val = uav.val; - const uav_ty = Type.fromInterned(ip.typeOf(uav_val)); - const target = zcu.getTarget(); + const uav_ty: Type = .fromInterned(ip.typeOf(uav_val)); switch (ip.indexToKey(uav_val)) { - .func => @panic("TODO"), - .@"extern" => @panic("TODO"), + .func => unreachable, // should be using a Nav ref + .@"extern" => unreachable, // should be using a Nav ref else => {}, } - const ptr_ty = Type.fromInterned(uav.orig_ty); - - if (!uav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) { - return o.lowerPtrToVoid(pt, ptr_ty); + if (!uav_ty.hasRuntimeBits(zcu)) { + return o.lowerPtrToVoid(@"align", @"addrspace"); } - assert(uav_ty.zigTypeTag(zcu) != .@"fn"); // should be using a Nav ref + const llvm_addrspace = toLlvmAddressSpace(@"addrspace", zcu.getTarget()); + const llvm_global = (try o.resolveGlobalUav(uav_val, llvm_addrspace, @"align")).ptrConst(&o.builder).global; - const llvm_addr_space = toLlvmAddressSpace(ptr_ty.ptrAddressSpace(zcu), target); - const alignment = ptr_ty.ptrAlignment(zcu); - const llvm_global = (try o.resolveGlobalUav(pt, uav.val, llvm_addr_space, alignment)).ptrConst(&o.builder).global; - - const llvm_val = try o.builder.convConst( + return o.builder.convConst( llvm_global.toConst(), - try o.builder.ptrType(llvm_addr_space), + try o.builder.ptrType(llvm_addrspace), ); - - return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty)); } - pub fn lowerNavRefValue(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant { - const zcu = pt.zcu; + pub fn lowerNavRefValue(o: *Object, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant { + const zcu = o.zcu; const ip = &zcu.intern_pool; const nav = ip.getNav(nav_index); const nav_ty: Type = .fromInterned(nav.resolved.?.type); - const ptr_ty = try pt.navPtrType(nav_index); if (nav.getExtern(ip) == null and !nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) { - return o.lowerPtrToVoid(pt, ptr_ty); + return o.lowerPtrToVoid(nav.resolved.?.@"align", nav.resolved.?.@"addrspace"); } const llvm_global = if (nav_ty.zigTypeTag(zcu) == .@"fn") - (try o.resolveLlvmFunction(pt, nav_index)).ptrConst(&o.builder).global + (try o.resolveLlvmFunction(nav_index)).ptrConst(&o.builder).global else - (try o.resolveGlobalNav(pt, nav_index)).ptrConst(&o.builder).global; + (try o.resolveGlobalNav(nav_index)).ptrConst(&o.builder).global; - const llvm_val = try o.builder.convConst( + return try o.builder.convConst( llvm_global.toConst(), try o.builder.ptrType(toLlvmAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget())), ); - - return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty)); } - pub fn lowerPtrToVoid(o: *Object, pt: Zcu.PerThread, ptr_ty: Type) Allocator.Error!Builder.Constant { - const zcu = pt.zcu; + pub fn lowerPtrToVoid( + o: *Object, + @"align": InternPool.Alignment, + @"addrspace": std.builtin.AddressSpace, + ) Allocator.Error!Builder.Constant { + const target = o.zcu.getTarget(); // Even though we are pointing at something which has zero bits (e.g. `void`), // Pointers are defined to have bits. So we must return something here. // The value cannot be undefined, because we use the `nonnull` annotation // for non-optional pointers. We also need to respect the alignment, even though // the address will never be dereferenced. - const int: u64 = ptr_ty.ptrInfo(zcu).flags.alignment.toByteUnits() orelse + const int: u64 = @"align".toByteUnits() orelse // Note that these 0xaa values are appropriate even in release-optimized builds // because we need a well-defined value that is not null, and LLVM does not // have an "undef_but_not_null" attribute. As an example, if this `alloc` AIR // instruction is followed by a `wrap_optional`, it will return this value // verbatim, and the result should test as non-null. - switch (zcu.getTarget().ptrBitWidth()) { + switch (target.ptrBitWidth()) { 16 => 0xaaaa, 32 => 0xaaaaaaaa, 64 => 0xaaaaaaaa_aaaaaaaa, else => unreachable, }; - const llvm_usize = try o.lowerType(pt, Type.usize); - const llvm_ptr_ty = try o.lowerType(pt, ptr_ty); + const llvm_usize = try o.lowerType(.usize); + const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", target)); return o.builder.castConst(.inttoptr, try o.builder.intConst(llvm_usize, int), llvm_ptr_ty); } @@ -4207,7 +4187,7 @@ pub const Object = struct { fn_info: InternPool.Key.FuncType, llvm_arg_i: u32, ) Allocator.Error!void { - const zcu = pt.zcu; + const zcu = o.zcu; if (param_ty.isPtrAtRuntime(zcu)) { const ptr_info = param_ty.ptrInfo(zcu); if (std.math.cast(u5, param_index)) |i| { @@ -4226,7 +4206,7 @@ pub const Object = struct { .x86_64_interrupt, .x86_interrupt, => { - const child_type = try lowerType(o, pt, Type.fromInterned(ptr_info.child)); + const child_type = try lowerType(o, Type.fromInterned(ptr_info.child)); try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder); }, } @@ -4292,8 +4272,8 @@ pub const Object = struct { } /// MLUGG TODO: this also needs incremental updates dumbass - pub fn getEnumTagNameFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index { - const zcu = pt.zcu; + pub fn getEnumTagNameFunction(o: *Object, enum_ty: Type) !Builder.Function.Index { + const zcu = o.zcu; const ip = &zcu.intern_pool; const enum_type = ip.loadEnumType(enum_ty.toIntern()); @@ -4301,11 +4281,12 @@ pub const Object = struct { if (gop.found_existing) return gop.value_ptr.ptrConst(&o.builder).kind.function; errdefer assert(o.enum_tag_name_map.remove(enum_ty.toIntern())); - const usize_ty = try o.lowerType(pt, Type.usize); - const ret_ty = try o.lowerType(pt, Type.slice_const_u8_sentinel_0); + const usize_ty = try o.lowerType(.usize); + const ret_ty = try o.lowerType(.slice_const_u8_sentinel_0); + const llvm_int_ty = try o.lowerType(.fromInterned(enum_type.int_tag_type)); const target = &zcu.root_mod.resolved_target.result; const function_index = try o.builder.addFunction( - try o.builder.fnType(ret_ty, &.{try o.lowerType(pt, Type.fromInterned(enum_type.int_tag_type))}, .normal), + try o.builder.fnType(ret_ty, &.{llvm_int_ty}, .normal), try o.builder.strtabStringFmt("__zig_tag_name_{f}", .{enum_type.name.fmt(ip)}), toLlvmAddressSpace(.generic, target), ); @@ -4353,11 +4334,11 @@ pub const Object = struct { }); const return_block = try wip.block(1, "Name"); - const this_tag_int_value = try o.lowerValue( - pt, - (try pt.enumValueFieldIndex(enum_ty, @intCast(field_index))).toIntern(), - ); - try wip_switch.addCase(this_tag_int_value, return_block, &wip); + const llvm_tag_val = switch (enum_type.field_values.getOrNone(ip, field_index)) { + .none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered + else => |tag_val_ip| try o.lowerValue(tag_val_ip), + }; + try wip_switch.addCase(llvm_tag_val, return_block, &wip); wip.cursor = .{ .block = return_block }; _ = try wip.ret(name_val); @@ -4375,8 +4356,8 @@ pub const Object = struct { return o.lazy_abi_aligns.items[@intFromEnum(index)]; } - pub fn getIsNamedEnumValueFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index { - const zcu = pt.zcu; + pub fn getIsNamedEnumValueFunction(o: *Object, enum_ty: Type) !Builder.Function.Index { + const zcu = o.zcu; const ip = &zcu.intern_pool; const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern()); @@ -4390,23 +4371,21 @@ pub const Object = struct { toLlvmAddressSpace(.generic, zcu.getTarget()), ); gop.value_ptr.* = function_index; - try o.updateIsNamedEnumValueFunction(pt, enum_ty, function_index); + try o.updateIsNamedEnumValueFunction(enum_ty, function_index); return function_index; } fn updateIsNamedEnumValueFunction( o: *Object, - pt: Zcu.PerThread, enum_ty: Type, function_index: Builder.Function.Index, ) Allocator.Error!void { - const zcu = pt.zcu; + const zcu = o.zcu; const builder = &o.builder; const loaded_enum = zcu.intern_pool.loadEnumType(enum_ty.toIntern()); - function_index.ptrConst(builder).global.ptr(builder).type = try builder.fnType( - .i1, - &.{try o.lowerType(pt, .fromInterned(loaded_enum.int_tag_type))}, - .normal, - ); + + const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type)); + function_index.ptrConst(builder).global.ptr(builder).type = + try builder.fnType(.i1, &.{llvm_int_ty}, .normal); var attributes: Builder.FunctionAttributes.Wip = .{}; defer attributes.deinit(builder); @@ -4429,13 +4408,19 @@ pub const Object = struct { var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(loaded_enum.field_names.len), .none); defer wip_switch.finish(&wip); - for (0..loaded_enum.field_names.len) |field_index| { - const this_tag_int_value = try o.lowerValue( - pt, - (try pt.enumValueFieldIndex(enum_ty, @intCast(field_index))).toIntern(), - ); - try wip_switch.addCase(this_tag_int_value, named_block, &wip); + if (loaded_enum.field_values.len > 0) { + for (loaded_enum.field_values.get(&zcu.intern_pool)) |tag_val_ip| { + const llvm_tag_val = try o.lowerValue(tag_val_ip); + try wip_switch.addCase(llvm_tag_val, named_block, &wip); + } + } else { + // Auto-numbered. + for (0..loaded_enum.field_names.len) |field_index| { + const llvm_tag_val = try o.builder.intConst(llvm_int_ty, field_index); + try wip_switch.addCase(llvm_tag_val, named_block, &wip); + } } + wip.cursor = .{ .block = named_block }; _ = try wip.ret(.true); diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index 4b1edf9ff9c1d8076c9c736f71ed73f1b69de636..ec8e0b1d0577dadd8a299bf0cb274be8723a76e9 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -73,7 +73,7 @@ const TodoError = Zcu.CodegenFailError; /// Avoid introducing new calls to this function---see documentation comment on `TodoError`. fn todo(fg: *FuncGen, comptime format: []const u8, args: anytype) TodoError { @branchHint(.cold); - return fg.pt.zcu.codegenFail( + return fg.object.zcu.codegenFail( fg.nav_index, "TODO (LLVM): " ++ format, args, @@ -81,7 +81,7 @@ fn todo(fg: *FuncGen, comptime format: []const u8, args: anytype) TodoError { } fn ownerModule(fg: *const FuncGen) *Package.Module { - return fg.pt.zcu.navFileScope(fg.nav_index).mod.?; + return fg.object.zcu.navFileScope(fg.nav_index).mod.?; } fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void { @@ -154,34 +154,35 @@ fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) Allocator.Error!Builder.Value const gop = try self.func_inst_table.getOrPut(gpa, inst); if (gop.found_existing) return gop.value_ptr.*; - const llvm_val = try self.resolveValue((try self.air.value(inst, self.pt)).?); + const llvm_val = try self.resolveValue(.fromInterned(inst.toInterned().?)); gop.value_ptr.* = llvm_val.toValue(); return llvm_val.toValue(); } fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty = val.typeOf(zcu); if (!isByRef(ty, zcu)) { - return o.lowerValue(pt, val.toIntern()); + return o.lowerValue(val.toIntern()); } else { // We need a pointer to a global constant, i.e. a UAV. - return o.lowerUavRef(pt, .{ - .val = val.toIntern(), - .orig_ty = (try pt.singleConstPtrType(ty)).toIntern(), - }); + return o.lowerUavRef( + val.toIntern(), + ty.abiAlignment(zcu), + target_util.defaultAddressSpace(zcu.getTarget(), .global_constant), + ); } } +/// MLUGG TODO okay yeah prolly delete this again fn lowerType(fg: *const FuncGen, ty: Type) Allocator.Error!Builder.Type { - return fg.object.lowerType(fg.pt, ty); + return fg.object.lowerType(ty); } pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void { const o = self.object; - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const ip = &zcu.intern_pool; const air_tags = self.air.instructions.items(.tag); switch (coverage_point) { @@ -514,8 +515,7 @@ fn genBodyDebugScope( defer self.scope = old_scope; if (maybe_inline_func) |inline_func| { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const func = zcu.funcInfo(inline_func); @@ -529,18 +529,13 @@ fn genBodyDebugScope( const line_number = self.base_line + 1; self.inlined_at = try self.wip.debug_location.toMetadata(&o.builder); - const fn_ty = try pt.funcType(.{ - .param_types = &.{}, - .return_type = .void_type, - }); - self.scope = try o.builder.debugSubprogram( self.file, try o.builder.metadataString(nav.name.toSlice(&zcu.intern_pool)), try o.builder.metadataString(nav.fqn.toSlice(&zcu.intern_pool)), line_number, line_number + func.lbrace_line, - try o.getDebugType(pt, fn_ty), + try o.builder.debugSubroutineType(null), .{ .di_flags = .{ .StaticMember = true }, .sp_flags = .{ @@ -582,7 +577,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif const args = air_call.args; const o = self.object; const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const callee_ty = self.typeOf(air_call.callee); const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) { @@ -628,7 +623,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif try llvm_args.append(self.err_ret_trace); } - var it = iterateParamTypes(o, pt, fn_info); + var it = iterateParamTypes(o, fn_info); while (try it.nextCall(self, args)) |lowering| switch (lowering) { .no_bits => continue, .byval => { @@ -761,7 +756,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif { // Add argument attributes. - it = iterateParamTypes(o, pt, fn_info); + it = iterateParamTypes(o, fn_info); it.llvm_index += @intFromBool(sret); it.llvm_index += @intFromBool(err_return_tracing); while (try it.next()) |lowering| switch (lowering) { @@ -852,7 +847,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif } } - const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); + const abi_ret_ty = try lowerFnRetTy(o, fn_info); if (abi_ret_ty != llvm_ret_ty) { // In this case the function return type is honoring the calling convention by having @@ -881,12 +876,11 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!void { const o = fg.object; - const pt = fg.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const target = zcu.getTarget(); const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin())); const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?; - const panic_global = try o.resolveLlvmFunction(pt, panic_func.owner_nav); + const panic_global = try o.resolveLlvmFunction(panic_func.owner_nav); const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto; if (has_err_trace) assert(fg.err_ret_trace != .none); @@ -905,30 +899,19 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!void { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const ret_ty = self.typeOf(un_op); if (self.ret_ptr != .none) { - const ptr_ty = try pt.singleMutPtrType(ret_ty); - const operand = try self.resolveInst(un_op); - const val_is_undef = if (try self.air.value(un_op, pt)) |val| val.isUndef(zcu) else false; - if (val_is_undef and safety) undef: { - const ptr_info = ptr_ty.ptrInfo(zcu); - const needs_bitmask = (ptr_info.packed_offset.host_size != 0); - if (needs_bitmask) { - // TODO: only some bits are to be undef, we cannot write with a simple memset. - // meanwhile, ignore the write rather than stomping over valid bits. - // https://github.com/ziglang/zig/issues/15337 - break :undef; - } + const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false; + if (val_is_undef and safety) { const len = try o.builder.intValue(try self.lowerType(.usize), ret_ty.abiSize(zcu)); _ = try self.wip.callMemSet( self.ret_ptr, - ptr_ty.ptrAlignment(zcu).toLlvm(), + ret_ty.abiAlignment(zcu).toLlvm(), try o.builder.intValue(.i8, 0xaa), len, .normal, @@ -951,7 +934,12 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo return; } - try self.store(self.ret_ptr, ptr_ty, operand, .none); + try self.store( + self.ret_ptr, + .none, + operand, + ret_ty, + ); _ = try self.wip.retVoid(); return; } @@ -968,15 +956,15 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo return; } - const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); + const abi_ret_ty = try lowerFnRetTy(o, fn_info); const operand = try self.resolveInst(un_op); - const val_is_undef = if (try self.air.value(un_op, pt)) |val| val.isUndef(zcu) else false; + const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false; const alignment = ret_ty.abiAlignment(zcu).toLlvm(); if (val_is_undef and safety) { const llvm_ret_ty = operand.typeOfWip(&self.wip); const rp = try self.buildAlloca(llvm_ret_ty, alignment); - const len = try o.builder.intValue(try self.lowerType(Type.usize), ret_ty.abiSize(zcu)); + const len = try o.builder.intValue(try self.lowerType(.usize), ret_ty.abiSize(zcu)); _ = try self.wip.callMemSet( rp, alignment, @@ -1014,8 +1002,7 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const ptr_ty = self.typeOf(un_op); @@ -1037,7 +1024,7 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void { return; } const ptr = try self.resolveInst(un_op); - const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info); + const abi_ret_ty = try lowerFnRetTy(o, fn_info); const alignment = ret_ty.abiAlignment(zcu).toLlvm(); _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, "")); return; @@ -1053,14 +1040,13 @@ fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const src_list = try self.resolveInst(ty_op.operand); const va_list_ty = ty_op.ty.toType(); const llvm_va_list_ty = try self.lowerType(va_list_ty); - const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); + const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm(); const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, ""); @@ -1079,12 +1065,11 @@ fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const va_list_ty = self.typeOfIndex(inst); const llvm_va_list_ty = try self.lowerType(va_list_ty); - const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); + const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm(); const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, ""); @@ -1145,8 +1130,7 @@ fn cmp( rhs: Builder.Value, ) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const scalar_ty = operand_ty.scalarType(zcu); const int_ty = switch (scalar_ty.zigTypeTag(zcu)) { .@"enum" => scalar_ty.intTagType(zcu), @@ -1244,8 +1228,7 @@ fn lowerBlock( maybe_inline_func: ?InternPool.Index, body: []const Air.Inst.Index, ) TodoError!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const inst_ty = self.typeOfIndex(inst); if (inst_ty.isNoReturn(zcu)) { @@ -1294,7 +1277,7 @@ fn lowerBlock( } fn airBr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br; const block = self.blocks.get(branch.block_inst).?; @@ -1324,12 +1307,12 @@ fn lowerSwitchDispatch( dispatch_info: SwitchDispatchInfo, ) Allocator.Error!void { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const cond_ty = self.typeOf(cond_ref); const switch_br = self.air.unwrapSwitch(switch_inst); - if (try self.air.value(cond_ref, pt)) |cond_val| { + if (cond_ref.toInterned()) |cond_ip_index| { + const cond_val: Value = .fromInterned(cond_ip_index); // Comptime-known dispatch. Iterate the cases to find the correct // one, and branch to the corresponding element of `case_blocks`. var it = switch_br.iterateCases(); @@ -1413,7 +1396,7 @@ fn lowerSwitchDispatch( // The switch prongs will correspond to our scalar cases. Ranges will // be handled by conditional branches in the `else` prong. - const llvm_usize = try self.lowerType(Type.usize); + const llvm_usize = try self.lowerType(.usize); const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder)) try self.wip.cast(.ptrtoint, cond, llvm_usize, "") else @@ -1581,7 +1564,7 @@ fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) TodoError!Builde } fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) TodoError!Builder.Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const unwrapped_try = self.air.unwrapTryPtr(inst); const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr); const body = unwrapped_try.else_body; @@ -1605,8 +1588,7 @@ fn lowerTry( err_cold: bool, ) TodoError!Builder.Value { const o = fg.object; - const pt = fg.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const payload_ty = err_union_ty.errorUnionPayload(zcu); const payload_has_bits = payload_ty.hasRuntimeBits(zcu); const error_type = try o.errorIntType(); @@ -1667,8 +1649,7 @@ fn lowerTry( fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) TodoError!void { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const switch_br = self.air.unwrapSwitch(inst); @@ -1773,8 +1754,8 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod const table_includes_else = item_count != table_len; break :jmp_table .{ - .min = try o.lowerValue(pt, min.toIntern()), - .max = try o.lowerValue(pt, max.toIntern()), + .min = try o.lowerValue(min.toIntern()), + .max = try o.lowerValue(max.toIntern()), .in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) { .none, .cold => .none, .unpredictable => .unpredictable, @@ -1883,7 +1864,7 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod } fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) ?[2]Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; var it = switch_br.iterateCases(); var min: ?Value = null; var max: ?Value = null; @@ -1928,8 +1909,7 @@ fn airLoop(self: *FuncGen, inst: Air.Inst.Index) TodoError!void { fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand_ty = self.typeOf(ty_op.operand); const array_ty = operand_ty.childType(zcu); @@ -1942,8 +1922,7 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); @@ -1964,7 +1943,7 @@ fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value ); const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu))) orelse { - return self.todo("float_from_int from '{f}' without intrinsics", .{operand_scalar_ty.fmt(pt)}); + return self.todo("float_from_int on {d} bit integer", .{operand_scalar_ty.bitSize(zcu)}); }; const rt_int_ty = try o.builder.intType(rt_int_bits); var extended = try self.wip.conv( @@ -2011,8 +1990,7 @@ fn airIntFromFloat( _ = fast; const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const target = zcu.getTarget(); const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; @@ -2035,7 +2013,7 @@ fn airIntFromFloat( } const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu))) orelse { - return self.todo("int_from_float to '{f}' without intrinsics", .{dest_scalar_ty.fmt(pt)}); + return self.todo("int_from_float to {d} bit integer", .{dest_scalar_ty.bitSize(zcu)}); }; const ret_ty = try o.builder.intType(rt_int_bits); const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: { @@ -2074,14 +2052,13 @@ fn airIntFromFloat( } fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value { - const zcu = fg.pt.zcu; + const zcu = fg.object.zcu; return if (ty.isSlice(zcu)) fg.wip.extractValue(ptr, &.{0}, "") else ptr; } fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value { const o = fg.object; - const pt = fg.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const llvm_usize = try fg.lowerType(.usize); switch (ty.ptrSize(zcu)) { .slice => { @@ -2109,15 +2086,14 @@ fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) Allocator.Err } fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: u1) Allocator.Error!Builder.Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const slice_ptr = try self.resolveInst(ty_op.operand); return self.ptraddConst(slice_ptr, index * Type.usize.abiSize(zcu)); } fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const slice_ty = self.typeOf(bin_op.lhs); const slice = try self.resolveInst(bin_op.lhs); @@ -2138,8 +2114,7 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder } fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; const slice_ty = self.typeOf(bin_op.lhs); @@ -2151,8 +2126,7 @@ fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder } fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const array_ty = self.typeOf(bin_op.lhs); @@ -2174,28 +2148,25 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder } fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const ptr_ty = self.typeOf(bin_op.lhs); const elem_ty = ptr_ty.indexableElem(zcu); const base_ptr = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); - const ptr = try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)); - if (isByRef(elem_ty, zcu)) { - self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm(); - return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal); - } self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - return self.load(ptr, ptr_ty); + return self.load( + try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)), + elem_ty, + ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)).toLlvm(), + if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, + ); } fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; const ptr_ty = self.typeOf(bin_op.lhs); @@ -2232,8 +2203,7 @@ fn airStructFieldPtrIndex( fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data; const struct_ty = self.typeOf(struct_field.struct_operand); @@ -2298,8 +2268,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; @@ -2310,7 +2279,7 @@ fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build if (field_offset == 0) return field_ptr; const res_ty = try self.lowerType(ty_pl.ty.toType()); - const llvm_usize = try self.lowerType(Type.usize); + const llvm_usize = try self.lowerType(.usize); const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, ""); const base_ptr_int = try self.wip.bin( @@ -2358,7 +2327,7 @@ fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const operand = try self.resolveInst(pl_op.operand); const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload); @@ -2415,7 +2384,7 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Er try o.getDebugType(pt, operand_ty), ); - const zcu = pt.zcu; + const zcu = o.zcu; const owner_mod = self.ownerModule(); if (isByRef(operand_ty, zcu)) { _ = try self.wip.callIntrinsic( @@ -2501,8 +2470,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value { // This stores whether we need to add an elementtype attribute and // if so, the element type itself. const llvm_param_attrs = try arena.alloc(Builder.Type, max_param_count); - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const target = zcu.getTarget(); @@ -2855,8 +2823,7 @@ fn airIsNonNull( cond: Builder.IntegerCondition, ) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); const operand_ty = self.typeOf(un_op); @@ -2905,8 +2872,7 @@ fn airIsErr( operand_is_ptr: bool, ) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); const operand_ty = self.typeOf(un_op); @@ -2959,8 +2925,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro comptime assert(optional_layout_version == 3); const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); const optional_ptr_ty = self.typeOf(ty_op.operand); @@ -3001,8 +2966,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro } fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); const optional_ty = self.typeOf(ty_op.operand); @@ -3018,8 +2982,7 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil } fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); const operand_ty = self.typeOf(ty_op.operand); @@ -3050,8 +3013,7 @@ fn airErrUnionErr( operand_is_ptr: bool, ) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); const operand_ty = self.typeOf(ty_op.operand); @@ -3093,8 +3055,7 @@ fn airErrUnionErr( fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); const err_union_ptr_ty = self.typeOf(ty_op.operand); @@ -3133,8 +3094,7 @@ fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Bu } fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const struct_ty = ty_pl.ty.toType(); @@ -3150,12 +3110,7 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er }; const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset); - - const field_ptr_ty = try pt.ptrType(.{ - .child = field_ty.toIntern(), - .flags = .{ .alignment = field_align }, - }); - return self.load(field_ptr, field_ptr_ty); + return self.load(field_ptr, field_ty, field_align.toLlvm(), .normal); } /// As an optimization, we want to avoid unnecessary copies of @@ -3183,8 +3138,7 @@ fn isNextRet( fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const inst = body_tail[0]; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const payload_ty = self.typeOf(ty_op.operand); @@ -3205,8 +3159,12 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator. }; const payload_ptr = optional_ptr; // payload always at offset 0 - const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); - try self.store(payload_ptr, payload_ptr_ty, operand, .none); + try self.store( + payload_ptr, + .none, + operand, + payload_ty, + ); // Non-null bit immediately after payload (no padding because the bit has alignment 1). const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu)); _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default); @@ -3215,8 +3173,7 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator. fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const inst = body_tail[0]; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const err_un_ty = self.typeOfIndex(inst); @@ -3230,23 +3187,26 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) All const result_ptr = if (self.isNextRet(body_tail)) self.ret_ptr else brk: { - const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm(); + const alignment = err_un_ty.abiAlignment(o.zcu).toLlvm(); const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment); break :brk result_ptr; }; const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu)); - const error_alignment = Type.anyerror.abiAlignment(pt.zcu).toLlvm(); + const error_alignment = Type.anyerror.abiAlignment(o.zcu).toLlvm(); _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment); const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu)); - const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); - try self.store(payload_ptr, payload_ptr_ty, operand, .none); + try self.store( + payload_ptr, + .none, + operand, + payload_ty, + ); return result_ptr; } fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const inst = body_tail[0]; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const err_un_ty = self.typeOfIndex(inst); @@ -3268,10 +3228,8 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocat const error_alignment = Type.anyerror.abiAlignment(zcu).toLlvm(); _ = try self.wip.store(.normal, operand, err_ptr, error_alignment); const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu)); - const payload_ptr_ty = try pt.singleMutPtrType(payload_ty); // TODO store undef to payload_ptr _ = payload_ptr; - _ = payload_ptr_ty; return result_ptr; } @@ -3279,7 +3237,7 @@ fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build const o = self.object; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const index = pl_op.payload; - const llvm_usize = try self.lowerType(Type.usize); + const llvm_usize = try self.lowerType(.usize); return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{ try o.builder.intValue(.i32, index), }, ""); @@ -3289,7 +3247,7 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build const o = self.object; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const index = pl_op.payload; - const llvm_isize = try self.lowerType(Type.isize); + const llvm_isize = try self.lowerType(.isize); return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{ try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand), }, ""); @@ -3297,15 +3255,13 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = fg.object; - const pt = fg.pt; const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; - const llvm_ptr_const = try o.lowerNavRefValue(pt, ty_nav.nav); + const llvm_ptr_const = try o.lowerNavRefValue(ty_nav.nav); return llvm_ptr_const.toValue(); } fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3324,8 +3280,7 @@ fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { } fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3353,7 +3308,7 @@ fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3371,8 +3326,7 @@ fn airSafeArithmetic( unsigned_intrinsic: Builder.Intrinsic, ) Allocator.Error!Builder.Value { const o = fg.object; - const pt = fg.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try fg.resolveInst(bin_op.lhs); @@ -3419,8 +3373,7 @@ fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu } fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3438,7 +3391,7 @@ fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3458,8 +3411,7 @@ fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu } fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3477,7 +3429,7 @@ fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3497,8 +3449,7 @@ fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu } fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3525,7 +3476,7 @@ fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) } fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3541,8 +3492,7 @@ fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3591,7 +3541,7 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) } fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3608,7 +3558,7 @@ fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) } fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3625,8 +3575,7 @@ fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3678,7 +3627,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo } fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; const ptr_or_slice = try self.resolveInst(bin_op.lhs); @@ -3694,8 +3643,7 @@ fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; const ptr_or_slice = try self.resolveInst(bin_op.lhs); @@ -3718,8 +3666,7 @@ fn airOverflow( signed_intrinsic: Builder.Intrinsic, unsigned_intrinsic: Builder.Intrinsic, ) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; @@ -3801,8 +3748,7 @@ fn buildFloatCmp( params: [2]Builder.Value, ) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const target = zcu.getTarget(); const scalar_ty = ty.scalarType(zcu); const scalar_llvm_ty = try self.lowerType(scalar_ty); @@ -3908,8 +3854,7 @@ fn buildFloatOp( params: [params_len]Builder.Value, ) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const target = zcu.getTarget(); const scalar_ty = ty.scalarType(zcu); const llvm_ty = try self.lowerType(ty); @@ -4049,8 +3994,7 @@ fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; @@ -4120,8 +4064,7 @@ fn airXor(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { } fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -4143,8 +4086,7 @@ fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val } fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -4162,8 +4104,7 @@ fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -4244,8 +4185,7 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -4269,8 +4209,7 @@ fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error! fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); const operand_ty = self.typeOf(ty_op.operand); @@ -4292,8 +4231,7 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value { const o = fg.object; - const pt = fg.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const dest_ty = fg.typeOfIndex(inst); const dest_llvm_ty = try fg.lowerType(dest_ty); @@ -4379,7 +4317,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error! }, operand, dest_llvm_ty, ""); if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) { - const llvm_fn = try o.getIsNamedEnumValueFunction(pt, dest_ty); + const llvm_fn = try o.getIsNamedEnumValueFunction(dest_ty); const is_valid_enum_val = try fg.wip.call( .normal, .fastcc, @@ -4409,8 +4347,7 @@ fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); const operand_ty = self.typeOf(ty_op.operand); @@ -4444,8 +4381,7 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); const operand_ty = self.typeOf(ty_op.operand); @@ -4493,8 +4429,7 @@ fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const operand_is_ref = isByRef(operand_ty, zcu); const result_is_ref = isByRef(inst_ty, zcu); const llvm_dest_ty = try self.lowerType(inst_ty); @@ -4599,7 +4534,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const arg_val = self.args[self.arg_index]; self.arg_index += 1; @@ -4688,13 +4623,13 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ptr_ty = self.typeOfIndex(inst); const pointee_type = ptr_ty.childType(zcu); - if (!pointee_type.hasRuntimeBits(zcu)) - return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue(); - + if (!pointee_type.hasRuntimeBits(zcu)) { + const ptr_info = ptr_ty.ptrInfo(zcu); + return (try o.lowerPtrToVoid(ptr_info.flags.alignment, ptr_info.flags.address_space)).toValue(); + } const pointee_llvm_ty = try self.lowerType(pointee_type); const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); return self.buildAlloca(pointee_llvm_ty, alignment); @@ -4702,12 +4637,13 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ptr_ty = self.typeOfIndex(inst); const ret_ty = ptr_ty.childType(zcu); - if (!ret_ty.hasRuntimeBits(zcu)) - return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue(); + if (!ret_ty.hasRuntimeBits(zcu)) { + const ptr_info = ptr_ty.ptrInfo(zcu); + return (try o.lowerPtrToVoid(ptr_info.flags.alignment, ptr_info.flags.address_space)).toValue(); + } if (self.ret_ptr != .none) return self.ret_ptr; const ret_llvm_ty = try self.lowerType(ret_ty); const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); @@ -4721,20 +4657,19 @@ fn buildAlloca( llvm_ty: Builder.Type, alignment: Builder.Alignment, ) Allocator.Error!Builder.Value { - const target = self.pt.zcu.getTarget(); + const target = self.object.zcu.getTarget(); return buildAllocaInner(&self.wip, llvm_ty, alignment, target); } fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const dest_ptr = try self.resolveInst(bin_op.lhs); const ptr_ty = self.typeOf(bin_op.lhs); const operand_ty = ptr_ty.childType(zcu); - const val_is_undef = if (try self.air.value(bin_op.rhs, pt)) |val| val.isUndef(zcu) else false; + const val_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false; if (val_is_undef) { const owner_mod = self.ownerModule(); @@ -4762,7 +4697,7 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error! self.maybeMarkAllowZeroAccess(ptr_info); - const len = try o.builder.intValue(try self.lowerType(Type.usize), operand_ty.abiSize(zcu)); + const len = try o.builder.intValue(try self.lowerType(.usize), operand_ty.abiSize(zcu)); _ = try self.wip.callMemSet( dest_ptr, ptr_ty.ptrAlignment(zcu).toLlvm(), @@ -4780,24 +4715,75 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error! self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); const src_operand = try self.resolveInst(bin_op.rhs); - try self.store(dest_ptr, ptr_ty, src_operand, .none); + try self.storeFull(dest_ptr, ptr_ty, src_operand, .none); return .none; } fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = fg.pt; - const zcu = pt.zcu; + const o = fg.object; + const zcu = o.zcu; const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const ptr_ty = fg.typeOf(ty_op.operand); const ptr_info = ptr_ty.ptrInfo(zcu); const ptr = try fg.resolveInst(ty_op.operand); + const elem_ty = ptr_ty.childType(zcu); + const llvm_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm(); + fg.maybeMarkAllowZeroAccess(ptr_info); - return fg.load(ptr, ptr_ty); + + const access_kind: Builder.MemoryAccessKind = + if (ptr_info.flags.is_volatile) .@"volatile" else .normal; + + if (ptr_info.flags.vector_index != .none) { + const index_u32 = try o.builder.intValue(.i32, ptr_info.flags.vector_index); + const vec_elem_ty = try fg.lowerType(elem_ty); + const vec_ty = try o.builder.vectorType(.normal, ptr_info.packed_offset.host_size, vec_elem_ty); + + const loaded_vector = try fg.wip.load(access_kind, vec_ty, ptr, llvm_ptr_align, ""); + return fg.wip.extractElement(loaded_vector, index_u32, ""); + } + + if (ptr_info.packed_offset.host_size == 0) { + return fg.load(ptr, elem_ty, llvm_ptr_align, access_kind); + } + + const containing_int_ty = try o.builder.intType(@intCast(ptr_info.packed_offset.host_size * 8)); + const containing_int = + try fg.wip.load(access_kind, containing_int_ty, ptr, llvm_ptr_align, ""); + + const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); + const shift_amt = try o.builder.intValue(containing_int_ty, ptr_info.packed_offset.bit_offset); + const shifted_value = try fg.wip.bin(.lshr, containing_int, shift_amt, ""); + const elem_llvm_ty = try fg.lowerType(elem_ty); + + if (isByRef(elem_ty, zcu)) { + const result_align = elem_ty.abiAlignment(zcu).toLlvm(); + const result_ptr = try fg.buildAlloca(elem_llvm_ty, result_align); + + const same_size_int = try o.builder.intType(@intCast(elem_bits)); + const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, ""); + _ = try fg.wip.store(.normal, truncated_int, result_ptr, result_align); + return result_ptr; + } + + if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) { + const same_size_int = try o.builder.intType(@intCast(elem_bits)); + const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, ""); + return fg.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); + } + + if (elem_ty.isPtrAtRuntime(zcu)) { + const same_size_int = try o.builder.intType(@intCast(elem_bits)); + const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, ""); + return fg.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, ""); + } + + return fg.wip.cast(.trunc, shifted_value, elem_llvm_ty, ""); } fn airTrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void { _ = inst; - const target = self.pt.zcu.getTarget(); + const target = self.object.zcu.getTarget(); if ((target.cpu.arch == .mips or target.cpu.arch == .mipsel) and target.cpu.has(.mips, .notraps)) { @@ -4828,8 +4814,8 @@ fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { _ = inst; const o = self.object; - const llvm_usize = try self.lowerType(Type.usize); - if (!target_util.supportsReturnAddress(self.pt.zcu.getTarget(), self.ownerModule().optimize_mode)) { + const llvm_usize = try self.lowerType(.usize); + if (!target_util.supportsReturnAddress(self.object.zcu.getTarget(), self.ownerModule().optimize_mode)) { // https://github.com/ziglang/zig/issues/11946 return o.builder.intValue(llvm_usize, 0); } @@ -4840,7 +4826,7 @@ fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { _ = inst; const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, ""); - return self.wip.cast(.ptrtoint, result, try self.lowerType(Type.usize), ""); + return self.wip.cast(.ptrtoint, result, try self.lowerType(.usize), ""); } fn airCmpxchg( @@ -4849,8 +4835,7 @@ fn airCmpxchg( kind: Builder.Function.Instruction.CmpXchg.Kind, ) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data; const ptr = try self.resolveInst(extra.ptr); @@ -4915,8 +4900,7 @@ fn airCmpxchg( fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data; const ptr = try self.resolveInst(pl_op.operand); @@ -4971,7 +4955,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va access_kind, op, ptr, - try self.wip.cast(.ptrtoint, operand, try self.lowerType(Type.usize), ""), + try self.wip.cast(.ptrtoint, operand, try self.lowerType(.usize), ""), self.sync_scope, ordering, ptr_alignment, @@ -4980,8 +4964,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va } fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; const ptr = try self.resolveInst(atomic_load.ptr); const ptr_ty = self.typeOf(atomic_load.ptr); @@ -5029,8 +5012,7 @@ fn airAtomicStore( inst: Air.Inst.Index, ordering: Builder.AtomicOrdering, ) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const ptr_ty = self.typeOf(bin_op.lhs); const operand_ty = ptr_ty.childType(zcu); @@ -5051,14 +5033,13 @@ fn airAtomicStore( self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - try self.store(ptr, ptr_ty, element, ordering); + try self.storeFull(ptr, ptr_ty, element, ordering); return .none; } fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const dest_slice = try self.resolveInst(bin_op.lhs); const ptr_ty = self.typeOf(bin_op.lhs); @@ -5070,7 +5051,8 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu)); - if (try self.air.value(bin_op.rhs, pt)) |elem_val| { + if (bin_op.rhs.toInterned()) |elem_ip_index| { + const elem_val: Value = .fromInterned(elem_ip_index); if (elem_val.isUndef(zcu)) { // Even if safety is disabled, we still emit a memset to undefined since it conveys // extra information to LLVM. However, safety makes the difference between using @@ -5099,7 +5081,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error // repeating byte pattern, for example, `@as(u64, 0)` has a // repeating byte pattern of 0 bytes. In such case, the memset // intrinsic can be used. - if (try elem_val.hasRepeatedByteRepr(pt)) |byte_val| { + if (try elem_val.hasRepeatedByteRepr(zcu)) |byte_val| { const fill_byte = try o.builder.intValue(.i8, byte_val); const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); _ = try self.wip.callMemSet( @@ -5154,7 +5136,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error const body_block = try self.wip.block(1, "InlineMemsetBody"); const end_block = try self.wip.block(1, "InlineMemsetEnd"); - const llvm_usize_ty = try self.lowerType(Type.usize); + const llvm_usize_ty = try self.lowerType(.usize); const end_ptr = switch (ptr_ty.ptrSize(zcu)) { .slice => try self.ptraddScaled( dest_ptr, @@ -5194,8 +5176,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error } fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const dest_slice = try self.resolveInst(bin_op.lhs); const dest_ptr_ty = self.typeOf(bin_op.lhs); @@ -5223,8 +5204,7 @@ fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const dest_slice = try self.resolveInst(bin_op.lhs); const dest_ptr_ty = self.typeOf(bin_op.lhs); @@ -5248,8 +5228,7 @@ fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu } fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const un_ptr_ty = self.typeOf(bin_op.lhs); const un_ty = un_ptr_ty.childType(zcu); @@ -5280,8 +5259,7 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder. fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const un_ty = self.typeOf(ty_op.operand); const layout = un_ty.unionGetLayout(zcu); @@ -5354,8 +5332,7 @@ fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand_ty = self.typeOf(ty_op.operand); var bits = operand_ty.intInfo(zcu).bits; @@ -5389,8 +5366,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); @@ -5426,7 +5402,7 @@ fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Bui const operand = try self.resolveInst(un_op); const enum_ty = self.typeOf(un_op); - const llvm_fn = try o.getIsNamedEnumValueFunction(self.pt, enum_ty); + const llvm_fn = try o.getIsNamedEnumValueFunction(enum_ty); return self.wip.call( .normal, .fastcc, @@ -5440,12 +5416,11 @@ fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Bui fn airTagName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); const enum_ty = self.typeOf(un_op); - const llvm_fn = try o.getEnumTagNameFunction(pt, enum_ty); + const llvm_fn = try o.getEnumTagNameFunction(enum_ty); return self.wip.call( .normal, .fastcc, @@ -5459,8 +5434,7 @@ fn airTagName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); const slice_ty = self.typeOfIndex(inst); @@ -5493,8 +5467,7 @@ fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = fg.object; - const pt = fg.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const gpa = zcu.gpa; const unwrapped = fg.air.unwrapShuffleOne(zcu, inst); @@ -5534,7 +5507,7 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val .elem => llvm_poison_elem, .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: { any_defined_comptime_value = true; - break :elem try o.lowerValue(pt, val); + break :elem try o.lowerValue(val); } else llvm_poison_elem, }; } @@ -5600,8 +5573,7 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = fg.object; - const pt = fg.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const gpa = zcu.gpa; const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst); @@ -5699,7 +5671,7 @@ fn buildReducedCall( accum_init: Builder.Value, ) Allocator.Error!Builder.Value { const o = self.object; - const usize_ty = try self.lowerType(Type.usize); + const usize_ty = try self.lowerType(.usize); const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len); const llvm_result_ty = accum_init.typeOfWip(&self.wip); @@ -5753,8 +5725,7 @@ fn buildReducedCall( fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const target = zcu.getTarget(); const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce; @@ -5863,8 +5834,7 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) A fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const result_ty = self.typeOfIndex(inst); @@ -5960,19 +5930,18 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment); const array_info = result_ty.arrayInfo(zcu); - const elem_ptr_ty = try pt.singleConstPtrType(array_info.elem_type); const elem_size = array_info.elem_type.abiSize(zcu); for (elements, 0..) |elem, i| { const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i); const llvm_elem = try self.resolveInst(elem); - try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none); + try self.store(elem_ptr, .none, llvm_elem, array_info.elem_type); } if (array_info.sentinel) |sent_val| { const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len); const llvm_elem = try self.resolveValue(sent_val); - try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none); + try self.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type); } return alloca_inst; @@ -5983,8 +5952,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const ip = &zcu.intern_pool; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; @@ -6006,18 +5974,19 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va assert(field_ty.hasRuntimeBits(zcu)); { - const payload_ptr_ty = try pt.ptrType(.{ - .child = field_ty.toIntern(), - .flags = .{ .alignment = layout.payload_align }, - }); const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset()); - try self.store(payload_ptr, payload_ptr_ty, llvm_payload, .none); + try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty); } if (layout.tag_size != 0) { - const tag_ty: Type = .fromInterned(union_obj.enum_tag_type); - const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index); - const llvm_tag_val = try o.lowerValue(pt, tag_val.toIntern()); + const loaded_enum = ip.loadEnumType(union_obj.enum_tag_type); + const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) { + .none => try o.builder.intConst( + try o.lowerType(.fromInterned(union_obj.enum_tag_type)), + extra.field_index, // auto-numbered + ), + else => |tag_val_ip| try o.lowerValue(tag_val_ip), + }; const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset()); _ = try self.wip.store(.normal, llvm_tag_val.toValue(), tag_ptr, layout.tag_align.toLlvm()); } @@ -6043,7 +6012,7 @@ fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val // by the target. // To work around this, don't emit llvm.prefetch in this case. // See https://bugs.llvm.org/show_bug.cgi?id=21037 - const zcu = self.pt.zcu; + const zcu = self.object.zcu; const target = zcu.getTarget(); switch (prefetch.cache) { .instruction => switch (target.cpu.arch) { @@ -6097,7 +6066,7 @@ fn workIntrinsic( } fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const target = self.pt.zcu.getTarget(); + const target = self.object.zcu.getTarget(); const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const dimension = pl_op.payload; @@ -6110,8 +6079,7 @@ fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V } fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const pt = self.pt; - const target = pt.zcu.getTarget(); + const target = self.object.zcu.getTarget(); const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const dimension = pl_op.payload; @@ -6138,7 +6106,7 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde } fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const target = self.pt.zcu.getTarget(); + const target = self.object.zcu.getTarget(); const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const dimension = pl_op.payload; @@ -6158,7 +6126,7 @@ fn optCmpNull( opt_ptr: Builder.Value, access_kind: Builder.MemoryAccessKind, ) Allocator.Error!Builder.Value { - const zcu = self.pt.zcu; + const zcu = self.object.zcu; assert(isByRef(opt_ty, zcu)); comptime assert(optional_layout_version == 3); // Non-null bit is always after the payload, with no padding because it has alignment 1. @@ -6174,8 +6142,7 @@ fn optPayloadHandle( opt_ty: Type, can_elide_load: bool, ) Allocator.Error!Builder.Value { - const pt = fg.pt; - const zcu = pt.zcu; + const zcu = fg.object.zcu; assert(isByRef(opt_ty, zcu)); const payload_ty = opt_ty.optionalChild(zcu); @@ -6197,8 +6164,7 @@ fn fieldPtr( aggregate_ptr_ty: Type, field_index: u32, ) Allocator.Error!Builder.Value { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.object.zcu; const aggregate_ty = aggregate_ptr_ty.childType(zcu); if (aggregate_ty.containerLayout(zcu) == .@"packed") { // A pointer to a bitpack field is equivalent to a pointer to the whole bitpack; the @@ -6226,8 +6192,7 @@ fn loadTruncate( // => so load the byte aligned value and trunc the unwanted bits. const o = fg.object; - const pt = fg.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const payload_llvm_ty = try fg.lowerType(payload_ty); const abi_size = payload_ty.abiSize(zcu); @@ -6256,12 +6221,11 @@ fn loadByRef( access_kind: Builder.MemoryAccessKind, ) Allocator.Error!Builder.Value { const o = fg.object; - const pt = fg.pt; const pointee_llvm_ty = try fg.lowerType(pointee_type); const result_align = InternPool.Alignment.fromLlvm(ptr_alignment) - .max(pointee_type.abiAlignment(pt.zcu)).toLlvm(); + .max(pointee_type.abiAlignment(o.zcu)).toLlvm(); const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align); - const size_bytes = pointee_type.abiSize(pt.zcu); + const size_bytes = pointee_type.abiSize(o.zcu); _ = try fg.wip.callMemCpy( result_ptr, result_align, @@ -6274,76 +6238,24 @@ fn loadByRef( return result_ptr; } -/// This function always performs a copy. For isByRef=true types, it creates a new -/// alloca and copies the value into it, then returns the alloca instruction. -/// For isByRef=false types, it creates a load instruction and returns it. -fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) Allocator.Error!Builder.Value { - const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; - const info = ptr_ty.ptrInfo(zcu); - const elem_ty = Type.fromInterned(info.child); - if (!elem_ty.hasRuntimeBits(zcu)) return .none; - - const ptr_alignment = (if (info.flags.alignment != .none) - @as(InternPool.Alignment, info.flags.alignment) - else - elem_ty.abiAlignment(zcu)).toLlvm(); - - const access_kind: Builder.MemoryAccessKind = - if (info.flags.is_volatile) .@"volatile" else .normal; - - if (info.flags.vector_index != .none) { - const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); - const vec_elem_ty = try self.lowerType(elem_ty); - const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); - - const loaded_vector = try self.wip.load(access_kind, vec_ty, ptr, ptr_alignment, ""); - return self.wip.extractElement(loaded_vector, index_u32, ""); - } - - if (info.packed_offset.host_size == 0) { - if (isByRef(elem_ty, zcu)) { - return self.loadByRef(ptr, elem_ty, ptr_alignment, access_kind); - } - return self.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment); - } - - const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8)); - const containing_int = - try self.wip.load(access_kind, containing_int_ty, ptr, ptr_alignment, ""); - - const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); - const shift_amt = try o.builder.intValue(containing_int_ty, info.packed_offset.bit_offset); - const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, ""); - const elem_llvm_ty = try self.lowerType(elem_ty); - +/// If `isByRef` returns `true` for `elem_ty`, this still performs a copy by memcpy'ing the value +/// into a new alloca. +fn load( + fg: *FuncGen, + ptr: Builder.Value, + elem_ty: Type, + ptr_alignment: Builder.Alignment, + access_kind: Builder.MemoryAccessKind, +) Allocator.Error!Builder.Value { + const zcu = fg.object.zcu; if (isByRef(elem_ty, zcu)) { - const result_align = elem_ty.abiAlignment(zcu).toLlvm(); - const result_ptr = try self.buildAlloca(elem_llvm_ty, result_align); - - const same_size_int = try o.builder.intType(@intCast(elem_bits)); - const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); - _ = try self.wip.store(.normal, truncated_int, result_ptr, result_align); - return result_ptr; + return fg.loadByRef(ptr, elem_ty, ptr_alignment, access_kind); + } else { + return fg.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment); } - - if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) { - const same_size_int = try o.builder.intType(@intCast(elem_bits)); - const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); - return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, ""); - } - - if (elem_ty.isPtrAtRuntime(zcu)) { - const same_size_int = try o.builder.intType(@intCast(elem_bits)); - const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, ""); - return self.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, ""); - } - - return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, ""); } -fn store( +fn storeFull( self: *FuncGen, ptr: Builder.Value, ptr_ty: Type, @@ -6351,8 +6263,7 @@ fn store( ordering: Builder.AtomicOrdering, ) Allocator.Error!void { const o = self.object; - const pt = self.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const info = ptr_ty.ptrInfo(zcu); const elem_ty = Type.fromInterned(info.child); if (!elem_ty.hasRuntimeBits(zcu)) { @@ -6433,12 +6344,51 @@ fn store( ptr_alignment, elem, elem_ty.abiAlignment(zcu).toLlvm(), - try o.builder.intValue(try self.lowerType(Type.usize), elem_ty.abiSize(zcu)), + try o.builder.intValue(try self.lowerType(.usize), elem_ty.abiSize(zcu)), access_kind, self.disable_intrinsics, ); } +/// Non-atomic, non-volatile, non-packed store. +fn store( + fg: *FuncGen, + ptr: Builder.Value, + ptr_align: InternPool.Alignment, + elem: Builder.Value, + elem_ty: Type, +) Allocator.Error!void { + const o = fg.object; + const zcu = o.zcu; + const llvm_ptr_align = switch (ptr_align) { + .none => elem_ty.abiAlignment(zcu).toLlvm(), + else => ptr_align.toLlvm(), + }; + if (isByRef(elem_ty, zcu)) { + _ = try fg.wip.callMemCpy( + ptr, + llvm_ptr_align, + elem, + elem_ty.abiAlignment(zcu).toLlvm(), + try o.builder.intValue( + try fg.lowerType(.usize), + elem_ty.abiSize(zcu), + ), + .normal, + fg.disable_intrinsics, + ); + } else { + _ = try fg.wip.storeAtomic( + .normal, + elem, + ptr, + fg.sync_scope, + .none, + llvm_ptr_align, + ); + } +} + fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void { const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545; const o = fg.object; @@ -6460,8 +6410,7 @@ fn valgrindClientRequest( a5: Builder.Value, ) Allocator.Error!Builder.Value { const o = fg.object; - const pt = fg.pt; - const zcu = pt.zcu; + const zcu = o.zcu; const target = zcu.getTarget(); if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value; @@ -6588,18 +6537,17 @@ fn valgrindClientRequest( } fn typeOf(fg: *FuncGen, inst: Air.Inst.Ref) Type { - const zcu = fg.pt.zcu; + const zcu = fg.object.zcu; return fg.air.typeOf(inst, &zcu.intern_pool); } fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type { - const zcu = fg.pt.zcu; + const zcu = fg.object.zcu; return fg.air.typeOfIndex(inst, &zcu.intern_pool); } const ParamTypeIterator = struct { object: *Object, - pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType, zig_index: u32, llvm_index: u32, @@ -6622,7 +6570,7 @@ const ParamTypeIterator = struct { pub fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering { if (it.zig_index >= it.fn_info.param_types.len) return null; - const ip = &it.pt.zcu.intern_pool; + const ip = &it.object.zcu.intern_pool; const ty = it.fn_info.param_types.get(ip)[it.zig_index]; it.byval_attr = false; return nextInner(it, Type.fromInterned(ty)); @@ -6630,8 +6578,7 @@ const ParamTypeIterator = struct { /// `airCall` uses this instead of `next` so that it can take into account variadic functions. fn nextCall(it: *ParamTypeIterator, fg: *FuncGen, args: []const Air.Inst.Ref) Allocator.Error!?Lowering { - assert(std.meta.eql(it.pt, fg.pt)); - const ip = &it.pt.zcu.intern_pool; + const ip = &it.object.zcu.intern_pool; if (it.zig_index >= it.fn_info.param_types.len) { if (it.zig_index >= args.len) { return null; @@ -6644,8 +6591,7 @@ const ParamTypeIterator = struct { } fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering { - const pt = it.pt; - const zcu = pt.zcu; + const zcu = it.object.zcu; const target = zcu.getTarget(); if (!ty.hasRuntimeBits(zcu)) { @@ -6744,7 +6690,7 @@ const ParamTypeIterator = struct { for (0..ty.structFieldCount(zcu)) |field_index| { const field_ty = ty.fieldType(field_index, zcu); if (!field_ty.hasRuntimeBits(zcu)) continue; - it.types_buffer[it.types_len] = try it.object.lowerType(pt, field_ty); + it.types_buffer[it.types_len] = try it.object.lowerType(field_ty); it.types_len += 1; } it.llvm_index += it.types_len - 1; @@ -6760,7 +6706,7 @@ const ParamTypeIterator = struct { return .byval; } else { var types_buffer: [8]Builder.Type = undefined; - types_buffer[0] = try it.object.lowerType(pt, scalar_ty); + types_buffer[0] = try it.object.lowerType(scalar_ty); it.types_buffer = types_buffer; it.types_len = 1; it.llvm_index += 1; @@ -6785,7 +6731,7 @@ const ParamTypeIterator = struct { } fn nextWin64(it: *ParamTypeIterator, ty: Type) ?Lowering { - const zcu = it.pt.zcu; + const zcu = it.object.zcu; switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) { .integer => { if (isScalar(zcu, ty)) { @@ -6818,7 +6764,7 @@ const ParamTypeIterator = struct { } fn nextSystemV(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering { - const zcu = it.pt.zcu; + const zcu = it.object.zcu; const ip = &zcu.intern_pool; ty.assertHasLayout(zcu); const classes = x86_64_abi.classifySystemV(ty, zcu, zcu.getTarget(), .arg); @@ -6909,10 +6855,9 @@ const ParamTypeIterator = struct { return .multiple_llvm_types; } }; -pub fn iterateParamTypes(object: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) ParamTypeIterator { +pub fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) ParamTypeIterator { return .{ .object = object, - .pt = pt, .fn_info = fn_info, .zig_index = 0, .llvm_index = 0, @@ -6976,8 +6921,8 @@ fn firstParamSRetSystemV(ty: Type, zcu: *Zcu, target: *const std.Target) bool { /// In order to support the C calling convention, some return types need to be lowered /// completely differently in the function prototype to honor the C ABI, and then /// be effectively bitcasted to the actual return type. -pub fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { - const zcu = pt.zcu; +pub fn lowerFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { + const zcu = o.zcu; const return_type = Type.fromInterned(fn_info.return_type); if (!return_type.hasRuntimeBits(zcu)) { assert(!return_type.isError(zcu)); @@ -6986,27 +6931,27 @@ pub fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncT const target = zcu.getTarget(); switch (fn_info.cc) { .@"inline" => unreachable, - .auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(pt, return_type), + .auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(return_type), - .x86_64_sysv => return lowerSystemVFnRetTy(o, pt, fn_info), - .x86_64_win => return lowerWin64FnRetTy(o, pt, fn_info), - .x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(pt, return_type) else .void, - .x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(pt, return_type), + .x86_64_sysv => return lowerSystemVFnRetTy(o, fn_info), + .x86_64_win => return lowerWin64FnRetTy(o, fn_info), + .x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(return_type) else .void, + .x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(return_type), .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) { .memory => return .void, - .float_array => return o.lowerType(pt, return_type), - .byval => return o.lowerType(pt, return_type), + .float_array => return o.lowerType(return_type), + .byval => return o.lowerType(return_type), .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))), .double_integer => return o.builder.arrayType(2, .i64), }, .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) { .memory, .i64_array => return .void, .i32_array => |len| return if (len == 1) .i32 else .void, - .byval => return o.lowerType(pt, return_type), + .byval => return o.lowerType(return_type), }, .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) { .memory, .i32_array => return .void, - .byval => return o.lowerType(pt, return_type), + .byval => return o.lowerType(return_type), }, .riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) { .memory => return .void, @@ -7019,53 +6964,53 @@ pub fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncT }; return o.builder.structType(.normal, &.{ integer, integer }); }, - .byval => return o.lowerType(pt, return_type), + .byval => return o.lowerType(return_type), .fields => { var types_len: usize = 0; var types: [8]Builder.Type = undefined; for (0..return_type.structFieldCount(zcu)) |field_index| { const field_ty = return_type.fieldType(field_index, zcu); if (!field_ty.hasRuntimeBits(zcu)) continue; - types[types_len] = try o.lowerType(pt, field_ty); + types[types_len] = try o.lowerType(field_ty); types_len += 1; } return o.builder.structType(.normal, types[0..types_len]); }, }, .wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) { - .direct => |scalar_ty| return o.lowerType(pt, scalar_ty), + .direct => |scalar_ty| return o.lowerType(scalar_ty), .indirect => return .void, }, // TODO investigate other callconvs - else => return o.lowerType(pt, return_type), + else => return o.lowerType(return_type), } } -fn lowerWin64FnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { - const zcu = pt.zcu; +fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { + const zcu = o.zcu; const return_type = Type.fromInterned(fn_info.return_type); switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) { .integer => { if (isScalar(zcu, return_type)) { - return o.lowerType(pt, return_type); + return o.lowerType(return_type); } else { return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8)); } }, .win_i128 => return o.builder.vectorType(.normal, 2, .i64), .memory => return .void, - .sse => return o.lowerType(pt, return_type), + .sse => return o.lowerType(return_type), else => unreachable, } } -fn lowerSystemVFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { - const zcu = pt.zcu; +fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { + const zcu = o.zcu; const ip = &zcu.intern_pool; const return_type = Type.fromInterned(fn_info.return_type); return_type.assertHasLayout(zcu); if (isScalar(zcu, return_type)) { - return o.lowerType(pt, return_type); + return o.lowerType(return_type); } const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret); var types_index: u32 = 0; @@ -7297,7 +7242,7 @@ pub fn isByRef(ty: Type, zcu: *const Zcu) bool { /// RMW exchange of floating-point values is bitcasted to same-sized integer /// types to work around a LLVM deficiency when targeting ARM/AArch64. fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type { - const zcu = fg.pt.zcu; + const zcu = fg.object.zcu; switch (ty.zigTypeTag(zcu)) { .int, .@"enum", .@"struct", .@"union" => {}, .float => { diff --git a/src/codegen/riscv64/CodeGen.zig b/src/codegen/riscv64/CodeGen.zig index 3ad6faf805259154ae35f347e1dee712bea196cf..d50f6a71254877ca7cad6b0c82af9b79381d7a7b 100644 --- a/src/codegen/riscv64/CodeGen.zig +++ b/src/codegen/riscv64/CodeGen.zig @@ -4956,8 +4956,8 @@ fn genCall( // on linking. switch (info) { .air => |callee| { - if (try func.air.value(callee, pt)) |func_value| { - const func_key = zcu.intern_pool.indexToKey(func_value.ip_index); + if (callee.toInterned()) |func_ip_index| { + const func_key = zcu.intern_pool.indexToKey(func_ip_index); switch (switch (func_key) { else => func_key, .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) { diff --git a/src/codegen/sparc64/CodeGen.zig b/src/codegen/sparc64/CodeGen.zig index 3b38d6319ac325cb462b805727197121be348bfd..d4744e0fd0921de6aac44c859c9f9363ef37d677 100644 --- a/src/codegen/sparc64/CodeGen.zig +++ b/src/codegen/sparc64/CodeGen.zig @@ -1310,7 +1310,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier // Due to incremental compilation, how function calls are generated depends // on linking. - if (try self.air.value(call.callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) { + if (call.callee.toInterned()) |func_ip_index| switch (ip.indexToKey(func_ip_index)) { .func => { return self.fail("TODO implement calling functions", .{}); }, @@ -4487,7 +4487,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue { return self.getResolvedInstValue(inst); } - return self.genTypedValue((try self.air.value(ref, pt)).?); + return self.genTypedValue(.fromInterned(ref.toInterned().?)); } fn ret(self: *Self, mcv: MCValue) !void { diff --git a/src/codegen/spirv/CodeGen.zig b/src/codegen/spirv/CodeGen.zig index 865b7f9dcff79a5181b85c30e207f998a664b4d0..4ce1681129568085499a611caea0d5f0c1b8eeef 100644 --- a/src/codegen/spirv/CodeGen.zig +++ b/src/codegen/spirv/CodeGen.zig @@ -387,13 +387,12 @@ fn importExtendedSet(cg: *CodeGen) !Id { /// Fetch the result-id for a previously generated instruction or constant. fn resolve(cg: *CodeGen, inst: Air.Inst.Ref) !Id { - const pt = cg.pt; const zcu = cg.module.zcu; const ip = &zcu.intern_pool; - if (try cg.air.value(inst, pt)) |val| { + if (inst.toInterned()) |val_ip_index| { const ty = cg.typeOf(inst); if (ty.zigTypeTag(zcu) == .@"fn") { - const fn_nav = switch (zcu.intern_pool.indexToKey(val.ip_index)) { + const fn_nav = switch (zcu.intern_pool.indexToKey(val_ip_index)) { .@"extern" => |@"extern"| @"extern".owner_nav, .func => |func| func.owner_nav, else => unreachable, @@ -403,7 +402,7 @@ fn resolve(cg: *CodeGen, inst: Air.Inst.Ref) !Id { return cg.module.declPtr(spv_decl_index).result_id; } - return try cg.constant(ty, val, .direct); + return try cg.constant(ty, .fromInterned(val_ip_index), .direct); } const index = inst.toIndex().?; return cg.inst_results.get(index).?; // Assertion means instruction does not dominate usage. @@ -5657,7 +5656,6 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) !?Id { fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index) !void { const gpa = cg.module.gpa; - const pt = cg.pt; const zcu = cg.module.zcu; const target = cg.module.zcu.getTarget(); const switch_br = cg.air.unwrapSwitch(inst); @@ -5732,7 +5730,7 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index) !void { const label = case_labels.at(case.idx); for (case.items) |item| { - const value = (try cg.air.value(item, pt)) orelse unreachable; + const value: Value = .fromInterned(item.toInterned().?); const int_val: u64 = switch (cond_ty.zigTypeTag(zcu)) { .bool, .int => if (cond_ty.isSignedInt(zcu)) @bitCast(value.toSignedInt(zcu)) else value.toUnsignedInt(zcu), .@"enum" => blk: { @@ -5875,9 +5873,9 @@ fn airAssembly(cg: *CodeGen, inst: Air.Inst.Index) !?Id { if (std.mem.eql(u8, in.constraint, "c")) { // constant - const val = (try cg.air.value(in.operand, cg.pt)) orelse { + const val: Value = .fromInterned(in.operand.toInterned() orelse { return cg.fail("assembly inputs with 'c' constraint have to be compile-time known", .{}); - }; + }); // TODO: This entire function should be handled a bit better... const ip = &zcu.intern_pool; @@ -5911,8 +5909,7 @@ fn airAssembly(cg: *CodeGen, inst: Air.Inst.Index) !?Id { if (input_ty.zigTypeTag(zcu) == .type) { // This assembly input is a type instead of a value. // That's fine for now, just make sure to resolve it as such. - const val = (try cg.air.value(in.operand, cg.pt)).?; - const ty_id = try cg.resolveType(val.toType(), .direct); + const ty_id = try cg.resolveType(in.operand.toType(), .direct); try ass.value_map.put(gpa, in.name, .{ .ty = ty_id }); } else { const ty_id = try cg.resolveType(input_ty, .direct); diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index c97892a10c22cacc4b7ecc3422694793fc1b7221..83049921ef7f6d4dca1cef1208266fb3f578118d 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -303,7 +303,7 @@ fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue { const pt = cg.pt; const zcu = pt.zcu; - const val = (try cg.air.value(ref, pt)).?; + const val: Value = .fromInterned(ref.toInterned().?); const ty = cg.typeOf(ref); if (!ty.hasRuntimeBits(zcu) and !ty.isInt(zcu) and !ty.isError(zcu)) { gop.value_ptr.* = .none; @@ -2006,7 +2006,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie const first_param_sret = firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target); const callee: ?InternPool.Nav.Index = blk: { - const func_val = (try cg.air.value(call.callee, pt)) orelse break :blk null; + const func_val: Value = .fromInterned(call.callee.toInterned() orelse break :blk null); switch (ip.indexToKey(func_val.toIntern())) { inline .func, .@"extern" => |x| break :blk x.owner_nav, @@ -4464,7 +4464,7 @@ fn lowerConstant(cg: *CodeGen, val: Value) InnerError!WValue { .vector_type => { assert(determineSimdStoreStrategy(ty, zcu, cg.target) == .direct); var buf: [16]u8 = undefined; - val.writeToMemory(pt, &buf) catch unreachable; + val.writeToMemory(zcu, &buf) catch unreachable; return cg.storeSimdImmd(buf); }, .struct_type => unreachable, // packed structs use `bitpack` diff --git a/src/codegen/x86_64/CodeGen.zig b/src/codegen/x86_64/CodeGen.zig index 9ba98c3e833cc11ff996a61a7d624a62ebdda3fa..69e6a94f08684905a47417c66a6518d544cdbbe7 100644 --- a/src/codegen/x86_64/CodeGen.zig +++ b/src/codegen/x86_64/CodeGen.zig @@ -176185,8 +176185,8 @@ fn genCall(self: *CodeGen, info: union(enum) { // Due to incremental compilation, how function calls are generated depends // on linking. switch (info) { - .air => |callee| if (try self.air.value(callee, pt)) |func_value| { - const func_key = ip.indexToKey(func_value.ip_index); + .air => |callee| if (callee.toInterned()) |func_ip_index| { + const func_key = ip.indexToKey(func_ip_index); switch (switch (func_key) { else => func_key, .ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) { -- 2.54.0 From fb224178aaeeb5d8e37392f5c168afca771147ca Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Mon, 23 Mar 2026 14:54:40 +0000 Subject: [PATCH 05/15] Air: change misleading instruction tag name --- src/Air.zig | 7 +++---- src/Air/Legalize.zig | 2 +- src/Air/Liveness.zig | 2 +- src/Air/Liveness/Verify.zig | 2 +- src/Air/print.zig | 2 +- src/Sema.zig | 4 ++-- src/codegen/aarch64/Select.zig | 4 ++-- src/codegen/c.zig | 4 ++-- src/codegen/llvm/FuncGen.zig | 5 ++--- src/codegen/riscv64/CodeGen.zig | 6 +++--- src/codegen/sparc64/CodeGen.zig | 6 +++--- src/codegen/wasm/CodeGen.zig | 4 ++-- src/codegen/x86_64/CodeGen.zig | 2 +- 13 files changed, 24 insertions(+), 26 deletions(-) diff --git a/src/Air.zig b/src/Air.zig index a3de9c84c867cc0e4ca158087b0e60c7efc6e1c2..275857214ebb3638bedc3e785620cd18939f031d 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -874,7 +874,6 @@ pub const Inst = struct { /// size as the error integer type, is less than *or equal to* the total /// number of errors in the Zcu. The "or equal to" is a consequence of /// value 0 being reserved for the "non-error" status in error unions. - /// MLUGG TODO: rename this instruction to `cmp_lte_errors_len` /// /// This instruction exists (as opposed to just using `cmp_lte` against /// a constant) because the number of errors in the Zcu is not known @@ -884,7 +883,7 @@ pub const Inst = struct { /// Result type is always `bool`. /// /// Uses the `un_op` field. - cmp_lt_errors_len, + cmp_lte_errors_len, /// Returns pointer to current error return trace. err_return_trace, @@ -1623,7 +1622,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool) .cmp_gte_optimized, .cmp_gt_optimized, .cmp_neq_optimized, - .cmp_lt_errors_len, + .cmp_lte_errors_len, .is_null, .is_non_null, .is_null_ptr, @@ -2059,7 +2058,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool { .mul_add, .field_parent_ptr, .wasm_memory_size, - .cmp_lt_errors_len, + .cmp_lte_errors_len, .err_return_trace, .addrspace_cast, .save_err_return_trace_index, diff --git a/src/Air/Legalize.zig b/src/Air/Legalize.zig index 3dcab2043581cefc7c79871e40d7717936747c88..01f4c53482b97eb9b2b620b5c967f748ff3d7b9b 100644 --- a/src/Air/Legalize.zig +++ b/src/Air/Legalize.zig @@ -884,7 +884,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void { .field_parent_ptr, .wasm_memory_size, .wasm_memory_grow, - .cmp_lt_errors_len, + .cmp_lte_errors_len, .err_return_trace, .set_err_return_trace, .addrspace_cast, diff --git a/src/Air/Liveness.zig b/src/Air/Liveness.zig index 384a056988b40b6a59fc4f9a582c7869bf994d3e..402873227a738953ee6ce9546115e39b75c0331a 100644 --- a/src/Air/Liveness.zig +++ b/src/Air/Liveness.zig @@ -565,7 +565,7 @@ fn analyzeInst( .trunc_float, .neg, .neg_optimized, - .cmp_lt_errors_len, + .cmp_lte_errors_len, .set_err_return_trace, .c_va_end, => { diff --git a/src/Air/Liveness/Verify.zig b/src/Air/Liveness/Verify.zig index 7f820e65981b1d4247b363c061269c8d893ce239..fd8f735741bf905ce80d5dbbf69e54425b1e2e63 100644 --- a/src/Air/Liveness/Verify.zig +++ b/src/Air/Liveness/Verify.zig @@ -152,7 +152,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void { .trunc_float, .neg, .neg_optimized, - .cmp_lt_errors_len, + .cmp_lte_errors_len, .set_err_return_trace, .c_va_end, => { diff --git a/src/Air/print.zig b/src/Air/print.zig index f6c0f5a03b8f1c957b0129978358888977690de7..b1114b39c3dfc154705b9e5c8324b200514a6a91 100644 --- a/src/Air/print.zig +++ b/src/Air/print.zig @@ -211,7 +211,7 @@ const Writer = struct { .trunc_float, .neg, .neg_optimized, - .cmp_lt_errors_len, + .cmp_lte_errors_len, .set_err_return_trace, .c_va_end, => try w.writeUnOp(s, inst), diff --git a/src/Sema.zig b/src/Sema.zig index 0629e827a97a3acd68dc811714020b600616e341..ab7e8b4c155fe804c09a17e6020c7955c2f86d5f 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -7832,10 +7832,10 @@ fn zirErrorFromInt(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstD } try sema.requireRuntimeBlock(block, src, operand_src); if (block.wantSafety()) { - const is_lt_len = try block.addUnOp(.cmp_lt_errors_len, operand); + const is_lte_len = try block.addUnOp(.cmp_lte_errors_len, operand); const zero_val = Air.internedToRef((try pt.intValue(err_int_ty, 0)).toIntern()); const is_non_zero = try block.addBinOp(.cmp_neq, operand, zero_val); - const ok = try block.addBinOp(.bool_and, is_lt_len, is_non_zero); + const ok = try block.addBinOp(.bool_and, is_lte_len, is_non_zero); try sema.addSafetyCheck(block, src, ok, .invalid_error_code); } return block.addInst(.{ diff --git a/src/codegen/aarch64/Select.zig b/src/codegen/aarch64/Select.zig index c8a9f41cb3d6c2906e7011373ede8fca90a031a3..2bfb3c4a552c2d453a7612ea07ada56eae490308 100644 --- a/src/codegen/aarch64/Select.zig +++ b/src/codegen/aarch64/Select.zig @@ -522,7 +522,7 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void { .is_named_enum_value, .tag_name, .error_name, - .cmp_lt_errors_len, + .cmp_lte_errors_len, => { const un_op = air_data[@intFromEnum(air_inst_index)].un_op; @@ -7175,7 +7175,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory, if (air.next()) |next_air_tag| continue :air_tag next_air_tag; }, .wasm_memory_size, .wasm_memory_grow => unreachable, - .cmp_lt_errors_len => { + .cmp_lte_errors_len => { if (isel.live_values.fetchRemove(air.inst_index)) |is_vi| unused: { defer is_vi.value.deref(isel); const is_ra = try is_vi.value.defReg(isel) orelse break :unused; diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 1b72c8759907bb004e0c31850bc1e53ccde57dea..99f15a902659004788477a58fa12004ef956b828 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -2722,7 +2722,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void { const extra = f.air.extraData(Air.VectorCmp, ty_pl.payload).data; break :blk try airCmpOp(f, inst, extra, extra.compareOperator()); }, - .cmp_lt_errors_len => try airCmpLtErrorsLen(f, inst), + .cmp_lte_errors_len => try airCmpLteErrorsLen(f, inst), // bool_and and bool_or are non-short-circuit operations .bool_and, .bit_and => try airBinOp(f, inst, "&", "and", .none), @@ -3728,7 +3728,7 @@ fn airEquality( return local; } -fn airCmpLtErrorsLen(f: *Function, inst: Air.Inst.Index) !CValue { +fn airCmpLteErrorsLen(f: *Function, inst: Air.Inst.Index) !CValue { const un_op = f.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try f.resolveInst(un_op); diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index ec8e0b1d0577dadd8a299bf0cb274be8723a76e9..2b4bef2bc76013936396c674bf1414a508b5824e 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -302,7 +302,7 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air .cmp_vector => try self.airCmpVector(inst, .normal), .cmp_vector_optimized => try self.airCmpVector(inst, .fast), - .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), + .cmp_lte_errors_len => try self.airCmpLteErrorsLen(inst), .is_non_null => try self.airIsNonNull(inst, false, .ne), .is_non_null_ptr => try self.airIsNonNull(inst, true , .ne), @@ -1105,7 +1105,7 @@ fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind return self.cmp(fast, cmp_op, vec_ty, lhs, rhs); } -fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { +fn airCmpLteErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = self.object; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); @@ -1117,7 +1117,6 @@ fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build Type.errorAbiAlignment(o.zcu).toLlvm(), "", ); - // Despite the name, this instruction is actually lte. MLUGG TODO RENAME return self.wip.icmp(.ule, operand, errors_len_val, ""); } diff --git a/src/codegen/riscv64/CodeGen.zig b/src/codegen/riscv64/CodeGen.zig index d50f6a71254877ca7cad6b0c82af9b79381d7a7b..5ab8bebed44c45d8eeb7aa071fe2d803c1f43839 100644 --- a/src/codegen/riscv64/CodeGen.zig +++ b/src/codegen/riscv64/CodeGen.zig @@ -1477,7 +1477,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void { => try func.airCmp(inst, tag), .cmp_vector => try func.airCmpVector(inst), - .cmp_lt_errors_len => try func.airCmpLtErrorsLen(inst), + .cmp_lte_errors_len => try func.airCmpLteErrorsLen(inst), .slice => try func.airSlice(inst), .array_to_slice => try func.airArrayToSlice(inst), @@ -5186,11 +5186,11 @@ fn airCmpVector(func: *Func, inst: Air.Inst.Index) !void { return func.fail("TODO implement airCmpVector for {}", .{func.target.cpu.arch}); } -fn airCmpLtErrorsLen(func: *Func, inst: Air.Inst.Index) !void { +fn airCmpLteErrorsLen(func: *Func, inst: Air.Inst.Index) !void { const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try func.resolveInst(un_op); _ = operand; - const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airCmpLtErrorsLen for {}", .{func.target.cpu.arch}); + const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airCmpLteErrorsLen for {}", .{func.target.cpu.arch}); return func.finishAir(inst, result, .{ un_op, .none, .none }); } diff --git a/src/codegen/sparc64/CodeGen.zig b/src/codegen/sparc64/CodeGen.zig index d4744e0fd0921de6aac44c859c9f9363ef37d677..c34e91bd96b0390cb063cf9bcd22261a364f2df1 100644 --- a/src/codegen/sparc64/CodeGen.zig +++ b/src/codegen/sparc64/CodeGen.zig @@ -545,7 +545,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .cmp_gt => try self.airCmp(inst, .gt), .cmp_neq => try self.airCmp(inst, .neq), .cmp_vector => @panic("TODO try self.airCmpVector(inst)"), - .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), + .cmp_lte_errors_len => try self.airCmpLteErrorsLen(inst), .alloc => try self.airAlloc(inst), .ret_ptr => try self.airRetPtr(inst), @@ -1425,11 +1425,11 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void { +fn airCmpLteErrorsLen(self: *Self, inst: Air.Inst.Index) !void { const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); _ = operand; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLtErrorsLen for {}", .{self.target.cpu.arch}); + const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLteErrorsLen for {}", .{self.target.cpu.arch}); return self.finishAir(inst, result, .{ un_op, .none, .none }); } diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index 83049921ef7f6d4dca1cef1208266fb3f578118d..c968cbf322978d098059f0ae5863166597ff26bd 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -1718,7 +1718,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .cmp_neq => cg.airCmp(inst, .neq), .cmp_vector => cg.airCmpVector(inst), - .cmp_lt_errors_len => cg.airCmpLtErrorsLen(inst), + .cmp_lte_errors_len => cg.airCmpLteErrorsLen(inst), .array_elem_val => cg.airArrayElemVal(inst), .array_to_slice => cg.airArrayToSlice(inst), @@ -4841,7 +4841,7 @@ fn airCmpVector(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { return cg.fail("TODO implement airCmpVector for wasm", .{}); } -fn airCmpLtErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { +fn airCmpLteErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try cg.resolveInst(un_op); diff --git a/src/codegen/x86_64/CodeGen.zig b/src/codegen/x86_64/CodeGen.zig index 69e6a94f08684905a47417c66a6518d544cdbbe7..48101500be89eab157b802c326970a99ca43ef7a 100644 --- a/src/codegen/x86_64/CodeGen.zig +++ b/src/codegen/x86_64/CodeGen.zig @@ -172921,7 +172921,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { try ops[0].finish(inst, &.{field_parent_ptr.field_ptr}, &ops, cg); }, .wasm_memory_size, .wasm_memory_grow => unreachable, - .cmp_lt_errors_len => |air_tag| { + .cmp_lte_errors_len => |air_tag| { const un_op = air_datas[@intFromEnum(inst)].un_op; var ops = try cg.tempsFromOperands(inst, .{un_op}); var res: [1]Temp = undefined; -- 2.54.0 From 9c0c65c313776fdc394d8e481ee4eb548a5333ed Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Mon, 23 Mar 2026 16:10:16 +0000 Subject: [PATCH 06/15] llvm: minor refactors, and incremental `@tagName` updates --- src/codegen/llvm.zig | 86 ++++++----- src/codegen/llvm/FuncGen.zig | 291 ++++++++++++++++++----------------- 2 files changed, 201 insertions(+), 176 deletions(-) diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index b34f630fc21e87c4e49d51dd9c9c27c991bbf0bf..1ce296f452e6c1cfb3d52cc4263f7d1692ccb3b9 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -562,7 +562,7 @@ pub const Object = struct { /// Same deal as `decl_map` but for anonymous declarations, which are always global constants. uav_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Global.Index), /// Maps enum types to their corresponding LLVM functions for implementing the `tag_name` instruction. - enum_tag_name_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Global.Index), + enum_tag_name_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index), /// Serves the same purpose as `enum_tag_name_map` but for the `is_named_enum_value` instruction. named_enum_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index), /// Maps Zig types to LLVM types. The table memory is backed by the GPA of @@ -1138,7 +1138,7 @@ pub const Object = struct { func_index: InternPool.Index, air: *const Air, liveness: *const ?Air.Liveness, - ) !void { + ) Zcu.CodegenFailError!void { const zcu = o.zcu; const comp = zcu.comp; const ip = &zcu.intern_pool; @@ -1514,10 +1514,7 @@ pub const Object = struct { defer fg.deinit(); deinit_wip = false; - fg.genBody(air.getMainBody(), .poi) catch |err| switch (err) { - error.CodegenFail => return, // MLUGG TODO - else => |e| return e, - }; + try fg.genBody(air.getMainBody(), .poi); // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole // function as considering null pointers valid so that LLVM's optimizers don't remove these @@ -1894,6 +1891,9 @@ pub const Object = struct { if (o.named_enum_map.get(ty)) |function_index| { try o.updateIsNamedEnumValueFunction(.fromInterned(ty), function_index); } + if (o.enum_tag_name_map.get(ty)) |function_index| { + try o.updateEnumTagNameFunction(.fromInterned(ty), function_index); + } } /// Should only be called by the `link.ConstPool` implementation. @@ -4271,25 +4271,39 @@ pub const Object = struct { return o.errors_len_variable; } - /// MLUGG TODO: this also needs incremental updates dumbass - pub fn getEnumTagNameFunction(o: *Object, enum_ty: Type) !Builder.Function.Index { + pub fn getEnumTagNameFunction(o: *Object, enum_ty: Type) Allocator.Error!Builder.Function.Index { const zcu = o.zcu; const ip = &zcu.intern_pool; - const enum_type = ip.loadEnumType(enum_ty.toIntern()); const gop = try o.enum_tag_name_map.getOrPut(o.gpa, enum_ty.toIntern()); - if (gop.found_existing) return gop.value_ptr.ptrConst(&o.builder).kind.function; + if (gop.found_existing) return gop.value_ptr.*; errdefer assert(o.enum_tag_name_map.remove(enum_ty.toIntern())); - - const usize_ty = try o.lowerType(.usize); - const ret_ty = try o.lowerType(.slice_const_u8_sentinel_0); - const llvm_int_ty = try o.lowerType(.fromInterned(enum_type.int_tag_type)); - const target = &zcu.root_mod.resolved_target.result; const function_index = try o.builder.addFunction( - try o.builder.fnType(ret_ty, &.{llvm_int_ty}, .normal), - try o.builder.strtabStringFmt("__zig_tag_name_{f}", .{enum_type.name.fmt(ip)}), - toLlvmAddressSpace(.generic, target), + // Dummy function type; `updateEnumTagNameFunction` will replace it with the correct type. + // TODO: change the builder API so we don't need to do this. + try o.builder.fnType(.void, &.{}, .normal), + try o.builder.strtabStringFmt("__zig_tag_name_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}), + toLlvmAddressSpace(.generic, zcu.getTarget()), ); + gop.value_ptr.* = function_index; + try o.updateEnumTagNameFunction(enum_ty, function_index); + return function_index; + } + fn updateEnumTagNameFunction( + o: *Object, + enum_ty: Type, + function_index: Builder.Function.Index, + ) Allocator.Error!void { + const zcu = o.zcu; + const ip = &zcu.intern_pool; + const loaded_enum = ip.loadEnumType(enum_ty.toIntern()); + + const llvm_usize_ty = try o.lowerType(.usize); + const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0); + const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type)); + + function_index.ptrConst(&o.builder).global.ptr(&o.builder).type = + try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal); var attributes: Builder.FunctionAttributes.Wip = .{}; defer attributes.deinit(&o.builder); @@ -4298,7 +4312,6 @@ pub const Object = struct { function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); function_index.setCallConv(.fastcc, &o.builder); function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); - gop.value_ptr.* = function_index.ptrConst(&o.builder).global; var wip = try Builder.WipFunction.init(&o.builder, .{ .function = function_index, @@ -4312,13 +4325,13 @@ pub const Object = struct { var wip_switch = try wip.@"switch"( tag_int_value, bad_value_block, - @intCast(enum_type.field_names.len), + @intCast(loaded_enum.field_names.len), .none, ); defer wip_switch.finish(&wip); - for (0..enum_type.field_names.len) |field_index| { - const name = try o.builder.stringNull(enum_type.field_names.get(ip)[field_index].toSlice(ip)); + for (0..loaded_enum.field_names.len) |field_index| { + const name = try o.builder.stringNull(loaded_enum.field_names.get(ip)[field_index].toSlice(ip)); const name_init = try o.builder.stringConst(name); const name_variable_index = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); @@ -4328,13 +4341,13 @@ pub const Object = struct { name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); - const name_val = try o.builder.structValue(ret_ty, &.{ + const name_val = try o.builder.structValue(llvm_ret_ty, &.{ name_variable_index.toConst(&o.builder), - try o.builder.intConst(usize_ty, name.slice(&o.builder).?.len - 1), + try o.builder.intConst(llvm_usize_ty, name.slice(&o.builder).?.len - 1), }); const return_block = try wip.block(1, "Name"); - const llvm_tag_val = switch (enum_type.field_values.getOrNone(ip, field_index)) { + const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) { .none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered else => |tag_val_ip| try o.lowerValue(tag_val_ip), }; @@ -4348,7 +4361,6 @@ pub const Object = struct { _ = try wip.@"unreachable"(); try wip.finish(); - return function_index; } pub fn lazyAbiAlignment(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Alignment.Lazy { @@ -4356,7 +4368,7 @@ pub const Object = struct { return o.lazy_abi_aligns.items[@intFromEnum(index)]; } - pub fn getIsNamedEnumValueFunction(o: *Object, enum_ty: Type) !Builder.Function.Index { + pub fn getIsNamedEnumValueFunction(o: *Object, enum_ty: Type) Allocator.Error!Builder.Function.Index { const zcu = o.zcu; const ip = &zcu.intern_pool; @@ -4380,22 +4392,22 @@ pub const Object = struct { function_index: Builder.Function.Index, ) Allocator.Error!void { const zcu = o.zcu; - const builder = &o.builder; - const loaded_enum = zcu.intern_pool.loadEnumType(enum_ty.toIntern()); + const ip = &zcu.intern_pool; + const loaded_enum = ip.loadEnumType(enum_ty.toIntern()); const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type)); - function_index.ptrConst(builder).global.ptr(builder).type = - try builder.fnType(.i1, &.{llvm_int_ty}, .normal); + function_index.ptrConst(&o.builder).global.ptr(&o.builder).type = + try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal); var attributes: Builder.FunctionAttributes.Wip = .{}; - defer attributes.deinit(builder); + defer attributes.deinit(&o.builder); try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer); - function_index.setLinkage(if (o.builder.strip) .private else .internal, builder); - function_index.setCallConv(.fastcc, builder); - function_index.setAttributes(try attributes.finish(builder), builder); + function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); + function_index.setCallConv(.fastcc, &o.builder); + function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); - var wip: Builder.WipFunction = try .init(builder, .{ + var wip: Builder.WipFunction = try .init(&o.builder, .{ .function = function_index, .strip = true, }); @@ -4409,7 +4421,7 @@ pub const Object = struct { defer wip_switch.finish(&wip); if (loaded_enum.field_values.len > 0) { - for (loaded_enum.field_values.get(&zcu.intern_pool)) |tag_val_ip| { + for (loaded_enum.field_values.get(ip)) |tag_val_ip| { const llvm_tag_val = try o.lowerValue(tag_val_ip); try wip_switch.addCase(llvm_tag_val, named_block, &wip); } diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index 2b4bef2bc76013936396c674bf1414a508b5824e..6f90cfd3c50b4c882eb39b0a0e37e0ecd28797a9 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -175,11 +175,6 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant { } } -/// MLUGG TODO okay yeah prolly delete this again -fn lowerType(fg: *const FuncGen, ty: Type) Allocator.Error!Builder.Type { - return fg.object.lowerType(ty); -} - pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void { const o = self.object; const zcu = self.object.zcu; @@ -608,7 +603,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif } const ret_ptr = if (!sret) null else blk: { - const llvm_ret_ty = try self.lowerType(return_type); + const llvm_ret_ty = try o.lowerType(return_type); try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder); const alignment = return_type.abiAlignment(zcu).toLlvm(); @@ -630,7 +625,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif const arg = args[it.zig_index - 1]; const param_ty = self.typeOf(arg); const llvm_arg = try self.resolveInst(arg); - const llvm_param_ty = try self.lowerType(param_ty); + const llvm_param_ty = try o.lowerType(param_ty); if (isByRef(param_ty, zcu)) { const alignment = param_ty.abiAlignment(zcu).toLlvm(); const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, ""); @@ -659,7 +654,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif const llvm_arg = try self.resolveInst(arg); const alignment = param_ty.abiAlignment(zcu).toLlvm(); - const param_llvm_ty = try self.lowerType(param_ty); + const param_llvm_ty = try o.lowerType(param_ty); const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment); if (isByRef(param_ty, zcu)) { const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, ""); @@ -729,7 +724,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif llvm_arg = ptr; } - const float_ty = try self.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?); + const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?); const array_ty = try o.builder.arrayType(count, float_ty); const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, ""); @@ -770,7 +765,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif .byref => { const param_index = it.zig_index - 1; const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); - const param_llvm_ty = try self.lowerType(param_ty); + const param_llvm_ty = try o.lowerType(param_ty); const alignment = param_ty.abiAlignment(zcu).toLlvm(); try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty); }, @@ -822,7 +817,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif }, toLlvmCallConvTag(fn_info.cc, target).?, try attributes.finish(&o.builder), - try self.lowerType(zig_fn_ty), + try o.lowerType(zig_fn_ty), llvm_fn, llvm_args.items, "", @@ -836,7 +831,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif return .none; } - const llvm_ret_ty = try self.lowerType(return_type); + const llvm_ret_ty = try o.lowerType(return_type); if (ret_ptr) |rp| { if (isByRef(return_type, zcu)) { return rp; @@ -908,7 +903,7 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo const operand = try self.resolveInst(un_op); const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false; if (val_is_undef and safety) { - const len = try o.builder.intValue(try self.lowerType(.usize), ret_ty.abiSize(zcu)); + const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu)); _ = try self.wip.callMemSet( self.ret_ptr, ret_ty.abiAlignment(zcu).toLlvm(), @@ -964,7 +959,7 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo if (val_is_undef and safety) { const llvm_ret_ty = operand.typeOfWip(&self.wip); const rp = try self.buildAlloca(llvm_ret_ty, alignment); - const len = try o.builder.intValue(try self.lowerType(.usize), ret_ty.abiSize(zcu)); + const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu)); _ = try self.wip.callMemSet( rp, alignment, @@ -1034,17 +1029,18 @@ fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const list = try self.resolveInst(ty_op.operand); const arg_ty = ty_op.ty.toType(); - const llvm_arg_ty = try self.lowerType(arg_ty); + const llvm_arg_ty = try self.object.lowerType(arg_ty); return self.wip.vaArg(list, llvm_arg_ty, ""); } fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const src_list = try self.resolveInst(ty_op.operand); const va_list_ty = ty_op.ty.toType(); - const llvm_va_list_ty = try self.lowerType(va_list_ty); + const llvm_va_list_ty = try o.lowerType(va_list_ty); const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm(); const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); @@ -1065,9 +1061,10 @@ fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const va_list_ty = self.typeOfIndex(inst); - const llvm_va_list_ty = try self.lowerType(va_list_ty); + const llvm_va_list_ty = try o.lowerType(va_list_ty); const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm(); const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); @@ -1227,7 +1224,8 @@ fn lowerBlock( maybe_inline_func: ?InternPool.Index, body: []const Air.Inst.Index, ) TodoError!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const inst_ty = self.typeOfIndex(inst); if (inst_ty.isNoReturn(zcu)) { @@ -1253,7 +1251,7 @@ fn lowerBlock( // Create a phi node only if the block returns a value. if (have_block_result) { - const raw_llvm_ty = try self.lowerType(inst_ty); + const raw_llvm_ty = try o.lowerType(inst_ty); const llvm_ty: Builder.Type = ty: { // If the zig tag type is a function, this represents an actual function body; not // a pointer to it. LLVM IR allows the call instruction to use function bodies instead @@ -1370,7 +1368,7 @@ fn lowerSwitchDispatch( const table_index = try self.wip.conv( .unsigned, try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""), - try self.lowerType(.usize), + try o.lowerType(.usize), "", ); const target_ptr_ptr = try self.ptraddScaled( @@ -1395,7 +1393,7 @@ fn lowerSwitchDispatch( // The switch prongs will correspond to our scalar cases. Ranges will // be handled by conditional branches in the `else` prong. - const llvm_usize = try self.lowerType(.usize); + const llvm_usize = try o.lowerType(.usize); const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder)) try self.wip.cast(.ptrtoint, cond, llvm_usize, "") else @@ -1642,7 +1640,7 @@ fn lowerTry( } else if (isByRef(payload_ty, zcu)) { return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal); } else { - return fg.wip.load(.normal, try fg.lowerType(payload_ty), payload_ptr, payload_align.toLlvm(), ""); + return fg.wip.load(.normal, try o.lowerType(payload_ty), payload_ptr, payload_align.toLlvm(), ""); } } @@ -1912,9 +1910,9 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand_ty = self.typeOf(ty_op.operand); const array_ty = operand_ty.childType(zcu); - const llvm_usize = try self.lowerType(.usize); + const llvm_usize = try o.lowerType(.usize); const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu)); - const slice_llvm_ty = try self.lowerType(self.typeOfIndex(inst)); + const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst)); const operand = try self.resolveInst(ty_op.operand); return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, ""); } @@ -1931,7 +1929,7 @@ fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value const dest_ty = self.typeOfIndex(inst); const dest_scalar_ty = dest_ty.scalarType(zcu); - const dest_llvm_ty = try self.lowerType(dest_ty); + const dest_llvm_ty = try o.lowerType(dest_ty); const target = zcu.getTarget(); if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv( @@ -1999,7 +1997,7 @@ fn airIntFromFloat( const dest_ty = self.typeOfIndex(inst); const dest_scalar_ty = dest_ty.scalarType(zcu); - const dest_llvm_ty = try self.lowerType(dest_ty); + const dest_llvm_ty = try o.lowerType(dest_ty); if (intrinsicsAllowed(operand_scalar_ty, target)) { // TODO set fast math flag @@ -2033,7 +2031,7 @@ fn airIntFromFloat( compiler_rt_dest_abbrev, }); - const operand_llvm_ty = try self.lowerType(operand_ty); + const operand_llvm_ty = try o.lowerType(operand_ty); const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty); var result = try self.wip.call( .normal, @@ -2058,7 +2056,7 @@ fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!B fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value { const o = fg.object; const zcu = o.zcu; - const llvm_usize = try fg.lowerType(.usize); + const llvm_usize = try o.lowerType(.usize); switch (ty.ptrSize(zcu)) { .slice => { const len = try fg.wip.extractValue(ptr, &.{1}, ""); @@ -2240,7 +2238,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build }, .float => { // bitcast int->float - return self.wip.cast(.bitcast, field_int_val, try self.lowerType(field_ty), ""); + return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty), ""); }, } } @@ -2277,8 +2275,8 @@ fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu); if (field_offset == 0) return field_ptr; - const res_ty = try self.lowerType(ty_pl.ty.toType()); - const llvm_usize = try self.lowerType(.usize); + const res_ty = try o.lowerType(ty_pl.ty.toType()); + const llvm_usize = try o.lowerType(.usize); const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, ""); const base_ptr_int = try self.wip.bin( @@ -2495,7 +2493,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value { const output_inst = try self.resolveInst(output.operand); const output_ty = self.typeOf(output.operand); assert(output_ty.zigTypeTag(zcu) == .pointer); - const elem_llvm_ty = try self.lowerType(output_ty.childType(zcu)); + const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu)); switch (constraint[0]) { '=' => {}, @@ -2533,7 +2531,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value { llvm_ret_indirect[output.index] = false; const ret_ty = self.typeOfIndex(inst); - llvm_ret_types[llvm_ret_i] = try self.lowerType(ret_ty); + llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty); llvm_ret_i += 1; } @@ -2572,7 +2570,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value { llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip); } else { const alignment = arg_ty.abiAlignment(zcu).toLlvm(); - const arg_llvm_ty = try self.lowerType(arg_ty); + const arg_llvm_ty = try o.lowerType(arg_ty); const load_inst = try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, ""); llvm_param_values[llvm_param_i] = load_inst; @@ -2613,7 +2611,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value { llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: { if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu)); - break :blk try self.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu)); + break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu)); } else .none; llvm_param_i += 1; @@ -2627,7 +2625,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value { if (constraint[0] != '+') continue; const rw_ty = self.typeOf(output.operand); - const llvm_elem_ty = try self.lowerType(rw_ty.childType(zcu)); + const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu)); if (llvm_ret_indirect[output.index]) { llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index]; llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip); @@ -2827,7 +2825,7 @@ fn airIsNonNull( const operand = try self.resolveInst(un_op); const operand_ty = self.typeOf(un_op); const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty; - const optional_llvm_ty = try self.lowerType(optional_ty); + const optional_llvm_ty = try o.lowerType(optional_ty); const payload_ty = optional_ty.optionalChild(zcu); const access_kind: Builder.MemoryAccessKind = @@ -2896,7 +2894,7 @@ fn airIsErr( if (!payload_ty.hasRuntimeBits(zcu)) { const loaded = if (operand_is_ptr) - try self.wip.load(access_kind, try self.lowerType(err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") + try self.wip.load(access_kind, try o.lowerType(err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "") else operand; return self.wip.icmp(cond, loaded, zero, ""); @@ -2981,7 +2979,8 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil } fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); const operand_ty = self.typeOf(ty_op.operand); @@ -3001,7 +3000,7 @@ fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool if (isByRef(payload_ty, zcu)) { return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal); } else { - const payload_llvm_ty = try self.lowerType(payload_ty); + const payload_llvm_ty = try o.lowerType(payload_ty); return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, ""); } } @@ -3148,7 +3147,7 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator. const optional_ty = self.typeOfIndex(inst); if (optional_ty.optionalReprIsPayload(zcu)) return operand; assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload` - const llvm_optional_ty = try self.lowerType(optional_ty); + const llvm_optional_ty = try o.lowerType(optional_ty); const optional_ptr = if (self.isNextRet(body_tail)) self.ret_ptr else brk: { @@ -3181,7 +3180,7 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) All assert(payload_ty.hasRuntimeBits(zcu)); assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref const ok_err_code = try o.builder.intValue(try o.errorIntType(), 0); - const err_un_llvm_ty = try self.lowerType(err_un_ty); + const err_un_llvm_ty = try o.lowerType(err_un_ty); const result_ptr = if (self.isNextRet(body_tail)) self.ret_ptr @@ -3205,7 +3204,8 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) All } fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const inst = body_tail[0]; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const err_un_ty = self.typeOfIndex(inst); @@ -3213,7 +3213,7 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocat const operand = try self.resolveInst(ty_op.operand); if (!payload_ty.hasRuntimeBits(zcu)) return operand; assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref - const err_un_llvm_ty = try self.lowerType(err_un_ty); + const err_un_llvm_ty = try o.lowerType(err_un_ty); const result_ptr = if (self.isNextRet(body_tail)) self.ret_ptr @@ -3236,7 +3236,7 @@ fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build const o = self.object; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const index = pl_op.payload; - const llvm_usize = try self.lowerType(.usize); + const llvm_usize = try o.lowerType(.usize); return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{ try o.builder.intValue(.i32, index), }, ""); @@ -3246,7 +3246,7 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build const o = self.object; const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op; const index = pl_op.payload; - const llvm_isize = try self.lowerType(.isize); + const llvm_isize = try o.lowerType(.isize); return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{ try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand), }, ""); @@ -3260,7 +3260,8 @@ fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder. } fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3272,14 +3273,15 @@ fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { .normal, .none, if (scalar_ty.isSignedInt(zcu)) .smin else .umin, - &.{try self.lowerType(inst_ty)}, + &.{try o.lowerType(inst_ty)}, &.{ lhs, rhs }, "", ); } fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3291,7 +3293,7 @@ fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { .normal, .none, if (scalar_ty.isSignedInt(zcu)) .smax else .umax, - &.{try self.lowerType(inst_ty)}, + &.{try o.lowerType(inst_ty)}, &.{ lhs, rhs }, "", ); @@ -3303,7 +3305,7 @@ fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const ptr = try self.resolveInst(bin_op.lhs); const len = try self.resolveInst(bin_op.rhs); const inst_ty = self.typeOfIndex(inst); - return self.wip.buildAggregate(try self.lowerType(inst_ty), &.{ ptr, len }, ""); + return self.wip.buildAggregate(try self.object.lowerType(inst_ty), &.{ ptr, len }, ""); } fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value { @@ -3334,7 +3336,7 @@ fn airSafeArithmetic( const scalar_ty = inst_ty.scalarType(zcu); const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; - const llvm_inst_ty = try fg.lowerType(inst_ty); + const llvm_inst_ty = try o.lowerType(inst_ty); const results = try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, ""); @@ -3372,7 +3374,8 @@ fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu } fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3383,7 +3386,7 @@ fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value .normal, .none, if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat", - &.{try self.lowerType(inst_ty)}, + &.{try o.lowerType(inst_ty)}, &.{ lhs, rhs }, "", ); @@ -3410,7 +3413,8 @@ fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu } fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3421,7 +3425,7 @@ fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value .normal, .none, if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat", - &.{try self.lowerType(inst_ty)}, + &.{try o.lowerType(inst_ty)}, &.{ lhs, rhs }, "", ); @@ -3448,7 +3452,8 @@ fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu } fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -3459,7 +3464,7 @@ fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value .normal, .none, if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat", - &.{try self.lowerType(inst_ty)}, + &.{try o.lowerType(inst_ty)}, &.{ lhs, rhs, .@"0" }, "", ); @@ -3503,7 +3508,7 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result}); } if (scalar_ty.isSignedInt(zcu)) { - const inst_llvm_ty = try self.lowerType(inst_ty); + const inst_llvm_ty = try o.lowerType(inst_ty); const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; var stack align(@max( @@ -3579,7 +3584,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); const inst_ty = self.typeOfIndex(inst); - const inst_llvm_ty = try self.lowerType(inst_ty); + const inst_llvm_ty = try o.lowerType(inst_ty); const scalar_ty = inst_ty.scalarType(zcu); if (scalar_ty.isRuntimeFloat()) { @@ -3646,7 +3651,7 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; const ptr_or_slice = try self.resolveInst(bin_op.lhs); - const llvm_usize_ty = try self.lowerType(.usize); + const llvm_usize_ty = try o.lowerType(.usize); const ptr_ty = self.typeOf(bin_op.lhs); const elem_ty = ptr_ty.indexableElem(zcu); const ptr = switch (ptr_ty.ptrSize(zcu)) { @@ -3665,7 +3670,8 @@ fn airOverflow( signed_intrinsic: Builder.Intrinsic, unsigned_intrinsic: Builder.Intrinsic, ) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; @@ -3678,8 +3684,8 @@ fn airOverflow( assert(isByRef(inst_ty, zcu)); // auto structs are by-ref const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic; - const llvm_inst_ty = try self.lowerType(inst_ty); - const llvm_lhs_ty = try self.lowerType(lhs_ty); + const llvm_inst_ty = try o.lowerType(inst_ty); + const llvm_lhs_ty = try o.lowerType(lhs_ty); const results = try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, ""); @@ -3750,7 +3756,7 @@ fn buildFloatCmp( const zcu = o.zcu; const target = zcu.getTarget(); const scalar_ty = ty.scalarType(zcu); - const scalar_llvm_ty = try self.lowerType(scalar_ty); + const scalar_llvm_ty = try o.lowerType(scalar_ty); if (intrinsicsAllowed(scalar_ty, target)) { const cond: Builder.FloatCondition = switch (pred) { @@ -3856,7 +3862,7 @@ fn buildFloatOp( const zcu = o.zcu; const target = zcu.getTarget(); const scalar_ty = ty.scalarType(zcu); - const llvm_ty = try self.lowerType(ty); + const llvm_ty = try o.lowerType(ty); if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) { // Some operations are dedicated LLVM instructions, not available as intrinsics @@ -3993,7 +3999,8 @@ fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; @@ -4011,9 +4018,9 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil const dest_ty = self.typeOfIndex(inst); assert(isByRef(dest_ty, zcu)); // auto structs are by-ref - const llvm_dest_ty = try self.lowerType(dest_ty); + const llvm_dest_ty = try o.lowerType(dest_ty); - const casted_rhs = try self.wip.conv(.unsigned, rhs, try self.lowerType(lhs_ty), ""); + const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), ""); const result = try self.wip.bin(.shl, lhs, casted_rhs, ""); const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu)) @@ -4063,7 +4070,8 @@ fn airXor(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { } fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -4077,7 +4085,7 @@ fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val } const lhs_scalar_ty = lhs_ty.scalarType(zcu); - const casted_rhs = try self.wip.conv(.unsigned, rhs, try self.lowerType(lhs_ty), ""); + const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), ""); return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu)) .@"shl nsw" else @@ -4085,7 +4093,8 @@ fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val } fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -4097,7 +4106,7 @@ fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { // features which we do not use. Therefore this branch is currently impossible. unreachable; } - const casted_rhs = try self.wip.conv(.unsigned, rhs, try self.lowerType(lhs_ty), ""); + const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), ""); return self.wip.bin(.shl, lhs, casted_rhs, ""); } @@ -4111,7 +4120,7 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const lhs_ty = self.typeOf(bin_op.lhs); const lhs_info = lhs_ty.intInfo(zcu); - const llvm_lhs_ty = try self.lowerType(lhs_ty); + const llvm_lhs_ty = try o.lowerType(lhs_ty); const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder); const rhs_ty = self.typeOf(bin_op.rhs); @@ -4122,7 +4131,7 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } const rhs_info = rhs_ty.intInfo(zcu); assert(rhs_info.signedness == .unsigned); - const llvm_rhs_ty = try self.lowerType(rhs_ty); + const llvm_rhs_ty = try o.lowerType(rhs_ty); const llvm_rhs_scalar_ty = llvm_rhs_ty.scalarType(&o.builder); const result = try self.wip.callIntrinsic( @@ -4184,7 +4193,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value } fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; const lhs = try self.resolveInst(bin_op.lhs); @@ -4198,7 +4208,7 @@ fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error! } const lhs_scalar_ty = lhs_ty.scalarType(zcu); - const casted_rhs = try self.wip.conv(.unsigned, rhs, try self.lowerType(lhs_ty), ""); + const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), ""); const is_signed_int = lhs_scalar_ty.isSignedInt(zcu); return self.wip.bin(if (is_exact) @@ -4219,7 +4229,7 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { .normal, .none, .abs, - &.{try self.lowerType(operand_ty)}, + &.{try o.lowerType(operand_ty)}, &.{ operand, try o.builder.intValue(.i1, 0) }, "", ), @@ -4233,7 +4243,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error! const zcu = o.zcu; const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const dest_ty = fg.typeOfIndex(inst); - const dest_llvm_ty = try fg.lowerType(dest_ty); + const dest_llvm_ty = try o.lowerType(dest_ty); const operand = try fg.resolveInst(ty_op.operand); const operand_ty = fg.typeOf(ty_op.operand); const operand_info = operand_ty.intInfo(zcu); @@ -4261,8 +4271,8 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error! if (!have_min_check and !have_max_check) break :bounds_check; - const operand_llvm_ty = try fg.lowerType(operand_ty); - const operand_scalar_llvm_ty = try fg.lowerType(operand_scalar); + const operand_llvm_ty = try o.lowerType(operand_ty); + const operand_scalar_llvm_ty = try o.lowerType(operand_scalar); const is_vector = operand_ty.zigTypeTag(zcu) == .vector; assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector)); @@ -4340,7 +4350,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error! fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const operand = try self.resolveInst(ty_op.operand); - const dest_llvm_ty = try self.lowerType(self.typeOfIndex(inst)); + const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst)); return self.wip.cast(.trunc, operand, dest_llvm_ty, ""); } @@ -4354,10 +4364,10 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu const target = zcu.getTarget(); if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) { - return self.wip.cast(.fptrunc, operand, try self.lowerType(dest_ty), ""); + return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty), ""); } else { - const operand_llvm_ty = try self.lowerType(operand_ty); - const dest_llvm_ty = try self.lowerType(dest_ty); + const operand_llvm_ty = try o.lowerType(operand_ty); + const dest_llvm_ty = try o.lowerType(dest_ty); const dest_bits = dest_ty.floatBits(target); const src_bits = operand_ty.floatBits(target); @@ -4388,10 +4398,10 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const target = zcu.getTarget(); if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) { - return self.wip.cast(.fpext, operand, try self.lowerType(dest_ty), ""); + return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty), ""); } else { - const operand_llvm_ty = try self.lowerType(operand_ty); - const dest_llvm_ty = try self.lowerType(dest_ty); + const operand_llvm_ty = try o.lowerType(operand_ty); + const dest_llvm_ty = try o.lowerType(dest_ty); const dest_bits = dest_ty.scalarType(zcu).floatBits(target); const src_bits = operand_ty.scalarType(zcu).floatBits(target); @@ -4431,7 +4441,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty const zcu = o.zcu; const operand_is_ref = isByRef(operand_ty, zcu); const result_is_ref = isByRef(inst_ty, zcu); - const llvm_dest_ty = try self.lowerType(inst_ty); + const llvm_dest_ty = try o.lowerType(inst_ty); if (operand_is_ref and result_is_ref) { // They are both pointers, so just return the same opaque pointer :) @@ -4477,7 +4487,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) { const elem_ty = operand_ty.childType(zcu); assert(operand_is_ref); // arrays are always by-ref provided they have runtime bits - const llvm_vector_ty = try self.lowerType(inst_ty); + const llvm_vector_ty = try o.lowerType(inst_ty); const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8; if (bitcast_ok) { @@ -4488,7 +4498,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty } else { // If the ABI size of the element type is not evenly divisible by size in bits; // a simple bitcast will not work, and we fall back to extractelement. - const elem_llvm_ty = try self.lowerType(elem_ty); + const elem_llvm_ty = try o.lowerType(elem_ty); const elem_size = elem_ty.abiSize(zcu); const vector_len = operand_ty.arrayLen(zcu); var vector = try o.builder.poisonValue(llvm_vector_ty); @@ -4629,7 +4639,7 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const ptr_info = ptr_ty.ptrInfo(zcu); return (try o.lowerPtrToVoid(ptr_info.flags.alignment, ptr_info.flags.address_space)).toValue(); } - const pointee_llvm_ty = try self.lowerType(pointee_type); + const pointee_llvm_ty = try o.lowerType(pointee_type); const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); return self.buildAlloca(pointee_llvm_ty, alignment); } @@ -4644,7 +4654,7 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value return (try o.lowerPtrToVoid(ptr_info.flags.alignment, ptr_info.flags.address_space)).toValue(); } if (self.ret_ptr != .none) return self.ret_ptr; - const ret_llvm_ty = try self.lowerType(ret_ty); + const ret_llvm_ty = try o.lowerType(ret_ty); const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); return self.buildAlloca(ret_llvm_ty, alignment); } @@ -4696,7 +4706,7 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error! self.maybeMarkAllowZeroAccess(ptr_info); - const len = try o.builder.intValue(try self.lowerType(.usize), operand_ty.abiSize(zcu)); + const len = try o.builder.intValue(try o.lowerType(.usize), operand_ty.abiSize(zcu)); _ = try self.wip.callMemSet( dest_ptr, ptr_ty.ptrAlignment(zcu).toLlvm(), @@ -4735,7 +4745,7 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { if (ptr_info.flags.vector_index != .none) { const index_u32 = try o.builder.intValue(.i32, ptr_info.flags.vector_index); - const vec_elem_ty = try fg.lowerType(elem_ty); + const vec_elem_ty = try o.lowerType(elem_ty); const vec_ty = try o.builder.vectorType(.normal, ptr_info.packed_offset.host_size, vec_elem_ty); const loaded_vector = try fg.wip.load(access_kind, vec_ty, ptr, llvm_ptr_align, ""); @@ -4753,7 +4763,7 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const elem_bits = ptr_ty.childType(zcu).bitSize(zcu); const shift_amt = try o.builder.intValue(containing_int_ty, ptr_info.packed_offset.bit_offset); const shifted_value = try fg.wip.bin(.lshr, containing_int, shift_amt, ""); - const elem_llvm_ty = try fg.lowerType(elem_ty); + const elem_llvm_ty = try o.lowerType(elem_ty); if (isByRef(elem_ty, zcu)) { const result_align = elem_ty.abiAlignment(zcu).toLlvm(); @@ -4813,7 +4823,7 @@ fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { _ = inst; const o = self.object; - const llvm_usize = try self.lowerType(.usize); + const llvm_usize = try o.lowerType(.usize); if (!target_util.supportsReturnAddress(self.object.zcu.getTarget(), self.ownerModule().optimize_mode)) { // https://github.com/ziglang/zig/issues/11946 return o.builder.intValue(llvm_usize, 0); @@ -4825,7 +4835,7 @@ fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { _ = inst; const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, ""); - return self.wip.cast(.ptrtoint, result, try self.lowerType(.usize), ""); + return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize), ""); } fn airCmpxchg( @@ -4842,7 +4852,7 @@ fn airCmpxchg( var expected_value = try self.resolveInst(extra.expected_value); var new_value = try self.resolveInst(extra.new_value); const operand_ty = ptr_ty.childType(zcu); - const llvm_operand_ty = try self.lowerType(operand_ty); + const llvm_operand_ty = try o.lowerType(operand_ty); const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false); if (llvm_abi_ty != .none) { // operand needs widening and truncating @@ -4885,7 +4895,7 @@ fn airCmpxchg( const non_null_bit = try self.wip.not(success_bit, ""); const payload_align = operand_ty.abiAlignment(zcu).toLlvm(); - const alloca_inst = try self.buildAlloca(try self.lowerType(optional_ty), payload_align); + const alloca_inst = try self.buildAlloca(try o.lowerType(optional_ty), payload_align); // Payload is always the first field at offset 0, so address is `alloca_inst` _ = try self.wip.store(.normal, payload, alloca_inst, payload_align); @@ -4911,7 +4921,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float); const ordering = toLlvmAtomicOrdering(extra.ordering()); const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg); - const llvm_operand_ty = try self.lowerType(operand_ty); + const llvm_operand_ty = try o.lowerType(operand_ty); const access_kind: Builder.MemoryAccessKind = if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; @@ -4954,7 +4964,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va access_kind, op, ptr, - try self.wip.cast(.ptrtoint, operand, try self.lowerType(.usize), ""), + try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize), ""), self.sync_scope, ordering, ptr_alignment, @@ -4963,7 +4973,8 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va } fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const zcu = self.object.zcu; + const o = self.object; + const zcu = o.zcu; const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load; const ptr = try self.resolveInst(atomic_load.ptr); const ptr_ty = self.typeOf(atomic_load.ptr); @@ -4978,7 +4989,7 @@ fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm(); const access_kind: Builder.MemoryAccessKind = if (info.flags.is_volatile) .@"volatile" else .normal; - const elem_llvm_ty = try self.lowerType(elem_ty); + const elem_llvm_ty = try o.lowerType(elem_ty); self.maybeMarkAllowZeroAccess(info); @@ -5135,7 +5146,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error const body_block = try self.wip.block(1, "InlineMemsetBody"); const end_block = try self.wip.block(1, "InlineMemsetEnd"); - const llvm_usize_ty = try self.lowerType(.usize); + const llvm_usize_ty = try o.lowerType(.usize); const end_ptr = switch (ptr_ty.ptrSize(zcu)) { .slice => try self.ptraddScaled( dest_ptr, @@ -5265,7 +5276,7 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder. assert(layout.tag_size != 0); const union_ptr = try self.resolveInst(ty_op.operand); if (isByRef(un_ty, zcu)) { - const llvm_un_ty = try self.lowerType(un_ty); + const llvm_un_ty = try o.lowerType(un_ty); if (layout.payload_size == 0) return self.wip.load(.normal, llvm_un_ty, union_ptr, .default, ""); const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); @@ -5296,6 +5307,7 @@ fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo } fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value { + const o = self.object; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const inst_ty = self.typeOfIndex(inst); const operand_ty = self.typeOf(ty_op.operand); @@ -5305,14 +5317,15 @@ fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) .normal, .none, intrinsic, - &.{try self.lowerType(operand_ty)}, + &.{try o.lowerType(operand_ty)}, &.{ operand, .false }, "", ); - return self.wip.conv(.unsigned, result, try self.lowerType(inst_ty), ""); + return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), ""); } fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value { + const o = self.object; const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const inst_ty = self.typeOfIndex(inst); const operand_ty = self.typeOf(ty_op.operand); @@ -5322,11 +5335,11 @@ fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) .normal, .none, intrinsic, - &.{try self.lowerType(operand_ty)}, + &.{try o.lowerType(operand_ty)}, &.{operand}, "", ); - return self.wip.conv(.unsigned, result, try self.lowerType(inst_ty), ""); + return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), ""); } fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { @@ -5339,7 +5352,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val const inst_ty = self.typeOfIndex(inst); var operand = try self.resolveInst(ty_op.operand); - var llvm_operand_ty = try self.lowerType(operand_ty); + var llvm_operand_ty = try o.lowerType(operand_ty); if (bits % 16 == 8) { // If not an even byte-multiple, we need zero-extend + shift-left 1 byte @@ -5360,7 +5373,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val const result = try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, ""); - return self.wip.conv(.unsigned, result, try self.lowerType(inst_ty), ""); + return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), ""); } fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { @@ -5437,10 +5450,10 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const operand = try self.resolveInst(un_op); const slice_ty = self.typeOfIndex(inst); - const slice_llvm_ty = try self.lowerType(slice_ty); + const slice_llvm_ty = try o.lowerType(slice_ty); // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed. - const operand_usize = try self.wip.conv(.unsigned, operand, try self.lowerType(.usize), ""); + const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize), ""); const error_name_table_ptr = try o.getErrorNameTable(); const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu)); @@ -5451,7 +5464,7 @@ fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const scalar = try self.resolveInst(ty_op.operand); const vector_ty = self.typeOfIndex(inst); - return self.wip.splatVector(try self.lowerType(vector_ty), scalar, ""); + return self.wip.splatVector(try self.object.lowerType(vector_ty), scalar, ""); } fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { @@ -5474,9 +5487,9 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val const operand = try fg.resolveInst(unwrapped.operand); const mask = unwrapped.mask; const operand_ty = fg.typeOf(unwrapped.operand); - const llvm_operand_ty = try fg.lowerType(operand_ty); - const llvm_result_ty = try fg.lowerType(unwrapped.result_ty); - const llvm_elem_ty = try fg.lowerType(unwrapped.result_ty.childType(zcu)); + const llvm_operand_ty = try o.lowerType(operand_ty); + const llvm_result_ty = try o.lowerType(unwrapped.result_ty); + const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu)); const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty); const llvm_poison_mask_elem = try o.builder.poisonConst(.i32); const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32); @@ -5578,7 +5591,7 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst); const mask = unwrapped.mask; - const llvm_elem_ty = try fg.lowerType(unwrapped.result_ty.childType(zcu)); + const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu)); const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32); const llvm_poison_mask_elem = try o.builder.poisonConst(.i32); @@ -5670,7 +5683,7 @@ fn buildReducedCall( accum_init: Builder.Value, ) Allocator.Error!Builder.Value { const o = self.object; - const usize_ty = try self.lowerType(.usize); + const usize_ty = try o.lowerType(.usize); const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len); const llvm_result_ty = accum_init.typeOfWip(&self.wip); @@ -5730,9 +5743,9 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) A const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce; const operand = try self.resolveInst(reduce.operand); const operand_ty = self.typeOf(reduce.operand); - const llvm_operand_ty = try self.lowerType(operand_ty); + const llvm_operand_ty = try o.lowerType(operand_ty); const scalar_ty = self.typeOfIndex(inst); - const llvm_scalar_ty = try self.lowerType(scalar_ty); + const llvm_scalar_ty = try o.lowerType(scalar_ty); switch (reduce.operation) { .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { @@ -5839,7 +5852,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde const result_ty = self.typeOfIndex(inst); const len: usize = @intCast(result_ty.arrayLen(zcu)); const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); - const llvm_result_ty = try self.lowerType(result_ty); + const llvm_result_ty = try o.lowerType(result_ty); switch (result_ty.zigTypeTag(zcu)) { .vector => { @@ -5905,7 +5918,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde field_ptr_align.toLlvm(), llvm_field_val, field_ty.abiAlignment(zcu).toLlvm(), - try o.builder.intValue(try self.lowerType(.usize), field_ty.abiSize(zcu)), + try o.builder.intValue(try o.lowerType(.usize), field_ty.abiSize(zcu)), .normal, self.disable_intrinsics, ); @@ -5956,7 +5969,7 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data; const union_ty = self.typeOfIndex(inst); - const union_llvm_ty = try self.lowerType(union_ty); + const union_llvm_ty = try o.lowerType(union_ty); const union_obj = zcu.typeToUnion(union_ty).?; assert(union_obj.layout != .@"packed"); @@ -6046,8 +6059,7 @@ fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const inst_ty = self.typeOfIndex(inst); const operand = try self.resolveInst(ty_op.operand); - - return self.wip.cast(.addrspacecast, operand, try self.lowerType(inst_ty), ""); + return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty), ""); } fn workIntrinsic( @@ -6192,7 +6204,7 @@ fn loadTruncate( const o = fg.object; const zcu = o.zcu; - const payload_llvm_ty = try fg.lowerType(payload_ty); + const payload_llvm_ty = try o.lowerType(payload_ty); const abi_size = payload_ty.abiSize(zcu); const load_llvm_ty = if (payload_ty.isAbiInt(zcu)) @@ -6220,7 +6232,7 @@ fn loadByRef( access_kind: Builder.MemoryAccessKind, ) Allocator.Error!Builder.Value { const o = fg.object; - const pointee_llvm_ty = try fg.lowerType(pointee_type); + const pointee_llvm_ty = try o.lowerType(pointee_type); const result_align = InternPool.Alignment.fromLlvm(ptr_alignment) .max(pointee_type.abiAlignment(o.zcu)).toLlvm(); const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align); @@ -6230,7 +6242,7 @@ fn loadByRef( result_align, ptr, ptr_alignment, - try o.builder.intValue(try fg.lowerType(.usize), size_bytes), + try o.builder.intValue(try o.lowerType(.usize), size_bytes), access_kind, fg.disable_intrinsics, ); @@ -6274,7 +6286,7 @@ fn storeFull( if (info.flags.vector_index != .none) { const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); - const vec_elem_ty = try self.lowerType(elem_ty); + const vec_elem_ty = try o.lowerType(elem_ty); const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty); const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, ""); @@ -6343,7 +6355,7 @@ fn storeFull( ptr_alignment, elem, elem_ty.abiAlignment(zcu).toLlvm(), - try o.builder.intValue(try self.lowerType(.usize), elem_ty.abiSize(zcu)), + try o.builder.intValue(try o.lowerType(.usize), elem_ty.abiSize(zcu)), access_kind, self.disable_intrinsics, ); @@ -6370,7 +6382,7 @@ fn store( elem, elem_ty.abiAlignment(zcu).toLlvm(), try o.builder.intValue( - try fg.lowerType(.usize), + try o.lowerType(.usize), elem_ty.abiSize(zcu), ), .normal, @@ -6391,7 +6403,7 @@ fn store( fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void { const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545; const o = fg.object; - const usize_ty = try fg.lowerType(.usize); + const usize_ty = try o.lowerType(.usize); const zero = try o.builder.intValue(usize_ty, 0); const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED); const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, ""); @@ -6413,7 +6425,7 @@ fn valgrindClientRequest( const target = zcu.getTarget(); if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value; - const llvm_usize = try fg.lowerType(.usize); + const llvm_usize = try o.lowerType(.usize); const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm(); const array_llvm_ty = try o.builder.arrayType(6, llvm_usize); @@ -7261,8 +7273,9 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value { if (offset == 0) return ptr; - const llvm_usize_ty = try fg.lowerType(.usize); - const offset_val = try fg.object.builder.intValue(llvm_usize_ty, offset); + const o = fg.object; + const llvm_usize_ty = try o.lowerType(.usize); + const offset_val = try o.builder.intValue(llvm_usize_ty, offset); return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, ""); } fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value { -- 2.54.0 From 500e6c7cfe7fe57db51b99a8e76e67a3441de7ac Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Tue, 24 Mar 2026 10:28:04 +0000 Subject: [PATCH 07/15] llvm: some more random enhancements Avoid directly querying `Builder.Type`s in favour of `lowerType` calls in a couple of places. The idea here is to avoid querying state stored in the `Builder` to try and move towards a world where codegen (essentially the logic in `codegen.llvm.FuncGen`) can happen on a separate thread to "linking" (which actually interacts with shared state on `codegen.llvm.Object` and `std.zig.llvm.Builder`). Don't clear the `Builder` state during `emit`; this is clearly incompatible with incremental compilation. With that line of code removed, incremental compilation is actually already somewhat functional with the LLVM backend. Also, don't use `c_uint` for source location state---I have no idea where this came from but it definitely isn't correct. --- src/codegen/llvm.zig | 1 - src/codegen/llvm/FuncGen.zig | 104 +++++++++++++++++------------------ 2 files changed, 49 insertions(+), 56 deletions(-) diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 1ce296f452e6c1cfb3d52cc4263f7d1692ccb3b9..6716b1caeaea2c80c8f201a43439bf7813fa841e 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -944,7 +944,6 @@ pub const Object = struct { .version = build_options.semver, }); defer o.gpa.free(bitcode); - o.builder.clearAndFree(); if (options.pre_bc_path) |path| { var file = Io.Dir.cwd().createFile(io, path, .{}) catch |err| diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index 6f90cfd3c50b4c882eb39b0a0e37e0ecd28797a9..e70091cebada83ad15da3c8ee87dd72f3981a4d5 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -16,8 +16,8 @@ scope: Builder.Metadata, inlined_at: Builder.Metadata.Optional, base_line: u32, -prev_dbg_line: c_uint, -prev_dbg_column: c_uint, +prev_dbg_line: u32, +prev_dbg_column: u32, /// This stores the LLVM values used in a function, such that they can be referred to /// in other instructions. This table is cleared before every function is generated. @@ -815,7 +815,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif .always_tail => .musttail, .no_suspend, .always_inline, .compile_time => unreachable, }, - toLlvmCallConvTag(fn_info.cc, target).?, + llvm.toLlvmCallConvTag(fn_info.cc, target).?, try attributes.finish(&o.builder), try o.lowerType(zig_fn_ty), llvm_fn, @@ -882,7 +882,7 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v _ = try fg.wip.callIntrinsicAssumeCold(); _ = try fg.wip.call( .normal, - toLlvmCallConvTag(fn_info.cc, target).?, + llvm.toLlvmCallConvTag(fn_info.cc, target).?, .none, panic_global.typeOf(&o.builder), panic_global.toValue(&o.builder), @@ -1394,7 +1394,7 @@ fn lowerSwitchDispatch( // be handled by conditional branches in the `else` prong. const llvm_usize = try o.lowerType(.usize); - const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder)) + const cond_int = if (cond_ty.zigTypeTag(zcu) == .pointer) try self.wip.cast(.ptrtoint, cond, llvm_usize, "") else cond; @@ -1433,7 +1433,7 @@ fn lowerSwitchDispatch( for (case.items) |item| { const llvm_item = (try self.resolveInst(item)).toConst().?; - const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder)) + const llvm_int_item = if (cond_ty.zigTypeTag(zcu) == .pointer) try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize) else llvm_item; @@ -2840,7 +2840,7 @@ fn airIsNonNull( operand; if (payload_ty.isSlice(zcu)) { const slice_ptr = try self.wip.extractValue(loaded, &.{0}, ""); - const ptr_ty = try o.builder.ptrType(toLlvmAddressSpace( + const ptr_ty = try o.builder.ptrType(llvm.toLlvmAddressSpace( payload_ty.ptrAddressSpace(zcu), zcu.getTarget(), )); @@ -3342,17 +3342,17 @@ fn airSafeArithmetic( const overflow_bits = try fg.wip.extractValue(results, &.{1}, ""); const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip); - const overflow_bit = if (overflow_bits_ty.isVector(&o.builder)) - try fg.wip.callIntrinsic( + const overflow_bit = switch (inst_ty.zigTypeTag(zcu)) { + .vector => try fg.wip.callIntrinsic( .normal, .none, .@"vector.reduce.or", &.{overflow_bits_ty}, &.{overflow_bits}, "", - ) - else - overflow_bits; + ), + else => overflow_bits, + }; const fail_block = try fg.wip.block(1, "OverflowFail"); const ok_block = try fg.wip.block(1, "OverflowOk"); @@ -3508,6 +3508,7 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result}); } if (scalar_ty.isSignedInt(zcu)) { + const scalar_llvm_ty = try o.lowerType(scalar_ty); const inst_llvm_ty = try o.lowerType(inst_ty); const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb; @@ -3517,7 +3518,7 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) )) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); const allocator = stack.get(); - const scalar_bits = inst_llvm_ty.scalarBits(&o.builder); + const scalar_bits = scalar_ty.intInfo(zcu).bits; var smin_big_int: std.math.big.int.Mutable = .{ .limbs = try allocator.alloc( std.math.big.Limb, @@ -3529,7 +3530,7 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) defer allocator.free(smin_big_int.limbs); smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits); const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst( - inst_llvm_ty.scalarType(&o.builder), + scalar_llvm_ty, smin_big_int.toConst(), )); @@ -3603,7 +3604,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo )) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); const allocator = stack.get(); - const scalar_bits = inst_llvm_ty.scalarBits(&o.builder); + const scalar_bits = scalar_ty.intInfo(zcu).bits; var smin_big_int: std.math.big.int.Mutable = .{ .limbs = try allocator.alloc( std.math.big.Limb, @@ -3615,7 +3616,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo defer allocator.free(smin_big_int.limbs); smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits); const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst( - inst_llvm_ty.scalarType(&o.builder), + try o.lowerType(scalar_ty), smin_big_int.toConst(), )); @@ -3929,7 +3930,10 @@ fn buildFloatOp( // In this case we can generate a softfloat negation by XORing the // bits with a constant. const int_ty = try o.builder.intType(@intCast(float_bits)); - const cast_ty = try llvm_ty.changeScalar(int_ty, &o.builder); + const cast_ty = switch (ty.zigTypeTag(zcu)) { + .vector => try o.builder.vectorType(.normal, ty.vectorLen(zcu), int_ty), + else => int_ty, + }; const sign_mask = try o.builder.splatValue( cast_ty, try o.builder.intConst(int_ty, @as(u128, 1) << @intCast(float_bits - 1)), @@ -3964,7 +3968,7 @@ fn buildFloatOp( }), }; - const scalar_llvm_ty = llvm_ty.scalarType(&o.builder); + const scalar_llvm_ty = try o.lowerType(scalar_ty); const libc_fn = try o.getLibcFunction( fn_name, ([1]Builder.Type{scalar_llvm_ty} ** 3)[0..params.len], @@ -4121,7 +4125,7 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const lhs_ty = self.typeOf(bin_op.lhs); const lhs_info = lhs_ty.intInfo(zcu); const llvm_lhs_ty = try o.lowerType(lhs_ty); - const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder); + const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu)); const rhs_ty = self.typeOf(bin_op.rhs); if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) { @@ -4132,7 +4136,7 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const rhs_info = rhs_ty.intInfo(zcu); assert(rhs_info.signedness == .unsigned); const llvm_rhs_ty = try o.lowerType(rhs_ty); - const llvm_rhs_scalar_ty = llvm_rhs_ty.scalarType(&o.builder); + const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu)); const result = try self.wip.callIntrinsic( .normal, @@ -4448,9 +4452,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty return operand; } - if (llvm_dest_ty.isInteger(&o.builder) and - operand.typeOfWip(&self.wip).isInteger(&o.builder)) - { + if (inst_ty.isAbiInt(zcu) and operand_ty.isAbiInt(zcu)) { return self.wip.conv(.unsigned, operand, llvm_dest_ty, ""); } @@ -4524,7 +4526,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty return result_ptr; } - if (llvm_dest_ty.isStruct(&o.builder) or + if (inst_ty.isSliceAtRuntime(zcu) or ((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu))) { @@ -4948,28 +4950,28 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va ), llvm_operand_ty, ""); } - if (!llvm_operand_ty.isPointer(&o.builder)) return self.wip.atomicrmw( + // If we are storing a pointer we need to convert to and from a plain old integer. + const non_ptr_operand = switch (operand_ty.zigTypeTag(zcu)) { + .pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize), ""), + else => operand, + }; + + const raw_result = try self.wip.atomicrmw( access_kind, op, ptr, - operand, + non_ptr_operand, self.sync_scope, ordering, ptr_alignment, "", ); - // It's a pointer but we need to treat it as an int. - return self.wip.cast(.inttoptr, try self.wip.atomicrmw( - access_kind, - op, - ptr, - try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize), ""), - self.sync_scope, - ordering, - ptr_alignment, - "", - ), llvm_operand_ty, ""); + // ...and then convert the result back. + switch (operand_ty.zigTypeTag(zcu)) { + .pointer => return self.wip.cast(.inttoptr, raw_result, llvm_operand_ty, ""), + else => return raw_result, + } } fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { @@ -5274,19 +5276,16 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder. const un_ty = self.typeOf(ty_op.operand); const layout = un_ty.unionGetLayout(zcu); assert(layout.tag_size != 0); - const union_ptr = try self.resolveInst(ty_op.operand); + const operand = try self.resolveInst(ty_op.operand); if (isByRef(un_ty, zcu)) { - const llvm_un_ty = try o.lowerType(un_ty); - if (layout.payload_size == 0) - return self.wip.load(.normal, llvm_un_ty, union_ptr, .default, ""); - const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); - const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset()); - const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index]; + const llvm_tag_ty = try o.lowerType(un_ty.unionTagTypeRuntime(zcu).?); + const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset()); return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, ""); } else { - if (layout.payload_size == 0) return union_ptr; - const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align)); - return self.wip.extractValue(union_ptr, &.{tag_index}, ""); + // This is only possible if all fields are zero-bit, in which case `operand` is already an + // integer value (the union is lowered as its enum tag). + assert(layout.payload_size == 0); + return operand; } } @@ -7356,9 +7355,9 @@ fn appendConstraints( /// may need to manually generate a compiler-rt call. fn intrinsicsAllowed(scalar_ty: Type, target: *const std.Target) bool { return switch (scalar_ty.toIntern()) { - .f16_type => backendSupportsF16(target), - .f80_type => (target.cTypeBitSize(.longdouble) == 80) and backendSupportsF80(target), - .f128_type => (target.cTypeBitSize(.longdouble) == 128) and backendSupportsF128(target), + .f16_type => llvm.backendSupportsF16(target), + .f80_type => (target.cTypeBitSize(.longdouble) == 80) and llvm.backendSupportsF80(target), + .f128_type => (target.cTypeBitSize(.longdouble) == 128) and llvm.backendSupportsF128(target), else => true, }; } @@ -7702,9 +7701,4 @@ const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev; const llvm = @import("../llvm.zig"); const Object = llvm.Object; -const toLlvmCallConvTag = llvm.toLlvmCallConvTag; -const toLlvmAddressSpace = llvm.toLlvmAddressSpace; const optional_layout_version = llvm.optional_layout_version; -const backendSupportsF16 = llvm.backendSupportsF16; -const backendSupportsF80 = llvm.backendSupportsF80; -const backendSupportsF128 = llvm.backendSupportsF128; -- 2.54.0 From d28c5069b8421628fcb333d8e558a41d2a2d93d5 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Tue, 24 Mar 2026 11:23:28 +0000 Subject: [PATCH 08/15] llvm: rework handling of globals The main goal here is to make incremental compilation work a bit better. I also slightly expanded some `std.zig.llvm.Builder` APIs so that we don't need to pointlessly create new `Global`s whenever e.g. a function turns into a variable or vice versa. Also, lean into aliases for exports! If we just use aliases for every export, everything becomes simpler. Besides, we can't just go around renaming the globals of `Nav`s: the export could disappear on a future update, in which case we'd have to somehow revert that change, which is easier said than done. --- lib/std/zig/llvm/Builder.zig | 155 +++-- src/Sema.zig | 1 - src/Zcu.zig | 4 +- src/Zcu/PerThread.zig | 9 +- src/codegen/llvm.zig | 1196 ++++++++++++++++------------------ src/codegen/llvm/FuncGen.zig | 66 +- 6 files changed, 703 insertions(+), 728 deletions(-) diff --git a/lib/std/zig/llvm/Builder.zig b/lib/std/zig/llvm/Builder.zig index 5001a0c375a55f2c49a26e5f697086d65f34c2dd..276012b09a9e0e41622fa28a5d47b95723a26222 100644 --- a/lib/std/zig/llvm/Builder.zig +++ b/lib/std/zig/llvm/Builder.zig @@ -2343,12 +2343,13 @@ pub const Global = struct { none = maxInt(u32), _, - pub fn unwrap(self: Index, builder: *const Builder) Index { - var cur = self; + pub fn unwrap(orig_index: Index, builder: *const Builder) Index { + var cur = orig_index; while (true) { - const replacement = cur.getReplacement(builder); - if (replacement == .none) return cur; - cur = replacement; + switch (builder.globals.values()[@intFromEnum(cur)].kind) { + .replaced => |replacement| cur = replacement, + else => return cur, + } } } @@ -2388,8 +2389,12 @@ pub const Global = struct { return self.ptrConst(builder).type; } - pub fn toConst(self: Index) Constant { - return @enumFromInt(@intFromEnum(Constant.first_global) + @intFromEnum(self)); + pub fn toConst(global: Index) Constant { + return @enumFromInt(@intFromEnum(Constant.first_global) + @intFromEnum(global)); + } + + pub fn toValue(global: Index) Value { + return global.toConst().toValue(); } pub fn setLinkage(self: Index, linkage: Linkage, builder: *Builder) void { @@ -2450,6 +2455,42 @@ pub const Global = struct { self.ptr(builder).kind = .{ .replaced = .none }; } + /// Replaces whatever this `Global` currently contains with a new `Function`. Similar to + /// `Builder.addFunction`, but the same `Global` is reused. + pub fn toNewFunction(global: Index, builder: *Builder) Allocator.Error!Function.Index { + try builder.functions.ensureUnusedCapacity(builder.gpa, 1); + errdefer comptime unreachable; + const function: Function.Index = @enumFromInt(builder.functions.items.len); + builder.functions.appendAssumeCapacity(.{ + .global = global, + .strip = undefined, + }); + global.ptr(builder).kind = .{ .function = function }; + return function; + } + + /// Replaces whatever this `Global` currently contains with a new `Variable`. Similar to + /// `Builder.addVariable`, but the same `Global` is reused. + pub fn toNewVariable(global: Index, builder: *Builder) Allocator.Error!Variable.Index { + try builder.variables.ensureUnusedCapacity(builder.gpa, 1); + errdefer comptime unreachable; + const variable: Variable.Index = @enumFromInt(builder.variables.items.len); + builder.variables.appendAssumeCapacity(.{ .global = global }); + global.ptr(builder).kind = .{ .variable = variable }; + return variable; + } + + /// Replaces whatever this `Global` currently contains with a new `Alias`. Similar to + /// `Builder.addAlias`, but the same `Global` is reused. + pub fn toNewAlias(global: Index, builder: *Builder) Allocator.Error!Alias.Index { + try builder.aliases.ensureUnusedCapacity(builder.gpa, 1); + errdefer comptime unreachable; + const alias: Alias.Index = @enumFromInt(builder.aliases.items.len); + builder.aliass.appendAssumeCapacity(.{ .global = global, .aliasee = .none }); + global.ptr(builder).kind = .{ .alias = alias }; + return alias; + } + fn updateDsoLocal(self: Index, builder: *Builder) void { const self_ptr = self.ptr(builder); switch (self_ptr.linkage) { @@ -2494,13 +2535,6 @@ pub const Global = struct { self.renameAssumeCapacity(builder.next_replaced_global, builder); self.ptr(builder).kind = .{ .replaced = other.unwrap(builder) }; } - - fn getReplacement(self: Index, builder: *const Builder) Index { - return switch (builder.globals.values()[@intFromEnum(self)].kind) { - .replaced => |replacement| replacement, - else => .none, - }; - } }; }; @@ -2593,22 +2627,6 @@ pub const Variable = struct { return self.toConst(builder).toValue(); } - pub fn setLinkage(self: Index, linkage: Linkage, builder: *Builder) void { - return self.ptrConst(builder).global.setLinkage(linkage, builder); - } - - pub fn setVisibility(self: Index, visibility: Visibility, builder: *Builder) void { - return self.ptrConst(builder).global.setVisibility(visibility, builder); - } - - pub fn setDllStorageClass(self: Index, class: DllStorageClass, builder: *Builder) void { - return self.ptrConst(builder).global.setDllStorageClass(class, builder); - } - - pub fn setUnnamedAddr(self: Index, unnamed_addr: UnnamedAddr, builder: *Builder) void { - return self.ptrConst(builder).global.setUnnamedAddr(unnamed_addr, builder); - } - pub fn setThreadLocal(self: Index, thread_local: ThreadLocal, builder: *Builder) void { self.ptr(builder).thread_local = thread_local; } @@ -9692,8 +9710,12 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void if (self.variables.items.len > 0) { if (need_newline) try w.writeByte('\n') else need_newline = true; - for (self.variables.items) |variable| { - if (variable.global.getReplacement(self) != .none) continue; + for (self.variables.items, 0..) |variable, variable_i| { + // Skip the variable if its global has been repurposed for something else. + switch (variable.global.ptrConst(self).kind) { + .variable => |v| if (@intFromEnum(v) != variable_i) continue, + else => continue, + } const global = variable.global.ptrConst(self); metadata_formatter.need_comma = true; defer metadata_formatter.need_comma = undefined; @@ -9723,8 +9745,12 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void if (self.aliases.items.len > 0) { if (need_newline) try w.writeByte('\n') else need_newline = true; - for (self.aliases.items) |alias| { - if (alias.global.getReplacement(self) != .none) continue; + for (self.aliases.items, 0..) |alias, alias_i| { + // Skip the alias if its global has been repurposed for something else. + switch (alias.global.ptrConst(self).kind) { + .alias => |a| if (@intFromEnum(a) != alias_i) continue, + else => continue, + } const global = alias.global.ptrConst(self); metadata_formatter.need_comma = true; defer metadata_formatter.need_comma = undefined; @@ -9750,7 +9776,11 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void defer attribute_groups.deinit(self.gpa); for (0.., self.functions.items) |function_i, function| { - if (function.global.getReplacement(self) != .none) continue; + // Skip the function if its global has been repurposed for something else. + switch (function.global.ptrConst(self).kind) { + .function => |f| if (@intFromEnum(f) != function_i) continue, + else => continue, + } if (need_newline) try w.writeByte('\n') else need_newline = true; const function_index: Function.Index = @enumFromInt(function_i); const global = function.global.ptrConst(self); @@ -13687,20 +13717,32 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco self.aliases.items.len, ); - for (self.variables.items) |variable| { - if (variable.global.getReplacement(self) != .none) continue; + for (self.variables.items, 0..) |variable, variable_i| { + // Skip the variable if its global has been repurposed for something else. + switch (variable.global.ptrConst(self).kind) { + .variable => |v| if (@intFromEnum(v) != variable_i) continue, + else => continue, + } globals.putAssumeCapacity(variable.global, {}); } - for (self.functions.items) |function| { - if (function.global.getReplacement(self) != .none) continue; + for (self.functions.items, 0..) |function, function_i| { + // Skip the function if its global has been repurposed for something else. + switch (function.global.ptrConst(self).kind) { + .function => |f| if (@intFromEnum(f) != function_i) continue, + else => continue, + } globals.putAssumeCapacity(function.global, {}); } - for (self.aliases.items) |alias| { - if (alias.global.getReplacement(self) != .none) continue; + for (self.aliases.items, 0..) |alias, alias_i| { + // Skip the alias if its global has been repurposed for something else. + switch (alias.global.ptrConst(self).kind) { + .alias => |a| if (@intFromEnum(a) != alias_i) continue, + else => continue, + } globals.putAssumeCapacity(alias.global, {}); } @@ -13742,8 +13784,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco defer section_map.deinit(self.gpa); try section_map.ensureUnusedCapacity(self.gpa, globals.count()); - for (self.variables.items) |variable| { - if (variable.global.getReplacement(self) != .none) continue; + for (self.variables.items, 0..) |variable, variable_i| { + // Skip the variable if its global has been repurposed for something else. + switch (variable.global.ptrConst(self).kind) { + .variable => |v| if (@intFromEnum(v) != variable_i) continue, + else => continue, + } const section = blk: { if (variable.section == .none) break :blk 0; @@ -13789,8 +13835,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco }); } - for (self.functions.items) |func| { - if (func.global.getReplacement(self) != .none) continue; + for (self.functions.items, 0..) |func, func_i| { + // Skip the function if its global has been repurposed for something else. + switch (func.global.ptrConst(self).kind) { + .function => |f| if (@intFromEnum(f) != func_i) continue, + else => continue, + } const section = blk: { if (func.section == .none) break :blk 0; @@ -13830,8 +13880,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco }); } - for (self.aliases.items) |alias| { - if (alias.global.getReplacement(self) != .none) continue; + for (self.aliases.items, 0..) |alias, alias_i| { + // Skip the alias if its global has been repurposed for something else. + switch (alias.global.ptrConst(self).kind) { + .alias => |a| if (@intFromEnum(a) != alias_i) continue, + else => continue, + } const strtab = alias.global.strtab(self); @@ -14635,8 +14689,13 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco }; for (self.functions.items, 0..) |func, func_index| { + // Skip the function if its global has been repurposed for something else. + switch (func.global.ptrConst(self).kind) { + .function => |f| if (@intFromEnum(f) != func_index) continue, + else => continue, + } + const FunctionBlock = ir.ModuleBlock.FunctionBlock; - if (func.global.getReplacement(self) != .none) continue; if (func.instructions.len == 0) continue; diff --git a/src/Sema.zig b/src/Sema.zig index ab7e8b4c155fe804c09a17e6020c7955c2f86d5f..bda2659d8b1cdc0a2a81f0141017bfaff16fb0b6 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -5751,7 +5751,6 @@ fn zirExport(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void if (ptr_info.byte_offset != 0) { return sema.fail(block, ptr_src, "TODO: export pointer in middle of value", .{}); } - if (zcu.llvm_object != null and options.linkage == .internal) return; try sema.exports.append(zcu.gpa, .{ .opts = options, .src = src, diff --git a/src/Zcu.zig b/src/Zcu.zig index 820524d718ad168253bc4e658eda1ddae2f7ba73..88258b236acf77c69a5b3e97b2d9e20ebc346c04 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -3731,7 +3731,9 @@ pub fn resetUnit(zcu: *Zcu, unit: AnalUnit) void { }; for (zcu.all_exports.items[base..][0..len], base..) |exp, exp_index_usize| { const exp_index: Export.Index = @enumFromInt(exp_index_usize); - if (zcu.comp.bin_file) |lf| { + if (zcu.llvm_object) |llvm_object| { + _ = llvm_object; // TODO: delete exports from LLVM + } else if (zcu.comp.bin_file) |lf| { lf.deleteExport(exp.exported, exp.opts.name); } if (zcu.failed_exports.fetchSwapRemove(exp_index)) |failed_kv| { diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index c9972a300abea99261410ce5ea2de893040a3b8a..7d7e029041bd2f6d5cd04f3082a4c76072159b0d 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -1910,14 +1910,7 @@ fn analyzeNavVal( try sema.flushExports(); - queue_codegen: { - if (!queue_linker_work) break :queue_codegen; - - if (!nav_ty.hasRuntimeBits(zcu)) { - if (comp.config.use_llvm) break :queue_codegen; - if (file.mod.?.strip) break :queue_codegen; - } - + if (queue_linker_work) { comp.link_prog_node.increaseEstimatedTotalItems(1); try comp.link_queue.enqueueZcu(comp, pt.tid, .{ .link_nav = nav_id }); } diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 6716b1caeaea2c80c8f201a43439bf7813fa841e..bd3be4f96f8f71b8301dd466b505c25df85b960e 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -550,17 +550,13 @@ pub const Object = struct { debug_anyerror_fwd_ref: Builder.Metadata.Optional, zcu: *Zcu, - /// Ideally we would use `llvm_module.getNamedFunction` to go from *Decl to LLVM function, - /// but that has some downsides: - /// * we have to compute the fully qualified name every time we want to do the lookup - /// * for externally linked functions, the name is not fully qualified, but when - /// a Decl goes from exported to not exported and vice-versa, we would use the wrong - /// version of the name and incorrectly get function not found in the llvm module. - /// * it works for functions not all globals. - /// Therefore, this table keeps track of the mapping. + /// Maps a `Nav` to the corresponding LLVM global. nav_map: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Builder.Global.Index), - /// Same deal as `decl_map` but for anonymous declarations, which are always global constants. - uav_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Global.Index), + /// Same as `nav_map` but for UAVs (which are always global constants). + uav_map: std.AutoHashMapUnmanaged(struct { + val: InternPool.Index, + @"addrspace": std.builtin.AddressSpace, + }, Builder.Variable.Index), /// Maps enum types to their corresponding LLVM functions for implementing the `tag_name` instruction. enum_tag_name_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index), /// Serves the same purpose as `enum_tag_name_map` but for the `is_named_enum_value` instruction. @@ -717,13 +713,13 @@ pub const Object = struct { for (llvm_errors[1..], error_name_list) |*llvm_error, name| { const name_string = try o.builder.stringNull(name.toSlice(ip)); const name_init = try o.builder.stringConst(name_string); - const name_variable_index = - try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); + const name_variable_index = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); try name_variable_index.setInitializer(name_init, &o.builder); - name_variable_index.setLinkage(.private, &o.builder); name_variable_index.setMutability(.constant, &o.builder); - name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); name_variable_index.setAlignment(comptime .fromByteUnits(1), &o.builder); + const global_index = name_variable_index.ptrConst(&o.builder).global; + global_index.setLinkage(.private, &o.builder); + global_index.setUnnamedAddr(.unnamed_addr, &o.builder); llvm_error.* = try o.builder.structConst(llvm_slice_ty, &.{ name_variable_index.toConst(&o.builder), @@ -790,9 +786,9 @@ pub const Object = struct { array_llvm_ty, .default, ); - compiler_used_variable.setLinkage(.appending, &o.builder); - compiler_used_variable.setSection(try o.builder.string("llvm.metadata"), &o.builder); try compiler_used_variable.setInitializer(init_val, &o.builder); + compiler_used_variable.setSection(try o.builder.string("llvm.metadata"), &o.builder); + compiler_used_variable.ptrConst(&o.builder).global.setLinkage(.appending, &o.builder); } if (!o.builder.strip) { @@ -1140,6 +1136,7 @@ pub const Object = struct { ) Zcu.CodegenFailError!void { const zcu = o.zcu; const comp = zcu.comp; + const gpa = comp.gpa; const ip = &zcu.intern_pool; const func = zcu.funcInfo(func_index); const nav = ip.getNav(func.owner_nav); @@ -1149,9 +1146,42 @@ pub const Object = struct { const fn_info = zcu.typeToFunc(fn_ty).?; const target = &owner_mod.resolved_target.result; - const function_index = try o.resolveLlvmFunction(func.owner_nav); + const gop = try o.nav_map.getOrPut(gpa, func.owner_nav); + if (!gop.found_existing) { + errdefer assert(o.nav_map.remove(func.owner_nav)); + // First time lowering this NAV! Create a fresh global. + const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip)); + gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{ + .type = .void, // placeholder; populated below + .kind = .{ .alias = .none }, // placeholder; populated below + }); + } + const llvm_global = gop.value_ptr.*; - var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); + const llvm_function: Builder.Function.Index = switch (llvm_global.ptrConst(&o.builder).kind) { + .function => |function| function, // re-use existing `Builder.Function` + .replaced, .alias, .variable => try llvm_global.toNewFunction(&o.builder), + }; + { + const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder); + global.type = try o.lowerType(fn_ty); + global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target); + global.linkage = if (o.builder.strip) .private else .internal; + global.visibility = .default; + global.dll_storage_class = .default; + global.unnamed_addr = .unnamed_addr; + } + llvm_function.setAlignment(switch (nav.resolved.?.@"align") { + .none => fn_ty.abiAlignment(zcu).toLlvm(), + else => |a| a.toLlvm(), + }, &o.builder); + llvm_function.setSection(s: { + const section = nav.resolved.?.@"linksection".toSlice(ip) orelse break :s .none; + break :s try o.builder.string(section); + }, &o.builder); + try o.addLlvmFunctionAttributes(pt, func.owner_nav, llvm_function); + + var attributes = try llvm_function.ptrConst(&o.builder).attributes.toWip(&o.builder); defer attributes.deinit(&o.builder); const func_analysis = func.analysisUnordered(ip); @@ -1221,47 +1251,41 @@ pub const Object = struct { } }, &o.builder); } - if (nav.resolved.?.@"linksection".toSlice(ip)) |section| - function_index.setSection(try o.builder.string(section), &o.builder); - var deinit_wip = true; var wip = try Builder.WipFunction.init(&o.builder, .{ - .function = function_index, + .function = llvm_function, .strip = owner_mod.strip, }); defer if (deinit_wip) wip.deinit(); wip.cursor = .{ .block = try wip.block(0, "Entry") }; - var llvm_arg_i: u32 = 0; - - const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: { - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; - break :param param; - } else .none; - if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) { .signed => try attributes.addRetAttr(.signext, &o.builder), .unsigned => try attributes.addRetAttr(.zeroext, &o.builder), }; - const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing; - - const err_ret_trace: Builder.Value = if (err_return_tracing) param: { - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; - break :param param; - } else .none; - // This is the list of args we will use that correspond directly to the AIR arg // instructions. Depending on the calling convention, this list is not necessarily // a bijection with the actual LLVM parameters of the function. - const gpa = o.gpa; var args: std.ArrayList(Builder.Value) = .empty; defer args.deinit(gpa); - { + const ret_ptr: Builder.Value, const err_ret_trace: Builder.Value = implicit_args: { var it = iterateParamTypes(o, fn_info); + + const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: { + const param = wip.arg(it.llvm_index); + it.llvm_index += 1; + break :param param; + } else .none; + + const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing; + const err_ret_trace: Builder.Value = if (err_return_tracing) param: { + const param = wip.arg(it.llvm_index); + it.llvm_index += 1; + break :param param; + } else .none; + while (try it.next()) |lowering| { try args.ensureUnusedCapacity(gpa, 1); @@ -1271,7 +1295,7 @@ pub const Object = struct { assert(!it.byval_attr); const param_index = it.zig_index - 1; const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); - const param = wip.arg(llvm_arg_i); + const param = wip.arg(it.llvm_index - 1); if (isByRef(param_ty, zcu)) { const alignment = param_ty.abiAlignment(zcu).toLlvm(); @@ -1281,146 +1305,116 @@ pub const Object = struct { args.appendAssumeCapacity(arg_ptr); } else { args.appendAssumeCapacity(param); - - try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, llvm_arg_i); } - llvm_arg_i += 1; }, .byref => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(param_ty); - const param = wip.arg(llvm_arg_i); - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - - try o.addByRefParamAttrs(&attributes, llvm_arg_i, alignment, it.byval_attr, param_llvm_ty); - llvm_arg_i += 1; + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param = wip.arg(it.llvm_index - 1); if (isByRef(param_ty, zcu)) { args.appendAssumeCapacity(param); } else { + const param_llvm_ty = try o.lowerType(param_ty); + const alignment = param_ty.abiAlignment(zcu).toLlvm(); args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, "")); } }, .byref_mut => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(param_ty); - const param = wip.arg(llvm_arg_i); - const alignment = param_ty.abiAlignment(zcu).toLlvm(); - - try attributes.addParamAttr(llvm_arg_i, .noundef, &o.builder); - llvm_arg_i += 1; + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param = wip.arg(it.llvm_index - 1); if (isByRef(param_ty, zcu)) { args.appendAssumeCapacity(param); } else { + const param_llvm_ty = try o.lowerType(param_ty); + const alignment = param_ty.abiAlignment(zcu).toLlvm(); args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, "")); } }, .abi_sized_int => { assert(!it.byval_attr); - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param = wip.arg(it.llvm_index - 1); const param_llvm_ty = try o.lowerType(param_ty); const alignment = param_ty.abiAlignment(zcu).toLlvm(); const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target); _ = try wip.store(.normal, param, arg_ptr, alignment); - args.appendAssumeCapacity(if (isByRef(param_ty, zcu)) - arg_ptr - else - try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + if (isByRef(param_ty, zcu)) { + args.appendAssumeCapacity(arg_ptr); + } else { + args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + } }, .slice => { assert(!it.byval_attr); - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const ptr_info = param_ty.ptrInfo(zcu); - - if (std.math.cast(u5, it.zig_index - 1)) |i| { - if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { - try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); - } - } - if (param_ty.zigTypeTag(zcu) != .optional and - !ptr_info.flags.is_allowzero and - ptr_info.flags.address_space == .generic) - { - try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); - } - if (ptr_info.flags.is_const) { - try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder); - } - const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) { - else => |a| .wrap(a.toLlvm()), - .none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)), - }; - try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder); - const ptr_param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; - const len_param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; - - const slice_llvm_ty = try o.lowerType(param_ty); - args.appendAssumeCapacity( - try wip.buildAggregate(slice_llvm_ty, &.{ ptr_param, len_param }, ""), + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + assert(!isByRef(param_ty, zcu)); + const slice_val = try wip.buildAggregate( + try o.lowerType(param_ty), + &.{ wip.arg(it.llvm_index - 2), wip.arg(it.llvm_index - 1) }, + "", ); + args.appendAssumeCapacity(slice_val); }, .multiple_llvm_types => { assert(!it.byval_attr); const field_types = it.types_buffer[0..it.types_len]; - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); const param_llvm_ty = try o.lowerType(param_ty); const param_alignment = param_ty.abiAlignment(zcu).toLlvm(); const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, param_alignment, target); const llvm_ty = try o.builder.structType(.normal, field_types); - for (0..field_types.len) |field_i| { - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; + const llvm_args_start = it.llvm_index - field_types.len; + for (0..field_types.len, llvm_args_start..) |field_i, llvm_arg_index| { + const param = wip.arg(@intCast(llvm_arg_index)); const field_ptr = try wip.gepStruct(llvm_ty, arg_ptr, field_i, ""); - const alignment = Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8)); + const alignment: Builder.Alignment = .fromByteUnits(@divExact(target.ptrBitWidth(), 8)); _ = try wip.store(.normal, param, field_ptr, alignment); } - const is_by_ref = isByRef(param_ty, zcu); - args.appendAssumeCapacity(if (is_by_ref) - arg_ptr - else - try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment, "")); + if (isByRef(param_ty, zcu)) { + args.appendAssumeCapacity(arg_ptr); + } else { + args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment, "")); + } }, .float_array => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); const param_llvm_ty = try o.lowerType(param_ty); - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; + const param = wip.arg(it.llvm_index - 1); const alignment = param_ty.abiAlignment(zcu).toLlvm(); const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target); _ = try wip.store(.normal, param, arg_ptr, alignment); - args.appendAssumeCapacity(if (isByRef(param_ty, zcu)) - arg_ptr - else - try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + if (isByRef(param_ty, zcu)) { + args.appendAssumeCapacity(arg_ptr); + } else { + args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + } }, .i32_array, .i64_array => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); const param_llvm_ty = try o.lowerType(param_ty); - const param = wip.arg(llvm_arg_i); - llvm_arg_i += 1; + const param = wip.arg(it.llvm_index - 1); const alignment = param_ty.abiAlignment(zcu).toLlvm(); const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target); _ = try wip.store(.normal, param, arg_ptr, alignment); - args.appendAssumeCapacity(if (isByRef(param_ty, zcu)) - arg_ptr - else - try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + if (isByRef(param_ty, zcu)) { + args.appendAssumeCapacity(arg_ptr); + } else { + args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, "")); + } }, } } - } + + break :implicit_args .{ ret_ptr, err_ret_trace }; + }; const file, const subprogram = if (!wip.strip) debug_info: { const file = try o.getDebugFile(file_scope); @@ -1432,7 +1426,7 @@ pub const Object = struct { const subprogram = try o.builder.debugSubprogram( file, try o.builder.metadataString(nav.name.toSlice(ip)), - try o.builder.metadataStringFromStrtabString(function_index.name(&o.builder)), + try o.builder.metadataString(nav.fqn.toSlice(ip)), line_number, line_number + func.lbrace_line, debug_decl_type, @@ -1449,7 +1443,7 @@ pub const Object = struct { }, o.debug_compile_unit.unwrap().?, ); - function_index.setSubprogram(subprogram, &o.builder); + llvm_function.setSubprogram(subprogram, &o.builder); break :debug_info .{ file, subprogram }; } else .{undefined} ** 2; @@ -1466,7 +1460,7 @@ pub const Object = struct { const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len}); const counters_variable = try o.builder.addVariable(anon_name, .void, .default); try o.used.append(gpa, counters_variable.toConst(&o.builder)); - counters_variable.setLinkage(.private, &o.builder); + counters_variable.ptrConst(&o.builder).global.setLinkage(.private, &o.builder); counters_variable.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); if (target.ofmt == .macho) { @@ -1524,7 +1518,7 @@ pub const Object = struct { _ = try attributes.removeFnAttr(.null_pointer_is_valid); } - function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); + llvm_function.setAttributes(try attributes.finish(&o.builder), &o.builder); if (fg.fuzz) |*f| { { @@ -1539,158 +1533,162 @@ pub const Object = struct { // Due to error "members of llvm.compiler.used must be named", this global needs a name. const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len}); const pcs_variable = try o.builder.addVariable(anon_name, array_llvm_ty, .default); - try o.used.append(gpa, pcs_variable.toConst(&o.builder)); - pcs_variable.setLinkage(.private, &o.builder); + try pcs_variable.setInitializer(init_val, &o.builder); pcs_variable.setMutability(.constant, &o.builder); + pcs_variable.setSection(switch (target.ofmt) { + .macho => try o.builder.string("__DATA,__sancov_pcs1"), + else => try o.builder.string("__sancov_pcs1"), + }, &o.builder); pcs_variable.setAlignment(Type.usize.abiAlignment(zcu).toLlvm(), &o.builder); - if (target.ofmt == .macho) { - pcs_variable.setSection(try o.builder.string("__DATA,__sancov_pcs1"), &o.builder); - } else { - pcs_variable.setSection(try o.builder.string("__sancov_pcs1"), &o.builder); - } - try pcs_variable.setInitializer(init_val, &o.builder); + const pcs_global = pcs_variable.ptrConst(&o.builder).global; + pcs_global.setLinkage(.private, &o.builder); + try o.used.append(gpa, pcs_global.toConst()); } try fg.wip.finish(); try o.flushTypePool(pt); } - pub fn updateNav(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void { + pub fn updateNav(o: *Object, pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) !void { const zcu = o.zcu; const ip = &zcu.intern_pool; + const comp = zcu.comp; + const gpa = comp.gpa; - const nav = ip.getNav(nav_index); + const nav = ip.getNav(nav_id); const resolved = nav.resolved.?; - const lib_name, const linkage, const visibility: Builder.Visibility, const is_dll_import, const init_val, const owner_nav = switch (ip.indexToKey(resolved.value)) { - else => .{ .none, .internal, .default, false, resolved.value, nav_index }, - .@"extern" => |e| .{ e.lib_name, e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import, .none, e.owner_nav }, + const opt_extern: ?InternPool.Key.Extern = switch (ip.indexToKey(resolved.value)) { + .@"extern" => |@"extern"| @"extern", + else => null, }; - const ty: Type = .fromInterned(nav.resolved.?.type); + const nav_ty: Type = .fromInterned(resolved.type); + const llvm_ty: Builder.Type = if (opt_extern != null) ty: { + // We *must* lower this declaration no matter what. If it has a type we can't actually + // represent (because it doesn't have runtime bits), we instead lower as the zero-size + // type `[0 x i8]`. I don't think the type on an extern declaration actually does much + // anyway. + if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty); + break :ty try o.builder.arrayType(0, .i8); + } else if (nav_ty.hasRuntimeBits(zcu)) ty: { + break :ty try o.lowerType(nav_ty); + } else { + // This is a non-extern zero-bit `Nav`---we're not interested in it. + // TODO: we might need to rethink this a little under incremental compilation. If a + // declaration becomes zero-bit, we can't just leave its old value there, because it + // might now be ill-formed. + return; + }; + + const gop = try o.nav_map.getOrPut(gpa, nav_id); + if (!gop.found_existing) { + errdefer assert(o.nav_map.remove(nav_id)); + // First time lowering this NAV! Create a fresh global. + const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip)); + gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{ + .type = .void, // placeholder; populated below + .kind = .{ .alias = .none }, // placeholder; populated below + }); + } + const llvm_global = gop.value_ptr.*; + + llvm_global.ptr(&o.builder).type = llvm_ty; + llvm_global.ptr(&o.builder).addr_space = toLlvmAddressSpace(resolved.@"addrspace", zcu.getTarget()); - if (linkage != .internal and ip.isFunctionType(ty.toIntern())) { - const function_index = try o.resolveLlvmFunction(owner_nav); - // Add parameter attributes which weren't set by `resolveLlvmFunction` - const fn_info = zcu.typeToFunc(ty).?; - var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); - defer attributes.deinit(&o.builder); - var it = iterateParamTypes(o, fn_info); - if (firstParamSRet(fn_info, zcu, zcu.getTarget())) it.llvm_index += 1; - if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) it.llvm_index += 1; - while (try it.next()) |lowering| switch (lowering) { - .byval => { - const param_index = it.zig_index - 1; - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]); - if (!isByRef(param_ty, zcu)) { - try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); + if (opt_extern) |@"extern"| { + const name = name: { + const name_slice = nav.name.toSlice(ip); + if (zcu.getTarget().cpu.arch.isWasm() and nav_ty.zigTypeTag(zcu) == .@"fn") { + if (@"extern".lib_name.toSlice(ip)) |lib_name_slice| { + if (!std.mem.eql(u8, lib_name_slice, "c")) { + break :name try o.builder.strtabStringFmt("{s}|{s}", .{ name_slice, lib_name_slice }); + } } - }, - .byref => { - const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); - const param_llvm_ty = try o.lowerType(param_ty); - const alignment = param_ty.abiAlignment(zcu); - try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty); - }, - .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), - // No attributes needed for these. - .no_bits, - .abi_sized_int, - .multiple_llvm_types, - .float_array, - .i32_array, - .i64_array, - => continue, - - .slice => unreachable, // extern functions do not support slice types. + } + break :name try o.builder.strtabString(name_slice); + }; + if (o.builder.getGlobal(name)) |other_global| { + if (other_global != llvm_global) { + // Another global already has this name; just use it in place of this global. + try llvm_global.replace(other_global, &o.builder); + return; + } + } + try llvm_global.rename(name, &o.builder); + llvm_global.ptr(&o.builder).unnamed_addr = .default; + llvm_global.ptr(&o.builder).dll_storage_class = switch (@"extern".is_dll_import) { + true => .dllimport, + false => .default, + }; + llvm_global.ptr(&o.builder).linkage = switch (@"extern".linkage) { + .internal => if (o.builder.strip) .private else .internal, + .strong => .external, + .weak => .extern_weak, + .link_once => unreachable, }; - function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); + llvm_global.ptr(&o.builder).visibility = .fromSymbolVisibility(@"extern".visibility); } else { - const variable_index = try o.resolveGlobalNav(nav_index); - variable_index.setAlignment(zcu.navAlignment(nav_index).toLlvm(), &o.builder); - if (resolved.@"linksection".toSlice(ip)) |section| - variable_index.setSection(try o.builder.string(section), &o.builder); - if (resolved.@"const") variable_index.setMutability(.constant, &o.builder); - try variable_index.setInitializer(switch (init_val) { - .none => .no_init, - else => try o.lowerValue(init_val), - }, &o.builder); - variable_index.setVisibility(visibility, &o.builder); + llvm_global.ptr(&o.builder).linkage = if (o.builder.strip) .private else .internal; + llvm_global.ptr(&o.builder).visibility = .default; + llvm_global.ptr(&o.builder).dll_storage_class = .default; + llvm_global.ptr(&o.builder).unnamed_addr = .unnamed_addr; + } - const file_scope = zcu.navFileScopeIndex(nav_index); + const llvm_align = switch (resolved.@"align") { + .none => nav_ty.abiAlignment(zcu).toLlvm(), + else => |a| a.toLlvm(), + }; + const llvm_section: Builder.String = if (resolved.@"linksection".toSlice(ip)) |section| s: { + break :s try o.builder.string(section); + } else .none; + + // Actual function bodies with AIR go through `updateFunc` instead, so the only functions we + // can see are extern functions or other comptime function body values (e.g. undefined). Of + // these, only extern functions need to be lowered to LLVM functions. + if (opt_extern != null and nav_ty.zigTypeTag(zcu) == .@"fn" and nav_ty.fnHasRuntimeBits(zcu)) { + const llvm_function: Builder.Function.Index = switch (llvm_global.ptrConst(&o.builder).kind) { + .function => |function| function, // re-use existing `Builder.Function` + .replaced, .alias, .variable => try llvm_global.toNewFunction(&o.builder), + }; + llvm_function.setAlignment(llvm_align, &o.builder); + llvm_function.setSection(llvm_section, &o.builder); + try o.addLlvmFunctionAttributes(pt, nav_id, llvm_function); + } else { + const file_scope = nav.srcInst(ip).resolveFile(ip); const mod = zcu.fileByIndex(file_scope).mod.?; - if (resolved.@"threadlocal" and !mod.single_threaded) - variable_index.setThreadLocal(.generaldynamic, &o.builder); - const line_number = zcu.navSrcLine(nav_index) + 1; + const llvm_variable: Builder.Variable.Index = switch (llvm_global.ptrConst(&o.builder).kind) { + .variable => |variable| variable, // re-use existing `Builder.Variable` + .replaced, .alias, .function => try llvm_global.toNewVariable(&o.builder), + }; + llvm_variable.setAlignment(llvm_align, &o.builder); + llvm_variable.setSection(llvm_section, &o.builder); + llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder); + try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value), &o.builder); + llvm_variable.setThreadLocal(tl: { + if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic; + break :tl .default; + }, &o.builder); if (!mod.strip) { const debug_file = try o.getDebugFile(file_scope); - - const debug_global_var = try o.builder.debugGlobalVar( - try o.builder.metadataString(nav.name.toSlice(ip)), // Name - try o.builder.metadataStringFromStrtabString(variable_index.name(&o.builder)), // Linkage name - debug_file, // File - debug_file, // Scope - line_number, - try o.getDebugType(pt, ty), - variable_index, - .{ .local = linkage == .internal }, + const debug_global_var_expr = try o.builder.debugGlobalVarExpression( + try o.builder.debugGlobalVar( + try o.builder.metadataString(nav.name.toSlice(ip)), // Name + try o.builder.metadataString(nav.fqn.toSlice(ip)), // Linkage name + debug_file, // File + debug_file, // Scope + zcu.navSrcLine(nav_id) + 1, + try o.getDebugType(pt, nav_ty), + llvm_variable, + .{ .local = llvm_global.ptrConst(&o.builder).linkage == .internal }, + ), + try o.builder.debugExpression(&.{}), ); - - const debug_expression = try o.builder.debugExpression(&.{}); - - const debug_global_var_expression = try o.builder.debugGlobalVarExpression( - debug_global_var, - debug_expression, - ); - - variable_index.setGlobalVariableExpression(debug_global_var_expression, &o.builder); - try o.debug_globals.append(o.gpa, debug_global_var_expression); + llvm_variable.setGlobalVariableExpression(debug_global_var_expr, &o.builder); + try o.debug_globals.append(o.gpa, debug_global_var_expr); } } - - switch (linkage) { - .internal => {}, - .strong, .weak => { - const global_index = o.nav_map.get(nav_index).?; - - const decl_name = decl_name: { - if (zcu.getTarget().cpu.arch.isWasm() and ty.zigTypeTag(zcu) == .@"fn") { - if (lib_name.toSlice(ip)) |lib_name_slice| { - if (!std.mem.eql(u8, lib_name_slice, "c")) { - break :decl_name try o.builder.strtabStringFmt("{f}|{s}", .{ nav.name.fmt(ip), lib_name_slice }); - } - } - } - break :decl_name try o.builder.strtabString(nav.name.toSlice(ip)); - }; - - if (o.builder.getGlobal(decl_name)) |other_global| { - if (other_global != global_index) { - // Another global already has this name; just use it in place of this global. - try global_index.replace(other_global, &o.builder); - return; - } - } - - try global_index.rename(decl_name, &o.builder); - global_index.setUnnamedAddr(.default, &o.builder); - if (is_dll_import) { - global_index.setDllStorageClass(.dllimport, &o.builder); - } else if (zcu.comp.config.dll_export_fns) { - global_index.setDllStorageClass(.default, &o.builder); - } - - global_index.setLinkage(switch (linkage) { - .internal => unreachable, - .strong => .external, - .weak => .extern_weak, - .link_once => unreachable, - }, &o.builder); - global_index.setVisibility(visibility, &o.builder); - }, - .link_once => unreachable, - } } fn flushTypePool(o: *Object, pt: Zcu.PerThread) Allocator.Error!void { @@ -1698,18 +1696,43 @@ pub const Object = struct { } pub fn updateExports( - self: *Object, + o: *Object, exported: Zcu.Exported, export_indices: []const Zcu.Export.Index, ) link.File.UpdateExportsError!void { - const zcu = self.zcu; - const nav_index = switch (exported) { - .nav => |nav| nav, - .uav => |uav| return updateExportedValue(self, uav, export_indices), + const zcu = o.zcu; + const ip = &zcu.intern_pool; + const ty: Type, const llvm_ptr: Builder.Constant = switch (exported) { + .nav => |nav| exp: { + const nav_ty: Type = .fromInterned(ip.getNav(nav).resolved.?.type); + const nav_ref = try o.lowerNavRef(nav); + break :exp .{ nav_ty, nav_ref }; + }, + .uav => |uav| exp: { + const uav_ty = Value.fromInterned(uav).typeOf(zcu); + const uav_ref = try o.lowerUavRef( + uav, + uav_ty.abiAlignment(zcu), + target_util.defaultAddressSpace(zcu.getTarget(), .global_constant), + ); + break :exp .{ uav_ty, uav_ref }; + }, }; - const ip = &zcu.intern_pool; - const global_index = self.nav_map.get(nav_index).?; + switch (llvm_ptr.unwrap()) { + .global => |global| return o.updateExportedGlobal(global, ty, export_indices), + .constant => @panic("LLVM TODO: export zero-bit value"), + } + } + + fn updateExportedGlobal( + o: *Object, + global_index: Builder.Global.Index, + ty: Type, + export_indices: []const Zcu.Export.Index, + ) link.File.UpdateExportsError!void { + const zcu = o.zcu; const comp = zcu.comp; + const ip = &zcu.intern_pool; // If we're on COFF and linking with LLD, the linker cares about our exports to determine the subsystem in use. coff_export_flags: { @@ -1719,7 +1742,7 @@ pub const Object = struct { .elf, .wasm => break :coff_export_flags, .coff => |*coff| coff, }; - if (!ip.isFunctionType(ip.getNav(nav_index).resolved.?.type)) break :coff_export_flags; + if (ty.zigTypeTag(zcu) != .@"fn") break :coff_export_flags; const flags = &coff.lld_export_flags; for (export_indices) |export_index| { const name = export_index.ptr(zcu).opts.name; @@ -1732,152 +1755,88 @@ pub const Object = struct { } } - if (export_indices.len != 0) { - return updateExportedGlobal(self, zcu, global_index, export_indices); - } else { - const fqn = try self.builder.strtabString(ip.getNav(nav_index).fqn.toSlice(ip)); - try global_index.rename(fqn, &self.builder); - global_index.setLinkage(if (self.builder.strip) .private else .internal, &self.builder); - if (comp.config.dll_export_fns) - global_index.setDllStorageClass(.default, &self.builder); - global_index.setUnnamedAddr(.unnamed_addr, &self.builder); + // If the first export specifies a linksection, set the exported variable's section to that + // one. This is kind of a hack because `std.builtin.ExportOptions.section` doesn't actually + // make much sense: the linksection should be associated with the declaration itself rather + // than some particular symbol it is exported as! + if (export_indices[0].ptr(zcu).opts.section.toSlice(ip)) |section_slice| { + const variable = &global_index.ptrConst(&o.builder).kind.variable; + variable.setSection(try o.builder.string(section_slice), &o.builder); } - } - fn updateExportedValue( - o: *Object, - exported_value: InternPool.Index, - export_indices: []const Zcu.Export.Index, - ) link.File.UpdateExportsError!void { - const zcu = o.zcu; - const gpa = zcu.gpa; - const ip = &zcu.intern_pool; - const main_exp_name = try o.builder.strtabString(export_indices[0].ptr(zcu).opts.name.toSlice(ip)); - const global_index = i: { - const gop = try o.uav_map.getOrPut(gpa, exported_value); - if (gop.found_existing) { - const global_index = gop.value_ptr.*; - try global_index.rename(main_exp_name, &o.builder); - break :i global_index; - } - const llvm_addr_space = toLlvmAddressSpace(.generic, zcu.getTarget()); - const variable_index = try o.builder.addVariable( - main_exp_name, - try o.lowerType(.fromInterned(ip.typeOf(exported_value))), - llvm_addr_space, - ); - const global_index = variable_index.ptrConst(&o.builder).global; - gop.value_ptr.* = global_index; - // This line invalidates `gop`. - const init_val = try o.lowerValue(exported_value); - try variable_index.setInitializer(init_val, &o.builder); - break :i global_index; - }; - return updateExportedGlobal(o, zcu, global_index, export_indices); - } + const llvm_global_ty = global_index.typeOf(&o.builder); - fn updateExportedGlobal( - o: *Object, - zcu: *Zcu, - global_index: Builder.Global.Index, - export_indices: []const Zcu.Export.Index, - ) link.File.UpdateExportsError!void { - const comp = zcu.comp; - const ip = &zcu.intern_pool; - const first_export = export_indices[0].ptr(zcu); + // All exports are represented as aliases to the original global. - // We will rename this global to have a name matching `first_export`. - // Successive exports become aliases. - // If the first export name already exists, then there is a corresponding - // extern global - we replace it with this global. - const first_exp_name = try o.builder.strtabString(first_export.opts.name.toSlice(ip)); - if (o.builder.getGlobal(first_exp_name)) |other_global| replace: { - if (other_global.toConst().getBase(&o.builder) == global_index.toConst().getBase(&o.builder)) { - break :replace; // this global already has the name we want - } - try global_index.takeName(other_global, &o.builder); - try other_global.replace(global_index, &o.builder); - // Problem: now we need to replace in the decl_map that - // the extern decl index points to this new global. However we don't - // know the decl index. - // Even if we did, a future incremental update to the extern would then - // treat the LLVM global as an extern rather than an export, so it would - // need a way to check that. - // This is a TODO that needs to be solved when making - // the LLVM backend support incremental compilation. - } else { - try global_index.rename(first_exp_name, &o.builder); - } - - global_index.setUnnamedAddr(.default, &o.builder); - if (comp.config.dll_export_fns and first_export.opts.visibility != .hidden) - global_index.setDllStorageClass(.dllexport, &o.builder); - global_index.setLinkage(switch (first_export.opts.linkage) { - .internal => unreachable, - .strong => .external, - .weak => .weak_odr, - .link_once => .linkonce_odr, - }, &o.builder); - global_index.setVisibility(switch (first_export.opts.visibility) { - .default => .default, - .hidden => .hidden, - .protected => .protected, - }, &o.builder); - if (first_export.opts.section.toSlice(ip)) |section| - switch (global_index.ptrConst(&o.builder).kind) { - .variable => |impl_index| impl_index.setSection( - try o.builder.string(section), - &o.builder, - ), - .function => unreachable, - .alias => unreachable, - .replaced => unreachable, - }; + // TODO: we currently do not delete old exports. To do that we'll need to track which + // globals actually *are* exports. - // If a Decl is exported more than one time (which is rare), - // we add aliases for all but the first export. - // TODO LLVM C API does not support deleting aliases. - // The planned solution to this is https://github.com/ziglang/zig/issues/13265 - // Until then we iterate over existing aliases and make them point - // to the correct decl, or otherwise add a new alias. Old aliases are leaked. - for (export_indices[1..]) |export_idx| { + for (export_indices) |export_idx| { const exp = export_idx.ptr(zcu); const exp_name = try o.builder.strtabString(exp.opts.name.toSlice(ip)); - if (o.builder.getGlobal(exp_name)) |global| { - switch (global.ptrConst(&o.builder).kind) { + + // Our goal is to make an alias with the name `exp_name`, but if that name is already + // taken by some existing global, we need to figure out what to do with that existing + // global. + // + // The name, aliasee, and type will be set within this block. Other properties of the + // alias will be set below. + const alias_global: Builder.Global.Index = global: { + const existing_global = o.builder.getGlobal(exp_name) orelse { + // There is no existing global with this name, so make a new alias. + const alias = try o.builder.addAlias( + exp_name, + llvm_global_ty, + .default, + global_index.toConst(), + ); + break :global alias.ptrConst(&o.builder).global; + }; + // There is an existing global with this name, so we can't just create an alias. We + // need to figure out what to do with the existing global instead. + switch (existing_global.ptrConst(&o.builder).kind) { .alias => |alias| { + // We can just repurpose the existing alias. alias.setAliasee(global_index.toConst(), &o.builder); - continue; + alias.ptrConst(&o.builder).global.ptr(&o.builder).type = global_index.typeOf(&o.builder); + break :global existing_global; }, .variable, .function => { - // This existing global is an `extern` corresponding to this export. - // Replace it with the global being exported. - // This existing global must be replaced with the alias. - try global.rename(.empty, &o.builder); - try global.replace(global_index, &o.builder); + // This must be an extern, which is no good to us---we need an alias. The + // extern should refer to the value we're exporting, so replace it with the + // exported value. That will free up the name for us to create a new alias. + // We need to make a new global which is an alias. Replace this existing one + // with the target global, making the name available and fixing references + // to this global to point to the target. + try existing_global.replace(global_index, &o.builder); + // The name is now free, so create an alias. + const alias = try o.builder.addAlias( + exp_name, + llvm_global_ty, + .default, + global_index.toConst(), + ); + break :global alias.ptrConst(&o.builder).global; }, - .replaced => unreachable, + .replaced => unreachable, // a replaced global would have lost the name `exp_name` } - } - const alias_index = try o.builder.addAlias( - .empty, - global_index.typeOf(&o.builder), - .default, - global_index.toConst(), - ); - try alias_index.rename(exp_name, &o.builder); + }; - const alias_global_index = alias_index.ptrConst(&o.builder).global; - alias_global_index.setUnnamedAddr(.default, &o.builder); - if (comp.config.dll_export_fns and first_export.opts.visibility != .hidden) - alias_global_index.setDllStorageClass(.dllexport, &o.builder); - alias_global_index.setLinkage(switch (first_export.opts.linkage) { - .internal => unreachable, + // Now for a bit of setup which + + // We need the alias to *not* be `unnamed_addr` to ensure that the alias address equals + // the address of the original global. + alias_global.setUnnamedAddr(.default, &o.builder); + + if (comp.config.dll_export_fns and exp.opts.visibility != .hidden) + alias_global.setDllStorageClass(.dllexport, &o.builder); + alias_global.setLinkage(switch (exp.opts.linkage) { + .internal => if (o.builder.strip) .private else .internal, // we still did useful work in replacing an existing symbol if there was one .strong => .external, .weak => .weak_odr, .link_once => .linkonce_odr, }, &o.builder); - alias_global_index.setVisibility(switch (first_export.opts.visibility) { + alias_global.setVisibility(switch (exp.opts.visibility) { .default => .default, .hidden => .hidden, .protected => .protected, @@ -1940,7 +1899,12 @@ pub const Object = struct { assert(val != .anyerror_type); const fwd_ref = o.debug_types.items[@intFromEnum(index)]; const name_str = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)}); - const debug_incomplete_type = try o.builder.debugSignedType(name_str, 0); + // If `ty` is a function, use a dummy *function* type to prevent existing debug + // subprograms from becoming ill-formed. + const debug_incomplete_type = switch (ty.zigTypeTag(zcu)) { + .@"fn" => try o.builder.debugSubroutineType(null), + else => try o.builder.debugSignedType(name_str, 0), + }; o.builder.resolveDebugForwardReference(fwd_ref, debug_incomplete_type); } } @@ -2269,7 +2233,9 @@ pub const Object = struct { }, .@"fn" => { if (!ty.fnHasRuntimeBits(zcu)) { - return o.builder.debugSignedType(name, 0); + // Use a dummy *function* type to prevent existing debug subprograms from + // becoming ill-formed. + return o.builder.debugSubroutineType(null); } const fn_info = zcu.typeToFunc(ty).?; @@ -2716,75 +2682,38 @@ pub const Object = struct { return o.getDebugType(pt, .fromInterned(namespace.owner_type)); } - /// If the llvm function does not exist, create it. - /// Note that this can be called before the function's semantic analysis has - /// completed, so if any attributes rely on that, they must be done in updateFunc, not here. - pub fn resolveLlvmFunction( + /// Sets the attributes and callconv of the given `Builder.Function`, which corresponds to the + /// given `Nav` (which is a function). + fn addLlvmFunctionAttributes( o: *Object, - nav_index: InternPool.Nav.Index, - ) Allocator.Error!Builder.Function.Index { + pt: Zcu.PerThread, + nav_id: InternPool.Nav.Index, + function_index: Builder.Function.Index, + ) Allocator.Error!void { const zcu = o.zcu; const ip = &zcu.intern_pool; - const gpa = o.gpa; - const nav = ip.getNav(nav_index); - const owner_mod = zcu.navFileScope(nav_index).mod.?; + const nav = ip.getNav(nav_id); + const owner_mod = zcu.navFileScope(nav_id).mod.?; const ty: Type = .fromInterned(nav.resolved.?.type); - const gop = try o.nav_map.getOrPut(gpa, nav_index); - if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.function; const fn_info = zcu.typeToFunc(ty).?; const target = &owner_mod.resolved_target.result; - const is_extern, const lib_name = if (nav.getExtern(ip)) |@"extern"| - .{ true, @"extern".lib_name } - else - .{ false, .none }; - const function_index = try o.builder.addFunction( - try o.lowerType(ty), - try o.builder.strtabString((if (is_extern) nav.name else nav.fqn).toSlice(ip)), - toLlvmAddressSpace(nav.resolved.?.@"addrspace", target), - ); - gop.value_ptr.* = function_index.ptrConst(&o.builder).global; - var attributes: Builder.FunctionAttributes.Wip = .{}; defer attributes.deinit(&o.builder); - if (!is_extern) { - function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); - function_index.setUnnamedAddr(.unnamed_addr, &o.builder); - } else { - if (target.cpu.arch.isWasm()) { - try attributes.addFnAttr(.{ .string = .{ - .kind = try o.builder.string("wasm-import-name"), - .value = try o.builder.string(nav.name.toSlice(ip)), + if (target.cpu.arch.isWasm()) if (nav.getExtern(ip)) |@"extern"| { + try attributes.addFnAttr(.{ .string = .{ + .kind = try o.builder.string("wasm-import-name"), + .value = try o.builder.string(nav.name.toSlice(ip)), + } }, &o.builder); + if (@"extern".lib_name.toSlice(ip)) |lib_name_slice| { + if (!std.mem.eql(u8, lib_name_slice, "c")) try attributes.addFnAttr(.{ .string = .{ + .kind = try o.builder.string("wasm-import-module"), + .value = try o.builder.string(lib_name_slice), } }, &o.builder); - if (lib_name.toSlice(ip)) |lib_name_slice| { - if (!std.mem.eql(u8, lib_name_slice, "c")) try attributes.addFnAttr(.{ .string = .{ - .kind = try o.builder.string("wasm-import-module"), - .value = try o.builder.string(lib_name_slice), - } }, &o.builder); - } } - } - - var llvm_arg_i: u32 = 0; - if (firstParamSRet(fn_info, zcu, target)) { - // Sret pointers must not be address 0 - try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); - try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder); - - const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type)); - try attributes.addParamAttr(llvm_arg_i, .{ .sret = raw_llvm_ret_ty }, &o.builder); - - llvm_arg_i += 1; - } - - const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; - - if (err_return_tracing) { - try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder); - llvm_arg_i += 1; - } + }; if (fn_info.cc == .async) { @panic("TODO: LLVM backend lower async function"); @@ -2859,9 +2788,6 @@ pub const Object = struct { } } - if (nav.resolved.?.@"align" != .none) - function_index.setAlignment(nav.resolved.?.@"align".toLlvm(), &o.builder); - // Function attributes that are independent of analysis results of the function body. try o.addCommonFnAttributes( &attributes, @@ -2877,8 +2803,71 @@ pub const Object = struct { if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder); + var it = iterateParamTypes(o, fn_info); + if (firstParamSRet(fn_info, zcu, target)) { + // Sret pointers must not be address 0 + try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder); + try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder); + + const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type)); + try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder); + it.llvm_index += 1; + } + const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing; + if (err_return_tracing) { + try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder); + it.llvm_index += 1; + } + while (try it.next()) |lowering| switch (lowering) { + .byval => { + const param_index = it.zig_index - 1; + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[param_index]); + if (!isByRef(param_ty, zcu)) { + try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1); + } + }, + .byref => { + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const param_llvm_ty = try o.lowerType(param_ty); + const alignment = param_ty.abiAlignment(zcu); + try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty); + }, + .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), + .slice => { + const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]); + const ptr_info = param_ty.ptrInfo(zcu); + const llvm_ptr_index = it.llvm_index - 2; + if (std.math.cast(u5, it.zig_index - 1)) |i| { + if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) { + try attributes.addParamAttr(llvm_ptr_index, .@"noalias", &o.builder); + } + } + if (param_ty.zigTypeTag(zcu) != .optional and + !ptr_info.flags.is_allowzero and + ptr_info.flags.address_space == .generic) + { + try attributes.addParamAttr(llvm_ptr_index, .nonnull, &o.builder); + } + if (ptr_info.flags.is_const) { + try attributes.addParamAttr(llvm_ptr_index, .readonly, &o.builder); + } + const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) { + else => |a| .wrap(a.toLlvm()), + .none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)), + }; + try attributes.addParamAttr(llvm_ptr_index, .{ .@"align" = elem_align }, &o.builder); + }, + // No attributes needed for these. + .no_bits, + .abi_sized_int, + .multiple_llvm_types, + .float_array, + .i32_array, + .i64_array, + => continue, + }; + function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); - return function_index; } fn addCommonFnAttributes( @@ -2951,97 +2940,6 @@ pub const Object = struct { } } - fn resolveGlobalUav( - o: *Object, - uav: InternPool.Index, - llvm_addr_space: Builder.AddrSpace, - alignment: InternPool.Alignment, - ) Allocator.Error!Builder.Variable.Index { - assert(alignment != .none); - // TODO: Add address space to the anon_decl_map - const gop = try o.uav_map.getOrPut(o.gpa, uav); - if (gop.found_existing) { - // Keep the greater of the two alignments. - const variable_index = gop.value_ptr.ptr(&o.builder).kind.variable; - const old_alignment = InternPool.Alignment.fromLlvm(variable_index.getAlignment(&o.builder)); - const max_alignment = old_alignment.maxStrict(alignment); - variable_index.setAlignment(max_alignment.toLlvm(), &o.builder); - return variable_index; - } - errdefer assert(o.uav_map.remove(uav)); - - const zcu = o.zcu; - const decl_ty = zcu.intern_pool.typeOf(uav); - - const variable_index = try o.builder.addVariable( - try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav)}), - try o.lowerType(.fromInterned(decl_ty)), - llvm_addr_space, - ); - gop.value_ptr.* = variable_index.ptrConst(&o.builder).global; - - try variable_index.setInitializer(try o.lowerValue(uav), &o.builder); - variable_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); - variable_index.setMutability(.constant, &o.builder); - variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); - variable_index.setAlignment(alignment.toLlvm(), &o.builder); - return variable_index; - } - - fn resolveGlobalNav( - o: *Object, - nav_index: InternPool.Nav.Index, - ) Allocator.Error!Builder.Variable.Index { - const gop = try o.nav_map.getOrPut(o.gpa, nav_index); - if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.variable; - errdefer assert(o.nav_map.remove(nav_index)); - - const zcu = o.zcu; - const ip = &zcu.intern_pool; - const nav = ip.getNav(nav_index); - const linkage: std.builtin.GlobalLinkage, const visibility: Builder.Visibility, const is_dll_import: bool = switch (nav.resolved.?.value) { - .none => .{ .internal, .default, false }, // this is a source declaration which is *not* marked `extern` - else => |val| switch (ip.indexToKey(val)) { - else => .{ .internal, .default, false }, - .@"extern" => |e| .{ e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import }, - }, - }; - - const variable_index = try o.builder.addVariable( - try o.builder.strtabString(switch (linkage) { - .internal => nav.fqn, - .strong, .weak => nav.name, - .link_once => unreachable, - }.toSlice(ip)), - try o.lowerType(.fromInterned(nav.resolved.?.type)), - toLlvmGlobalAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget()), - ); - gop.value_ptr.* = variable_index.ptrConst(&o.builder).global; - - // This is needed for declarations created by `@extern`. - switch (linkage) { - .internal => { - variable_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); - variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); - }, - .strong, .weak => { - variable_index.setLinkage(switch (linkage) { - .internal => unreachable, - .strong => .external, - .weak => .extern_weak, - .link_once => unreachable, - }, &o.builder); - variable_index.setUnnamedAddr(.default, &o.builder); - if (nav.resolved.?.@"threadlocal" and !zcu.navFileScope(nav_index).mod.?.single_threaded) - variable_index.setThreadLocal(.generaldynamic, &o.builder); - if (is_dll_import) variable_index.setDllStorageClass(.dllimport, &o.builder); - }, - .link_once => unreachable, - } - variable_index.setVisibility(visibility, &o.builder); - return variable_index; - } - pub fn errorIntType(o: *Object) Allocator.Error!Builder.Type { return o.builder.intType(o.zcu.errorSetBits()); } @@ -3051,7 +2949,7 @@ pub const Object = struct { const target = zcu.getTarget(); const ip = &zcu.intern_pool; return switch (t.toIntern()) { - .u0_type, .i0_type => unreachable, + .u0_type, .i0_type => unreachable, // no runtime bits inline .u1_type, .u8_type, .i8_type, @@ -3100,18 +2998,18 @@ pub const Object = struct { return .i8; }, .bool_type => .i1, - .void_type => .void, - .type_type => unreachable, .anyerror_type => try o.errorIntType(), - .comptime_int_type, - .comptime_float_type, - .noreturn_type, - => unreachable, + .void_type => unreachable, // no runtime bits + .type_type => unreachable, // no runtime bits + .comptime_int_type => unreachable, // no runtime bits + .comptime_float_type => unreachable, // no runtime bits + .noreturn_type => unreachable, // no runtime bits + .null_type => unreachable, // no runtime bits + .undefined_type => unreachable, // no runtime bits + .enum_literal_type => unreachable, // no runtime bits + .optional_noreturn_type => unreachable, // no runtime bits + .empty_tuple_type => unreachable, // no runtime bits .anyframe_type => @panic("TODO implement lowerType for AnyFrame types"), - .null_type, - .undefined_type, - .enum_literal_type, - => unreachable, .ptr_usize_type, .ptr_const_comptime_int_type, .manyptr_u8_type, @@ -3121,13 +3019,10 @@ pub const Object = struct { .slice_const_u8_type, .slice_const_u8_sentinel_0_type, => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize) }), - .optional_noreturn_type => unreachable, .anyerror_void_error_union_type, .adhoc_inferred_error_set_type, => try o.errorIntType(), - .generic_poison_type, - .empty_tuple_type, - => unreachable, + .generic_poison_type => unreachable, // values, not types .undef, .undef_bool, @@ -3176,7 +3071,11 @@ pub const Object = struct { ), .opt_type => |child_ty| { // Must stay in sync with `opt_payload` logic in `lowerPtr`. - if (!Type.fromInterned(child_ty).hasRuntimeBits(zcu)) return .i8; + switch (Type.fromInterned(child_ty).classify(zcu)) { + .no_possible_value, .fully_comptime => unreachable, + .one_possible_value => return .i8, + .runtime, .partially_comptime => {}, + } const payload_ty = try o.lowerType(.fromInterned(child_ty)); if (t.optionalReprIsPayload(zcu)) return payload_ty; @@ -3198,8 +3097,13 @@ pub const Object = struct { // Must stay in sync with `codegen.errUnionPayloadOffset`. // See logic in `lowerPtr`. const error_type = try o.errorIntType(); - if (!Type.fromInterned(error_union_type.payload_type).hasRuntimeBits(zcu)) - return error_type; + + switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) { + .fully_comptime => unreachable, + .no_possible_value, .one_possible_value => return error_type, + .runtime, .partially_comptime => {}, + } + const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type)); const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu); @@ -3245,6 +3149,8 @@ pub const Object = struct { return int_ty; } + assert(struct_type.size > 0); + var llvm_field_types: std.ArrayList(Builder.Type) = .empty; defer llvm_field_types.deinit(o.gpa); // Although we can estimate how much capacity to add, these cannot be @@ -3311,7 +3217,7 @@ pub const Object = struct { comptime assert(struct_layout_version == 2); var offset: u64 = 0; - var big_align: InternPool.Alignment = .none; + var big_align: InternPool.Alignment = .@"1"; for ( tuple_type.types.get(ip), @@ -3345,6 +3251,7 @@ pub const Object = struct { try o.builder.arrayType(padding_len, .i8), ); } + assert(offset > 0); return o.builder.structType(.normal, llvm_field_types.items); }, .union_type => { @@ -3358,6 +3265,8 @@ pub const Object = struct { return int_ty; } + assert(union_obj.size > 0); + const layout = Type.getUnionLayout(union_obj, zcu); if (layout.payload_size == 0) { @@ -3421,15 +3330,9 @@ pub const Object = struct { ); return ty; }, - .opaque_type => { - const gop = try o.type_map.getOrPut(o.gpa, t.toIntern()); - if (!gop.found_existing) { - gop.value_ptr.* = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip))); - } - return gop.value_ptr.*; - }, + .opaque_type => unreachable, // no runtime bits .enum_type => try o.lowerType(t.intTagType(zcu)), - .func_type => |func_type| try o.lowerFnType(func_type), + .func_type => |func_type| try o.lowerFnType(t, func_type), .error_set_type, .inferred_error_set_type => try o.errorIntType(), // values, not types .undef, @@ -3455,10 +3358,13 @@ pub const Object = struct { }; } - fn lowerFnType(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { + fn lowerFnType(o: *Object, fn_ty: Type, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type { const zcu = o.zcu; const ip = &zcu.intern_pool; const target = zcu.getTarget(); + + assert(fn_ty.fnHasRuntimeBits(zcu)); + const ret_ty = try lowerFnRetTy(o, fn_info); var llvm_params: std.ArrayList(Builder.Type) = .empty; @@ -3526,15 +3432,12 @@ pub const Object = struct { const ip = &zcu.intern_pool; const target = zcu.getTarget(); - const val = Value.fromInterned(arg_val); + const val: Value = .fromInterned(arg_val); const val_key = ip.indexToKey(val.toIntern()); - if (val.isUndef(zcu)) { - return o.builder.undefConst(try o.lowerType(.fromInterned(val_key.typeOf()))); - } - const ty: Type = .fromInterned(val_key.typeOf()); ty.assertHasLayout(zcu); + assert(ty.hasRuntimeBits(zcu)); return switch (val_key) { .int_type, @@ -3555,7 +3458,7 @@ pub const Object = struct { .inferred_error_set_type, => unreachable, // types, not values - .undef => unreachable, // handled above + .undef => return o.builder.undefConst(try o.lowerType(ty)), .simple_value => |simple_value| switch (simple_value) { .void => unreachable, // non-runtime value .null => unreachable, // non-runtime value @@ -3565,14 +3468,8 @@ pub const Object = struct { .true => .true, }, .enum_literal => unreachable, // non-runtime value - .@"extern" => |@"extern"| { - const function_index = try o.resolveLlvmFunction(@"extern".owner_nav); - return function_index.ptrConst(&o.builder).global.toConst(); - }, - .func => |func| { - const function_index = try o.resolveLlvmFunction(func.owner_nav); - return function_index.ptrConst(&o.builder).global.toConst(); - }, + .@"extern" => unreachable, // non-runtime value + .func => unreachable, // non-runtime value .int => { var bigint_space: Value.BigIntSpace = undefined; const bigint = val.toBigInt(&bigint_space, zcu); @@ -3815,7 +3712,7 @@ pub const Object = struct { comptime assert(struct_layout_version == 2); var llvm_index: usize = 0; var offset: u64 = 0; - var big_align: InternPool.Alignment = .none; + var big_align: InternPool.Alignment = .@"1"; var need_unnamed = false; for ( tuple.types.get(ip), @@ -4033,7 +3930,7 @@ pub const Object = struct { const offset: u64 = prev_offset + ptr.byte_offset; return switch (ptr.base_addr) { .nav => |nav| { - const base_ptr = try o.lowerNavRefValue(nav); + const base_ptr = try o.lowerNavRef(nav); return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{ try o.builder.intConst(.i64, offset), }); @@ -4092,8 +3989,19 @@ pub const Object = struct { }; } - /// This logic is very similar to `lowerNavRefValue` but for anonymous declarations. - /// Maybe the logic could be unified. + pub fn lowerPtrToVoid( + o: *Object, + /// Must not be `.none`. + @"align": InternPool.Alignment, + @"addrspace": std.builtin.AddressSpace, + ) Allocator.Error!Builder.Constant { + const addr: u64 = @"align".toByteUnits().?; + const llvm_usize = try o.lowerType(.usize); + const llvm_addr = try o.builder.intConst(llvm_usize, addr); + const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget())); + return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty); + } + pub fn lowerUavRef( o: *Object, uav_val: InternPool.Index, @@ -4105,6 +4013,8 @@ pub const Object = struct { const zcu = o.zcu; const ip = &zcu.intern_pool; + const gpa = zcu.comp.gpa; + const uav_ty: Type = .fromInterned(ip.typeOf(uav_val)); switch (ip.indexToKey(uav_val)) { @@ -4118,63 +4028,63 @@ pub const Object = struct { } const llvm_addrspace = toLlvmAddressSpace(@"addrspace", zcu.getTarget()); - const llvm_global = (try o.resolveGlobalUav(uav_val, llvm_addrspace, @"align")).ptrConst(&o.builder).global; - return o.builder.convConst( - llvm_global.toConst(), - try o.builder.ptrType(llvm_addrspace), - ); + const gop = try o.uav_map.getOrPut(gpa, .{ .val = uav_val, .@"addrspace" = @"addrspace" }); + if (gop.found_existing) { + // Keep the greater of the two alignments. + const llvm_variable = gop.value_ptr.*; + const old_align: InternPool.Alignment = .fromLlvm(llvm_variable.getAlignment(&o.builder)); + llvm_variable.setAlignment(old_align.maxStrict(@"align").toLlvm(), &o.builder); + return llvm_variable.ptrConst(&o.builder).global.toConst(); + } + errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" })); + + const llvm_ty = try o.lowerType(uav_ty); + const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)}); + const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace); + gop.value_ptr.* = llvm_variable; + try llvm_variable.setInitializer(try o.lowerValue(uav_val), &o.builder); + llvm_variable.setMutability(.constant, &o.builder); + llvm_variable.setAlignment(@"align".toLlvm(), &o.builder); + const llvm_global = llvm_variable.ptrConst(&o.builder).global; + llvm_global.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); + llvm_global.setUnnamedAddr(.unnamed_addr, &o.builder); + return llvm_global.toConst(); } - pub fn lowerNavRefValue(o: *Object, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant { + pub fn lowerNavRef(o: *Object, nav_id: InternPool.Nav.Index) Allocator.Error!Builder.Constant { const zcu = o.zcu; const ip = &zcu.intern_pool; + const gpa = zcu.comp.gpa; - const nav = ip.getNav(nav_index); - + const nav = ip.getNav(nav_id); const nav_ty: Type = .fromInterned(nav.resolved.?.type); - - if (nav.getExtern(ip) == null and !nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) { - return o.lowerPtrToVoid(nav.resolved.?.@"align", nav.resolved.?.@"addrspace"); - } - - const llvm_global = if (nav_ty.zigTypeTag(zcu) == .@"fn") - (try o.resolveLlvmFunction(nav_index)).ptrConst(&o.builder).global - else - (try o.resolveGlobalNav(nav_index)).ptrConst(&o.builder).global; - - return try o.builder.convConst( - llvm_global.toConst(), - try o.builder.ptrType(toLlvmAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget())), - ); - } - - pub fn lowerPtrToVoid( - o: *Object, - @"align": InternPool.Alignment, - @"addrspace": std.builtin.AddressSpace, - ) Allocator.Error!Builder.Constant { - const target = o.zcu.getTarget(); - // Even though we are pointing at something which has zero bits (e.g. `void`), - // Pointers are defined to have bits. So we must return something here. - // The value cannot be undefined, because we use the `nonnull` annotation - // for non-optional pointers. We also need to respect the alignment, even though - // the address will never be dereferenced. - const int: u64 = @"align".toByteUnits() orelse - // Note that these 0xaa values are appropriate even in release-optimized builds - // because we need a well-defined value that is not null, and LLVM does not - // have an "undef_but_not_null" attribute. As an example, if this `alloc` AIR - // instruction is followed by a `wrap_optional`, it will return this value - // verbatim, and the result should test as non-null. - switch (target.ptrBitWidth()) { - 16 => 0xaaaa, - 32 => 0xaaaaaaaa, - 64 => 0xaaaaaaaa_aaaaaaaa, - else => unreachable, + if (!nav_ty.isRuntimeFnOrHasRuntimeBits(zcu) and nav.getExtern(ip) == null) { + const nav_align = switch (nav.resolved.?.@"align") { + .none => nav_ty.abiAlignment(zcu), + else => |a| a, }; - const llvm_usize = try o.lowerType(.usize); - const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", target)); - return o.builder.castConst(.inttoptr, try o.builder.intConst(llvm_usize, int), llvm_ptr_ty); + return o.lowerPtrToVoid(nav_align, nav.resolved.?.@"addrspace"); + } + + const gop = try o.nav_map.getOrPut(gpa, nav_id); + if (!gop.found_existing) { + errdefer assert(o.nav_map.remove(nav_id)); + // The NAV hasn't been lowered yet, so generate a placeholder global whose details will + // be filled in later. + const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip)); + gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{ + .type = .void, // placeholder; populated by `updateNav`/`updateFunc` + .kind = .{ .alias = .none }, // placeholder; populated by `updateNav`/`updateFunc` + }); + } + const llvm_global = gop.value_ptr.*; + + // We need to make sure the global's address space is up to date, because that affects the + // type of a pointer to this global. But everything else about the global will be populated + // by `updateNav` or `updateFunc`. + llvm_global.ptr(&o.builder).addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget()); + return llvm_global.toConst(); } pub fn addByValParamAttrs( @@ -4243,13 +4153,14 @@ pub const Object = struct { const name = try o.builder.strtabString("__zig_error_name_table"); // TODO: Address space const variable_index = try o.builder.addVariable(name, .ptr, .default); - variable_index.setLinkage(.private, &o.builder); variable_index.setMutability(.constant, &o.builder); - variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); variable_index.setAlignment( Type.slice_const_u8_sentinel_0.abiAlignment(o.zcu).toLlvm(), &o.builder, ); + const global_index = variable_index.ptrConst(&o.builder).global; + global_index.setLinkage(.private, &o.builder); + global_index.setUnnamedAddr(.unnamed_addr, &o.builder); o.error_name_table = variable_index; return variable_index; @@ -4261,10 +4172,11 @@ pub const Object = struct { const llvm_err_int_ty = try o.errorIntType(); const name = try builder.strtabString("__zig_errors_len"); const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default); - variable_index.setLinkage(.private, builder); variable_index.setMutability(.constant, builder); - variable_index.setUnnamedAddr(.unnamed_addr, builder); variable_index.setAlignment(Type.errorAbiAlignment(o.zcu).toLlvm(), builder); + const global_index = variable_index.ptrConst(&o.builder).global; + global_index.setLinkage(.private, builder); + global_index.setUnnamedAddr(.unnamed_addr, builder); o.errors_len_variable = variable_index; } return o.errors_len_variable; @@ -4332,16 +4244,16 @@ pub const Object = struct { for (0..loaded_enum.field_names.len) |field_index| { const name = try o.builder.stringNull(loaded_enum.field_names.get(ip)[field_index].toSlice(ip)); const name_init = try o.builder.stringConst(name); - const name_variable_index = - try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); + const name_variable_index = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); try name_variable_index.setInitializer(name_init, &o.builder); - name_variable_index.setLinkage(.private, &o.builder); name_variable_index.setMutability(.constant, &o.builder); - name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); + const name_global_index = name_variable_index.ptrConst(&o.builder).global; + name_global_index.setLinkage(.private, &o.builder); + name_global_index.setUnnamedAddr(.unnamed_addr, &o.builder); const name_val = try o.builder.structValue(llvm_ret_ty, &.{ - name_variable_index.toConst(&o.builder), + name_global_index.toConst(), try o.builder.intConst(llvm_usize_ty, name.slice(&o.builder).?.len - 1), }); diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index e70091cebada83ad15da3c8ee87dd72f3981a4d5..0c7e794c78d7c55c0c9d63954476e810b3554dce 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -581,8 +581,17 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif else => unreachable, }; const fn_info = zcu.typeToFunc(zig_fn_ty).?; - const return_type = Type.fromInterned(fn_info.return_type); - const llvm_fn = try self.resolveInst(air_call.callee); + const return_type: Type = .fromInterned(fn_info.return_type); + const llvm_fn = llvm_fn: { + // If the callee is a function *body*, we need to use a pointer to the global. + if (air_call.callee.toInterned()) |ip_index| switch (ip.indexToKey(ip_index)) { + .@"extern" => |e| break :llvm_fn (try o.lowerNavRef(e.owner_nav)).toValue(), + .func => |f| break :llvm_fn (try o.lowerNavRef(f.owner_nav)).toValue(), + else => {}, + }; + // Otherwise, the operand is already a function pointer (possibly runtime-known). + break :llvm_fn try self.resolveInst(air_call.callee); + }; const target = zcu.getTarget(); const sret = firstParamSRet(fn_info, zcu, target); @@ -875,7 +884,9 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v const target = zcu.getTarget(); const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin())); const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?; - const panic_global = try o.resolveLlvmFunction(panic_func.owner_nav); + const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty)); + + const llvm_panic_fn_ref = try o.lowerNavRef(panic_func.owner_nav); const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto; if (has_err_trace) assert(fg.err_ret_trace != .none); @@ -884,8 +895,8 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v .normal, llvm.toLlvmCallConvTag(fn_info.cc, target).?, .none, - panic_global.typeOf(&o.builder), - panic_global.toValue(&o.builder), + llvm_panic_fn_ty, + llvm_panic_fn_ref.toValue(), if (has_err_trace) &.{fg.err_ret_trace} else &.{}, "", ); @@ -1745,8 +1756,9 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod .default, ); try table_variable.setInitializer(table_val, &o.builder); - table_variable.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); - table_variable.setUnnamedAddr(.unnamed_addr, &o.builder); + const table_global = table_variable.ptrConst(&o.builder).global; + table_global.setLinkage(if (o.builder.strip) .private else .internal, &o.builder); + table_global.setUnnamedAddr(.unnamed_addr, &o.builder); const table_includes_else = item_count != table_len; @@ -1759,7 +1771,7 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod .likely => .likely, .unlikely => .unlikely, }, - .table = table_variable.toConst(&o.builder), + .table = table_global.toConst(), .table_includes_else = table_includes_else, }; }; @@ -3255,8 +3267,8 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { const o = fg.object; const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; - const llvm_ptr_const = try o.lowerNavRefValue(ty_nav.nav); - return llvm_ptr_const.toValue(); + const llvm_ptr = try o.lowerNavRef(ty_nav.nav); + return llvm_ptr.toValue(); } fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { @@ -4636,29 +4648,27 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value const o = self.object; const zcu = o.zcu; const ptr_ty = self.typeOfIndex(inst); - const pointee_type = ptr_ty.childType(zcu); - if (!pointee_type.hasRuntimeBits(zcu)) { - const ptr_info = ptr_ty.ptrInfo(zcu); - return (try o.lowerPtrToVoid(ptr_info.flags.alignment, ptr_info.flags.address_space)).toValue(); + const ptr_align = ptr_ty.ptrAlignment(zcu); + const elem_ty = ptr_ty.childType(zcu); + if (!elem_ty.hasRuntimeBits(zcu)) { + return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue(); } - const pointee_llvm_ty = try o.lowerType(pointee_type); - const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); - return self.buildAlloca(pointee_llvm_ty, alignment); + const llvm_elem_ty = try o.lowerType(elem_ty); + return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm()); } fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value { - const o = self.object; - const zcu = o.zcu; - const ptr_ty = self.typeOfIndex(inst); - const ret_ty = ptr_ty.childType(zcu); - if (!ret_ty.hasRuntimeBits(zcu)) { - const ptr_info = ptr_ty.ptrInfo(zcu); - return (try o.lowerPtrToVoid(ptr_info.flags.alignment, ptr_info.flags.address_space)).toValue(); - } if (self.ret_ptr != .none) return self.ret_ptr; - const ret_llvm_ty = try o.lowerType(ret_ty); - const alignment = ptr_ty.ptrAlignment(zcu).toLlvm(); - return self.buildAlloca(ret_llvm_ty, alignment); + const o = self.object; + const zcu = o.zcu; + const ptr_ty = self.typeOfIndex(inst); + const ptr_align = ptr_ty.ptrAlignment(zcu); + const elem_ty = ptr_ty.childType(zcu); + if (!elem_ty.hasRuntimeBits(zcu)) { + return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue(); + } + const llvm_elem_ty = try o.lowerType(elem_ty); + return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm()); } /// Use this instead of builder.buildAlloca, because this function makes sure to -- 2.54.0 From 4a494a8cf94596e73a7ace576111500be7a6f758 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Wed, 25 Mar 2026 20:17:09 +0000 Subject: [PATCH 09/15] tests: enable incremental tests for x86_64-linux-llvm These all pass now! I have also removed the warning about the LLVM backend not supporting incremental compilation; I expect it will work sort of okay in practice by now. --- src/main.zig | 4 ---- test/incremental/add_decl | 1 + test/incremental/add_decl_namespaced | 1 + test/incremental/add_remove_struct_fields | 1 + test/incremental/add_remove_toplevel_fields | 1 + test/incremental/analysis_error_and_syntax_error | 1 + test/incremental/bad_import | 1 + test/incremental/change_embed_file | 1 + test/incremental/change_enum_tag_type | 1 + test/incremental/change_exports | 1 + test/incremental/change_fn_type | 1 + test/incremental/change_generic_line_number | 1 + test/incremental/change_line_number | 1 + test/incremental/change_module | 1 + test/incremental/change_panic_handler | 1 + test/incremental/change_panic_handler_explicit | 1 + test/incremental/change_shift_op | 1 + test/incremental/change_struct_same_fields | 1 + test/incremental/change_zon_file | 1 + test/incremental/change_zon_file_no_result_type | 1 + test/incremental/compile_error_then_log | 1 + test/incremental/compile_log | 1 + test/incremental/delete_comptime_decls | 1 + test/incremental/dependency_on_type_of_inferred_global | 1 + test/incremental/fix_astgen_failure | 1 + test/incremental/function_becomes_inline | 1 + test/incremental/hello | 1 + test/incremental/make_decl_pub | 1 + test/incremental/modify_inline_fn | 1 + test/incremental/move_src | 1 + test/incremental/no_change_preserves_tag_names | 1 + test/incremental/recursive_function_becomes_non_recursive | 1 + test/incremental/remove_enum_field | 1 + test/incremental/remove_invalid_union_backing_enum | 1 + test/incremental/temporary_parse_error | 1 + test/incremental/type_becomes_comptime_only | 1 + test/incremental/type_dependency_loop | 1 + test/incremental/unreferenced_error | 1 + 38 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/main.zig b/src/main.zig index 5554e87c1d263fa31aec9917bde9c1794a80193a..4a69edfe82d9b2a11e0247691e13c4a9482ec9e8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3528,10 +3528,6 @@ fn buildOutputType( fatal("--debug-incremental requires -fincremental", .{}); } - if (incremental and create_module.resolved_options.use_llvm) { - warn("-fincremental is currently unsupported by the LLVM backend; crashes or miscompilations are likely", .{}); - } - const cache_mode: Compilation.CacheMode = b: { // Once incremental compilation is the default, we'll want some smarter logic here, // considering things like the backend in use and whether there's a ZCU. diff --git a/test/incremental/add_decl b/test/incremental/add_decl index 662160fc2dc1038babaab18a234d0917d82675bc..e91eda1930a5c90485112ef01c08f9ea8fab18e9 100644 --- a/test/incremental/add_decl +++ b/test/incremental/add_decl @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm //#target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/add_decl_namespaced b/test/incremental/add_decl_namespaced index aed78815c81aec020f3a4e4dc6a7bbb8c0054ffe..389aef6fb8d289a480260811b7d51a0c6a32858d 100644 --- a/test/incremental/add_decl_namespaced +++ b/test/incremental/add_decl_namespaced @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm //#target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/add_remove_struct_fields b/test/incremental/add_remove_struct_fields index f046229101ee2c1873cc7a0b0f1e0488a25b3c64..34ad260ae1aa479686eec3fd68d4061e2afa8cc5 100644 --- a/test/incremental/add_remove_struct_fields +++ b/test/incremental/add_remove_struct_fields @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm //#target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/add_remove_toplevel_fields b/test/incremental/add_remove_toplevel_fields index 2d5483778ae350bc0de545cc2c97711e32236bd2..83775ae3e3cafa535ebfd79ec9d0b21b250fed66 100644 --- a/test/incremental/add_remove_toplevel_fields +++ b/test/incremental/add_remove_toplevel_fields @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm //#target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/analysis_error_and_syntax_error b/test/incremental/analysis_error_and_syntax_error index 43ba7480526a216c92184ec834a48d3e9fbcd957..c41fecb94572d09c8f8b947fa1cf18300678c115 100644 --- a/test/incremental/analysis_error_and_syntax_error +++ b/test/incremental/analysis_error_and_syntax_error @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/bad_import b/test/incremental/bad_import index b9437714033b679afeb2fdea126c7f4d822f9177..15b9c17fd1f0063bc73783fa6c2a27bc8ac0e3f6 100644 --- a/test/incremental/bad_import +++ b/test/incremental/bad_import @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version diff --git a/test/incremental/change_embed_file b/test/incremental/change_embed_file index 92b9ec23e370e047973886cd613e8287adca5214..74cd2ac146b6844209ceb23762386c463d535b57 100644 --- a/test/incremental/change_embed_file +++ b/test/incremental/change_embed_file @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/change_enum_tag_type b/test/incremental/change_enum_tag_type index 46681407cb6a259e44b1e6071281ecc302b14967..678e9c8c4c11d4c3061d5d9628da4d072a787272 100644 --- a/test/incremental/change_enum_tag_type +++ b/test/incremental/change_enum_tag_type @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/change_exports b/test/incremental/change_exports index 05e3111cebe4903384284aaf0ee0daf9abc9c388..c44bd324888e2a67abd9aaa29aa73683b605e14d 100644 --- a/test/incremental/change_exports +++ b/test/incremental/change_exports @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #update=initial version #file=main.zig diff --git a/test/incremental/change_fn_type b/test/incremental/change_fn_type index 0c512e416e49b208939d07d1c8d80bf42f525afd..69e25a5a4aa7e896e44583867873bc34f5da07b7 100644 --- a/test/incremental/change_fn_type +++ b/test/incremental/change_fn_type @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #update=initial version #file=main.zig pub fn main() !void { diff --git a/test/incremental/change_generic_line_number b/test/incremental/change_generic_line_number index 45b3d2f0d02ff724012328c84db643de027d11a6..d5f7e4b5c181dcdb72c00a4e89af14ad84913f5b 100644 --- a/test/incremental/change_generic_line_number +++ b/test/incremental/change_generic_line_number @@ -1,5 +1,6 @@ #target=x86_64-linux-selfhosted #target=x86_64-windows-selfhosted +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/change_line_number b/test/incremental/change_line_number index c95b690f7f07003c1fcafdc473334540dda5bd93..a06aa3a9006054493549ba8c89389bbe5607a892 100644 --- a/test/incremental/change_line_number +++ b/test/incremental/change_line_number @@ -1,5 +1,6 @@ #target=x86_64-linux-selfhosted #target=x86_64-windows-selfhosted +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/change_module b/test/incremental/change_module index 6275536ad4a8f3bc362eebad43a14f7533ee578b..94cd39bb378b3966977650cfba73c0db4023ad53 100644 --- a/test/incremental/change_module +++ b/test/incremental/change_module @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #module=foo=foo.zig diff --git a/test/incremental/change_panic_handler b/test/incremental/change_panic_handler index 2d8e95f6e11b352b791ff66d927cc626f9bfc48b..13685406dff01786fe8ff1fe17ad0133a162f448 100644 --- a/test/incremental/change_panic_handler +++ b/test/incremental/change_panic_handler @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #update=initial version #file=main.zig pub fn main() !u8 { diff --git a/test/incremental/change_panic_handler_explicit b/test/incremental/change_panic_handler_explicit index bf57a076282d7c609e307c333b5ccfb660b6c88e..ed4cfcfca44cad7f2c7085b6855a3312bdf7542c 100644 --- a/test/incremental/change_panic_handler_explicit +++ b/test/incremental/change_panic_handler_explicit @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #update=initial version #file=main.zig pub fn main() !u8 { diff --git a/test/incremental/change_shift_op b/test/incremental/change_shift_op index 159ef6f11d1ab0dbd6e7074cd49afa75b02af67e..31a8b67c687680f45aec4bf5fb0be385b6bbdc84 100644 --- a/test/incremental/change_shift_op +++ b/test/incremental/change_shift_op @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/change_struct_same_fields b/test/incremental/change_struct_same_fields index 180948337020cba4922e030e2e01ef0a0c189a38..d686688172ff030864ab4d2ea524fabc8c33118e 100644 --- a/test/incremental/change_struct_same_fields +++ b/test/incremental/change_struct_same_fields @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/change_zon_file b/test/incremental/change_zon_file index beeffec4373c4ddf6ce48a7e566cd29f32d4e38f..822996ac74966971b237718461d555ce461d194f 100644 --- a/test/incremental/change_zon_file +++ b/test/incremental/change_zon_file @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm //#target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/change_zon_file_no_result_type b/test/incremental/change_zon_file_no_result_type index d05aecedfbccf72fe96709b3aa42a02042b745a9..f9a3c3006577c64a6b8725e90fef02e67364130f 100644 --- a/test/incremental/change_zon_file_no_result_type +++ b/test/incremental/change_zon_file_no_result_type @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm //#target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/compile_error_then_log b/test/incremental/compile_error_then_log index 9bd306fbf75fe325f50b1f948d4efe0eb08a6187..91053ff45c5086a5fa550d1078aac30c34ebe3ea 100644 --- a/test/incremental/compile_error_then_log +++ b/test/incremental/compile_error_then_log @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version with compile error #file=main.zig diff --git a/test/incremental/compile_log b/test/incremental/compile_log index 3ed5467a9a175c559bef5b910215a05175fa8d5c..41fc344d1724d5a76371d60291ea5e290cafff2e 100644 --- a/test/incremental/compile_log +++ b/test/incremental/compile_log @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version with no compile log diff --git a/test/incremental/delete_comptime_decls b/test/incremental/delete_comptime_decls index c8b68e31ed101633dcb64b63fefdfbc19b5494cb..261c03343ccfb28f52fbfa69e5bd2699dcf6344a 100644 --- a/test/incremental/delete_comptime_decls +++ b/test/incremental/delete_comptime_decls @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/dependency_on_type_of_inferred_global b/test/incremental/dependency_on_type_of_inferred_global index 9d5ab28034d3be7c13e2f2261c9bc4a42b92c31d..b7b0f7d979aa4451d6a6decd14c347bc15e03723 100644 --- a/test/incremental/dependency_on_type_of_inferred_global +++ b/test/incremental/dependency_on_type_of_inferred_global @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/fix_astgen_failure b/test/incremental/fix_astgen_failure index 701e9973a73e5df91b3d2cda86188c15b96ad67b..fcfc51e8a2c6656964c18a8af5ec6cabd44b24f1 100644 --- a/test/incremental/fix_astgen_failure +++ b/test/incremental/fix_astgen_failure @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version with error #file=main.zig diff --git a/test/incremental/function_becomes_inline b/test/incremental/function_becomes_inline index eefb4f1a077d437389c79631f6bcd2953821e3f0..b63aa3e5697a600d2463dfb3af9282ea62b223a4 100644 --- a/test/incremental/function_becomes_inline +++ b/test/incremental/function_becomes_inline @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #update=non-inline version #file=main.zig pub fn main() !void { diff --git a/test/incremental/hello b/test/incremental/hello index e2146e52be590779e8132e65ba9e5503a2d80e98..7699f6f7a27f13725a06b4899bee61d247578275 100644 --- a/test/incremental/hello +++ b/test/incremental/hello @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/make_decl_pub b/test/incremental/make_decl_pub index a2c87b8f50114fa9b77216f0b4036e8a55e81fc7..f985690cae0719caf1693f7b29b2bed6083fcb06 100644 --- a/test/incremental/make_decl_pub +++ b/test/incremental/make_decl_pub @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/modify_inline_fn b/test/incremental/modify_inline_fn index 6bf1f0baf29729ca12729955fe1683a6d3cf3706..96d483233895be5766fcdaf3d73418620d91a845 100644 --- a/test/incremental/modify_inline_fn +++ b/test/incremental/modify_inline_fn @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/move_src b/test/incremental/move_src index a60211e31b86f3dbf3746b28ee39df4c92e83803..200b6767cf1f7269b61251cfe6cdada5779b8f93 100644 --- a/test/incremental/move_src +++ b/test/incremental/move_src @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/no_change_preserves_tag_names b/test/incremental/no_change_preserves_tag_names index 06dc2f069c23634ad1d70ea30b997c555c483241..7d7e0b20ea4d0756258815d477368ec68d715664 100644 --- a/test/incremental/no_change_preserves_tag_names +++ b/test/incremental/no_change_preserves_tag_names @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm //#target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/recursive_function_becomes_non_recursive b/test/incremental/recursive_function_becomes_non_recursive index ad8c0059fabba099e12f0e2b6055127aab0dc9a2..6830d9c5f92ffd19aac10bd0ff4ff327a6db7fcb 100644 --- a/test/incremental/recursive_function_becomes_non_recursive +++ b/test/incremental/recursive_function_becomes_non_recursive @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/remove_enum_field b/test/incremental/remove_enum_field index a1bbab8fd0fdaaf65115cb29d2b0210ccc77f862..fe1db0974aa225a72e6f28ac930988cfcdc2c4ac 100644 --- a/test/incremental/remove_enum_field +++ b/test/incremental/remove_enum_field @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/remove_invalid_union_backing_enum b/test/incremental/remove_invalid_union_backing_enum index 84abedcf7b418a04ed22a1c578b0a007d6da132c..51dc76fd07377d621c59ba136e4b00c925b79e8d 100644 --- a/test/incremental/remove_invalid_union_backing_enum +++ b/test/incremental/remove_invalid_union_backing_enum @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/temporary_parse_error b/test/incremental/temporary_parse_error index 956ed61225701ca155e5ebfa5743a0298858f099..3b408b40488f18d96b810855f151ecc14409b01a 100644 --- a/test/incremental/temporary_parse_error +++ b/test/incremental/temporary_parse_error @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/type_becomes_comptime_only b/test/incremental/type_becomes_comptime_only index 8e712042f586dbc77d26697a27cedeece19f4b8e..6d4502a555b053579ebde54c2e307ab87f056cb5 100644 --- a/test/incremental/type_becomes_comptime_only +++ b/test/incremental/type_becomes_comptime_only @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/type_dependency_loop b/test/incremental/type_dependency_loop index 40f41bf19fe4bb4d7a19b5a4a40a0a10fb361ca9..c58ec93e75cbc84f78ad1df875728362bceb69a4 100644 --- a/test/incremental/type_dependency_loop +++ b/test/incremental/type_dependency_loop @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig diff --git a/test/incremental/unreferenced_error b/test/incremental/unreferenced_error index 25790ed0447950ebefa41f677d594e5dbc4b76f4..d52819797672eb752f6c9d8e43d6254e81b4e726 100644 --- a/test/incremental/unreferenced_error +++ b/test/incremental/unreferenced_error @@ -2,6 +2,7 @@ #target=x86_64-windows-selfhosted #target=x86_64-linux-cbe #target=x86_64-windows-cbe +#target=x86_64-linux-llvm #target=wasm32-wasi-selfhosted #update=initial version #file=main.zig -- 2.54.0 From 6ed9c0521055add99097fd43598d5a8e1e3665ff Mon Sep 17 00:00:00 2001 From: Pavel Verigo Date: Thu, 26 Mar 2026 11:19:33 +0000 Subject: [PATCH 10/15] link.Wasm: fix indirect function table handling The changes to the LLVM backend here changed the compiler_rt object which LLVM emits, and exposed some buggy behavior in the self-hosted WASM linker when parsing that object. --- src/link/Wasm/Object.zig | 66 +++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/src/link/Wasm/Object.zig b/src/link/Wasm/Object.zig index c55d36f05a18787c1fd0050a73d3f74cec09b1e1..8885fb5247b94c365a8a251743420bea2a0682d7 100644 --- a/src/link/Wasm/Object.zig +++ b/src/link/Wasm/Object.zig @@ -969,6 +969,41 @@ pub fn parse( func.type_index = func_type.ptr(ss).*; } + // Check for indirect function table in case of an MVP object file. + legacy_indirect_function_table: { + // If there is a symbol for each import table, this is not a legacy object file. + if (ss.table_imports.items.len == table_import_symbol_count) break :legacy_indirect_function_table; + if (table_import_symbol_count != 0) { + return diags.failParse(path, "expected a table entry symbol for each of the {d} table(s), but instead got {d} symbols.", .{ + ss.table_imports.items.len, table_import_symbol_count, + }); + } + // MVP object files cannot have any table definitions, only imports + // (for the indirect function table). + const tables = wasm.object_tables.items[tables_start..]; + if (tables.len > 0) { + return diags.failParse(path, "table definition without representing table symbols", .{}); + } + if (ss.table_imports.items.len != 1) { + return diags.failParse(path, "found more than one table import, but no representing table symbols", .{}); + } + const table_import_name = ss.table_imports.items[0].name; + if (table_import_name != wasm.preloaded_strings.__indirect_function_table) { + return diags.failParse(path, "non-indirect function table import '{s}' is missing a corresponding symbol", .{ + table_import_name.slice(wasm), + }); + } + + try ss.symbol_table.append(gpa, .{ + .flags = .{ + .undefined = true, + .no_strip = true, + }, + .name = table_import_name.toOptional(), + .pointee = .{ .table_import = @enumFromInt(0) }, + }); + } + // Apply symbol table information. for (ss.symbol_table.items) |symbol| switch (symbol.pointee) { .function_import => |index| { @@ -1331,37 +1366,6 @@ pub fn parse( }; } - // Check for indirect function table in case of an MVP object file. - legacy_indirect_function_table: { - // If there is a symbol for each import table, this is not a legacy object file. - if (ss.table_imports.items.len == table_import_symbol_count) break :legacy_indirect_function_table; - if (table_import_symbol_count != 0) { - return diags.failParse(path, "expected a table entry symbol for each of the {d} table(s), but instead got {d} symbols.", .{ - ss.table_imports.items.len, table_import_symbol_count, - }); - } - // MVP object files cannot have any table definitions, only imports - // (for the indirect function table). - const tables = wasm.object_tables.items[tables_start..]; - if (tables.len > 0) { - return diags.failParse(path, "table definition without representing table symbols", .{}); - } - if (ss.table_imports.items.len != 1) { - return diags.failParse(path, "found more than one table import, but no representing table symbols", .{}); - } - const table_import_name = ss.table_imports.items[0].name; - if (table_import_name != wasm.preloaded_strings.__indirect_function_table) { - return diags.failParse(path, "non-indirect function table import '{s}' is missing a corresponding symbol", .{ - table_import_name.slice(wasm), - }); - } - const ptr = wasm.object_table_imports.getPtr(table_import_name).?; - ptr.flags = .{ - .undefined = true, - .no_strip = true, - }; - } - for (wasm.object_init_funcs.items[init_funcs_start..]) |init_func| { const func = init_func.function_index.ptr(wasm); const params = func.type_index.ptr(wasm).params.slice(wasm); -- 2.54.0 From fca3f6f62e515669cb3aa22b8512e2cc4e1994f2 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Fri, 27 Mar 2026 14:01:24 +0000 Subject: [PATCH 11/15] MachO: don't split subsections on N_ALT_ENTRY symbols MachO has a mechanism where symbols can introduce "subsections", which (as I understand it) allows a linker to garbage-collect parts of sections without pulling in the heavy machinery of `-fdata-sections` and `-ffunction-sections`. Essentially, symbols can be considered to partition a section, and these boundaries are not allowed to be crossed by memory accesses, so the linker can detect symbols which are unused and drop the corresponding input section regions. However, the symbol flag `N_ALT_ENTRY` indicates that a symbol should not participate in this "splitting", and is instead an "alternate entry point" to the previous subsection, which should continue through this symbol. The Mach-O linker was failing to ignore `N_ALT_ENTRY` symbols when creating subsections, which meant that for certain link inputs, it would create additional subsection splits, and then garbage collect the extra sections (due to the `N_ALT_ENTRY` symbol being unused). Naturally, this silent dropping of parts of input sections led to miscompilations. --- src/link/MachO/Object.zig | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/link/MachO/Object.zig b/src/link/MachO/Object.zig index e81ef5558b64fd878044322fb82e9882150af970..76aa4f5db0087b5bfc416cfb24943aa691a153de 100644 --- a/src/link/MachO/Object.zig +++ b/src/link/MachO/Object.zig @@ -328,7 +328,9 @@ fn initSubsections(self: *Object, allocator: Allocator, nlists: anytype) !void { if (isPtrLiteral(sect)) continue; const nlist_start = for (nlists, 0..) |nlist, i| { - if (nlist.nlist.n_sect - 1 == n_sect) break i; + // We must ignore `alt_entry` (N_ALT_ENTRY) symbols here, because that flag indicates + // that a symbol should *not* split subsections. + if (nlist.nlist.n_sect - 1 == n_sect and !nlist.nlist.n_desc.alt_entry) break i; } else nlists.len; const nlist_end = for (nlists[nlist_start..], nlist_start..) |nlist, i| { if (nlist.nlist.n_sect - 1 != n_sect) break i; @@ -359,9 +361,24 @@ fn initSubsections(self: *Object, allocator: Allocator, nlists: anytype) !void { const alias_start = idx; const nlist = nlists[alias_start]; - while (idx < nlist_end and - nlists[idx].nlist.n_value == nlist.nlist.n_value) : (idx += 1) - {} + // Skip past any symbols which shouldn't terminate this subsection. + while (true) { + idx += 1; + if (idx == nlist_end) { + // This subsection contains the full remainder of the section. + break; + } + if (nlists[idx].nlist.n_value == nlist.nlist.n_value) { + // Multiple symbols at the same address---don't create zero-length subsections. + continue; + } + if (nlists[idx].nlist.n_desc.alt_entry) { + // N_ALT_ENTRY indicates that this symbol does not split subsections, and is + // instead an "alternate entry point" into an existing subsection. + continue; + } + break; + } const size = if (idx < nlist_end) nlists[idx].nlist.n_value - nlist.nlist.n_value @@ -385,7 +402,9 @@ fn initSubsections(self: *Object, allocator: Allocator, nlists: anytype) !void { }); for (alias_start..idx) |i| { - self.symtab.items(.size)[nlists[i].idx] = size; + if (!nlists[i].nlist.n_desc.alt_entry) { + self.symtab.items(.size)[nlists[i].idx] = size; + } } } -- 2.54.0 From 29faeeee9de95a327bbb864dd861f15386c4ccff Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Fri, 27 Mar 2026 20:27:11 +0000 Subject: [PATCH 12/15] tests: fix passing -fno-builtin to module tests --- test/tests.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests.zig b/test/tests.zig index 8a417f7fc3c22849042e017402565feccfecfe28..c2445037e40ce1138af8aad9c6ebaeb89e654905 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -2487,7 +2487,7 @@ fn addOneModuleTest( .zig_lib_dir = b.path("lib"), }); these_tests.linkage = test_target.linkage; - if (options.no_builtin) these_tests.root_module.no_builtin = false; + if (options.no_builtin) these_tests.root_module.no_builtin = true; if (options.build_options) |build_options| { these_tests.root_module.addOptions("build_options", build_options); } -- 2.54.0 From 2ac47fe314f0a88a2e5ac6c0116392504369b466 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Fri, 27 Mar 2026 20:27:30 +0000 Subject: [PATCH 13/15] tests: disable safe compiler-rt tests on LLVM Disabling due to multiple LLVM bugs: * https://codeberg.org/ziglang/zig/issues/31701 * https://codeberg.org/ziglang/zig/issues/31702 --- test/tests.zig | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/tests.zig b/test/tests.zig index c2445037e40ce1138af8aad9c6ebaeb89e654905..ad2e4b24784eebec0217c8b8df54a0bca869bd34 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -2413,6 +2413,19 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step { const would_use_llvm = wouldUseLlvm(test_target.use_llvm, test_target.target, test_target.optimize_mode); if (options.skip_llvm and would_use_llvm) continue; + if (would_use_llvm and (mem.eql(u8, options.name, "compiler-rt") or mem.eql(u8, options.name, "zigc"))) { + switch (test_target.optimize_mode) { + .Debug, .ReleaseSafe => { + // LLVM 21 is affected by multiple bugs in safe builds of compiler-rt: + // * https://codeberg.org/ziglang/zig/issues/31701 + // * https://codeberg.org/ziglang/zig/issues/31702 + // ...so for now, skip these tests. + continue; + }, + .ReleaseSmall, .ReleaseFast => {}, + } + } + const triple_txt = resolved_target.query.zigTriple(b.allocator) catch @panic("OOM"); if (options.test_target_filters.len > 0) { -- 2.54.0 From b36cfc6352df4c5c5005c7a720dbc84894097d11 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Sat, 28 Mar 2026 09:53:13 +0000 Subject: [PATCH 14/15] compiler-rt: work around LLVM not respecting -fno-builtin --- lib/compiler_rt/mulo.zig | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/compiler_rt/mulo.zig b/lib/compiler_rt/mulo.zig index 979a5452a53e3497f48af7bf4abe8af9eb7ed884..bd8e3e28953931f9a85e7e599c9b0184fc432d23 100644 --- a/lib/compiler_rt/mulo.zig +++ b/lib/compiler_rt/mulo.zig @@ -19,7 +19,12 @@ comptime { inline fn muloXi4_genericSmall(comptime ST: type, a: ST, b: ST, overflow: *c_int) ST { overflow.* = 0; const min = math.minInt(ST); - const res: ST = a *% b; + const res: ST = if (ST == i128 and builtin.target.cpu.arch.isWasm()) res: { + // Despite compiler-rt being built with `-fno-builtin`, LLVM still converts this function to + // a call to `__muloti4` on WASM. This is an upstream bug: circumvent it by directly calling + // the "lower-level" compiler-rt routine for this wrapping multiplication. + break :res @import("mulXi3.zig").__multi3(a, b); + } else a *% b; // Hacker's Delight section Overflow subsection Multiplication // case a=-2^{31}, b=-1 problem, because // on some machines a*b = -2^{31} with overflow -- 2.54.0 From abaf3dfbe9b79dcdaf742fa1c08ce6f54061e237 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Sat, 28 Mar 2026 19:24:09 +0000 Subject: [PATCH 15/15] tests: fix `wouldUseLlvm` on x86_64-windows --- test/tests.zig | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/tests.zig b/test/tests.zig index ad2e4b24784eebec0217c8b8df54a0bca869bd34..9cb7a879b251fd4c224dc141f9df7327faf03789 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -2647,12 +2647,19 @@ pub fn wouldUseLlvm(use_llvm: ?bool, query: std.Target.Query, optimize_mode: Opt } const cpu_arch = query.cpu_arch orelse builtin.cpu.arch; const os_tag = query.os_tag orelse builtin.os.tag; + const ofmt: std.Target.ObjectFormat = query.ofmt orelse .default(os_tag, cpu_arch); switch (cpu_arch) { - .x86_64 => if (os_tag.isBSD() or os_tag == .illumos or std.Target.ptrBitWidth_arch_abi(cpu_arch, query.abi orelse .none) != 64) return true, + .x86_64 => { + if (std.Target.ptrBitWidth_arch_abi(cpu_arch, query.abi orelse .none) != 64) return true; + if (os_tag.isBSD() or os_tag == .illumos) return true; + return switch (ofmt) { + .elf, .macho => return false, + else => return true, + }; + }, .spirv32, .spirv64 => return false, else => return true, } - return false; } const CAbiTestOptions = struct { -- 2.54.0