authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-06-10 09:30:45+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-06-24 10:20:52+01:00
logfa3a9fcdfaee42ef304f662d65decbb685c705a0
tree6839b76688eda3e09f99af6f7fc81078f28e523a
parentf26cdb2771a4bb4d5f1d5acc446ec51c3e177f75
signaturelock-open Commit is signed but in an unrecognized format.

llvm: only load/store ABI-sized integers to/from memory

In theory, for any integer type containing padding bits (e.g. `i3`), LLVM *does* support storing and loading that type to and from memory, and the documented semantics for this in the LLVM langref are a valid implementation of Zig's integer semantics. However, in practice, this support is frequently buggy, because Clang never emits such operations and so they are poorly tested in LLVM. The most recent example of such a bug is https://codeberg.org/ziglang/zig/issues/35560. In addition, LLVM's semantics here lead to suboptimal codegen, because loads/stores are only rounded up to the next byte (rather than the next ABI integer type) and padding bits are unspecified (or, on LLVM master, padding bits are all zero, even for a negative signed integer). We can both mitigate LLVM bugs and get better codegen by employing the strategy used by the self-hosted backends, wherein we zero- or sign-extend to an integer of bit size `@sizeOf(WeirdInt) * 8`. This type has the same size in memory but has no padding bits. When loading, we load the extended type and then use `trunc nuw` or `trunc nsw` (depending on the signedness) to convert to the correct value type. The name of the game here is to minimize occurrences of `wip.load` and `wip.store` in the LLVM backend, with most accesses instead going through higher-level functions which handle this extension/truncation for us. Those higher-level functions are `codegen.llvm.FuncGen.load` and `codegen.llvm.FuncGen.store`. I changed their signatures slightly to make them consistent and more generally useful, and changed the majority of calls to `wip.load`/`wip.store` to use them. They are the only functions which include the integer extension/truncation logic. The remaining occurrences of `wip.load` and `wip.store` outside of those two functions are all in ABI handling. These code paths cannot occur for non-ABI types due to how we lower `CallingConvention.auto`, therefore they cannot be reached for non-ABI-sized integer types. This diff was not purely mechanical---I had to make a handful of non-trivial logic changes and refactors. The biggest logic change is in lowering a "select" operation to a manual loop: we now use phi nodes to track the loop iterator and accumulator instead of an `alloca`. (To be honest, this wasn't really necessary, but I felt dirty migrating the old logic when this approach is clearly better.) The most significant refactor is that the logic for extracting function call arguments has been moved into `FuncGen`: the `codegen.llvm.Object.updateFunc` implementation will not initialize the three fields which depend on function arguments, and a new function `FuncGen.genMainBody` will populate those fields before calling `genBody` on the main AIR body. This change was made so that I could use member functions on `FuncGen` in this logic without jumping through hoops. The reproduction given in https://codeberg.org/ziglang/zig/issues/35560 is apparently quite delicate, so I haven't added a test case because it realistically would not be very useful. However, I have manually tested the reproduction, and can confirm that the bug *does* trigger before this commit (targeting both baseline and native CPU), and does *not* trigger after this commit (again, for both baseline and native CPU). Resolves: https://codeberg.org/ziglang/zig/issues/35560

5 files changed, 762 insertions(+), 807 deletions(-)

lib/std/zig/llvm/Builder.zig+48-2
......@@ -2061,6 +2061,13 @@ pub const Alignment = enum(u6) {
20612061 };
20622062 }
20632063
2064 /// Asserts that neither `a` nor `b` is `.default`.
2065 pub fn max(a: Alignment, b: Alignment) Alignment {
2066 assert(a != .default);
2067 assert(b != .default);
2068 return @enumFromInt(@max(@intFromEnum(a), @intFromEnum(b)));
2069 }
2070
20642071 pub fn toLlvm(self: Alignment) u6 {
20652072 return switch (self) {
20662073 .default => 0,
......@@ -4314,6 +4321,9 @@ pub const Function = struct {
43144321 @"tail call",
43154322 @"tail call fast",
43164323 trunc,
4324 @"trunc nuw",
4325 @"trunc nsw",
4326 @"trunc nuw nsw",
43174327 udiv,
43184328 @"udiv exact",
43194329 urem,
......@@ -4377,7 +4387,10 @@ pub const Function = struct {
43774387 };
43784388 }
43794389
4380 pub fn toCastOpcode(self: Tag) CastOpcode {
4390 /// Does not accept `.@"trunc nuw"`, `.@"trunc nsw"`, or `.@"trunc nuw nsw"`, because
4391 /// they do not have distinct `CastOpcode` values, and are instead encoded in bitcode
4392 /// using flags on a normal `trunc` operation.
4393 fn toCastOpcode(self: Tag) CastOpcode {
43814394 return switch (self) {
43824395 .trunc => .trunc,
43834396 .zext => .zext,
......@@ -4572,6 +4585,9 @@ pub const Function = struct {
45724585 .sext,
45734586 .sitofp,
45744587 .trunc,
4588 .@"trunc nuw",
4589 .@"trunc nsw",
4590 .@"trunc nuw nsw",
45754591 .uitofp,
45764592 .zext,
45774593 => wip.extraData(Cast, instruction.data).type,
......@@ -4758,6 +4774,9 @@ pub const Function = struct {
47584774 .sext,
47594775 .sitofp,
47604776 .trunc,
4777 .@"trunc nuw",
4778 .@"trunc nsw",
4779 .@"trunc nuw nsw",
47614780 .uitofp,
47624781 .zext,
47634782 => function.extraData(Cast, instruction.data).type,
......@@ -5975,6 +5994,9 @@ pub const WipFunction = struct {
59755994 .sext,
59765995 .sitofp,
59775996 .trunc,
5997 .@"trunc nuw",
5998 .@"trunc nsw",
5999 .@"trunc nuw nsw",
59786000 .uitofp,
59796001 .zext,
59806002 => {},
......@@ -6583,6 +6605,9 @@ pub const WipFunction = struct {
65836605 .sext,
65846606 .sitofp,
65856607 .trunc,
6608 .@"trunc nuw",
6609 .@"trunc nsw",
6610 .@"trunc nuw nsw",
65866611 .uitofp,
65876612 .zext,
65886613 => {
......@@ -9975,6 +10000,9 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void
997510000 .sext,
997610001 .sitofp,
997710002 .trunc,
10003 .@"trunc nuw",
10004 .@"trunc nsw",
10005 .@"trunc nuw nsw",
997810006 .uitofp,
997910007 .zext,
998010008 => |tag| {
......@@ -11649,7 +11677,11 @@ fn convTag(
1164911677 .unneeded => unreachable,
1165011678 },
1165111679 .eq => unreachable,
11652 .gt => .trunc,
11680 .gt => switch (signedness) {
11681 .unsigned => .@"trunc nuw",
11682 .signed => .@"trunc nsw",
11683 .unneeded => .trunc,
11684 },
1165311685 },
1165411686 .pointer => .inttoptr,
1165511687 else => unreachable,
......@@ -14962,6 +14994,20 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
1496214994 .opcode = kind.toCastOpcode(),
1496314995 });
1496414996 },
14997 .@"trunc nuw",
14998 .@"trunc nsw",
14999 .@"trunc nuw nsw",
15000 => |kind| {
15001 const extra = func.extraData(Function.Instruction.Cast, data);
15002 try function_block.writeAbbrev(FunctionBlock.TruncNoWrap{
15003 .val = adapter.getOffsetValueIndex(extra.val),
15004 .type_index = extra.type,
15005 .flags = .{
15006 .no_unsigned_wrap = kind == .@"trunc nuw" or kind == .@"trunc nuw nsw",
15007 .no_signed_wrap = kind == .@"trunc nsw" or kind == .@"trunc nuw nsw",
15008 },
15009 });
15010 },
1496515011 .@"fcmp false",
1496615012 .@"fcmp oeq",
1496715013 .@"fcmp oge",
lib/std/zig/llvm/ir.zig+19
......@@ -696,6 +696,7 @@ pub const ModuleBlock = struct {
696696 ModuleBlock.FunctionBlock.Select,
697697 ModuleBlock.FunctionBlock.SelectFast,
698698 ModuleBlock.FunctionBlock.Cast,
699 ModuleBlock.FunctionBlock.TruncNoWrap,
699700 ModuleBlock.FunctionBlock.Alloca,
700701 ModuleBlock.FunctionBlock.GetElementPtr,
701702 ModuleBlock.FunctionBlock.ExtractValue,
......@@ -1086,6 +1087,24 @@ pub const ModuleBlock = struct {
10861087 opcode: CastOpcode,
10871088 };
10881089
1090 pub const TruncNoWrap = struct {
1091 pub const Flags = packed struct(u2) {
1092 no_unsigned_wrap: bool,
1093 no_signed_wrap: bool,
1094 };
1095 pub const ops = [_]AbbrevOp{
1096 .{ .literal = @intFromEnum(ModuleBlock.FunctionBlock.Code.INST_CAST) },
1097 ValueAbbrev,
1098 .{ .fixed_runtime = Builder.Type },
1099 .{ .literal = @intFromEnum(Builder.CastOpcode.trunc) },
1100 .{ .fixed = @bitSizeOf(Flags) },
1101 };
1102
1103 val: u32,
1104 type_index: Builder.Type,
1105 flags: Flags,
1106 };
1107
10891108 pub const Alloca = struct {
10901109 pub const Flags = packed struct(u11) {
10911110 align_lower: u5,
src/InternPool.zig+1-7
......@@ -6007,13 +6007,7 @@ pub const Alignment = enum(u6) {
60076007 return r;
60086008 }
60096009
6010 const LlvmBuilderAlignment = std.zig.llvm.Builder.Alignment;
6011
6012 pub fn toLlvm(a: Alignment) LlvmBuilderAlignment {
6013 return @enumFromInt(@intFromEnum(a));
6014 }
6015
6016 pub fn fromLlvm(a: LlvmBuilderAlignment) Alignment {
6010 pub fn toLlvm(a: Alignment) std.zig.llvm.Builder.Alignment {
60176011 return @enumFromInt(@intFromEnum(a));
60186012 }
60196013};
src/codegen/llvm.zig+15-175
......@@ -20,7 +20,6 @@ const Value = @import("../Value.zig");
2020const Zcu = @import("../Zcu.zig");
2121const aarch64_c_abi = @import("aarch64/abi.zig");
2222const FuncGen = @import("llvm/FuncGen.zig");
23const buildAllocaInner = FuncGen.buildAllocaInner;
2423const isByRef = FuncGen.isByRef;
2524const firstParamSRet = FuncGen.firstParamSRet;
2625const lowerFnRetTy = FuncGen.lowerFnRetTy;
......@@ -1274,165 +1273,7 @@ pub const Object = struct {
12741273 } }, &o.builder);
12751274 }
12761275
1277 var deinit_wip = true;
1278 var wip = try Builder.WipFunction.init(&o.builder, .{
1279 .function = llvm_function,
1280 .strip = owner_mod.strip,
1281 });
1282 defer if (deinit_wip) wip.deinit();
1283 wip.cursor = .{ .block = try wip.block(0, "Entry") };
1284
1285 // This is the list of args we will use that correspond directly to the AIR arg
1286 // instructions. Depending on the calling convention, this list is not necessarily
1287 // a bijection with the actual LLVM parameters of the function.
1288 var args: std.ArrayList(Builder.Value) = .empty;
1289 defer args.deinit(gpa);
1290
1291 const ret_ptr: Builder.Value, const err_ret_trace: Builder.Value = implicit_args: {
1292 var it = iterateParamTypes(o, fn_info);
1293
1294 const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: {
1295 const param = wip.arg(it.llvm_index);
1296 it.llvm_index += 1;
1297 break :param param;
1298 } else .none;
1299
1300 const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing;
1301 const err_ret_trace: Builder.Value = if (err_return_tracing) param: {
1302 const param = wip.arg(it.llvm_index);
1303 it.llvm_index += 1;
1304 break :param param;
1305 } else .none;
1306
1307 while (try it.next()) |lowering| {
1308 try args.ensureUnusedCapacity(gpa, 1);
1309
1310 switch (lowering) {
1311 .no_bits => continue,
1312 .byval => {
1313 assert(!it.byval_attr);
1314 const param_index = it.zig_index - 1;
1315 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
1316 const param = wip.arg(it.llvm_index - 1);
1317
1318 if (isByRef(param_ty, zcu)) {
1319 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1320 const param_llvm_ty = param.typeOfWip(&wip);
1321 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1322 _ = try wip.store(.normal, param, arg_ptr, alignment);
1323 args.appendAssumeCapacity(arg_ptr);
1324 } else {
1325 args.appendAssumeCapacity(param);
1326 }
1327 },
1328 .byref => {
1329 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1330 const param = wip.arg(it.llvm_index - 1);
1331
1332 if (isByRef(param_ty, zcu)) {
1333 args.appendAssumeCapacity(param);
1334 } else {
1335 const param_llvm_ty = try o.lowerType(param_ty);
1336 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1337 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1338 }
1339 },
1340 .byref_mut => {
1341 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1342 const param = wip.arg(it.llvm_index - 1);
1343
1344 if (isByRef(param_ty, zcu)) {
1345 args.appendAssumeCapacity(param);
1346 } else {
1347 const param_llvm_ty = try o.lowerType(param_ty);
1348 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1349 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1350 }
1351 },
1352 .abi_sized_int => {
1353 assert(!it.byval_attr);
1354 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1355 const param = wip.arg(it.llvm_index - 1);
1356
1357 const param_llvm_ty = try o.lowerType(param_ty);
1358 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1359 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1360 _ = try wip.store(.normal, param, arg_ptr, alignment);
1361
1362 if (isByRef(param_ty, zcu)) {
1363 args.appendAssumeCapacity(arg_ptr);
1364 } else {
1365 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1366 }
1367 },
1368 .slice => {
1369 assert(!it.byval_attr);
1370 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1371 assert(!isByRef(param_ty, zcu));
1372 const slice_val = try wip.buildAggregate(
1373 try o.lowerType(param_ty),
1374 &.{ wip.arg(it.llvm_index - 2), wip.arg(it.llvm_index - 1) },
1375 "",
1376 );
1377 args.appendAssumeCapacity(slice_val);
1378 },
1379 .multiple_llvm_types => {
1380 assert(!it.byval_attr);
1381 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1382 const param_llvm_ty = try o.lowerType(param_ty);
1383 const param_alignment = param_ty.abiAlignment(zcu);
1384 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
1385 const arg_ptr = try buildAllocaInner(&wip, llvm_ty, param_alignment.toLlvm(), target);
1386 const llvm_args_start = it.llvm_index - it.types_len;
1387 for (llvm_args_start.., it.offsets_buffer[0..it.types_len]) |llvm_arg_index, offset| {
1388 const param = wip.arg(@intCast(llvm_arg_index));
1389 const part_ptr = try o.ptraddConst(&wip, arg_ptr, offset);
1390 _ = try wip.store(.normal, param, part_ptr, param_alignment.offset(offset).toLlvm());
1391 }
1392
1393 if (isByRef(param_ty, zcu)) {
1394 args.appendAssumeCapacity(arg_ptr);
1395 } else {
1396 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment.toLlvm(), ""));
1397 }
1398 },
1399 .float_array => {
1400 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1401 const param_llvm_ty = try o.lowerType(param_ty);
1402 const param = wip.arg(it.llvm_index - 1);
1403
1404 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1405 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1406 _ = try wip.store(.normal, param, arg_ptr, alignment);
1407
1408 if (isByRef(param_ty, zcu)) {
1409 args.appendAssumeCapacity(arg_ptr);
1410 } else {
1411 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1412 }
1413 },
1414 .i32_array, .i64_array => {
1415 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1416 const param_llvm_ty = try o.lowerType(param_ty);
1417 const param = wip.arg(it.llvm_index - 1);
1418
1419 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1420 const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target);
1421 _ = try wip.store(.normal, param, arg_ptr, alignment);
1422
1423 if (isByRef(param_ty, zcu)) {
1424 args.appendAssumeCapacity(arg_ptr);
1425 } else {
1426 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1427 }
1428 },
1429 }
1430 }
1431
1432 break :implicit_args .{ ret_ptr, err_ret_trace };
1433 };
1434
1435 const file, const subprogram = if (!wip.strip) debug_info: {
1276 const file, const subprogram = if (!owner_mod.strip) debug_info: {
14361277 const file = try o.getDebugFile(file_scope);
14371278
14381279 const line_number = zcu.navSrcLine(func.owner_nav) + 1;
......@@ -1498,11 +1339,12 @@ pub const Object = struct {
14981339 .gpa = gpa,
14991340 .air = air.*,
15001341 .liveness = liveness.*.?,
1501 .wip = wip,
1342 .wip = try .init(&o.builder, .{
1343 .function = llvm_function,
1344 .strip = owner_mod.strip,
1345 }),
15021346 .is_naked = fn_info.cc == .naked,
15031347 .fuzz = fuzz,
1504 .ret_ptr = ret_ptr,
1505 .args = args.items,
15061348 .arg_index = 0,
15071349 .arg_inline_index = 0,
15081350 .func_inst_table = .empty,
......@@ -1516,14 +1358,18 @@ pub const Object = struct {
15161358 .base_line = zcu.navSrcLine(func.owner_nav),
15171359 .prev_dbg_line = 0,
15181360 .prev_dbg_column = 0,
1519 .err_ret_trace = err_ret_trace,
15201361 .disable_intrinsics = disable_intrinsics,
15211362 .allowzero_access = false,
1363
1364 .ret_ptr = undefined, // populated by `genMainBody`
1365 .err_ret_trace = undefined, // populated by `genMainBody`
1366 .args = undefined, // populated by `genMainBody`
15221367 };
15231368 defer fg.deinit();
1524 deinit_wip = false;
15251369
1526 try fg.genBody(air.getMainBody(), .poi);
1370 fg.wip.cursor = .{ .block = try fg.wip.block(0, "Entry") };
1371
1372 try fg.genMainBody();
15271373
15281374 // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole
15291375 // function as considering null pointers valid so that LLVM's optimizers don't remove these
......@@ -4074,8 +3920,9 @@ pub const Object = struct {
40743920 if (gop.found_existing) {
40753921 // Keep the greater of the two alignments.
40763922 const llvm_variable = gop.value_ptr.*;
4077 const old_align: InternPool.Alignment = .fromLlvm(llvm_variable.getAlignment(&o.builder));
4078 llvm_variable.setAlignment(old_align.maxStrict(@"align").toLlvm(), &o.builder);
3923 const llvm_old_align = llvm_variable.getAlignment(&o.builder);
3924 const llvm_new_align = llvm_old_align.max(@"align".toLlvm());
3925 llvm_variable.setAlignment(llvm_new_align, &o.builder);
40793926 return llvm_variable.ptrConst(&o.builder).global.toConst();
40803927 }
40813928 errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));
......@@ -4411,13 +4258,6 @@ pub const Object = struct {
44114258 toLlvmAddressSpace(.generic, o.zcu.getTarget()),
44124259 );
44134260 }
4414
4415 pub fn ptraddConst(o: *Object, wip: *Builder.WipFunction, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
4416 if (offset == 0) return ptr;
4417 const llvm_usize_ty = try o.lowerType(.usize);
4418 const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
4419 return wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
4420 }
44214261};
44224262
44234263const CallingConventionInfo = struct {
src/codegen/llvm/FuncGen.zig+679-623
......@@ -175,7 +175,158 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
175175 }
176176}
177177
178pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void {
178/// Populates `fg.ret_ptr`, `fg.err_ret_trace`, and `fg.args` based on the parameters of the
179/// function type, then generates the entire function body.
180///
181/// The caller may initialize `fg.ret_ptr`, `fg.err_ret_trace`, and `fg.args` to undefined.
182pub fn genMainBody(fg: *FuncGen) TodoError!void {
183 const o = fg.object;
184 const zcu = o.zcu;
185 const ip = &zcu.intern_pool;
186 const comp = zcu.comp;
187 const gpa = comp.gpa;
188
189 const fn_ty: Type = .fromInterned(ip.getNav(fg.nav_index).resolved.?.type);
190 const fn_info = zcu.typeToFunc(fn_ty).?;
191 const param_types = fn_info.param_types.get(ip);
192
193 var it = iterateParamTypes(o, fn_info);
194
195 // Populate `fg.ret_ptr`...
196 if (firstParamSRet(fn_info, zcu, zcu.getTarget())) {
197 fg.ret_ptr = fg.wip.arg(it.llvm_index);
198 it.llvm_index += 1;
199 } else {
200 fg.ret_ptr = .none;
201 }
202 // ...and `fg.err_ret_trace`...
203 if (fn_info.cc == .auto and comp.config.any_error_tracing) {
204 fg.err_ret_trace = fg.wip.arg(it.llvm_index);
205 it.llvm_index += 1;
206 } else {
207 fg.err_ret_trace = .none;
208 }
209 // ...and as for `fg.args`, we'll put all of the arguments into this ArrayList, and once that's
210 // done we'll use its buffer as `fg.args`.
211 var args: std.ArrayList(Builder.Value) = .empty;
212 defer args.deinit(gpa);
213
214 while (try it.next()) |lowering| {
215 try args.ensureUnusedCapacity(gpa, 1);
216
217 switch (lowering) {
218 .no_bits => continue,
219 .byval => {
220 assert(!it.byval_attr);
221 const param_index = it.zig_index - 1;
222 const param_ty: Type = .fromInterned(param_types[param_index]);
223 const param = fg.wip.arg(it.llvm_index - 1);
224
225 if (isByRef(param_ty, zcu)) {
226 const alignment = param_ty.abiAlignment(zcu).toLlvm();
227 const arg_ptr = try fg.buildAlloca(try o.lowerType(param_ty), alignment);
228 // We don't need to handle non-ABI-sized integer types in memory here since they
229 // are never by-ref.
230 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
231 args.appendAssumeCapacity(arg_ptr);
232 } else {
233 args.appendAssumeCapacity(param);
234 }
235 },
236 .byref, .byref_mut => {
237 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
238 const param = fg.wip.arg(it.llvm_index - 1);
239
240 if (isByRef(param_ty, zcu)) {
241 args.appendAssumeCapacity(param);
242 } else {
243 args.appendAssumeCapacity(try fg.load(param, .none, param_ty, .normal));
244 }
245 },
246 .abi_sized_int => {
247 assert(!it.byval_attr);
248 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
249 const param = fg.wip.arg(it.llvm_index - 1);
250
251 const param_llvm_ty = try o.lowerType(param_ty);
252 const alignment = param_ty.abiAlignment(zcu).toLlvm();
253 const arg_ptr = try fg.buildAlloca(param_llvm_ty, alignment);
254 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
255
256 if (isByRef(param_ty, zcu)) {
257 args.appendAssumeCapacity(arg_ptr);
258 } else {
259 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
260 }
261 },
262 .slice => {
263 assert(!it.byval_attr);
264 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
265 assert(!isByRef(param_ty, zcu));
266 const slice_val = try fg.wip.buildAggregate(
267 try o.lowerType(param_ty),
268 &.{ fg.wip.arg(it.llvm_index - 2), fg.wip.arg(it.llvm_index - 1) },
269 "",
270 );
271 args.appendAssumeCapacity(slice_val);
272 },
273 .multiple_llvm_types => {
274 assert(!it.byval_attr);
275 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
276 const param_alignment = param_ty.abiAlignment(zcu);
277 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
278 const arg_ptr = try fg.buildAlloca(llvm_ty, param_alignment.toLlvm());
279 const llvm_args_start = it.llvm_index - it.types_len;
280 for (llvm_args_start.., it.offsets_buffer[0..it.types_len]) |llvm_arg_index, offset| {
281 const param = fg.wip.arg(@intCast(llvm_arg_index));
282 const part_ptr = try fg.ptraddConst(arg_ptr, offset);
283 _ = try fg.wip.store(.normal, param, part_ptr, param_alignment.offset(offset).toLlvm());
284 }
285
286 if (isByRef(param_ty, zcu)) {
287 args.appendAssumeCapacity(arg_ptr);
288 } else {
289 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
290 }
291 },
292 .float_array => {
293 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
294 const param_llvm_ty = try o.lowerType(param_ty);
295 const param = fg.wip.arg(it.llvm_index - 1);
296
297 const alignment = param_ty.abiAlignment(zcu).toLlvm();
298 const arg_ptr = try fg.buildAlloca(param_llvm_ty, alignment);
299 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
300
301 if (isByRef(param_ty, zcu)) {
302 args.appendAssumeCapacity(arg_ptr);
303 } else {
304 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
305 }
306 },
307 .i32_array, .i64_array => {
308 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
309 const param = fg.wip.arg(it.llvm_index - 1);
310
311 const alignment = param_ty.abiAlignment(zcu).toLlvm();
312 const arg_ptr = try fg.buildAlloca(param.typeOfWip(&fg.wip), alignment);
313 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
314
315 if (isByRef(param_ty, zcu)) {
316 args.appendAssumeCapacity(arg_ptr);
317 } else {
318 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
319 }
320 },
321 }
322 }
323
324 fg.args = args.items;
325
326 try fg.genBody(fg.air.getMainBody(), .poi);
327}
328
329fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void {
179330 const o = self.object;
180331 const zcu = self.object.zcu;
181332 const ip = &zcu.intern_pool;
......@@ -400,8 +551,8 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
400551 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
401552 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
402553
403 .unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),
404 .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),
554 .unwrap_errunion_payload => try self.airErrUnionPayload(inst),
555 .unwrap_errunion_payload_ptr => try self.airErrUnionPayloadPtr(inst),
405556 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),
406557 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
407558 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
......@@ -644,6 +795,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
644795 const llvm_param_ty = try o.lowerType(param_ty);
645796 if (isByRef(param_ty, zcu)) {
646797 const alignment = param_ty.abiAlignment(zcu).toLlvm();
798 // We don't need to handle non-ABI-sized integer types in memory here since they are
799 // never by-ref.
647800 const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
648801 try llvm_args.append(loaded);
649802 } else {
......@@ -660,7 +813,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
660813 const alignment = param_ty.abiAlignment(zcu).toLlvm();
661814 const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);
662815 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
663 _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
816 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
664817 try llvm_args.append(arg_ptr);
665818 }
666819 },
......@@ -672,12 +825,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
672825 const alignment = param_ty.abiAlignment(zcu).toLlvm();
673826 const param_llvm_ty = try o.lowerType(param_ty);
674827 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
675 if (isByRef(param_ty, zcu)) {
676 const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
677 _ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
678 } else {
679 _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
680 }
828 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
681829 try llvm_args.append(arg_ptr);
682830 },
683831 .abi_sized_int => {
......@@ -694,9 +842,9 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
694842 // LLVM does not allow bitcasting structs so we must allocate
695843 // a local, store as one type, and then load as another type.
696844 const alignment = param_ty.abiAlignment(zcu).toLlvm();
697 const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);
698 _ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);
699 const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");
845 const ptr = try self.buildAlloca(int_llvm_ty, alignment);
846 try self.store(ptr, .none, llvm_arg, param_ty, .normal);
847 const loaded = try self.wip.load(.normal, int_llvm_ty, ptr, alignment, "");
700848 try llvm_args.append(loaded);
701849 }
702850 },
......@@ -711,19 +859,10 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
711859 const arg = args[it.zig_index - 1];
712860 const param_ty = self.typeOf(arg);
713861 const llvm_arg = try self.resolveInst(arg);
714 const is_by_ref = isByRef(param_ty, zcu);
715862 const param_alignment = param_ty.abiAlignment(zcu);
716863 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
717864 const arg_ptr = try self.buildAlloca(llvm_ty, param_alignment.toLlvm());
718 if (is_by_ref) _ = try self.wip.callMemCpy(
719 arg_ptr,
720 param_alignment.toLlvm(),
721 llvm_arg,
722 param_alignment.toLlvm(),
723 try o.builder.intValue(try o.lowerType(.usize), param_ty.abiSize(zcu)),
724 .normal,
725 self.disable_intrinsics,
726 ) else _ = try self.wip.store(.normal, llvm_arg, arg_ptr, param_alignment.toLlvm());
865 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
727866
728867 try llvm_args.ensureUnusedCapacity(it.types_len);
729868 for (it.types_buffer[0..it.types_len], it.offsets_buffer[0..it.types_len]) |field_ty, offset| {
......@@ -735,35 +874,38 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
735874 .float_array => |count| {
736875 const arg = args[it.zig_index - 1];
737876 const arg_ty = self.typeOf(arg);
738 var llvm_arg = try self.resolveInst(arg);
739 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
740 if (!isByRef(arg_ty, zcu)) {
741 const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
742 _ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
743 llvm_arg = ptr;
744 }
877 const arg_val = try self.resolveInst(arg);
878
879 const arg_align = arg_ty.abiAlignment(zcu);
880
881 const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
882 const ptr = try self.buildAlloca(try o.lowerType(arg_ty), arg_align.toLlvm());
883 try self.store(ptr, .none, arg_val, arg_ty, .normal);
884 break :ptr ptr;
885 } else arg_val;
745886
746887 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);
747888 const array_ty = try o.builder.arrayType(count, float_ty);
748889
749 const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
890 const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_align.toLlvm(), "");
750891 try llvm_args.append(loaded);
751892 },
752893 .i32_array, .i64_array => |arr_len| {
753894 const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
754895 const arg = args[it.zig_index - 1];
755896 const arg_ty = self.typeOf(arg);
756 var llvm_arg = try self.resolveInst(arg);
757 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
758 if (!isByRef(arg_ty, zcu)) {
759 const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
760 _ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
761 llvm_arg = ptr;
762 }
897 const arg_val = try self.resolveInst(arg);
763898
764 const array_ty =
765 try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
766 const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
899 const arg_align = arg_ty.abiAlignment(zcu);
900
901 const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
902 const ptr = try self.buildAlloca(try o.lowerType(arg_ty), arg_align.toLlvm());
903 try self.store(ptr, .none, arg_val, arg_ty, .normal);
904 break :ptr ptr;
905 } else arg_val;
906
907 const array_ty = try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
908 const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_align.toLlvm(), "");
767909 try llvm_args.append(loaded);
768910 },
769911 };
......@@ -875,8 +1017,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
8751017 return rp;
8761018 } else {
8771019 // our by-ref status disagrees with sret so we must load.
878 const return_alignment = return_type.abiAlignment(zcu).toLlvm();
879 return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, "");
1020 return self.load(rp, .none, return_type, .normal);
8801021 }
8811022 }
8821023
......@@ -888,11 +1029,14 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
8881029 // by using our canonical type, then loading it if necessary.
8891030 const alignment = return_type.abiAlignment(zcu).toLlvm();
8901031 const rp = try self.buildAlloca(abi_ret_ty, alignment);
1032 // We don't need to handle non-ABI-sized integer types in memory here since they can only be
1033 // returned from `CallingConvention.auto` functions, in which case `abi_ret_ty` will equal
1034 // `llvm_ret_ty` anyway.
8911035 _ = try self.wip.store(.normal, call, rp, alignment);
8921036 return if (isByRef(return_type, zcu))
8931037 rp
8941038 else
895 try self.wip.load(.normal, llvm_ret_ty, rp, alignment, "");
1039 try self.load(rp, .none, return_type, .normal);
8961040 }
8971041
8981042 if (isByRef(return_type, zcu)) {
......@@ -900,6 +1044,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
9001044 // and return the allocation pointer.
9011045 const alignment = return_type.abiAlignment(zcu).toLlvm();
9021046 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1047 // We don't need to handle non-ABI-sized integer types here since they are never by-ref.
9031048 _ = try self.wip.store(.normal, call, rp, alignment);
9041049 return rp;
9051050 } else {
......@@ -969,12 +1114,7 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
9691114 return;
9701115 }
9711116
972 try self.store(
973 self.ret_ptr,
974 .none,
975 operand,
976 ret_ty,
977 );
1117 try self.store(self.ret_ptr, .none, operand, ret_ty, .normal);
9781118 _ = try self.wip.retVoid();
9791119 return;
9801120 }
......@@ -991,18 +1131,18 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
9911131 return;
9921132 }
9931133
1134 const llvm_ret_ty = try o.lowerType(ret_ty);
9941135 const abi_ret_ty = try lowerFnRetTy(o, fn_info);
9951136 const operand = try self.resolveInst(un_op);
9961137 const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
997 const alignment = ret_ty.abiAlignment(zcu).toLlvm();
1138 const ret_ty_align = ret_ty.abiAlignment(zcu);
9981139
9991140 if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
1000 const llvm_ret_ty = operand.typeOfWip(&self.wip);
1001 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1141 const rp = try self.buildAlloca(llvm_ret_ty, ret_ty_align.toLlvm());
10021142 const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));
10031143 _ = try self.wip.callMemSet(
10041144 rp,
1005 alignment,
1145 ret_ty_align.toLlvm(),
10061146 try o.builder.intValue(.i8, 0xaa),
10071147 len,
10081148 .normal,
......@@ -1012,27 +1152,37 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
10121152 if (owner_mod.valgrind) {
10131153 try self.valgrindMarkUndef(rp, len);
10141154 }
1015 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
1155 if (fn_info.cc == .auto and abi_ret_ty == llvm_ret_ty) {
1156 assert(!isByRef(ret_ty, zcu));
1157 // The return type could be a non-ABI-sized integer, so use `FuncGen.load` to make sure
1158 // we load it from memory correctly.
1159 const loaded = try self.load(rp, .none, ret_ty, .normal);
1160 _ = try self.wip.ret(loaded);
1161 } else {
1162 const loaded = try self.wip.load(.normal, abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
1163 _ = try self.wip.ret(loaded);
1164 }
10161165 return;
10171166 }
10181167
10191168 if (isByRef(ret_ty, zcu)) {
1020 // operand is a pointer however self.ret_ptr is null so that means
1021 // we need to return a value.
1022 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, ""));
1169 // operand is a pointer however self.ret_ptr is null so that means we need to return a value.
1170 // No need to handle non-ABI-sized integer types in memory here since they are never by-ref.
1171 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, ret_ty_align.toLlvm(), ""));
10231172 return;
10241173 }
10251174
1026 const llvm_ret_ty = operand.typeOfWip(&self.wip);
10271175 if (abi_ret_ty == llvm_ret_ty) {
10281176 _ = try self.wip.ret(operand);
1029 return;
1177 } else {
1178 const rp = try self.buildAlloca(llvm_ret_ty, ret_ty_align.toLlvm());
1179 try self.store(rp, .none, operand, ret_ty, .normal);
1180 // No need to handle non-ABI-sized integer types in memory here since they can only be
1181 // returned from `CallingConvention.auto` functions, in which case `abi_ret_ty` will equal
1182 // `llvm_ret_ty` anyway.
1183 const ret_val = try self.wip.load(.normal, abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
1184 _ = try self.wip.ret(ret_val);
10301185 }
1031
1032 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1033 _ = try self.wip.store(.normal, operand, rp, alignment);
1034 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
1035 return;
10361186}
10371187
10381188fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
......@@ -1043,26 +1193,24 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
10431193 const ptr_ty = self.typeOf(un_op);
10441194 const ret_ty = ptr_ty.childType(zcu);
10451195 const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1046 if (!ret_ty.hasRuntimeBits(zcu)) {
1047 if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
1048 // Functions with an empty error set are emitted with an error code
1049 // return type and return zero so they can be function pointers coerced
1050 // to functions that return anyerror.
1051 _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0));
1052 } else {
1053 _ = try self.wip.retVoid();
1054 }
1055 return;
1056 }
1057 if (self.ret_ptr != .none) {
1196 if (!ret_ty.hasRuntimeBits(zcu) or self.ret_ptr != .none) {
10581197 _ = try self.wip.retVoid();
10591198 return;
10601199 }
10611200 const ptr = try self.resolveInst(un_op);
1201 const llvm_ret_ty = try o.lowerType(ret_ty);
10621202 const abi_ret_ty = try lowerFnRetTy(o, fn_info);
1063 const alignment = ret_ty.abiAlignment(zcu).toLlvm();
1064 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, ""));
1065 return;
1203 if (fn_info.cc == .auto and abi_ret_ty == llvm_ret_ty) {
1204 assert(!isByRef(ret_ty, zcu));
1205 // The return type could be a non-ABI-sized integer, so use `FuncGen.load` to make sure we
1206 // load it from memory correctly.
1207 const loaded = try self.load(ptr, .none, ret_ty, .normal);
1208 _ = try self.wip.ret(loaded);
1209 } else {
1210 const ret_ty_align = ret_ty.abiAlignment(zcu);
1211 const loaded = try self.wip.load(.normal, abi_ret_ty, ptr, ret_ty_align.toLlvm(), "");
1212 _ = try self.wip.ret(loaded);
1213 }
10661214}
10671215
10681216fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
......@@ -1089,7 +1237,7 @@ fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
10891237 return if (isByRef(va_list_ty, zcu))
10901238 dest_list
10911239 else
1092 try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
1240 try self.load(dest_list, .none, va_list_ty, .normal);
10931241}
10941242
10951243fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
......@@ -1113,7 +1261,7 @@ fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
11131261 return if (isByRef(va_list_ty, zcu))
11141262 dest_list
11151263 else
1116 try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
1264 try self.load(dest_list, .none, va_list_ty, .normal);
11171265}
11181266
11191267fn airCmp(
......@@ -1147,13 +1295,7 @@ fn airCmpLteErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
11471295 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
11481296 const operand = try self.resolveInst(un_op);
11491297 const errors_len_ptr = try o.getErrorsLen();
1150 const errors_len_val = try self.wip.load(
1151 .normal,
1152 try o.errorIntType(),
1153 errors_len_ptr.toValue(&o.builder),
1154 Type.errorAbiAlignment(o.zcu).toLlvm(),
1155 "",
1156 );
1298 const errors_len_val = try self.load(errors_len_ptr.toValue(&o.builder), .none, .anyerror, .normal);
11571299 return self.wip.icmp(.ule, operand, errors_len_val, "");
11581300}
11591301
......@@ -1630,32 +1772,23 @@ fn lowerTry(
16301772 const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
16311773 const error_type = try o.errorIntType();
16321774
1633 const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{
1634 operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)),
1635 operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)),
1636 } else .{ .none, .none };
1775 const operand_align: InternPool.Alignment = if (operand_is_ptr) operand_ptr_align else err_union_ty.abiAlignment(zcu);
16371776
16381777 if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
16391778 const loaded = loaded: {
1640 const access_kind: Builder.MemoryAccessKind =
1641 if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
1642
1643 if (!payload_has_bits) {
1644 break :loaded if (operand_is_ptr)
1645 try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "")
1646 else
1647 err_union;
1779 if (payload_has_bits) {
1780 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1781 } else if (!operand_is_ptr) {
1782 break :loaded err_union;
16481783 }
16491784
1650 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
16511785 const offset = codegen.errUnionErrorOffset(payload_ty, zcu);
16521786 const err_field_ptr = try fg.ptraddConst(err_union, offset);
1653 break :loaded try fg.wip.load(
1654 if (operand_is_ptr) access_kind else .normal,
1655 error_type,
1787 break :loaded try fg.load(
16561788 err_field_ptr,
1657 err_set_align.toLlvm(),
1658 "",
1789 operand_align.offset(offset),
1790 .anyerror,
1791 if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
16591792 );
16601793 };
16611794 const zero = try o.builder.intValue(error_type, 0);
......@@ -1672,15 +1805,18 @@ fn lowerTry(
16721805 fg.wip.cursor = .{ .block = continue_block };
16731806 }
16741807 if (is_unused) return .none;
1675 if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;
1676 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1677 const payload_ptr = try fg.ptraddConst(err_union, codegen.errUnionPayloadOffset(payload_ty, zcu));
1808
1809 if (!operand_is_ptr) {
1810 assert(payload_has_bits); // otherwise the result should be comptime-known
1811 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1812 }
1813
1814 const offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
1815 const payload_ptr = try fg.ptraddConst(err_union, offset);
16781816 if (operand_is_ptr) {
16791817 return payload_ptr;
1680 } else if (isByRef(payload_ty, zcu)) {
1681 return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
16821818 } else {
1683 return fg.wip.load(.normal, try o.lowerType(payload_ty), payload_ptr, payload_align.toLlvm(), "");
1819 return fg.load(payload_ptr, operand_align.offset(offset), payload_ty, .normal);
16841820 }
16851821}
16861822
......@@ -2144,11 +2280,7 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
21442280 const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
21452281 const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
21462282 self.maybeMarkAllowZeroAccess(slice_info);
2147 if (isByRef(elem_ty, zcu)) {
2148 return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind);
2149 } else {
2150 return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm());
2151 }
2283 return self.load(ptr, elem_align, elem_ty, access_kind);
21522284}
21532285
21542286fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
......@@ -2173,12 +2305,7 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
21732305 const elem_ty = array_ty.childType(zcu);
21742306 if (isByRef(array_ty, zcu)) {
21752307 const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu));
2176 if (isByRef(elem_ty, zcu)) {
2177 const elem_align = elem_ty.abiAlignment(zcu).toLlvm();
2178 return self.loadByRef(elem_ptr, elem_ty, elem_align, .normal);
2179 } else {
2180 return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
2181 }
2308 return self.load(elem_ptr, .none, elem_ty, .normal);
21822309 }
21832310
21842311 // This branch can be reached for vectors, which are always by-value.
......@@ -2197,8 +2324,8 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
21972324
21982325 return self.load(
21992326 try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)),
2327 ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)),
22002328 elem_ty,
2201 ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)).toLlvm(),
22022329 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
22032330 );
22042331}
......@@ -2294,11 +2421,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
22942421 const field_ptr = try self.ptraddConst(struct_llvm_val, offset);
22952422 const field_ptr_align = struct_ptr_align.offset(offset);
22962423
2297 if (isByRef(field_ty, zcu)) {
2298 return self.loadByRef(field_ptr, field_ty, field_ptr_align.toLlvm(), .normal);
2299 } else {
2300 return self.loadTruncate(.normal, field_ty, field_ptr, field_ptr_align.toLlvm());
2301 }
2424 return self.load(field_ptr, field_ptr_align, field_ty, .normal);
23022425}
23032426
23042427fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
......@@ -2439,8 +2562,8 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Er
24392562 // functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
24402563
24412564 const alignment = operand_ty.abiAlignment(zcu).toLlvm();
2442 const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment);
2443 _ = try self.wip.store(.normal, operand, alloca, alignment);
2565 const alloca = try self.buildAlloca(try o.lowerType(operand_ty), alignment);
2566 try self.store(alloca, .none, operand, operand_ty, .normal);
24442567 _ = try self.wip.callIntrinsic(
24452568 .normal,
24462569 .none,
......@@ -2609,8 +2732,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
26092732 } else {
26102733 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
26112734 const arg_llvm_ty = try o.lowerType(arg_ty);
2612 const load_inst =
2613 try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
2735 const load_inst = try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
26142736 llvm_param_values[llvm_param_i] = load_inst;
26152737 llvm_param_types[llvm_param_i] = arg_llvm_ty;
26162738 }
......@@ -2621,7 +2743,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
26212743 } else {
26222744 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
26232745 const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);
2624 _ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment);
2746 try self.store(arg_ptr, .none, arg_llvm_value, arg_ty, .normal);
26252747 llvm_param_values[llvm_param_i] = arg_ptr;
26262748 llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);
26272749 }
......@@ -2668,14 +2790,8 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
26682790 llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
26692791 llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
26702792 } else {
2671 const alignment = rw_ty.abiAlignment(zcu).toLlvm();
2672 const loaded = try self.wip.load(
2673 if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2674 llvm_elem_ty,
2675 llvm_rw_vals[output.index],
2676 alignment,
2677 "",
2678 );
2793 const access_kind: Builder.MemoryAccessKind = if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
2794 const loaded = try self.load(llvm_rw_vals[output.index], .none, rw_ty.childType(zcu), access_kind);
26792795 llvm_param_values[llvm_param_i] = loaded;
26802796 llvm_param_types[llvm_param_i] = llvm_elem_ty;
26812797 }
......@@ -2835,12 +2951,12 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
28352951 if (output != .none) {
28362952 const output_ptr = try self.resolveInst(output);
28372953 const output_ptr_ty = self.typeOf(output);
2838 const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm();
2839 _ = try self.wip.store(
2840 if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2841 output_value,
2954 try self.store(
28422955 output_ptr,
2843 alignment,
2956 output_ptr_ty.ptrAlignment(zcu),
2957 output_value,
2958 output_ptr_ty.childType(zcu),
2959 if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
28442960 );
28452961 } else {
28462962 ret_val = output_value;
......@@ -2863,7 +2979,6 @@ fn airIsNonNull(
28632979 const operand = try self.resolveInst(un_op);
28642980 const operand_ty = self.typeOf(un_op);
28652981 const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
2866 const optional_llvm_ty = try o.lowerType(optional_ty);
28672982 const payload_ty = optional_ty.optionalChild(zcu);
28682983
28692984 const access_kind: Builder.MemoryAccessKind =
......@@ -2873,7 +2988,7 @@ fn airIsNonNull(
28732988
28742989 if (optional_ty.optionalReprIsPayload(zcu)) {
28752990 const loaded = if (operand_is_ptr)
2876 try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
2991 try self.load(operand, operand_ty.ptrAlignment(zcu), optional_ty, access_kind)
28772992 else
28782993 operand;
28792994 if (payload_ty.isSlice(zcu)) {
......@@ -2884,14 +2999,14 @@ fn airIsNonNull(
28842999 ));
28853000 return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
28863001 }
2887 return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), "");
3002 return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(try o.lowerType(optional_ty)), "");
28883003 }
28893004
28903005 comptime assert(optional_layout_version == 3);
28913006
28923007 if (!payload_ty.hasRuntimeBits(zcu)) {
28933008 const loaded = if (operand_is_ptr)
2894 try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
3009 try self.load(operand, operand_ty.ptrAlignment(zcu), optional_ty, access_kind)
28953010 else
28963011 operand;
28973012 return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
......@@ -2932,7 +3047,7 @@ fn airIsErr(
29323047
29333048 if (!payload_ty.hasRuntimeBits(zcu)) {
29343049 const loaded = if (operand_is_ptr)
2935 try self.wip.load(access_kind, try o.lowerType(err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
3050 try self.load(operand, operand_ty.ptrAlignment(zcu), err_union_ty, access_kind)
29363051 else
29373052 operand;
29383053 return self.wip.icmp(cond, loaded, zero, "");
......@@ -2944,7 +3059,7 @@ fn airIsErr(
29443059 else
29453060 .none;
29463061 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
2947 const loaded = try self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");
3062 const loaded = try self.load(err_field_ptr, err_align, .anyerror, access_kind);
29483063 return self.wip.icmp(cond, loaded, zero, "");
29493064}
29503065
......@@ -2966,7 +3081,6 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
29663081 const optional_ptr_ty = self.typeOf(ty_op.operand);
29673082 const optional_ty = optional_ptr_ty.childType(zcu);
29683083 const payload_ty = optional_ty.optionalChild(zcu);
2969 const non_null_bit = try o.builder.intValue(.i8, 1);
29703084
29713085 const access_kind: Builder.MemoryAccessKind =
29723086 if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
......@@ -2976,7 +3090,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
29763090
29773091 // We have a pointer to a i8. We need to set it to 1 and then return the same pointer.
29783092 // Default alignment store because align of the non null bit is 1 anyway.
2979 _ = try self.wip.store(access_kind, non_null_bit, operand, .default);
3093 try self.store(operand, .@"1", .true, .bool, access_kind);
29803094 return operand;
29813095 }
29823096 if (optional_ty.optionalReprIsPayload(zcu)) {
......@@ -2992,7 +3106,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
29923106 self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
29933107
29943108 // Default alignment store because align of the non null bit is 1 anyway.
2995 _ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default);
3109 try self.store(non_null_ptr, .@"1", .true, .bool, access_kind);
29963110
29973111 // Then return the payload pointer (only if it's used).
29983112 if (self.liveness.isUnused(inst)) return .none;
......@@ -3016,31 +3130,29 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
30163130 return self.optPayloadHandle(operand, optional_ty, false);
30173131}
30183132
3019fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) Allocator.Error!Builder.Value {
3020 const o = self.object;
3133fn airErrUnionPayload(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3134 const o = fg.object;
30213135 const zcu = o.zcu;
3022 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3023 const operand = try self.resolveInst(ty_op.operand);
3024 const operand_ty = self.typeOf(ty_op.operand);
3025 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3026 const result_ty = self.typeOfIndex(inst);
3027 const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty;
3136 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3137 const operand = try fg.resolveInst(ty_op.operand);
3138 const err_union_ty = fg.typeOf(ty_op.operand);
3139 const payload_ty = fg.typeOfIndex(inst);
30283140
3029 if (!payload_ty.hasRuntimeBits(zcu)) {
3030 return if (operand_is_ptr) operand else .none;
3031 }
3032 const payload_ptr = try self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3033 if (operand_is_ptr) {
3034 return payload_ptr;
3035 }
3141 assert(payload_ty.hasRuntimeBits(zcu));
30363142 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3037 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
3038 if (isByRef(payload_ty, zcu)) {
3039 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
3040 } else {
3041 const payload_llvm_ty = try o.lowerType(payload_ty);
3042 return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");
3043 }
3143
3144 const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
3145 const payload_ptr = try fg.ptraddConst(operand, payload_offset);
3146 return fg.load(payload_ptr, err_union_ty.abiAlignment(zcu).offset(payload_offset), payload_ty, .normal);
3147}
3148
3149fn airErrUnionPayloadPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3150 const o = fg.object;
3151 const zcu = o.zcu;
3152 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3153 const operand = try fg.resolveInst(ty_op.operand);
3154 const payload_ty = fg.typeOfIndex(inst).childType(zcu);
3155 return fg.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
30443156}
30453157
30463158fn airErrUnionErr(
......@@ -3053,40 +3165,28 @@ fn airErrUnionErr(
30533165 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
30543166 const operand = try self.resolveInst(ty_op.operand);
30553167 const operand_ty = self.typeOf(ty_op.operand);
3056 const error_type = try o.errorIntType();
30573168 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3058 if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
3059 if (operand_is_ptr) {
3060 return operand;
3061 } else {
3062 return o.builder.intValue(error_type, 0);
3063 }
3064 }
30653169
30663170 const access_kind: Builder.MemoryAccessKind =
30673171 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
30683172
30693173 const payload_ty = err_union_ty.errorUnionPayload(zcu);
3070 if (!payload_ty.hasRuntimeBits(zcu)) {
3071 if (!operand_is_ptr) return operand;
3072
3073 self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
30743174
3075 return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");
3175 if (payload_ty.hasRuntimeBits(zcu)) {
3176 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3177 } else if (!operand_is_ptr) {
3178 return operand;
30763179 }
30773180
3078 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3079
30803181 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
30813182
3082 const err_align: InternPool.Alignment = a: {
3083 const err_abi_align = Type.anyerror.abiAlignment(zcu);
3084 if (!operand_is_ptr) break :a err_abi_align;
3085 break :a err_abi_align.minStrict(operand_ty.ptrAlignment(zcu));
3086 };
3183 const ptr_align = if (operand_is_ptr) operand_ty.ptrAlignment(zcu) else err_union_ty.abiAlignment(zcu);
30873184
3088 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3089 return self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");
3185 const err_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
3186 const err_align = ptr_align.offset(err_offset);
3187 const err_ptr = try self.ptraddConst(operand, err_offset);
3188
3189 return self.load(err_ptr, err_align, .anyerror, access_kind);
30903190}
30913191
30923192fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
......@@ -3107,10 +3207,10 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
31073207 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
31083208
31093209 {
3110 const error_align = Type.anyerror.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
31113210 // First set the non-error value.
3112 const error_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3113 _ = try self.wip.store(access_kind, non_error_val, error_ptr, error_align);
3211 const error_off = codegen.errUnionErrorOffset(payload_ty, zcu);
3212 const error_ptr = try self.ptraddConst(operand, error_off);
3213 try self.store(error_ptr, err_union_ptr_align.offset(error_off), non_error_val, .anyerror, access_kind);
31143214 }
31153215
31163216 // Then return the payload pointer (only if it is used).
......@@ -3142,7 +3242,7 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er
31423242 const field_offset = struct_ty.structFieldOffset(field_index, zcu);
31433243 const field_align = struct_ty.abiAlignment(zcu).offset(field_offset);
31443244 const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset);
3145 return self.load(field_ptr, field_ty, field_align.toLlvm(), .normal);
3245 return self.load(field_ptr, field_align, field_ty, .normal);
31463246}
31473247
31483248/// As an optimization, we want to avoid unnecessary copies of
......@@ -3174,7 +3274,6 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.
31743274 const inst = body_tail[0];
31753275 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
31763276 const payload_ty = self.typeOf(ty_op.operand);
3177 const non_null_bit = try o.builder.intValue(.i8, 1);
31783277 comptime assert(optional_layout_version == 3);
31793278 assert(payload_ty.hasRuntimeBits(zcu));
31803279 const operand = try self.resolveInst(ty_op.operand);
......@@ -3191,15 +3290,12 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.
31913290 };
31923291
31933292 const payload_ptr = optional_ptr; // payload always at offset 0
3194 try self.store(
3195 payload_ptr,
3196 .none,
3197 operand,
3198 payload_ty,
3199 );
3293 try self.store(payload_ptr, .none, operand, payload_ty, .normal);
3294
32003295 // Non-null bit immediately after payload (no padding because the bit has alignment 1).
32013296 const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu));
3202 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);
3297 try self.store(non_null_ptr, .none, .true, .bool, .normal);
3298
32033299 return optional_ptr;
32043300}
32053301
......@@ -3225,15 +3321,11 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) All
32253321 };
32263322
32273323 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3228 const error_alignment = Type.anyerror.abiAlignment(o.zcu).toLlvm();
3229 _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);
3324 try self.store(err_ptr, .none, ok_err_code, .anyerror, .normal);
3325
32303326 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3231 try self.store(
3232 payload_ptr,
3233 .none,
3234 operand,
3235 payload_ty,
3236 );
3327 try self.store(payload_ptr, .none, operand, payload_ty, .normal);
3328
32373329 return result_ptr;
32383330}
32393331
......@@ -3258,11 +3350,12 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocat
32583350 };
32593351
32603352 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3261 const error_alignment = Type.anyerror.abiAlignment(zcu).toLlvm();
3262 _ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
3353 try self.store(err_ptr, .none, operand, .anyerror, .normal);
3354
32633355 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
32643356 // TODO store undef to payload_ptr
32653357 _ = payload_ptr;
3358
32663359 return result_ptr;
32673360}
32683361
......@@ -3723,19 +3816,21 @@ fn airOverflow(
37233816 const result_val = try self.wip.extractValue(results, &.{0}, "");
37243817 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
37253818
3726 const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();
3727 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
3819 const result_alignment = inst_ty.abiAlignment(zcu);
3820 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment.toLlvm());
37283821
37293822 {
37303823 // Store to 'result: IntType' field
3731 const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(0, zcu));
3732 _ = try self.wip.store(.normal, result_val, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());
3824 const field_off = inst_ty.structFieldOffset(0, zcu);
3825 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
3826 try self.store(field_ptr, result_alignment.offset(field_off), result_val, lhs_ty, .normal);
37333827 }
37343828
37353829 {
37363830 // Store to 'overflow: u1' field
3737 const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(1, zcu));
3738 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
3831 const field_off = inst_ty.structFieldOffset(1, zcu);
3832 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
3833 try self.store(field_ptr, result_alignment.offset(field_off), overflow_bit, inst_ty.fieldType(1, zcu), .normal);
37393834 }
37403835
37413836 return alloca_inst;
......@@ -4064,19 +4159,21 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
40644159
40654160 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
40664161
4067 const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();
4068 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
4162 const result_alignment = dest_ty.abiAlignment(zcu);
4163 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment.toLlvm());
40694164
40704165 {
40714166 // Store to 'result: IntType' field
4072 const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(0, zcu));
4073 _ = try self.wip.store(.normal, result, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());
4167 const field_off = dest_ty.structFieldOffset(0, zcu);
4168 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
4169 try self.store(field_ptr, result_alignment.offset(field_off), result, lhs_ty, .normal);
40744170 }
40754171
40764172 {
40774173 // Store to 'overflow: u1' field
4078 const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(1, zcu));
4079 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
4174 const field_off = dest_ty.structFieldOffset(1, zcu);
4175 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
4176 try self.store(field_ptr, result_alignment.offset(field_off), overflow_bit, dest_ty.fieldType(1, zcu), .normal);
40804177 }
40814178
40824179 return alloca_inst;
......@@ -4264,7 +4361,7 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
42644361 .none,
42654362 .abs,
42664363 &.{try o.lowerType(operand_ty)},
4267 &.{ operand, try o.builder.intValue(.i1, 0) },
4364 &.{ operand, .false },
42684365 "",
42694366 ),
42704367 .float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),
......@@ -4483,7 +4580,8 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
44834580 }
44844581
44854582 if (inst_ty.isAbiInt(zcu) and operand_ty.isAbiInt(zcu)) {
4486 return self.wip.conv(.unsigned, operand, llvm_dest_ty, "");
4583 assert(inst_ty.bitSize(zcu) == operand_ty.bitSize(zcu));
4584 return operand;
44874585 }
44884586
44894587 const operand_scalar_ty = operand_ty.scalarType(zcu);
......@@ -4498,11 +4596,11 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
44984596 if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
44994597 const elem_ty = operand_scalar_ty;
45004598 assert(result_is_ref); // arrays are always by-ref provided they have runtime bits
4501 const alignment = inst_ty.abiAlignment(zcu).toLlvm();
4502 const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4599 const alignment = inst_ty.abiAlignment(zcu);
4600 const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment.toLlvm());
45034601 const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
45044602 if (bitcast_ok) {
4505 _ = try self.wip.store(.normal, operand, array_ptr, alignment);
4603 try self.store(array_ptr, alignment, operand, operand_ty, .normal);
45064604 } else {
45074605 // If the ABI size of the element type is not evenly divisible by size in bits;
45084606 // a simple bitcast will not work, and we fall back to extractelement.
......@@ -4512,7 +4610,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
45124610 while (i < vector_len) : (i += 1) {
45134611 const arr_elem_ptr = try self.ptraddConst(array_ptr, i * elem_size);
45144612 const vec_elem = try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
4515 _ = try self.wip.store(.normal, vec_elem, arr_elem_ptr, .default);
4613 try self.store(arr_elem_ptr, .none, vec_elem, elem_ty, .normal);
45164614 }
45174615 }
45184616 return array_ptr;
......@@ -4525,19 +4623,17 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
45254623 if (bitcast_ok) {
45264624 // The array is aligned to the element's alignment, while the vector might have a completely
45274625 // different alignment. This means we need to enforce the alignment of this load.
4528 const alignment = elem_ty.abiAlignment(zcu).toLlvm();
4529 return self.wip.load(.normal, llvm_vector_ty, operand, alignment, "");
4626 return self.load(operand, elem_ty.abiAlignment(zcu), inst_ty, .normal);
45304627 } else {
45314628 // If the ABI size of the element type is not evenly divisible by size in bits;
45324629 // a simple bitcast will not work, and we fall back to extractelement.
4533 const elem_llvm_ty = try o.lowerType(elem_ty);
45344630 const elem_size = elem_ty.abiSize(zcu);
45354631 const vector_len = operand_ty.arrayLen(zcu);
45364632 var vector = try o.builder.poisonValue(llvm_vector_ty);
45374633 var i: u64 = 0;
45384634 while (i < vector_len) : (i += 1) {
45394635 const arr_elem_ptr = try self.ptraddConst(operand, i * elem_size);
4540 const arr_elem = try self.wip.load(.normal, elem_llvm_ty, arr_elem_ptr, .default, "");
4636 const arr_elem = try self.load(arr_elem_ptr, .none, elem_ty, .normal);
45414637 vector = try self.wip.insertElement(vector, arr_elem, try o.builder.intValue(.i32, i), "");
45424638 }
45434639 return vector;
......@@ -4545,14 +4641,17 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
45454641 }
45464642
45474643 if (operand_is_ref) {
4548 const alignment = operand_ty.abiAlignment(zcu).toLlvm();
4549 return self.wip.load(.normal, llvm_dest_ty, operand, alignment, "");
4644 return self.load(operand, operand_ty.abiAlignment(zcu), inst_ty, .normal);
45504645 }
45514646
45524647 if (result_is_ref) {
4553 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
4554 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4555 _ = try self.wip.store(.normal, operand, result_ptr, alignment);
4648 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu));
4649 const llvm_alloc_ty = if (operand_ty.abiSize(zcu) > inst_ty.abiSize(zcu))
4650 try o.lowerType(operand_ty)
4651 else
4652 llvm_dest_ty;
4653 const result_ptr = try self.buildAlloca(llvm_alloc_ty, alignment.toLlvm());
4654 try self.store(result_ptr, alignment, operand, operand_ty, .normal);
45564655 return result_ptr;
45574656 }
45584657
......@@ -4563,10 +4662,10 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
45634662 // Both our operand and our result are values, not pointers,
45644663 // but LLVM won't let us bitcast struct values or vectors with padding bits.
45654664 // Therefore, we store operand to alloca, then load for result.
4566 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
4567 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4568 _ = try self.wip.store(.normal, operand, result_ptr, alignment);
4569 return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");
4665 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu));
4666 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment.toLlvm());
4667 try self.store(result_ptr, alignment, operand, operand_ty, .normal);
4668 return self.load(result_ptr, alignment, inst_ty, .normal);
45704669 }
45714670
45724671 return self.wip.cast(.bitcast, operand, llvm_dest_ty, "");
......@@ -4629,8 +4728,8 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
46294728 );
46304729 } else if (mod.optimize_mode == .Debug) {
46314730 const alignment = inst_ty.abiAlignment(zcu).toLlvm();
4632 const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment);
4633 _ = try self.wip.store(.normal, arg_val, alloca, alignment);
4731 const alloca = try self.buildAlloca(try o.lowerType(inst_ty), alignment);
4732 try self.store(alloca, .none, arg_val, inst_ty, .normal);
46344733 _ = try self.wip.callIntrinsic(
46354734 .normal,
46364735 .none,
......@@ -4689,28 +4788,56 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
46894788 return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
46904789}
46914790
4692/// Use this instead of builder.buildAlloca, because this function makes sure to
4693/// put the alloca instruction at the top of the function!
4791/// Unlike `WipFunction.alloca`, this puts the alloca instruction at the top of the function.
46944792fn buildAlloca(
4695 self: *FuncGen,
4793 fg: *FuncGen,
46964794 llvm_ty: Builder.Type,
46974795 alignment: Builder.Alignment,
46984796) Allocator.Error!Builder.Value {
4699 const target = self.object.zcu.getTarget();
4700 return buildAllocaInner(&self.wip, llvm_ty, alignment, target);
4797 const wip = &fg.wip;
4798
4799 const alloca = blk: {
4800 const prev_cursor = wip.cursor;
4801 const prev_debug_location = wip.debug_location;
4802 defer {
4803 wip.cursor = prev_cursor;
4804 if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
4805 wip.debug_location = prev_debug_location;
4806 }
4807
4808 wip.cursor = .{ .block = .entry };
4809 wip.debug_location = .no_location;
4810 const address_space = llvmAllocaAddressSpace(fg.object.zcu.getTarget());
4811 break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
4812 };
4813
4814 // The pointer returned from this function should have the generic address space,
4815 // if this isn't the case then cast it to the generic address space.
4816 return fg.wip.conv(.unneeded, alloca, .ptr, "");
47014817}
47024818
4703fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
4704 const o = self.object;
4819fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
4820 const o = fg.object;
47054821 const zcu = o.zcu;
4706 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4707 const dest_ptr = try self.resolveInst(bin_op.lhs);
4708 const ptr_ty = self.typeOf(bin_op.lhs);
4709 const operand_ty = ptr_ty.childType(zcu);
4822 const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4823 const ptr = try fg.resolveInst(bin_op.lhs);
4824 const ptr_ty = fg.typeOf(bin_op.lhs);
4825 const ptr_info = ptr_ty.ptrInfo(zcu);
4826 const ptr_alignment = ptr_ty.ptrAlignment(zcu);
4827
4828 const elem_ty = fg.typeOf(bin_op.rhs);
4829 assert(elem_ty.hasRuntimeBits(zcu));
4830
4831 fg.maybeMarkAllowZeroAccess(ptr_info);
4832
4833 const access_kind: Builder.MemoryAccessKind = switch (ptr_info.flags.is_volatile) {
4834 true => .@"volatile",
4835 false => .normal,
4836 };
47104837
47114838 const val_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
4712 if (val_is_undef and !self.needMemsetWorkaround(operand_ty.abiSize(zcu))) {
4713 const owner_mod = self.ownerModule();
4839 if (val_is_undef and !fg.needMemsetWorkaround(elem_ty.abiSize(zcu))) {
4840 const owner_mod = fg.ownerModule();
47144841
47154842 // Even if safety is disabled, we still emit a memset to undefined since it conveys
47164843 // extra information to LLVM, and LLVM will optimize it out. Safety makes the difference
......@@ -4725,7 +4852,6 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
47254852 return .none;
47264853 }
47274854
4728 const ptr_info = ptr_ty.ptrInfo(zcu);
47294855 const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
47304856 if (needs_bitmask) {
47314857 // TODO: only some bits are to be undef, we cannot write with a simple memset.
......@@ -4734,27 +4860,82 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
47344860 return .none;
47354861 }
47364862
4737 self.maybeMarkAllowZeroAccess(ptr_info);
4738
4739 const len = try o.builder.intValue(try o.lowerType(.usize), operand_ty.abiSize(zcu));
4740 _ = try self.wip.callMemSet(
4741 dest_ptr,
4742 ptr_ty.ptrAlignment(zcu).toLlvm(),
4863 const len = try o.builder.intValue(try o.lowerType(.usize), elem_ty.abiSize(zcu));
4864 _ = try fg.wip.callMemSet(
4865 ptr,
4866 ptr_alignment.toLlvm(),
47434867 if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
47444868 len,
4745 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
4746 self.disable_intrinsics,
4869 access_kind,
4870 fg.disable_intrinsics,
47474871 );
47484872 if (safety and owner_mod.valgrind) {
4749 try self.valgrindMarkUndef(dest_ptr, len);
4873 try fg.valgrindMarkUndef(ptr, len);
47504874 }
47514875 return .none;
47524876 }
47534877
4754 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
4878 const elem = try fg.resolveInst(bin_op.rhs);
4879
4880 if (ptr_info.flags.vector_index != .none) {
4881 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4882 const vec_ty = try fg.pt.vectorType(.{
4883 .len = ptr_info.packed_offset.host_size,
4884 .child = elem_ty.toIntern(),
4885 });
4886
4887 const loaded_vector = try fg.load(ptr, ptr_alignment, vec_ty, access_kind);
4888 const index_val = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
4889 const modified_vector = try fg.wip.insertElement(loaded_vector, elem, index_val, "");
4890
4891 try fg.store(ptr, ptr_alignment, modified_vector, vec_ty, access_kind);
4892 return .none;
4893 }
4894
4895 if (ptr_info.packed_offset.host_size != 0) {
4896 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4897 const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
4898 const llvm_backing_int_ty = try o.lowerType(backing_int_ty);
4899
4900 const backing_int_val = try fg.load(ptr, ptr_alignment, backing_int_ty, access_kind);
47554901
4756 const src_operand = try self.resolveInst(bin_op.rhs);
4757 try self.storeFull(dest_ptr, ptr_ty, src_operand, .none);
4902 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
4903 const shift_amt = try o.builder.intConst(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
4904
4905 // Convert to equally-sized integer type in order to perform the bit
4906 // operations on the value to store
4907 const new_val_bits_type = try o.builder.intType(@intCast(elem_bits));
4908 const new_val_bits = if (elem_ty.isPtrAtRuntime(zcu))
4909 try fg.wip.cast(.ptrtoint, elem, new_val_bits_type, "")
4910 else
4911 try fg.wip.cast(.bitcast, elem, new_val_bits_type, "");
4912
4913 const mask_val = blk: {
4914 const zext = try fg.wip.cast(
4915 .zext,
4916 try o.builder.intValue(new_val_bits_type, -1),
4917 llvm_backing_int_ty,
4918 "",
4919 );
4920 const shl = try fg.wip.bin(.shl, zext, shift_amt.toValue(), "");
4921 break :blk try fg.wip.bin(
4922 .xor,
4923 shl,
4924 try o.builder.intValue(llvm_backing_int_ty, -1),
4925 "",
4926 );
4927 };
4928
4929 const masked_backing_int_val = try fg.wip.bin(.@"and", backing_int_val, mask_val, "");
4930 const extended_new_val = try fg.wip.cast(.zext, new_val_bits, llvm_backing_int_ty, "");
4931 const shifted_new_val = try fg.wip.bin(.shl, extended_new_val, shift_amt.toValue(), "");
4932 const new_backing_int_val = try fg.wip.bin(.@"or", shifted_new_val, masked_backing_int_val, "");
4933
4934 try fg.store(ptr, ptr_alignment, new_backing_int_val, backing_int_ty, access_kind);
4935 return .none;
4936 }
4937
4938 try fg.store(ptr, ptr_alignment, elem, elem_ty, access_kind);
47584939 return .none;
47594940}
47604941
......@@ -4766,7 +4947,7 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
47664947 const ptr_info = ptr_ty.ptrInfo(zcu);
47674948 const ptr = try fg.resolveInst(ty_op.operand);
47684949 const elem_ty = ptr_ty.childType(zcu);
4769 const llvm_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();
4950 const ptr_align = ptr_ty.ptrAlignment(zcu);
47704951
47714952 fg.maybeMarkAllowZeroAccess(ptr_info);
47724953
......@@ -4774,36 +4955,32 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
47744955 if (ptr_info.flags.is_volatile) .@"volatile" else .normal;
47754956
47764957 if (ptr_info.flags.vector_index != .none) {
4777 const index_u32 = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
4778 const vec_elem_ty = try o.lowerType(elem_ty);
4779 const vec_ty = try o.builder.vectorType(.normal, ptr_info.packed_offset.host_size, vec_elem_ty);
4780
4781 const loaded_vector = try fg.wip.load(access_kind, vec_ty, ptr, llvm_ptr_align, "");
4782 return fg.wip.extractElement(loaded_vector, index_u32, "");
4958 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4959 const vec_ty = try fg.pt.vectorType(.{
4960 .len = ptr_info.packed_offset.host_size,
4961 .child = elem_ty.toIntern(),
4962 });
4963 const vector_val = try fg.load(ptr, ptr_align, vec_ty, access_kind);
4964 const index_val = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
4965 return fg.wip.extractElement(vector_val, index_val, "");
47834966 }
47844967
47854968 if (ptr_info.packed_offset.host_size == 0) {
4786 return fg.load(ptr, elem_ty, llvm_ptr_align, access_kind);
4969 return fg.load(ptr, ptr_align, elem_ty, access_kind);
47874970 }
47884971
4789 const containing_int_ty = try o.builder.intType(@intCast(ptr_info.packed_offset.host_size * 8));
4790 const containing_int =
4791 try fg.wip.load(access_kind, containing_int_ty, ptr, llvm_ptr_align, "");
4972 assert(!isByRef(elem_ty, zcu)); // all packable types are by-val
47924973
4793 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
4794 const shift_amt = try o.builder.intValue(containing_int_ty, ptr_info.packed_offset.bit_offset);
4795 const shifted_value = try fg.wip.bin(.lshr, containing_int, shift_amt, "");
4796 const elem_llvm_ty = try o.lowerType(elem_ty);
4974 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4975 const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
4976 const llvm_backing_int_ty = try o.lowerType(backing_int_ty);
47974977
4798 if (isByRef(elem_ty, zcu)) {
4799 const result_align = elem_ty.abiAlignment(zcu).toLlvm();
4800 const result_ptr = try fg.buildAlloca(elem_llvm_ty, result_align);
4978 const backing_int_val = try fg.load(ptr, ptr_align, backing_int_ty, .normal);
48014979
4802 const same_size_int = try o.builder.intType(@intCast(elem_bits));
4803 const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, "");
4804 _ = try fg.wip.store(.normal, truncated_int, result_ptr, result_align);
4805 return result_ptr;
4806 }
4980 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
4981 const shift_amt = try o.builder.intValue(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
4982 const shifted_value = try fg.wip.bin(.lshr, backing_int_val, shift_amt, "");
4983 const elem_llvm_ty = try o.lowerType(elem_ty);
48074984
48084985 if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
48094986 const same_size_int = try o.builder.intType(@intCast(elem_bits));
......@@ -4918,21 +5095,22 @@ fn airCmpxchg(
49185095 return self.wip.select(.normal, success_bit, zero, payload, "");
49195096 }
49205097
4921 assert(isByRef(optional_ty, zcu));
5098 assert(!isByRef(operand_ty, zcu)); // can only cmpxchg non-by-ref types
5099 assert(isByRef(optional_ty, zcu)); // all optionals are by-ref
49225100
49235101 comptime assert(optional_layout_version == 3);
49245102
49255103 const non_null_bit = try self.wip.not(success_bit, "");
49265104
4927 const payload_align = operand_ty.abiAlignment(zcu).toLlvm();
4928 const alloca_inst = try self.buildAlloca(try o.lowerType(optional_ty), payload_align);
5105 const payload_align = operand_ty.abiAlignment(zcu);
5106 const alloca_inst = try self.buildAlloca(try o.lowerType(optional_ty), payload_align.toLlvm());
49295107
49305108 // Payload is always the first field at offset 0, so address is `alloca_inst`
4931 _ = try self.wip.store(.normal, payload, alloca_inst, payload_align);
5109 try self.store(alloca_inst, .none, payload, operand_ty, .normal);
49325110
49335111 // Non-null bit is after payload with no padding because it has alignment 1
49345112 const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu));
4935 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, comptime .fromByteUnits(1));
5113 try self.store(non_null_ptr, payload_align, non_null_bit, .bool, .normal);
49365114
49375115 return alloca_inst;
49385116}
......@@ -5073,7 +5251,17 @@ fn airAtomicStore(
50735251
50745252 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
50755253
5076 try self.storeFull(ptr, ptr_ty, element, ordering);
5254 assert(!isByRef(operand_ty, zcu));
5255
5256 _ = try self.wip.storeAtomic(
5257 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
5258 element,
5259 ptr,
5260 self.sync_scope,
5261 ordering,
5262 ptr_ty.ptrAlignment(zcu).toLlvm(),
5263 );
5264
50775265 return .none;
50785266}
50795267
......@@ -5084,7 +5272,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
50845272 const dest_slice = try self.resolveInst(bin_op.lhs);
50855273 const ptr_ty = self.typeOf(bin_op.lhs);
50865274 const elem_ty = self.typeOf(bin_op.rhs);
5087 const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();
5275 const dest_ptr_align = ptr_ty.ptrAlignment(zcu);
50885276 const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);
50895277 const access_kind: Builder.MemoryAccessKind =
50905278 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
......@@ -5110,7 +5298,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
51105298 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
51115299 _ = try self.wip.callMemSet(
51125300 dest_ptr,
5113 dest_ptr_align,
5301 dest_ptr_align.toLlvm(),
51145302 fill_byte,
51155303 len,
51165304 access_kind,
......@@ -5132,7 +5320,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
51325320 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
51335321 _ = try self.wip.callMemSet(
51345322 dest_ptr,
5135 dest_ptr_align,
5323 dest_ptr_align.toLlvm(),
51365324 fill_byte,
51375325 len,
51385326 access_kind,
......@@ -5152,7 +5340,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
51525340
51535341 _ = try self.wip.callMemSet(
51545342 dest_ptr,
5155 dest_ptr_align,
5343 dest_ptr_align.toLlvm(),
51565344 fill_byte,
51575345 len,
51585346 access_kind,
......@@ -5182,7 +5370,6 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
51825370 const body_block = try self.wip.block(1, "InlineMemsetBody");
51835371 const end_block = try self.wip.block(1, "InlineMemsetEnd");
51845372
5185 const llvm_usize_ty = try o.lowerType(.usize);
51865373 const end_ptr = switch (ptr_ty.ptrSize(zcu)) {
51875374 .slice => try self.ptraddScaled(
51885375 dest_ptr,
......@@ -5201,18 +5388,8 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
52015388
52025389 self.wip.cursor = .{ .block = body_block };
52035390 const elem_abi_align = elem_ty.abiAlignment(zcu);
5204 const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm();
5205 if (isByRef(elem_ty, zcu)) {
5206 _ = try self.wip.callMemCpy(
5207 it_ptr.toValue(),
5208 it_ptr_align,
5209 value,
5210 elem_abi_align.toLlvm(),
5211 try o.builder.intValue(llvm_usize_ty, elem_abi_size),
5212 access_kind,
5213 self.disable_intrinsics,
5214 );
5215 } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
5391 const it_ptr_align: InternPool.Alignment = dest_ptr_align.min(elem_abi_align);
5392 try self.store(it_ptr.toValue(), it_ptr_align, value, elem_ty, access_kind);
52165393 const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size);
52175394 _ = try self.wip.br(loop_block);
52185395
......@@ -5289,14 +5466,11 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.
52895466
52905467 const union_ptr = try self.resolveInst(bin_op.lhs);
52915468 const new_tag = try self.resolveInst(bin_op.rhs);
5469 const tag_ty = self.typeOf(bin_op.rhs);
52925470 const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);
5293 if (layout.payload_size == 0) {
5294 _ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());
5295 return .none;
5296 }
52975471 const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());
52985472 const tag_ptr_align = union_ptr_align.offset(layout.tagOffset());
5299 _ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm());
5473 try self.store(tag_field_ptr, tag_ptr_align, new_tag, tag_ty, access_kind);
53005474 return .none;
53015475}
53025476
......@@ -5309,9 +5483,8 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.
53095483 assert(layout.tag_size != 0);
53105484 const operand = try self.resolveInst(ty_op.operand);
53115485 if (isByRef(un_ty, zcu)) {
5312 const llvm_tag_ty = try o.lowerType(un_ty.unionTagTypeRuntime(zcu).?);
53135486 const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());
5314 return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");
5487 return self.load(tag_field_ptr, .none, un_ty.unionTagTypeRuntime(zcu).?, .normal);
53155488 } else {
53165489 // This is only possible if all fields are zero-bit, in which case `operand` is already an
53175490 // integer value (the union is lowered as its enum tag).
......@@ -5480,14 +5653,13 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
54805653 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
54815654 const operand = try self.resolveInst(un_op);
54825655 const slice_ty = self.typeOfIndex(inst);
5483 const slice_llvm_ty = try o.lowerType(slice_ty);
54845656
54855657 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
54865658 const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize), "");
54875659
54885660 const error_name_table_ptr = try o.getErrorNameTable();
54895661 const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));
5490 return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");
5662 return self.load(error_name_ptr, .none, slice_ty, .normal);
54915663}
54925664
54935665fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
......@@ -5696,15 +5868,13 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
56965868/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.
56975869///
56985870/// Equivalent to:
5699/// reduce: {
5700/// var i: usize = 0;
5701/// var accum: T = init;
5702/// while (i < vec.len) : (i += 1) {
5703/// accum = llvm_fn(accum, vec[i]);
5704/// }
5705/// break :reduce accum;
5706/// }
5707///
5871/// ```
5872/// var accum: T = init;
5873/// for (0..i) |i| {
5874/// accum = llvm_fn(accum, vec[i]);
5875/// }
5876/// // result is 'accum'
5877/// ```
57085878fn buildReducedCall(
57095879 self: *FuncGen,
57105880 llvm_fn: Builder.Function.Index,
......@@ -5713,56 +5883,54 @@ fn buildReducedCall(
57135883 accum_init: Builder.Value,
57145884) Allocator.Error!Builder.Value {
57155885 const o = self.object;
5716 const usize_ty = try o.lowerType(.usize);
5717 const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len);
5886 const llvm_usize_ty = try o.lowerType(.usize);
5887 const llvm_vector_len = try o.builder.intValue(llvm_usize_ty, vector_len);
57185888 const llvm_result_ty = accum_init.typeOfWip(&self.wip);
57195889
5720 // Allocate and initialize our mutable variables
5721 const i_ptr = try self.buildAlloca(usize_ty, .default);
5722 _ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
5723 const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
5724 _ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
5725
5726 // Setup the loop
5727 const loop = try self.wip.block(2, "ReduceLoop");
5728 const loop_exit = try self.wip.block(1, "AfterReduce");
5729 _ = try self.wip.br(loop);
5730 {
5731 self.wip.cursor = .{ .block = loop };
5732
5733 // while (i < vec.len)
5734 const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, "");
5735 const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");
5736 const loop_then = try self.wip.block(1, "ReduceLoopThen");
5737
5738 _ = try self.wip.brCond(cond, loop_then, loop_exit, .none);
5739
5740 {
5741 self.wip.cursor = .{ .block = loop_then };
5890 const entry_block = self.wip.cursor.block;
57425891
5743 // accum = f(accum, vec[i]);
5744 const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
5745 const element = try self.wip.extractElement(operand_vector, i, "");
5746 const new_accum = try self.wip.call(
5747 .normal,
5748 .ccc,
5749 .none,
5750 llvm_fn.typeOf(&o.builder),
5751 llvm_fn.toValue(&o.builder),
5752 &.{ accum, element },
5753 "",
5754 );
5755 _ = try self.wip.store(.normal, new_accum, accum_ptr, .default);
5892 const cond_block = try self.wip.block(2, "ReduceLoopCond");
5893 const body_block = try self.wip.block(1, "ReduceLoopBody");
5894 const exit_block = try self.wip.block(1, "ReduceLoopExit");
5895
5896 _ = try self.wip.br(cond_block);
5897
5898 // ReduceLoopCond:
5899 // %index = phi iN [0, %Entry], [%new_index, %ReduceLoopBody]
5900 // %accum = phi T [%accum_init, %Entry], [%new_accum, %ReduceLoopBody]
5901 // %cond = icmp ult iN %index, %vector_len
5902 // br i1 %cond, label %ReduceLoopBody, label %ReduceLoopExit
5903 self.wip.cursor = .{ .block = cond_block };
5904 const index = try self.wip.phi(llvm_usize_ty, "");
5905 const accum = try self.wip.phi(llvm_result_ty, "");
5906 const cond = try self.wip.icmp(.ult, index.toValue(), llvm_vector_len, "");
5907 _ = try self.wip.brCond(cond, body_block, exit_block, .none);
5908
5909 // ReduceLoopBody:
5910 // %elem = extractelement <n x T> %operand_vec, iN %index
5911 // %new_accum = call T @llvm_fn(T %accum, T %elem)
5912 // %new_index = add nuw iN %index, 1
5913 // br label %ReduceLoopCond
5914 self.wip.cursor = .{ .block = body_block };
5915 const elem = try self.wip.extractElement(operand_vector, index.toValue(), "");
5916 const new_accum = try self.wip.call(
5917 .normal,
5918 .ccc,
5919 .none,
5920 llvm_fn.typeOf(&o.builder),
5921 llvm_fn.toValue(&o.builder),
5922 &.{ accum.toValue(), elem },
5923 "",
5924 );
5925 const new_index = try self.wip.bin(.@"add nuw", index.toValue(), try o.builder.intValue(llvm_usize_ty, 1), "");
5926 _ = try self.wip.br(cond_block);
57565927
5757 // i += 1
5758 const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), "");
5759 _ = try self.wip.store(.normal, new_i, i_ptr, .default);
5760 _ = try self.wip.br(loop);
5761 }
5762 }
5928 const index_init = try o.builder.intValue(llvm_usize_ty, 0);
5929 index.finish(&.{ index_init, new_index }, &.{ entry_block, body_block }, &self.wip);
5930 accum.finish(&.{ accum_init, new_accum }, &.{ entry_block, body_block }, &self.wip);
57635931
5764 self.wip.cursor = .{ .block = loop_exit };
5765 return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
5932 self.wip.cursor = .{ .block = exit_block };
5933 return new_accum;
57665934}
57675935
57685936fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
......@@ -5939,24 +6107,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
59396107
59406108 const llvm_field_val = try self.resolveInst(elem);
59416109
5942 if (isByRef(field_ty, zcu)) {
5943 _ = try self.wip.callMemCpy(
5944 field_ptr,
5945 field_ptr_align.toLlvm(),
5946 llvm_field_val,
5947 field_ty.abiAlignment(zcu).toLlvm(),
5948 try o.builder.intValue(try o.lowerType(.usize), field_ty.abiSize(zcu)),
5949 .normal,
5950 self.disable_intrinsics,
5951 );
5952 } else {
5953 _ = try self.wip.store(
5954 .normal,
5955 llvm_field_val,
5956 field_ptr,
5957 field_ptr_align.toLlvm(),
5958 );
5959 }
6110 try self.store(field_ptr, field_ptr_align, llvm_field_val, field_ty, .normal);
59606111 }
59616112
59626113 return alloca_inst;
......@@ -5975,12 +6126,12 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
59756126 for (elements, 0..) |elem, i| {
59766127 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i);
59776128 const llvm_elem = try self.resolveInst(elem);
5978 try self.store(elem_ptr, .none, llvm_elem, array_info.elem_type);
6129 try self.store(elem_ptr, .none, llvm_elem, array_info.elem_type, .normal);
59796130 }
59806131 if (array_info.sentinel) |sent_val| {
59816132 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len);
59826133 const llvm_elem = try self.resolveValue(sent_val);
5983 try self.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type);
6134 try self.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type, .normal);
59846135 }
59856136
59866137 return alloca_inst;
......@@ -6014,11 +6165,12 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
60146165
60156166 {
60166167 const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset());
6017 try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty);
6168 try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty, .normal);
60186169 }
60196170
60206171 if (layout.tag_size != 0) {
6021 const loaded_enum = ip.loadEnumType(union_obj.enum_tag_type);
6172 const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
6173 const loaded_enum = ip.loadEnumType(tag_ty.toIntern());
60226174 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {
60236175 .none => try o.builder.intConst(
60246176 try o.lowerType(.fromInterned(union_obj.enum_tag_type)),
......@@ -6027,7 +6179,7 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
60276179 else => |tag_val_ip| try o.lowerValue(tag_val_ip),
60286180 };
60296181 const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());
6030 _ = try self.wip.store(.normal, llvm_tag_val.toValue(), tag_ptr, layout.tag_align.toLlvm());
6182 try self.store(tag_ptr, layout.tag_align, llvm_tag_val.toValue(), tag_ty, .normal);
60316183 }
60326184
60336185 return result_ptr;
......@@ -6134,7 +6286,7 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
61346286 // Load the work_group_* member from the struct as u16.
61356287 // Just treat the dispatch pointer as an array of u16 to keep things simple.
61366288 const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2);
6137 return self.wip.load(.normal, .i16, workgroup_size_ptr, comptime .fromByteUnits(2), "");
6289 return self.load(workgroup_size_ptr, .@"2", .u16, .normal);
61386290 },
61396291 .nvptx, .nvptx64 => {
61406292 return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
......@@ -6169,8 +6321,8 @@ fn optCmpNull(
61696321 comptime assert(optional_layout_version == 3);
61706322 // Non-null bit is always after the payload, with no padding because it has alignment 1.
61716323 const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu));
6172 const non_null = try self.wip.load(access_kind, .i8, non_null_ptr, .default, "");
6173 return self.wip.icmp(cond, non_null, try self.object.builder.intValue(.i8, 0), "");
6324 const non_null = try self.load(non_null_ptr, .@"1", .bool, access_kind);
6325 return self.wip.icmp(cond, non_null, .false, "");
61746326}
61756327
61766328/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
......@@ -6187,13 +6339,9 @@ fn optPayloadHandle(
61876339 // Payload is first field so always at the same address as the optional itself.
61886340 const payload_ptr = opt_ptr;
61896341
6190 const payload_align = payload_ty.abiAlignment(zcu).toLlvm();
6191 if (isByRef(payload_ty, zcu)) {
6192 if (can_elide_load) return payload_ptr;
6193 return fg.loadByRef(payload_ptr, payload_ty, payload_align, .normal);
6194 } else {
6195 return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_align);
6196 }
6342 if (can_elide_load and isByRef(payload_ty, zcu)) return payload_ptr;
6343
6344 return fg.load(payload_ptr, .none, payload_ty, .normal);
61976345}
61986346
61996347fn fieldPtr(
......@@ -6217,214 +6365,145 @@ fn fieldPtr(
62176365 return self.ptraddConst(aggregate_ptr, offset);
62186366}
62196367
6220/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.
6221fn loadTruncate(
6222 fg: *FuncGen,
6223 access_kind: Builder.MemoryAccessKind,
6224 payload_ty: Type,
6225 payload_ptr: Builder.Value,
6226 payload_alignment: Builder.Alignment,
6227) Allocator.Error!Builder.Value {
6228 // from https://llvm.org/docs/LangRef.html#load-instruction :
6229 // "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. "
6230 // => so load the byte aligned value and trunc the unwanted bits.
6231
6232 const o = fg.object;
6233 const zcu = o.zcu;
6234 const payload_llvm_ty = try o.lowerType(payload_ty);
6235 const abi_size = payload_ty.abiSize(zcu);
6236
6237 const load_llvm_ty = if (payload_ty.isAbiInt(zcu))
6238 try o.builder.intType(@intCast(abi_size * 8))
6239 else
6240 payload_llvm_ty;
6241 const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, "");
6242 const shifted = if (payload_llvm_ty != load_llvm_ty and zcu.getTarget().cpu.arch.endian() == .big)
6243 try fg.wip.bin(.lshr, loaded, try o.builder.intValue(
6244 load_llvm_ty,
6245 (payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8,
6246 ), "")
6247 else
6248 loaded;
6249
6250 return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, "");
6251}
6252
6253/// Load a by-ref type by constructing a new alloca and performing a memcpy.
6254fn loadByRef(
6255 fg: *FuncGen,
6256 ptr: Builder.Value,
6257 pointee_type: Type,
6258 ptr_alignment: Builder.Alignment,
6259 access_kind: Builder.MemoryAccessKind,
6260) Allocator.Error!Builder.Value {
6261 const o = fg.object;
6262 const pointee_llvm_ty = try o.lowerType(pointee_type);
6263 const result_align = InternPool.Alignment.fromLlvm(ptr_alignment)
6264 .max(pointee_type.abiAlignment(o.zcu)).toLlvm();
6265 const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);
6266 const size_bytes = pointee_type.abiSize(o.zcu);
6267 _ = try fg.wip.callMemCpy(
6268 result_ptr,
6269 result_align,
6270 ptr,
6271 ptr_alignment,
6272 try o.builder.intValue(try o.lowerType(.usize), size_bytes),
6273 access_kind,
6274 fg.disable_intrinsics,
6275 );
6276 return result_ptr;
6277}
6278
6279/// If `isByRef` returns `true` for `elem_ty`, this still performs a copy by memcpy'ing the value
6280/// into a new alloca.
6368/// Non-atomic, non-bitpacked load of type `load_ty` from pointer `ptr`.
6369///
6370/// `ptr` has alignment `ptr_align`, or `load_ty.abiAlignment(zcu)` if `ptr_align` is `.none`.
6371///
6372/// If `load_ty` is a by-ref type, then the value is copied to a new alloca with a memcpy, and a
6373/// pointer to that alloca is returned.
62816374fn load(
62826375 fg: *FuncGen,
62836376 ptr: Builder.Value,
6284 elem_ty: Type,
6285 ptr_alignment: Builder.Alignment,
6377 ptr_align: InternPool.Alignment,
6378 load_ty: Type,
62866379 access_kind: Builder.MemoryAccessKind,
62876380) Allocator.Error!Builder.Value {
6288 const zcu = fg.object.zcu;
6289 if (isByRef(elem_ty, zcu)) {
6290 return fg.loadByRef(ptr, elem_ty, ptr_alignment, access_kind);
6291 } else {
6292 return fg.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment);
6293 }
6294}
6295
6296fn storeFull(
6297 self: *FuncGen,
6298 ptr: Builder.Value,
6299 ptr_ty: Type,
6300 elem: Builder.Value,
6301 ordering: Builder.AtomicOrdering,
6302) Allocator.Error!void {
6303 const o = self.object;
6381 const o = fg.object;
63046382 const zcu = o.zcu;
6305 const info = ptr_ty.ptrInfo(zcu);
6306 const elem_ty = Type.fromInterned(info.child);
6307 if (!elem_ty.hasRuntimeBits(zcu)) {
6308 return;
6309 }
6310 const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
6311 const access_kind: Builder.MemoryAccessKind =
6312 if (info.flags.is_volatile) .@"volatile" else .normal;
6313
6314 if (info.flags.vector_index != .none) {
6315 const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
6316 const vec_elem_ty = try o.lowerType(elem_ty);
6317 const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
63186383
6319 const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, "");
6384 const abi_align = load_ty.abiAlignment(zcu);
6385 const abi_size = load_ty.abiSize(zcu);
63206386
6321 const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, "");
6322
6323 assert(ordering == .none);
6324 _ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment);
6325 return;
6326 }
6327
6328 if (info.packed_offset.host_size != 0) {
6329 const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
6330 assert(ordering == .none);
6331 const containing_int =
6332 try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, "");
6333 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
6334 const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset);
6335 // Convert to equally-sized integer type in order to perform the bit
6336 // operations on the value to store
6337 const value_bits_type = try o.builder.intType(@intCast(elem_bits));
6338 const value_bits = if (elem_ty.isPtrAtRuntime(zcu))
6339 try self.wip.cast(.ptrtoint, elem, value_bits_type, "")
6340 else
6341 try self.wip.cast(.bitcast, elem, value_bits_type, "");
6342
6343 const mask_val = blk: {
6344 const zext = try self.wip.cast(
6345 .zext,
6346 try o.builder.intValue(value_bits_type, -1),
6347 containing_int_ty,
6348 "",
6349 );
6350 const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), "");
6351 break :blk try self.wip.bin(
6352 .xor,
6353 shl,
6354 try o.builder.intValue(containing_int_ty, -1),
6355 "",
6356 );
6357 };
6358
6359 const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, "");
6360 const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, "");
6361 const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), "");
6362 const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, "");
6387 const llvm_load_ty = try o.lowerType(load_ty);
6388 const llvm_ptr_align: Builder.Alignment = switch (ptr_align) {
6389 .none => abi_align.toLlvm(),
6390 else => |a| a.toLlvm(),
6391 };
63636392
6364 assert(ordering == .none);
6365 _ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment);
6366 return;
6367 }
6368 if (!isByRef(elem_ty, zcu)) {
6369 _ = try self.wip.storeAtomic(
6370 access_kind,
6371 elem,
6393 if (isByRef(load_ty, zcu)) {
6394 const llvm_usize_ty = try o.lowerType(.usize);
6395 const result_ptr = try fg.buildAlloca(llvm_load_ty, abi_align.toLlvm());
6396 _ = try fg.wip.callMemCpy(
6397 result_ptr,
6398 abi_align.toLlvm(),
63726399 ptr,
6373 self.sync_scope,
6374 ordering,
6375 ptr_alignment,
6400 llvm_ptr_align,
6401 try o.builder.intValue(llvm_usize_ty, abi_size),
6402 access_kind,
6403 fg.disable_intrinsics,
63766404 );
6377 return;
6405 return result_ptr;
63786406 }
6379 assert(ordering == .none);
6380 _ = try self.wip.callMemCpy(
6381 ptr,
6382 ptr_alignment,
6383 elem,
6384 elem_ty.abiAlignment(zcu).toLlvm(),
6385 try o.builder.intValue(try o.lowerType(.usize), elem_ty.abiSize(zcu)),
6386 access_kind,
6387 self.disable_intrinsics,
6388 );
6407
6408 if (load_ty.isAbiInt(zcu) and load_ty.bitSize(zcu) != abi_size * 8) {
6409 // `load_ty` is an integer type with padding bits. In theory, we shouldn't need any special
6410 // handling for these, as LLVM's documented semantics are a valid implementation of Zig's
6411 // semantics. However:
6412 //
6413 // * LLVM's lowering for these integer types generally leads to poor codegen, as integers
6414 // are only extended to the next byte, instead of to the next "natural" integer type.
6415 //
6416 // * Clang never emits loads or stores of these types, so LLVM's support for them is rather
6417 // flaky---we have encountered several LLVM bugs caused by incorrect handling of them.
6418 //
6419 // Therefore, we handle these memory accesses specially: in this case we will actually load
6420 // the next-largest "natural" integer type and then truncate to `load_ty`.
6421 const llvm_abi_ty = try o.builder.intType(@intCast(abi_size * 8));
6422 const loaded = try fg.wip.load(access_kind, llvm_abi_ty, ptr, llvm_ptr_align, "");
6423 // For packed structs, current Zig semantics don't really allow us to make the padding bits
6424 // well-defined. This should be solved once https://github.com/ziglang/zig/issues/24061 is
6425 // implemented, but until then, do a normal trunc for packed types.
6426 return fg.wip.cast(switch (load_ty.zigTypeTag(zcu)) {
6427 .@"struct", .@"union" => .trunc,
6428 else => switch (load_ty.intInfo(zcu).signedness) {
6429 .unsigned => .@"trunc nuw",
6430 .signed => .@"trunc nsw",
6431 },
6432 }, loaded, llvm_load_ty, "");
6433 }
6434
6435 // `load_ty` is a simple by-val type which requires no special handling.
6436 return fg.wip.load(access_kind, llvm_load_ty, ptr, llvm_ptr_align, "");
63896437}
63906438
6391/// Non-atomic, non-volatile, non-packed store.
6439/// Non-atomic, non-bitpacked store of `elem` to pointer `ptr`.
6440///
6441/// `ptr` has alignment `ptr_align`, or `elem_ty.abiAlignment(zcu)` if `ptr_align` is `.none`.
6442///
6443/// If `elem_ty` is a by-ref type, then `elem` is itself a pointer, and a memcpy is emitted.
63926444fn store(
63936445 fg: *FuncGen,
63946446 ptr: Builder.Value,
63956447 ptr_align: InternPool.Alignment,
63966448 elem: Builder.Value,
63976449 elem_ty: Type,
6450 access_kind: Builder.MemoryAccessKind,
63986451) Allocator.Error!void {
63996452 const o = fg.object;
64006453 const zcu = o.zcu;
6454
6455 const abi_align = elem_ty.abiAlignment(zcu);
6456 const abi_size = elem_ty.abiSize(zcu);
6457
64016458 const llvm_ptr_align = switch (ptr_align) {
6402 .none => elem_ty.abiAlignment(zcu).toLlvm(),
6459 .none => abi_align.toLlvm(),
64036460 else => ptr_align.toLlvm(),
64046461 };
6462
64056463 if (isByRef(elem_ty, zcu)) {
6464 const llvm_usize_ty = try o.lowerType(.usize);
64066465 _ = try fg.wip.callMemCpy(
64076466 ptr,
64086467 llvm_ptr_align,
64096468 elem,
6410 elem_ty.abiAlignment(zcu).toLlvm(),
6411 try o.builder.intValue(
6412 try o.lowerType(.usize),
6413 elem_ty.abiSize(zcu),
6414 ),
6415 .normal,
6469 abi_align.toLlvm(),
6470 try o.builder.intValue(llvm_usize_ty, abi_size),
6471 access_kind,
64166472 fg.disable_intrinsics,
64176473 );
6418 } else {
6474 return;
6475 }
6476
6477 assert(elem.typeOfWip(&fg.wip) == try o.lowerType(elem_ty));
6478
6479 if (elem_ty.isAbiInt(zcu) and elem_ty.bitSize(zcu) != abi_size * 8) {
6480 // `elem_ty` is an integer type with padding bits, so we need to handle it specially---see
6481 // the corresponding comment in `FuncGen.load` for more details.
6482 const llvm_abi_ty = try o.builder.intType(@intCast(abi_size * 8));
6483 const extended = try fg.wip.cast(switch (elem_ty.intInfo(zcu).signedness) {
6484 .unsigned => .zext,
6485 .signed => .sext,
6486 }, elem, llvm_abi_ty, "");
64196487 _ = try fg.wip.storeAtomic(
6420 .normal,
6421 elem,
6488 access_kind,
6489 extended,
64226490 ptr,
64236491 fg.sync_scope,
64246492 .none,
64256493 llvm_ptr_align,
64266494 );
6495 return;
64276496 }
6497
6498 // `elem_ty` is a simple by-val type which requires no special handling.
6499 _ = try fg.wip.storeAtomic(
6500 access_kind,
6501 elem,
6502 ptr,
6503 fg.sync_scope,
6504 .none,
6505 llvm_ptr_align,
6506 );
64286507}
64296508
64306509fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
......@@ -6453,18 +6532,18 @@ fn valgrindClientRequest(
64536532 if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
64546533
64556534 const llvm_usize = try o.lowerType(.usize);
6456 const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm();
6535 const usize_align = Type.usize.abiAlignment(zcu).toLlvm();
64576536
64586537 const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
64596538 const array_ptr = if (fg.valgrind_client_request_array == .none) a: {
6460 const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment);
6539 const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_align);
64616540 fg.valgrind_client_request_array = array_ptr;
64626541 break :a array_ptr;
64636542 } else fg.valgrind_client_request_array;
64646543 const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
64656544 for (array_elements, 0..) |elem, i| {
64666545 const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu));
6467 _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);
6546 try fg.store(elem_ptr, .none, elem, .usize, .normal);
64686547 }
64696548
64706549 const arch_specific: struct {
......@@ -7283,33 +7362,6 @@ fn isScalar(zcu: *Zcu, ty: Type) bool {
72837362 };
72847363}
72857364
7286pub fn buildAllocaInner(
7287 wip: *Builder.WipFunction,
7288 llvm_ty: Builder.Type,
7289 alignment: Builder.Alignment,
7290 target: *const std.Target,
7291) Allocator.Error!Builder.Value {
7292 const address_space = llvmAllocaAddressSpace(target);
7293
7294 const alloca = blk: {
7295 const prev_cursor = wip.cursor;
7296 const prev_debug_location = wip.debug_location;
7297 defer {
7298 wip.cursor = prev_cursor;
7299 if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
7300 wip.debug_location = prev_debug_location;
7301 }
7302
7303 wip.cursor = .{ .block = .entry };
7304 wip.debug_location = .no_location;
7305 break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
7306 };
7307
7308 // The pointer returned from this function should have the generic address space,
7309 // if this isn't the case then cast it to the generic address space.
7310 return wip.conv(.unneeded, alloca, .ptr, "");
7311}
7312
73137365/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
73147366/// or as an LLVM value.
73157367pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
......@@ -7380,7 +7432,11 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E
73807432}
73817433
73827434fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
7383 return fg.object.ptraddConst(&fg.wip, ptr, offset);
7435 if (offset == 0) return ptr;
7436 const o = fg.object;
7437 const llvm_usize_ty = try o.lowerType(.usize);
7438 const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
7439 return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
73847440}
73857441fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value {
73867442 if (scale == 0) return ptr;