authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-03-22 20:11:35+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-03-28 16:45:55+00:00
log5941c9da08ae0d8ba4add37f0baeafdcd160dbd4
tree15f1ad30949184585ea9d658485fb5e0bdd2e033
parente3ee37f983ffe655b5e9516c12a1a0d69a967e12
signaturelock-open Commit is signed but in an unrecognized format.

llvm: remove almost all GEPs

LLVM is gradually transitioning from the `getelementptr` instruction to a new `ptradd` instruction. The latter instruction doesn't actually exist yet, but for now, LLVM is considering `getelementptr i8` to be equivalent. LLVM is already internally canonicalizing `getelementptr` usages to this pattern in many cases, and it's far easier for us to emit that, so... let's do so! For runtime indexing this does sometimes require an explicit multiplication to scale an index to a byte offset. The helper function `llvm.FuncGen.ptraddScaled` makes this common pattern more convenient. A particularly nice side effect from this is that after removing some dead code (left over from before we made all `struct`s etc by-ref), it has eliminated the need to maintain that nasty mapping between Zig field indices and LLVM field indices. `FuncGen` no longer cares at all how aggregate types are lowered! Slices are still by-val at least for now, but they never lived in that mapping because their structure is simple and consistent (they always have a pointer at field index 0 and a usize at field index 1, with no explicit padding necessary).

3 files changed, 406 insertions(+), 764 deletions(-)

src/Type.zig+1-1
...@@ -1594,7 +1594,7 @@ pub fn unionTagFieldIndex(ty: Type, enum_tag: Value, zcu: *const Zcu) ?u32 {...@@ -1594,7 +1594,7 @@ pub fn unionTagFieldIndex(ty: Type, enum_tag: Value, zcu: *const Zcu) ?u32 {
1594 return zcu.unionTagFieldIndex(union_obj, enum_tag);1594 return zcu.unionTagFieldIndex(union_obj, enum_tag);
1595}1595}
15961596
1597pub fn unionHasAllZeroBitFieldTypes(ty: Type, zcu: *Zcu) bool {1597pub fn unionHasAllZeroBitFieldTypes(ty: Type, zcu: *const Zcu) bool {
1598 assertHasLayout(ty, zcu);1598 assertHasLayout(ty, zcu);
1599 const ip = &zcu.intern_pool;1599 const ip = &zcu.intern_pool;
1600 const union_obj = zcu.typeToUnion(ty).?;1600 const union_obj = zcu.typeToUnion(ty).?;
src/codegen/llvm.zig+2-59
...@@ -578,24 +578,9 @@ pub const Object = struct {...@@ -578,24 +578,9 @@ pub const Object = struct {
578 /// Memoizes a null `?usize` value.578 /// Memoizes a null `?usize` value.
579 null_opt_usize: Builder.Constant,579 null_opt_usize: Builder.Constant,
580580
581 /// When an LLVM struct type is created, an entry is inserted into this
582 /// table for every zig source field of the struct that has a corresponding
583 /// LLVM struct field. comptime fields are not included. Zero-bit fields are
584 /// mapped to a field at the correct byte, which may be a padding field, or
585 /// are not mapped, in which case they are semantically at the end of the
586 /// struct.
587 /// The value is the LLVM struct field index.
588 /// This is denormalized data.
589 struct_field_map: std.AutoHashMapUnmanaged(ZigStructField, c_uint),
590
591 /// Values for `@llvm.used`.581 /// Values for `@llvm.used`.
592 used: std.ArrayList(Builder.Constant),582 used: std.ArrayList(Builder.Constant),
593583
594 const ZigStructField = struct {
595 struct_ty: InternPool.Index,
596 field_index: u32,
597 };
598
599 pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn;584 pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn;
600585
601 const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);586 const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);
...@@ -688,7 +673,6 @@ pub const Object = struct {...@@ -688,7 +673,6 @@ pub const Object = struct {
688 .type_map = .empty,673 .type_map = .empty,
689 .error_name_table = .none,674 .error_name_table = .none,
690 .null_opt_usize = .no_init,675 .null_opt_usize = .no_init,
691 .struct_field_map = .empty,
692 .used = .empty,676 .used = .empty,
693 };677 };
694 return obj;678 return obj;
...@@ -708,7 +692,6 @@ pub const Object = struct {...@@ -708,7 +692,6 @@ pub const Object = struct {
708 self.named_enum_map.deinit(gpa);692 self.named_enum_map.deinit(gpa);
709 self.type_map.deinit(gpa);693 self.type_map.deinit(gpa);
710 self.builder.deinit();694 self.builder.deinit();
711 self.struct_field_map.deinit(gpa);
712 self.* = undefined;695 self.* = undefined;
713 }696 }
714697
...@@ -3311,7 +3294,6 @@ pub const Object = struct {...@@ -3311,7 +3294,6 @@ pub const Object = struct {
3311 // Although we can estimate how much capacity to add, these cannot be3294 // Although we can estimate how much capacity to add, these cannot be
3312 // relied upon because of the recursive calls to lowerType below.3295 // relied upon because of the recursive calls to lowerType below.
3313 try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_type.field_types.len);3296 try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_type.field_types.len);
3314 try o.struct_field_map.ensureUnusedCapacity(o.gpa, struct_type.field_types.len);
33153297
3316 comptime assert(struct_layout_version == 2);3298 comptime assert(struct_layout_version == 2);
3317 var offset: u64 = 0;3299 var offset: u64 = 0;
...@@ -3336,23 +3318,8 @@ pub const Object = struct {...@@ -3336,23 +3318,8 @@ pub const Object = struct {
3336 try o.builder.arrayType(padding_len, .i8),3318 try o.builder.arrayType(padding_len, .i8),
3337 );3319 );
33383320
3339 if (!field_ty.hasRuntimeBits(zcu)) {3321 if (!field_ty.hasRuntimeBits(zcu)) continue;
3340 // This is a zero-bit field. If there are runtime bits after this field,
3341 // map to the next LLVM field (which we know exists): otherwise, don't
3342 // map the field, indicating it's at the end of the struct.
3343 if (offset != struct_type.size) {
3344 try o.struct_field_map.put(o.gpa, .{
3345 .struct_ty = t.toIntern(),
3346 .field_index = field_index,
3347 }, @intCast(llvm_field_types.items.len));
3348 }
3349 continue;
3350 }
33513322
3352 try o.struct_field_map.put(o.gpa, .{
3353 .struct_ty = t.toIntern(),
3354 .field_index = field_index,
3355 }, @intCast(llvm_field_types.items.len));
3356 try llvm_field_types.append(o.gpa, try o.lowerType(pt, field_ty));3323 try llvm_field_types.append(o.gpa, try o.lowerType(pt, field_ty));
33573324
3358 offset += field_ty.abiSize(zcu);3325 offset += field_ty.abiSize(zcu);
...@@ -3385,19 +3352,15 @@ pub const Object = struct {...@@ -3385,19 +3352,15 @@ pub const Object = struct {
3385 // Although we can estimate how much capacity to add, these cannot be3352 // Although we can estimate how much capacity to add, these cannot be
3386 // relied upon because of the recursive calls to lowerType below.3353 // relied upon because of the recursive calls to lowerType below.
3387 try llvm_field_types.ensureUnusedCapacity(o.gpa, tuple_type.types.len);3354 try llvm_field_types.ensureUnusedCapacity(o.gpa, tuple_type.types.len);
3388 try o.struct_field_map.ensureUnusedCapacity(o.gpa, tuple_type.types.len);
33893355
3390 comptime assert(struct_layout_version == 2);3356 comptime assert(struct_layout_version == 2);
3391 var offset: u64 = 0;3357 var offset: u64 = 0;
3392 var big_align: InternPool.Alignment = .none;3358 var big_align: InternPool.Alignment = .none;
33933359
3394 const struct_size = t.abiSize(zcu);
3395
3396 for (3360 for (
3397 tuple_type.types.get(ip),3361 tuple_type.types.get(ip),
3398 tuple_type.values.get(ip),3362 tuple_type.values.get(ip),
3399 0..,3363 ) |field_ty, field_val| {
3400 ) |field_ty, field_val, field_index| {
3401 if (field_val != .none) continue;3364 if (field_val != .none) continue;
34023365
3403 const field_align = Type.fromInterned(field_ty).abiAlignment(zcu);3366 const field_align = Type.fromInterned(field_ty).abiAlignment(zcu);
...@@ -3411,21 +3374,8 @@ pub const Object = struct {...@@ -3411,21 +3374,8 @@ pub const Object = struct {
3411 try o.builder.arrayType(padding_len, .i8),3374 try o.builder.arrayType(padding_len, .i8),
3412 );3375 );
3413 if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {3376 if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {
3414 // This is a zero-bit field. If there are runtime bits after this field,
3415 // map to the next LLVM field (which we know exists): otherwise, don't
3416 // map the field, indicating it's at the end of the struct.
3417 if (offset != struct_size) {
3418 try o.struct_field_map.put(o.gpa, .{
3419 .struct_ty = t.toIntern(),
3420 .field_index = @intCast(field_index),
3421 }, @intCast(llvm_field_types.items.len));
3422 }
3423 continue;3377 continue;
3424 }3378 }
3425 try o.struct_field_map.put(o.gpa, .{
3426 .struct_ty = t.toIntern(),
3427 .field_index = @intCast(field_index),
3428 }, @intCast(llvm_field_types.items.len));
3429 try llvm_field_types.append(o.gpa, try o.lowerType(pt, Type.fromInterned(field_ty)));3379 try llvm_field_types.append(o.gpa, try o.lowerType(pt, Type.fromInterned(field_ty)));
34303380
3431 offset += Type.fromInterned(field_ty).abiSize(zcu);3381 offset += Type.fromInterned(field_ty).abiSize(zcu);
...@@ -4338,13 +4288,6 @@ pub const Object = struct {...@@ -4338,13 +4288,6 @@ pub const Object = struct {
4338 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);4288 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);
4339 }4289 }
43404290
4341 pub fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint {
4342 return o.struct_field_map.get(.{
4343 .struct_ty = struct_ty.toIntern(),
4344 .field_index = @intCast(field_index),
4345 });
4346 }
4347
4348 /// MLUGG TODO: this is super dumb4291 /// MLUGG TODO: this is super dumb
4349 pub fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index {4292 pub fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index {
4350 const name = try o.builder.strtabString(lt_errors_fn_name);4293 const name = try o.builder.strtabString(lt_errors_fn_name);
src/codegen/llvm/FuncGen.zig+403-704
...@@ -141,7 +141,7 @@ pub fn deinit(self: *FuncGen) void {...@@ -141,7 +141,7 @@ pub fn deinit(self: *FuncGen) void {
141 self.switch_dispatch_info.deinit(gpa);141 self.switch_dispatch_info.deinit(gpa);
142}142}
143143
144fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value {144fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) Allocator.Error!Builder.Value {
145 const gpa = self.gpa;145 const gpa = self.gpa;
146 const gop = try self.func_inst_table.getOrPut(gpa, inst);146 const gop = try self.func_inst_table.getOrPut(gpa, inst);
147 if (gop.found_existing) return gop.value_ptr.*;147 if (gop.found_existing) return gop.value_ptr.*;
...@@ -151,7 +151,7 @@ fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value {...@@ -151,7 +151,7 @@ fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value {
151 return llvm_val.toValue();151 return llvm_val.toValue();
152}152}
153153
154fn resolveValue(self: *FuncGen, val: Value) Error!Builder.Constant {154fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
155 const o = self.object;155 const o = self.object;
156 const pt = self.pt;156 const pt = self.pt;
157 const zcu = pt.zcu;157 const zcu = pt.zcu;
...@@ -177,9 +177,7 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air...@@ -177,9 +177,7 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
177 .poi => if (self.fuzz) |*fuzz| {177 .poi => if (self.fuzz) |*fuzz| {
178 const poi_index = fuzz.pcs.items.len;178 const poi_index = fuzz.pcs.items.len;
179 const base_ptr = fuzz.counters_variable.toValue(&o.builder);179 const base_ptr = fuzz.counters_variable.toValue(&o.builder);
180 const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{180 const ptr = try self.ptraddConst(base_ptr, poi_index);
181 try o.builder.intValue(.i32, poi_index),
182 }, "");
183 const one = try o.builder.intValue(.i8, 1);181 const one = try o.builder.intValue(.i8, 1);
184 _ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, "");182 _ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, "");
185183
...@@ -704,8 +702,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif...@@ -704,8 +702,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModif
704 const llvm_ty = try o.builder.structType(.normal, llvm_types);702 const llvm_ty = try o.builder.structType(.normal, llvm_types);
705 try llvm_args.ensureUnusedCapacity(it.types_len);703 try llvm_args.ensureUnusedCapacity(it.types_len);
706 for (llvm_types, 0..) |field_ty, i| {704 for (llvm_types, 0..) |field_ty, i| {
707 const alignment =705 const alignment: Builder.Alignment = .fromByteUnits(@divExact(target.ptrBitWidth(), 8));
708 Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8));
709 const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, "");706 const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, "");
710 const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, "");707 const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, "");
711 llvm_args.appendAssumeCapacity(loaded);708 llvm_args.appendAssumeCapacity(loaded);
...@@ -1153,10 +1150,8 @@ fn cmp(...@@ -1153,10 +1150,8 @@ fn cmp(
1153 }1150 }
1154 // We need to emit instructions to check for equality/inequality1151 // We need to emit instructions to check for equality/inequality
1155 // of optionals that are not pointers.1152 // of optionals that are not pointers.
1156 const is_by_ref = isByRef(scalar_ty, zcu);1153 const lhs_non_null = try self.optCmpNull(.ne, scalar_ty, lhs, .normal);
1157 const opt_llvm_ty = try o.lowerType(pt, scalar_ty);1154 const rhs_non_null = try self.optCmpNull(.ne, scalar_ty, rhs, .normal);
1158 const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal);
1159 const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal);
1160 const llvm_i2 = try o.builder.intType(2);1155 const llvm_i2 = try o.builder.intType(2);
1161 const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, "");1156 const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, "");
1162 const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, "");1157 const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, "");
...@@ -1186,8 +1181,8 @@ fn cmp(...@@ -1186,8 +1181,8 @@ fn cmp(
1186 _ = try self.wip.br(end_block);1181 _ = try self.wip.br(end_block);
11871182
1188 self.wip.cursor = .{ .block = both_pl_block };1183 self.wip.cursor = .{ .block = both_pl_block };
1189 const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true);1184 const lhs_payload = try self.optPayloadHandle(lhs, scalar_ty, true);
1190 const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true);1185 const rhs_payload = try self.optPayloadHandle(rhs, scalar_ty, true);
1191 const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload);1186 const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload);
1192 _ = try self.wip.br(end_block);1187 _ = try self.wip.br(end_block);
1193 const both_pl_block_end = self.wip.cursor.block;1188 const both_pl_block_end = self.wip.cursor.block;
...@@ -1392,12 +1387,10 @@ fn lowerSwitchDispatch(...@@ -1392,12 +1387,10 @@ fn lowerSwitchDispatch(
1392 try o.lowerType(pt, .usize),1387 try o.lowerType(pt, .usize),
1393 "",1388 "",
1394 );1389 );
1395 const target_ptr_ptr = try self.wip.gep(1390 const target_ptr_ptr = try self.ptraddScaled(
1396 .inbounds,
1397 .ptr,
1398 jmp_table.table.toValue(),1391 jmp_table.table.toValue(),
1399 &.{table_index},1392 table_index,
1400 "",1393 Type.usize.abiSize(zcu),
1401 );1394 );
1402 const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, "");1395 const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, "");
14031396
...@@ -1580,7 +1573,7 @@ fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {...@@ -1580,7 +1573,7 @@ fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
1580 const body = unwrapped_try.else_body;1573 const body = unwrapped_try.else_body;
1581 const err_union_ty = self.typeOf(unwrapped_try.error_union);1574 const err_union_ty = self.typeOf(unwrapped_try.error_union);
1582 const is_unused = self.liveness.isUnused(inst);1575 const is_unused = self.liveness.isUnused(inst);
1583 return lowerTry(self, err_union, body, err_union_ty, false, .none, false, is_unused, err_cold);1576 return lowerTry(self, err_union, body, err_union_ty, false, .none, is_unused, err_cold);
1584}1577}
15851578
1586fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {1579fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
...@@ -1594,7 +1587,7 @@ fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Valu...@@ -1594,7 +1587,7 @@ fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Valu
15941587
1595 self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu));1588 self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu));
15961589
1597 return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), true, is_unused, err_cold);1590 return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), is_unused, err_cold);
1598}1591}
15991592
1600fn lowerTry(1593fn lowerTry(
...@@ -1604,7 +1597,6 @@ fn lowerTry(...@@ -1604,7 +1597,6 @@ fn lowerTry(
1604 err_union_ty: Type,1597 err_union_ty: Type,
1605 operand_is_ptr: bool,1598 operand_is_ptr: bool,
1606 operand_ptr_align: InternPool.Alignment,1599 operand_ptr_align: InternPool.Alignment,
1607 can_elide_load: bool,
1608 is_unused: bool,1600 is_unused: bool,
1609 err_cold: bool,1601 err_cold: bool,
1610) !Builder.Value {1602) !Builder.Value {
...@@ -1613,7 +1605,6 @@ fn lowerTry(...@@ -1613,7 +1605,6 @@ fn lowerTry(
1613 const zcu = pt.zcu;1605 const zcu = pt.zcu;
1614 const payload_ty = err_union_ty.errorUnionPayload(zcu);1606 const payload_ty = err_union_ty.errorUnionPayload(zcu);
1615 const payload_has_bits = payload_ty.hasRuntimeBits(zcu);1607 const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
1616 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
1617 const error_type = try o.errorIntType(pt);1608 const error_type = try o.errorIntType(pt);
16181609
1619 const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{1610 const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{
...@@ -1632,19 +1623,17 @@ fn lowerTry(...@@ -1632,19 +1623,17 @@ fn lowerTry(
1632 else1623 else
1633 err_union;1624 err_union;
1634 }1625 }
1635 const err_field_index = try errUnionErrorFieldIndex(payload_ty, pt);1626
1636 if (operand_is_ptr or isByRef(err_union_ty, zcu)) {1627 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1637 const err_field_ptr =1628 const offset = codegen.errUnionErrorOffset(payload_ty, zcu);
1638 try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, "");1629 const err_field_ptr = try fg.ptraddConst(err_union, offset);
1639 break :loaded try fg.wip.load(1630 break :loaded try fg.wip.load(
1640 if (operand_is_ptr) access_kind else .normal,1631 if (operand_is_ptr) access_kind else .normal,
1641 error_type,1632 error_type,
1642 err_field_ptr,1633 err_field_ptr,
1643 err_set_align.toLlvm(),1634 err_set_align.toLlvm(),
1644 "",1635 "",
1645 );1636 );
1646 }
1647 break :loaded try fg.wip.extractValue(err_union, &.{err_field_index}, "");
1648 };1637 };
1649 const zero = try o.builder.intValue(error_type, 0);1638 const zero = try o.builder.intValue(error_type, 0);
1650 const is_err = try fg.wip.icmp(.ne, loaded, zero, "");1639 const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
...@@ -1661,21 +1650,15 @@ fn lowerTry(...@@ -1661,21 +1650,15 @@ fn lowerTry(
1661 }1650 }
1662 if (is_unused) return .none;1651 if (is_unused) return .none;
1663 if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;1652 if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;
1664 const offset = try errUnionPayloadFieldIndex(payload_ty, pt);1653 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1654 const payload_ptr = try fg.ptraddConst(err_union, codegen.errUnionPayloadOffset(payload_ty, zcu));
1665 if (operand_is_ptr) {1655 if (operand_is_ptr) {
1666 return fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");1656 return payload_ptr;
1667 } else if (isByRef(err_union_ty, zcu)) {1657 } else if (isByRef(payload_ty, zcu)) {
1668 const payload_ptr = try fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");1658 return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
1669 if (isByRef(payload_ty, zcu)) {1659 } else {
1670 if (can_elide_load)1660 return fg.wip.load(.normal, try o.lowerType(pt, payload_ty), payload_ptr, payload_align.toLlvm(), "");
1671 return payload_ptr;
1672
1673 return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
1674 }
1675 const load_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
1676 return fg.wip.load(.normal, load_ty, payload_ptr, payload_align.toLlvm(), "");
1677 }1661 }
1678 return fg.wip.extractValue(err_union, &.{offset}, "");
1679}1662}
16801663
1681fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void {1664fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void {
...@@ -1950,12 +1933,7 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -1950,12 +1933,7 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
1950 const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));1933 const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
1951 const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));1934 const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));
1952 const operand = try self.resolveInst(ty_op.operand);1935 const operand = try self.resolveInst(ty_op.operand);
1953 if (!array_ty.hasRuntimeBits(zcu))1936 return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
1954 return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
1955 const ptr = try self.wip.gep(.inbounds, try o.lowerType(pt, array_ty), operand, &.{
1956 try o.builder.intValue(llvm_usize, 0), try o.builder.intValue(llvm_usize, 0),
1957 }, "");
1958 return self.wip.buildAggregate(slice_llvm_ty, &.{ ptr, len }, "");
1959}1937}
19601938
1961fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {1939fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...@@ -2126,20 +2104,14 @@ fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Valu...@@ -2126,20 +2104,14 @@ fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Valu
2126 return self.wip.extractValue(operand, &.{index}, "");2104 return self.wip.extractValue(operand, &.{index}, "");
2127}2105}
21282106
2129fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: c_uint) !Builder.Value {2107fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: u1) !Builder.Value {
2130 const o = self.object;2108 const zcu = self.pt.zcu;
2131 const pt = self.pt;
2132 const zcu = pt.zcu;
2133 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;2109 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2134 const slice_ptr = try self.resolveInst(ty_op.operand);2110 const slice_ptr = try self.resolveInst(ty_op.operand);
2135 const slice_ptr_ty = self.typeOf(ty_op.operand);2111 return self.ptraddConst(slice_ptr, index * Type.usize.abiSize(zcu));
2136 const slice_llvm_ty = try o.lowerType(pt, slice_ptr_ty.childType(zcu));
2137
2138 return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, "");
2139}2112}
21402113
2141fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {2114fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2142 const o = self.object;
2143 const pt = self.pt;2115 const pt = self.pt;
2144 const zcu = pt.zcu;2116 const zcu = pt.zcu;
2145 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;2117 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -2149,9 +2121,8 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -2149,9 +2121,8 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2149 const slice_info = slice_ty.ptrInfo(zcu);2121 const slice_info = slice_ty.ptrInfo(zcu);
2150 assert(slice_info.flags.size == .slice);2122 assert(slice_info.flags.size == .slice);
2151 const elem_ty: Type = .fromInterned(slice_info.child);2123 const elem_ty: Type = .fromInterned(slice_info.child);
2152 const llvm_elem_ty = try o.lowerType(pt, elem_ty);
2153 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");2124 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
2154 const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");2125 const ptr = try self.ptraddScaled(base_ptr, index, elem_ty.abiSize(zcu));
2155 const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));2126 const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
2156 const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;2127 const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
2157 self.maybeMarkAllowZeroAccess(slice_info);2128 self.maybeMarkAllowZeroAccess(slice_info);
...@@ -2163,7 +2134,6 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -2163,7 +2134,6 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2163}2134}
21642135
2165fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {2136fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2166 const o = self.object;
2167 const pt = self.pt;2137 const pt = self.pt;
2168 const zcu = pt.zcu;2138 const zcu = pt.zcu;
2169 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;2139 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...@@ -2172,13 +2142,11 @@ fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -2172,13 +2142,11 @@ fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
21722142
2173 const slice = try self.resolveInst(bin_op.lhs);2143 const slice = try self.resolveInst(bin_op.lhs);
2174 const index = try self.resolveInst(bin_op.rhs);2144 const index = try self.resolveInst(bin_op.rhs);
2175 const llvm_elem_ty = try o.lowerType(pt, slice_ty.childType(zcu));
2176 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");2145 const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
2177 return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");2146 return self.ptraddScaled(base_ptr, index, slice_ty.childType(zcu).abiSize(zcu));
2178}2147}
21792148
2180fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {2149fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2181 const o = self.object;
2182 const pt = self.pt;2150 const pt = self.pt;
2183 const zcu = pt.zcu;2151 const zcu = pt.zcu;
21842152
...@@ -2186,16 +2154,12 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -2186,16 +2154,12 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2186 const array_ty = self.typeOf(bin_op.lhs);2154 const array_ty = self.typeOf(bin_op.lhs);
2187 const array_llvm_val = try self.resolveInst(bin_op.lhs);2155 const array_llvm_val = try self.resolveInst(bin_op.lhs);
2188 const rhs = try self.resolveInst(bin_op.rhs);2156 const rhs = try self.resolveInst(bin_op.rhs);
2189 const array_llvm_ty = try o.lowerType(pt, array_ty);
2190 const elem_ty = array_ty.childType(zcu);2157 const elem_ty = array_ty.childType(zcu);
2191 if (isByRef(array_ty, zcu)) {2158 if (isByRef(array_ty, zcu)) {
2192 const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{2159 const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu));
2193 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0),
2194 rhs,
2195 }, "");
2196 if (isByRef(elem_ty, zcu)) {2160 if (isByRef(elem_ty, zcu)) {
2197 const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();2161 const elem_align = elem_ty.abiAlignment(zcu).toLlvm();
2198 return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);2162 return self.loadByRef(elem_ptr, elem_ty, elem_align, .normal);
2199 } else {2163 } else {
2200 return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);2164 return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
2201 }2165 }
...@@ -2206,16 +2170,14 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -2206,16 +2170,14 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2206}2170}
22072171
2208fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {2172fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2209 const o = self.object;
2210 const pt = self.pt;2173 const pt = self.pt;
2211 const zcu = pt.zcu;2174 const zcu = pt.zcu;
2212 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;2175 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2213 const ptr_ty = self.typeOf(bin_op.lhs);2176 const ptr_ty = self.typeOf(bin_op.lhs);
2214 const elem_ty = ptr_ty.indexableElem(zcu);2177 const elem_ty = ptr_ty.indexableElem(zcu);
2215 const llvm_elem_ty = try o.lowerType(pt, elem_ty);
2216 const base_ptr = try self.resolveInst(bin_op.lhs);2178 const base_ptr = try self.resolveInst(bin_op.lhs);
2217 const rhs = try self.resolveInst(bin_op.rhs);2179 const rhs = try self.resolveInst(bin_op.rhs);
2218 const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, "");2180 const ptr = try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));
2219 if (isByRef(elem_ty, zcu)) {2181 if (isByRef(elem_ty, zcu)) {
2220 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));2182 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
2221 const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();2183 const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
...@@ -2228,7 +2190,6 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -2228,7 +2190,6 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2228}2190}
22292191
2230fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {2192fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2231 const o = self.object;
2232 const pt = self.pt;2193 const pt = self.pt;
2233 const zcu = pt.zcu;2194 const zcu = pt.zcu;
2234 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;2195 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...@@ -2243,8 +2204,7 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -2243,8 +2204,7 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2243 const elem_ptr = ty_pl.ty.toType();2204 const elem_ptr = ty_pl.ty.toType();
2244 if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;2205 if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
22452206
2246 const llvm_elem_ty = try o.lowerType(pt, elem_ty);2207 return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));
2247 return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, "");
2248}2208}
22492209
2250fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {2210fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...@@ -2276,86 +2236,59 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -2276,86 +2236,59 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
2276 const struct_llvm_val = try self.resolveInst(struct_field.struct_operand);2236 const struct_llvm_val = try self.resolveInst(struct_field.struct_operand);
2277 const field_index = struct_field.field_index;2237 const field_index = struct_field.field_index;
2278 const field_ty = struct_ty.fieldType(field_index, zcu);2238 const field_ty = struct_ty.fieldType(field_index, zcu);
2279 if (!field_ty.hasRuntimeBits(zcu)) return .none;2239 assert(field_ty.hasRuntimeBits(zcu));
22802240
2281 if (!isByRef(struct_ty, zcu)) {2241 if (!isByRef(struct_ty, zcu)) {
2242 // All auto/extern struct/union types are by-ref, unless they have no runtime bits, in which
2243 // case we shouldn't be seeing this instruction to begin with. Therefore we must be dealing
2244 // with a `packed struct` or `packed union`.
2245 assert(struct_ty.containerLayout(zcu) == .@"packed");
2282 assert(!isByRef(field_ty, zcu));2246 assert(!isByRef(field_ty, zcu));
2283 switch (struct_ty.zigTypeTag(zcu)) {2247 const field_int_val: Builder.Value = switch (struct_ty.zigTypeTag(zcu)) {
2284 .@"struct" => switch (struct_ty.containerLayout(zcu)) {2248 .@"struct" => field_int_val: {
2285 .@"packed" => {2249 const llvm_field_int_ty = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
2286 const struct_type = zcu.typeToStruct(struct_ty).?;2250 const bit_offset = zcu.structPackedFieldBitOffset(
2287 const bit_offset = zcu.structPackedFieldBitOffset(struct_type, field_index);2251 zcu.intern_pool.loadStructType(struct_ty.toIntern()),
2288 const containing_int = struct_llvm_val;2252 field_index,
2289 const shift_amt =2253 );
2290 try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset);2254 const shift_bits = try o.builder.intValue(struct_llvm_val.typeOfWip(&self.wip), bit_offset);
2291 const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, "");2255 const shifted = try self.wip.bin(.lshr, struct_llvm_val, shift_bits, "");
2292 const elem_llvm_ty = try o.lowerType(pt, field_ty);2256 break :field_int_val try self.wip.cast(.trunc, shifted, llvm_field_int_ty, "");
2293 if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
2294 const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
2295 const truncated_int =
2296 try self.wip.cast(.trunc, shifted_value, same_size_int, "");
2297 return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
2298 }
2299 return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
2300 },
2301 else => {
2302 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
2303 return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, "");
2304 },
2305 },
2306 .@"union" => {
2307 assert(struct_ty.containerLayout(zcu) == .@"packed");
2308 const containing_int = struct_llvm_val;
2309 const elem_llvm_ty = try o.lowerType(pt, field_ty);
2310 if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
2311 const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
2312 const truncated_int =
2313 try self.wip.cast(.trunc, containing_int, same_size_int, "");
2314 return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
2315 }
2316 return self.wip.cast(.trunc, containing_int, elem_llvm_ty, "");
2317 },2257 },
2258 .@"union" => struct_llvm_val,
2318 else => unreachable,2259 else => unreachable,
2260 };
2261 switch (field_ty.zigTypeTag(zcu)) {
2262 else => unreachable, // not packable
2263 .void => unreachable, // opv bug in sema
2264 .int, .bool, .@"enum", .@"struct", .@"union" => {
2265 // Represented as integers, so already done
2266 return field_int_val;
2267 },
2268 .float => {
2269 // bitcast int->float
2270 return self.wip.cast(.bitcast, field_int_val, try o.lowerType(pt, field_ty), "");
2271 },
2319 }2272 }
2320 }2273 }
23212274
2322 switch (struct_ty.zigTypeTag(zcu)) {2275 const offset: u64 = switch (struct_ty.zigTypeTag(zcu)) {
2323 .@"struct" => {2276 .@"struct" => struct_ty.structFieldOffset(field_index, zcu),
2324 const layout = struct_ty.containerLayout(zcu);2277 .@"union" => struct_ty.unionGetLayout(zcu).payloadOffset(),
2325 assert(layout != .@"packed");
2326 const struct_llvm_ty = try o.lowerType(pt, struct_ty);
2327 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
2328 const field_ptr =
2329 try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, "");
2330 const explicit_alignment = struct_ty.explicitFieldAlignment(field_index, zcu);
2331 const field_ptr_ty = try pt.ptrType(.{
2332 .child = field_ty.toIntern(),
2333 .flags = .{ .alignment = explicit_alignment },
2334 });
2335 if (isByRef(field_ty, zcu)) {
2336 const alignment = switch (explicit_alignment) {
2337 .none => field_ty.abiAlignment(zcu),
2338 else => |a| a,
2339 };
2340 return self.loadByRef(field_ptr, field_ty, alignment.toLlvm(), .normal);
2341 } else {
2342 return self.load(field_ptr, field_ptr_ty);
2343 }
2344 },
2345 .@"union" => {
2346 const union_llvm_ty = try o.lowerType(pt, struct_ty);
2347 const layout = struct_ty.unionGetLayout(zcu);
2348 const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align));
2349 const field_ptr =
2350 try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, "");
2351 const payload_alignment = layout.payload_align.toLlvm();
2352 if (isByRef(field_ty, zcu)) {
2353 return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal);
2354 } else {
2355 return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment);
2356 }
2357 },
2358 else => unreachable,2278 else => unreachable,
2279 };
2280
2281 const struct_ptr_align = struct_ty.abiAlignment(zcu);
2282 const field_ptr = try self.ptraddConst(struct_llvm_val, offset);
2283 const field_ptr_align: InternPool.Alignment = switch (offset) {
2284 0 => struct_ptr_align,
2285 else => struct_ptr_align.minStrict(.fromLog2Units(@ctz(offset))),
2286 };
2287
2288 if (isByRef(field_ty, zcu)) {
2289 return self.loadByRef(field_ptr, field_ty, field_ptr_align.toLlvm(), .normal);
2290 } else {
2291 return self.loadTruncate(.normal, field_ty, field_ptr, field_ptr_align.toLlvm());
2359 }2292 }
2360}2293}
23612294
...@@ -2964,8 +2897,7 @@ fn airIsNonNull(...@@ -2964,8 +2897,7 @@ fn airIsNonNull(
2964 return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");2897 return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
2965 }2898 }
29662899
2967 const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu);2900 return self.optCmpNull(cond, optional_ty, operand, access_kind);
2968 return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind);
2969}2901}
29702902
2971fn airIsErr(2903fn airIsErr(
...@@ -3006,40 +2938,23 @@ fn airIsErr(...@@ -3006,40 +2938,23 @@ fn airIsErr(
3006 operand;2938 operand;
3007 return self.wip.icmp(cond, loaded, zero, "");2939 return self.wip.icmp(cond, loaded, zero, "");
3008 }2940 }
2941 assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref
30092942
3010 const err_field_index = try errUnionErrorFieldIndex(payload_ty, pt);2943 const err_align = if (operand_is_ptr)
30112944 operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu))
3012 const loaded = if (operand_is_ptr or isByRef(err_union_ty, zcu)) loaded: {2945 else
3013 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);2946 .none;
3014 const err_alignment = if (operand_is_ptr)2947 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3015 operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu))2948 const loaded = try self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");
3016 else
3017 .none;
3018 const err_field_ptr =
3019 try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, "");
3020 break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, err_alignment.toLlvm(), "");
3021 } else try self.wip.extractValue(operand, &.{err_field_index}, "");
3022 return self.wip.icmp(cond, loaded, zero, "");2949 return self.wip.icmp(cond, loaded, zero, "");
3023}2950}
30242951
3025fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {2952fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3026 const o = self.object;
3027 const pt = self.pt;
3028 const zcu = pt.zcu;
3029 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;2953 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3030 const operand = try self.resolveInst(ty_op.operand);2954 const operand = try self.resolveInst(ty_op.operand);
3031 const optional_ty = self.typeOf(ty_op.operand).childType(zcu);2955 // If `Type.optionalReprIsPayload`, then the address should be the same. Otherwise, optional
3032 const payload_ty = optional_ty.optionalChild(zcu);2956 // layouts always put the payload at offset 0, so... the address should still be the same.
3033 if (!payload_ty.hasRuntimeBits(zcu)) {2957 return operand;
3034 // We have a pointer to a zero-bit value and we need to return
3035 // a pointer to a zero-bit value.
3036 return operand;
3037 }
3038 if (optional_ty.optionalReprIsPayload(zcu)) {
3039 // The payload and the optional are the same value.
3040 return operand;
3041 }
3042 return self.wip.gepStruct(try o.lowerType(pt, optional_ty), operand, 0, "");
3043}2958}
30442959
3045fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {2960fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...@@ -3072,9 +2987,9 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value...@@ -3072,9 +2987,9 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value
3072 return operand;2987 return operand;
3073 }2988 }
30742989
3075 // First set the non-null bit.2990 // First set the non-null bit. It's always immediately after the payload (no padding) because it
3076 const optional_llvm_ty = try o.lowerType(pt, optional_ty);2991 // has alignment 1.
3077 const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, "");2992 const non_null_ptr = try self.ptraddConst(operand, payload_ty.abiSize(zcu));
30782993
3079 self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));2994 self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
30802995
...@@ -3084,11 +2999,10 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value...@@ -3084,11 +2999,10 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value
3084 // Then return the payload pointer (only if it's used).2999 // Then return the payload pointer (only if it's used).
3085 if (self.liveness.isUnused(inst)) return .none;3000 if (self.liveness.isUnused(inst)) return .none;
30863001
3087 return self.wip.gepStruct(optional_llvm_ty, operand, 0, "");3002 return operand; // payload is at offset 0
3088}3003}
30893004
3090fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {3005fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3091 const o = self.object;
3092 const pt = self.pt;3006 const pt = self.pt;
3093 const zcu = pt.zcu;3007 const zcu = pt.zcu;
3094 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3008 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -3102,8 +3016,7 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -3102,8 +3016,7 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3102 return operand;3016 return operand;
3103 }3017 }
31043018
3105 const opt_llvm_ty = try o.lowerType(pt, optional_ty);3019 return self.optPayloadHandle(operand, optional_ty, false);
3106 return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false);
3107}3020}
31083021
3109fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value {3022fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value {
...@@ -3120,20 +3033,18 @@ fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool...@@ -3120,20 +3033,18 @@ fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool
3120 if (!payload_ty.hasRuntimeBits(zcu)) {3033 if (!payload_ty.hasRuntimeBits(zcu)) {
3121 return if (operand_is_ptr) operand else .none;3034 return if (operand_is_ptr) operand else .none;
3122 }3035 }
3123 const offset = try errUnionPayloadFieldIndex(payload_ty, pt);3036 const payload_ptr = try self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3124 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
3125 if (operand_is_ptr) {3037 if (operand_is_ptr) {
3126 return self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");3038 return payload_ptr;
3127 } else if (isByRef(err_union_ty, zcu)) {3039 }
3128 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();3040 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3129 const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");3041 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
3130 if (isByRef(payload_ty, zcu)) {3042 if (isByRef(payload_ty, zcu)) {
3131 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);3043 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
3132 }3044 } else {
3133 const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset];3045 const payload_llvm_ty = try o.lowerType(pt, payload_ty);
3134 return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");3046 return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");
3135 }3047 }
3136 return self.wip.extractValue(operand, &.{offset}, "");
3137}3048}
31383049
3139fn airErrUnionErr(3050fn airErrUnionErr(
...@@ -3169,17 +3080,18 @@ fn airErrUnionErr(...@@ -3169,17 +3080,18 @@ fn airErrUnionErr(
3169 return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");3080 return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");
3170 }3081 }
31713082
3172 const offset = try errUnionErrorFieldIndex(payload_ty, pt);3083 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
31733084
3174 if (operand_is_ptr or isByRef(err_union_ty, zcu)) {3085 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
3175 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
31763086
3177 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);3087 const err_align: InternPool.Alignment = a: {
3178 const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");3088 const err_abi_align = Type.anyerror.abiAlignment(zcu);
3179 return self.wip.load(access_kind, error_type, err_field_ptr, .default, "");3089 if (!operand_is_ptr) break :a err_abi_align;
3180 }3090 break :a err_abi_align.minStrict(operand_ty.ptrAlignment(zcu));
3091 };
31813092
3182 return self.wip.extractValue(operand, &.{offset}, "");3093 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3094 return self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");
3183}3095}
31843096
3185fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {3097fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...@@ -3198,27 +3110,18 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value...@@ -3198,27 +3110,18 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value
3198 const access_kind: Builder.MemoryAccessKind =3110 const access_kind: Builder.MemoryAccessKind =
3199 if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;3111 if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
32003112
3201 if (!payload_ty.hasRuntimeBits(zcu)) {3113 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
3202 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
3203 _ = try self.wip.store(access_kind, non_error_val, operand, err_union_ptr_align.toLlvm());
3204 return operand;
3205 }
3206 const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
3207 {
3208 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
32093114
3210 const err_int_ty = try pt.errorIntType();3115 {
3211 const error_alignment = err_int_ty.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();3116 const error_align = Type.anyerror.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
3212 const error_offset = try errUnionErrorFieldIndex(payload_ty, pt);
3213 // First set the non-error value.3117 // First set the non-error value.
3214 const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, "");3118 const error_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3215 _ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment);3119 _ = try self.wip.store(access_kind, non_error_val, error_ptr, error_align);
3216 }3120 }
3121
3217 // Then return the payload pointer (only if it is used).3122 // Then return the payload pointer (only if it is used).
3218 if (self.liveness.isUnused(inst)) return .none;3123 if (self.liveness.isUnused(inst)) return .none;
32193124 return self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3220 const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt);
3221 return self.wip.gepStruct(err_union_llvm_ty, operand, payload_offset, "");
3222}3125}
32233126
3224fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value {3127fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value {
...@@ -3233,7 +3136,6 @@ fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -3233,7 +3136,6 @@ fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3233}3136}
32343137
3235fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {3138fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3236 const o = self.object;
3237 const pt = self.pt;3139 const pt = self.pt;
3238 const zcu = pt.zcu;3140 const zcu = pt.zcu;
32393141
...@@ -3241,15 +3143,20 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Val...@@ -3241,15 +3143,20 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Val
3241 const struct_ty = ty_pl.ty.toType();3143 const struct_ty = ty_pl.ty.toType();
3242 const field_index = ty_pl.payload;3144 const field_index = ty_pl.payload;
32433145
3244 const struct_llvm_ty = try o.lowerType(pt, struct_ty);
3245 const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
3246 assert(self.err_ret_trace != .none);3146 assert(self.err_ret_trace != .none);
3247 const field_ptr = try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, "");3147
3248 const field_alignment = struct_ty.explicitFieldAlignment(field_index, zcu);
3249 const field_ty = struct_ty.fieldType(field_index, zcu);3148 const field_ty = struct_ty.fieldType(field_index, zcu);
3149 const field_offset = struct_ty.structFieldOffset(field_index, zcu);
3150 const field_align = switch (field_offset) {
3151 0 => struct_ty.abiAlignment(zcu),
3152 else => struct_ty.abiAlignment(zcu).minStrict(.fromLog2Units(@ctz(field_offset))),
3153 };
3154
3155 const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset);
3156
3250 const field_ptr_ty = try pt.ptrType(.{3157 const field_ptr_ty = try pt.ptrType(.{
3251 .child = field_ty.toIntern(),3158 .child = field_ty.toIntern(),
3252 .flags = .{ .alignment = field_alignment },3159 .flags = .{ .alignment = field_align },
3253 });3160 });
3254 return self.load(field_ptr, field_ptr_ty);3161 return self.load(field_ptr, field_ptr_ty);
3255}3162}
...@@ -3290,25 +3197,23 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.V...@@ -3290,25 +3197,23 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.V
3290 const operand = try self.resolveInst(ty_op.operand);3197 const operand = try self.resolveInst(ty_op.operand);
3291 const optional_ty = self.typeOfIndex(inst);3198 const optional_ty = self.typeOfIndex(inst);
3292 if (optional_ty.optionalReprIsPayload(zcu)) return operand;3199 if (optional_ty.optionalReprIsPayload(zcu)) return operand;
3200 assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload`
3293 const llvm_optional_ty = try o.lowerType(pt, optional_ty);3201 const llvm_optional_ty = try o.lowerType(pt, optional_ty);
3294 if (isByRef(optional_ty, zcu)) {3202 const optional_ptr = if (self.isNextRet(body_tail))
3295 const directReturn = self.isNextRet(body_tail);3203 self.ret_ptr
3296 const optional_ptr = if (directReturn)3204 else brk: {
3297 self.ret_ptr3205 const alignment = optional_ty.abiAlignment(zcu).toLlvm();
3298 else brk: {3206 const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
3299 const alignment = optional_ty.abiAlignment(zcu).toLlvm();3207 break :brk optional_ptr;
3300 const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);3208 };
3301 break :brk optional_ptr;
3302 };
33033209
3304 const payload_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 0, "");3210 const payload_ptr = optional_ptr; // payload always at offset 0
3305 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);3211 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3306 try self.store(payload_ptr, payload_ptr_ty, operand, .none);3212 try self.store(payload_ptr, payload_ptr_ty, operand, .none);
3307 const non_null_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 1, "");3213 // Non-null bit immediately after payload (no padding because the bit has alignment 1).
3308 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);3214 const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu));
3309 return optional_ptr;3215 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);
3310 }3216 return optional_ptr;
3311 return self.wip.buildAggregate(llvm_optional_ty, &.{ operand, non_null_bit }, "");
3312}3217}
33133218
3314fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {3219fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
...@@ -3321,34 +3226,25 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Bu...@@ -3321,34 +3226,25 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Bu
3321 const operand = try self.resolveInst(ty_op.operand);3226 const operand = try self.resolveInst(ty_op.operand);
3322 const payload_ty = self.typeOf(ty_op.operand);3227 const payload_ty = self.typeOf(ty_op.operand);
3323 assert(payload_ty.hasRuntimeBits(zcu));3228 assert(payload_ty.hasRuntimeBits(zcu));
3229 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3324 const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0);3230 const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0);
3325 const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);3231 const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
33263232
3327 const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt);3233 const result_ptr = if (self.isNextRet(body_tail))
3328 const error_offset = try errUnionErrorFieldIndex(payload_ty, pt);3234 self.ret_ptr
3329 if (isByRef(err_un_ty, zcu)) {3235 else brk: {
3330 const directReturn = self.isNextRet(body_tail);3236 const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm();
3331 const result_ptr = if (directReturn)3237 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3332 self.ret_ptr3238 break :brk result_ptr;
3333 else brk: {3239 };
3334 const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm();
3335 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3336 break :brk result_ptr;
3337 };
33383240
3339 const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");3241 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3340 const err_int_ty = try pt.errorIntType();3242 const error_alignment = Type.anyerror.abiAlignment(pt.zcu).toLlvm();
3341 const error_alignment = err_int_ty.abiAlignment(pt.zcu).toLlvm();3243 _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);
3342 _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);3244 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3343 const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");3245 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3344 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);3246 try self.store(payload_ptr, payload_ptr_ty, operand, .none);
3345 try self.store(payload_ptr, payload_ptr_ty, operand, .none);3247 return result_ptr;
3346 return result_ptr;
3347 }
3348 var fields: [2]Builder.Value = undefined;
3349 fields[payload_offset] = operand;
3350 fields[error_offset] = ok_err_code;
3351 return self.wip.buildAggregate(err_un_llvm_ty, &fields, "");
3352}3248}
33533249
3354fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {3250fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
...@@ -3361,35 +3257,26 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builde...@@ -3361,35 +3257,26 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builde
3361 const payload_ty = err_un_ty.errorUnionPayload(zcu);3257 const payload_ty = err_un_ty.errorUnionPayload(zcu);
3362 const operand = try self.resolveInst(ty_op.operand);3258 const operand = try self.resolveInst(ty_op.operand);
3363 if (!payload_ty.hasRuntimeBits(zcu)) return operand;3259 if (!payload_ty.hasRuntimeBits(zcu)) return operand;
3260 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3364 const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);3261 const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
33653262
3366 const payload_offset = try errUnionPayloadFieldIndex(payload_ty, pt);3263 const result_ptr = if (self.isNextRet(body_tail))
3367 const error_offset = try errUnionErrorFieldIndex(payload_ty, pt);3264 self.ret_ptr
3368 if (isByRef(err_un_ty, zcu)) {3265 else brk: {
3369 const directReturn = self.isNextRet(body_tail);3266 const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
3370 const result_ptr = if (directReturn)3267 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3371 self.ret_ptr3268 break :brk result_ptr;
3372 else brk: {3269 };
3373 const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
3374 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3375 break :brk result_ptr;
3376 };
3377
3378 const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
3379 const err_int_ty = try pt.errorIntType();
3380 const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm();
3381 _ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
3382 const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
3383 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3384 // TODO store undef to payload_ptr
3385 _ = payload_ptr;
3386 _ = payload_ptr_ty;
3387 return result_ptr;
3388 }
33893270
3390 // TODO set payload bytes to undef3271 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3391 const undef = try o.builder.undefValue(err_un_llvm_ty);3272 const error_alignment = Type.anyerror.abiAlignment(zcu).toLlvm();
3392 return self.wip.insertValue(undef, operand, &.{error_offset}, "");3273 _ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
3274 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3275 const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
3276 // TODO store undef to payload_ptr
3277 _ = payload_ptr;
3278 _ = payload_ptr_ty;
3279 return result_ptr;
3393}3280}
33943281
3395fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {3282fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...@@ -3804,26 +3691,18 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Bui...@@ -3804,26 +3691,18 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Bui
3804}3691}
38053692
3806fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {3693fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3807 const o = self.object;3694 const zcu = self.pt.zcu;
3808 const pt = self.pt;
3809 const zcu = pt.zcu;
3810 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;3695 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3811 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;3696 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3812 const ptr = try self.resolveInst(bin_op.lhs);3697 const ptr_or_slice = try self.resolveInst(bin_op.lhs);
3813 const offset = try self.resolveInst(bin_op.rhs);3698 const index = try self.resolveInst(bin_op.rhs);
3814 const ptr_ty = self.typeOf(bin_op.lhs);3699 const ptr_ty = self.typeOf(bin_op.lhs);
3815 const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu));3700 const elem_ty = ptr_ty.indexableElem(zcu);
3816 switch (ptr_ty.ptrSize(zcu)) {3701 const ptr = switch (ptr_ty.ptrSize(zcu)) {
3817 // It's a pointer to an array, so according to LLVM we need an extra GEP index.3702 .one, .many, .c => ptr_or_slice,
3818 .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{3703 .slice => try self.wip.extractValue(ptr_or_slice, &.{0}, ""),
3819 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), offset,3704 };
3820 }, ""),3705 return self.ptraddScaled(ptr, index, elem_ty.abiSize(zcu));
3821 .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{offset}, ""),
3822 .slice => {
3823 const base = try self.wip.extractValue(ptr, &.{0}, "");
3824 return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{offset}, "");
3825 },
3826 }
3827}3706}
38283707
3829fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {3708fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...@@ -3832,22 +3711,18 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -3832,22 +3711,18 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
3832 const zcu = pt.zcu;3711 const zcu = pt.zcu;
3833 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;3712 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3834 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;3713 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3835 const ptr = try self.resolveInst(bin_op.lhs);3714 const ptr_or_slice = try self.resolveInst(bin_op.lhs);
3836 const offset = try self.resolveInst(bin_op.rhs);3715 const llvm_usize_ty = try o.lowerType(pt, .usize);
3837 const negative_offset = try self.wip.neg(offset, "");
3838 const ptr_ty = self.typeOf(bin_op.lhs);3716 const ptr_ty = self.typeOf(bin_op.lhs);
3839 const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu));3717 const elem_ty = ptr_ty.indexableElem(zcu);
3840 switch (ptr_ty.ptrSize(zcu)) {3718 const ptr = switch (ptr_ty.ptrSize(zcu)) {
3841 // It's a pointer to an array, so according to LLVM we need an extra GEP index.3719 .one, .many, .c => ptr_or_slice,
3842 .one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{3720 .slice => try self.wip.extractValue(ptr_or_slice, &.{0}, ""),
3843 try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), negative_offset,3721 };
3844 }, ""),3722 const scale_val = try o.builder.intValue(llvm_usize_ty, -@as(i65, elem_ty.abiSize(zcu)));
3845 .c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{negative_offset}, ""),3723 const positive_index = try self.resolveInst(bin_op.rhs);
3846 .slice => {3724 const negative_offset = try self.wip.bin(.@"mul nsw", positive_index, scale_val, "");
3847 const base = try self.wip.extractValue(ptr, &.{0}, "");3725 return self.ptradd(ptr, negative_offset);
3848 return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{negative_offset}, "");
3849 },
3850 }
3851}3726}
38523727
3853fn airOverflow(3728fn airOverflow(
...@@ -3868,6 +3743,7 @@ fn airOverflow(...@@ -3868,6 +3743,7 @@ fn airOverflow(
3868 const lhs_ty = self.typeOf(extra.lhs);3743 const lhs_ty = self.typeOf(extra.lhs);
3869 const scalar_ty = lhs_ty.scalarType(zcu);3744 const scalar_ty = lhs_ty.scalarType(zcu);
3870 const inst_ty = self.typeOfIndex(inst);3745 const inst_ty = self.typeOfIndex(inst);
3746 assert(isByRef(inst_ty, zcu)); // auto structs are by-ref
38713747
3872 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;3748 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3873 const llvm_inst_ty = try o.lowerType(pt, inst_ty);3749 const llvm_inst_ty = try o.lowerType(pt, inst_ty);
...@@ -3878,28 +3754,22 @@ fn airOverflow(...@@ -3878,28 +3754,22 @@ fn airOverflow(
3878 const result_val = try self.wip.extractValue(results, &.{0}, "");3754 const result_val = try self.wip.extractValue(results, &.{0}, "");
3879 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");3755 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
38803756
3881 const result_index = o.llvmFieldIndex(inst_ty, 0).?;3757 const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();
3882 const overflow_index = o.llvmFieldIndex(inst_ty, 1).?;3758 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
38833759
3884 if (isByRef(inst_ty, zcu)) {3760 {
3885 const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();3761 // Store to 'result: IntType' field
3886 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);3762 const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(0, zcu));
3887 {3763 _ = try self.wip.store(.normal, result_val, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());
3888 const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, "");3764 }
3889 _ = try self.wip.store(.normal, result_val, field_ptr, result_alignment);
3890 }
3891 {
3892 const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, overflow_index, "");
3893 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
3894 }
38953765
3896 return alloca_inst;3766 {
3767 // Store to 'overflow: u1' field
3768 const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(1, zcu));
3769 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
3897 }3770 }
38983771
3899 var fields: [2]Builder.Value = undefined;3772 return alloca_inst;
3900 fields[result_index] = result_val;
3901 fields[overflow_index] = overflow_bit;
3902 return self.wip.buildAggregate(llvm_inst_ty, &fields, "");
3903}3773}
39043774
3905fn buildElementwiseCall(3775fn buildElementwiseCall(
...@@ -4227,6 +4097,7 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -4227,6 +4097,7 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
4227 const lhs_scalar_ty = lhs_ty.scalarType(zcu);4097 const lhs_scalar_ty = lhs_ty.scalarType(zcu);
42284098
4229 const dest_ty = self.typeOfIndex(inst);4099 const dest_ty = self.typeOfIndex(inst);
4100 assert(isByRef(dest_ty, zcu)); // auto structs are by-ref
4230 const llvm_dest_ty = try o.lowerType(pt, dest_ty);4101 const llvm_dest_ty = try o.lowerType(pt, dest_ty);
42314102
4232 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");4103 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
...@@ -4239,27 +4110,22 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -4239,27 +4110,22 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
42394110
4240 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");4111 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
42414112
4242 const result_index = o.llvmFieldIndex(dest_ty, 0).?;4113 const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();
4243 const overflow_index = o.llvmFieldIndex(dest_ty, 1).?;4114 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
42444115
4245 if (isByRef(dest_ty, zcu)) {4116 {
4246 const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();4117 // Store to 'result: IntType' field
4247 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);4118 const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(0, zcu));
4248 {4119 _ = try self.wip.store(.normal, result, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());
4249 const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, "");
4250 _ = try self.wip.store(.normal, result, field_ptr, result_alignment);
4251 }
4252 {
4253 const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, overflow_index, "");
4254 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
4255 }
4256 return alloca_inst;
4257 }4120 }
42584121
4259 var fields: [2]Builder.Value = undefined;4122 {
4260 fields[result_index] = result;4123 // Store to 'overflow: u1' field
4261 fields[overflow_index] = overflow_bit;4124 const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(1, zcu));
4262 return self.wip.buildAggregate(llvm_dest_ty, &fields, "");4125 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
4126 }
4127
4128 return alloca_inst;
4263}4129}
42644130
4265fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {4131fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...@@ -4678,7 +4544,8 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty...@@ -4678,7 +4544,8 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
4678 }4544 }
46794545
4680 if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {4546 if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
4681 const elem_ty = operand_ty.childType(zcu);4547 const elem_ty = inst_ty.childType(zcu);
4548 assert(elem_ty.toIntern() == operand_scalar_ty.toIntern());
4682 if (!result_is_ref) {4549 if (!result_is_ref) {
4683 return self.todo("implement bitcast vector to non-ref array", .{});4550 return self.todo("implement bitcast vector to non-ref array", .{});
4684 }4551 }
...@@ -4690,22 +4557,19 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty...@@ -4690,22 +4557,19 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
4690 } else {4557 } else {
4691 // If the ABI size of the element type is not evenly divisible by size in bits;4558 // If the ABI size of the element type is not evenly divisible by size in bits;
4692 // a simple bitcast will not work, and we fall back to extractelement.4559 // a simple bitcast will not work, and we fall back to extractelement.
4693 const llvm_usize = try o.lowerType(pt, Type.usize);4560 const elem_size = elem_ty.abiSize(zcu);
4694 const usize_zero = try o.builder.intValue(llvm_usize, 0);
4695 const vector_len = operand_ty.arrayLen(zcu);4561 const vector_len = operand_ty.arrayLen(zcu);
4696 var i: u64 = 0;4562 var i: u64 = 0;
4697 while (i < vector_len) : (i += 1) {4563 while (i < vector_len) : (i += 1) {
4698 const elem_ptr = try self.wip.gep(.inbounds, llvm_dest_ty, array_ptr, &.{4564 const arr_elem_ptr = try self.ptraddConst(array_ptr, i * elem_size);
4699 usize_zero, try o.builder.intValue(llvm_usize, i),4565 const vec_elem = try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
4700 }, "");4566 _ = try self.wip.store(.normal, vec_elem, arr_elem_ptr, .default);
4701 const elem =
4702 try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
4703 _ = try self.wip.store(.normal, elem, elem_ptr, .default);
4704 }4567 }
4705 }4568 }
4706 return array_ptr;4569 return array_ptr;
4707 } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) {4570 } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) {
4708 const elem_ty = operand_ty.childType(zcu);4571 const elem_ty = operand_ty.childType(zcu);
4572 assert(elem_ty.toIntern() == inst_scalar_ty.toIntern());
4709 const llvm_vector_ty = try o.lowerType(pt, inst_ty);4573 const llvm_vector_ty = try o.lowerType(pt, inst_ty);
4710 if (!operand_is_ref) return self.todo("implement bitcast non-ref array to vector", .{});4574 if (!operand_is_ref) return self.todo("implement bitcast non-ref array to vector", .{});
47114575
...@@ -4718,20 +4582,15 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty...@@ -4718,20 +4582,15 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
4718 } else {4582 } else {
4719 // If the ABI size of the element type is not evenly divisible by size in bits;4583 // If the ABI size of the element type is not evenly divisible by size in bits;
4720 // a simple bitcast will not work, and we fall back to extractelement.4584 // a simple bitcast will not work, and we fall back to extractelement.
4721 const array_llvm_ty = try o.lowerType(pt, operand_ty);
4722 const elem_llvm_ty = try o.lowerType(pt, elem_ty);4585 const elem_llvm_ty = try o.lowerType(pt, elem_ty);
4723 const llvm_usize = try o.lowerType(pt, Type.usize);4586 const elem_size = elem_ty.abiSize(zcu);
4724 const usize_zero = try o.builder.intValue(llvm_usize, 0);
4725 const vector_len = operand_ty.arrayLen(zcu);4587 const vector_len = operand_ty.arrayLen(zcu);
4726 var vector = try o.builder.poisonValue(llvm_vector_ty);4588 var vector = try o.builder.poisonValue(llvm_vector_ty);
4727 var i: u64 = 0;4589 var i: u64 = 0;
4728 while (i < vector_len) : (i += 1) {4590 while (i < vector_len) : (i += 1) {
4729 const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, operand, &.{4591 const arr_elem_ptr = try self.ptraddConst(operand, i * elem_size);
4730 usize_zero, try o.builder.intValue(llvm_usize, i),4592 const arr_elem = try self.wip.load(.normal, elem_llvm_ty, arr_elem_ptr, .default, "");
4731 }, "");4593 vector = try self.wip.insertElement(vector, arr_elem, try o.builder.intValue(.i32, i), "");
4732 const elem = try self.wip.load(.normal, elem_llvm_ty, elem_ptr, .default, "");
4733 vector =
4734 try self.wip.insertElement(vector, elem, try o.builder.intValue(.i32, i), "");
4735 }4594 }
4736 return vector;4595 return vector;
4737 }4596 }
...@@ -5066,10 +4925,23 @@ fn airCmpxchg(...@@ -5066,10 +4925,23 @@ fn airCmpxchg(
5066 return self.wip.select(.normal, success_bit, zero, payload, "");4925 return self.wip.select(.normal, success_bit, zero, payload, "");
5067 }4926 }
50684927
4928 assert(isByRef(optional_ty, zcu));
4929
5069 comptime assert(optional_layout_version == 3);4930 comptime assert(optional_layout_version == 3);
50704931
5071 const non_null_bit = try self.wip.not(success_bit, "");4932 const non_null_bit = try self.wip.not(success_bit, "");
5072 return buildOptional(self, optional_ty, payload, non_null_bit);4933
4934 const payload_align = operand_ty.abiAlignment(zcu).toLlvm();
4935 const alloca_inst = try self.buildAlloca(try o.lowerType(pt, optional_ty), payload_align);
4936
4937 // Payload is always the first field at offset 0, so address is `alloca_inst`
4938 _ = try self.wip.store(.normal, payload, alloca_inst, payload_align);
4939
4940 // Non-null bit is after payload with no padding because it has alignment 1
4941 const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu));
4942 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, comptime .fromByteUnits(1));
4943
4944 return alloca_inst;
5073}4945}
50744946
5075fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {4947fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
...@@ -5315,13 +5187,15 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value...@@ -5315,13 +5187,15 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value
5315 const end_block = try self.wip.block(1, "InlineMemsetEnd");5187 const end_block = try self.wip.block(1, "InlineMemsetEnd");
53165188
5317 const llvm_usize_ty = try o.lowerType(pt, Type.usize);5189 const llvm_usize_ty = try o.lowerType(pt, Type.usize);
5318 const len = switch (ptr_ty.ptrSize(zcu)) {5190 const end_ptr = switch (ptr_ty.ptrSize(zcu)) {
5319 .slice => try self.wip.extractValue(dest_slice, &.{1}, ""),5191 .slice => try self.ptraddScaled(
5320 .one => try o.builder.intValue(llvm_usize_ty, ptr_ty.childType(zcu).arrayLen(zcu)),5192 dest_ptr,
5193 try self.wip.extractValue(dest_slice, &.{1}, ""),
5194 elem_abi_size,
5195 ),
5196 .one => try self.ptraddConst(dest_ptr, ptr_ty.childType(zcu).abiSize(zcu)),
5321 .many, .c => unreachable,5197 .many, .c => unreachable,
5322 };5198 };
5323 const elem_llvm_ty = try o.lowerType(pt, elem_ty);
5324 const end_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, dest_ptr, &.{len}, "");
5325 _ = try self.wip.br(loop_block);5199 _ = try self.wip.br(loop_block);
53265200
5327 self.wip.cursor = .{ .block = loop_block };5201 self.wip.cursor = .{ .block = loop_block };
...@@ -5343,9 +5217,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value...@@ -5343,9 +5217,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value
5343 self.disable_intrinsics,5217 self.disable_intrinsics,
5344 );5218 );
5345 } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);5219 } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
5346 const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{5220 const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size);
5347 try o.builder.intValue(llvm_usize_ty, 1),
5348 }, "");
5349 _ = try self.wip.br(loop_block);5221 _ = try self.wip.br(loop_block);
53505222
5351 self.wip.cursor = .{ .block = end_block };5223 self.wip.cursor = .{ .block = end_block };
...@@ -5408,14 +5280,13 @@ fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -5408,14 +5280,13 @@ fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5408}5280}
54095281
5410fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {5282fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5411 const o = self.object;
5412 const pt = self.pt;5283 const pt = self.pt;
5413 const zcu = pt.zcu;5284 const zcu = pt.zcu;
5414 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;5285 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5415 const un_ptr_ty = self.typeOf(bin_op.lhs);5286 const un_ptr_ty = self.typeOf(bin_op.lhs);
5416 const un_ty = un_ptr_ty.childType(zcu);5287 const un_ty = un_ptr_ty.childType(zcu);
5417 const layout = un_ty.unionGetLayout(zcu);5288 const layout = un_ty.unionGetLayout(zcu);
5418 if (layout.tag_size == 0) return .none;5289 assert(layout.tag_size != 0);
54195290
5420 const access_kind: Builder.MemoryAccessKind =5291 const access_kind: Builder.MemoryAccessKind =
5421 if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;5292 if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -5429,8 +5300,7 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -5429,8 +5300,7 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5429 _ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());5300 _ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());
5430 return .none;5301 return .none;
5431 }5302 }
5432 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));5303 const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());
5433 const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(pt, un_ty), union_ptr, tag_index, "");
5434 const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) {5304 const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) {
5435 0 => union_ptr_align,5305 0 => union_ptr_align,
5436 else => |off| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))),5306 else => |off| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))),
...@@ -5446,20 +5316,20 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -5446,20 +5316,20 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5446 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5316 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5447 const un_ty = self.typeOf(ty_op.operand);5317 const un_ty = self.typeOf(ty_op.operand);
5448 const layout = un_ty.unionGetLayout(zcu);5318 const layout = un_ty.unionGetLayout(zcu);
5449 if (layout.tag_size == 0) return .none;5319 assert(layout.tag_size != 0);
5450 const union_handle = try self.resolveInst(ty_op.operand);5320 const union_ptr = try self.resolveInst(ty_op.operand);
5451 if (isByRef(un_ty, zcu)) {5321 if (isByRef(un_ty, zcu)) {
5452 const llvm_un_ty = try o.lowerType(pt, un_ty);5322 const llvm_un_ty = try o.lowerType(pt, un_ty);
5453 if (layout.payload_size == 0)5323 if (layout.payload_size == 0)
5454 return self.wip.load(.normal, llvm_un_ty, union_handle, .default, "");5324 return self.wip.load(.normal, llvm_un_ty, union_ptr, .default, "");
5455 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));5325 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
5456 const tag_field_ptr = try self.wip.gepStruct(llvm_un_ty, union_handle, tag_index, "");5326 const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());
5457 const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index];5327 const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index];
5458 return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");5328 return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");
5459 } else {5329 } else {
5460 if (layout.payload_size == 0) return union_handle;5330 if (layout.payload_size == 0) return union_ptr;
5461 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));5331 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
5462 return self.wip.extractValue(union_handle, &.{tag_index}, "");5332 return self.wip.extractValue(union_ptr, &.{tag_index}, "");
5463 }5333 }
5464}5334}
54655335
...@@ -5646,19 +5516,18 @@ fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -5646,19 +5516,18 @@ fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5646fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {5516fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5647 const o = self.object;5517 const o = self.object;
5648 const pt = self.pt;5518 const pt = self.pt;
5519 const zcu = pt.zcu;
5649 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;5520 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5650 const operand = try self.resolveInst(un_op);5521 const operand = try self.resolveInst(un_op);
5651 const slice_ty = self.typeOfIndex(inst);5522 const slice_ty = self.typeOfIndex(inst);
5652 const slice_llvm_ty = try o.lowerType(pt, slice_ty);5523 const slice_llvm_ty = try o.lowerType(pt, slice_ty);
56535524
5654 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.5525 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
5655 const extended_operand = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), "");5526 const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), "");
56565527
5657 const error_name_table_ptr = try self.getErrorNameTable();5528 const error_name_table_ptr = try self.getErrorNameTable();
5658 const error_name_table =5529 const error_name_table = try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, "");
5659 try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, "");5530 const error_name_ptr = try self.ptraddScaled(error_name_table, operand_usize, slice_ty.abiSize(zcu));
5660 const error_name_ptr =
5661 try self.wip.gep(.inbounds, slice_llvm_ty, error_name_table, &.{extended_operand}, "");
5662 return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");5531 return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");
5663}5532}
56645533
...@@ -6074,8 +5943,9 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -6074,8 +5943,9 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6074 }5943 }
6075 return vector;5944 return vector;
6076 },5945 },
6077 .@"struct" => {5946 .@"struct" => switch (result_ty.containerLayout(zcu)) {
6078 if (zcu.typeToPackedStruct(result_ty)) |struct_type| {5947 .@"packed" => {
5948 const struct_type = ip.loadStructType(result_ty.toIntern());
6079 const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type);5949 const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type);
6080 const big_bits = backing_int_ty.bitSize(zcu);5950 const big_bits = backing_int_ty.bitSize(zcu);
6081 const int_ty = try o.builder.intType(@intCast(big_bits));5951 const int_ty = try o.builder.intType(@intCast(big_bits));
...@@ -6100,69 +5970,68 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -6100,69 +5970,68 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6100 running_bits += ty_bit_size;5970 running_bits += ty_bit_size;
6101 }5971 }
6102 return running_int;5972 return running_int;
6103 }5973 },
61045974 .auto, .@"extern" => {
6105 assert(result_ty.containerLayout(zcu) != .@"packed");5975 assert(isByRef(result_ty, zcu));
6106
6107 if (isByRef(result_ty, zcu)) {
6108 // TODO in debug builds init to undef so that the padding will be 0xaa5976 // TODO in debug builds init to undef so that the padding will be 0xaa
6109 // even if we fully populate the fields.5977 // even if we fully populate the fields.
6110 const alignment = result_ty.abiAlignment(zcu).toLlvm();5978 const struct_align = result_ty.abiAlignment(zcu);
6111 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);5979 const alloca_inst = try self.buildAlloca(llvm_result_ty, struct_align.toLlvm());
6112
6113 for (elements, 0..) |elem, i| {
6114 if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
6115
6116 const llvm_elem = try self.resolveInst(elem);
6117 const llvm_i = o.llvmFieldIndex(result_ty, i).?;
6118 const field_ptr = try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, "");
6119
6120 const field_ptr_ty = try pt.ptrType(.{
6121 .child = self.typeOf(elem).toIntern(),
6122 .flags = .{
6123 .alignment = result_ty.explicitFieldAlignment(i, zcu),
6124 },
6125 });
6126 try self.store(field_ptr, field_ptr_ty, llvm_elem, .none);
6127 }
61285980
6129 return alloca_inst;5981 for (elements, 0..) |elem, field_index| {
6130 } else {5982 if (result_ty.structFieldIsComptime(field_index, zcu)) continue;
6131 var result = try o.builder.poisonValue(llvm_result_ty);5983 const field_ty = result_ty.fieldType(field_index, zcu);
6132 for (elements, 0..) |elem, i| {5984 if (!field_ty.hasRuntimeBits(zcu)) continue;
6133 if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;5985 const offset = result_ty.structFieldOffset(field_index, zcu);
5986 const field_ptr = try self.ptraddConst(alloca_inst, offset);
5987 const field_ptr_align: InternPool.Alignment = switch (offset) {
5988 0 => struct_align,
5989 else => struct_align.minStrict(.fromLog2Units(@ctz(offset))),
5990 };
61345991
6135 const llvm_elem = try self.resolveInst(elem);5992 const llvm_field_val = try self.resolveInst(elem);
6136 const llvm_i = o.llvmFieldIndex(result_ty, i).?;5993
6137 result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, "");5994 if (isByRef(field_ty, zcu)) {
5995 _ = try self.wip.callMemCpy(
5996 field_ptr,
5997 field_ptr_align.toLlvm(),
5998 llvm_field_val,
5999 field_ty.abiAlignment(zcu).toLlvm(),
6000 try o.builder.intValue(try o.lowerType(pt, .usize), field_ty.abiSize(zcu)),
6001 .normal,
6002 self.disable_intrinsics,
6003 );
6004 } else {
6005 _ = try self.wip.store(
6006 .normal,
6007 llvm_field_val,
6008 field_ptr,
6009 field_ptr_align.toLlvm(),
6010 );
6011 }
6138 }6012 }
6139 return result;6013
6140 }6014 return alloca_inst;
6015 },
6141 },6016 },
6142 .array => {6017 .array => {
6143 assert(isByRef(result_ty, zcu));6018 assert(isByRef(result_ty, zcu));
61446019
6145 const llvm_usize = try o.lowerType(pt, Type.usize);
6146 const usize_zero = try o.builder.intValue(llvm_usize, 0);
6147 const alignment = result_ty.abiAlignment(zcu).toLlvm();6020 const alignment = result_ty.abiAlignment(zcu).toLlvm();
6148 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);6021 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
61496022
6150 const array_info = result_ty.arrayInfo(zcu);6023 const array_info = result_ty.arrayInfo(zcu);
6151 const elem_ptr_ty = try pt.ptrType(.{6024 const elem_ptr_ty = try pt.singleConstPtrType(array_info.elem_type);
6152 .child = array_info.elem_type.toIntern(),6025
6153 });6026 const elem_size = array_info.elem_type.abiSize(zcu);
61546027
6155 for (elements, 0..) |elem, i| {6028 for (elements, 0..) |elem, i| {
6156 const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{6029 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i);
6157 usize_zero, try o.builder.intValue(llvm_usize, i),
6158 }, "");
6159 const llvm_elem = try self.resolveInst(elem);6030 const llvm_elem = try self.resolveInst(elem);
6160 try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none);6031 try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none);
6161 }6032 }
6162 if (array_info.sentinel) |sent_val| {6033 if (array_info.sentinel) |sent_val| {
6163 const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{6034 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len);
6164 usize_zero, try o.builder.intValue(llvm_usize, array_info.len),
6165 }, "");
6166 const llvm_elem = try self.resolveValue(sent_val);6035 const llvm_elem = try self.resolveValue(sent_val);
6167 try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none);6036 try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none);
6168 }6037 }
...@@ -6188,95 +6057,30 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -6188,95 +6057,30 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
61886057
6189 const layout = Type.getUnionLayout(union_obj, zcu);6058 const layout = Type.getUnionLayout(union_obj, zcu);
61906059
6191 const tag_int_val = blk: {6060 assert(layout.payload_size != 0); // otherwise the value would be comptime-known
6192 const tag_ty = union_ty.unionTagTypeHypothetical(zcu);
6193 const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index);
6194 break :blk tag_val.intFromEnum(zcu);
6195 };
6196 if (layout.payload_size == 0) {
6197 if (layout.tag_size == 0) {
6198 return .none;
6199 }
6200 assert(!isByRef(union_ty, zcu));
6201 var big_int_space: Value.BigIntSpace = undefined;
6202 const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
6203 return try o.builder.bigIntValue(union_llvm_ty, tag_big_int);
6204 }
6205 assert(isByRef(union_ty, zcu));6061 assert(isByRef(union_ty, zcu));
6206 // The llvm type of the alloca will be the named LLVM union type, and will not6062
6207 // necessarily match the format that we need, depending on which tag is active.
6208 // We must construct the correct unnamed struct type here, in order to then set
6209 // the fields appropriately.
6210 const alignment = layout.abi_align.toLlvm();6063 const alignment = layout.abi_align.toLlvm();
6211 const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);6064 const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
6212 const llvm_payload = try self.resolveInst(extra.init);6065 const llvm_payload = try self.resolveInst(extra.init);
6213 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);6066 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6214 const field_llvm_ty = try o.lowerType(pt, field_ty);
6215 const field_size = field_ty.abiSize(zcu);
6216 const field_align = union_ty.explicitFieldAlignment(extra.field_index, zcu);
6217 const llvm_usize = try o.lowerType(pt, Type.usize);
6218 const usize_zero = try o.builder.intValue(llvm_usize, 0);
6219
6220 assert(field_ty.hasRuntimeBits(zcu));6067 assert(field_ty.hasRuntimeBits(zcu));
62216068
6222 const llvm_union_ty = t: {
6223 const payload_ty = p: {
6224 if (field_size == layout.payload_size) {
6225 break :p field_llvm_ty;
6226 }
6227 const padding_len = layout.payload_size - field_size;
6228 break :p try o.builder.structType(.@"packed", &.{
6229 field_llvm_ty, try o.builder.arrayType(padding_len, .i8),
6230 });
6231 };
6232 if (layout.tag_size == 0) break :t try o.builder.structType(.normal, &.{payload_ty});
6233 const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
6234 var fields: [3]Builder.Type = undefined;
6235 var fields_len: usize = 2;
6236 if (layout.tag_align.compare(.gte, layout.payload_align)) {
6237 fields = .{ tag_ty, payload_ty, undefined };
6238 } else {
6239 fields = .{ payload_ty, tag_ty, undefined };
6240 }
6241 if (layout.padding != 0) {
6242 fields[fields_len] = try o.builder.arrayType(layout.padding, .i8);
6243 fields_len += 1;
6244 }
6245 break :t try o.builder.structType(.normal, fields[0..fields_len]);
6246 };
6247
6248 // Now we follow the layout as expressed above with GEP instructions to set the
6249 // tag and the payload.
6250 const field_ptr_ty = try pt.ptrType(.{
6251 .child = field_ty.toIntern(),
6252 .flags = .{ .alignment = field_align },
6253 });
6254 if (layout.tag_size == 0) {
6255 const indices = [3]Builder.Value{ usize_zero, .@"0", .@"0" };
6256 const len: usize = if (field_size == layout.payload_size) 2 else 3;
6257 const field_ptr =
6258 try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
6259 try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
6260 return result_ptr;
6261 }
6262
6263 {6069 {
6264 const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align));6070 const payload_ptr_ty = try pt.ptrType(.{
6265 const indices: [3]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, payload_index), .@"0" };6071 .child = field_ty.toIntern(),
6266 const len: usize = if (field_size == layout.payload_size) 2 else 3;6072 .flags = .{ .alignment = layout.payload_align },
6267 const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");6073 });
6268 try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);6074 const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset());
6075 try self.store(payload_ptr, payload_ptr_ty, llvm_payload, .none);
6269 }6076 }
6270 {6077
6271 const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));6078 if (layout.tag_size != 0) {
6272 const indices: [2]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, tag_index) };6079 const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
6273 const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, &indices, "");6080 const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index);
6274 const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));6081 const llvm_tag_val = try o.lowerValue(pt, tag_val.toIntern());
6275 var big_int_space: Value.BigIntSpace = undefined;6082 const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());
6276 const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);6083 _ = try self.wip.store(.normal, llvm_tag_val.toValue(), tag_ptr, layout.tag_align.toLlvm());
6277 const llvm_tag = try o.builder.bigIntValue(tag_ty, tag_big_int);
6278 const tag_alignment = Type.fromInterned(union_obj.enum_tag_type).abiAlignment(zcu).toLlvm();
6279 _ = try self.wip.store(.normal, llvm_tag, field_ptr, tag_alignment);
6280 }6084 }
62816085
6282 return result_ptr;6086 return result_ptr;
...@@ -6369,7 +6173,6 @@ fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -6369,7 +6173,6 @@ fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6369}6173}
63706174
6371fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {6175fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6372 const o = self.object;
6373 const pt = self.pt;6176 const pt = self.pt;
6374 const target = pt.zcu.getTarget();6177 const target = pt.zcu.getTarget();
63756178
...@@ -6387,11 +6190,8 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {...@@ -6387,11 +6190,8 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
63876190
6388 // Load the work_group_* member from the struct as u16.6191 // Load the work_group_* member from the struct as u16.
6389 // Just treat the dispatch pointer as an array of u16 to keep things simple.6192 // Just treat the dispatch pointer as an array of u16 to keep things simple.
6390 const workgroup_size_ptr = try self.wip.gep(.inbounds, .i16, dispatch_ptr, &.{6193 const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2);
6391 try o.builder.intValue(try o.lowerType(pt, Type.usize), 2 + dimension),6194 return self.wip.load(.normal, .i16, workgroup_size_ptr, comptime .fromByteUnits(2), "");
6392 }, "");
6393 const workgroup_size_alignment = comptime Builder.Alignment.fromByteUnits(2);
6394 return self.wip.load(.normal, .i16, workgroup_size_ptr, workgroup_size_alignment, "");
6395 },6195 },
6396 .nvptx, .nvptx64 => {6196 .nvptx, .nvptx64 => {
6397 return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");6197 return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
...@@ -6435,88 +6235,45 @@ fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index {...@@ -6435,88 +6235,45 @@ fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index {
6435 return variable_index;6235 return variable_index;
6436}6236}
64376237
6438/// Assumes the optional is not pointer-like and payload has bits.6238/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
6439fn optCmpNull(6239fn optCmpNull(
6440 self: *FuncGen,6240 self: *FuncGen,
6441 cond: Builder.IntegerCondition,6241 cond: Builder.IntegerCondition,
6442 opt_llvm_ty: Builder.Type,6242 opt_ty: Type,
6443 opt_handle: Builder.Value,6243 opt_ptr: Builder.Value,
6444 is_by_ref: bool,
6445 access_kind: Builder.MemoryAccessKind,6244 access_kind: Builder.MemoryAccessKind,
6446) Allocator.Error!Builder.Value {6245) Allocator.Error!Builder.Value {
6447 const o = self.object;6246 const zcu = self.pt.zcu;
6448 const field = b: {6247 assert(isByRef(opt_ty, zcu));
6449 if (is_by_ref) {
6450 const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, "");
6451 break :b try self.wip.load(access_kind, .i8, field_ptr, .default, "");
6452 }
6453 break :b try self.wip.extractValue(opt_handle, &.{1}, "");
6454 };
6455 comptime assert(optional_layout_version == 3);6248 comptime assert(optional_layout_version == 3);
64566249 // Non-null bit is always after the payload, with no padding because it has alignment 1.
6457 return self.wip.icmp(cond, field, try o.builder.intValue(.i8, 0), "");6250 const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu));
6251 const non_null = try self.wip.load(access_kind, .i8, non_null_ptr, .default, "");
6252 return self.wip.icmp(cond, non_null, try self.object.builder.intValue(.i8, 0), "");
6458}6253}
64596254
6460/// Assumes the optional is not pointer-like and payload has bits.6255/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
6461fn optPayloadHandle(6256fn optPayloadHandle(
6462 fg: *FuncGen,6257 fg: *FuncGen,
6463 opt_llvm_ty: Builder.Type,6258 opt_ptr: Builder.Value,
6464 opt_handle: Builder.Value,
6465 opt_ty: Type,6259 opt_ty: Type,
6466 can_elide_load: bool,6260 can_elide_load: bool,
6467) !Builder.Value {6261) !Builder.Value {
6468 const pt = fg.pt;6262 const pt = fg.pt;
6469 const zcu = pt.zcu;6263 const zcu = pt.zcu;
6264 assert(isByRef(opt_ty, zcu));
6470 const payload_ty = opt_ty.optionalChild(zcu);6265 const payload_ty = opt_ty.optionalChild(zcu);
64716266
6472 if (isByRef(opt_ty, zcu)) {6267 // Payload is first field so always at the same address as the optional itself.
6473 // We have a pointer and we need to return a pointer to the first field.6268 const payload_ptr = opt_ptr;
6474 const payload_ptr = try fg.wip.gepStruct(opt_llvm_ty, opt_handle, 0, "");
6475
6476 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
6477 if (isByRef(payload_ty, zcu)) {
6478 if (can_elide_load)
6479 return payload_ptr;
64806269
6481 return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);6270 const payload_align = payload_ty.abiAlignment(zcu).toLlvm();
6482 }6271 if (isByRef(payload_ty, zcu)) {
6483 return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_alignment);6272 if (can_elide_load) return payload_ptr;
6484 }6273 return fg.loadByRef(payload_ptr, payload_ty, payload_align, .normal);
64856274 } else {
6486 assert(!isByRef(payload_ty, zcu));6275 return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_align);
6487 return fg.wip.extractValue(opt_handle, &.{0}, "");
6488}
6489
6490fn buildOptional(
6491 self: *FuncGen,
6492 optional_ty: Type,
6493 payload: Builder.Value,
6494 non_null_bit: Builder.Value,
6495) !Builder.Value {
6496 const o = self.object;
6497 const pt = self.pt;
6498 const zcu = pt.zcu;
6499 const optional_llvm_ty = try o.lowerType(pt, optional_ty);
6500 const non_null_field = try self.wip.cast(.zext, non_null_bit, .i8, "");
6501
6502 if (isByRef(optional_ty, zcu)) {
6503 const payload_alignment = optional_ty.abiAlignment(pt.zcu).toLlvm();
6504 const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment);
6505
6506 {
6507 const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, "");
6508 _ = try self.wip.store(.normal, payload, field_ptr, payload_alignment);
6509 }
6510 {
6511 const non_null_alignment = comptime Builder.Alignment.fromByteUnits(1);
6512 const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 1, "");
6513 _ = try self.wip.store(.normal, non_null_field, field_ptr, non_null_alignment);
6514 }
6515
6516 return alloca_inst;
6517 }6276 }
6518
6519 return self.wip.buildAggregate(optional_llvm_ty, &.{ payload, non_null_field }, "");
6520}6277}
65216278
6522fn fieldPtr(6279fn fieldPtr(
...@@ -6525,7 +6282,6 @@ fn fieldPtr(...@@ -6525,7 +6282,6 @@ fn fieldPtr(
6525 aggregate_ptr_ty: Type,6282 aggregate_ptr_ty: Type,
6526 field_index: u32,6283 field_index: u32,
6527) !Builder.Value {6284) !Builder.Value {
6528 const o = self.object;
6529 const pt = self.pt;6285 const pt = self.pt;
6530 const zcu = pt.zcu;6286 const zcu = pt.zcu;
6531 const aggregate_ty = aggregate_ptr_ty.childType(zcu);6287 const aggregate_ty = aggregate_ptr_ty.childType(zcu);
...@@ -6534,35 +6290,12 @@ fn fieldPtr(...@@ -6534,35 +6290,12 @@ fn fieldPtr(
6534 // bit offset is represented in the pointer *type*.6290 // bit offset is represented in the pointer *type*.
6535 return aggregate_ptr;6291 return aggregate_ptr;
6536 }6292 }
6537 switch (aggregate_ty.zigTypeTag(zcu)) {6293 const offset: u64 = switch (aggregate_ty.zigTypeTag(zcu)) {
6538 .@"struct" => {6294 .@"struct" => aggregate_ty.structFieldOffset(field_index, zcu),
6539 if (!aggregate_ty.hasRuntimeBits(zcu)) {6295 .@"union" => aggregate_ty.unionGetLayout(zcu).payloadOffset(),
6540 return aggregate_ptr;
6541 }
6542 const struct_llvm_ty = try o.lowerType(pt, aggregate_ty);
6543 if (o.llvmFieldIndex(aggregate_ty, field_index)) |llvm_field_index| {
6544 return self.wip.gepStruct(struct_llvm_ty, aggregate_ptr, llvm_field_index, "");
6545 } else {
6546 // If we found no index then this means this is a zero sized field at the
6547 // end of the struct. Treat our struct pointer as an array of two and get
6548 // the index to the element at index `1` to get a pointer to the end of
6549 // the struct.
6550 const llvm_index = try o.builder.intValue(
6551 try o.lowerType(pt, Type.usize),
6552 @intFromBool(aggregate_ty.hasRuntimeBits(zcu)),
6553 );
6554 return self.wip.gep(.inbounds, struct_llvm_ty, aggregate_ptr, &.{llvm_index}, "");
6555 }
6556 },
6557 .@"union" => {
6558 const layout = aggregate_ty.unionGetLayout(zcu);
6559 if (layout.payload_size == 0) return aggregate_ptr;
6560 const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align));
6561 const union_llvm_ty = try o.lowerType(pt, aggregate_ty);
6562 return self.wip.gepStruct(union_llvm_ty, aggregate_ptr, payload_index, "");
6563 },
6564 else => unreachable,6296 else => unreachable,
6565 }6297 };
6298 return self.ptraddConst(aggregate_ptr, offset);
6566}6299}
65676300
6568/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.6301/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.
...@@ -6828,11 +6561,8 @@ fn valgrindClientRequest(...@@ -6828,11 +6561,8 @@ fn valgrindClientRequest(
6828 break :a array_ptr;6561 break :a array_ptr;
6829 } else fg.valgrind_client_request_array;6562 } else fg.valgrind_client_request_array;
6830 const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };6563 const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
6831 const zero = try o.builder.intValue(llvm_usize, 0);
6832 for (array_elements, 0..) |elem, i| {6564 for (array_elements, 0..) |elem, i| {
6833 const elem_ptr = try fg.wip.gep(.inbounds, array_llvm_ty, array_ptr, &.{6565 const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu));
6834 zero, try o.builder.intValue(llvm_usize, i),
6835 }, "");
6836 _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);6566 _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);
6837 }6567 }
68386568
...@@ -7605,13 +7335,8 @@ pub fn buildAllocaInner(...@@ -7605,13 +7335,8 @@ pub fn buildAllocaInner(
76057335
7606/// This is the one source of truth for whether a type is passed around as an LLVM pointer,7336/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
7607/// or as an LLVM value.7337/// or as an LLVM value.
7608pub fn isByRef(ty: Type, zcu: *Zcu) bool {7338pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
7609 // For tuples and structs, if there are more than this many non-void7339 return switch (ty.zigTypeTag(zcu)) {
7610 // fields, then we make it byref, otherwise byval.
7611 const max_fields_byval = 0;
7612 const ip = &zcu.intern_pool;
7613
7614 switch (ty.zigTypeTag(zcu)) {
7615 .type,7340 .type,
7616 .comptime_int,7341 .comptime_int,
7617 .comptime_float,7342 .comptime_float,
...@@ -7632,62 +7357,24 @@ pub fn isByRef(ty: Type, zcu: *Zcu) bool {...@@ -7632,62 +7357,24 @@ pub fn isByRef(ty: Type, zcu: *Zcu) bool {
7632 .@"enum",7357 .@"enum",
7633 .vector,7358 .vector,
7634 .@"anyframe",7359 .@"anyframe",
7635 => return false,7360 => false,
7636
7637 .array, .frame => return ty.hasRuntimeBits(zcu),
7638 .@"struct" => {
7639 const struct_type = switch (ip.indexToKey(ty.toIntern())) {
7640 .tuple_type => |tuple| {
7641 var count: usize = 0;
7642 for (tuple.types.get(ip), tuple.values.get(ip)) |field_ty, field_val| {
7643 if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
7644
7645 count += 1;
7646 if (count > max_fields_byval) return true;
7647 if (isByRef(Type.fromInterned(field_ty), zcu)) return true;
7648 }
7649 return false;
7650 },
7651 .struct_type => ip.loadStructType(ty.toIntern()),
7652 else => unreachable,
7653 };
76547361
7655 // Packed structs are represented to LLVM as integers.7362 .array,
7656 if (struct_type.layout == .@"packed") return false;7363 .error_union,
76577364 .frame,
7658 const field_types = struct_type.field_types.get(ip);7365 => ty.hasRuntimeBits(zcu),
7659 var it = struct_type.iterateRuntimeOrder(ip);7366
7660 var count: usize = 0;7367 .optional => ty.hasRuntimeBits(zcu) and !ty.optionalReprIsPayload(zcu),
7661 while (it.next()) |field_index| {7368
7662 count += 1;7369 .@"struct" => switch (ty.containerLayout(zcu)) {
7663 if (count > max_fields_byval) return true;7370 .@"packed" => false,
7664 const field_ty = Type.fromInterned(field_types[field_index]);7371 .auto, .@"extern" => ty.hasRuntimeBits(zcu),
7665 if (isByRef(field_ty, zcu)) return true;
7666 }
7667 return false;
7668 },7372 },
7669 .@"union" => switch (ty.containerLayout(zcu)) {7373 .@"union" => switch (ty.containerLayout(zcu)) {
7670 .@"packed" => return false,7374 .@"packed" => false,
7671 else => return ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),7375 else => ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),
7672 },
7673 .error_union => {
7674 const payload_ty = ty.errorUnionPayload(zcu);
7675 if (!payload_ty.hasRuntimeBits(zcu)) {
7676 return false;
7677 }
7678 return true;
7679 },7376 },
7680 .optional => {7377 };
7681 const payload_ty = ty.optionalChild(zcu);
7682 if (!payload_ty.hasRuntimeBits(zcu)) {
7683 return false;
7684 }
7685 if (ty.optionalReprIsPayload(zcu)) {
7686 return false;
7687 }
7688 return true;
7689 },
7690 }
7691}7378}
76927379
7693/// If the operand type of an atomic operation is not byte sized we need to7380/// If the operand type of an atomic operation is not byte sized we need to
...@@ -7713,16 +7400,27 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E...@@ -7713,16 +7400,27 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E
7713 }7400 }
7714}7401}
77157402
7716fn errUnionPayloadFieldIndex(payload_ty: Type, pt: Zcu.PerThread) !u1 {7403fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
7717 const zcu = pt.zcu;7404 if (offset == 0) return ptr;
7718 const err_int_ty = try pt.errorIntType();7405 const llvm_usize_ty = try fg.object.lowerType(fg.pt, .usize);
7719 return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.gt, payload_ty.abiAlignment(zcu)));7406 const offset_val = try fg.object.builder.intValue(llvm_usize_ty, offset);
7407 return fg.ptradd(ptr, offset_val);
7720}7408}
77217409fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value {
7722fn errUnionErrorFieldIndex(payload_ty: Type, pt: Zcu.PerThread) !u1 {7410 switch (scale) {
7723 const zcu = pt.zcu;7411 0 => return ptr,
7724 const err_int_ty = try pt.errorIntType();7412 1 => return fg.ptradd(ptr, index),
7725 return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.lte, payload_ty.abiAlignment(zcu)));7413 else => {
7414 const o = fg.object;
7415 const llvm_usize_ty = try o.lowerType(fg.pt, .usize);
7416 const scale_val = try o.builder.intValue(llvm_usize_ty, scale);
7417 const offset = try fg.wip.bin(.@"mul nuw", index, scale_val, "");
7418 return fg.ptradd(ptr, offset);
7419 },
7420 }
7421}
7422fn ptradd(fg: *FuncGen, ptr: Builder.Value, offset: Builder.Value) Allocator.Error!Builder.Value {
7423 return fg.wip.gep(.inbounds, .i8, ptr, &.{offset}, "");
7726}7424}
77277425
7728fn compilerRtIntBits(bits: u16) ?u16 {7426fn compilerRtIntBits(bits: u16) ?u16 {
...@@ -8128,6 +7826,7 @@ const Package = @import("../../Package.zig");...@@ -8128,6 +7826,7 @@ const Package = @import("../../Package.zig");
8128const InternPool = @import("../../InternPool.zig");7826const InternPool = @import("../../InternPool.zig");
8129const Value = @import("../../Value.zig");7827const Value = @import("../../Value.zig");
8130const Type = @import("../../Type.zig");7828const Type = @import("../../Type.zig");
7829const codegen = @import("../../codegen.zig");
81317830
8132const target_util = @import("../../target.zig");7831const target_util = @import("../../target.zig");
8133const libcFloatPrefix = target_util.libcFloatPrefix;7832const libcFloatPrefix = target_util.libcFloatPrefix;