authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-23 23:07:12+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-05-24 15:34:52-07:00
log41f517e5f506500c4e3f0bea53d73db0a1daf456
treef0eb1bcce81e29e8aa732611c5acd7db5699a7e6
parentb42100c70fc306c6d6f69a55e9225a9a91e363ef

x64: update for new error union layout


2 files changed, 175 insertions(+), 78 deletions(-)

src/arch/x86_64/CodeGen.zig+171-77
......@@ -854,7 +854,7 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !u32 {
854854 const ptr_ty = self.air.typeOfIndex(inst);
855855 const elem_ty = ptr_ty.elemType();
856856
857 if (!elem_ty.hasRuntimeBits()) {
857 if (!elem_ty.hasRuntimeBitsIgnoreComptime()) {
858858 return self.allocMem(inst, @sizeOf(usize), @alignOf(usize));
859859 }
860860
......@@ -1786,21 +1786,34 @@ fn airUnwrapErrErr(self: *Self, inst: Air.Inst.Index) !void {
17861786 const err_ty = err_union_ty.errorUnionSet();
17871787 const payload_ty = err_union_ty.errorUnionPayload();
17881788 const operand = try self.resolveInst(ty_op.operand);
1789 const operand_lock: ?RegisterLock = switch (operand) {
1790 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1791 else => null,
1792 };
1793 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
17941789
17951790 const result: MCValue = result: {
1796 if (!payload_ty.hasRuntimeBits()) break :result operand;
1791 if (err_ty.errorSetCardinality() == .zero) {
1792 break :result MCValue{ .immediate = 0 };
1793 }
1794
1795 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
1796 break :result operand;
1797 }
1798
1799 const err_off = errUnionErrOffset(err_union_ty, self.target.*);
17971800 switch (operand) {
17981801 .stack_offset => |off| {
1799 break :result MCValue{ .stack_offset = off };
1802 const offset = off - @intCast(i32, err_off);
1803 break :result MCValue{ .stack_offset = offset };
18001804 },
1801 .register => {
1805 .register => |reg| {
18021806 // TODO reuse operand
1803 break :result try self.copyToRegisterWithInstTracking(inst, err_ty, operand);
1807 const lock = self.register_manager.lockRegAssumeUnused(reg);
1808 defer self.register_manager.unlockReg(lock);
1809 const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
1810 if (err_off > 0) {
1811 const shift = @intCast(u6, err_off * 8);
1812 try self.genShiftBinOpMir(.shr, err_union_ty, result.register, .{ .immediate = shift });
1813 } else {
1814 try self.truncateRegister(Type.anyerror, result.register);
1815 }
1816 break :result result;
18041817 },
18051818 else => return self.fail("TODO implement unwrap_err_err for {}", .{operand}),
18061819 }
......@@ -1815,32 +1828,37 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void {
18151828 }
18161829 const err_union_ty = self.air.typeOf(ty_op.operand);
18171830 const payload_ty = err_union_ty.errorUnionPayload();
1831 const err_ty = err_union_ty.errorUnionSet();
1832 const operand = try self.resolveInst(ty_op.operand);
1833
18181834 const result: MCValue = result: {
1819 if (!payload_ty.hasRuntimeBits()) break :result MCValue.none;
1835 if (err_ty.errorSetCardinality() == .zero) {
1836 // TODO check if we can reuse
1837 break :result operand;
1838 }
18201839
1821 const operand = try self.resolveInst(ty_op.operand);
1822 const operand_lock: ?RegisterLock = switch (operand) {
1823 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1824 else => null,
1825 };
1826 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
1840 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
1841 break :result MCValue.none;
1842 }
18271843
1828 const abi_align = err_union_ty.abiAlignment(self.target.*);
1829 const err_ty = err_union_ty.errorUnionSet();
1830 const err_abi_size = mem.alignForwardGeneric(u32, @intCast(u32, err_ty.abiSize(self.target.*)), abi_align);
1844 const payload_off = errUnionPayloadOffset(err_union_ty, self.target.*);
18311845 switch (operand) {
18321846 .stack_offset => |off| {
1833 const offset = off - @intCast(i32, err_abi_size);
1847 const offset = off - @intCast(i32, payload_off);
18341848 break :result MCValue{ .stack_offset = offset };
18351849 },
1836 .register => {
1850 .register => |reg| {
18371851 // TODO reuse operand
1838 const shift = @intCast(u6, err_abi_size * @sizeOf(usize));
1852 const lock = self.register_manager.lockRegAssumeUnused(reg);
1853 defer self.register_manager.unlockReg(lock);
18391854 const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
1840 try self.genShiftBinOpMir(.shr, Type.usize, result.register, .{ .immediate = shift });
1841 break :result MCValue{
1842 .register = registerAlias(result.register, @intCast(u32, payload_ty.abiSize(self.target.*))),
1843 };
1855 if (payload_off > 0) {
1856 const shift = @intCast(u6, payload_off * 8);
1857 try self.genShiftBinOpMir(.shr, err_union_ty, result.register, .{ .immediate = shift });
1858 } else {
1859 try self.truncateRegister(payload_ty, result.register);
1860 }
1861 break :result result;
18441862 },
18451863 else => return self.fail("TODO implement unwrap_err_payload for {}", .{operand}),
18461864 }
......@@ -1935,24 +1953,37 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void {
19351953/// T to E!T
19361954fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void {
19371955 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
1956
19381957 if (self.liveness.isUnused(inst)) {
19391958 return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
19401959 }
1960
19411961 const error_union_ty = self.air.getRefType(ty_op.ty);
19421962 const error_ty = error_union_ty.errorUnionSet();
19431963 const payload_ty = error_union_ty.errorUnionPayload();
19441964 const operand = try self.resolveInst(ty_op.operand);
1945 assert(payload_ty.hasRuntimeBits());
19461965
1947 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
1948 const abi_align = error_union_ty.abiAlignment(self.target.*);
1949 const err_abi_size = @intCast(u32, error_ty.abiSize(self.target.*));
1950 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
1951 const offset = mem.alignForwardGeneric(u32, err_abi_size, abi_align);
1952 try self.genSetStack(error_ty, stack_offset, .{ .immediate = 0 }, .{});
1953 try self.genSetStack(payload_ty, stack_offset - @intCast(i32, offset), operand, .{});
1966 const result: MCValue = result: {
1967 if (error_ty.errorSetCardinality() == .zero) {
1968 break :result operand;
1969 }
1970
1971 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
1972 break :result operand;
1973 }
1974
1975 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
1976 const abi_align = error_union_ty.abiAlignment(self.target.*);
1977 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
1978 const payload_off = errUnionPayloadOffset(error_union_ty, self.target.*);
1979 const err_off = errUnionErrOffset(error_union_ty, self.target.*);
1980 try self.genSetStack(payload_ty, stack_offset - @intCast(i32, payload_off), operand, .{});
1981 try self.genSetStack(Type.anyerror, stack_offset - @intCast(i32, err_off), .{ .immediate = 0 }, .{});
1982
1983 break :result MCValue{ .stack_offset = stack_offset };
1984 };
19541985
1955 return self.finishAir(inst, .{ .stack_offset = stack_offset }, .{ ty_op.operand, .none, .none });
1986 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
19561987}
19571988
19581989/// E to E!T
......@@ -1962,19 +1993,22 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
19621993 return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none });
19631994 }
19641995 const error_union_ty = self.air.getRefType(ty_op.ty);
1965 const error_ty = error_union_ty.errorUnionSet();
19661996 const payload_ty = error_union_ty.errorUnionPayload();
1967 const err = try self.resolveInst(ty_op.operand);
1997 const operand = try self.resolveInst(ty_op.operand);
1998
19681999 const result: MCValue = result: {
1969 if (!payload_ty.hasRuntimeBits()) break :result err;
2000 if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
2001 break :result operand;
2002 }
19702003
19712004 const abi_size = @intCast(u32, error_union_ty.abiSize(self.target.*));
19722005 const abi_align = error_union_ty.abiAlignment(self.target.*);
1973 const err_abi_size = @intCast(u32, error_ty.abiSize(self.target.*));
19742006 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
1975 const offset = mem.alignForwardGeneric(u32, err_abi_size, abi_align);
1976 try self.genSetStack(error_ty, stack_offset, err, .{});
1977 try self.genSetStack(payload_ty, stack_offset - @intCast(i32, offset), .undef, .{});
2007 const payload_off = errUnionPayloadOffset(error_union_ty, self.target.*);
2008 const err_off = errUnionErrOffset(error_union_ty, self.target.*);
2009 try self.genSetStack(Type.anyerror, stack_offset - @intCast(i32, err_off), operand, .{});
2010 try self.genSetStack(payload_ty, stack_offset - @intCast(i32, payload_off), .undef, .{});
2011
19782012 break :result MCValue{ .stack_offset = stack_offset };
19792013 };
19802014
......@@ -2535,7 +2569,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
25352569 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
25362570 const elem_ty = self.air.typeOfIndex(inst);
25372571 const result: MCValue = result: {
2538 if (!elem_ty.hasRuntimeBits())
2572 if (!elem_ty.hasRuntimeBitsIgnoreComptime())
25392573 break :result MCValue.none;
25402574
25412575 const ptr = try self.resolveInst(ty_op.operand);
......@@ -4102,6 +4136,9 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
41024136 const operand = try self.resolveInst(un_op);
41034137 const ret_ty = self.fn_type.fnReturnType();
41044138 switch (self.ret_mcv) {
4139 .immediate => {
4140 assert(ret_ty.isError());
4141 },
41054142 .stack_offset => {
41064143 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
41074144 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
......@@ -4134,6 +4171,9 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
41344171 const ptr_ty = self.air.typeOf(un_op);
41354172 const elem_ty = ptr_ty.elemType();
41364173 switch (self.ret_mcv) {
4174 .immediate => {
4175 assert(elem_ty.isError());
4176 },
41374177 .stack_offset => {
41384178 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
41394179 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
......@@ -4603,7 +4643,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValu
46034643 const cmp_ty: Type = if (!ty.isPtrLikeOptional()) blk: {
46044644 var buf: Type.Payload.ElemType = undefined;
46054645 const payload_ty = ty.optionalChild(&buf);
4606 break :blk if (payload_ty.hasRuntimeBits()) Type.bool else ty;
4646 break :blk if (payload_ty.hasRuntimeBitsIgnoreComptime()) Type.bool else ty;
46074647 } else ty;
46084648
46094649 try self.genBinOpMir(.cmp, cmp_ty, operand, MCValue{ .immediate = 0 });
......@@ -4619,25 +4659,36 @@ fn isNonNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCV
46194659
46204660fn isErr(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
46214661 const err_type = ty.errorUnionSet();
4622 const payload_type = ty.errorUnionPayload();
4623 if (!err_type.hasRuntimeBits()) {
4662
4663 if (err_type.errorSetCardinality() == .zero) {
46244664 return MCValue{ .immediate = 0 }; // always false
46254665 }
46264666
46274667 try self.spillCompareFlagsIfOccupied();
46284668 self.compare_flags_inst = inst;
46294669
4630 if (!payload_type.hasRuntimeBits()) {
4631 if (err_type.abiSize(self.target.*) <= 8) {
4632 try self.genBinOpMir(.cmp, err_type, operand, MCValue{ .immediate = 0 });
4633 return MCValue{ .compare_flags_unsigned = .gt };
4634 } else {
4635 return self.fail("TODO isErr for errors with size larger than register size", .{});
4636 }
4637 } else {
4638 try self.genBinOpMir(.cmp, err_type, operand, MCValue{ .immediate = 0 });
4639 return MCValue{ .compare_flags_unsigned = .gt };
4670 const err_off = errUnionErrOffset(ty, self.target.*);
4671 switch (operand) {
4672 .stack_offset => |off| {
4673 const offset = off - @intCast(i32, err_off);
4674 try self.genBinOpMir(.cmp, Type.anyerror, .{ .stack_offset = offset }, .{ .immediate = 0 });
4675 },
4676 .register => |reg| {
4677 const maybe_lock = self.register_manager.lockReg(reg);
4678 defer if (maybe_lock) |lock| self.register_manager.unlockReg(lock);
4679 const tmp_reg = try self.copyToTmpRegister(ty, operand);
4680 if (err_off > 0) {
4681 const shift = @intCast(u6, err_off * 8);
4682 try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = shift });
4683 } else {
4684 try self.truncateRegister(Type.anyerror, tmp_reg);
4685 }
4686 try self.genBinOpMir(.cmp, Type.anyerror, .{ .register = tmp_reg }, .{ .immediate = 0 });
4687 },
4688 else => return self.fail("TODO implement isErr for {}", .{operand}),
46404689 }
4690
4691 return MCValue{ .compare_flags_unsigned = .gt };
46414692}
46424693
46434694fn isNonErr(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue {
......@@ -5460,6 +5511,21 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
54605511 .immediate => |x_big| {
54615512 const base_reg = opts.dest_stack_base orelse .rbp;
54625513 switch (abi_size) {
5514 0 => {
5515 assert(ty.isError());
5516 const payload = try self.addExtra(Mir.ImmPair{
5517 .dest_off = @bitCast(u32, -stack_offset),
5518 .operand = @truncate(u32, x_big),
5519 });
5520 _ = try self.addInst(.{
5521 .tag = .mov_mem_imm,
5522 .ops = Mir.Inst.Ops.encode(.{
5523 .reg1 = base_reg,
5524 .flags = 0b00,
5525 }),
5526 .data = .{ .payload = payload },
5527 });
5528 },
54635529 1, 2, 4 => {
54645530 const payload = try self.addExtra(Mir.ImmPair{
54655531 .dest_off = @bitCast(u32, -stack_offset),
......@@ -6642,7 +6708,7 @@ pub fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
66426708 const ref_int = @enumToInt(inst);
66436709 if (ref_int < Air.Inst.Ref.typed_value_map.len) {
66446710 const tv = Air.Inst.Ref.typed_value_map[ref_int];
6645 if (!tv.ty.hasRuntimeBits()) {
6711 if (!tv.ty.hasRuntimeBitsIgnoreComptime() and !tv.ty.isError()) {
66466712 return MCValue{ .none = {} };
66476713 }
66486714 return self.genTypedValue(tv);
......@@ -6650,7 +6716,7 @@ pub fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
66506716
66516717 // If the type has no codegen bits, no need to store it.
66526718 const inst_ty = self.air.typeOf(inst);
6653 if (!inst_ty.hasRuntimeBits())
6719 if (!inst_ty.hasRuntimeBitsIgnoreComptime() and !inst_ty.isError())
66546720 return MCValue{ .none = {} };
66556721
66566722 const inst_index = @intCast(Air.Inst.Index, ref_int - Air.Inst.Ref.typed_value_map.len);
......@@ -6779,6 +6845,7 @@ fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue {
67796845 const target = self.target.*;
67806846
67816847 switch (typed_value.ty.zigTypeTag()) {
6848 .Void => return MCValue{ .none = {} },
67826849 .Pointer => switch (typed_value.ty.ptrSize()) {
67836850 .Slice => {},
67846851 else => {
......@@ -6840,26 +6907,35 @@ fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue {
68406907 }
68416908 },
68426909 .ErrorSet => {
6843 const err_name = typed_value.val.castTag(.@"error").?.data.name;
6844 const module = self.bin_file.options.module.?;
6845 const global_error_set = module.global_error_set;
6846 const error_index = global_error_set.get(err_name).?;
6847 return MCValue{ .immediate = error_index };
6910 switch (typed_value.val.tag()) {
6911 .@"error" => {
6912 const err_name = typed_value.val.castTag(.@"error").?.data.name;
6913 const module = self.bin_file.options.module.?;
6914 const global_error_set = module.global_error_set;
6915 const error_index = global_error_set.get(err_name).?;
6916 return MCValue{ .immediate = error_index };
6917 },
6918 else => {
6919 // In this case we are rendering an error union which has a 0 bits payload.
6920 return MCValue{ .immediate = 0 };
6921 },
6922 }
68486923 },
68496924 .ErrorUnion => {
68506925 const error_type = typed_value.ty.errorUnionSet();
68516926 const payload_type = typed_value.ty.errorUnionPayload();
68526927
6853 if (typed_value.val.castTag(.eu_payload)) |_| {
6854 if (!payload_type.hasRuntimeBits()) {
6855 // We use the error type directly as the type.
6856 return MCValue{ .immediate = 0 };
6857 }
6858 } else {
6859 if (!payload_type.hasRuntimeBits()) {
6860 // We use the error type directly as the type.
6861 return self.genTypedValue(.{ .ty = error_type, .val = typed_value.val });
6862 }
6928 if (error_type.errorSetCardinality() == .zero) {
6929 const payload_val = typed_value.val.castTag(.eu_payload).?.data;
6930 return self.genTypedValue(.{ .ty = payload_type, .val = payload_val });
6931 }
6932
6933 const is_pl = typed_value.val.errorUnionIsPayload();
6934
6935 if (!payload_type.hasRuntimeBitsIgnoreComptime()) {
6936 // We use the error type directly as the type.
6937 const err_val = if (!is_pl) typed_value.val else Value.initTag(.zero);
6938 return self.genTypedValue(.{ .ty = error_type, .val = err_val });
68636939 }
68646940 },
68656941
......@@ -6867,7 +6943,6 @@ fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue {
68676943 .ComptimeFloat => unreachable,
68686944 .Type => unreachable,
68696945 .EnumLiteral => unreachable,
6870 .Void => unreachable,
68716946 .NoReturn => unreachable,
68726947 .Undefined => unreachable,
68736948 .Null => unreachable,
......@@ -6921,11 +6996,14 @@ fn resolveCallingConventionValues(self: *Self, fn_ty: Type) !CallMCValues {
69216996 // Return values
69226997 if (ret_ty.zigTypeTag() == .NoReturn) {
69236998 result.return_value = .{ .unreach = {} };
6924 } else if (!ret_ty.hasRuntimeBits()) {
6999 } else if (!ret_ty.hasRuntimeBitsIgnoreComptime() and !ret_ty.isError()) {
69257000 result.return_value = .{ .none = {} };
69267001 } else {
69277002 const ret_ty_size = @intCast(u32, ret_ty.abiSize(self.target.*));
6928 if (ret_ty_size <= 8) {
7003 if (ret_ty_size == 0) {
7004 assert(ret_ty.isError());
7005 result.return_value = .{ .immediate = 0 };
7006 } else if (ret_ty_size <= 8) {
69297007 const aliased_reg = registerAlias(c_abi_int_return_regs[0], ret_ty_size);
69307008 result.return_value = .{ .register = aliased_reg };
69317009 } else {
......@@ -7105,3 +7183,19 @@ fn intrinsicsAllowed(target: Target, ty: Type) bool {
71057183fn hasAvxSupport(target: Target) bool {
71067184 return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 });
71077185}
7186
7187fn errUnionPayloadOffset(ty: Type, target: std.Target) u64 {
7188 const payload_ty = ty.errorUnionPayload();
7189 return if (Type.anyerror.abiAlignment(target) >= payload_ty.abiAlignment(target))
7190 Type.anyerror.abiSize(target)
7191 else
7192 0;
7193}
7194
7195fn errUnionErrOffset(ty: Type, target: std.Target) u64 {
7196 const payload_ty = ty.errorUnionPayload();
7197 return if (Type.anyerror.abiAlignment(target) >= payload_ty.abiAlignment(target))
7198 0
7199 else
7200 payload_ty.abiSize(target);
7201}
src/codegen.zig+4-1
......@@ -442,7 +442,10 @@ pub fn generateSymbol(
442442 .Int => {
443443 const info = typed_value.ty.intInfo(target);
444444 if (info.bits <= 8) {
445 const x = @intCast(u8, typed_value.val.toUnsignedInt(target));
445 const x: u8 = switch (info.signedness) {
446 .unsigned => @intCast(u8, typed_value.val.toUnsignedInt(target)),
447 .signed => @bitCast(u8, @intCast(i8, typed_value.val.toSignedInt())),
448 };
446449 try code.append(x);
447450 return Result{ .appended = {} };
448451 }