authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-04-13 13:53:01+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-04-13 13:53:01+02:00
log31738de2817f7932fa9237492f20fb736bc07dd3
tree6613ba68b7236f1fa3d3def5b854ffeba2677694
parente2fe1907ecac075e4d4a37776359144318b6055a
parente72c41a32af41c15f4f73b46c902c01ed838462f
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15184 from jacobly0/x86_64

x86_64: fix various miscompilations

6 files changed, 252 insertions(+), 156 deletions(-)

src/arch/x86_64/CodeGen.zig+243-147
...@@ -1080,7 +1080,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -1080,7 +1080,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
10801080
1081 .constant => unreachable, // excluded from function bodies1081 .constant => unreachable, // excluded from function bodies
1082 .const_ty => unreachable, // excluded from function bodies1082 .const_ty => unreachable, // excluded from function bodies
1083 .unreach => self.finishAirBookkeeping(),1083 .unreach => if (self.wantSafety()) try self.airTrap() else self.finishAirBookkeeping(),
10841084
1085 .optional_payload => try self.airOptionalPayload(inst),1085 .optional_payload => try self.airOptionalPayload(inst),
1086 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),1086 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
...@@ -1734,10 +1734,10 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1734,10 +1734,10 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
1734 };1734 };
1735 try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);1735 try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);
17361736
1737 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));1737 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
1738 try self.asmCmovccRegisterRegister(1738 try self.asmCmovccRegisterRegister(
1739 registerAlias(dst_reg, abi_size),1739 registerAlias(dst_reg, cmov_abi_size),
1740 registerAlias(limit_reg, abi_size),1740 registerAlias(limit_reg, cmov_abi_size),
1741 cc,1741 cc,
1742 );1742 );
1743 break :result dst_mcv;1743 break :result dst_mcv;
...@@ -1785,10 +1785,10 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1785,10 +1785,10 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
1785 };1785 };
1786 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);1786 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
17871787
1788 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));1788 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
1789 try self.asmCmovccRegisterRegister(1789 try self.asmCmovccRegisterRegister(
1790 registerAlias(dst_reg, abi_size),1790 registerAlias(dst_reg, cmov_abi_size),
1791 registerAlias(limit_reg, abi_size),1791 registerAlias(limit_reg, cmov_abi_size),
1792 cc,1792 cc,
1793 );1793 );
1794 break :result dst_mcv;1794 break :result dst_mcv;
...@@ -1841,10 +1841,10 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -1841,10 +1841,10 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1841 };1841 };
18421842
1843 const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv);1843 const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv);
1844 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));1844 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
1845 try self.asmCmovccRegisterRegister(1845 try self.asmCmovccRegisterRegister(
1846 registerAlias(dst_mcv.register, abi_size),1846 registerAlias(dst_mcv.register, cmov_abi_size),
1847 registerAlias(limit_reg, abi_size),1847 registerAlias(limit_reg, cmov_abi_size),
1848 cc,1848 cc,
1849 );1849 );
1850 break :result dst_mcv;1850 break :result dst_mcv;
...@@ -3083,7 +3083,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -3083,7 +3083,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3083 };3083 };
3084 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);3084 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
30853085
3086 const dst_reg = try self.register_manager.allocReg(null, gp);3086 const dst_reg = try self.register_manager.allocReg(inst, gp);
3087 const dst_mcv = MCValue{ .register = dst_reg };3087 const dst_mcv = MCValue{ .register = dst_reg };
3088 const dst_lock = self.register_manager.lockReg(dst_reg);3088 const dst_lock = self.register_manager.lockReg(dst_reg);
3089 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);3089 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
...@@ -3098,19 +3098,37 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -3098,19 +3098,37 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
3098 }3098 }
30993099
3100 const src_bits = src_ty.bitSize(self.target.*);3100 const src_bits = src_ty.bitSize(self.target.*);
3101 const width_mcv =3101 if (math.isPowerOfTwo(src_bits)) {
3102 try self.copyToRegisterWithInstTracking(inst, dst_ty, .{ .immediate = src_bits });3102 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
3103 try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);3103 .immediate = src_bits ^ (src_bits - 1),
3104 });
3105 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
31043106
3105 const dst_abi_size = @intCast(u32, @max(dst_ty.abiSize(self.target.*), 2));3107 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3106 try self.asmCmovccRegisterRegister(3108 try self.asmCmovccRegisterRegister(
3107 registerAlias(dst_reg, dst_abi_size),3109 registerAlias(dst_reg, cmov_abi_size),
3108 registerAlias(width_mcv.register, dst_abi_size),3110 registerAlias(imm_reg, cmov_abi_size),
3109 .z,3111 .z,
3110 );3112 );
3113
3114 try self.genBinOpMir(.xor, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
3115 } else {
3116 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
3117 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),
3118 });
3119 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
3120
3121 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3122 try self.asmCmovccRegisterRegister(
3123 registerAlias(imm_reg, cmov_abi_size),
3124 registerAlias(dst_reg, cmov_abi_size),
3125 .nz,
3126 );
31113127
3112 try self.genBinOpMir(.sub, dst_ty, width_mcv, dst_mcv);3128 try self.genSetReg(dst_ty, dst_reg, .{ .immediate = src_bits - 1 });
3113 break :result width_mcv;3129 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });
3130 }
3131 break :result dst_mcv;
3114 };3132 };
3115 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });3133 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
3116}3134}
...@@ -3162,10 +3180,10 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -3162,10 +3180,10 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
3162 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });3180 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
3163 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);3181 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
31643182
3165 const abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);3183 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3166 try self.asmCmovccRegisterRegister(3184 try self.asmCmovccRegisterRegister(
3167 registerAlias(dst_reg, abi_size),3185 registerAlias(dst_reg, cmov_abi_size),
3168 registerAlias(width_reg, abi_size),3186 registerAlias(width_reg, cmov_abi_size),
3169 .z,3187 .z,
3170 );3188 );
3171 break :result dst_mcv;3189 break :result dst_mcv;
...@@ -3569,7 +3587,12 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo...@@ -3569,7 +3587,12 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
3569 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});3587 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
3570 }3588 }
35713589
3572 try self.genInlineMemcpy(dst_mcv, ptr, .{ .immediate = abi_size }, .{});3590 try self.genInlineMemcpy(
3591 .{ .ptr_stack_offset = off },
3592 ptr,
3593 .{ .immediate = abi_size },
3594 .{},
3595 );
3573 },3596 },
3574 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),3597 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
3575 }3598 }
...@@ -3745,22 +3768,47 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type...@@ -3745,22 +3768,47 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
3745 -@intCast(i32, overflow_bit_offset),3768 -@intCast(i32, overflow_bit_offset),
3746 );3769 );
3747 },3770 },
3748 .linker_load, .memory, .stack_offset => if (abi_size <= 8) {3771 .memory, .linker_load => if (abi_size <= 8) {
3772 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3773 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3774 defer self.register_manager.unlockReg(tmp_lock);
3775
3776 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3777 } else {
3778 const addr_reg = try self.register_manager.allocReg(null, gp);
3779 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3780 defer self.register_manager.unlockReg(addr_lock);
3781
3782 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, value);
3783 try self.genInlineMemcpy(
3784 ptr,
3785 .{ .register = addr_reg },
3786 .{ .immediate = abi_size },
3787 .{},
3788 );
3789 },
3790 .stack_offset => |off| if (abi_size <= 8) {
3749 const tmp_reg = try self.copyToTmpRegister(value_ty, value);3791 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3792 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3793 defer self.register_manager.unlockReg(tmp_lock);
3794
3750 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);3795 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3751 } else try self.genInlineMemcpy(3796 } else try self.genInlineMemcpy(
3752 .{ .stack_offset = 0 },3797 ptr,
3753 value,3798 .{ .ptr_stack_offset = off },
3754 .{ .immediate = abi_size },3799 .{ .immediate = abi_size },
3755 .{ .source_stack_base = .rbp, .dest_stack_base = reg.to64() },3800 .{},
3756 ),3801 ),
3757 .ptr_stack_offset => {3802 .ptr_stack_offset => {
3758 const tmp_reg = try self.copyToTmpRegister(value_ty, value);3803 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3804 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3805 defer self.register_manager.unlockReg(tmp_lock);
3806
3759 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);3807 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3760 },3808 },
3761 }3809 }
3762 },3810 },
3763 .linker_load, .memory => {3811 .memory, .linker_load => {
3764 const value_lock: ?RegisterLock = switch (value) {3812 const value_lock: ?RegisterLock = switch (value) {
3765 .register => |reg| self.register_manager.lockReg(reg),3813 .register => |reg| self.register_manager.lockReg(reg),
3766 else => null,3814 else => null,
...@@ -4736,7 +4784,7 @@ fn genBinOp(...@@ -4736,7 +4784,7 @@ fn genBinOp(
4736 },4784 },
4737 };4785 };
47384786
4739 const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));4787 const cmov_abi_size = @max(@intCast(u32, lhs_ty.abiSize(self.target.*)), 2);
4740 const tmp_reg = switch (dst_mcv) {4788 const tmp_reg = switch (dst_mcv) {
4741 .register => |reg| reg,4789 .register => |reg| reg,
4742 else => try self.copyToTmpRegister(lhs_ty, dst_mcv),4790 else => try self.copyToTmpRegister(lhs_ty, dst_mcv),
...@@ -4754,13 +4802,13 @@ fn genBinOp(...@@ -4754,13 +4802,13 @@ fn genBinOp(
4754 .ptr_stack_offset,4802 .ptr_stack_offset,
4755 => unreachable,4803 => unreachable,
4756 .register => |src_reg| try self.asmCmovccRegisterRegister(4804 .register => |src_reg| try self.asmCmovccRegisterRegister(
4757 registerAlias(tmp_reg, abi_size),4805 registerAlias(tmp_reg, cmov_abi_size),
4758 registerAlias(src_reg, abi_size),4806 registerAlias(src_reg, cmov_abi_size),
4759 cc,4807 cc,
4760 ),4808 ),
4761 .stack_offset => |off| try self.asmCmovccRegisterMemory(4809 .stack_offset => |off| try self.asmCmovccRegisterMemory(
4762 registerAlias(tmp_reg, abi_size),4810 registerAlias(tmp_reg, cmov_abi_size),
4763 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{4811 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{
4764 .base = .rbp,4812 .base = .rbp,
4765 .disp = -off,4813 .disp = -off,
4766 }),4814 }),
...@@ -4773,8 +4821,8 @@ fn genBinOp(...@@ -4773,8 +4821,8 @@ fn genBinOp(
47734821
4774 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mat_src_mcv);4822 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mat_src_mcv);
4775 try self.asmCmovccRegisterMemory(4823 try self.asmCmovccRegisterMemory(
4776 registerAlias(tmp_reg, abi_size),4824 registerAlias(tmp_reg, cmov_abi_size),
4777 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),4825 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{ .base = addr_reg }),
4778 cc,4826 cc,
4779 );4827 );
4780 },4828 },
...@@ -5530,14 +5578,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {...@@ -5530,14 +5578,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
5530 assert(ret_ty.isError());5578 assert(ret_ty.isError());
5531 },5579 },
5532 .stack_offset => {5580 .stack_offset => {
5533 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);5581 try self.store(self.ret_mcv, operand, Type.usize, ret_ty);
5534 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
5535 defer self.register_manager.unlockReg(reg_lock);
5536
5537 try self.genSetStack(ret_ty, 0, operand, .{
5538 .source_stack_base = .rbp,
5539 .dest_stack_base = reg,
5540 });
5541 },5582 },
5542 else => {5583 else => {
5543 try self.setRegOrMem(ret_ty, self.ret_mcv, operand);5584 try self.setRegOrMem(ret_ty, self.ret_mcv, operand);
...@@ -5556,6 +5597,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -5556,6 +5597,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
5556 const ptr = try self.resolveInst(un_op);5597 const ptr = try self.resolveInst(un_op);
5557 const ptr_ty = self.air.typeOf(un_op);5598 const ptr_ty = self.air.typeOf(un_op);
5558 const elem_ty = ptr_ty.elemType();5599 const elem_ty = ptr_ty.elemType();
5600 const abi_size = elem_ty.abiSize(self.target.*);
5559 switch (self.ret_mcv) {5601 switch (self.ret_mcv) {
5560 .immediate => {5602 .immediate => {
5561 assert(elem_ty.isError());5603 assert(elem_ty.isError());
...@@ -5565,10 +5607,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -5565,10 +5607,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
5565 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);5607 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
5566 defer self.register_manager.unlockReg(reg_lock);5608 defer self.register_manager.unlockReg(reg_lock);
55675609
5568 try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{5610 try self.genInlineMemcpy(.{ .register = reg }, ptr, .{ .immediate = abi_size }, .{});
5569 .source_stack_base = .rbp,
5570 .dest_stack_base = reg,
5571 });
5572 },5611 },
5573 else => {5612 else => {
5574 try self.load(self.ret_mcv, ptr, ptr_ty);5613 try self.load(self.ret_mcv, ptr, ptr_ty);
...@@ -6252,6 +6291,15 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {...@@ -6252,6 +6291,15 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
62526291
6253 const block_data = self.blocks.getPtr(inst).?;6292 const block_data = self.blocks.getPtr(inst).?;
6254 const target_branch = self.branch_stack.pop();6293 const target_branch = self.branch_stack.pop();
6294
6295 log.debug("airBlock: %{d}", .{inst});
6296 log.debug("Upper branches:", .{});
6297 for (self.branch_stack.items) |bs| {
6298 log.debug("{}", .{bs.fmtDebug()});
6299 }
6300 log.debug("Block branch: {}", .{block_data.branch.fmtDebug()});
6301 log.debug("Target branch: {}", .{target_branch.fmtDebug()});
6302
6255 try self.canonicaliseBranches(true, &block_data.branch, &target_branch, false, false);6303 try self.canonicaliseBranches(true, &block_data.branch, &target_branch, false, false);
62566304
6257 for (block_data.relocs.items) |reloc| try self.performReloc(reloc);6305 for (block_data.relocs.items) |reloc| try self.performReloc(reloc);
...@@ -6423,7 +6471,7 @@ fn canonicaliseBranches(...@@ -6423,7 +6471,7 @@ fn canonicaliseBranches(
6423 // If integer overflow occurs, the question is: why wasn't the instruction marked dead?6471 // If integer overflow occurs, the question is: why wasn't the instruction marked dead?
6424 break :blk self.getResolvedInstValue(target_key).?.*;6472 break :blk self.getResolvedInstValue(target_key).?.*;
6425 };6473 };
6426 log.debug("consolidating target_entry {d} {}=>{}", .{ target_key, target_value, canon_mcv });6474 log.debug("consolidating target_entry %{d} {}=>{}", .{ target_key, target_value, canon_mcv });
6427 // TODO handle the case where the destination stack offset / register has something6475 // TODO handle the case where the destination stack offset / register has something
6428 // going on there.6476 // going on there.
6429 assert(!hazard_map.contains(target_value));6477 assert(!hazard_map.contains(target_value));
...@@ -6445,7 +6493,7 @@ fn canonicaliseBranches(...@@ -6445,7 +6493,7 @@ fn canonicaliseBranches(
6445 const parent_mcv =6493 const parent_mcv =
6446 if (canon_value != .dead) self.getResolvedInstValue(canon_key).?.* else undefined;6494 if (canon_value != .dead) self.getResolvedInstValue(canon_key).?.* else undefined;
6447 if (canon_value != .dead) {6495 if (canon_value != .dead) {
6448 log.debug("consolidating canon_entry {d} {}=>{}", .{ canon_key, parent_mcv, canon_value });6496 log.debug("consolidating canon_entry %{d} {}=>{}", .{ canon_key, parent_mcv, canon_value });
6449 // TODO handle the case where the destination stack offset / register has something6497 // TODO handle the case where the destination stack offset / register has something
6450 // going on there.6498 // going on there.
6451 assert(!hazard_map.contains(parent_mcv));6499 assert(!hazard_map.contains(parent_mcv));
...@@ -6838,7 +6886,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE...@@ -6838,7 +6886,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
6838 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });6886 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
6839 }6887 }
6840 try self.genInlineMemset(6888 try self.genInlineMemset(
6841 .{ .stack_offset = stack_offset },6889 .{ .ptr_stack_offset = stack_offset },
6842 .{ .immediate = 0xaa },6890 .{ .immediate = 0xaa },
6843 .{ .immediate = abi_size },6891 .{ .immediate = abi_size },
6844 .{ .dest_stack_base = .rsp },6892 .{ .dest_stack_base = .rsp },
...@@ -6877,10 +6925,17 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE...@@ -6877,10 +6925,17 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
6877 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });6925 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
6878 }6926 }
68796927
6880 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{6928 const addr_reg = try self.register_manager.allocReg(null, gp);
6881 .source_stack_base = .rbp,6929 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6882 .dest_stack_base = .rsp,6930 defer self.register_manager.unlockReg(addr_lock);
6883 });6931
6932 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mcv);
6933 try self.genInlineMemcpy(
6934 .{ .ptr_stack_offset = stack_offset },
6935 .{ .register = addr_reg },
6936 .{ .immediate = abi_size },
6937 .{ .dest_stack_base = .rsp },
6938 );
6884 },6939 },
6885 .register => |reg| {6940 .register => |reg| {
6886 switch (ty.zigTypeTag()) {6941 switch (ty.zigTypeTag()) {
...@@ -6920,16 +6975,18 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE...@@ -6920,16 +6975,18 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
6920 const reg = try self.copyToTmpRegister(ty, mcv);6975 const reg = try self.copyToTmpRegister(ty, mcv);
6921 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });6976 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
6922 },6977 },
6923 .stack_offset => {6978 .stack_offset => |mcv_off| {
6924 if (abi_size <= 8) {6979 if (abi_size <= 8) {
6925 const reg = try self.copyToTmpRegister(ty, mcv);6980 const reg = try self.copyToTmpRegister(ty, mcv);
6926 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });6981 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
6927 }6982 }
69286983
6929 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{6984 try self.genInlineMemcpy(
6930 .source_stack_base = .rbp,6985 .{ .ptr_stack_offset = stack_offset },
6931 .dest_stack_base = .rsp,6986 .{ .ptr_stack_offset = mcv_off },
6932 });6987 .{ .immediate = abi_size },
6988 .{ .dest_stack_base = .rsp },
6989 );
6933 },6990 },
6934 }6991 }
6935}6992}
...@@ -6963,7 +7020,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl...@@ -6963,7 +7020,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
6963 opts,7020 opts,
6964 ),7021 ),
6965 else => |x| return self.genInlineMemset(7022 else => |x| return self.genInlineMemset(
6966 .{ .stack_offset = stack_offset },7023 .{ .ptr_stack_offset = stack_offset },
6967 .{ .immediate = 0xaa },7024 .{ .immediate = 0xaa },
6968 .{ .immediate = x },7025 .{ .immediate = x },
6969 opts,7026 opts,
...@@ -7072,22 +7129,40 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl...@@ -7072,22 +7129,40 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
7072 },7129 },
7073 }7130 }
7074 },7131 },
7075 .memory, .linker_load, .stack_offset, .ptr_stack_offset => {7132 .memory, .linker_load => if (abi_size <= 8) {
7076 switch (mcv) {7133 const reg = try self.copyToTmpRegister(ty, mcv);
7077 else => unreachable,7134 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
7078 .memory, .linker_load, .ptr_stack_offset => {},7135 } else {
7079 .stack_offset => |src_off| if (stack_offset == src_off) {7136 const addr_reg = try self.register_manager.allocReg(null, gp);
7080 // Copy stack variable to itself; nothing to do.7137 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
7081 return;7138 defer self.register_manager.unlockReg(addr_lock);
7082 },
7083 }
70847139
7085 if (abi_size <= 8) {7140 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mcv);
7086 const reg = try self.copyToTmpRegister(ty, mcv);7141 try self.genInlineMemcpy(
7087 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);7142 .{ .ptr_stack_offset = stack_offset },
7088 }7143 .{ .register = addr_reg },
7144 .{ .immediate = abi_size },
7145 .{},
7146 );
7147 },
7148 .stack_offset => |off| if (abi_size <= 8) {
7149 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7150 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7151 defer self.register_manager.unlockReg(tmp_lock);
7152
7153 try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
7154 } else try self.genInlineMemcpy(
7155 .{ .ptr_stack_offset = stack_offset },
7156 .{ .ptr_stack_offset = off },
7157 .{ .immediate = abi_size },
7158 .{},
7159 ),
7160 .ptr_stack_offset => {
7161 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7162 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7163 defer self.register_manager.unlockReg(tmp_lock);
70897164
7090 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts);7165 try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
7091 },7166 },
7092 }7167 }
7093}7168}
...@@ -7134,10 +7209,14 @@ fn genInlineMemcpyRegisterRegister(...@@ -7134,10 +7209,14 @@ fn genInlineMemcpyRegisterRegister(
7134 next_offset -= nearest_power_of_two;7209 next_offset -= nearest_power_of_two;
7135 }7210 }
7136 } else {7211 } else {
7137 try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{7212 try self.asmMemoryRegister(
7138 .base = dst_reg,7213 switch (src_reg.class()) {
7139 .disp = -offset,7214 .general_purpose, .segment => .mov,
7140 }), registerAlias(src_reg, abi_size));7215 .floating_point => .movss,
7216 },
7217 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
7218 registerAlias(src_reg, abi_size),
7219 );
7141 }7220 }
7142}7221}
71437222
...@@ -7170,9 +7249,15 @@ fn genInlineMemcpy(...@@ -7170,9 +7249,15 @@ fn genInlineMemcpy(
7170 switch (dst_ptr) {7249 switch (dst_ptr) {
7171 .memory, .linker_load => {7250 .memory, .linker_load => {
7172 try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);7251 try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
7252 // Load the pointer, which is stored in memory
7253 try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
7173 },7254 },
7174 .ptr_stack_offset, .stack_offset => |off| {7255 .stack_offset, .ptr_stack_offset => |off| {
7175 try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{7256 try self.asmRegisterMemory(switch (dst_ptr) {
7257 .stack_offset => .mov,
7258 .ptr_stack_offset => .lea,
7259 else => unreachable,
7260 }, .rdi, Memory.sib(.qword, .{
7176 .base = opts.dest_stack_base orelse .rbp,7261 .base = opts.dest_stack_base orelse .rbp,
7177 .disp = -off,7262 .disp = -off,
7178 }));7263 }));
...@@ -7192,9 +7277,15 @@ fn genInlineMemcpy(...@@ -7192,9 +7277,15 @@ fn genInlineMemcpy(
7192 switch (src_ptr) {7277 switch (src_ptr) {
7193 .memory, .linker_load => {7278 .memory, .linker_load => {
7194 try self.loadMemPtrIntoRegister(.rsi, Type.usize, src_ptr);7279 try self.loadMemPtrIntoRegister(.rsi, Type.usize, src_ptr);
7280 // Load the pointer, which is stored in memory
7281 try self.asmRegisterMemory(.mov, .rsi, Memory.sib(.qword, .{ .base = .rsi }));
7195 },7282 },
7196 .ptr_stack_offset, .stack_offset => |off| {7283 .stack_offset, .ptr_stack_offset => |off| {
7197 try self.asmRegisterMemory(.lea, .rsi, Memory.sib(.qword, .{7284 try self.asmRegisterMemory(switch (src_ptr) {
7285 .stack_offset => .mov,
7286 .ptr_stack_offset => .lea,
7287 else => unreachable,
7288 }, .rsi, Memory.sib(.qword, .{
7198 .base = opts.source_stack_base orelse .rbp,7289 .base = opts.source_stack_base orelse .rbp,
7199 .disp = -off,7290 .disp = -off,
7200 }));7291 }));
...@@ -7237,9 +7328,15 @@ fn genInlineMemset(...@@ -7237,9 +7328,15 @@ fn genInlineMemset(
7237 switch (dst_ptr) {7328 switch (dst_ptr) {
7238 .memory, .linker_load => {7329 .memory, .linker_load => {
7239 try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);7330 try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
7331 // Load the pointer, which is stored in memory
7332 try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
7240 },7333 },
7241 .ptr_stack_offset, .stack_offset => |off| {7334 .stack_offset, .ptr_stack_offset => |off| {
7242 try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{7335 try self.asmRegisterMemory(switch (dst_ptr) {
7336 .stack_offset => .mov,
7337 .ptr_stack_offset => .lea,
7338 else => unreachable,
7339 }, .rdi, Memory.sib(.qword, .{
7243 .base = opts.dest_stack_base orelse .rbp,7340 .base = opts.dest_stack_base orelse .rbp,
7244 .disp = -off,7341 .disp = -off,
7245 }));7342 }));
...@@ -7657,7 +7754,6 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -7657,7 +7754,6 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
76577754
7658fn atomicOp(7755fn atomicOp(
7659 self: *Self,7756 self: *Self,
7660 dst_reg: Register,
7661 ptr_mcv: MCValue,7757 ptr_mcv: MCValue,
7662 val_mcv: MCValue,7758 val_mcv: MCValue,
7663 ptr_ty: Type,7759 ptr_ty: Type,
...@@ -7665,11 +7761,7 @@ fn atomicOp(...@@ -7665,11 +7761,7 @@ fn atomicOp(
7665 unused: bool,7761 unused: bool,
7666 rmw_op: ?std.builtin.AtomicRmwOp,7762 rmw_op: ?std.builtin.AtomicRmwOp,
7667 order: std.builtin.AtomicOrder,7763 order: std.builtin.AtomicOrder,
7668) InnerError!void {7764) InnerError!MCValue {
7669 const dst_mcv = MCValue{ .register = dst_reg };
7670 const dst_lock = self.register_manager.lockReg(dst_reg);
7671 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
7672
7673 const ptr_lock = switch (ptr_mcv) {7765 const ptr_lock = switch (ptr_mcv) {
7674 .register => |reg| self.register_manager.lockReg(reg),7766 .register => |reg| self.register_manager.lockReg(reg),
7675 else => null,7767 else => null,
...@@ -7724,9 +7816,14 @@ fn atomicOp(...@@ -7724,9 +7816,14 @@ fn atomicOp(
7724 .SeqCst => .xchg,7816 .SeqCst => .xchg,
7725 };7817 };
77267818
7819 const dst_reg = try self.register_manager.allocReg(null, gp);
7820 const dst_mcv = MCValue{ .register = dst_reg };
7821 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
7822 defer self.register_manager.unlockReg(dst_lock);
7823
7727 try self.genSetReg(val_ty, dst_reg, val_mcv);7824 try self.genSetReg(val_ty, dst_reg, val_mcv);
7728 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {7825 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
7729 try self.genUnOpMir(.neg, val_ty, .{ .register = dst_reg });7826 try self.genUnOpMir(.neg, val_ty, dst_mcv);
7730 }7827 }
7731 _ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {7828 _ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {
7732 .mov, .xchg => .mr_sib,7829 .mov, .xchg => .mr_sib,
...@@ -7736,25 +7833,31 @@ fn atomicOp(...@@ -7736,25 +7833,31 @@ fn atomicOp(
7736 .r = registerAlias(dst_reg, val_abi_size),7833 .r = registerAlias(dst_reg, val_abi_size),
7737 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),7834 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
7738 } } });7835 } } });
7836
7837 return if (unused) .none else dst_mcv;
7739 },7838 },
7740 .loop => _ = try self.asmJccReloc(if (val_abi_size <= 8) loop: {7839 .loop => _ = if (val_abi_size <= 8) {
7741 try self.genSetReg(val_ty, dst_reg, val_mcv);7840 const tmp_reg = try self.register_manager.allocReg(null, gp);
7841 const tmp_mcv = MCValue{ .register = tmp_reg };
7842 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7843 defer self.register_manager.unlockReg(tmp_lock);
7844
7742 try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);7845 try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);
7743 const loop = @intCast(u32, self.mir_instructions.len);7846 const loop = @intCast(u32, self.mir_instructions.len);
7744 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {7847 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
7745 try self.genSetReg(val_ty, dst_reg, .{ .register = .rax });7848 try self.genSetReg(val_ty, tmp_reg, .{ .register = .rax });
7746 }7849 }
7747 if (rmw_op) |op| switch (op) {7850 if (rmw_op) |op| switch (op) {
7748 .Xchg => try self.genSetReg(val_ty, dst_reg, val_mcv),7851 .Xchg => try self.genSetReg(val_ty, tmp_reg, val_mcv),
7749 .Add => try self.genBinOpMir(.add, val_ty, dst_mcv, val_mcv),7852 .Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),
7750 .Sub => try self.genBinOpMir(.sub, val_ty, dst_mcv, val_mcv),7853 .Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),
7751 .And => try self.genBinOpMir(.@"and", val_ty, dst_mcv, val_mcv),7854 .And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),
7752 .Nand => {7855 .Nand => {
7753 try self.genBinOpMir(.@"and", val_ty, dst_mcv, val_mcv);7856 try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv);
7754 try self.genUnOpMir(.not, val_ty, dst_mcv);7857 try self.genUnOpMir(.not, val_ty, tmp_mcv);
7755 },7858 },
7756 .Or => try self.genBinOpMir(.@"or", val_ty, dst_mcv, val_mcv),7859 .Or => try self.genBinOpMir(.@"or", val_ty, tmp_mcv, val_mcv),
7757 .Xor => try self.genBinOpMir(.xor, val_ty, dst_mcv, val_mcv),7860 .Xor => try self.genBinOpMir(.xor, val_ty, tmp_mcv, val_mcv),
7758 .Min, .Max => {7861 .Min, .Max => {
7759 const cc: Condition = switch (if (val_ty.isAbiInt())7862 const cc: Condition = switch (if (val_ty.isAbiInt())
7760 val_ty.intInfo(self.target.*).signedness7863 val_ty.intInfo(self.target.*).signedness
...@@ -7772,17 +7875,18 @@ fn atomicOp(...@@ -7772,17 +7875,18 @@ fn atomicOp(
7772 },7875 },
7773 };7876 };
77747877
7775 try self.genBinOpMir(.cmp, val_ty, dst_mcv, val_mcv);7878 try self.genBinOpMir(.cmp, val_ty, tmp_mcv, val_mcv);
7879 const cmov_abi_size = @max(val_abi_size, 2);
7776 switch (val_mcv) {7880 switch (val_mcv) {
7777 .register => |val_reg| try self.asmCmovccRegisterRegister(7881 .register => |val_reg| try self.asmCmovccRegisterRegister(
7778 registerAlias(dst_reg, val_abi_size),7882 registerAlias(tmp_reg, cmov_abi_size),
7779 registerAlias(val_reg, val_abi_size),7883 registerAlias(val_reg, cmov_abi_size),
7780 cc,7884 cc,
7781 ),7885 ),
7782 .stack_offset => |val_off| try self.asmCmovccRegisterMemory(7886 .stack_offset => |val_off| try self.asmCmovccRegisterMemory(
7783 registerAlias(dst_reg, val_abi_size),7887 registerAlias(tmp_reg, cmov_abi_size),
7784 Memory.sib(7888 Memory.sib(
7785 Memory.PtrSize.fromSize(val_abi_size),7889 Memory.PtrSize.fromSize(cmov_abi_size),
7786 .{ .base = .rbp, .disp = -val_off },7890 .{ .base = .rbp, .disp = -val_off },
7787 ),7891 ),
7788 cc,7892 cc,
...@@ -7790,8 +7894,8 @@ fn atomicOp(...@@ -7790,8 +7894,8 @@ fn atomicOp(
7790 else => {7894 else => {
7791 const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);7895 const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);
7792 try self.asmCmovccRegisterRegister(7896 try self.asmCmovccRegisterRegister(
7793 registerAlias(dst_reg, val_abi_size),7897 registerAlias(tmp_reg, cmov_abi_size),
7794 registerAlias(val_reg, val_abi_size),7898 registerAlias(val_reg, cmov_abi_size),
7795 cc,7899 cc,
7796 );7900 );
7797 },7901 },
...@@ -7799,11 +7903,12 @@ fn atomicOp(...@@ -7799,11 +7903,12 @@ fn atomicOp(
7799 },7903 },
7800 };7904 };
7801 _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{7905 _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
7802 .r = registerAlias(dst_reg, val_abi_size),7906 .r = registerAlias(tmp_reg, val_abi_size),
7803 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),7907 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
7804 } } });7908 } } });
7805 break :loop loop;7909 _ = try self.asmJccReloc(loop, .ne);
7806 } else loop: {7910 return if (unused) .none else .{ .register = .rax };
7911 } else {
7807 try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{7912 try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{
7808 .base = ptr_mem.sib.base,7913 .base = ptr_mem.sib.base,
7809 .scale_index = ptr_mem.sib.scale_index,7914 .scale_index = ptr_mem.sib.scale_index,
...@@ -7869,8 +7974,22 @@ fn atomicOp(...@@ -7869,8 +7974,22 @@ fn atomicOp(
7869 _ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{7974 _ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
7870 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),7975 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
7871 } });7976 } });
7872 break :loop loop;7977 _ = try self.asmJccReloc(loop, .ne);
7873 }, .ne),7978
7979 if (unused) return .none;
7980 const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
7981 try self.asmMemoryRegister(
7982 .mov,
7983 Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - dst_mcv.stack_offset }),
7984 .rax,
7985 );
7986 try self.asmMemoryRegister(
7987 .mov,
7988 Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - dst_mcv.stack_offset }),
7989 .rdx,
7990 );
7991 return dst_mcv;
7992 },
7874 .libcall => return self.fail("TODO implement x86 atomic libcall", .{}),7993 .libcall => return self.fail("TODO implement x86 atomic libcall", .{}),
7875 }7994 }
7876}7995}
...@@ -7884,7 +8003,6 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {...@@ -7884,7 +8003,6 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
7884 defer for (regs_lock) |lock| self.register_manager.unlockReg(lock);8003 defer for (regs_lock) |lock| self.register_manager.unlockReg(lock);
78858004
7886 const unused = self.liveness.isUnused(inst);8005 const unused = self.liveness.isUnused(inst);
7887 const dst_reg = try self.register_manager.allocReg(if (unused) null else inst, gp);
78888006
7889 const ptr_ty = self.air.typeOf(pl_op.operand);8007 const ptr_ty = self.air.typeOf(pl_op.operand);
7890 const ptr_mcv = try self.resolveInst(pl_op.operand);8008 const ptr_mcv = try self.resolveInst(pl_op.operand);
...@@ -7892,8 +8010,8 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {...@@ -7892,8 +8010,8 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
7892 const val_ty = self.air.typeOf(extra.operand);8010 const val_ty = self.air.typeOf(extra.operand);
7893 const val_mcv = try self.resolveInst(extra.operand);8011 const val_mcv = try self.resolveInst(extra.operand);
78948012
7895 try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering());8013 const result =
7896 const result: MCValue = if (unused) .dead else .{ .register = dst_reg };8014 try self.atomicOp(ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering());
7897 return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });8015 return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
7898}8016}
78998017
...@@ -7926,16 +8044,14 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {...@@ -7926,16 +8044,14 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
7926fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {8044fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
7927 const bin_op = self.air.instructions.items(.data)[inst].bin_op;8045 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
79288046
7929 const dst_reg = try self.register_manager.allocReg(null, gp);
7930
7931 const ptr_ty = self.air.typeOf(bin_op.lhs);8047 const ptr_ty = self.air.typeOf(bin_op.lhs);
7932 const ptr_mcv = try self.resolveInst(bin_op.lhs);8048 const ptr_mcv = try self.resolveInst(bin_op.lhs);
79338049
7934 const val_ty = self.air.typeOf(bin_op.rhs);8050 const val_ty = self.air.typeOf(bin_op.rhs);
7935 const val_mcv = try self.resolveInst(bin_op.rhs);8051 const val_mcv = try self.resolveInst(bin_op.rhs);
79368052
7937 try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order);8053 const result = try self.atomicOp(ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order);
7938 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });8054 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
7939}8055}
79408056
7941fn airMemset(self: *Self, inst: Air.Inst.Index) !void {8057fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
...@@ -7979,7 +8095,6 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -7979,7 +8095,6 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
7979 };8095 };
7980 defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock);8096 defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock);
79818097
7982 const src_ty = self.air.typeOf(extra.lhs);
7983 const src_ptr = try self.resolveInst(extra.lhs);8098 const src_ptr = try self.resolveInst(extra.lhs);
7984 const src_ptr_lock: ?RegisterLock = switch (src_ptr) {8099 const src_ptr_lock: ?RegisterLock = switch (src_ptr) {
7985 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),8100 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
...@@ -7994,25 +8109,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -7994,25 +8109,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
7994 };8109 };
7995 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);8110 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
79968111
7997 // TODO Is this the only condition for pointer dereference for memcpy?8112 try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{});
7998 const src: MCValue = blk: {
7999 switch (src_ptr) {
8000 .linker_load, .memory => {
8001 const reg = try self.register_manager.allocReg(null, gp);
8002 try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
8003 try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{ .base = reg }));
8004 break :blk MCValue{ .register = reg };
8005 },
8006 else => break :blk src_ptr,
8007 }
8008 };
8009 const src_lock: ?RegisterLock = switch (src) {
8010 .register => |reg| self.register_manager.lockReg(reg),
8011 else => null,
8012 };
8013 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
8014
8015 try self.genInlineMemcpy(dst_ptr, src, len, .{});
80168113
8017 return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });8114 return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
8018}8115}
...@@ -8156,11 +8253,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8156,11 +8253,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8156 switch (result_ty.zigTypeTag()) {8253 switch (result_ty.zigTypeTag()) {
8157 .Struct => {8254 .Struct => {
8158 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));8255 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
8159 const dst_mcv = MCValue{ .stack_offset = stack_offset };
8160 if (result_ty.containerLayout() == .Packed) {8256 if (result_ty.containerLayout() == .Packed) {
8161 const struct_obj = result_ty.castTag(.@"struct").?.data;8257 const struct_obj = result_ty.castTag(.@"struct").?.data;
8162 try self.genInlineMemset(8258 try self.genInlineMemset(
8163 dst_mcv,8259 .{ .ptr_stack_offset = stack_offset },
8164 .{ .immediate = 0 },8260 .{ .immediate = 0 },
8165 .{ .immediate = abi_size },8261 .{ .immediate = abi_size },
8166 .{},8262 .{},
...@@ -8236,7 +8332,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -8236,7 +8332,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
8236 const elem_mcv = try self.resolveInst(elem);8332 const elem_mcv = try self.resolveInst(elem);
8237 try self.genSetStack(elem_ty, stack_offset - elem_off, elem_mcv, .{});8333 try self.genSetStack(elem_ty, stack_offset - elem_off, elem_mcv, .{});
8238 }8334 }
8239 break :res dst_mcv;8335 break :res .{ .stack_offset = stack_offset };
8240 },8336 },
8241 .Array => {8337 .Array => {
8242 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));8338 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
src/codegen.zig+9-5
...@@ -966,7 +966,7 @@ fn genDeclRef(...@@ -966,7 +966,7 @@ fn genDeclRef(
966 const module = bin_file.options.module.?;966 const module = bin_file.options.module.?;
967 const decl = module.declPtr(decl_index);967 const decl = module.declPtr(decl_index);
968968
969 if (decl.ty.zigTypeTag() != .Fn and !decl.ty.hasRuntimeBitsIgnoreComptime()) {969 if (!decl.ty.isFnOrHasRuntimeBitsIgnoreComptime()) {
970 const imm: u64 = switch (ptr_bytes) {970 const imm: u64 = switch (ptr_bytes) {
971 1 => 0xaa,971 1 => 0xaa,
972 2 => 0xaaaa,972 2 => 0xaaaa,
...@@ -978,10 +978,14 @@ fn genDeclRef(...@@ -978,10 +978,14 @@ fn genDeclRef(
978 }978 }
979979
980 // TODO this feels clunky. Perhaps we should check for it in `genTypedValue`?980 // TODO this feels clunky. Perhaps we should check for it in `genTypedValue`?
981 if (tv.ty.zigTypeTag() == .Pointer) blk: {981 if (tv.ty.castPtrToFn()) |fn_ty| {
982 if (tv.ty.castPtrToFn()) |_| break :blk;982 if (fn_ty.fnInfo().is_generic) {
983 if (!tv.ty.elemType2().hasRuntimeBits()) {983 return GenResult.mcv(.{ .immediate = fn_ty.abiAlignment(target) });
984 return GenResult.mcv(.none);984 }
985 } else if (tv.ty.zigTypeTag() == .Pointer) {
986 const elem_ty = tv.ty.elemType2();
987 if (!elem_ty.hasRuntimeBits()) {
988 return GenResult.mcv(.{ .immediate = elem_ty.abiAlignment(target) });
985 }989 }
986 }990 }
987991
test/behavior/array.zig-1
...@@ -190,7 +190,6 @@ test "nested arrays of strings" {...@@ -190,7 +190,6 @@ test "nested arrays of strings" {
190 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;190 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
191 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;191 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
192 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO192 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
193 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
194193
195 const array_of_strings = [_][]const u8{ "hello", "this", "is", "my", "thing" };194 const array_of_strings = [_][]const u8{ "hello", "this", "is", "my", "thing" };
196 for (array_of_strings, 0..) |s, i| {195 for (array_of_strings, 0..) |s, i| {
test/behavior/atomics.zig-1
...@@ -232,7 +232,6 @@ fn testAtomicRmwFloat() !void {...@@ -232,7 +232,6 @@ fn testAtomicRmwFloat() !void {
232232
233test "atomicrmw with ints" {233test "atomicrmw with ints" {
234 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO234 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
235 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
236 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO235 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
237 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO236 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
238 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO237 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
test/behavior/bugs/718.zig-1
...@@ -12,7 +12,6 @@ var keys: Keys = undefined;...@@ -12,7 +12,6 @@ var keys: Keys = undefined;
12test "zero keys with @memset" {12test "zero keys with @memset" {
13 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO13 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
14 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO14 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
15 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
16 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO15 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1716
18 @memset(@ptrCast([*]u8, &keys), 0, @sizeOf(@TypeOf(keys)));17 @memset(@ptrCast([*]u8, &keys), 0, @sizeOf(@TypeOf(keys)));
test/behavior/bugs/7325.zig-1
...@@ -79,7 +79,6 @@ fn genExpression(expr: Expression) !ExpressionResult {...@@ -79,7 +79,6 @@ fn genExpression(expr: Expression) !ExpressionResult {
7979
80test {80test {
81 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO81 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
82 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
83 if (builtin.zig_backend == .stage2_x86) return error.SkipZigTest; // TODO82 if (builtin.zig_backend == .stage2_x86) return error.SkipZigTest; // TODO
84 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO83 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
85 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO84 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO