authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-04-13 13:53:01+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-04-13 13:53:01+02:00
log31738de2817f7932fa9237492f20fb736bc07dd3
tree6613ba68b7236f1fa3d3def5b854ffeba2677694
parente2fe1907ecac075e4d4a37776359144318b6055a
parente72c41a32af41c15f4f73b46c902c01ed838462f
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15184 from jacobly0/x86_64

x86_64: fix various miscompilations

6 files changed, 252 insertions(+), 156 deletions(-)

src/arch/x86_64/CodeGen.zig+243-147
......@@ -1080,7 +1080,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
10801080
10811081 .constant => unreachable, // excluded from function bodies
10821082 .const_ty => unreachable, // excluded from function bodies
1083 .unreach => self.finishAirBookkeeping(),
1083 .unreach => if (self.wantSafety()) try self.airTrap() else self.finishAirBookkeeping(),
10841084
10851085 .optional_payload => try self.airOptionalPayload(inst),
10861086 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
......@@ -1734,10 +1734,10 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
17341734 };
17351735 try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);
17361736
1737 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1737 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
17381738 try self.asmCmovccRegisterRegister(
1739 registerAlias(dst_reg, abi_size),
1740 registerAlias(limit_reg, abi_size),
1739 registerAlias(dst_reg, cmov_abi_size),
1740 registerAlias(limit_reg, cmov_abi_size),
17411741 cc,
17421742 );
17431743 break :result dst_mcv;
......@@ -1785,10 +1785,10 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
17851785 };
17861786 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
17871787
1788 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1788 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
17891789 try self.asmCmovccRegisterRegister(
1790 registerAlias(dst_reg, abi_size),
1791 registerAlias(limit_reg, abi_size),
1790 registerAlias(dst_reg, cmov_abi_size),
1791 registerAlias(limit_reg, cmov_abi_size),
17921792 cc,
17931793 );
17941794 break :result dst_mcv;
......@@ -1841,10 +1841,10 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
18411841 };
18421842
18431843 const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv);
1844 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1844 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
18451845 try self.asmCmovccRegisterRegister(
1846 registerAlias(dst_mcv.register, abi_size),
1847 registerAlias(limit_reg, abi_size),
1846 registerAlias(dst_mcv.register, cmov_abi_size),
1847 registerAlias(limit_reg, cmov_abi_size),
18481848 cc,
18491849 );
18501850 break :result dst_mcv;
......@@ -3083,7 +3083,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
30833083 };
30843084 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
30853085
3086 const dst_reg = try self.register_manager.allocReg(null, gp);
3086 const dst_reg = try self.register_manager.allocReg(inst, gp);
30873087 const dst_mcv = MCValue{ .register = dst_reg };
30883088 const dst_lock = self.register_manager.lockReg(dst_reg);
30893089 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
......@@ -3098,19 +3098,37 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
30983098 }
30993099
31003100 const src_bits = src_ty.bitSize(self.target.*);
3101 const width_mcv =
3102 try self.copyToRegisterWithInstTracking(inst, dst_ty, .{ .immediate = src_bits });
3103 try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
3101 if (math.isPowerOfTwo(src_bits)) {
3102 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
3103 .immediate = src_bits ^ (src_bits - 1),
3104 });
3105 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
31043106
3105 const dst_abi_size = @intCast(u32, @max(dst_ty.abiSize(self.target.*), 2));
3106 try self.asmCmovccRegisterRegister(
3107 registerAlias(dst_reg, dst_abi_size),
3108 registerAlias(width_mcv.register, dst_abi_size),
3109 .z,
3110 );
3107 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3108 try self.asmCmovccRegisterRegister(
3109 registerAlias(dst_reg, cmov_abi_size),
3110 registerAlias(imm_reg, cmov_abi_size),
3111 .z,
3112 );
3113
3114 try self.genBinOpMir(.xor, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
3115 } else {
3116 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
3117 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),
3118 });
3119 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
3120
3121 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3122 try self.asmCmovccRegisterRegister(
3123 registerAlias(imm_reg, cmov_abi_size),
3124 registerAlias(dst_reg, cmov_abi_size),
3125 .nz,
3126 );
31113127
3112 try self.genBinOpMir(.sub, dst_ty, width_mcv, dst_mcv);
3113 break :result width_mcv;
3128 try self.genSetReg(dst_ty, dst_reg, .{ .immediate = src_bits - 1 });
3129 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .register = imm_reg });
3130 }
3131 break :result dst_mcv;
31143132 };
31153133 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
31163134}
......@@ -3162,10 +3180,10 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
31623180 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
31633181 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
31643182
3165 const abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
3183 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
31663184 try self.asmCmovccRegisterRegister(
3167 registerAlias(dst_reg, abi_size),
3168 registerAlias(width_reg, abi_size),
3185 registerAlias(dst_reg, cmov_abi_size),
3186 registerAlias(width_reg, cmov_abi_size),
31693187 .z,
31703188 );
31713189 break :result dst_mcv;
......@@ -3569,7 +3587,12 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
35693587 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
35703588 }
35713589
3572 try self.genInlineMemcpy(dst_mcv, ptr, .{ .immediate = abi_size }, .{});
3590 try self.genInlineMemcpy(
3591 .{ .ptr_stack_offset = off },
3592 ptr,
3593 .{ .immediate = abi_size },
3594 .{},
3595 );
35733596 },
35743597 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
35753598 }
......@@ -3745,22 +3768,47 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
37453768 -@intCast(i32, overflow_bit_offset),
37463769 );
37473770 },
3748 .linker_load, .memory, .stack_offset => if (abi_size <= 8) {
3771 .memory, .linker_load => if (abi_size <= 8) {
3772 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3773 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3774 defer self.register_manager.unlockReg(tmp_lock);
3775
3776 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3777 } else {
3778 const addr_reg = try self.register_manager.allocReg(null, gp);
3779 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3780 defer self.register_manager.unlockReg(addr_lock);
3781
3782 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, value);
3783 try self.genInlineMemcpy(
3784 ptr,
3785 .{ .register = addr_reg },
3786 .{ .immediate = abi_size },
3787 .{},
3788 );
3789 },
3790 .stack_offset => |off| if (abi_size <= 8) {
37493791 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3792 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3793 defer self.register_manager.unlockReg(tmp_lock);
3794
37503795 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
37513796 } else try self.genInlineMemcpy(
3752 .{ .stack_offset = 0 },
3753 value,
3797 ptr,
3798 .{ .ptr_stack_offset = off },
37543799 .{ .immediate = abi_size },
3755 .{ .source_stack_base = .rbp, .dest_stack_base = reg.to64() },
3800 .{},
37563801 ),
37573802 .ptr_stack_offset => {
37583803 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3804 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3805 defer self.register_manager.unlockReg(tmp_lock);
3806
37593807 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
37603808 },
37613809 }
37623810 },
3763 .linker_load, .memory => {
3811 .memory, .linker_load => {
37643812 const value_lock: ?RegisterLock = switch (value) {
37653813 .register => |reg| self.register_manager.lockReg(reg),
37663814 else => null,
......@@ -4736,7 +4784,7 @@ fn genBinOp(
47364784 },
47374785 };
47384786
4739 const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
4787 const cmov_abi_size = @max(@intCast(u32, lhs_ty.abiSize(self.target.*)), 2);
47404788 const tmp_reg = switch (dst_mcv) {
47414789 .register => |reg| reg,
47424790 else => try self.copyToTmpRegister(lhs_ty, dst_mcv),
......@@ -4754,13 +4802,13 @@ fn genBinOp(
47544802 .ptr_stack_offset,
47554803 => unreachable,
47564804 .register => |src_reg| try self.asmCmovccRegisterRegister(
4757 registerAlias(tmp_reg, abi_size),
4758 registerAlias(src_reg, abi_size),
4805 registerAlias(tmp_reg, cmov_abi_size),
4806 registerAlias(src_reg, cmov_abi_size),
47594807 cc,
47604808 ),
47614809 .stack_offset => |off| try self.asmCmovccRegisterMemory(
4762 registerAlias(tmp_reg, abi_size),
4763 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
4810 registerAlias(tmp_reg, cmov_abi_size),
4811 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{
47644812 .base = .rbp,
47654813 .disp = -off,
47664814 }),
......@@ -4773,8 +4821,8 @@ fn genBinOp(
47734821
47744822 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mat_src_mcv);
47754823 try self.asmCmovccRegisterMemory(
4776 registerAlias(tmp_reg, abi_size),
4777 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
4824 registerAlias(tmp_reg, cmov_abi_size),
4825 Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{ .base = addr_reg }),
47784826 cc,
47794827 );
47804828 },
......@@ -5530,14 +5578,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
55305578 assert(ret_ty.isError());
55315579 },
55325580 .stack_offset => {
5533 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
5534 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
5535 defer self.register_manager.unlockReg(reg_lock);
5536
5537 try self.genSetStack(ret_ty, 0, operand, .{
5538 .source_stack_base = .rbp,
5539 .dest_stack_base = reg,
5540 });
5581 try self.store(self.ret_mcv, operand, Type.usize, ret_ty);
55415582 },
55425583 else => {
55435584 try self.setRegOrMem(ret_ty, self.ret_mcv, operand);
......@@ -5556,6 +5597,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
55565597 const ptr = try self.resolveInst(un_op);
55575598 const ptr_ty = self.air.typeOf(un_op);
55585599 const elem_ty = ptr_ty.elemType();
5600 const abi_size = elem_ty.abiSize(self.target.*);
55595601 switch (self.ret_mcv) {
55605602 .immediate => {
55615603 assert(elem_ty.isError());
......@@ -5565,10 +5607,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
55655607 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
55665608 defer self.register_manager.unlockReg(reg_lock);
55675609
5568 try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{
5569 .source_stack_base = .rbp,
5570 .dest_stack_base = reg,
5571 });
5610 try self.genInlineMemcpy(.{ .register = reg }, ptr, .{ .immediate = abi_size }, .{});
55725611 },
55735612 else => {
55745613 try self.load(self.ret_mcv, ptr, ptr_ty);
......@@ -6252,6 +6291,15 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void {
62526291
62536292 const block_data = self.blocks.getPtr(inst).?;
62546293 const target_branch = self.branch_stack.pop();
6294
6295 log.debug("airBlock: %{d}", .{inst});
6296 log.debug("Upper branches:", .{});
6297 for (self.branch_stack.items) |bs| {
6298 log.debug("{}", .{bs.fmtDebug()});
6299 }
6300 log.debug("Block branch: {}", .{block_data.branch.fmtDebug()});
6301 log.debug("Target branch: {}", .{target_branch.fmtDebug()});
6302
62556303 try self.canonicaliseBranches(true, &block_data.branch, &target_branch, false, false);
62566304
62576305 for (block_data.relocs.items) |reloc| try self.performReloc(reloc);
......@@ -6423,7 +6471,7 @@ fn canonicaliseBranches(
64236471 // If integer overflow occurs, the question is: why wasn't the instruction marked dead?
64246472 break :blk self.getResolvedInstValue(target_key).?.*;
64256473 };
6426 log.debug("consolidating target_entry {d} {}=>{}", .{ target_key, target_value, canon_mcv });
6474 log.debug("consolidating target_entry %{d} {}=>{}", .{ target_key, target_value, canon_mcv });
64276475 // TODO handle the case where the destination stack offset / register has something
64286476 // going on there.
64296477 assert(!hazard_map.contains(target_value));
......@@ -6445,7 +6493,7 @@ fn canonicaliseBranches(
64456493 const parent_mcv =
64466494 if (canon_value != .dead) self.getResolvedInstValue(canon_key).?.* else undefined;
64476495 if (canon_value != .dead) {
6448 log.debug("consolidating canon_entry {d} {}=>{}", .{ canon_key, parent_mcv, canon_value });
6496 log.debug("consolidating canon_entry %{d} {}=>{}", .{ canon_key, parent_mcv, canon_value });
64496497 // TODO handle the case where the destination stack offset / register has something
64506498 // going on there.
64516499 assert(!hazard_map.contains(parent_mcv));
......@@ -6838,7 +6886,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
68386886 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
68396887 }
68406888 try self.genInlineMemset(
6841 .{ .stack_offset = stack_offset },
6889 .{ .ptr_stack_offset = stack_offset },
68426890 .{ .immediate = 0xaa },
68436891 .{ .immediate = abi_size },
68446892 .{ .dest_stack_base = .rsp },
......@@ -6877,10 +6925,17 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
68776925 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
68786926 }
68796927
6880 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{
6881 .source_stack_base = .rbp,
6882 .dest_stack_base = .rsp,
6883 });
6928 const addr_reg = try self.register_manager.allocReg(null, gp);
6929 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6930 defer self.register_manager.unlockReg(addr_lock);
6931
6932 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mcv);
6933 try self.genInlineMemcpy(
6934 .{ .ptr_stack_offset = stack_offset },
6935 .{ .register = addr_reg },
6936 .{ .immediate = abi_size },
6937 .{ .dest_stack_base = .rsp },
6938 );
68846939 },
68856940 .register => |reg| {
68866941 switch (ty.zigTypeTag()) {
......@@ -6920,16 +6975,18 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
69206975 const reg = try self.copyToTmpRegister(ty, mcv);
69216976 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
69226977 },
6923 .stack_offset => {
6978 .stack_offset => |mcv_off| {
69246979 if (abi_size <= 8) {
69256980 const reg = try self.copyToTmpRegister(ty, mcv);
69266981 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
69276982 }
69286983
6929 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{
6930 .source_stack_base = .rbp,
6931 .dest_stack_base = .rsp,
6932 });
6984 try self.genInlineMemcpy(
6985 .{ .ptr_stack_offset = stack_offset },
6986 .{ .ptr_stack_offset = mcv_off },
6987 .{ .immediate = abi_size },
6988 .{ .dest_stack_base = .rsp },
6989 );
69336990 },
69346991 }
69356992}
......@@ -6963,7 +7020,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
69637020 opts,
69647021 ),
69657022 else => |x| return self.genInlineMemset(
6966 .{ .stack_offset = stack_offset },
7023 .{ .ptr_stack_offset = stack_offset },
69677024 .{ .immediate = 0xaa },
69687025 .{ .immediate = x },
69697026 opts,
......@@ -7072,22 +7129,40 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
70727129 },
70737130 }
70747131 },
7075 .memory, .linker_load, .stack_offset, .ptr_stack_offset => {
7076 switch (mcv) {
7077 else => unreachable,
7078 .memory, .linker_load, .ptr_stack_offset => {},
7079 .stack_offset => |src_off| if (stack_offset == src_off) {
7080 // Copy stack variable to itself; nothing to do.
7081 return;
7082 },
7083 }
7132 .memory, .linker_load => if (abi_size <= 8) {
7133 const reg = try self.copyToTmpRegister(ty, mcv);
7134 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
7135 } else {
7136 const addr_reg = try self.register_manager.allocReg(null, gp);
7137 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
7138 defer self.register_manager.unlockReg(addr_lock);
70847139
7085 if (abi_size <= 8) {
7086 const reg = try self.copyToTmpRegister(ty, mcv);
7087 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
7088 }
7140 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mcv);
7141 try self.genInlineMemcpy(
7142 .{ .ptr_stack_offset = stack_offset },
7143 .{ .register = addr_reg },
7144 .{ .immediate = abi_size },
7145 .{},
7146 );
7147 },
7148 .stack_offset => |off| if (abi_size <= 8) {
7149 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7150 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7151 defer self.register_manager.unlockReg(tmp_lock);
7152
7153 try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
7154 } else try self.genInlineMemcpy(
7155 .{ .ptr_stack_offset = stack_offset },
7156 .{ .ptr_stack_offset = off },
7157 .{ .immediate = abi_size },
7158 .{},
7159 ),
7160 .ptr_stack_offset => {
7161 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7162 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7163 defer self.register_manager.unlockReg(tmp_lock);
70897164
7090 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts);
7165 try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
70917166 },
70927167 }
70937168}
......@@ -7134,10 +7209,14 @@ fn genInlineMemcpyRegisterRegister(
71347209 next_offset -= nearest_power_of_two;
71357210 }
71367211 } else {
7137 try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7138 .base = dst_reg,
7139 .disp = -offset,
7140 }), registerAlias(src_reg, abi_size));
7212 try self.asmMemoryRegister(
7213 switch (src_reg.class()) {
7214 .general_purpose, .segment => .mov,
7215 .floating_point => .movss,
7216 },
7217 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
7218 registerAlias(src_reg, abi_size),
7219 );
71417220 }
71427221}
71437222
......@@ -7170,9 +7249,15 @@ fn genInlineMemcpy(
71707249 switch (dst_ptr) {
71717250 .memory, .linker_load => {
71727251 try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
7252 // Load the pointer, which is stored in memory
7253 try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
71737254 },
7174 .ptr_stack_offset, .stack_offset => |off| {
7175 try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{
7255 .stack_offset, .ptr_stack_offset => |off| {
7256 try self.asmRegisterMemory(switch (dst_ptr) {
7257 .stack_offset => .mov,
7258 .ptr_stack_offset => .lea,
7259 else => unreachable,
7260 }, .rdi, Memory.sib(.qword, .{
71767261 .base = opts.dest_stack_base orelse .rbp,
71777262 .disp = -off,
71787263 }));
......@@ -7192,9 +7277,15 @@ fn genInlineMemcpy(
71927277 switch (src_ptr) {
71937278 .memory, .linker_load => {
71947279 try self.loadMemPtrIntoRegister(.rsi, Type.usize, src_ptr);
7280 // Load the pointer, which is stored in memory
7281 try self.asmRegisterMemory(.mov, .rsi, Memory.sib(.qword, .{ .base = .rsi }));
71957282 },
7196 .ptr_stack_offset, .stack_offset => |off| {
7197 try self.asmRegisterMemory(.lea, .rsi, Memory.sib(.qword, .{
7283 .stack_offset, .ptr_stack_offset => |off| {
7284 try self.asmRegisterMemory(switch (src_ptr) {
7285 .stack_offset => .mov,
7286 .ptr_stack_offset => .lea,
7287 else => unreachable,
7288 }, .rsi, Memory.sib(.qword, .{
71987289 .base = opts.source_stack_base orelse .rbp,
71997290 .disp = -off,
72007291 }));
......@@ -7237,9 +7328,15 @@ fn genInlineMemset(
72377328 switch (dst_ptr) {
72387329 .memory, .linker_load => {
72397330 try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
7331 // Load the pointer, which is stored in memory
7332 try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
72407333 },
7241 .ptr_stack_offset, .stack_offset => |off| {
7242 try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{
7334 .stack_offset, .ptr_stack_offset => |off| {
7335 try self.asmRegisterMemory(switch (dst_ptr) {
7336 .stack_offset => .mov,
7337 .ptr_stack_offset => .lea,
7338 else => unreachable,
7339 }, .rdi, Memory.sib(.qword, .{
72437340 .base = opts.dest_stack_base orelse .rbp,
72447341 .disp = -off,
72457342 }));
......@@ -7657,7 +7754,6 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
76577754
76587755fn atomicOp(
76597756 self: *Self,
7660 dst_reg: Register,
76617757 ptr_mcv: MCValue,
76627758 val_mcv: MCValue,
76637759 ptr_ty: Type,
......@@ -7665,11 +7761,7 @@ fn atomicOp(
76657761 unused: bool,
76667762 rmw_op: ?std.builtin.AtomicRmwOp,
76677763 order: std.builtin.AtomicOrder,
7668) InnerError!void {
7669 const dst_mcv = MCValue{ .register = dst_reg };
7670 const dst_lock = self.register_manager.lockReg(dst_reg);
7671 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
7672
7764) InnerError!MCValue {
76737765 const ptr_lock = switch (ptr_mcv) {
76747766 .register => |reg| self.register_manager.lockReg(reg),
76757767 else => null,
......@@ -7724,9 +7816,14 @@ fn atomicOp(
77247816 .SeqCst => .xchg,
77257817 };
77267818
7819 const dst_reg = try self.register_manager.allocReg(null, gp);
7820 const dst_mcv = MCValue{ .register = dst_reg };
7821 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
7822 defer self.register_manager.unlockReg(dst_lock);
7823
77277824 try self.genSetReg(val_ty, dst_reg, val_mcv);
77287825 if (rmw_op == std.builtin.AtomicRmwOp.Sub and tag == .xadd) {
7729 try self.genUnOpMir(.neg, val_ty, .{ .register = dst_reg });
7826 try self.genUnOpMir(.neg, val_ty, dst_mcv);
77307827 }
77317828 _ = try self.addInst(.{ .tag = tag, .ops = switch (tag) {
77327829 .mov, .xchg => .mr_sib,
......@@ -7736,25 +7833,31 @@ fn atomicOp(
77367833 .r = registerAlias(dst_reg, val_abi_size),
77377834 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
77387835 } } });
7836
7837 return if (unused) .none else dst_mcv;
77397838 },
7740 .loop => _ = try self.asmJccReloc(if (val_abi_size <= 8) loop: {
7741 try self.genSetReg(val_ty, dst_reg, val_mcv);
7839 .loop => _ = if (val_abi_size <= 8) {
7840 const tmp_reg = try self.register_manager.allocReg(null, gp);
7841 const tmp_mcv = MCValue{ .register = tmp_reg };
7842 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7843 defer self.register_manager.unlockReg(tmp_lock);
7844
77427845 try self.asmRegisterMemory(.mov, registerAlias(.rax, val_abi_size), ptr_mem);
77437846 const loop = @intCast(u32, self.mir_instructions.len);
77447847 if (rmw_op != std.builtin.AtomicRmwOp.Xchg) {
7745 try self.genSetReg(val_ty, dst_reg, .{ .register = .rax });
7848 try self.genSetReg(val_ty, tmp_reg, .{ .register = .rax });
77467849 }
77477850 if (rmw_op) |op| switch (op) {
7748 .Xchg => try self.genSetReg(val_ty, dst_reg, val_mcv),
7749 .Add => try self.genBinOpMir(.add, val_ty, dst_mcv, val_mcv),
7750 .Sub => try self.genBinOpMir(.sub, val_ty, dst_mcv, val_mcv),
7751 .And => try self.genBinOpMir(.@"and", val_ty, dst_mcv, val_mcv),
7851 .Xchg => try self.genSetReg(val_ty, tmp_reg, val_mcv),
7852 .Add => try self.genBinOpMir(.add, val_ty, tmp_mcv, val_mcv),
7853 .Sub => try self.genBinOpMir(.sub, val_ty, tmp_mcv, val_mcv),
7854 .And => try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv),
77527855 .Nand => {
7753 try self.genBinOpMir(.@"and", val_ty, dst_mcv, val_mcv);
7754 try self.genUnOpMir(.not, val_ty, dst_mcv);
7856 try self.genBinOpMir(.@"and", val_ty, tmp_mcv, val_mcv);
7857 try self.genUnOpMir(.not, val_ty, tmp_mcv);
77557858 },
7756 .Or => try self.genBinOpMir(.@"or", val_ty, dst_mcv, val_mcv),
7757 .Xor => try self.genBinOpMir(.xor, val_ty, dst_mcv, val_mcv),
7859 .Or => try self.genBinOpMir(.@"or", val_ty, tmp_mcv, val_mcv),
7860 .Xor => try self.genBinOpMir(.xor, val_ty, tmp_mcv, val_mcv),
77587861 .Min, .Max => {
77597862 const cc: Condition = switch (if (val_ty.isAbiInt())
77607863 val_ty.intInfo(self.target.*).signedness
......@@ -7772,17 +7875,18 @@ fn atomicOp(
77727875 },
77737876 };
77747877
7775 try self.genBinOpMir(.cmp, val_ty, dst_mcv, val_mcv);
7878 try self.genBinOpMir(.cmp, val_ty, tmp_mcv, val_mcv);
7879 const cmov_abi_size = @max(val_abi_size, 2);
77767880 switch (val_mcv) {
77777881 .register => |val_reg| try self.asmCmovccRegisterRegister(
7778 registerAlias(dst_reg, val_abi_size),
7779 registerAlias(val_reg, val_abi_size),
7882 registerAlias(tmp_reg, cmov_abi_size),
7883 registerAlias(val_reg, cmov_abi_size),
77807884 cc,
77817885 ),
77827886 .stack_offset => |val_off| try self.asmCmovccRegisterMemory(
7783 registerAlias(dst_reg, val_abi_size),
7887 registerAlias(tmp_reg, cmov_abi_size),
77847888 Memory.sib(
7785 Memory.PtrSize.fromSize(val_abi_size),
7889 Memory.PtrSize.fromSize(cmov_abi_size),
77867890 .{ .base = .rbp, .disp = -val_off },
77877891 ),
77887892 cc,
......@@ -7790,8 +7894,8 @@ fn atomicOp(
77907894 else => {
77917895 const val_reg = try self.copyToTmpRegister(val_ty, val_mcv);
77927896 try self.asmCmovccRegisterRegister(
7793 registerAlias(dst_reg, val_abi_size),
7794 registerAlias(val_reg, val_abi_size),
7897 registerAlias(tmp_reg, cmov_abi_size),
7898 registerAlias(val_reg, cmov_abi_size),
77957899 cc,
77967900 );
77977901 },
......@@ -7799,11 +7903,12 @@ fn atomicOp(
77997903 },
78007904 };
78017905 _ = try self.addInst(.{ .tag = .cmpxchg, .ops = .lock_mr_sib, .data = .{ .rx = .{
7802 .r = registerAlias(dst_reg, val_abi_size),
7906 .r = registerAlias(tmp_reg, val_abi_size),
78037907 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
78047908 } } });
7805 break :loop loop;
7806 } else loop: {
7909 _ = try self.asmJccReloc(loop, .ne);
7910 return if (unused) .none else .{ .register = .rax };
7911 } else {
78077912 try self.asmRegisterMemory(.mov, .rax, Memory.sib(.qword, .{
78087913 .base = ptr_mem.sib.base,
78097914 .scale_index = ptr_mem.sib.scale_index,
......@@ -7869,8 +7974,22 @@ fn atomicOp(
78697974 _ = try self.addInst(.{ .tag = .cmpxchgb, .ops = .lock_m_sib, .data = .{
78707975 .payload = try self.addExtra(Mir.MemorySib.encode(ptr_mem)),
78717976 } });
7872 break :loop loop;
7873 }, .ne),
7977 _ = try self.asmJccReloc(loop, .ne);
7978
7979 if (unused) return .none;
7980 const dst_mcv = try self.allocTempRegOrMem(val_ty, false);
7981 try self.asmMemoryRegister(
7982 .mov,
7983 Memory.sib(.qword, .{ .base = .rbp, .disp = 0 - dst_mcv.stack_offset }),
7984 .rax,
7985 );
7986 try self.asmMemoryRegister(
7987 .mov,
7988 Memory.sib(.qword, .{ .base = .rbp, .disp = 8 - dst_mcv.stack_offset }),
7989 .rdx,
7990 );
7991 return dst_mcv;
7992 },
78747993 .libcall => return self.fail("TODO implement x86 atomic libcall", .{}),
78757994 }
78767995}
......@@ -7884,7 +8003,6 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
78848003 defer for (regs_lock) |lock| self.register_manager.unlockReg(lock);
78858004
78868005 const unused = self.liveness.isUnused(inst);
7887 const dst_reg = try self.register_manager.allocReg(if (unused) null else inst, gp);
78888006
78898007 const ptr_ty = self.air.typeOf(pl_op.operand);
78908008 const ptr_mcv = try self.resolveInst(pl_op.operand);
......@@ -7892,8 +8010,8 @@ fn airAtomicRmw(self: *Self, inst: Air.Inst.Index) !void {
78928010 const val_ty = self.air.typeOf(extra.operand);
78938011 const val_mcv = try self.resolveInst(extra.operand);
78948012
7895 try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering());
7896 const result: MCValue = if (unused) .dead else .{ .register = dst_reg };
8013 const result =
8014 try self.atomicOp(ptr_mcv, val_mcv, ptr_ty, val_ty, unused, extra.op(), extra.ordering());
78978015 return self.finishAir(inst, result, .{ pl_op.operand, extra.operand, .none });
78988016}
78998017
......@@ -7926,16 +8044,14 @@ fn airAtomicLoad(self: *Self, inst: Air.Inst.Index) !void {
79268044fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOrder) !void {
79278045 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
79288046
7929 const dst_reg = try self.register_manager.allocReg(null, gp);
7930
79318047 const ptr_ty = self.air.typeOf(bin_op.lhs);
79328048 const ptr_mcv = try self.resolveInst(bin_op.lhs);
79338049
79348050 const val_ty = self.air.typeOf(bin_op.rhs);
79358051 const val_mcv = try self.resolveInst(bin_op.rhs);
79368052
7937 try self.atomicOp(dst_reg, ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order);
7938 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
8053 const result = try self.atomicOp(ptr_mcv, val_mcv, ptr_ty, val_ty, true, null, order);
8054 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
79398055}
79408056
79418057fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
......@@ -7979,7 +8095,6 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
79798095 };
79808096 defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock);
79818097
7982 const src_ty = self.air.typeOf(extra.lhs);
79838098 const src_ptr = try self.resolveInst(extra.lhs);
79848099 const src_ptr_lock: ?RegisterLock = switch (src_ptr) {
79858100 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
......@@ -7994,25 +8109,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
79948109 };
79958110 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
79968111
7997 // TODO Is this the only condition for pointer dereference for memcpy?
7998 const src: MCValue = blk: {
7999 switch (src_ptr) {
8000 .linker_load, .memory => {
8001 const reg = try self.register_manager.allocReg(null, gp);
8002 try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
8003 try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{ .base = reg }));
8004 break :blk MCValue{ .register = reg };
8005 },
8006 else => break :blk src_ptr,
8007 }
8008 };
8009 const src_lock: ?RegisterLock = switch (src) {
8010 .register => |reg| self.register_manager.lockReg(reg),
8011 else => null,
8012 };
8013 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
8014
8015 try self.genInlineMemcpy(dst_ptr, src, len, .{});
8112 try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{});
80168113
80178114 return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
80188115}
......@@ -8156,11 +8253,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
81568253 switch (result_ty.zigTypeTag()) {
81578254 .Struct => {
81588255 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
8159 const dst_mcv = MCValue{ .stack_offset = stack_offset };
81608256 if (result_ty.containerLayout() == .Packed) {
81618257 const struct_obj = result_ty.castTag(.@"struct").?.data;
81628258 try self.genInlineMemset(
8163 dst_mcv,
8259 .{ .ptr_stack_offset = stack_offset },
81648260 .{ .immediate = 0 },
81658261 .{ .immediate = abi_size },
81668262 .{},
......@@ -8236,7 +8332,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
82368332 const elem_mcv = try self.resolveInst(elem);
82378333 try self.genSetStack(elem_ty, stack_offset - elem_off, elem_mcv, .{});
82388334 }
8239 break :res dst_mcv;
8335 break :res .{ .stack_offset = stack_offset };
82408336 },
82418337 .Array => {
82428338 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
src/codegen.zig+9-5
......@@ -966,7 +966,7 @@ fn genDeclRef(
966966 const module = bin_file.options.module.?;
967967 const decl = module.declPtr(decl_index);
968968
969 if (decl.ty.zigTypeTag() != .Fn and !decl.ty.hasRuntimeBitsIgnoreComptime()) {
969 if (!decl.ty.isFnOrHasRuntimeBitsIgnoreComptime()) {
970970 const imm: u64 = switch (ptr_bytes) {
971971 1 => 0xaa,
972972 2 => 0xaaaa,
......@@ -978,10 +978,14 @@ fn genDeclRef(
978978 }
979979
980980 // TODO this feels clunky. Perhaps we should check for it in `genTypedValue`?
981 if (tv.ty.zigTypeTag() == .Pointer) blk: {
982 if (tv.ty.castPtrToFn()) |_| break :blk;
983 if (!tv.ty.elemType2().hasRuntimeBits()) {
984 return GenResult.mcv(.none);
981 if (tv.ty.castPtrToFn()) |fn_ty| {
982 if (fn_ty.fnInfo().is_generic) {
983 return GenResult.mcv(.{ .immediate = fn_ty.abiAlignment(target) });
984 }
985 } else if (tv.ty.zigTypeTag() == .Pointer) {
986 const elem_ty = tv.ty.elemType2();
987 if (!elem_ty.hasRuntimeBits()) {
988 return GenResult.mcv(.{ .immediate = elem_ty.abiAlignment(target) });
985989 }
986990 }
987991
test/behavior/array.zig-1
......@@ -190,7 +190,6 @@ test "nested arrays of strings" {
190190 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
191191 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
192192 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
193 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
194193
195194 const array_of_strings = [_][]const u8{ "hello", "this", "is", "my", "thing" };
196195 for (array_of_strings, 0..) |s, i| {
test/behavior/atomics.zig-1
......@@ -232,7 +232,6 @@ fn testAtomicRmwFloat() !void {
232232
233233test "atomicrmw with ints" {
234234 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
235 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
236235 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
237236 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
238237 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
test/behavior/bugs/718.zig-1
......@@ -12,7 +12,6 @@ var keys: Keys = undefined;
1212test "zero keys with @memset" {
1313 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1414 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
15 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1615 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1716
1817 @memset(@ptrCast([*]u8, &keys), 0, @sizeOf(@TypeOf(keys)));
test/behavior/bugs/7325.zig-1
......@@ -79,7 +79,6 @@ fn genExpression(expr: Expression) !ExpressionResult {
7979
8080test {
8181 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
82 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8382 if (builtin.zig_backend == .stage2_x86) return error.SkipZigTest; // TODO
8483 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
8584 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO