authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-03 12:12:36-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-11-04 09:15:05-04:00
log095c4294aa8b275da0627adefad046923fcaae46
tree069ca321e822ea95d7265ece0b79173832a0745a
parent6553359f398ccc3be10a4975d99cb325a5b7c1fd

x86_64: fix miscompilations

Closes #17618

2 files changed, 102 insertions(+), 29 deletions(-)

lib/std/multi_array_list.zig+1-1
......@@ -106,7 +106,7 @@ pub fn MultiArrayList(comptime T: type) type {
106106 }
107107
108108 pub fn toMultiArrayList(self: Slice) Self {
109 if (self.ptrs.len == 0) {
109 if (self.ptrs.len == 0 or self.capacity == 0) {
110110 return .{};
111111 }
112112 const unaligned_ptr = self.ptrs[sizes.fields[0]];
src/arch/x86_64/CodeGen.zig+101-28
......@@ -2533,6 +2533,19 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
25332533 ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation);
25342534 for (deaths) |death| try self.processDeath(death);
25352535
2536 const ExpectedContents = [@typeInfo(RegisterManager.TrackedRegisters).Array.len]RegisterLock;
2537 var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
2538 if (opts.update_tracking) ({}) else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
2539
2540 var reg_locks = if (opts.update_tracking) {} else try std.ArrayList(RegisterLock).initCapacity(
2541 stack.get(),
2542 @typeInfo(ExpectedContents).Array.len,
2543 );
2544 defer if (!opts.update_tracking) {
2545 for (reg_locks.items) |lock| self.register_manager.unlockReg(lock);
2546 reg_locks.deinit();
2547 };
2548
25362549 for (0..state.registers.len) |index| {
25372550 const current_maybe_inst = if (self.register_manager.free_registers.isSet(index))
25382551 null
......@@ -2549,11 +2562,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
25492562 try self.inst_tracking.getPtr(current_inst).?.spill(self, current_inst);
25502563 }
25512564 if (target_maybe_inst) |target_inst| {
2552 try self.inst_tracking.getPtr(target_inst).?.materialize(
2553 self,
2554 target_inst,
2555 state.reg_tracking[index],
2556 );
2565 const target_tracking = self.inst_tracking.getPtr(target_inst).?;
2566 try target_tracking.materialize(self, target_inst, state.reg_tracking[index]);
25572567 }
25582568 }
25592569 if (opts.update_tracking) {
......@@ -2571,7 +2581,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt
25712581 state.reg_tracking[index],
25722582 );
25732583 }
2574 }
2584 } else if (target_maybe_inst) |_|
2585 try reg_locks.append(self.register_manager.lockRegIndexAssumeUnused(@intCast(index)));
25752586 }
25762587 if (opts.emit_instructions) if (self.eflags_inst) |inst|
25772588 try self.inst_tracking.getPtr(inst).?.spill(self, inst);
......@@ -6746,6 +6757,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
67466757 .disp = val_byte_off,
67476758 } },
67486759 });
6760 try self.spillEflagsIfOccupied();
67496761 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off));
67506762 } else {
67516763 const tmp_reg =
......@@ -6768,6 +6780,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
67686780 .disp = val_byte_off + 1,
67696781 } },
67706782 });
6783 try self.spillEflagsIfOccupied();
67716784 try self.asmRegisterRegisterImmediate(
67726785 .{ ._rd, .sh },
67736786 dst_alias,
......@@ -6851,6 +6864,27 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
68516864 } else {
68526865 try self.load(dst_mcv, ptr_ty, ptr_mcv);
68536866 }
6867
6868 if (elem_ty.isAbiInt(mod) and elem_size * 8 > elem_ty.bitSize(mod)) {
6869 const high_mcv: MCValue = switch (dst_mcv) {
6870 .register => |dst_reg| .{ .register = dst_reg },
6871 .register_pair => |dst_regs| .{ .register = dst_regs[1] },
6872 else => dst_mcv.address().offset(@intCast((elem_size - 1) / 8 * 8)).deref(),
6873 };
6874 const high_reg = if (high_mcv.isRegister())
6875 high_mcv.getReg().?
6876 else
6877 try self.copyToTmpRegister(Type.usize, high_mcv);
6878 const high_lock = self.register_manager.lockReg(high_reg);
6879 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);
6880
6881 try self.truncateRegister(elem_ty, high_reg);
6882 if (!high_mcv.isRegister()) try self.genCopy(
6883 if (elem_size <= 8) elem_ty else Type.usize,
6884 high_mcv,
6885 .{ .register = high_reg },
6886 );
6887 }
68546888 break :result dst_mcv;
68556889 };
68566890 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
......@@ -6996,6 +7030,11 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
69967030 } else {
69977031 // TODO if the value is undef, don't lower this instruction
69987032 }
7033
7034 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
7035 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
7036 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
7037
69997038 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
70007039 const ptr_mcv = try self.resolveInst(bin_op.lhs);
70017040 const ptr_ty = self.typeOf(bin_op.lhs);
......@@ -7088,12 +7127,15 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
70887127 const dst_lock = self.register_manager.lockReg(dst_reg);
70897128 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
70907129
7091 if (field_off > 0) try self.genShiftBinOpMir(
7092 .{ ._r, .sh },
7093 Type.usize,
7094 dst_mcv,
7095 .{ .immediate = field_off },
7096 );
7130 if (field_off > 0) {
7131 try self.spillEflagsIfOccupied();
7132 try self.genShiftBinOpMir(
7133 .{ ._r, .sh },
7134 Type.usize,
7135 dst_mcv,
7136 .{ .immediate = field_off },
7137 );
7138 }
70977139 if (abi.RegisterClass.gp.isSet(RegisterManager.indexOfRegIntoTracked(dst_reg).?) and
70987140 container_ty.abiSize(mod) * 8 > field_ty.bitSize(mod))
70997141 try self.truncateRegister(field_ty, dst_reg);
......@@ -7128,12 +7170,15 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
71287170 defer for (dst_locks) |dst_lock| if (dst_lock) |lock|
71297171 self.register_manager.unlockReg(lock);
71307172
7131 if (field_off > 0) try self.genShiftBinOpMir(
7132 .{ ._r, .sh },
7133 Type.u128,
7134 dst_mcv,
7135 .{ .immediate = field_off },
7136 );
7173 if (field_off > 0) {
7174 try self.spillEflagsIfOccupied();
7175 try self.genShiftBinOpMir(
7176 .{ ._r, .sh },
7177 Type.u128,
7178 dst_mcv,
7179 .{ .immediate = field_off },
7180 );
7181 }
71377182
71387183 if (field_bit_size <= 64) {
71397184 if (self.regExtraBits(field_ty) > 0)
......@@ -7161,12 +7206,15 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
71617206 const dst_lock = self.register_manager.lockReg(dst_reg);
71627207 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
71637208
7164 if (field_off % 64 > 0) try self.genShiftBinOpMir(
7165 .{ ._r, .sh },
7166 Type.usize,
7167 dst_mcv,
7168 .{ .immediate = field_off % 64 },
7169 );
7209 if (field_off % 64 > 0) {
7210 try self.spillEflagsIfOccupied();
7211 try self.genShiftBinOpMir(
7212 .{ ._r, .sh },
7213 Type.usize,
7214 dst_mcv,
7215 .{ .immediate = field_off % 64 },
7216 );
7217 }
71707218 if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);
71717219
71727220 break :result if (field_rc.supersetOf(abi.RegisterClass.gp))
......@@ -7272,6 +7320,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
72727320 .disp = frame_addr.off + field_byte_off,
72737321 } },
72747322 });
7323 try self.spillEflagsIfOccupied();
72757324 try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off));
72767325 } else {
72777326 const tmp_reg = registerAlias(
......@@ -7300,6 +7349,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
73007349 .disp = frame_addr.off + field_byte_off + limb_abi_size,
73017350 } },
73027351 });
7352 try self.spillEflagsIfOccupied();
73037353 try self.asmRegisterRegisterImmediate(
73047354 .{ ._rd, .sh },
73057355 dst_alias,
......@@ -7344,13 +7394,18 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void {
73447394fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
73457395 const mod = self.bin_file.options.module.?;
73467396 const src_ty = self.typeOf(src_air);
7347 const src_mcv = try self.resolveInst(src_air);
73487397 if (src_ty.zigTypeTag(mod) == .Vector)
73497398 return self.fail("TODO implement genUnOp for {}", .{src_ty.fmt(mod)});
73507399
7400 var src_mcv = try self.resolveInst(src_air);
73517401 switch (src_mcv) {
73527402 .eflags => |cc| switch (tag) {
7353 .not => return .{ .eflags = cc.negate() },
7403 .not => {
7404 if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv))
7405 return .{ .eflags = cc.negate() };
7406 try self.spillEflagsIfOccupied();
7407 src_mcv = try self.resolveInst(src_air);
7408 },
73547409 else => {},
73557410 },
73567411 else => {},
......@@ -11873,8 +11928,18 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void {
1187311928 try self.spillEflagsIfOccupied();
1187411929 for (items, relocs, 0..) |item, *reloc, i| {
1187511930 const item_mcv = try self.resolveInst(item);
11876 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
11877 reloc.* = try self.asmJccReloc(if (i < relocs.len - 1) .e else .ne, undefined);
11931 const cc: Condition = switch (condition) {
11932 .eflags => |cc| switch (item_mcv.immediate) {
11933 0 => cc.negate(),
11934 1 => cc,
11935 else => unreachable,
11936 },
11937 else => cc: {
11938 try self.genBinOpMir(.{ ._, .cmp }, condition_ty, condition, item_mcv);
11939 break :cc .e;
11940 },
11941 };
11942 reloc.* = try self.asmJccReloc(if (i < relocs.len - 1) cc else cc.negate(), undefined);
1187811943 }
1187911944
1188011945 for (liveness.deaths[case_i]) |operand| try self.processDeath(operand);
......@@ -14380,6 +14445,10 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
1438014445
1438114446 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1438214447
14448 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
14449 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
14450 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
14451
1438314452 const dst_ptr = try self.resolveInst(bin_op.lhs);
1438414453 const dst_ptr_ty = self.typeOf(bin_op.lhs);
1438514454 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {
......@@ -14497,6 +14566,10 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
1449714566 const mod = self.bin_file.options.module.?;
1449814567 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1449914568
14569 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
14570 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx });
14571 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
14572
1450014573 const dst_ptr = try self.resolveInst(bin_op.lhs);
1450114574 const dst_ptr_ty = self.typeOf(bin_op.lhs);
1450214575 const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) {