authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-25 23:06:54-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-27 04:31:39-04:00
log86625c5a752796b64966541187a5843e11add615
tree8e2fb0dd71dd6cd945deeff6eac26e239c036e15
parent1de64dba236d9496ae5f31b59b7a4947ff4d90be

x86_64: enable mem dst bin ops, and fix uncovered bugs

1 files changed, 119 insertions(+), 101 deletions(-)

src/arch/x86_64/CodeGen.zig+119-101
......@@ -1628,7 +1628,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
16281628 const reg_bits = self.regBitSize(ty);
16291629 const cc: Condition = if (ty.isSignedInt()) cc: {
16301630 try self.genSetReg(ty, limit_reg, dst_mcv);
1631 try self.genBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1631 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
16321632 try self.genBinOpMir(.xor, ty, limit_mcv, .{
16331633 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
16341634 });
......@@ -1681,7 +1681,7 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
16811681 const reg_bits = self.regBitSize(ty);
16821682 const cc: Condition = if (ty.isSignedInt()) cc: {
16831683 try self.genSetReg(ty, limit_reg, dst_mcv);
1684 try self.genBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1684 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
16851685 try self.genBinOpMir(.xor, ty, limit_mcv, .{
16861686 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
16871687 });
......@@ -1735,7 +1735,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
17351735 const cc: Condition = if (ty.isSignedInt()) cc: {
17361736 try self.genSetReg(ty, limit_reg, lhs_mcv);
17371737 try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);
1738 try self.genBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1738 try self.genShiftBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
17391739 try self.genBinOpMir(.xor, ty, limit_mcv, .{
17401740 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
17411741 });
......@@ -2509,16 +2509,13 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void {
25092509
25102510fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void {
25112511 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2512 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
2513 const operand = try self.resolveInst(ty_op.operand);
2514 const dst_mcv: MCValue = blk: {
2515 switch (operand) {
2516 .stack_offset => |off| {
2517 break :blk MCValue{ .stack_offset = off };
2518 },
2519 else => return self.fail("TODO implement slice_ptr for {}", .{operand}),
2520 }
2521 };
2512 const result = if (self.liveness.isUnused(inst)) .dead else result: {
2513 const src_mcv = try self.resolveInst(ty_op.operand);
2514 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
2515
2516 const dst_mcv = try self.allocRegOrMem(inst, true);
2517 const dst_ty = self.air.typeOfIndex(inst);
2518 try self.setRegOrMem(dst_ty, dst_mcv, src_mcv);
25222519 break :result dst_mcv;
25232520 };
25242521 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
......@@ -4402,7 +4399,7 @@ fn genBinOp(
44024399 else => {},
44034400 }
44044401
4405 const is_commutative: bool = switch (tag) {
4402 const is_commutative = switch (tag) {
44064403 .add,
44074404 .addwrap,
44084405 .bool_or,
......@@ -4416,6 +4413,20 @@ fn genBinOp(
44164413
44174414 else => false,
44184415 };
4416 const needs_reg_dst = switch (tag) {
4417 .add,
4418 .addwrap,
4419 .sub,
4420 .subwrap,
4421 .mul,
4422 .div_float,
4423 .div_exact,
4424 .div_trunc,
4425 .div_floor,
4426 => lhs_ty.isRuntimeFloat(),
4427
4428 else => false,
4429 };
44194430
44204431 const lhs_lock: ?RegisterLock = switch (lhs) {
44214432 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
......@@ -4432,10 +4443,10 @@ fn genBinOp(
44324443 var flipped: bool = false;
44334444 const dst_mcv: MCValue = blk: {
44344445 if (maybe_inst) |inst| {
4435 if (lhs.isRegister() and self.reuseOperand(inst, lhs_air, 0, lhs)) {
4446 if ((!needs_reg_dst or lhs.isRegister()) and self.reuseOperand(inst, lhs_air, 0, lhs)) {
44364447 break :blk lhs;
44374448 }
4438 if (is_commutative and rhs.isRegister() and self.reuseOperand(inst, rhs_air, 1, rhs)) {
4449 if (is_commutative and (!needs_reg_dst or rhs.isRegister()) and self.reuseOperand(inst, rhs_air, 1, rhs)) {
44394450 flipped = true;
44404451 break :blk rhs;
44414452 }
......@@ -4485,33 +4496,37 @@ fn genBinOp(
44854496
44864497 .div_float,
44874498 .div_exact,
4488 => try self.genBinOpMir(switch (lhs_ty.tag()) {
4489 .f32 => .divss,
4490 .f64 => .divsd,
4491 else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
4492 }, lhs_ty, dst_mcv, src_mcv),
4493
44944499 .div_trunc,
44954500 .div_floor,
44964501 => {
44974502 try self.genBinOpMir(switch (lhs_ty.tag()) {
44984503 .f32 => .divss,
44994504 .f64 => .divsd,
4500 else => return self.fail("TODO implement genBinOp for {s} {}", .{ @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?) }),
4505 else => return self.fail("TODO implement genBinOp for {s} {}", .{
4506 @tagName(tag), lhs_ty.fmt(self.bin_file.options.module.?),
4507 }),
45014508 }, lhs_ty, dst_mcv, src_mcv);
4502 if (Target.x86.featureSetHas(self.target.cpu.features, .sse4_1)) {
4503 const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
4504 const dst_alias = registerAlias(dst_mcv.register, abi_size);
4505 try self.asmRegisterRegisterImmediate(switch (lhs_ty.tag()) {
4506 .f32 => .roundss,
4507 .f64 => .roundsd,
4508 else => unreachable,
4509 }, dst_alias, dst_alias, Immediate.u(switch (tag) {
4510 .div_trunc => 0b1_0_11,
4511 .div_floor => 0b1_0_01,
4512 else => unreachable,
4513 }));
4514 } else return self.fail("TODO implement round without sse4_1", .{});
4509 switch (tag) {
4510 .div_float,
4511 .div_exact,
4512 => {},
4513 .div_trunc,
4514 .div_floor,
4515 => if (Target.x86.featureSetHas(self.target.cpu.features, .sse4_1)) {
4516 const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
4517 const dst_alias = registerAlias(dst_mcv.register, abi_size);
4518 try self.asmRegisterRegisterImmediate(switch (lhs_ty.tag()) {
4519 .f32 => .roundss,
4520 .f64 => .roundsd,
4521 else => unreachable,
4522 }, dst_alias, dst_alias, Immediate.u(switch (tag) {
4523 .div_trunc => 0b1_0_11,
4524 .div_floor => 0b1_0_01,
4525 else => unreachable,
4526 }));
4527 } else return self.fail("TODO implement round without sse4_1", .{}),
4528 else => unreachable,
4529 }
45154530 },
45164531
45174532 .ptr_add,
......@@ -4568,7 +4583,13 @@ fn genBinOp(
45684583 };
45694584
45704585 const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
4571 switch (dst_mcv) {
4586 const tmp_reg = switch (dst_mcv) {
4587 .register => |reg| reg,
4588 else => try self.copyToTmpRegister(lhs_ty, dst_mcv),
4589 };
4590 const tmp_lock = self.register_manager.lockReg(tmp_reg);
4591 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
4592 switch (mat_src_mcv) {
45724593 .none,
45734594 .undef,
45744595 .dead,
......@@ -4576,57 +4597,44 @@ fn genBinOp(
45764597 .immediate,
45774598 .eflags,
45784599 .register_overflow,
4579 .stack_offset,
45804600 .ptr_stack_offset,
4581 .memory,
4582 .linker_load,
45834601 => unreachable,
4584 .register => |dst_reg| switch (mat_src_mcv) {
4585 .none,
4586 .undef,
4587 .dead,
4588 .unreach,
4589 .immediate,
4590 .eflags,
4591 .register_overflow,
4592 .ptr_stack_offset,
4593 => unreachable,
4594 .register => |src_reg| try self.asmCmovccRegisterRegister(
4595 registerAlias(dst_reg, abi_size),
4596 registerAlias(src_reg, abi_size),
4597 cc,
4598 ),
4599 .stack_offset => |off| try self.asmCmovccRegisterMemory(
4600 registerAlias(dst_reg, abi_size),
4601 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
4602 .base = .rbp,
4603 .disp = -off,
4604 }),
4605 cc,
4606 ),
4607 .memory, .linker_load => {
4608 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
4609 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4610 defer self.register_manager.unlockReg(addr_reg_lock);
4602 .register => |src_reg| try self.asmCmovccRegisterRegister(
4603 registerAlias(tmp_reg, abi_size),
4604 registerAlias(src_reg, abi_size),
4605 cc,
4606 ),
4607 .stack_offset => |off| try self.asmCmovccRegisterMemory(
4608 registerAlias(tmp_reg, abi_size),
4609 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
4610 .base = .rbp,
4611 .disp = -off,
4612 }),
4613 cc,
4614 ),
4615 .memory, .linker_load => {
4616 const addr_reg = (try self.register_manager.allocReg(null, gp)).to64();
4617 const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
4618 defer self.register_manager.unlockReg(addr_reg_lock);
46114619
4612 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_mcv);
4620 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mat_src_mcv);
46134621
4614 // To get the actual address of the value we want to modify we
4615 // we have to go through the GOT
4616 try self.asmRegisterMemory(
4617 .mov,
4618 addr_reg,
4619 Memory.sib(.qword, .{ .base = addr_reg }),
4620 );
4622 // To get the actual address of the value we want to modify we
4623 // we have to go through the GOT
4624 try self.asmRegisterMemory(
4625 .mov,
4626 addr_reg,
4627 Memory.sib(.qword, .{ .base = addr_reg }),
4628 );
46214629
4622 try self.asmCmovccRegisterMemory(
4623 registerAlias(dst_reg, abi_size),
4624 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
4625 cc,
4626 );
4627 },
4630 try self.asmCmovccRegisterMemory(
4631 registerAlias(tmp_reg, abi_size),
4632 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }),
4633 cc,
4634 );
46284635 },
46294636 }
4637 try self.setRegOrMem(lhs_ty, dst_mcv, .{ .register = tmp_reg });
46304638 },
46314639 .Float => try self.genBinOpMir(switch (lhs_ty.tag()) {
46324640 .f32 => switch (tag) {
......@@ -4649,8 +4657,8 @@ fn genBinOp(
46494657 return dst_mcv;
46504658}
46514659
4652fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
4653 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
4660fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void {
4661 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
46544662 switch (dst_mcv) {
46554663 .none => unreachable,
46564664 .undef => unreachable,
......@@ -4667,12 +4675,12 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
46674675 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
46684676 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
46694677
4670 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
4671 return self.genBinOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
4678 const reg = try self.copyToTmpRegister(ty, src_mcv);
4679 return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });
46724680 },
4673 .register => |src_reg| switch (dst_ty.zigTypeTag()) {
4681 .register => |src_reg| switch (ty.zigTypeTag()) {
46744682 .Float => {
4675 if (intrinsicsAllowed(self.target.*, dst_ty)) {
4683 if (intrinsicsAllowed(self.target.*, ty)) {
46764684 return self.asmRegisterRegister(mir_tag, dst_reg.to128(), src_reg.to128());
46774685 }
46784686
......@@ -4685,7 +4693,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
46854693 ),
46864694 },
46874695 .immediate => |imm| {
4688 switch (self.regBitSize(dst_ty)) {
4696 switch (self.regBitSize(ty)) {
46894697 8, 16, 32 => {
46904698 try self.asmRegisterImmediate(
46914699 mir_tag,
......@@ -4704,7 +4712,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
47044712 try self.asmRegisterRegister(
47054713 mir_tag,
47064714 registerAlias(dst_reg, abi_size),
4707 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
4715 registerAlias(try self.copyToTmpRegister(ty, src_mcv), abi_size),
47084716 );
47094717 }
47104718 },
......@@ -4719,8 +4727,8 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
47194727 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
47204728 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
47214729
4722 const reg = try self.copyToTmpRegister(dst_ty, src_mcv);
4723 return self.genBinOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg });
4730 const reg = try self.copyToTmpRegister(ty, src_mcv);
4731 return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = reg });
47244732 },
47254733 .stack_offset => |off| {
47264734 if (off > math.maxInt(i32)) {
......@@ -4754,7 +4762,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
47544762 }), registerAlias(src_reg, abi_size));
47554763 },
47564764 .immediate => |imm| {
4757 switch (self.regBitSize(dst_ty)) {
4765 switch (self.regBitSize(ty)) {
47584766 8, 16, 32 => {
47594767 try self.asmMemoryImmediate(
47604768 mir_tag,
......@@ -4785,7 +4793,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
47854793 .base = .rbp,
47864794 .disp = -off,
47874795 }),
4788 registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size),
4796 registerAlias(try self.copyToTmpRegister(ty, src_mcv), abi_size),
47894797 );
47904798 }
47914799 },
......@@ -4793,16 +4801,18 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
47934801 }
47944802 },
47954803 .memory,
4804 .linker_load,
47964805 .stack_offset,
47974806 .ptr_stack_offset,
4807 .eflags,
47984808 => {
4799 return self.fail("TODO implement x86 genBinOpMir source memory", .{});
4800 },
4801 .linker_load => {
4802 return self.fail("TODO implement x86 genBinOpMir source symbol at index in linker", .{});
4803 },
4804 .eflags => {
4805 return self.fail("TODO implement x86 genBinOpMir source eflags", .{});
4809 assert(abi_size <= 8);
4810
4811 const tmp_reg = try self.copyToTmpRegister(ty, src_mcv);
4812 const tmp_lock = self.register_manager.lockReg(tmp_reg);
4813 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
4814
4815 return self.genBinOpMir(mir_tag, ty, dst_mcv, .{ .register = tmp_reg });
48064816 },
48074817 }
48084818 },
......@@ -7151,7 +7161,15 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
71517161
71527162fn airPtrToInt(self: *Self, inst: Air.Inst.Index) !void {
71537163 const un_op = self.air.instructions.items(.data)[inst].un_op;
7154 const result = try self.resolveInst(un_op);
7164 const result = if (self.liveness.isUnused(inst)) .dead else result: {
7165 const src_mcv = try self.resolveInst(un_op);
7166 if (self.reuseOperand(inst, un_op, 0, src_mcv)) break :result src_mcv;
7167
7168 const dst_mcv = try self.allocRegOrMem(inst, true);
7169 const dst_ty = self.air.typeOfIndex(inst);
7170 try self.setRegOrMem(dst_ty, dst_mcv, src_mcv);
7171 break :result dst_mcv;
7172 };
71557173 return self.finishAir(inst, result, .{ un_op, .none, .none });
71567174}
71577175