authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-18 17:28:19-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-04-18 17:28:19-07:00
log30fc160874d743da74f7cf7c974c4c8cce03909e
tree0a7880e93d3636c3563b358c3bc9cf3372ac9932
parent2568da2f41d3403b2cd91bbb84862c86932b63e6
parent780f654e1907f8d6d194ca41e49fce09fd7066c5
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15302 from jacobly0/x86_64-non-liveness

x86_64: liveness independent fixes

6 files changed, 148 insertions(+), 27 deletions(-)

src/arch/x86_64/CodeGen.zig+147-13
......@@ -1690,7 +1690,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
16901690 .unsigned => .int_unsigned,
16911691 } }, .data = switch (tag) {
16921692 else => unreachable,
1693 .mul, .mulwrap => std.math.max3(
1693 .mul, .mulwrap => math.max3(
16941694 self.activeIntBits(bin_op.lhs),
16951695 self.activeIntBits(bin_op.rhs),
16961696 dst_info.bits / 2,
......@@ -1743,7 +1743,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
17431743 break :cc .o;
17441744 } else cc: {
17451745 try self.genSetReg(ty, limit_reg, .{
1746 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1746 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
17471747 });
17481748 break :cc .c;
17491749 };
......@@ -1850,7 +1850,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
18501850 break :cc .o;
18511851 } else cc: {
18521852 try self.genSetReg(ty, limit_reg, .{
1853 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1853 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
18541854 });
18551855 break :cc .c;
18561856 };
......@@ -2067,7 +2067,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
20672067 var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {
20682068 .signed => .int_signed,
20692069 .unsigned => .int_unsigned,
2070 } }, .data = std.math.max3(
2070 } }, .data = math.max3(
20712071 self.activeIntBits(bin_op.lhs),
20722072 self.activeIntBits(bin_op.rhs),
20732073 dst_info.bits / 2,
......@@ -2087,12 +2087,14 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
20872087 else => {},
20882088 }
20892089
2090 // For now, this is the only supported multiply that doesn't fit in a register.
2091 assert(dst_info.bits == 128 and src_pl.data == 64);
20902092 const dst_abi_size = @intCast(i32, dst_ty.abiSize(self.target.*));
20912093 const dst_mcv = try self.allocRegOrMem(inst, false);
20922094 try self.genSetStack(
20932095 Type.u1,
20942096 dst_mcv.stack_offset - dst_abi_size,
2095 .{ .eflags = cc },
2097 .{ .immediate = 0 }, // 64x64 -> 128 never overflows
20962098 .{},
20972099 );
20982100 try self.genSetStack(dst_ty, dst_mcv.stack_offset, partial_mcv, .{});
......@@ -3122,7 +3124,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
31223124 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
31233125 .immediate = src_bits ^ (src_bits - 1),
31243126 });
3125 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
3127 try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
31263128
31273129 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
31283130 try self.asmCmovccRegisterRegister(
......@@ -3136,7 +3138,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
31363138 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
31373139 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - self.regBitSize(dst_ty)),
31383140 });
3139 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
3141 try self.genBinOpMir(.bsr, src_ty, dst_mcv, mat_src_mcv);
31403142
31413143 const cmov_abi_size = @max(@intCast(u32, dst_ty.abiSize(self.target.*)), 2);
31423144 try self.asmCmovccRegisterRegister(
......@@ -3565,6 +3567,62 @@ fn reuseOperand(
35653567 return true;
35663568}
35673569
3570fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_mcv: MCValue, ptr_ty: Type) InnerError!void {
3571 const ptr_info = ptr_ty.ptrInfo().data;
3572
3573 const val_ty = ptr_info.pointee_type;
3574 const val_abi_size = @intCast(u32, val_ty.abiSize(self.target.*));
3575 const limb_abi_size = @min(val_abi_size, 8);
3576 const limb_abi_bits = limb_abi_size * 8;
3577 const val_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);
3578 const val_bit_off = ptr_info.bit_offset % limb_abi_bits;
3579 const val_extra_bits = self.regExtraBits(val_ty);
3580
3581 if (val_abi_size > 8) return self.fail("TODO implement packed load of {}", .{
3582 val_ty.fmt(self.bin_file.options.module.?),
3583 });
3584
3585 const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3586 const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);
3587 defer self.register_manager.unlockReg(ptr_lock);
3588
3589 const dst_reg = switch (dst_mcv) {
3590 .register => |reg| reg,
3591 else => try self.register_manager.allocReg(null, gp),
3592 };
3593 const dst_lock = self.register_manager.lockReg(dst_reg);
3594 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
3595
3596 const load_abi_size =
3597 if (val_bit_off < val_extra_bits) val_abi_size else val_abi_size * 2;
3598 if (load_abi_size <= 8) {
3599 const load_reg = registerAlias(dst_reg, load_abi_size);
3600 try self.asmRegisterMemory(.mov, load_reg, Memory.sib(
3601 Memory.PtrSize.fromSize(load_abi_size),
3602 .{ .base = ptr_reg, .disp = val_byte_off },
3603 ));
3604 try self.asmRegisterImmediate(.shr, load_reg, Immediate.u(val_bit_off));
3605 } else {
3606 const tmp_reg = registerAlias(try self.register_manager.allocReg(null, gp), val_abi_size);
3607 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3608 defer self.register_manager.unlockReg(tmp_lock);
3609
3610 const dst_alias = registerAlias(dst_reg, val_abi_size);
3611 try self.asmRegisterMemory(.mov, dst_alias, Memory.sib(
3612 Memory.PtrSize.fromSize(val_abi_size),
3613 .{ .base = ptr_reg, .disp = val_byte_off },
3614 ));
3615 try self.asmRegisterMemory(.mov, tmp_reg, Memory.sib(
3616 Memory.PtrSize.fromSize(val_abi_size),
3617 .{ .base = ptr_reg, .disp = val_byte_off + 1 },
3618 ));
3619 try self.asmRegisterRegisterImmediate(.shrd, dst_alias, tmp_reg, Immediate.u(val_bit_off));
3620 }
3621
3622 if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg);
3623 try self.setRegOrMem(val_ty, dst_mcv, .{ .register = dst_reg });
3624}
3625
35683626fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void {
35693627 const elem_ty = ptr_ty.elemType();
35703628 const abi_size = @intCast(u32, elem_ty.abiSize(self.target.*));
......@@ -3653,12 +3711,84 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void {
36533711 ptr
36543712 else
36553713 try self.allocRegOrMem(inst, true);
3656 try self.load(dst_mcv, ptr, self.air.typeOf(ty_op.operand));
3714
3715 const ptr_ty = self.air.typeOf(ty_op.operand);
3716 if (ptr_ty.ptrInfo().data.host_size > 0) {
3717 try self.packedLoad(dst_mcv, ptr, ptr_ty);
3718 } else {
3719 try self.load(dst_mcv, ptr, ptr_ty);
3720 }
36573721 break :result dst_mcv;
36583722 };
36593723 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
36603724}
36613725
3726fn packedStore(
3727 self: *Self,
3728 ptr_mcv: MCValue,
3729 val_mcv: MCValue,
3730 ptr_ty: Type,
3731 val_ty: Type,
3732) InnerError!void {
3733 const ptr_info = ptr_ty.ptrInfo().data;
3734
3735 const limb_abi_size = @min(ptr_info.host_size, 8);
3736 const limb_abi_bits = limb_abi_size * 8;
3737
3738 const val_bit_size = val_ty.bitSize(self.target.*);
3739 const val_byte_off = @intCast(i32, ptr_info.bit_offset / limb_abi_bits * limb_abi_size);
3740 const val_bit_off = ptr_info.bit_offset % limb_abi_bits;
3741
3742 const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv);
3743 const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg);
3744 defer self.register_manager.unlockReg(ptr_lock);
3745
3746 var limb_i: u16 = 0;
3747 while (limb_i * limb_abi_bits < val_bit_off + val_bit_size) : (limb_i += 1) {
3748 const part_bit_off = if (limb_i == 0) val_bit_off else 0;
3749 const part_bit_size =
3750 @min(val_bit_off + val_bit_size - limb_i * limb_abi_bits, limb_abi_bits) - part_bit_off;
3751 const limb_mem = Memory.sib(
3752 Memory.PtrSize.fromSize(limb_abi_size),
3753 .{ .base = ptr_reg, .disp = val_byte_off + limb_i * limb_abi_bits },
3754 );
3755
3756 const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - part_bit_size)) <<
3757 @intCast(u6, part_bit_off);
3758 const part_mask_not = part_mask ^
3759 (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - limb_abi_bits));
3760 if (limb_abi_size <= 4) {
3761 try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.u(part_mask_not));
3762 } else if (math.cast(i32, @bitCast(i64, part_mask_not))) |small| {
3763 try self.asmMemoryImmediate(.@"and", limb_mem, Immediate.s(small));
3764 } else {
3765 const part_mask_reg = try self.register_manager.allocReg(null, gp);
3766 try self.asmRegisterImmediate(.mov, part_mask_reg, Immediate.u(part_mask_not));
3767 try self.asmMemoryRegister(.@"and", limb_mem, part_mask_reg);
3768 }
3769
3770 if (val_bit_size <= 64) {
3771 const tmp_reg = try self.register_manager.allocReg(null, gp);
3772 const tmp_mcv = MCValue{ .register = tmp_reg };
3773 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3774 defer self.register_manager.unlockReg(tmp_lock);
3775
3776 try self.genSetReg(val_ty, tmp_reg, val_mcv);
3777 switch (limb_i) {
3778 0 => try self.genShiftBinOpMir(.shl, val_ty, tmp_mcv, .{ .immediate = val_bit_off }),
3779 1 => try self.genShiftBinOpMir(.shr, val_ty, tmp_mcv, .{
3780 .immediate = limb_abi_bits - val_bit_off,
3781 }),
3782 else => unreachable,
3783 }
3784 try self.genBinOpMir(.@"and", val_ty, tmp_mcv, .{ .immediate = part_mask });
3785 try self.asmMemoryRegister(.@"or", limb_mem, registerAlias(tmp_reg, limb_abi_size));
3786 } else return self.fail("TODO: implement packed store of {}", .{
3787 val_ty.fmt(self.bin_file.options.module.?),
3788 });
3789 }
3790}
3791
36623792fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void {
36633793 const abi_size = @intCast(u32, value_ty.abiSize(self.target.*));
36643794 switch (ptr) {
......@@ -3850,7 +3980,11 @@ fn airStore(self: *Self, inst: Air.Inst.Index) !void {
38503980 const value = try self.resolveInst(bin_op.rhs);
38513981 const value_ty = self.air.typeOf(bin_op.rhs);
38523982 log.debug("airStore(%{d}): {} <- {}", .{ inst, ptr, value });
3853 try self.store(ptr, value, ptr_ty, value_ty);
3983 if (ptr_ty.ptrInfo().data.host_size > 0) {
3984 try self.packedStore(ptr, value, ptr_ty, value_ty);
3985 } else {
3986 try self.store(ptr, value, ptr_ty, value_ty);
3987 }
38543988 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
38553989}
38563990
......@@ -5214,7 +5348,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M
52145348 registerAlias(src_reg, abi_size),
52155349 ),
52165350 .immediate => |imm| {
5217 if (std.math.cast(i32, imm)) |small| {
5351 if (math.cast(i32, imm)) |small| {
52185352 try self.asmRegisterRegisterImmediate(
52195353 .imul,
52205354 dst_alias,
......@@ -6820,7 +6954,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
68206954 } else if (mem.startsWith(u8, op_str, "$")) {
68216955 if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| {
68226956 if (mnem_size) |size| {
6823 const max = @as(u64, std.math.maxInt(u64)) >>
6957 const max = @as(u64, math.maxInt(u64)) >>
68246958 @intCast(u6, 64 - (size.bitSize() - 1));
68256959 if ((if (s < 0) ~s else s) > max)
68266960 return self.fail("Invalid immediate size: '{s}'", .{op_str});
......@@ -6828,7 +6962,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void {
68286962 op.* = .{ .imm = Immediate.s(s) };
68296963 } else |_| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| {
68306964 if (mnem_size) |size| {
6831 const max = @as(u64, std.math.maxInt(u64)) >>
6965 const max = @as(u64, math.maxInt(u64)) >>
68326966 @intCast(u6, 64 - size.bitSize());
68336967 if (u > max)
68346968 return self.fail("Invalid immediate size: '{s}'", .{op_str});
......@@ -7167,7 +7301,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
71677301 else => {
71687302 // 64 bit write to memory would take two mov's anyways so we
71697303 // insted just use two 32 bit writes to avoid register allocation
7170 if (std.math.cast(i32, @bitCast(i64, imm))) |small| {
7304 if (math.cast(i32, @bitCast(i64, imm))) |small| {
71717305 try self.asmMemoryImmediate(.mov, Memory.sib(
71727306 Memory.PtrSize.fromSize(abi_size),
71737307 .{ .base = base_reg, .disp = -stack_offset },
test/behavior/bugs/12776.zig+1
......@@ -32,6 +32,7 @@ test {
3232 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
3333 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
3434 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
35 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
3536
3637 var ram = try RAM.new();
3738 var cpu = try CPU.new(&ram);
test/behavior/bugs/1851.zig-1
......@@ -4,7 +4,6 @@ const expect = std.testing.expect;
44
55test "allocation and looping over 3-byte integer" {
66 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
7 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
87 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
98 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
109 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/enum.zig-1
......@@ -1071,7 +1071,6 @@ const bit_field_1 = BitFieldOfEnums{
10711071
10721072test "bit field access with enum fields" {
10731073 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1074 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
10751074 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
10761075 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
10771076 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/struct.zig-4
......@@ -456,7 +456,6 @@ test "packed struct 24bits" {
456456test "runtime struct initialization of bitfield" {
457457 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
458458 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
459 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
460459 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
461460
462461 const s1 = Nibbles{
......@@ -577,7 +576,6 @@ const bit_field_1 = BitField1{
577576test "bit field access" {
578577 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
579578 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
580 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
581579 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
582580
583581 var data = bit_field_1;
......@@ -696,7 +694,6 @@ const FooArrayOfAligned = packed struct {
696694};
697695
698696test "pointer to packed struct member in a stack variable" {
699 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
700697 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
701698 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
702699 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
......@@ -1259,7 +1256,6 @@ test "packed struct aggregate init" {
12591256}
12601257
12611258test "packed struct field access via pointer" {
1262 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
12631259 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
12641260 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
12651261 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/tests.zig-8
......@@ -943,14 +943,6 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
943943 if (test_target.use_llvm == false and mem.eql(u8, options.name, "compiler-rt"))
944944 continue;
945945
946 // TODO get the x86_64 self-hosted backend tests passing on Windows
947 if (test_target.target.getCpuArch() == .x86_64 and
948 test_target.target.getOsTag() == .windows and
949 test_target.use_llvm == false)
950 {
951 continue;
952 }
953
954946 // TODO get compiler-rt tests passing for wasm32-wasi
955947 // currently causes "LLVM ERROR: Unable to expand fixed point multiplication."
956948 if (test_target.target.getCpuArch() == .wasm32 and