authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-05 02:18:18-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-13 04:17:47-04:00
log0165187cd07b26f15c2ca7e021747d0989d1956b
treec64cef85e67a46fd5adbc5fb8b5f3734c694a8cc
parente2fe1907ecac075e4d4a37776359144318b6055a

x86_64: fix some of the mass confusion about the meaning of `MCValue`

4 files changed, 126 insertions(+), 79 deletions(-)

src/arch/x86_64/CodeGen.zig+126-76
......@@ -3569,7 +3569,12 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
35693569 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
35703570 }
35713571
3572 try self.genInlineMemcpy(dst_mcv, ptr, .{ .immediate = abi_size }, .{});
3572 try self.genInlineMemcpy(
3573 .{ .ptr_stack_offset = off },
3574 ptr,
3575 .{ .immediate = abi_size },
3576 .{},
3577 );
35733578 },
35743579 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
35753580 }
......@@ -3745,22 +3750,47 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
37453750 -@intCast(i32, overflow_bit_offset),
37463751 );
37473752 },
3748 .linker_load, .memory, .stack_offset => if (abi_size <= 8) {
3753 .memory, .linker_load => if (abi_size <= 8) {
3754 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3755 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3756 defer self.register_manager.unlockReg(tmp_lock);
3757
3758 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
3759 } else {
3760 const addr_reg = try self.register_manager.allocReg(null, gp);
3761 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
3762 defer self.register_manager.unlockReg(addr_lock);
3763
3764 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, value);
3765 try self.genInlineMemcpy(
3766 ptr,
3767 .{ .register = addr_reg },
3768 .{ .immediate = abi_size },
3769 .{},
3770 );
3771 },
3772 .stack_offset => |off| if (abi_size <= 8) {
37493773 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3774 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3775 defer self.register_manager.unlockReg(tmp_lock);
3776
37503777 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
37513778 } else try self.genInlineMemcpy(
3752 .{ .stack_offset = 0 },
3753 value,
3779 ptr,
3780 .{ .ptr_stack_offset = off },
37543781 .{ .immediate = abi_size },
3755 .{ .source_stack_base = .rbp, .dest_stack_base = reg.to64() },
3782 .{},
37563783 ),
37573784 .ptr_stack_offset => {
37583785 const tmp_reg = try self.copyToTmpRegister(value_ty, value);
3786 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3787 defer self.register_manager.unlockReg(tmp_lock);
3788
37593789 try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
37603790 },
37613791 }
37623792 },
3763 .linker_load, .memory => {
3793 .memory, .linker_load => {
37643794 const value_lock: ?RegisterLock = switch (value) {
37653795 .register => |reg| self.register_manager.lockReg(reg),
37663796 else => null,
......@@ -5530,14 +5560,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void {
55305560 assert(ret_ty.isError());
55315561 },
55325562 .stack_offset => {
5533 const reg = try self.copyToTmpRegister(Type.usize, self.ret_mcv);
5534 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
5535 defer self.register_manager.unlockReg(reg_lock);
5536
5537 try self.genSetStack(ret_ty, 0, operand, .{
5538 .source_stack_base = .rbp,
5539 .dest_stack_base = reg,
5540 });
5563 try self.store(self.ret_mcv, operand, Type.usize, ret_ty);
55415564 },
55425565 else => {
55435566 try self.setRegOrMem(ret_ty, self.ret_mcv, operand);
......@@ -5556,6 +5579,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
55565579 const ptr = try self.resolveInst(un_op);
55575580 const ptr_ty = self.air.typeOf(un_op);
55585581 const elem_ty = ptr_ty.elemType();
5582 const abi_size = elem_ty.abiSize(self.target.*);
55595583 switch (self.ret_mcv) {
55605584 .immediate => {
55615585 assert(elem_ty.isError());
......@@ -5565,10 +5589,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
55655589 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);
55665590 defer self.register_manager.unlockReg(reg_lock);
55675591
5568 try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{
5569 .source_stack_base = .rbp,
5570 .dest_stack_base = reg,
5571 });
5592 try self.genInlineMemcpy(.{ .register = reg }, ptr, .{ .immediate = abi_size }, .{});
55725593 },
55735594 else => {
55745595 try self.load(self.ret_mcv, ptr, ptr_ty);
......@@ -6838,7 +6859,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
68386859 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
68396860 }
68406861 try self.genInlineMemset(
6841 .{ .stack_offset = stack_offset },
6862 .{ .ptr_stack_offset = stack_offset },
68426863 .{ .immediate = 0xaa },
68436864 .{ .immediate = abi_size },
68446865 .{ .dest_stack_base = .rsp },
......@@ -6877,10 +6898,17 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
68776898 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
68786899 }
68796900
6880 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{
6881 .source_stack_base = .rbp,
6882 .dest_stack_base = .rsp,
6883 });
6901 const addr_reg = try self.register_manager.allocReg(null, gp);
6902 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
6903 defer self.register_manager.unlockReg(addr_lock);
6904
6905 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mcv);
6906 try self.genInlineMemcpy(
6907 .{ .ptr_stack_offset = stack_offset },
6908 .{ .register = addr_reg },
6909 .{ .immediate = abi_size },
6910 .{ .dest_stack_base = .rsp },
6911 );
68846912 },
68856913 .register => |reg| {
68866914 switch (ty.zigTypeTag()) {
......@@ -6920,16 +6948,18 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
69206948 const reg = try self.copyToTmpRegister(ty, mcv);
69216949 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
69226950 },
6923 .stack_offset => {
6951 .stack_offset => |mcv_off| {
69246952 if (abi_size <= 8) {
69256953 const reg = try self.copyToTmpRegister(ty, mcv);
69266954 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
69276955 }
69286956
6929 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{
6930 .source_stack_base = .rbp,
6931 .dest_stack_base = .rsp,
6932 });
6957 try self.genInlineMemcpy(
6958 .{ .ptr_stack_offset = stack_offset },
6959 .{ .ptr_stack_offset = mcv_off },
6960 .{ .immediate = abi_size },
6961 .{ .dest_stack_base = .rsp },
6962 );
69336963 },
69346964 }
69356965}
......@@ -6963,7 +6993,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
69636993 opts,
69646994 ),
69656995 else => |x| return self.genInlineMemset(
6966 .{ .stack_offset = stack_offset },
6996 .{ .ptr_stack_offset = stack_offset },
69676997 .{ .immediate = 0xaa },
69686998 .{ .immediate = x },
69696999 opts,
......@@ -7072,22 +7102,40 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
70727102 },
70737103 }
70747104 },
7075 .memory, .linker_load, .stack_offset, .ptr_stack_offset => {
7076 switch (mcv) {
7077 else => unreachable,
7078 .memory, .linker_load, .ptr_stack_offset => {},
7079 .stack_offset => |src_off| if (stack_offset == src_off) {
7080 // Copy stack variable to itself; nothing to do.
7081 return;
7082 },
7083 }
7105 .memory, .linker_load => if (abi_size <= 8) {
7106 const reg = try self.copyToTmpRegister(ty, mcv);
7107 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
7108 } else {
7109 const addr_reg = try self.register_manager.allocReg(null, gp);
7110 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
7111 defer self.register_manager.unlockReg(addr_lock);
70847112
7085 if (abi_size <= 8) {
7086 const reg = try self.copyToTmpRegister(ty, mcv);
7087 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
7088 }
7113 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mcv);
7114 try self.genInlineMemcpy(
7115 .{ .ptr_stack_offset = stack_offset },
7116 .{ .register = addr_reg },
7117 .{ .immediate = abi_size },
7118 .{},
7119 );
7120 },
7121 .stack_offset => |off| if (abi_size <= 8) {
7122 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7123 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7124 defer self.register_manager.unlockReg(tmp_lock);
70897125
7090 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts);
7126 try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
7127 } else try self.genInlineMemcpy(
7128 .{ .ptr_stack_offset = stack_offset },
7129 .{ .ptr_stack_offset = off },
7130 .{ .immediate = abi_size },
7131 .{},
7132 ),
7133 .ptr_stack_offset => {
7134 const tmp_reg = try self.copyToTmpRegister(ty, mcv);
7135 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
7136 defer self.register_manager.unlockReg(tmp_lock);
7137
7138 try self.genSetStack(ty, stack_offset, .{ .register = tmp_reg }, opts);
70917139 },
70927140 }
70937141}
......@@ -7134,10 +7182,14 @@ fn genInlineMemcpyRegisterRegister(
71347182 next_offset -= nearest_power_of_two;
71357183 }
71367184 } else {
7137 try self.asmMemoryRegister(.mov, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{
7138 .base = dst_reg,
7139 .disp = -offset,
7140 }), registerAlias(src_reg, abi_size));
7185 try self.asmMemoryRegister(
7186 switch (src_reg.class()) {
7187 .general_purpose, .segment => .mov,
7188 .floating_point => .movss,
7189 },
7190 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
7191 registerAlias(src_reg, abi_size),
7192 );
71417193 }
71427194}
71437195
......@@ -7170,9 +7222,15 @@ fn genInlineMemcpy(
71707222 switch (dst_ptr) {
71717223 .memory, .linker_load => {
71727224 try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
7225 // Load the pointer, which is stored in memory
7226 try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
71737227 },
7174 .ptr_stack_offset, .stack_offset => |off| {
7175 try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{
7228 .stack_offset, .ptr_stack_offset => |off| {
7229 try self.asmRegisterMemory(switch (dst_ptr) {
7230 .stack_offset => .mov,
7231 .ptr_stack_offset => .lea,
7232 else => unreachable,
7233 }, .rdi, Memory.sib(.qword, .{
71767234 .base = opts.dest_stack_base orelse .rbp,
71777235 .disp = -off,
71787236 }));
......@@ -7192,9 +7250,15 @@ fn genInlineMemcpy(
71927250 switch (src_ptr) {
71937251 .memory, .linker_load => {
71947252 try self.loadMemPtrIntoRegister(.rsi, Type.usize, src_ptr);
7253 // Load the pointer, which is stored in memory
7254 try self.asmRegisterMemory(.mov, .rsi, Memory.sib(.qword, .{ .base = .rsi }));
71957255 },
7196 .ptr_stack_offset, .stack_offset => |off| {
7197 try self.asmRegisterMemory(.lea, .rsi, Memory.sib(.qword, .{
7256 .stack_offset, .ptr_stack_offset => |off| {
7257 try self.asmRegisterMemory(switch (src_ptr) {
7258 .stack_offset => .mov,
7259 .ptr_stack_offset => .lea,
7260 else => unreachable,
7261 }, .rsi, Memory.sib(.qword, .{
71987262 .base = opts.source_stack_base orelse .rbp,
71997263 .disp = -off,
72007264 }));
......@@ -7237,9 +7301,15 @@ fn genInlineMemset(
72377301 switch (dst_ptr) {
72387302 .memory, .linker_load => {
72397303 try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr);
7304 // Load the pointer, which is stored in memory
7305 try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi }));
72407306 },
7241 .ptr_stack_offset, .stack_offset => |off| {
7242 try self.asmRegisterMemory(.lea, .rdi, Memory.sib(.qword, .{
7307 .stack_offset, .ptr_stack_offset => |off| {
7308 try self.asmRegisterMemory(switch (dst_ptr) {
7309 .stack_offset => .mov,
7310 .ptr_stack_offset => .lea,
7311 else => unreachable,
7312 }, .rdi, Memory.sib(.qword, .{
72437313 .base = opts.dest_stack_base orelse .rbp,
72447314 .disp = -off,
72457315 }));
......@@ -7979,7 +8049,6 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
79798049 };
79808050 defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock);
79818051
7982 const src_ty = self.air.typeOf(extra.lhs);
79838052 const src_ptr = try self.resolveInst(extra.lhs);
79848053 const src_ptr_lock: ?RegisterLock = switch (src_ptr) {
79858054 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
......@@ -7994,25 +8063,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
79948063 };
79958064 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
79968065
7997 // TODO Is this the only condition for pointer dereference for memcpy?
7998 const src: MCValue = blk: {
7999 switch (src_ptr) {
8000 .linker_load, .memory => {
8001 const reg = try self.register_manager.allocReg(null, gp);
8002 try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
8003 try self.asmRegisterMemory(.mov, reg, Memory.sib(.qword, .{ .base = reg }));
8004 break :blk MCValue{ .register = reg };
8005 },
8006 else => break :blk src_ptr,
8007 }
8008 };
8009 const src_lock: ?RegisterLock = switch (src) {
8010 .register => |reg| self.register_manager.lockReg(reg),
8011 else => null,
8012 };
8013 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
8014
8015 try self.genInlineMemcpy(dst_ptr, src, len, .{});
8066 try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{});
80168067
80178068 return self.finishAir(inst, .none, .{ pl_op.operand, extra.lhs, extra.rhs });
80188069}
......@@ -8156,11 +8207,10 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
81568207 switch (result_ty.zigTypeTag()) {
81578208 .Struct => {
81588209 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
8159 const dst_mcv = MCValue{ .stack_offset = stack_offset };
81608210 if (result_ty.containerLayout() == .Packed) {
81618211 const struct_obj = result_ty.castTag(.@"struct").?.data;
81628212 try self.genInlineMemset(
8163 dst_mcv,
8213 .{ .ptr_stack_offset = stack_offset },
81648214 .{ .immediate = 0 },
81658215 .{ .immediate = abi_size },
81668216 .{},
......@@ -8236,7 +8286,7 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
82368286 const elem_mcv = try self.resolveInst(elem);
82378287 try self.genSetStack(elem_ty, stack_offset - elem_off, elem_mcv, .{});
82388288 }
8239 break :res dst_mcv;
8289 break :res .{ .stack_offset = stack_offset };
82408290 },
82418291 .Array => {
82428292 const stack_offset = @intCast(i32, try self.allocMem(inst, abi_size, abi_align));
test/behavior/array.zig-1
......@@ -190,7 +190,6 @@ test "nested arrays of strings" {
190190 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
191191 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
192192 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
193 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
194193
195194 const array_of_strings = [_][]const u8{ "hello", "this", "is", "my", "thing" };
196195 for (array_of_strings, 0..) |s, i| {
test/behavior/bugs/718.zig-1
......@@ -12,7 +12,6 @@ var keys: Keys = undefined;
1212test "zero keys with @memset" {
1313 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1414 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
15 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1615 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1716
1817 @memset(@ptrCast([*]u8, &keys), 0, @sizeOf(@TypeOf(keys)));
test/behavior/bugs/7325.zig-1
......@@ -79,7 +79,6 @@ fn genExpression(expr: Expression) !ExpressionResult {
7979
8080test {
8181 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
82 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8382 if (builtin.zig_backend == .stage2_x86) return error.SkipZigTest; // TODO
8483 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
8584 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO