authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-03-01 15:21:10+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-03-01 15:21:10+01:00
logcaa4e30ef458ee150a9ce9950293827957afb87c
treec820beca9cfac30ea1ea659f2f21f63e93a16406
parent5a6f4395e6569399b228030913b79cd73d5bad02

x64: impl airMemset using inline memset

3 files changed, 100 insertions(+), 42 deletions(-)

src/arch/x86_64/CodeGen.zig+99-39
...@@ -4413,9 +4413,12 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE...@@ -4413,9 +4413,12 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
4413 const reg = try self.copyToTmpRegister(ty, mcv);4413 const reg = try self.copyToTmpRegister(ty, mcv);
4414 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });4414 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
4415 }4415 }
4416 try self.genInlineMemset(stack_offset, ty, .{ .immediate = 0xaa }, .{4416 try self.genInlineMemset(
4417 .dest_stack_base = .rsp,4417 .{ .stack_offset = stack_offset },
4418 });4418 .{ .immediate = 0xaa },
4419 .{ .immediate = abi_size },
4420 .{ .dest_stack_base = .rsp },
4421 );
4419 },4422 },
4420 .compare_flags_unsigned,4423 .compare_flags_unsigned,
4421 .compare_flags_signed,4424 .compare_flags_signed,
...@@ -4519,7 +4522,12 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl...@@ -4519,7 +4522,12 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
4519 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }, opts),4522 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }, opts),
4520 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }, opts),4523 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }, opts),
4521 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, opts),4524 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, opts),
4522 else => return self.genInlineMemset(stack_offset, ty, .{ .immediate = 0xaa }, opts),4525 else => |x| return self.genInlineMemset(
4526 .{ .stack_offset = stack_offset },
4527 .{ .immediate = 0xaa },
4528 .{ .immediate = x },
4529 opts,
4530 ),
4523 }4531 }
4524 },4532 },
4525 .compare_flags_unsigned,4533 .compare_flags_unsigned,
...@@ -4701,8 +4709,8 @@ fn genInlineMemcpy(...@@ -4701,8 +4709,8 @@ fn genInlineMemcpy(
47014709
4702 const dst_addr_reg = try self.register_manager.allocReg(null);4710 const dst_addr_reg = try self.register_manager.allocReg(null);
4703 switch (dst_ptr) {4711 switch (dst_ptr) {
4704 .got_load => unreachable,
4705 .memory,4712 .memory,
4713 .got_load,
4706 .direct_load,4714 .direct_load,
4707 => {4715 => {
4708 try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);4716 try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);
...@@ -4737,8 +4745,8 @@ fn genInlineMemcpy(...@@ -4737,8 +4745,8 @@ fn genInlineMemcpy(
47374745
4738 const src_addr_reg = try self.register_manager.allocReg(null);4746 const src_addr_reg = try self.register_manager.allocReg(null);
4739 switch (src_ptr) {4747 switch (src_ptr) {
4740 .got_load => unreachable,
4741 .memory,4748 .memory,
4749 .got_load,
4742 .direct_load,4750 .direct_load,
4743 => {4751 => {
4744 try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);4752 try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);
...@@ -4872,39 +4880,52 @@ fn genInlineMemcpy(...@@ -4872,39 +4880,52 @@ fn genInlineMemcpy(
48724880
4873fn genInlineMemset(4881fn genInlineMemset(
4874 self: *Self,4882 self: *Self,
4875 stack_offset: i32,4883 dst_ptr: MCValue,
4876 ty: Type,
4877 value: MCValue,4884 value: MCValue,
4885 len: MCValue,
4878 opts: InlineMemcpyOpts,4886 opts: InlineMemcpyOpts,
4879) InnerError!void {4887) InnerError!void {
4880 try self.register_manager.getReg(.rax, null);4888 try self.register_manager.getReg(.rax, null);
4889 self.register_manager.freezeRegs(&.{.rax});
4890 defer self.register_manager.unfreezeRegs(&.{.rax});
48814891
4882 const abi_size = ty.abiSize(self.target.*);4892 const addr_reg = try self.register_manager.allocReg(null);
4883 const negative_offset = @bitCast(u32, -stack_offset);4893 switch (dst_ptr) {
48844894 .memory,
4885 // We are actually counting `abi_size` bytes; however, we reuse the index register4895 .got_load,
4886 // as both the counter and offset scaler, hence we need to subtract one from `abi_size`4896 .direct_load,
4887 // and count until -1.4897 => {
4888 if (abi_size > math.maxInt(i32)) {4898 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr);
4889 // movabs rax, abi_size - 14899 },
4890 const payload = try self.addExtra(Mir.Imm64.encode(abi_size - 1));4900 .ptr_stack_offset, .stack_offset => |off| {
4891 _ = try self.addInst(.{4901 _ = try self.addInst(.{
4892 .tag = .movabs,4902 .tag = .lea,
4893 .ops = (Mir.Ops{4903 .ops = (Mir.Ops{
4894 .reg1 = .rax,4904 .reg1 = addr_reg.to64(),
4895 }).encode(),4905 .reg2 = opts.dest_stack_base orelse .rbp,
4896 .data = .{ .payload = payload },4906 }).encode(),
4897 });4907 .data = .{ .imm = @bitCast(u32, -off) },
4898 } else {4908 });
4899 // mov rax, abi_size - 14909 },
4900 _ = try self.addInst(.{4910 .register => |reg| {
4901 .tag = .mov,4911 _ = try self.addInst(.{
4902 .ops = (Mir.Ops{4912 .tag = .mov,
4903 .reg1 = .rax,4913 .ops = (Mir.Ops{
4904 }).encode(),4914 .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
4905 .data = .{ .imm = @truncate(u32, abi_size - 1) },4915 .reg2 = reg,
4906 });4916 }).encode(),
4917 .data = undefined,
4918 });
4919 },
4920 else => {
4921 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
4922 },
4907 }4923 }
4924 self.register_manager.freezeRegs(&.{addr_reg});
4925 defer self.register_manager.unfreezeRegs(&.{addr_reg});
4926
4927 try self.genSetReg(Type.usize, .rax, len);
4928 try self.genBinMathOpMir(.sub, Type.usize, .{ .register = .rax }, .{ .immediate = 1 });
49084929
4909 // loop:4930 // loop:
4910 // cmp rax, -14931 // cmp rax, -1
...@@ -4930,13 +4951,13 @@ fn genInlineMemset(...@@ -4930,13 +4951,13 @@ fn genInlineMemset(
4930 }4951 }
4931 // mov byte ptr [rbp + rax + stack_offset], imm4952 // mov byte ptr [rbp + rax + stack_offset], imm
4932 const payload = try self.addExtra(Mir.ImmPair{4953 const payload = try self.addExtra(Mir.ImmPair{
4933 .dest_off = negative_offset,4954 .dest_off = 0,
4934 .operand = @truncate(u32, x),4955 .operand = @truncate(u32, x),
4935 });4956 });
4936 _ = try self.addInst(.{4957 _ = try self.addInst(.{
4937 .tag = .mov_mem_index_imm,4958 .tag = .mov_mem_index_imm,
4938 .ops = (Mir.Ops{4959 .ops = (Mir.Ops{
4939 .reg1 = opts.dest_stack_base orelse .rbp,4960 .reg1 = addr_reg,
4940 }).encode(),4961 }).encode(),
4941 .data = .{ .payload = payload },4962 .data = .{ .payload = payload },
4942 });4963 });
...@@ -5301,8 +5322,24 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr...@@ -5301,8 +5322,24 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr
5301}5322}
53025323
5303fn airMemset(self: *Self, inst: Air.Inst.Index) !void {5324fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
5304 _ = inst;5325 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
5305 return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});5326 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
5327
5328 const dst_ptr = try self.resolveInst(pl_op.operand);
5329 dst_ptr.freezeIfRegister(&self.register_manager);
5330 defer dst_ptr.unfreezeIfRegister(&self.register_manager);
5331
5332 const src_val = try self.resolveInst(extra.lhs);
5333 src_val.freezeIfRegister(&self.register_manager);
5334 defer src_val.unfreezeIfRegister(&self.register_manager);
5335
5336 const len = try self.resolveInst(extra.rhs);
5337 len.freezeIfRegister(&self.register_manager);
5338 defer len.unfreezeIfRegister(&self.register_manager);
5339
5340 try self.genInlineMemset(dst_ptr, src_val, len, .{});
5341
5342 return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
5306}5343}
53075344
5308fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {5345fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
...@@ -5313,6 +5350,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -5313,6 +5350,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
5313 dst_ptr.freezeIfRegister(&self.register_manager);5350 dst_ptr.freezeIfRegister(&self.register_manager);
5314 defer dst_ptr.unfreezeIfRegister(&self.register_manager);5351 defer dst_ptr.unfreezeIfRegister(&self.register_manager);
53155352
5353 const src_ty = self.air.typeOf(extra.lhs);
5316 const src_ptr = try self.resolveInst(extra.lhs);5354 const src_ptr = try self.resolveInst(extra.lhs);
5317 src_ptr.freezeIfRegister(&self.register_manager);5355 src_ptr.freezeIfRegister(&self.register_manager);
5318 defer src_ptr.unfreezeIfRegister(&self.register_manager);5356 defer src_ptr.unfreezeIfRegister(&self.register_manager);
...@@ -5321,8 +5359,30 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {...@@ -5321,8 +5359,30 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
5321 len.freezeIfRegister(&self.register_manager);5359 len.freezeIfRegister(&self.register_manager);
5322 defer len.unfreezeIfRegister(&self.register_manager);5360 defer len.unfreezeIfRegister(&self.register_manager);
53235361
5324 log.warn("dst_ptr = {}, src_ptr = {}, len = {}", .{ dst_ptr, src_ptr, len });5362 // TODO Is this the only condition for pointer dereference for memcpy?
5325 try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{});5363 const src: MCValue = blk: {
5364 switch (src_ptr) {
5365 .got_load, .direct_load, .memory => {
5366 const reg = try self.register_manager.allocReg(null);
5367 try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
5368 _ = try self.addInst(.{
5369 .tag = .mov,
5370 .ops = (Mir.Ops{
5371 .reg1 = reg,
5372 .reg2 = reg,
5373 .flags = 0b01,
5374 }).encode(),
5375 .data = .{ .imm = 0 },
5376 });
5377 break :blk MCValue{ .register = reg };
5378 },
5379 else => break :blk src_ptr,
5380 }
5381 };
5382 src.freezeIfRegister(&self.register_manager);
5383 defer src.unfreezeIfRegister(&self.register_manager);
5384
5385 try self.genInlineMemcpy(dst_ptr, src, len, .{});
53265386
5327 return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });5387 return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
5328}5388}
test/behavior/basic.zig-1
...@@ -341,7 +341,6 @@ fn f2(x: bool) []const u8 {...@@ -341,7 +341,6 @@ fn f2(x: bool) []const u8 {
341test "memcpy and memset intrinsics" {341test "memcpy and memset intrinsics" {
342 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;342 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
343 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;343 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
344 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
345 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO344 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
346345
347 try testMemcpyMemset();346 try testMemcpyMemset();
test/behavior/struct.zig+1-2
...@@ -80,12 +80,11 @@ const StructWithNoFields = struct {...@@ -80,12 +80,11 @@ const StructWithNoFields = struct {
80const StructFoo = struct {80const StructFoo = struct {
81 a: i32,81 a: i32,
82 b: bool,82 b: bool,
83 c: f32,83 c: u64,
84};84};
8585
86test "structs" {86test "structs" {
87 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;87 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
88 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
89 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;88 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
9089
91 var foo: StructFoo = undefined;90 var foo: StructFoo = undefined;