authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-03-01 17:59:21+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-03-01 17:59:21+01:00
logad03fef71222ad4335107257451c1828fb9adc88
treec820beca9cfac30ea1ea659f2f21f63e93a16406
parentca97caab8a02813eb76388e72e2a9dab1f5fa047
parentcaa4e30ef458ee150a9ce9950293827957afb87c
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11019 from ziglang/x64-air-memcpy

x64: memcpy and memset

3 files changed, 184 insertions(+), 86 deletions(-)

src/arch/x86_64/CodeGen.zig+183-83
......@@ -2346,7 +2346,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
23462346 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
23472347 }
23482348
2349 try self.genInlineMemcpy(off, elem_ty, ptr, .{});
2349 try self.genInlineMemcpy(dst_mcv, ptr, .{ .immediate = abi_size }, .{});
23502350 },
23512351 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
23522352 }
......@@ -2392,6 +2392,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
23922392 .got_load,
23932393 .direct_load,
23942394 => |sym_index| {
2395 const abi_size = @intCast(u32, ptr_ty.abiSize(self.target.*));
23952396 const flags: u2 = switch (ptr) {
23962397 .got_load => 0b00,
23972398 .direct_load => 0b01,
......@@ -2400,7 +2401,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
24002401 _ = try self.addInst(.{
24012402 .tag = .lea_pie,
24022403 .ops = (Mir.Ops{
2403 .reg1 = registerAlias(reg, @intCast(u32, ptr_ty.abiSize(self.target.*))),
2404 .reg1 = registerAlias(reg, abi_size),
24042405 .flags = flags,
24052406 }).encode(),
24062407 .data = .{
......@@ -2524,7 +2525,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
25242525 return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
25252526 }
25262527
2527 try self.genInlineMemcpy(0, value_ty, value, .{
2528 try self.genInlineMemcpy(.{ .stack_offset = 0 }, value, .{ .immediate = abi_size }, .{
25282529 .source_stack_base = .rbp,
25292530 .dest_stack_base = reg.to64(),
25302531 });
......@@ -3464,7 +3465,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
34643465 .data = undefined,
34653466 });
34663467 try self.ret_backpatches.append(self.gpa, backpatch);
3467 try self.genInlineMemcpy(0, elem_ty, ptr, .{
3468 try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{
34683469 .source_stack_base = .rbp,
34693470 .dest_stack_base = reg,
34703471 });
......@@ -4412,9 +4413,12 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
44124413 const reg = try self.copyToTmpRegister(ty, mcv);
44134414 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
44144415 }
4415 try self.genInlineMemset(stack_offset, ty, .{ .immediate = 0xaa }, .{
4416 .dest_stack_base = .rsp,
4417 });
4416 try self.genInlineMemset(
4417 .{ .stack_offset = stack_offset },
4418 .{ .immediate = 0xaa },
4419 .{ .immediate = abi_size },
4420 .{ .dest_stack_base = .rsp },
4421 );
44184422 },
44194423 .compare_flags_unsigned,
44204424 .compare_flags_signed,
......@@ -4469,7 +4473,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
44694473 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
44704474 }
44714475
4472 try self.genInlineMemcpy(stack_offset, ty, mcv, .{
4476 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{
44734477 .source_stack_base = .rbp,
44744478 .dest_stack_base = .rsp,
44754479 });
......@@ -4495,7 +4499,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
44954499 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
44964500 }
44974501
4498 try self.genInlineMemcpy(stack_offset, ty, mcv, .{
4502 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{
44994503 .source_stack_base = .rbp,
45004504 .dest_stack_base = .rsp,
45014505 });
......@@ -4518,7 +4522,12 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
45184522 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }, opts),
45194523 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }, opts),
45204524 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, opts),
4521 else => return self.genInlineMemset(stack_offset, ty, .{ .immediate = 0xaa }, opts),
4525 else => |x| return self.genInlineMemset(
4526 .{ .stack_offset = stack_offset },
4527 .{ .immediate = 0xaa },
4528 .{ .immediate = x },
4529 opts,
4530 ),
45224531 }
45234532 },
45244533 .compare_flags_unsigned,
......@@ -4650,7 +4659,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
46504659 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
46514660 }
46524661
4653 try self.genInlineMemcpy(stack_offset, ty, mcv, opts);
4662 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts);
46544663 },
46554664 .ptr_stack_offset => {
46564665 const reg = try self.copyToTmpRegister(ty, mcv);
......@@ -4667,7 +4676,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
46674676 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
46684677 }
46694678
4670 try self.genInlineMemcpy(stack_offset, ty, mcv, opts);
4679 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts);
46714680 },
46724681 }
46734682}
......@@ -4677,9 +4686,13 @@ const InlineMemcpyOpts = struct {
46774686 dest_stack_base: ?Register = null,
46784687};
46794688
4680fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: InlineMemcpyOpts) InnerError!void {
4681 const abi_size = ty.abiSize(self.target.*);
4682
4689fn genInlineMemcpy(
4690 self: *Self,
4691 dst_ptr: MCValue,
4692 src_ptr: MCValue,
4693 len: MCValue,
4694 opts: InlineMemcpyOpts,
4695) InnerError!void {
46834696 // TODO this is wrong. We should check first if any of the operands is in `.rax` or `.rcx` before
46844697 // spilling. Consolidate with other TODOs regarding register allocation mechanics.
46854698 try self.register_manager.getReg(.rax, null);
......@@ -4694,19 +4707,55 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts:
46944707 if (opts.dest_stack_base) |reg| self.register_manager.freezeRegs(&.{reg});
46954708 defer if (opts.dest_stack_base) |reg| self.register_manager.unfreezeRegs(&.{reg});
46964709
4697 const addr_reg = try self.register_manager.allocReg(null);
4698 switch (val) {
4710 const dst_addr_reg = try self.register_manager.allocReg(null);
4711 switch (dst_ptr) {
46994712 .memory,
4713 .got_load,
47004714 .direct_load,
4715 => {
4716 try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);
4717 },
4718 .ptr_stack_offset, .stack_offset => |off| {
4719 _ = try self.addInst(.{
4720 .tag = .lea,
4721 .ops = (Mir.Ops{
4722 .reg1 = dst_addr_reg.to64(),
4723 .reg2 = opts.dest_stack_base orelse .rbp,
4724 }).encode(),
4725 .data = .{ .imm = @bitCast(u32, -off) },
4726 });
4727 },
4728 .register => |reg| {
4729 _ = try self.addInst(.{
4730 .tag = .mov,
4731 .ops = (Mir.Ops{
4732 .reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)),
4733 .reg2 = reg,
4734 }).encode(),
4735 .data = undefined,
4736 });
4737 },
4738 else => {
4739 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
4740 },
4741 }
4742
4743 self.register_manager.freezeRegs(&.{dst_addr_reg});
4744 defer self.register_manager.unfreezeRegs(&.{dst_addr_reg});
4745
4746 const src_addr_reg = try self.register_manager.allocReg(null);
4747 switch (src_ptr) {
4748 .memory,
47014749 .got_load,
4750 .direct_load,
47024751 => {
4703 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, val);
4752 try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);
47044753 },
47054754 .ptr_stack_offset, .stack_offset => |off| {
47064755 _ = try self.addInst(.{
47074756 .tag = .lea,
47084757 .ops = (Mir.Ops{
4709 .reg1 = addr_reg.to64(),
4758 .reg1 = src_addr_reg.to64(),
47104759 .reg2 = opts.source_stack_base orelse .rbp,
47114760 }).encode(),
47124761 .data = .{ .imm = @bitCast(u32, -off) },
......@@ -4716,25 +4765,25 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts:
47164765 _ = try self.addInst(.{
47174766 .tag = .mov,
47184767 .ops = (Mir.Ops{
4719 .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
4768 .reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)),
47204769 .reg2 = reg,
47214770 }).encode(),
47224771 .data = undefined,
47234772 });
47244773 },
47254774 else => {
4726 return self.fail("TODO implement memcpy for setting stack from {}", .{val});
4775 return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
47274776 },
47284777 }
47294778
4730 self.register_manager.freezeRegs(&.{addr_reg});
4731 defer self.register_manager.unfreezeRegs(&.{addr_reg});
4779 self.register_manager.freezeRegs(&.{src_addr_reg});
4780 defer self.register_manager.unfreezeRegs(&.{src_addr_reg});
47324781
47334782 const regs = try self.register_manager.allocRegs(2, .{ null, null });
47344783 const count_reg = regs[0].to64();
47354784 const tmp_reg = regs[1].to8();
47364785
4737 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
4786 try self.genSetReg(Type.usize, count_reg, len);
47384787
47394788 // mov rcx, 0
47404789 _ = try self.addInst(.{
......@@ -4776,7 +4825,7 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts:
47764825 .tag = .mov_scale_src,
47774826 .ops = (Mir.Ops{
47784827 .reg1 = tmp_reg.to8(),
4779 .reg2 = addr_reg,
4828 .reg2 = src_addr_reg,
47804829 }).encode(),
47814830 .data = .{ .imm = 0 },
47824831 });
......@@ -4785,10 +4834,10 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts:
47854834 _ = try self.addInst(.{
47864835 .tag = .mov_scale_dst,
47874836 .ops = (Mir.Ops{
4788 .reg1 = opts.dest_stack_base orelse .rbp,
4837 .reg1 = dst_addr_reg,
47894838 .reg2 = tmp_reg.to8(),
47904839 }).encode(),
4791 .data = .{ .imm = @bitCast(u32, -stack_offset) },
4840 .data = .{ .imm = 0 },
47924841 });
47934842
47944843 // add rcx, 1
......@@ -4831,39 +4880,52 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts:
48314880
48324881fn genInlineMemset(
48334882 self: *Self,
4834 stack_offset: i32,
4835 ty: Type,
4883 dst_ptr: MCValue,
48364884 value: MCValue,
4885 len: MCValue,
48374886 opts: InlineMemcpyOpts,
48384887) InnerError!void {
48394888 try self.register_manager.getReg(.rax, null);
4889 self.register_manager.freezeRegs(&.{.rax});
4890 defer self.register_manager.unfreezeRegs(&.{.rax});
48404891
4841 const abi_size = ty.abiSize(self.target.*);
4842 const negative_offset = @bitCast(u32, -stack_offset);
4843
4844 // We are actually counting `abi_size` bytes; however, we reuse the index register
4845 // as both the counter and offset scaler, hence we need to subtract one from `abi_size`
4846 // and count until -1.
4847 if (abi_size > math.maxInt(i32)) {
4848 // movabs rax, abi_size - 1
4849 const payload = try self.addExtra(Mir.Imm64.encode(abi_size - 1));
4850 _ = try self.addInst(.{
4851 .tag = .movabs,
4852 .ops = (Mir.Ops{
4853 .reg1 = .rax,
4854 }).encode(),
4855 .data = .{ .payload = payload },
4856 });
4857 } else {
4858 // mov rax, abi_size - 1
4859 _ = try self.addInst(.{
4860 .tag = .mov,
4861 .ops = (Mir.Ops{
4862 .reg1 = .rax,
4863 }).encode(),
4864 .data = .{ .imm = @truncate(u32, abi_size - 1) },
4865 });
4892 const addr_reg = try self.register_manager.allocReg(null);
4893 switch (dst_ptr) {
4894 .memory,
4895 .got_load,
4896 .direct_load,
4897 => {
4898 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr);
4899 },
4900 .ptr_stack_offset, .stack_offset => |off| {
4901 _ = try self.addInst(.{
4902 .tag = .lea,
4903 .ops = (Mir.Ops{
4904 .reg1 = addr_reg.to64(),
4905 .reg2 = opts.dest_stack_base orelse .rbp,
4906 }).encode(),
4907 .data = .{ .imm = @bitCast(u32, -off) },
4908 });
4909 },
4910 .register => |reg| {
4911 _ = try self.addInst(.{
4912 .tag = .mov,
4913 .ops = (Mir.Ops{
4914 .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
4915 .reg2 = reg,
4916 }).encode(),
4917 .data = undefined,
4918 });
4919 },
4920 else => {
4921 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
4922 },
48664923 }
4924 self.register_manager.freezeRegs(&.{addr_reg});
4925 defer self.register_manager.unfreezeRegs(&.{addr_reg});
4926
4927 try self.genSetReg(Type.usize, .rax, len);
4928 try self.genBinMathOpMir(.sub, Type.usize, .{ .register = .rax }, .{ .immediate = 1 });
48674929
48684930 // loop:
48694931 // cmp rax, -1
......@@ -4889,13 +4951,13 @@ fn genInlineMemset(
48894951 }
48904952 // mov byte ptr [rbp + rax + stack_offset], imm
48914953 const payload = try self.addExtra(Mir.ImmPair{
4892 .dest_off = negative_offset,
4954 .dest_off = 0,
48934955 .operand = @truncate(u32, x),
48944956 });
48954957 _ = try self.addInst(.{
48964958 .tag = .mov_mem_index_imm,
48974959 .ops = (Mir.Ops{
4898 .reg1 = opts.dest_stack_base orelse .rbp,
4960 .reg1 = addr_reg,
48994961 }).encode(),
49004962 .data = .{ .payload = payload },
49014963 });
......@@ -5076,33 +5138,15 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
50765138 .data = undefined,
50775139 });
50785140 },
5079 .got_load,
50805141 .direct_load,
5081 => |sym_index| {
5082 const flags: u2 = switch (mcv) {
5083 .got_load => 0b00,
5084 .direct_load => 0b01,
5085 else => unreachable,
5086 };
5087 _ = try self.addInst(.{
5088 .tag = .lea_pie,
5089 .ops = (Mir.Ops{
5090 .reg1 = reg,
5091 .flags = flags,
5092 }).encode(),
5093 .data = .{
5094 .load_reloc = .{
5095 .atom_index = self.mod_fn.owner_decl.link.macho.local_sym_index,
5096 .sym_index = sym_index,
5097 },
5098 },
5099 });
5100 // MOV reg, [reg]
5142 .got_load,
5143 => {
5144 try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);
51015145 _ = try self.addInst(.{
51025146 .tag = .mov,
51035147 .ops = (Mir.Ops{
5104 .reg1 = reg,
5105 .reg2 = reg,
5148 .reg1 = reg.to64(),
5149 .reg2 = reg.to64(),
51065150 .flags = 0b01,
51075151 }).encode(),
51085152 .data = .{ .imm = 0 },
......@@ -5278,13 +5322,69 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr
52785322}
52795323
52805324fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
5281 _ = inst;
5282 return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch});
5325 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
5326 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
5327
5328 const dst_ptr = try self.resolveInst(pl_op.operand);
5329 dst_ptr.freezeIfRegister(&self.register_manager);
5330 defer dst_ptr.unfreezeIfRegister(&self.register_manager);
5331
5332 const src_val = try self.resolveInst(extra.lhs);
5333 src_val.freezeIfRegister(&self.register_manager);
5334 defer src_val.unfreezeIfRegister(&self.register_manager);
5335
5336 const len = try self.resolveInst(extra.rhs);
5337 len.freezeIfRegister(&self.register_manager);
5338 defer len.unfreezeIfRegister(&self.register_manager);
5339
5340 try self.genInlineMemset(dst_ptr, src_val, len, .{});
5341
5342 return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
52835343}
52845344
52855345fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
5286 _ = inst;
5287 return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch});
5346 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
5347 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
5348
5349 const dst_ptr = try self.resolveInst(pl_op.operand);
5350 dst_ptr.freezeIfRegister(&self.register_manager);
5351 defer dst_ptr.unfreezeIfRegister(&self.register_manager);
5352
5353 const src_ty = self.air.typeOf(extra.lhs);
5354 const src_ptr = try self.resolveInst(extra.lhs);
5355 src_ptr.freezeIfRegister(&self.register_manager);
5356 defer src_ptr.unfreezeIfRegister(&self.register_manager);
5357
5358 const len = try self.resolveInst(extra.rhs);
5359 len.freezeIfRegister(&self.register_manager);
5360 defer len.unfreezeIfRegister(&self.register_manager);
5361
5362 // TODO Is this the only condition for pointer dereference for memcpy?
5363 const src: MCValue = blk: {
5364 switch (src_ptr) {
5365 .got_load, .direct_load, .memory => {
5366 const reg = try self.register_manager.allocReg(null);
5367 try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr);
5368 _ = try self.addInst(.{
5369 .tag = .mov,
5370 .ops = (Mir.Ops{
5371 .reg1 = reg,
5372 .reg2 = reg,
5373 .flags = 0b01,
5374 }).encode(),
5375 .data = .{ .imm = 0 },
5376 });
5377 break :blk MCValue{ .register = reg };
5378 },
5379 else => break :blk src_ptr,
5380 }
5381 };
5382 src.freezeIfRegister(&self.register_manager);
5383 defer src.unfreezeIfRegister(&self.register_manager);
5384
5385 try self.genInlineMemcpy(dst_ptr, src, len, .{});
5386
5387 return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
52885388}
52895389
52905390fn airTagName(self: *Self, inst: Air.Inst.Index) !void {
test/behavior/basic.zig-1
......@@ -341,7 +341,6 @@ fn f2(x: bool) []const u8 {
341341test "memcpy and memset intrinsics" {
342342 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
343343 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
344 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
345344 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
346345
347346 try testMemcpyMemset();
test/behavior/struct.zig+1-2
......@@ -80,12 +80,11 @@ const StructWithNoFields = struct {
8080const StructFoo = struct {
8181 a: i32,
8282 b: bool,
83 c: f32,
83 c: u64,
8484};
8585
8686test "structs" {
8787 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
88 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
8988 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
9089
9190 var foo: StructFoo = undefined;