authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-03-01 14:04:49+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-03-01 14:29:48+01:00
log5a6f4395e6569399b228030913b79cd73d5bad02
treec747bead4eadacf8bd1d1bfe02b1fc8993d42275
parentca97caab8a02813eb76388e72e2a9dab1f5fa047

x64: impl airMemcpy using inline memcpy

1 files changed, 89 insertions(+), 49 deletions(-)

src/arch/x86_64/CodeGen.zig+89-49
......@@ -2346,7 +2346,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo
23462346 return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{});
23472347 }
23482348
2349 try self.genInlineMemcpy(off, elem_ty, ptr, .{});
2349 try self.genInlineMemcpy(dst_mcv, ptr, .{ .immediate = abi_size }, .{});
23502350 },
23512351 else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}),
23522352 }
......@@ -2392,6 +2392,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
23922392 .got_load,
23932393 .direct_load,
23942394 => |sym_index| {
2395 const abi_size = @intCast(u32, ptr_ty.abiSize(self.target.*));
23952396 const flags: u2 = switch (ptr) {
23962397 .got_load => 0b00,
23972398 .direct_load => 0b01,
......@@ -2400,7 +2401,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue
24002401 _ = try self.addInst(.{
24012402 .tag = .lea_pie,
24022403 .ops = (Mir.Ops{
2403 .reg1 = registerAlias(reg, @intCast(u32, ptr_ty.abiSize(self.target.*))),
2404 .reg1 = registerAlias(reg, abi_size),
24042405 .flags = flags,
24052406 }).encode(),
24062407 .data = .{
......@@ -2524,7 +2525,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type
25242525 return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty);
25252526 }
25262527
2527 try self.genInlineMemcpy(0, value_ty, value, .{
2528 try self.genInlineMemcpy(.{ .stack_offset = 0 }, value, .{ .immediate = abi_size }, .{
25282529 .source_stack_base = .rbp,
25292530 .dest_stack_base = reg.to64(),
25302531 });
......@@ -3464,7 +3465,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void {
34643465 .data = undefined,
34653466 });
34663467 try self.ret_backpatches.append(self.gpa, backpatch);
3467 try self.genInlineMemcpy(0, elem_ty, ptr, .{
3468 try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{
34683469 .source_stack_base = .rbp,
34693470 .dest_stack_base = reg,
34703471 });
......@@ -4469,7 +4470,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
44694470 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
44704471 }
44714472
4472 try self.genInlineMemcpy(stack_offset, ty, mcv, .{
4473 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{
44734474 .source_stack_base = .rbp,
44744475 .dest_stack_base = .rsp,
44754476 });
......@@ -4495,7 +4496,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
44954496 return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg });
44964497 }
44974498
4498 try self.genInlineMemcpy(stack_offset, ty, mcv, .{
4499 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{
44994500 .source_stack_base = .rbp,
45004501 .dest_stack_base = .rsp,
45014502 });
......@@ -4650,7 +4651,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
46504651 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
46514652 }
46524653
4653 try self.genInlineMemcpy(stack_offset, ty, mcv, opts);
4654 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts);
46544655 },
46554656 .ptr_stack_offset => {
46564657 const reg = try self.copyToTmpRegister(ty, mcv);
......@@ -4667,7 +4668,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
46674668 return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts);
46684669 }
46694670
4670 try self.genInlineMemcpy(stack_offset, ty, mcv, opts);
4671 try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts);
46714672 },
46724673 }
46734674}
......@@ -4677,9 +4678,13 @@ const InlineMemcpyOpts = struct {
46774678 dest_stack_base: ?Register = null,
46784679};
46794680
4680fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: InlineMemcpyOpts) InnerError!void {
4681 const abi_size = ty.abiSize(self.target.*);
4682
4681fn genInlineMemcpy(
4682 self: *Self,
4683 dst_ptr: MCValue,
4684 src_ptr: MCValue,
4685 len: MCValue,
4686 opts: InlineMemcpyOpts,
4687) InnerError!void {
46834688 // TODO this is wrong. We should check first if any of the operands is in `.rax` or `.rcx` before
46844689 // spilling. Consolidate with other TODOs regarding register allocation mechanics.
46854690 try self.register_manager.getReg(.rax, null);
......@@ -4694,19 +4699,55 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts:
46944699 if (opts.dest_stack_base) |reg| self.register_manager.freezeRegs(&.{reg});
46954700 defer if (opts.dest_stack_base) |reg| self.register_manager.unfreezeRegs(&.{reg});
46964701
4697 const addr_reg = try self.register_manager.allocReg(null);
4698 switch (val) {
4702 const dst_addr_reg = try self.register_manager.allocReg(null);
4703 switch (dst_ptr) {
4704 .got_load => unreachable,
46994705 .memory,
47004706 .direct_load,
4701 .got_load,
47024707 => {
4703 try self.loadMemPtrIntoRegister(addr_reg, Type.usize, val);
4708 try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr);
47044709 },
47054710 .ptr_stack_offset, .stack_offset => |off| {
47064711 _ = try self.addInst(.{
47074712 .tag = .lea,
47084713 .ops = (Mir.Ops{
4709 .reg1 = addr_reg.to64(),
4714 .reg1 = dst_addr_reg.to64(),
4715 .reg2 = opts.dest_stack_base orelse .rbp,
4716 }).encode(),
4717 .data = .{ .imm = @bitCast(u32, -off) },
4718 });
4719 },
4720 .register => |reg| {
4721 _ = try self.addInst(.{
4722 .tag = .mov,
4723 .ops = (Mir.Ops{
4724 .reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)),
4725 .reg2 = reg,
4726 }).encode(),
4727 .data = undefined,
4728 });
4729 },
4730 else => {
4731 return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr});
4732 },
4733 }
4734
4735 self.register_manager.freezeRegs(&.{dst_addr_reg});
4736 defer self.register_manager.unfreezeRegs(&.{dst_addr_reg});
4737
4738 const src_addr_reg = try self.register_manager.allocReg(null);
4739 switch (src_ptr) {
4740 .got_load => unreachable,
4741 .memory,
4742 .direct_load,
4743 => {
4744 try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr);
4745 },
4746 .ptr_stack_offset, .stack_offset => |off| {
4747 _ = try self.addInst(.{
4748 .tag = .lea,
4749 .ops = (Mir.Ops{
4750 .reg1 = src_addr_reg.to64(),
47104751 .reg2 = opts.source_stack_base orelse .rbp,
47114752 }).encode(),
47124753 .data = .{ .imm = @bitCast(u32, -off) },
......@@ -4716,25 +4757,25 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts:
47164757 _ = try self.addInst(.{
47174758 .tag = .mov,
47184759 .ops = (Mir.Ops{
4719 .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)),
4760 .reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)),
47204761 .reg2 = reg,
47214762 }).encode(),
47224763 .data = undefined,
47234764 });
47244765 },
47254766 else => {
4726 return self.fail("TODO implement memcpy for setting stack from {}", .{val});
4767 return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr});
47274768 },
47284769 }
47294770
4730 self.register_manager.freezeRegs(&.{addr_reg});
4731 defer self.register_manager.unfreezeRegs(&.{addr_reg});
4771 self.register_manager.freezeRegs(&.{src_addr_reg});
4772 defer self.register_manager.unfreezeRegs(&.{src_addr_reg});
47324773
47334774 const regs = try self.register_manager.allocRegs(2, .{ null, null });
47344775 const count_reg = regs[0].to64();
47354776 const tmp_reg = regs[1].to8();
47364777
4737 try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) });
4778 try self.genSetReg(Type.usize, count_reg, len);
47384779
47394780 // mov rcx, 0
47404781 _ = try self.addInst(.{
......@@ -4776,7 +4817,7 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts:
47764817 .tag = .mov_scale_src,
47774818 .ops = (Mir.Ops{
47784819 .reg1 = tmp_reg.to8(),
4779 .reg2 = addr_reg,
4820 .reg2 = src_addr_reg,
47804821 }).encode(),
47814822 .data = .{ .imm = 0 },
47824823 });
......@@ -4785,10 +4826,10 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts:
47854826 _ = try self.addInst(.{
47864827 .tag = .mov_scale_dst,
47874828 .ops = (Mir.Ops{
4788 .reg1 = opts.dest_stack_base orelse .rbp,
4829 .reg1 = dst_addr_reg,
47894830 .reg2 = tmp_reg.to8(),
47904831 }).encode(),
4791 .data = .{ .imm = @bitCast(u32, -stack_offset) },
4832 .data = .{ .imm = 0 },
47924833 });
47934834
47944835 // add rcx, 1
......@@ -5076,33 +5117,15 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
50765117 .data = undefined,
50775118 });
50785119 },
5079 .got_load,
50805120 .direct_load,
5081 => |sym_index| {
5082 const flags: u2 = switch (mcv) {
5083 .got_load => 0b00,
5084 .direct_load => 0b01,
5085 else => unreachable,
5086 };
5087 _ = try self.addInst(.{
5088 .tag = .lea_pie,
5089 .ops = (Mir.Ops{
5090 .reg1 = reg,
5091 .flags = flags,
5092 }).encode(),
5093 .data = .{
5094 .load_reloc = .{
5095 .atom_index = self.mod_fn.owner_decl.link.macho.local_sym_index,
5096 .sym_index = sym_index,
5097 },
5098 },
5099 });
5100 // MOV reg, [reg]
5121 .got_load,
5122 => {
5123 try self.loadMemPtrIntoRegister(reg, Type.usize, mcv);
51015124 _ = try self.addInst(.{
51025125 .tag = .mov,
51035126 .ops = (Mir.Ops{
5104 .reg1 = reg,
5105 .reg2 = reg,
5127 .reg1 = reg.to64(),
5128 .reg2 = reg.to64(),
51065129 .flags = 0b01,
51075130 }).encode(),
51085131 .data = .{ .imm = 0 },
......@@ -5283,8 +5306,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void {
52835306}
52845307
52855308fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void {
5286 _ = inst;
5287 return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch});
5309 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
5310 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
5311
5312 const dst_ptr = try self.resolveInst(pl_op.operand);
5313 dst_ptr.freezeIfRegister(&self.register_manager);
5314 defer dst_ptr.unfreezeIfRegister(&self.register_manager);
5315
5316 const src_ptr = try self.resolveInst(extra.lhs);
5317 src_ptr.freezeIfRegister(&self.register_manager);
5318 defer src_ptr.unfreezeIfRegister(&self.register_manager);
5319
5320 const len = try self.resolveInst(extra.rhs);
5321 len.freezeIfRegister(&self.register_manager);
5322 defer len.unfreezeIfRegister(&self.register_manager);
5323
5324 log.warn("dst_ptr = {}, src_ptr = {}, len = {}", .{ dst_ptr, src_ptr, len });
5325 try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{});
5326
5327 return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none });
52885328}
52895329
52905330fn airTagName(self: *Self, inst: Air.Inst.Index) !void {