| ... | @@ -2346,7 +2346,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo | ... | @@ -2346,7 +2346,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo |
| 2346 | return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{}); | 2346 | return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{}); |
| 2347 | } | 2347 | } |
| 2348 | | 2348 | |
| 2349 | try self.genInlineMemcpy(off, elem_ty, ptr, .{}); | 2349 | try self.genInlineMemcpy(dst_mcv, ptr, .{ .immediate = abi_size }, .{}); |
| 2350 | }, | 2350 | }, |
| 2351 | else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}), | 2351 | else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}), |
| 2352 | } | 2352 | } |
| ... | @@ -2392,6 +2392,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue | ... | @@ -2392,6 +2392,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue |
| 2392 | .got_load, | 2392 | .got_load, |
| 2393 | .direct_load, | 2393 | .direct_load, |
| 2394 | => |sym_index| { | 2394 | => |sym_index| { |
| | 2395 | const abi_size = @intCast(u32, ptr_ty.abiSize(self.target.*)); |
| 2395 | const flags: u2 = switch (ptr) { | 2396 | const flags: u2 = switch (ptr) { |
| 2396 | .got_load => 0b00, | 2397 | .got_load => 0b00, |
| 2397 | .direct_load => 0b01, | 2398 | .direct_load => 0b01, |
| ... | @@ -2400,7 +2401,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue | ... | @@ -2400,7 +2401,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue |
| 2400 | _ = try self.addInst(.{ | 2401 | _ = try self.addInst(.{ |
| 2401 | .tag = .lea_pie, | 2402 | .tag = .lea_pie, |
| 2402 | .ops = (Mir.Ops{ | 2403 | .ops = (Mir.Ops{ |
| 2403 | .reg1 = registerAlias(reg, @intCast(u32, ptr_ty.abiSize(self.target.*))), | 2404 | .reg1 = registerAlias(reg, abi_size), |
| 2404 | .flags = flags, | 2405 | .flags = flags, |
| 2405 | }).encode(), | 2406 | }).encode(), |
| 2406 | .data = .{ | 2407 | .data = .{ |
| ... | @@ -2524,7 +2525,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type | ... | @@ -2524,7 +2525,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type |
| 2524 | return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty); | 2525 | return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty); |
| 2525 | } | 2526 | } |
| 2526 | | 2527 | |
| 2527 | try self.genInlineMemcpy(0, value_ty, value, .{ | 2528 | try self.genInlineMemcpy(.{ .stack_offset = 0 }, value, .{ .immediate = abi_size }, .{ |
| 2528 | .source_stack_base = .rbp, | 2529 | .source_stack_base = .rbp, |
| 2529 | .dest_stack_base = reg.to64(), | 2530 | .dest_stack_base = reg.to64(), |
| 2530 | }); | 2531 | }); |
| ... | @@ -3464,7 +3465,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -3464,7 +3465,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void { |
| 3464 | .data = undefined, | 3465 | .data = undefined, |
| 3465 | }); | 3466 | }); |
| 3466 | try self.ret_backpatches.append(self.gpa, backpatch); | 3467 | try self.ret_backpatches.append(self.gpa, backpatch); |
| 3467 | try self.genInlineMemcpy(0, elem_ty, ptr, .{ | 3468 | try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{ |
| 3468 | .source_stack_base = .rbp, | 3469 | .source_stack_base = .rbp, |
| 3469 | .dest_stack_base = reg, | 3470 | .dest_stack_base = reg, |
| 3470 | }); | 3471 | }); |
| ... | @@ -4469,7 +4470,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE | ... | @@ -4469,7 +4470,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE |
| 4469 | return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg }); | 4470 | return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg }); |
| 4470 | } | 4471 | } |
| 4471 | | 4472 | |
| 4472 | try self.genInlineMemcpy(stack_offset, ty, mcv, .{ | 4473 | try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{ |
| 4473 | .source_stack_base = .rbp, | 4474 | .source_stack_base = .rbp, |
| 4474 | .dest_stack_base = .rsp, | 4475 | .dest_stack_base = .rsp, |
| 4475 | }); | 4476 | }); |
| ... | @@ -4495,7 +4496,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE | ... | @@ -4495,7 +4496,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE |
| 4495 | return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg }); | 4496 | return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg }); |
| 4496 | } | 4497 | } |
| 4497 | | 4498 | |
| 4498 | try self.genInlineMemcpy(stack_offset, ty, mcv, .{ | 4499 | try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{ |
| 4499 | .source_stack_base = .rbp, | 4500 | .source_stack_base = .rbp, |
| 4500 | .dest_stack_base = .rsp, | 4501 | .dest_stack_base = .rsp, |
| 4501 | }); | 4502 | }); |
| ... | @@ -4650,7 +4651,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl | ... | @@ -4650,7 +4651,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 4650 | return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts); | 4651 | return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts); |
| 4651 | } | 4652 | } |
| 4652 | | 4653 | |
| 4653 | try self.genInlineMemcpy(stack_offset, ty, mcv, opts); | 4654 | try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts); |
| 4654 | }, | 4655 | }, |
| 4655 | .ptr_stack_offset => { | 4656 | .ptr_stack_offset => { |
| 4656 | const reg = try self.copyToTmpRegister(ty, mcv); | 4657 | const reg = try self.copyToTmpRegister(ty, mcv); |
| ... | @@ -4667,7 +4668,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl | ... | @@ -4667,7 +4668,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl |
| 4667 | return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts); | 4668 | return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts); |
| 4668 | } | 4669 | } |
| 4669 | | 4670 | |
| 4670 | try self.genInlineMemcpy(stack_offset, ty, mcv, opts); | 4671 | try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts); |
| 4671 | }, | 4672 | }, |
| 4672 | } | 4673 | } |
| 4673 | } | 4674 | } |
| ... | @@ -4677,9 +4678,13 @@ const InlineMemcpyOpts = struct { | ... | @@ -4677,9 +4678,13 @@ const InlineMemcpyOpts = struct { |
| 4677 | dest_stack_base: ?Register = null, | 4678 | dest_stack_base: ?Register = null, |
| 4678 | }; | 4679 | }; |
| 4679 | | 4680 | |
| 4680 | fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: InlineMemcpyOpts) InnerError!void { | 4681 | fn genInlineMemcpy( |
| 4681 | const abi_size = ty.abiSize(self.target.*); | 4682 | self: *Self, |
| 4682 | | 4683 | dst_ptr: MCValue, |
| | 4684 | src_ptr: MCValue, |
| | 4685 | len: MCValue, |
| | 4686 | opts: InlineMemcpyOpts, |
| | 4687 | ) InnerError!void { |
| 4683 | // TODO this is wrong. We should check first if any of the operands is in `.rax` or `.rcx` before | 4688 | // TODO this is wrong. We should check first if any of the operands is in `.rax` or `.rcx` before |
| 4684 | // spilling. Consolidate with other TODOs regarding register allocation mechanics. | 4689 | // spilling. Consolidate with other TODOs regarding register allocation mechanics. |
| 4685 | try self.register_manager.getReg(.rax, null); | 4690 | try self.register_manager.getReg(.rax, null); |
| ... | @@ -4694,19 +4699,55 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: | ... | @@ -4694,19 +4699,55 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: |
| 4694 | if (opts.dest_stack_base) |reg| self.register_manager.freezeRegs(&.{reg}); | 4699 | if (opts.dest_stack_base) |reg| self.register_manager.freezeRegs(&.{reg}); |
| 4695 | defer if (opts.dest_stack_base) |reg| self.register_manager.unfreezeRegs(&.{reg}); | 4700 | defer if (opts.dest_stack_base) |reg| self.register_manager.unfreezeRegs(&.{reg}); |
| 4696 | | 4701 | |
| 4697 | const addr_reg = try self.register_manager.allocReg(null); | 4702 | const dst_addr_reg = try self.register_manager.allocReg(null); |
| 4698 | switch (val) { | 4703 | switch (dst_ptr) { |
| | 4704 | .got_load => unreachable, |
| 4699 | .memory, | 4705 | .memory, |
| 4700 | .direct_load, | 4706 | .direct_load, |
| 4701 | .got_load, | | |
| 4702 | => { | 4707 | => { |
| 4703 | try self.loadMemPtrIntoRegister(addr_reg, Type.usize, val); | 4708 | try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr); |
| 4704 | }, | 4709 | }, |
| 4705 | .ptr_stack_offset, .stack_offset => |off| { | 4710 | .ptr_stack_offset, .stack_offset => |off| { |
| 4706 | _ = try self.addInst(.{ | 4711 | _ = try self.addInst(.{ |
| 4707 | .tag = .lea, | 4712 | .tag = .lea, |
| 4708 | .ops = (Mir.Ops{ | 4713 | .ops = (Mir.Ops{ |
| 4709 | .reg1 = addr_reg.to64(), | 4714 | .reg1 = dst_addr_reg.to64(), |
| | 4715 | .reg2 = opts.dest_stack_base orelse .rbp, |
| | 4716 | }).encode(), |
| | 4717 | .data = .{ .imm = @bitCast(u32, -off) }, |
| | 4718 | }); |
| | 4719 | }, |
| | 4720 | .register => |reg| { |
| | 4721 | _ = try self.addInst(.{ |
| | 4722 | .tag = .mov, |
| | 4723 | .ops = (Mir.Ops{ |
| | 4724 | .reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)), |
| | 4725 | .reg2 = reg, |
| | 4726 | }).encode(), |
| | 4727 | .data = undefined, |
| | 4728 | }); |
| | 4729 | }, |
| | 4730 | else => { |
| | 4731 | return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr}); |
| | 4732 | }, |
| | 4733 | } |
| | 4734 | |
| | 4735 | self.register_manager.freezeRegs(&.{dst_addr_reg}); |
| | 4736 | defer self.register_manager.unfreezeRegs(&.{dst_addr_reg}); |
| | 4737 | |
| | 4738 | const src_addr_reg = try self.register_manager.allocReg(null); |
| | 4739 | switch (src_ptr) { |
| | 4740 | .got_load => unreachable, |
| | 4741 | .memory, |
| | 4742 | .direct_load, |
| | 4743 | => { |
| | 4744 | try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr); |
| | 4745 | }, |
| | 4746 | .ptr_stack_offset, .stack_offset => |off| { |
| | 4747 | _ = try self.addInst(.{ |
| | 4748 | .tag = .lea, |
| | 4749 | .ops = (Mir.Ops{ |
| | 4750 | .reg1 = src_addr_reg.to64(), |
| 4710 | .reg2 = opts.source_stack_base orelse .rbp, | 4751 | .reg2 = opts.source_stack_base orelse .rbp, |
| 4711 | }).encode(), | 4752 | }).encode(), |
| 4712 | .data = .{ .imm = @bitCast(u32, -off) }, | 4753 | .data = .{ .imm = @bitCast(u32, -off) }, |
| ... | @@ -4716,25 +4757,25 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: | ... | @@ -4716,25 +4757,25 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: |
| 4716 | _ = try self.addInst(.{ | 4757 | _ = try self.addInst(.{ |
| 4717 | .tag = .mov, | 4758 | .tag = .mov, |
| 4718 | .ops = (Mir.Ops{ | 4759 | .ops = (Mir.Ops{ |
| 4719 | .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)), | 4760 | .reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)), |
| 4720 | .reg2 = reg, | 4761 | .reg2 = reg, |
| 4721 | }).encode(), | 4762 | }).encode(), |
| 4722 | .data = undefined, | 4763 | .data = undefined, |
| 4723 | }); | 4764 | }); |
| 4724 | }, | 4765 | }, |
| 4725 | else => { | 4766 | else => { |
| 4726 | return self.fail("TODO implement memcpy for setting stack from {}", .{val}); | 4767 | return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr}); |
| 4727 | }, | 4768 | }, |
| 4728 | } | 4769 | } |
| 4729 | | 4770 | |
| 4730 | self.register_manager.freezeRegs(&.{addr_reg}); | 4771 | self.register_manager.freezeRegs(&.{src_addr_reg}); |
| 4731 | defer self.register_manager.unfreezeRegs(&.{addr_reg}); | 4772 | defer self.register_manager.unfreezeRegs(&.{src_addr_reg}); |
| 4732 | | 4773 | |
| 4733 | const regs = try self.register_manager.allocRegs(2, .{ null, null }); | 4774 | const regs = try self.register_manager.allocRegs(2, .{ null, null }); |
| 4734 | const count_reg = regs[0].to64(); | 4775 | const count_reg = regs[0].to64(); |
| 4735 | const tmp_reg = regs[1].to8(); | 4776 | const tmp_reg = regs[1].to8(); |
| 4736 | | 4777 | |
| 4737 | try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) }); | 4778 | try self.genSetReg(Type.usize, count_reg, len); |
| 4738 | | 4779 | |
| 4739 | // mov rcx, 0 | 4780 | // mov rcx, 0 |
| 4740 | _ = try self.addInst(.{ | 4781 | _ = try self.addInst(.{ |
| ... | @@ -4776,7 +4817,7 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: | ... | @@ -4776,7 +4817,7 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: |
| 4776 | .tag = .mov_scale_src, | 4817 | .tag = .mov_scale_src, |
| 4777 | .ops = (Mir.Ops{ | 4818 | .ops = (Mir.Ops{ |
| 4778 | .reg1 = tmp_reg.to8(), | 4819 | .reg1 = tmp_reg.to8(), |
| 4779 | .reg2 = addr_reg, | 4820 | .reg2 = src_addr_reg, |
| 4780 | }).encode(), | 4821 | }).encode(), |
| 4781 | .data = .{ .imm = 0 }, | 4822 | .data = .{ .imm = 0 }, |
| 4782 | }); | 4823 | }); |
| ... | @@ -4785,10 +4826,10 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: | ... | @@ -4785,10 +4826,10 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: |
| 4785 | _ = try self.addInst(.{ | 4826 | _ = try self.addInst(.{ |
| 4786 | .tag = .mov_scale_dst, | 4827 | .tag = .mov_scale_dst, |
| 4787 | .ops = (Mir.Ops{ | 4828 | .ops = (Mir.Ops{ |
| 4788 | .reg1 = opts.dest_stack_base orelse .rbp, | 4829 | .reg1 = dst_addr_reg, |
| 4789 | .reg2 = tmp_reg.to8(), | 4830 | .reg2 = tmp_reg.to8(), |
| 4790 | }).encode(), | 4831 | }).encode(), |
| 4791 | .data = .{ .imm = @bitCast(u32, -stack_offset) }, | 4832 | .data = .{ .imm = 0 }, |
| 4792 | }); | 4833 | }); |
| 4793 | | 4834 | |
| 4794 | // add rcx, 1 | 4835 | // add rcx, 1 |
| ... | @@ -5076,33 +5117,15 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void | ... | @@ -5076,33 +5117,15 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void |
| 5076 | .data = undefined, | 5117 | .data = undefined, |
| 5077 | }); | 5118 | }); |
| 5078 | }, | 5119 | }, |
| 5079 | .got_load, | | |
| 5080 | .direct_load, | 5120 | .direct_load, |
| 5081 | => |sym_index| { | 5121 | .got_load, |
| 5082 | const flags: u2 = switch (mcv) { | 5122 | => { |
| 5083 | .got_load => 0b00, | 5123 | try self.loadMemPtrIntoRegister(reg, Type.usize, mcv); |
| 5084 | .direct_load => 0b01, | | |
| 5085 | else => unreachable, | | |
| 5086 | }; | | |
| 5087 | _ = try self.addInst(.{ | | |
| 5088 | .tag = .lea_pie, | | |
| 5089 | .ops = (Mir.Ops{ | | |
| 5090 | .reg1 = reg, | | |
| 5091 | .flags = flags, | | |
| 5092 | }).encode(), | | |
| 5093 | .data = .{ | | |
| 5094 | .load_reloc = .{ | | |
| 5095 | .atom_index = self.mod_fn.owner_decl.link.macho.local_sym_index, | | |
| 5096 | .sym_index = sym_index, | | |
| 5097 | }, | | |
| 5098 | }, | | |
| 5099 | }); | | |
| 5100 | // MOV reg, [reg] | | |
| 5101 | _ = try self.addInst(.{ | 5124 | _ = try self.addInst(.{ |
| 5102 | .tag = .mov, | 5125 | .tag = .mov, |
| 5103 | .ops = (Mir.Ops{ | 5126 | .ops = (Mir.Ops{ |
| 5104 | .reg1 = reg, | 5127 | .reg1 = reg.to64(), |
| 5105 | .reg2 = reg, | 5128 | .reg2 = reg.to64(), |
| 5106 | .flags = 0b01, | 5129 | .flags = 0b01, |
| 5107 | }).encode(), | 5130 | }).encode(), |
| 5108 | .data = .{ .imm = 0 }, | 5131 | .data = .{ .imm = 0 }, |
| ... | @@ -5283,8 +5306,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5283,8 +5306,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index) !void { |
| 5283 | } | 5306 | } |
| 5284 | | 5307 | |
| 5285 | fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { | 5308 | fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { |
| 5286 | _ = inst; | 5309 | const pl_op = self.air.instructions.items(.data)[inst].pl_op; |
| 5287 | return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch}); | 5310 | const extra = self.air.extraData(Air.Bin, pl_op.payload).data; |
| | 5311 | |
| | 5312 | const dst_ptr = try self.resolveInst(pl_op.operand); |
| | 5313 | dst_ptr.freezeIfRegister(&self.register_manager); |
| | 5314 | defer dst_ptr.unfreezeIfRegister(&self.register_manager); |
| | 5315 | |
| | 5316 | const src_ptr = try self.resolveInst(extra.lhs); |
| | 5317 | src_ptr.freezeIfRegister(&self.register_manager); |
| | 5318 | defer src_ptr.unfreezeIfRegister(&self.register_manager); |
| | 5319 | |
| | 5320 | const len = try self.resolveInst(extra.rhs); |
| | 5321 | len.freezeIfRegister(&self.register_manager); |
| | 5322 | defer len.unfreezeIfRegister(&self.register_manager); |
| | 5323 | |
| | 5324 | log.warn("dst_ptr = {}, src_ptr = {}, len = {}", .{ dst_ptr, src_ptr, len }); |
| | 5325 | try self.genInlineMemcpy(dst_ptr, src_ptr, len, .{}); |
| | 5326 | |
| | 5327 | return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none }); |
| 5288 | } | 5328 | } |
| 5289 | | 5329 | |
| 5290 | fn airTagName(self: *Self, inst: Air.Inst.Index) !void { | 5330 | fn airTagName(self: *Self, inst: Air.Inst.Index) !void { |
Jakub Konka <kubkon@jakubkonka.com> 2022-03-01 14:04:49+01:00
src