diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 1c86b794d7526f1f19548721f0a539f0f627432f..23d99c72bd7e5efc8afb4872ba7153c0f9496c9b 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -2346,7 +2346,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{}); } - try self.genInlineMemcpy(off, elem_ty, ptr, .{}); + try self.genInlineMemcpy(dst_mcv, ptr, .{ .immediate = abi_size }, .{}); }, else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}), } @@ -2392,6 +2392,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue .got_load, .direct_load, => |sym_index| { + const abi_size = @intCast(u32, ptr_ty.abiSize(self.target.*)); const flags: u2 = switch (ptr) { .got_load => 0b00, .direct_load => 0b01, @@ -2400,7 +2401,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue _ = try self.addInst(.{ .tag = .lea_pie, .ops = (Mir.Ops{ - .reg1 = registerAlias(reg, @intCast(u32, ptr_ty.abiSize(self.target.*))), + .reg1 = registerAlias(reg, abi_size), .flags = flags, }).encode(), .data = .{ @@ -2524,7 +2525,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type return self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty); } - try self.genInlineMemcpy(0, value_ty, value, .{ + try self.genInlineMemcpy(.{ .stack_offset = 0 }, value, .{ .immediate = abi_size }, .{ .source_stack_base = .rbp, .dest_stack_base = reg.to64(), }); @@ -3464,7 +3465,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void { .data = undefined, }); try self.ret_backpatches.append(self.gpa, backpatch); - try self.genInlineMemcpy(0, elem_ty, ptr, .{ + try self.genInlineMemcpy(.{ .stack_offset = 0 }, ptr, .{ .immediate = elem_ty.abiSize(self.target.*) }, .{ .source_stack_base = .rbp, .dest_stack_base = reg, }); @@ -4412,9 +4413,12 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE const reg = try self.copyToTmpRegister(ty, mcv); return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg }); } - try self.genInlineMemset(stack_offset, ty, .{ .immediate = 0xaa }, .{ - .dest_stack_base = .rsp, - }); + try self.genInlineMemset( + .{ .stack_offset = stack_offset }, + .{ .immediate = 0xaa }, + .{ .immediate = abi_size }, + .{ .dest_stack_base = .rsp }, + ); }, .compare_flags_unsigned, .compare_flags_signed, @@ -4469,7 +4473,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg }); } - try self.genInlineMemcpy(stack_offset, ty, mcv, .{ + try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{ .source_stack_base = .rbp, .dest_stack_base = .rsp, }); @@ -4495,7 +4499,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg }); } - try self.genInlineMemcpy(stack_offset, ty, mcv, .{ + try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, .{ .source_stack_base = .rbp, .dest_stack_base = .rsp, }); @@ -4518,7 +4522,12 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl 2 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaa }, opts), 4 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaa }, opts), 8 => return self.genSetStack(ty, stack_offset, .{ .immediate = 0xaaaaaaaaaaaaaaaa }, opts), - else => return self.genInlineMemset(stack_offset, ty, .{ .immediate = 0xaa }, opts), + else => |x| return self.genInlineMemset( + .{ .stack_offset = stack_offset }, + .{ .immediate = 0xaa }, + .{ .immediate = x }, + opts, + ), } }, .compare_flags_unsigned, @@ -4650,7 +4659,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts); } - try self.genInlineMemcpy(stack_offset, ty, mcv, opts); + try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts); }, .ptr_stack_offset => { const reg = try self.copyToTmpRegister(ty, mcv); @@ -4667,7 +4676,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts); } - try self.genInlineMemcpy(stack_offset, ty, mcv, opts); + try self.genInlineMemcpy(.{ .stack_offset = stack_offset }, mcv, .{ .immediate = abi_size }, opts); }, } } @@ -4677,9 +4686,13 @@ const InlineMemcpyOpts = struct { dest_stack_base: ?Register = null, }; -fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: InlineMemcpyOpts) InnerError!void { - const abi_size = ty.abiSize(self.target.*); - +fn genInlineMemcpy( + self: *Self, + dst_ptr: MCValue, + src_ptr: MCValue, + len: MCValue, + opts: InlineMemcpyOpts, +) InnerError!void { // TODO this is wrong. We should check first if any of the operands is in `.rax` or `.rcx` before // spilling. Consolidate with other TODOs regarding register allocation mechanics. try self.register_manager.getReg(.rax, null); @@ -4694,19 +4707,55 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: if (opts.dest_stack_base) |reg| self.register_manager.freezeRegs(&.{reg}); defer if (opts.dest_stack_base) |reg| self.register_manager.unfreezeRegs(&.{reg}); - const addr_reg = try self.register_manager.allocReg(null); - switch (val) { + const dst_addr_reg = try self.register_manager.allocReg(null); + switch (dst_ptr) { .memory, + .got_load, .direct_load, + => { + try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_ptr); + }, + .ptr_stack_offset, .stack_offset => |off| { + _ = try self.addInst(.{ + .tag = .lea, + .ops = (Mir.Ops{ + .reg1 = dst_addr_reg.to64(), + .reg2 = opts.dest_stack_base orelse .rbp, + }).encode(), + .data = .{ .imm = @bitCast(u32, -off) }, + }); + }, + .register => |reg| { + _ = try self.addInst(.{ + .tag = .mov, + .ops = (Mir.Ops{ + .reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)), + .reg2 = reg, + }).encode(), + .data = undefined, + }); + }, + else => { + return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr}); + }, + } + + self.register_manager.freezeRegs(&.{dst_addr_reg}); + defer self.register_manager.unfreezeRegs(&.{dst_addr_reg}); + + const src_addr_reg = try self.register_manager.allocReg(null); + switch (src_ptr) { + .memory, .got_load, + .direct_load, => { - try self.loadMemPtrIntoRegister(addr_reg, Type.usize, val); + try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_ptr); }, .ptr_stack_offset, .stack_offset => |off| { _ = try self.addInst(.{ .tag = .lea, .ops = (Mir.Ops{ - .reg1 = addr_reg.to64(), + .reg1 = src_addr_reg.to64(), .reg2 = opts.source_stack_base orelse .rbp, }).encode(), .data = .{ .imm = @bitCast(u32, -off) }, @@ -4716,25 +4765,25 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: _ = try self.addInst(.{ .tag = .mov, .ops = (Mir.Ops{ - .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)), + .reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)), .reg2 = reg, }).encode(), .data = undefined, }); }, else => { - return self.fail("TODO implement memcpy for setting stack from {}", .{val}); + return self.fail("TODO implement memcpy for setting stack when src is {}", .{src_ptr}); }, } - self.register_manager.freezeRegs(&.{addr_reg}); - defer self.register_manager.unfreezeRegs(&.{addr_reg}); + self.register_manager.freezeRegs(&.{src_addr_reg}); + defer self.register_manager.unfreezeRegs(&.{src_addr_reg}); const regs = try self.register_manager.allocRegs(2, .{ null, null }); const count_reg = regs[0].to64(); const tmp_reg = regs[1].to8(); - try self.genSetReg(Type.u32, count_reg, .{ .immediate = @intCast(u32, abi_size) }); + try self.genSetReg(Type.usize, count_reg, len); // mov rcx, 0 _ = try self.addInst(.{ @@ -4776,7 +4825,7 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: .tag = .mov_scale_src, .ops = (Mir.Ops{ .reg1 = tmp_reg.to8(), - .reg2 = addr_reg, + .reg2 = src_addr_reg, }).encode(), .data = .{ .imm = 0 }, }); @@ -4785,10 +4834,10 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: _ = try self.addInst(.{ .tag = .mov_scale_dst, .ops = (Mir.Ops{ - .reg1 = opts.dest_stack_base orelse .rbp, + .reg1 = dst_addr_reg, .reg2 = tmp_reg.to8(), }).encode(), - .data = .{ .imm = @bitCast(u32, -stack_offset) }, + .data = .{ .imm = 0 }, }); // add rcx, 1 @@ -4831,39 +4880,52 @@ fn genInlineMemcpy(self: *Self, stack_offset: i32, ty: Type, val: MCValue, opts: fn genInlineMemset( self: *Self, - stack_offset: i32, - ty: Type, + dst_ptr: MCValue, value: MCValue, + len: MCValue, opts: InlineMemcpyOpts, ) InnerError!void { try self.register_manager.getReg(.rax, null); + self.register_manager.freezeRegs(&.{.rax}); + defer self.register_manager.unfreezeRegs(&.{.rax}); - const abi_size = ty.abiSize(self.target.*); - const negative_offset = @bitCast(u32, -stack_offset); - - // We are actually counting `abi_size` bytes; however, we reuse the index register - // as both the counter and offset scaler, hence we need to subtract one from `abi_size` - // and count until -1. - if (abi_size > math.maxInt(i32)) { - // movabs rax, abi_size - 1 - const payload = try self.addExtra(Mir.Imm64.encode(abi_size - 1)); - _ = try self.addInst(.{ - .tag = .movabs, - .ops = (Mir.Ops{ - .reg1 = .rax, - }).encode(), - .data = .{ .payload = payload }, - }); - } else { - // mov rax, abi_size - 1 - _ = try self.addInst(.{ - .tag = .mov, - .ops = (Mir.Ops{ - .reg1 = .rax, - }).encode(), - .data = .{ .imm = @truncate(u32, abi_size - 1) }, - }); + const addr_reg = try self.register_manager.allocReg(null); + switch (dst_ptr) { + .memory, + .got_load, + .direct_load, + => { + try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_ptr); + }, + .ptr_stack_offset, .stack_offset => |off| { + _ = try self.addInst(.{ + .tag = .lea, + .ops = (Mir.Ops{ + .reg1 = addr_reg.to64(), + .reg2 = opts.dest_stack_base orelse .rbp, + }).encode(), + .data = .{ .imm = @bitCast(u32, -off) }, + }); + }, + .register => |reg| { + _ = try self.addInst(.{ + .tag = .mov, + .ops = (Mir.Ops{ + .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)), + .reg2 = reg, + }).encode(), + .data = undefined, + }); + }, + else => { + return self.fail("TODO implement memcpy for setting stack when dest is {}", .{dst_ptr}); + }, } + self.register_manager.freezeRegs(&.{addr_reg}); + defer self.register_manager.unfreezeRegs(&.{addr_reg}); + + try self.genSetReg(Type.usize, .rax, len); + try self.genBinMathOpMir(.sub, Type.usize, .{ .register = .rax }, .{ .immediate = 1 }); // loop: // cmp rax, -1 @@ -4889,13 +4951,13 @@ fn genInlineMemset( } // mov byte ptr [rbp + rax + stack_offset], imm const payload = try self.addExtra(Mir.ImmPair{ - .dest_off = negative_offset, + .dest_off = 0, .operand = @truncate(u32, x), }); _ = try self.addInst(.{ .tag = .mov_mem_index_imm, .ops = (Mir.Ops{ - .reg1 = opts.dest_stack_base orelse .rbp, + .reg1 = addr_reg, }).encode(), .data = .{ .payload = payload }, }); @@ -5076,33 +5138,15 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .data = undefined, }); }, - .got_load, .direct_load, - => |sym_index| { - const flags: u2 = switch (mcv) { - .got_load => 0b00, - .direct_load => 0b01, - else => unreachable, - }; - _ = try self.addInst(.{ - .tag = .lea_pie, - .ops = (Mir.Ops{ - .reg1 = reg, - .flags = flags, - }).encode(), - .data = .{ - .load_reloc = .{ - .atom_index = self.mod_fn.owner_decl.link.macho.local_sym_index, - .sym_index = sym_index, - }, - }, - }); - // MOV reg, [reg] + .got_load, + => { + try self.loadMemPtrIntoRegister(reg, Type.usize, mcv); _ = try self.addInst(.{ .tag = .mov, .ops = (Mir.Ops{ - .reg1 = reg, - .reg2 = reg, + .reg1 = reg.to64(), + .reg2 = reg.to64(), .flags = 0b01, }).encode(), .data = .{ .imm = 0 }, @@ -5278,13 +5322,69 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr } fn airMemset(self: *Self, inst: Air.Inst.Index) !void { - _ = inst; - return self.fail("TODO implement airMemset for {}", .{self.target.cpu.arch}); + const pl_op = self.air.instructions.items(.data)[inst].pl_op; + const extra = self.air.extraData(Air.Bin, pl_op.payload).data; + + const dst_ptr = try self.resolveInst(pl_op.operand); + dst_ptr.freezeIfRegister(&self.register_manager); + defer dst_ptr.unfreezeIfRegister(&self.register_manager); + + const src_val = try self.resolveInst(extra.lhs); + src_val.freezeIfRegister(&self.register_manager); + defer src_val.unfreezeIfRegister(&self.register_manager); + + const len = try self.resolveInst(extra.rhs); + len.freezeIfRegister(&self.register_manager); + defer len.unfreezeIfRegister(&self.register_manager); + + try self.genInlineMemset(dst_ptr, src_val, len, .{}); + + return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none }); } fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { - _ = inst; - return self.fail("TODO implement airMemcpy for {}", .{self.target.cpu.arch}); + const pl_op = self.air.instructions.items(.data)[inst].pl_op; + const extra = self.air.extraData(Air.Bin, pl_op.payload).data; + + const dst_ptr = try self.resolveInst(pl_op.operand); + dst_ptr.freezeIfRegister(&self.register_manager); + defer dst_ptr.unfreezeIfRegister(&self.register_manager); + + const src_ty = self.air.typeOf(extra.lhs); + const src_ptr = try self.resolveInst(extra.lhs); + src_ptr.freezeIfRegister(&self.register_manager); + defer src_ptr.unfreezeIfRegister(&self.register_manager); + + const len = try self.resolveInst(extra.rhs); + len.freezeIfRegister(&self.register_manager); + defer len.unfreezeIfRegister(&self.register_manager); + + // TODO Is this the only condition for pointer dereference for memcpy? + const src: MCValue = blk: { + switch (src_ptr) { + .got_load, .direct_load, .memory => { + const reg = try self.register_manager.allocReg(null); + try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr); + _ = try self.addInst(.{ + .tag = .mov, + .ops = (Mir.Ops{ + .reg1 = reg, + .reg2 = reg, + .flags = 0b01, + }).encode(), + .data = .{ .imm = 0 }, + }); + break :blk MCValue{ .register = reg }; + }, + else => break :blk src_ptr, + } + }; + src.freezeIfRegister(&self.register_manager); + defer src.unfreezeIfRegister(&self.register_manager); + + try self.genInlineMemcpy(dst_ptr, src, len, .{}); + + return self.finishAir(inst, .none, .{ pl_op.operand, .none, .none }); } fn airTagName(self: *Self, inst: Air.Inst.Index) !void { diff --git a/test/behavior/basic.zig b/test/behavior/basic.zig index 939ba277a8e7b4d871d3b946ec5b7c5ce8feabfc..7b2fa0948c081ee9282b97d7c1180eeee39cffbc 100644 --- a/test/behavior/basic.zig +++ b/test/behavior/basic.zig @@ -341,7 +341,6 @@ fn f2(x: bool) []const u8 { test "memcpy and memset intrinsics" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO try testMemcpyMemset(); diff --git a/test/behavior/struct.zig b/test/behavior/struct.zig index a505f5b04bb1255b49ab3f27339b8ed00f8185e8..a5e39ad0712ada048757a97d3f78cf6c58381e2c 100644 --- a/test/behavior/struct.zig +++ b/test/behavior/struct.zig @@ -80,12 +80,11 @@ const StructWithNoFields = struct { const StructFoo = struct { a: i32, b: bool, - c: f32, + c: u64, }; test "structs" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; var foo: StructFoo = undefined;