authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-23 20:52:43-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-25 11:23:41-07:00
log881e931ee1758dee469c3f87700413bb321c9f43
treece3f2e3982362f9fcde2edf3719a3b40736e3bb6
parent7c56145a7651124dafdc87176ba85c814cd97d2d

x86_64 backend: implement `@memset` for element ABI size > 1

* make memset and memset_safe guarantee that if the length is comptime-known then it will be nonzero.

4 files changed, 74 insertions(+), 25 deletions(-)

src/Air.zig+5
...@@ -641,6 +641,8 @@ pub const Inst = struct {...@@ -641,6 +641,8 @@ pub const Inst = struct {
641 /// The element value may be undefined, in which case the destination641 /// The element value may be undefined, in which case the destination
642 /// memory region has undefined bytes after this function executes. In642 /// memory region has undefined bytes after this function executes. In
643 /// such case ignoring this instruction is legal lowering.643 /// such case ignoring this instruction is legal lowering.
644 /// If the length is compile-time known (due to the destination being a
645 /// pointer-to-array), then it is guaranteed to be greater than zero.
644 memset,646 memset,
645 /// Same as `memset`, except if the element value is undefined, the memory region647 /// Same as `memset`, except if the element value is undefined, the memory region
646 /// should be filled with 0xaa bytes, and any other safety metadata such as Valgrind648 /// should be filled with 0xaa bytes, and any other safety metadata such as Valgrind
...@@ -654,6 +656,9 @@ pub const Inst = struct {...@@ -654,6 +656,9 @@ pub const Inst = struct {
654 /// The two memory regions must not overlap.656 /// The two memory regions must not overlap.
655 /// Result type is always void.657 /// Result type is always void.
656 /// Uses the `bin_op` field. LHS is the dest slice. RHS is the source pointer.658 /// Uses the `bin_op` field. LHS is the dest slice. RHS is the source pointer.
659 /// If the length is compile-time known (due to the destination or
660 /// source being a pointer-to-array), then it is guaranteed to be
661 /// greater than zero.
657 memcpy,662 memcpy,
658663
659 /// Uses the `ty_pl` field with payload `Cmpxchg`.664 /// Uses the `ty_pl` field with payload `Cmpxchg`.
src/Sema.zig+16-7
...@@ -21918,8 +21918,6 @@ fn zirMemcpy(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void...@@ -21918,8 +21918,6 @@ fn zirMemcpy(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void
21918 } else break :rs src_src;21918 } else break :rs src_src;
21919 } else dest_src;21919 } else dest_src;
2192021920
21921 try sema.requireRuntimeBlock(block, src, runtime_src);
21922
21923 const dest_ty = sema.typeOf(dest_ptr);21921 const dest_ty = sema.typeOf(dest_ptr);
21924 const src_ty = sema.typeOf(src_ptr);21922 const src_ty = sema.typeOf(src_ptr);
2192521923
...@@ -21946,10 +21944,16 @@ fn zirMemcpy(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void...@@ -21946,10 +21944,16 @@ fn zirMemcpy(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void
21946 var new_src_ptr = src_ptr;21944 var new_src_ptr = src_ptr;
21947 if (len_val) |val| {21945 if (len_val) |val| {
21948 const len = val.toUnsignedInt(target);21946 const len = val.toUnsignedInt(target);
21947 if (len == 0) {
21948 // This AIR instruction guarantees length > 0 if it is comptime-known.
21949 return;
21950 }
21949 new_dest_ptr = try upgradeToArrayPtr(sema, block, dest_ptr, len);21951 new_dest_ptr = try upgradeToArrayPtr(sema, block, dest_ptr, len);
21950 new_src_ptr = try upgradeToArrayPtr(sema, block, src_ptr, len);21952 new_src_ptr = try upgradeToArrayPtr(sema, block, src_ptr, len);
21951 }21953 }
2195221954
21955 try sema.requireRuntimeBlock(block, src, runtime_src);
21956
21953 // Aliasing safety check.21957 // Aliasing safety check.
21954 if (block.wantSafety()) {21958 if (block.wantSafety()) {
21955 const dest_int = try block.addUnOp(.ptrtoint, new_dest_ptr);21959 const dest_int = try block.addUnOp(.ptrtoint, new_dest_ptr);
...@@ -21995,13 +21999,18 @@ fn zirMemset(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void...@@ -21995,13 +21999,18 @@ fn zirMemset(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!void
21995 const target = sema.mod.getTarget();21999 const target = sema.mod.getTarget();
2199622000
21997 const runtime_src = if (try sema.resolveDefinedValue(block, dest_src, dest_ptr)) |ptr_val| rs: {22001 const runtime_src = if (try sema.resolveDefinedValue(block, dest_src, dest_ptr)) |ptr_val| rs: {
22002 const len_air_ref = try sema.fieldVal(block, src, dest_ptr, "len", dest_src);
22003 const len_val = (try sema.resolveDefinedValue(block, dest_src, len_air_ref)) orelse
22004 break :rs dest_src;
22005 const len_u64 = (try len_val.getUnsignedIntAdvanced(target, sema)).?;
22006 const len = try sema.usizeCast(block, dest_src, len_u64);
22007 if (len == 0) {
22008 // This AIR instruction guarantees length > 0 if it is comptime-known.
22009 return;
22010 }
22011
21998 if (!ptr_val.isComptimeMutablePtr()) break :rs dest_src;22012 if (!ptr_val.isComptimeMutablePtr()) break :rs dest_src;
21999 if (try sema.resolveMaybeUndefVal(uncoerced_elem)) |_| {22013 if (try sema.resolveMaybeUndefVal(uncoerced_elem)) |_| {
22000 const len_air_ref = try sema.fieldVal(block, src, dest_ptr, "len", dest_src);
22001 const len_val = (try sema.resolveDefinedValue(block, dest_src, len_air_ref)) orelse
22002 break :rs dest_src;
22003 const len_u64 = (try len_val.getUnsignedIntAdvanced(target, sema)).?;
22004 const len = try sema.usizeCast(block, dest_src, len_u64);
22005 for (0..len) |i| {22014 for (0..len) |i| {
22006 const elem_index = try sema.addIntUnsigned(Type.usize, i);22015 const elem_index = try sema.addIntUnsigned(Type.usize, i);
22007 const elem_ptr = try sema.elemPtr(22016 const elem_ptr = try sema.elemPtr(
src/arch/x86_64/CodeGen.zig+53-14
...@@ -8175,23 +8175,62 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {...@@ -8175,23 +8175,62 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void {
8175 };8175 };
8176 defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock);8176 defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock);
81778177
8178 if (elem_ty.abiSize(self.target.*) != 1) {8178 if (elem_ty.abiSize(self.target.*) == 1) {
8179 return self.fail("TODO implement airMemset when element ABI size > 1", .{});8179 const len = switch (dst_ptr_ty.ptrSize()) {
8180 // TODO: this only handles slices stored in the stack
8181 .Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),
8182 .One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),
8183 .C, .Many => unreachable,
8184 };
8185 const len_lock: ?RegisterLock = switch (len) {
8186 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
8187 else => null,
8188 };
8189 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);
8190
8191 // TODO: dst_ptr could be a slice rather than raw pointer
8192 try self.genInlineMemset(dst_ptr, src_val, len, .{});
8193 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
8180 }8194 }
81818195
8182 const len = switch (dst_ptr_ty.ptrSize()) {8196 // Store the first element, and then rely on memcpy copying forwards.
8183 .Slice => @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 }),8197 // Length zero requires a runtime check - so we handle arrays specially
8184 .One => @as(MCValue, .{ .immediate = dst_ptr_ty.childType().arrayLen() }),8198 // here to elide it.
8185 .C, .Many => unreachable,8199 switch (dst_ptr_ty.ptrSize()) {
8186 };8200 .Slice => {
8187 const len_lock: ?RegisterLock = switch (len) {8201 // TODO: this only handles slices stored in the stack
8188 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),8202 const ptr = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 0 });
8189 else => null,8203 const len = @as(MCValue, .{ .stack_offset = dst_ptr.stack_offset - 8 });
8190 };8204 _ = ptr;
8191 defer if (len_lock) |lock| self.register_manager.unlockReg(lock);8205 _ = len;
8206 return self.fail("TODO implement airMemset for x86_64 with ABI size > 1 using a slice", .{});
8207 },
8208 .One => {
8209 const len = dst_ptr_ty.childType().arrayLen();
8210 assert(len != 0); // prevented by Sema
8211 try self.store(dst_ptr, src_val, dst_ptr_ty, elem_ty);
8212
8213 const second_elem_ptr_reg = try self.register_manager.allocReg(null, gp);
8214 const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg };
8215 const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg);
8216 defer self.register_manager.unlockReg(second_elem_ptr_lock);
81928217
8193 // TODO: dst_ptr could be a slice rather than raw pointer8218 const elem_abi_size = @intCast(u31, elem_ty.abiSize(self.target.*));
8194 try self.genInlineMemset(dst_ptr, src_val, len, .{});8219
8220 try self.asmRegisterMemory(
8221 .lea,
8222 second_elem_ptr_reg,
8223 Memory.sib(.qword, .{
8224 .base = try self.copyToTmpRegister(Type.usize, dst_ptr),
8225 .disp = elem_abi_size,
8226 }),
8227 );
8228
8229 const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) };
8230 try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy, .{});
8231 },
8232 .C, .Many => unreachable,
8233 }
81958234
8196 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });8235 return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none });
8197}8236}
test/behavior/basic.zig-4
...@@ -361,10 +361,6 @@ test "@memset on array pointers" {...@@ -361,10 +361,6 @@ test "@memset on array pointers" {
361 // TODO: implement memset when element ABI size > 1361 // TODO: implement memset when element ABI size > 1
362 return error.SkipZigTest;362 return error.SkipZigTest;
363 }363 }
364 if (builtin.zig_backend == .stage2_x86_64) {
365 // TODO: implement memset when element ABI size > 1
366 return error.SkipZigTest;
367 }
368364
369 try testMemsetArray();365 try testMemsetArray();
370 try comptime testMemsetArray();366 try comptime testMemsetArray();