authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-09-25 03:45:47+01:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-09-27 18:30:52-04:00
log0c476191a4ad3514b2d10ba083ba2f86fee9b223
tree6caf5db939c65417d0e48f67afb74a1d290a84bf
parent1b0bde0d8dedfef324ba16fba5aa3a24b5fb9bbd

x86_64: generate better constant memcpy code

`rep movsb` isn't usually a great idea here. This commit makes the logic which tentatively existed in `genInlineMemcpy` apply in more cases, and in particular applies it to the "new" backend logic. Put simply, all copies of 128 bytes or fewer will now attempt this path first, where---provided there is an SSE register and/or a general-purpose register available---we will lower the operation using a sequence of 32, 16, 8, 4, 2, and 1 byte copy operations. The feedback I got on this diff was "Push it to master and if it miscomps I'll revert it" so don't blame me when it explodes

1 files changed, 146 insertions(+), 81 deletions(-)

src/codegen/x86_64/CodeGen.zig+146-81
...@@ -182106,68 +182106,135 @@ fn genSetMem(...@@ -182106,68 +182106,135 @@ fn genSetMem(
182106 }182106 }
182107}182107}
182108182108
182109fn genInlineMemcpy(self: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue, opts: struct {182109fn genInlineMemcpy(cg: *CodeGen, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue, opts: struct {
182110 no_alias: bool,182110 no_alias: bool,
182111}) InnerError!void {182111}) InnerError!void {
182112 if (opts.no_alias and dst_ptr.isAddress() and src_ptr.isAddress()) switch (len) {182112 if (opts.no_alias and len == .immediate and cg.useConstMemcpyForSize(len.immediate)) {
182113 else => {},182113 if (try cg.genConstMemcpy(dst_ptr, src_ptr, len.immediate)) {
182114 .immediate => |len_imm| switch (len_imm) {182114 return;
182115 else => {},182115 }
182116 1 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {182116 }
182117 try self.asmRegisterMemory(.{ ._, .mov }, reg.to8(), try src_ptr.deref().mem(self, .{ .size = .byte }));182117
182118 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .byte }), reg.to8());182118 try cg.spillRegisters(&.{ .rsi, .rdi, .rcx });
182119 return;182119 try cg.genSetReg(.rsi, .usize, src_ptr, .{});
182120 },182120 try cg.genSetReg(.rdi, .usize, dst_ptr, .{});
182121 2 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {182121 try cg.genSetReg(.rcx, .usize, len, .{});
182122 try self.asmRegisterMemory(.{ ._, .mov }, reg.to16(), try src_ptr.deref().mem(self, .{ .size = .word }));182122 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
182123 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .word }), reg.to16());182123}
182124 return;182124
182125 },182125/// Returns `true` iff it would be efficient to use `genConstMemcpy` for a copy
182126 4 => if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {182126/// of `len` bytes instead of using `rep movsb`.
182127 try self.asmRegisterMemory(.{ ._, .mov }, reg.to32(), try src_ptr.deref().mem(self, .{ .size = .dword }));182127fn useConstMemcpyForSize(cg: *CodeGen, len: u64) bool {
182128 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .dword }), reg.to32());182128 // LLVM's threshold here is 8 load/store pairs. However, after that threshold, it typically
182129 return;182129 // calls through to `memcpy` instead of emitting `rep movsb`; if we don't have the `fsrm` CPU
182130 },182130 // feature, `rep movsb` has a high startup cost, so is inefficient for small copies. Since we
182131 8 => if (self.target.cpu.arch == .x86_64) {182131 // don't currently call compiler_rt in the case of longer copies, we use a higher threshold if
182132 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |reg| {182132 // we don't have the `fsrm` feature.
182133 try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), try src_ptr.deref().mem(self, .{ .size = .qword }));182133 return switch (len) {
182134 try self.asmMemoryRegister(.{ ._, .mov }, try dst_ptr.deref().mem(self, .{ .size = .qword }), reg.to64());182134 0...128 => true,
182135 return;182135 129...256 => cg.hasFeature(.avx) or !cg.hasFeature(.fsrm), // if we have AVX, each pair is 32 bytes
182136 }182136 257...512 => !cg.hasFeature(.fsrm), // the threshold should probably be higher, but I'm being cautious
182137 },182137 else => false,
182138 16 => if (self.hasFeature(.avx)) {
182139 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
182140 try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
182141 try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
182142 return;
182143 }
182144 } else if (self.hasFeature(.sse2)) {
182145 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
182146 try self.asmRegisterMemory(.{ ._dqu, .mov }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
182147 try self.asmMemoryRegister(.{ ._dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
182148 return;
182149 }
182150 } else if (self.hasFeature(.sse)) {
182151 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
182152 try self.asmRegisterMemory(.{ ._ps, .movu }, reg.to128(), try src_ptr.deref().mem(self, .{ .size = .xword }));
182153 try self.asmMemoryRegister(.{ ._ps, .movu }, try dst_ptr.deref().mem(self, .{ .size = .xword }), reg.to128());
182154 return;
182155 }
182156 },
182157 32 => if (self.hasFeature(.avx)) {
182158 if (self.register_manager.tryAllocReg(null, abi.RegisterClass.sse)) |reg| {
182159 try self.asmRegisterMemory(.{ .v_dqu, .mov }, reg.to256(), try src_ptr.deref().mem(self, .{ .size = .yword }));
182160 try self.asmMemoryRegister(.{ .v_dqu, .mov }, try dst_ptr.deref().mem(self, .{ .size = .yword }), reg.to256());
182161 return;
182162 }
182163 },
182164 },
182165 };182138 };
182166 try self.spillRegisters(&.{ .rsi, .rdi, .rcx });182139}
182167 try self.genSetReg(.rsi, .usize, src_ptr, .{});182140/// If we can trivially copy `len` bytes from `orig_src_ptr` to `dst_ptr` using simple loads and
182168 try self.genSetReg(.rdi, .usize, dst_ptr, .{});182141/// stored through already-free register[s] (i.e. without spilling anything), emits that code and
182169 try self.genSetReg(.rcx, .usize, len, .{});182142/// returns `true`. Otherwise, returns `false`, and the caller must lower the operation themselves,
182170 try self.asmOpOnly(.{ .@"rep _sb", .mov });182143/// for instance with `rep movsb`.
182144fn genConstMemcpy(
182145 cg: *CodeGen,
182146 dst_ptr: MCValue,
182147 orig_src_ptr: MCValue,
182148 len: u64,
182149) !bool {
182150 if (!dst_ptr.isAddress()) return false;
182151
182152 const src_reg: ?Register = r: {
182153 if (orig_src_ptr.isAddress()) break :r null;
182154 if (orig_src_ptr == .lea_uav or orig_src_ptr == .lea_nav) {
182155 // To "hack around linker relocation bugs", a `lea_uav` isn't considered an address, but
182156 // we want to avoid pessimising that case because it's common (e.g. returning constant
182157 // values over 8 bytes). So if there's a register free, load the source address into it.
182158 if (cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |src_reg| {
182159 // We'll actually do the load a little later, in case another register alloc fails.
182160 break :r src_reg;
182161 }
182162 }
182163 return false;
182164 };
182165
182166 // Prevent `src_reg` from aliasing `gp_reg` below.
182167 const src_lock: ?RegisterLock = if (src_reg) |r| cg.register_manager.lockReg(r) else null;
182168 defer if (src_lock) |l| cg.register_manager.unlockReg(l);
182169
182170 const want_sse_reg = len >= 16 and cg.hasFeature(.sse);
182171 const sse_reg: ?Register = if (want_sse_reg) r: {
182172 break :r cg.register_manager.tryAllocReg(null, abi.RegisterClass.sse);
182173 } else null;
182174
182175 const need_gp_reg = len % 16 != 0 or sse_reg == null;
182176 const gp_reg: Register = if (need_gp_reg) r: {
182177 break :r cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp) orelse return false;
182178 } else undefined;
182179
182180 const src_ptr: MCValue = src_ptr: {
182181 const reg = src_reg orelse break :src_ptr orig_src_ptr;
182182 try cg.asmRegisterMemory(.{ ._, .lea }, reg.to64(), switch (orig_src_ptr) {
182183 .lea_uav => |uav| .{ .base = .{ .uav = uav } },
182184 .lea_nav => |nav| .{ .base = .{ .nav = nav } },
182185 else => unreachable,
182186 });
182187 break :src_ptr .{ .register = reg.to64() };
182188 };
182189
182190 var offset: u64 = 0;
182191
182192 if (sse_reg) |r| {
182193 if (cg.hasFeature(.avx)) {
182194 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, r, .yword);
182195 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, r, .xword);
182196 } else if (cg.hasFeature(.sse2)) {
182197 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._dqu, .mov }, r, .xword);
182198 } else if (cg.hasFeature(.sse)) {
182199 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._ps, .movu }, r, .xword);
182200 }
182201 }
182202
182203 if (need_gp_reg) {
182204 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .qword);
182205 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .dword);
182206 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .word);
182207 try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .byte);
182208 }
182209
182210 assert(offset == len);
182211
182212 return true;
182213}
182214fn memcpyPart(
182215 cg: *CodeGen,
182216 dst_ptr: MCValue,
182217 src_ptr: MCValue,
182218 len: u64,
182219 offset: *u64,
182220 tag: Mir.Inst.FixedTag,
182221 temp_reg: Register,
182222 size: Memory.Size,
182223) !void {
182224 const bytes_len = @divExact(size.bitSize(cg.target), 8);
182225 while (len - offset.* >= bytes_len) {
182226 try cg.asmRegisterMemory(
182227 tag,
182228 temp_reg.toSize(size, cg.target),
182229 try src_ptr.deref().mem(cg, .{ .size = size, .disp = @intCast(offset.*) }),
182230 );
182231 try cg.asmMemoryRegister(
182232 tag,
182233 try dst_ptr.deref().mem(cg, .{ .size = size, .disp = @intCast(offset.*) }),
182234 temp_reg.toSize(size, cg.target),
182235 );
182236 offset.* += bytes_len;
182237 }
182171}182238}
182172182239
182173fn genInlineMemset(182240fn genInlineMemset(
...@@ -186834,10 +186901,8 @@ const Temp = struct {...@@ -186834,10 +186901,8 @@ const Temp = struct {
186834 },186901 },
186835 .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => {186902 .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => {
186836 var val_ptr = try cg.tempInit(.usize, val_mcv.address());186903 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
186837 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });186904 try val_ptr.memcpy(ptr, val_ty.abiSize(cg.pt.zcu), cg);
186838 try val_ptr.memcpy(ptr, &len, cg);
186839 try val_ptr.die(cg);186905 try val_ptr.die(cg);
186840 try len.die(cg);
186841 },186906 },
186842 }186907 }
186843 return val;186908 return val;
...@@ -186939,10 +187004,8 @@ const Temp = struct {...@@ -186939,10 +187004,8 @@ const Temp = struct {
186939 .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => continue :val_to_gpr,187004 .lea_frame, .lea_nav, .lea_uav, .lea_lazy_sym => continue :val_to_gpr,
186940 .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => {187005 .memory, .indirect, .load_frame, .load_nav, .load_uav, .load_lazy_sym => {
186941 var val_ptr = try cg.tempInit(.usize, val_mcv.address());187006 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
186942 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });187007 try ptr.memcpy(&val_ptr, val_ty.abiSize(cg.pt.zcu), cg);
186943 try ptr.memcpy(&val_ptr, &len, cg);
186944 try val_ptr.die(cg);187008 try val_ptr.die(cg);
186945 try len.die(cg);
186946 },187009 },
186947 }187010 }
186948 break;187011 break;
...@@ -186981,11 +187044,9 @@ const Temp = struct {...@@ -186981,11 +187044,9 @@ const Temp = struct {
186981 var val_ptr = try cg.tempInit(.usize, val_mcv.address());187044 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
186982 var src_ptr =187045 var src_ptr =
186983 try cg.tempInit(.usize, src.tracking(cg).short.address().offset(opts.disp));187046 try cg.tempInit(.usize, src.tracking(cg).short.address().offset(opts.disp));
186984 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });187047 try val_ptr.memcpy(&src_ptr, val_ty.abiSize(cg.pt.zcu), cg);
186985 try val_ptr.memcpy(&src_ptr, &len, cg);
186986 try val_ptr.die(cg);187048 try val_ptr.die(cg);
186987 try src_ptr.die(cg);187049 try src_ptr.die(cg);
186988 try len.die(cg);
186989 },187050 },
186990 }187051 }
186991 }187052 }
...@@ -187076,11 +187137,9 @@ const Temp = struct {...@@ -187076,11 +187137,9 @@ const Temp = struct {
187076 var dst_ptr =187137 var dst_ptr =
187077 try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));187138 try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
187078 var val_ptr = try cg.tempInit(.usize, val_mcv.address());187139 var val_ptr = try cg.tempInit(.usize, val_mcv.address());
187079 var len = try cg.tempInit(.usize, .{ .immediate = val_ty.abiSize(cg.pt.zcu) });187140 try dst_ptr.memcpy(&val_ptr, val_ty.abiSize(cg.pt.zcu), cg);
187080 try dst_ptr.memcpy(&val_ptr, &len, cg);
187081 try dst_ptr.die(cg);187141 try dst_ptr.die(cg);
187082 try val_ptr.die(cg);187142 try val_ptr.die(cg);
187083 try len.die(cg);
187084 },187143 },
187085 }187144 }
187086 break;187145 break;
...@@ -187182,10 +187241,8 @@ const Temp = struct {...@@ -187182,10 +187241,8 @@ const Temp = struct {
187182 try ptr.toOffset(deferred_disp, cg);187241 try ptr.toOffset(deferred_disp, cg);
187183 deferred_disp = 0;187242 deferred_disp = 0;
187184 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });187243 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
187185 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });187244 try ptr.memcpy(&src_ptr, src_abi_size, cg);
187186 try ptr.memcpy(&src_ptr, &len, cg);
187187 try src_ptr.die(cg);187245 try src_ptr.die(cg);
187188 try len.die(cg);
187189 }187246 }
187190 part_disp += part_size;187247 part_disp += part_size;
187191 deferred_disp += part_size;187248 deferred_disp += part_size;
...@@ -187224,11 +187281,9 @@ const Temp = struct {...@@ -187224,11 +187281,9 @@ const Temp = struct {
187224 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address());187281 var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address());
187225 try dst_ptr.toOffset(disp, cg);187282 try dst_ptr.toOffset(disp, cg);
187226 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });187283 var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
187227 var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });187284 try dst_ptr.memcpy(&src_ptr, src_abi_size, cg);
187228 try dst_ptr.memcpy(&src_ptr, &len, cg);
187229 try dst_ptr.die(cg);187285 try dst_ptr.die(cg);
187230 try src_ptr.die(cg);187286 try src_ptr.die(cg);
187231 try len.die(cg);
187232 }187287 }
187233 }187288 }
187234187289
...@@ -187273,11 +187328,21 @@ const Temp = struct {...@@ -187273,11 +187328,21 @@ const Temp = struct {
187273 assert(next_class_index == classes.len);187328 assert(next_class_index == classes.len);
187274 }187329 }
187275187330
187276 fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) InnerError!void {187331 fn memcpy(dst: *Temp, src: *Temp, len: u64, cg: *CodeGen) InnerError!void {
187277 while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| {187332 if (cg.useConstMemcpyForSize(len)) {
187278 if (try temp.toReg(reg, cg)) break;187333 while (try dst.toLea(cg) or try src.toLea(cg)) {} // `genConstMemcpy` wants these values to be address operands
187279 } else break;187334 if (try cg.genConstMemcpy(dst.tracking(cg).short, src.tracking(cg).short, len)) {
187335 return;
187336 }
187337 }
187338 var temp = try cg.tempAllocReg(.usize, abi.RegisterClass.gp);
187339 while (try dst.toReg(.rdi, cg) or
187340 try src.toReg(.rsi, cg) or
187341 try temp.toReg(.rcx, cg))
187342 {}
187343 try cg.asmRegisterImmediate(.{ ._, .mov }, .rcx, .{ .unsigned = len });
187280 try cg.asmOpOnly(.{ .@"rep _sb", .mov });187344 try cg.asmOpOnly(.{ .@"rep _sb", .mov });
187345 try temp.die(cg);
187281 }187346 }
187282187347
187283 fn memset(dst: *Temp, val: *Temp, len: *Temp, cg: *CodeGen) InnerError!void {187348 fn memset(dst: *Temp, val: *Temp, len: *Temp, cg: *CodeGen) InnerError!void {