| ... | ... | @@ -545,15 +545,18 @@ pub const MCValue = union(enum) { |
| 545 | 545 | .lea_lazy_sym, |
| 546 | 546 | .lea_extern_func, |
| 547 | 547 | => unreachable, |
| 548 | | .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{ |
| 549 | | .base = .{ .reg = .ds }, |
| 550 | | .mod = .{ .rm = .{ |
| 551 | | .size = mod_rm.size, |
| 552 | | .index = mod_rm.index, |
| 553 | | .scale = mod_rm.scale, |
| 554 | | .disp = small_addr + mod_rm.disp, |
| 555 | | } }, |
| 556 | | } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } }, |
| 548 | .memory => |base_addr| { |
| 549 | const addr = @as(i64, @bitCast(base_addr)) +% mod_rm.disp; |
| 550 | return .{ |
| 551 | .base = .{ .reg = .ds }, |
| 552 | .mod = if (std.math.cast(i32, addr)) |small_addr| .{ .rm = .{ |
| 553 | .size = mod_rm.size, |
| 554 | .index = mod_rm.index, |
| 555 | .scale = mod_rm.scale, |
| 556 | .disp = small_addr, |
| 557 | } } else .{ .off = @bitCast(addr) }, |
| 558 | }; |
| 559 | }, |
| 557 | 560 | .indirect => |reg_off| .{ |
| 558 | 561 | .base = .{ .reg = reg_off.reg.toSize(.ptr, function.target) }, |
| 559 | 562 | .mod = .{ .rm = .{ |
| ... | ... | @@ -89205,19 +89208,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89205 | 89208 | try cg.spillEflagsIfOccupied(); |
| 89206 | 89209 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89207 | 89210 | while (try ops[0].toBase(false, cg)) {} |
| 89208 | | try cg.asmMemoryImmediate( |
| 89209 | | .{ ._, .cmp }, |
| 89210 | | try ops[0].tracking(cg).short.mem(cg, .{ |
| 89211 | | .size = if (!opt_repr_is_pl) |
| 89212 | | .byte |
| 89213 | | else if (opt_child_ty.isSlice(zcu)) |
| 89214 | | .ptr |
| 89215 | | else |
| 89216 | | .fromSize(opt_child_abi_size), |
| 89217 | | .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, |
| 89218 | | }), |
| 89219 | | .u(0), |
| 89220 | | ); |
| 89211 | const mem_size: Memory.Size = if (!opt_repr_is_pl) |
| 89212 | .byte |
| 89213 | else if (opt_child_ty.isSlice(zcu)) |
| 89214 | .ptr |
| 89215 | else |
| 89216 | .fromSize(opt_child_abi_size); |
| 89217 | const dst_mem = try ops[0].tracking(cg).short.mem(cg, .{ |
| 89218 | .size = mem_size, |
| 89219 | .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, |
| 89220 | }); |
| 89221 | switch (dst_mem.mod) { |
| 89222 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89223 | .off => { |
| 89224 | try cg.register_manager.getKnownReg(.rax, inst); |
| 89225 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| 89226 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| 89227 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| 89228 | }, |
| 89229 | } |
| 89221 | 89230 | const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 89222 | 89231 | try is_null.finish(inst, &.{un_op}, &ops, cg); |
| 89223 | 89232 | }, |
| ... | ... | @@ -89230,19 +89239,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89230 | 89239 | try cg.spillEflagsIfOccupied(); |
| 89231 | 89240 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89232 | 89241 | while (try ops[0].toBase(false, cg)) {} |
| 89233 | | try cg.asmMemoryImmediate( |
| 89234 | | .{ ._, .cmp }, |
| 89235 | | try ops[0].tracking(cg).short.mem(cg, .{ |
| 89236 | | .size = if (!opt_repr_is_pl) |
| 89237 | | .byte |
| 89238 | | else if (opt_child_ty.isSlice(zcu)) |
| 89239 | | .ptr |
| 89240 | | else |
| 89241 | | .fromSize(opt_child_abi_size), |
| 89242 | | .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, |
| 89243 | | }), |
| 89244 | | .u(0), |
| 89245 | | ); |
| 89242 | const mem_size: Memory.Size = if (!opt_repr_is_pl) |
| 89243 | .byte |
| 89244 | else if (opt_child_ty.isSlice(zcu)) |
| 89245 | .ptr |
| 89246 | else |
| 89247 | .fromSize(opt_child_abi_size); |
| 89248 | const dst_mem = try ops[0].tracking(cg).short.mem(cg, .{ |
| 89249 | .size = mem_size, |
| 89250 | .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, |
| 89251 | }); |
| 89252 | switch (dst_mem.mod) { |
| 89253 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89254 | .off => { |
| 89255 | try cg.register_manager.getKnownReg(.rax, inst); |
| 89256 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| 89257 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| 89258 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| 89259 | }, |
| 89260 | } |
| 89246 | 89261 | const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 89247 | 89262 | try is_non_null.finish(inst, &.{un_op}, &ops, cg); |
| 89248 | 89263 | }, |
| ... | ... | @@ -89256,16 +89271,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89256 | 89271 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89257 | 89272 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); |
| 89258 | 89273 | while (try ops[0].toLea(cg)) {} |
| 89259 | | try cg.asmMemoryImmediate( |
| 89260 | | .{ ._, .cmp }, |
| 89261 | | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) |
| 89262 | | .byte |
| 89263 | | else if (opt_child_ty.isSlice(zcu)) |
| 89264 | | .ptr |
| 89265 | | else |
| 89266 | | .fromSize(opt_child_abi_size) }), |
| 89267 | | .u(0), |
| 89268 | | ); |
| 89274 | const mem_size: Memory.Size = if (!opt_repr_is_pl) |
| 89275 | .byte |
| 89276 | else if (opt_child_ty.isSlice(zcu)) |
| 89277 | .ptr |
| 89278 | else |
| 89279 | .fromSize(opt_child_abi_size); |
| 89280 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = mem_size }); |
| 89281 | switch (dst_mem.mod) { |
| 89282 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89283 | .off => { |
| 89284 | try cg.register_manager.getKnownReg(.rax, inst); |
| 89285 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| 89286 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| 89287 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| 89288 | }, |
| 89289 | } |
| 89269 | 89290 | const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 89270 | 89291 | try is_null.finish(inst, &.{un_op}, &ops, cg); |
| 89271 | 89292 | }, |
| ... | ... | @@ -89279,16 +89300,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89279 | 89300 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89280 | 89301 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); |
| 89281 | 89302 | while (try ops[0].toLea(cg)) {} |
| 89282 | | try cg.asmMemoryImmediate( |
| 89283 | | .{ ._, .cmp }, |
| 89284 | | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) |
| 89285 | | .byte |
| 89286 | | else if (opt_child_ty.isSlice(zcu)) |
| 89287 | | .ptr |
| 89288 | | else |
| 89289 | | .fromSize(opt_child_abi_size) }), |
| 89290 | | .u(0), |
| 89291 | | ); |
| 89303 | const mem_size: Memory.Size = if (!opt_repr_is_pl) |
| 89304 | .byte |
| 89305 | else if (opt_child_ty.isSlice(zcu)) |
| 89306 | .ptr |
| 89307 | else |
| 89308 | .fromSize(opt_child_abi_size); |
| 89309 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = mem_size }); |
| 89310 | switch (dst_mem.mod) { |
| 89311 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89312 | .off => { |
| 89313 | try cg.register_manager.getKnownReg(.rax, inst); |
| 89314 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| 89315 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| 89316 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| 89317 | }, |
| 89318 | } |
| 89292 | 89319 | const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 89293 | 89320 | try is_non_null.finish(inst, &.{un_op}, &ops, cg); |
| 89294 | 89321 | }, |
| ... | ... | @@ -89301,10 +89328,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89301 | 89328 | try cg.spillEflagsIfOccupied(); |
| 89302 | 89329 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89303 | 89330 | while (try ops[0].toBase(false, cg)) {} |
| 89304 | | try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.mem(cg, .{ |
| 89305 | | .size = cg.memSize(eu_err_ty, .general_purpose), |
| 89331 | const mem_size: Memory.Size = cg.memSize(eu_err_ty, .general_purpose); |
| 89332 | const dst_mem = try ops[0].tracking(cg).short.mem(cg, .{ |
| 89333 | .size = mem_size, |
| 89306 | 89334 | .disp = eu_err_off, |
| 89307 | | }), .u(0)); |
| 89335 | }); |
| 89336 | switch (dst_mem.mod) { |
| 89337 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89338 | .off => { |
| 89339 | try cg.register_manager.getKnownReg(.rax, inst); |
| 89340 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| 89341 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| 89342 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| 89343 | }, |
| 89344 | } |
| 89308 | 89345 | const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 89309 | 89346 | try is_err.finish(inst, &.{un_op}, &ops, cg); |
| 89310 | 89347 | }, |
| ... | ... | @@ -89317,10 +89354,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89317 | 89354 | try cg.spillEflagsIfOccupied(); |
| 89318 | 89355 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89319 | 89356 | while (try ops[0].toBase(false, cg)) {} |
| 89320 | | try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.mem(cg, .{ |
| 89321 | | .size = cg.memSize(eu_err_ty, .general_purpose), |
| 89357 | const mem_size = cg.memSize(eu_err_ty, .general_purpose); |
| 89358 | const dst_mem = try ops[0].tracking(cg).short.mem(cg, .{ |
| 89359 | .size = mem_size, |
| 89322 | 89360 | .disp = eu_err_off, |
| 89323 | | }), .u(0)); |
| 89361 | }); |
| 89362 | switch (dst_mem.mod) { |
| 89363 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89364 | .off => { |
| 89365 | try cg.register_manager.getKnownReg(.rax, inst); |
| 89366 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| 89367 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| 89368 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| 89369 | }, |
| 89370 | } |
| 89324 | 89371 | const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 89325 | 89372 | try is_non_err.finish(inst, &.{un_op}, &ops, cg); |
| 89326 | 89373 | }, |
| ... | ... | @@ -89334,9 +89381,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89334 | 89381 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89335 | 89382 | try ops[0].toOffset(eu_err_off, cg); |
| 89336 | 89383 | while (try ops[0].toLea(cg)) {} |
| 89337 | | try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.deref().mem(cg, .{ |
| 89338 | | .size = cg.memSize(eu_err_ty, .general_purpose), |
| 89339 | | }), .u(0)); |
| 89384 | const mem_size = cg.memSize(eu_err_ty, .general_purpose); |
| 89385 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = mem_size }); |
| 89386 | switch (dst_mem.mod) { |
| 89387 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89388 | .off => { |
| 89389 | try cg.register_manager.getKnownReg(.rax, inst); |
| 89390 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| 89391 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| 89392 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| 89393 | }, |
| 89394 | } |
| 89340 | 89395 | const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 89341 | 89396 | try is_err.finish(inst, &.{un_op}, &ops, cg); |
| 89342 | 89397 | }, |
| ... | ... | @@ -89350,9 +89405,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89350 | 89405 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89351 | 89406 | try ops[0].toOffset(eu_err_off, cg); |
| 89352 | 89407 | while (try ops[0].toLea(cg)) {} |
| 89353 | | try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.deref().mem(cg, .{ |
| 89354 | | .size = cg.memSize(eu_err_ty, .general_purpose), |
| 89355 | | }), .u(0)); |
| 89408 | const mem_size = cg.memSize(eu_err_ty, .general_purpose); |
| 89409 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = mem_size }); |
| 89410 | switch (dst_mem.mod) { |
| 89411 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89412 | .off => { |
| 89413 | try cg.register_manager.getKnownReg(.rax, inst); |
| 89414 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| 89415 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| 89416 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| 89417 | }, |
| 89418 | } |
| 89356 | 89419 | const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 89357 | 89420 | try is_non_err.finish(inst, &.{un_op}, &ops, cg); |
| 89358 | 89421 | }, |
| ... | ... | @@ -104194,11 +104257,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 104194 | 104257 | rhs_reg, |
| 104195 | 104258 | .u(elem_size), |
| 104196 | 104259 | ); |
| 104197 | | try cg.asmRegisterMemory( |
| 104198 | | .{ ._, .lea }, |
| 104199 | | base_reg, |
| 104200 | | try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }), |
| 104201 | | ); |
| 104260 | const src_mem = try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }); |
| 104261 | switch (src_mem.mod) { |
| 104262 | .rm => try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, src_mem), |
| 104263 | .off => |src_addr| { |
| 104264 | try cg.asmRegisterImmediate(.{ ._, .mov }, base_reg, .u(src_addr)); |
| 104265 | try cg.asmRegisterRegister(.{ ._, .add }, base_reg, rhs_reg); |
| 104266 | }, |
| 104267 | } |
| 104202 | 104268 | } else if (elem_size > 8) { |
| 104203 | 104269 | try cg.spillEflagsIfOccupied(); |
| 104204 | 104270 | try cg.asmRegisterImmediate( |
| ... | ... | @@ -104206,19 +104272,33 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 104206 | 104272 | rhs_reg, |
| 104207 | 104273 | .u(std.math.log2_int(u64, elem_size)), |
| 104208 | 104274 | ); |
| 104209 | | try cg.asmRegisterMemory( |
| 104210 | | .{ ._, .lea }, |
| 104211 | | base_reg, |
| 104212 | | try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }), |
| 104213 | | ); |
| 104214 | | } else try cg.asmRegisterMemory( |
| 104215 | | .{ ._, .lea }, |
| 104216 | | base_reg, |
| 104217 | | try ops[0].tracking(cg).short.mem(cg, .{ |
| 104275 | const src_mem = try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }); |
| 104276 | switch (src_mem.mod) { |
| 104277 | .rm => try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, src_mem), |
| 104278 | .off => |src_addr| { |
| 104279 | try cg.asmRegisterImmediate(.{ ._, .mov }, base_reg, .u(src_addr)); |
| 104280 | try cg.asmRegisterRegister(.{ ._, .add }, base_reg, rhs_reg); |
| 104281 | }, |
| 104282 | } |
| 104283 | } else { |
| 104284 | const src_mem = try ops[0].tracking(cg).short.mem(cg, .{ |
| 104218 | 104285 | .index = rhs_reg, |
| 104219 | 104286 | .scale = .fromFactor(@intCast(elem_size)), |
| 104220 | | }), |
| 104221 | | ); |
| 104287 | }); |
| 104288 | switch (src_mem.mod) { |
| 104289 | .rm => try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, src_mem), |
| 104290 | .off => |src_addr| { |
| 104291 | try cg.spillEflagsIfOccupied(); |
| 104292 | try cg.asmRegisterImmediate( |
| 104293 | .{ ._l, .sh }, |
| 104294 | rhs_reg, |
| 104295 | .u(std.math.log2_int(u64, elem_size)), |
| 104296 | ); |
| 104297 | try cg.asmRegisterImmediate(.{ ._, .mov }, base_reg, .u(src_addr)); |
| 104298 | try cg.asmRegisterRegister(.{ ._, .add }, base_reg, rhs_reg); |
| 104299 | }, |
| 104300 | } |
| 104301 | } |
| 104222 | 104302 | // Hack around Sema insanity: lhs could be an arbitrarily large comptime-known array |
| 104223 | 104303 | // which could easily get spilled by the upcoming `load`, which would infinite recurse |
| 104224 | 104304 | // since spilling an array requires the same operation that triggered the spill. |
| ... | ... | @@ -171517,19 +171597,37 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 171517 | 171597 | .data => if (prefetch.rw == .write and prefetch.locality <= 2 and cg.hasFeature(.prefetchwt1)) { |
| 171518 | 171598 | try ops[0].toSlicePtr(cg); |
| 171519 | 171599 | while (try ops[0].toLea(cg)) {} |
| 171520 | | try cg.asmMemory(.{ ._wt1, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })); |
| 171600 | while (true) { |
| 171601 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte }); |
| 171602 | switch (dst_mem.mod) { |
| 171603 | .rm => break try cg.asmMemory(.{ ._wt1, .prefetch }, dst_mem), |
| 171604 | .off => while (try ops[0].toRegClass(true, .general_purpose, cg)) {}, |
| 171605 | } |
| 171606 | } |
| 171521 | 171607 | } else if (prefetch.rw == .write and cg.hasFeature(.prfchw)) { |
| 171522 | 171608 | try ops[0].toSlicePtr(cg); |
| 171523 | 171609 | while (try ops[0].toLea(cg)) {} |
| 171524 | | try cg.asmMemory(.{ ._w, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })); |
| 171610 | while (true) { |
| 171611 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte }); |
| 171612 | switch (dst_mem.mod) { |
| 171613 | .rm => break try cg.asmMemory(.{ ._w, .prefetch }, dst_mem), |
| 171614 | .off => while (try ops[0].toRegClass(true, .general_purpose, cg)) {}, |
| 171615 | } |
| 171616 | } |
| 171525 | 171617 | } else if (cg.hasFeature(.sse) or cg.hasFeature(.prfchw) or cg.hasFeature(.prefetchi) or cg.hasFeature(.prefetchwt1)) { |
| 171526 | 171618 | try ops[0].toSlicePtr(cg); |
| 171527 | 171619 | while (try ops[0].toLea(cg)) {} |
| 171528 | | switch (prefetch.locality) { |
| 171529 | | 0 => try cg.asmMemory(.{ ._nta, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), |
| 171530 | | 1 => try cg.asmMemory(.{ ._t2, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), |
| 171531 | | 2 => try cg.asmMemory(.{ ._t1, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), |
| 171532 | | 3 => try cg.asmMemory(.{ ._t0, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), |
| 171620 | while (true) { |
| 171621 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte }); |
| 171622 | switch (dst_mem.mod) { |
| 171623 | .rm => break try cg.asmMemory(.{ switch (prefetch.locality) { |
| 171624 | 0 => ._nta, |
| 171625 | 1 => ._t2, |
| 171626 | 2 => ._t1, |
| 171627 | 3 => ._t0, |
| 171628 | }, .prefetch }, dst_mem), |
| 171629 | .off => while (try ops[0].toRegClass(true, .general_purpose, cg)) {}, |
| 171630 | } |
| 171533 | 171631 | } |
| 171534 | 171632 | }, |
| 171535 | 171633 | } |
| ... | ... | @@ -179871,18 +179969,35 @@ fn genConstMemcpy( |
| 179871 | 179969 | orig_src_ptr: MCValue, |
| 179872 | 179970 | len: u64, |
| 179873 | 179971 | ) !bool { |
| 179972 | assert(len > 0 and std.math.cast(u32, len) != null); // ensured by `useConstMemcpyForSize`, needed by immediate bounds checks below |
| 179973 | |
| 179974 | switch (dst_ptr) { |
| 179975 | else => return false, |
| 179976 | .immediate => |dst_addr| if (std.math.cast(i32, @as(i64, @bitCast(dst_addr))) == null or |
| 179977 | std.math.cast(i32, @as(i64, @bitCast(dst_addr + (len - 1)))) == null) return false, |
| 179978 | .register, .register_offset, .lea_frame => {}, |
| 179979 | } |
| 179874 | 179980 | if (!dst_ptr.isAddress()) return false; |
| 179875 | 179981 | |
| 179876 | | const src_reg: ?Register = r: { |
| 179877 | | if (orig_src_ptr.isAddress()) break :r null; |
| 179878 | | if (orig_src_ptr == .lea_uav or orig_src_ptr == .lea_nav) { |
| 179879 | | // To "hack around linker relocation bugs", a `lea_uav` isn't considered an address, but |
| 179880 | | // we want to avoid pessimising that case because it's common (e.g. returning constant |
| 179881 | | // values over 8 bytes). So if there's a register free, load the source address into it. |
| 179882 | | if (cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |src_reg| { |
| 179883 | | // We'll actually do the load a little later, in case another register alloc fails. |
| 179884 | | break :r src_reg; |
| 179885 | | } |
| 179982 | const sse_reg = if (len >= 16 and cg.hasFeature(.sse)) |
| 179983 | cg.register_manager.tryAllocReg(null, abi.RegisterClass.sse) |
| 179984 | else |
| 179985 | null; |
| 179986 | |
| 179987 | const src_reg = src_reg: { |
| 179988 | switch (orig_src_ptr) { |
| 179989 | else => return false, |
| 179990 | .immediate => |src_addr| if (std.math.cast(i32, @as(i64, @bitCast(src_addr))) != null and |
| 179991 | std.math.cast(i32, @as(i64, @bitCast(src_addr + (len - 1)))) != null) break :src_reg null, |
| 179992 | .register, .register_offset, .lea_frame => break :src_reg null, |
| 179993 | .lea_uav, .lea_nav => if (sse_reg == null) break :src_reg null, |
| 179994 | } |
| 179995 | // To "hack around linker relocation bugs", a `lea_uav` isn't considered an address, but |
| 179996 | // we want to avoid pessimising that case because it's common (e.g. returning constant |
| 179997 | // values over 8 bytes). So if there's a register free, load the source address into it. |
| 179998 | if (cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |src_reg| { |
| 179999 | // We'll actually do the load a little later, in case another register alloc fails. |
| 180000 | break :src_reg src_reg; |
| 179886 | 180001 | } |
| 179887 | 180002 | return false; |
| 179888 | 180003 | }; |
| ... | ... | @@ -179891,44 +180006,40 @@ fn genConstMemcpy( |
| 179891 | 180006 | const src_lock: ?RegisterLock = if (src_reg) |r| cg.register_manager.lockReg(r) else null; |
| 179892 | 180007 | defer if (src_lock) |l| cg.register_manager.unlockReg(l); |
| 179893 | 180008 | |
| 179894 | | const want_sse_reg = len >= 16 and cg.hasFeature(.sse); |
| 179895 | | const sse_reg: ?Register = if (want_sse_reg) r: { |
| 179896 | | break :r cg.register_manager.tryAllocReg(null, abi.RegisterClass.sse); |
| 179897 | | } else null; |
| 179898 | | |
| 179899 | | const need_gp_reg = len % 16 != 0 or sse_reg == null; |
| 179900 | | const gp_reg: Register = if (need_gp_reg) r: { |
| 179901 | | break :r cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp) orelse return false; |
| 179902 | | } else undefined; |
| 180009 | const gp_reg = if (len % 16 != 0 or sse_reg == null) |
| 180010 | cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp) orelse return false |
| 180011 | else |
| 180012 | null; |
| 179903 | 180013 | |
| 179904 | 180014 | const src_ptr: MCValue = src_ptr: { |
| 179905 | 180015 | const reg = src_reg orelse break :src_ptr orig_src_ptr; |
| 179906 | | try cg.asmRegisterMemory(.{ ._, .lea }, reg.to64(), switch (orig_src_ptr) { |
| 179907 | | .lea_uav => |uav| .{ .base = .{ .uav = uav } }, |
| 179908 | | .lea_nav => |nav| .{ .base = .{ .nav = nav } }, |
| 180016 | switch (orig_src_ptr) { |
| 180017 | .immediate => |imm| try cg.asmRegisterImmediate(.{ ._, .mov }, reg.to64(), .u(imm)), |
| 180018 | .lea_uav => |uav| try cg.asmRegisterMemory(.{ ._, .lea }, reg.to64(), .{ .base = .{ .uav = uav } }), |
| 180019 | .lea_nav => |nav| try cg.asmRegisterMemory(.{ ._, .lea }, reg.to64(), .{ .base = .{ .nav = nav } }), |
| 179909 | 180020 | else => unreachable, |
| 179910 | | }); |
| 180021 | } |
| 179911 | 180022 | break :src_ptr .{ .register = reg.to64() }; |
| 179912 | 180023 | }; |
| 179913 | 180024 | |
| 179914 | 180025 | var offset: u64 = 0; |
| 179915 | 180026 | |
| 179916 | | if (sse_reg) |r| { |
| 180027 | if (sse_reg) |temp_reg| { |
| 179917 | 180028 | if (cg.hasFeature(.avx)) { |
| 179918 | | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, r, .yword); |
| 179919 | | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, r, .xword); |
| 180029 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, temp_reg, .yword); |
| 180030 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, temp_reg, .xword); |
| 179920 | 180031 | } else if (cg.hasFeature(.sse2)) { |
| 179921 | | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._dqu, .mov }, r, .xword); |
| 180032 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._dqu, .mov }, temp_reg, .xword); |
| 179922 | 180033 | } else if (cg.hasFeature(.sse)) { |
| 179923 | | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._ps, .movu }, r, .xword); |
| 180034 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._ps, .movu }, temp_reg, .xword); |
| 179924 | 180035 | } |
| 179925 | 180036 | } |
| 179926 | 180037 | |
| 179927 | | if (need_gp_reg) { |
| 179928 | | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .qword); |
| 179929 | | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .dword); |
| 179930 | | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .word); |
| 179931 | | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .byte); |
| 180038 | if (gp_reg) |temp_reg| { |
| 180039 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, temp_reg, .qword); |
| 180040 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, temp_reg, .dword); |
| 180041 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, temp_reg, .word); |
| 180042 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, temp_reg, .byte); |
| 179932 | 180043 | } |
| 179933 | 180044 | |
| 179934 | 180045 | assert(offset == len); |
| ... | ... | @@ -183219,13 +183330,20 @@ const Temp = struct { |
| 183219 | 183330 | .lea_extern_func, |
| 183220 | 183331 | => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, |
| 183221 | 183332 | } |
| 183222 | | try cg.asmMemoryImmediate( |
| 183223 | | .{ ._, .mov }, |
| 183224 | | try ptr.tracking(cg).short.deref().mem(cg, .{ |
| 183225 | | .size = cg.memSize(val_ty, .general_purpose), |
| 183226 | | }), |
| 183227 | | val_op, |
| 183228 | | ); |
| 183333 | const dst_mem = try ptr.tracking(cg).short.deref().mem(cg, .{ |
| 183334 | .size = cg.memSize(val_ty, .general_purpose), |
| 183335 | }); |
| 183336 | switch (dst_mem.mod) { |
| 183337 | .rm => try cg.asmMemoryImmediate(.{ ._, .mov }, dst_mem, val_op), |
| 183338 | .off => { |
| 183339 | while (try val.toReg(.rax, cg)) {} |
| 183340 | try cg.asmMemoryRegister( |
| 183341 | .{ ._, .mov }, |
| 183342 | dst_mem, |
| 183343 | registerAlias(.rax, @intCast(val_ty.abiSize(zcu))), |
| 183344 | ); |
| 183345 | }, |
| 183346 | } |
| 183229 | 183347 | }, |
| 183230 | 183348 | .eflags => |cc| { |
| 183231 | 183349 | // hack around linker relocation bugs |
| ... | ... | @@ -183429,9 +183547,15 @@ const Temp = struct { |
| 183429 | 183547 | .lea_nav, .lea_uav, .lea_lazy_sym => if (dst_rc != .general_purpose) |
| 183430 | 183548 | while (try ptr.toRegClass(false, .general_purpose, cg)) {}, |
| 183431 | 183549 | } |
| 183432 | | try strat.read(cg, dst_reg, try ptr.tracking(cg).short.deref().mem(cg, .{ |
| 183550 | const src_mem = try ptr.tracking(cg).short.deref().mem(cg, .{ |
| 183433 | 183551 | .size = cg.memSize(dst_ty, dst_rc), |
| 183434 | | })); |
| 183552 | }); |
| 183553 | if (src_mem.mod == .off and dst_reg.id() != comptime Register.rax.id()) { |
| 183554 | const tmp_reg = Register.rax.toSize(dst_reg.size(), cg.target); |
| 183555 | try cg.register_manager.getKnownReg(.rax, null); |
| 183556 | try strat.read(cg, tmp_reg, src_mem); |
| 183557 | try cg.asmRegisterRegister(.{ ._, .mov }, dst_reg, tmp_reg); |
| 183558 | } else try strat.read(cg, dst_reg, src_mem); |
| 183435 | 183559 | } |
| 183436 | 183560 | |
| 183437 | 183561 | fn storeRegs(ptr: *Temp, src_ty: Type, src_regs: []const Register, cg: *CodeGen) InnerError!void { |
| ... | ... | @@ -183500,7 +183624,7 @@ const Temp = struct { |
| 183500 | 183624 | .lea_nav, .lea_uav, .lea_lazy_sym => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, |
| 183501 | 183625 | } |
| 183502 | 183626 | const strat = try cg.moveStrategy(part_ty, src_rc, false); |
| 183503 | | try strat.write(cg, try ptr.tracking(cg).short.deref().mem(cg, .{ |
| 183627 | const dst_mem = try ptr.tracking(cg).short.deref().mem(cg, .{ |
| 183504 | 183628 | .size = switch (src_rc) { |
| 183505 | 183629 | else => .fromBitSize(8 * part_size), |
| 183506 | 183630 | .x87 => switch (abi.classifySystemV(src_ty, zcu, cg.target, .other)[part_index]) { |
| ... | ... | @@ -183512,7 +183636,13 @@ const Temp = struct { |
| 183512 | 183636 | }, |
| 183513 | 183637 | }, |
| 183514 | 183638 | .disp = part_disp, |
| 183515 | | }), registerAlias(src_reg, part_size)); |
| 183639 | }); |
| 183640 | if (dst_mem.mod == .off and src_reg.id() != comptime Register.rax.id()) { |
| 183641 | const tmp_reg = registerAlias(.rax, part_size); |
| 183642 | try cg.register_manager.getKnownReg(.rax, null); |
| 183643 | try cg.asmRegisterRegister(.{ ._, .mov }, tmp_reg, registerAlias(src_reg, part_size)); |
| 183644 | try strat.write(cg, dst_mem, tmp_reg); |
| 183645 | } else try strat.write(cg, dst_mem, registerAlias(src_reg, part_size)); |
| 183516 | 183646 | } else { |
| 183517 | 183647 | const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size); |
| 183518 | 183648 | const frame_index = try cg.allocFrameIndex(.init(.{ |
Jacob Young <jacobly@ziglang.org> 2026-10-01 12:46:25-04:00
Andrew Kelley <andrew@ziglang.org> 2026-10-01 23:30:21+02:00