| ... | @@ -545,15 +545,18 @@ pub const MCValue = union(enum) { | ... | @@ -545,15 +545,18 @@ pub const MCValue = union(enum) { |
| 545 | .lea_lazy_sym, | 545 | .lea_lazy_sym, |
| 546 | .lea_extern_func, | 546 | .lea_extern_func, |
| 547 | => unreachable, | 547 | => unreachable, |
| 548 | .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{ | 548 | .memory => |base_addr| { |
| 549 | .base = .{ .reg = .ds }, | 549 | const addr = @as(i64, @bitCast(base_addr)) +% mod_rm.disp; |
| 550 | .mod = .{ .rm = .{ | 550 | return .{ |
| 551 | .size = mod_rm.size, | 551 | .base = .{ .reg = .ds }, |
| 552 | .index = mod_rm.index, | 552 | .mod = if (std.math.cast(i32, addr)) |small_addr| .{ .rm = .{ |
| 553 | .scale = mod_rm.scale, | 553 | .size = mod_rm.size, |
| 554 | .disp = small_addr + mod_rm.disp, | 554 | .index = mod_rm.index, |
| 555 | } }, | 555 | .scale = mod_rm.scale, |
| 556 | } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } }, | 556 | .disp = small_addr, |
| | 557 | } } else .{ .off = @bitCast(addr) }, |
| | 558 | }; |
| | 559 | }, |
| 557 | .indirect => |reg_off| .{ | 560 | .indirect => |reg_off| .{ |
| 558 | .base = .{ .reg = reg_off.reg.toSize(.ptr, function.target) }, | 561 | .base = .{ .reg = reg_off.reg.toSize(.ptr, function.target) }, |
| 559 | .mod = .{ .rm = .{ | 562 | .mod = .{ .rm = .{ |
| ... | @@ -89205,19 +89208,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -89205,19 +89208,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89205 | try cg.spillEflagsIfOccupied(); | 89208 | try cg.spillEflagsIfOccupied(); |
| 89206 | var ops = try cg.tempsFromOperands(inst, .{un_op}); | 89209 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89207 | while (try ops[0].toBase(false, cg)) {} | 89210 | while (try ops[0].toBase(false, cg)) {} |
| 89208 | try cg.asmMemoryImmediate( | 89211 | const mem_size: Memory.Size = if (!opt_repr_is_pl) |
| 89209 | .{ ._, .cmp }, | 89212 | .byte |
| 89210 | try ops[0].tracking(cg).short.mem(cg, .{ | 89213 | else if (opt_child_ty.isSlice(zcu)) |
| 89211 | .size = if (!opt_repr_is_pl) | 89214 | .ptr |
| 89212 | .byte | 89215 | else |
| 89213 | else if (opt_child_ty.isSlice(zcu)) | 89216 | .fromSize(opt_child_abi_size); |
| 89214 | .ptr | 89217 | const dst_mem = try ops[0].tracking(cg).short.mem(cg, .{ |
| 89215 | else | 89218 | .size = mem_size, |
| 89216 | .fromSize(opt_child_abi_size), | 89219 | .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, |
| 89217 | .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, | 89220 | }); |
| 89218 | }), | 89221 | switch (dst_mem.mod) { |
| 89219 | .u(0), | 89222 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89220 | ); | 89223 | .off => { |
| | 89224 | try cg.register_manager.getKnownReg(.rax, inst); |
| | 89225 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| | 89226 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| | 89227 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| | 89228 | }, |
| | 89229 | } |
| 89221 | const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); | 89230 | const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 89222 | try is_null.finish(inst, &.{un_op}, &ops, cg); | 89231 | try is_null.finish(inst, &.{un_op}, &ops, cg); |
| 89223 | }, | 89232 | }, |
| ... | @@ -89230,19 +89239,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -89230,19 +89239,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89230 | try cg.spillEflagsIfOccupied(); | 89239 | try cg.spillEflagsIfOccupied(); |
| 89231 | var ops = try cg.tempsFromOperands(inst, .{un_op}); | 89240 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89232 | while (try ops[0].toBase(false, cg)) {} | 89241 | while (try ops[0].toBase(false, cg)) {} |
| 89233 | try cg.asmMemoryImmediate( | 89242 | const mem_size: Memory.Size = if (!opt_repr_is_pl) |
| 89234 | .{ ._, .cmp }, | 89243 | .byte |
| 89235 | try ops[0].tracking(cg).short.mem(cg, .{ | 89244 | else if (opt_child_ty.isSlice(zcu)) |
| 89236 | .size = if (!opt_repr_is_pl) | 89245 | .ptr |
| 89237 | .byte | 89246 | else |
| 89238 | else if (opt_child_ty.isSlice(zcu)) | 89247 | .fromSize(opt_child_abi_size); |
| 89239 | .ptr | 89248 | const dst_mem = try ops[0].tracking(cg).short.mem(cg, .{ |
| 89240 | else | 89249 | .size = mem_size, |
| 89241 | .fromSize(opt_child_abi_size), | 89250 | .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, |
| 89242 | .disp = if (opt_repr_is_pl) 0 else opt_child_abi_size, | 89251 | }); |
| 89243 | }), | 89252 | switch (dst_mem.mod) { |
| 89244 | .u(0), | 89253 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89245 | ); | 89254 | .off => { |
| | 89255 | try cg.register_manager.getKnownReg(.rax, inst); |
| | 89256 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| | 89257 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| | 89258 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| | 89259 | }, |
| | 89260 | } |
| 89246 | const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); | 89261 | const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 89247 | try is_non_null.finish(inst, &.{un_op}, &ops, cg); | 89262 | try is_non_null.finish(inst, &.{un_op}, &ops, cg); |
| 89248 | }, | 89263 | }, |
| ... | @@ -89256,16 +89271,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -89256,16 +89271,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89256 | var ops = try cg.tempsFromOperands(inst, .{un_op}); | 89271 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89257 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); | 89272 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); |
| 89258 | while (try ops[0].toLea(cg)) {} | 89273 | while (try ops[0].toLea(cg)) {} |
| 89259 | try cg.asmMemoryImmediate( | 89274 | const mem_size: Memory.Size = if (!opt_repr_is_pl) |
| 89260 | .{ ._, .cmp }, | 89275 | .byte |
| 89261 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) | 89276 | else if (opt_child_ty.isSlice(zcu)) |
| 89262 | .byte | 89277 | .ptr |
| 89263 | else if (opt_child_ty.isSlice(zcu)) | 89278 | else |
| 89264 | .ptr | 89279 | .fromSize(opt_child_abi_size); |
| 89265 | else | 89280 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = mem_size }); |
| 89266 | .fromSize(opt_child_abi_size) }), | 89281 | switch (dst_mem.mod) { |
| 89267 | .u(0), | 89282 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89268 | ); | 89283 | .off => { |
| | 89284 | try cg.register_manager.getKnownReg(.rax, inst); |
| | 89285 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| | 89286 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| | 89287 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| | 89288 | }, |
| | 89289 | } |
| 89269 | const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); | 89290 | const is_null = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 89270 | try is_null.finish(inst, &.{un_op}, &ops, cg); | 89291 | try is_null.finish(inst, &.{un_op}, &ops, cg); |
| 89271 | }, | 89292 | }, |
| ... | @@ -89279,16 +89300,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -89279,16 +89300,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89279 | var ops = try cg.tempsFromOperands(inst, .{un_op}); | 89300 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89280 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); | 89301 | if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, cg); |
| 89281 | while (try ops[0].toLea(cg)) {} | 89302 | while (try ops[0].toLea(cg)) {} |
| 89282 | try cg.asmMemoryImmediate( | 89303 | const mem_size: Memory.Size = if (!opt_repr_is_pl) |
| 89283 | .{ ._, .cmp }, | 89304 | .byte |
| 89284 | try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = if (!opt_repr_is_pl) | 89305 | else if (opt_child_ty.isSlice(zcu)) |
| 89285 | .byte | 89306 | .ptr |
| 89286 | else if (opt_child_ty.isSlice(zcu)) | 89307 | else |
| 89287 | .ptr | 89308 | .fromSize(opt_child_abi_size); |
| 89288 | else | 89309 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = mem_size }); |
| 89289 | .fromSize(opt_child_abi_size) }), | 89310 | switch (dst_mem.mod) { |
| 89290 | .u(0), | 89311 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| 89291 | ); | 89312 | .off => { |
| | 89313 | try cg.register_manager.getKnownReg(.rax, inst); |
| | 89314 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| | 89315 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| | 89316 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| | 89317 | }, |
| | 89318 | } |
| 89292 | const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); | 89319 | const is_non_null = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 89293 | try is_non_null.finish(inst, &.{un_op}, &ops, cg); | 89320 | try is_non_null.finish(inst, &.{un_op}, &ops, cg); |
| 89294 | }, | 89321 | }, |
| ... | @@ -89301,10 +89328,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -89301,10 +89328,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89301 | try cg.spillEflagsIfOccupied(); | 89328 | try cg.spillEflagsIfOccupied(); |
| 89302 | var ops = try cg.tempsFromOperands(inst, .{un_op}); | 89329 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89303 | while (try ops[0].toBase(false, cg)) {} | 89330 | while (try ops[0].toBase(false, cg)) {} |
| 89304 | try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.mem(cg, .{ | 89331 | const mem_size: Memory.Size = cg.memSize(eu_err_ty, .general_purpose); |
| 89305 | .size = cg.memSize(eu_err_ty, .general_purpose), | 89332 | const dst_mem = try ops[0].tracking(cg).short.mem(cg, .{ |
| | 89333 | .size = mem_size, |
| 89306 | .disp = eu_err_off, | 89334 | .disp = eu_err_off, |
| 89307 | }), .u(0)); | 89335 | }); |
| | 89336 | switch (dst_mem.mod) { |
| | 89337 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| | 89338 | .off => { |
| | 89339 | try cg.register_manager.getKnownReg(.rax, inst); |
| | 89340 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| | 89341 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| | 89342 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| | 89343 | }, |
| | 89344 | } |
| 89308 | const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); | 89345 | const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 89309 | try is_err.finish(inst, &.{un_op}, &ops, cg); | 89346 | try is_err.finish(inst, &.{un_op}, &ops, cg); |
| 89310 | }, | 89347 | }, |
| ... | @@ -89317,10 +89354,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -89317,10 +89354,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89317 | try cg.spillEflagsIfOccupied(); | 89354 | try cg.spillEflagsIfOccupied(); |
| 89318 | var ops = try cg.tempsFromOperands(inst, .{un_op}); | 89355 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89319 | while (try ops[0].toBase(false, cg)) {} | 89356 | while (try ops[0].toBase(false, cg)) {} |
| 89320 | try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.mem(cg, .{ | 89357 | const mem_size = cg.memSize(eu_err_ty, .general_purpose); |
| 89321 | .size = cg.memSize(eu_err_ty, .general_purpose), | 89358 | const dst_mem = try ops[0].tracking(cg).short.mem(cg, .{ |
| | 89359 | .size = mem_size, |
| 89322 | .disp = eu_err_off, | 89360 | .disp = eu_err_off, |
| 89323 | }), .u(0)); | 89361 | }); |
| | 89362 | switch (dst_mem.mod) { |
| | 89363 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| | 89364 | .off => { |
| | 89365 | try cg.register_manager.getKnownReg(.rax, inst); |
| | 89366 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| | 89367 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| | 89368 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| | 89369 | }, |
| | 89370 | } |
| 89324 | const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); | 89371 | const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 89325 | try is_non_err.finish(inst, &.{un_op}, &ops, cg); | 89372 | try is_non_err.finish(inst, &.{un_op}, &ops, cg); |
| 89326 | }, | 89373 | }, |
| ... | @@ -89334,9 +89381,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -89334,9 +89381,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89334 | var ops = try cg.tempsFromOperands(inst, .{un_op}); | 89381 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89335 | try ops[0].toOffset(eu_err_off, cg); | 89382 | try ops[0].toOffset(eu_err_off, cg); |
| 89336 | while (try ops[0].toLea(cg)) {} | 89383 | while (try ops[0].toLea(cg)) {} |
| 89337 | try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.deref().mem(cg, .{ | 89384 | const mem_size = cg.memSize(eu_err_ty, .general_purpose); |
| 89338 | .size = cg.memSize(eu_err_ty, .general_purpose), | 89385 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = mem_size }); |
| 89339 | }), .u(0)); | 89386 | switch (dst_mem.mod) { |
| | 89387 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| | 89388 | .off => { |
| | 89389 | try cg.register_manager.getKnownReg(.rax, inst); |
| | 89390 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| | 89391 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| | 89392 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| | 89393 | }, |
| | 89394 | } |
| 89340 | const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); | 89395 | const is_err = try cg.tempInit(.bool, .{ .eflags = .ne }); |
| 89341 | try is_err.finish(inst, &.{un_op}, &ops, cg); | 89396 | try is_err.finish(inst, &.{un_op}, &ops, cg); |
| 89342 | }, | 89397 | }, |
| ... | @@ -89350,9 +89405,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -89350,9 +89405,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 89350 | var ops = try cg.tempsFromOperands(inst, .{un_op}); | 89405 | var ops = try cg.tempsFromOperands(inst, .{un_op}); |
| 89351 | try ops[0].toOffset(eu_err_off, cg); | 89406 | try ops[0].toOffset(eu_err_off, cg); |
| 89352 | while (try ops[0].toLea(cg)) {} | 89407 | while (try ops[0].toLea(cg)) {} |
| 89353 | try cg.asmMemoryImmediate(.{ ._, .cmp }, try ops[0].tracking(cg).short.deref().mem(cg, .{ | 89408 | const mem_size = cg.memSize(eu_err_ty, .general_purpose); |
| 89354 | .size = cg.memSize(eu_err_ty, .general_purpose), | 89409 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = mem_size }); |
| 89355 | }), .u(0)); | 89410 | switch (dst_mem.mod) { |
| | 89411 | .rm => try cg.asmMemoryImmediate(.{ ._, .cmp }, dst_mem, .u(0)), |
| | 89412 | .off => { |
| | 89413 | try cg.register_manager.getKnownReg(.rax, inst); |
| | 89414 | const tmp_reg = Register.rax.toSize(mem_size, cg.target); |
| | 89415 | try cg.asmRegisterMemory(.{ ._, .mov }, tmp_reg, dst_mem); |
| | 89416 | try cg.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); |
| | 89417 | }, |
| | 89418 | } |
| 89356 | const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); | 89419 | const is_non_err = try cg.tempInit(.bool, .{ .eflags = .e }); |
| 89357 | try is_non_err.finish(inst, &.{un_op}, &ops, cg); | 89420 | try is_non_err.finish(inst, &.{un_op}, &ops, cg); |
| 89358 | }, | 89421 | }, |
| ... | @@ -104194,11 +104257,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -104194,11 +104257,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 104194 | rhs_reg, | 104257 | rhs_reg, |
| 104195 | .u(elem_size), | 104258 | .u(elem_size), |
| 104196 | ); | 104259 | ); |
| 104197 | try cg.asmRegisterMemory( | 104260 | const src_mem = try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }); |
| 104198 | .{ ._, .lea }, | 104261 | switch (src_mem.mod) { |
| 104199 | base_reg, | 104262 | .rm => try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, src_mem), |
| 104200 | try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }), | 104263 | .off => |src_addr| { |
| 104201 | ); | 104264 | try cg.asmRegisterImmediate(.{ ._, .mov }, base_reg, .u(src_addr)); |
| | 104265 | try cg.asmRegisterRegister(.{ ._, .add }, base_reg, rhs_reg); |
| | 104266 | }, |
| | 104267 | } |
| 104202 | } else if (elem_size > 8) { | 104268 | } else if (elem_size > 8) { |
| 104203 | try cg.spillEflagsIfOccupied(); | 104269 | try cg.spillEflagsIfOccupied(); |
| 104204 | try cg.asmRegisterImmediate( | 104270 | try cg.asmRegisterImmediate( |
| ... | @@ -104206,19 +104272,33 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -104206,19 +104272,33 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 104206 | rhs_reg, | 104272 | rhs_reg, |
| 104207 | .u(std.math.log2_int(u64, elem_size)), | 104273 | .u(std.math.log2_int(u64, elem_size)), |
| 104208 | ); | 104274 | ); |
| 104209 | try cg.asmRegisterMemory( | 104275 | const src_mem = try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }); |
| 104210 | .{ ._, .lea }, | 104276 | switch (src_mem.mod) { |
| 104211 | base_reg, | 104277 | .rm => try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, src_mem), |
| 104212 | try ops[0].tracking(cg).short.mem(cg, .{ .index = rhs_reg }), | 104278 | .off => |src_addr| { |
| 104213 | ); | 104279 | try cg.asmRegisterImmediate(.{ ._, .mov }, base_reg, .u(src_addr)); |
| 104214 | } else try cg.asmRegisterMemory( | 104280 | try cg.asmRegisterRegister(.{ ._, .add }, base_reg, rhs_reg); |
| 104215 | .{ ._, .lea }, | 104281 | }, |
| 104216 | base_reg, | 104282 | } |
| 104217 | try ops[0].tracking(cg).short.mem(cg, .{ | 104283 | } else { |
| | 104284 | const src_mem = try ops[0].tracking(cg).short.mem(cg, .{ |
| 104218 | .index = rhs_reg, | 104285 | .index = rhs_reg, |
| 104219 | .scale = .fromFactor(@intCast(elem_size)), | 104286 | .scale = .fromFactor(@intCast(elem_size)), |
| 104220 | }), | 104287 | }); |
| 104221 | ); | 104288 | switch (src_mem.mod) { |
| | 104289 | .rm => try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, src_mem), |
| | 104290 | .off => |src_addr| { |
| | 104291 | try cg.spillEflagsIfOccupied(); |
| | 104292 | try cg.asmRegisterImmediate( |
| | 104293 | .{ ._l, .sh }, |
| | 104294 | rhs_reg, |
| | 104295 | .u(std.math.log2_int(u64, elem_size)), |
| | 104296 | ); |
| | 104297 | try cg.asmRegisterImmediate(.{ ._, .mov }, base_reg, .u(src_addr)); |
| | 104298 | try cg.asmRegisterRegister(.{ ._, .add }, base_reg, rhs_reg); |
| | 104299 | }, |
| | 104300 | } |
| | 104301 | } |
| 104222 | // Hack around Sema insanity: lhs could be an arbitrarily large comptime-known array | 104302 | // Hack around Sema insanity: lhs could be an arbitrarily large comptime-known array |
| 104223 | // which could easily get spilled by the upcoming `load`, which would infinite recurse | 104303 | // which could easily get spilled by the upcoming `load`, which would infinite recurse |
| 104224 | // since spilling an array requires the same operation that triggered the spill. | 104304 | // since spilling an array requires the same operation that triggered the spill. |
| ... | @@ -171517,19 +171597,37 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -171517,19 +171597,37 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 171517 | .data => if (prefetch.rw == .write and prefetch.locality <= 2 and cg.hasFeature(.prefetchwt1)) { | 171597 | .data => if (prefetch.rw == .write and prefetch.locality <= 2 and cg.hasFeature(.prefetchwt1)) { |
| 171518 | try ops[0].toSlicePtr(cg); | 171598 | try ops[0].toSlicePtr(cg); |
| 171519 | while (try ops[0].toLea(cg)) {} | 171599 | while (try ops[0].toLea(cg)) {} |
| 171520 | try cg.asmMemory(.{ ._wt1, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })); | 171600 | while (true) { |
| | 171601 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte }); |
| | 171602 | switch (dst_mem.mod) { |
| | 171603 | .rm => break try cg.asmMemory(.{ ._wt1, .prefetch }, dst_mem), |
| | 171604 | .off => while (try ops[0].toRegClass(true, .general_purpose, cg)) {}, |
| | 171605 | } |
| | 171606 | } |
| 171521 | } else if (prefetch.rw == .write and cg.hasFeature(.prfchw)) { | 171607 | } else if (prefetch.rw == .write and cg.hasFeature(.prfchw)) { |
| 171522 | try ops[0].toSlicePtr(cg); | 171608 | try ops[0].toSlicePtr(cg); |
| 171523 | while (try ops[0].toLea(cg)) {} | 171609 | while (try ops[0].toLea(cg)) {} |
| 171524 | try cg.asmMemory(.{ ._w, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })); | 171610 | while (true) { |
| | 171611 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte }); |
| | 171612 | switch (dst_mem.mod) { |
| | 171613 | .rm => break try cg.asmMemory(.{ ._w, .prefetch }, dst_mem), |
| | 171614 | .off => while (try ops[0].toRegClass(true, .general_purpose, cg)) {}, |
| | 171615 | } |
| | 171616 | } |
| 171525 | } else if (cg.hasFeature(.sse) or cg.hasFeature(.prfchw) or cg.hasFeature(.prefetchi) or cg.hasFeature(.prefetchwt1)) { | 171617 | } else if (cg.hasFeature(.sse) or cg.hasFeature(.prfchw) or cg.hasFeature(.prefetchi) or cg.hasFeature(.prefetchwt1)) { |
| 171526 | try ops[0].toSlicePtr(cg); | 171618 | try ops[0].toSlicePtr(cg); |
| 171527 | while (try ops[0].toLea(cg)) {} | 171619 | while (try ops[0].toLea(cg)) {} |
| 171528 | switch (prefetch.locality) { | 171620 | while (true) { |
| 171529 | 0 => try cg.asmMemory(.{ ._nta, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), | 171621 | const dst_mem = try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte }); |
| 171530 | 1 => try cg.asmMemory(.{ ._t2, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), | 171622 | switch (dst_mem.mod) { |
| 171531 | 2 => try cg.asmMemory(.{ ._t1, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), | 171623 | .rm => break try cg.asmMemory(.{ switch (prefetch.locality) { |
| 171532 | 3 => try cg.asmMemory(.{ ._t0, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })), | 171624 | 0 => ._nta, |
| | 171625 | 1 => ._t2, |
| | 171626 | 2 => ._t1, |
| | 171627 | 3 => ._t0, |
| | 171628 | }, .prefetch }, dst_mem), |
| | 171629 | .off => while (try ops[0].toRegClass(true, .general_purpose, cg)) {}, |
| | 171630 | } |
| 171533 | } | 171631 | } |
| 171534 | }, | 171632 | }, |
| 171535 | } | 171633 | } |
| ... | @@ -179871,18 +179969,35 @@ fn genConstMemcpy( | ... | @@ -179871,18 +179969,35 @@ fn genConstMemcpy( |
| 179871 | orig_src_ptr: MCValue, | 179969 | orig_src_ptr: MCValue, |
| 179872 | len: u64, | 179970 | len: u64, |
| 179873 | ) !bool { | 179971 | ) !bool { |
| | 179972 | assert(len > 0 and std.math.cast(u32, len) != null); // ensured by `useConstMemcpyForSize`, needed by immediate bounds checks below |
| | 179973 | |
| | 179974 | switch (dst_ptr) { |
| | 179975 | else => return false, |
| | 179976 | .immediate => |dst_addr| if (std.math.cast(i32, @as(i64, @bitCast(dst_addr))) == null or |
| | 179977 | std.math.cast(i32, @as(i64, @bitCast(dst_addr + (len - 1)))) == null) return false, |
| | 179978 | .register, .register_offset, .lea_frame => {}, |
| | 179979 | } |
| 179874 | if (!dst_ptr.isAddress()) return false; | 179980 | if (!dst_ptr.isAddress()) return false; |
| 179875 | | 179981 | |
| 179876 | const src_reg: ?Register = r: { | 179982 | const sse_reg = if (len >= 16 and cg.hasFeature(.sse)) |
| 179877 | if (orig_src_ptr.isAddress()) break :r null; | 179983 | cg.register_manager.tryAllocReg(null, abi.RegisterClass.sse) |
| 179878 | if (orig_src_ptr == .lea_uav or orig_src_ptr == .lea_nav) { | 179984 | else |
| 179879 | // To "hack around linker relocation bugs", a `lea_uav` isn't considered an address, but | 179985 | null; |
| 179880 | // we want to avoid pessimising that case because it's common (e.g. returning constant | 179986 | |
| 179881 | // values over 8 bytes). So if there's a register free, load the source address into it. | 179987 | const src_reg = src_reg: { |
| 179882 | if (cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |src_reg| { | 179988 | switch (orig_src_ptr) { |
| 179883 | // We'll actually do the load a little later, in case another register alloc fails. | 179989 | else => return false, |
| 179884 | break :r src_reg; | 179990 | .immediate => |src_addr| if (std.math.cast(i32, @as(i64, @bitCast(src_addr))) != null and |
| 179885 | } | 179991 | std.math.cast(i32, @as(i64, @bitCast(src_addr + (len - 1)))) != null) break :src_reg null, |
| | 179992 | .register, .register_offset, .lea_frame => break :src_reg null, |
| | 179993 | .lea_uav, .lea_nav => if (sse_reg == null) break :src_reg null, |
| | 179994 | } |
| | 179995 | // To "hack around linker relocation bugs", a `lea_uav` isn't considered an address, but |
| | 179996 | // we want to avoid pessimising that case because it's common (e.g. returning constant |
| | 179997 | // values over 8 bytes). So if there's a register free, load the source address into it. |
| | 179998 | if (cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp)) |src_reg| { |
| | 179999 | // We'll actually do the load a little later, in case another register alloc fails. |
| | 180000 | break :src_reg src_reg; |
| 179886 | } | 180001 | } |
| 179887 | return false; | 180002 | return false; |
| 179888 | }; | 180003 | }; |
| ... | @@ -179891,44 +180006,40 @@ fn genConstMemcpy( | ... | @@ -179891,44 +180006,40 @@ fn genConstMemcpy( |
| 179891 | const src_lock: ?RegisterLock = if (src_reg) |r| cg.register_manager.lockReg(r) else null; | 180006 | const src_lock: ?RegisterLock = if (src_reg) |r| cg.register_manager.lockReg(r) else null; |
| 179892 | defer if (src_lock) |l| cg.register_manager.unlockReg(l); | 180007 | defer if (src_lock) |l| cg.register_manager.unlockReg(l); |
| 179893 | | 180008 | |
| 179894 | const want_sse_reg = len >= 16 and cg.hasFeature(.sse); | 180009 | const gp_reg = if (len % 16 != 0 or sse_reg == null) |
| 179895 | const sse_reg: ?Register = if (want_sse_reg) r: { | 180010 | cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp) orelse return false |
| 179896 | break :r cg.register_manager.tryAllocReg(null, abi.RegisterClass.sse); | 180011 | else |
| 179897 | } else null; | 180012 | null; |
| 179898 | | | |
| 179899 | const need_gp_reg = len % 16 != 0 or sse_reg == null; | | |
| 179900 | const gp_reg: Register = if (need_gp_reg) r: { | | |
| 179901 | break :r cg.register_manager.tryAllocReg(null, abi.RegisterClass.gp) orelse return false; | | |
| 179902 | } else undefined; | | |
| 179903 | | 180013 | |
| 179904 | const src_ptr: MCValue = src_ptr: { | 180014 | const src_ptr: MCValue = src_ptr: { |
| 179905 | const reg = src_reg orelse break :src_ptr orig_src_ptr; | 180015 | const reg = src_reg orelse break :src_ptr orig_src_ptr; |
| 179906 | try cg.asmRegisterMemory(.{ ._, .lea }, reg.to64(), switch (orig_src_ptr) { | 180016 | switch (orig_src_ptr) { |
| 179907 | .lea_uav => |uav| .{ .base = .{ .uav = uav } }, | 180017 | .immediate => |imm| try cg.asmRegisterImmediate(.{ ._, .mov }, reg.to64(), .u(imm)), |
| 179908 | .lea_nav => |nav| .{ .base = .{ .nav = nav } }, | 180018 | .lea_uav => |uav| try cg.asmRegisterMemory(.{ ._, .lea }, reg.to64(), .{ .base = .{ .uav = uav } }), |
| | 180019 | .lea_nav => |nav| try cg.asmRegisterMemory(.{ ._, .lea }, reg.to64(), .{ .base = .{ .nav = nav } }), |
| 179909 | else => unreachable, | 180020 | else => unreachable, |
| 179910 | }); | 180021 | } |
| 179911 | break :src_ptr .{ .register = reg.to64() }; | 180022 | break :src_ptr .{ .register = reg.to64() }; |
| 179912 | }; | 180023 | }; |
| 179913 | | 180024 | |
| 179914 | var offset: u64 = 0; | 180025 | var offset: u64 = 0; |
| 179915 | | 180026 | |
| 179916 | if (sse_reg) |r| { | 180027 | if (sse_reg) |temp_reg| { |
| 179917 | if (cg.hasFeature(.avx)) { | 180028 | if (cg.hasFeature(.avx)) { |
| 179918 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, r, .yword); | 180029 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, temp_reg, .yword); |
| 179919 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, r, .xword); | 180030 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ .v_dqu, .mov }, temp_reg, .xword); |
| 179920 | } else if (cg.hasFeature(.sse2)) { | 180031 | } else if (cg.hasFeature(.sse2)) { |
| 179921 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._dqu, .mov }, r, .xword); | 180032 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._dqu, .mov }, temp_reg, .xword); |
| 179922 | } else if (cg.hasFeature(.sse)) { | 180033 | } else if (cg.hasFeature(.sse)) { |
| 179923 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._ps, .movu }, r, .xword); | 180034 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._ps, .movu }, temp_reg, .xword); |
| 179924 | } | 180035 | } |
| 179925 | } | 180036 | } |
| 179926 | | 180037 | |
| 179927 | if (need_gp_reg) { | 180038 | if (gp_reg) |temp_reg| { |
| 179928 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .qword); | 180039 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, temp_reg, .qword); |
| 179929 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .dword); | 180040 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, temp_reg, .dword); |
| 179930 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .word); | 180041 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, temp_reg, .word); |
| 179931 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, gp_reg, .byte); | 180042 | try cg.memcpyPart(dst_ptr, src_ptr, len, &offset, .{ ._, .mov }, temp_reg, .byte); |
| 179932 | } | 180043 | } |
| 179933 | | 180044 | |
| 179934 | assert(offset == len); | 180045 | assert(offset == len); |
| ... | @@ -183219,13 +183330,20 @@ const Temp = struct { | ... | @@ -183219,13 +183330,20 @@ const Temp = struct { |
| 183219 | .lea_extern_func, | 183330 | .lea_extern_func, |
| 183220 | => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, | 183331 | => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, |
| 183221 | } | 183332 | } |
| 183222 | try cg.asmMemoryImmediate( | 183333 | const dst_mem = try ptr.tracking(cg).short.deref().mem(cg, .{ |
| 183223 | .{ ._, .mov }, | 183334 | .size = cg.memSize(val_ty, .general_purpose), |
| 183224 | try ptr.tracking(cg).short.deref().mem(cg, .{ | 183335 | }); |
| 183225 | .size = cg.memSize(val_ty, .general_purpose), | 183336 | switch (dst_mem.mod) { |
| 183226 | }), | 183337 | .rm => try cg.asmMemoryImmediate(.{ ._, .mov }, dst_mem, val_op), |
| 183227 | val_op, | 183338 | .off => { |
| 183228 | ); | 183339 | while (try val.toReg(.rax, cg)) {} |
| | 183340 | try cg.asmMemoryRegister( |
| | 183341 | .{ ._, .mov }, |
| | 183342 | dst_mem, |
| | 183343 | registerAlias(.rax, @intCast(val_ty.abiSize(zcu))), |
| | 183344 | ); |
| | 183345 | }, |
| | 183346 | } |
| 183229 | }, | 183347 | }, |
| 183230 | .eflags => |cc| { | 183348 | .eflags => |cc| { |
| 183231 | // hack around linker relocation bugs | 183349 | // hack around linker relocation bugs |
| ... | @@ -183429,9 +183547,15 @@ const Temp = struct { | ... | @@ -183429,9 +183547,15 @@ const Temp = struct { |
| 183429 | .lea_nav, .lea_uav, .lea_lazy_sym => if (dst_rc != .general_purpose) | 183547 | .lea_nav, .lea_uav, .lea_lazy_sym => if (dst_rc != .general_purpose) |
| 183430 | while (try ptr.toRegClass(false, .general_purpose, cg)) {}, | 183548 | while (try ptr.toRegClass(false, .general_purpose, cg)) {}, |
| 183431 | } | 183549 | } |
| 183432 | try strat.read(cg, dst_reg, try ptr.tracking(cg).short.deref().mem(cg, .{ | 183550 | const src_mem = try ptr.tracking(cg).short.deref().mem(cg, .{ |
| 183433 | .size = cg.memSize(dst_ty, dst_rc), | 183551 | .size = cg.memSize(dst_ty, dst_rc), |
| 183434 | })); | 183552 | }); |
| | 183553 | if (src_mem.mod == .off and dst_reg.id() != comptime Register.rax.id()) { |
| | 183554 | const tmp_reg = Register.rax.toSize(dst_reg.size(), cg.target); |
| | 183555 | try cg.register_manager.getKnownReg(.rax, null); |
| | 183556 | try strat.read(cg, tmp_reg, src_mem); |
| | 183557 | try cg.asmRegisterRegister(.{ ._, .mov }, dst_reg, tmp_reg); |
| | 183558 | } else try strat.read(cg, dst_reg, src_mem); |
| 183435 | } | 183559 | } |
| 183436 | | 183560 | |
| 183437 | fn storeRegs(ptr: *Temp, src_ty: Type, src_regs: []const Register, cg: *CodeGen) InnerError!void { | 183561 | fn storeRegs(ptr: *Temp, src_ty: Type, src_regs: []const Register, cg: *CodeGen) InnerError!void { |
| ... | @@ -183500,7 +183624,7 @@ const Temp = struct { | ... | @@ -183500,7 +183624,7 @@ const Temp = struct { |
| 183500 | .lea_nav, .lea_uav, .lea_lazy_sym => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, | 183624 | .lea_nav, .lea_uav, .lea_lazy_sym => while (try ptr.toRegClass(false, .general_purpose, cg)) {}, |
| 183501 | } | 183625 | } |
| 183502 | const strat = try cg.moveStrategy(part_ty, src_rc, false); | 183626 | const strat = try cg.moveStrategy(part_ty, src_rc, false); |
| 183503 | try strat.write(cg, try ptr.tracking(cg).short.deref().mem(cg, .{ | 183627 | const dst_mem = try ptr.tracking(cg).short.deref().mem(cg, .{ |
| 183504 | .size = switch (src_rc) { | 183628 | .size = switch (src_rc) { |
| 183505 | else => .fromBitSize(8 * part_size), | 183629 | else => .fromBitSize(8 * part_size), |
| 183506 | .x87 => switch (abi.classifySystemV(src_ty, zcu, cg.target, .other)[part_index]) { | 183630 | .x87 => switch (abi.classifySystemV(src_ty, zcu, cg.target, .other)[part_index]) { |
| ... | @@ -183512,7 +183636,13 @@ const Temp = struct { | ... | @@ -183512,7 +183636,13 @@ const Temp = struct { |
| 183512 | }, | 183636 | }, |
| 183513 | }, | 183637 | }, |
| 183514 | .disp = part_disp, | 183638 | .disp = part_disp, |
| 183515 | }), registerAlias(src_reg, part_size)); | 183639 | }); |
| | 183640 | if (dst_mem.mod == .off and src_reg.id() != comptime Register.rax.id()) { |
| | 183641 | const tmp_reg = registerAlias(.rax, part_size); |
| | 183642 | try cg.register_manager.getKnownReg(.rax, null); |
| | 183643 | try cg.asmRegisterRegister(.{ ._, .mov }, tmp_reg, registerAlias(src_reg, part_size)); |
| | 183644 | try strat.write(cg, dst_mem, tmp_reg); |
| | 183645 | } else try strat.write(cg, dst_mem, registerAlias(src_reg, part_size)); |
| 183516 | } else { | 183646 | } else { |
| 183517 | const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size); | 183647 | const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size); |
| 183518 | const frame_index = try cg.allocFrameIndex(.init(.{ | 183648 | const frame_index = try cg.allocFrameIndex(.init(.{ |
Jacob Young <jacobly@ziglang.org> 2026-10-01 12:46:25-04:00
Andrew Kelley <andrew@ziglang.org> 2026-10-01 23:30:21+02:00