authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-04-29 17:58:09-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-01 19:22:52-04:00
log3c2636a83dbb964f80f93131b30222ad3889a7e9
tree8bf9bc7eb7eae2f26f4fd511b2a09a492ec1db1d
parentc3889600424e3720dc07b22397129f82f2111d72

x86_64: implement more forms of wide mul with overflow

1 files changed, 122 insertions(+), 141 deletions(-)

src/arch/x86_64/CodeGen.zig+122-141
......@@ -2434,12 +2434,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
24342434
24352435 const frame_index =
24362436 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2437 try self.genSetFrameTruncatedOverflowCompare(
2438 tuple_ty,
2439 frame_index,
2440 partial_mcv.register,
2441 cc,
2442 );
2437 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
24432438 break :result .{ .load_frame = .{ .index = frame_index } };
24442439 },
24452440 else => unreachable,
......@@ -2511,12 +2506,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
25112506
25122507 const frame_index =
25132508 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2514 try self.genSetFrameTruncatedOverflowCompare(
2515 tuple_ty,
2516 frame_index,
2517 partial_mcv.register,
2518 cc,
2519 );
2509 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
25202510 break :result .{ .load_frame = .{ .index = frame_index } };
25212511 },
25222512 else => unreachable,
......@@ -2529,173 +2519,164 @@ fn genSetFrameTruncatedOverflowCompare(
25292519 self: *Self,
25302520 tuple_ty: Type,
25312521 frame_index: FrameIndex,
2532 reg: Register,
2522 src_mcv: MCValue,
25332523 cc: Condition,
25342524) !void {
2535 const reg_lock = self.register_manager.lockReg(reg);
2536 defer if (reg_lock) |lock| self.register_manager.unlockReg(lock);
2525 const src_lock = switch (src_mcv) {
2526 .register => |reg| self.register_manager.lockReg(reg),
2527 else => null,
2528 };
2529 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
25372530
25382531 const ty = tuple_ty.structFieldType(0);
25392532 const int_info = ty.intInfo(self.target.*);
2540 const extended_ty = switch (int_info.signedness) {
2541 .signed => Type.isize,
2542 .unsigned => ty,
2533
2534 var hi_limb_pl = Type.Payload.Bits{
2535 .base = .{ .tag = switch (int_info.signedness) {
2536 .signed => .int_signed,
2537 .unsigned => .int_unsigned,
2538 } },
2539 .data = (int_info.bits - 1) % 64 + 1,
25432540 };
2541 const hi_limb_ty = Type.initPayload(&hi_limb_pl.base);
25442542
2545 const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp);
2546 const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
2547 defer for (temp_regs_locks) |rreg| {
2548 self.register_manager.unlockReg(rreg);
2543 var rest_pl = Type.Payload.Bits{
2544 .base = .{ .tag = .int_unsigned },
2545 .data = int_info.bits - hi_limb_pl.data,
25492546 };
2547 const rest_ty = Type.initPayload(&rest_pl.base);
2548
2549 const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp);
2550 const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
2551 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
25502552
25512553 const overflow_reg = temp_regs[0];
25522554 try self.asmSetccRegister(overflow_reg.to8(), cc);
25532555
25542556 const scratch_reg = temp_regs[1];
2555 try self.genSetReg(scratch_reg, extended_ty, .{ .register = reg });
2556 try self.truncateRegister(ty, scratch_reg);
2557 try self.genBinOpMir(
2558 .cmp,
2559 extended_ty,
2560 .{ .register = reg },
2561 .{ .register = scratch_reg },
2562 );
2557 const hi_limb_off = if (int_info.bits <= 64) 0 else (int_info.bits - 1) / 64 * 8;
2558 const hi_limb_mcv = if (hi_limb_off > 0)
2559 src_mcv.address().offset(int_info.bits / 64 * 8).deref()
2560 else
2561 src_mcv;
2562 try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv);
2563 try self.truncateRegister(hi_limb_ty, scratch_reg);
2564 try self.genBinOpMir(.cmp, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
25632565
25642566 const eq_reg = temp_regs[2];
25652567 try self.asmSetccRegister(eq_reg.to8(), .ne);
2566 try self.genBinOpMir(
2567 .@"or",
2568 Type.u8,
2569 .{ .register = overflow_reg },
2570 .{ .register = eq_reg },
2571 );
2568 try self.genBinOpMir(.@"or", Type.u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
25722569
2570 const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*));
2571 if (hi_limb_off > 0) try self.genSetMem(.{ .frame = frame_index }, payload_off, rest_ty, src_mcv);
25732572 try self.genSetMem(
25742573 .{ .frame = frame_index },
2575 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
2576 tuple_ty.structFieldType(1),
2577 .{ .register = overflow_reg.to8() },
2574 payload_off + hi_limb_off,
2575 hi_limb_ty,
2576 .{ .register = scratch_reg },
25782577 );
25792578 try self.genSetMem(
25802579 .{ .frame = frame_index },
2581 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2582 ty,
2583 .{ .register = scratch_reg },
2580 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
2581 tuple_ty.structFieldType(1),
2582 .{ .register = overflow_reg.to8() },
25842583 );
25852584}
25862585
25872586fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
25882587 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
25892588 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
2590 const result: MCValue = result: {
2591 const dst_ty = self.air.typeOf(bin_op.lhs);
2592 switch (dst_ty.zigTypeTag()) {
2593 .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),
2594 .Int => {
2595 try self.spillEflagsIfOccupied();
2589 const dst_ty = self.air.typeOf(bin_op.lhs);
2590 const result: MCValue = switch (dst_ty.zigTypeTag()) {
2591 .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}),
2592 .Int => result: {
2593 try self.spillEflagsIfOccupied();
2594 try self.spillRegisters(&.{ .rax, .rdx });
25962595
2597 const dst_info = dst_ty.intInfo(self.target.*);
2598 const cc: Condition = switch (dst_info.signedness) {
2599 .unsigned => .c,
2600 .signed => .o,
2596 const dst_info = dst_ty.intInfo(self.target.*);
2597 const cc: Condition = switch (dst_info.signedness) {
2598 .unsigned => .c,
2599 .signed => .o,
2600 };
2601
2602 const lhs_active_bits = self.activeIntBits(bin_op.lhs);
2603 const rhs_active_bits = self.activeIntBits(bin_op.rhs);
2604 var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {
2605 .signed => .int_signed,
2606 .unsigned => .int_unsigned,
2607 } }, .data = math.max3(lhs_active_bits, rhs_active_bits, dst_info.bits / 2) };
2608 const src_ty = Type.initPayload(&src_pl.base);
2609
2610 const lhs = try self.resolveInst(bin_op.lhs);
2611 const rhs = try self.resolveInst(bin_op.rhs);
2612
2613 const tuple_ty = self.air.typeOfIndex(inst);
2614 const extra_bits = if (dst_info.bits <= 64)
2615 self.regExtraBits(dst_ty)
2616 else
2617 dst_info.bits % 64;
2618 const partial_mcv = if (dst_info.signedness == .signed and extra_bits > 0) dst: {
2619 const rhs_lock: ?RegisterLock = switch (rhs) {
2620 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2621 else => null,
26012622 };
2623 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
26022624
2603 const tuple_ty = self.air.typeOfIndex(inst);
2604 if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) {
2605 var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) {
2606 .signed => .int_signed,
2607 .unsigned => .int_unsigned,
2608 } }, .data = math.max3(
2609 self.activeIntBits(bin_op.lhs),
2610 self.activeIntBits(bin_op.rhs),
2611 dst_info.bits / 2,
2612 ) };
2613 const src_ty = Type.initPayload(&src_pl.base);
2625 const dst_reg: Register = blk: {
2626 if (lhs.isRegister()) break :blk lhs.register;
2627 break :blk try self.copyToTmpRegister(dst_ty, lhs);
2628 };
2629 const dst_mcv = MCValue{ .register = dst_reg };
2630 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
2631 defer self.register_manager.unlockReg(dst_reg_lock);
26142632
2615 try self.spillRegisters(&.{ .rax, .rdx });
2616 const lhs = try self.resolveInst(bin_op.lhs);
2617 const rhs = try self.resolveInst(bin_op.rhs);
2633 const rhs_mcv: MCValue = blk: {
2634 if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;
2635 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };
2636 };
2637 const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
2638 .register => |reg| self.register_manager.lockReg(reg),
2639 else => null,
2640 };
2641 defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock);
26182642
2619 const partial_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs);
2620 switch (partial_mcv) {
2621 .register => |reg| {
2622 self.eflags_inst = inst;
2623 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
2624 },
2625 else => {},
2626 }
2643 try self.genIntMulComplexOpMir(Type.isize, dst_mcv, rhs_mcv);
2644 break :dst dst_mcv;
2645 } else try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs);
26272646
2628 // For now, this is the only supported multiply that doesn't fit in a register.
2629 assert(dst_info.bits == 128 and src_pl.data == 64);
2647 switch (partial_mcv) {
2648 .register => |reg| if (extra_bits == 0) {
2649 self.eflags_inst = inst;
2650 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
2651 } else {
26302652 const frame_index =
26312653 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2632 try self.genSetMem(
2633 .{ .frame = frame_index },
2634 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
2635 tuple_ty.structFieldType(1),
2636 .{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit
2637 );
2638 try self.genSetMem(
2639 .{ .frame = frame_index },
2640 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2641 tuple_ty.structFieldType(0),
2642 partial_mcv,
2643 );
2654 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
26442655 break :result .{ .load_frame = .{ .index = frame_index } };
2645 }
2646
2647 const dst_reg: Register = dst_reg: {
2648 switch (dst_info.signedness) {
2649 .signed => {
2650 const lhs = try self.resolveInst(bin_op.lhs);
2651 const rhs = try self.resolveInst(bin_op.rhs);
2652
2653 const rhs_lock: ?RegisterLock = switch (rhs) {
2654 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2655 else => null,
2656 };
2657 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
2658
2659 const dst_reg: Register = blk: {
2660 if (lhs.isRegister()) break :blk lhs.register;
2661 break :blk try self.copyToTmpRegister(dst_ty, lhs);
2662 };
2663 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
2664 defer self.register_manager.unlockReg(dst_reg_lock);
2665
2666 const rhs_mcv: MCValue = blk: {
2667 if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;
2668 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };
2669 };
2670 const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
2671 .register => |reg| self.register_manager.lockReg(reg),
2672 else => null,
2673 };
2674 defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock);
2675
2676 try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv);
2677
2678 break :dst_reg dst_reg;
2679 },
2680 .unsigned => {
2681 try self.spillRegisters(&.{ .rax, .rdx });
2682
2683 const lhs = try self.resolveInst(bin_op.lhs);
2684 const rhs = try self.resolveInst(bin_op.rhs);
2685
2686 const dst_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, dst_ty, lhs, rhs);
2687 break :dst_reg dst_mcv.register;
2688 },
2689 }
2690 };
2656 },
2657 // For now, this is the only supported multiply that doesn't fit in a register.
2658 else => assert(dst_info.bits <= 128 and src_pl.data == 64),
2659 }
26912660
2692 const frame_index =
2693 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2694 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, dst_reg, cc);
2695 break :result .{ .load_frame = .{ .index = frame_index } };
2696 },
2697 else => unreachable,
2698 }
2661 const frame_index =
2662 try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*));
2663 if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
2664 try self.genSetMem(
2665 .{ .frame = frame_index },
2666 @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)),
2667 tuple_ty.structFieldType(0),
2668 partial_mcv,
2669 );
2670 try self.genSetMem(
2671 .{ .frame = frame_index },
2672 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
2673 tuple_ty.structFieldType(1),
2674 .{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit
2675 );
2676 } else try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
2677 break :result .{ .load_frame = .{ .index = frame_index } };
2678 },
2679 else => unreachable,
26992680 };
27002681 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
27012682}