| ... | @@ -2434,12 +2434,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2434,12 +2434,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 2434 | | 2434 | |
| 2435 | const frame_index = | 2435 | const frame_index = |
| 2436 | try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*)); | 2436 | try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*)); |
| 2437 | try self.genSetFrameTruncatedOverflowCompare( | 2437 | try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc); |
| 2438 | tuple_ty, | | |
| 2439 | frame_index, | | |
| 2440 | partial_mcv.register, | | |
| 2441 | cc, | | |
| 2442 | ); | | |
| 2443 | break :result .{ .load_frame = .{ .index = frame_index } }; | 2438 | break :result .{ .load_frame = .{ .index = frame_index } }; |
| 2444 | }, | 2439 | }, |
| 2445 | else => unreachable, | 2440 | else => unreachable, |
| ... | @@ -2511,12 +2506,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -2511,12 +2506,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 2511 | | 2506 | |
| 2512 | const frame_index = | 2507 | const frame_index = |
| 2513 | try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*)); | 2508 | try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*)); |
| 2514 | try self.genSetFrameTruncatedOverflowCompare( | 2509 | try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc); |
| 2515 | tuple_ty, | | |
| 2516 | frame_index, | | |
| 2517 | partial_mcv.register, | | |
| 2518 | cc, | | |
| 2519 | ); | | |
| 2520 | break :result .{ .load_frame = .{ .index = frame_index } }; | 2510 | break :result .{ .load_frame = .{ .index = frame_index } }; |
| 2521 | }, | 2511 | }, |
| 2522 | else => unreachable, | 2512 | else => unreachable, |
| ... | @@ -2529,173 +2519,164 @@ fn genSetFrameTruncatedOverflowCompare( | ... | @@ -2529,173 +2519,164 @@ fn genSetFrameTruncatedOverflowCompare( |
| 2529 | self: *Self, | 2519 | self: *Self, |
| 2530 | tuple_ty: Type, | 2520 | tuple_ty: Type, |
| 2531 | frame_index: FrameIndex, | 2521 | frame_index: FrameIndex, |
| 2532 | reg: Register, | 2522 | src_mcv: MCValue, |
| 2533 | cc: Condition, | 2523 | cc: Condition, |
| 2534 | ) !void { | 2524 | ) !void { |
| 2535 | const reg_lock = self.register_manager.lockReg(reg); | 2525 | const src_lock = switch (src_mcv) { |
| 2536 | defer if (reg_lock) |lock| self.register_manager.unlockReg(lock); | 2526 | .register => |reg| self.register_manager.lockReg(reg), |
| | 2527 | else => null, |
| | 2528 | }; |
| | 2529 | defer if (src_lock) |lock| self.register_manager.unlockReg(lock); |
| 2537 | | 2530 | |
| 2538 | const ty = tuple_ty.structFieldType(0); | 2531 | const ty = tuple_ty.structFieldType(0); |
| 2539 | const int_info = ty.intInfo(self.target.*); | 2532 | const int_info = ty.intInfo(self.target.*); |
| 2540 | const extended_ty = switch (int_info.signedness) { | 2533 | |
| 2541 | .signed => Type.isize, | 2534 | var hi_limb_pl = Type.Payload.Bits{ |
| 2542 | .unsigned => ty, | 2535 | .base = .{ .tag = switch (int_info.signedness) { |
| | 2536 | .signed => .int_signed, |
| | 2537 | .unsigned => .int_unsigned, |
| | 2538 | } }, |
| | 2539 | .data = (int_info.bits - 1) % 64 + 1, |
| 2543 | }; | 2540 | }; |
| | 2541 | const hi_limb_ty = Type.initPayload(&hi_limb_pl.base); |
| 2544 | | 2542 | |
| 2545 | const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp); | 2543 | var rest_pl = Type.Payload.Bits{ |
| 2546 | const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs); | 2544 | .base = .{ .tag = .int_unsigned }, |
| 2547 | defer for (temp_regs_locks) |rreg| { | 2545 | .data = int_info.bits - hi_limb_pl.data, |
| 2548 | self.register_manager.unlockReg(rreg); | | |
| 2549 | }; | 2546 | }; |
| | 2547 | const rest_ty = Type.initPayload(&rest_pl.base); |
| | 2548 | |
| | 2549 | const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp); |
| | 2550 | const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs); |
| | 2551 | defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); |
| 2550 | | 2552 | |
| 2551 | const overflow_reg = temp_regs[0]; | 2553 | const overflow_reg = temp_regs[0]; |
| 2552 | try self.asmSetccRegister(overflow_reg.to8(), cc); | 2554 | try self.asmSetccRegister(overflow_reg.to8(), cc); |
| 2553 | | 2555 | |
| 2554 | const scratch_reg = temp_regs[1]; | 2556 | const scratch_reg = temp_regs[1]; |
| 2555 | try self.genSetReg(scratch_reg, extended_ty, .{ .register = reg }); | 2557 | const hi_limb_off = if (int_info.bits <= 64) 0 else (int_info.bits - 1) / 64 * 8; |
| 2556 | try self.truncateRegister(ty, scratch_reg); | 2558 | const hi_limb_mcv = if (hi_limb_off > 0) |
| 2557 | try self.genBinOpMir( | 2559 | src_mcv.address().offset(int_info.bits / 64 * 8).deref() |
| 2558 | .cmp, | 2560 | else |
| 2559 | extended_ty, | 2561 | src_mcv; |
| 2560 | .{ .register = reg }, | 2562 | try self.genSetReg(scratch_reg, hi_limb_ty, hi_limb_mcv); |
| 2561 | .{ .register = scratch_reg }, | 2563 | try self.truncateRegister(hi_limb_ty, scratch_reg); |
| 2562 | ); | 2564 | try self.genBinOpMir(.cmp, hi_limb_ty, .{ .register = scratch_reg }, hi_limb_mcv); |
| 2563 | | 2565 | |
| 2564 | const eq_reg = temp_regs[2]; | 2566 | const eq_reg = temp_regs[2]; |
| 2565 | try self.asmSetccRegister(eq_reg.to8(), .ne); | 2567 | try self.asmSetccRegister(eq_reg.to8(), .ne); |
| 2566 | try self.genBinOpMir( | 2568 | try self.genBinOpMir(.@"or", Type.u8, .{ .register = overflow_reg }, .{ .register = eq_reg }); |
| 2567 | .@"or", | | |
| 2568 | Type.u8, | | |
| 2569 | .{ .register = overflow_reg }, | | |
| 2570 | .{ .register = eq_reg }, | | |
| 2571 | ); | | |
| 2572 | | 2569 | |
| | 2570 | const payload_off = @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)); |
| | 2571 | if (hi_limb_off > 0) try self.genSetMem(.{ .frame = frame_index }, payload_off, rest_ty, src_mcv); |
| 2573 | try self.genSetMem( | 2572 | try self.genSetMem( |
| 2574 | .{ .frame = frame_index }, | 2573 | .{ .frame = frame_index }, |
| 2575 | @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)), | 2574 | payload_off + hi_limb_off, |
| 2576 | tuple_ty.structFieldType(1), | 2575 | hi_limb_ty, |
| 2577 | .{ .register = overflow_reg.to8() }, | 2576 | .{ .register = scratch_reg }, |
| 2578 | ); | 2577 | ); |
| 2579 | try self.genSetMem( | 2578 | try self.genSetMem( |
| 2580 | .{ .frame = frame_index }, | 2579 | .{ .frame = frame_index }, |
| 2581 | @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)), | 2580 | @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)), |
| 2582 | ty, | 2581 | tuple_ty.structFieldType(1), |
| 2583 | .{ .register = scratch_reg }, | 2582 | .{ .register = overflow_reg.to8() }, |
| 2584 | ); | 2583 | ); |
| 2585 | } | 2584 | } |
| 2586 | | 2585 | |
| 2587 | fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { | 2586 | fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 2588 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; | 2587 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 2589 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | 2588 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 2590 | const result: MCValue = result: { | 2589 | const dst_ty = self.air.typeOf(bin_op.lhs); |
| 2591 | const dst_ty = self.air.typeOf(bin_op.lhs); | 2590 | const result: MCValue = switch (dst_ty.zigTypeTag()) { |
| 2592 | switch (dst_ty.zigTypeTag()) { | 2591 | .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}), |
| 2593 | .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}), | 2592 | .Int => result: { |
| 2594 | .Int => { | 2593 | try self.spillEflagsIfOccupied(); |
| 2595 | try self.spillEflagsIfOccupied(); | 2594 | try self.spillRegisters(&.{ .rax, .rdx }); |
| 2596 | | 2595 | |
| 2597 | const dst_info = dst_ty.intInfo(self.target.*); | 2596 | const dst_info = dst_ty.intInfo(self.target.*); |
| 2598 | const cc: Condition = switch (dst_info.signedness) { | 2597 | const cc: Condition = switch (dst_info.signedness) { |
| 2599 | .unsigned => .c, | 2598 | .unsigned => .c, |
| 2600 | .signed => .o, | 2599 | .signed => .o, |
| | 2600 | }; |
| | 2601 | |
| | 2602 | const lhs_active_bits = self.activeIntBits(bin_op.lhs); |
| | 2603 | const rhs_active_bits = self.activeIntBits(bin_op.rhs); |
| | 2604 | var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) { |
| | 2605 | .signed => .int_signed, |
| | 2606 | .unsigned => .int_unsigned, |
| | 2607 | } }, .data = math.max3(lhs_active_bits, rhs_active_bits, dst_info.bits / 2) }; |
| | 2608 | const src_ty = Type.initPayload(&src_pl.base); |
| | 2609 | |
| | 2610 | const lhs = try self.resolveInst(bin_op.lhs); |
| | 2611 | const rhs = try self.resolveInst(bin_op.rhs); |
| | 2612 | |
| | 2613 | const tuple_ty = self.air.typeOfIndex(inst); |
| | 2614 | const extra_bits = if (dst_info.bits <= 64) |
| | 2615 | self.regExtraBits(dst_ty) |
| | 2616 | else |
| | 2617 | dst_info.bits % 64; |
| | 2618 | const partial_mcv = if (dst_info.signedness == .signed and extra_bits > 0) dst: { |
| | 2619 | const rhs_lock: ?RegisterLock = switch (rhs) { |
| | 2620 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| | 2621 | else => null, |
| 2601 | }; | 2622 | }; |
| | 2623 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); |
| 2602 | | 2624 | |
| 2603 | const tuple_ty = self.air.typeOfIndex(inst); | 2625 | const dst_reg: Register = blk: { |
| 2604 | if (dst_info.bits >= 8 and math.isPowerOfTwo(dst_info.bits)) { | 2626 | if (lhs.isRegister()) break :blk lhs.register; |
| 2605 | var src_pl = Type.Payload.Bits{ .base = .{ .tag = switch (dst_info.signedness) { | 2627 | break :blk try self.copyToTmpRegister(dst_ty, lhs); |
| 2606 | .signed => .int_signed, | 2628 | }; |
| 2607 | .unsigned => .int_unsigned, | 2629 | const dst_mcv = MCValue{ .register = dst_reg }; |
| 2608 | } }, .data = math.max3( | 2630 | const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); |
| 2609 | self.activeIntBits(bin_op.lhs), | 2631 | defer self.register_manager.unlockReg(dst_reg_lock); |
| 2610 | self.activeIntBits(bin_op.rhs), | | |
| 2611 | dst_info.bits / 2, | | |
| 2612 | ) }; | | |
| 2613 | const src_ty = Type.initPayload(&src_pl.base); | | |
| 2614 | | 2632 | |
| 2615 | try self.spillRegisters(&.{ .rax, .rdx }); | 2633 | const rhs_mcv: MCValue = blk: { |
| 2616 | const lhs = try self.resolveInst(bin_op.lhs); | 2634 | if (rhs.isRegister() or rhs.isMemory()) break :blk rhs; |
| 2617 | const rhs = try self.resolveInst(bin_op.rhs); | 2635 | break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) }; |
| | 2636 | }; |
| | 2637 | const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) { |
| | 2638 | .register => |reg| self.register_manager.lockReg(reg), |
| | 2639 | else => null, |
| | 2640 | }; |
| | 2641 | defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock); |
| 2618 | | 2642 | |
| 2619 | const partial_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs); | 2643 | try self.genIntMulComplexOpMir(Type.isize, dst_mcv, rhs_mcv); |
| 2620 | switch (partial_mcv) { | 2644 | break :dst dst_mcv; |
| 2621 | .register => |reg| { | 2645 | } else try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs); |
| 2622 | self.eflags_inst = inst; | | |
| 2623 | break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } }; | | |
| 2624 | }, | | |
| 2625 | else => {}, | | |
| 2626 | } | | |
| 2627 | | 2646 | |
| 2628 | // For now, this is the only supported multiply that doesn't fit in a register. | 2647 | switch (partial_mcv) { |
| 2629 | assert(dst_info.bits == 128 and src_pl.data == 64); | 2648 | .register => |reg| if (extra_bits == 0) { |
| | 2649 | self.eflags_inst = inst; |
| | 2650 | break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } }; |
| | 2651 | } else { |
| 2630 | const frame_index = | 2652 | const frame_index = |
| 2631 | try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*)); | 2653 | try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*)); |
| 2632 | try self.genSetMem( | 2654 | try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc); |
| 2633 | .{ .frame = frame_index }, | | |
| 2634 | @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)), | | |
| 2635 | tuple_ty.structFieldType(1), | | |
| 2636 | .{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit | | |
| 2637 | ); | | |
| 2638 | try self.genSetMem( | | |
| 2639 | .{ .frame = frame_index }, | | |
| 2640 | @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)), | | |
| 2641 | tuple_ty.structFieldType(0), | | |
| 2642 | partial_mcv, | | |
| 2643 | ); | | |
| 2644 | break :result .{ .load_frame = .{ .index = frame_index } }; | 2655 | break :result .{ .load_frame = .{ .index = frame_index } }; |
| 2645 | } | 2656 | }, |
| 2646 | | 2657 | // For now, this is the only supported multiply that doesn't fit in a register. |
| 2647 | const dst_reg: Register = dst_reg: { | 2658 | else => assert(dst_info.bits <= 128 and src_pl.data == 64), |
| 2648 | switch (dst_info.signedness) { | 2659 | } |
| 2649 | .signed => { | | |
| 2650 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 2651 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 2652 | | | |
| 2653 | const rhs_lock: ?RegisterLock = switch (rhs) { | | |
| 2654 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | | |
| 2655 | else => null, | | |
| 2656 | }; | | |
| 2657 | defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 2658 | | | |
| 2659 | const dst_reg: Register = blk: { | | |
| 2660 | if (lhs.isRegister()) break :blk lhs.register; | | |
| 2661 | break :blk try self.copyToTmpRegister(dst_ty, lhs); | | |
| 2662 | }; | | |
| 2663 | const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); | | |
| 2664 | defer self.register_manager.unlockReg(dst_reg_lock); | | |
| 2665 | | | |
| 2666 | const rhs_mcv: MCValue = blk: { | | |
| 2667 | if (rhs.isRegister() or rhs.isMemory()) break :blk rhs; | | |
| 2668 | break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) }; | | |
| 2669 | }; | | |
| 2670 | const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) { | | |
| 2671 | .register => |reg| self.register_manager.lockReg(reg), | | |
| 2672 | else => null, | | |
| 2673 | }; | | |
| 2674 | defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 2675 | | | |
| 2676 | try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv); | | |
| 2677 | | | |
| 2678 | break :dst_reg dst_reg; | | |
| 2679 | }, | | |
| 2680 | .unsigned => { | | |
| 2681 | try self.spillRegisters(&.{ .rax, .rdx }); | | |
| 2682 | | | |
| 2683 | const lhs = try self.resolveInst(bin_op.lhs); | | |
| 2684 | const rhs = try self.resolveInst(bin_op.rhs); | | |
| 2685 | | | |
| 2686 | const dst_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, dst_ty, lhs, rhs); | | |
| 2687 | break :dst_reg dst_mcv.register; | | |
| 2688 | }, | | |
| 2689 | } | | |
| 2690 | }; | | |
| 2691 | | 2660 | |
| 2692 | const frame_index = | 2661 | const frame_index = |
| 2693 | try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*)); | 2662 | try self.allocFrameIndex(FrameAlloc.initType(tuple_ty, self.target.*)); |
| 2694 | try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, dst_reg, cc); | 2663 | if (dst_info.bits >= lhs_active_bits + rhs_active_bits) { |
| 2695 | break :result .{ .load_frame = .{ .index = frame_index } }; | 2664 | try self.genSetMem( |
| 2696 | }, | 2665 | .{ .frame = frame_index }, |
| 2697 | else => unreachable, | 2666 | @intCast(i32, tuple_ty.structFieldOffset(0, self.target.*)), |
| 2698 | } | 2667 | tuple_ty.structFieldType(0), |
| | 2668 | partial_mcv, |
| | 2669 | ); |
| | 2670 | try self.genSetMem( |
| | 2671 | .{ .frame = frame_index }, |
| | 2672 | @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)), |
| | 2673 | tuple_ty.structFieldType(1), |
| | 2674 | .{ .immediate = 0 }, // overflow is impossible for 64-bit*64-bit -> 128-bit |
| | 2675 | ); |
| | 2676 | } else try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc); |
| | 2677 | break :result .{ .load_frame = .{ .index = frame_index } }; |
| | 2678 | }, |
| | 2679 | else => unreachable, |
| 2699 | }; | 2680 | }; |
| 2700 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); | 2681 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 2701 | } | 2682 | } |
Jacob Young <jacobly@ziglang.org> 2023-04-29 17:58:09-04:00
src