| ... | ... | @@ -123,6 +123,16 @@ const MCValue = union(enum) { |
| 123 | 123 | immediate: u64, |
| 124 | 124 | /// The value is in a target-specific register. |
| 125 | 125 | register: Register, |
| 126 | /// The value is a tuple { wrapped, overflow } where |
| 127 | /// wrapped is stored in the register and the overflow bit is |
| 128 | /// stored in the C (signed) or V (unsigned) flag of the CCR. |
| 129 | /// |
| 130 | /// This MCValue is only generated by a add_with_overflow or |
| 131 | /// sub_with_overflow instruction operating on 32- or 64-bit values. |
| 132 | register_with_overflow: struct { |
| 133 | reg: Register, |
| 134 | flag: struct { cond: Instruction.ICondition, ccr: Instruction.CCR }, |
| 135 | }, |
| 126 | 136 | /// The value is in memory at a hard-coded address. |
| 127 | 137 | /// If the type is a pointer, it means the pointer address is at this memory location. |
| 128 | 138 | memory: u64, |
| ... | ... | @@ -131,12 +141,18 @@ const MCValue = union(enum) { |
| 131 | 141 | stack_offset: u32, |
| 132 | 142 | /// The value is a pointer to one of the stack variables (payload is stack offset). |
| 133 | 143 | ptr_stack_offset: u32, |
| 134 | | /// The value is in the compare flags assuming an unsigned operation, |
| 135 | | /// with this operator applied on top of it. |
| 136 | | compare_flags_unsigned: math.CompareOperator, |
| 137 | | /// The value is in the compare flags assuming a signed operation, |
| 138 | | /// with this operator applied on top of it. |
| 139 | | compare_flags_signed: math.CompareOperator, |
| 144 | /// The value is in the specified CCR assuming an unsigned operation, |
| 145 | /// with the operator applied on top of it. |
| 146 | compare_flags_unsigned: struct { |
| 147 | cmp: math.CompareOperator, |
| 148 | ccr: Instruction.CCR, |
| 149 | }, |
| 150 | /// The value is in the specified CCR assuming an signed operation, |
| 151 | /// with the operator applied on top of it. |
| 152 | compare_flags_signed: struct { |
| 153 | cmp: math.CompareOperator, |
| 154 | ccr: Instruction.CCR, |
| 155 | }, |
| 140 | 156 | |
| 141 | 157 | fn isMemory(mcv: MCValue) bool { |
| 142 | 158 | return switch (mcv) { |
| ... | ... | @@ -501,7 +517,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 501 | 517 | .shl_sat => @panic("TODO try self.airShlSat(inst)"), |
| 502 | 518 | .min => @panic("TODO try self.airMin(inst)"), |
| 503 | 519 | .max => @panic("TODO try self.airMax(inst)"), |
| 504 | | .slice => @panic("TODO try self.airSlice(inst)"), |
| 520 | .slice => try self.airSlice(inst), |
| 505 | 521 | |
| 506 | 522 | .sqrt, |
| 507 | 523 | .sin, |
| ... | ... | @@ -519,8 +535,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 519 | 535 | .trunc_float, |
| 520 | 536 | => @panic("TODO try self.airUnaryMath(inst)"), |
| 521 | 537 | |
| 522 | | .add_with_overflow => @panic("TODO try self.airAddWithOverflow(inst)"), |
| 523 | | .sub_with_overflow => @panic("TODO try self.airSubWithOverflow(inst)"), |
| 538 | .add_with_overflow => try self.airAddSubWithOverflow(inst), |
| 539 | .sub_with_overflow => try self.airAddSubWithOverflow(inst), |
| 524 | 540 | .mul_with_overflow => @panic("TODO try self.airMulWithOverflow(inst)"), |
| 525 | 541 | .shl_with_overflow => @panic("TODO try self.airShlWithOverflow(inst)"), |
| 526 | 542 | |
| ... | ... | @@ -570,14 +586,14 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 570 | 586 | .is_err_ptr => @panic("TODO try self.airIsErrPtr(inst)"), |
| 571 | 587 | .load => try self.airLoad(inst), |
| 572 | 588 | .loop => try self.airLoop(inst), |
| 573 | | .not => @panic("TODO try self.airNot(inst)"), |
| 589 | .not => try self.airNot(inst), |
| 574 | 590 | .ptrtoint => @panic("TODO try self.airPtrToInt(inst)"), |
| 575 | 591 | .ret => try self.airRet(inst), |
| 576 | 592 | .ret_load => try self.airRetLoad(inst), |
| 577 | 593 | .store => try self.airStore(inst), |
| 578 | 594 | .struct_field_ptr=> @panic("TODO try self.airStructFieldPtr(inst)"), |
| 579 | | .struct_field_val=> @panic("TODO try self.airStructFieldVal(inst)"), |
| 580 | | .array_to_slice => @panic("TODO try self.airArrayToSlice(inst)"), |
| 595 | .struct_field_val=> try self.airStructFieldVal(inst), |
| 596 | .array_to_slice => try self.airArrayToSlice(inst), |
| 581 | 597 | .int_to_float => @panic("TODO try self.airIntToFloat(inst)"), |
| 582 | 598 | .float_to_int => @panic("TODO try self.airFloatToInt(inst)"), |
| 583 | 599 | .cmpxchg_strong => @panic("TODO try self.airCmpxchg(inst)"), |
| ... | ... | @@ -647,7 +663,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 647 | 663 | .slice_elem_val => try self.airSliceElemVal(inst), |
| 648 | 664 | .slice_elem_ptr => @panic("TODO try self.airSliceElemPtr(inst)"), |
| 649 | 665 | .ptr_elem_val => @panic("TODO try self.airPtrElemVal(inst)"), |
| 650 | | .ptr_elem_ptr => @panic("TODO try self.airPtrElemPtr(inst)"), |
| 666 | .ptr_elem_ptr => try self.airPtrElemPtr(inst), |
| 651 | 667 | |
| 652 | 668 | .constant => unreachable, // excluded from function bodies |
| 653 | 669 | .const_ty => unreachable, // excluded from function bodies |
| ... | ... | @@ -666,13 +682,15 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 666 | 682 | |
| 667 | 683 | .wrap_optional => @panic("TODO try self.airWrapOptional(inst)"), |
| 668 | 684 | .wrap_errunion_payload => @panic("TODO try self.airWrapErrUnionPayload(inst)"), |
| 669 | | .wrap_errunion_err => @panic("TODO try self.airWrapErrUnionErr(inst)"), |
| 685 | .wrap_errunion_err => try self.airWrapErrUnionErr(inst), |
| 670 | 686 | |
| 671 | 687 | .wasm_memory_size => unreachable, |
| 672 | 688 | .wasm_memory_grow => unreachable, |
| 673 | 689 | // zig fmt: on |
| 674 | 690 | } |
| 675 | 691 | |
| 692 | assert(!self.register_manager.lockedRegsExist()); |
| 693 | |
| 676 | 694 | if (std.debug.runtime_safety) { |
| 677 | 695 | if (self.air_bookkeeping < old_air_bookkeeping + 1) { |
| 678 | 696 | std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, air_tags[inst] }); |
| ... | ... | @@ -681,11 +699,112 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 681 | 699 | } |
| 682 | 700 | } |
| 683 | 701 | |
| 702 | fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { |
| 703 | const tag = self.air.instructions.items(.tag)[inst]; |
| 704 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 705 | const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 706 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { |
| 707 | const lhs = try self.resolveInst(extra.lhs); |
| 708 | const rhs = try self.resolveInst(extra.rhs); |
| 709 | const lhs_ty = self.air.typeOf(extra.lhs); |
| 710 | const rhs_ty = self.air.typeOf(extra.rhs); |
| 711 | |
| 712 | switch (lhs_ty.zigTypeTag()) { |
| 713 | .Vector => return self.fail("TODO implement add_with_overflow/sub_with_overflow for vectors", .{}), |
| 714 | .Int => { |
| 715 | const mod = self.bin_file.options.module.?; |
| 716 | assert(lhs_ty.eql(rhs_ty, mod)); |
| 717 | const int_info = lhs_ty.intInfo(self.target.*); |
| 718 | switch (int_info.bits) { |
| 719 | 32, 64 => { |
| 720 | // Only say yes if the operation is |
| 721 | // commutative, i.e. we can swap both of the |
| 722 | // operands |
| 723 | const lhs_immediate_ok = switch (tag) { |
| 724 | .add_with_overflow => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), |
| 725 | .sub_with_overflow => false, |
| 726 | else => unreachable, |
| 727 | }; |
| 728 | const rhs_immediate_ok = switch (tag) { |
| 729 | .add_with_overflow, |
| 730 | .sub_with_overflow, |
| 731 | => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12), |
| 732 | else => unreachable, |
| 733 | }; |
| 734 | |
| 735 | const mir_tag: Mir.Inst.Tag = switch (tag) { |
| 736 | .add_with_overflow => .addcc, |
| 737 | .sub_with_overflow => .subcc, |
| 738 | else => unreachable, |
| 739 | }; |
| 740 | |
| 741 | try self.spillCompareFlagsIfOccupied(); |
| 742 | self.compare_flags_inst = inst; |
| 743 | |
| 744 | const dest = blk: { |
| 745 | if (rhs_immediate_ok) { |
| 746 | break :blk try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, null); |
| 747 | } else if (lhs_immediate_ok) { |
| 748 | // swap lhs and rhs |
| 749 | break :blk try self.binOpImmediate(mir_tag, rhs, lhs, rhs_ty, true, null); |
| 750 | } else { |
| 751 | break :blk try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, null); |
| 752 | } |
| 753 | }; |
| 754 | |
| 755 | const cond = switch (int_info.signedness) { |
| 756 | .unsigned => switch (tag) { |
| 757 | .add_with_overflow => Instruction.ICondition.cs, |
| 758 | .sub_with_overflow => Instruction.ICondition.cc, |
| 759 | else => unreachable, |
| 760 | }, |
| 761 | .signed => Instruction.ICondition.vs, |
| 762 | }; |
| 763 | |
| 764 | const ccr = switch (int_info.bits) { |
| 765 | 32 => Instruction.CCR.icc, |
| 766 | 64 => Instruction.CCR.xcc, |
| 767 | else => unreachable, |
| 768 | }; |
| 769 | |
| 770 | break :result MCValue{ .register_with_overflow = .{ |
| 771 | .reg = dest.register, |
| 772 | .flag = .{ .cond = cond, .ccr = ccr }, |
| 773 | } }; |
| 774 | }, |
| 775 | else => return self.fail("TODO overflow operations on other integer sizes", .{}), |
| 776 | } |
| 777 | }, |
| 778 | else => unreachable, |
| 779 | } |
| 780 | }; |
| 781 | return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); |
| 782 | } |
| 783 | |
| 684 | 784 | fn airAlloc(self: *Self, inst: Air.Inst.Index) !void { |
| 685 | 785 | const stack_offset = try self.allocMemPtr(inst); |
| 686 | 786 | return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none }); |
| 687 | 787 | } |
| 688 | 788 | |
| 789 | fn airArrayToSlice(self: *Self, inst: Air.Inst.Index) !void { |
| 790 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; |
| 791 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { |
| 792 | const ptr_ty = self.air.typeOf(ty_op.operand); |
| 793 | const ptr = try self.resolveInst(ty_op.operand); |
| 794 | const array_ty = ptr_ty.childType(); |
| 795 | const array_len = @intCast(u32, array_ty.arrayLen()); |
| 796 | |
| 797 | const ptr_bits = self.target.cpu.arch.ptrBitWidth(); |
| 798 | const ptr_bytes = @divExact(ptr_bits, 8); |
| 799 | |
| 800 | const stack_offset = try self.allocMem(inst, ptr_bytes * 2, ptr_bytes * 2); |
| 801 | try self.genSetStack(ptr_ty, stack_offset, ptr); |
| 802 | try self.genSetStack(Type.initTag(.usize), stack_offset - ptr_bytes, .{ .immediate = array_len }); |
| 803 | break :result MCValue{ .stack_offset = stack_offset }; |
| 804 | }; |
| 805 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 806 | } |
| 807 | |
| 689 | 808 | fn airAsm(self: *Self, inst: Air.Inst.Index) !void { |
| 690 | 809 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 691 | 810 | const extra = self.air.extraData(Air.Asm, ty_pl.payload); |
| ... | ... | @@ -896,9 +1015,14 @@ fn airBlock(self: *Self, inst: Air.Inst.Index) !void { |
| 896 | 1015 | const relocs = &self.blocks.getPtr(inst).?.relocs; |
| 897 | 1016 | if (relocs.items.len > 0 and relocs.items[relocs.items.len - 1] == self.mir_instructions.len - 1) { |
| 898 | 1017 | // If the last Mir instruction is the last relocation (which |
| 899 | | // would just jump one instruction further), it can be safely |
| 1018 | // would just jump two instruction further), it can be safely |
| 900 | 1019 | // removed |
| 901 | | self.mir_instructions.orderedRemove(relocs.pop()); |
| 1020 | const index = relocs.pop(); |
| 1021 | |
| 1022 | // First, remove the delay slot, then remove |
| 1023 | // the branch instruction itself. |
| 1024 | self.mir_instructions.orderedRemove(index + 1); |
| 1025 | self.mir_instructions.orderedRemove(index); |
| 902 | 1026 | } |
| 903 | 1027 | for (relocs.items) |reloc| { |
| 904 | 1028 | try self.performReloc(reloc); |
| ... | ... | @@ -945,6 +1069,11 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. |
| 945 | 1069 | |
| 946 | 1070 | var info = try self.resolveCallingConventionValues(fn_ty, .caller); |
| 947 | 1071 | defer info.deinit(self); |
| 1072 | |
| 1073 | // CCR is volatile across function calls |
| 1074 | // (SCD 2.4.1, page 3P-10) |
| 1075 | try self.spillCompareFlagsIfOccupied(); |
| 1076 | |
| 948 | 1077 | for (info.args) |mc_arg, arg_i| { |
| 949 | 1078 | const arg = args[arg_i]; |
| 950 | 1079 | const arg_ty = self.air.typeOf(arg); |
| ... | ... | @@ -952,13 +1081,6 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. |
| 952 | 1081 | |
| 953 | 1082 | switch (mc_arg) { |
| 954 | 1083 | .none => continue, |
| 955 | | .undef => unreachable, |
| 956 | | .immediate => unreachable, |
| 957 | | .unreach => unreachable, |
| 958 | | .dead => unreachable, |
| 959 | | .memory => unreachable, |
| 960 | | .compare_flags_signed => unreachable, |
| 961 | | .compare_flags_unsigned => unreachable, |
| 962 | 1084 | .register => |reg| { |
| 963 | 1085 | try self.register_manager.getReg(reg, null); |
| 964 | 1086 | try self.genSetReg(arg_ty, reg, arg_mcv); |
| ... | ... | @@ -969,6 +1091,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. |
| 969 | 1091 | .ptr_stack_offset => { |
| 970 | 1092 | return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{}); |
| 971 | 1093 | }, |
| 1094 | else => unreachable, |
| 972 | 1095 | } |
| 973 | 1096 | } |
| 974 | 1097 | |
| ... | ... | @@ -1089,8 +1212,8 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { |
| 1089 | 1212 | self.compare_flags_inst = inst; |
| 1090 | 1213 | |
| 1091 | 1214 | break :result switch (int_info.signedness) { |
| 1092 | | .signed => MCValue{ .compare_flags_signed = op }, |
| 1093 | | .unsigned => MCValue{ .compare_flags_unsigned = op }, |
| 1215 | .signed => MCValue{ .compare_flags_signed = .{ .cmp = op, .ccr = .xcc } }, |
| 1216 | .unsigned => MCValue{ .compare_flags_unsigned = .{ .cmp = op, .ccr = .xcc } }, |
| 1094 | 1217 | }; |
| 1095 | 1218 | } else { |
| 1096 | 1219 | return self.fail("TODO SPARCv9 cmp for ints > 64 bits", .{}); |
| ... | ... | @@ -1116,16 +1239,20 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { |
| 1116 | 1239 | .tag = .bpcc, |
| 1117 | 1240 | .data = .{ |
| 1118 | 1241 | .branch_predict_int = .{ |
| 1119 | | .ccr = .xcc, |
| 1242 | .ccr = switch (cond) { |
| 1243 | .compare_flags_signed => |cmp_op| cmp_op.ccr, |
| 1244 | .compare_flags_unsigned => |cmp_op| cmp_op.ccr, |
| 1245 | else => unreachable, |
| 1246 | }, |
| 1120 | 1247 | .cond = switch (cond) { |
| 1121 | 1248 | .compare_flags_signed => |cmp_op| blk: { |
| 1122 | 1249 | // Here we map to the opposite condition because the jump is to the false branch. |
| 1123 | | const condition = Instruction.ICondition.fromCompareOperatorSigned(cmp_op); |
| 1250 | const condition = Instruction.ICondition.fromCompareOperatorSigned(cmp_op.cmp); |
| 1124 | 1251 | break :blk condition.negate(); |
| 1125 | 1252 | }, |
| 1126 | 1253 | .compare_flags_unsigned => |cmp_op| blk: { |
| 1127 | 1254 | // Here we map to the opposite condition because the jump is to the false branch. |
| 1128 | | const condition = Instruction.ICondition.fromCompareOperatorUnsigned(cmp_op); |
| 1255 | const condition = Instruction.ICondition.fromCompareOperatorUnsigned(cmp_op.cmp); |
| 1129 | 1256 | break :blk condition.negate(); |
| 1130 | 1257 | }, |
| 1131 | 1258 | else => unreachable, |
| ... | ... | @@ -1402,6 +1529,133 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { |
| 1402 | 1529 | return self.finishAirBookkeeping(); |
| 1403 | 1530 | } |
| 1404 | 1531 | |
| 1532 | fn airNot(self: *Self, inst: Air.Inst.Index) !void { |
| 1533 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; |
| 1534 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { |
| 1535 | const operand = try self.resolveInst(ty_op.operand); |
| 1536 | const operand_ty = self.air.typeOf(ty_op.operand); |
| 1537 | switch (operand) { |
| 1538 | .dead => unreachable, |
| 1539 | .unreach => unreachable, |
| 1540 | .compare_flags_unsigned => |op| { |
| 1541 | const r = MCValue{ |
| 1542 | .compare_flags_unsigned = .{ |
| 1543 | .cmp = switch (op.cmp) { |
| 1544 | .gte => .lt, |
| 1545 | .gt => .lte, |
| 1546 | .neq => .eq, |
| 1547 | .lt => .gte, |
| 1548 | .lte => .gt, |
| 1549 | .eq => .neq, |
| 1550 | }, |
| 1551 | .ccr = op.ccr, |
| 1552 | }, |
| 1553 | }; |
| 1554 | break :result r; |
| 1555 | }, |
| 1556 | .compare_flags_signed => |op| { |
| 1557 | const r = MCValue{ |
| 1558 | .compare_flags_signed = .{ |
| 1559 | .cmp = switch (op.cmp) { |
| 1560 | .gte => .lt, |
| 1561 | .gt => .lte, |
| 1562 | .neq => .eq, |
| 1563 | .lt => .gte, |
| 1564 | .lte => .gt, |
| 1565 | .eq => .neq, |
| 1566 | }, |
| 1567 | .ccr = op.ccr, |
| 1568 | }, |
| 1569 | }; |
| 1570 | break :result r; |
| 1571 | }, |
| 1572 | else => { |
| 1573 | switch (operand_ty.zigTypeTag()) { |
| 1574 | .Bool => { |
| 1575 | // TODO convert this to mvn + and |
| 1576 | const op_reg = switch (operand) { |
| 1577 | .register => |r| r, |
| 1578 | else => try self.copyToTmpRegister(operand_ty, operand), |
| 1579 | }; |
| 1580 | const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg); |
| 1581 | defer self.register_manager.unlockReg(reg_lock); |
| 1582 | |
| 1583 | const dest_reg = blk: { |
| 1584 | if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { |
| 1585 | break :blk op_reg; |
| 1586 | } |
| 1587 | |
| 1588 | const reg = try self.register_manager.allocReg(null, gp); |
| 1589 | break :blk reg; |
| 1590 | }; |
| 1591 | |
| 1592 | _ = try self.addInst(.{ |
| 1593 | .tag = .xor, |
| 1594 | .data = .{ |
| 1595 | .arithmetic_3op = .{ |
| 1596 | .is_imm = true, |
| 1597 | .rd = dest_reg, |
| 1598 | .rs1 = op_reg, |
| 1599 | .rs2_or_imm = .{ .imm = 1 }, |
| 1600 | }, |
| 1601 | }, |
| 1602 | }); |
| 1603 | |
| 1604 | break :result MCValue{ .register = dest_reg }; |
| 1605 | }, |
| 1606 | .Vector => return self.fail("TODO bitwise not for vectors", .{}), |
| 1607 | .Int => { |
| 1608 | const int_info = operand_ty.intInfo(self.target.*); |
| 1609 | if (int_info.bits <= 64) { |
| 1610 | const op_reg = switch (operand) { |
| 1611 | .register => |r| r, |
| 1612 | else => try self.copyToTmpRegister(operand_ty, operand), |
| 1613 | }; |
| 1614 | const reg_lock = self.register_manager.lockRegAssumeUnused(op_reg); |
| 1615 | defer self.register_manager.unlockReg(reg_lock); |
| 1616 | |
| 1617 | const dest_reg = blk: { |
| 1618 | if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { |
| 1619 | break :blk op_reg; |
| 1620 | } |
| 1621 | |
| 1622 | const reg = try self.register_manager.allocReg(null, gp); |
| 1623 | break :blk reg; |
| 1624 | }; |
| 1625 | |
| 1626 | _ = try self.addInst(.{ |
| 1627 | .tag = .not, |
| 1628 | .data = .{ |
| 1629 | .arithmetic_2op = .{ |
| 1630 | .is_imm = false, |
| 1631 | .rs1 = dest_reg, |
| 1632 | .rs2_or_imm = .{ .rs2 = op_reg }, |
| 1633 | }, |
| 1634 | }, |
| 1635 | }); |
| 1636 | |
| 1637 | try self.truncRegister(dest_reg, dest_reg, int_info.signedness, int_info.bits); |
| 1638 | |
| 1639 | break :result MCValue{ .register = dest_reg }; |
| 1640 | } else { |
| 1641 | return self.fail("TODO AArch64 not on integers > u64/i64", .{}); |
| 1642 | } |
| 1643 | }, |
| 1644 | else => unreachable, |
| 1645 | } |
| 1646 | }, |
| 1647 | } |
| 1648 | }; |
| 1649 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 1650 | } |
| 1651 | |
| 1652 | fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void { |
| 1653 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 1654 | const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 1655 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement ptr_elem_ptr for {}", .{self.target.cpu.arch}); |
| 1656 | return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); |
| 1657 | } |
| 1658 | |
| 1405 | 1659 | fn airRet(self: *Self, inst: Air.Inst.Index) !void { |
| 1406 | 1660 | const un_op = self.air.instructions.items(.data)[inst].un_op; |
| 1407 | 1661 | const operand = try self.resolveInst(un_op); |
| ... | ... | @@ -1422,6 +1676,26 @@ fn airRetPtr(self: *Self, inst: Air.Inst.Index) !void { |
| 1422 | 1676 | return self.finishAir(inst, .{ .ptr_stack_offset = stack_offset }, .{ .none, .none, .none }); |
| 1423 | 1677 | } |
| 1424 | 1678 | |
| 1679 | fn airSlice(self: *Self, inst: Air.Inst.Index) !void { |
| 1680 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 1681 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 1682 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { |
| 1683 | const ptr = try self.resolveInst(bin_op.lhs); |
| 1684 | const ptr_ty = self.air.typeOf(bin_op.lhs); |
| 1685 | const len = try self.resolveInst(bin_op.rhs); |
| 1686 | const len_ty = self.air.typeOf(bin_op.rhs); |
| 1687 | |
| 1688 | const ptr_bits = self.target.cpu.arch.ptrBitWidth(); |
| 1689 | const ptr_bytes = @divExact(ptr_bits, 8); |
| 1690 | |
| 1691 | const stack_offset = try self.allocMem(inst, ptr_bytes * 2, ptr_bytes * 2); |
| 1692 | try self.genSetStack(ptr_ty, stack_offset, ptr); |
| 1693 | try self.genSetStack(len_ty, stack_offset - ptr_bytes, len); |
| 1694 | break :result MCValue{ .stack_offset = stack_offset }; |
| 1695 | }; |
| 1696 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 1697 | } |
| 1698 | |
| 1425 | 1699 | fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { |
| 1426 | 1700 | const is_volatile = false; // TODO |
| 1427 | 1701 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| ... | ... | @@ -1505,6 +1779,75 @@ fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void { |
| 1505 | 1779 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 1506 | 1780 | } |
| 1507 | 1781 | |
| 1782 | fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { |
| 1783 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 1784 | const extra = self.air.extraData(Air.StructField, ty_pl.payload).data; |
| 1785 | const operand = extra.struct_operand; |
| 1786 | const index = extra.field_index; |
| 1787 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { |
| 1788 | const mcv = try self.resolveInst(operand); |
| 1789 | const struct_ty = self.air.typeOf(operand); |
| 1790 | const struct_field_offset = @intCast(u32, struct_ty.structFieldOffset(index, self.target.*)); |
| 1791 | |
| 1792 | switch (mcv) { |
| 1793 | .dead, .unreach => unreachable, |
| 1794 | .stack_offset => |off| { |
| 1795 | break :result MCValue{ .stack_offset = off - struct_field_offset }; |
| 1796 | }, |
| 1797 | .memory => |addr| { |
| 1798 | break :result MCValue{ .memory = addr + struct_field_offset }; |
| 1799 | }, |
| 1800 | .register_with_overflow => |rwo| { |
| 1801 | switch (index) { |
| 1802 | 0 => { |
| 1803 | // get wrapped value: return register |
| 1804 | break :result MCValue{ .register = rwo.reg }; |
| 1805 | }, |
| 1806 | 1 => { |
| 1807 | // TODO return special MCValue condition flags |
| 1808 | // get overflow bit: set register to C flag |
| 1809 | // resp. V flag |
| 1810 | const dest_reg = try self.register_manager.allocReg(null, gp); |
| 1811 | |
| 1812 | // TODO handle floating point CCRs |
| 1813 | assert(rwo.flag.ccr == .xcc or rwo.flag.ccr == .icc); |
| 1814 | |
| 1815 | _ = try self.addInst(.{ |
| 1816 | .tag = .mov, |
| 1817 | .data = .{ |
| 1818 | .arithmetic_2op = .{ |
| 1819 | .is_imm = false, |
| 1820 | .rs1 = dest_reg, |
| 1821 | .rs2_or_imm = .{ .rs2 = .g0 }, |
| 1822 | }, |
| 1823 | }, |
| 1824 | }); |
| 1825 | |
| 1826 | _ = try self.addInst(.{ |
| 1827 | .tag = .movcc, |
| 1828 | .data = .{ |
| 1829 | .conditional_move = .{ |
| 1830 | .ccr = rwo.flag.ccr, |
| 1831 | .cond = .{ .icond = rwo.flag.cond }, |
| 1832 | .is_imm = true, |
| 1833 | .rd = dest_reg, |
| 1834 | .rs2_or_imm = .{ .imm = 1 }, |
| 1835 | }, |
| 1836 | }, |
| 1837 | }); |
| 1838 | |
| 1839 | break :result MCValue{ .register = dest_reg }; |
| 1840 | }, |
| 1841 | else => unreachable, |
| 1842 | } |
| 1843 | }, |
| 1844 | else => return self.fail("TODO implement codegen struct_field_val for {}", .{mcv}), |
| 1845 | } |
| 1846 | }; |
| 1847 | |
| 1848 | return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); |
| 1849 | } |
| 1850 | |
| 1508 | 1851 | fn airSwitch(self: *Self, inst: Air.Inst.Index) !void { |
| 1509 | 1852 | _ = self; |
| 1510 | 1853 | _ = inst; |
| ... | ... | @@ -1537,6 +1880,20 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void { |
| 1537 | 1880 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 1538 | 1881 | } |
| 1539 | 1882 | |
| 1883 | /// E to E!T |
| 1884 | fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void { |
| 1885 | const ty_op = self.air.instructions.items(.data)[inst].ty_op; |
| 1886 | const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { |
| 1887 | const error_union_ty = self.air.getRefType(ty_op.ty); |
| 1888 | const payload_ty = error_union_ty.errorUnionPayload(); |
| 1889 | const mcv = try self.resolveInst(ty_op.operand); |
| 1890 | if (!payload_ty.hasRuntimeBits()) break :result mcv; |
| 1891 | |
| 1892 | return self.fail("TODO implement wrap errunion error for non-empty payloads", .{}); |
| 1893 | }; |
| 1894 | return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); |
| 1895 | } |
| 1896 | |
| 1540 | 1897 | // Common helper functions |
| 1541 | 1898 | |
| 1542 | 1899 | /// Adds a Type to the .debug_info at the current position. The bytes will be populated later, |
| ... | ... | @@ -1571,8 +1928,8 @@ fn allocMem(self: *Self, inst: Air.Inst.Index, abi_size: u32, abi_align: u32) !u |
| 1571 | 1928 | if (abi_align > self.stack_align) |
| 1572 | 1929 | self.stack_align = abi_align; |
| 1573 | 1930 | // TODO find a free slot instead of always appending |
| 1574 | | const offset = mem.alignForwardGeneric(u32, self.next_stack_offset, abi_align); |
| 1575 | | self.next_stack_offset = offset + abi_size; |
| 1931 | const offset = mem.alignForwardGeneric(u32, self.next_stack_offset, abi_align) + abi_size; |
| 1932 | self.next_stack_offset = offset; |
| 1576 | 1933 | if (self.next_stack_offset > self.max_end_stack) |
| 1577 | 1934 | self.max_end_stack = self.next_stack_offset; |
| 1578 | 1935 | try self.stack.putNoClobber(self.gpa, offset, .{ |
| ... | ... | @@ -1708,21 +2065,34 @@ fn binOp( |
| 1708 | 2065 | assert(lhs_ty.eql(rhs_ty, mod)); |
| 1709 | 2066 | const int_info = lhs_ty.intInfo(self.target.*); |
| 1710 | 2067 | if (int_info.bits <= 64) { |
| 1711 | | // If LHS is immediate, then swap it with RHS. |
| 1712 | | const lhs_is_imm = lhs == .immediate; |
| 1713 | | const new_lhs = if (lhs_is_imm) rhs else lhs; |
| 1714 | | const new_rhs = if (lhs_is_imm) lhs else rhs; |
| 1715 | | const new_lhs_ty = if (lhs_is_imm) rhs_ty else lhs_ty; |
| 1716 | | const new_rhs_ty = if (lhs_is_imm) lhs_ty else rhs_ty; |
| 1717 | | |
| 1718 | | // At this point, RHS might be an immediate |
| 1719 | | // If it's a power of two immediate then we emit an shl instead |
| 1720 | | // TODO add similar checks for LHS |
| 1721 | | if (new_rhs == .immediate and math.isPowerOfTwo(new_rhs.immediate)) { |
| 1722 | | return try self.binOp(.shl, new_lhs, .{ .immediate = math.log2(new_rhs.immediate) }, new_lhs_ty, Type.usize, metadata); |
| 1723 | | } |
| 2068 | // Only say yes if the operation is |
| 2069 | // commutative, i.e. we can swap both of the |
| 2070 | // operands |
| 2071 | const lhs_immediate_ok = switch (tag) { |
| 2072 | .mul => lhs == .immediate and lhs.immediate <= std.math.maxInt(u12), |
| 2073 | else => unreachable, |
| 2074 | }; |
| 2075 | const rhs_immediate_ok = switch (tag) { |
| 2076 | .mul => rhs == .immediate and rhs.immediate <= std.math.maxInt(u12), |
| 2077 | else => unreachable, |
| 2078 | }; |
| 1724 | 2079 | |
| 1725 | | return try self.binOpRegister(.mulx, new_lhs, new_rhs, new_lhs_ty, new_rhs_ty, metadata); |
| 2080 | const mir_tag: Mir.Inst.Tag = switch (tag) { |
| 2081 | .mul => .mulx, |
| 2082 | else => unreachable, |
| 2083 | }; |
| 2084 | |
| 2085 | if (rhs_immediate_ok) { |
| 2086 | // At this point, rhs is an immediate |
| 2087 | return try self.binOpImmediate(mir_tag, lhs, rhs, lhs_ty, false, metadata); |
| 2088 | } else if (lhs_immediate_ok) { |
| 2089 | // swap lhs and rhs |
| 2090 | // At this point, lhs is an immediate |
| 2091 | return try self.binOpImmediate(mir_tag, rhs, lhs, rhs_ty, true, metadata); |
| 2092 | } else { |
| 2093 | // TODO convert large immediates to register before adding |
| 2094 | return try self.binOpRegister(mir_tag, lhs, rhs, lhs_ty, rhs_ty, metadata); |
| 2095 | } |
| 1726 | 2096 | } else { |
| 1727 | 2097 | return self.fail("TODO binary operations on int with bits > 64", .{}); |
| 1728 | 2098 | } |
| ... | ... | @@ -1867,6 +2237,7 @@ fn binOpImmediate( |
| 1867 | 2237 | defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg); |
| 1868 | 2238 | |
| 1869 | 2239 | const dest_reg = switch (mir_tag) { |
| 2240 | .cmp => undefined, // cmp has no destination register |
| 1870 | 2241 | else => if (metadata) |md| blk: { |
| 1871 | 2242 | if (lhs_is_register and self.reuseOperand( |
| 1872 | 2243 | md.inst, |
| ... | ... | @@ -1887,6 +2258,7 @@ fn binOpImmediate( |
| 1887 | 2258 | |
| 1888 | 2259 | const mir_data: Mir.Inst.Data = switch (mir_tag) { |
| 1889 | 2260 | .add, |
| 2261 | .addcc, |
| 1890 | 2262 | .mulx, |
| 1891 | 2263 | .subcc, |
| 1892 | 2264 | => .{ |
| ... | ... | @@ -1985,6 +2357,7 @@ fn binOpRegister( |
| 1985 | 2357 | defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg); |
| 1986 | 2358 | |
| 1987 | 2359 | const dest_reg = switch (mir_tag) { |
| 2360 | .cmp => undefined, // cmp has no destination register |
| 1988 | 2361 | else => if (metadata) |md| blk: { |
| 1989 | 2362 | if (lhs_is_register and self.reuseOperand(md.inst, md.lhs, 0, lhs)) { |
| 1990 | 2363 | break :blk lhs_reg; |
| ... | ... | @@ -2003,6 +2376,7 @@ fn binOpRegister( |
| 2003 | 2376 | |
| 2004 | 2377 | const mir_data: Mir.Inst.Data = switch (mir_tag) { |
| 2005 | 2378 | .add, |
| 2379 | .addcc, |
| 2006 | 2380 | .mulx, |
| 2007 | 2381 | .subcc, |
| 2008 | 2382 | => .{ |
| ... | ... | @@ -2293,8 +2667,47 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void |
| 2293 | 2667 | switch (mcv) { |
| 2294 | 2668 | .dead => unreachable, |
| 2295 | 2669 | .unreach, .none => return, // Nothing to do. |
| 2296 | | .compare_flags_signed => return self.fail("TODO: genSetReg for compare_flags_signed", .{}), |
| 2297 | | .compare_flags_unsigned => return self.fail("TODO: genSetReg for compare_flags_unsigned", .{}), |
| 2670 | .compare_flags_signed, |
| 2671 | .compare_flags_unsigned, |
| 2672 | => { |
| 2673 | const condition = switch (mcv) { |
| 2674 | .compare_flags_unsigned => |op| Instruction.ICondition.fromCompareOperatorUnsigned(op.cmp), |
| 2675 | .compare_flags_signed => |op| Instruction.ICondition.fromCompareOperatorSigned(op.cmp), |
| 2676 | else => unreachable, |
| 2677 | }; |
| 2678 | |
| 2679 | const ccr = switch (mcv) { |
| 2680 | .compare_flags_unsigned => |op| op.ccr, |
| 2681 | .compare_flags_signed => |op| op.ccr, |
| 2682 | else => unreachable, |
| 2683 | }; |
| 2684 | // TODO handle floating point CCRs |
| 2685 | assert(ccr == .xcc or ccr == .icc); |
| 2686 | |
| 2687 | _ = try self.addInst(.{ |
| 2688 | .tag = .mov, |
| 2689 | .data = .{ |
| 2690 | .arithmetic_2op = .{ |
| 2691 | .is_imm = false, |
| 2692 | .rs1 = reg, |
| 2693 | .rs2_or_imm = .{ .rs2 = .g0 }, |
| 2694 | }, |
| 2695 | }, |
| 2696 | }); |
| 2697 | |
| 2698 | _ = try self.addInst(.{ |
| 2699 | .tag = .movcc, |
| 2700 | .data = .{ |
| 2701 | .conditional_move = .{ |
| 2702 | .ccr = ccr, |
| 2703 | .cond = .{ .icond = condition }, |
| 2704 | .is_imm = true, |
| 2705 | .rd = reg, |
| 2706 | .rs2_or_imm = .{ .imm = 1 }, |
| 2707 | }, |
| 2708 | }, |
| 2709 | }); |
| 2710 | }, |
| 2298 | 2711 | .undef => { |
| 2299 | 2712 | if (!self.wantSafety()) |
| 2300 | 2713 | return; // The already existing value will do just fine. |
| ... | ... | @@ -2302,8 +2715,9 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void |
| 2302 | 2715 | return self.genSetReg(ty, reg, .{ .immediate = 0xaaaaaaaaaaaaaaaa }); |
| 2303 | 2716 | }, |
| 2304 | 2717 | .ptr_stack_offset => |off| { |
| 2305 | | const simm13 = math.cast(u12, off + abi.stack_bias + abi.stack_reserved_area) orelse |
| 2306 | | return self.fail("TODO larger stack offsets", .{}); |
| 2718 | const real_offset = off + abi.stack_bias + abi.stack_reserved_area; |
| 2719 | const simm13 = math.cast(i13, real_offset) orelse |
| 2720 | return self.fail("TODO larger stack offsets: {}", .{real_offset}); |
| 2307 | 2721 | |
| 2308 | 2722 | _ = try self.addInst(.{ |
| 2309 | 2723 | .tag = .add, |
| ... | ... | @@ -2427,6 +2841,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void |
| 2427 | 2841 | }, |
| 2428 | 2842 | }); |
| 2429 | 2843 | }, |
| 2844 | .register_with_overflow => unreachable, |
| 2430 | 2845 | .memory => |addr| { |
| 2431 | 2846 | // The value is in memory at a hard-coded address. |
| 2432 | 2847 | // If the type is a pointer, it means the pointer address is at this memory location. |
| ... | ... | @@ -2436,7 +2851,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void |
| 2436 | 2851 | .stack_offset => |off| { |
| 2437 | 2852 | const real_offset = off + abi.stack_bias + abi.stack_reserved_area; |
| 2438 | 2853 | const simm13 = math.cast(i13, real_offset) orelse |
| 2439 | | return self.fail("TODO larger stack offsets", .{}); |
| 2854 | return self.fail("TODO larger stack offsets: {}", .{real_offset}); |
| 2440 | 2855 | try self.genLoad(reg, .sp, i13, simm13, ty.abiSize(self.target.*)); |
| 2441 | 2856 | }, |
| 2442 | 2857 | } |
| ... | ... | @@ -2470,9 +2885,50 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro |
| 2470 | 2885 | .register => |reg| { |
| 2471 | 2886 | const real_offset = stack_offset + abi.stack_bias + abi.stack_reserved_area; |
| 2472 | 2887 | const simm13 = math.cast(i13, real_offset) orelse |
| 2473 | | return self.fail("TODO larger stack offsets", .{}); |
| 2888 | return self.fail("TODO larger stack offsets: {}", .{real_offset}); |
| 2474 | 2889 | return self.genStore(reg, .sp, i13, simm13, abi_size); |
| 2475 | 2890 | }, |
| 2891 | .register_with_overflow => |rwo| { |
| 2892 | const reg_lock = self.register_manager.lockReg(rwo.reg); |
| 2893 | defer if (reg_lock) |locked_reg| self.register_manager.unlockReg(locked_reg); |
| 2894 | |
| 2895 | const wrapped_ty = ty.structFieldType(0); |
| 2896 | try self.genSetStack(wrapped_ty, stack_offset, .{ .register = rwo.reg }); |
| 2897 | |
| 2898 | const overflow_bit_ty = ty.structFieldType(1); |
| 2899 | const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*)); |
| 2900 | const cond_reg = try self.register_manager.allocReg(null, gp); |
| 2901 | |
| 2902 | // TODO handle floating point CCRs |
| 2903 | assert(rwo.flag.ccr == .xcc or rwo.flag.ccr == .icc); |
| 2904 | |
| 2905 | _ = try self.addInst(.{ |
| 2906 | .tag = .mov, |
| 2907 | .data = .{ |
| 2908 | .arithmetic_2op = .{ |
| 2909 | .is_imm = false, |
| 2910 | .rs1 = cond_reg, |
| 2911 | .rs2_or_imm = .{ .rs2 = .g0 }, |
| 2912 | }, |
| 2913 | }, |
| 2914 | }); |
| 2915 | |
| 2916 | _ = try self.addInst(.{ |
| 2917 | .tag = .movcc, |
| 2918 | .data = .{ |
| 2919 | .conditional_move = .{ |
| 2920 | .ccr = rwo.flag.ccr, |
| 2921 | .cond = .{ .icond = rwo.flag.cond }, |
| 2922 | .is_imm = true, |
| 2923 | .rd = cond_reg, |
| 2924 | .rs2_or_imm = .{ .imm = 1 }, |
| 2925 | }, |
| 2926 | }, |
| 2927 | }); |
| 2928 | try self.genSetStack(overflow_bit_ty, stack_offset - overflow_bit_offset, .{ |
| 2929 | .register = cond_reg, |
| 2930 | }); |
| 2931 | }, |
| 2476 | 2932 | .memory, .stack_offset => { |
| 2477 | 2933 | switch (mcv) { |
| 2478 | 2934 | .stack_offset => |off| { |
| ... | ... | @@ -2647,7 +3103,7 @@ fn isErr(self: *Self, ty: Type, operand: MCValue) !MCValue { |
| 2647 | 3103 | } }, |
| 2648 | 3104 | }); |
| 2649 | 3105 | |
| 2650 | | return MCValue{ .compare_flags_unsigned = .gt }; |
| 3106 | return MCValue{ .compare_flags_unsigned = .{ .cmp = .gt, .ccr = .xcc } }; |
| 2651 | 3107 | } else { |
| 2652 | 3108 | return self.fail("TODO isErr for errors with size > 8", .{}); |
| 2653 | 3109 | } |
| ... | ... | @@ -2661,8 +3117,8 @@ fn isNonErr(self: *Self, ty: Type, operand: MCValue) !MCValue { |
| 2661 | 3117 | const is_err_result = try self.isErr(ty, operand); |
| 2662 | 3118 | switch (is_err_result) { |
| 2663 | 3119 | .compare_flags_unsigned => |op| { |
| 2664 | | assert(op == .gt); |
| 2665 | | return MCValue{ .compare_flags_unsigned = .lte }; |
| 3120 | assert(op.cmp == .gt); |
| 3121 | return MCValue{ .compare_flags_unsigned = .{ .cmp = .lte, .ccr = op.ccr } }; |
| 2666 | 3122 | }, |
| 2667 | 3123 | .immediate => |imm| { |
| 2668 | 3124 | assert(imm == 0); |
| ... | ... | @@ -2714,6 +3170,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo |
| 2714 | 3170 | .dead => unreachable, |
| 2715 | 3171 | .compare_flags_unsigned, |
| 2716 | 3172 | .compare_flags_signed, |
| 3173 | .register_with_overflow, |
| 2717 | 3174 | => unreachable, // cannot hold an address |
| 2718 | 3175 | .immediate => |imm| try self.setRegOrMem(elem_ty, dst_mcv, .{ .memory = imm }), |
| 2719 | 3176 | .ptr_stack_offset => |off| try self.setRegOrMem(elem_ty, dst_mcv, .{ .stack_offset = off }), |
| ... | ... | @@ -2801,6 +3258,7 @@ fn performReloc(self: *Self, inst: Mir.Inst.Index) !void { |
| 2801 | 3258 | const tag = self.mir_instructions.items(.tag)[inst]; |
| 2802 | 3259 | switch (tag) { |
| 2803 | 3260 | .bpcc => self.mir_instructions.items(.data)[inst].branch_predict_int.inst = @intCast(Mir.Inst.Index, self.mir_instructions.len), |
| 3261 | .bpr => self.mir_instructions.items(.data)[inst].branch_predict_reg.inst = @intCast(Mir.Inst.Index, self.mir_instructions.len), |
| 2804 | 3262 | else => unreachable, |
| 2805 | 3263 | } |
| 2806 | 3264 | } |
| ... | ... | @@ -2817,6 +3275,10 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) void { |
| 2817 | 3275 | .register => |reg| { |
| 2818 | 3276 | self.register_manager.freeReg(reg); |
| 2819 | 3277 | }, |
| 3278 | .register_with_overflow => |rwo| { |
| 3279 | self.register_manager.freeReg(rwo.reg); |
| 3280 | self.compare_flags_inst = null; |
| 3281 | }, |
| 2820 | 3282 | .compare_flags_signed, .compare_flags_unsigned => { |
| 2821 | 3283 | self.compare_flags_inst = null; |
| 2822 | 3284 | }, |
| ... | ... | @@ -2918,7 +3380,7 @@ fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue { |
| 2918 | 3380 | const ref_int = @enumToInt(inst); |
| 2919 | 3381 | if (ref_int < Air.Inst.Ref.typed_value_map.len) { |
| 2920 | 3382 | const tv = Air.Inst.Ref.typed_value_map[ref_int]; |
| 2921 | | if (!tv.ty.hasRuntimeBits()) { |
| 3383 | if (!tv.ty.hasRuntimeBits() and !tv.ty.isError()) { |
| 2922 | 3384 | return MCValue{ .none = {} }; |
| 2923 | 3385 | } |
| 2924 | 3386 | return self.genTypedValue(tv); |
| ... | ... | @@ -2926,7 +3388,7 @@ fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue { |
| 2926 | 3388 | |
| 2927 | 3389 | // If the type has no codegen bits, no need to store it. |
| 2928 | 3390 | const inst_ty = self.air.typeOf(inst); |
| 2929 | | if (!inst_ty.hasRuntimeBits()) |
| 3391 | if (!inst_ty.hasRuntimeBits() and !inst_ty.isError()) |
| 2930 | 3392 | return MCValue{ .none = {} }; |
| 2931 | 3393 | |
| 2932 | 3394 | const inst_index = @intCast(Air.Inst.Index, ref_int - Air.Inst.Ref.typed_value_map.len); |
| ... | ... | @@ -3017,14 +3479,14 @@ fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void { |
| 3017 | 3479 | fn spillCompareFlagsIfOccupied(self: *Self) !void { |
| 3018 | 3480 | if (self.compare_flags_inst) |inst_to_save| { |
| 3019 | 3481 | const mcv = self.getResolvedInstValue(inst_to_save); |
| 3020 | | switch (mcv) { |
| 3482 | const new_mcv = switch (mcv) { |
| 3021 | 3483 | .compare_flags_signed, |
| 3022 | 3484 | .compare_flags_unsigned, |
| 3023 | | => {}, |
| 3485 | => try self.allocRegOrMem(inst_to_save, true), |
| 3486 | .register_with_overflow => try self.allocRegOrMem(inst_to_save, false), |
| 3024 | 3487 | else => unreachable, // mcv doesn't occupy the compare flags |
| 3025 | | } |
| 3488 | }; |
| 3026 | 3489 | |
| 3027 | | const new_mcv = try self.allocRegOrMem(inst_to_save, true); |
| 3028 | 3490 | try self.setRegOrMem(self.air.typeOfIndex(inst_to_save), new_mcv, mcv); |
| 3029 | 3491 | log.debug("spilling {d} to mcv {any}", .{ inst_to_save, new_mcv }); |
| 3030 | 3492 | |
| ... | ... | @@ -3032,6 +3494,13 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void { |
| 3032 | 3494 | try branch.inst_table.put(self.gpa, inst_to_save, new_mcv); |
| 3033 | 3495 | |
| 3034 | 3496 | self.compare_flags_inst = null; |
| 3497 | |
| 3498 | // TODO consolidate with register manager and spillInstruction |
| 3499 | // this call should really belong in the register manager! |
| 3500 | switch (mcv) { |
| 3501 | .register_with_overflow => |rwo| self.register_manager.freeReg(rwo.reg), |
| 3502 | else => {}, |
| 3503 | } |
| 3035 | 3504 | } |
| 3036 | 3505 | } |
| 3037 | 3506 | |
| ... | ... | @@ -3055,6 +3524,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type |
| 3055 | 3524 | .dead => unreachable, |
| 3056 | 3525 | .compare_flags_unsigned, |
| 3057 | 3526 | .compare_flags_signed, |
| 3527 | .register_with_overflow, |
| 3058 | 3528 | => unreachable, // cannot hold an address |
| 3059 | 3529 | .immediate => |imm| { |
| 3060 | 3530 | try self.setRegOrMem(value_ty, .{ .memory = imm }, value); |
| ... | ... | @@ -3175,11 +3645,13 @@ fn truncRegister( |
| 3175 | 3645 | }); |
| 3176 | 3646 | }, |
| 3177 | 3647 | 64 => { |
| 3648 | if (dest_reg == operand_reg) |
| 3649 | return; // Copy register to itself; nothing to do. |
| 3178 | 3650 | _ = try self.addInst(.{ |
| 3179 | 3651 | .tag = .mov, |
| 3180 | 3652 | .data = .{ |
| 3181 | 3653 | .arithmetic_2op = .{ |
| 3182 | | .is_imm = true, |
| 3654 | .is_imm = false, |
| 3183 | 3655 | .rs1 = dest_reg, |
| 3184 | 3656 | .rs2_or_imm = .{ .rs2 = operand_reg }, |
| 3185 | 3657 | }, |