| ... | ... | @@ -5133,74 +5133,114 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { |
| 5133 | 5133 | const mod = self.bin_file.options.module.?; |
| 5134 | 5134 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 5135 | 5135 | |
| 5136 | | const array_ty = self.typeOf(bin_op.lhs); |
| 5137 | | const array = try self.resolveInst(bin_op.lhs); |
| 5138 | | const array_lock: ?RegisterLock = switch (array) { |
| 5139 | | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 5140 | | else => null, |
| 5141 | | }; |
| 5142 | | defer if (array_lock) |lock| self.register_manager.unlockReg(lock); |
| 5136 | const result: MCValue = result: { |
| 5137 | const array_ty = self.typeOf(bin_op.lhs); |
| 5138 | const elem_ty = array_ty.childType(mod); |
| 5143 | 5139 | |
| 5144 | | const elem_ty = array_ty.childType(mod); |
| 5145 | | const elem_abi_size = elem_ty.abiSize(mod); |
| 5140 | const array_mcv = try self.resolveInst(bin_op.lhs); |
| 5141 | const array_lock: ?RegisterLock = switch (array_mcv) { |
| 5142 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 5143 | else => null, |
| 5144 | }; |
| 5145 | defer if (array_lock) |lock| self.register_manager.unlockReg(lock); |
| 5146 | 5146 | |
| 5147 | | const index_ty = self.typeOf(bin_op.rhs); |
| 5148 | | const index = try self.resolveInst(bin_op.rhs); |
| 5149 | | const index_lock: ?RegisterLock = switch (index) { |
| 5150 | | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 5151 | | else => null, |
| 5152 | | }; |
| 5153 | | defer if (index_lock) |lock| self.register_manager.unlockReg(lock); |
| 5147 | const index_ty = self.typeOf(bin_op.rhs); |
| 5148 | const index_mcv = try self.resolveInst(bin_op.rhs); |
| 5149 | const index_lock: ?RegisterLock = switch (index_mcv) { |
| 5150 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 5151 | else => null, |
| 5152 | }; |
| 5153 | defer if (index_lock) |lock| self.register_manager.unlockReg(lock); |
| 5154 | 5154 | |
| 5155 | | const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 5156 | | const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); |
| 5157 | | defer self.register_manager.unlockReg(addr_lock); |
| 5155 | try self.spillEflagsIfOccupied(); |
| 5156 | if (array_ty.isVector(mod) and elem_ty.toIntern() == .bool_type) { |
| 5157 | const index_reg = switch (index_mcv) { |
| 5158 | .register => |reg| reg, |
| 5159 | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| 5160 | }; |
| 5161 | switch (array_mcv) { |
| 5162 | .register => |array_reg| try self.asmRegisterRegister( |
| 5163 | .{ ._, .bt }, |
| 5164 | array_reg.to64(), |
| 5165 | index_reg.to64(), |
| 5166 | ), |
| 5167 | .load_frame => try self.asmMemoryRegister( |
| 5168 | .{ ._, .bt }, |
| 5169 | try array_mcv.mem(self, .qword), |
| 5170 | index_reg.to64(), |
| 5171 | ), |
| 5172 | .memory, .load_symbol, .load_direct, .load_got, .load_tlv => try self.asmMemoryRegister( |
| 5173 | .{ ._, .bt }, |
| 5174 | .{ |
| 5175 | .base = .{ |
| 5176 | .reg = try self.copyToTmpRegister(Type.usize, array_mcv.address()), |
| 5177 | }, |
| 5178 | .mod = .{ .rm = .{ .size = .qword } }, |
| 5179 | }, |
| 5180 | index_reg.to64(), |
| 5181 | ), |
| 5182 | else => return self.fail("TODO airArrayElemVal for {s} of {}", .{ |
| 5183 | @tagName(array_mcv), array_ty.fmt(mod), |
| 5184 | }), |
| 5185 | } |
| 5158 | 5186 | |
| 5159 | | switch (array) { |
| 5160 | | .register => { |
| 5161 | | const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod)); |
| 5162 | | try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array); |
| 5163 | | try self.asmRegisterMemory( |
| 5187 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); |
| 5188 | try self.asmSetccRegister(.c, dst_reg.to8()); |
| 5189 | break :result .{ .register = dst_reg }; |
| 5190 | } |
| 5191 | |
| 5192 | const elem_abi_size = elem_ty.abiSize(mod); |
| 5193 | const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 5194 | const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); |
| 5195 | defer self.register_manager.unlockReg(addr_lock); |
| 5196 | |
| 5197 | switch (array_mcv) { |
| 5198 | .register => { |
| 5199 | const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod)); |
| 5200 | try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv); |
| 5201 | try self.asmRegisterMemory( |
| 5202 | .{ ._, .lea }, |
| 5203 | addr_reg, |
| 5204 | .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } }, |
| 5205 | ); |
| 5206 | }, |
| 5207 | .load_frame => |frame_addr| try self.asmRegisterMemory( |
| 5164 | 5208 | .{ ._, .lea }, |
| 5165 | 5209 | addr_reg, |
| 5166 | | .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } }, |
| 5167 | | ); |
| 5168 | | }, |
| 5169 | | .load_frame => |frame_addr| try self.asmRegisterMemory( |
| 5170 | | .{ ._, .lea }, |
| 5171 | | addr_reg, |
| 5172 | | .{ |
| 5173 | | .base = .{ .frame = frame_addr.index }, |
| 5174 | | .mod = .{ .rm = .{ .size = .qword, .disp = frame_addr.off } }, |
| 5175 | | }, |
| 5176 | | ), |
| 5177 | | .memory, |
| 5178 | | .load_symbol, |
| 5179 | | .load_direct, |
| 5180 | | .load_got, |
| 5181 | | .load_tlv, |
| 5182 | | => try self.genSetReg(addr_reg, Type.usize, array.address()), |
| 5183 | | .lea_symbol, .lea_direct, .lea_tlv => unreachable, |
| 5184 | | else => return self.fail("TODO implement array_elem_val when array is {}", .{array}), |
| 5185 | | } |
| 5186 | | |
| 5187 | | const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size); |
| 5188 | | const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg); |
| 5189 | | defer self.register_manager.unlockReg(offset_lock); |
| 5210 | .{ |
| 5211 | .base = .{ .frame = frame_addr.index }, |
| 5212 | .mod = .{ .rm = .{ .size = .qword, .disp = frame_addr.off } }, |
| 5213 | }, |
| 5214 | ), |
| 5215 | .memory, |
| 5216 | .load_symbol, |
| 5217 | .load_direct, |
| 5218 | .load_got, |
| 5219 | .load_tlv, |
| 5220 | => try self.genSetReg(addr_reg, Type.usize, array_mcv.address()), |
| 5221 | .lea_symbol, .lea_direct, .lea_tlv => unreachable, |
| 5222 | else => return self.fail("TODO airArrayElemVal_val for {s} of {}", .{ |
| 5223 | @tagName(array_mcv), array_ty.fmt(mod), |
| 5224 | }), |
| 5225 | } |
| 5190 | 5226 | |
| 5191 | | // TODO we could allocate register here, but need to expect addr register and potentially |
| 5192 | | // offset register. |
| 5193 | | try self.spillEflagsIfOccupied(); |
| 5194 | | const dst_mcv = try self.allocRegOrMem(inst, false); |
| 5195 | | try self.genBinOpMir( |
| 5196 | | .{ ._, .add }, |
| 5197 | | Type.usize, |
| 5198 | | .{ .register = addr_reg }, |
| 5199 | | .{ .register = offset_reg }, |
| 5200 | | ); |
| 5201 | | try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }); |
| 5227 | const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size); |
| 5228 | const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg); |
| 5229 | defer self.register_manager.unlockReg(offset_lock); |
| 5202 | 5230 | |
| 5203 | | return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 5231 | // TODO we could allocate register here, but need to expect addr register and potentially |
| 5232 | // offset register. |
| 5233 | const dst_mcv = try self.allocRegOrMem(inst, false); |
| 5234 | try self.genBinOpMir( |
| 5235 | .{ ._, .add }, |
| 5236 | Type.usize, |
| 5237 | .{ .register = addr_reg }, |
| 5238 | .{ .register = offset_reg }, |
| 5239 | ); |
| 5240 | try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }); |
| 5241 | break :result dst_mcv; |
| 5242 | }; |
| 5243 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 5204 | 5244 | } |
| 5205 | 5245 | |
| 5206 | 5246 | fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void { |
| ... | ... | @@ -5258,32 +5298,44 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void { |
| 5258 | 5298 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 5259 | 5299 | const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 5260 | 5300 | |
| 5261 | | const ptr_ty = self.typeOf(extra.lhs); |
| 5262 | | const ptr = try self.resolveInst(extra.lhs); |
| 5263 | | const ptr_lock: ?RegisterLock = switch (ptr) { |
| 5264 | | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 5265 | | else => null, |
| 5266 | | }; |
| 5267 | | defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); |
| 5301 | const result = result: { |
| 5302 | const elem_ptr_ty = self.typeOfIndex(inst); |
| 5303 | const base_ptr_ty = self.typeOf(extra.lhs); |
| 5268 | 5304 | |
| 5269 | | const elem_ty = ptr_ty.elemType2(mod); |
| 5270 | | const elem_abi_size = elem_ty.abiSize(mod); |
| 5271 | | const index_ty = self.typeOf(extra.rhs); |
| 5272 | | const index = try self.resolveInst(extra.rhs); |
| 5273 | | const index_lock: ?RegisterLock = switch (index) { |
| 5274 | | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 5275 | | else => null, |
| 5276 | | }; |
| 5277 | | defer if (index_lock) |lock| self.register_manager.unlockReg(lock); |
| 5305 | const base_ptr_mcv = try self.resolveInst(extra.lhs); |
| 5306 | const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { |
| 5307 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 5308 | else => null, |
| 5309 | }; |
| 5310 | defer if (base_ptr_lock) |lock| self.register_manager.unlockReg(lock); |
| 5278 | 5311 | |
| 5279 | | const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size); |
| 5280 | | const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); |
| 5281 | | defer self.register_manager.unlockReg(offset_reg_lock); |
| 5312 | if (elem_ptr_ty.ptrInfo(mod).flags.vector_index != .none) { |
| 5313 | break :result if (self.reuseOperand(inst, extra.lhs, 0, base_ptr_mcv)) |
| 5314 | base_ptr_mcv |
| 5315 | else |
| 5316 | try self.copyToRegisterWithInstTracking(inst, elem_ptr_ty, base_ptr_mcv); |
| 5317 | } |
| 5282 | 5318 | |
| 5283 | | const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr); |
| 5284 | | try self.genBinOpMir(.{ ._, .add }, ptr_ty, dst_mcv, .{ .register = offset_reg }); |
| 5319 | const elem_ty = base_ptr_ty.elemType2(mod); |
| 5320 | const elem_abi_size = elem_ty.abiSize(mod); |
| 5321 | const index_ty = self.typeOf(extra.rhs); |
| 5322 | const index_mcv = try self.resolveInst(extra.rhs); |
| 5323 | const index_lock: ?RegisterLock = switch (index_mcv) { |
| 5324 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 5325 | else => null, |
| 5326 | }; |
| 5327 | defer if (index_lock) |lock| self.register_manager.unlockReg(lock); |
| 5285 | 5328 | |
| 5286 | | return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none }); |
| 5329 | const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size); |
| 5330 | const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); |
| 5331 | defer self.register_manager.unlockReg(offset_reg_lock); |
| 5332 | |
| 5333 | const dst_mcv = try self.copyToRegisterWithInstTracking(inst, elem_ptr_ty, base_ptr_mcv); |
| 5334 | try self.genBinOpMir(.{ ._, .add }, elem_ptr_ty, dst_mcv, .{ .register = offset_reg }); |
| 5335 | |
| 5336 | break :result dst_mcv; |
| 5337 | }; |
| 5338 | return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); |
| 5287 | 5339 | } |
| 5288 | 5340 | |
| 5289 | 5341 | fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void { |
| ... | ... | @@ -6712,19 +6764,20 @@ fn reuseOperandAdvanced( |
| 6712 | 6764 | |
| 6713 | 6765 | fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void { |
| 6714 | 6766 | const mod = self.bin_file.options.module.?; |
| 6715 | | const ptr_info = ptr_ty.ptrInfo(mod); |
| 6716 | 6767 | |
| 6768 | const ptr_info = ptr_ty.ptrInfo(mod); |
| 6717 | 6769 | const val_ty = Type.fromInterned(ptr_info.child); |
| 6718 | 6770 | if (!val_ty.hasRuntimeBitsIgnoreComptime(mod)) return; |
| 6719 | 6771 | const val_abi_size: u32 = @intCast(val_ty.abiSize(mod)); |
| 6720 | 6772 | |
| 6721 | | if (ptr_info.packed_offset.bit_offset % 8 == 0) { |
| 6722 | | try self.load( |
| 6723 | | dst_mcv, |
| 6724 | | ptr_ty, |
| 6725 | | ptr_mcv.offset(@intCast(@divExact(ptr_info.packed_offset.bit_offset, 8))), |
| 6726 | | ); |
| 6727 | | const val_bit_size: u32 = @intCast(val_ty.bitSize(mod)); |
| 6773 | const val_bit_size: u32 = @intCast(val_ty.bitSize(mod)); |
| 6774 | const ptr_bit_off = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) { |
| 6775 | .none => 0, |
| 6776 | .runtime => unreachable, |
| 6777 | else => |vector_index| @intFromEnum(vector_index) * val_bit_size, |
| 6778 | }; |
| 6779 | if (ptr_bit_off % 8 == 0) { |
| 6780 | try self.load(dst_mcv, ptr_ty, ptr_mcv.offset(@intCast(@divExact(ptr_bit_off, 8)))); |
| 6728 | 6781 | if (val_abi_size * 8 > val_bit_size) { |
| 6729 | 6782 | if (dst_mcv.isRegister()) { |
| 6730 | 6783 | try self.truncateRegister(val_ty, dst_mcv.getReg().?); |
| ... | ... | @@ -6746,9 +6799,8 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn |
| 6746 | 6799 | |
| 6747 | 6800 | const limb_abi_size: u32 = @min(val_abi_size, 8); |
| 6748 | 6801 | const limb_abi_bits = limb_abi_size * 8; |
| 6749 | | const val_byte_off: i32 = |
| 6750 | | @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size); |
| 6751 | | const val_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits; |
| 6802 | const val_byte_off: i32 = @intCast(ptr_bit_off / limb_abi_bits * limb_abi_size); |
| 6803 | const val_bit_off = ptr_bit_off % limb_abi_bits; |
| 6752 | 6804 | const val_extra_bits = self.regExtraBits(val_ty); |
| 6753 | 6805 | |
| 6754 | 6806 | const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); |
| ... | ... | @@ -6875,7 +6927,8 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void { |
| 6875 | 6927 | else |
| 6876 | 6928 | try self.allocRegOrMem(inst, true); |
| 6877 | 6929 | |
| 6878 | | if (ptr_ty.ptrInfo(mod).packed_offset.host_size > 0) { |
| 6930 | const ptr_info = ptr_ty.ptrInfo(mod); |
| 6931 | if (ptr_info.flags.vector_index != .none or ptr_info.packed_offset.host_size > 0) { |
| 6879 | 6932 | try self.packedLoad(dst_mcv, ptr_ty, ptr_mcv); |
| 6880 | 6933 | } else { |
| 6881 | 6934 | try self.load(dst_mcv, ptr_ty, ptr_mcv); |
| ... | ... | @@ -6909,7 +6962,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void { |
| 6909 | 6962 | fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void { |
| 6910 | 6963 | const mod = self.bin_file.options.module.?; |
| 6911 | 6964 | const ptr_info = ptr_ty.ptrInfo(mod); |
| 6912 | | const src_ty = ptr_ty.childType(mod); |
| 6965 | const src_ty = Type.fromInterned(ptr_info.child); |
| 6913 | 6966 | if (!src_ty.hasRuntimeBitsIgnoreComptime(mod)) return; |
| 6914 | 6967 | |
| 6915 | 6968 | const limb_abi_size: u16 = @min(ptr_info.packed_offset.host_size, 8); |
| ... | ... | @@ -6917,8 +6970,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In |
| 6917 | 6970 | const limb_ty = try mod.intType(.unsigned, limb_abi_bits); |
| 6918 | 6971 | |
| 6919 | 6972 | const src_bit_size = src_ty.bitSize(mod); |
| 6920 | | const src_byte_off: i32 = @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size); |
| 6921 | | const src_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits; |
| 6973 | const ptr_bit_off = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) { |
| 6974 | .none => 0, |
| 6975 | .runtime => unreachable, |
| 6976 | else => |vector_index| @intFromEnum(vector_index) * src_bit_size, |
| 6977 | }; |
| 6978 | const src_byte_off: i32 = @intCast(ptr_bit_off / limb_abi_bits * limb_abi_size); |
| 6979 | const src_bit_off = ptr_bit_off % limb_abi_bits; |
| 6922 | 6980 | |
| 6923 | 6981 | const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); |
| 6924 | 6982 | const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg); |
| ... | ... | @@ -7055,11 +7113,14 @@ fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void { |
| 7055 | 7113 | const ptr_mcv = try self.resolveInst(bin_op.lhs); |
| 7056 | 7114 | const ptr_ty = self.typeOf(bin_op.lhs); |
| 7057 | 7115 | const src_mcv = try self.resolveInst(bin_op.rhs); |
| 7058 | | if (ptr_ty.ptrInfo(mod).packed_offset.host_size > 0) { |
| 7116 | |
| 7117 | const ptr_info = ptr_ty.ptrInfo(mod); |
| 7118 | if (ptr_info.flags.vector_index != .none or ptr_info.packed_offset.host_size > 0) { |
| 7059 | 7119 | try self.packedStore(ptr_ty, ptr_mcv, src_mcv); |
| 7060 | 7120 | } else { |
| 7061 | 7121 | try self.store(ptr_ty, ptr_mcv, src_mcv); |
| 7062 | 7122 | } |
| 7123 | |
| 7063 | 7124 | return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 7064 | 7125 | } |
| 7065 | 7126 | |
| ... | ... | @@ -12777,7 +12838,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo |
| 12777 | 12838 | else => { |
| 12778 | 12839 | const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none); |
| 12779 | 12840 | assert(std.mem.indexOfNone(abi.Class, classes, &.{ |
| 12780 | | .integer, .sse, .float, .float_combine, |
| 12841 | .integer, .sse, .memory, .float, .float_combine, |
| 12781 | 12842 | }) == null); |
| 12782 | 12843 | const abi_size = ty.abiSize(mod); |
| 12783 | 12844 | if (abi_size < 4 or |
Jacob Young <jacobly@ziglang.org> 2023-12-01 23:27:35-05:00
src