| ... | @@ -2478,8 +2478,11 @@ fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet { | ... | @@ -2478,8 +2478,11 @@ fn regClassForType(self: *Self, ty: Type) RegisterManager.RegisterBitSet { |
| 2478 | else => abi.RegisterClass.sse, | 2478 | else => abi.RegisterClass.sse, |
| 2479 | }, | 2479 | }, |
| 2480 | .Vector => switch (ty.childType(mod).toIntern()) { | 2480 | .Vector => switch (ty.childType(mod).toIntern()) { |
| 2481 | .bool_type => abi.RegisterClass.gp, | 2481 | .bool_type, .u1_type => abi.RegisterClass.gp, |
| 2482 | else => abi.RegisterClass.sse, | 2482 | else => if (ty.isAbiInt(mod) and ty.intInfo(mod).bits == 1) |
| | 2483 | abi.RegisterClass.gp |
| | 2484 | else |
| | 2485 | abi.RegisterClass.sse, |
| 2483 | }, | 2486 | }, |
| 2484 | else => abi.RegisterClass.gp, | 2487 | else => abi.RegisterClass.gp, |
| 2485 | }; | 2488 | }; |
| ... | @@ -3104,18 +3107,17 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -3104,18 +3107,17 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void { |
| 3104 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; | 3107 | const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 3105 | | 3108 | |
| 3106 | const slice_ty = self.typeOfIndex(inst); | 3109 | const slice_ty = self.typeOfIndex(inst); |
| 3107 | const ptr = try self.resolveInst(bin_op.lhs); | 3110 | const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod)); |
| | 3111 | |
| 3108 | const ptr_ty = self.typeOf(bin_op.lhs); | 3112 | const ptr_ty = self.typeOf(bin_op.lhs); |
| 3109 | const len = try self.resolveInst(bin_op.rhs); | 3113 | try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }); |
| 3110 | const len_ty = self.typeOf(bin_op.rhs); | | |
| 3111 | | 3114 | |
| 3112 | const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(slice_ty, mod)); | 3115 | const len_ty = self.typeOf(bin_op.rhs); |
| 3113 | try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr); | | |
| 3114 | try self.genSetMem( | 3116 | try self.genSetMem( |
| 3115 | .{ .frame = frame_index }, | 3117 | .{ .frame = frame_index }, |
| 3116 | @intCast(ptr_ty.abiSize(mod)), | 3118 | @intCast(ptr_ty.abiSize(mod)), |
| 3117 | len_ty, | 3119 | len_ty, |
| 3118 | len, | 3120 | .{ .air_ref = bin_op.rhs }, |
| 3119 | ); | 3121 | ); |
| 3120 | | 3122 | |
| 3121 | const result = MCValue{ .load_frame = .{ .index = frame_index } }; | 3123 | const result = MCValue{ .load_frame = .{ .index = frame_index } }; |
| ... | @@ -5133,74 +5135,114 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5133,74 +5135,114 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { |
| 5133 | const mod = self.bin_file.options.module.?; | 5135 | const mod = self.bin_file.options.module.?; |
| 5134 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | 5136 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 5135 | | 5137 | |
| 5136 | const array_ty = self.typeOf(bin_op.lhs); | 5138 | const result: MCValue = result: { |
| 5137 | const array = try self.resolveInst(bin_op.lhs); | 5139 | const array_ty = self.typeOf(bin_op.lhs); |
| 5138 | const array_lock: ?RegisterLock = switch (array) { | 5140 | const elem_ty = array_ty.childType(mod); |
| 5139 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | | |
| 5140 | else => null, | | |
| 5141 | }; | | |
| 5142 | defer if (array_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 5143 | | 5141 | |
| 5144 | const elem_ty = array_ty.childType(mod); | 5142 | const array_mcv = try self.resolveInst(bin_op.lhs); |
| 5145 | const elem_abi_size = elem_ty.abiSize(mod); | 5143 | const array_lock: ?RegisterLock = switch (array_mcv) { |
| | 5144 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| | 5145 | else => null, |
| | 5146 | }; |
| | 5147 | defer if (array_lock) |lock| self.register_manager.unlockReg(lock); |
| 5146 | | 5148 | |
| 5147 | const index_ty = self.typeOf(bin_op.rhs); | 5149 | const index_ty = self.typeOf(bin_op.rhs); |
| 5148 | const index = try self.resolveInst(bin_op.rhs); | 5150 | const index_mcv = try self.resolveInst(bin_op.rhs); |
| 5149 | const index_lock: ?RegisterLock = switch (index) { | 5151 | const index_lock: ?RegisterLock = switch (index_mcv) { |
| 5150 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | 5152 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 5151 | else => null, | 5153 | else => null, |
| 5152 | }; | 5154 | }; |
| 5153 | defer if (index_lock) |lock| self.register_manager.unlockReg(lock); | 5155 | defer if (index_lock) |lock| self.register_manager.unlockReg(lock); |
| 5154 | | 5156 | |
| 5155 | const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | 5157 | try self.spillEflagsIfOccupied(); |
| 5156 | const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); | 5158 | if (array_ty.isVector(mod) and elem_ty.bitSize(mod) == 1) { |
| 5157 | defer self.register_manager.unlockReg(addr_lock); | 5159 | const index_reg = switch (index_mcv) { |
| | 5160 | .register => |reg| reg, |
| | 5161 | else => try self.copyToTmpRegister(index_ty, index_mcv), |
| | 5162 | }; |
| | 5163 | switch (array_mcv) { |
| | 5164 | .register => |array_reg| try self.asmRegisterRegister( |
| | 5165 | .{ ._, .bt }, |
| | 5166 | array_reg.to64(), |
| | 5167 | index_reg.to64(), |
| | 5168 | ), |
| | 5169 | .load_frame => try self.asmMemoryRegister( |
| | 5170 | .{ ._, .bt }, |
| | 5171 | try array_mcv.mem(self, .qword), |
| | 5172 | index_reg.to64(), |
| | 5173 | ), |
| | 5174 | .memory, .load_symbol, .load_direct, .load_got, .load_tlv => try self.asmMemoryRegister( |
| | 5175 | .{ ._, .bt }, |
| | 5176 | .{ |
| | 5177 | .base = .{ |
| | 5178 | .reg = try self.copyToTmpRegister(Type.usize, array_mcv.address()), |
| | 5179 | }, |
| | 5180 | .mod = .{ .rm = .{ .size = .qword } }, |
| | 5181 | }, |
| | 5182 | index_reg.to64(), |
| | 5183 | ), |
| | 5184 | else => return self.fail("TODO airArrayElemVal for {s} of {}", .{ |
| | 5185 | @tagName(array_mcv), array_ty.fmt(mod), |
| | 5186 | }), |
| | 5187 | } |
| | 5188 | |
| | 5189 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); |
| | 5190 | try self.asmSetccRegister(.c, dst_reg.to8()); |
| | 5191 | break :result .{ .register = dst_reg }; |
| | 5192 | } |
| | 5193 | |
| | 5194 | const elem_abi_size = elem_ty.abiSize(mod); |
| | 5195 | const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| | 5196 | const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); |
| | 5197 | defer self.register_manager.unlockReg(addr_lock); |
| 5158 | | 5198 | |
| 5159 | switch (array) { | 5199 | switch (array_mcv) { |
| 5160 | .register => { | 5200 | .register => { |
| 5161 | const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod)); | 5201 | const frame_index = try self.allocFrameIndex(FrameAlloc.initType(array_ty, mod)); |
| 5162 | try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array); | 5202 | try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv); |
| 5163 | try self.asmRegisterMemory( | 5203 | try self.asmRegisterMemory( |
| | 5204 | .{ ._, .lea }, |
| | 5205 | addr_reg, |
| | 5206 | .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } }, |
| | 5207 | ); |
| | 5208 | }, |
| | 5209 | .load_frame => |frame_addr| try self.asmRegisterMemory( |
| 5164 | .{ ._, .lea }, | 5210 | .{ ._, .lea }, |
| 5165 | addr_reg, | 5211 | addr_reg, |
| 5166 | .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } }, | 5212 | .{ |
| 5167 | ); | 5213 | .base = .{ .frame = frame_addr.index }, |
| 5168 | }, | 5214 | .mod = .{ .rm = .{ .size = .qword, .disp = frame_addr.off } }, |
| 5169 | .load_frame => |frame_addr| try self.asmRegisterMemory( | 5215 | }, |
| 5170 | .{ ._, .lea }, | 5216 | ), |
| 5171 | addr_reg, | 5217 | .memory, |
| 5172 | .{ | 5218 | .load_symbol, |
| 5173 | .base = .{ .frame = frame_addr.index }, | 5219 | .load_direct, |
| 5174 | .mod = .{ .rm = .{ .size = .qword, .disp = frame_addr.off } }, | 5220 | .load_got, |
| 5175 | }, | 5221 | .load_tlv, |
| 5176 | ), | 5222 | => try self.genSetReg(addr_reg, Type.usize, array_mcv.address()), |
| 5177 | .memory, | 5223 | .lea_symbol, .lea_direct, .lea_tlv => unreachable, |
| 5178 | .load_symbol, | 5224 | else => return self.fail("TODO airArrayElemVal_val for {s} of {}", .{ |
| 5179 | .load_direct, | 5225 | @tagName(array_mcv), array_ty.fmt(mod), |
| 5180 | .load_got, | 5226 | }), |
| 5181 | .load_tlv, | 5227 | } |
| 5182 | => try self.genSetReg(addr_reg, Type.usize, array.address()), | | |
| 5183 | .lea_symbol, .lea_direct, .lea_tlv => unreachable, | | |
| 5184 | else => return self.fail("TODO implement array_elem_val when array is {}", .{array}), | | |
| 5185 | } | | |
| 5186 | | | |
| 5187 | const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size); | | |
| 5188 | const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg); | | |
| 5189 | defer self.register_manager.unlockReg(offset_lock); | | |
| 5190 | | 5228 | |
| 5191 | // TODO we could allocate register here, but need to expect addr register and potentially | 5229 | const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size); |
| 5192 | // offset register. | 5230 | const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg); |
| 5193 | try self.spillEflagsIfOccupied(); | 5231 | defer self.register_manager.unlockReg(offset_lock); |
| 5194 | const dst_mcv = try self.allocRegOrMem(inst, false); | | |
| 5195 | try self.genBinOpMir( | | |
| 5196 | .{ ._, .add }, | | |
| 5197 | Type.usize, | | |
| 5198 | .{ .register = addr_reg }, | | |
| 5199 | .{ .register = offset_reg }, | | |
| 5200 | ); | | |
| 5201 | try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }); | | |
| 5202 | | 5232 | |
| 5203 | return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); | 5233 | // TODO we could allocate register here, but need to expect addr register and potentially |
| | 5234 | // offset register. |
| | 5235 | const dst_mcv = try self.allocRegOrMem(inst, false); |
| | 5236 | try self.genBinOpMir( |
| | 5237 | .{ ._, .add }, |
| | 5238 | Type.usize, |
| | 5239 | .{ .register = addr_reg }, |
| | 5240 | .{ .register = offset_reg }, |
| | 5241 | ); |
| | 5242 | try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }); |
| | 5243 | break :result dst_mcv; |
| | 5244 | }; |
| | 5245 | return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 5204 | } | 5246 | } |
| 5205 | | 5247 | |
| 5206 | fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void { | 5248 | fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void { |
| ... | @@ -5258,32 +5300,44 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -5258,32 +5300,44 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void { |
| 5258 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; | 5300 | const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; |
| 5259 | const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; | 5301 | const extra = self.air.extraData(Air.Bin, ty_pl.payload).data; |
| 5260 | | 5302 | |
| 5261 | const ptr_ty = self.typeOf(extra.lhs); | 5303 | const result = result: { |
| 5262 | const ptr = try self.resolveInst(extra.lhs); | 5304 | const elem_ptr_ty = self.typeOfIndex(inst); |
| 5263 | const ptr_lock: ?RegisterLock = switch (ptr) { | 5305 | const base_ptr_ty = self.typeOf(extra.lhs); |
| 5264 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | | |
| 5265 | else => null, | | |
| 5266 | }; | | |
| 5267 | defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 5268 | | 5306 | |
| 5269 | const elem_ty = ptr_ty.elemType2(mod); | 5307 | const base_ptr_mcv = try self.resolveInst(extra.lhs); |
| 5270 | const elem_abi_size = elem_ty.abiSize(mod); | 5308 | const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) { |
| 5271 | const index_ty = self.typeOf(extra.rhs); | 5309 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 5272 | const index = try self.resolveInst(extra.rhs); | 5310 | else => null, |
| 5273 | const index_lock: ?RegisterLock = switch (index) { | 5311 | }; |
| 5274 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | 5312 | defer if (base_ptr_lock) |lock| self.register_manager.unlockReg(lock); |
| 5275 | else => null, | | |
| 5276 | }; | | |
| 5277 | defer if (index_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 5278 | | 5313 | |
| 5279 | const offset_reg = try self.elemOffset(index_ty, index, elem_abi_size); | 5314 | if (elem_ptr_ty.ptrInfo(mod).flags.vector_index != .none) { |
| 5280 | const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); | 5315 | break :result if (self.reuseOperand(inst, extra.lhs, 0, base_ptr_mcv)) |
| 5281 | defer self.register_manager.unlockReg(offset_reg_lock); | 5316 | base_ptr_mcv |
| | 5317 | else |
| | 5318 | try self.copyToRegisterWithInstTracking(inst, elem_ptr_ty, base_ptr_mcv); |
| | 5319 | } |
| 5282 | | 5320 | |
| 5283 | const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr); | 5321 | const elem_ty = base_ptr_ty.elemType2(mod); |
| 5284 | try self.genBinOpMir(.{ ._, .add }, ptr_ty, dst_mcv, .{ .register = offset_reg }); | 5322 | const elem_abi_size = elem_ty.abiSize(mod); |
| | 5323 | const index_ty = self.typeOf(extra.rhs); |
| | 5324 | const index_mcv = try self.resolveInst(extra.rhs); |
| | 5325 | const index_lock: ?RegisterLock = switch (index_mcv) { |
| | 5326 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| | 5327 | else => null, |
| | 5328 | }; |
| | 5329 | defer if (index_lock) |lock| self.register_manager.unlockReg(lock); |
| 5285 | | 5330 | |
| 5286 | return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none }); | 5331 | const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size); |
| | 5332 | const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); |
| | 5333 | defer self.register_manager.unlockReg(offset_reg_lock); |
| | 5334 | |
| | 5335 | const dst_mcv = try self.copyToRegisterWithInstTracking(inst, elem_ptr_ty, base_ptr_mcv); |
| | 5336 | try self.genBinOpMir(.{ ._, .add }, elem_ptr_ty, dst_mcv, .{ .register = offset_reg }); |
| | 5337 | |
| | 5338 | break :result dst_mcv; |
| | 5339 | }; |
| | 5340 | return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none }); |
| 5287 | } | 5341 | } |
| 5288 | | 5342 | |
| 5289 | fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void { | 5343 | fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void { |
| ... | @@ -6712,19 +6766,20 @@ fn reuseOperandAdvanced( | ... | @@ -6712,19 +6766,20 @@ fn reuseOperandAdvanced( |
| 6712 | | 6766 | |
| 6713 | fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void { | 6767 | fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerError!void { |
| 6714 | const mod = self.bin_file.options.module.?; | 6768 | const mod = self.bin_file.options.module.?; |
| 6715 | const ptr_info = ptr_ty.ptrInfo(mod); | | |
| 6716 | | 6769 | |
| | 6770 | const ptr_info = ptr_ty.ptrInfo(mod); |
| 6717 | const val_ty = Type.fromInterned(ptr_info.child); | 6771 | const val_ty = Type.fromInterned(ptr_info.child); |
| 6718 | if (!val_ty.hasRuntimeBitsIgnoreComptime(mod)) return; | 6772 | if (!val_ty.hasRuntimeBitsIgnoreComptime(mod)) return; |
| 6719 | const val_abi_size: u32 = @intCast(val_ty.abiSize(mod)); | 6773 | const val_abi_size: u32 = @intCast(val_ty.abiSize(mod)); |
| 6720 | | 6774 | |
| 6721 | if (ptr_info.packed_offset.bit_offset % 8 == 0) { | 6775 | const val_bit_size: u32 = @intCast(val_ty.bitSize(mod)); |
| 6722 | try self.load( | 6776 | const ptr_bit_off = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) { |
| 6723 | dst_mcv, | 6777 | .none => 0, |
| 6724 | ptr_ty, | 6778 | .runtime => unreachable, |
| 6725 | ptr_mcv.offset(@intCast(@divExact(ptr_info.packed_offset.bit_offset, 8))), | 6779 | else => |vector_index| @intFromEnum(vector_index) * val_bit_size, |
| 6726 | ); | 6780 | }; |
| 6727 | const val_bit_size: u32 = @intCast(val_ty.bitSize(mod)); | 6781 | if (ptr_bit_off % 8 == 0) { |
| | 6782 | try self.load(dst_mcv, ptr_ty, ptr_mcv.offset(@intCast(@divExact(ptr_bit_off, 8)))); |
| 6728 | if (val_abi_size * 8 > val_bit_size) { | 6783 | if (val_abi_size * 8 > val_bit_size) { |
| 6729 | if (dst_mcv.isRegister()) { | 6784 | if (dst_mcv.isRegister()) { |
| 6730 | try self.truncateRegister(val_ty, dst_mcv.getReg().?); | 6785 | try self.truncateRegister(val_ty, dst_mcv.getReg().?); |
| ... | @@ -6746,9 +6801,8 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn | ... | @@ -6746,9 +6801,8 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn |
| 6746 | | 6801 | |
| 6747 | const limb_abi_size: u32 = @min(val_abi_size, 8); | 6802 | const limb_abi_size: u32 = @min(val_abi_size, 8); |
| 6748 | const limb_abi_bits = limb_abi_size * 8; | 6803 | const limb_abi_bits = limb_abi_size * 8; |
| 6749 | const val_byte_off: i32 = | 6804 | const val_byte_off: i32 = @intCast(ptr_bit_off / limb_abi_bits * limb_abi_size); |
| 6750 | @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size); | 6805 | const val_bit_off = ptr_bit_off % limb_abi_bits; |
| 6751 | const val_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits; | | |
| 6752 | const val_extra_bits = self.regExtraBits(val_ty); | 6806 | const val_extra_bits = self.regExtraBits(val_ty); |
| 6753 | | 6807 | |
| 6754 | const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); | 6808 | const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); |
| ... | @@ -6875,7 +6929,8 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -6875,7 +6929,8 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void { |
| 6875 | else | 6929 | else |
| 6876 | try self.allocRegOrMem(inst, true); | 6930 | try self.allocRegOrMem(inst, true); |
| 6877 | | 6931 | |
| 6878 | if (ptr_ty.ptrInfo(mod).packed_offset.host_size > 0) { | 6932 | const ptr_info = ptr_ty.ptrInfo(mod); |
| | 6933 | if (ptr_info.flags.vector_index != .none or ptr_info.packed_offset.host_size > 0) { |
| 6879 | try self.packedLoad(dst_mcv, ptr_ty, ptr_mcv); | 6934 | try self.packedLoad(dst_mcv, ptr_ty, ptr_mcv); |
| 6880 | } else { | 6935 | } else { |
| 6881 | try self.load(dst_mcv, ptr_ty, ptr_mcv); | 6936 | try self.load(dst_mcv, ptr_ty, ptr_mcv); |
| ... | @@ -6909,7 +6964,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -6909,7 +6964,7 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void { |
| 6909 | fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void { | 6964 | fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerError!void { |
| 6910 | const mod = self.bin_file.options.module.?; | 6965 | const mod = self.bin_file.options.module.?; |
| 6911 | const ptr_info = ptr_ty.ptrInfo(mod); | 6966 | const ptr_info = ptr_ty.ptrInfo(mod); |
| 6912 | const src_ty = ptr_ty.childType(mod); | 6967 | const src_ty = Type.fromInterned(ptr_info.child); |
| 6913 | if (!src_ty.hasRuntimeBitsIgnoreComptime(mod)) return; | 6968 | if (!src_ty.hasRuntimeBitsIgnoreComptime(mod)) return; |
| 6914 | | 6969 | |
| 6915 | const limb_abi_size: u16 = @min(ptr_info.packed_offset.host_size, 8); | 6970 | const limb_abi_size: u16 = @min(ptr_info.packed_offset.host_size, 8); |
| ... | @@ -6917,8 +6972,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In | ... | @@ -6917,8 +6972,13 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In |
| 6917 | const limb_ty = try mod.intType(.unsigned, limb_abi_bits); | 6972 | const limb_ty = try mod.intType(.unsigned, limb_abi_bits); |
| 6918 | | 6973 | |
| 6919 | const src_bit_size = src_ty.bitSize(mod); | 6974 | const src_bit_size = src_ty.bitSize(mod); |
| 6920 | const src_byte_off: i32 = @intCast(ptr_info.packed_offset.bit_offset / limb_abi_bits * limb_abi_size); | 6975 | const ptr_bit_off = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) { |
| 6921 | const src_bit_off = ptr_info.packed_offset.bit_offset % limb_abi_bits; | 6976 | .none => 0, |
| | 6977 | .runtime => unreachable, |
| | 6978 | else => |vector_index| @intFromEnum(vector_index) * src_bit_size, |
| | 6979 | }; |
| | 6980 | const src_byte_off: i32 = @intCast(ptr_bit_off / limb_abi_bits * limb_abi_size); |
| | 6981 | const src_bit_off = ptr_bit_off % limb_abi_bits; |
| 6922 | | 6982 | |
| 6923 | const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); | 6983 | const ptr_reg = try self.copyToTmpRegister(ptr_ty, ptr_mcv); |
| 6924 | const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg); | 6984 | const ptr_lock = self.register_manager.lockRegAssumeUnused(ptr_reg); |
| ... | @@ -7041,24 +7101,25 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr | ... | @@ -7041,24 +7101,25 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr |
| 7041 | | 7101 | |
| 7042 | fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void { | 7102 | fn airStore(self: *Self, inst: Air.Inst.Index, safety: bool) !void { |
| 7043 | const mod = self.bin_file.options.module.?; | 7103 | const mod = self.bin_file.options.module.?; |
| 7044 | if (safety) { | 7104 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 7045 | // TODO if the value is undef, write 0xaa bytes to dest | | |
| 7046 | } else { | | |
| 7047 | // TODO if the value is undef, don't lower this instruction | | |
| 7048 | } | | |
| 7049 | | 7105 | |
| 7050 | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); | 7106 | result: { |
| 7051 | const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx }); | 7107 | if (!safety and (try self.resolveInst(bin_op.rhs)) == .undef) break :result; |
| 7052 | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); | | |
| 7053 | | 7108 | |
| 7054 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | 7109 | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); |
| 7055 | const ptr_mcv = try self.resolveInst(bin_op.lhs); | 7110 | const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx }); |
| 7056 | const ptr_ty = self.typeOf(bin_op.lhs); | 7111 | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); |
| 7057 | const src_mcv = try self.resolveInst(bin_op.rhs); | 7112 | |
| 7058 | if (ptr_ty.ptrInfo(mod).packed_offset.host_size > 0) { | 7113 | const src_mcv = try self.resolveInst(bin_op.rhs); |
| 7059 | try self.packedStore(ptr_ty, ptr_mcv, src_mcv); | 7114 | const ptr_mcv = try self.resolveInst(bin_op.lhs); |
| 7060 | } else { | 7115 | const ptr_ty = self.typeOf(bin_op.lhs); |
| 7061 | try self.store(ptr_ty, ptr_mcv, src_mcv); | 7116 | |
| | 7117 | const ptr_info = ptr_ty.ptrInfo(mod); |
| | 7118 | if (ptr_info.flags.vector_index != .none or ptr_info.packed_offset.host_size > 0) { |
| | 7119 | try self.packedStore(ptr_ty, ptr_mcv, src_mcv); |
| | 7120 | } else { |
| | 7121 | try self.store(ptr_ty, ptr_mcv, src_mcv); |
| | 7122 | } |
| 7062 | } | 7123 | } |
| 7063 | return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); | 7124 | return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 7064 | } | 7125 | } |
| ... | @@ -7263,8 +7324,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -7263,8 +7324,8 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { |
| 7263 | .load_frame => |frame_addr| { | 7324 | .load_frame => |frame_addr| { |
| 7264 | const field_abi_size: u32 = @intCast(field_ty.abiSize(mod)); | 7325 | const field_abi_size: u32 = @intCast(field_ty.abiSize(mod)); |
| 7265 | if (field_off % 8 == 0) { | 7326 | if (field_off % 8 == 0) { |
| 7266 | const off_mcv = | 7327 | const field_byte_off = @divExact(field_off, 8); |
| 7267 | src_mcv.address().offset(@intCast(@divExact(field_off, 8))).deref(); | 7328 | const off_mcv = src_mcv.address().offset(@intCast(field_byte_off)).deref(); |
| 7268 | const field_bit_size = field_ty.bitSize(mod); | 7329 | const field_bit_size = field_ty.bitSize(mod); |
| 7269 | | 7330 | |
| 7270 | if (field_abi_size <= 8) { | 7331 | if (field_abi_size <= 8) { |
| ... | @@ -7289,7 +7350,9 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -7289,7 +7350,9 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { |
| 7289 | try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv); | 7350 | try self.copyToRegisterWithInstTracking(inst, field_ty, dst_mcv); |
| 7290 | } | 7351 | } |
| 7291 | | 7352 | |
| 7292 | const dst_mcv = if (self.reuseOperand(inst, operand, 0, src_mcv)) | 7353 | const container_abi_size: u32 = @intCast(container_ty.abiSize(mod)); |
| | 7354 | const dst_mcv = if (field_byte_off + field_abi_size <= container_abi_size and |
| | 7355 | self.reuseOperand(inst, operand, 0, src_mcv)) |
| 7293 | off_mcv | 7356 | off_mcv |
| 7294 | else dst: { | 7357 | else dst: { |
| 7295 | const dst_mcv = try self.allocRegOrMem(inst, true); | 7358 | const dst_mcv = try self.allocRegOrMem(inst, true); |
| ... | @@ -8422,7 +8485,8 @@ fn genBinOp( | ... | @@ -8422,7 +8485,8 @@ fn genBinOp( |
| 8422 | if (maybe_mask_reg) |mask_reg| self.register_manager.lockRegAssumeUnused(mask_reg) else null; | 8485 | if (maybe_mask_reg) |mask_reg| self.register_manager.lockRegAssumeUnused(mask_reg) else null; |
| 8423 | defer if (mask_lock) |lock| self.register_manager.unlockReg(lock); | 8486 | defer if (mask_lock) |lock| self.register_manager.unlockReg(lock); |
| 8424 | | 8487 | |
| 8425 | const ordered_air = if (lhs_ty.isVector(mod) and switch (lhs_ty.childType(mod).zigTypeTag(mod)) { | 8488 | const ordered_air: [2]Air.Inst.Ref = if (lhs_ty.isVector(mod) and |
| | 8489 | switch (lhs_ty.childType(mod).zigTypeTag(mod)) { |
| 8426 | .Int => switch (air_tag) { | 8490 | .Int => switch (air_tag) { |
| 8427 | .cmp_lt, .cmp_gte => true, | 8491 | .cmp_lt, .cmp_gte => true, |
| 8428 | else => false, | 8492 | else => false, |
| ... | @@ -8432,14 +8496,24 @@ fn genBinOp( | ... | @@ -8432,14 +8496,24 @@ fn genBinOp( |
| 8432 | else => false, | 8496 | else => false, |
| 8433 | }, | 8497 | }, |
| 8434 | else => unreachable, | 8498 | else => unreachable, |
| 8435 | }) .{ .lhs = rhs_air, .rhs = lhs_air } else .{ .lhs = lhs_air, .rhs = rhs_air }; | 8499 | }) .{ rhs_air, lhs_air } else .{ lhs_air, rhs_air }; |
| | 8500 | |
| | 8501 | if (lhs_ty.isAbiInt(mod)) for (ordered_air) |op_air| { |
| | 8502 | switch (try self.resolveInst(op_air)) { |
| | 8503 | .register => |op_reg| switch (op_reg.class()) { |
| | 8504 | .sse => try self.register_manager.getReg(op_reg, null), |
| | 8505 | else => {}, |
| | 8506 | }, |
| | 8507 | else => {}, |
| | 8508 | } |
| | 8509 | }; |
| 8436 | | 8510 | |
| 8437 | const lhs_mcv = try self.resolveInst(ordered_air.lhs); | 8511 | const lhs_mcv = try self.resolveInst(ordered_air[0]); |
| 8438 | var rhs_mcv = try self.resolveInst(ordered_air.rhs); | 8512 | var rhs_mcv = try self.resolveInst(ordered_air[1]); |
| 8439 | switch (lhs_mcv) { | 8513 | switch (lhs_mcv) { |
| 8440 | .immediate => |imm| switch (imm) { | 8514 | .immediate => |imm| switch (imm) { |
| 8441 | 0 => switch (air_tag) { | 8515 | 0 => switch (air_tag) { |
| 8442 | .sub, .sub_wrap => return self.genUnOp(maybe_inst, .neg, ordered_air.rhs), | 8516 | .sub, .sub_wrap => return self.genUnOp(maybe_inst, .neg, ordered_air[1]), |
| 8443 | else => {}, | 8517 | else => {}, |
| 8444 | }, | 8518 | }, |
| 8445 | else => {}, | 8519 | else => {}, |
| ... | @@ -8491,10 +8565,10 @@ fn genBinOp( | ... | @@ -8491,10 +8565,10 @@ fn genBinOp( |
| 8491 | }; | 8565 | }; |
| 8492 | if (maybe_inst) |inst| { | 8566 | if (maybe_inst) |inst| { |
| 8493 | if ((!sse_op or lhs_mcv.isRegister()) and | 8567 | if ((!sse_op or lhs_mcv.isRegister()) and |
| 8494 | self.reuseOperandAdvanced(inst, ordered_air.lhs, 0, lhs_mcv, tracked_inst)) | 8568 | self.reuseOperandAdvanced(inst, ordered_air[0], 0, lhs_mcv, tracked_inst)) |
| 8495 | break :dst lhs_mcv; | 8569 | break :dst lhs_mcv; |
| 8496 | if (is_commutative and (!sse_op or rhs_mcv.isRegister()) and | 8570 | if (is_commutative and (!sse_op or rhs_mcv.isRegister()) and |
| 8497 | self.reuseOperandAdvanced(inst, ordered_air.rhs, 1, rhs_mcv, tracked_inst)) | 8571 | self.reuseOperandAdvanced(inst, ordered_air[1], 1, rhs_mcv, tracked_inst)) |
| 8498 | { | 8572 | { |
| 8499 | flipped = true; | 8573 | flipped = true; |
| 8500 | break :dst rhs_mcv; | 8574 | break :dst rhs_mcv; |
| ... | @@ -8505,7 +8579,7 @@ fn genBinOp( | ... | @@ -8505,7 +8579,7 @@ fn genBinOp( |
| 8505 | copied_to_dst = false | 8579 | copied_to_dst = false |
| 8506 | else | 8580 | else |
| 8507 | try self.genCopy(lhs_ty, dst_mcv, lhs_mcv); | 8581 | try self.genCopy(lhs_ty, dst_mcv, lhs_mcv); |
| 8508 | rhs_mcv = try self.resolveInst(ordered_air.rhs); | 8582 | rhs_mcv = try self.resolveInst(ordered_air[1]); |
| 8509 | break :dst dst_mcv; | 8583 | break :dst dst_mcv; |
| 8510 | }; | 8584 | }; |
| 8511 | const dst_locks: [2]?RegisterLock = switch (dst_mcv) { | 8585 | const dst_locks: [2]?RegisterLock = switch (dst_mcv) { |
| ... | @@ -9905,12 +9979,25 @@ fn genBinOp( | ... | @@ -9905,12 +9979,25 @@ fn genBinOp( |
| 9905 | defer self.register_manager.unlockReg(gp_lock); | 9979 | defer self.register_manager.unlockReg(gp_lock); |
| 9906 | | 9980 | |
| 9907 | try self.asmRegisterRegister(switch (mir_tag[0]) { | 9981 | try self.asmRegisterRegister(switch (mir_tag[0]) { |
| 9908 | ._pd, ._sd => .{ ._pd, .movmsk }, | 9982 | ._pd, ._sd, .p_q => .{ ._pd, .movmsk }, |
| 9909 | ._ps, ._ss => .{ ._ps, .movmsk }, | 9983 | ._ps, ._ss, .p_d => .{ ._ps, .movmsk }, |
| 9910 | .p_b, .p_d, .p_q, .p_w => .{ .p_b, .movmsk }, | 9984 | .p_b => .{ .p_b, .movmsk }, |
| 9911 | .v_pd, .v_sd => .{ .v_pd, .movmsk }, | 9985 | .p_w => movmsk: { |
| 9912 | .v_ps, .v_ss => .{ .v_ps, .movmsk }, | 9986 | try self.asmRegisterRegister(.{ .p_b, .ackssw }, dst_reg, dst_reg); |
| 9913 | .vp_b, .vp_d, .vp_q, .vp_w => .{ .vp_b, .movmsk }, | 9987 | break :movmsk .{ .p_b, .movmsk }; |
| | 9988 | }, |
| | 9989 | .v_pd, .v_sd, .vp_q => .{ .v_pd, .movmsk }, |
| | 9990 | .v_ps, .v_ss, .vp_d => .{ .v_ps, .movmsk }, |
| | 9991 | .vp_b => .{ .vp_b, .movmsk }, |
| | 9992 | .vp_w => movmsk: { |
| | 9993 | try self.asmRegisterRegisterRegister( |
| | 9994 | .{ .vp_b, .ackssw }, |
| | 9995 | dst_reg, |
| | 9996 | dst_reg, |
| | 9997 | dst_reg, |
| | 9998 | ); |
| | 9999 | break :movmsk .{ .vp_b, .movmsk }; |
| | 10000 | }, |
| 9914 | else => unreachable, | 10001 | else => unreachable, |
| 9915 | }, gp_reg.to32(), dst_reg); | 10002 | }, gp_reg.to32(), dst_reg); |
| 9916 | return .{ .register = gp_reg }; | 10003 | return .{ .register = gp_reg }; |
| ... | @@ -11488,7 +11575,6 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index { | ... | @@ -11488,7 +11575,6 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index { |
| 11488 | }, | 11575 | }, |
| 11489 | else => return self.fail("TODO implement condbr when condition is {s}", .{@tagName(mcv)}), | 11576 | else => return self.fail("TODO implement condbr when condition is {s}", .{@tagName(mcv)}), |
| 11490 | } | 11577 | } |
| 11491 | return 0; // TODO | | |
| 11492 | } | 11578 | } |
| 11493 | | 11579 | |
| 11494 | fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { | 11580 | fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { |
| ... | @@ -12275,7 +12361,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -12275,7 +12361,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { |
| 12275 | // for the string, we still use the next u32 for the null terminator. | 12361 | // for the string, we still use the next u32 for the null terminator. |
| 12276 | extra_i += clobber.len / 4 + 1; | 12362 | extra_i += clobber.len / 4 + 1; |
| 12277 | | 12363 | |
| 12278 | // TODO honor these | 12364 | if (std.mem.eql(u8, clobber, "") or std.mem.eql(u8, clobber, "memory")) { |
| | 12365 | // ok, sure |
| | 12366 | } else if (std.mem.eql(u8, clobber, "cc") or |
| | 12367 | std.mem.eql(u8, clobber, "flags") or |
| | 12368 | std.mem.eql(u8, clobber, "eflags") or |
| | 12369 | std.mem.eql(u8, clobber, "rflags")) |
| | 12370 | { |
| | 12371 | try self.spillEflagsIfOccupied(); |
| | 12372 | } else { |
| | 12373 | try self.register_manager.getReg(parseRegName(clobber) orelse |
| | 12374 | return self.fail("invalid clobber: '{s}'", .{clobber}), null); |
| | 12375 | } |
| 12279 | } | 12376 | } |
| 12280 | } | 12377 | } |
| 12281 | | 12378 | |
| ... | @@ -12777,7 +12874,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo | ... | @@ -12777,7 +12874,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo |
| 12777 | else => { | 12874 | else => { |
| 12778 | const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none); | 12875 | const classes = mem.sliceTo(&abi.classifySystemV(ty, mod, .other), .none); |
| 12779 | assert(std.mem.indexOfNone(abi.Class, classes, &.{ | 12876 | assert(std.mem.indexOfNone(abi.Class, classes, &.{ |
| 12780 | .integer, .sse, .float, .float_combine, | 12877 | .integer, .sse, .memory, .float, .float_combine, |
| 12781 | }) == null); | 12878 | }) == null); |
| 12782 | const abi_size = ty.abiSize(mod); | 12879 | const abi_size = ty.abiSize(mod); |
| 12783 | if (abi_size < 4 or | 12880 | if (abi_size < 4 or |
| ... | @@ -12967,6 +13064,24 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo | ... | @@ -12967,6 +13064,24 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo |
| 12967 | }, | 13064 | }, |
| 12968 | else => {}, | 13065 | else => {}, |
| 12969 | }, | 13066 | }, |
| | 13067 | .Pointer, .Optional => if (ty.childType(mod).isPtrAtRuntime(mod)) |
| | 13068 | switch (ty.vectorLen(mod)) { |
| | 13069 | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| | 13070 | .{ .v_q, .mov } |
| | 13071 | else |
| | 13072 | .{ ._q, .mov } }, |
| | 13073 | 2 => return .{ .move = if (self.hasFeature(.avx)) |
| | 13074 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } |
| | 13075 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, |
| | 13076 | 3...4 => if (self.hasFeature(.avx)) |
| | 13077 | return .{ .move = if (aligned) |
| | 13078 | .{ .v_, .movdqa } |
| | 13079 | else |
| | 13080 | .{ .v_, .movdqu } }, |
| | 13081 | else => {}, |
| | 13082 | } |
| | 13083 | else |
| | 13084 | unreachable, |
| 12970 | .Float => switch (ty.childType(mod).floatBits(self.target.*)) { | 13085 | .Float => switch (ty.childType(mod).floatBits(self.target.*)) { |
| 12971 | 16 => switch (ty.vectorLen(mod)) { | 13086 | 16 => switch (ty.vectorLen(mod)) { |
| 12972 | 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ | 13087 | 1 => return if (self.hasFeature(.avx)) .{ .vex_insert_extract = .{ |
| ... | @@ -13456,7 +13571,11 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal | ... | @@ -13456,7 +13571,11 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal |
| 13456 | }; | 13571 | }; |
| 13457 | switch (src_mcv) { | 13572 | switch (src_mcv) { |
| 13458 | .none, .unreach, .dead, .reserved_frame => unreachable, | 13573 | .none, .unreach, .dead, .reserved_frame => unreachable, |
| 13459 | .undef => {}, | 13574 | .undef => try self.genInlineMemset( |
| | 13575 | dst_ptr_mcv, |
| | 13576 | .{ .immediate = 0xaa }, |
| | 13577 | .{ .immediate = abi_size }, |
| | 13578 | ), |
| 13460 | .immediate => |imm| switch (abi_size) { | 13579 | .immediate => |imm| switch (abi_size) { |
| 13461 | 1, 2, 4 => { | 13580 | 1, 2, 4 => { |
| 13462 | const immediate = switch (if (ty.isAbiInt(mod)) | 13581 | const immediate = switch (if (ty.isAbiInt(mod)) |
| ... | @@ -14162,9 +14281,8 @@ fn atomicOp( | ... | @@ -14162,9 +14281,8 @@ fn atomicOp( |
| 14162 | }; | 14281 | }; |
| 14163 | defer if (mem_lock) |lock| self.register_manager.unlockReg(lock); | 14282 | defer if (mem_lock) |lock| self.register_manager.unlockReg(lock); |
| 14164 | | 14283 | |
| 14165 | const method: enum { lock, loop, libcall } = if (val_ty.isRuntimeFloat()) | 14284 | const use_sse = rmw_op orelse .Xchg != .Xchg and val_ty.isRuntimeFloat(); |
| 14166 | .loop | 14285 | const strat: enum { lock, loop, libcall } = if (use_sse) .loop else switch (rmw_op orelse .Xchg) { |
| 14167 | else switch (rmw_op orelse .Xchg) { | | |
| 14168 | .Xchg, | 14286 | .Xchg, |
| 14169 | .Add, | 14287 | .Add, |
| 14170 | .Sub, | 14288 | .Sub, |
| ... | @@ -14178,7 +14296,7 @@ fn atomicOp( | ... | @@ -14178,7 +14296,7 @@ fn atomicOp( |
| 14178 | .Min, | 14296 | .Min, |
| 14179 | => if (val_abi_size <= 16) .loop else .libcall, | 14297 | => if (val_abi_size <= 16) .loop else .libcall, |
| 14180 | }; | 14298 | }; |
| 14181 | switch (method) { | 14299 | switch (strat) { |
| 14182 | .lock => { | 14300 | .lock => { |
| 14183 | const tag: Mir.Inst.Tag = if (rmw_op) |op| switch (op) { | 14301 | const tag: Mir.Inst.Tag = if (rmw_op) |op| switch (op) { |
| 14184 | .Xchg => if (unused) .mov else .xchg, | 14302 | .Xchg => if (unused) .mov else .xchg, |
| ... | @@ -14216,6 +14334,14 @@ fn atomicOp( | ... | @@ -14216,6 +14334,14 @@ fn atomicOp( |
| 14216 | return if (unused) .unreach else dst_mcv; | 14334 | return if (unused) .unreach else dst_mcv; |
| 14217 | }, | 14335 | }, |
| 14218 | .loop => _ = if (val_abi_size <= 8) { | 14336 | .loop => _ = if (val_abi_size <= 8) { |
| | 14337 | const sse_reg: Register = if (use_sse) |
| | 14338 | try self.register_manager.allocReg(null, abi.RegisterClass.sse) |
| | 14339 | else |
| | 14340 | undefined; |
| | 14341 | const sse_lock = |
| | 14342 | if (use_sse) self.register_manager.lockRegAssumeUnused(sse_reg) else undefined; |
| | 14343 | defer if (use_sse) self.register_manager.unlockReg(sse_lock); |
| | 14344 | |
| 14219 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | 14345 | const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 14220 | const tmp_mcv = MCValue{ .register = tmp_reg }; | 14346 | const tmp_mcv = MCValue{ .register = tmp_reg }; |
| 14221 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); | 14347 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| ... | @@ -14223,10 +14349,67 @@ fn atomicOp( | ... | @@ -14223,10 +14349,67 @@ fn atomicOp( |
| 14223 | | 14349 | |
| 14224 | try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem); | 14350 | try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(.rax, val_abi_size), ptr_mem); |
| 14225 | const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); | 14351 | const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); |
| 14226 | if (rmw_op != std.builtin.AtomicRmwOp.Xchg) { | 14352 | if (!use_sse and rmw_op orelse .Xchg != .Xchg) { |
| 14227 | try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax }); | 14353 | try self.genSetReg(tmp_reg, val_ty, .{ .register = .rax }); |
| 14228 | } | 14354 | } |
| 14229 | if (rmw_op) |op| switch (op) { | 14355 | if (rmw_op) |op| if (use_sse) { |
| | 14356 | const mir_tag = @as(?Mir.Inst.FixedTag, switch (op) { |
| | 14357 | .Add => switch (val_ty.floatBits(self.target.*)) { |
| | 14358 | 32 => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add }, |
| | 14359 | 64 => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add }, |
| | 14360 | else => null, |
| | 14361 | }, |
| | 14362 | .Sub => switch (val_ty.floatBits(self.target.*)) { |
| | 14363 | 32 => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub }, |
| | 14364 | 64 => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub }, |
| | 14365 | else => null, |
| | 14366 | }, |
| | 14367 | .Min => switch (val_ty.floatBits(self.target.*)) { |
| | 14368 | 32 => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min }, |
| | 14369 | 64 => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min }, |
| | 14370 | else => null, |
| | 14371 | }, |
| | 14372 | .Max => switch (val_ty.floatBits(self.target.*)) { |
| | 14373 | 32 => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max }, |
| | 14374 | 64 => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max }, |
| | 14375 | else => null, |
| | 14376 | }, |
| | 14377 | else => unreachable, |
| | 14378 | }) orelse return self.fail("TODO implement atomicOp of {s} for {}", .{ |
| | 14379 | @tagName(op), val_ty.fmt(mod), |
| | 14380 | }); |
| | 14381 | try self.genSetReg(sse_reg, val_ty, .{ .register = .rax }); |
| | 14382 | switch (mir_tag[0]) { |
| | 14383 | .v_ss, .v_sd => if (val_mcv.isMemory()) try self.asmRegisterRegisterMemory( |
| | 14384 | mir_tag, |
| | 14385 | sse_reg.to128(), |
| | 14386 | sse_reg.to128(), |
| | 14387 | try val_mcv.mem(self, self.memSize(val_ty)), |
| | 14388 | ) else try self.asmRegisterRegisterRegister( |
| | 14389 | mir_tag, |
| | 14390 | sse_reg.to128(), |
| | 14391 | sse_reg.to128(), |
| | 14392 | (if (val_mcv.isRegister()) |
| | 14393 | val_mcv.getReg().? |
| | 14394 | else |
| | 14395 | try self.copyToTmpRegister(val_ty, val_mcv)).to128(), |
| | 14396 | ), |
| | 14397 | ._ss, ._sd => if (val_mcv.isMemory()) try self.asmRegisterMemory( |
| | 14398 | mir_tag, |
| | 14399 | sse_reg.to128(), |
| | 14400 | try val_mcv.mem(self, self.memSize(val_ty)), |
| | 14401 | ) else try self.asmRegisterRegister( |
| | 14402 | mir_tag, |
| | 14403 | sse_reg.to128(), |
| | 14404 | (if (val_mcv.isRegister()) |
| | 14405 | val_mcv.getReg().? |
| | 14406 | else |
| | 14407 | try self.copyToTmpRegister(val_ty, val_mcv)).to128(), |
| | 14408 | ), |
| | 14409 | else => unreachable, |
| | 14410 | } |
| | 14411 | try self.genSetReg(tmp_reg, val_ty, .{ .register = sse_reg }); |
| | 14412 | } else switch (op) { |
| 14230 | .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv), | 14413 | .Xchg => try self.genSetReg(tmp_reg, val_ty, val_mcv), |
| 14231 | .Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv), | 14414 | .Add => try self.genBinOpMir(.{ ._, .add }, val_ty, tmp_mcv, val_mcv), |
| 14232 | .Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv), | 14415 | .Sub => try self.genBinOpMir(.{ ._, .sub }, val_ty, tmp_mcv, val_mcv), |
| ... | @@ -14362,9 +14545,32 @@ fn atomicOp( | ... | @@ -14362,9 +14545,32 @@ fn atomicOp( |
| 14362 | try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem); | 14545 | try self.asmRegisterMemory(.{ ._, .xor }, .rbx, val_lo_mem); |
| 14363 | try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem); | 14546 | try self.asmRegisterMemory(.{ ._, .xor }, .rcx, val_hi_mem); |
| 14364 | }, | 14547 | }, |
| 14365 | else => return self.fail("TODO implement x86 atomic loop for {} {s}", .{ | 14548 | .Min, .Max => { |
| 14366 | val_ty.fmt(mod), @tagName(op), | 14549 | const cc: Condition = switch (if (val_ty.isAbiInt(mod)) |
| 14367 | }), | 14550 | val_ty.intInfo(mod).signedness |
| | 14551 | else |
| | 14552 | .unsigned) { |
| | 14553 | .unsigned => switch (op) { |
| | 14554 | .Min => .a, |
| | 14555 | .Max => .b, |
| | 14556 | else => unreachable, |
| | 14557 | }, |
| | 14558 | .signed => switch (op) { |
| | 14559 | .Min => .g, |
| | 14560 | .Max => .l, |
| | 14561 | else => unreachable, |
| | 14562 | }, |
| | 14563 | }; |
| | 14564 | |
| | 14565 | const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .register = .rcx }); |
| | 14566 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| | 14567 | defer self.register_manager.unlockReg(tmp_lock); |
| | 14568 | |
| | 14569 | try self.asmRegisterMemory(.{ ._, .cmp }, .rbx, val_lo_mem); |
| | 14570 | try self.asmRegisterMemory(.{ ._, .sbb }, tmp_reg, val_hi_mem); |
| | 14571 | try self.asmCmovccRegisterMemory(cc, .rbx, val_lo_mem); |
| | 14572 | try self.asmCmovccRegisterMemory(cc, .rcx, val_hi_mem); |
| | 14573 | }, |
| 14368 | }; | 14574 | }; |
| 14369 | try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem); | 14575 | try self.asmMemory(.{ .@"lock _16b", .cmpxchg }, ptr_mem); |
| 14370 | _ = try self.asmJccReloc(.ne, loop); | 14576 | _ = try self.asmJccReloc(.ne, loop); |
| ... | @@ -14448,128 +14654,129 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr | ... | @@ -14448,128 +14654,129 @@ fn airAtomicStore(self: *Self, inst: Air.Inst.Index, order: std.builtin.AtomicOr |
| 14448 | | 14654 | |
| 14449 | fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void { | 14655 | fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void { |
| 14450 | const mod = self.bin_file.options.module.?; | 14656 | const mod = self.bin_file.options.module.?; |
| 14451 | if (safety) { | | |
| 14452 | // TODO if the value is undef, write 0xaa bytes to dest | | |
| 14453 | } else { | | |
| 14454 | // TODO if the value is undef, don't lower this instruction | | |
| 14455 | } | | |
| 14456 | | | |
| 14457 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; | 14657 | const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; |
| 14458 | | 14658 | |
| 14459 | try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); | 14659 | result: { |
| 14460 | const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx }); | 14660 | if (!safety and (try self.resolveInst(bin_op.rhs)) == .undef) break :result; |
| 14461 | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); | | |
| 14462 | | | |
| 14463 | const dst_ptr = try self.resolveInst(bin_op.lhs); | | |
| 14464 | const dst_ptr_ty = self.typeOf(bin_op.lhs); | | |
| 14465 | const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) { | | |
| 14466 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | | |
| 14467 | else => null, | | |
| 14468 | }; | | |
| 14469 | defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 14470 | | | |
| 14471 | const src_val = try self.resolveInst(bin_op.rhs); | | |
| 14472 | const elem_ty = self.typeOf(bin_op.rhs); | | |
| 14473 | const src_val_lock: ?RegisterLock = switch (src_val) { | | |
| 14474 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | | |
| 14475 | else => null, | | |
| 14476 | }; | | |
| 14477 | defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock); | | |
| 14478 | | 14661 | |
| 14479 | const elem_abi_size: u31 = @intCast(elem_ty.abiSize(mod)); | 14662 | try self.spillRegisters(&.{ .rax, .rdi, .rsi, .rcx }); |
| | 14663 | const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx }); |
| | 14664 | defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); |
| 14480 | | 14665 | |
| 14481 | if (elem_abi_size == 1) { | 14666 | const dst_ptr = try self.resolveInst(bin_op.lhs); |
| 14482 | const ptr: MCValue = switch (dst_ptr_ty.ptrSize(mod)) { | 14667 | const dst_ptr_ty = self.typeOf(bin_op.lhs); |
| 14483 | // TODO: this only handles slices stored in the stack | 14668 | const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) { |
| 14484 | .Slice => dst_ptr, | | |
| 14485 | .One => dst_ptr, | | |
| 14486 | .C, .Many => unreachable, | | |
| 14487 | }; | | |
| 14488 | const len: MCValue = switch (dst_ptr_ty.ptrSize(mod)) { | | |
| 14489 | // TODO: this only handles slices stored in the stack | | |
| 14490 | .Slice => dst_ptr.address().offset(8).deref(), | | |
| 14491 | .One => .{ .immediate = dst_ptr_ty.childType(mod).arrayLen(mod) }, | | |
| 14492 | .C, .Many => unreachable, | | |
| 14493 | }; | | |
| 14494 | const len_lock: ?RegisterLock = switch (len) { | | |
| 14495 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), | 14669 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 14496 | else => null, | 14670 | else => null, |
| 14497 | }; | 14671 | }; |
| 14498 | defer if (len_lock) |lock| self.register_manager.unlockReg(lock); | 14672 | defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock); |
| 14499 | | | |
| 14500 | try self.genInlineMemset(ptr, src_val, len); | | |
| 14501 | return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); | | |
| 14502 | } | | |
| 14503 | | 14673 | |
| 14504 | // Store the first element, and then rely on memcpy copying forwards. | 14674 | const src_val = try self.resolveInst(bin_op.rhs); |
| 14505 | // Length zero requires a runtime check - so we handle arrays specially | 14675 | const elem_ty = self.typeOf(bin_op.rhs); |
| 14506 | // here to elide it. | 14676 | const src_val_lock: ?RegisterLock = switch (src_val) { |
| 14507 | switch (dst_ptr_ty.ptrSize(mod)) { | 14677 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| 14508 | .Slice => { | 14678 | else => null, |
| 14509 | const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(mod); | 14679 | }; |
| 14510 | | 14680 | defer if (src_val_lock) |lock| self.register_manager.unlockReg(lock); |
| 14511 | // TODO: this only handles slices stored in the stack | | |
| 14512 | const ptr = dst_ptr; | | |
| 14513 | const len = dst_ptr.address().offset(8).deref(); | | |
| 14514 | | | |
| 14515 | // Used to store the number of elements for comparison. | | |
| 14516 | // After comparison, updated to store number of bytes needed to copy. | | |
| 14517 | const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | | |
| 14518 | const len_mcv: MCValue = .{ .register = len_reg }; | | |
| 14519 | const len_lock = self.register_manager.lockRegAssumeUnused(len_reg); | | |
| 14520 | defer self.register_manager.unlockReg(len_lock); | | |
| 14521 | | | |
| 14522 | try self.genSetReg(len_reg, Type.usize, len); | | |
| 14523 | try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg); | | |
| 14524 | | 14681 | |
| 14525 | const skip_reloc = try self.asmJccReloc(.z, undefined); | 14682 | const elem_abi_size: u31 = @intCast(elem_ty.abiSize(mod)); |
| 14526 | try self.store(slice_ptr_ty, ptr, src_val); | | |
| 14527 | | 14683 | |
| 14528 | const second_elem_ptr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | 14684 | if (elem_abi_size == 1) { |
| 14529 | const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; | 14685 | const ptr: MCValue = switch (dst_ptr_ty.ptrSize(mod)) { |
| 14530 | const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); | 14686 | // TODO: this only handles slices stored in the stack |
| 14531 | defer self.register_manager.unlockReg(second_elem_ptr_lock); | 14687 | .Slice => dst_ptr, |
| | 14688 | .One => dst_ptr, |
| | 14689 | .C, .Many => unreachable, |
| | 14690 | }; |
| | 14691 | const len: MCValue = switch (dst_ptr_ty.ptrSize(mod)) { |
| | 14692 | // TODO: this only handles slices stored in the stack |
| | 14693 | .Slice => dst_ptr.address().offset(8).deref(), |
| | 14694 | .One => .{ .immediate = dst_ptr_ty.childType(mod).arrayLen(mod) }, |
| | 14695 | .C, .Many => unreachable, |
| | 14696 | }; |
| | 14697 | const len_lock: ?RegisterLock = switch (len) { |
| | 14698 | .register => |reg| self.register_manager.lockRegAssumeUnused(reg), |
| | 14699 | else => null, |
| | 14700 | }; |
| | 14701 | defer if (len_lock) |lock| self.register_manager.unlockReg(lock); |
| 14532 | | 14702 | |
| 14533 | try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{ | 14703 | try self.genInlineMemset(ptr, src_val, len); |
| 14534 | .reg = try self.copyToTmpRegister(Type.usize, ptr), | 14704 | break :result; |
| 14535 | .off = elem_abi_size, | 14705 | } |
| 14536 | } }); | | |
| 14537 | | 14706 | |
| 14538 | try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 }); | 14707 | // Store the first element, and then rely on memcpy copying forwards. |
| 14539 | try self.asmRegisterRegisterImmediate( | 14708 | // Length zero requires a runtime check - so we handle arrays specially |
| 14540 | .{ .i_, .mul }, | 14709 | // here to elide it. |
| 14541 | len_reg, | 14710 | switch (dst_ptr_ty.ptrSize(mod)) { |
| 14542 | len_reg, | 14711 | .Slice => { |
| 14543 | Immediate.s(elem_abi_size), | 14712 | const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(mod); |
| 14544 | ); | 14713 | |
| 14545 | try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv); | 14714 | // TODO: this only handles slices stored in the stack |
| | 14715 | const ptr = dst_ptr; |
| | 14716 | const len = dst_ptr.address().offset(8).deref(); |
| | 14717 | |
| | 14718 | // Used to store the number of elements for comparison. |
| | 14719 | // After comparison, updated to store number of bytes needed to copy. |
| | 14720 | const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| | 14721 | const len_mcv: MCValue = .{ .register = len_reg }; |
| | 14722 | const len_lock = self.register_manager.lockRegAssumeUnused(len_reg); |
| | 14723 | defer self.register_manager.unlockReg(len_lock); |
| | 14724 | |
| | 14725 | try self.genSetReg(len_reg, Type.usize, len); |
| | 14726 | try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg); |
| | 14727 | |
| | 14728 | const skip_reloc = try self.asmJccReloc(.z, undefined); |
| | 14729 | try self.store(slice_ptr_ty, ptr, src_val); |
| | 14730 | |
| | 14731 | const second_elem_ptr_reg = |
| | 14732 | try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| | 14733 | const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; |
| | 14734 | const second_elem_ptr_lock = |
| | 14735 | self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); |
| | 14736 | defer self.register_manager.unlockReg(second_elem_ptr_lock); |
| | 14737 | |
| | 14738 | try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{ |
| | 14739 | .reg = try self.copyToTmpRegister(Type.usize, ptr), |
| | 14740 | .off = elem_abi_size, |
| | 14741 | } }); |
| | 14742 | |
| | 14743 | try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 }); |
| | 14744 | try self.asmRegisterRegisterImmediate( |
| | 14745 | .{ .i_, .mul }, |
| | 14746 | len_reg, |
| | 14747 | len_reg, |
| | 14748 | Immediate.s(elem_abi_size), |
| | 14749 | ); |
| | 14750 | try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv); |
| 14546 | | 14751 | |
| 14547 | try self.performReloc(skip_reloc); | 14752 | try self.performReloc(skip_reloc); |
| 14548 | }, | 14753 | }, |
| 14549 | .One => { | 14754 | .One => { |
| 14550 | const elem_ptr_ty = try mod.singleMutPtrType(elem_ty); | 14755 | const elem_ptr_ty = try mod.singleMutPtrType(elem_ty); |
| 14551 | | 14756 | |
| 14552 | const len = dst_ptr_ty.childType(mod).arrayLen(mod); | 14757 | const len = dst_ptr_ty.childType(mod).arrayLen(mod); |
| 14553 | | 14758 | |
| 14554 | assert(len != 0); // prevented by Sema | 14759 | assert(len != 0); // prevented by Sema |
| 14555 | try self.store(elem_ptr_ty, dst_ptr, src_val); | 14760 | try self.store(elem_ptr_ty, dst_ptr, src_val); |
| 14556 | | 14761 | |
| 14557 | const second_elem_ptr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); | 14762 | const second_elem_ptr_reg = |
| 14558 | const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; | 14763 | try self.register_manager.allocReg(null, abi.RegisterClass.gp); |
| 14559 | const second_elem_ptr_lock = self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); | 14764 | const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; |
| 14560 | defer self.register_manager.unlockReg(second_elem_ptr_lock); | 14765 | const second_elem_ptr_lock = |
| | 14766 | self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); |
| | 14767 | defer self.register_manager.unlockReg(second_elem_ptr_lock); |
| 14561 | | 14768 | |
| 14562 | try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{ | 14769 | try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{ |
| 14563 | .reg = try self.copyToTmpRegister(Type.usize, dst_ptr), | 14770 | .reg = try self.copyToTmpRegister(Type.usize, dst_ptr), |
| 14564 | .off = elem_abi_size, | 14771 | .off = elem_abi_size, |
| 14565 | } }); | 14772 | } }); |
| 14566 | | 14773 | |
| 14567 | const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; | 14774 | const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; |
| 14568 | try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy); | 14775 | try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy); |
| 14569 | }, | 14776 | }, |
| 14570 | .C, .Many => unreachable, | 14777 | .C, .Many => unreachable, |
| | 14778 | } |
| 14571 | } | 14779 | } |
| 14572 | | | |
| 14573 | return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); | 14780 | return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); |
| 14574 | } | 14781 | } |
| 14575 | | 14782 | |
| ... | @@ -15285,29 +15492,61 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -15285,29 +15492,61 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void { |
| 15285 | } | 15492 | } |
| 15286 | break :result .{ .load_frame = .{ .index = frame_index } }; | 15493 | break :result .{ .load_frame = .{ .index = frame_index } }; |
| 15287 | }, | 15494 | }, |
| 15288 | .Array => { | 15495 | .Array, .Vector => { |
| 15289 | const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, mod)); | | |
| 15290 | const elem_ty = result_ty.childType(mod); | 15496 | const elem_ty = result_ty.childType(mod); |
| 15291 | const elem_size: u32 = @intCast(elem_ty.abiSize(mod)); | 15497 | if (result_ty.isVector(mod) and elem_ty.bitSize(mod) == 1) { |
| | 15498 | const result_size: u32 = @intCast(result_ty.abiSize(mod)); |
| | 15499 | const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); |
| | 15500 | try self.asmRegisterRegister( |
| | 15501 | .{ ._, .xor }, |
| | 15502 | registerAlias(dst_reg, @min(result_size, 4)), |
| | 15503 | registerAlias(dst_reg, @min(result_size, 4)), |
| | 15504 | ); |
| 15292 | | 15505 | |
| 15293 | for (elements, 0..) |elem, elem_i| { | 15506 | for (elements, 0..) |elem, elem_i| { |
| 15294 | const elem_mcv = try self.resolveInst(elem); | 15507 | const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem }); |
| 15295 | const mat_elem_mcv = switch (elem_mcv) { | 15508 | const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg); |
| 15296 | .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index }, | 15509 | defer self.register_manager.unlockReg(elem_lock); |
| 15297 | else => elem_mcv, | 15510 | |
| 15298 | }; | 15511 | try self.asmRegisterImmediate( |
| 15299 | const elem_off: i32 = @intCast(elem_size * elem_i); | 15512 | .{ ._, .@"and" }, |
| 15300 | try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv); | 15513 | registerAlias(elem_reg, @min(result_size, 4)), |
| | 15514 | Immediate.u(1), |
| | 15515 | ); |
| | 15516 | if (elem_i > 0) try self.asmRegisterImmediate( |
| | 15517 | .{ ._l, .sh }, |
| | 15518 | registerAlias(elem_reg, result_size), |
| | 15519 | Immediate.u(@intCast(elem_i)), |
| | 15520 | ); |
| | 15521 | try self.asmRegisterRegister( |
| | 15522 | .{ ._, .@"or" }, |
| | 15523 | registerAlias(dst_reg, result_size), |
| | 15524 | registerAlias(elem_reg, result_size), |
| | 15525 | ); |
| | 15526 | } |
| | 15527 | break :result .{ .register = dst_reg }; |
| | 15528 | } else { |
| | 15529 | const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(result_ty, mod)); |
| | 15530 | const elem_size: u32 = @intCast(elem_ty.abiSize(mod)); |
| | 15531 | |
| | 15532 | for (elements, 0..) |elem, elem_i| { |
| | 15533 | const elem_mcv = try self.resolveInst(elem); |
| | 15534 | const mat_elem_mcv = switch (elem_mcv) { |
| | 15535 | .load_tlv => |sym_index| MCValue{ .lea_tlv = sym_index }, |
| | 15536 | else => elem_mcv, |
| | 15537 | }; |
| | 15538 | const elem_off: i32 = @intCast(elem_size * elem_i); |
| | 15539 | try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv); |
| | 15540 | } |
| | 15541 | if (result_ty.sentinel(mod)) |sentinel| try self.genSetMem( |
| | 15542 | .{ .frame = frame_index }, |
| | 15543 | @intCast(elem_size * elements.len), |
| | 15544 | elem_ty, |
| | 15545 | try self.genTypedValue(.{ .ty = elem_ty, .val = sentinel }), |
| | 15546 | ); |
| | 15547 | break :result .{ .load_frame = .{ .index = frame_index } }; |
| 15301 | } | 15548 | } |
| 15302 | if (result_ty.sentinel(mod)) |sentinel| try self.genSetMem( | | |
| 15303 | .{ .frame = frame_index }, | | |
| 15304 | @intCast(elem_size * elements.len), | | |
| 15305 | elem_ty, | | |
| 15306 | try self.genTypedValue(.{ .ty = elem_ty, .val = sentinel }), | | |
| 15307 | ); | | |
| 15308 | break :result .{ .load_frame = .{ .index = frame_index } }; | | |
| 15309 | }, | 15549 | }, |
| 15310 | .Vector => return self.fail("TODO implement aggregate_init for vectors", .{}), | | |
| 15311 | else => unreachable, | 15550 | else => unreachable, |
| 15312 | } | 15551 | } |
| 15313 | }; | 15552 | }; |
Jacob Young <jacobly@ziglang.org> 2023-12-04 18:37:39-05:00
GitHub <noreply@github.com> 2023-12-04 18:37:39-05:00
Signed by PGP key 4AEE18F83AFDEB23
src