| ... | ... | @@ -6570,6 +6570,34 @@ fn genBinOp( |
| 6570 | 6570 | const lhs_ty = self.typeOf(lhs_air); |
| 6571 | 6571 | const rhs_ty = self.typeOf(rhs_air); |
| 6572 | 6572 | const abi_size: u32 = @intCast(lhs_ty.abiSize(mod)); |
| 6573 | |
| 6574 | if (lhs_ty.isRuntimeFloat() and switch (lhs_ty.floatBits(self.target.*)) { |
| 6575 | 16 => !self.hasFeature(.f16c), |
| 6576 | 32, 64 => false, |
| 6577 | 80, 128 => true, |
| 6578 | else => unreachable, |
| 6579 | }) { |
| 6580 | var callee: ["__add?f3".len]u8 = undefined; |
| 6581 | return self.genCall(.{ .lib = .{ |
| 6582 | .return_type = lhs_ty.toIntern(), |
| 6583 | .param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() }, |
| 6584 | .callee = switch (air_tag) { |
| 6585 | .add, .sub, .mul, .div_float => std.fmt.bufPrint(&callee, "__{s}{c}f3", .{ |
| 6586 | @tagName(air_tag)[0..3], |
| 6587 | floatCompilerRtAbiName(lhs_ty.floatBits(self.target.*)), |
| 6588 | }), |
| 6589 | .min, .max => std.fmt.bufPrint(&callee, "{s}f{s}{s}", .{ |
| 6590 | floatLibcAbiPrefix(lhs_ty), |
| 6591 | @tagName(air_tag), |
| 6592 | floatLibcAbiSuffix(lhs_ty), |
| 6593 | }), |
| 6594 | else => return self.fail("TODO implement genBinOp for {s} {}", .{ |
| 6595 | @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?), |
| 6596 | }), |
| 6597 | } catch unreachable, |
| 6598 | } }, &.{ lhs_ty, rhs_ty }, &.{ .{ .air_ref = lhs_air }, .{ .air_ref = rhs_air } }); |
| 6599 | } |
| 6600 | |
| 6573 | 6601 | if ((lhs_ty.scalarType(mod).isRuntimeFloat() and |
| 6574 | 6602 | lhs_ty.scalarType(mod).floatBits(self.target.*) == 80) or |
| 6575 | 6603 | lhs_ty.abiSize(mod) > @as(u6, if (self.hasFeature(.avx)) 32 else 16)) |
| ... | ... | @@ -6830,7 +6858,8 @@ fn genBinOp( |
| 6830 | 6858 | const mir_tag = @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(mod)) { |
| 6831 | 6859 | else => unreachable, |
| 6832 | 6860 | .Float => switch (lhs_ty.floatBits(self.target.*)) { |
| 6833 | | 16 => if (self.hasFeature(.f16c)) { |
| 6861 | 16 => { |
| 6862 | assert(self.hasFeature(.f16c)); |
| 6834 | 6863 | const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128(); |
| 6835 | 6864 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 6836 | 6865 | defer self.register_manager.unlockReg(tmp_lock); |
| ... | ... | @@ -6873,7 +6902,7 @@ fn genBinOp( |
| 6873 | 6902 | Immediate.u(0b1_00), |
| 6874 | 6903 | ); |
| 6875 | 6904 | return dst_mcv; |
| 6876 | | } else null, |
| 6905 | }, |
| 6877 | 6906 | 32 => switch (air_tag) { |
| 6878 | 6907 | .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add }, |
| 6879 | 6908 | .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub }, |
| ... | ... | @@ -7134,181 +7163,184 @@ fn genBinOp( |
| 7134 | 7163 | else => null, |
| 7135 | 7164 | }, |
| 7136 | 7165 | .Float => switch (lhs_ty.childType(mod).floatBits(self.target.*)) { |
| 7137 | | 16 => if (self.hasFeature(.f16c)) switch (lhs_ty.vectorLen(mod)) { |
| 7138 | | 1 => { |
| 7139 | | const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128(); |
| 7140 | | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7141 | | defer self.register_manager.unlockReg(tmp_lock); |
| 7166 | 16 => tag: { |
| 7167 | assert(self.hasFeature(.f16c)); |
| 7168 | switch (lhs_ty.vectorLen(mod)) { |
| 7169 | 1 => { |
| 7170 | const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128(); |
| 7171 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7172 | defer self.register_manager.unlockReg(tmp_lock); |
| 7142 | 7173 | |
| 7143 | | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 7144 | | .{ .vp_w, .insr }, |
| 7145 | | dst_reg, |
| 7146 | | dst_reg, |
| 7147 | | src_mcv.mem(.word), |
| 7148 | | Immediate.u(1), |
| 7149 | | ) else try self.asmRegisterRegisterRegister( |
| 7150 | | .{ .vp_, .unpcklwd }, |
| 7151 | | dst_reg, |
| 7152 | | dst_reg, |
| 7153 | | (if (src_mcv.isRegister()) |
| 7154 | | src_mcv.getReg().? |
| 7155 | | else |
| 7156 | | try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), |
| 7157 | | ); |
| 7158 | | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); |
| 7159 | | try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg); |
| 7160 | | try self.asmRegisterRegisterRegister( |
| 7161 | | switch (air_tag) { |
| 7162 | | .add => .{ .v_ss, .add }, |
| 7163 | | .sub => .{ .v_ss, .sub }, |
| 7164 | | .mul => .{ .v_ss, .mul }, |
| 7165 | | .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div }, |
| 7166 | | .max => .{ .v_ss, .max }, |
| 7167 | | .min => .{ .v_ss, .max }, |
| 7168 | | else => unreachable, |
| 7169 | | }, |
| 7170 | | dst_reg, |
| 7171 | | dst_reg, |
| 7172 | | tmp_reg, |
| 7173 | | ); |
| 7174 | | try self.asmRegisterRegisterImmediate( |
| 7175 | | .{ .v_, .cvtps2ph }, |
| 7176 | | dst_reg, |
| 7177 | | dst_reg, |
| 7178 | | Immediate.u(0b1_00), |
| 7179 | | ); |
| 7180 | | return dst_mcv; |
| 7181 | | }, |
| 7182 | | 2 => { |
| 7183 | | const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128(); |
| 7184 | | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7185 | | defer self.register_manager.unlockReg(tmp_lock); |
| 7174 | if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate( |
| 7175 | .{ .vp_w, .insr }, |
| 7176 | dst_reg, |
| 7177 | dst_reg, |
| 7178 | src_mcv.mem(.word), |
| 7179 | Immediate.u(1), |
| 7180 | ) else try self.asmRegisterRegisterRegister( |
| 7181 | .{ .vp_, .unpcklwd }, |
| 7182 | dst_reg, |
| 7183 | dst_reg, |
| 7184 | (if (src_mcv.isRegister()) |
| 7185 | src_mcv.getReg().? |
| 7186 | else |
| 7187 | try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), |
| 7188 | ); |
| 7189 | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); |
| 7190 | try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg); |
| 7191 | try self.asmRegisterRegisterRegister( |
| 7192 | switch (air_tag) { |
| 7193 | .add => .{ .v_ss, .add }, |
| 7194 | .sub => .{ .v_ss, .sub }, |
| 7195 | .mul => .{ .v_ss, .mul }, |
| 7196 | .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div }, |
| 7197 | .max => .{ .v_ss, .max }, |
| 7198 | .min => .{ .v_ss, .max }, |
| 7199 | else => unreachable, |
| 7200 | }, |
| 7201 | dst_reg, |
| 7202 | dst_reg, |
| 7203 | tmp_reg, |
| 7204 | ); |
| 7205 | try self.asmRegisterRegisterImmediate( |
| 7206 | .{ .v_, .cvtps2ph }, |
| 7207 | dst_reg, |
| 7208 | dst_reg, |
| 7209 | Immediate.u(0b1_00), |
| 7210 | ); |
| 7211 | return dst_mcv; |
| 7212 | }, |
| 7213 | 2 => { |
| 7214 | const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128(); |
| 7215 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7216 | defer self.register_manager.unlockReg(tmp_lock); |
| 7186 | 7217 | |
| 7187 | | if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 7188 | | .{ .vp_d, .insr }, |
| 7189 | | dst_reg, |
| 7190 | | src_mcv.mem(.dword), |
| 7191 | | Immediate.u(1), |
| 7192 | | ) else try self.asmRegisterRegisterRegister( |
| 7193 | | .{ .v_ps, .unpckl }, |
| 7194 | | dst_reg, |
| 7195 | | dst_reg, |
| 7196 | | (if (src_mcv.isRegister()) |
| 7197 | | src_mcv.getReg().? |
| 7198 | | else |
| 7199 | | try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), |
| 7200 | | ); |
| 7201 | | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); |
| 7202 | | try self.asmRegisterRegisterRegister( |
| 7203 | | .{ .v_ps, .movhl }, |
| 7204 | | tmp_reg, |
| 7205 | | dst_reg, |
| 7206 | | dst_reg, |
| 7207 | | ); |
| 7208 | | try self.asmRegisterRegisterRegister( |
| 7209 | | switch (air_tag) { |
| 7210 | | .add => .{ .v_ps, .add }, |
| 7211 | | .sub => .{ .v_ps, .sub }, |
| 7212 | | .mul => .{ .v_ps, .mul }, |
| 7213 | | .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, |
| 7214 | | .max => .{ .v_ps, .max }, |
| 7215 | | .min => .{ .v_ps, .max }, |
| 7216 | | else => unreachable, |
| 7217 | | }, |
| 7218 | | dst_reg, |
| 7219 | | dst_reg, |
| 7220 | | tmp_reg, |
| 7221 | | ); |
| 7222 | | try self.asmRegisterRegisterImmediate( |
| 7223 | | .{ .v_, .cvtps2ph }, |
| 7224 | | dst_reg, |
| 7225 | | dst_reg, |
| 7226 | | Immediate.u(0b1_00), |
| 7227 | | ); |
| 7228 | | return dst_mcv; |
| 7229 | | }, |
| 7230 | | 3...4 => { |
| 7231 | | const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128(); |
| 7232 | | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7233 | | defer self.register_manager.unlockReg(tmp_lock); |
| 7218 | if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 7219 | .{ .vp_d, .insr }, |
| 7220 | dst_reg, |
| 7221 | src_mcv.mem(.dword), |
| 7222 | Immediate.u(1), |
| 7223 | ) else try self.asmRegisterRegisterRegister( |
| 7224 | .{ .v_ps, .unpckl }, |
| 7225 | dst_reg, |
| 7226 | dst_reg, |
| 7227 | (if (src_mcv.isRegister()) |
| 7228 | src_mcv.getReg().? |
| 7229 | else |
| 7230 | try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), |
| 7231 | ); |
| 7232 | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); |
| 7233 | try self.asmRegisterRegisterRegister( |
| 7234 | .{ .v_ps, .movhl }, |
| 7235 | tmp_reg, |
| 7236 | dst_reg, |
| 7237 | dst_reg, |
| 7238 | ); |
| 7239 | try self.asmRegisterRegisterRegister( |
| 7240 | switch (air_tag) { |
| 7241 | .add => .{ .v_ps, .add }, |
| 7242 | .sub => .{ .v_ps, .sub }, |
| 7243 | .mul => .{ .v_ps, .mul }, |
| 7244 | .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, |
| 7245 | .max => .{ .v_ps, .max }, |
| 7246 | .min => .{ .v_ps, .max }, |
| 7247 | else => unreachable, |
| 7248 | }, |
| 7249 | dst_reg, |
| 7250 | dst_reg, |
| 7251 | tmp_reg, |
| 7252 | ); |
| 7253 | try self.asmRegisterRegisterImmediate( |
| 7254 | .{ .v_, .cvtps2ph }, |
| 7255 | dst_reg, |
| 7256 | dst_reg, |
| 7257 | Immediate.u(0b1_00), |
| 7258 | ); |
| 7259 | return dst_mcv; |
| 7260 | }, |
| 7261 | 3...4 => { |
| 7262 | const tmp_reg = (try self.register_manager.allocReg(null, sse)).to128(); |
| 7263 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7264 | defer self.register_manager.unlockReg(tmp_lock); |
| 7234 | 7265 | |
| 7235 | | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); |
| 7236 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 7237 | | .{ .v_ps, .cvtph2 }, |
| 7238 | | tmp_reg, |
| 7239 | | src_mcv.mem(.qword), |
| 7240 | | ) else try self.asmRegisterRegister( |
| 7241 | | .{ .v_ps, .cvtph2 }, |
| 7242 | | tmp_reg, |
| 7243 | | (if (src_mcv.isRegister()) |
| 7244 | | src_mcv.getReg().? |
| 7245 | | else |
| 7246 | | try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), |
| 7247 | | ); |
| 7248 | | try self.asmRegisterRegisterRegister( |
| 7249 | | switch (air_tag) { |
| 7250 | | .add => .{ .v_ps, .add }, |
| 7251 | | .sub => .{ .v_ps, .sub }, |
| 7252 | | .mul => .{ .v_ps, .mul }, |
| 7253 | | .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, |
| 7254 | | .max => .{ .v_ps, .max }, |
| 7255 | | .min => .{ .v_ps, .max }, |
| 7256 | | else => unreachable, |
| 7257 | | }, |
| 7258 | | dst_reg, |
| 7259 | | dst_reg, |
| 7260 | | tmp_reg, |
| 7261 | | ); |
| 7262 | | try self.asmRegisterRegisterImmediate( |
| 7263 | | .{ .v_, .cvtps2ph }, |
| 7264 | | dst_reg, |
| 7265 | | dst_reg, |
| 7266 | | Immediate.u(0b1_00), |
| 7267 | | ); |
| 7268 | | return dst_mcv; |
| 7269 | | }, |
| 7270 | | 5...8 => { |
| 7271 | | const tmp_reg = (try self.register_manager.allocReg(null, sse)).to256(); |
| 7272 | | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7273 | | defer self.register_manager.unlockReg(tmp_lock); |
| 7266 | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg); |
| 7267 | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 7268 | .{ .v_ps, .cvtph2 }, |
| 7269 | tmp_reg, |
| 7270 | src_mcv.mem(.qword), |
| 7271 | ) else try self.asmRegisterRegister( |
| 7272 | .{ .v_ps, .cvtph2 }, |
| 7273 | tmp_reg, |
| 7274 | (if (src_mcv.isRegister()) |
| 7275 | src_mcv.getReg().? |
| 7276 | else |
| 7277 | try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), |
| 7278 | ); |
| 7279 | try self.asmRegisterRegisterRegister( |
| 7280 | switch (air_tag) { |
| 7281 | .add => .{ .v_ps, .add }, |
| 7282 | .sub => .{ .v_ps, .sub }, |
| 7283 | .mul => .{ .v_ps, .mul }, |
| 7284 | .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, |
| 7285 | .max => .{ .v_ps, .max }, |
| 7286 | .min => .{ .v_ps, .max }, |
| 7287 | else => unreachable, |
| 7288 | }, |
| 7289 | dst_reg, |
| 7290 | dst_reg, |
| 7291 | tmp_reg, |
| 7292 | ); |
| 7293 | try self.asmRegisterRegisterImmediate( |
| 7294 | .{ .v_, .cvtps2ph }, |
| 7295 | dst_reg, |
| 7296 | dst_reg, |
| 7297 | Immediate.u(0b1_00), |
| 7298 | ); |
| 7299 | return dst_mcv; |
| 7300 | }, |
| 7301 | 5...8 => { |
| 7302 | const tmp_reg = (try self.register_manager.allocReg(null, sse)).to256(); |
| 7303 | const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); |
| 7304 | defer self.register_manager.unlockReg(tmp_lock); |
| 7274 | 7305 | |
| 7275 | | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), dst_reg); |
| 7276 | | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 7277 | | .{ .v_ps, .cvtph2 }, |
| 7278 | | tmp_reg, |
| 7279 | | src_mcv.mem(.xword), |
| 7280 | | ) else try self.asmRegisterRegister( |
| 7281 | | .{ .v_ps, .cvtph2 }, |
| 7282 | | tmp_reg, |
| 7283 | | (if (src_mcv.isRegister()) |
| 7284 | | src_mcv.getReg().? |
| 7285 | | else |
| 7286 | | try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), |
| 7287 | | ); |
| 7288 | | try self.asmRegisterRegisterRegister( |
| 7289 | | switch (air_tag) { |
| 7290 | | .add => .{ .v_ps, .add }, |
| 7291 | | .sub => .{ .v_ps, .sub }, |
| 7292 | | .mul => .{ .v_ps, .mul }, |
| 7293 | | .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, |
| 7294 | | .max => .{ .v_ps, .max }, |
| 7295 | | .min => .{ .v_ps, .max }, |
| 7296 | | else => unreachable, |
| 7297 | | }, |
| 7298 | | dst_reg.to256(), |
| 7299 | | dst_reg.to256(), |
| 7300 | | tmp_reg, |
| 7301 | | ); |
| 7302 | | try self.asmRegisterRegisterImmediate( |
| 7303 | | .{ .v_, .cvtps2ph }, |
| 7304 | | dst_reg, |
| 7305 | | dst_reg.to256(), |
| 7306 | | Immediate.u(0b1_00), |
| 7307 | | ); |
| 7308 | | return dst_mcv; |
| 7309 | | }, |
| 7310 | | else => null, |
| 7311 | | } else null, |
| 7306 | try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), dst_reg); |
| 7307 | if (src_mcv.isMemory()) try self.asmRegisterMemory( |
| 7308 | .{ .v_ps, .cvtph2 }, |
| 7309 | tmp_reg, |
| 7310 | src_mcv.mem(.xword), |
| 7311 | ) else try self.asmRegisterRegister( |
| 7312 | .{ .v_ps, .cvtph2 }, |
| 7313 | tmp_reg, |
| 7314 | (if (src_mcv.isRegister()) |
| 7315 | src_mcv.getReg().? |
| 7316 | else |
| 7317 | try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(), |
| 7318 | ); |
| 7319 | try self.asmRegisterRegisterRegister( |
| 7320 | switch (air_tag) { |
| 7321 | .add => .{ .v_ps, .add }, |
| 7322 | .sub => .{ .v_ps, .sub }, |
| 7323 | .mul => .{ .v_ps, .mul }, |
| 7324 | .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div }, |
| 7325 | .max => .{ .v_ps, .max }, |
| 7326 | .min => .{ .v_ps, .max }, |
| 7327 | else => unreachable, |
| 7328 | }, |
| 7329 | dst_reg.to256(), |
| 7330 | dst_reg.to256(), |
| 7331 | tmp_reg, |
| 7332 | ); |
| 7333 | try self.asmRegisterRegisterImmediate( |
| 7334 | .{ .v_, .cvtps2ph }, |
| 7335 | dst_reg, |
| 7336 | dst_reg.to256(), |
| 7337 | Immediate.u(0b1_00), |
| 7338 | ); |
| 7339 | return dst_mcv; |
| 7340 | }, |
| 7341 | else => break :tag null, |
| 7342 | } |
| 7343 | }, |
| 7312 | 7344 | 32 => switch (lhs_ty.vectorLen(mod)) { |
| 7313 | 7345 | 1 => switch (air_tag) { |
| 7314 | 7346 | .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add }, |