authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-07 10:04:56-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-08 07:36:20-04:00
logf8708e2c4d93eece5b3e131fd2d1b5b210806cd6
tree06385bc6d2841e74a90225f98ac89460430f38e7
parent057139fda575e0e6038b821256a45669cd70a073

x86_64: implement `@floor`, `@ceil`, and `@trunc` for float vectors

2 files changed, 93 insertions(+), 92 deletions(-)

src/arch/x86_64/CodeGen.zig+87-89
......@@ -1587,9 +1587,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
15871587 .round,
15881588 => try self.airUnaryMath(inst),
15891589
1590 .floor => try self.airRound(inst, Immediate.u(0b1_0_01)),
1591 .ceil => try self.airRound(inst, Immediate.u(0b1_0_10)),
1592 .trunc_float => try self.airRound(inst, Immediate.u(0b1_0_11)),
1590 .floor => try self.airRound(inst, 0b1_0_01),
1591 .ceil => try self.airRound(inst, 0b1_0_10),
1592 .trunc_float => try self.airRound(inst, 0b1_0_11),
15931593 .sqrt => try self.airSqrt(inst),
15941594 .neg, .fabs => try self.airFloatSign(inst),
15951595
......@@ -4509,49 +4509,91 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void {
45094509 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
45104510}
45114511
4512fn airRound(self: *Self, inst: Air.Inst.Index, mode: Immediate) !void {
4512fn airRound(self: *Self, inst: Air.Inst.Index, mode: u4) !void {
45134513 const un_op = self.air.instructions.items(.data)[inst].un_op;
45144514 const ty = self.air.typeOf(un_op);
45154515
4516 if (!self.hasFeature(.sse4_1))
4517 return self.fail("TODO implement airRound without sse4_1 feature", .{});
4518
45194516 const src_mcv = try self.resolveInst(un_op);
45204517 const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
45214518 src_mcv
45224519 else
45234520 try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
4521 const dst_reg = dst_mcv.getReg().?;
4522 const dst_lock = self.register_manager.lockReg(dst_reg);
4523 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
4524 try self.genRound(ty, dst_reg, src_mcv, mode);
4525 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
4526}
45244527
4525 const mir_tag: Mir.Inst.Tag = switch (ty.zigTypeTag()) {
4528fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4) !void {
4529 if (!self.hasFeature(.sse4_1))
4530 return self.fail("TODO implement genRound without sse4_1 feature", .{});
4531
4532 const mir_tag = if (@as(?Mir.Inst.Tag, switch (ty.zigTypeTag()) {
45264533 .Float => switch (ty.floatBits(self.target.*)) {
4527 32 => .roundss,
4528 64 => .roundsd,
4529 else => return self.fail("TODO implement airRound for {}", .{
4530 ty.fmt(self.bin_file.options.module.?),
4531 }),
4534 32 => if (self.hasFeature(.avx)) .vroundss else .roundss,
4535 64 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
4536 16, 80, 128 => null,
4537 else => unreachable,
45324538 },
4533 else => return self.fail("TODO implement airRound for {}", .{
4534 ty.fmt(self.bin_file.options.module.?),
4535 }),
4536 };
4537 assert(dst_mcv.isRegister());
4539 .Vector => switch (ty.childType().zigTypeTag()) {
4540 .Float => switch (ty.childType().floatBits(self.target.*)) {
4541 32 => switch (ty.vectorLen()) {
4542 1 => if (self.hasFeature(.avx)) .vroundss else .roundss,
4543 2...4 => if (self.hasFeature(.avx)) .vroundps else .roundps,
4544 5...8 => if (self.hasFeature(.avx)) .vroundps else null,
4545 else => null,
4546 },
4547 64 => switch (ty.vectorLen()) {
4548 1 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
4549 2 => if (self.hasFeature(.avx)) .vroundpd else .roundpd,
4550 3...4 => if (self.hasFeature(.avx)) .vroundpd else null,
4551 else => null,
4552 },
4553 16, 80, 128 => null,
4554 else => unreachable,
4555 },
4556 else => null,
4557 },
4558 else => unreachable,
4559 })) |tag| tag else return self.fail("TODO implement genRound for {}", .{
4560 ty.fmt(self.bin_file.options.module.?),
4561 });
4562
45384563 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4539 const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
4540 if (src_mcv.isRegister())
4541 try self.asmRegisterRegisterImmediate(
4564 const dst_alias = registerAlias(dst_reg, abi_size);
4565 switch (mir_tag) {
4566 .vroundss, .vroundsd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(
45424567 mir_tag,
4543 dst_reg,
4544 registerAlias(src_mcv.getReg().?, abi_size),
4545 mode,
4546 )
4547 else
4548 try self.asmRegisterMemoryImmediate(
4568 dst_alias,
4569 dst_alias,
4570 src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
4571 Immediate.u(mode),
4572 ) else try self.asmRegisterRegisterRegisterImmediate(
45494573 mir_tag,
4550 dst_reg,
4551 src_mcv.mem(Memory.PtrSize.fromSize(@intCast(u32, ty.abiSize(self.target.*)))),
4552 mode,
4553 );
4554 return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none });
4574 dst_alias,
4575 dst_alias,
4576 registerAlias(if (src_mcv.isRegister())
4577 src_mcv.getReg().?
4578 else
4579 try self.copyToTmpRegister(ty, src_mcv), abi_size),
4580 Immediate.u(mode),
4581 ),
4582 else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate(
4583 mir_tag,
4584 dst_alias,
4585 src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
4586 Immediate.u(mode),
4587 ) else try self.asmRegisterRegisterImmediate(
4588 mir_tag,
4589 dst_alias,
4590 registerAlias(if (src_mcv.isRegister())
4591 src_mcv.getReg().?
4592 else
4593 try self.copyToTmpRegister(ty, src_mcv), abi_size),
4594 Immediate.u(mode),
4595 ),
4596 }
45554597}
45564598
45574599fn airSqrt(self: *Self, inst: Air.Inst.Index) !void {
......@@ -6188,18 +6230,18 @@ fn genBinOp(
61886230 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
61896231 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
61906232 });
6191 const dst_alias = registerAlias(dst_mcv.getReg().?, abi_size);
6233 const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
61926234 if (self.hasFeature(.avx)) {
61936235 const src1_alias =
6194 if (copied_to_dst) dst_alias else registerAlias(lhs_mcv.getReg().?, abi_size);
6236 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
61956237 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
61966238 mir_tag,
6197 dst_alias,
6239 dst_reg,
61986240 src1_alias,
61996241 src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
62006242 ) else try self.asmRegisterRegisterRegister(
62016243 mir_tag,
6202 dst_alias,
6244 dst_reg,
62036245 src1_alias,
62046246 registerAlias(if (src_mcv.isRegister())
62056247 src_mcv.getReg().?
......@@ -6210,11 +6252,11 @@ fn genBinOp(
62106252 assert(copied_to_dst);
62116253 if (src_mcv.isMemory()) try self.asmRegisterMemory(
62126254 mir_tag,
6213 dst_alias,
6255 dst_reg,
62146256 src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
62156257 ) else try self.asmRegisterRegister(
62166258 mir_tag,
6217 dst_alias,
6259 dst_reg,
62186260 registerAlias(if (src_mcv.isRegister())
62196261 src_mcv.getReg().?
62206262 else
......@@ -6223,60 +6265,16 @@ fn genBinOp(
62236265 }
62246266 switch (air_tag) {
62256267 .add, .sub, .mul, .div_float, .div_exact => {},
6226 .div_trunc, .div_floor => if (self.hasFeature(.sse4_1)) {
6227 const round_tag = if (@as(?Mir.Inst.Tag, switch (lhs_ty.zigTypeTag()) {
6228 .Float => switch (lhs_ty.floatBits(self.target.*)) {
6229 32 => if (self.hasFeature(.avx)) .vroundss else .roundss,
6230 64 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
6231 16, 80, 128 => null,
6232 else => unreachable,
6233 },
6234 .Vector => switch (lhs_ty.childType().zigTypeTag()) {
6235 .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
6236 32 => switch (lhs_ty.vectorLen()) {
6237 1 => if (self.hasFeature(.avx)) .vroundss else .roundss,
6238 2...4 => if (self.hasFeature(.avx)) .vroundps else .roundps,
6239 5...8 => if (self.hasFeature(.avx)) .vroundps else null,
6240 else => null,
6241 },
6242 64 => switch (lhs_ty.vectorLen()) {
6243 1 => if (self.hasFeature(.avx)) .vroundsd else .roundsd,
6244 2 => if (self.hasFeature(.avx)) .vroundpd else .roundpd,
6245 3...4 => if (self.hasFeature(.avx)) .vroundpd else null,
6246 else => null,
6247 },
6248 16, 80, 128 => null,
6249 else => unreachable,
6250 },
6251 else => null,
6252 },
6253 else => unreachable,
6254 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
6255 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
6256 });
6257 const round_mode = Immediate.u(switch (air_tag) {
6268 .div_trunc, .div_floor => try self.genRound(
6269 lhs_ty,
6270 dst_reg,
6271 .{ .register = dst_reg },
6272 switch (air_tag) {
62586273 .div_trunc => 0b1_0_11,
62596274 .div_floor => 0b1_0_01,
62606275 else => unreachable,
6261 });
6262 switch (round_tag) {
6263 .vroundss, .vroundsd => try self.asmRegisterRegisterRegisterImmediate(
6264 round_tag,
6265 dst_alias,
6266 dst_alias,
6267 dst_alias,
6268 round_mode,
6269 ),
6270 else => try self.asmRegisterRegisterImmediate(
6271 round_tag,
6272 dst_alias,
6273 dst_alias,
6274 round_mode,
6275 ),
6276 }
6277 } else return self.fail("TODO implement genBinOp for {s} {} without sse4_1", .{
6278 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
6279 }),
6276 },
6277 ),
62806278 .max, .min => {}, // TODO: unordered select
62816279 else => unreachable,
62826280 }
test/behavior/floatop.zig+6-3
......@@ -617,7 +617,8 @@ fn testFloor() !void {
617617
618618test "@floor with vectors" {
619619 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
620 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
620 if (builtin.zig_backend == .stage2_x86_64 and
621 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
621622 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
622623 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
623624
......@@ -707,7 +708,8 @@ fn testCeil() !void {
707708
708709test "@ceil with vectors" {
709710 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
710 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
711 if (builtin.zig_backend == .stage2_x86_64 and
712 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
711713 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
712714 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
713715
......@@ -797,7 +799,8 @@ fn testTrunc() !void {
797799
798800test "@trunc with vectors" {
799801 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
800 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
802 if (builtin.zig_backend == .stage2_x86_64 and
803 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
801804 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
802805 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
803806