authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-19 17:36:04+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-05-19 19:39:34+02:00
log5cbfd5819e423cdc6b092d1eb687189fb204b075
tree813b04e2f28f59af65d3d69709def69f6dd7ce08
parent283f40e4e9c44986353ba8abcc760684e9adf6cc

x64: check for floating-point intrinsics in codegen

4 files changed, 179 insertions(+), 134 deletions(-)

src/arch/x86_64/CodeGen.zig+164-119
......@@ -39,7 +39,7 @@ const RegisterLock = RegisterManager.RegisterLock;
3939const Register = bits.Register;
4040
4141const gp = abi.RegisterClass.gp;
42const avx = abi.RegisterClass.avx;
42const sse = abi.RegisterClass.sse;
4343
4444const InnerError = error{
4545 OutOfMemory,
......@@ -881,15 +881,18 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue {
881881 switch (elem_ty.zigTypeTag()) {
882882 .Vector => return self.fail("TODO allocRegOrMem for Vector type", .{}),
883883 .Float => {
884 // TODO check if AVX available
885 const ptr_bytes: u64 = 32;
886 if (abi_size <= ptr_bytes) {
887 if (self.register_manager.tryAllocReg(inst, .{
888 .selector_mask = avx,
889 })) |reg| {
890 return MCValue{ .register = registerAlias(reg, abi_size) };
884 if (self.intrinsicsAllowed(elem_ty)) {
885 const ptr_bytes: u64 = 32;
886 if (abi_size <= ptr_bytes) {
887 if (self.register_manager.tryAllocReg(inst, .{
888 .selector_mask = sse,
889 })) |reg| {
890 return MCValue{ .register = registerAlias(reg, abi_size) };
891 }
891892 }
892893 }
894
895 return self.fail("TODO allocRegOrMem for Float type without SSE/AVX support", .{});
893896 },
894897 else => {
895898 // Make sure the type can fit in a register before we try to allocate one.
......@@ -969,8 +972,11 @@ pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [cou
969972/// allocated. A second call to `copyToTmpRegister` may return the same register.
970973/// This can have a side effect of spilling instructions to the stack to free up a register.
971974fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
972 const mask = switch (ty.zigTypeTag()) {
973 .Float => avx,
975 const mask: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {
976 .Float => blk: {
977 if (self.intrinsicsAllowed(ty)) break :blk sse;
978 return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
979 },
974980 else => gp,
975981 };
976982 const reg: Register = try self.register_manager.allocReg(null, .{
......@@ -985,8 +991,11 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register {
985991/// This can have a side effect of spilling instructions to the stack to free up a register.
986992/// WARNING make sure that the allocated register matches the returned MCValue from an instruction!
987993fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue {
988 const mask = switch (ty.zigTypeTag()) {
989 .Float => avx,
994 const mask: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) {
995 .Float => blk: {
996 if (self.intrinsicsAllowed(ty)) break :blk sse;
997 return self.fail("TODO copy {} to register", .{ty.fmtDebug()});
998 },
990999 else => gp,
9911000 };
9921001 const reg: Register = try self.register_manager.allocReg(reg_owner, .{
......@@ -3469,27 +3478,32 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu
34693478 },
34703479 .register => |src_reg| switch (dst_ty.zigTypeTag()) {
34713480 .Float => {
3472 const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
3473 .f32 => switch (mir_tag) {
3474 .add => Mir.Inst.Tag.add_f32,
3475 .cmp => Mir.Inst.Tag.cmp_f32,
3476 else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}),
3477 },
3478 .f64 => switch (mir_tag) {
3479 .add => Mir.Inst.Tag.add_f64,
3480 .cmp => Mir.Inst.Tag.cmp_f64,
3481 else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}),
3482 },
3483 else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}),
3484 };
3485 _ = try self.addInst(.{
3486 .tag = actual_tag,
3487 .ops = Mir.Inst.Ops.encode(.{
3488 .reg1 = dst_reg.to128(),
3489 .reg2 = src_reg.to128(),
3490 }),
3491 .data = undefined,
3492 });
3481 if (self.intrinsicsAllowed(dst_ty)) {
3482 const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) {
3483 .f32 => switch (mir_tag) {
3484 .add => Mir.Inst.Tag.add_f32_avx,
3485 .cmp => Mir.Inst.Tag.cmp_f32_avx,
3486 else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}),
3487 },
3488 .f64 => switch (mir_tag) {
3489 .add => Mir.Inst.Tag.add_f64_avx,
3490 .cmp => Mir.Inst.Tag.cmp_f64_avx,
3491 else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}),
3492 },
3493 else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}),
3494 };
3495 _ = try self.addInst(.{
3496 .tag = actual_tag,
3497 .ops = Mir.Inst.Ops.encode(.{
3498 .reg1 = dst_reg.to128(),
3499 .reg2 = src_reg.to128(),
3500 }),
3501 .data = undefined,
3502 });
3503 return;
3504 }
3505
3506 return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{});
34933507 },
34943508 else => {
34953509 _ = try self.addInst(.{
......@@ -5326,24 +5340,29 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE
53265340 .register => |reg| {
53275341 switch (ty.zigTypeTag()) {
53285342 .Float => {
5329 const tag: Mir.Inst.Tag = switch (ty.tag()) {
5330 .f32 => .mov_f32,
5331 .f64 => .mov_f64,
5332 else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}),
5333 };
5334 _ = try self.addInst(.{
5335 .tag = tag,
5336 .ops = Mir.Inst.Ops.encode(.{
5337 .reg1 = switch (ty.tag()) {
5338 .f32 => .esp,
5339 .f64 => .rsp,
5340 else => unreachable,
5341 },
5342 .reg2 = reg.to128(),
5343 .flags = 0b01,
5344 }),
5345 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5346 });
5343 if (self.intrinsicsAllowed(ty)) {
5344 const tag: Mir.Inst.Tag = switch (ty.tag()) {
5345 .f32 => .mov_f32_avx,
5346 .f64 => .mov_f64_avx,
5347 else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}),
5348 };
5349 _ = try self.addInst(.{
5350 .tag = tag,
5351 .ops = Mir.Inst.Ops.encode(.{
5352 .reg1 = switch (ty.tag()) {
5353 .f32 => .esp,
5354 .f64 => .rsp,
5355 else => unreachable,
5356 },
5357 .reg2 = reg.to128(),
5358 .flags = 0b01,
5359 }),
5360 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5361 });
5362 return;
5363 }
5364
5365 return self.fail("TODO genSetStackArg for register with no intrinsics", .{});
53475366 },
53485367 else => {
53495368 _ = try self.addInst(.{
......@@ -5505,24 +5524,29 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl
55055524
55065525 switch (ty.zigTypeTag()) {
55075526 .Float => {
5508 const tag: Mir.Inst.Tag = switch (ty.tag()) {
5509 .f32 => .mov_f32,
5510 .f64 => .mov_f64,
5511 else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}),
5512 };
5513 _ = try self.addInst(.{
5514 .tag = tag,
5515 .ops = Mir.Inst.Ops.encode(.{
5516 .reg1 = switch (ty.tag()) {
5517 .f32 => base_reg.to32(),
5518 .f64 => base_reg.to64(),
5519 else => unreachable,
5520 },
5521 .reg2 = reg.to128(),
5522 .flags = 0b01,
5523 }),
5524 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5525 });
5527 if (self.intrinsicsAllowed(ty)) {
5528 const tag: Mir.Inst.Tag = switch (ty.tag()) {
5529 .f32 => .mov_f32_avx,
5530 .f64 => .mov_f64_avx,
5531 else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}),
5532 };
5533 _ = try self.addInst(.{
5534 .tag = tag,
5535 .ops = Mir.Inst.Ops.encode(.{
5536 .reg1 = switch (ty.tag()) {
5537 .f32 => base_reg.to32(),
5538 .f64 => base_reg.to64(),
5539 else => unreachable,
5540 },
5541 .reg2 = reg.to128(),
5542 .flags = 0b01,
5543 }),
5544 .data = .{ .imm = @bitCast(u32, -stack_offset) },
5545 });
5546 return;
5547 }
5548
5549 return self.fail("TODO genSetStack for register for type float with no intrinsics", .{});
55265550 },
55275551 else => {
55285552 if (!math.isPowerOfTwo(abi_size)) {
......@@ -6026,21 +6050,25 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
60266050 },
60276051 },
60286052 .Float => {
6029 const tag: Mir.Inst.Tag = switch (ty.tag()) {
6030 .f32 => .mov_f32,
6031 .f64 => .mov_f64,
6032 else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
6033 };
6034 _ = try self.addInst(.{
6035 .tag = tag,
6036 .ops = Mir.Inst.Ops.encode(.{
6037 .reg1 = reg.to128(),
6038 .reg2 = src_reg.to128(),
6039 .flags = 0b10,
6040 }),
6041 .data = undefined,
6042 });
6043 return;
6053 if (self.intrinsicsAllowed(ty)) {
6054 const tag: Mir.Inst.Tag = switch (ty.tag()) {
6055 .f32 => .mov_f32_avx,
6056 .f64 => .mov_f64_avx,
6057 else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}),
6058 };
6059 _ = try self.addInst(.{
6060 .tag = tag,
6061 .ops = Mir.Inst.Ops.encode(.{
6062 .reg1 = reg.to128(),
6063 .reg2 = src_reg.to128(),
6064 .flags = 0b10,
6065 }),
6066 .data = undefined,
6067 });
6068 return;
6069 }
6070
6071 return self.fail("TODO genSetReg from register for float with no intrinsics", .{});
60446072 },
60456073 else => {},
60466074 }
......@@ -6073,24 +6101,29 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
60736101 const base_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp });
60746102 try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv);
60756103
6076 const tag: Mir.Inst.Tag = switch (ty.tag()) {
6077 .f32 => .mov_f32,
6078 .f64 => .mov_f64,
6079 else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6080 };
6104 if (self.intrinsicsAllowed(ty)) {
6105 const tag: Mir.Inst.Tag = switch (ty.tag()) {
6106 .f32 => .mov_f32_avx,
6107 .f64 => .mov_f64_avx,
6108 else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}),
6109 };
60816110
6082 _ = try self.addInst(.{
6083 .tag = tag,
6084 .ops = Mir.Inst.Ops.encode(.{
6085 .reg1 = reg.to128(),
6086 .reg2 = switch (ty.tag()) {
6087 .f32 => base_reg.to32(),
6088 .f64 => base_reg.to64(),
6089 else => unreachable,
6090 },
6091 }),
6092 .data = .{ .imm = 0 },
6093 });
6111 _ = try self.addInst(.{
6112 .tag = tag,
6113 .ops = Mir.Inst.Ops.encode(.{
6114 .reg1 = reg.to128(),
6115 .reg2 = switch (ty.tag()) {
6116 .f32 => base_reg.to32(),
6117 .f64 => base_reg.to64(),
6118 else => unreachable,
6119 },
6120 }),
6121 .data = .{ .imm = 0 },
6122 });
6123 return;
6124 }
6125
6126 return self.fail("TODO genSetReg from memory for float with no intrinsics", .{});
60946127 },
60956128 else => {
60966129 if (x <= math.maxInt(i32)) {
......@@ -6183,24 +6216,27 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void
61836216 },
61846217 },
61856218 .Float => {
6186 const tag: Mir.Inst.Tag = switch (ty.tag()) {
6187 .f32 => .mov_f32,
6188 .f64 => .mov_f64,
6189 else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}),
6190 };
6191 _ = try self.addInst(.{
6192 .tag = tag,
6193 .ops = Mir.Inst.Ops.encode(.{
6194 .reg1 = reg.to128(),
6195 .reg2 = switch (ty.tag()) {
6196 .f32 => .ebp,
6197 .f64 => .rbp,
6198 else => unreachable,
6199 },
6200 }),
6201 .data = .{ .imm = @bitCast(u32, -off) },
6202 });
6203 return;
6219 if (self.intrinsicsAllowed(ty)) {
6220 const tag: Mir.Inst.Tag = switch (ty.tag()) {
6221 .f32 => .mov_f32_avx,
6222 .f64 => .mov_f64_avx,
6223 else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}),
6224 };
6225 _ = try self.addInst(.{
6226 .tag = tag,
6227 .ops = Mir.Inst.Ops.encode(.{
6228 .reg1 = reg.to128(),
6229 .reg2 = switch (ty.tag()) {
6230 .f32 => .ebp,
6231 .f64 => .rbp,
6232 else => unreachable,
6233 },
6234 }),
6235 .data = .{ .imm = @bitCast(u32, -off) },
6236 });
6237 return;
6238 }
6239 return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{});
62046240 },
62056241 else => {},
62066242 }
......@@ -6995,3 +7031,12 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void {
69957031 },
69967032 }
69977033}
7034
7035fn intrinsicsAllowed(self: *Self, ty: Type) bool {
7036 return switch (ty.tag()) {
7037 .f32,
7038 .f64,
7039 => Target.x86.featureSetHasAny(self.target.cpu.features, .{ .avx, .avx2 }),
7040 else => unreachable, // TODO finish this off
7041 };
7042}
src/arch/x86_64/Emit.zig+6-6
......@@ -183,14 +183,14 @@ pub fn lowerMir(emit: *Emit) InnerError!void {
183183 .nop => try emit.mirNop(),
184184
185185 // AVX instructions
186 .mov_f64 => try emit.mirMovFloatAvx(.vmovsd, inst),
187 .mov_f32 => try emit.mirMovFloatAvx(.vmovss, inst),
186 .mov_f64_avx => try emit.mirMovFloatAvx(.vmovsd, inst),
187 .mov_f32_avx => try emit.mirMovFloatAvx(.vmovss, inst),
188188
189 .add_f64 => try emit.mirAddFloatAvx(.vaddsd, inst),
190 .add_f32 => try emit.mirAddFloatAvx(.vaddss, inst),
189 .add_f64_avx => try emit.mirAddFloatAvx(.vaddsd, inst),
190 .add_f32_avx => try emit.mirAddFloatAvx(.vaddss, inst),
191191
192 .cmp_f64 => try emit.mirCmpFloatAvx(.vucomisd, inst),
193 .cmp_f32 => try emit.mirCmpFloatAvx(.vucomiss, inst),
192 .cmp_f64_avx => try emit.mirCmpFloatAvx(.vucomisd, inst),
193 .cmp_f32_avx => try emit.mirCmpFloatAvx(.vucomiss, inst),
194194
195195 // Pseudo-instructions
196196 .call_extern => try emit.mirCallExtern(inst),
src/arch/x86_64/Mir.zig+6-6
......@@ -350,18 +350,18 @@ pub const Inst = struct {
350350 /// 0b00 reg1, qword ptr [reg2 + imm32]
351351 /// 0b01 qword ptr [reg1 + imm32], reg2
352352 /// 0b10 reg1, reg2
353 mov_f64,
354 mov_f32,
353 mov_f64_avx,
354 mov_f32_avx,
355355
356356 /// ops flags: form:
357357 /// 0b00 reg1, reg1, reg2
358 add_f64,
359 add_f32,
358 add_f64_avx,
359 add_f32_avx,
360360
361361 /// ops flags: form:
362362 ///
363 cmp_f64,
364 cmp_f32,
363 cmp_f64_avx,
364 cmp_f32_avx,
365365
366366 /// Pseudo-instructions
367367 /// call extern function
src/arch/x86_64/abi.zig+3-3
......@@ -383,11 +383,11 @@ pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8
383383pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 };
384384pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx };
385385
386const avx_regs = [_]Register{
386const sse_avx_regs = [_]Register{
387387 .ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7,
388388 .ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15,
389389};
390const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ avx_regs;
390const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ sse_avx_regs;
391391pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers);
392392
393393// Register classes
......@@ -401,7 +401,7 @@ pub const RegisterClass = struct {
401401 }, true);
402402 break :blk set;
403403 };
404 pub const avx: RegisterBitSet = blk: {
404 pub const sse: RegisterBitSet = blk: {
405405 var set = RegisterBitSet.initEmpty();
406406 set.setRangeValue(.{
407407 .start = caller_preserved_regs.len + callee_preserved_regs.len,