diff --git a/src/codegen/x86_64/CodeGen.zig b/src/codegen/x86_64/CodeGen.zig index 8e1df5e433e2e3bb8797aa6f548d5b8706657df4..4f9b9ef0b0a2c6d3df51f642256e83b7cdda8a24 100644 --- a/src/codegen/x86_64/CodeGen.zig +++ b/src/codegen/x86_64/CodeGen.zig @@ -156319,10 +156319,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, .each = .{ .once = &.{ .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .add, .dst0x, .dst0x, .tmp0x, ._ }, .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, } }, }, .{ @@ -156614,9 +156614,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, .{ ._, .v_ss, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, .{ ._, .v_ss, .add, .dst0x, .tmp0x, .dst0x, ._ }, } }, }, .{ @@ -156646,6 +156646,33 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, .h_ps, .add, .dst0x, .src0x, ._, ._ }, .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movshdup, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, + } }, }, .{ .required_features = .{ .sse, null, null, null }, .dst_constraints = .{ .{ .float = .dword }, .any }, @@ -156668,10 +156695,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .dst_temps = .{ .{ .ref = .src0 }, .unused }, .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ss, .add, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, .{ ._, ._ss, .add, .dst0x, .tmp0x, ._, ._ }, } }, }, .{ @@ -156853,9 +156880,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, .{ ._, .v_sd, .add, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, .{ ._, .v_sd, .add, .dst0x, .tmp0x, .dst0x, ._ }, } }, }, .{ @@ -156917,6 +156944,44 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, } }, }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, + .{ ._, ._, .@"and", .memd(.tmp0b, 1), .ui(0b111_1_11_10), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .mem(.src0q), ._, ._, ._ }, + .{ .@"0:", .f_, .add, .memia(.src0q, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, .required_features = .{ .x87, null, null, null }, .dst_constraints = .{ .{ .float = .qword }, .any }, .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, @@ -160086,10 +160151,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, .each = .{ .once = &.{ .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ }, - .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, - .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b01) }, .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .dst0x, .dst0x, .ui(0b11_10) }, + .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0x, ._ }, .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ }, } }, }, .{ @@ -160300,11 +160365,38 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, .each = .{ .once = &.{ - .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, + .{ ._, .v_ps, .shuf, .tmp0x, .src0x, .src0x, .ui(0b01) }, .{ ._, .v_ss, .mul, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .shuf, .dst0x, .src0x, .src0x, .ui(0b01) }, + .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, .{ ._, .v_ss, .mul, .dst0x, .tmp0x, .dst0x, ._ }, } }, + }, .{ + .required_features = .{ .sse3, null, null, null }, + .dst_constraints = .{ .{ .float = .dword }, .any }, + .src_constraints = .{ .{ .exclusive_scalar_float = .{ .of = .xword, .is = .dword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_sse, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .movshdup, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .movhl, .tmp0x, .dst0x, ._, ._ }, + .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, + } }, }, .{ .required_features = .{ .sse, null, null, null }, .dst_constraints = .{ .{ .float = .dword }, .any }, @@ -160327,10 +160419,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .dst_temps = .{ .{ .ref = .src0 }, .unused }, .each = .{ .once = &.{ - .{ ._, ._ps, .xor, .tmp0x, .tmp0x, ._, ._ }, + .{ ._, ._ps, .mova, .tmp0x, .src0x, ._, ._ }, + .{ ._, ._ps, .shuf, .tmp0x, .src0x, .ui(0b01), ._ }, + .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, .{ ._, ._ps, .movhl, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ss, .mul, .tmp0x, .src0x, ._, ._ }, - .{ ._, ._ps, .shuf, .dst0x, .src0x, .ui(0b01), ._ }, .{ ._, ._ss, .mul, .dst0x, .tmp0x, ._, ._ }, } }, }, .{ @@ -160464,9 +160556,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }, .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused }, .each = .{ .once = &.{ - .{ ._, .v_f128, .extract, .tmp0x, .src0y, .ui(1), ._ }, + .{ ._, .v_ps, .movhl, .tmp0x, .src0x, .src0x, ._ }, .{ ._, .v_sd, .mul, .tmp0x, .src0x, .tmp0x, ._ }, - .{ ._, .v_ps, .movhl, .dst0x, .src0x, .src0x, ._ }, + .{ ._, .v_f128, .extract, .dst0x, .src0y, .ui(1), ._ }, .{ ._, .v_sd, .mul, .dst0x, .tmp0x, .dst0x, ._ }, } }, }, .{ @@ -160528,6 +160620,44 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, } }, }, .{ + .required_abi = .gnu, + .required_features = .{ .x87, null, null, null }, + .dst_constraints = .{ .{ .float = .qword }, .any }, + .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .to_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u16, .kind = .mem }, + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .f64, .kind = .{ .reg = .st7 } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, .fn_cw, .st, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .movzx, .tmp1d, .memd(.tmp0b, 1), ._, ._ }, + .{ ._, ._, .@"and", .memd(.tmp0b, 1), .ui(0b111_1_11_10), ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + .{ ._, ._, .mov, .memd(.tmp0b, 1), .tmp1b, ._, ._ }, + .{ ._, ._, .mov, .tmp1p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, + .{ ._, .f_, .ld, .mem(.src0q), ._, ._, ._ }, + .{ .@"0:", .f_, .mul, .memia(.src0q, .tmp1, .add_unaligned_size), ._, ._, ._ }, + .{ ._, ._, .add, .tmp1p, .si(8), ._, ._ }, + .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, + .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ }, + } }, + }, .{ + .required_abi = .msvc, .required_features = .{ .x87, null, null, null }, .dst_constraints = .{ .{ .float = .qword }, .any }, .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any, .any }, @@ -160552,13 +160682,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .each = .{ .once = &.{ .{ ._, ._, .mov, .tmp0p, .sia(8, .src0, .sub_unaligned_size), ._, ._ }, .{ ._, .f_, .ld, .mem(.src0q), ._, ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ }, - .{ .@"0:", .f_, .ld, .dst0q, ._, ._, ._ }, - .{ .@"1:", .f_, .mul, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, + .{ .@"0:", .f_, .mul, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ }, .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ }, - .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, + .{ ._, .f_p, .st, .dst0q, ._, ._, ._ }, } }, }, .{ .required_abi = .gnu, diff --git a/test/behavior/x86_64/math.zig b/test/behavior/x86_64/math.zig index afb1ebbd558cfc94ef47c454394ac681fc355cd2..0d5539244b4a0e26b58494733dbea7b3efd4c9a8 100644 --- a/test/behavior/x86_64/math.zig +++ b/test/behavior/x86_64/math.zig @@ -22,6 +22,10 @@ pub const Sse = if (builtin.cpu.has(.x86, .avx)) else @Vector(16, u8); +pub fn fracBits(comptime Float: type) comptime_int { + return std.math.floatFractionalBits(Float); +} + pub fn Scalar(comptime Type: type) type { return switch (@typeInfo(Type)) { else => Type, diff --git a/test/behavior/x86_64/unary.zig b/test/behavior/x86_64/unary.zig index 263505f090d09aa3b76eddb309b0d6bd531057dc..9b1edc4b0559c3527b483444d785e80cef8c8720 100644 --- a/test/behavior/x86_64/unary.zig +++ b/test/behavior/x86_64/unary.zig @@ -5,6 +5,7 @@ const Compare = math.Compare; const fmax = math.fmax; const fmin = math.fmin; const Gpr = math.Gpr; +const fracBits = math.fracBits; const inf = math.inf; const Log2IntCeil = math.Log2IntCeil; const math = @import("math.zig"); @@ -4487,34 +4488,65 @@ fn unary(comptime op: anytype, comptime opts: struct { fn testFloatVectorTypes() !void { try testArgs(@Vector(1, f16), undefined); try testArgs(@Vector(2, f16), undefined); + try testArgs(@Vector(3, f16), undefined); try testArgs(@Vector(4, f16), undefined); + try testArgs(@Vector(5, f16), undefined); + try testArgs(@Vector(7, f16), undefined); try testArgs(@Vector(8, f16), undefined); + try testArgs(@Vector(9, f16), undefined); + try testArgs(@Vector(15, f16), undefined); try testArgs(@Vector(16, f16), undefined); + try testArgs(@Vector(17, f16), undefined); + try testArgs(@Vector(31, f16), undefined); try testArgs(@Vector(32, f16), undefined); + try testArgs(@Vector(33, f16), undefined); + try testArgs(@Vector(63, f16), undefined); try testArgs(@Vector(64, f16), undefined); + try testArgs(@Vector(65, f16), undefined); try testArgs(@Vector(1, f32), undefined); try testArgs(@Vector(2, f32), undefined); + try testArgs(@Vector(3, f32), undefined); try testArgs(@Vector(4, f32), undefined); + try testArgs(@Vector(5, f32), undefined); + try testArgs(@Vector(7, f32), undefined); try testArgs(@Vector(8, f32), undefined); + try testArgs(@Vector(9, f32), undefined); + try testArgs(@Vector(15, f32), undefined); try testArgs(@Vector(16, f32), undefined); + try testArgs(@Vector(17, f32), undefined); + try testArgs(@Vector(31, f32), undefined); try testArgs(@Vector(32, f32), undefined); + try testArgs(@Vector(33, f32), undefined); try testArgs(@Vector(1, f64), undefined); try testArgs(@Vector(2, f64), undefined); + try testArgs(@Vector(3, f64), undefined); try testArgs(@Vector(4, f64), undefined); + try testArgs(@Vector(5, f64), undefined); + try testArgs(@Vector(7, f64), undefined); try testArgs(@Vector(8, f64), undefined); + try testArgs(@Vector(9, f64), undefined); + try testArgs(@Vector(15, f64), undefined); try testArgs(@Vector(16, f64), undefined); + try testArgs(@Vector(17, f64), undefined); try testArgs(@Vector(1, f80), undefined); try testArgs(@Vector(2, f80), undefined); + try testArgs(@Vector(3, f80), undefined); try testArgs(@Vector(4, f80), undefined); + try testArgs(@Vector(5, f80), undefined); + try testArgs(@Vector(7, f80), undefined); try testArgs(@Vector(8, f80), undefined); + try testArgs(@Vector(9, f80), undefined); try testArgs(@Vector(1, f128), undefined); try testArgs(@Vector(2, f128), undefined); + try testArgs(@Vector(3, f128), undefined); try testArgs(@Vector(4, f128), undefined); + try testArgs(@Vector(7, f128), undefined); try testArgs(@Vector(8, f128), undefined); + try testArgs(@Vector(9, f128), undefined); } fn testFloatVectors() !void { try testArgs(@Vector(1, f16), .{ @@ -5257,6 +5289,20 @@ test reduceMulOptimized { try test_reduce_mul_optimized.testFloatVectors(); } +inline fn reduceAddOrder(comptime Type: type, _: Type) @typeInfo(Type).vector.child { + const vector = @typeInfo(Type).vector; + const full_mant_bits = 1 + fracBits(vector.child); + comptime var rhs: Type = @splat(1); + rhs[0] = @exp2(full_mant_bits); + rhs[@as(usize, vector.len) -| 2] = -rhs[0]; + rhs[vector.len - 1] = @exp2(-full_mant_bits); + return @reduce(.Add, runtime(Type, rhs)); +} +test reduceAddOrder { + const test_reduce_add_order = unary(reduceAddOrder, .{}); + try test_reduce_add_order.testFloatVectorTypes(); +} + inline fn splat(comptime Type: type, rhs: Type) Type { return @splat(rhs[0]); }