authorgravatar for john.schmidt.h@gmail.comJohn Schmidt <john.schmidt.h@gmail.com> 2022-03-03 21:54:01+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-03-06 15:36:56-07:00
log6637335981f7179b449fced78cfd4052b1618051
treebb5bd8eaaf3b1534f1087630d5c30dbee42600ed
parentc7e4c711fc5795e66f974316611922a0b962eb99

stage2: implement `@mulAdd` for scalar floats


13 files changed, 242 insertions(+), 26 deletions(-)

src/Air.zig+11
......@@ -579,6 +579,10 @@ pub const Inst = struct {
579579 /// Uses the `prefetch` field.
580580 prefetch,
581581
582 /// Computes `(a * b) + c`, but only rounds once.
583 /// Uses the `ty_pl` field.
584 mul_add,
585
582586 /// Implements @fieldParentPtr builtin.
583587 /// Uses the `ty_pl` field.
584588 field_parent_ptr,
......@@ -724,6 +728,12 @@ pub const Bin = struct {
724728 rhs: Inst.Ref,
725729};
726730
731pub const MulAdd = struct {
732 mulend1: Inst.Ref,
733 mulend2: Inst.Ref,
734 addend: Inst.Ref,
735};
736
727737pub const FieldParentPtr = struct {
728738 field_ptr: Inst.Ref,
729739 field_index: u32,
......@@ -889,6 +899,7 @@ pub fn typeOfIndex(air: Air, inst: Air.Inst.Index) Type {
889899 .aggregate_init,
890900 .union_init,
891901 .field_parent_ptr,
902 .mul_add,
892903 => return air.getRefType(datas[inst].ty_pl.ty),
893904
894905 .not,
src/Liveness.zig+4
......@@ -464,6 +464,10 @@ fn analyzeInst(
464464 const extra = a.air.extraData(Air.Cmpxchg, inst_datas[inst].ty_pl.payload).data;
465465 return trackOperands(a, new_set, inst, main_tomb, .{ extra.ptr, extra.expected_value, extra.new_value });
466466 },
467 .mul_add => {
468 const extra = a.air.extraData(Air.MulAdd, inst_datas[inst].ty_pl.payload).data;
469 return trackOperands(a, new_set, inst, main_tomb, .{ extra.mulend1, extra.mulend2, extra.addend });
470 },
467471 .atomic_load => {
468472 const ptr = inst_datas[inst].atomic_load.ptr;
469473 return trackOperands(a, new_set, inst, main_tomb, .{ ptr, .none, .none });
src/Sema.zig+77-1
......@@ -13518,8 +13518,84 @@ fn zirAtomicStore(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
1351813518
1351913519fn zirMulAdd(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
1352013520 const inst_data = sema.code.instructions.items(.data)[inst].pl_node;
13521 const extra = sema.code.extraData(Zir.Inst.MulAdd, inst_data.payload_index).data;
1352113522 const src = inst_data.src();
13522 return sema.fail(block, src, "TODO: Sema.zirMulAdd", .{});
13523
13524 const mulend1_src: LazySrcLoc = .{ .node_offset_builtin_call_arg1 = inst_data.src_node };
13525 const mulend2_src: LazySrcLoc = .{ .node_offset_builtin_call_arg2 = inst_data.src_node };
13526 const addend_src: LazySrcLoc = .{ .node_offset_builtin_call_arg3 = inst_data.src_node };
13527
13528 const mulend1 = sema.resolveInst(extra.mulend1);
13529 const mulend2 = sema.resolveInst(extra.mulend2);
13530 const addend = sema.resolveInst(extra.addend);
13531 // All args have the same type
13532 const ty = sema.typeOf(mulend1);
13533 switch (ty.zigTypeTag()) {
13534 .ComptimeFloat, .Float => {},
13535 .Vector => {
13536 const scalar_ty = ty.scalarType();
13537 switch (scalar_ty.zigTypeTag()) {
13538 .ComptimeFloat, .Float => {},
13539 else => return sema.fail(block, src, "expected vector of floats or float type, found '{}'", .{scalar_ty}),
13540 }
13541 },
13542 else => return sema.fail(block, src, "expected vector of floats or float type, found '{}'", .{ty}),
13543 }
13544
13545 const target = sema.mod.getTarget();
13546 switch (ty.zigTypeTag()) {
13547 .ComptimeFloat, .Float => {
13548 const maybe_mulend1 = try sema.resolveMaybeUndefVal(block, mulend1_src, mulend1);
13549 const maybe_mulend2 = try sema.resolveMaybeUndefVal(block, mulend2_src, mulend2);
13550 const maybe_addend = try sema.resolveMaybeUndefVal(block, addend_src, addend);
13551
13552 if (maybe_mulend1) |mulend1_val| {
13553 if (mulend1_val.isUndef())
13554 return sema.addConstUndef(ty);
13555 }
13556
13557 if (maybe_mulend2) |mulend2_val| {
13558 if (mulend2_val.isUndef())
13559 return sema.addConstUndef(ty);
13560 }
13561
13562 if (maybe_addend) |addend_val| {
13563 if (addend_val.isUndef())
13564 return sema.addConstUndef(ty);
13565 }
13566
13567 if (maybe_mulend1) |mulend1_val| {
13568 if (maybe_mulend2) |mulend2_val| {
13569 if (maybe_addend) |addend_val| {
13570 const result_val = try Value.mulAdd(
13571 ty,
13572 mulend1_val,
13573 mulend2_val,
13574 addend_val,
13575 sema.arena,
13576 target,
13577 );
13578 return sema.addConstant(ty, result_val);
13579 }
13580 }
13581 }
13582
13583 try sema.requireRuntimeBlock(block, src);
13584 return block.addInst(.{
13585 .tag = .mul_add,
13586 .data = .{ .ty_pl = .{
13587 .ty = try sema.addType(ty),
13588 .payload = try sema.addExtra(Air.MulAdd{
13589 .mulend1 = mulend1,
13590 .mulend2 = mulend2,
13591 .addend = addend,
13592 }),
13593 } },
13594 });
13595 },
13596 .Vector => return sema.fail(block, src, "TODO: implement @mulAdd for vectors", .{}),
13597 else => unreachable,
13598 }
1352313599}
1352413600
1352513601fn zirBuiltinCall(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
src/arch/aarch64/CodeGen.zig+6
......@@ -632,6 +632,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
632632 .aggregate_init => try self.airAggregateInit(inst),
633633 .union_init => try self.airUnionInit(inst),
634634 .prefetch => try self.airPrefetch(inst),
635 .mul_add => try self.airMulAdd(inst),
635636
636637 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
637638 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
......@@ -3652,6 +3653,11 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
36523653 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
36533654}
36543655
3656fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
3657 _ = inst;
3658 return self.fail("TODO implement airMulAdd for aarch64", .{});
3659}
3660
36553661fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
36563662 // First section of indexes correspond to a set number of constant values.
36573663 const ref_int = @enumToInt(inst);
src/arch/arm/CodeGen.zig+6
......@@ -628,6 +628,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
628628 .aggregate_init => try self.airAggregateInit(inst),
629629 .union_init => try self.airUnionInit(inst),
630630 .prefetch => try self.airPrefetch(inst),
631 .mul_add => try self.airMulAdd(inst),
631632
632633 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
633634 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
......@@ -4086,6 +4087,11 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
40864087 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
40874088}
40884089
4090fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
4091 _ = inst;
4092 return self.fail("TODO implement airMulAdd for arm", .{});
4093}
4094
40894095fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
40904096 // First section of indexes correspond to a set number of constant values.
40914097 const ref_int = @enumToInt(inst);
src/arch/riscv64/CodeGen.zig+6
......@@ -600,6 +600,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
600600 .aggregate_init => try self.airAggregateInit(inst),
601601 .union_init => try self.airUnionInit(inst),
602602 .prefetch => try self.airPrefetch(inst),
603 .mul_add => try self.airMulAdd(inst),
603604
604605 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
605606 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
......@@ -2203,6 +2204,11 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
22032204 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
22042205}
22052206
2207fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
2208 _ = inst;
2209 return self.fail("TODO implement airMulAdd for riscv64", .{});
2210}
2211
22062212fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
22072213 // First section of indexes correspond to a set number of constant values.
22082214 const ref_int = @enumToInt(inst);
src/arch/wasm/CodeGen.zig+1
......@@ -1333,6 +1333,7 @@ fn genInst(self: *Self, inst: Air.Inst.Index) !WValue {
13331333 .error_name,
13341334 .errunion_payload_ptr_set,
13351335 .field_parent_ptr,
1336 .mul_add,
13361337
13371338 // For these 4, probably best to wait until https://github.com/ziglang/zig/issues/10248
13381339 // is implemented in the frontend before implementing them here in the wasm backend.
src/arch/x86_64/CodeGen.zig+6
......@@ -717,6 +717,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
717717 .aggregate_init => try self.airAggregateInit(inst),
718718 .union_init => try self.airUnionInit(inst),
719719 .prefetch => try self.airPrefetch(inst),
720 .mul_add => try self.airMulAdd(inst),
720721
721722 .atomic_store_unordered => try self.airAtomicStore(inst, .Unordered),
722723 .atomic_store_monotonic => try self.airAtomicStore(inst, .Monotonic),
......@@ -5559,6 +5560,11 @@ fn airPrefetch(self: *Self, inst: Air.Inst.Index) !void {
55595560 return self.finishAir(inst, MCValue.dead, .{ prefetch.ptr, .none, .none });
55605561}
55615562
5563fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void {
5564 _ = inst;
5565 return self.fail("TODO implement airMulAdd for x86_64", .{});
5566}
5567
55625568fn resolveInst(self: *Self, inst: Air.Inst.Ref) InnerError!MCValue {
55635569 // First section of indexes correspond to a set number of constant values.
55645570 const ref_int = @enumToInt(inst);
src/codegen/c.zig+2
......@@ -1635,6 +1635,8 @@ fn genBody(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, OutO
16351635 .trunc_float,
16361636 => |tag| return f.fail("TODO: C backend: implement unary op for tag '{s}'", .{@tagName(tag)}),
16371637
1638 .mul_add => return f.fail("TODO: C backend: implement @mulAdd", .{}),
1639
16381640 .add_with_overflow => try airAddWithOverflow(f, inst),
16391641 .sub_with_overflow => try airSubWithOverflow(f, inst),
16401642 .mul_with_overflow => try airMulWithOverflow(f, inst),
src/codegen/llvm.zig+41
......@@ -2194,6 +2194,7 @@ pub const FuncGen = struct {
21942194 .sub_with_overflow => try self.airOverflow(inst, "llvm.ssub.with.overflow", "llvm.usub.with.overflow"),
21952195 .mul_with_overflow => try self.airOverflow(inst, "llvm.smul.with.overflow", "llvm.umul.with.overflow"),
21962196 .shl_with_overflow => try self.airShlWithOverflow(inst),
2197 .mul_add => try self.airMulAdd(inst),
21972198
21982199 .bit_and, .bool_and => try self.airAnd(inst),
21992200 .bit_or, .bool_or => try self.airOr(inst),
......@@ -3842,6 +3843,46 @@ pub const FuncGen = struct {
38423843 return overflow_bit;
38433844 }
38443845
3846 fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
3847 if (self.liveness.isUnused(inst))
3848 return null;
3849
3850 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
3851 const extra = self.air.extraData(Air.MulAdd, ty_pl.payload).data;
3852
3853 const mulend1 = try self.resolveInst(extra.mulend1);
3854 const mulend2 = try self.resolveInst(extra.mulend2);
3855 const addend = try self.resolveInst(extra.addend);
3856
3857 const ty = self.air.typeOfIndex(inst);
3858 const llvm_ty = try self.dg.llvmType(ty);
3859 const target = self.dg.module.getTarget();
3860
3861 const fn_val = switch (ty.floatBits(target)) {
3862 16, 32, 64 => blk: {
3863 break :blk self.getIntrinsic("llvm.fma", &.{llvm_ty});
3864 },
3865 // TODO: using `llvm.fma` for f80 does not seem to work for all targets, needs further investigation.
3866 80 => return self.dg.todo("Implement mulAdd for f80", .{}),
3867 128 => blk: {
3868 // LLVM incorrectly lowers the fma builtin for f128 to fmal, which is for
3869 // `long double`. On some targets this will be correct; on others it will be incorrect.
3870 if (target.longDoubleIsF128()) {
3871 break :blk self.getIntrinsic("llvm.fma", &.{llvm_ty});
3872 } else {
3873 break :blk self.dg.object.llvm_module.getNamedFunction("fmaq") orelse fn_blk: {
3874 const param_types = [_]*const llvm.Type{ llvm_ty, llvm_ty, llvm_ty };
3875 const fn_type = llvm.functionType(llvm_ty, &param_types, param_types.len, .False);
3876 break :fn_blk self.dg.object.llvm_module.addFunction("fmaq", fn_type);
3877 };
3878 }
3879 },
3880 else => unreachable,
3881 };
3882 const params = [_]*const llvm.Value{ mulend1, mulend2, addend };
3883 return self.builder.buildCall(fn_val, &params, params.len, .C, .Auto, "");
3884 }
3885
38453886 fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
38463887 if (self.liveness.isUnused(inst))
38473888 return null;
src/print_air.zig+12
......@@ -252,6 +252,7 @@ const Writer = struct {
252252 .field_parent_ptr => try w.writeFieldParentPtr(s, inst),
253253 .wasm_memory_size => try w.writeWasmMemorySize(s, inst),
254254 .wasm_memory_grow => try w.writeWasmMemoryGrow(s, inst),
255 .mul_add => try w.writeMulAdd(s, inst),
255256
256257 .add_with_overflow,
257258 .sub_with_overflow,
......@@ -358,6 +359,17 @@ const Writer = struct {
358359 });
359360 }
360361
362 fn writeMulAdd(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
363 const ty_pl = w.air.instructions.items(.data)[inst].ty_pl;
364 const extra = w.air.extraData(Air.MulAdd, ty_pl.payload).data;
365
366 try w.writeOperand(s, inst, 0, extra.mulend1);
367 try s.writeAll(", ");
368 try w.writeOperand(s, inst, 1, extra.mulend2);
369 try s.writeAll(", ");
370 try w.writeOperand(s, inst, 2, extra.addend);
371 }
372
361373 fn writeFence(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void {
362374 const atomic_order = w.air.instructions.items(.data)[inst].fence;
363375
src/value.zig+40-4
......@@ -2931,7 +2931,7 @@ pub const Value = extern union {
29312931 return fromBigInt(arena, result_bigint.toConst());
29322932 }
29332933
2934 /// operands must be integers; handles undefined.
2934 /// operands must be integers; handles undefined.
29352935 pub fn bitwiseAnd(lhs: Value, rhs: Value, arena: Allocator) !Value {
29362936 if (lhs.isUndef() or rhs.isUndef()) return Value.initTag(.undef);
29372937
......@@ -2951,7 +2951,7 @@ pub const Value = extern union {
29512951 return fromBigInt(arena, result_bigint.toConst());
29522952 }
29532953
2954 /// operands must be integers; handles undefined.
2954 /// operands must be integers; handles undefined.
29552955 pub fn bitwiseNand(lhs: Value, rhs: Value, ty: Type, arena: Allocator, target: Target) !Value {
29562956 if (lhs.isUndef() or rhs.isUndef()) return Value.initTag(.undef);
29572957
......@@ -2965,7 +2965,7 @@ pub const Value = extern union {
29652965 return bitwiseXor(anded, all_ones, arena);
29662966 }
29672967
2968 /// operands must be integers; handles undefined.
2968 /// operands must be integers; handles undefined.
29692969 pub fn bitwiseOr(lhs: Value, rhs: Value, arena: Allocator) !Value {
29702970 if (lhs.isUndef() or rhs.isUndef()) return Value.initTag(.undef);
29712971
......@@ -2984,7 +2984,7 @@ pub const Value = extern union {
29842984 return fromBigInt(arena, result_bigint.toConst());
29852985 }
29862986
2987 /// operands must be integers; handles undefined.
2987 /// operands must be integers; handles undefined.
29882988 pub fn bitwiseXor(lhs: Value, rhs: Value, arena: Allocator) !Value {
29892989 if (lhs.isUndef() or rhs.isUndef()) return Value.initTag(.undef);
29902990
......@@ -4020,6 +4020,42 @@ pub const Value = extern union {
40204020 }
40214021 }
40224022
4023 pub fn mulAdd(float_type: Type, mulend1: Value, mulend2: Value, addend: Value, arena: Allocator, target: Target) Allocator.Error!Value {
4024 switch (float_type.floatBits(target)) {
4025 16 => {
4026 if (true) {
4027 // TODO: missing f16 implementation of FMA in `std.math.fma` or compiler-rt
4028 @panic("TODO implement mulAdd for f16");
4029 }
4030 },
4031 32 => {
4032 const m1 = mulend1.toFloat(f32);
4033 const m2 = mulend2.toFloat(f32);
4034 const a = addend.toFloat(f32);
4035 return Value.Tag.float_32.create(arena, std.math.fma(f32, m1, m2, a));
4036 },
4037 64 => {
4038 const m1 = mulend1.toFloat(f64);
4039 const m2 = mulend2.toFloat(f64);
4040 const a = addend.toFloat(f64);
4041 return Value.Tag.float_64.create(arena, std.math.fma(f64, m1, m2, a));
4042 },
4043 80 => {
4044 if (true) {
4045 // TODO: missing f80 implementation of FMA in `std.math.fma` or compiler-rt
4046 @panic("TODO implement mulAdd for f80");
4047 }
4048 },
4049 128 => {
4050 const m1 = mulend1.toFloat(f128);
4051 const m2 = mulend2.toFloat(f128);
4052 const a = addend.toFloat(f128);
4053 return Value.Tag.float_128.create(arena, std.math.fma(f128, m1, m2, a));
4054 },
4055 else => unreachable,
4056 }
4057 }
4058
40234059 /// This type is not copyable since it may contain pointers to its inner data.
40244060 pub const Payload = struct {
40254061 tag: Tag,
test/behavior/muladd.zig+30-21
......@@ -2,29 +2,33 @@ const builtin = @import("builtin");
22const expect = @import("std").testing.expect;
33
44test "@mulAdd" {
5 if (builtin.zig_backend != .stage1) return error.SkipZigTest; // TODO
5 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
6 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
7 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
8 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
9 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
610
711 comptime try testMulAdd();
812 try testMulAdd();
913}
1014
1115fn testMulAdd() !void {
12 {
13 var a: f16 = 5.5;
14 var b: f16 = 2.5;
15 var c: f16 = 6.25;
16 if (builtin.zig_backend == .stage1) {
17 const a: f16 = 5.5;
18 const b: f16 = 2.5;
19 const c: f16 = 6.25;
1620 try expect(@mulAdd(f16, a, b, c) == 20);
1721 }
1822 {
19 var a: f32 = 5.5;
20 var b: f32 = 2.5;
21 var c: f32 = 6.25;
23 const a: f32 = 5.5;
24 const b: f32 = 2.5;
25 const c: f32 = 6.25;
2226 try expect(@mulAdd(f32, a, b, c) == 20);
2327 }
2428 {
25 var a: f64 = 5.5;
26 var b: f64 = 2.5;
27 var c: f64 = 6.25;
29 const a: f64 = 5.5;
30 const b: f64 = 2.5;
31 const c: f64 = 6.25;
2832 try expect(@mulAdd(f64, a, b, c) == 20);
2933 }
3034}
......@@ -35,7 +39,9 @@ test "@mulAdd f80" {
3539 return error.SkipZigTest;
3640 }
3741
38 comptime try testMulAdd80();
42 // TODO: missing f80 implementation of FMA in `std.math.fma` or compiler-rt
43 // comptime try testMulAdd80();
44
3945 try testMulAdd80();
4046}
4147
......@@ -43,24 +49,27 @@ fn testMulAdd80() !void {
4349 var a: f16 = 5.5;
4450 var b: f80 = 2.5;
4551 var c: f80 = 6.25;
46 try expect(@mulAdd(f80, a, b, c) == 20);
52 try expect(@mulAdd(f80, a, b, c) == 20.0);
4753}
4854
4955test "@mulAdd f128" {
50 if (builtin.zig_backend != .stage1) return error.SkipZigTest; // TODO
51
56 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
57 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
58 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
59 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
60 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
5261 if (builtin.os.tag == .macos and builtin.cpu.arch == .aarch64) {
5362 // https://github.com/ziglang/zig/issues/9900
5463 return error.SkipZigTest;
5564 }
5665
57 comptime try testMullAdd128();
58 try testMullAdd128();
66 comptime try testMulAdd128();
67 try testMulAdd128();
5968}
6069
61fn testMullAdd128() !void {
62 var a: f16 = 5.5;
63 var b: f128 = 2.5;
64 var c: f128 = 6.25;
70fn testMulAdd128() !void {
71 const a: f16 = 5.5;
72 const b: f128 = 2.5;
73 const c: f128 = 6.25;
6574 try expect(@mulAdd(f128, a, b, c) == 20);
6675}