authorgravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-04-26 15:19:55-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-04-27 12:22:20-07:00
log7d6a6ce87fde465ffc3bd6b0a8bb6e039c67268f
tree0f082b98f1693e7bbff34b27a3a256041f8edc7c
parent0e8242b905514f838f36757f2f890241e7448554

stage2: Manually lower softfloat ops when needed

Updates stage2 to manually lower softfloat operations for all unary floating point operations and arithmetic. Softfloat support still needs to be added for conversion operators (float<->float and int<->float)

4 files changed, 280 insertions(+), 136 deletions(-)

src/codegen/llvm.zig+264-136
...@@ -3518,19 +3518,19 @@ pub const FuncGen = struct {...@@ -3518,19 +3518,19 @@ pub const FuncGen = struct {
3518 .shr => try self.airShr(inst, false),3518 .shr => try self.airShr(inst, false),
3519 .shr_exact => try self.airShr(inst, true),3519 .shr_exact => try self.airShr(inst, true),
35203520
3521 .sqrt => try self.airUnaryOp(inst, "sqrt"),3521 .sqrt => try self.airUnaryOp(inst, .sqrt),
3522 .sin => try self.airUnaryOp(inst, "sin"),3522 .sin => try self.airUnaryOp(inst, .sin),
3523 .cos => try self.airUnaryOp(inst, "cos"),3523 .cos => try self.airUnaryOp(inst, .cos),
3524 .exp => try self.airUnaryOp(inst, "exp"),3524 .exp => try self.airUnaryOp(inst, .exp),
3525 .exp2 => try self.airUnaryOp(inst, "exp2"),3525 .exp2 => try self.airUnaryOp(inst, .exp2),
3526 .log => try self.airUnaryOp(inst, "log"),3526 .log => try self.airUnaryOp(inst, .log),
3527 .log2 => try self.airUnaryOp(inst, "log2"),3527 .log2 => try self.airUnaryOp(inst, .log2),
3528 .log10 => try self.airUnaryOp(inst, "log10"),3528 .log10 => try self.airUnaryOp(inst, .log10),
3529 .fabs => try self.airUnaryOp(inst, "fabs"),3529 .fabs => try self.airUnaryOp(inst, .fabs),
3530 .floor => try self.airUnaryOp(inst, "floor"),3530 .floor => try self.airUnaryOp(inst, .floor),
3531 .ceil => try self.airUnaryOp(inst, "ceil"),3531 .ceil => try self.airUnaryOp(inst, .ceil),
3532 .round => try self.airUnaryOp(inst, "round"),3532 .round => try self.airUnaryOp(inst, .round),
3533 .trunc_float => try self.airUnaryOp(inst, "trunc"),3533 .trunc_float => try self.airUnaryOp(inst, .trunc),
35343534
3535 .cmp_eq => try self.airCmp(inst, .eq),3535 .cmp_eq => try self.airCmp(inst, .eq),
3536 .cmp_gt => try self.airCmp(inst, .gt),3536 .cmp_gt => try self.airCmp(inst, .gt),
...@@ -3905,7 +3905,7 @@ pub const FuncGen = struct {...@@ -3905,7 +3905,7 @@ pub const FuncGen = struct {
3905 rhs: *const llvm.Value,3905 rhs: *const llvm.Value,
3906 operand_ty: Type,3906 operand_ty: Type,
3907 op: math.CompareOperator,3907 op: math.CompareOperator,
3908 ) *const llvm.Value {3908 ) Allocator.Error!*const llvm.Value {
3909 var int_buffer: Type.Payload.Bits = undefined;3909 var int_buffer: Type.Payload.Bits = undefined;
3910 var opt_buffer: Type.Payload.ElemType = undefined;3910 var opt_buffer: Type.Payload.ElemType = undefined;
39113911
...@@ -3947,7 +3947,7 @@ pub const FuncGen = struct {...@@ -3947,7 +3947,7 @@ pub const FuncGen = struct {
3947 self.builder.positionBuilderAtEnd(both_pl_block);3947 self.builder.positionBuilderAtEnd(both_pl_block);
3948 const lhs_payload = self.optPayloadHandle(lhs, is_by_ref);3948 const lhs_payload = self.optPayloadHandle(lhs, is_by_ref);
3949 const rhs_payload = self.optPayloadHandle(rhs, is_by_ref);3949 const rhs_payload = self.optPayloadHandle(rhs, is_by_ref);
3950 const payload_cmp = self.cmp(lhs_payload, rhs_payload, payload_ty, op);3950 const payload_cmp = try self.cmp(lhs_payload, rhs_payload, payload_ty, op);
3951 _ = self.builder.buildBr(end_block);3951 _ = self.builder.buildBr(end_block);
3952 const both_pl_block_end = self.builder.getInsertBlock();3952 const both_pl_block_end = self.builder.getInsertBlock();
39533953
...@@ -3983,17 +3983,7 @@ pub const FuncGen = struct {...@@ -3983,17 +3983,7 @@ pub const FuncGen = struct {
3983 );3983 );
3984 return phi_node;3984 return phi_node;
3985 },3985 },
3986 .Float => {3986 .Float => return self.buildFloatCmp(op, operand_ty, &.{ lhs, rhs }),
3987 const operation: llvm.RealPredicate = switch (op) {
3988 .eq => .OEQ,
3989 .neq => .UNE,
3990 .lt => .OLT,
3991 .lte => .OLE,
3992 .gt => .OGT,
3993 .gte => .OGE,
3994 };
3995 return self.builder.buildFCmp(operation, lhs, rhs, "");
3996 },
3997 else => unreachable,3987 else => unreachable,
3998 };3988 };
3999 const is_signed = int_ty.isSignedInt();3989 const is_signed = int_ty.isSignedInt();
...@@ -5221,7 +5211,7 @@ pub const FuncGen = struct {...@@ -5221,7 +5211,7 @@ pub const FuncGen = struct {
5221 const inst_ty = self.air.typeOfIndex(inst);5211 const inst_ty = self.air.typeOfIndex(inst);
5222 const scalar_ty = inst_ty.scalarType();5212 const scalar_ty = inst_ty.scalarType();
52235213
5224 if (scalar_ty.isAnyFloat()) return self.builder.buildFAdd(lhs, rhs, "");5214 if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, inst_ty, &.{ lhs, rhs });
5225 if (scalar_ty.isSignedInt()) return self.builder.buildNSWAdd(lhs, rhs, "");5215 if (scalar_ty.isSignedInt()) return self.builder.buildNSWAdd(lhs, rhs, "");
5226 return self.builder.buildNUWAdd(lhs, rhs, "");5216 return self.builder.buildNUWAdd(lhs, rhs, "");
5227 }5217 }
...@@ -5260,7 +5250,7 @@ pub const FuncGen = struct {...@@ -5260,7 +5250,7 @@ pub const FuncGen = struct {
5260 const inst_ty = self.air.typeOfIndex(inst);5250 const inst_ty = self.air.typeOfIndex(inst);
5261 const scalar_ty = inst_ty.scalarType();5251 const scalar_ty = inst_ty.scalarType();
52625252
5263 if (scalar_ty.isAnyFloat()) return self.builder.buildFSub(lhs, rhs, "");5253 if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, inst_ty, &.{ lhs, rhs });
5264 if (scalar_ty.isSignedInt()) return self.builder.buildNSWSub(lhs, rhs, "");5254 if (scalar_ty.isSignedInt()) return self.builder.buildNSWSub(lhs, rhs, "");
5265 return self.builder.buildNUWSub(lhs, rhs, "");5255 return self.builder.buildNUWSub(lhs, rhs, "");
5266 }5256 }
...@@ -5298,7 +5288,7 @@ pub const FuncGen = struct {...@@ -5298,7 +5288,7 @@ pub const FuncGen = struct {
5298 const inst_ty = self.air.typeOfIndex(inst);5288 const inst_ty = self.air.typeOfIndex(inst);
5299 const scalar_ty = inst_ty.scalarType();5289 const scalar_ty = inst_ty.scalarType();
53005290
5301 if (scalar_ty.isAnyFloat()) return self.builder.buildFMul(lhs, rhs, "");5291 if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, inst_ty, &.{ lhs, rhs });
5302 if (scalar_ty.isSignedInt()) return self.builder.buildNSWMul(lhs, rhs, "");5292 if (scalar_ty.isSignedInt()) return self.builder.buildNSWMul(lhs, rhs, "");
5303 return self.builder.buildNUWMul(lhs, rhs, "");5293 return self.builder.buildNUWMul(lhs, rhs, "");
5304 }5294 }
...@@ -5333,8 +5323,9 @@ pub const FuncGen = struct {...@@ -5333,8 +5323,9 @@ pub const FuncGen = struct {
5333 const bin_op = self.air.instructions.items(.data)[inst].bin_op;5323 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
5334 const lhs = try self.resolveInst(bin_op.lhs);5324 const lhs = try self.resolveInst(bin_op.lhs);
5335 const rhs = try self.resolveInst(bin_op.rhs);5325 const rhs = try self.resolveInst(bin_op.rhs);
5326 const inst_ty = self.air.typeOfIndex(inst);
53365327
5337 return self.builder.buildFDiv(lhs, rhs, "");5328 return self.buildFloatOp(.div, inst_ty, &.{ lhs, rhs });
5338 }5329 }
53395330
5340 fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {5331 fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
...@@ -5347,8 +5338,8 @@ pub const FuncGen = struct {...@@ -5347,8 +5338,8 @@ pub const FuncGen = struct {
5347 const scalar_ty = inst_ty.scalarType();5338 const scalar_ty = inst_ty.scalarType();
53485339
5349 if (scalar_ty.isRuntimeFloat()) {5340 if (scalar_ty.isRuntimeFloat()) {
5350 const result = self.builder.buildFDiv(lhs, rhs, "");5341 const result = try self.buildFloatOp(.div, inst_ty, &.{ lhs, rhs });
5351 return self.callTrunc(result, inst_ty);5342 return self.buildFloatOp(.trunc, inst_ty, &.{result});
5352 }5343 }
5353 if (scalar_ty.isSignedInt()) return self.builder.buildSDiv(lhs, rhs, "");5344 if (scalar_ty.isSignedInt()) return self.builder.buildSDiv(lhs, rhs, "");
5354 return self.builder.buildUDiv(lhs, rhs, "");5345 return self.builder.buildUDiv(lhs, rhs, "");
...@@ -5364,8 +5355,8 @@ pub const FuncGen = struct {...@@ -5364,8 +5355,8 @@ pub const FuncGen = struct {
5364 const scalar_ty = inst_ty.scalarType();5355 const scalar_ty = inst_ty.scalarType();
53655356
5366 if (scalar_ty.isRuntimeFloat()) {5357 if (scalar_ty.isRuntimeFloat()) {
5367 const result = self.builder.buildFDiv(lhs, rhs, "");5358 const result = try self.buildFloatOp(.div, inst_ty, &.{ lhs, rhs });
5368 return try self.callFloor(result, inst_ty);5359 return self.buildFloatOp(.floor, inst_ty, &.{result});
5369 }5360 }
5370 if (scalar_ty.isSignedInt()) {5361 if (scalar_ty.isSignedInt()) {
5371 // const d = @divTrunc(a, b);5362 // const d = @divTrunc(a, b);
...@@ -5395,7 +5386,7 @@ pub const FuncGen = struct {...@@ -5395,7 +5386,7 @@ pub const FuncGen = struct {
5395 const inst_ty = self.air.typeOfIndex(inst);5386 const inst_ty = self.air.typeOfIndex(inst);
5396 const scalar_ty = inst_ty.scalarType();5387 const scalar_ty = inst_ty.scalarType();
53975388
5398 if (scalar_ty.isRuntimeFloat()) return self.builder.buildFDiv(lhs, rhs, "");5389 if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, inst_ty, &.{ lhs, rhs });
5399 if (scalar_ty.isSignedInt()) return self.builder.buildExactSDiv(lhs, rhs, "");5390 if (scalar_ty.isSignedInt()) return self.builder.buildExactSDiv(lhs, rhs, "");
5400 return self.builder.buildExactUDiv(lhs, rhs, "");5391 return self.builder.buildExactUDiv(lhs, rhs, "");
5401 }5392 }
...@@ -5409,7 +5400,7 @@ pub const FuncGen = struct {...@@ -5409,7 +5400,7 @@ pub const FuncGen = struct {
5409 const inst_ty = self.air.typeOfIndex(inst);5400 const inst_ty = self.air.typeOfIndex(inst);
5410 const scalar_ty = inst_ty.scalarType();5401 const scalar_ty = inst_ty.scalarType();
54115402
5412 if (scalar_ty.isRuntimeFloat()) return self.builder.buildFRem(lhs, rhs, "");5403 if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.rem, inst_ty, &.{ lhs, rhs });
5413 if (scalar_ty.isSignedInt()) return self.builder.buildSRem(lhs, rhs, "");5404 if (scalar_ty.isSignedInt()) return self.builder.buildSRem(lhs, rhs, "");
5414 return self.builder.buildURem(lhs, rhs, "");5405 return self.builder.buildURem(lhs, rhs, "");
5415 }5406 }
...@@ -5425,11 +5416,11 @@ pub const FuncGen = struct {...@@ -5425,11 +5416,11 @@ pub const FuncGen = struct {
5425 const scalar_ty = inst_ty.scalarType();5416 const scalar_ty = inst_ty.scalarType();
54265417
5427 if (scalar_ty.isRuntimeFloat()) {5418 if (scalar_ty.isRuntimeFloat()) {
5428 const a = self.builder.buildFRem(lhs, rhs, "");5419 const a = try self.buildFloatOp(.rem, inst_ty, &.{ lhs, rhs });
5429 const b = self.builder.buildFAdd(a, rhs, "");5420 const b = try self.buildFloatOp(.add, inst_ty, &.{ a, rhs });
5430 const c = self.builder.buildFRem(b, rhs, "");5421 const c = try self.buildFloatOp(.rem, inst_ty, &.{ b, rhs });
5431 const zero = inst_llvm_ty.constNull();5422 const zero = inst_llvm_ty.constNull();
5432 const ltz = self.builder.buildFCmp(.OLT, lhs, zero, "");5423 const ltz = try self.buildFloatCmp(.lt, inst_ty, &.{ lhs, zero });
5433 return self.builder.buildSelect(ltz, c, a, "");5424 return self.builder.buildSelect(ltz, c, a, "");
5434 }5425 }
5435 if (scalar_ty.isSignedInt()) {5426 if (scalar_ty.isSignedInt()) {
...@@ -5508,75 +5499,253 @@ pub const FuncGen = struct {...@@ -5508,75 +5499,253 @@ pub const FuncGen = struct {
5508 return result_struct;5499 return result_struct;
5509 }5500 }
55105501
5511 fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {5502 fn buildElementwiseCall(
5512 if (self.liveness.isUnused(inst)) return null;5503 self: *FuncGen,
5504 llvm_fn: *const llvm.Value,
5505 args_vectors: []const *const llvm.Value,
5506 result_vector: *const llvm.Value,
5507 vector_len: usize,
5508 ) !*const llvm.Value {
5509 const args_len = @intCast(c_uint, args_vectors.len);
5510 const llvm_i32 = self.context.intType(32);
5511 assert(args_len <= 8);
55135512
5514 const pl_op = self.air.instructions.items(.data)[inst].pl_op;5513 var i: usize = 0;
5515 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;5514 var result = result_vector;
5515 while (i < vector_len) : (i += 1) {
5516 const index_i32 = llvm_i32.constInt(i, .False);
55165517
5517 const mulend1 = try self.resolveInst(extra.lhs);5518 var args: [8]*const llvm.Value = undefined;
5518 const mulend2 = try self.resolveInst(extra.rhs);5519 for (args_vectors) |arg_vector, k| {
5519 const addend = try self.resolveInst(pl_op.operand);5520 args[k] = self.builder.buildExtractElement(arg_vector, index_i32, "");
5521 }
5522 const result_elem = self.builder.buildCall(llvm_fn, args[0..], args_len, .C, .Auto, "");
5523 result = self.builder.buildInsertElement(result, result_elem, index_i32, "");
5524 }
5525 return result;
5526 }
55205527
5521 const ty = self.air.typeOfIndex(inst);5528 fn getLibcFunction(
5522 const llvm_ty = try self.dg.llvmType(ty);5529 self: *FuncGen,
5523 const scalar_ty = ty.scalarType();5530 fn_name: [:0]const u8,
5524 const target = self.dg.module.getTarget();5531 param_types: []const *const llvm.Type,
5532 return_type: *const llvm.Type,
5533 ) *const llvm.Value {
5534 return self.dg.object.llvm_module.getNamedFunction(fn_name.ptr) orelse b: {
5535 const alias = self.dg.object.llvm_module.getNamedGlobalAlias(fn_name.ptr, fn_name.len);
5536 break :b if (alias) |a| a.getAliasee() else null;
5537 } orelse b: {
5538 const params_len = @intCast(c_uint, param_types.len);
5539 const fn_type = llvm.functionType(return_type, param_types.ptr, params_len, .False);
5540 const f = self.dg.object.llvm_module.addFunction(fn_name, fn_type);
5541 break :b f;
5542 };
5543 }
55255544
5526 const Strat = union(enum) {5545 fn getMathHTypeAbbrev(ty: Type) []const u8 {
5527 intrinsic,5546 return switch (ty.tag()) {
5528 libc: [*:0]const u8,5547 .f16 => "h", // Non-standard
5548 .f32 => "s",
5549 .f64 => "",
5550 .f80 => "x", // Non-standard
5551 .c_longdouble => "l",
5552 .f128 => "q", // Non-standard (mimics convention in GCC libquadmath)
5553 else => unreachable,
5529 };5554 };
5555 }
55305556
5531 const strat: Strat = switch (scalar_ty.floatBits(target)) {5557 fn getCompilerRtTypeAbbrev(ty: Type, target: std.Target) []const u8 {
5532 16, 32, 64 => Strat.intrinsic,5558 return switch (ty.floatBits(target)) {
5533 80 => if (CType.longdouble.sizeInBits(target) == 80) Strat{ .intrinsic = {} } else Strat{ .libc = "__fmax" },5559 16 => "h",
5534 // LLVM always lowers the fma builtin for f128 to fmal, which is for `long double`.5560 32 => "s",
5535 // On some targets this will be correct; on others it will be incorrect.5561 64 => "d",
5536 128 => if (CType.longdouble.sizeInBits(target) == 128) Strat{ .intrinsic = {} } else Strat{ .libc = "fmaq" },5562 80 => "x",
5563 128 => "t",
5537 else => unreachable,5564 else => unreachable,
5538 };5565 };
5566 }
5567
5568 /// Creates a floating point comparison by lowering to the appropriate
5569 /// hardware instruction or softfloat routine for the target
5570 fn buildFloatCmp(
5571 self: *FuncGen,
5572 pred: math.CompareOperator,
5573 ty: Type,
5574 params: []const *const llvm.Value,
5575 ) !*const llvm.Value {
5576 const target = self.dg.module.getTarget();
5577 const scalar_ty = ty.scalarType();
5578 const scalar_llvm_ty = try self.dg.llvmType(scalar_ty);
5579
5580 // LLVM does not support all floating point comparisons for all targets, so we
5581 // may need to manually generate a libc call
5582 const intrinsics_allowed = switch (scalar_ty.tag()) {
5583 .f80 => target.longDoubleIs(f80) and backendSupportsF80(target),
5584 .f128 => target.longDoubleIs(f128),
5585 else => true,
5586 };
5587 if (intrinsics_allowed) {
5588 const llvm_predicate: llvm.RealPredicate = switch (pred) {
5589 .eq => .OEQ,
5590 .neq => .UNE,
5591 .lt => .OLT,
5592 .lte => .OLE,
5593 .gt => .OGT,
5594 .gte => .OGE,
5595 };
5596 return self.builder.buildFCmp(llvm_predicate, params[0], params[1], "");
5597 }
5598
5599 const compiler_rt_type_abbrev = getCompilerRtTypeAbbrev(scalar_ty, target);
5600 var fn_name_buf: [64]u8 = undefined;
5601 const fn_base_name = switch (pred) {
5602 .neq => "ne",
5603 .eq => "eq",
5604 .lt => "lt",
5605 .lte => "le",
5606 .gt => "gt",
5607 .gte => "ge",
5608 };
5609 const fn_name = std.fmt.bufPrintZ(&fn_name_buf, "__{s}{s}f2", .{ fn_base_name, compiler_rt_type_abbrev }) catch unreachable;
55395610
5540 switch (strat) {5611 assert(params.len == 2);
5541 .intrinsic => {5612 const param_types = [2]*const llvm.Type{ scalar_llvm_ty, scalar_llvm_ty };
5542 const llvm_fn = self.getIntrinsic("llvm.fma", &.{llvm_ty});5613 const llvm_i32 = self.context.intType(32);
5543 const params = [_]*const llvm.Value{ mulend1, mulend2, addend };5614 const libc_fn = self.getLibcFunction(fn_name, param_types[0..], llvm_i32);
5544 return self.builder.buildCall(llvm_fn, &params, params.len, .C, .Auto, "");
5545 },
5546 .libc => |fn_name| {
5547 const scalar_llvm_ty = try self.dg.llvmType(scalar_ty);
5548 const llvm_fn = self.dg.object.llvm_module.getNamedFunction(fn_name) orelse b: {
5549 const param_types = [_]*const llvm.Type{ scalar_llvm_ty, scalar_llvm_ty, scalar_llvm_ty };
5550 const fn_type = llvm.functionType(scalar_llvm_ty, &param_types, param_types.len, .False);
5551 break :b self.dg.object.llvm_module.addFunction(fn_name, fn_type);
5552 };
55535615
5554 if (ty.zigTypeTag() == .Vector) {5616 const zero = llvm_i32.constInt(0, .False);
5555 const llvm_i32 = self.context.intType(32);5617 const int_pred: llvm.IntPredicate = switch (pred) {
5556 const vector_llvm_ty = try self.dg.llvmType(ty);5618 .eq => .EQ,
5619 .neq => .NE,
5620 .lt => .SLT,
5621 .lte => .SLE,
5622 .gt => .SGT,
5623 .gte => .SGE,
5624 };
55575625
5558 var i: usize = 0;5626 if (ty.zigTypeTag() == .Vector) {
5559 var vector = vector_llvm_ty.getUndef();5627 const vec_len = ty.vectorLen();
5560 while (i < ty.vectorLen()) : (i += 1) {5628 const vector_result_ty = llvm_i32.vectorType(vec_len);
5561 const index_i32 = llvm_i32.constInt(i, .False);
55625629
5563 const mulend1_elem = self.builder.buildExtractElement(mulend1, index_i32, "");5630 var result = vector_result_ty.getUndef();
5564 const mulend2_elem = self.builder.buildExtractElement(mulend2, index_i32, "");5631 result = try self.buildElementwiseCall(libc_fn, params[0..], result, vec_len);
5565 const addend_elem = self.builder.buildExtractElement(addend, index_i32, "");
55665632
5567 const params = [_]*const llvm.Value{ mulend1_elem, mulend2_elem, addend_elem };5633 const zero_vector = self.builder.buildVectorSplat(zero, vec_len, "");
5568 const mul_add = self.builder.buildCall(llvm_fn, &params, params.len, .C, .Auto, "");5634 return self.builder.buildICmp(int_pred, result, zero_vector, "");
5635 }
55695636
5570 vector = self.builder.buildInsertElement(vector, mul_add, index_i32, "");5637 const result = self.builder.buildCall(libc_fn, params.ptr, 2, .C, .Auto, "");
5571 }5638 return self.builder.buildICmp(int_pred, result, zero, "");
5639 }
55725640
5573 return vector;5641 /// Creates a floating point operation (add, sub, fma, sqrt, exp, etc.)
5574 } else {5642 /// by lowering to the appropriate hardware instruction or softfloat
5575 const params = [_]*const llvm.Value{ mulend1, mulend2, addend };5643 /// routine for the target
5576 return self.builder.buildCall(llvm_fn, &params, params.len, .C, .Auto, "");5644 fn buildFloatOp(
5645 self: *FuncGen,
5646 comptime op: @TypeOf(.EnumLiteral),
5647 ty: Type,
5648 params: []const *const llvm.Value,
5649 ) !*const llvm.Value {
5650 const target = self.dg.module.getTarget();
5651 const scalar_ty = ty.scalarType();
5652 const llvm_ty = try self.dg.llvmType(ty);
5653 const scalar_llvm_ty = try self.dg.llvmType(scalar_ty);
5654
5655 const Strat = union(enum) {
5656 intrinsic: []const u8,
5657 libc: [:0]const u8,
5658 };
5659
5660 // LLVM does not support all relevant intrinsics for all targets, so we
5661 // may need to manually generate a libc call
5662 const intrinsics_allowed = switch (scalar_ty.tag()) {
5663 .f80 => target.longDoubleIs(f80) and backendSupportsF80(target),
5664 .f128 => target.longDoubleIs(f128),
5665 else => true,
5666 };
5667 const strat: Strat = if (intrinsics_allowed) b: {
5668 // Some operations are dedicated LLVM instructions, not available as intrinsics
5669 switch (op) {
5670 .add => return self.builder.buildFAdd(params[0], params[1], ""),
5671 .sub => return self.builder.buildFSub(params[0], params[1], ""),
5672 .mul => return self.builder.buildFMul(params[0], params[1], ""),
5673 .div => return self.builder.buildFDiv(params[0], params[1], ""),
5674 .rem => return self.builder.buildFRem(params[0], params[1], ""),
5675 else => {},
5676 }
5677 // All other operations are available as intrinsics
5678 break :b .{
5679 .intrinsic = "llvm." ++ switch (op) {
5680 .max => "maximum",
5681 .min => "minimum",
5682 .fma, .sqrt, .sin, .cos, .exp, .exp2, .log, .log2, .log10, .fabs, .floor, .ceil, .round, .trunc => @tagName(op),
5683 .add, .sub, .mul, .div, .rem => unreachable,
5684 else => unreachable,
5685 },
5686 };
5687 } else b: {
5688 const math_h_type_abbrev = getMathHTypeAbbrev(scalar_ty);
5689 const compiler_rt_type_abbrev = getCompilerRtTypeAbbrev(scalar_ty, target);
5690 var fn_name_buf: [64]u8 = undefined;
5691 break :b switch (op) {
5692 .fma => Strat{
5693 .libc = switch (scalar_ty.floatBits(target)) {
5694 80 => "__fmax",
5695 else => std.fmt.bufPrintZ(&fn_name_buf, "fma{s}", .{math_h_type_abbrev}) catch unreachable,
5696 },
5697 },
5698 .add, .sub, .div, .mul => Strat{
5699 .libc = std.fmt.bufPrintZ(&fn_name_buf, "__{s}{s}f3", .{ @tagName(op), compiler_rt_type_abbrev }) catch unreachable,
5700 },
5701 .rem => Strat{
5702 .libc = std.fmt.bufPrintZ(&fn_name_buf, "fmod{s}", .{math_h_type_abbrev}) catch unreachable,
5703 },
5704 .max, .min => Strat{
5705 .libc = std.fmt.bufPrintZ(&fn_name_buf, "f{s}{s}", .{ @tagName(op), math_h_type_abbrev }) catch unreachable,
5706 },
5707 .sqrt, .sin, .cos, .exp, .exp2, .log, .log2, .log10, .fabs, .floor, .ceil, .round, .trunc => Strat{
5708 .libc = std.fmt.bufPrintZ(&fn_name_buf, "{s}{s}", .{ @tagName(op), math_h_type_abbrev }) catch unreachable,
5709 },
5710 else => unreachable,
5711 };
5712 };
5713
5714 var llvm_fn: *const llvm.Value = switch (strat) {
5715 .intrinsic => |fn_name| self.getIntrinsic(fn_name, &.{llvm_ty}),
5716 .libc => |fn_name| b: {
5717 assert(params.len == switch (op) {
5718 .fma => 3,
5719 .add, .sub, .div, .mul, .rem, .max, .min => 2,
5720 .sqrt, .sin, .cos, .exp, .exp2, .log, .log2, .log10, .fabs, .floor, .ceil, .round, .trunc => 1,
5721 else => unreachable,
5722 });
5723 const param_types = [3]*const llvm.Type{ scalar_llvm_ty, scalar_llvm_ty, scalar_llvm_ty };
5724 const libc_fn = self.getLibcFunction(fn_name, param_types[0..params.len], scalar_llvm_ty);
5725 if (ty.zigTypeTag() == .Vector) {
5726 const result = llvm_ty.getUndef();
5727 return self.buildElementwiseCall(libc_fn, params[0..], result, ty.vectorLen());
5577 }5728 }
5729
5730 break :b libc_fn;
5578 },5731 },
5579 }5732 };
5733 const params_len = @intCast(c_uint, params.len);
5734 return self.builder.buildCall(llvm_fn, params.ptr, params_len, .C, .Auto, "");
5735 }
5736
5737 fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
5738 if (self.liveness.isUnused(inst)) return null;
5739
5740 const pl_op = self.air.instructions.items(.data)[inst].pl_op;
5741 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
5742
5743 const mulend1 = try self.resolveInst(extra.lhs);
5744 const mulend2 = try self.resolveInst(extra.rhs);
5745 const addend = try self.resolveInst(pl_op.operand);
5746
5747 const ty = self.air.typeOfIndex(inst);
5748 return self.buildFloatOp(.fma, ty, &.{ mulend1, mulend2, addend });
5580 }5749 }
55815750
5582 fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {5751 fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
...@@ -6381,14 +6550,15 @@ pub const FuncGen = struct {...@@ -6381,14 +6550,15 @@ pub const FuncGen = struct {
6381 }6550 }
6382 }6551 }
63836552
6384 fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, llvm_fn_name: []const u8) !?*const llvm.Value {6553 fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: @TypeOf(.EnumLiteral)) !?*const llvm.Value {
6385 if (self.liveness.isUnused(inst)) return null;6554 if (self.liveness.isUnused(inst)) return null;
63866555
6387 const un_op = self.air.instructions.items(.data)[inst].un_op;6556 const un_op = self.air.instructions.items(.data)[inst].un_op;
6388 const operand = try self.resolveInst(un_op);6557 const operand = try self.resolveInst(un_op);
6389 const operand_ty = self.air.typeOf(un_op);6558 const operand_ty = self.air.typeOf(un_op);
63906559
6391 return self.callFloatUnary(operand, operand_ty, llvm_fn_name);6560 const params = [_]*const llvm.Value{operand};
6561 return self.buildFloatOp(op, operand_ty, &params);
6392 }6562 }
63936563
6394 fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, llvm_fn_name: []const u8) !?*const llvm.Value {6564 fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, llvm_fn_name: []const u8) !?*const llvm.Value {
...@@ -7191,48 +7361,6 @@ pub const FuncGen = struct {...@@ -7191,48 +7361,6 @@ pub const FuncGen = struct {
7191 return self.builder.buildExtractValue(opt_handle, 0, "");7361 return self.builder.buildExtractValue(opt_handle, 0, "");
7192 }7362 }
71937363
7194 fn callFloor(self: *FuncGen, arg: *const llvm.Value, ty: Type) !*const llvm.Value {
7195 return self.callFloatUnary(arg, ty, "floor");
7196 }
7197
7198 fn callCeil(self: *FuncGen, arg: *const llvm.Value, ty: Type) !*const llvm.Value {
7199 return self.callFloatUnary(arg, ty, "ceil");
7200 }
7201
7202 fn callTrunc(self: *FuncGen, arg: *const llvm.Value, ty: Type) !*const llvm.Value {
7203 return self.callFloatUnary(arg, ty, "trunc");
7204 }
7205
7206 fn callFloatUnary(
7207 self: *FuncGen,
7208 arg: *const llvm.Value,
7209 ty: Type,
7210 name: []const u8,
7211 ) !*const llvm.Value {
7212 const target = self.dg.module.getTarget();
7213
7214 var fn_name_buf: [100]u8 = undefined;
7215 const llvm_fn_name = switch (ty.zigTypeTag()) {
7216 .Vector => std.fmt.bufPrintZ(&fn_name_buf, "llvm.{s}.v{d}f{d}", .{
7217 name, ty.vectorLen(), ty.childType().floatBits(target),
7218 }) catch unreachable,
7219 .Float => std.fmt.bufPrintZ(&fn_name_buf, "llvm.{s}.f{d}", .{
7220 name, ty.floatBits(target),
7221 }) catch unreachable,
7222 else => unreachable,
7223 };
7224
7225 const llvm_fn = self.dg.object.llvm_module.getNamedFunction(llvm_fn_name) orelse blk: {
7226 const operand_llvm_ty = try self.dg.llvmType(ty);
7227 const param_types = [_]*const llvm.Type{operand_llvm_ty};
7228 const fn_type = llvm.functionType(operand_llvm_ty, &param_types, param_types.len, .False);
7229 break :blk self.dg.object.llvm_module.addFunction(llvm_fn_name, fn_type);
7230 };
7231
7232 const args: [1]*const llvm.Value = .{arg};
7233 return self.builder.buildCall(llvm_fn, &args, args.len, .C, .Auto, "");
7234 }
7235
7236 fn fieldPtr(7364 fn fieldPtr(
7237 self: *FuncGen,7365 self: *FuncGen,
7238 inst: Air.Inst.Index,7366 inst: Air.Inst.Index,
src/codegen/llvm/bindings.zig+11
...@@ -295,6 +295,9 @@ pub const Type = opaque {...@@ -295,6 +295,9 @@ pub const Type = opaque {
295295
296 pub const countStructElementTypes = LLVMCountStructElementTypes;296 pub const countStructElementTypes = LLVMCountStructElementTypes;
297 extern fn LLVMCountStructElementTypes(StructTy: *const Type) c_uint;297 extern fn LLVMCountStructElementTypes(StructTy: *const Type) c_uint;
298
299 pub const getVectorSize = LLVMGetVectorSize;
300 extern fn LLVMGetVectorSize(VectorTy: *const Type) c_uint;
298};301};
299302
300pub const Module = opaque {303pub const Module = opaque {
...@@ -675,6 +678,14 @@ pub const Builder = opaque {...@@ -675,6 +678,14 @@ pub const Builder = opaque {
675 Name: [*:0]const u8,678 Name: [*:0]const u8,
676 ) *const Value;679 ) *const Value;
677680
681 pub const buildVectorSplat = LLVMBuildVectorSplat;
682 extern fn LLVMBuildVectorSplat(
683 *const Builder,
684 EltVal: *const Value,
685 ElementCount: c_uint,
686 Name: [*:0]const u8,
687 ) *const Value;
688
678 pub const buildPtrToInt = LLVMBuildPtrToInt;689 pub const buildPtrToInt = LLVMBuildPtrToInt;
679 extern fn LLVMBuildPtrToInt(690 extern fn LLVMBuildPtrToInt(
680 *const Builder,691 *const Builder,
src/zig_llvm.cpp+4
...@@ -541,6 +541,10 @@ LLVMValueRef ZigLLVMBuildUShlSat(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRe...@@ -541,6 +541,10 @@ LLVMValueRef ZigLLVMBuildUShlSat(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRe
541 return wrap(call_inst);541 return wrap(call_inst);
542}542}
543543
544LLVMValueRef LLVMBuildVectorSplat(LLVMBuilderRef B, unsigned elem_count, LLVMValueRef V, const char *Name) {
545 return wrap(unwrap(B)->CreateVectorSplat(elem_count, unwrap(V), Name));
546}
547
544void ZigLLVMFnSetSubprogram(LLVMValueRef fn, ZigLLVMDISubprogram *subprogram) {548void ZigLLVMFnSetSubprogram(LLVMValueRef fn, ZigLLVMDISubprogram *subprogram) {
545 assert( isa<Function>(unwrap(fn)) );549 assert( isa<Function>(unwrap(fn)) );
546 Function *unwrapped_function = reinterpret_cast<Function*>(unwrap(fn));550 Function *unwrapped_function = reinterpret_cast<Function*>(unwrap(fn));
src/zig_llvm.h+1
...@@ -149,6 +149,7 @@ ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildSMulFixSat(LLVMBuilderRef B, LLVMValueRef...@@ -149,6 +149,7 @@ ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildSMulFixSat(LLVMBuilderRef B, LLVMValueRef
149ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildUMulFixSat(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS, const char *name);149ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildUMulFixSat(LLVMBuilderRef B, LLVMValueRef LHS, LLVMValueRef RHS, const char *name);
150ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildUShlSat(LLVMBuilderRef builder, LLVMValueRef LHS, LLVMValueRef RHS, const char* name);150ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildUShlSat(LLVMBuilderRef builder, LLVMValueRef LHS, LLVMValueRef RHS, const char* name);
151ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildSShlSat(LLVMBuilderRef builder, LLVMValueRef LHS, LLVMValueRef RHS, const char* name);151ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildSShlSat(LLVMBuilderRef builder, LLVMValueRef LHS, LLVMValueRef RHS, const char* name);
152ZIG_EXTERN_C LLVMValueRef LLVMBuildVectorSplat(LLVMBuilderRef B, unsigned elem_count, LLVMValueRef V, const char *Name);
152153
153154
154ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildNSWShl(LLVMBuilderRef builder, LLVMValueRef LHS, LLVMValueRef RHS,155ZIG_EXTERN_C LLVMValueRef ZigLLVMBuildNSWShl(LLVMBuilderRef builder, LLVMValueRef LHS, LLVMValueRef RHS,