| ... | ... | @@ -215,6 +215,7 @@ pub const MCValue = union(enum) { |
| 215 | 215 | /// but it has not been spilled there yet in the current control flow. |
| 216 | 216 | /// Payload is a frame index. |
| 217 | 217 | reserved_frame: FrameIndex, |
| 218 | air_ref: Air.Inst.Ref, |
| 218 | 219 | |
| 219 | 220 | fn isMemory(mcv: MCValue) bool { |
| 220 | 221 | return switch (mcv) { |
| ... | ... | @@ -278,6 +279,7 @@ pub const MCValue = union(enum) { |
| 278 | 279 | .lea_tlv, |
| 279 | 280 | .lea_frame, |
| 280 | 281 | .reserved_frame, |
| 282 | .air_ref, |
| 281 | 283 | => unreachable, // not in memory |
| 282 | 284 | .memory => |addr| .{ .immediate = addr }, |
| 283 | 285 | .indirect => |reg_off| switch (reg_off.off) { |
| ... | ... | @@ -306,6 +308,7 @@ pub const MCValue = union(enum) { |
| 306 | 308 | .load_tlv, |
| 307 | 309 | .load_frame, |
| 308 | 310 | .reserved_frame, |
| 311 | .air_ref, |
| 309 | 312 | => unreachable, // not dereferenceable |
| 310 | 313 | .immediate => |addr| .{ .memory = addr }, |
| 311 | 314 | .register => |reg| .{ .indirect = .{ .reg = reg } }, |
| ... | ... | @@ -335,6 +338,7 @@ pub const MCValue = union(enum) { |
| 335 | 338 | .lea_tlv, |
| 336 | 339 | .load_frame, |
| 337 | 340 | .reserved_frame, |
| 341 | .air_ref, |
| 338 | 342 | => unreachable, // not offsettable |
| 339 | 343 | .immediate => |imm| .{ .immediate = @bitCast(@as(i64, @bitCast(imm)) +% off) }, |
| 340 | 344 | .register => |reg| .{ .register_offset = .{ .reg = reg, .off = off } }, |
| ... | ... | @@ -366,6 +370,7 @@ pub const MCValue = union(enum) { |
| 366 | 370 | .lea_tlv, |
| 367 | 371 | .lea_frame, |
| 368 | 372 | .reserved_frame, |
| 373 | .air_ref, |
| 369 | 374 | => unreachable, |
| 370 | 375 | .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| |
| 371 | 376 | Memory.sib(ptr_size, .{ .base = .{ .reg = .ds }, .disp = small_addr }) |
| ... | ... | @@ -405,6 +410,7 @@ pub const MCValue = union(enum) { |
| 405 | 410 | .load_frame => |pl| try writer.print("[{} + 0x{x}]", .{ pl.index, pl.off }), |
| 406 | 411 | .lea_frame => |pl| try writer.print("{} + 0x{x}", .{ pl.index, pl.off }), |
| 407 | 412 | .reserved_frame => |pl| try writer.print("(dead:{})", .{pl}), |
| 413 | .air_ref => |pl| try writer.print("(air:0x{x})", .{@intFromEnum(pl)}), |
| 408 | 414 | } |
| 409 | 415 | } |
| 410 | 416 | }; |
| ... | ... | @@ -433,6 +439,7 @@ const InstTracking = struct { |
| 433 | 439 | => result, |
| 434 | 440 | .dead, |
| 435 | 441 | .reserved_frame, |
| 442 | .air_ref, |
| 436 | 443 | => unreachable, |
| 437 | 444 | .eflags, |
| 438 | 445 | .register, |
| ... | ... | @@ -491,6 +498,7 @@ const InstTracking = struct { |
| 491 | 498 | .register_overflow, |
| 492 | 499 | .indirect, |
| 493 | 500 | .reserved_frame, |
| 501 | .air_ref, |
| 494 | 502 | => unreachable, |
| 495 | 503 | }; |
| 496 | 504 | } |
| ... | ... | @@ -531,6 +539,7 @@ const InstTracking = struct { |
| 531 | 539 | .register_offset, |
| 532 | 540 | .register_overflow, |
| 533 | 541 | .indirect, |
| 542 | .air_ref, |
| 534 | 543 | => unreachable, |
| 535 | 544 | } |
| 536 | 545 | } |
| ... | ... | @@ -1809,9 +1818,9 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { |
| 1809 | 1818 | .round, |
| 1810 | 1819 | => |tag| try self.airUnaryMath(inst, tag), |
| 1811 | 1820 | |
| 1812 | | .floor => try self.airRound(inst, 0b1_0_01), |
| 1813 | | .ceil => try self.airRound(inst, 0b1_0_10), |
| 1814 | | .trunc_float => try self.airRound(inst, 0b1_0_11), |
| 1821 | .floor => try self.airRound(inst, .{ .mode = .down, .precision = .inexact }), |
| 1822 | .ceil => try self.airRound(inst, .{ .mode = .up, .precision = .inexact }), |
| 1823 | .trunc_float => try self.airRound(inst, .{ .mode = .zero, .precision = .inexact }), |
| 1815 | 1824 | .sqrt => try self.airSqrt(inst), |
| 1816 | 1825 | .neg => try self.airFloatSign(inst), |
| 1817 | 1826 | |
| ... | ... | @@ -2439,7 +2448,7 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt |
| 2439 | 2448 | } |
| 2440 | 2449 | |
| 2441 | 2450 | pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void { |
| 2442 | | const tracking = self.inst_tracking.getPtr(inst).?; |
| 2451 | const tracking = self.inst_tracking.getPtr(inst) orelse return; |
| 2443 | 2452 | assert(tracking.getReg().?.id() == reg.id()); |
| 2444 | 2453 | try tracking.spill(self, inst); |
| 2445 | 2454 | tracking.trackSpill(self, inst); |
| ... | ... | @@ -2545,7 +2554,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void { |
| 2545 | 2554 | floatCompilerRtAbiName(src_bits), |
| 2546 | 2555 | floatCompilerRtAbiName(dst_bits), |
| 2547 | 2556 | }) catch unreachable, |
| 2548 | | } }, &.{ty_op.operand}); |
| 2557 | } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}); |
| 2549 | 2558 | } |
| 2550 | 2559 | |
| 2551 | 2560 | const src_mcv = try self.resolveInst(ty_op.operand); |
| ... | ... | @@ -2644,7 +2653,7 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void { |
| 2644 | 2653 | floatCompilerRtAbiName(src_bits), |
| 2645 | 2654 | floatCompilerRtAbiName(dst_bits), |
| 2646 | 2655 | }) catch unreachable, |
| 2647 | | } }, &.{ty_op.operand}); |
| 2656 | } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}); |
| 2648 | 2657 | } |
| 2649 | 2658 | |
| 2650 | 2659 | const src_mcv = try self.resolveInst(ty_op.operand); |
| ... | ... | @@ -5073,7 +5082,26 @@ fn airFloatSign(self: *Self, inst: Air.Inst.Index) !void { |
| 5073 | 5082 | return self.floatSign(inst, un_op, ty); |
| 5074 | 5083 | } |
| 5075 | 5084 | |
| 5076 | | fn airRound(self: *Self, inst: Air.Inst.Index, mode: u4) !void { |
| 5085 | const RoundMode = packed struct(u5) { |
| 5086 | mode: enum(u4) { |
| 5087 | /// Round to nearest (even) |
| 5088 | nearest = 0b0_00, |
| 5089 | /// Round down (toward -∞) |
| 5090 | down = 0b0_01, |
| 5091 | /// Round up (toward +∞) |
| 5092 | up = 0b0_10, |
| 5093 | /// Round toward zero (truncate) |
| 5094 | zero = 0b0_11, |
| 5095 | /// Use current rounding mode of MXCSR.RC |
| 5096 | mxcsr = 0b1_00, |
| 5097 | }, |
| 5098 | precision: enum(u1) { |
| 5099 | normal = 0b0, |
| 5100 | inexact = 0b1, |
| 5101 | }, |
| 5102 | }; |
| 5103 | |
| 5104 | fn airRound(self: *Self, inst: Air.Inst.Index, mode: RoundMode) !void { |
| 5077 | 5105 | const un_op = self.air.instructions.items(.data)[inst].un_op; |
| 5078 | 5106 | const ty = self.typeOf(un_op); |
| 5079 | 5107 | |
| ... | ... | @@ -5089,12 +5117,9 @@ fn airRound(self: *Self, inst: Air.Inst.Index, mode: u4) !void { |
| 5089 | 5117 | return self.finishAir(inst, dst_mcv, .{ un_op, .none, .none }); |
| 5090 | 5118 | } |
| 5091 | 5119 | |
| 5092 | | fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4) !void { |
| 5120 | fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: RoundMode) !void { |
| 5093 | 5121 | const mod = self.bin_file.options.module.?; |
| 5094 | | if (!self.hasFeature(.sse4_1)) |
| 5095 | | return self.fail("TODO implement genRound without sse4_1 feature", .{}); |
| 5096 | | |
| 5097 | | const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(mod)) { |
| 5122 | const mir_tag = @as(?Mir.Inst.FixedTag, if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(mod)) { |
| 5098 | 5123 | .Float => switch (ty.floatBits(self.target.*)) { |
| 5099 | 5124 | 32 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round }, |
| 5100 | 5125 | 64 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round }, |
| ... | ... | @@ -5121,9 +5146,28 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4 |
| 5121 | 5146 | else => null, |
| 5122 | 5147 | }, |
| 5123 | 5148 | else => unreachable, |
| 5124 | | }) orelse return self.fail("TODO implement genRound for {}", .{ |
| 5125 | | ty.fmt(self.bin_file.options.module.?), |
| 5126 | | }); |
| 5149 | } else null) orelse { |
| 5150 | if (ty.zigTypeTag(mod) != .Float) return self.fail("TODO implement genRound for {}", .{ |
| 5151 | ty.fmt(self.bin_file.options.module.?), |
| 5152 | }); |
| 5153 | |
| 5154 | var callee: ["__trunc?".len]u8 = undefined; |
| 5155 | const res = try self.genCall(.{ .lib = .{ |
| 5156 | .return_type = ty.toIntern(), |
| 5157 | .param_types = &.{ty.toIntern()}, |
| 5158 | .callee = std.fmt.bufPrint(&callee, "{s}{s}{s}", .{ |
| 5159 | floatLibcAbiPrefix(ty), |
| 5160 | switch (mode.mode) { |
| 5161 | .down => "floor", |
| 5162 | .up => "ceil", |
| 5163 | .zero => "trunc", |
| 5164 | else => unreachable, |
| 5165 | }, |
| 5166 | floatLibcAbiSuffix(ty), |
| 5167 | }) catch unreachable, |
| 5168 | } }, &.{ty}, &.{src_mcv}); |
| 5169 | return self.genSetReg(dst_reg, ty, res); |
| 5170 | }; |
| 5127 | 5171 | const abi_size: u32 = @intCast(ty.abiSize(mod)); |
| 5128 | 5172 | const dst_alias = registerAlias(dst_reg, abi_size); |
| 5129 | 5173 | switch (mir_tag[0]) { |
| ... | ... | @@ -5132,7 +5176,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4 |
| 5132 | 5176 | dst_alias, |
| 5133 | 5177 | dst_alias, |
| 5134 | 5178 | src_mcv.mem(Memory.PtrSize.fromSize(abi_size)), |
| 5135 | | Immediate.u(mode), |
| 5179 | Immediate.u(@as(u5, @bitCast(mode))), |
| 5136 | 5180 | ) else try self.asmRegisterRegisterRegisterImmediate( |
| 5137 | 5181 | mir_tag, |
| 5138 | 5182 | dst_alias, |
| ... | ... | @@ -5141,13 +5185,13 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4 |
| 5141 | 5185 | src_mcv.getReg().? |
| 5142 | 5186 | else |
| 5143 | 5187 | try self.copyToTmpRegister(ty, src_mcv), abi_size), |
| 5144 | | Immediate.u(mode), |
| 5188 | Immediate.u(@as(u5, @bitCast(mode))), |
| 5145 | 5189 | ), |
| 5146 | 5190 | else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( |
| 5147 | 5191 | mir_tag, |
| 5148 | 5192 | dst_alias, |
| 5149 | 5193 | src_mcv.mem(Memory.PtrSize.fromSize(abi_size)), |
| 5150 | | Immediate.u(mode), |
| 5194 | Immediate.u(@as(u5, @bitCast(mode))), |
| 5151 | 5195 | ) else try self.asmRegisterRegisterImmediate( |
| 5152 | 5196 | mir_tag, |
| 5153 | 5197 | dst_alias, |
| ... | ... | @@ -5155,7 +5199,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: u4 |
| 5155 | 5199 | src_mcv.getReg().? |
| 5156 | 5200 | else |
| 5157 | 5201 | try self.copyToTmpRegister(ty, src_mcv), abi_size), |
| 5158 | | Immediate.u(mode), |
| 5202 | Immediate.u(@as(u5, @bitCast(mode))), |
| 5159 | 5203 | ), |
| 5160 | 5204 | } |
| 5161 | 5205 | } |
| ... | ... | @@ -5353,34 +5397,29 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void { |
| 5353 | 5397 | |
| 5354 | 5398 | fn airUnaryMath(self: *Self, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void { |
| 5355 | 5399 | const un_op = self.air.instructions.items(.data)[inst].un_op; |
| 5356 | | const ty = self.typeOf(un_op).toIntern(); |
| 5400 | const ty = self.typeOf(un_op); |
| 5401 | var callee: ["__round?".len]u8 = undefined; |
| 5357 | 5402 | const result = try self.genCall(.{ .lib = .{ |
| 5358 | | .return_type = ty, |
| 5359 | | .param_types = &.{ty}, |
| 5360 | | .callee = switch (tag) { |
| 5361 | | inline .sin, |
| 5362 | | .cos, |
| 5363 | | .tan, |
| 5364 | | .exp, |
| 5365 | | .exp2, |
| 5366 | | .log, |
| 5367 | | .log2, |
| 5368 | | .log10, |
| 5369 | | .round, |
| 5370 | | => |comptime_tag| switch (ty) { |
| 5371 | | .f16_type => "__" ++ @tagName(comptime_tag) ++ "h", |
| 5372 | | .f32_type => @tagName(comptime_tag) ++ "f", |
| 5373 | | .f64_type => @tagName(comptime_tag), |
| 5374 | | .f80_type => "__" ++ @tagName(comptime_tag) ++ "x", |
| 5375 | | .f128_type => @tagName(comptime_tag) ++ "q", |
| 5376 | | .c_longdouble_type => @tagName(comptime_tag) ++ "l", |
| 5377 | | else => return self.fail("TODO implement airUnaryMath for {s} of {}", .{ |
| 5378 | | @tagName(tag), ty.toType().fmt(self.bin_file.options.module.?), |
| 5379 | | }), |
| 5403 | .return_type = ty.toIntern(), |
| 5404 | .param_types = &.{ty.toIntern()}, |
| 5405 | .callee = std.fmt.bufPrint(&callee, "{s}{s}{s}", .{ |
| 5406 | floatLibcAbiPrefix(ty), |
| 5407 | switch (tag) { |
| 5408 | .sin, |
| 5409 | .cos, |
| 5410 | .tan, |
| 5411 | .exp, |
| 5412 | .exp2, |
| 5413 | .log, |
| 5414 | .log2, |
| 5415 | .log10, |
| 5416 | .round, |
| 5417 | => @tagName(tag), |
| 5418 | else => unreachable, |
| 5380 | 5419 | }, |
| 5381 | | else => unreachable, |
| 5382 | | }, |
| 5383 | | } }, &.{un_op}); |
| 5420 | floatLibcAbiSuffix(ty), |
| 5421 | }) catch unreachable, |
| 5422 | } }, &.{ty}, &.{.{ .air_ref = un_op }}); |
| 5384 | 5423 | return self.finishAir(inst, result, .{ un_op, .none, .none }); |
| 5385 | 5424 | } |
| 5386 | 5425 | |
| ... | ... | @@ -5534,6 +5573,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerErro |
| 5534 | 5573 | |
| 5535 | 5574 | try self.genCopy(dst_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }); |
| 5536 | 5575 | }, |
| 5576 | .air_ref => |ref| try self.load(dst_mcv, ptr_ty, try self.resolveInst(ref)), |
| 5537 | 5577 | } |
| 5538 | 5578 | } |
| 5539 | 5579 | |
| ... | ... | @@ -5679,6 +5719,7 @@ fn store(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) InnerErr |
| 5679 | 5719 | |
| 5680 | 5720 | try self.genCopy(src_ty, .{ .indirect = .{ .reg = addr_reg } }, src_mcv); |
| 5681 | 5721 | }, |
| 5722 | .air_ref => |ref| try self.store(ptr_ty, ptr_mcv, try self.resolveInst(ref)), |
| 5682 | 5723 | } |
| 5683 | 5724 | } |
| 5684 | 5725 | |
| ... | ... | @@ -6020,6 +6061,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MC |
| 6020 | 6061 | .lea_tlv, |
| 6021 | 6062 | .lea_frame, |
| 6022 | 6063 | .reserved_frame, |
| 6064 | .air_ref, |
| 6023 | 6065 | => unreachable, // unmodifiable destination |
| 6024 | 6066 | .register => |dst_reg| try self.asmRegister(mir_tag, registerAlias(dst_reg, abi_size)), |
| 6025 | 6067 | .memory, .load_got, .load_direct, .load_tlv => { |
| ... | ... | @@ -6679,7 +6721,11 @@ fn genBinOp( |
| 6679 | 6721 | .min, |
| 6680 | 6722 | .max, |
| 6681 | 6723 | => { |
| 6682 | | const mat_src_mcv: MCValue = if (switch (src_mcv) { |
| 6724 | const resolved_src_mcv = switch (src_mcv) { |
| 6725 | else => src_mcv, |
| 6726 | .air_ref => |ref| try self.resolveInst(ref), |
| 6727 | }; |
| 6728 | const mat_src_mcv: MCValue = if (switch (resolved_src_mcv) { |
| 6683 | 6729 | .immediate, |
| 6684 | 6730 | .eflags, |
| 6685 | 6731 | .register_offset, |
| ... | ... | @@ -6693,7 +6739,10 @@ fn genBinOp( |
| 6693 | 6739 | => true, |
| 6694 | 6740 | .memory => |addr| math.cast(i32, @as(i64, @bitCast(addr))) == null, |
| 6695 | 6741 | else => false, |
| 6696 | | }) .{ .register = try self.copyToTmpRegister(rhs_ty, src_mcv) } else src_mcv; |
| 6742 | }) |
| 6743 | .{ .register = try self.copyToTmpRegister(rhs_ty, resolved_src_mcv) } |
| 6744 | else |
| 6745 | resolved_src_mcv; |
| 6697 | 6746 | const mat_mcv_lock = switch (mat_src_mcv) { |
| 6698 | 6747 | .register => |reg| self.register_manager.lockReg(reg), |
| 6699 | 6748 | else => null, |
| ... | ... | @@ -6740,6 +6789,7 @@ fn genBinOp( |
| 6740 | 6789 | .lea_tlv, |
| 6741 | 6790 | .lea_frame, |
| 6742 | 6791 | .reserved_frame, |
| 6792 | .air_ref, |
| 6743 | 6793 | => unreachable, |
| 6744 | 6794 | .register => |src_reg| try self.asmCmovccRegisterRegister( |
| 6745 | 6795 | registerAlias(tmp_reg, cmov_abi_size), |
| ... | ... | @@ -7389,11 +7439,11 @@ fn genBinOp( |
| 7389 | 7439 | lhs_ty, |
| 7390 | 7440 | dst_reg, |
| 7391 | 7441 | .{ .register = dst_reg }, |
| 7392 | | switch (air_tag) { |
| 7393 | | .div_trunc => 0b1_0_11, |
| 7394 | | .div_floor => 0b1_0_01, |
| 7442 | .{ .mode = switch (air_tag) { |
| 7443 | .div_trunc => .zero, |
| 7444 | .div_floor => .down, |
| 7395 | 7445 | else => unreachable, |
| 7396 | | }, |
| 7446 | }, .precision = .inexact }, |
| 7397 | 7447 | ), |
| 7398 | 7448 | .bit_and, .bit_or, .xor => {}, |
| 7399 | 7449 | .max, .min => if (maybe_mask_reg) |mask_reg| if (self.hasFeature(.avx)) { |
| ... | ... | @@ -7651,6 +7701,7 @@ fn genBinOpMir( |
| 7651 | 7701 | .lea_tlv, |
| 7652 | 7702 | .lea_frame, |
| 7653 | 7703 | .reserved_frame, |
| 7704 | .air_ref, |
| 7654 | 7705 | => unreachable, // unmodifiable destination |
| 7655 | 7706 | .register, .register_offset => { |
| 7656 | 7707 | assert(dst_mcv.isRegister()); |
| ... | ... | @@ -7766,6 +7817,7 @@ fn genBinOpMir( |
| 7766 | 7817 | else => unreachable, |
| 7767 | 7818 | } |
| 7768 | 7819 | }, |
| 7820 | .air_ref => |ref| try self.genBinOpMir(mir_tag, ty, dst_mcv, try self.resolveInst(ref)), |
| 7769 | 7821 | } |
| 7770 | 7822 | }, |
| 7771 | 7823 | .memory, .indirect, .load_got, .load_direct, .load_tlv, .load_frame => { |
| ... | ... | @@ -7789,13 +7841,18 @@ fn genBinOpMir( |
| 7789 | 7841 | }; |
| 7790 | 7842 | defer if (dst_info) |info| self.register_manager.unlockReg(info.addr_lock); |
| 7791 | 7843 | |
| 7792 | | const src_info: OpInfo = switch (src_mcv) { |
| 7844 | const resolved_src_mcv = switch (src_mcv) { |
| 7845 | else => src_mcv, |
| 7846 | .air_ref => |ref| try self.resolveInst(ref), |
| 7847 | }; |
| 7848 | const src_info: OpInfo = switch (resolved_src_mcv) { |
| 7793 | 7849 | .none, |
| 7794 | 7850 | .unreach, |
| 7795 | 7851 | .dead, |
| 7796 | 7852 | .undef, |
| 7797 | 7853 | .register_overflow, |
| 7798 | 7854 | .reserved_frame, |
| 7855 | .air_ref, |
| 7799 | 7856 | => unreachable, |
| 7800 | 7857 | .immediate, |
| 7801 | 7858 | .register, |
| ... | ... | @@ -7809,7 +7866,7 @@ fn genBinOpMir( |
| 7809 | 7866 | .lea_frame, |
| 7810 | 7867 | => null, |
| 7811 | 7868 | .memory, .load_got, .load_direct, .load_tlv => src: { |
| 7812 | | switch (src_mcv) { |
| 7869 | switch (resolved_src_mcv) { |
| 7813 | 7870 | .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr))) != null and |
| 7814 | 7871 | math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null) |
| 7815 | 7872 | break :src null, |
| ... | ... | @@ -7821,7 +7878,7 @@ fn genBinOpMir( |
| 7821 | 7878 | const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg); |
| 7822 | 7879 | errdefer self.register_manager.unlockReg(src_addr_lock); |
| 7823 | 7880 | |
| 7824 | | try self.genSetReg(src_addr_reg, Type.usize, src_mcv.address()); |
| 7881 | try self.genSetReg(src_addr_reg, Type.usize, resolved_src_mcv.address()); |
| 7825 | 7882 | break :src .{ |
| 7826 | 7883 | .addr_reg = src_addr_reg, |
| 7827 | 7884 | .addr_lock = src_addr_lock, |
| ... | ... | @@ -7868,13 +7925,14 @@ fn genBinOpMir( |
| 7868 | 7925 | else => unreachable, |
| 7869 | 7926 | }, |
| 7870 | 7927 | ); |
| 7871 | | switch (src_mcv) { |
| 7928 | switch (resolved_src_mcv) { |
| 7872 | 7929 | .none, |
| 7873 | 7930 | .unreach, |
| 7874 | 7931 | .dead, |
| 7875 | 7932 | .undef, |
| 7876 | 7933 | .register_overflow, |
| 7877 | 7934 | .reserved_frame, |
| 7935 | .air_ref, |
| 7878 | 7936 | => unreachable, |
| 7879 | 7937 | .register => |src_reg| switch (off) { |
| 7880 | 7938 | 0 => try self.asmMemoryRegister( |
| ... | ... | @@ -7950,7 +8008,7 @@ fn genBinOpMir( |
| 7950 | 8008 | => { |
| 7951 | 8009 | const src_limb_reg = try self.copyToTmpRegister(limb_ty, if (src_info) |info| .{ |
| 7952 | 8010 | .indirect = .{ .reg = info.addr_reg, .off = off }, |
| 7953 | | } else switch (src_mcv) { |
| 8011 | } else switch (resolved_src_mcv) { |
| 7954 | 8012 | .eflags, |
| 7955 | 8013 | .register_offset, |
| 7956 | 8014 | .lea_direct, |
| ... | ... | @@ -7958,7 +8016,7 @@ fn genBinOpMir( |
| 7958 | 8016 | .lea_tlv, |
| 7959 | 8017 | .lea_frame, |
| 7960 | 8018 | => switch (off) { |
| 7961 | | 0 => src_mcv, |
| 8019 | 0 => resolved_src_mcv, |
| 7962 | 8020 | else => .{ .immediate = 0 }, |
| 7963 | 8021 | }, |
| 7964 | 8022 | .memory => |addr| .{ .memory = @bitCast(@as(i64, @bitCast(addr)) + off) }, |
| ... | ... | @@ -8003,19 +8061,25 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M |
| 8003 | 8061 | .lea_tlv, |
| 8004 | 8062 | .lea_frame, |
| 8005 | 8063 | .reserved_frame, |
| 8064 | .air_ref, |
| 8006 | 8065 | => unreachable, // unmodifiable destination |
| 8007 | 8066 | .register => |dst_reg| { |
| 8008 | 8067 | const dst_alias = registerAlias(dst_reg, abi_size); |
| 8009 | 8068 | const dst_lock = self.register_manager.lockReg(dst_reg); |
| 8010 | 8069 | defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); |
| 8011 | 8070 | |
| 8012 | | switch (src_mcv) { |
| 8071 | const resolved_src_mcv = switch (src_mcv) { |
| 8072 | else => src_mcv, |
| 8073 | .air_ref => |ref| try self.resolveInst(ref), |
| 8074 | }; |
| 8075 | switch (resolved_src_mcv) { |
| 8013 | 8076 | .none, |
| 8014 | 8077 | .unreach, |
| 8015 | 8078 | .dead, |
| 8016 | 8079 | .undef, |
| 8017 | 8080 | .register_overflow, |
| 8018 | 8081 | .reserved_frame, |
| 8082 | .air_ref, |
| 8019 | 8083 | => unreachable, |
| 8020 | 8084 | .register => |src_reg| try self.asmRegisterRegister( |
| 8021 | 8085 | .{ .i_, .mul }, |
| ... | ... | @@ -8031,7 +8095,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M |
| 8031 | 8095 | Immediate.s(small), |
| 8032 | 8096 | ); |
| 8033 | 8097 | } else { |
| 8034 | | const src_reg = try self.copyToTmpRegister(dst_ty, src_mcv); |
| 8098 | const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv); |
| 8035 | 8099 | return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg }); |
| 8036 | 8100 | } |
| 8037 | 8101 | }, |
| ... | ... | @@ -8047,19 +8111,22 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M |
| 8047 | 8111 | => try self.asmRegisterRegister( |
| 8048 | 8112 | .{ .i_, .mul }, |
| 8049 | 8113 | dst_alias, |
| 8050 | | registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size), |
| 8114 | registerAlias(try self.copyToTmpRegister(dst_ty, resolved_src_mcv), abi_size), |
| 8051 | 8115 | ), |
| 8052 | 8116 | .memory, .indirect, .load_frame => try self.asmRegisterMemory( |
| 8053 | 8117 | .{ .i_, .mul }, |
| 8054 | 8118 | dst_alias, |
| 8055 | | Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (src_mcv) { |
| 8119 | Memory.sib(Memory.PtrSize.fromSize(abi_size), switch (resolved_src_mcv) { |
| 8056 | 8120 | .memory => |addr| .{ |
| 8057 | 8121 | .base = .{ .reg = .ds }, |
| 8058 | 8122 | .disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse |
| 8059 | 8123 | return self.asmRegisterRegister( |
| 8060 | 8124 | .{ .i_, .mul }, |
| 8061 | 8125 | dst_alias, |
| 8062 | | registerAlias(try self.copyToTmpRegister(dst_ty, src_mcv), abi_size), |
| 8126 | registerAlias( |
| 8127 | try self.copyToTmpRegister(dst_ty, resolved_src_mcv), |
| 8128 | abi_size, |
| 8129 | ), |
| 8063 | 8130 | ), |
| 8064 | 8131 | }, |
| 8065 | 8132 | .indirect => |reg_off| .{ |
| ... | ... | @@ -8221,13 +8288,30 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier |
| 8221 | 8288 | |
| 8222 | 8289 | const pl_op = self.air.instructions.items(.data)[inst].pl_op; |
| 8223 | 8290 | const extra = self.air.extraData(Air.Call, pl_op.payload); |
| 8224 | | const args: []const Air.Inst.Ref = @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]); |
| 8291 | const arg_refs: []const Air.Inst.Ref = |
| 8292 | @ptrCast(self.air.extra[extra.end..][0..extra.data.args_len]); |
| 8293 | |
| 8294 | const ExpectedContents = extern struct { |
| 8295 | tys: [16][@sizeOf(Type)]u8 align(@alignOf(Type)), |
| 8296 | vals: [16][@sizeOf(MCValue)]u8 align(@alignOf(MCValue)), |
| 8297 | }; |
| 8298 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = |
| 8299 | std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); |
| 8300 | const allocator = stack.get(); |
| 8301 | |
| 8302 | const arg_tys = try allocator.alloc(Type, arg_refs.len); |
| 8303 | defer allocator.free(arg_tys); |
| 8304 | for (arg_tys, arg_refs) |*arg_ty, arg_ref| arg_ty.* = self.typeOf(arg_ref); |
| 8225 | 8305 | |
| 8226 | | const ret = try self.genCall(.{ .air = pl_op.operand }, args); |
| 8306 | const arg_vals = try allocator.alloc(MCValue, arg_refs.len); |
| 8307 | defer allocator.free(arg_vals); |
| 8308 | for (arg_vals, arg_refs) |*arg_val, arg_ref| arg_val.* = .{ .air_ref = arg_ref }; |
| 8309 | |
| 8310 | const ret = try self.genCall(.{ .air = pl_op.operand }, arg_tys, arg_vals); |
| 8227 | 8311 | |
| 8228 | 8312 | var bt = self.liveness.iterateBigTomb(inst); |
| 8229 | 8313 | self.feed(&bt, pl_op.operand); |
| 8230 | | for (args) |arg| self.feed(&bt, arg); |
| 8314 | for (arg_refs) |arg_ref| self.feed(&bt, arg_ref); |
| 8231 | 8315 | |
| 8232 | 8316 | const result = if (self.liveness.isUnused(inst)) .unreach else ret; |
| 8233 | 8317 | return self.finishAirResult(inst, result); |
| ... | ... | @@ -8241,7 +8325,7 @@ fn genCall(self: *Self, info: union(enum) { |
| 8241 | 8325 | lib: ?[]const u8 = null, |
| 8242 | 8326 | callee: []const u8, |
| 8243 | 8327 | }, |
| 8244 | | }, args: []const Air.Inst.Ref) !MCValue { |
| 8328 | }, arg_types: []const Type, args: []const MCValue) !MCValue { |
| 8245 | 8329 | const mod = self.bin_file.options.module.?; |
| 8246 | 8330 | |
| 8247 | 8331 | const fn_ty = switch (info) { |
| ... | ... | @@ -8261,8 +8345,17 @@ fn genCall(self: *Self, info: union(enum) { |
| 8261 | 8345 | }; |
| 8262 | 8346 | const fn_info = mod.typeToFunc(fn_ty).?; |
| 8263 | 8347 | |
| 8348 | const ExpectedContents = [16]Type; |
| 8349 | var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = |
| 8350 | std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); |
| 8351 | const allocator = stack.get(); |
| 8352 | |
| 8353 | const var_args = try allocator.alloc(Type, args.len - fn_info.param_types.len); |
| 8354 | defer allocator.free(var_args); |
| 8355 | for (var_args, arg_types[fn_info.param_types.len..]) |*var_arg, arg_ty| var_arg.* = arg_ty; |
| 8356 | |
| 8264 | 8357 | var call_info = |
| 8265 | | try self.resolveCallingConventionValues(fn_info, args[fn_info.param_types.len..], .call_frame); |
| 8358 | try self.resolveCallingConventionValues(fn_info, var_args, .call_frame); |
| 8266 | 8359 | defer call_info.deinit(self); |
| 8267 | 8360 | |
| 8268 | 8361 | // We need a properly aligned and sized call frame to be able to call this function. |
| ... | ... | @@ -8290,10 +8383,10 @@ fn genCall(self: *Self, info: union(enum) { |
| 8290 | 8383 | .indirect => |reg_off| try self.spillRegisters(&.{reg_off.reg}), |
| 8291 | 8384 | else => unreachable, |
| 8292 | 8385 | } |
| 8293 | | for (call_info.args, args) |dst_arg, src_arg| switch (dst_arg) { |
| 8386 | for (call_info.args, arg_types, args) |dst_arg, arg_ty, src_arg| switch (dst_arg) { |
| 8294 | 8387 | .none => {}, |
| 8295 | 8388 | .register => |reg| try self.spillRegisters(&.{reg}), |
| 8296 | | .load_frame => try self.genCopy(self.typeOf(src_arg), dst_arg, try self.resolveInst(src_arg)), |
| 8389 | .load_frame => try self.genCopy(arg_ty, dst_arg, src_arg), |
| 8297 | 8390 | else => unreachable, |
| 8298 | 8391 | }; |
| 8299 | 8392 | |
| ... | ... | @@ -8313,10 +8406,10 @@ fn genCall(self: *Self, info: union(enum) { |
| 8313 | 8406 | }; |
| 8314 | 8407 | defer if (ret_lock) |lock| self.register_manager.unlockReg(lock); |
| 8315 | 8408 | |
| 8316 | | for (call_info.args, args) |dst_arg, src_arg| { |
| 8409 | for (call_info.args, arg_types, args) |dst_arg, arg_ty, src_arg| { |
| 8317 | 8410 | switch (dst_arg) { |
| 8318 | 8411 | .none, .load_frame => {}, |
| 8319 | | .register => try self.genCopy(self.typeOf(src_arg), dst_arg, try self.resolveInst(src_arg)), |
| 8412 | .register => try self.genCopy(arg_ty, dst_arg, src_arg), |
| 8320 | 8413 | else => unreachable, |
| 8321 | 8414 | } |
| 8322 | 8415 | } |
| ... | ... | @@ -8374,7 +8467,7 @@ fn genCall(self: *Self, info: union(enum) { |
| 8374 | 8467 | } |
| 8375 | 8468 | } else { |
| 8376 | 8469 | assert(self.typeOf(callee).zigTypeTag(mod) == .Pointer); |
| 8377 | | try self.genSetReg(.rax, Type.usize, try self.resolveInst(callee)); |
| 8470 | try self.genSetReg(.rax, Type.usize, .{ .air_ref = callee }); |
| 8378 | 8471 | try self.asmRegister(.{ ._, .call }, .rax); |
| 8379 | 8472 | }, |
| 8380 | 8473 | .lib => |lib| try self.genExternSymbolRef(.call, lib.lib, lib.callee), |
| ... | ... | @@ -8454,7 +8547,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { |
| 8454 | 8547 | }, |
| 8455 | 8548 | floatCompilerRtAbiName(float_bits), |
| 8456 | 8549 | }) catch unreachable, |
| 8457 | | } }, &.{ bin_op.lhs, bin_op.rhs }); |
| 8550 | } }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } }); |
| 8458 | 8551 | try self.genBinOpMir(.{ ._, .@"test" }, Type.i32, ret, ret); |
| 8459 | 8552 | break :result switch (op) { |
| 8460 | 8553 | .eq => .e, |
| ... | ... | @@ -8908,6 +9001,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC |
| 8908 | 9001 | .lea_tlv, |
| 8909 | 9002 | .lea_frame, |
| 8910 | 9003 | .reserved_frame, |
| 9004 | .air_ref, |
| 8911 | 9005 | => unreachable, |
| 8912 | 9006 | |
| 8913 | 9007 | .register => |opt_reg| { |
| ... | ... | @@ -9933,6 +10027,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError |
| 9933 | 10027 | .lea_tlv, |
| 9934 | 10028 | .lea_frame, |
| 9935 | 10029 | .reserved_frame, |
| 10030 | .air_ref, |
| 9936 | 10031 | => unreachable, // unmodifiable destination |
| 9937 | 10032 | .register => |reg| try self.genSetReg(reg, ty, src_mcv), |
| 9938 | 10033 | .register_offset => |dst_reg_off| try self.genSetReg(dst_reg_off.reg, ty, switch (src_mcv) { |
| ... | ... | @@ -10258,6 +10353,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr |
| 10258 | 10353 | @tagName(self.bin_file.tag), |
| 10259 | 10354 | }); |
| 10260 | 10355 | }, |
| 10356 | .air_ref => |ref| try self.genSetReg(dst_reg, ty, try self.resolveInst(ref)), |
| 10261 | 10357 | } |
| 10262 | 10358 | } |
| 10263 | 10359 | |
| ... | ... | @@ -10370,6 +10466,7 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal |
| 10370 | 10466 | }, |
| 10371 | 10467 | else => try self.genInlineMemcpy(dst_ptr_mcv, src_mcv.address(), .{ .immediate = abi_size }), |
| 10372 | 10468 | }, |
| 10469 | .air_ref => |ref| try self.genSetMem(base, disp, ty, try self.resolveInst(ref)), |
| 10373 | 10470 | } |
| 10374 | 10471 | } |
| 10375 | 10472 | |
| ... | ... | @@ -10663,7 +10760,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void { |
| 10663 | 10760 | intCompilerRtAbiName(src_bits), |
| 10664 | 10761 | floatCompilerRtAbiName(dst_bits), |
| 10665 | 10762 | }) catch unreachable, |
| 10666 | | } }, &.{ty_op.operand}); |
| 10763 | } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}); |
| 10667 | 10764 | } |
| 10668 | 10765 | |
| 10669 | 10766 | if (src_size > 8) return self.fail("TODO implement airFloatFromInt from {} to {}", .{ |
| ... | ... | @@ -10742,7 +10839,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void { |
| 10742 | 10839 | floatCompilerRtAbiName(src_bits), |
| 10743 | 10840 | intCompilerRtAbiName(dst_bits), |
| 10744 | 10841 | }) catch unreachable, |
| 10745 | | } }, &.{ty_op.operand}); |
| 10842 | } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}); |
| 10746 | 10843 | } |
| 10747 | 10844 | |
| 10748 | 10845 | if (dst_size > 8) return self.fail("TODO implement airIntFromFloat from {} to {}", .{ |
| ... | ... | @@ -12103,7 +12200,7 @@ const CallMCValues = struct { |
| 12103 | 12200 | fn resolveCallingConventionValues( |
| 12104 | 12201 | self: *Self, |
| 12105 | 12202 | fn_info: InternPool.Key.FuncType, |
| 12106 | | var_args: []const Air.Inst.Ref, |
| 12203 | var_args: []const Type, |
| 12107 | 12204 | stack_frame_base: FrameIndex, |
| 12108 | 12205 | ) !CallMCValues { |
| 12109 | 12206 | const mod = self.bin_file.options.module.?; |
| ... | ... | @@ -12116,9 +12213,7 @@ fn resolveCallingConventionValues( |
| 12116 | 12213 | dest.* = src.toType(); |
| 12117 | 12214 | } |
| 12118 | 12215 | // TODO: promote var arg types |
| 12119 | | for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg| { |
| 12120 | | param_ty.* = self.typeOf(arg); |
| 12121 | | } |
| 12216 | for (param_types[fn_info.param_types.len..], var_args) |*param_ty, arg_ty| param_ty.* = arg_ty; |
| 12122 | 12217 | |
| 12123 | 12218 | var result: CallMCValues = .{ |
| 12124 | 12219 | .args = try self.gpa.alloc(MCValue, param_types.len), |
| ... | ... | @@ -12454,3 +12549,25 @@ fn floatCompilerRtAbiType(self: *Self, ty: Type, other_ty: Type) Type { |
| 12454 | 12549 | self.target.isDarwin()) return Type.u16; |
| 12455 | 12550 | return ty; |
| 12456 | 12551 | } |
| 12552 | |
| 12553 | fn floatLibcAbiPrefix(ty: Type) []const u8 { |
| 12554 | return switch (ty.toIntern()) { |
| 12555 | .f16_type, |
| 12556 | .f80_type, |
| 12557 | => "__", |
| 12558 | .f32_type, .f64_type, .f128_type, .c_longdouble_type => "", |
| 12559 | else => unreachable, |
| 12560 | }; |
| 12561 | } |
| 12562 | |
| 12563 | fn floatLibcAbiSuffix(ty: Type) []const u8 { |
| 12564 | return switch (ty.toIntern()) { |
| 12565 | .f16_type => "h", |
| 12566 | .f32_type => "f", |
| 12567 | .f64_type => "", |
| 12568 | .f80_type => "x", |
| 12569 | .f128_type => "q", |
| 12570 | .c_longdouble_type => "l", |
| 12571 | else => unreachable, |
| 12572 | }; |
| 12573 | } |