authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-05-19 00:56:59-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-05-19 00:56:59-07:00
logb873ce1e0e527a74fab9e55975ff1503e8f70b53
tree812ebfef4c58ef9ef9e4684b747570db2861256a
parent503302ceef7290027eedcc78c903be32fad90433
parent47405b1a1c29b8a907997af18291eb6eb1cf3e02
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15727 from jacobly0/x86_64-behavior

x86_64: behavior

17 files changed, 861 insertions(+), 293 deletions(-)

src/arch/x86_64/CodeGen.zig+706-245
...@@ -1271,6 +1271,27 @@ fn asmRegisterRegisterRegister(...@@ -1271,6 +1271,27 @@ fn asmRegisterRegisterRegister(
1271 });1271 });
1272}1272}
12731273
1274fn asmRegisterRegisterRegisterRegister(
1275 self: *Self,
1276 tag: Mir.Inst.FixedTag,
1277 reg1: Register,
1278 reg2: Register,
1279 reg3: Register,
1280 reg4: Register,
1281) !void {
1282 _ = try self.addInst(.{
1283 .tag = tag[1],
1284 .ops = .rrrr,
1285 .data = .{ .rrrr = .{
1286 .fixes = tag[0],
1287 .r1 = reg1,
1288 .r2 = reg2,
1289 .r3 = reg3,
1290 .r4 = reg4,
1291 } },
1292 });
1293}
1294
1274fn asmRegisterRegisterRegisterImmediate(1295fn asmRegisterRegisterRegisterImmediate(
1275 self: *Self,1296 self: *Self,
1276 tag: Mir.Inst.FixedTag,1297 tag: Mir.Inst.FixedTag,
...@@ -2203,6 +2224,10 @@ fn getFrameAddrAlignment(self: *Self, frame_addr: FrameAddr) u32 {...@@ -2203,6 +2224,10 @@ fn getFrameAddrAlignment(self: *Self, frame_addr: FrameAddr) u32 {
2203 return @min(alloc_align, @bitCast(u32, frame_addr.off) & (alloc_align - 1));2224 return @min(alloc_align, @bitCast(u32, frame_addr.off) & (alloc_align - 1));
2204}2225}
22052226
2227fn getFrameAddrSize(self: *Self, frame_addr: FrameAddr) u32 {
2228 return self.frame_allocs.get(@enumToInt(frame_addr.index)).abi_size - @intCast(u31, frame_addr.off);
2229}
2230
2206fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {2231fn allocFrameIndex(self: *Self, alloc: FrameAlloc) !FrameIndex {
2207 const frame_allocs_slice = self.frame_allocs.slice();2232 const frame_allocs_slice = self.frame_allocs.slice();
2208 const frame_size = frame_allocs_slice.items(.abi_size);2233 const frame_size = frame_allocs_slice.items(.abi_size);
...@@ -2594,115 +2619,202 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {...@@ -2594,115 +2619,202 @@ fn airFpext(self: *Self, inst: Air.Inst.Index) !void {
25942619
2595fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {2620fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
2596 const ty_op = self.air.instructions.items(.data)[inst].ty_op;2621 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2622 const result: MCValue = result: {
2623 const src_ty = self.air.typeOf(ty_op.operand);
2624 const src_int_info = src_ty.intInfo(self.target.*);
25972625
2598 const src_ty = self.air.typeOf(ty_op.operand);2626 const dst_ty = self.air.typeOfIndex(inst);
2599 const src_int_info = src_ty.intInfo(self.target.*);2627 const dst_int_info = dst_ty.intInfo(self.target.*);
2600 const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));2628 const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
2601 const src_mcv = try self.resolveInst(ty_op.operand);
2602 const src_lock = switch (src_mcv) {
2603 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
2604 else => null,
2605 };
2606 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
26072629
2608 const dst_ty = self.air.typeOfIndex(inst);2630 const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
2609 const dst_int_info = dst_ty.intInfo(self.target.*);2631 const extend = switch (src_int_info.signedness) {
2610 const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));2632 .signed => dst_int_info,
2611 const dst_mcv = if (dst_abi_size <= src_abi_size and2633 .unsigned => src_int_info,
2612 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))2634 }.signedness;
2613 src_mcv
2614 else
2615 try self.allocRegOrMem(inst, true);
26162635
2617 const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;2636 const src_mcv = try self.resolveInst(ty_op.operand);
2618 const signedness: std.builtin.Signedness = if (dst_int_info.signedness == .signed and2637 const src_storage_bits = switch (src_mcv) {
2619 src_int_info.signedness == .signed) .signed else .unsigned;2638 .register, .register_offset => 64,
2620 switch (dst_mcv) {2639 .load_frame => |frame_addr| self.getFrameAddrSize(frame_addr) * 8,
2621 .register => |dst_reg| {2640 else => src_int_info.bits,
2622 const min_abi_size = @min(dst_abi_size, src_abi_size);2641 };
2623 const tag: Mir.Inst.FixedTag = switch (signedness) {2642
2624 .signed => if (min_abi_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },2643 const dst_mcv = if (dst_int_info.bits <= src_storage_bits and
2625 .unsigned => if (min_abi_size >= 4) .{ ._, .mov } else .{ ._, .movzx },2644 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
2626 };2645 const dst_mcv = try self.allocRegOrMem(inst, true);
2627 const dst_alias = switch (tag[1]) {2646 try self.genCopy(min_ty, dst_mcv, src_mcv);
2628 .movsx => dst_reg.to64(),2647 break :dst dst_mcv;
2629 .mov, .movzx => if (min_abi_size > 4) dst_reg.to64() else dst_reg.to32(),2648 };
2630 else => unreachable,2649
2650 if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister())
2651 .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) }
2652 else
2653 dst_mcv;
2654
2655 if (dst_mcv.isRegister()) {
2656 try self.truncateRegister(src_ty, dst_mcv.getReg().?);
2657 break :result .{ .register = registerAlias(dst_mcv.getReg().?, abi_size) };
2658 }
2659
2660 const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable;
2661 const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable;
2662
2663 const high_mcv = dst_mcv.address().offset((src_limbs_len - 1) * 8).deref();
2664 const high_reg = try self.copyToTmpRegister(switch (src_int_info.signedness) {
2665 .signed => Type.isize,
2666 .unsigned => Type.usize,
2667 }, high_mcv);
2668 const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);
2669 defer self.register_manager.unlockReg(high_lock);
2670
2671 const high_bits = src_int_info.bits % 64;
2672 if (high_bits > 0) {
2673 var high_pl = Type.Payload.Bits{
2674 .base = .{ .tag = switch (extend) {
2675 .signed => .int_signed,
2676 .unsigned => .int_unsigned,
2677 } },
2678 .data = high_bits,
2631 };2679 };
2632 switch (src_mcv) {2680 const high_ty = Type.initPayload(&high_pl.base);
2633 .register => |src_reg| {2681 try self.truncateRegister(high_ty, high_reg);
2634 try self.asmRegisterRegister(2682 try self.genCopy(Type.usize, high_mcv, .{ .register = high_reg });
2635 tag,2683 }
2636 dst_alias,2684
2637 registerAlias(src_reg, min_abi_size),2685 if (dst_limbs_len > src_limbs_len) try self.genInlineMemset(
2686 dst_mcv.address().offset(src_limbs_len * 8),
2687 switch (extend) {
2688 .signed => extend: {
2689 const extend_mcv = MCValue{ .register = high_reg };
2690 try self.genShiftBinOpMir(
2691 .{ ._r, .sa },
2692 Type.isize,
2693 extend_mcv,
2694 .{ .immediate = 63 },
2638 );2695 );
2696 break :extend extend_mcv;
2639 },2697 },
2640 .memory, .indirect, .load_frame => try self.asmRegisterMemory(2698 .unsigned => .{ .immediate = 0 },
2641 tag,2699 },
2642 dst_alias,2700 .{ .immediate = (dst_limbs_len - src_limbs_len) * 8 },
2643 src_mcv.mem(Memory.PtrSize.fromSize(min_abi_size)),2701 );
2644 ),2702
2645 else => return self.fail("TODO airIntCast from {s} to {s}", .{2703 break :result dst_mcv;
2646 @tagName(src_mcv),2704 };
2647 @tagName(dst_mcv),2705 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2648 }),
2649 }
2650 if (self.regExtraBits(min_ty) > 0) try self.truncateRegister(min_ty, dst_reg);
2651 },
2652 else => {
2653 try self.genCopy(min_ty, dst_mcv, src_mcv);
2654 const extra = dst_abi_size * 8 - dst_int_info.bits;
2655 if (extra > 0) {
2656 try self.genShiftBinOpMir(
2657 switch (signedness) {
2658 .signed => .{ ._l, .sa },
2659 .unsigned => .{ ._l, .sh },
2660 },
2661 dst_ty,
2662 dst_mcv,
2663 .{ .immediate = extra },
2664 );
2665 try self.genShiftBinOpMir(
2666 switch (signedness) {
2667 .signed => .{ ._r, .sa },
2668 .unsigned => .{ ._r, .sh },
2669 },
2670 dst_ty,
2671 dst_mcv,
2672 .{ .immediate = extra },
2673 );
2674 }
2675 },
2676 }
2677 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
2678}2706}
26792707
2680fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {2708fn airTrunc(self: *Self, inst: Air.Inst.Index) !void {
2681 const ty_op = self.air.instructions.items(.data)[inst].ty_op;2709 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
26822710
2683 const dst_ty = self.air.typeOfIndex(inst);2711 const dst_ty = self.air.typeOfIndex(inst);
2684 const dst_abi_size = dst_ty.abiSize(self.target.*);2712 const dst_abi_size = @intCast(u32, dst_ty.abiSize(self.target.*));
2685 if (dst_abi_size > 8) {2713 const src_ty = self.air.typeOf(ty_op.operand);
2686 return self.fail("TODO implement trunc for abi sizes larger than 8", .{});2714 const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
2687 }
26882715
2689 const src_mcv = try self.resolveInst(ty_op.operand);2716 const result = result: {
2690 const src_lock = switch (src_mcv) {2717 const src_mcv = try self.resolveInst(ty_op.operand);
2691 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),2718 const src_lock =
2692 else => null,2719 if (src_mcv.getReg()) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
2693 };2720 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
2694 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
26952721
2696 const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))2722 const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2697 src_mcv2723 src_mcv
2698 else2724 else
2699 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);2725 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
2726
2727 if (dst_ty.zigTypeTag() == .Vector) {
2728 assert(src_ty.zigTypeTag() == .Vector and dst_ty.vectorLen() == src_ty.vectorLen());
2729 const dst_info = dst_ty.childType().intInfo(self.target.*);
2730 const src_info = src_ty.childType().intInfo(self.target.*);
2731 const mir_tag = if (@as(?Mir.Inst.FixedTag, switch (dst_info.bits) {
2732 8 => switch (src_info.bits) {
2733 16 => switch (dst_ty.vectorLen()) {
2734 1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
2735 9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null,
2736 else => null,
2737 },
2738 else => null,
2739 },
2740 16 => switch (src_info.bits) {
2741 32 => switch (dst_ty.vectorLen()) {
2742 1...4 => if (self.hasFeature(.avx))
2743 .{ .vp_w, .ackusd }
2744 else if (self.hasFeature(.sse4_1))
2745 .{ .p_w, .ackusd }
2746 else
2747 null,
2748 5...8 => if (self.hasFeature(.avx2)) .{ .vp_w, .ackusd } else null,
2749 else => null,
2750 },
2751 else => null,
2752 },
2753 else => null,
2754 })) |tag| tag else return self.fail("TODO implement airTrunc for {}", .{
2755 dst_ty.fmt(self.bin_file.options.module.?),
2756 });
27002757
2701 // when truncating a `u16` to `u5`, for example, those top 3 bits in the result2758 var mask_pl = Value.Payload.U64{
2702 // have to be removed. this only happens if the dst if not a power-of-two size.2759 .base = .{ .tag = .int_u64 },
2703 if (self.regExtraBits(dst_ty) > 0) try self.truncateRegister(dst_ty, dst_mcv.register.to64());2760 .data = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - dst_info.bits),
2761 };
2762 const mask_val = Value.initPayload(&mask_pl.base);
27042763
2705 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });2764 var splat_pl = Value.Payload.SubValue{
2765 .base = .{ .tag = .repeated },
2766 .data = mask_val,
2767 };
2768 const splat_val = Value.initPayload(&splat_pl.base);
2769
2770 var full_pl = Type.Payload.Array{
2771 .base = .{ .tag = .vector },
2772 .data = .{
2773 .len = @divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits),
2774 .elem_type = src_ty.childType(),
2775 },
2776 };
2777 const full_ty = Type.initPayload(&full_pl.base);
2778 const full_abi_size = @intCast(u32, full_ty.abiSize(self.target.*));
2779
2780 const splat_mcv = try self.genTypedValue(.{ .ty = full_ty, .val = splat_val });
2781 const splat_addr_mcv: MCValue = switch (splat_mcv) {
2782 .memory, .indirect, .load_frame => splat_mcv.address(),
2783 else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) },
2784 };
2785
2786 const dst_reg = registerAlias(dst_mcv.getReg().?, src_abi_size);
2787 if (self.hasFeature(.avx)) {
2788 try self.asmRegisterRegisterMemory(
2789 .{ .vp_, .@"and" },
2790 dst_reg,
2791 dst_reg,
2792 splat_addr_mcv.deref().mem(Memory.PtrSize.fromSize(full_abi_size)),
2793 );
2794 try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg);
2795 } else {
2796 try self.asmRegisterMemory(
2797 .{ .p_, .@"and" },
2798 dst_reg,
2799 splat_addr_mcv.deref().mem(Memory.PtrSize.fromSize(full_abi_size)),
2800 );
2801 try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg);
2802 }
2803 break :result dst_mcv;
2804 }
2805
2806 if (dst_abi_size > 8) {
2807 return self.fail("TODO implement trunc for abi sizes larger than 8", .{});
2808 }
2809
2810 // when truncating a `u16` to `u5`, for example, those top 3 bits in the result
2811 // have to be removed. this only happens if the dst if not a power-of-two size.
2812 if (self.regExtraBits(dst_ty) > 0)
2813 try self.truncateRegister(dst_ty, dst_mcv.register.to64());
2814
2815 break :result dst_mcv;
2816 };
2817 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
2706}2818}
27072819
2708fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void {2820fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void {
...@@ -2855,20 +2967,43 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -2855,20 +2967,43 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2855 defer self.register_manager.unlockReg(limit_lock);2967 defer self.register_manager.unlockReg(limit_lock);
28562968
2857 const reg_bits = self.regBitSize(ty);2969 const reg_bits = self.regBitSize(ty);
2970 const reg_extra_bits = self.regExtraBits(ty);
2858 const cc: Condition = if (ty.isSignedInt()) cc: {2971 const cc: Condition = if (ty.isSignedInt()) cc: {
2972 if (reg_extra_bits > 0) {
2973 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .{ .immediate = reg_extra_bits });
2974 }
2859 try self.genSetReg(limit_reg, ty, dst_mcv);2975 try self.genSetReg(limit_reg, ty, dst_mcv);
2860 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });2976 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2861 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{2977 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2862 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,2978 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2863 });2979 });
2980 if (reg_extra_bits > 0) {
2981 const shifted_rhs_reg = try self.copyToTmpRegister(ty, rhs_mcv);
2982 const shifted_rhs_mcv = MCValue{ .register = shifted_rhs_reg };
2983 const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);
2984 defer self.register_manager.unlockReg(shifted_rhs_lock);
2985
2986 try self.genShiftBinOpMir(
2987 .{ ._l, .sa },
2988 ty,
2989 shifted_rhs_mcv,
2990 .{ .immediate = reg_extra_bits },
2991 );
2992 try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, shifted_rhs_mcv);
2993 } else try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
2864 break :cc .o;2994 break :cc .o;
2865 } else cc: {2995 } else cc: {
2866 try self.genSetReg(limit_reg, ty, .{2996 try self.genSetReg(limit_reg, ty, .{
2867 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),2997 .immediate = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - ty.bitSize(self.target.*)),
2868 });2998 });
2999
3000 try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
3001 if (reg_extra_bits > 0) {
3002 try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, limit_mcv);
3003 break :cc .a;
3004 }
2869 break :cc .c;3005 break :cc .c;
2870 };3006 };
2871 try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
28723007
2873 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);3008 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
2874 try self.asmCmovccRegisterRegister(3009 try self.asmCmovccRegisterRegister(
...@@ -2877,6 +3012,10 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -2877,6 +3012,10 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
2877 cc,3012 cc,
2878 );3013 );
28793014
3015 if (reg_extra_bits > 0 and ty.isSignedInt()) {
3016 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .{ .immediate = reg_extra_bits });
3017 }
3018
2880 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3019 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2881}3020}
28823021
...@@ -2906,18 +3045,36 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -2906,18 +3045,36 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2906 defer self.register_manager.unlockReg(limit_lock);3045 defer self.register_manager.unlockReg(limit_lock);
29073046
2908 const reg_bits = self.regBitSize(ty);3047 const reg_bits = self.regBitSize(ty);
3048 const reg_extra_bits = self.regExtraBits(ty);
2909 const cc: Condition = if (ty.isSignedInt()) cc: {3049 const cc: Condition = if (ty.isSignedInt()) cc: {
3050 if (reg_extra_bits > 0) {
3051 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .{ .immediate = reg_extra_bits });
3052 }
2910 try self.genSetReg(limit_reg, ty, dst_mcv);3053 try self.genSetReg(limit_reg, ty, dst_mcv);
2911 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });3054 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
2912 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{3055 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
2913 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,3056 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
2914 });3057 });
3058 if (reg_extra_bits > 0) {
3059 const shifted_rhs_reg = try self.copyToTmpRegister(ty, rhs_mcv);
3060 const shifted_rhs_mcv = MCValue{ .register = shifted_rhs_reg };
3061 const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);
3062 defer self.register_manager.unlockReg(shifted_rhs_lock);
3063
3064 try self.genShiftBinOpMir(
3065 .{ ._l, .sa },
3066 ty,
3067 shifted_rhs_mcv,
3068 .{ .immediate = reg_extra_bits },
3069 );
3070 try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, shifted_rhs_mcv);
3071 } else try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
2915 break :cc .o;3072 break :cc .o;
2916 } else cc: {3073 } else cc: {
2917 try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });3074 try self.genSetReg(limit_reg, ty, .{ .immediate = 0 });
3075 try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
2918 break :cc .c;3076 break :cc .c;
2919 };3077 };
2920 try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
29213078
2922 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);3079 const cmov_abi_size = @max(@intCast(u32, ty.abiSize(self.target.*)), 2);
2923 try self.asmCmovccRegisterRegister(3080 try self.asmCmovccRegisterRegister(
...@@ -2926,6 +3083,10 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {...@@ -2926,6 +3083,10 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
2926 cc,3083 cc,
2927 );3084 );
29283085
3086 if (reg_extra_bits > 0 and ty.isSignedInt()) {
3087 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .{ .immediate = reg_extra_bits });
3088 }
3089
2929 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });3090 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
2930}3091}
29313092
...@@ -3222,34 +3383,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3222,34 +3383,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3222 self.regExtraBits(dst_ty)3383 self.regExtraBits(dst_ty)
3223 else3384 else
3224 dst_info.bits % 64;3385 dst_info.bits % 64;
3225 const partial_mcv = if (dst_info.signedness == .signed and extra_bits > 0) dst: {3386 const partial_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs);
3226 const rhs_lock: ?RegisterLock = switch (rhs) {
3227 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
3228 else => null,
3229 };
3230 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
3231
3232 const dst_reg: Register = blk: {
3233 if (lhs.isRegister()) break :blk lhs.register;
3234 break :blk try self.copyToTmpRegister(dst_ty, lhs);
3235 };
3236 const dst_mcv = MCValue{ .register = dst_reg };
3237 const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
3238 defer self.register_manager.unlockReg(dst_reg_lock);
3239
3240 const rhs_mcv: MCValue = blk: {
3241 if (rhs.isRegister() or rhs.isMemory()) break :blk rhs;
3242 break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) };
3243 };
3244 const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) {
3245 .register => |reg| self.register_manager.lockReg(reg),
3246 else => null,
3247 };
3248 defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock);
3249
3250 try self.genIntMulComplexOpMir(Type.isize, dst_mcv, rhs_mcv);
3251 break :dst dst_mcv;
3252 } else try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs);
32533387
3254 switch (partial_mcv) {3388 switch (partial_mcv) {
3255 .register => |reg| if (extra_bits == 0) {3389 .register => |reg| if (extra_bits == 0) {
...@@ -3262,9 +3396,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3262,9 +3396,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3262 break :result .{ .load_frame = .{ .index = frame_index } };3396 break :result .{ .load_frame = .{ .index = frame_index } };
3263 },3397 },
3264 else => {3398 else => {
3265 // For now, this is the only supported multiply that doesn't fit in a register,3399 // For now, this is the only supported multiply that doesn't fit in a register.
3266 // so cc being set is impossible.
3267
3268 assert(dst_info.bits <= 128 and src_pl.data == 64);3400 assert(dst_info.bits <= 128 and src_pl.data == 64);
32693401
3270 const frame_index =3402 const frame_index =
...@@ -3280,7 +3412,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {...@@ -3280,7 +3412,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void {
3280 .{ .frame = frame_index },3412 .{ .frame = frame_index },
3281 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),3413 @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)),
3282 tuple_ty.structFieldType(1),3414 tuple_ty.structFieldType(1),
3283 .{ .immediate = 0 },3415 .{ .immediate = 0 }, // cc being set is impossible
3284 );3416 );
3285 } else try self.genSetFrameTruncatedOverflowCompare(3417 } else try self.genSetFrameTruncatedOverflowCompare(
3286 tuple_ty,3418 tuple_ty,
...@@ -5558,31 +5690,13 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {...@@ -5558,31 +5690,13 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void {
5558 const dst_lock = self.register_manager.lockReg(dst_reg);5690 const dst_lock = self.register_manager.lockReg(dst_reg);
5559 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);5691 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
55605692
5561 // Shift by struct_field_offset.
5562 try self.genShiftBinOpMir(5693 try self.genShiftBinOpMir(
5563 .{ ._r, .sh },5694 .{ ._r, .sh },
5564 Type.usize,5695 Type.usize,
5565 dst_mcv,5696 dst_mcv,
5566 .{ .immediate = field_off },5697 .{ .immediate = field_off },
5567 );5698 );
55685699 if (self.regExtraBits(field_ty) > 0) try self.truncateRegister(field_ty, dst_reg);
5569 // Mask to field_bit_size bits
5570 const field_bit_size = field_ty.bitSize(self.target.*);
5571 const mask = ~@as(u64, 0) >> @intCast(u6, 64 - field_bit_size);
5572
5573 const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask });
5574 try self.genBinOpMir(.{ ._, .@"and" }, Type.usize, dst_mcv, .{ .register = tmp_reg });
5575
5576 const signedness =
5577 if (field_ty.isAbiInt()) field_ty.intInfo(self.target.*).signedness else .unsigned;
5578 const field_byte_size = @intCast(u32, field_ty.abiSize(self.target.*));
5579 if (signedness == .signed and field_byte_size < 8) {
5580 try self.asmRegisterRegister(
5581 if (field_byte_size >= 4) .{ ._d, .movsx } else .{ ._, .movsx },
5582 dst_mcv.register,
5583 registerAlias(dst_mcv.register, field_byte_size),
5584 );
5585 }
55865700
5587 break :result if (field_rc.supersetOf(gp))5701 break :result if (field_rc.supersetOf(gp))
5588 dst_mcv5702 dst_mcv
...@@ -6224,12 +6338,26 @@ fn genBinOp(...@@ -6224,12 +6338,26 @@ fn genBinOp(
6224 lhs_air: Air.Inst.Ref,6338 lhs_air: Air.Inst.Ref,
6225 rhs_air: Air.Inst.Ref,6339 rhs_air: Air.Inst.Ref,
6226) !MCValue {6340) !MCValue {
6227 const lhs_mcv = try self.resolveInst(lhs_air);
6228 const rhs_mcv = try self.resolveInst(rhs_air);
6229 const lhs_ty = self.air.typeOf(lhs_air);6341 const lhs_ty = self.air.typeOf(lhs_air);
6230 const rhs_ty = self.air.typeOf(rhs_air);6342 const rhs_ty = self.air.typeOf(rhs_air);
6231 const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));6343 const abi_size = @intCast(u32, lhs_ty.abiSize(self.target.*));
62326344
6345 const maybe_mask_reg = switch (air_tag) {
6346 else => null,
6347 .max, .min => if (lhs_ty.scalarType().isRuntimeFloat()) registerAlias(
6348 if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: {
6349 try self.register_manager.getReg(.xmm0, null);
6350 break :mask .xmm0;
6351 } else try self.register_manager.allocReg(null, sse),
6352 abi_size,
6353 ) else null,
6354 };
6355 const mask_lock =
6356 if (maybe_mask_reg) |mask_reg| self.register_manager.lockRegAssumeUnused(mask_reg) else null;
6357 defer if (mask_lock) |lock| self.register_manager.unlockReg(lock);
6358
6359 const lhs_mcv = try self.resolveInst(lhs_air);
6360 const rhs_mcv = try self.resolveInst(rhs_air);
6233 switch (lhs_mcv) {6361 switch (lhs_mcv) {
6234 .immediate => |imm| switch (imm) {6362 .immediate => |imm| switch (imm) {
6235 0 => switch (air_tag) {6363 0 => switch (air_tag) {
...@@ -6300,7 +6428,16 @@ fn genBinOp(...@@ -6300,7 +6428,16 @@ fn genBinOp(
6300 };6428 };
6301 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);6429 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
63026430
6303 const src_mcv = if (flipped) lhs_mcv else rhs_mcv;6431 const unmat_src_mcv = if (flipped) lhs_mcv else rhs_mcv;
6432 const src_mcv: MCValue = if (maybe_mask_reg) |mask_reg|
6433 if (self.hasFeature(.avx) and unmat_src_mcv.isRegister() and maybe_inst != null and
6434 self.liveness.operandDies(maybe_inst.?, if (flipped) 0 else 1)) unmat_src_mcv else src: {
6435 try self.genSetReg(mask_reg, rhs_ty, unmat_src_mcv);
6436 break :src .{ .register = mask_reg };
6437 }
6438 else
6439 unmat_src_mcv;
6440
6304 if (!vec_op) {6441 if (!vec_op) {
6305 switch (air_tag) {6442 switch (air_tag) {
6306 .add,6443 .add,
...@@ -7009,18 +7146,26 @@ fn genBinOp(...@@ -7009,18 +7146,26 @@ fn genBinOp(
7009 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{7146 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
7010 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),7147 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7011 });7148 });
7149
7150 const lhs_copy_reg = if (maybe_mask_reg) |_| registerAlias(
7151 if (copied_to_dst) try self.copyToTmpRegister(lhs_ty, dst_mcv) else lhs_mcv.getReg().?,
7152 abi_size,
7153 ) else null;
7154 const lhs_copy_lock = if (lhs_copy_reg) |reg| self.register_manager.lockReg(reg) else null;
7155 defer if (lhs_copy_lock) |lock| self.register_manager.unlockReg(lock);
7156
7012 if (self.hasFeature(.avx)) {7157 if (self.hasFeature(.avx)) {
7013 const src1_alias =7158 const lhs_reg =
7014 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);7159 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
7015 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(7160 if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory(
7016 mir_tag,7161 mir_tag,
7017 dst_reg,7162 dst_reg,
7018 src1_alias,7163 lhs_reg,
7019 src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),7164 src_mcv.mem(Memory.PtrSize.fromSize(abi_size)),
7020 ) else try self.asmRegisterRegisterRegister(7165 ) else try self.asmRegisterRegisterRegister(
7021 mir_tag,7166 mir_tag,
7022 dst_reg,7167 dst_reg,
7023 src1_alias,7168 lhs_reg,
7024 registerAlias(if (src_mcv.isRegister())7169 registerAlias(if (src_mcv.isRegister())
7025 src_mcv.getReg().?7170 src_mcv.getReg().?
7026 else7171 else
...@@ -7041,9 +7186,10 @@ fn genBinOp(...@@ -7041,9 +7186,10 @@ fn genBinOp(
7041 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),7186 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
7042 );7187 );
7043 }7188 }
7189
7044 switch (air_tag) {7190 switch (air_tag) {
7045 .add, .addwrap, .sub, .subwrap, .mul, .mulwrap, .div_float, .div_exact => {},7191 .add, .addwrap, .sub, .subwrap, .mul, .mulwrap, .div_float, .div_exact => {},
7046 .div_trunc, .div_floor => try self.genRound(7192 .div_trunc, .div_floor => if (self.hasFeature(.sse4_1)) try self.genRound(
7047 lhs_ty,7193 lhs_ty,
7048 dst_reg,7194 dst_reg,
7049 .{ .register = dst_reg },7195 .{ .register = dst_reg },
...@@ -7052,11 +7198,240 @@ fn genBinOp(...@@ -7052,11 +7198,240 @@ fn genBinOp(
7052 .div_floor => 0b1_0_01,7198 .div_floor => 0b1_0_01,
7053 else => unreachable,7199 else => unreachable,
7054 },7200 },
7055 ),7201 ) else return self.fail("TODO implement genBinOp for {s} {} without sse4_1 feature", .{
7202 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7203 }),
7056 .bit_and, .bit_or, .xor => {},7204 .bit_and, .bit_or, .xor => {},
7057 .max, .min => {}, // TODO: unordered select7205 .max, .min => if (maybe_mask_reg) |mask_reg| if (self.hasFeature(.avx)) {
7206 const rhs_copy_reg = registerAlias(src_mcv.getReg().?, abi_size);
7207
7208 try self.asmRegisterRegisterRegisterImmediate(
7209 if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
7210 .Float => switch (lhs_ty.floatBits(self.target.*)) {
7211 32 => .{ .v_ss, .cmp },
7212 64 => .{ .v_sd, .cmp },
7213 16, 80, 128 => null,
7214 else => unreachable,
7215 },
7216 .Vector => switch (lhs_ty.childType().zigTypeTag()) {
7217 .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
7218 32 => switch (lhs_ty.vectorLen()) {
7219 1 => .{ .v_ss, .cmp },
7220 2...8 => .{ .v_ps, .cmp },
7221 else => null,
7222 },
7223 64 => switch (lhs_ty.vectorLen()) {
7224 1 => .{ .v_sd, .cmp },
7225 2...4 => .{ .v_pd, .cmp },
7226 else => null,
7227 },
7228 16, 80, 128 => null,
7229 else => unreachable,
7230 },
7231 else => unreachable,
7232 },
7233 else => unreachable,
7234 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
7235 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7236 }),
7237 mask_reg,
7238 rhs_copy_reg,
7239 rhs_copy_reg,
7240 Immediate.u(3), // unord
7241 );
7242 try self.asmRegisterRegisterRegisterRegister(
7243 if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
7244 .Float => switch (lhs_ty.floatBits(self.target.*)) {
7245 32 => .{ .v_ps, .blendv },
7246 64 => .{ .v_pd, .blendv },
7247 16, 80, 128 => null,
7248 else => unreachable,
7249 },
7250 .Vector => switch (lhs_ty.childType().zigTypeTag()) {
7251 .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
7252 32 => switch (lhs_ty.vectorLen()) {
7253 1...8 => .{ .v_ps, .blendv },
7254 else => null,
7255 },
7256 64 => switch (lhs_ty.vectorLen()) {
7257 1...4 => .{ .v_pd, .blendv },
7258 else => null,
7259 },
7260 16, 80, 128 => null,
7261 else => unreachable,
7262 },
7263 else => unreachable,
7264 },
7265 else => unreachable,
7266 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
7267 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7268 }),
7269 dst_reg,
7270 dst_reg,
7271 lhs_copy_reg.?,
7272 mask_reg,
7273 );
7274 } else {
7275 const has_blend = self.hasFeature(.sse4_1);
7276 try self.asmRegisterRegisterImmediate(
7277 if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
7278 .Float => switch (lhs_ty.floatBits(self.target.*)) {
7279 32 => .{ ._ss, .cmp },
7280 64 => .{ ._sd, .cmp },
7281 16, 80, 128 => null,
7282 else => unreachable,
7283 },
7284 .Vector => switch (lhs_ty.childType().zigTypeTag()) {
7285 .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
7286 32 => switch (lhs_ty.vectorLen()) {
7287 1 => .{ ._ss, .cmp },
7288 2...4 => .{ ._ps, .cmp },
7289 else => null,
7290 },
7291 64 => switch (lhs_ty.vectorLen()) {
7292 1 => .{ ._sd, .cmp },
7293 2 => .{ ._pd, .cmp },
7294 else => null,
7295 },
7296 16, 80, 128 => null,
7297 else => unreachable,
7298 },
7299 else => unreachable,
7300 },
7301 else => unreachable,
7302 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
7303 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7304 }),
7305 mask_reg,
7306 mask_reg,
7307 Immediate.u(if (has_blend) 3 else 7), // unord, ord
7308 );
7309 if (has_blend) try self.asmRegisterRegisterRegister(
7310 if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
7311 .Float => switch (lhs_ty.floatBits(self.target.*)) {
7312 32 => .{ ._ps, .blendv },
7313 64 => .{ ._pd, .blendv },
7314 16, 80, 128 => null,
7315 else => unreachable,
7316 },
7317 .Vector => switch (lhs_ty.childType().zigTypeTag()) {
7318 .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
7319 32 => switch (lhs_ty.vectorLen()) {
7320 1...4 => .{ ._ps, .blendv },
7321 else => null,
7322 },
7323 64 => switch (lhs_ty.vectorLen()) {
7324 1...2 => .{ ._pd, .blendv },
7325 else => null,
7326 },
7327 16, 80, 128 => null,
7328 else => unreachable,
7329 },
7330 else => unreachable,
7331 },
7332 else => unreachable,
7333 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
7334 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7335 }),
7336 dst_reg,
7337 lhs_copy_reg.?,
7338 mask_reg,
7339 ) else {
7340 try self.asmRegisterRegister(
7341 if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
7342 .Float => switch (lhs_ty.floatBits(self.target.*)) {
7343 32 => .{ ._ps, .@"and" },
7344 64 => .{ ._pd, .@"and" },
7345 16, 80, 128 => null,
7346 else => unreachable,
7347 },
7348 .Vector => switch (lhs_ty.childType().zigTypeTag()) {
7349 .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
7350 32 => switch (lhs_ty.vectorLen()) {
7351 1...4 => .{ ._ps, .@"and" },
7352 else => null,
7353 },
7354 64 => switch (lhs_ty.vectorLen()) {
7355 1...2 => .{ ._pd, .@"and" },
7356 else => null,
7357 },
7358 16, 80, 128 => null,
7359 else => unreachable,
7360 },
7361 else => unreachable,
7362 },
7363 else => unreachable,
7364 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
7365 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7366 }),
7367 dst_reg,
7368 mask_reg,
7369 );
7370 try self.asmRegisterRegister(
7371 if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
7372 .Float => switch (lhs_ty.floatBits(self.target.*)) {
7373 32 => .{ ._ps, .andn },
7374 64 => .{ ._pd, .andn },
7375 16, 80, 128 => null,
7376 else => unreachable,
7377 },
7378 .Vector => switch (lhs_ty.childType().zigTypeTag()) {
7379 .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
7380 32 => switch (lhs_ty.vectorLen()) {
7381 1...4 => .{ ._ps, .andn },
7382 else => null,
7383 },
7384 64 => switch (lhs_ty.vectorLen()) {
7385 1...2 => .{ ._pd, .andn },
7386 else => null,
7387 },
7388 16, 80, 128 => null,
7389 else => unreachable,
7390 },
7391 else => unreachable,
7392 },
7393 else => unreachable,
7394 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
7395 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7396 }),
7397 mask_reg,
7398 lhs_copy_reg.?,
7399 );
7400 try self.asmRegisterRegister(
7401 if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag()) {
7402 .Float => switch (lhs_ty.floatBits(self.target.*)) {
7403 32 => .{ ._ps, .@"or" },
7404 64 => .{ ._pd, .@"or" },
7405 16, 80, 128 => null,
7406 else => unreachable,
7407 },
7408 .Vector => switch (lhs_ty.childType().zigTypeTag()) {
7409 .Float => switch (lhs_ty.childType().floatBits(self.target.*)) {
7410 32 => switch (lhs_ty.vectorLen()) {
7411 1...4 => .{ ._ps, .@"or" },
7412 else => null,
7413 },
7414 64 => switch (lhs_ty.vectorLen()) {
7415 1...2 => .{ ._pd, .@"or" },
7416 else => null,
7417 },
7418 16, 80, 128 => null,
7419 else => unreachable,
7420 },
7421 else => unreachable,
7422 },
7423 else => unreachable,
7424 })) |tag| tag else return self.fail("TODO implement genBinOp for {s} {}", .{
7425 @tagName(air_tag), lhs_ty.fmt(self.bin_file.options.module.?),
7426 }),
7427 dst_reg,
7428 mask_reg,
7429 );
7430 }
7431 },
7058 else => unreachable,7432 else => unreachable,
7059 }7433 }
7434
7060 return dst_mcv;7435 return dst_mcv;
7061}7436}
70627437
...@@ -7882,24 +8257,105 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {...@@ -7882,24 +8257,105 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
7882 const result = MCValue{8257 const result = MCValue{
7883 .eflags = switch (ty.zigTypeTag()) {8258 .eflags = switch (ty.zigTypeTag()) {
7884 else => result: {8259 else => result: {
7885 var flipped = false;8260 const abi_size = @intCast(u16, ty.abiSize(self.target.*));
7886 const dst_mcv: MCValue = if (lhs_mcv.isRegister() or lhs_mcv.isMemory())8261 const may_flip: enum {
7887 lhs_mcv8262 may_flip,
7888 else if (rhs_mcv.isRegister() or rhs_mcv.isMemory()) dst: {8263 must_flip,
7889 flipped = true;8264 must_not_flip,
7890 break :dst rhs_mcv;8265 } = if (abi_size > 8) switch (op) {
7891 } else .{ .register = try self.copyToTmpRegister(ty, lhs_mcv) };8266 .lt, .gte => .must_not_flip,
7892 const dst_lock = switch (dst_mcv) {8267 .lte, .gt => .must_flip,
7893 .register => |reg| self.register_manager.lockReg(reg),8268 .eq, .neq => .may_flip,
7894 else => null,8269 } else .may_flip;
8270
8271 const flipped = switch (may_flip) {
8272 .may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isMemory(),
8273 .must_flip => true,
8274 .must_not_flip => false,
8275 };
8276 const unmat_dst_mcv = if (flipped) rhs_mcv else lhs_mcv;
8277 const dst_mcv = if (unmat_dst_mcv.isRegister() or
8278 (abi_size <= 8 and unmat_dst_mcv.isMemory())) unmat_dst_mcv else dst: {
8279 const dst_mcv = try self.allocTempRegOrMem(ty, true);
8280 try self.genCopy(ty, dst_mcv, unmat_dst_mcv);
8281 break :dst dst_mcv;
7895 };8282 };
8283 const dst_lock =
8284 if (dst_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
7896 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);8285 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
8286
7897 const src_mcv = if (flipped) lhs_mcv else rhs_mcv;8287 const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
8288 const src_lock =
8289 if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
8290 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
78988291
7899 try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, src_mcv);
7900 break :result Condition.fromCompareOperator(8292 break :result Condition.fromCompareOperator(
7901 if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned,8293 if (ty.isAbiInt()) ty.intInfo(self.target.*).signedness else .unsigned,
7902 if (flipped) op.reverse() else op,8294 result_op: {
8295 const flipped_op = if (flipped) op.reverse() else op;
8296 if (abi_size > 8) switch (flipped_op) {
8297 .lt, .gte => {},
8298 .lte, .gt => unreachable,
8299 .eq, .neq => {
8300 const dst_addr_mcv: MCValue = switch (dst_mcv) {
8301 .memory, .indirect, .load_frame => dst_mcv.address(),
8302 else => .{ .register = try self.copyToTmpRegister(
8303 Type.usize,
8304 dst_mcv.address(),
8305 ) },
8306 };
8307 const dst_addr_lock = if (dst_addr_mcv.getReg()) |reg|
8308 self.register_manager.lockReg(reg)
8309 else
8310 null;
8311 defer if (dst_addr_lock) |lock| self.register_manager.unlockReg(lock);
8312
8313 const src_addr_mcv: MCValue = switch (src_mcv) {
8314 .memory, .indirect, .load_frame => src_mcv.address(),
8315 else => .{ .register = try self.copyToTmpRegister(
8316 Type.usize,
8317 src_mcv.address(),
8318 ) },
8319 };
8320 const src_addr_lock = if (src_addr_mcv.getReg()) |reg|
8321 self.register_manager.lockReg(reg)
8322 else
8323 null;
8324 defer if (src_addr_lock) |lock| self.register_manager.unlockReg(lock);
8325
8326 const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp);
8327 const acc_reg = regs[0].to64();
8328 const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
8329 defer for (locks) |lock| self.register_manager.unlockReg(lock);
8330
8331 const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable;
8332 var limb_i: u16 = 0;
8333 while (limb_i < limbs_len) : (limb_i += 1) {
8334 const tmp_reg = regs[@min(limb_i, 1)].to64();
8335 try self.genSetReg(
8336 tmp_reg,
8337 Type.usize,
8338 dst_addr_mcv.offset(limb_i * 8).deref(),
8339 );
8340 try self.genBinOpMir(
8341 .{ ._, .xor },
8342 Type.usize,
8343 .{ .register = tmp_reg },
8344 src_addr_mcv.offset(limb_i * 8).deref(),
8345 );
8346 if (limb_i > 0) try self.asmRegisterRegister(
8347 .{ ._, .@"or" },
8348 acc_reg,
8349 tmp_reg,
8350 );
8351 }
8352 try self.asmRegisterRegister(.{ ._, .@"test" }, acc_reg, acc_reg);
8353 break :result_op flipped_op;
8354 },
8355 };
8356 try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, src_mcv);
8357 break :result_op flipped_op;
8358 },
7903 );8359 );
7904 },8360 },
7905 .Float => result: {8361 .Float => result: {
...@@ -9282,7 +9738,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr...@@ -9282,7 +9738,7 @@ fn genSetReg(self: *Self, dst_reg: Register, ty: Type, src_mcv: MCValue) InnerEr
9282 17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null,9738 17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null,
9283 else => null,9739 else => null,
9284 },9740 },
9285 .Float => switch (ty.floatBits(self.target.*)) {9741 .Float => switch (ty.scalarType().floatBits(self.target.*)) {
9286 16, 128 => switch (abi_size) {9742 16, 128 => switch (abi_size) {
9287 2...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },9743 2...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov },
9288 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },9744 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov },
...@@ -9597,63 +10053,6 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal...@@ -9597,63 +10053,6 @@ fn genSetMem(self: *Self, base: Memory.Base, disp: i32, ty: Type, src_mcv: MCVal
9597 }10053 }
9598}10054}
959910055
9600/// Like `genInlineMemcpy` but copies value from a register to an address via dereferencing
9601/// of destination register.
9602/// Boils down to MOV r/m64, r64.
9603fn genInlineMemcpyRegisterRegister(
9604 self: *Self,
9605 ty: Type,
9606 dst_reg: Register,
9607 src_reg: Register,
9608 offset: i32,
9609) InnerError!void {
9610 assert(dst_reg.bitSize() == 64);
9611
9612 const dst_reg_lock = self.register_manager.lockReg(dst_reg);
9613 defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock);
9614
9615 const src_reg_lock = self.register_manager.lockReg(src_reg);
9616 defer if (src_reg_lock) |lock| self.register_manager.unlockReg(lock);
9617
9618 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
9619
9620 if (!math.isPowerOfTwo(abi_size)) {
9621 const tmp_reg = try self.copyToTmpRegister(ty, .{ .register = src_reg });
9622
9623 var next_offset = offset;
9624 var remainder = abi_size;
9625 while (remainder > 0) {
9626 const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder));
9627 try self.asmMemoryRegister(
9628 .{ ._, .mov },
9629 Memory.sib(Memory.PtrSize.fromSize(nearest_power_of_two), .{
9630 .base = dst_reg,
9631 .disp = -next_offset,
9632 }),
9633 registerAlias(tmp_reg, nearest_power_of_two),
9634 );
9635
9636 if (nearest_power_of_two > 1) {
9637 try self.genShiftBinOpMir(.{ ._r, .sh }, ty, .{ .register = tmp_reg }, .{
9638 .immediate = nearest_power_of_two * 8,
9639 });
9640 }
9641
9642 remainder -= nearest_power_of_two;
9643 next_offset -= nearest_power_of_two;
9644 }
9645 } else {
9646 try self.asmMemoryRegister(
9647 switch (src_reg.class()) {
9648 .general_purpose, .segment => .{ ._, .mov },
9649 .sse => .{ ._ss, .mov },
9650 },
9651 Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = dst_reg, .disp = -offset }),
9652 registerAlias(src_reg, abi_size),
9653 );
9654 }
9655}
9656
9657fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {10056fn genInlineMemcpy(self: *Self, dst_ptr: MCValue, src_ptr: MCValue, len: MCValue) InnerError!void {
9658 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });10057 try self.spillRegisters(&.{ .rdi, .rsi, .rcx });
9659 try self.genSetReg(.rdi, Type.usize, dst_ptr);10058 try self.genSetReg(.rdi, Type.usize, dst_ptr);
...@@ -9754,20 +10153,60 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -9754,20 +10153,60 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void {
9754 const result = result: {10153 const result = result: {
9755 const dst_rc = regClassForType(dst_ty);10154 const dst_rc = regClassForType(dst_ty);
9756 const src_rc = regClassForType(src_ty);10155 const src_rc = regClassForType(src_ty);
9757 const operand = try self.resolveInst(ty_op.operand);10156 const src_mcv = try self.resolveInst(ty_op.operand);
9758 if (dst_rc.supersetOf(src_rc) and self.reuseOperand(inst, ty_op.operand, 0, operand))
9759 break :result operand;
976010157
9761 const operand_lock = switch (operand) {10158 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
9762 .register => |reg| self.register_manager.lockReg(reg),10159 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
9763 .register_overflow => |ro| self.register_manager.lockReg(ro.reg),10160
9764 else => null,10161 const dst_mcv = if (dst_rc.supersetOf(src_rc) and
10162 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
10163 src_mcv
10164 else dst: {
10165 const dst_mcv = try self.allocRegOrMem(inst, true);
10166 try self.genCopy(
10167 if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty,
10168 dst_mcv,
10169 src_mcv,
10170 );
10171 break :dst dst_mcv;
9765 };10172 };
9766 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
976710173
9768 const dest = try self.allocRegOrMem(inst, true);10174 const dst_signedness =
9769 try self.genCopy(if (!dest.isMemory() or operand.isMemory()) dst_ty else src_ty, dest, operand);10175 if (dst_ty.isAbiInt()) dst_ty.intInfo(self.target.*).signedness else .unsigned;
9770 break :result dest;10176 const src_signedness =
10177 if (src_ty.isAbiInt()) src_ty.intInfo(self.target.*).signedness else .unsigned;
10178 const abi_size = @intCast(u16, dst_ty.abiSize(self.target.*));
10179 const bit_size = @intCast(u16, dst_ty.bitSize(self.target.*));
10180 const dst_limbs_len = math.divCeil(u16, bit_size, 64) catch unreachable;
10181 if (dst_signedness != src_signedness and abi_size * 8 > bit_size) {
10182 const high_reg = if (dst_mcv.isRegister())
10183 dst_mcv.getReg().?
10184 else
10185 try self.copyToTmpRegister(
10186 Type.usize,
10187 dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
10188 );
10189 const high_lock = self.register_manager.lockReg(high_reg);
10190 defer if (high_lock) |lock| self.register_manager.unlockReg(lock);
10191
10192 var high_pl = Type.Payload.Bits{
10193 .base = .{ .tag = switch (dst_signedness) {
10194 .signed => .int_signed,
10195 .unsigned => .int_unsigned,
10196 } },
10197 .data = bit_size % 64,
10198 };
10199 const high_ty = Type.initPayload(&high_pl.base);
10200
10201 try self.truncateRegister(high_ty, high_reg);
10202 if (!dst_mcv.isRegister()) try self.genCopy(
10203 Type.usize,
10204 dst_mcv.address().offset((dst_limbs_len - 1) * 8).deref(),
10205 .{ .register = high_reg },
10206 );
10207 }
10208
10209 break :result dst_mcv;
9771 };10210 };
9772 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });10211 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
9773}10212}
...@@ -9803,7 +10242,7 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {...@@ -9803,7 +10242,7 @@ fn airIntToFloat(self: *Self, inst: Air.Inst.Index) !void {
9803 if (src_ty.isAbiInt()) src_ty.intInfo(self.target.*).signedness else .unsigned;10242 if (src_ty.isAbiInt()) src_ty.intInfo(self.target.*).signedness else .unsigned;
9804 const dst_ty = self.air.typeOfIndex(inst);10243 const dst_ty = self.air.typeOfIndex(inst);
980510244
9806 const src_size = std.math.divCeil(u32, @max(switch (src_signedness) {10245 const src_size = math.divCeil(u32, @max(switch (src_signedness) {
9807 .signed => src_bits,10246 .signed => src_bits,
9808 .unsigned => src_bits + 1,10247 .unsigned => src_bits + 1,
9809 }, 32), 8) catch unreachable;10248 }, 32), 8) catch unreachable;
...@@ -9856,7 +10295,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {...@@ -9856,7 +10295,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void {
9856 const dst_signedness =10295 const dst_signedness =
9857 if (dst_ty.isAbiInt()) dst_ty.intInfo(self.target.*).signedness else .unsigned;10296 if (dst_ty.isAbiInt()) dst_ty.intInfo(self.target.*).signedness else .unsigned;
985810297
9859 const dst_size = std.math.divCeil(u32, @max(switch (dst_signedness) {10298 const dst_size = math.divCeil(u32, @max(switch (dst_signedness) {
9860 .signed => dst_bits,10299 .signed => dst_bits,
9861 .unsigned => dst_bits + 1,10300 .unsigned => dst_bits + 1,
9862 }, 32), 8) catch unreachable;10301 }, 32), 8) catch unreachable;
...@@ -9912,14 +10351,30 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {...@@ -9912,14 +10351,30 @@ fn airCmpxchg(self: *Self, inst: Air.Inst.Index) !void {
991210351
9913 const exp_mcv = try self.resolveInst(extra.expected_value);10352 const exp_mcv = try self.resolveInst(extra.expected_value);
9914 if (val_abi_size > 8) {10353 if (val_abi_size > 8) {
9915 try self.genSetReg(.rax, Type.usize, exp_mcv);10354 const exp_addr_mcv: MCValue = switch (exp_mcv) {
9916 try self.genSetReg(.rdx, Type.usize, exp_mcv.address().offset(8).deref());10355 .memory, .indirect, .load_frame => exp_mcv.address(),
10356 else => .{ .register = try self.copyToTmpRegister(Type.usize, exp_mcv.address()) },
10357 };
10358 const exp_addr_lock =
10359 if (exp_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
10360 defer if (exp_addr_lock) |lock| self.register_manager.unlockReg(lock);
10361
10362 try self.genSetReg(.rax, Type.usize, exp_addr_mcv.deref());
10363 try self.genSetReg(.rdx, Type.usize, exp_addr_mcv.offset(8).deref());
9917 } else try self.genSetReg(.rax, val_ty, exp_mcv);10364 } else try self.genSetReg(.rax, val_ty, exp_mcv);
991810365
9919 const new_mcv = try self.resolveInst(extra.new_value);10366 const new_mcv = try self.resolveInst(extra.new_value);
9920 const new_reg = if (val_abi_size > 8) new: {10367 const new_reg = if (val_abi_size > 8) new: {
9921 try self.genSetReg(.rbx, Type.usize, new_mcv);10368 const new_addr_mcv: MCValue = switch (new_mcv) {
9922 try self.genSetReg(.rcx, Type.usize, new_mcv.address().offset(8).deref());10369 .memory, .indirect, .load_frame => new_mcv.address(),
10370 else => .{ .register = try self.copyToTmpRegister(Type.usize, new_mcv.address()) },
10371 };
10372 const new_addr_lock =
10373 if (new_addr_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
10374 defer if (new_addr_lock) |lock| self.register_manager.unlockReg(lock);
10375
10376 try self.genSetReg(.rbx, Type.usize, new_addr_mcv.deref());
10377 try self.genSetReg(.rcx, Type.usize, new_addr_mcv.offset(8).deref());
9923 break :new null;10378 break :new null;
9924 } else try self.copyToTmpRegister(val_ty, new_mcv);10379 } else try self.copyToTmpRegister(val_ty, new_mcv);
9925 const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;10380 const new_lock = if (new_reg) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
...@@ -10763,8 +11218,8 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {...@@ -10763,8 +11218,8 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void {
10763}11218}
1076411219
10765fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {11220fn airShuffle(self: *Self, inst: Air.Inst.Index) !void {
10766 const ty_op = self.air.instructions.items(.data)[inst].ty_op;11221 const ty_pl = self.air.instructions.items(.data)[inst].ty_pl;
10767 _ = ty_op;11222 _ = ty_pl;
10768 return self.fail("TODO implement airShuffle for x86_64", .{});11223 return self.fail("TODO implement airShuffle for x86_64", .{});
10769 //return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });11224 //return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
10770}11225}
...@@ -10894,6 +11349,12 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {...@@ -10894,6 +11349,12 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void {
10894 const elem_off = @intCast(i32, elem_size * elem_i);11349 const elem_off = @intCast(i32, elem_size * elem_i);
10895 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);11350 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, mat_elem_mcv);
10896 }11351 }
11352 if (result_ty.sentinel()) |sentinel| try self.genSetMem(
11353 .{ .frame = frame_index },
11354 @intCast(i32, elem_size * elements.len),
11355 elem_ty,
11356 try self.genTypedValue(.{ .ty = elem_ty, .val = sentinel }),
11357 );
10897 break :result .{ .load_frame = .{ .index = frame_index } };11358 break :result .{ .load_frame = .{ .index = frame_index } };
10898 },11359 },
10899 .Vector => return self.fail("TODO implement aggregate_init for vectors", .{}),11360 .Vector => return self.fail("TODO implement aggregate_init for vectors", .{}),
src/arch/x86_64/Encoding.zig+23-13
...@@ -178,7 +178,7 @@ pub fn format(...@@ -178,7 +178,7 @@ pub fn format(
178 try writer.print("+{s} ", .{tag});178 try writer.print("+{s} ", .{tag});
179 },179 },
180 .m, .mi, .m1, .mc, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}),180 .m, .mi, .m1, .mc, .vmi => try writer.print("/{d} ", .{encoding.modRmExt()}),
181 .mr, .rm, .rmi, .mri, .mrc, .rvm, .rvmi, .mvr => try writer.writeAll("/r "),181 .mr, .rm, .rmi, .mri, .mrc, .rm0, .rvm, .rvmr, .rvmi, .mvr => try writer.writeAll("/r "),
182 }182 }
183183
184 switch (encoding.data.op_en) {184 switch (encoding.data.op_en) {
...@@ -202,7 +202,8 @@ pub fn format(...@@ -202,7 +202,8 @@ pub fn format(
202 };202 };
203 try writer.print("{s} ", .{tag});203 try writer.print("{s} ", .{tag});
204 },204 },
205 .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rvm, .mvr => {},205 .rvmr => try writer.writeAll("/is4 "),
206 .np, .fd, .td, .o, .m, .m1, .mc, .mr, .rm, .mrc, .rm0, .rvm, .mvr => {},
206 }207 }
207208
208 try writer.print("{s} ", .{@tagName(encoding.mnemonic)});209 try writer.print("{s} ", .{@tagName(encoding.mnemonic)});
...@@ -262,6 +263,7 @@ pub const Mnemonic = enum {...@@ -262,6 +263,7 @@ pub const Mnemonic = enum {
262 fisttp, fld,263 fisttp, fld,
263 // MMX264 // MMX
264 movd, movq,265 movd, movq,
266 packssdw, packsswb, packuswb,
265 paddb, paddd, paddq, paddsb, paddsw, paddusb, paddusw, paddw,267 paddb, paddd, paddq, paddsb, paddsw, paddusb, paddusw, paddw,
266 pand, pandn, por, pxor,268 pand, pandn, por, pxor,
267 pmulhw, pmullw,269 pmulhw, pmullw,
...@@ -270,7 +272,7 @@ pub const Mnemonic = enum {...@@ -270,7 +272,7 @@ pub const Mnemonic = enum {
270 addps, addss,272 addps, addss,
271 andps,273 andps,
272 andnps,274 andnps,
273 cmpss,275 cmpps, cmpss,
274 cvtpi2ps, cvtps2pi, cvtsi2ss, cvtss2si, cvttps2pi, cvttss2si,276 cvtpi2ps, cvtps2pi, cvtsi2ss, cvtss2si, cvttps2pi, cvttss2si,
275 divps, divss,277 divps, divss,
276 maxps, maxss,278 maxps, maxss,
...@@ -290,7 +292,7 @@ pub const Mnemonic = enum {...@@ -290,7 +292,7 @@ pub const Mnemonic = enum {
290 addpd, addsd,292 addpd, addsd,
291 andpd,293 andpd,
292 andnpd,294 andnpd,
293 //cmpsd,295 cmppd, //cmpsd,
294 cvtdq2pd, cvtdq2ps, cvtpd2dq, cvtpd2pi, cvtpd2ps, cvtpi2pd,296 cvtdq2pd, cvtdq2ps, cvtpd2dq, cvtpd2pi, cvtpd2ps, cvtpi2pd,
295 cvtps2dq, cvtps2pd, cvtsd2si, cvtsd2ss, cvtsi2sd, cvtss2sd,297 cvtps2dq, cvtps2pd, cvtsd2si, cvtsd2ss, cvtsi2sd, cvtss2sd,
296 cvttpd2dq, cvttpd2pi, cvttps2dq, cvttsd2si,298 cvttpd2dq, cvttpd2pi, cvttps2dq, cvttsd2si,
...@@ -315,8 +317,10 @@ pub const Mnemonic = enum {...@@ -315,8 +317,10 @@ pub const Mnemonic = enum {
315 // SSE3317 // SSE3
316 movddup, movshdup, movsldup,318 movddup, movshdup, movsldup,
317 // SSE4.1319 // SSE4.1
320 blendpd, blendps, blendvpd, blendvps,
318 extractps,321 extractps,
319 insertps,322 insertps,
323 packusdw,
320 pextrb, pextrd, pextrq,324 pextrb, pextrd, pextrq,
321 pinsrb, pinsrd, pinsrq,325 pinsrb, pinsrd, pinsrq,
322 pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw,326 pmaxsb, pmaxsd, pmaxud, pmaxuw, pminsb, pminsd, pminud, pminuw,
...@@ -325,7 +329,9 @@ pub const Mnemonic = enum {...@@ -325,7 +329,9 @@ pub const Mnemonic = enum {
325 // AVX329 // AVX
326 vaddpd, vaddps, vaddsd, vaddss,330 vaddpd, vaddps, vaddsd, vaddss,
327 vandnpd, vandnps, vandpd, vandps,331 vandnpd, vandnps, vandpd, vandps,
332 vblendpd, vblendps, vblendvpd, vblendvps,
328 vbroadcastf128, vbroadcastsd, vbroadcastss,333 vbroadcastf128, vbroadcastsd, vbroadcastss,
334 vcmppd, vcmpps, vcmpsd, vcmpss,
329 vcvtdq2pd, vcvtdq2ps, vcvtpd2dq, vcvtpd2ps,335 vcvtdq2pd, vcvtdq2ps, vcvtpd2dq, vcvtpd2ps,
330 vcvtps2dq, vcvtps2pd, vcvtsd2si, vcvtsd2ss,336 vcvtps2dq, vcvtps2pd, vcvtsd2si, vcvtsd2ss,
331 vcvtsi2sd, vcvtsi2ss, vcvtss2sd, vcvtss2si,337 vcvtsi2sd, vcvtsi2ss, vcvtss2sd, vcvtss2si,
...@@ -347,6 +353,7 @@ pub const Mnemonic = enum {...@@ -347,6 +353,7 @@ pub const Mnemonic = enum {
347 vmovupd, vmovups,353 vmovupd, vmovups,
348 vmulpd, vmulps, vmulsd, vmulss,354 vmulpd, vmulps, vmulsd, vmulss,
349 vorpd, vorps,355 vorpd, vorps,
356 vpackssdw, vpacksswb, vpackusdw, vpackuswb,
350 vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw,357 vpaddb, vpaddd, vpaddq, vpaddsb, vpaddsw, vpaddusb, vpaddusw, vpaddw,
351 vpand, vpandn,358 vpand, vpandn,
352 vpextrb, vpextrd, vpextrq, vpextrw,359 vpextrb, vpextrd, vpextrq, vpextrw,
...@@ -385,7 +392,7 @@ pub const OpEn = enum {...@@ -385,7 +392,7 @@ pub const OpEn = enum {
385 fd, td,392 fd, td,
386 m1, mc, mi, mr, rm,393 m1, mc, mi, mr, rm,
387 rmi, mri, mrc,394 rmi, mri, mrc,
388 vmi, rvm, rvmi, mvr,395 rm0, vmi, rvm, rvmr, rvmi, mvr,
389 // zig fmt: on396 // zig fmt: on
390};397};
391398
...@@ -407,7 +414,7 @@ pub const Op = enum {...@@ -407,7 +414,7 @@ pub const Op = enum {
407 moffs,414 moffs,
408 sreg,415 sreg,
409 st, mm, mm_m64,416 st, mm, mm_m64,
410 xmm, xmm_m32, xmm_m64, xmm_m128,417 xmm0, xmm, xmm_m32, xmm_m64, xmm_m128,
411 ymm, ymm_m256,418 ymm, ymm_m256,
412 // zig fmt: on419 // zig fmt: on
413420
...@@ -436,7 +443,9 @@ pub const Op = enum {...@@ -436,7 +443,9 @@ pub const Op = enum {
436 .segment => .sreg,443 .segment => .sreg,
437 .x87 => .st,444 .x87 => .st,
438 .mmx => .mm,445 .mmx => .mm,
439 .sse => switch (reg.bitSize()) {446 .sse => if (reg == .xmm0)
447 .xmm0
448 else switch (reg.bitSize()) {
440 128 => .xmm,449 128 => .xmm,
441 256 => .ymm,450 256 => .ymm,
442 else => unreachable,451 else => unreachable,
...@@ -494,7 +503,7 @@ pub const Op = enum {...@@ -494,7 +503,7 @@ pub const Op = enum {
494 .eax, .r32, .rm32, .r32_m16 => unreachable,503 .eax, .r32, .rm32, .r32_m16 => unreachable,
495 .rax, .r64, .rm64, .r64_m16 => unreachable,504 .rax, .r64, .rm64, .r64_m16 => unreachable,
496 .st, .mm, .mm_m64 => unreachable,505 .st, .mm, .mm_m64 => unreachable,
497 .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,506 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => unreachable,
498 .ymm, .ymm_m256 => unreachable,507 .ymm, .ymm_m256 => unreachable,
499 .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,508 .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
500 .unity => 1,509 .unity => 1,
...@@ -516,7 +525,7 @@ pub const Op = enum {...@@ -516,7 +525,7 @@ pub const Op = enum {
516 .eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,525 .eax, .r32, .rm32, .r32_m8, .r32_m16 => 32,
517 .rax, .r64, .rm64, .r64_m16, .mm, .mm_m64 => 64,526 .rax, .r64, .rm64, .r64_m16, .mm, .mm_m64 => 64,
518 .st => 80,527 .st => 80,
519 .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,528 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => 128,
520 .ymm, .ymm_m256 => 256,529 .ymm, .ymm_m256 => 256,
521 };530 };
522 }531 }
...@@ -526,7 +535,8 @@ pub const Op = enum {...@@ -526,7 +535,8 @@ pub const Op = enum {
526 .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,535 .none, .o16, .o32, .o64, .moffs, .m, .sreg => unreachable,
527 .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,536 .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
528 .rel8, .rel16, .rel32 => unreachable,537 .rel8, .rel16, .rel32 => unreachable,
529 .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64, .st, .mm, .xmm, .ymm => unreachable,538 .al, .cl, .r8, .ax, .r16, .eax, .r32, .rax, .r64 => unreachable,
539 .st, .mm, .xmm0, .xmm, .ymm => unreachable,
530 .m8, .rm8, .r32_m8 => 8,540 .m8, .rm8, .r32_m8 => 8,
531 .m16, .rm16, .r32_m16, .r64_m16 => 16,541 .m16, .rm16, .r32_m16, .r64_m16 => 16,
532 .m32, .rm32, .xmm_m32 => 32,542 .m32, .rm32, .xmm_m32 => 32,
...@@ -558,7 +568,7 @@ pub const Op = enum {...@@ -558,7 +568,7 @@ pub const Op = enum {
558 .rm8, .rm16, .rm32, .rm64,568 .rm8, .rm16, .rm32, .rm64,
559 .r32_m8, .r32_m16, .r64_m16,569 .r32_m8, .r32_m16, .r64_m16,
560 .st, .mm, .mm_m64,570 .st, .mm, .mm_m64,
561 .xmm, .xmm_m32, .xmm_m64, .xmm_m128,571 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128,
562 .ymm, .ymm_m256,572 .ymm, .ymm_m256,
563 => true,573 => true,
564 else => false,574 else => false,
...@@ -612,7 +622,7 @@ pub const Op = enum {...@@ -612,7 +622,7 @@ pub const Op = enum {
612 .sreg => .segment,622 .sreg => .segment,
613 .st => .x87,623 .st => .x87,
614 .mm, .mm_m64 => .mmx,624 .mm, .mm_m64 => .mmx,
615 .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,625 .xmm0, .xmm, .xmm_m32, .xmm_m64, .xmm_m128 => .sse,
616 .ymm, .ymm_m256 => .sse,626 .ymm, .ymm_m256 => .sse,
617 };627 };
618 }628 }
...@@ -629,7 +639,7 @@ pub const Op = enum {...@@ -629,7 +639,7 @@ pub const Op = enum {
629 else => {639 else => {
630 if (op.isRegister() and target.isRegister()) {640 if (op.isRegister() and target.isRegister()) {
631 return switch (target) {641 return switch (target) {
632 .cl, .al, .ax, .eax, .rax => op == target,642 .cl, .al, .ax, .eax, .rax, .xmm0 => op == target,
633 else => op.class() == target.class() and op.regBitSize() == target.regBitSize(),643 else => op.class() == target.class() and op.regBitSize() == target.regBitSize(),
634 };644 };
635 }645 }
src/arch/x86_64/Lower.zig+7
...@@ -377,6 +377,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -377,6 +377,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
377 .r => inst.data.r.fixes,377 .r => inst.data.r.fixes,
378 .rr => inst.data.rr.fixes,378 .rr => inst.data.rr.fixes,
379 .rrr => inst.data.rrr.fixes,379 .rrr => inst.data.rrr.fixes,
380 .rrrr => inst.data.rrrr.fixes,
380 .rrri => inst.data.rrri.fixes,381 .rrri => inst.data.rrri.fixes,
381 .rri_s, .rri_u => inst.data.rri.fixes,382 .rri_s, .rri_u => inst.data.rri.fixes,
382 .ri_s, .ri_u => inst.data.ri.fixes,383 .ri_s, .ri_u => inst.data.ri.fixes,
...@@ -430,6 +431,12 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {...@@ -430,6 +431,12 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
430 .{ .reg = inst.data.rrr.r2 },431 .{ .reg = inst.data.rrr.r2 },
431 .{ .reg = inst.data.rrr.r3 },432 .{ .reg = inst.data.rrr.r3 },
432 },433 },
434 .rrrr => &.{
435 .{ .reg = inst.data.rrrr.r1 },
436 .{ .reg = inst.data.rrrr.r2 },
437 .{ .reg = inst.data.rrrr.r3 },
438 .{ .reg = inst.data.rrrr.r4 },
439 },
433 .rrri => &.{440 .rrri => &.{
434 .{ .reg = inst.data.rrri.r1 },441 .{ .reg = inst.data.rrri.r1 },
435 .{ .reg = inst.data.rrri.r2 },442 .{ .reg = inst.data.rrri.r2 },
src/arch/x86_64/Mir.zig+28
...@@ -446,6 +446,12 @@ pub const Inst = struct {...@@ -446,6 +446,12 @@ pub const Inst = struct {
446 /// Bitwise logical xor of packed double-precision floating-point values446 /// Bitwise logical xor of packed double-precision floating-point values
447 xor,447 xor,
448448
449 /// Pack with signed saturation
450 ackssw,
451 /// Pack with signed saturation
452 ackssd,
453 /// Pack with unsigned saturation
454 ackusw,
449 /// Add packed signed integers with signed saturation455 /// Add packed signed integers with signed saturation
450 adds,456 adds,
451 /// Add packed unsigned integers with unsigned saturation457 /// Add packed unsigned integers with unsigned saturation
...@@ -596,6 +602,18 @@ pub const Inst = struct {...@@ -596,6 +602,18 @@ pub const Inst = struct {
596 /// Replicate single floating-point values602 /// Replicate single floating-point values
597 movsldup,603 movsldup,
598604
605 /// Pack with unsigned saturation
606 ackusd,
607 /// Blend packed single-precision floating-point values
608 /// Blend scalar single-precision floating-point values
609 /// Blend packed double-precision floating-point values
610 /// Blend scalar double-precision floating-point values
611 blend,
612 /// Variable blend packed single-precision floating-point values
613 /// Variable blend scalar single-precision floating-point values
614 /// Variable blend packed double-precision floating-point values
615 /// Variable blend scalar double-precision floating-point values
616 blendv,
599 /// Extract packed floating-point values617 /// Extract packed floating-point values
600 extract,618 extract,
601 /// Insert scalar single-precision floating-point value619 /// Insert scalar single-precision floating-point value
...@@ -651,6 +669,9 @@ pub const Inst = struct {...@@ -651,6 +669,9 @@ pub const Inst = struct {
651 /// Register, register, register operands.669 /// Register, register, register operands.
652 /// Uses `rrr` payload.670 /// Uses `rrr` payload.
653 rrr,671 rrr,
672 /// Register, register, register, register operands.
673 /// Uses `rrrr` payload.
674 rrrr,
654 /// Register, register, register, immediate (byte) operands.675 /// Register, register, register, immediate (byte) operands.
655 /// Uses `rrri` payload.676 /// Uses `rrri` payload.
656 rrri,677 rrri,
...@@ -870,6 +891,13 @@ pub const Inst = struct {...@@ -870,6 +891,13 @@ pub const Inst = struct {
870 r2: Register,891 r2: Register,
871 r3: Register,892 r3: Register,
872 },893 },
894 rrrr: struct {
895 fixes: Fixes = ._,
896 r1: Register,
897 r2: Register,
898 r3: Register,
899 r4: Register,
900 },
873 rrri: struct {901 rrri: struct {
874 fixes: Fixes = ._,902 fixes: Fixes = ._,
875 r1: Register,903 r1: Register,
src/arch/x86_64/encoder.zig+16-15
...@@ -226,8 +226,8 @@ pub const Instruction = struct {...@@ -226,8 +226,8 @@ pub const Instruction = struct {
226 else => {226 else => {
227 const mem_op = switch (data.op_en) {227 const mem_op = switch (data.op_en) {
228 .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],228 .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
229 .rm, .rmi, .vmi => inst.ops[1],229 .rm, .rmi, .rm0, .vmi => inst.ops[1],
230 .rvm, .rvmi => inst.ops[2],230 .rvm, .rvmr, .rvmi => inst.ops[2],
231 else => unreachable,231 else => unreachable,
232 };232 };
233 switch (mem_op) {233 switch (mem_op) {
...@@ -235,7 +235,7 @@ pub const Instruction = struct {...@@ -235,7 +235,7 @@ pub const Instruction = struct {
235 const rm = switch (data.op_en) {235 const rm = switch (data.op_en) {
236 .m, .mi, .m1, .mc, .vmi => enc.modRmExt(),236 .m, .mi, .m1, .mc, .vmi => enc.modRmExt(),
237 .mr, .mri, .mrc => inst.ops[1].reg.lowEnc(),237 .mr, .mri, .mrc => inst.ops[1].reg.lowEnc(),
238 .rm, .rmi, .rvm, .rvmi => inst.ops[0].reg.lowEnc(),238 .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi => inst.ops[0].reg.lowEnc(),
239 .mvr => inst.ops[2].reg.lowEnc(),239 .mvr => inst.ops[2].reg.lowEnc(),
240 else => unreachable,240 else => unreachable,
241 };241 };
...@@ -245,7 +245,7 @@ pub const Instruction = struct {...@@ -245,7 +245,7 @@ pub const Instruction = struct {
245 const op = switch (data.op_en) {245 const op = switch (data.op_en) {
246 .m, .mi, .m1, .mc, .vmi => .none,246 .m, .mi, .m1, .mc, .vmi => .none,
247 .mr, .mri, .mrc => inst.ops[1],247 .mr, .mri, .mrc => inst.ops[1],
248 .rm, .rmi, .rvm, .rvmi => inst.ops[0],248 .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi => inst.ops[0],
249 .mvr => inst.ops[2],249 .mvr => inst.ops[2],
250 else => unreachable,250 else => unreachable,
251 };251 };
...@@ -257,6 +257,7 @@ pub const Instruction = struct {...@@ -257,6 +257,7 @@ pub const Instruction = struct {
257 switch (data.op_en) {257 switch (data.op_en) {
258 .mi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder),258 .mi => try encodeImm(inst.ops[1].imm, data.ops[1], encoder),
259 .rmi, .mri, .vmi => try encodeImm(inst.ops[2].imm, data.ops[2], encoder),259 .rmi, .mri, .vmi => try encodeImm(inst.ops[2].imm, data.ops[2], encoder),
260 .rvmr => try encoder.imm8(@as(u8, inst.ops[3].reg.enc()) << 4),
260 .rvmi => try encodeImm(inst.ops[3].imm, data.ops[3], encoder),261 .rvmi => try encodeImm(inst.ops[3].imm, data.ops[3], encoder),
261 else => {},262 else => {},
262 }263 }
...@@ -298,7 +299,7 @@ pub const Instruction = struct {...@@ -298,7 +299,7 @@ pub const Instruction = struct {
298 .i, .zi, .o, .oi, .d, .np => null,299 .i, .zi, .o, .oi, .d, .np => null,
299 .fd => inst.ops[1].mem.base().reg,300 .fd => inst.ops[1].mem.base().reg,
300 .td => inst.ops[0].mem.base().reg,301 .td => inst.ops[0].mem.base().reg,
301 .rm, .rmi => if (inst.ops[1].isSegmentRegister())302 .rm, .rmi, .rm0 => if (inst.ops[1].isSegmentRegister())
302 switch (inst.ops[1]) {303 switch (inst.ops[1]) {
303 .reg => |reg| reg,304 .reg => |reg| reg,
304 .mem => |mem| mem.base().reg,305 .mem => |mem| mem.base().reg,
...@@ -314,7 +315,7 @@ pub const Instruction = struct {...@@ -314,7 +315,7 @@ pub const Instruction = struct {
314 }315 }
315 else316 else
316 null,317 null,
317 .vmi, .rvm, .rvmi, .mvr => unreachable,318 .vmi, .rvm, .rvmr, .rvmi, .mvr => unreachable,
318 };319 };
319 if (segment_override) |seg| {320 if (segment_override) |seg| {
320 legacy.setSegmentOverride(seg);321 legacy.setSegmentOverride(seg);
...@@ -333,23 +334,23 @@ pub const Instruction = struct {...@@ -333,23 +334,23 @@ pub const Instruction = struct {
333 switch (op_en) {334 switch (op_en) {
334 .np, .i, .zi, .fd, .td, .d => {},335 .np, .i, .zi, .fd, .td, .d => {},
335 .o, .oi => rex.b = inst.ops[0].reg.isExtended(),336 .o, .oi => rex.b = inst.ops[0].reg.isExtended(),
336 .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc => {337 .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0 => {
337 const r_op = switch (op_en) {338 const r_op = switch (op_en) {
338 .rm, .rmi => inst.ops[0],339 .rm, .rmi, .rm0 => inst.ops[0],
339 .mr, .mri, .mrc => inst.ops[1],340 .mr, .mri, .mrc => inst.ops[1],
340 else => .none,341 else => .none,
341 };342 };
342 rex.r = r_op.isBaseExtended();343 rex.r = r_op.isBaseExtended();
343344
344 const b_x_op = switch (op_en) {345 const b_x_op = switch (op_en) {
345 .rm, .rmi => inst.ops[1],346 .rm, .rmi, .rm0 => inst.ops[1],
346 .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],347 .m, .mi, .m1, .mc, .mr, .mri, .mrc => inst.ops[0],
347 else => unreachable,348 else => unreachable,
348 };349 };
349 rex.b = b_x_op.isBaseExtended();350 rex.b = b_x_op.isBaseExtended();
350 rex.x = b_x_op.isIndexExtended();351 rex.x = b_x_op.isIndexExtended();
351 },352 },
352 .vmi, .rvm, .rvmi, .mvr => unreachable,353 .vmi, .rvm, .rvmr, .rvmi, .mvr => unreachable,
353 }354 }
354355
355 try encoder.rex(rex);356 try encoder.rex(rex);
...@@ -367,9 +368,9 @@ pub const Instruction = struct {...@@ -367,9 +368,9 @@ pub const Instruction = struct {
367 switch (op_en) {368 switch (op_en) {
368 .np, .i, .zi, .fd, .td, .d => {},369 .np, .i, .zi, .fd, .td, .d => {},
369 .o, .oi => vex.b = inst.ops[0].reg.isExtended(),370 .o, .oi => vex.b = inst.ops[0].reg.isExtended(),
370 .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .vmi, .rvm, .rvmi, .mvr => {371 .m, .mi, .m1, .mc, .mr, .rm, .rmi, .mri, .mrc, .rm0, .vmi, .rvm, .rvmr, .rvmi, .mvr => {
371 const r_op = switch (op_en) {372 const r_op = switch (op_en) {
372 .rm, .rmi, .rvm, .rvmi => inst.ops[0],373 .rm, .rmi, .rm0, .rvm, .rvmr, .rvmi => inst.ops[0],
373 .mr, .mri, .mrc => inst.ops[1],374 .mr, .mri, .mrc => inst.ops[1],
374 .mvr => inst.ops[2],375 .mvr => inst.ops[2],
375 .m, .mi, .m1, .mc, .vmi => .none,376 .m, .mi, .m1, .mc, .vmi => .none,
...@@ -378,9 +379,9 @@ pub const Instruction = struct {...@@ -378,9 +379,9 @@ pub const Instruction = struct {
378 vex.r = r_op.isBaseExtended();379 vex.r = r_op.isBaseExtended();
379380
380 const b_x_op = switch (op_en) {381 const b_x_op = switch (op_en) {
381 .rm, .rmi, .vmi => inst.ops[1],382 .rm, .rmi, .rm0, .vmi => inst.ops[1],
382 .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],383 .m, .mi, .m1, .mc, .mr, .mri, .mrc, .mvr => inst.ops[0],
383 .rvm, .rvmi => inst.ops[2],384 .rvm, .rvmr, .rvmi => inst.ops[2],
384 else => unreachable,385 else => unreachable,
385 };386 };
386 vex.b = b_x_op.isBaseExtended();387 vex.b = b_x_op.isBaseExtended();
...@@ -408,7 +409,7 @@ pub const Instruction = struct {...@@ -408,7 +409,7 @@ pub const Instruction = struct {
408 switch (op_en) {409 switch (op_en) {
409 else => {},410 else => {},
410 .vmi => vex.v = inst.ops[0].reg,411 .vmi => vex.v = inst.ops[0].reg,
411 .rvm, .rvmi => vex.v = inst.ops[1].reg,412 .rvm, .rvmr, .rvmi => vex.v = inst.ops[1].reg,
412 }413 }
413414
414 try encoder.vex(vex);415 try encoder.vex(vex);
src/arch/x86_64/encodings.zig+55
...@@ -846,6 +846,8 @@ pub const table = [_]Entry{...@@ -846,6 +846,8 @@ pub const table = [_]Entry{
846846
847 .{ .andps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .none, .sse },847 .{ .andps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .none, .sse },
848848
849 .{ .cmpps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x0f, 0xc2 }, 0, .none, .sse },
850
849 .{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .none, .sse },851 .{ .cmpss, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .none, .sse },
850852
851 .{ .cvtpi2ps, .rm, &.{ .xmm, .mm_m64 }, &.{ 0x0f, 0x2a }, 0, .none, .sse },853 .{ .cvtpi2ps, .rm, &.{ .xmm, .mm_m64 }, &.{ 0x0f, 0x2a }, 0, .none, .sse },
...@@ -917,6 +919,8 @@ pub const table = [_]Entry{...@@ -917,6 +919,8 @@ pub const table = [_]Entry{
917919
918 .{ .andpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x54 }, 0, .none, .sse2 },920 .{ .andpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x54 }, 0, .none, .sse2 },
919921
922 .{ .cmppd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0xc2 }, 0, .none, .sse2 },
923
920 .{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .none, .sse2 },924 .{ .cmpsd, .rmi, &.{ .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .none, .sse2 },
921925
922 .{ .cvtdq2pd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0xe6 }, 0, .none, .sse2 },926 .{ .cvtdq2pd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0xe6 }, 0, .none, .sse2 },
...@@ -992,6 +996,11 @@ pub const table = [_]Entry{...@@ -992,6 +996,11 @@ pub const table = [_]Entry{
992996
993 .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 },997 .{ .orpd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x56 }, 0, .none, .sse2 },
994998
999 .{ .packsswb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x63 }, 0, .none, .sse2 },
1000 .{ .packssdw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6b }, 0, .none, .sse2 },
1001
1002 .{ .packuswb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x67 }, 0, .none, .sse2 },
1003
995 .{ .paddb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfc }, 0, .none, .sse2 },1004 .{ .paddb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfc }, 0, .none, .sse2 },
996 .{ .paddw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfd }, 0, .none, .sse2 },1005 .{ .paddw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfd }, 0, .none, .sse2 },
997 .{ .paddd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfe }, 0, .none, .sse2 },1006 .{ .paddd, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfe }, 0, .none, .sse2 },
...@@ -1085,10 +1094,20 @@ pub const table = [_]Entry{...@@ -1085,10 +1094,20 @@ pub const table = [_]Entry{
1085 .{ .movsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 },1094 .{ .movsldup, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0x12 }, 0, .none, .sse3 },
10861095
1087 // SSE4.11096 // SSE4.1
1097 .{ .blendpd, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .none, .sse4_1 },
1098
1099 .{ .blendps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0c }, 0, .none, .sse4_1 },
1100
1101 .{ .blendvpd, .rm0, &.{ .xmm, .xmm_m128, .xmm0 }, &.{ 0x66, 0x0f, 0x38, 0x15 }, 0, .none, .sse4_1 },
1102
1103 .{ .blendvps, .rm0, &.{ .xmm, .xmm_m128, .xmm0 }, &.{ 0x66, 0x0f, 0x38, 0x14 }, 0, .none, .sse4_1 },
1104
1088 .{ .extractps, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x17 }, 0, .none, .sse4_1 },1105 .{ .extractps, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x17 }, 0, .none, .sse4_1 },
10891106
1090 .{ .insertps, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x21 }, 0, .none, .sse4_1 },1107 .{ .insertps, .rmi, &.{ .xmm, .xmm_m32, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x21 }, 0, .none, .sse4_1 },
10911108
1109 .{ .packusdw, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .none, .sse4_1 },
1110
1092 .{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },1111 .{ .pextrb, .mri, &.{ .r32_m8, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x14 }, 0, .none, .sse4_1 },
1093 .{ .pextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .none, .sse4_1 },1112 .{ .pextrd, .mri, &.{ .rm32, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .none, .sse4_1 },
1094 .{ .pextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .long, .sse4_1 },1113 .{ .pextrq, .mri, &.{ .rm64, .xmm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x16 }, 0, .long, .sse4_1 },
...@@ -1146,11 +1165,33 @@ pub const table = [_]Entry{...@@ -1146,11 +1165,33 @@ pub const table = [_]Entry{
1146 .{ .vandps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .vex_128_wig, .avx },1165 .{ .vandps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x54 }, 0, .vex_128_wig, .avx },
1147 .{ .vandps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x54 }, 0, .vex_256_wig, .avx },1166 .{ .vandps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x54 }, 0, .vex_256_wig, .avx },
11481167
1168 .{ .vblendpd, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .vex_128_wig, .avx },
1169 .{ .vblendpd, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0d }, 0, .vex_256_wig, .avx },
1170
1171 .{ .vblendps, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0c }, 0, .vex_128_wig, .avx },
1172 .{ .vblendps, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x0c }, 0, .vex_256_wig, .avx },
1173
1174 .{ .vblendvpd, .rvmr, &.{ .xmm, .xmm, .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x3a, 0x4b }, 0, .vex_128_w0, .avx },
1175 .{ .vblendvpd, .rvmr, &.{ .ymm, .ymm, .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x3a, 0x4b }, 0, .vex_256_w0, .avx },
1176
1177 .{ .vblendvps, .rvmr, &.{ .xmm, .xmm, .xmm_m128, .xmm }, &.{ 0x66, 0x0f, 0x3a, 0x4a }, 0, .vex_128_w0, .avx },
1178 .{ .vblendvps, .rvmr, &.{ .ymm, .ymm, .ymm_m256, .ymm }, &.{ 0x66, 0x0f, 0x3a, 0x4a }, 0, .vex_256_w0, .avx },
1179
1149 .{ .vbroadcastss, .rm, &.{ .xmm, .m32 }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_128_w0, .avx },1180 .{ .vbroadcastss, .rm, &.{ .xmm, .m32 }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_128_w0, .avx },
1150 .{ .vbroadcastss, .rm, &.{ .ymm, .m32 }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx },1181 .{ .vbroadcastss, .rm, &.{ .ymm, .m32 }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx },
1151 .{ .vbroadcastsd, .rm, &.{ .ymm, .m64 }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx },1182 .{ .vbroadcastsd, .rm, &.{ .ymm, .m64 }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx },
1152 .{ .vbroadcastf128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x1a }, 0, .vex_256_w0, .avx },1183 .{ .vbroadcastf128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x1a }, 0, .vex_256_w0, .avx },
11531184
1185 .{ .vcmppd, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0xc2 }, 0, .vex_128_wig, .avx },
1186 .{ .vcmppd, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x66, 0x0f, 0xc2 }, 0, .vex_256_wig, .avx },
1187
1188 .{ .vcmpps, .rvmi, &.{ .xmm, .xmm, .xmm_m128, .imm8 }, &.{ 0x0f, 0xc2 }, 0, .vex_128_wig, .avx },
1189 .{ .vcmpps, .rvmi, &.{ .ymm, .ymm, .ymm_m256, .imm8 }, &.{ 0x0f, 0xc2 }, 0, .vex_256_wig, .avx },
1190
1191 .{ .vcmpsd, .rvmi, &.{ .xmm, .xmm, .xmm_m64, .imm8 }, &.{ 0xf2, 0x0f, 0xc2 }, 0, .vex_lig_wig, .avx },
1192
1193 .{ .vcmpss, .rvmi, &.{ .xmm, .xmm, .xmm_m32, .imm8 }, &.{ 0xf3, 0x0f, 0xc2 }, 0, .vex_lig_wig, .avx },
1194
1154 .{ .vcvtdq2pd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx },1195 .{ .vcvtdq2pd, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0xf3, 0x0f, 0xe6 }, 0, .vex_128_wig, .avx },
1155 .{ .vcvtdq2pd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx },1196 .{ .vcvtdq2pd, .rm, &.{ .ymm, .xmm_m128 }, &.{ 0xf3, 0x0f, 0xe6 }, 0, .vex_256_wig, .avx },
11561197
...@@ -1312,6 +1353,13 @@ pub const table = [_]Entry{...@@ -1312,6 +1353,13 @@ pub const table = [_]Entry{
1312 .{ .vorps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .vex_128_wig, .avx },1353 .{ .vorps, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x0f, 0x56 }, 0, .vex_128_wig, .avx },
1313 .{ .vorps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x56 }, 0, .vex_256_wig, .avx },1354 .{ .vorps, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x0f, 0x56 }, 0, .vex_256_wig, .avx },
13141355
1356 .{ .vpacksswb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x63 }, 0, .vex_128_wig, .avx },
1357 .{ .vpackssdw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x6b }, 0, .vex_128_wig, .avx },
1358
1359 .{ .vpackusdw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .vex_128_wig, .avx },
1360
1361 .{ .vpackuswb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x67 }, 0, .vex_128_wig, .avx },
1362
1315 .{ .vpaddb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfc }, 0, .vex_128_wig, .avx },1363 .{ .vpaddb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfc }, 0, .vex_128_wig, .avx },
1316 .{ .vpaddw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfd }, 0, .vex_128_wig, .avx },1364 .{ .vpaddw, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfd }, 0, .vex_128_wig, .avx },
1317 .{ .vpaddd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfe }, 0, .vex_128_wig, .avx },1365 .{ .vpaddd, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0xfe }, 0, .vex_128_wig, .avx },
...@@ -1474,6 +1522,13 @@ pub const table = [_]Entry{...@@ -1474,6 +1522,13 @@ pub const table = [_]Entry{
1474 .{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },1522 .{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 },
1475 .{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },1523 .{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 },
14761524
1525 .{ .vpacksswb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x63 }, 0, .vex_256_wig, .avx2 },
1526 .{ .vpackssdw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x6b }, 0, .vex_256_wig, .avx2 },
1527
1528 .{ .vpackusdw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x2b }, 0, .vex_256_wig, .avx2 },
1529
1530 .{ .vpackuswb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x67 }, 0, .vex_256_wig, .avx2 },
1531
1477 .{ .vpaddb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfc }, 0, .vex_256_wig, .avx2 },1532 .{ .vpaddb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfc }, 0, .vex_256_wig, .avx2 },
1478 .{ .vpaddw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfd }, 0, .vex_256_wig, .avx2 },1533 .{ .vpaddw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfd }, 0, .vex_256_wig, .avx2 },
1479 .{ .vpaddd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfe }, 0, .vex_256_wig, .avx2 },1534 .{ .vpaddd, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0xfe }, 0, .vex_256_wig, .avx2 },
src/codegen.zig+22-5
...@@ -747,15 +747,23 @@ pub fn generateSymbol(...@@ -747,15 +747,23 @@ pub fn generateSymbol(
747 .Vector => switch (typed_value.val.tag()) {747 .Vector => switch (typed_value.val.tag()) {
748 .bytes => {748 .bytes => {
749 const bytes = typed_value.val.castTag(.bytes).?.data;749 const bytes = typed_value.val.castTag(.bytes).?.data;
750 const len = @intCast(usize, typed_value.ty.arrayLen());750 const len = math.cast(usize, typed_value.ty.arrayLen()) orelse return error.Overflow;
751 try code.ensureUnusedCapacity(len);751 const padding = math.cast(usize, typed_value.ty.abiSize(target) - len) orelse
752 return error.Overflow;
753 try code.ensureUnusedCapacity(len + padding);
752 code.appendSliceAssumeCapacity(bytes[0..len]);754 code.appendSliceAssumeCapacity(bytes[0..len]);
755 if (padding > 0) try code.writer().writeByteNTimes(0, padding);
753 return Result.ok;756 return Result.ok;
754 },757 },
755 .aggregate => {758 .aggregate => {
756 const elem_vals = typed_value.val.castTag(.aggregate).?.data;759 const elem_vals = typed_value.val.castTag(.aggregate).?.data;
757 const elem_ty = typed_value.ty.elemType();760 const elem_ty = typed_value.ty.elemType();
758 const len = @intCast(usize, typed_value.ty.arrayLen());761 const len = math.cast(usize, typed_value.ty.arrayLen()) orelse return error.Overflow;
762 const padding = math.cast(usize, typed_value.ty.abiSize(target) -
763 (math.divCeil(u64, elem_ty.bitSize(target) * len, 8) catch |err| switch (err) {
764 error.DivisionByZero => unreachable,
765 else => |e| return e,
766 })) orelse return error.Overflow;
759 for (elem_vals[0..len]) |elem_val| {767 for (elem_vals[0..len]) |elem_val| {
760 switch (try generateSymbol(bin_file, src_loc, .{768 switch (try generateSymbol(bin_file, src_loc, .{
761 .ty = elem_ty,769 .ty = elem_ty,
...@@ -765,13 +773,18 @@ pub fn generateSymbol(...@@ -765,13 +773,18 @@ pub fn generateSymbol(
765 .fail => |em| return Result{ .fail = em },773 .fail => |em| return Result{ .fail = em },
766 }774 }
767 }775 }
776 if (padding > 0) try code.writer().writeByteNTimes(0, padding);
768 return Result.ok;777 return Result.ok;
769 },778 },
770 .repeated => {779 .repeated => {
771 const array = typed_value.val.castTag(.repeated).?.data;780 const array = typed_value.val.castTag(.repeated).?.data;
772 const elem_ty = typed_value.ty.childType();781 const elem_ty = typed_value.ty.childType();
773 const len = typed_value.ty.arrayLen();782 const len = typed_value.ty.arrayLen();
774783 const padding = math.cast(usize, typed_value.ty.abiSize(target) -
784 (math.divCeil(u64, elem_ty.bitSize(target) * len, 8) catch |err| switch (err) {
785 error.DivisionByZero => unreachable,
786 else => |e| return e,
787 })) orelse return error.Overflow;
775 var index: u64 = 0;788 var index: u64 = 0;
776 while (index < len) : (index += 1) {789 while (index < len) : (index += 1) {
777 switch (try generateSymbol(bin_file, src_loc, .{790 switch (try generateSymbol(bin_file, src_loc, .{
...@@ -782,13 +795,17 @@ pub fn generateSymbol(...@@ -782,13 +795,17 @@ pub fn generateSymbol(
782 .fail => |em| return Result{ .fail = em },795 .fail => |em| return Result{ .fail = em },
783 }796 }
784 }797 }
798 if (padding > 0) try code.writer().writeByteNTimes(0, padding);
785 return Result.ok;799 return Result.ok;
786 },800 },
787 .str_lit => {801 .str_lit => {
788 const str_lit = typed_value.val.castTag(.str_lit).?.data;802 const str_lit = typed_value.val.castTag(.str_lit).?.data;
789 const bytes = mod.string_literal_bytes.items[str_lit.index..][0..str_lit.len];803 const bytes = mod.string_literal_bytes.items[str_lit.index..][0..str_lit.len];
790 try code.ensureUnusedCapacity(str_lit.len);804 const padding = math.cast(usize, typed_value.ty.abiSize(target) - str_lit.len) orelse
805 return error.Overflow;
806 try code.ensureUnusedCapacity(str_lit.len + padding);
791 code.appendSliceAssumeCapacity(bytes);807 code.appendSliceAssumeCapacity(bytes);
808 if (padding > 0) try code.writer().writeByteNTimes(0, padding);
792 return Result.ok;809 return Result.ok;
793 },810 },
794 else => unreachable,811 else => unreachable,
test/behavior/bitcast.zig-2
...@@ -35,7 +35,6 @@ test "@bitCast iX -> uX (8, 16, 128)" {...@@ -35,7 +35,6 @@ test "@bitCast iX -> uX (8, 16, 128)" {
3535
36test "@bitCast iX -> uX exotic integers" {36test "@bitCast iX -> uX exotic integers" {
37 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;37 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
38 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
39 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;38 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
40 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;39 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
41 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO40 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -82,7 +81,6 @@ fn conv_uN(comptime N: usize, x: std.meta.Int(.unsigned, N)) std.meta.Int(.signe...@@ -82,7 +81,6 @@ fn conv_uN(comptime N: usize, x: std.meta.Int(.unsigned, N)) std.meta.Int(.signe
8281
83test "bitcast uX to bytes" {82test "bitcast uX to bytes" {
84 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;83 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
85 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
86 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;84 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
87 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;85 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
88 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO86 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/bugs/13128.zig-1
...@@ -14,7 +14,6 @@ fn foo(val: U) !void {...@@ -14,7 +14,6 @@ fn foo(val: U) !void {
14test "runtime union init, most-aligned field != largest" {14test "runtime union init, most-aligned field != largest" {
15 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO15 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
16 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO16 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
17 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
18 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO17 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
19 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO18 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
20 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;19 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
test/behavior/eval.zig-2
...@@ -816,7 +816,6 @@ test "array concatenation peer resolves element types - pointer" {...@@ -816,7 +816,6 @@ test "array concatenation peer resolves element types - pointer" {
816816
817test "array concatenation sets the sentinel - value" {817test "array concatenation sets the sentinel - value" {
818 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;818 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
819 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
820 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;819 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
821 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;820 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
822 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO821 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -855,7 +854,6 @@ test "array concatenation sets the sentinel - pointer" {...@@ -855,7 +854,6 @@ test "array concatenation sets the sentinel - pointer" {
855854
856test "array multiplication sets the sentinel - value" {855test "array multiplication sets the sentinel - value" {
857 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;856 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
858 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
859 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;857 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
860 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;858 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
861 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO859 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/floatop.zig-1
...@@ -1145,7 +1145,6 @@ test "nan negation f64" {...@@ -1145,7 +1145,6 @@ test "nan negation f64" {
11451145
1146test "nan negation f128" {1146test "nan negation f128" {
1147 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO1147 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1148 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1149 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1148 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1150 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1149 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1151 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO1150 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/math.zig-2
...@@ -783,7 +783,6 @@ test "basic @mulWithOverflow" {...@@ -783,7 +783,6 @@ test "basic @mulWithOverflow" {
783test "extensive @mulWithOverflow" {783test "extensive @mulWithOverflow" {
784 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO784 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
785 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO785 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
786 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
787 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;786 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
788787
789 {788 {
...@@ -1055,7 +1054,6 @@ test "@subWithOverflow" {...@@ -1055,7 +1054,6 @@ test "@subWithOverflow" {
1055test "@shlWithOverflow" {1054test "@shlWithOverflow" {
1056 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1055 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1057 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1056 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1058 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1059 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;1057 if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest;
10601058
1061 {1059 {
test/behavior/maximum_minimum.zig+4-2
...@@ -24,7 +24,8 @@ test "@max" {...@@ -24,7 +24,8 @@ test "@max" {
2424
25test "@max on vectors" {25test "@max on vectors" {
26 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO26 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
27 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO27 if (builtin.zig_backend == .stage2_x86_64 and
28 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
28 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO29 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
29 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO30 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
30 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO31 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -72,7 +73,8 @@ test "@min" {...@@ -72,7 +73,8 @@ test "@min" {
7273
73test "@min for vectors" {74test "@min for vectors" {
74 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO75 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
75 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO76 if (builtin.zig_backend == .stage2_x86_64 and
77 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest; // TODO
76 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO78 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
77 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO79 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
78 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO80 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/saturating_arithmetic.zig-2
...@@ -5,7 +5,6 @@ const maxInt = std.math.maxInt;...@@ -5,7 +5,6 @@ const maxInt = std.math.maxInt;
5const expect = std.testing.expect;5const expect = std.testing.expect;
66
7test "saturating add" {7test "saturating add" {
8 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
9 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO8 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO9 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
11 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO10 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
...@@ -79,7 +78,6 @@ test "saturating add 128bit" {...@@ -79,7 +78,6 @@ test "saturating add 128bit" {
79}78}
8079
81test "saturating subtraction" {80test "saturating subtraction" {
82 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
83 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO81 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
84 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO82 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
85 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO83 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/truncate.zig-1
...@@ -61,7 +61,6 @@ test "truncate on comptime integer" {...@@ -61,7 +61,6 @@ test "truncate on comptime integer" {
6161
62test "truncate on vectors" {62test "truncate on vectors" {
63 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;63 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
64 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
65 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;64 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
66 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;65 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
67 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO66 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/vector.zig-1
...@@ -1142,7 +1142,6 @@ test "loading the second vector from a slice of vectors" {...@@ -1142,7 +1142,6 @@ test "loading the second vector from a slice of vectors" {
11421142
1143test "array of vectors is copied" {1143test "array of vectors is copied" {
1144 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO1144 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
1145 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
1146 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO1145 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
1147 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1146 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1148 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO1147 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/widening.zig-1
...@@ -5,7 +5,6 @@ const builtin = @import("builtin");...@@ -5,7 +5,6 @@ const builtin = @import("builtin");
5const has_f80_rt = @import("builtin").cpu.arch == .x86_64;5const has_f80_rt = @import("builtin").cpu.arch == .x86_64;
66
7test "integer widening" {7test "integer widening" {
8 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
9 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO8 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
10 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO9 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
11 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO10 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO