| ... | @@ -465,7 +465,7 @@ pub const MCValue = union(enum) { | ... | @@ -465,7 +465,7 @@ pub const MCValue = union(enum) { |
| 465 | } }, | 465 | } }, |
| 466 | } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } }, | 466 | } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } }, |
| 467 | .indirect => |reg_off| .{ | 467 | .indirect => |reg_off| .{ |
| 468 | .base = .{ .reg = reg_off.reg }, | 468 | .base = .{ .reg = registerAlias(reg_off.reg, @divExact(function.target.ptrBitWidth(), 8)) }, |
| 469 | .mod = .{ .rm = .{ | 469 | .mod = .{ .rm = .{ |
| 470 | .size = mod_rm.size, | 470 | .size = mod_rm.size, |
| 471 | .index = mod_rm.index, | 471 | .index = mod_rm.index, |
| ... | @@ -986,6 +986,7 @@ pub fn generate( | ... | @@ -986,6 +986,7 @@ pub fn generate( |
| 986 | .air = function.air, | 986 | .air = function.air, |
| 987 | .lower = .{ | 987 | .lower = .{ |
| 988 | .bin_file = bin_file, | 988 | .bin_file = bin_file, |
| | 989 | .target = function.target, |
| 989 | .allocator = gpa, | 990 | .allocator = gpa, |
| 990 | .mir = mir, | 991 | .mir = mir, |
| 991 | .cc = cc, | 992 | .cc = cc, |
| ... | @@ -1074,6 +1075,7 @@ pub fn generateLazy( | ... | @@ -1074,6 +1075,7 @@ pub fn generateLazy( |
| 1074 | .air = function.air, | 1075 | .air = function.air, |
| 1075 | .lower = .{ | 1076 | .lower = .{ |
| 1076 | .bin_file = bin_file, | 1077 | .bin_file = bin_file, |
| | 1078 | .target = function.target, |
| 1077 | .allocator = gpa, | 1079 | .allocator = gpa, |
| 1078 | .mir = mir, | 1080 | .mir = mir, |
| 1079 | .cc = abi.resolveCallingConvention(.auto, function.target.*), | 1081 | .cc = abi.resolveCallingConvention(.auto, function.target.*), |
| ... | @@ -1154,6 +1156,7 @@ fn formatWipMir( | ... | @@ -1154,6 +1156,7 @@ fn formatWipMir( |
| 1154 | const mod = comp.root_mod; | 1156 | const mod = comp.root_mod; |
| 1155 | var lower: Lower = .{ | 1157 | var lower: Lower = .{ |
| 1156 | .bin_file = data.self.bin_file, | 1158 | .bin_file = data.self.bin_file, |
| | 1159 | .target = data.self.target, |
| 1157 | .allocator = data.self.gpa, | 1160 | .allocator = data.self.gpa, |
| 1158 | .mir = .{ | 1161 | .mir = .{ |
| 1159 | .instructions = data.self.mir_instructions.slice(), | 1162 | .instructions = data.self.mir_instructions.slice(), |
| ... | @@ -2514,7 +2517,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2514,7 +2517,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2514 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; | 2517 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; |
| 2515 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); | 2518 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 2516 | var res: [1]Temp = undefined; | 2519 | var res: [1]Temp = undefined; |
| 2517 | cg.select2(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Mir.Inst.Tag, switch (air_tag) { | 2520 | cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Mir.Inst.Tag, switch (air_tag) { |
| 2518 | else => unreachable, | 2521 | else => unreachable, |
| 2519 | .bit_and => .@"and", | 2522 | .bit_and => .@"and", |
| 2520 | .bit_or => .@"or", | 2523 | .bit_or => .@"or", |
| ... | @@ -2530,7 +2533,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2530,7 +2533,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2530 | }, | 2533 | }, |
| 2531 | .dst_temps = .{.{ .rc = .sse }}, | 2534 | .dst_temps = .{.{ .rc = .sse }}, |
| 2532 | .each = .{ .once = &.{ | 2535 | .each = .{ .once = &.{ |
| 2533 | .{ .vp_, mir_tag, .ydst0, .ysrc0, .ysrc1, .none }, | 2536 | .{ ._, .vp_, mir_tag, .dst0y, .src0y, .src1y, ._ }, |
| 2534 | } }, | 2537 | } }, |
| 2535 | }, .{ | 2538 | }, .{ |
| 2536 | .required_features = .{ .avx, null }, | 2539 | .required_features = .{ .avx, null }, |
| ... | @@ -2541,7 +2544,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2541,7 +2544,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2541 | }, | 2544 | }, |
| 2542 | .dst_temps = .{.{ .rc = .sse }}, | 2545 | .dst_temps = .{.{ .rc = .sse }}, |
| 2543 | .each = .{ .once = &.{ | 2546 | .each = .{ .once = &.{ |
| 2544 | .{ .v_pd, mir_tag, .ydst0, .ysrc0, .ysrc1, .none }, | 2547 | .{ ._, .v_pd, mir_tag, .dst0y, .src0y, .src1y, ._ }, |
| 2545 | } }, | 2548 | } }, |
| 2546 | }, .{ | 2549 | }, .{ |
| 2547 | .required_features = .{ .avx, null }, | 2550 | .required_features = .{ .avx, null }, |
| ... | @@ -2552,7 +2555,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2552,7 +2555,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2552 | }, | 2555 | }, |
| 2553 | .dst_temps = .{.{ .rc = .sse }}, | 2556 | .dst_temps = .{.{ .rc = .sse }}, |
| 2554 | .each = .{ .once = &.{ | 2557 | .each = .{ .once = &.{ |
| 2555 | .{ .vp_, mir_tag, .xdst0, .xsrc0, .xsrc1, .none }, | 2558 | .{ ._, .vp_, mir_tag, .dst0x, .src0x, .src1x, ._ }, |
| 2556 | } }, | 2559 | } }, |
| 2557 | }, .{ | 2560 | }, .{ |
| 2558 | .required_features = .{ .sse2, null }, | 2561 | .required_features = .{ .sse2, null }, |
| ... | @@ -2561,9 +2564,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2561,9 +2564,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2561 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, | 2564 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, |
| 2562 | .{ .src = .{ .mut_xmm, .xmm } }, | 2565 | .{ .src = .{ .mut_xmm, .xmm } }, |
| 2563 | }, | 2566 | }, |
| 2564 | .dst_temps = .{.{ .src = 0 }}, | 2567 | .dst_temps = .{.{ .ref = .src0 }}, |
| 2565 | .each = .{ .once = &.{ | 2568 | .each = .{ .once = &.{ |
| 2566 | .{ .p_, mir_tag, .xdst0, .xsrc1, .none, .none }, | 2569 | .{ ._, .p_, mir_tag, .dst0x, .src1x, ._, ._ }, |
| 2567 | } }, | 2570 | } }, |
| 2568 | }, .{ | 2571 | }, .{ |
| 2569 | .required_features = .{ .sse, null }, | 2572 | .required_features = .{ .sse, null }, |
| ... | @@ -2572,9 +2575,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2572,9 +2575,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2572 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, | 2575 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, |
| 2573 | .{ .src = .{ .mut_xmm, .xmm } }, | 2576 | .{ .src = .{ .mut_xmm, .xmm } }, |
| 2574 | }, | 2577 | }, |
| 2575 | .dst_temps = .{.{ .src = 0 }}, | 2578 | .dst_temps = .{.{ .ref = .src0 }}, |
| 2576 | .each = .{ .once = &.{ | 2579 | .each = .{ .once = &.{ |
| 2577 | .{ ._ps, mir_tag, .xdst0, .xsrc1, .none, .none }, | 2580 | .{ ._, ._ps, mir_tag, .dst0x, .src1x, ._, ._ }, |
| 2578 | } }, | 2581 | } }, |
| 2579 | }, .{ | 2582 | }, .{ |
| 2580 | .required_features = .{ .mmx, null }, | 2583 | .required_features = .{ .mmx, null }, |
| ... | @@ -2583,12 +2586,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2583,12 +2586,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2583 | .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } }, | 2586 | .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } }, |
| 2584 | .{ .src = .{ .mut_mm, .mm } }, | 2587 | .{ .src = .{ .mut_mm, .mm } }, |
| 2585 | }, | 2588 | }, |
| 2586 | .dst_temps = .{.{ .src = 0 }}, | 2589 | .dst_temps = .{.{ .ref = .src0 }}, |
| 2587 | .each = .{ .once = &.{ | 2590 | .each = .{ .once = &.{ |
| 2588 | .{ .p_, mir_tag, .rdst0, .rsrc1, .none, .none }, | 2591 | .{ ._, .p_, mir_tag, .dst0q, .src1q, ._, ._ }, |
| 2589 | } }, | 2592 | } }, |
| 2590 | }, .{ | 2593 | }, .{ |
| 2591 | .constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, | 2594 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| 2592 | .patterns = &.{ | 2595 | .patterns = &.{ |
| 2593 | .{ .src = .{ .mut_mem, .imm8 } }, | 2596 | .{ .src = .{ .mut_mem, .imm8 } }, |
| 2594 | .{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } }, | 2597 | .{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -2601,12 +2604,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2601,12 +2604,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2601 | .{ .src = .{ .mut_gpr, .gpr } }, | 2604 | .{ .src = .{ .mut_gpr, .gpr } }, |
| 2602 | }, | 2605 | }, |
| 2603 | .clobbers = .{ .eflags = true }, | 2606 | .clobbers = .{ .eflags = true }, |
| 2604 | .dst_temps = .{.{ .src = 0 }}, | 2607 | .dst_temps = .{.{ .ref = .src0 }}, |
| 2605 | .each = .{ .once = &.{ | 2608 | .each = .{ .once = &.{ |
| 2606 | .{ ._, mir_tag, .dst0b, .src1b, .none, .none }, | 2609 | .{ ._, ._, mir_tag, .dst0b, .src1b, ._, ._ }, |
| 2607 | } }, | 2610 | } }, |
| 2608 | }, .{ | 2611 | }, .{ |
| 2609 | .constraints = .{ .{ .int = .word }, .{ .int = .word } }, | 2612 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| 2610 | .patterns = &.{ | 2613 | .patterns = &.{ |
| 2611 | .{ .src = .{ .mut_mem, .imm16 } }, | 2614 | .{ .src = .{ .mut_mem, .imm16 } }, |
| 2612 | .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } }, | 2615 | .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -2619,12 +2622,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2619,12 +2622,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2619 | .{ .src = .{ .mut_gpr, .gpr } }, | 2622 | .{ .src = .{ .mut_gpr, .gpr } }, |
| 2620 | }, | 2623 | }, |
| 2621 | .clobbers = .{ .eflags = true }, | 2624 | .clobbers = .{ .eflags = true }, |
| 2622 | .dst_temps = .{.{ .src = 0 }}, | 2625 | .dst_temps = .{.{ .ref = .src0 }}, |
| 2623 | .each = .{ .once = &.{ | 2626 | .each = .{ .once = &.{ |
| 2624 | .{ ._, mir_tag, .dst0w, .src1w, .none, .none }, | 2627 | .{ ._, ._, mir_tag, .dst0w, .src1w, ._, ._ }, |
| 2625 | } }, | 2628 | } }, |
| 2626 | }, .{ | 2629 | }, .{ |
| 2627 | .constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, | 2630 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| 2628 | .patterns = &.{ | 2631 | .patterns = &.{ |
| 2629 | .{ .src = .{ .mut_mem, .imm32 } }, | 2632 | .{ .src = .{ .mut_mem, .imm32 } }, |
| 2630 | .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } }, | 2633 | .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -2637,13 +2640,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2637,13 +2640,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2637 | .{ .src = .{ .mut_gpr, .gpr } }, | 2640 | .{ .src = .{ .mut_gpr, .gpr } }, |
| 2638 | }, | 2641 | }, |
| 2639 | .clobbers = .{ .eflags = true }, | 2642 | .clobbers = .{ .eflags = true }, |
| 2640 | .dst_temps = .{.{ .src = 0 }}, | 2643 | .dst_temps = .{.{ .ref = .src0 }}, |
| 2641 | .each = .{ .once = &.{ | 2644 | .each = .{ .once = &.{ |
| 2642 | .{ ._, mir_tag, .edst0, .esrc1, .none, .none }, | 2645 | .{ ._, ._, mir_tag, .dst0d, .src1d, ._, ._ }, |
| 2643 | } }, | 2646 | } }, |
| 2644 | }, .{ | 2647 | }, .{ |
| 2645 | .required_features = .{ .@"64bit", null }, | 2648 | .required_features = .{ .@"64bit", null }, |
| 2646 | .constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, | 2649 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| 2647 | .patterns = &.{ | 2650 | .patterns = &.{ |
| 2648 | .{ .src = .{ .mut_mem, .simm32 } }, | 2651 | .{ .src = .{ .mut_mem, .simm32 } }, |
| 2649 | .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } }, | 2652 | .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -2656,9 +2659,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2656,9 +2659,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2656 | .{ .src = .{ .mut_gpr, .gpr } }, | 2659 | .{ .src = .{ .mut_gpr, .gpr } }, |
| 2657 | }, | 2660 | }, |
| 2658 | .clobbers = .{ .eflags = true }, | 2661 | .clobbers = .{ .eflags = true }, |
| 2659 | .dst_temps = .{.{ .src = 0 }}, | 2662 | .dst_temps = .{.{ .ref = .src0 }}, |
| 2660 | .each = .{ .once = &.{ | 2663 | .each = .{ .once = &.{ |
| 2661 | .{ ._, mir_tag, .rdst0, .rsrc1, .none, .none }, | 2664 | .{ ._, ._, mir_tag, .dst0q, .src1q, ._, ._ }, |
| 2662 | } }, | 2665 | } }, |
| 2663 | }, .{ | 2666 | }, .{ |
| 2664 | .required_features = .{ .avx2, null }, | 2667 | .required_features = .{ .avx2, null }, |
| ... | @@ -2669,14 +2672,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2669,14 +2672,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2669 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 2672 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 2670 | .{ .kind = .{ .rc = .sse } }, | 2673 | .{ .kind = .{ .rc = .sse } }, |
| 2671 | .unused, | 2674 | .unused, |
| | 2675 | .unused, |
| | 2676 | .unused, |
| | 2677 | .unused, |
| 2672 | }, | 2678 | }, |
| 2673 | .dst_temps = .{.mem}, | 2679 | .dst_temps = .{.mem}, |
| 2674 | .each = .{ .limb = .{ | 2680 | .each = .{ .limb = .{ |
| 2675 | .of = .ysrc0, | 2681 | .of = .src0y, |
| 2676 | .body = &.{ | 2682 | .body = &.{ |
| 2677 | .{ .v_, .movdqu, .ytmp1, .{ .src_limb = 0 }, .none, .none }, | 2683 | .{ ._, .v_dqu, .mov, .tmp1y, .limb(.src0y), ._, ._ }, |
| 2678 | .{ .vp_, mir_tag, .ytmp1, .ytmp1, .{ .src_limb = 1 }, .none }, | 2684 | .{ ._, .vp_, mir_tag, .tmp1y, .tmp1y, .limb(.src1y), ._ }, |
| 2679 | .{ .v_, .movdqu, .{ .dst_limb = 0 }, .ytmp1, .none, .none }, | 2685 | .{ ._, .v_dqu, .mov, .limb(.dst0y), .tmp1y, ._, ._ }, |
| 2680 | }, | 2686 | }, |
| 2681 | } }, | 2687 | } }, |
| 2682 | }, .{ | 2688 | }, .{ |
| ... | @@ -2688,14 +2694,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2688,14 +2694,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2688 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 2694 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 2689 | .{ .kind = .{ .rc = .sse } }, | 2695 | .{ .kind = .{ .rc = .sse } }, |
| 2690 | .unused, | 2696 | .unused, |
| | 2697 | .unused, |
| | 2698 | .unused, |
| | 2699 | .unused, |
| 2691 | }, | 2700 | }, |
| 2692 | .dst_temps = .{.mem}, | 2701 | .dst_temps = .{.mem}, |
| 2693 | .each = .{ .limb = .{ | 2702 | .each = .{ .limb = .{ |
| 2694 | .of = .ysrc0, | 2703 | .of = .src0y, |
| 2695 | .body = &.{ | 2704 | .body = &.{ |
| 2696 | .{ .v_pd, .movu, .ytmp1, .{ .src_limb = 0 }, .none, .none }, | 2705 | .{ ._, .v_pd, .movu, .tmp1y, .limb(.src0y), ._, ._ }, |
| 2697 | .{ .v_pd, mir_tag, .ytmp1, .ytmp1, .{ .src_limb = 1 }, .none }, | 2706 | .{ ._, .v_pd, mir_tag, .tmp1y, .tmp1y, .limb(.src1y), ._ }, |
| 2698 | .{ .v_pd, .movu, .{ .dst_limb = 0 }, .ytmp1, .none, .none }, | 2707 | .{ ._, .v_pd, .movu, .limb(.dst0y), .tmp1y, ._, ._ }, |
| 2699 | }, | 2708 | }, |
| 2700 | } }, | 2709 | } }, |
| 2701 | }, .{ | 2710 | }, .{ |
| ... | @@ -2707,14 +2716,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2707,14 +2716,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2707 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 2716 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 2708 | .{ .kind = .{ .rc = .sse } }, | 2717 | .{ .kind = .{ .rc = .sse } }, |
| 2709 | .unused, | 2718 | .unused, |
| | 2719 | .unused, |
| | 2720 | .unused, |
| | 2721 | .unused, |
| 2710 | }, | 2722 | }, |
| 2711 | .dst_temps = .{.mem}, | 2723 | .dst_temps = .{.mem}, |
| 2712 | .each = .{ .limb = .{ | 2724 | .each = .{ .limb = .{ |
| 2713 | .of = .xsrc0, | 2725 | .of = .src0x, |
| 2714 | .body = &.{ | 2726 | .body = &.{ |
| 2715 | .{ .v_, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none }, | 2727 | .{ ._, .v_dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ }, |
| 2716 | .{ .vp_, mir_tag, .xtmp1, .xtmp1, .{ .src_limb = 1 }, .none }, | 2728 | .{ ._, .vp_, mir_tag, .tmp1x, .tmp1x, .limb(.src1x), ._ }, |
| 2717 | .{ .v_, .movdqu, .{ .dst_limb = 0 }, .xtmp1, .none, .none }, | 2729 | .{ ._, .v_dqu, .mov, .limb(.dst0x), .tmp1x, ._, ._ }, |
| 2718 | }, | 2730 | }, |
| 2719 | } }, | 2731 | } }, |
| 2720 | }, .{ | 2732 | }, .{ |
| ... | @@ -2726,14 +2738,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2726,14 +2738,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2726 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 2738 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 2727 | .{ .kind = .{ .rc = .sse } }, | 2739 | .{ .kind = .{ .rc = .sse } }, |
| 2728 | .unused, | 2740 | .unused, |
| | 2741 | .unused, |
| | 2742 | .unused, |
| | 2743 | .unused, |
| 2729 | }, | 2744 | }, |
| 2730 | .dst_temps = .{.mem}, | 2745 | .dst_temps = .{.mem}, |
| 2731 | .each = .{ .limb = .{ | 2746 | .each = .{ .limb = .{ |
| 2732 | .of = .xsrc0, | 2747 | .of = .src0x, |
| 2733 | .body = &.{ | 2748 | .body = &.{ |
| 2734 | .{ ._, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none }, | 2749 | .{ ._, ._dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ }, |
| 2735 | .{ .p_, mir_tag, .xtmp1, .{ .src_limb = 1 }, .none, .none }, | 2750 | .{ ._, .p_, mir_tag, .tmp1x, .limb(.src1x), ._, ._ }, |
| 2736 | .{ ._, .movdqu, .{ .dst_limb = 0 }, .xtmp1, .none, .none }, | 2751 | .{ ._, ._dqu, .mov, .limb(.dst0x), .tmp1x, ._, ._ }, |
| 2737 | }, | 2752 | }, |
| 2738 | } }, | 2753 | } }, |
| 2739 | }, .{ | 2754 | }, .{ |
| ... | @@ -2745,14 +2760,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2745,14 +2760,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2745 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 2760 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 2746 | .{ .kind = .{ .rc = .sse } }, | 2761 | .{ .kind = .{ .rc = .sse } }, |
| 2747 | .unused, | 2762 | .unused, |
| | 2763 | .unused, |
| | 2764 | .unused, |
| | 2765 | .unused, |
| 2748 | }, | 2766 | }, |
| 2749 | .dst_temps = .{.mem}, | 2767 | .dst_temps = .{.mem}, |
| 2750 | .each = .{ .limb = .{ | 2768 | .each = .{ .limb = .{ |
| 2751 | .of = .xsrc0, | 2769 | .of = .src0x, |
| 2752 | .body = &.{ | 2770 | .body = &.{ |
| 2753 | .{ ._ps, .movu, .xtmp1, .{ .src_limb = 0 }, .none, .none }, | 2771 | .{ ._, ._ps, .movu, .tmp1x, .limb(.src0x), ._, ._ }, |
| 2754 | .{ ._ps, mir_tag, .xtmp1, .{ .src_limb = 1 }, .none, .none }, | 2772 | .{ ._, ._ps, mir_tag, .tmp1x, .limb(.src1x), ._, ._ }, |
| 2755 | .{ ._ps, .movu, .{ .dst_limb = 0 }, .xtmp1, .none, .none }, | 2773 | .{ ._, ._ps, .movu, .limb(.dst0x), .tmp1x, ._, ._ }, |
| 2756 | }, | 2774 | }, |
| 2757 | } }, | 2775 | } }, |
| 2758 | }, .{ | 2776 | }, .{ |
| ... | @@ -2764,33 +2782,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2764,33 +2782,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2764 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 2782 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 2765 | .{ .kind = .{ .rc = .mmx } }, | 2783 | .{ .kind = .{ .rc = .mmx } }, |
| 2766 | .unused, | 2784 | .unused, |
| 2767 | }, | 2785 | .unused, |
| 2768 | .dst_temps = .{.mem}, | 2786 | .unused, |
| 2769 | .each = .{ .limb = .{ | | |
| 2770 | .of = .rsrc0, | | |
| 2771 | .body = &.{ | | |
| 2772 | .{ ._q, .mov, .rtmp1, .{ .src_limb = 0 }, .none, .none }, | | |
| 2773 | .{ .p_, mir_tag, .rtmp1, .{ .src_limb = 1 }, .none, .none }, | | |
| 2774 | .{ ._q, .mov, .{ .dst_limb = 0 }, .rtmp1, .none, .none }, | | |
| 2775 | }, | | |
| 2776 | } }, | | |
| 2777 | }, .{ | | |
| 2778 | .required_features = .{ .@"64bit", null }, | | |
| 2779 | .patterns = &.{ | | |
| 2780 | .{ .src = .{ .to_mem, .to_mem } }, | | |
| 2781 | }, | | |
| 2782 | .extra_temps = .{ | | |
| 2783 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | | |
| 2784 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | | |
| 2785 | .unused, | 2787 | .unused, |
| 2786 | }, | 2788 | }, |
| 2787 | .dst_temps = .{.mem}, | 2789 | .dst_temps = .{.mem}, |
| 2788 | .each = .{ .limb = .{ | 2790 | .each = .{ .limb = .{ |
| 2789 | .of = .rsrc0, | 2791 | .of = .src0q, |
| 2790 | .body = &.{ | 2792 | .body = &.{ |
| 2791 | .{ ._, .mov, .rtmp1, .{ .src_limb = 0 }, .none, .none }, | 2793 | .{ ._, ._q, .mov, .tmp1q, .limb(.src0q), ._, ._ }, |
| 2792 | .{ ._, mir_tag, .rtmp1, .{ .src_limb = 1 }, .none, .none }, | 2794 | .{ ._, .p_, mir_tag, .tmp1q, .limb(.src1q), ._, ._ }, |
| 2793 | .{ ._, .mov, .{ .dst_limb = 0 }, .rtmp1, .none, .none }, | 2795 | .{ ._, ._q, .mov, .limb(.dst0q), .tmp1q, ._, ._ }, |
| 2794 | }, | 2796 | }, |
| 2795 | } }, | 2797 | } }, |
| 2796 | }, .{ | 2798 | }, .{ |
| ... | @@ -2799,21 +2801,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2799,21 +2801,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2799 | }, | 2801 | }, |
| 2800 | .extra_temps = .{ | 2802 | .extra_temps = .{ |
| 2801 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 2803 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 2802 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | 2804 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 2805 | .unused, |
| | 2806 | .unused, |
| | 2807 | .unused, |
| 2803 | .unused, | 2808 | .unused, |
| 2804 | }, | 2809 | }, |
| 2805 | .dst_temps = .{.mem}, | 2810 | .dst_temps = .{.mem}, |
| 2806 | .each = .{ .limb = .{ | 2811 | .each = .{ .limb = .{ |
| 2807 | .of = .esrc0, | 2812 | .of = .src0p, |
| 2808 | .body = &.{ | 2813 | .body = &.{ |
| 2809 | .{ ._, .mov, .etmp1, .{ .src_limb = 0 }, .none, .none }, | 2814 | .{ ._, ._, .mov, .tmp1p, .limb(.src0p), ._, ._ }, |
| 2810 | .{ ._, mir_tag, .etmp1, .{ .src_limb = 1 }, .none, .none }, | 2815 | .{ ._, ._, mir_tag, .tmp1p, .limb(.src1p), ._, ._ }, |
| 2811 | .{ ._, .mov, .{ .dst_limb = 0 }, .etmp1, .none, .none }, | 2816 | .{ ._, ._, .mov, .limb(.dst0p), .tmp1p, ._, ._ }, |
| 2812 | }, | 2817 | }, |
| 2813 | } }, | 2818 | } }, |
| 2814 | } }, | 2819 | } }, |
| 2815 | }) catch |err2| switch (err2) { | 2820 | }) catch |err2| switch (err2) { |
| 2816 | error.Select2Failed => return cg.fail("failed to select2 {s} {} {} {}", .{ | 2821 | error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ |
| 2817 | @tagName(air_tag), | 2822 | @tagName(air_tag), |
| 2818 | cg.typeOf(bin_op.lhs).fmt(pt), | 2823 | cg.typeOf(bin_op.lhs).fmt(pt), |
| 2819 | ops[0].tracking(cg), | 2824 | ops[0].tracking(cg), |
| ... | @@ -2875,7 +2880,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2875,7 +2880,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2875 | .call_never_tail => try cg.airCall(inst, .never_tail), | 2880 | .call_never_tail => try cg.airCall(inst, .never_tail), |
| 2876 | .call_never_inline => try cg.airCall(inst, .never_inline), | 2881 | .call_never_inline => try cg.airCall(inst, .never_inline), |
| 2877 | | 2882 | |
| 2878 | .cmp_vector, .cmp_vector_optimized => if (use_old) try cg.airCmpVector(inst) else fallback: { | 2883 | .cmp_vector, .cmp_vector_optimized => |air_tag| if (use_old) try cg.airCmpVector(inst) else fallback: { |
| 2879 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; | 2884 | const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; |
| 2880 | const extra = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data; | 2885 | const extra = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data; |
| 2881 | switch (extra.compareOperator()) { | 2886 | switch (extra.compareOperator()) { |
| ... | @@ -2887,7 +2892,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2887,7 +2892,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2887 | switch (extra.compareOperator()) { | 2892 | switch (extra.compareOperator()) { |
| 2888 | .lt => unreachable, | 2893 | .lt => unreachable, |
| 2889 | .lte => unreachable, | 2894 | .lte => unreachable, |
| 2890 | .eq, .neq => |cmp_op| cg.select2(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (cmp_op) { | 2895 | .eq, .neq => |cmp_op| cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (cmp_op) { |
| 2891 | else => unreachable, | 2896 | else => unreachable, |
| 2892 | .eq => .e, | 2897 | .eq => .e, |
| 2893 | .neq => .ne, | 2898 | .neq => .ne, |
| ... | @@ -2895,7 +2900,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2895,7 +2900,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2895 | else => unreachable, | 2900 | else => unreachable, |
| 2896 | inline .e, .ne => |cc| comptime &.{ .{ | 2901 | inline .e, .ne => |cc| comptime &.{ .{ |
| 2897 | .required_features = .{ .avx2, null }, | 2902 | .required_features = .{ .avx2, null }, |
| 2898 | .constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, | 2903 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| 2899 | .patterns = &.{ | 2904 | .patterns = &.{ |
| 2900 | .{ .src = .{ .ymm, .mem } }, | 2905 | .{ .src = .{ .ymm, .mem } }, |
| 2901 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, | 2906 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, |
| ... | @@ -2911,11 +2916,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2911,11 +2916,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2911 | .scalar = .byte, | 2916 | .scalar = .byte, |
| 2912 | } } }}, | 2917 | } } }}, |
| 2913 | .each = .{ .once = &.{ | 2918 | .each = .{ .once = &.{ |
| 2914 | .{ .vp_b, .cmpeq, .ydst0, .ysrc0, .ysrc1, .none }, | 2919 | .{ ._, .vp_b, .cmpeq, .dst0y, .src0y, .src1y, ._ }, |
| 2915 | } }, | 2920 | } }, |
| 2916 | }, .{ | 2921 | }, .{ |
| 2917 | .required_features = .{ .avx2, null }, | 2922 | .required_features = .{ .avx2, null }, |
| 2918 | .constraints = .{ .{ .int = .word }, .{ .int = .word } }, | 2923 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| 2919 | .patterns = &.{ | 2924 | .patterns = &.{ |
| 2920 | .{ .src = .{ .ymm, .mem } }, | 2925 | .{ .src = .{ .ymm, .mem } }, |
| 2921 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, | 2926 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, |
| ... | @@ -2931,11 +2936,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2931,11 +2936,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2931 | .scalar = .word, | 2936 | .scalar = .word, |
| 2932 | } } }}, | 2937 | } } }}, |
| 2933 | .each = .{ .once = &.{ | 2938 | .each = .{ .once = &.{ |
| 2934 | .{ .vp_w, .cmpeq, .ydst0, .ysrc0, .ysrc1, .none }, | 2939 | .{ ._, .vp_w, .cmpeq, .dst0y, .src0y, .src1y, ._ }, |
| 2935 | } }, | 2940 | } }, |
| 2936 | }, .{ | 2941 | }, .{ |
| 2937 | .required_features = .{ .avx2, null }, | 2942 | .required_features = .{ .avx2, null }, |
| 2938 | .constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, | 2943 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| 2939 | .patterns = &.{ | 2944 | .patterns = &.{ |
| 2940 | .{ .src = .{ .ymm, .mem } }, | 2945 | .{ .src = .{ .ymm, .mem } }, |
| 2941 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, | 2946 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, |
| ... | @@ -2951,11 +2956,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2951,11 +2956,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2951 | .scalar = .dword, | 2956 | .scalar = .dword, |
| 2952 | } } }}, | 2957 | } } }}, |
| 2953 | .each = .{ .once = &.{ | 2958 | .each = .{ .once = &.{ |
| 2954 | .{ .vp_d, .cmpeq, .ydst0, .ysrc0, .ysrc1, .none }, | 2959 | .{ ._, .vp_d, .cmpeq, .dst0y, .src0y, .src1y, ._ }, |
| 2955 | } }, | 2960 | } }, |
| 2956 | }, .{ | 2961 | }, .{ |
| 2957 | .required_features = .{ .avx2, null }, | 2962 | .required_features = .{ .avx2, null }, |
| 2958 | .constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, | 2963 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| 2959 | .patterns = &.{ | 2964 | .patterns = &.{ |
| 2960 | .{ .src = .{ .ymm, .mem } }, | 2965 | .{ .src = .{ .ymm, .mem } }, |
| 2961 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, | 2966 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, |
| ... | @@ -2971,11 +2976,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2971,11 +2976,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2971 | .scalar = .qword, | 2976 | .scalar = .qword, |
| 2972 | } } }}, | 2977 | } } }}, |
| 2973 | .each = .{ .once = &.{ | 2978 | .each = .{ .once = &.{ |
| 2974 | .{ .vp_q, .cmpeq, .ydst0, .ysrc0, .ysrc1, .none }, | 2979 | .{ ._, .vp_q, .cmpeq, .dst0y, .src0y, .src1y, ._ }, |
| 2975 | } }, | 2980 | } }, |
| 2976 | }, .{ | 2981 | }, .{ |
| 2977 | .required_features = .{ .avx, null }, | 2982 | .required_features = .{ .avx, null }, |
| 2978 | .constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, | 2983 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| 2979 | .patterns = &.{ | 2984 | .patterns = &.{ |
| 2980 | .{ .src = .{ .xmm, .mem } }, | 2985 | .{ .src = .{ .xmm, .mem } }, |
| 2981 | .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } }, | 2986 | .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } }, |
| ... | @@ -2991,11 +2996,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -2991,11 +2996,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 2991 | .scalar = .byte, | 2996 | .scalar = .byte, |
| 2992 | } } }}, | 2997 | } } }}, |
| 2993 | .each = .{ .once = &.{ | 2998 | .each = .{ .once = &.{ |
| 2994 | .{ .vp_b, .cmpeq, .xdst0, .xsrc0, .xsrc1, .none }, | 2999 | .{ ._, .vp_b, .cmpeq, .dst0x, .src0x, .src1x, ._ }, |
| 2995 | } }, | 3000 | } }, |
| 2996 | }, .{ | 3001 | }, .{ |
| 2997 | .required_features = .{ .avx, null }, | 3002 | .required_features = .{ .avx, null }, |
| 2998 | .constraints = .{ .{ .int = .word }, .{ .int = .word } }, | 3003 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| 2999 | .patterns = &.{ | 3004 | .patterns = &.{ |
| 3000 | .{ .src = .{ .xmm, .mem } }, | 3005 | .{ .src = .{ .xmm, .mem } }, |
| 3001 | .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } }, | 3006 | .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } }, |
| ... | @@ -3011,11 +3016,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3011,11 +3016,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3011 | .scalar = .word, | 3016 | .scalar = .word, |
| 3012 | } } }}, | 3017 | } } }}, |
| 3013 | .each = .{ .once = &.{ | 3018 | .each = .{ .once = &.{ |
| 3014 | .{ .vp_w, .cmpeq, .xdst0, .xsrc0, .xsrc1, .none }, | 3019 | .{ ._, .vp_w, .cmpeq, .dst0x, .src0x, .src1x, ._ }, |
| 3015 | } }, | 3020 | } }, |
| 3016 | }, .{ | 3021 | }, .{ |
| 3017 | .required_features = .{ .avx, null }, | 3022 | .required_features = .{ .avx, null }, |
| 3018 | .constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, | 3023 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| 3019 | .patterns = &.{ | 3024 | .patterns = &.{ |
| 3020 | .{ .src = .{ .xmm, .mem } }, | 3025 | .{ .src = .{ .xmm, .mem } }, |
| 3021 | .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } }, | 3026 | .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } }, |
| ... | @@ -3031,11 +3036,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3031,11 +3036,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3031 | .scalar = .dword, | 3036 | .scalar = .dword, |
| 3032 | } } }}, | 3037 | } } }}, |
| 3033 | .each = .{ .once = &.{ | 3038 | .each = .{ .once = &.{ |
| 3034 | .{ .vp_d, .cmpeq, .xdst0, .xsrc0, .xsrc1, .none }, | 3039 | .{ ._, .vp_d, .cmpeq, .dst0x, .src0x, .src1x, ._ }, |
| 3035 | } }, | 3040 | } }, |
| 3036 | }, .{ | 3041 | }, .{ |
| 3037 | .required_features = .{ .avx, null }, | 3042 | .required_features = .{ .avx, null }, |
| 3038 | .constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, | 3043 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| 3039 | .patterns = &.{ | 3044 | .patterns = &.{ |
| 3040 | .{ .src = .{ .xmm, .mem } }, | 3045 | .{ .src = .{ .xmm, .mem } }, |
| 3041 | .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } }, | 3046 | .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } }, |
| ... | @@ -3051,17 +3056,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3051,17 +3056,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3051 | .scalar = .qword, | 3056 | .scalar = .qword, |
| 3052 | } } }}, | 3057 | } } }}, |
| 3053 | .each = .{ .once = &.{ | 3058 | .each = .{ .once = &.{ |
| 3054 | .{ .vp_q, .cmpeq, .xdst0, .xsrc0, .xsrc1, .none }, | 3059 | .{ ._, .vp_q, .cmpeq, .dst0x, .src0x, .src1x, ._ }, |
| 3055 | } }, | 3060 | } }, |
| 3056 | }, .{ | 3061 | }, .{ |
| 3057 | .required_features = .{ .sse2, null }, | 3062 | .required_features = .{ .sse2, null }, |
| 3058 | .constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, | 3063 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| 3059 | .patterns = &.{ | 3064 | .patterns = &.{ |
| 3060 | .{ .src = .{ .mut_xmm, .mem } }, | 3065 | .{ .src = .{ .mut_xmm, .mem } }, |
| 3061 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, | 3066 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, |
| 3062 | .{ .src = .{ .mut_xmm, .xmm } }, | 3067 | .{ .src = .{ .mut_xmm, .xmm } }, |
| 3063 | }, | 3068 | }, |
| 3064 | .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{ | 3069 | .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{ |
| 3065 | .kind = .all, | 3070 | .kind = .all, |
| 3066 | .inverted = switch (cc) { | 3071 | .inverted = switch (cc) { |
| 3067 | else => unreachable, | 3072 | else => unreachable, |
| ... | @@ -3071,17 +3076,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3071,17 +3076,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3071 | .scalar = .byte, | 3076 | .scalar = .byte, |
| 3072 | } } }}, | 3077 | } } }}, |
| 3073 | .each = .{ .once = &.{ | 3078 | .each = .{ .once = &.{ |
| 3074 | .{ .p_b, .cmpeq, .xdst0, .xsrc1, .none, .none }, | 3079 | .{ ._, .p_b, .cmpeq, .dst0x, .src1x, ._, ._ }, |
| 3075 | } }, | 3080 | } }, |
| 3076 | }, .{ | 3081 | }, .{ |
| 3077 | .required_features = .{ .sse2, null }, | 3082 | .required_features = .{ .sse2, null }, |
| 3078 | .constraints = .{ .{ .int = .word }, .{ .int = .word } }, | 3083 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| 3079 | .patterns = &.{ | 3084 | .patterns = &.{ |
| 3080 | .{ .src = .{ .mut_xmm, .mem } }, | 3085 | .{ .src = .{ .mut_xmm, .mem } }, |
| 3081 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, | 3086 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, |
| 3082 | .{ .src = .{ .mut_xmm, .xmm } }, | 3087 | .{ .src = .{ .mut_xmm, .xmm } }, |
| 3083 | }, | 3088 | }, |
| 3084 | .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{ | 3089 | .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{ |
| 3085 | .kind = .all, | 3090 | .kind = .all, |
| 3086 | .inverted = switch (cc) { | 3091 | .inverted = switch (cc) { |
| 3087 | else => unreachable, | 3092 | else => unreachable, |
| ... | @@ -3091,17 +3096,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3091,17 +3096,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3091 | .scalar = .word, | 3096 | .scalar = .word, |
| 3092 | } } }}, | 3097 | } } }}, |
| 3093 | .each = .{ .once = &.{ | 3098 | .each = .{ .once = &.{ |
| 3094 | .{ .p_w, .cmpeq, .xdst0, .xsrc1, .none, .none }, | 3099 | .{ ._, .p_w, .cmpeq, .dst0x, .src1x, ._, ._ }, |
| 3095 | } }, | 3100 | } }, |
| 3096 | }, .{ | 3101 | }, .{ |
| 3097 | .required_features = .{ .sse2, null }, | 3102 | .required_features = .{ .sse2, null }, |
| 3098 | .constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, | 3103 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| 3099 | .patterns = &.{ | 3104 | .patterns = &.{ |
| 3100 | .{ .src = .{ .mut_xmm, .mem } }, | 3105 | .{ .src = .{ .mut_xmm, .mem } }, |
| 3101 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, | 3106 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, |
| 3102 | .{ .src = .{ .mut_xmm, .xmm } }, | 3107 | .{ .src = .{ .mut_xmm, .xmm } }, |
| 3103 | }, | 3108 | }, |
| 3104 | .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{ | 3109 | .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{ |
| 3105 | .kind = .all, | 3110 | .kind = .all, |
| 3106 | .inverted = switch (cc) { | 3111 | .inverted = switch (cc) { |
| 3107 | else => unreachable, | 3112 | else => unreachable, |
| ... | @@ -3111,17 +3116,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3111,17 +3116,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3111 | .scalar = .dword, | 3116 | .scalar = .dword, |
| 3112 | } } }}, | 3117 | } } }}, |
| 3113 | .each = .{ .once = &.{ | 3118 | .each = .{ .once = &.{ |
| 3114 | .{ .p_d, .cmpeq, .xdst0, .xsrc1, .none, .none }, | 3119 | .{ ._, .p_d, .cmpeq, .dst0x, .src1x, ._, ._ }, |
| 3115 | } }, | 3120 | } }, |
| 3116 | }, .{ | 3121 | }, .{ |
| 3117 | .required_features = .{ .sse4_1, null }, | 3122 | .required_features = .{ .sse4_1, null }, |
| 3118 | .constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, | 3123 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| 3119 | .patterns = &.{ | 3124 | .patterns = &.{ |
| 3120 | .{ .src = .{ .mut_xmm, .mem } }, | 3125 | .{ .src = .{ .mut_xmm, .mem } }, |
| 3121 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, | 3126 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, |
| 3122 | .{ .src = .{ .mut_xmm, .xmm } }, | 3127 | .{ .src = .{ .mut_xmm, .xmm } }, |
| 3123 | }, | 3128 | }, |
| 3124 | .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{ | 3129 | .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{ |
| 3125 | .kind = .all, | 3130 | .kind = .all, |
| 3126 | .inverted = switch (cc) { | 3131 | .inverted = switch (cc) { |
| 3127 | else => unreachable, | 3132 | else => unreachable, |
| ... | @@ -3131,17 +3136,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3131,17 +3136,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3131 | .scalar = .qword, | 3136 | .scalar = .qword, |
| 3132 | } } }}, | 3137 | } } }}, |
| 3133 | .each = .{ .once = &.{ | 3138 | .each = .{ .once = &.{ |
| 3134 | .{ .p_q, .cmpeq, .xdst0, .xsrc1, .none, .none }, | 3139 | .{ ._, .p_q, .cmpeq, .dst0x, .src1x, ._, ._ }, |
| 3135 | } }, | 3140 | } }, |
| 3136 | }, .{ | 3141 | }, .{ |
| 3137 | .required_features = .{ .mmx, null }, | 3142 | .required_features = .{ .mmx, null }, |
| 3138 | .constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, | 3143 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| 3139 | .patterns = &.{ | 3144 | .patterns = &.{ |
| 3140 | .{ .src = .{ .mut_mm, .mem } }, | 3145 | .{ .src = .{ .mut_mm, .mem } }, |
| 3141 | .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } }, | 3146 | .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } }, |
| 3142 | .{ .src = .{ .mut_mm, .mm } }, | 3147 | .{ .src = .{ .mut_mm, .mm } }, |
| 3143 | }, | 3148 | }, |
| 3144 | .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{ | 3149 | .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{ |
| 3145 | .kind = .all, | 3150 | .kind = .all, |
| 3146 | .inverted = switch (cc) { | 3151 | .inverted = switch (cc) { |
| 3147 | else => unreachable, | 3152 | else => unreachable, |
| ... | @@ -3151,17 +3156,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3151,17 +3156,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3151 | .scalar = .byte, | 3156 | .scalar = .byte, |
| 3152 | } } }}, | 3157 | } } }}, |
| 3153 | .each = .{ .once = &.{ | 3158 | .each = .{ .once = &.{ |
| 3154 | .{ .p_b, .cmpeq, .rdst0, .rsrc1, .none, .none }, | 3159 | .{ ._, .p_b, .cmpeq, .dst0q, .src1q, ._, ._ }, |
| 3155 | } }, | 3160 | } }, |
| 3156 | }, .{ | 3161 | }, .{ |
| 3157 | .required_features = .{ .mmx, null }, | 3162 | .required_features = .{ .mmx, null }, |
| 3158 | .constraints = .{ .{ .int = .word }, .{ .int = .word } }, | 3163 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| 3159 | .patterns = &.{ | 3164 | .patterns = &.{ |
| 3160 | .{ .src = .{ .mut_mm, .mem } }, | 3165 | .{ .src = .{ .mut_mm, .mem } }, |
| 3161 | .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } }, | 3166 | .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } }, |
| 3162 | .{ .src = .{ .mut_mm, .mm } }, | 3167 | .{ .src = .{ .mut_mm, .mm } }, |
| 3163 | }, | 3168 | }, |
| 3164 | .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{ | 3169 | .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{ |
| 3165 | .kind = .all, | 3170 | .kind = .all, |
| 3166 | .inverted = switch (cc) { | 3171 | .inverted = switch (cc) { |
| 3167 | else => unreachable, | 3172 | else => unreachable, |
| ... | @@ -3171,17 +3176,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3171,17 +3176,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3171 | .scalar = .word, | 3176 | .scalar = .word, |
| 3172 | } } }}, | 3177 | } } }}, |
| 3173 | .each = .{ .once = &.{ | 3178 | .each = .{ .once = &.{ |
| 3174 | .{ .p_w, .cmpeq, .rdst0, .rsrc1, .none, .none }, | 3179 | .{ ._, .p_w, .cmpeq, .dst0q, .src1q, ._, ._ }, |
| 3175 | } }, | 3180 | } }, |
| 3176 | }, .{ | 3181 | }, .{ |
| 3177 | .required_features = .{ .mmx, null }, | 3182 | .required_features = .{ .mmx, null }, |
| 3178 | .constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, | 3183 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| 3179 | .patterns = &.{ | 3184 | .patterns = &.{ |
| 3180 | .{ .src = .{ .mut_mm, .mem } }, | 3185 | .{ .src = .{ .mut_mm, .mem } }, |
| 3181 | .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } }, | 3186 | .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } }, |
| 3182 | .{ .src = .{ .mut_mm, .mm } }, | 3187 | .{ .src = .{ .mut_mm, .mm } }, |
| 3183 | }, | 3188 | }, |
| 3184 | .dst_temps = .{.{ .src_mask = .{ .src = 0, .info = .{ | 3189 | .dst_temps = .{.{ .ref_mask = .{ .ref = .src0, .info = .{ |
| 3185 | .kind = .all, | 3190 | .kind = .all, |
| 3186 | .inverted = switch (cc) { | 3191 | .inverted = switch (cc) { |
| 3187 | else => unreachable, | 3192 | else => unreachable, |
| ... | @@ -3191,10 +3196,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3191,10 +3196,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3191 | .scalar = .dword, | 3196 | .scalar = .dword, |
| 3192 | } } }}, | 3197 | } } }}, |
| 3193 | .each = .{ .once = &.{ | 3198 | .each = .{ .once = &.{ |
| 3194 | .{ .p_d, .cmpeq, .rdst0, .rsrc1, .none, .none }, | 3199 | .{ ._, .p_d, .cmpeq, .dst0q, .src1q, ._, ._ }, |
| 3195 | } }, | 3200 | } }, |
| 3196 | }, .{ | 3201 | }, .{ |
| 3197 | .constraints = .{ .{ .bool_vec = .byte }, .{ .bool_vec = .byte } }, | 3202 | .src_constraints = .{ .{ .bool_vec = .byte }, .{ .bool_vec = .byte } }, |
| 3198 | .patterns = &.{ | 3203 | .patterns = &.{ |
| 3199 | .{ .src = .{ .mut_mem, .imm8 } }, | 3204 | .{ .src = .{ .mut_mem, .imm8 } }, |
| 3200 | .{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } }, | 3205 | .{ .src = .{ .imm8, .mut_mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -3207,19 +3212,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3207,19 +3212,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3207 | .{ .src = .{ .mut_gpr, .gpr } }, | 3212 | .{ .src = .{ .mut_gpr, .gpr } }, |
| 3208 | }, | 3213 | }, |
| 3209 | .clobbers = .{ .eflags = true }, | 3214 | .clobbers = .{ .eflags = true }, |
| 3210 | .dst_temps = .{.{ .src = 0 }}, | 3215 | .dst_temps = .{.{ .ref = .src0 }}, |
| 3211 | .each = .{ .once = switch (cc) { | 3216 | .each = .{ .once = switch (cc) { |
| 3212 | else => unreachable, | 3217 | else => unreachable, |
| 3213 | .e => &.{ | 3218 | .e => &.{ |
| 3214 | .{ ._, .xor, .dst0b, .src1b, .none, .none }, | 3219 | .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ }, |
| 3215 | .{ ._, .not, .dst0b, .none, .none, .none }, | 3220 | .{ ._, ._, .not, .dst0b, ._, ._, ._ }, |
| 3216 | }, | 3221 | }, |
| 3217 | .ne => &.{ | 3222 | .ne => &.{ |
| 3218 | .{ ._, .xor, .dst0b, .src1b, .none, .none }, | 3223 | .{ ._, ._, .xor, .dst0b, .src1b, ._, ._ }, |
| 3219 | }, | 3224 | }, |
| 3220 | } }, | 3225 | } }, |
| 3221 | }, .{ | 3226 | }, .{ |
| 3222 | .constraints = .{ .{ .bool_vec = .word }, .{ .bool_vec = .word } }, | 3227 | .src_constraints = .{ .{ .bool_vec = .word }, .{ .bool_vec = .word } }, |
| 3223 | .patterns = &.{ | 3228 | .patterns = &.{ |
| 3224 | .{ .src = .{ .mut_mem, .imm16 } }, | 3229 | .{ .src = .{ .mut_mem, .imm16 } }, |
| 3225 | .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } }, | 3230 | .{ .src = .{ .imm16, .mut_mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -3232,19 +3237,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3232,19 +3237,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3232 | .{ .src = .{ .mut_gpr, .gpr } }, | 3237 | .{ .src = .{ .mut_gpr, .gpr } }, |
| 3233 | }, | 3238 | }, |
| 3234 | .clobbers = .{ .eflags = true }, | 3239 | .clobbers = .{ .eflags = true }, |
| 3235 | .dst_temps = .{.{ .src = 0 }}, | 3240 | .dst_temps = .{.{ .ref = .src0 }}, |
| 3236 | .each = .{ .once = switch (cc) { | 3241 | .each = .{ .once = switch (cc) { |
| 3237 | else => unreachable, | 3242 | else => unreachable, |
| 3238 | .e => &.{ | 3243 | .e => &.{ |
| 3239 | .{ ._, .xor, .dst0w, .src1w, .none, .none }, | 3244 | .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ }, |
| 3240 | .{ ._, .not, .dst0w, .none, .none, .none }, | 3245 | .{ ._, ._, .not, .dst0w, ._, ._, ._ }, |
| 3241 | }, | 3246 | }, |
| 3242 | .ne => &.{ | 3247 | .ne => &.{ |
| 3243 | .{ ._, .xor, .dst0w, .src1w, .none, .none }, | 3248 | .{ ._, ._, .xor, .dst0w, .src1w, ._, ._ }, |
| 3244 | }, | 3249 | }, |
| 3245 | } }, | 3250 | } }, |
| 3246 | }, .{ | 3251 | }, .{ |
| 3247 | .constraints = .{ .{ .bool_vec = .dword }, .{ .bool_vec = .dword } }, | 3252 | .src_constraints = .{ .{ .bool_vec = .dword }, .{ .bool_vec = .dword } }, |
| 3248 | .patterns = &.{ | 3253 | .patterns = &.{ |
| 3249 | .{ .src = .{ .mut_mem, .imm32 } }, | 3254 | .{ .src = .{ .mut_mem, .imm32 } }, |
| 3250 | .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } }, | 3255 | .{ .src = .{ .imm32, .mut_mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -3257,20 +3262,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3257,20 +3262,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3257 | .{ .src = .{ .mut_gpr, .gpr } }, | 3262 | .{ .src = .{ .mut_gpr, .gpr } }, |
| 3258 | }, | 3263 | }, |
| 3259 | .clobbers = .{ .eflags = true }, | 3264 | .clobbers = .{ .eflags = true }, |
| 3260 | .dst_temps = .{.{ .src = 0 }}, | 3265 | .dst_temps = .{.{ .ref = .src0 }}, |
| 3261 | .each = .{ .once = switch (cc) { | 3266 | .each = .{ .once = switch (cc) { |
| 3262 | else => unreachable, | 3267 | else => unreachable, |
| 3263 | .e => &.{ | 3268 | .e => &.{ |
| 3264 | .{ ._, .xor, .edst0, .esrc1, .none, .none }, | 3269 | .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ }, |
| 3265 | .{ ._, .not, .edst0, .none, .none, .none }, | 3270 | .{ ._, ._, .not, .dst0d, ._, ._, ._ }, |
| 3266 | }, | 3271 | }, |
| 3267 | .ne => &.{ | 3272 | .ne => &.{ |
| 3268 | .{ ._, .xor, .edst0, .esrc1, .none, .none }, | 3273 | .{ ._, ._, .xor, .dst0d, .src1d, ._, ._ }, |
| 3269 | }, | 3274 | }, |
| 3270 | } }, | 3275 | } }, |
| 3271 | }, .{ | 3276 | }, .{ |
| 3272 | .required_features = .{ .@"64bit", null }, | 3277 | .required_features = .{ .@"64bit", null }, |
| 3273 | .constraints = .{ .{ .bool_vec = .qword }, .{ .bool_vec = .qword } }, | 3278 | .src_constraints = .{ .{ .bool_vec = .qword }, .{ .bool_vec = .qword } }, |
| 3274 | .patterns = &.{ | 3279 | .patterns = &.{ |
| 3275 | .{ .src = .{ .mut_mem, .simm32 } }, | 3280 | .{ .src = .{ .mut_mem, .simm32 } }, |
| 3276 | .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } }, | 3281 | .{ .src = .{ .simm32, .mut_mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -3283,326 +3288,1665 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3283,326 +3288,1665 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3283 | .{ .src = .{ .mut_gpr, .gpr } }, | 3288 | .{ .src = .{ .mut_gpr, .gpr } }, |
| 3284 | }, | 3289 | }, |
| 3285 | .clobbers = .{ .eflags = true }, | 3290 | .clobbers = .{ .eflags = true }, |
| 3286 | .dst_temps = .{.{ .src = 0 }}, | 3291 | .dst_temps = .{.{ .ref = .src0 }}, |
| 3287 | .each = .{ .once = switch (cc) { | 3292 | .each = .{ .once = switch (cc) { |
| 3288 | else => unreachable, | 3293 | else => unreachable, |
| 3289 | .e => &.{ | 3294 | .e => &.{ |
| 3290 | .{ ._, .xor, .rdst0, .rsrc1, .none, .none }, | 3295 | .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ }, |
| 3291 | .{ ._, .not, .rdst0, .none, .none, .none }, | 3296 | .{ ._, ._, .not, .dst0q, ._, ._, ._ }, |
| 3292 | }, | 3297 | }, |
| 3293 | .ne => &.{ | 3298 | .ne => &.{ |
| 3294 | .{ ._, .xor, .rdst0, .rsrc1, .none, .none }, | 3299 | .{ ._, ._, .xor, .dst0q, .src1q, ._, ._ }, |
| 3295 | }, | 3300 | }, |
| 3296 | } }, | 3301 | } }, |
| 3297 | } }, | 3302 | }, .{ |
| 3298 | }) catch |err2| switch (err2) { | 3303 | .src_constraints = .{ .any_bool_vec, .any_bool_vec }, |
| 3299 | error.Select2Failed => cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, &.{ | 3304 | .patterns = &.{ |
| 3300 | .{ | 3305 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3301 | .required_features = &.{.avx2}, | | |
| 3302 | .scalar = .{ .any_int = .byte }, | | |
| 3303 | .loop = .elementwise, | | |
| 3304 | .mir_tag = .{ .vp_b, .cmpeq }, | | |
| 3305 | .patterns = &.{ | | |
| 3306 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, | | |
| 3307 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, | | |
| 3308 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, | | |
| 3309 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, | | |
| 3310 | }, | | |
| 3311 | }, | 3306 | }, |
| 3312 | .{ | 3307 | .clobbers = .{ .eflags = true }, |
| 3313 | .required_features = &.{.avx2}, | 3308 | .extra_temps = .{ |
| 3314 | .scalar = .{ .any_int = .word }, | 3309 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3315 | .loop = .elementwise, | 3310 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3316 | .mir_tag = .{ .vp_w, .cmpeq }, | 3311 | .unused, |
| 3317 | .patterns = &.{ | 3312 | .unused, |
| 3318 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, | 3313 | .unused, |
| 3319 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, | 3314 | .unused, |
| 3320 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, | | |
| 3321 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, | | |
| 3322 | }, | | |
| 3323 | }, | 3315 | }, |
| 3324 | .{ | 3316 | .dst_temps = .{.mem}, |
| 3325 | .required_features = &.{.avx2}, | 3317 | .each = .{ .once = switch (cc) { |
| 3326 | .scalar = .{ .any_int = .dword }, | 3318 | else => unreachable, |
| 3327 | .loop = .elementwise, | 3319 | .e => &.{ |
| 3328 | .mir_tag = .{ .vp_d, .cmpeq }, | 3320 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| 3329 | .patterns = &.{ | 3321 | .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ }, |
| 3330 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, | 3322 | .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ }, |
| 3331 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, | 3323 | .{ ._, ._, .not, .tmp1p, ._, ._, ._ }, |
| 3332 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, | 3324 | .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ }, |
| 3333 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, | 3325 | .{ ._, ._, .add, .tmp0p, .a(.tmp1, .add_size), ._, ._ }, |
| | 3326 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| 3334 | }, | 3327 | }, |
| 3335 | }, | 3328 | .ne => &.{ |
| 3336 | .{ | 3329 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| 3337 | .required_features = &.{.avx2}, | 3330 | .{ .@"0:", ._, .mov, .tmp1p, .memia(.src0p, .tmp0, .add_size), ._, ._ }, |
| 3338 | .scalar = .{ .any_int = .qword }, | 3331 | .{ ._, ._, .xor, .tmp1p, .memia(.src1p, .tmp0, .add_size), ._, ._ }, |
| 3339 | .loop = .elementwise, | 3332 | .{ ._, ._, .mov, .memia(.dst0p, .tmp0, .add_size), .tmp1p, ._, ._ }, |
| 3340 | .mir_tag = .{ .vp_q, .cmpeq }, | 3333 | .{ ._, ._, .add, .tmp0p, .a(.tmp1, .add_size), ._, ._ }, |
| 3341 | .patterns = &.{ | 3334 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| 3342 | .{ .ops = &.{ .ymm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, | | |
| 3343 | .{ .ops = &.{ .ymm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, | | |
| 3344 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .mem_limb } }, | | |
| 3345 | .{ .ops = &.{ .ymm_mask_limb, .ymm_limb, .ymm_limb } }, | | |
| 3346 | }, | 3335 | }, |
| | 3336 | } }, |
| | 3337 | }, .{ |
| | 3338 | .required_features = .{ .avx2, null }, |
| | 3339 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 3340 | .patterns = &.{ |
| | 3341 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3347 | }, | 3342 | }, |
| 3348 | .{ | 3343 | .extra_temps = .{ |
| 3349 | .required_features = &.{.avx}, | 3344 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3350 | .scalar = .{ .any_int = .byte }, | 3345 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3351 | .loop = .elementwise, | 3346 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| 3352 | .mir_tag = .{ .vp_b, .cmpeq }, | 3347 | .{ .kind = .{ .rc = .sse } }, |
| 3353 | .patterns = &.{ | 3348 | .unused, |
| 3354 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, | 3349 | .unused, |
| 3355 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, | | |
| 3356 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, | | |
| 3357 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, | | |
| 3358 | }, | | |
| 3359 | }, | 3350 | }, |
| 3360 | .{ | 3351 | .dst_temps = .{.mem}, |
| 3361 | .required_features = &.{.avx}, | 3352 | .each = .{ .limb_and_mask_limb = .{ |
| 3362 | .scalar = .{ .any_int = .word }, | 3353 | .of = .src0y, |
| 3363 | .loop = .elementwise, | 3354 | .of_mask = .dst0b, |
| 3364 | .mir_tag = .{ .vp_w, .cmpeq }, | 3355 | .body = switch (cc) { |
| 3365 | .patterns = &.{ | 3356 | else => unreachable, |
| 3366 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, | 3357 | .e => &.{ |
| 3367 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, | 3358 | .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ }, |
| 3368 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, | 3359 | .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ }, |
| 3369 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, | 3360 | .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ }, |
| | 3361 | .{ ._, ._, .mov, .maskLimb(.dst0d), .tmp2d, ._, ._ }, |
| | 3362 | }, |
| | 3363 | .ne => &.{ |
| | 3364 | .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ }, |
| | 3365 | .{ ._, .vp_b, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ }, |
| | 3366 | .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ }, |
| | 3367 | .{ ._, ._, .not, .tmp2d, ._, ._, ._ }, |
| | 3368 | .{ ._, ._, .mov, .maskLimb(.dst0d), .tmp2d, ._, ._ }, |
| | 3369 | }, |
| 3370 | }, | 3370 | }, |
| | 3371 | } }, |
| | 3372 | }, .{ |
| | 3373 | .required_features = .{ .avx2, null }, |
| | 3374 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| | 3375 | .patterns = &.{ |
| | 3376 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3371 | }, | 3377 | }, |
| 3372 | .{ | 3378 | .extra_temps = .{ |
| 3373 | .required_features = &.{.avx}, | 3379 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3374 | .scalar = .{ .any_int = .dword }, | 3380 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3375 | .loop = .elementwise, | 3381 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| 3376 | .mir_tag = .{ .vp_d, .cmpeq }, | 3382 | .{ .kind = .{ .rc = .sse } }, |
| 3377 | .patterns = &.{ | 3383 | .unused, |
| 3378 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, | 3384 | .unused, |
| 3379 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, | 3385 | }, |
| 3380 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, | 3386 | .dst_temps = .{.mem}, |
| 3381 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, | 3387 | .each = .{ .limb_and_mask_limb = .{ |
| | 3388 | .of = .src0y, |
| | 3389 | .of_mask = .dst0w, |
| | 3390 | .body = switch (cc) { |
| | 3391 | else => unreachable, |
| | 3392 | .e => &.{ |
| | 3393 | .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ }, |
| | 3394 | .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ }, |
| | 3395 | .{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ }, |
| | 3396 | .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ }, |
| | 3397 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ }, |
| | 3398 | }, |
| | 3399 | .ne => &.{ |
| | 3400 | .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ }, |
| | 3401 | .{ ._, .vp_w, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ }, |
| | 3402 | .{ ._, .vp_b, .ackssw, .tmp3y, .tmp3y, .tmp3y, ._ }, |
| | 3403 | .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3y, ._, ._ }, |
| | 3404 | .{ ._, ._, .not, .tmp2w, ._, ._, ._ }, |
| | 3405 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ }, |
| | 3406 | }, |
| 3382 | }, | 3407 | }, |
| | 3408 | } }, |
| | 3409 | }, .{ |
| | 3410 | .required_features = .{ .avx2, null }, |
| | 3411 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| | 3412 | .patterns = &.{ |
| | 3413 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3383 | }, | 3414 | }, |
| 3384 | .{ | 3415 | .extra_temps = .{ |
| 3385 | .required_features = &.{.avx}, | 3416 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3386 | .scalar = .{ .any_int = .qword }, | 3417 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3387 | .loop = .elementwise, | 3418 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| 3388 | .mir_tag = .{ .vp_q, .cmpeq }, | 3419 | .{ .kind = .{ .rc = .sse } }, |
| 3389 | .patterns = &.{ | 3420 | .unused, |
| 3390 | .{ .ops = &.{ .xmm_mask_limb, .{ .explicit = 0 }, .mem_limb } }, | 3421 | .unused, |
| 3391 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .explicit = 0 } }, .commute = .{ 1, 2 } }, | 3422 | }, |
| 3392 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .mem_limb } }, | 3423 | .dst_temps = .{.mem}, |
| 3393 | .{ .ops = &.{ .xmm_mask_limb, .xmm_limb, .xmm_limb } }, | 3424 | .each = .{ .limb_and_mask_limb = .{ |
| | 3425 | .of = .src0y, |
| | 3426 | .of_mask = .dst0d, |
| | 3427 | .body = switch (cc) { |
| | 3428 | else => unreachable, |
| | 3429 | .e => &.{ |
| | 3430 | .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ }, |
| | 3431 | .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ }, |
| | 3432 | .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ }, |
| | 3433 | .{ ._, ._, .mov, .maskLimb(.dst0b), .tmp2b, ._, ._ }, |
| | 3434 | }, |
| | 3435 | .ne => &.{ |
| | 3436 | .{ ._, .v_dqu, .mov, .tmp3y, .limb(.src0y), ._, ._ }, |
| | 3437 | .{ ._, .vp_d, .cmpeq, .tmp3y, .tmp3y, .limb(.src1y), ._ }, |
| | 3438 | .{ ._, .v_ps, .movmsk, .tmp2d, .tmp3y, ._, ._ }, |
| | 3439 | .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, |
| | 3440 | .{ ._, ._, .mov, .maskLimb(.dst0b), .tmp2b, ._, ._ }, |
| | 3441 | }, |
| 3394 | }, | 3442 | }, |
| | 3443 | } }, |
| | 3444 | }, .{ |
| | 3445 | .required_features = .{ .avx2, null }, |
| | 3446 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| | 3447 | .patterns = &.{ |
| | 3448 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3395 | }, | 3449 | }, |
| 3396 | .{ | 3450 | .extra_temps = .{ |
| 3397 | .required_features = &.{.sse2}, | 3451 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3398 | .scalar = .{ .any_int = .byte }, | 3452 | .{ .type = .usize, .kind = .{ .reg = .rcx } }, |
| 3399 | .loop = .elementwise, | 3453 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3400 | .mir_tag = .{ .p_b, .cmpeq }, | 3454 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3401 | .patterns = &.{ | 3455 | .{ .kind = .{ .rc = .sse } }, |
| 3402 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, | 3456 | .unused, |
| 3403 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, | 3457 | }, |
| 3404 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, | 3458 | .dst_temps = .{.mem}, |
| | 3459 | .each = .{ .limb_and_mask_limb = .{ |
| | 3460 | .of = .src0y, |
| | 3461 | .of_mask = .dst0q, |
| | 3462 | .body = switch (cc) { |
| | 3463 | else => unreachable, |
| | 3464 | .e => &.{ |
| | 3465 | .{ ._, .v_dqu, .mov, .tmp4y, .limb(.src0y), ._, ._ }, |
| | 3466 | .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .limb(.src1y), ._ }, |
| | 3467 | .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ }, |
| | 3468 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3469 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3470 | }, |
| | 3471 | .ne => &.{ |
| | 3472 | .{ ._, .v_dqu, .mov, .tmp4y, .limb(.src0y), ._, ._ }, |
| | 3473 | .{ ._, .vp_q, .cmpeq, .tmp4y, .tmp4y, .limb(.src1y), ._ }, |
| | 3474 | .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4y, ._, ._ }, |
| | 3475 | .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ }, |
| | 3476 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3477 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3478 | }, |
| 3405 | }, | 3479 | }, |
| | 3480 | } }, |
| | 3481 | }, .{ |
| | 3482 | .required_features = .{ .avx, null }, |
| | 3483 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 3484 | .patterns = &.{ |
| | 3485 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3406 | }, | 3486 | }, |
| 3407 | .{ | 3487 | .extra_temps = .{ |
| 3408 | .required_features = &.{.sse2}, | 3488 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3409 | .scalar = .{ .any_int = .word }, | 3489 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3410 | .loop = .elementwise, | 3490 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| 3411 | .mir_tag = .{ .p_w, .cmpeq }, | 3491 | .{ .kind = .{ .rc = .sse } }, |
| 3412 | .patterns = &.{ | 3492 | .unused, |
| 3413 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, | 3493 | .unused, |
| 3414 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, | 3494 | }, |
| 3415 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, | 3495 | .dst_temps = .{.mem}, |
| | 3496 | .each = .{ .limb_and_mask_limb = .{ |
| | 3497 | .of = .src0x, |
| | 3498 | .of_mask = .dst0b, |
| | 3499 | .body = switch (cc) { |
| | 3500 | else => unreachable, |
| | 3501 | .e => &.{ |
| | 3502 | .{ ._, .v_dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ }, |
| | 3503 | .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .limb(.src1x), ._ }, |
| | 3504 | .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, |
| | 3505 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ }, |
| | 3506 | }, |
| | 3507 | .ne => &.{ |
| | 3508 | .{ ._, .v_dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ }, |
| | 3509 | .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .limb(.src1x), ._ }, |
| | 3510 | .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, |
| | 3511 | .{ ._, ._, .not, .tmp2w, ._, ._, ._ }, |
| | 3512 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ }, |
| | 3513 | }, |
| 3416 | }, | 3514 | }, |
| | 3515 | } }, |
| | 3516 | }, .{ |
| | 3517 | .required_features = .{ .avx, null }, |
| | 3518 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| | 3519 | .patterns = &.{ |
| | 3520 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3417 | }, | 3521 | }, |
| 3418 | .{ | 3522 | .extra_temps = .{ |
| 3419 | .required_features = &.{.sse2}, | 3523 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3420 | .scalar = .{ .any_int = .dword }, | 3524 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3421 | .loop = .elementwise, | 3525 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| 3422 | .mir_tag = .{ .p_d, .cmpeq }, | 3526 | .{ .kind = .{ .rc = .sse } }, |
| 3423 | .patterns = &.{ | 3527 | .unused, |
| 3424 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, | 3528 | .unused, |
| 3425 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, | 3529 | }, |
| 3426 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, | 3530 | .dst_temps = .{.mem}, |
| | 3531 | .each = .{ .limb_and_mask_limb = .{ |
| | 3532 | .of = .src0x, |
| | 3533 | .of_mask = .dst0w, |
| | 3534 | .body = switch (cc) { |
| | 3535 | else => unreachable, |
| | 3536 | .e => &.{ |
| | 3537 | .{ ._, .v_dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ }, |
| | 3538 | .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .limb(.src1x), ._ }, |
| | 3539 | .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ }, |
| | 3540 | .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, |
| | 3541 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ }, |
| | 3542 | }, |
| | 3543 | .ne => &.{ |
| | 3544 | .{ ._, .v_dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ }, |
| | 3545 | .{ ._, .vp_w, .cmpeq, .tmp3x, .tmp3x, .limb(.src1x), ._ }, |
| | 3546 | .{ ._, .vp_b, .ackssw, .tmp3x, .tmp3x, .tmp3x, ._ }, |
| | 3547 | .{ ._, .vp_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, |
| | 3548 | .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, |
| | 3549 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ }, |
| | 3550 | }, |
| 3427 | }, | 3551 | }, |
| | 3552 | } }, |
| | 3553 | }, .{ |
| | 3554 | .required_features = .{ .avx, null }, |
| | 3555 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| | 3556 | .patterns = &.{ |
| | 3557 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3428 | }, | 3558 | }, |
| 3429 | .{ | 3559 | .extra_temps = .{ |
| 3430 | .required_features = &.{.sse4_1}, | 3560 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3431 | .scalar = .{ .any_int = .qword }, | 3561 | .{ .type = .usize, .kind = .{ .reg = .rcx } }, |
| 3432 | .loop = .elementwise, | 3562 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3433 | .mir_tag = .{ .p_q, .cmpeq }, | 3563 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3434 | .patterns = &.{ | 3564 | .{ .kind = .{ .rc = .sse } }, |
| 3435 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, | 3565 | .unused, |
| 3436 | .{ .ops = &.{ .xmm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, | 3566 | }, |
| 3437 | .{ .ops = &.{ .xmm_mask_limb, .{ .implicit = 0 }, .xmm_limb } }, | 3567 | .dst_temps = .{.mem}, |
| | 3568 | .each = .{ .limb_and_mask_limb = .{ |
| | 3569 | .of = .src0x, |
| | 3570 | .of_mask = .dst0d, |
| | 3571 | .body = switch (cc) { |
| | 3572 | else => unreachable, |
| | 3573 | .e => &.{ |
| | 3574 | .{ ._, .v_dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ }, |
| | 3575 | .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .limb(.src1x), ._ }, |
| | 3576 | .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, |
| | 3577 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3578 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3579 | }, |
| | 3580 | .ne => &.{ |
| | 3581 | .{ ._, .v_dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ }, |
| | 3582 | .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .limb(.src1x), ._ }, |
| | 3583 | .{ ._, .v_ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, |
| | 3584 | .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ }, |
| | 3585 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3586 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3587 | }, |
| 3438 | }, | 3588 | }, |
| | 3589 | } }, |
| | 3590 | }, .{ |
| | 3591 | .required_features = .{ .avx, null }, |
| | 3592 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| | 3593 | .patterns = &.{ |
| | 3594 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3439 | }, | 3595 | }, |
| 3440 | .{ | 3596 | .extra_temps = .{ |
| 3441 | .required_features = &.{.mmx}, | 3597 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3442 | .scalar = .{ .any_int = .byte }, | 3598 | .{ .type = .usize, .kind = .{ .reg = .rcx } }, |
| 3443 | .loop = .elementwise, | 3599 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3444 | .mir_tag = .{ .p_b, .cmpeq }, | 3600 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3445 | .patterns = &.{ | 3601 | .{ .kind = .{ .rc = .sse } }, |
| 3446 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, | 3602 | .unused, |
| 3447 | .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, | 3603 | }, |
| 3448 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } }, | 3604 | .dst_temps = .{.mem}, |
| | 3605 | .each = .{ .limb_and_mask_limb = .{ |
| | 3606 | .of = .src0x, |
| | 3607 | .of_mask = .dst0q, |
| | 3608 | .body = switch (cc) { |
| | 3609 | else => unreachable, |
| | 3610 | .e => &.{ |
| | 3611 | .{ ._, .v_dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ }, |
| | 3612 | .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .limb(.src1x), ._ }, |
| | 3613 | .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, |
| | 3614 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3615 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3616 | }, |
| | 3617 | .ne => &.{ |
| | 3618 | .{ ._, .v_dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ }, |
| | 3619 | .{ ._, .vp_q, .cmpeq, .tmp4x, .tmp4x, .limb(.src1x), ._ }, |
| | 3620 | .{ ._, .v_pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, |
| | 3621 | .{ ._, ._, .xor, .tmp3b, .i(0b11), ._, ._ }, |
| | 3622 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3623 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3624 | }, |
| 3449 | }, | 3625 | }, |
| | 3626 | } }, |
| | 3627 | }, .{ |
| | 3628 | .required_features = .{ .sse2, null }, |
| | 3629 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 3630 | .patterns = &.{ |
| | 3631 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3450 | }, | 3632 | }, |
| 3451 | .{ | 3633 | .extra_temps = .{ |
| 3452 | .required_features = &.{.mmx}, | 3634 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3453 | .scalar = .{ .any_int = .word }, | 3635 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3454 | .loop = .elementwise, | 3636 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| 3455 | .mir_tag = .{ .p_w, .cmpeq }, | 3637 | .{ .kind = .{ .rc = .sse } }, |
| 3456 | .patterns = &.{ | 3638 | .unused, |
| 3457 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, | 3639 | .unused, |
| 3458 | .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, | 3640 | }, |
| 3459 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } }, | 3641 | .dst_temps = .{.mem}, |
| | 3642 | .each = .{ .limb_and_mask_limb = .{ |
| | 3643 | .of = .src0x, |
| | 3644 | .of_mask = .dst0b, |
| | 3645 | .body = switch (cc) { |
| | 3646 | else => unreachable, |
| | 3647 | .e => &.{ |
| | 3648 | .{ ._, ._dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ }, |
| | 3649 | .{ ._, .p_b, .cmpeq, .tmp3x, .limb(.src1x), ._, ._ }, |
| | 3650 | .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, |
| | 3651 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ }, |
| | 3652 | }, |
| | 3653 | .ne => &.{ |
| | 3654 | .{ ._, ._dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ }, |
| | 3655 | .{ ._, .p_b, .cmpeq, .tmp3x, .limb(.src1x), ._, ._ }, |
| | 3656 | .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, |
| | 3657 | .{ ._, ._, .not, .tmp2w, ._, ._, ._ }, |
| | 3658 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ }, |
| | 3659 | }, |
| 3460 | }, | 3660 | }, |
| | 3661 | } }, |
| | 3662 | }, .{ |
| | 3663 | .required_features = .{ .sse2, null }, |
| | 3664 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| | 3665 | .patterns = &.{ |
| | 3666 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3461 | }, | 3667 | }, |
| 3462 | .{ | 3668 | .extra_temps = .{ |
| 3463 | .required_features = &.{.mmx}, | 3669 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3464 | .scalar = .{ .any_int = .dword }, | 3670 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3465 | .loop = .elementwise, | 3671 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| 3466 | .mir_tag = .{ .p_d, .cmpeq }, | 3672 | .{ .kind = .{ .rc = .sse } }, |
| 3467 | .patterns = &.{ | 3673 | .unused, |
| 3468 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mem_limb } }, | 3674 | .unused, |
| 3469 | .{ .ops = &.{ .mm_mask_limb, .mem_limb, .{ .implicit = 0 } } }, | 3675 | }, |
| 3470 | .{ .ops = &.{ .mm_mask_limb, .{ .implicit = 0 }, .mm_limb } }, | 3676 | .dst_temps = .{.mem}, |
| | 3677 | .each = .{ .limb_and_mask_limb = .{ |
| | 3678 | .of = .src0x, |
| | 3679 | .of_mask = .dst0w, |
| | 3680 | .body = switch (cc) { |
| | 3681 | else => unreachable, |
| | 3682 | .e => &.{ |
| | 3683 | .{ ._, ._dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ }, |
| | 3684 | .{ ._, .p_w, .cmpeq, .tmp3x, .limb(.src1x), ._, ._ }, |
| | 3685 | .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ }, |
| | 3686 | .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, |
| | 3687 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ }, |
| | 3688 | }, |
| | 3689 | .ne => &.{ |
| | 3690 | .{ ._, ._dqu, .mov, .tmp3x, .limb(.src0x), ._, ._ }, |
| | 3691 | .{ ._, .p_w, .cmpeq, .tmp3x, .limb(.src1x), ._, ._ }, |
| | 3692 | .{ ._, .p_b, .ackssw, .tmp3x, .tmp3x, ._, ._ }, |
| | 3693 | .{ ._, .p_b, .movmsk, .tmp2d, .tmp3x, ._, ._ }, |
| | 3694 | .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, |
| | 3695 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ }, |
| | 3696 | }, |
| 3471 | }, | 3697 | }, |
| | 3698 | } }, |
| | 3699 | }, .{ |
| | 3700 | .required_features = .{ .sse2, null }, |
| | 3701 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| | 3702 | .patterns = &.{ |
| | 3703 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3472 | }, | 3704 | }, |
| 3473 | .{ | 3705 | .extra_temps = .{ |
| 3474 | .scalar = .bool, | 3706 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3475 | .clobbers = .{ .eflags = true }, | 3707 | .{ .type = .usize, .kind = .{ .reg = .rcx } }, |
| 3476 | .invert_result = true, | 3708 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3477 | .loop = .elementwise, | 3709 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3478 | .mir_tag = .{ ._, .xor }, | 3710 | .{ .kind = .{ .rc = .sse } }, |
| 3479 | .patterns = &.{ | 3711 | .unused, |
| 3480 | .{ .ops = &.{ .mem_limb, .{ .implicit = 0 }, .gpr_limb } }, | 3712 | }, |
| 3481 | .{ .ops = &.{ .mem_limb, .gpr_limb, .{ .implicit = 0 } } }, | 3713 | .dst_temps = .{.mem}, |
| 3482 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .mem_limb } }, | 3714 | .each = .{ .limb_and_mask_limb = .{ |
| 3483 | .{ .ops = &.{ .gpr_limb, .mem_limb, .{ .implicit = 0 } } }, | 3715 | .of = .src0x, |
| 3484 | .{ .ops = &.{ .gpr_limb, .{ .implicit = 0 }, .gpr_limb } }, | 3716 | .of_mask = .dst0d, |
| | 3717 | .body = switch (cc) { |
| | 3718 | else => unreachable, |
| | 3719 | .e => &.{ |
| | 3720 | .{ ._, ._dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ }, |
| | 3721 | .{ ._, .p_q, .cmpeq, .tmp4x, .limb(.src1x), ._, ._ }, |
| | 3722 | .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, |
| | 3723 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3724 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3725 | }, |
| | 3726 | .ne => &.{ |
| | 3727 | .{ ._, ._dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ }, |
| | 3728 | .{ ._, .p_d, .cmpeq, .tmp4x, .limb(.src1x), ._, ._ }, |
| | 3729 | .{ ._, ._ps, .movmsk, .tmp3d, .tmp4x, ._, ._ }, |
| | 3730 | .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ }, |
| | 3731 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3732 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3733 | }, |
| 3485 | }, | 3734 | }, |
| | 3735 | } }, |
| | 3736 | }, .{ |
| | 3737 | .required_features = .{ .sse4_1, null }, |
| | 3738 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| | 3739 | .patterns = &.{ |
| | 3740 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3486 | }, | 3741 | }, |
| 3487 | .{ | 3742 | .extra_temps = .{ |
| 3488 | .scalar = .{ .any_int = .byte }, | 3743 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3489 | .clobbers = .{ .eflags = true }, | 3744 | .{ .type = .usize, .kind = .{ .reg = .rcx } }, |
| 3490 | .loop = .elementwise, | 3745 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3491 | .mir_tag = .{ ._, .cmp }, | 3746 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3492 | .patterns = &.{ | 3747 | .{ .kind = .{ .rc = .sse } }, |
| 3493 | .{ .ops = &.{ .cc_elem, .mem_elem, .gpr_elem } }, | 3748 | .unused, |
| 3494 | .{ .ops = &.{ .cc_elem, .gpr_elem, .mem_elem } }, | 3749 | }, |
| 3495 | .{ .ops = &.{ .cc_elem, .gpr_elem, .gpr_elem } }, | 3750 | .dst_temps = .{.mem}, |
| | 3751 | .each = .{ .limb_and_mask_limb = .{ |
| | 3752 | .of = .src0x, |
| | 3753 | .of_mask = .dst0q, |
| | 3754 | .body = switch (cc) { |
| | 3755 | else => unreachable, |
| | 3756 | .e => &.{ |
| | 3757 | .{ ._, ._dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ }, |
| | 3758 | .{ ._, .p_q, .cmpeq, .tmp4x, .limb(.src1x), ._, ._ }, |
| | 3759 | .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, |
| | 3760 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3761 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3762 | }, |
| | 3763 | .ne => &.{ |
| | 3764 | .{ ._, ._dqu, .mov, .tmp4x, .limb(.src0x), ._, ._ }, |
| | 3765 | .{ ._, .p_q, .cmpeq, .tmp4x, .limb(.src1x), ._, ._ }, |
| | 3766 | .{ ._, ._pd, .movmsk, .tmp3d, .tmp4x, ._, ._ }, |
| | 3767 | .{ ._, ._, .xor, .tmp3b, .i(0b11), ._, ._ }, |
| | 3768 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3769 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3770 | }, |
| 3496 | }, | 3771 | }, |
| | 3772 | } }, |
| | 3773 | }, .{ |
| | 3774 | .required_features = .{ .mmx, null }, |
| | 3775 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 3776 | .patterns = &.{ |
| | 3777 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3497 | }, | 3778 | }, |
| 3498 | .{ | 3779 | .extra_temps = .{ |
| 3499 | .scalar = .{ .any_int = .word }, | 3780 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3500 | .clobbers = .{ .eflags = true }, | 3781 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3501 | .loop = .elementwise, | 3782 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| 3502 | .mir_tag = .{ ._, .cmp }, | 3783 | .{ .kind = .{ .rc = .mmx } }, |
| 3503 | .patterns = &.{ | 3784 | .unused, |
| 3504 | .{ .ops = &.{ .cc_elem, .mem_elem, .gpr_elem } }, | 3785 | .unused, |
| 3505 | .{ .ops = &.{ .cc_elem, .gpr_elem, .mem_elem } }, | 3786 | }, |
| 3506 | .{ .ops = &.{ .cc_elem, .gpr_elem, .gpr_elem } }, | 3787 | .dst_temps = .{.mem}, |
| | 3788 | .each = .{ .limb_and_mask_limb = .{ |
| | 3789 | .of = .src0q, |
| | 3790 | .of_mask = .dst0b, |
| | 3791 | .body = switch (cc) { |
| | 3792 | else => unreachable, |
| | 3793 | .e => &.{ |
| | 3794 | .{ ._, ._dqu, .mov, .tmp3q, .limb(.src0q), ._, ._ }, |
| | 3795 | .{ ._, .p_b, .cmpeq, .tmp3q, .limb(.src1q), ._, ._ }, |
| | 3796 | .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ }, |
| | 3797 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ }, |
| | 3798 | }, |
| | 3799 | .ne => &.{ |
| | 3800 | .{ ._, ._dqu, .mov, .tmp3q, .limb(.src0q), ._, ._ }, |
| | 3801 | .{ ._, .p_b, .cmpeq, .tmp3q, .limb(.src1q), ._, ._ }, |
| | 3802 | .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ }, |
| | 3803 | .{ ._, ._, .not, .tmp2w, ._, ._, ._ }, |
| | 3804 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2w, ._, ._ }, |
| | 3805 | }, |
| 3507 | }, | 3806 | }, |
| | 3807 | } }, |
| | 3808 | }, .{ |
| | 3809 | .required_features = .{ .mmx, null }, |
| | 3810 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| | 3811 | .patterns = &.{ |
| | 3812 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3508 | }, | 3813 | }, |
| 3509 | .{ | 3814 | .extra_temps = .{ |
| 3510 | .scalar = .{ .any_int = .dword }, | 3815 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3511 | .clobbers = .{ .eflags = true }, | 3816 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3512 | .loop = .elementwise, | 3817 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| 3513 | .mir_tag = .{ ._, .cmp }, | 3818 | .{ .kind = .{ .rc = .mmx } }, |
| 3514 | .patterns = &.{ | 3819 | .unused, |
| 3515 | .{ .ops = &.{ .cc_elem, .mem_elem, .gpr_elem } }, | 3820 | .unused, |
| 3516 | .{ .ops = &.{ .cc_elem, .gpr_elem, .mem_elem } }, | 3821 | }, |
| 3517 | .{ .ops = &.{ .cc_elem, .gpr_elem, .gpr_elem } }, | 3822 | .dst_temps = .{.mem}, |
| | 3823 | .each = .{ .limb_and_mask_limb = .{ |
| | 3824 | .of = .src0q, |
| | 3825 | .of_mask = .dst0w, |
| | 3826 | .body = switch (cc) { |
| | 3827 | else => unreachable, |
| | 3828 | .e => &.{ |
| | 3829 | .{ ._, ._dqu, .mov, .tmp3q, .limb(.src0q), ._, ._ }, |
| | 3830 | .{ ._, .p_w, .cmpeq, .tmp3q, .limb(.src1q), ._, ._ }, |
| | 3831 | .{ ._, .p_b, .ackssw, .tmp3q, .tmp3q, ._, ._ }, |
| | 3832 | .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ }, |
| | 3833 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ }, |
| | 3834 | }, |
| | 3835 | .ne => &.{ |
| | 3836 | .{ ._, ._dqu, .mov, .tmp3q, .limb(.src0q), ._, ._ }, |
| | 3837 | .{ ._, .p_w, .cmpeq, .tmp3q, .limb(.src1q), ._, ._ }, |
| | 3838 | .{ ._, .p_b, .ackssw, .tmp3q, .tmp3q, ._, ._ }, |
| | 3839 | .{ ._, .p_b, .movmsk, .tmp2d, .tmp3q, ._, ._ }, |
| | 3840 | .{ ._, ._, .not, .tmp2b, ._, ._, ._ }, |
| | 3841 | .{ ._, ._, .mov, .maskLimb(.dst0w), .tmp2b, ._, ._ }, |
| | 3842 | }, |
| 3518 | }, | 3843 | }, |
| | 3844 | } }, |
| | 3845 | }, .{ |
| | 3846 | .required_features = .{ .mmx, null }, |
| | 3847 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| | 3848 | .patterns = &.{ |
| | 3849 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3519 | }, | 3850 | }, |
| 3520 | .{ | 3851 | .extra_temps = .{ |
| 3521 | .scalar = .{ .any_int = .qword }, | 3852 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3522 | .clobbers = .{ .eflags = true }, | 3853 | .{ .type = .usize, .kind = .{ .reg = .rcx } }, |
| 3523 | .loop = .elementwise, | 3854 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3524 | .mir_tag = .{ ._, .cmp }, | 3855 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3525 | .patterns = &.{ | 3856 | .{ .kind = .{ .rc = .mmx } }, |
| 3526 | .{ .ops = &.{ .cc_elem, .mem_elem, .gpr_elem } }, | 3857 | .unused, |
| 3527 | .{ .ops = &.{ .cc_elem, .gpr_elem, .mem_elem } }, | 3858 | }, |
| 3528 | .{ .ops = &.{ .cc_elem, .gpr_elem, .gpr_elem } }, | 3859 | .dst_temps = .{.mem}, |
| | 3860 | .each = .{ .limb_and_mask_limb = .{ |
| | 3861 | .of = .src0q, |
| | 3862 | .of_mask = .dst0d, |
| | 3863 | .body = switch (cc) { |
| | 3864 | else => unreachable, |
| | 3865 | .e => &.{ |
| | 3866 | .{ ._, ._dqu, .mov, .tmp4q, .limb(.src0q), ._, ._ }, |
| | 3867 | .{ ._, .p_q, .cmpeq, .tmp4q, .limb(.src1q), ._, ._ }, |
| | 3868 | .{ ._, ._ps, .movmsk, .tmp3d, .tmp4q, ._, ._ }, |
| | 3869 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3870 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3871 | }, |
| | 3872 | .ne => &.{ |
| | 3873 | .{ ._, ._dqu, .mov, .tmp4q, .limb(.src0q), ._, ._ }, |
| | 3874 | .{ ._, .p_q, .cmpeq, .tmp4q, .limb(.src1q), ._, ._ }, |
| | 3875 | .{ ._, ._ps, .movmsk, .tmp3d, .tmp4q, ._, ._ }, |
| | 3876 | .{ ._, ._, .xor, .tmp3b, .i(0b1111), ._, ._ }, |
| | 3877 | .{ ._, ._l, .ro, .tmp3b, .tmp1b, ._, ._ }, |
| | 3878 | .{ ._, ._, .@"or", .tmp2b, .tmp3b, ._, ._ }, |
| | 3879 | }, |
| 3529 | }, | 3880 | }, |
| | 3881 | } }, |
| | 3882 | }, .{ |
| | 3883 | .required_features = .{ .slow_incdec, null }, |
| | 3884 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| | 3885 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 3886 | .patterns = &.{ |
| | 3887 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3530 | }, | 3888 | }, |
| | 3889 | .extra_temps = .{ |
| | 3890 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 3891 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 3892 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 3893 | .unused, |
| | 3894 | .unused, |
| | 3895 | .unused, |
| | 3896 | }, |
| | 3897 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 3898 | .each = .{ .once = &.{ |
| | 3899 | .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, |
| | 3900 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 3901 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 3902 | .{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, |
| | 3903 | .{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, |
| | 3904 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 3905 | .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, |
| | 3906 | .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, |
| | 3907 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 3908 | .{ ._, ._, .add, .tmp0p, .i(1), ._, ._ }, |
| | 3909 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 3910 | } }, |
| 3531 | }, .{ | 3911 | }, .{ |
| 3532 | .cc = .e, | 3912 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| 3533 | .invert_result = switch (cmp_op) { | 3913 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| 3534 | .eq => false, | 3914 | .patterns = &.{ |
| 3535 | .neq => true, | 3915 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3536 | else => unreachable, | | |
| 3537 | }, | 3916 | }, |
| 3538 | }) catch |err| switch (err) { | 3917 | .extra_temps = .{ |
| 3539 | error.SelectFailed => return cg.fail("failed to select", .{}), | 3918 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3540 | else => |e| return e, | 3919 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| 3541 | }, | 3920 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| 3542 | else => |e| return e, | 3921 | .unused, |
| 3543 | }, | 3922 | .unused, |
| 3544 | .gte => unreachable, | 3923 | .unused, |
| 3545 | .gt => unreachable, | 3924 | }, |
| 3546 | } | 3925 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| 3547 | if (ops[0].index != res[0].index) try ops[0].die(cg); | 3926 | .each = .{ .once = &.{ |
| 3548 | if (ops[1].index != res[0].index) try ops[1].die(cg); | 3927 | .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, |
| 3549 | try res[0].moveTo(inst, cg); | 3928 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| 3550 | }, | 3929 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| 3551 | | 3930 | .{ .@"0:", ._, .mov, .tmp2b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, |
| 3552 | .cmp_lt, | 3931 | .{ ._, ._, .cmp, .tmp2b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, |
| 3553 | .cmp_lt_optimized, | 3932 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| 3554 | .cmp_lte, | 3933 | .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, |
| 3555 | .cmp_lte_optimized, | 3934 | .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, |
| 3556 | .cmp_gte, | 3935 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| 3557 | .cmp_gte_optimized, | 3936 | .{ ._, ._, .inc, .tmp0p, ._, ._, ._ }, |
| 3558 | .cmp_gt, | 3937 | .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, |
| 3559 | .cmp_gt_optimized, | 3938 | } }, |
| 3560 | => |air_tag| if (use_old) try cg.airCmp(inst, switch (air_tag) { | 3939 | }, .{ |
| 3561 | else => unreachable, | 3940 | .required_features = .{ .slow_incdec, null }, |
| 3562 | .cmp_lt, .cmp_lt_optimized => .lt, | 3941 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| 3563 | .cmp_lte, .cmp_lte_optimized => .lte, | 3942 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| 3564 | .cmp_gte, .cmp_gte_optimized => .gte, | 3943 | .patterns = &.{ |
| 3565 | .cmp_gt, .cmp_gt_optimized => .gt, | 3944 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3566 | }) else { | 3945 | }, |
| 3567 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; | 3946 | .extra_temps = .{ |
| 3568 | const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu); | 3947 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3569 | const signedness = if (scalar_ty.isAbiInt(zcu)) | 3948 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| 3570 | scalar_ty.intInfo(zcu).signedness | 3949 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| 3571 | else | 3950 | .unused, |
| 3572 | .unsigned; | 3951 | .unused, |
| 3573 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); | 3952 | .unused, |
| 3574 | var res: [1]Temp = undefined; | 3953 | }, |
| 3575 | cg.select2(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (signedness) { | 3954 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| 3576 | .signed => switch (air_tag) { | 3955 | .each = .{ .once = &.{ |
| 3577 | else => unreachable, | 3956 | .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, |
| 3578 | .cmp_lt, .cmp_lt_optimized => .l, | 3957 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| 3579 | .cmp_lte, .cmp_lte_optimized => .le, | 3958 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| 3580 | .cmp_gte, .cmp_gte_optimized => .ge, | 3959 | .{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, |
| 3581 | .cmp_gt, .cmp_gt_optimized => .g, | 3960 | .{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, |
| 3582 | }, | 3961 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| 3583 | .unsigned => switch (air_tag) { | 3962 | .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, |
| 3584 | else => unreachable, | 3963 | .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, |
| 3585 | .cmp_lt, .cmp_lt_optimized => .b, | 3964 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| 3586 | .cmp_lte, .cmp_lte_optimized => .be, | 3965 | .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ }, |
| 3587 | .cmp_gte, .cmp_gte_optimized => .ae, | 3966 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| 3588 | .cmp_gt, .cmp_gt_optimized => .a, | 3967 | } }, |
| 3589 | }, | 3968 | }, .{ |
| 3590 | })) { | 3969 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| 3591 | else => unreachable, | 3970 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| 3592 | inline .l, .le, .ge, .g, .b, .be, .ae, .a => |cc| comptime &.{ .{ | 3971 | .patterns = &.{ |
| 3593 | .constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, | 3972 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3594 | .patterns = &.{ | 3973 | }, |
| 3595 | .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } }, | 3974 | .extra_temps = .{ |
| 3596 | .{ .src = .{ .imm8, .gpr }, .commute = .{ 0, 1 } }, | 3975 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3597 | .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } }, | 3976 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| 3598 | }, | 3977 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| 3599 | .clobbers = .{ .eflags = true }, | 3978 | .unused, |
| 3600 | .dst_temps = .{.{ .cc = cc.commute() }}, | 3979 | .unused, |
| 3601 | .each = .{ .once = &.{ | 3980 | .unused, |
| 3602 | .{ ._, .cmp, .src0b, .src1b, .none, .none }, | 3981 | }, |
| 3603 | } }, | 3982 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| 3604 | }, .{ | 3983 | .each = .{ .once = &.{ |
| 3605 | .constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, | 3984 | .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, |
| | 3985 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 3986 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 3987 | .{ .@"0:", ._, .mov, .tmp2w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, |
| | 3988 | .{ ._, ._, .cmp, .tmp2w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, |
| | 3989 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 3990 | .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, |
| | 3991 | .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, |
| | 3992 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 3993 | .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ }, |
| | 3994 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 3995 | } }, |
| | 3996 | }, .{ |
| | 3997 | .required_features = .{ .slow_incdec, null }, |
| | 3998 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| | 3999 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| | 4000 | .patterns = &.{ |
| | 4001 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4002 | }, |
| | 4003 | .extra_temps = .{ |
| | 4004 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4005 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4006 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4007 | .unused, |
| | 4008 | .unused, |
| | 4009 | .unused, |
| | 4010 | }, |
| | 4011 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4012 | .each = .{ .once = &.{ |
| | 4013 | .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, |
| | 4014 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4015 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4016 | .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, |
| | 4017 | .{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, |
| | 4018 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4019 | .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, |
| | 4020 | .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, |
| | 4021 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4022 | .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ }, |
| | 4023 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4024 | } }, |
| | 4025 | }, .{ |
| | 4026 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| | 4027 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| | 4028 | .patterns = &.{ |
| | 4029 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4030 | }, |
| | 4031 | .extra_temps = .{ |
| | 4032 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4033 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4034 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4035 | .unused, |
| | 4036 | .unused, |
| | 4037 | .unused, |
| | 4038 | }, |
| | 4039 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4040 | .each = .{ .once = &.{ |
| | 4041 | .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, |
| | 4042 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4043 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4044 | .{ .@"0:", ._, .mov, .tmp2d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, |
| | 4045 | .{ ._, ._, .cmp, .tmp2d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, |
| | 4046 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4047 | .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, |
| | 4048 | .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, |
| | 4049 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4050 | .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ }, |
| | 4051 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4052 | } }, |
| | 4053 | }, .{ |
| | 4054 | .required_features = .{ .@"64bit", .slow_incdec }, |
| | 4055 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| | 4056 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| | 4057 | .patterns = &.{ |
| | 4058 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4059 | }, |
| | 4060 | .extra_temps = .{ |
| | 4061 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4062 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4063 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4064 | .unused, |
| | 4065 | .unused, |
| | 4066 | .unused, |
| | 4067 | }, |
| | 4068 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4069 | .each = .{ .once = &.{ |
| | 4070 | .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, |
| | 4071 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4072 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4073 | .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, |
| | 4074 | .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, |
| | 4075 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4076 | .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, |
| | 4077 | .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, |
| | 4078 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4079 | .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ }, |
| | 4080 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4081 | } }, |
| | 4082 | }, .{ |
| | 4083 | .required_features = .{ .@"64bit", null }, |
| | 4084 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| | 4085 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| | 4086 | .patterns = &.{ |
| | 4087 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4088 | }, |
| | 4089 | .extra_temps = .{ |
| | 4090 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4091 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4092 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4093 | .unused, |
| | 4094 | .unused, |
| | 4095 | .unused, |
| | 4096 | }, |
| | 4097 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4098 | .each = .{ .once = &.{ |
| | 4099 | .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, |
| | 4100 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4101 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4102 | .{ .@"0:", ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, |
| | 4103 | .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, |
| | 4104 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4105 | .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, |
| | 4106 | .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, |
| | 4107 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4108 | .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ }, |
| | 4109 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4110 | } }, |
| | 4111 | }, .{ |
| | 4112 | .required_features = .{ .slow_incdec, null }, |
| | 4113 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| | 4114 | .patterns = &.{ |
| | 4115 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4116 | }, |
| | 4117 | .extra_temps = .{ |
| | 4118 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4119 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4120 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4121 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4122 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4123 | .unused, |
| | 4124 | }, |
| | 4125 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4126 | .each = .{ .once = &.{ |
| | 4127 | .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, |
| | 4128 | .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, |
| | 4129 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4130 | .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ }, |
| | 4131 | .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, |
| | 4132 | .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, |
| | 4133 | .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, |
| | 4134 | .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, |
| | 4135 | .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ }, |
| | 4136 | .{ ._, ._, .sub, .tmp2d, .i(1), ._, ._ }, |
| | 4137 | .{ ._, ._b, .j, .@"1b", ._, ._, ._ }, |
| | 4138 | .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, |
| | 4139 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4140 | .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, |
| | 4141 | .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, |
| | 4142 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4143 | .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ }, |
| | 4144 | .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, |
| | 4145 | } }, |
| | 4146 | }, .{ |
| | 4147 | .dst_constraints = .{.{ .bool_vec = .byte }}, |
| | 4148 | .patterns = &.{ |
| | 4149 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4150 | }, |
| | 4151 | .extra_temps = .{ |
| | 4152 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4153 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4154 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4155 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4156 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4157 | .unused, |
| | 4158 | }, |
| | 4159 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4160 | .each = .{ .once = &.{ |
| | 4161 | .{ ._, ._, .xor, .dst0b, .dst0b, ._, ._ }, |
| | 4162 | .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, |
| | 4163 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4164 | .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ }, |
| | 4165 | .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, |
| | 4166 | .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, |
| | 4167 | .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, |
| | 4168 | .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, |
| | 4169 | .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ }, |
| | 4170 | .{ ._, ._, .dec, .tmp2d, ._, ._, ._ }, |
| | 4171 | .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, |
| | 4172 | .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, |
| | 4173 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4174 | .{ ._, ._l, .sh, .tmp2b, .tmp1b, ._, ._ }, |
| | 4175 | .{ ._, ._, .@"or", .dst0b, .tmp2b, ._, ._ }, |
| | 4176 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4177 | .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ }, |
| | 4178 | .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, |
| | 4179 | } }, |
| | 4180 | }, .{ |
| | 4181 | .required_features = .{ .slow_incdec, null }, |
| | 4182 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| | 4183 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 4184 | .patterns = &.{ |
| | 4185 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4186 | }, |
| | 4187 | .extra_temps = .{ |
| | 4188 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4189 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4190 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4191 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 4192 | .unused, |
| | 4193 | .unused, |
| | 4194 | }, |
| | 4195 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4196 | .each = .{ .once = &.{ |
| | 4197 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4198 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4199 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4200 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4201 | .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, |
| | 4202 | .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, |
| | 4203 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4204 | .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, |
| | 4205 | .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, |
| | 4206 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4207 | .{ ._, ._, .add, .tmp0p, .i(1), ._, ._ }, |
| | 4208 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4209 | } }, |
| | 4210 | }, .{ |
| | 4211 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| | 4212 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 4213 | .patterns = &.{ |
| | 4214 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4215 | }, |
| | 4216 | .extra_temps = .{ |
| | 4217 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4218 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4219 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4220 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 4221 | .unused, |
| | 4222 | .unused, |
| | 4223 | }, |
| | 4224 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4225 | .each = .{ .once = &.{ |
| | 4226 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4227 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4228 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4229 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4230 | .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, |
| | 4231 | .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, |
| | 4232 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4233 | .{ ._, ._l, .sh, .tmp2d, .tmp1d, ._, ._ }, |
| | 4234 | .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, |
| | 4235 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4236 | .{ ._, ._, .inc, .tmp0p, ._, ._, ._ }, |
| | 4237 | .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, |
| | 4238 | } }, |
| | 4239 | }, .{ |
| | 4240 | .required_features = .{ .slow_incdec, null }, |
| | 4241 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| | 4242 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| | 4243 | .patterns = &.{ |
| | 4244 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4245 | }, |
| | 4246 | .extra_temps = .{ |
| | 4247 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4248 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4249 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4250 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| | 4251 | .unused, |
| | 4252 | .unused, |
| | 4253 | }, |
| | 4254 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4255 | .each = .{ .once = &.{ |
| | 4256 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4257 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4258 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4259 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4260 | .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, |
| | 4261 | .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, |
| | 4262 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4263 | .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, |
| | 4264 | .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, |
| | 4265 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4266 | .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ }, |
| | 4267 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4268 | } }, |
| | 4269 | }, .{ |
| | 4270 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| | 4271 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| | 4272 | .patterns = &.{ |
| | 4273 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4274 | }, |
| | 4275 | .extra_temps = .{ |
| | 4276 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4277 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4278 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4279 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| | 4280 | .unused, |
| | 4281 | .unused, |
| | 4282 | }, |
| | 4283 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4284 | .each = .{ .once = &.{ |
| | 4285 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4286 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4287 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4288 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4289 | .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, |
| | 4290 | .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, |
| | 4291 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4292 | .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, |
| | 4293 | .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, |
| | 4294 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4295 | .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ }, |
| | 4296 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4297 | } }, |
| | 4298 | }, .{ |
| | 4299 | .required_features = .{ .slow_incdec, null }, |
| | 4300 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| | 4301 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| | 4302 | .patterns = &.{ |
| | 4303 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4304 | }, |
| | 4305 | .extra_temps = .{ |
| | 4306 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4307 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4308 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4309 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4310 | .unused, |
| | 4311 | .unused, |
| | 4312 | }, |
| | 4313 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4314 | .each = .{ .once = &.{ |
| | 4315 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4316 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4317 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4318 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4319 | .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, |
| | 4320 | .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, |
| | 4321 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4322 | .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, |
| | 4323 | .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, |
| | 4324 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4325 | .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ }, |
| | 4326 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4327 | } }, |
| | 4328 | }, .{ |
| | 4329 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| | 4330 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| | 4331 | .patterns = &.{ |
| | 4332 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4333 | }, |
| | 4334 | .extra_temps = .{ |
| | 4335 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4336 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4337 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4338 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4339 | .unused, |
| | 4340 | .unused, |
| | 4341 | }, |
| | 4342 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4343 | .each = .{ .once = &.{ |
| | 4344 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4345 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4346 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4347 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4348 | .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, |
| | 4349 | .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, |
| | 4350 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4351 | .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, |
| | 4352 | .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, |
| | 4353 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4354 | .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ }, |
| | 4355 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4356 | } }, |
| | 4357 | }, .{ |
| | 4358 | .required_features = .{ .@"64bit", .slow_incdec }, |
| | 4359 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| | 4360 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| | 4361 | .patterns = &.{ |
| | 4362 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4363 | }, |
| | 4364 | .extra_temps = .{ |
| | 4365 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4366 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4367 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4368 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4369 | .unused, |
| | 4370 | .unused, |
| | 4371 | }, |
| | 4372 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4373 | .each = .{ .once = &.{ |
| | 4374 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4375 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4376 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4377 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4378 | .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, |
| | 4379 | .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, |
| | 4380 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4381 | .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, |
| | 4382 | .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, |
| | 4383 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4384 | .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ }, |
| | 4385 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4386 | } }, |
| | 4387 | }, .{ |
| | 4388 | .required_features = .{ .@"64bit", null }, |
| | 4389 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| | 4390 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| | 4391 | .patterns = &.{ |
| | 4392 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4393 | }, |
| | 4394 | .extra_temps = .{ |
| | 4395 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4396 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4397 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4398 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4399 | .unused, |
| | 4400 | .unused, |
| | 4401 | }, |
| | 4402 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4403 | .each = .{ .once = &.{ |
| | 4404 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4405 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4406 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4407 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4408 | .{ ._, ._, .mov, .tmp3q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, |
| | 4409 | .{ ._, ._, .cmp, .tmp3q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, |
| | 4410 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4411 | .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, |
| | 4412 | .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, |
| | 4413 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4414 | .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ }, |
| | 4415 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4416 | } }, |
| | 4417 | }, .{ |
| | 4418 | .required_features = .{ .slow_incdec, null }, |
| | 4419 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| | 4420 | .patterns = &.{ |
| | 4421 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4422 | }, |
| | 4423 | .extra_temps = .{ |
| | 4424 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4425 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4426 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4427 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4428 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4429 | .unused, |
| | 4430 | }, |
| | 4431 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4432 | .each = .{ .once = &.{ |
| | 4433 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4434 | .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, |
| | 4435 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4436 | .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ }, |
| | 4437 | .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, |
| | 4438 | .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, |
| | 4439 | .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, |
| | 4440 | .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, |
| | 4441 | .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ }, |
| | 4442 | .{ ._, ._, .sub, .tmp2d, .i(1), ._, ._ }, |
| | 4443 | .{ ._, ._b, .j, .@"1b", ._, ._, ._ }, |
| | 4444 | .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4445 | .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, |
| | 4446 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4447 | .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, |
| | 4448 | .{ ._, ._, .@"or", .dst0d, .tmp2d, ._, ._ }, |
| | 4449 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4450 | .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ }, |
| | 4451 | .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, |
| | 4452 | } }, |
| | 4453 | }, .{ |
| | 4454 | .dst_constraints = .{.{ .bool_vec = .dword }}, |
| | 4455 | .patterns = &.{ |
| | 4456 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4457 | }, |
| | 4458 | .extra_temps = .{ |
| | 4459 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4460 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4461 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4462 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4463 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4464 | .unused, |
| | 4465 | }, |
| | 4466 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4467 | .each = .{ .once = &.{ |
| | 4468 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4469 | .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, |
| | 4470 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4471 | .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ }, |
| | 4472 | .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, |
| | 4473 | .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, |
| | 4474 | .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, |
| | 4475 | .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, |
| | 4476 | .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ }, |
| | 4477 | .{ ._, ._, .dec, .tmp2d, ._, ._, ._ }, |
| | 4478 | .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, |
| | 4479 | .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4480 | .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, |
| | 4481 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4482 | .{ ._, ._l, .sh, .tmp2d, .tmp1b, ._, ._ }, |
| | 4483 | .{ ._, ._, .@"or", .dst0d, .tmp2b, ._, ._ }, |
| | 4484 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4485 | .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ }, |
| | 4486 | .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, |
| | 4487 | } }, |
| | 4488 | }, .{ |
| | 4489 | .required_features = .{ .@"64bit", .slow_incdec }, |
| | 4490 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| | 4491 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 4492 | .patterns = &.{ |
| | 4493 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4494 | }, |
| | 4495 | .extra_temps = .{ |
| | 4496 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4497 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4498 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4499 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 4500 | .unused, |
| | 4501 | .unused, |
| | 4502 | }, |
| | 4503 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4504 | .each = .{ .once = &.{ |
| | 4505 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4506 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4507 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4508 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4509 | .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, |
| | 4510 | .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, |
| | 4511 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4512 | .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, |
| | 4513 | .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, |
| | 4514 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4515 | .{ ._, ._, .add, .tmp0p, .i(1), ._, ._ }, |
| | 4516 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4517 | } }, |
| | 4518 | }, .{ |
| | 4519 | .required_features = .{ .@"64bit", null }, |
| | 4520 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| | 4521 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 4522 | .patterns = &.{ |
| | 4523 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4524 | }, |
| | 4525 | .extra_temps = .{ |
| | 4526 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4527 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4528 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4529 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 4530 | .unused, |
| | 4531 | .unused, |
| | 4532 | }, |
| | 4533 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4534 | .each = .{ .once = &.{ |
| | 4535 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4536 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4537 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4538 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4539 | .{ ._, ._, .mov, .tmp3b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, |
| | 4540 | .{ ._, ._, .cmp, .tmp3b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, |
| | 4541 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4542 | .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, |
| | 4543 | .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, |
| | 4544 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4545 | .{ ._, ._, .inc, .tmp0p, ._, ._, ._ }, |
| | 4546 | .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, |
| | 4547 | } }, |
| | 4548 | }, .{ |
| | 4549 | .required_features = .{ .@"64bit", .slow_incdec }, |
| | 4550 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| | 4551 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| | 4552 | .patterns = &.{ |
| | 4553 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4554 | }, |
| | 4555 | .extra_temps = .{ |
| | 4556 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4557 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4558 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4559 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| | 4560 | .unused, |
| | 4561 | .unused, |
| | 4562 | }, |
| | 4563 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4564 | .each = .{ .once = &.{ |
| | 4565 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4566 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4567 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4568 | .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, |
| | 4569 | .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, |
| | 4570 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4571 | .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, |
| | 4572 | .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, |
| | 4573 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4574 | .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ }, |
| | 4575 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4576 | } }, |
| | 4577 | }, .{ |
| | 4578 | .required_features = .{ .@"64bit", null }, |
| | 4579 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| | 4580 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| | 4581 | .patterns = &.{ |
| | 4582 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4583 | }, |
| | 4584 | .extra_temps = .{ |
| | 4585 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4586 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4587 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4588 | .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, |
| | 4589 | .unused, |
| | 4590 | .unused, |
| | 4591 | }, |
| | 4592 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4593 | .each = .{ .once = &.{ |
| | 4594 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4595 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4596 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4597 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4598 | .{ ._, ._, .mov, .tmp3w, .memia(.src0w, .tmp0, .add_size), ._, ._ }, |
| | 4599 | .{ ._, ._, .cmp, .tmp3w, .memia(.src1w, .tmp0, .add_size), ._, ._ }, |
| | 4600 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4601 | .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, |
| | 4602 | .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, |
| | 4603 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4604 | .{ ._, ._, .add, .tmp0p, .i(2), ._, ._ }, |
| | 4605 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4606 | } }, |
| | 4607 | }, .{ |
| | 4608 | .required_features = .{ .@"64bit", .slow_incdec }, |
| | 4609 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| | 4610 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| | 4611 | .patterns = &.{ |
| | 4612 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4613 | }, |
| | 4614 | .extra_temps = .{ |
| | 4615 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4616 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4617 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4618 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4619 | .unused, |
| | 4620 | .unused, |
| | 4621 | }, |
| | 4622 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4623 | .each = .{ .once = &.{ |
| | 4624 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4625 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4626 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4627 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4628 | .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, |
| | 4629 | .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, |
| | 4630 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4631 | .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, |
| | 4632 | .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, |
| | 4633 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4634 | .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ }, |
| | 4635 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4636 | } }, |
| | 4637 | }, .{ |
| | 4638 | .required_features = .{ .@"64bit", null }, |
| | 4639 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| | 4640 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| | 4641 | .patterns = &.{ |
| | 4642 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4643 | }, |
| | 4644 | .extra_temps = .{ |
| | 4645 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4646 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4647 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4648 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| | 4649 | .unused, |
| | 4650 | .unused, |
| | 4651 | }, |
| | 4652 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4653 | .each = .{ .once = &.{ |
| | 4654 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4655 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4656 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4657 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4658 | .{ ._, ._, .mov, .tmp3d, .memia(.src0d, .tmp0, .add_size), ._, ._ }, |
| | 4659 | .{ ._, ._, .cmp, .tmp3d, .memia(.src1d, .tmp0, .add_size), ._, ._ }, |
| | 4660 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4661 | .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, |
| | 4662 | .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, |
| | 4663 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4664 | .{ ._, ._, .add, .tmp0p, .i(4), ._, ._ }, |
| | 4665 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4666 | } }, |
| | 4667 | }, .{ |
| | 4668 | .required_features = .{ .@"64bit", .slow_incdec }, |
| | 4669 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| | 4670 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| | 4671 | .patterns = &.{ |
| | 4672 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4673 | }, |
| | 4674 | .extra_temps = .{ |
| | 4675 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4676 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4677 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4678 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4679 | .unused, |
| | 4680 | .unused, |
| | 4681 | }, |
| | 4682 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4683 | .each = .{ .once = &.{ |
| | 4684 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4685 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4686 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4687 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4688 | .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, |
| | 4689 | .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, |
| | 4690 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4691 | .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, |
| | 4692 | .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, |
| | 4693 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4694 | .{ ._, ._, .add, .tmp0p, .i(8), ._, ._ }, |
| | 4695 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4696 | } }, |
| | 4697 | }, .{ |
| | 4698 | .required_features = .{ .@"64bit", null }, |
| | 4699 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| | 4700 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| | 4701 | .patterns = &.{ |
| | 4702 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4703 | }, |
| | 4704 | .extra_temps = .{ |
| | 4705 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4706 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4707 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4708 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, |
| | 4709 | .unused, |
| | 4710 | .unused, |
| | 4711 | }, |
| | 4712 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4713 | .each = .{ .once = &.{ |
| | 4714 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4715 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4716 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4717 | .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4718 | .{ ._, ._, .mov, .tmp2q, .memia(.src0q, .tmp0, .add_size), ._, ._ }, |
| | 4719 | .{ ._, ._, .cmp, .tmp2q, .memia(.src1q, .tmp0, .add_size), ._, ._ }, |
| | 4720 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4721 | .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, |
| | 4722 | .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, |
| | 4723 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4724 | .{ ._, ._, .add, .tmp0p, .i(8), ._, ._ }, |
| | 4725 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4726 | } }, |
| | 4727 | }, .{ |
| | 4728 | .required_features = .{ .@"64bit", .slow_incdec }, |
| | 4729 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| | 4730 | .patterns = &.{ |
| | 4731 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4732 | }, |
| | 4733 | .extra_temps = .{ |
| | 4734 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4735 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4736 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4737 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4738 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4739 | .unused, |
| | 4740 | }, |
| | 4741 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4742 | .each = .{ .once = &.{ |
| | 4743 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4744 | .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, |
| | 4745 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4746 | .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ }, |
| | 4747 | .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, |
| | 4748 | .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, |
| | 4749 | .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, |
| | 4750 | .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, |
| | 4751 | .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ }, |
| | 4752 | .{ ._, ._, .sub, .tmp2d, .i(1), ._, ._ }, |
| | 4753 | .{ ._, ._b, .j, .@"1b", ._, ._, ._ }, |
| | 4754 | .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4755 | .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, |
| | 4756 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4757 | .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, |
| | 4758 | .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, |
| | 4759 | .{ ._, ._, .add, .tmp1b, .i(1), ._, ._ }, |
| | 4760 | .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ }, |
| | 4761 | .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, |
| | 4762 | } }, |
| | 4763 | }, .{ |
| | 4764 | .required_features = .{ .@"64bit", null }, |
| | 4765 | .dst_constraints = .{.{ .bool_vec = .qword }}, |
| | 4766 | .patterns = &.{ |
| | 4767 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4768 | }, |
| | 4769 | .extra_temps = .{ |
| | 4770 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4771 | .{ .type = .u8, .kind = .{ .reg = .cl } }, |
| | 4772 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4773 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4774 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4775 | .unused, |
| | 4776 | }, |
| | 4777 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| | 4778 | .each = .{ .once = &.{ |
| | 4779 | .{ ._, ._, .xor, .dst0d, .dst0d, ._, ._ }, |
| | 4780 | .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ }, |
| | 4781 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| | 4782 | .{ .@"0:", ._, .mov, .tmp2d, .a(.src0p, .add_elem_limbs), ._, ._ }, |
| | 4783 | .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ }, |
| | 4784 | .{ .@"1:", ._, .mov, .tmp4p, .memi(.src0p, .tmp0), ._, ._ }, |
| | 4785 | .{ ._, ._, .xor, .tmp4p, .memi(.src1p, .tmp0), ._, ._ }, |
| | 4786 | .{ ._, ._, .@"or", .tmp3p, .tmp4p, ._, ._ }, |
| | 4787 | .{ ._, ._, .add, .tmp0p, .a(.tmp4, .add_size), ._, ._ }, |
| | 4788 | .{ ._, ._, .dec, .tmp2d, ._, ._, ._ }, |
| | 4789 | .{ ._, ._nz, .j, .@"1b", ._, ._, ._ }, |
| | 4790 | .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4791 | .{ ._, ._, .@"test", .tmp3p, .tmp3p, ._, ._ }, |
| | 4792 | .{ ._, .fromCondition(cc), .set, .tmp2b, ._, ._, ._ }, |
| | 4793 | .{ ._, ._l, .sh, .tmp2q, .tmp1b, ._, ._ }, |
| | 4794 | .{ ._, ._, .@"or", .dst0q, .tmp2q, ._, ._ }, |
| | 4795 | .{ ._, ._, .inc, .tmp1b, ._, ._, ._ }, |
| | 4796 | .{ ._, ._, .cmp, .tmp1b, .a(.dst0, .add_len), ._, ._ }, |
| | 4797 | .{ ._, ._b, .j, .@"0b", ._, ._, ._ }, |
| | 4798 | } }, |
| | 4799 | }, .{ |
| | 4800 | .required_features = .{ .slow_incdec, null }, |
| | 4801 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 4802 | .patterns = &.{ |
| | 4803 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4804 | }, |
| | 4805 | .extra_temps = .{ |
| | 4806 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4807 | .{ .type = .u32, .kind = .{ .reg = .ecx } }, |
| | 4808 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4809 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4810 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 4811 | .unused, |
| | 4812 | }, |
| | 4813 | .dst_temps = .{.mem}, |
| | 4814 | .each = .{ .once = &.{ |
| | 4815 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4816 | .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, |
| | 4817 | .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4818 | .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, |
| | 4819 | .{ ._, ._, .mov, .tmp4b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, |
| | 4820 | .{ ._, ._, .cmp, .tmp4b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, |
| | 4821 | .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ }, |
| | 4822 | .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ }, |
| | 4823 | .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ }, |
| | 4824 | .{ ._, ._, .add, .tmp1d, .i(1), ._, ._ }, |
| | 4825 | .{ ._, ._, .@"test", .tmp1d, .ia(-1, .tmp2, .add_bit_size), ._, ._ }, |
| | 4826 | .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, |
| | 4827 | .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, |
| | 4828 | .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ }, |
| | 4829 | .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_access_size), .tmp2p, ._, ._ }, |
| | 4830 | .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4831 | .{ .@"1:", ._, .add, .tmp0p, .i(1), ._, ._ }, |
| | 4832 | .{ ._, ._nc, .j, .@"0b", ._, ._, ._ }, |
| | 4833 | .{ ._, ._, .@"test", .tmp1d, .ia(-1, .tmp2, .add_bit_size), ._, ._ }, |
| | 4834 | .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, |
| | 4835 | .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, |
| | 4836 | .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ }, |
| | 4837 | .{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ }, |
| | 4838 | } }, |
| | 4839 | }, .{ |
| | 4840 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 4841 | .patterns = &.{ |
| | 4842 | .{ .src = .{ .to_mem, .to_mem } }, |
| | 4843 | }, |
| | 4844 | .extra_temps = .{ |
| | 4845 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4846 | .{ .type = .u32, .kind = .{ .reg = .ecx } }, |
| | 4847 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4848 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| | 4849 | .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, |
| | 4850 | .unused, |
| | 4851 | }, |
| | 4852 | .dst_temps = .{.mem}, |
| | 4853 | .each = .{ .once = &.{ |
| | 4854 | .{ ._, ._, .mov, .tmp0p, .a(.src0, .sub_size), ._, ._ }, |
| | 4855 | .{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }, |
| | 4856 | .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4857 | .{ .@"0:", ._, .xor, .tmp3d, .tmp3d, ._, ._ }, |
| | 4858 | .{ ._, ._, .mov, .tmp4b, .memia(.src0b, .tmp0, .add_size), ._, ._ }, |
| | 4859 | .{ ._, ._, .cmp, .tmp4b, .memia(.src1b, .tmp0, .add_size), ._, ._ }, |
| | 4860 | .{ ._, .fromCondition(cc), .set, .tmp3b, ._, ._, ._ }, |
| | 4861 | .{ ._, ._l, .sh, .tmp3p, .tmp1b, ._, ._ }, |
| | 4862 | .{ ._, ._, .@"or", .tmp2p, .tmp3p, ._, ._ }, |
| | 4863 | .{ ._, ._, .inc, .tmp1d, ._, ._, ._ }, |
| | 4864 | .{ ._, ._, .@"test", .tmp1d, .ia(-1, .tmp2, .add_bit_size), ._, ._ }, |
| | 4865 | .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, |
| | 4866 | .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, |
| | 4867 | .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ }, |
| | 4868 | .{ ._, ._, .mov, .memia(.dst0p, .tmp3, .sub_access_size), .tmp2p, ._, ._ }, |
| | 4869 | .{ ._, ._, .xor, .tmp2d, .tmp2d, ._, ._ }, |
| | 4870 | .{ .@"1:", ._, .inc, .tmp0p, ._, ._, ._ }, |
| | 4871 | .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, |
| | 4872 | .{ ._, ._, .@"test", .tmp1d, .ia(-1, .tmp2, .add_bit_size), ._, ._ }, |
| | 4873 | .{ ._, ._z, .j, .@"0f", ._, ._, ._ }, |
| | 4874 | .{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }, |
| | 4875 | .{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ }, |
| | 4876 | .{ ._, ._, .mov, .memi(.dst0p, .tmp3), .tmp2p, ._, ._ }, |
| | 4877 | } }, |
| | 4878 | } }, |
| | 4879 | }) catch |err2| switch (err2) { |
| | 4880 | error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ |
| | 4881 | @tagName(air_tag), |
| | 4882 | cg.typeOf(extra.lhs).fmt(pt), |
| | 4883 | ops[0].tracking(cg), |
| | 4884 | ops[1].tracking(cg), |
| | 4885 | }), |
| | 4886 | else => |e| return e, |
| | 4887 | }, |
| | 4888 | .gte => unreachable, |
| | 4889 | .gt => unreachable, |
| | 4890 | } |
| | 4891 | if (ops[0].index != res[0].index) try ops[0].die(cg); |
| | 4892 | if (ops[1].index != res[0].index) try ops[1].die(cg); |
| | 4893 | try res[0].moveTo(inst, cg); |
| | 4894 | }, |
| | 4895 | |
| | 4896 | .cmp_lt, |
| | 4897 | .cmp_lt_optimized, |
| | 4898 | .cmp_lte, |
| | 4899 | .cmp_lte_optimized, |
| | 4900 | .cmp_gte, |
| | 4901 | .cmp_gte_optimized, |
| | 4902 | .cmp_gt, |
| | 4903 | .cmp_gt_optimized, |
| | 4904 | => |air_tag| if (use_old) try cg.airCmp(inst, switch (air_tag) { |
| | 4905 | else => unreachable, |
| | 4906 | .cmp_lt, .cmp_lt_optimized => .lt, |
| | 4907 | .cmp_lte, .cmp_lte_optimized => .lte, |
| | 4908 | .cmp_gte, .cmp_gte_optimized => .gte, |
| | 4909 | .cmp_gt, .cmp_gt_optimized => .gt, |
| | 4910 | }) else { |
| | 4911 | const bin_op = air_datas[@intFromEnum(inst)].bin_op; |
| | 4912 | const scalar_ty = cg.typeOf(bin_op.lhs).scalarType(zcu); |
| | 4913 | const signedness = if (scalar_ty.isAbiInt(zcu)) |
| | 4914 | scalar_ty.intInfo(zcu).signedness |
| | 4915 | else |
| | 4916 | .unsigned; |
| | 4917 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| | 4918 | var res: [1]Temp = undefined; |
| | 4919 | cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (signedness) { |
| | 4920 | .signed => switch (air_tag) { |
| | 4921 | else => unreachable, |
| | 4922 | .cmp_lt, .cmp_lt_optimized => .l, |
| | 4923 | .cmp_lte, .cmp_lte_optimized => .le, |
| | 4924 | .cmp_gte, .cmp_gte_optimized => .ge, |
| | 4925 | .cmp_gt, .cmp_gt_optimized => .g, |
| | 4926 | }, |
| | 4927 | .unsigned => switch (air_tag) { |
| | 4928 | else => unreachable, |
| | 4929 | .cmp_lt, .cmp_lt_optimized => .b, |
| | 4930 | .cmp_lte, .cmp_lte_optimized => .be, |
| | 4931 | .cmp_gte, .cmp_gte_optimized => .ae, |
| | 4932 | .cmp_gt, .cmp_gt_optimized => .a, |
| | 4933 | }, |
| | 4934 | })) { |
| | 4935 | else => unreachable, |
| | 4936 | inline .l, .le, .ge, .g, .b, .be, .ae, .a => |cc| comptime &.{ .{ |
| | 4937 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| | 4938 | .patterns = &.{ |
| | 4939 | .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } }, |
| | 4940 | .{ .src = .{ .imm8, .gpr }, .commute = .{ 0, 1 } }, |
| | 4941 | .{ .src = .{ .mem, .gpr }, .commute = .{ 0, 1 } }, |
| | 4942 | }, |
| | 4943 | .clobbers = .{ .eflags = true }, |
| | 4944 | .dst_temps = .{.{ .cc = cc.commute() }}, |
| | 4945 | .each = .{ .once = &.{ |
| | 4946 | .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, |
| | 4947 | } }, |
| | 4948 | }, .{ |
| | 4949 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| 3606 | .patterns = &.{ | 4950 | .patterns = &.{ |
| 3607 | .{ .src = .{ .mem, .imm8 } }, | 4951 | .{ .src = .{ .mem, .imm8 } }, |
| 3608 | .{ .src = .{ .gpr, .imm8 } }, | 4952 | .{ .src = .{ .gpr, .imm8 } }, |
| ... | @@ -3612,10 +4956,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3612,10 +4956,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3612 | .clobbers = .{ .eflags = true }, | 4956 | .clobbers = .{ .eflags = true }, |
| 3613 | .dst_temps = .{.{ .cc = cc }}, | 4957 | .dst_temps = .{.{ .cc = cc }}, |
| 3614 | .each = .{ .once = &.{ | 4958 | .each = .{ .once = &.{ |
| 3615 | .{ ._, .cmp, .src0b, .src1b, .none, .none }, | 4959 | .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, |
| 3616 | } }, | 4960 | } }, |
| 3617 | }, .{ | 4961 | }, .{ |
| 3618 | .constraints = .{ .{ .int = .word }, .{ .int = .word } }, | 4962 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| 3619 | .patterns = &.{ | 4963 | .patterns = &.{ |
| 3620 | .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } }, | 4964 | .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } }, |
| 3621 | .{ .src = .{ .imm16, .gpr }, .commute = .{ 0, 1 } }, | 4965 | .{ .src = .{ .imm16, .gpr }, .commute = .{ 0, 1 } }, |
| ... | @@ -3624,10 +4968,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3624,10 +4968,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3624 | .clobbers = .{ .eflags = true }, | 4968 | .clobbers = .{ .eflags = true }, |
| 3625 | .dst_temps = .{.{ .cc = cc.commute() }}, | 4969 | .dst_temps = .{.{ .cc = cc.commute() }}, |
| 3626 | .each = .{ .once = &.{ | 4970 | .each = .{ .once = &.{ |
| 3627 | .{ ._, .cmp, .src0w, .src1w, .none, .none }, | 4971 | .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, |
| 3628 | } }, | 4972 | } }, |
| 3629 | }, .{ | 4973 | }, .{ |
| 3630 | .constraints = .{ .{ .int = .word }, .{ .int = .word } }, | 4974 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| 3631 | .patterns = &.{ | 4975 | .patterns = &.{ |
| 3632 | .{ .src = .{ .mem, .imm16 } }, | 4976 | .{ .src = .{ .mem, .imm16 } }, |
| 3633 | .{ .src = .{ .gpr, .imm16 } }, | 4977 | .{ .src = .{ .gpr, .imm16 } }, |
| ... | @@ -3637,10 +4981,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3637,10 +4981,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3637 | .clobbers = .{ .eflags = true }, | 4981 | .clobbers = .{ .eflags = true }, |
| 3638 | .dst_temps = .{.{ .cc = cc }}, | 4982 | .dst_temps = .{.{ .cc = cc }}, |
| 3639 | .each = .{ .once = &.{ | 4983 | .each = .{ .once = &.{ |
| 3640 | .{ ._, .cmp, .src0w, .src1w, .none, .none }, | 4984 | .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, |
| 3641 | } }, | 4985 | } }, |
| 3642 | }, .{ | 4986 | }, .{ |
| 3643 | .constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, | 4987 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| 3644 | .patterns = &.{ | 4988 | .patterns = &.{ |
| 3645 | .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } }, | 4989 | .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } }, |
| 3646 | .{ .src = .{ .imm32, .gpr }, .commute = .{ 0, 1 } }, | 4990 | .{ .src = .{ .imm32, .gpr }, .commute = .{ 0, 1 } }, |
| ... | @@ -3649,10 +4993,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3649,10 +4993,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3649 | .clobbers = .{ .eflags = true }, | 4993 | .clobbers = .{ .eflags = true }, |
| 3650 | .dst_temps = .{.{ .cc = cc.commute() }}, | 4994 | .dst_temps = .{.{ .cc = cc.commute() }}, |
| 3651 | .each = .{ .once = &.{ | 4995 | .each = .{ .once = &.{ |
| 3652 | .{ ._, .cmp, .esrc0, .esrc1, .none, .none }, | 4996 | .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, |
| 3653 | } }, | 4997 | } }, |
| 3654 | }, .{ | 4998 | }, .{ |
| 3655 | .constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, | 4999 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| 3656 | .patterns = &.{ | 5000 | .patterns = &.{ |
| 3657 | .{ .src = .{ .mem, .imm32 } }, | 5001 | .{ .src = .{ .mem, .imm32 } }, |
| 3658 | .{ .src = .{ .gpr, .imm32 } }, | 5002 | .{ .src = .{ .gpr, .imm32 } }, |
| ... | @@ -3662,11 +5006,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3662,11 +5006,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3662 | .clobbers = .{ .eflags = true }, | 5006 | .clobbers = .{ .eflags = true }, |
| 3663 | .dst_temps = .{.{ .cc = cc }}, | 5007 | .dst_temps = .{.{ .cc = cc }}, |
| 3664 | .each = .{ .once = &.{ | 5008 | .each = .{ .once = &.{ |
| 3665 | .{ ._, .cmp, .esrc0, .esrc1, .none, .none }, | 5009 | .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, |
| 3666 | } }, | 5010 | } }, |
| 3667 | }, .{ | 5011 | }, .{ |
| 3668 | .required_features = .{ .@"64bit", null }, | 5012 | .required_features = .{ .@"64bit", null }, |
| 3669 | .constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, | 5013 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| 3670 | .patterns = &.{ | 5014 | .patterns = &.{ |
| 3671 | .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } }, | 5015 | .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } }, |
| 3672 | .{ .src = .{ .simm32, .gpr }, .commute = .{ 0, 1 } }, | 5016 | .{ .src = .{ .simm32, .gpr }, .commute = .{ 0, 1 } }, |
| ... | @@ -3675,11 +5019,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3675,11 +5019,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3675 | .clobbers = .{ .eflags = true }, | 5019 | .clobbers = .{ .eflags = true }, |
| 3676 | .dst_temps = .{.{ .cc = cc.commute() }}, | 5020 | .dst_temps = .{.{ .cc = cc.commute() }}, |
| 3677 | .each = .{ .once = &.{ | 5021 | .each = .{ .once = &.{ |
| 3678 | .{ ._, .cmp, .rsrc0, .rsrc1, .none, .none }, | 5022 | .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, |
| 3679 | } }, | 5023 | } }, |
| 3680 | }, .{ | 5024 | }, .{ |
| 3681 | .required_features = .{ .@"64bit", null }, | 5025 | .required_features = .{ .@"64bit", null }, |
| 3682 | .constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, | 5026 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| 3683 | .patterns = &.{ | 5027 | .patterns = &.{ |
| 3684 | .{ .src = .{ .mem, .simm32 } }, | 5028 | .{ .src = .{ .mem, .simm32 } }, |
| 3685 | .{ .src = .{ .gpr, .simm32 } }, | 5029 | .{ .src = .{ .gpr, .simm32 } }, |
| ... | @@ -3689,60 +5033,38 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3689,60 +5033,38 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3689 | .clobbers = .{ .eflags = true }, | 5033 | .clobbers = .{ .eflags = true }, |
| 3690 | .dst_temps = .{.{ .cc = cc }}, | 5034 | .dst_temps = .{.{ .cc = cc }}, |
| 3691 | .each = .{ .once = &.{ | 5035 | .each = .{ .once = &.{ |
| 3692 | .{ ._, .cmp, .rsrc0, .rsrc1, .none, .none }, | 5036 | .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, |
| 3693 | } }, | 5037 | } }, |
| 3694 | }, .{ | 5038 | }, .{ |
| 3695 | .required_features = .{ .@"64bit", null }, | | |
| 3696 | .patterns = &.{ | 5039 | .patterns = &.{ |
| 3697 | .{ .src = .{ .to_mem, .to_mem } }, | 5040 | .{ .src = .{ .to_mem, .to_mem } }, |
| 3698 | }, | 5041 | }, |
| 3699 | .extra_temps = .{ | 5042 | .extra_temps = .{ |
| 3700 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | 5043 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3701 | .{ .type = .bool, .kind = .{ .rc = .general_purpose } }, | 5044 | .{ .type = .bool, .kind = .{ .rc = .general_purpose } }, |
| 3702 | .unused, | 5045 | .unused, |
| 3703 | }, | 5046 | .unused, |
| 3704 | .clobbers = .{ .eflags = true }, | 5047 | .unused, |
| 3705 | .dst_temps = .{.{ .rc = .general_purpose }}, | | |
| 3706 | .each = .{ .limb = .{ | | |
| 3707 | .of = .rsrc0, | | |
| 3708 | .header = &.{ | | |
| 3709 | .{ ._, .xor, .tmp1b, .tmp1b, .none, .none }, | | |
| 3710 | }, | | |
| 3711 | .body = &.{ | | |
| 3712 | .{ ._, .mov, .rtmp0, .{ .src_limb = 0 }, .none, .none }, | | |
| 3713 | .{ ._r, .sh, .tmp1b, .{ .simm32 = 1 }, .none, .none }, | | |
| 3714 | .{ ._, .sbb, .rtmp0, .{ .src_limb = 1 }, .none, .none }, | | |
| 3715 | .{ ._c, .set, .tmp1b, .none, .none, .none }, | | |
| 3716 | .{ .fromCondition(cc), .set, .dst0b, .none, .none, .none }, | | |
| 3717 | }, | | |
| 3718 | } }, | | |
| 3719 | }, .{ | | |
| 3720 | .patterns = &.{ | | |
| 3721 | .{ .src = .{ .to_mem, .to_mem } }, | | |
| 3722 | }, | | |
| 3723 | .extra_temps = .{ | | |
| 3724 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | | |
| 3725 | .{ .type = .bool, .kind = .{ .rc = .general_purpose } }, | | |
| 3726 | .unused, | 5048 | .unused, |
| 3727 | }, | 5049 | }, |
| 3728 | .clobbers = .{ .eflags = true }, | 5050 | .clobbers = .{ .eflags = true }, |
| 3729 | .dst_temps = .{.{ .rc = .general_purpose }}, | 5051 | .dst_temps = .{.{ .rc = .general_purpose }}, |
| 3730 | .each = .{ .limb = .{ | 5052 | .each = .{ .limb = .{ |
| 3731 | .of = .esrc0, | 5053 | .of = .src0p, |
| 3732 | .header = &.{ | 5054 | .header = &.{ |
| 3733 | .{ ._, .xor, .tmp1b, .tmp1b, .none, .none }, | 5055 | .{ ._, ._, .xor, .tmp1b, .tmp1b, ._, ._ }, |
| 3734 | }, | 5056 | }, |
| 3735 | .body = &.{ | 5057 | .body = &.{ |
| 3736 | .{ ._, .mov, .etmp0, .{ .src_limb = 0 }, .none, .none }, | 5058 | .{ ._, ._, .mov, .tmp0p, .limb(.src0p), ._, ._ }, |
| 3737 | .{ ._r, .sh, .tmp1b, .{ .simm32 = 1 }, .none, .none }, | 5059 | .{ ._, ._r, .sh, .tmp1b, .i(1), ._, ._ }, |
| 3738 | .{ ._, .sbb, .etmp0, .{ .src_limb = 1 }, .none, .none }, | 5060 | .{ ._, ._, .sbb, .tmp0p, .limb(.src1p), ._, ._ }, |
| 3739 | .{ ._c, .set, .tmp1b, .none, .none, .none }, | 5061 | .{ ._, ._c, .set, .tmp1b, ._, ._, ._ }, |
| 3740 | .{ .fromCondition(cc), .set, .dst0b, .none, .none, .none }, | 5062 | .{ ._, .fromCondition(cc), .set, .dst0b, ._, ._, ._ }, |
| 3741 | }, | 5063 | }, |
| 3742 | } }, | 5064 | } }, |
| 3743 | } }, | 5065 | } }, |
| 3744 | }) catch |err| switch (err) { | 5066 | }) catch |err| switch (err) { |
| 3745 | error.Select2Failed => return cg.fail("failed to select2 {s} {} {} {}", .{ | 5067 | error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ |
| 3746 | @tagName(air_tag), | 5068 | @tagName(air_tag), |
| 3747 | cg.typeOf(bin_op.lhs).fmt(pt), | 5069 | cg.typeOf(bin_op.lhs).fmt(pt), |
| 3748 | ops[0].tracking(cg), | 5070 | ops[0].tracking(cg), |
| ... | @@ -3767,7 +5089,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3767,7 +5089,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3767 | }); | 5089 | }); |
| 3768 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); | 5090 | var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); |
| 3769 | var res: [1]Temp = undefined; | 5091 | var res: [1]Temp = undefined; |
| 3770 | cg.select2(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (air_tag) { | 5092 | cg.select(&res, &.{cg.typeOfIndex(inst)}, &ops, switch (@as(Condition, switch (air_tag) { |
| 3771 | else => unreachable, | 5093 | else => unreachable, |
| 3772 | .cmp_eq, .cmp_eq_optimized => .e, | 5094 | .cmp_eq, .cmp_eq_optimized => .e, |
| 3773 | .cmp_neq, .cmp_neq_optimized => .ne, | 5095 | .cmp_neq, .cmp_neq_optimized => .ne, |
| ... | @@ -3775,52 +5097,73 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3775,52 +5097,73 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3775 | else => unreachable, | 5097 | else => unreachable, |
| 3776 | inline .e, .ne => |cc| comptime &.{ .{ | 5098 | inline .e, .ne => |cc| comptime &.{ .{ |
| 3777 | .required_features = .{ .avx2, null }, | 5099 | .required_features = .{ .avx2, null }, |
| 3778 | .constraints = .{ .any_int, .any_int }, | 5100 | .src_constraints = .{ .any_int, .any_int }, |
| 3779 | .patterns = &.{ | 5101 | .patterns = &.{ |
| 3780 | .{ .src = .{ .ymm, .mem } }, | 5102 | .{ .src = .{ .ymm, .mem } }, |
| 3781 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, | 5103 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, |
| 3782 | .{ .src = .{ .ymm, .ymm } }, | 5104 | .{ .src = .{ .ymm, .ymm } }, |
| 3783 | }, | 5105 | }, |
| 3784 | .clobbers = .{ .eflags = true }, | 5106 | .clobbers = .{ .eflags = true }, |
| 3785 | .extra_temps = .{ .{ .kind = .{ .rc = .sse } }, .unused, .unused }, | 5107 | .extra_temps = .{ |
| | 5108 | .{ .kind = .{ .rc = .sse } }, |
| | 5109 | .unused, |
| | 5110 | .unused, |
| | 5111 | .unused, |
| | 5112 | .unused, |
| | 5113 | .unused, |
| | 5114 | }, |
| 3786 | .dst_temps = .{.{ .cc = cc }}, | 5115 | .dst_temps = .{.{ .cc = cc }}, |
| 3787 | .each = .{ .once = &.{ | 5116 | .each = .{ .once = &.{ |
| 3788 | .{ .vp_, .xor, .ytmp0, .ysrc0, .ysrc1, .none }, | 5117 | .{ ._, .vp_, .xor, .tmp0y, .src0y, .src1y, ._ }, |
| 3789 | .{ .vp_, .@"test", .ytmp0, .ytmp0, .none, .none }, | 5118 | .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ }, |
| 3790 | } }, | 5119 | } }, |
| 3791 | }, .{ | 5120 | }, .{ |
| 3792 | .required_features = .{ .avx, null }, | 5121 | .required_features = .{ .avx, null }, |
| 3793 | .constraints = .{ .any_int, .any_int }, | 5122 | .src_constraints = .{ .any_int, .any_int }, |
| 3794 | .patterns = &.{ | 5123 | .patterns = &.{ |
| 3795 | .{ .src = .{ .ymm, .mem } }, | 5124 | .{ .src = .{ .ymm, .mem } }, |
| 3796 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, | 5125 | .{ .src = .{ .mem, .ymm }, .commute = .{ 0, 1 } }, |
| 3797 | .{ .src = .{ .ymm, .ymm } }, | 5126 | .{ .src = .{ .ymm, .ymm } }, |
| 3798 | }, | 5127 | }, |
| 3799 | .clobbers = .{ .eflags = true }, | 5128 | .clobbers = .{ .eflags = true }, |
| 3800 | .extra_temps = .{ .{ .kind = .{ .rc = .sse } }, .unused, .unused }, | 5129 | .extra_temps = .{ |
| | 5130 | .{ .kind = .{ .rc = .sse } }, |
| | 5131 | .unused, |
| | 5132 | .unused, |
| | 5133 | .unused, |
| | 5134 | .unused, |
| | 5135 | .unused, |
| | 5136 | }, |
| 3801 | .dst_temps = .{.{ .cc = cc }}, | 5137 | .dst_temps = .{.{ .cc = cc }}, |
| 3802 | .each = .{ .once = &.{ | 5138 | .each = .{ .once = &.{ |
| 3803 | .{ .v_pd, .xor, .ytmp0, .ysrc0, .ysrc1, .none }, | 5139 | .{ ._, .v_pd, .xor, .tmp0y, .src0y, .src1y, ._ }, |
| 3804 | .{ .vp_, .@"test", .ytmp0, .ytmp0, .none, .none }, | 5140 | .{ ._, .vp_, .@"test", .tmp0y, .tmp0y, ._, ._ }, |
| 3805 | } }, | 5141 | } }, |
| 3806 | }, .{ | 5142 | }, .{ |
| 3807 | .required_features = .{ .avx, null }, | 5143 | .required_features = .{ .avx, null }, |
| 3808 | .constraints = .{ .any_int, .any_int }, | 5144 | .src_constraints = .{ .any_int, .any_int }, |
| 3809 | .patterns = &.{ | 5145 | .patterns = &.{ |
| 3810 | .{ .src = .{ .xmm, .mem } }, | 5146 | .{ .src = .{ .xmm, .mem } }, |
| 3811 | .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } }, | 5147 | .{ .src = .{ .mem, .xmm }, .commute = .{ 0, 1 } }, |
| 3812 | .{ .src = .{ .xmm, .xmm } }, | 5148 | .{ .src = .{ .xmm, .xmm } }, |
| 3813 | }, | 5149 | }, |
| 3814 | .clobbers = .{ .eflags = true }, | 5150 | .clobbers = .{ .eflags = true }, |
| 3815 | .extra_temps = .{ .{ .kind = .{ .rc = .sse } }, .unused, .unused }, | 5151 | .extra_temps = .{ |
| | 5152 | .{ .kind = .{ .rc = .sse } }, |
| | 5153 | .unused, |
| | 5154 | .unused, |
| | 5155 | .unused, |
| | 5156 | .unused, |
| | 5157 | .unused, |
| | 5158 | }, |
| 3816 | .dst_temps = .{.{ .cc = cc }}, | 5159 | .dst_temps = .{.{ .cc = cc }}, |
| 3817 | .each = .{ .once = &.{ | 5160 | .each = .{ .once = &.{ |
| 3818 | .{ .vp_, .xor, .xtmp0, .xsrc0, .xsrc1, .none }, | 5161 | .{ ._, .vp_, .xor, .tmp0x, .src0x, .src1x, ._ }, |
| 3819 | .{ .vp_, .@"test", .xtmp0, .xtmp0, .none, .none }, | 5162 | .{ ._, .vp_, .@"test", .tmp0x, .tmp0x, ._, ._ }, |
| 3820 | } }, | 5163 | } }, |
| 3821 | }, .{ | 5164 | }, .{ |
| 3822 | .required_features = .{ .sse4_1, null }, | 5165 | .required_features = .{ .sse4_1, null }, |
| 3823 | .constraints = .{ .any_int, .any_int }, | 5166 | .src_constraints = .{ .any_int, .any_int }, |
| 3824 | .patterns = &.{ | 5167 | .patterns = &.{ |
| 3825 | .{ .src = .{ .mut_xmm, .mem } }, | 5168 | .{ .src = .{ .mut_xmm, .mem } }, |
| 3826 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, | 5169 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, |
| ... | @@ -3829,12 +5172,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3829,12 +5172,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3829 | .clobbers = .{ .eflags = true }, | 5172 | .clobbers = .{ .eflags = true }, |
| 3830 | .dst_temps = .{.{ .cc = cc }}, | 5173 | .dst_temps = .{.{ .cc = cc }}, |
| 3831 | .each = .{ .once = &.{ | 5174 | .each = .{ .once = &.{ |
| 3832 | .{ .p_, .xor, .xsrc0, .xsrc1, .none, .none }, | 5175 | .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ }, |
| 3833 | .{ .p_, .@"test", .xsrc0, .xsrc0, .none, .none }, | 5176 | .{ ._, .p_, .@"test", .src0x, .src0x, ._, ._ }, |
| 3834 | } }, | 5177 | } }, |
| 3835 | }, .{ | 5178 | }, .{ |
| 3836 | .required_features = .{ .sse2, null }, | 5179 | .required_features = .{ .sse2, null }, |
| 3837 | .constraints = .{ .any_int, .any_int }, | 5180 | .src_constraints = .{ .any_int, .any_int }, |
| 3838 | .patterns = &.{ | 5181 | .patterns = &.{ |
| 3839 | .{ .src = .{ .mut_xmm, .mem } }, | 5182 | .{ .src = .{ .mut_xmm, .mem } }, |
| 3840 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, | 5183 | .{ .src = .{ .mem, .mut_xmm }, .commute = .{ 0, 1 } }, |
| ... | @@ -3845,18 +5188,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3845,18 +5188,21 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3845 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | 5188 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| 3846 | .{ .kind = .{ .rc = .sse } }, | 5189 | .{ .kind = .{ .rc = .sse } }, |
| 3847 | .unused, | 5190 | .unused, |
| | 5191 | .unused, |
| | 5192 | .unused, |
| | 5193 | .unused, |
| 3848 | }, | 5194 | }, |
| 3849 | .dst_temps = .{.{ .cc = cc }}, | 5195 | .dst_temps = .{.{ .cc = cc }}, |
| 3850 | .each = .{ .once = &.{ | 5196 | .each = .{ .once = &.{ |
| 3851 | .{ .p_, .xor, .xtmp1, .xtmp1, .none, .none }, | 5197 | .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, |
| 3852 | .{ .p_, .xor, .xsrc0, .xsrc1, .none, .none }, | 5198 | .{ ._, .p_, .xor, .src0x, .src1x, ._, ._ }, |
| 3853 | .{ .p_b, .cmpeq, .xtmp1, .xsrc0, .none, .none }, | 5199 | .{ ._, .p_b, .cmpeq, .tmp1x, .src0x, ._, ._ }, |
| 3854 | .{ .p_b, .movmsk, .etmp0, .xtmp1, .none, .none }, | 5200 | .{ ._, .p_b, .movmsk, .tmp0d, .tmp1x, ._, ._ }, |
| 3855 | .{ ._, .xor, .etmp0, .{ .simm32 = std.math.maxInt(u16) }, .none, .none }, | 5201 | .{ ._, ._, .xor, .tmp0d, .i(0xffff), ._, ._ }, |
| 3856 | } }, | 5202 | } }, |
| 3857 | }, .{ | 5203 | }, .{ |
| 3858 | .required_features = .{ .sse2, .mmx }, | 5204 | .required_features = .{ .sse2, .mmx }, |
| 3859 | .constraints = .{ .any_int, .any_int }, | 5205 | .src_constraints = .{ .any_int, .any_int }, |
| 3860 | .patterns = &.{ | 5206 | .patterns = &.{ |
| 3861 | .{ .src = .{ .mut_mm, .mem } }, | 5207 | .{ .src = .{ .mut_mm, .mem } }, |
| 3862 | .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } }, | 5208 | .{ .src = .{ .mem, .mut_mm }, .commute = .{ 0, 1 } }, |
| ... | @@ -3867,17 +5213,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3867,17 +5213,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3867 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | 5213 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, |
| 3868 | .{ .kind = .{ .rc = .mmx } }, | 5214 | .{ .kind = .{ .rc = .mmx } }, |
| 3869 | .unused, | 5215 | .unused, |
| | 5216 | .unused, |
| | 5217 | .unused, |
| | 5218 | .unused, |
| 3870 | }, | 5219 | }, |
| 3871 | .dst_temps = .{.{ .cc = cc }}, | 5220 | .dst_temps = .{.{ .cc = cc }}, |
| 3872 | .each = .{ .once = &.{ | 5221 | .each = .{ .once = &.{ |
| 3873 | .{ .p_, .xor, .rtmp1, .rtmp1, .none, .none }, | 5222 | .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ }, |
| 3874 | .{ .p_, .xor, .rsrc0, .rsrc1, .none, .none }, | 5223 | .{ ._, .p_, .xor, .src0q, .src1q, ._, ._ }, |
| 3875 | .{ .p_b, .cmpeq, .rtmp1, .rsrc0, .none, .none }, | 5224 | .{ ._, .p_b, .cmpeq, .tmp1q, .src0q, ._, ._ }, |
| 3876 | .{ .p_b, .movmsk, .etmp0, .rtmp1, .none, .none }, | 5225 | .{ ._, .p_b, .movmsk, .tmp0d, .tmp1q, ._, ._ }, |
| 3877 | .{ ._, .xor, .etmp0, .{ .simm32 = std.math.maxInt(u8) }, .none, .none }, | 5226 | .{ ._, ._, .xor, .tmp0d, .i(0xff), ._, ._ }, |
| 3878 | } }, | 5227 | } }, |
| 3879 | }, .{ | 5228 | }, .{ |
| 3880 | .constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, | 5229 | .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte } }, |
| 3881 | .patterns = &.{ | 5230 | .patterns = &.{ |
| 3882 | .{ .src = .{ .mem, .imm8 } }, | 5231 | .{ .src = .{ .mem, .imm8 } }, |
| 3883 | .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } }, | 5232 | .{ .src = .{ .imm8, .mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -3890,10 +5239,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3890,10 +5239,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3890 | .clobbers = .{ .eflags = true }, | 5239 | .clobbers = .{ .eflags = true }, |
| 3891 | .dst_temps = .{.{ .cc = cc }}, | 5240 | .dst_temps = .{.{ .cc = cc }}, |
| 3892 | .each = .{ .once = &.{ | 5241 | .each = .{ .once = &.{ |
| 3893 | .{ ._, .cmp, .src0b, .src1b, .none, .none }, | 5242 | .{ ._, ._, .cmp, .src0b, .src1b, ._, ._ }, |
| 3894 | } }, | 5243 | } }, |
| 3895 | }, .{ | 5244 | }, .{ |
| 3896 | .constraints = .{ .{ .int = .word }, .{ .int = .word } }, | 5245 | .src_constraints = .{ .{ .int = .word }, .{ .int = .word } }, |
| 3897 | .patterns = &.{ | 5246 | .patterns = &.{ |
| 3898 | .{ .src = .{ .mem, .imm16 } }, | 5247 | .{ .src = .{ .mem, .imm16 } }, |
| 3899 | .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } }, | 5248 | .{ .src = .{ .imm16, .mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -3906,10 +5255,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3906,10 +5255,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3906 | .clobbers = .{ .eflags = true }, | 5255 | .clobbers = .{ .eflags = true }, |
| 3907 | .dst_temps = .{.{ .cc = cc }}, | 5256 | .dst_temps = .{.{ .cc = cc }}, |
| 3908 | .each = .{ .once = &.{ | 5257 | .each = .{ .once = &.{ |
| 3909 | .{ ._, .cmp, .src0w, .src1w, .none, .none }, | 5258 | .{ ._, ._, .cmp, .src0w, .src1w, ._, ._ }, |
| 3910 | } }, | 5259 | } }, |
| 3911 | }, .{ | 5260 | }, .{ |
| 3912 | .constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, | 5261 | .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword } }, |
| 3913 | .patterns = &.{ | 5262 | .patterns = &.{ |
| 3914 | .{ .src = .{ .mem, .imm32 } }, | 5263 | .{ .src = .{ .mem, .imm32 } }, |
| 3915 | .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } }, | 5264 | .{ .src = .{ .imm32, .mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -3922,11 +5271,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3922,11 +5271,11 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3922 | .clobbers = .{ .eflags = true }, | 5271 | .clobbers = .{ .eflags = true }, |
| 3923 | .dst_temps = .{.{ .cc = cc }}, | 5272 | .dst_temps = .{.{ .cc = cc }}, |
| 3924 | .each = .{ .once = &.{ | 5273 | .each = .{ .once = &.{ |
| 3925 | .{ ._, .cmp, .esrc0, .esrc1, .none, .none }, | 5274 | .{ ._, ._, .cmp, .src0d, .src1d, ._, ._ }, |
| 3926 | } }, | 5275 | } }, |
| 3927 | }, .{ | 5276 | }, .{ |
| 3928 | .required_features = .{ .@"64bit", null }, | 5277 | .required_features = .{ .@"64bit", null }, |
| 3929 | .constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, | 5278 | .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword } }, |
| 3930 | .patterns = &.{ | 5279 | .patterns = &.{ |
| 3931 | .{ .src = .{ .mem, .simm32 } }, | 5280 | .{ .src = .{ .mem, .simm32 } }, |
| 3932 | .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } }, | 5281 | .{ .src = .{ .simm32, .mem }, .commute = .{ 0, 1 } }, |
| ... | @@ -3939,7 +5288,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3939,7 +5288,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3939 | .clobbers = .{ .eflags = true }, | 5288 | .clobbers = .{ .eflags = true }, |
| 3940 | .dst_temps = .{.{ .cc = cc }}, | 5289 | .dst_temps = .{.{ .cc = cc }}, |
| 3941 | .each = .{ .once = &.{ | 5290 | .each = .{ .once = &.{ |
| 3942 | .{ ._, .cmp, .rsrc0, .rsrc1, .none, .none }, | 5291 | .{ ._, ._, .cmp, .src0q, .src1q, ._, ._ }, |
| 3943 | } }, | 5292 | } }, |
| 3944 | }, .{ | 5293 | }, .{ |
| 3945 | .required_features = .{ .avx2, null }, | 5294 | .required_features = .{ .avx2, null }, |
| ... | @@ -3950,20 +5299,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3950,20 +5299,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3950 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 5299 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3951 | .{ .kind = .{ .rc = .sse } }, | 5300 | .{ .kind = .{ .rc = .sse } }, |
| 3952 | .{ .kind = .{ .rc = .sse } }, | 5301 | .{ .kind = .{ .rc = .sse } }, |
| | 5302 | .unused, |
| | 5303 | .unused, |
| | 5304 | .unused, |
| 3953 | }, | 5305 | }, |
| 3954 | .dst_temps = .{.{ .cc = cc }}, | 5306 | .dst_temps = .{.{ .cc = cc }}, |
| 3955 | .each = .{ .limb = .{ | 5307 | .each = .{ .limb = .{ |
| 3956 | .of = .ysrc0, | 5308 | .of = .src0y, |
| 3957 | .header = &.{ | 5309 | .header = &.{ |
| 3958 | .{ .vp_, .xor, .ytmp2, .ytmp2, .ytmp2, .none }, | 5310 | .{ ._, .vp_, .xor, .tmp2y, .tmp2y, .tmp2y, ._ }, |
| 3959 | }, | 5311 | }, |
| 3960 | .body = &.{ | 5312 | .body = &.{ |
| 3961 | .{ .v_, .movdqu, .ytmp1, .{ .src_limb = 0 }, .none, .none }, | 5313 | .{ ._, .v_dqu, .mov, .tmp1y, .limb(.src0y), ._, ._ }, |
| 3962 | .{ .vp_, .xor, .ytmp1, .ytmp1, .{ .src_limb = 1 }, .none }, | 5314 | .{ ._, .vp_, .xor, .tmp1y, .tmp1y, .limb(.src1y), ._ }, |
| 3963 | .{ .vp_, .@"or", .ytmp2, .ytmp2, .ytmp1, .none }, | 5315 | .{ ._, .vp_, .@"or", .tmp2y, .tmp2y, .tmp1y, ._ }, |
| 3964 | }, | 5316 | }, |
| 3965 | .trailer = &.{ | 5317 | .trailer = &.{ |
| 3966 | .{ .vp_, .@"test", .ytmp2, .ytmp2, .none, .none }, | 5318 | .{ ._, .vp_, .@"test", .tmp2y, .tmp2y, ._, ._ }, |
| 3967 | }, | 5319 | }, |
| 3968 | } }, | 5320 | } }, |
| 3969 | }, .{ | 5321 | }, .{ |
| ... | @@ -3975,20 +5327,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -3975,20 +5327,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 3975 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 5327 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 3976 | .{ .kind = .{ .rc = .sse } }, | 5328 | .{ .kind = .{ .rc = .sse } }, |
| 3977 | .{ .kind = .{ .rc = .sse } }, | 5329 | .{ .kind = .{ .rc = .sse } }, |
| | 5330 | .unused, |
| | 5331 | .unused, |
| | 5332 | .unused, |
| 3978 | }, | 5333 | }, |
| 3979 | .dst_temps = .{.{ .cc = cc }}, | 5334 | .dst_temps = .{.{ .cc = cc }}, |
| 3980 | .each = .{ .limb = .{ | 5335 | .each = .{ .limb = .{ |
| 3981 | .of = .ysrc0, | 5336 | .of = .src0y, |
| 3982 | .header = &.{ | 5337 | .header = &.{ |
| 3983 | .{ .v_pd, .xor, .ytmp2, .ytmp2, .ytmp2, .none }, | 5338 | .{ ._, .v_pd, .xor, .tmp2y, .tmp2y, .tmp2y, ._ }, |
| 3984 | }, | 5339 | }, |
| 3985 | .body = &.{ | 5340 | .body = &.{ |
| 3986 | .{ .v_pd, .movu, .ytmp1, .{ .src_limb = 0 }, .none, .none }, | 5341 | .{ ._, .v_pd, .movu, .tmp1y, .limb(.src0y), ._, ._ }, |
| 3987 | .{ .v_pd, .xor, .ytmp1, .ytmp1, .{ .src_limb = 1 }, .none }, | 5342 | .{ ._, .v_pd, .xor, .tmp1y, .tmp1y, .limb(.src1y), ._ }, |
| 3988 | .{ .v_pd, .@"or", .ytmp2, .ytmp2, .ytmp1, .none }, | 5343 | .{ ._, .v_pd, .@"or", .tmp2y, .tmp2y, .tmp1y, ._ }, |
| 3989 | }, | 5344 | }, |
| 3990 | .trailer = &.{ | 5345 | .trailer = &.{ |
| 3991 | .{ .vp_, .@"test", .ytmp2, .ytmp2, .none, .none }, | 5346 | .{ ._, .vp_, .@"test", .tmp2y, .tmp2y, ._, ._ }, |
| 3992 | }, | 5347 | }, |
| 3993 | } }, | 5348 | } }, |
| 3994 | }, .{ | 5349 | }, .{ |
| ... | @@ -4000,20 +5355,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -4000,20 +5355,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 4000 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 5355 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 4001 | .{ .kind = .{ .rc = .sse } }, | 5356 | .{ .kind = .{ .rc = .sse } }, |
| 4002 | .{ .kind = .{ .rc = .sse } }, | 5357 | .{ .kind = .{ .rc = .sse } }, |
| | 5358 | .unused, |
| | 5359 | .unused, |
| | 5360 | .unused, |
| 4003 | }, | 5361 | }, |
| 4004 | .dst_temps = .{.{ .cc = cc }}, | 5362 | .dst_temps = .{.{ .cc = cc }}, |
| 4005 | .each = .{ .limb = .{ | 5363 | .each = .{ .limb = .{ |
| 4006 | .of = .xsrc0, | 5364 | .of = .src0x, |
| 4007 | .header = &.{ | 5365 | .header = &.{ |
| 4008 | .{ .vp_, .xor, .xtmp2, .xtmp2, .xtmp2, .none }, | 5366 | .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, |
| 4009 | }, | 5367 | }, |
| 4010 | .body = &.{ | 5368 | .body = &.{ |
| 4011 | .{ .v_, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none }, | 5369 | .{ ._, .v_dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ }, |
| 4012 | .{ .vp_, .xor, .xtmp1, .xtmp1, .{ .src_limb = 1 }, .none }, | 5370 | .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .limb(.src1x), ._ }, |
| 4013 | .{ .vp_, .@"or", .xtmp2, .xtmp2, .xtmp1, .none }, | 5371 | .{ ._, .vp_, .@"or", .tmp2x, .tmp2x, .tmp1x, ._ }, |
| 4014 | }, | 5372 | }, |
| 4015 | .trailer = &.{ | 5373 | .trailer = &.{ |
| 4016 | .{ .vp_, .@"test", .xtmp2, .xtmp2, .none, .none }, | 5374 | .{ ._, .vp_, .@"test", .tmp2x, .tmp2x, ._, ._ }, |
| 4017 | }, | 5375 | }, |
| 4018 | } }, | 5376 | } }, |
| 4019 | }, .{ | 5377 | }, .{ |
| ... | @@ -4025,20 +5383,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -4025,20 +5383,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 4025 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 5383 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 4026 | .{ .kind = .{ .rc = .sse } }, | 5384 | .{ .kind = .{ .rc = .sse } }, |
| 4027 | .{ .kind = .{ .rc = .sse } }, | 5385 | .{ .kind = .{ .rc = .sse } }, |
| | 5386 | .unused, |
| | 5387 | .unused, |
| | 5388 | .unused, |
| 4028 | }, | 5389 | }, |
| 4029 | .dst_temps = .{.{ .cc = cc }}, | 5390 | .dst_temps = .{.{ .cc = cc }}, |
| 4030 | .each = .{ .limb = .{ | 5391 | .each = .{ .limb = .{ |
| 4031 | .of = .xsrc0, | 5392 | .of = .src0x, |
| 4032 | .header = &.{ | 5393 | .header = &.{ |
| 4033 | .{ .vp_, .xor, .xtmp2, .xtmp2, .xtmp2, .none }, | 5394 | .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ }, |
| 4034 | }, | 5395 | }, |
| 4035 | .body = &.{ | 5396 | .body = &.{ |
| 4036 | .{ .v_, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none }, | 5397 | .{ ._, .v_dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ }, |
| 4037 | .{ .vp_, .xor, .xtmp1, .xtmp1, .{ .src_limb = 1 }, .none }, | 5398 | .{ ._, .vp_, .xor, .tmp1x, .tmp1x, .limb(.src1x), ._ }, |
| 4038 | .{ .vp_, .@"or", .xtmp2, .xtmp2, .xtmp1, .none }, | 5399 | .{ ._, .vp_, .@"or", .tmp2x, .tmp2x, .tmp1x, ._ }, |
| 4039 | }, | 5400 | }, |
| 4040 | .trailer = &.{ | 5401 | .trailer = &.{ |
| 4041 | .{ .vp_, .@"test", .xtmp2, .xtmp2, .none, .none }, | 5402 | .{ ._, .vp_, .@"test", .tmp2x, .tmp2x, ._, ._ }, |
| 4042 | }, | 5403 | }, |
| 4043 | } }, | 5404 | } }, |
| 4044 | }, .{ | 5405 | }, .{ |
| ... | @@ -4050,20 +5411,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -4050,20 +5411,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 4050 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 5411 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 4051 | .{ .kind = .{ .rc = .sse } }, | 5412 | .{ .kind = .{ .rc = .sse } }, |
| 4052 | .{ .kind = .{ .rc = .sse } }, | 5413 | .{ .kind = .{ .rc = .sse } }, |
| | 5414 | .unused, |
| | 5415 | .unused, |
| | 5416 | .unused, |
| 4053 | }, | 5417 | }, |
| 4054 | .dst_temps = .{.{ .cc = cc }}, | 5418 | .dst_temps = .{.{ .cc = cc }}, |
| 4055 | .each = .{ .limb = .{ | 5419 | .each = .{ .limb = .{ |
| 4056 | .of = .xsrc0, | 5420 | .of = .src0x, |
| 4057 | .header = &.{ | 5421 | .header = &.{ |
| 4058 | .{ .p_, .xor, .xtmp2, .xtmp2, .none, .none }, | 5422 | .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, |
| 4059 | }, | 5423 | }, |
| 4060 | .body = &.{ | 5424 | .body = &.{ |
| 4061 | .{ ._, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none }, | 5425 | .{ ._, ._dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ }, |
| 4062 | .{ .p_, .xor, .xtmp1, .{ .src_limb = 1 }, .none, .none }, | 5426 | .{ ._, .p_, .xor, .tmp1x, .limb(.src1x), ._, ._ }, |
| 4063 | .{ .p_, .@"or", .xtmp2, .xtmp1, .none, .none }, | 5427 | .{ ._, .p_, .@"or", .tmp2x, .tmp1x, ._, ._ }, |
| 4064 | }, | 5428 | }, |
| 4065 | .trailer = &.{ | 5429 | .trailer = &.{ |
| 4066 | .{ .p_, .@"test", .xtmp2, .xtmp2, .none, .none }, | 5430 | .{ ._, .p_, .@"test", .tmp2x, .tmp2x, ._, ._ }, |
| 4067 | }, | 5431 | }, |
| 4068 | } }, | 5432 | } }, |
| 4069 | }, .{ | 5433 | }, .{ |
| ... | @@ -4075,23 +5439,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -4075,23 +5439,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 4075 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 5439 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 4076 | .{ .kind = .{ .rc = .sse } }, | 5440 | .{ .kind = .{ .rc = .sse } }, |
| 4077 | .{ .kind = .{ .rc = .sse } }, | 5441 | .{ .kind = .{ .rc = .sse } }, |
| | 5442 | .unused, |
| | 5443 | .unused, |
| | 5444 | .unused, |
| 4078 | }, | 5445 | }, |
| 4079 | .dst_temps = .{.{ .cc = cc }}, | 5446 | .dst_temps = .{.{ .cc = cc }}, |
| 4080 | .each = .{ .limb = .{ | 5447 | .each = .{ .limb = .{ |
| 4081 | .of = .xsrc0, | 5448 | .of = .src0x, |
| 4082 | .header = &.{ | 5449 | .header = &.{ |
| 4083 | .{ .p_, .xor, .xtmp2, .xtmp2, .none, .none }, | 5450 | .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ }, |
| 4084 | }, | 5451 | }, |
| 4085 | .body = &.{ | 5452 | .body = &.{ |
| 4086 | .{ ._, .movdqu, .xtmp1, .{ .src_limb = 0 }, .none, .none }, | 5453 | .{ ._, ._dqu, .mov, .tmp1x, .limb(.src0x), ._, ._ }, |
| 4087 | .{ .p_, .xor, .xtmp1, .{ .src_limb = 1 }, .none, .none }, | 5454 | .{ ._, .p_, .xor, .tmp1x, .limb(.src1x), ._, ._ }, |
| 4088 | .{ .p_, .@"or", .xtmp2, .xtmp1, .none, .none }, | 5455 | .{ ._, .p_, .@"or", .tmp2x, .tmp1x, ._, ._ }, |
| 4089 | }, | 5456 | }, |
| 4090 | .trailer = &.{ | 5457 | .trailer = &.{ |
| 4091 | .{ .p_, .xor, .xtmp1, .xtmp1, .none, .none }, | 5458 | .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ }, |
| 4092 | .{ .p_b, .cmpeq, .xtmp2, .xtmp1, .none, .none }, | 5459 | .{ ._, .p_b, .cmpeq, .tmp2x, .tmp1x, ._, ._ }, |
| 4093 | .{ .p_b, .movmsk, .etmp0, .xtmp2, .none, .none }, | 5460 | .{ ._, .p_b, .movmsk, .tmp0d, .tmp2x, ._, ._ }, |
| 4094 | .{ ._, .xor, .etmp0, .{ .simm32 = std.math.maxInt(u16) }, .none, .none }, | 5461 | .{ ._, ._, .xor, .tmp0d, .i(0xffff), ._, ._ }, |
| 4095 | }, | 5462 | }, |
| 4096 | } }, | 5463 | } }, |
| 4097 | }, .{ | 5464 | }, .{ |
| ... | @@ -4103,77 +5470,58 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { | ... | @@ -4103,77 +5470,58 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { |
| 4103 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 5470 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 4104 | .{ .kind = .{ .rc = .mmx } }, | 5471 | .{ .kind = .{ .rc = .mmx } }, |
| 4105 | .{ .kind = .{ .rc = .mmx } }, | 5472 | .{ .kind = .{ .rc = .mmx } }, |
| | 5473 | .unused, |
| | 5474 | .unused, |
| | 5475 | .unused, |
| 4106 | }, | 5476 | }, |
| 4107 | .dst_temps = .{.{ .cc = cc }}, | 5477 | .dst_temps = .{.{ .cc = cc }}, |
| 4108 | .each = .{ .limb = .{ | 5478 | .each = .{ .limb = .{ |
| 4109 | .of = .rsrc0, | 5479 | .of = .src0q, |
| 4110 | .header = &.{ | 5480 | .header = &.{ |
| 4111 | .{ .p_, .xor, .rtmp2, .rtmp2, .none, .none }, | 5481 | .{ ._, .p_, .xor, .tmp2q, .tmp2q, ._, ._ }, |
| 4112 | }, | 5482 | }, |
| 4113 | .body = &.{ | 5483 | .body = &.{ |
| 4114 | .{ ._q, .mov, .rtmp1, .{ .src_limb = 0 }, .none, .none }, | 5484 | .{ ._, ._q, .mov, .tmp1q, .limb(.src0q), ._, ._ }, |
| 4115 | .{ .p_, .xor, .rtmp1, .{ .src_limb = 1 }, .none, .none }, | 5485 | .{ ._, .p_, .xor, .tmp1q, .limb(.src1q), ._, ._ }, |
| 4116 | .{ .p_, .@"or", .rtmp2, .rtmp1, .none, .none }, | 5486 | .{ ._, .p_, .@"or", .tmp2q, .tmp1q, ._, ._ }, |
| 4117 | }, | 5487 | }, |
| 4118 | .trailer = &.{ | 5488 | .trailer = &.{ |
| 4119 | .{ .p_, .xor, .rtmp1, .rtmp1, .none, .none }, | 5489 | .{ ._, .p_, .xor, .tmp1q, .tmp1q, ._, ._ }, |
| 4120 | .{ .p_b, .cmpeq, .rtmp2, .rtmp1, .none, .none }, | 5490 | .{ ._, .p_b, .cmpeq, .tmp2q, .tmp1q, ._, ._ }, |
| 4121 | .{ .p_b, .movmsk, .etmp0, .rtmp2, .none, .none }, | 5491 | .{ ._, .p_b, .movmsk, .tmp0d, .tmp2q, ._, ._ }, |
| 4122 | .{ ._, .xor, .etmp0, .{ .simm32 = std.math.maxInt(u8) }, .none, .none }, | 5492 | .{ ._, ._, .xor, .tmp0d, .i(0xff), ._, ._ }, |
| 4123 | }, | 5493 | }, |
| 4124 | } }, | 5494 | } }, |
| 4125 | }, .{ | 5495 | }, .{ |
| 4126 | .required_features = .{ .@"64bit", null }, | | |
| 4127 | .patterns = &.{ | 5496 | .patterns = &.{ |
| 4128 | .{ .src = .{ .to_mem, .to_mem } }, | 5497 | .{ .src = .{ .to_mem, .to_mem } }, |
| 4129 | }, | 5498 | }, |
| 4130 | .extra_temps = .{ | 5499 | .extra_temps = .{ |
| 4131 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 5500 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 4132 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | | |
| 4133 | .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, | | |
| 4134 | }, | | |
| 4135 | .dst_temps = .{.{ .cc = cc }}, | | |
| 4136 | .each = .{ .limb = .{ | | |
| 4137 | .of = .rsrc0, | | |
| 4138 | .header = &.{ | | |
| 4139 | .{ ._, .xor, .rtmp2, .rtmp2, .none, .none }, | | |
| 4140 | }, | | |
| 4141 | .body = &.{ | | |
| 4142 | .{ ._, .mov, .rtmp1, .{ .src_limb = 0 }, .none, .none }, | | |
| 4143 | .{ ._, .xor, .rtmp1, .{ .src_limb = 1 }, .none, .none }, | | |
| 4144 | .{ ._, .@"or", .rtmp2, .rtmp1, .none, .none }, | | |
| 4145 | }, | | |
| 4146 | .trailer = &.{ | | |
| 4147 | .{ ._, .@"test", .rtmp2, .rtmp2, .none, .none }, | | |
| 4148 | }, | | |
| 4149 | } }, | | |
| 4150 | }, .{ | | |
| 4151 | .patterns = &.{ | | |
| 4152 | .{ .src = .{ .to_mem, .to_mem } }, | | |
| 4153 | }, | | |
| 4154 | .extra_temps = .{ | | |
| 4155 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, | 5501 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 4156 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | 5502 | .{ .type = .usize, .kind = .{ .rc = .general_purpose } }, |
| 4157 | .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, | 5503 | .unused, |
| | 5504 | .unused, |
| | 5505 | .unused, |
| 4158 | }, | 5506 | }, |
| 4159 | .dst_temps = .{.{ .cc = cc }}, | 5507 | .dst_temps = .{.{ .cc = cc }}, |
| 4160 | .each = .{ .limb = .{ | 5508 | .each = .{ .limb = .{ |
| 4161 | .of = .esrc0, | 5509 | .of = .src0p, |
| 4162 | .header = &.{ | 5510 | .header = &.{ |
| 4163 | .{ ._, .xor, .etmp2, .etmp2, .none, .none }, | 5511 | .{ ._, ._, .xor, .tmp2p, .tmp2p, ._, ._ }, |
| 4164 | }, | 5512 | }, |
| 4165 | .body = &.{ | 5513 | .body = &.{ |
| 4166 | .{ ._, .mov, .etmp1, .{ .src_limb = 0 }, .none, .none }, | 5514 | .{ ._, ._, .mov, .tmp1p, .limb(.src0p), ._, ._ }, |
| 4167 | .{ ._, .xor, .etmp1, .{ .src_limb = 1 }, .none, .none }, | 5515 | .{ ._, ._, .xor, .tmp1p, .limb(.src1p), ._, ._ }, |
| 4168 | .{ ._, .@"or", .etmp2, .etmp1, .none, .none }, | 5516 | .{ ._, ._, .@"or", .tmp2p, .tmp1p, ._, ._ }, |
| 4169 | }, | 5517 | }, |
| 4170 | .trailer = &.{ | 5518 | .trailer = &.{ |
| 4171 | .{ ._, .@"test", .etmp2, .etmp2, .none, .none }, | 5519 | .{ ._, ._, .@"test", .tmp2p, .tmp2p, ._, ._ }, |
| 4172 | }, | 5520 | }, |
| 4173 | } }, | 5521 | } }, |
| 4174 | } }, | 5522 | } }, |
| 4175 | }) catch |err| switch (err) { | 5523 | }) catch |err| switch (err) { |
| 4176 | error.Select2Failed => return cg.fail("failed to select2 {s} {} {} {}", .{ | 5524 | error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ |
| 4177 | @tagName(air_tag), | 5525 | @tagName(air_tag), |
| 4178 | cg.typeOf(bin_op.lhs).fmt(pt), | 5526 | cg.typeOf(bin_op.lhs).fmt(pt), |
| 4179 | ops[0].tracking(cg), | 5527 | ops[0].tracking(cg), |
| ... | @@ -16456,7 +17804,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -16456,7 +17804,7 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 16456 | } }, | 17804 | } }, |
| 16457 | } }; | 17805 | } }; |
| 16458 | } else { | 17806 | } else { |
| 16459 | if (mnem_size) |size| if (reg.bitSize() != size.bitSize()) | 17807 | if (mnem_size) |size| if (reg.bitSize() != size.bitSize(self.target)) |
| 16460 | return self.fail("invalid register size: '{s}'", .{op_str}); | 17808 | return self.fail("invalid register size: '{s}'", .{op_str}); |
| 16461 | op.* = .{ .reg = reg }; | 17809 | op.* = .{ .reg = reg }; |
| 16462 | } | 17810 | } |
| ... | @@ -16524,14 +17872,14 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { | ... | @@ -16524,14 +17872,14 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void { |
| 16524 | } else if (std.mem.startsWith(u8, op_str, "$")) { | 17872 | } else if (std.mem.startsWith(u8, op_str, "$")) { |
| 16525 | if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| { | 17873 | if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| { |
| 16526 | if (mnem_size) |size| { | 17874 | if (mnem_size) |size| { |
| 16527 | const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1)); | 17875 | const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - (size.bitSize(self.target) - 1)); |
| 16528 | if ((if (s < 0) ~s else s) > max) | 17876 | if ((if (s < 0) ~s else s) > max) |
| 16529 | return self.fail("invalid immediate size: '{s}'", .{op_str}); | 17877 | return self.fail("invalid immediate size: '{s}'", .{op_str}); |
| 16530 | } | 17878 | } |
| 16531 | op.* = .{ .imm = .s(s) }; | 17879 | op.* = .{ .imm = .s(s) }; |
| 16532 | } else |_| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| { | 17880 | } else |_| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| { |
| 16533 | if (mnem_size) |size| { | 17881 | if (mnem_size) |size| { |
| 16534 | const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - size.bitSize()); | 17882 | const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - size.bitSize(self.target)); |
| 16535 | if (u > max) | 17883 | if (u > max) |
| 16536 | return self.fail("invalid immediate size: '{s}'", .{op_str}); | 17884 | return self.fail("invalid immediate size: '{s}'", .{op_str}); |
| 16537 | } | 17885 | } |
| ... | @@ -16827,10 +18175,13 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) | ... | @@ -16827,10 +18175,13 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) |
| 16827 | else | 18175 | else |
| 16828 | .{ ._q, .mov } }, | 18176 | .{ ._q, .mov } }, |
| 16829 | 9...16 => return .{ .move = if (self.hasFeature(.avx)) | 18177 | 9...16 => return .{ .move = if (self.hasFeature(.avx)) |
| 16830 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } | 18178 | .{ if (aligned) .v_dqa else .v_dqu, .mov } |
| 16831 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, | 18179 | else if (self.hasFeature(.sse2)) |
| | 18180 | .{ if (aligned) ._dqa else ._dqu, .mov } |
| | 18181 | else |
| | 18182 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 16832 | 17...32 => if (self.hasFeature(.avx)) | 18183 | 17...32 => if (self.hasFeature(.avx)) |
| 16833 | return .{ .move = if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } }, | 18184 | return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, |
| 16834 | else => {}, | 18185 | else => {}, |
| 16835 | } else switch (abi_size) { | 18186 | } else switch (abi_size) { |
| 16836 | 4 => return .{ .move = if (self.hasFeature(.avx)) | 18187 | 4 => return .{ .move = if (self.hasFeature(.avx)) |
| ... | @@ -16842,12 +18193,13 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) | ... | @@ -16842,12 +18193,13 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) |
| 16842 | else | 18193 | else |
| 16843 | .{ ._sd, .mov } }, | 18194 | .{ ._sd, .mov } }, |
| 16844 | 9...16 => return .{ .move = if (self.hasFeature(.avx)) | 18195 | 9...16 => return .{ .move = if (self.hasFeature(.avx)) |
| 16845 | if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu } | 18196 | .{ .v_pd, if (aligned) .mova else .movu } |
| 16846 | else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } }, | 18197 | else if (self.hasFeature(.sse2)) |
| 16847 | 17...32 => if (self.hasFeature(.avx)) return .{ .move = if (aligned) | 18198 | .{ ._pd, if (aligned) .mova else .movu } |
| 16848 | .{ .v_pd, .mova } | | |
| 16849 | else | 18199 | else |
| 16850 | .{ .v_pd, .movu } }, | 18200 | .{ ._ps, if (aligned) .mova else .movu } }, |
| | 18201 | 17...32 => if (self.hasFeature(.avx)) |
| | 18202 | return .{ .move = .{ .v_pd, if (aligned) .mova else .movu } }, |
| 16851 | else => {}, | 18203 | else => {}, |
| 16852 | } | 18204 | } |
| 16853 | }, | 18205 | }, |
| ... | @@ -16868,8 +18220,11 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) | ... | @@ -16868,8 +18220,11 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) |
| 16868 | else | 18220 | else |
| 16869 | .{ ._sd, .mov } }, | 18221 | .{ ._sd, .mov } }, |
| 16870 | 128 => return .{ .move = if (self.hasFeature(.avx)) | 18222 | 128 => return .{ .move = if (self.hasFeature(.avx)) |
| 16871 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } | 18223 | .{ if (aligned) .v_dqa else .v_dqu, .mov } |
| 16872 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, | 18224 | else if (self.hasFeature(.sse2)) |
| | 18225 | .{ if (aligned) ._dqa else ._dqu, .mov } |
| | 18226 | else |
| | 18227 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 16873 | else => {}, | 18228 | else => {}, |
| 16874 | }, | 18229 | }, |
| 16875 | .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { | 18230 | .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) { |
| ... | @@ -16883,65 +18238,62 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) | ... | @@ -16883,65 +18238,62 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) |
| 16883 | .int => switch (ty.childType(zcu).intInfo(zcu).bits) { | 18238 | .int => switch (ty.childType(zcu).intInfo(zcu).bits) { |
| 16884 | 1...8 => switch (ty.vectorLen(zcu)) { | 18239 | 1...8 => switch (ty.vectorLen(zcu)) { |
| 16885 | 1...16 => return .{ .move = if (self.hasFeature(.avx)) | 18240 | 1...16 => return .{ .move = if (self.hasFeature(.avx)) |
| 16886 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } | 18241 | .{ if (aligned) .v_dqa else .v_dqu, .mov } |
| 16887 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, | 18242 | else if (self.hasFeature(.sse2)) |
| | 18243 | .{ if (aligned) ._dqa else ._dqu, .mov } |
| | 18244 | else |
| | 18245 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 16888 | 17...32 => if (self.hasFeature(.avx)) | 18246 | 17...32 => if (self.hasFeature(.avx)) |
| 16889 | return .{ .move = if (aligned) | 18247 | return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, |
| 16890 | .{ .v_, .movdqa } | | |
| 16891 | else | | |
| 16892 | .{ .v_, .movdqu } }, | | |
| 16893 | else => {}, | 18248 | else => {}, |
| 16894 | }, | 18249 | }, |
| 16895 | 9...16 => switch (ty.vectorLen(zcu)) { | 18250 | 9...16 => switch (ty.vectorLen(zcu)) { |
| 16896 | 1...8 => return .{ .move = if (self.hasFeature(.avx)) | 18251 | 1...8 => return .{ .move = if (self.hasFeature(.avx)) |
| 16897 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } | 18252 | .{ if (aligned) .v_dqa else .v_dqu, .mov } |
| 16898 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, | 18253 | else if (self.hasFeature(.sse2)) |
| | 18254 | .{ if (aligned) ._dqa else ._dqu, .mov } |
| | 18255 | else |
| | 18256 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 16899 | 9...16 => if (self.hasFeature(.avx)) | 18257 | 9...16 => if (self.hasFeature(.avx)) |
| 16900 | return .{ .move = if (aligned) | 18258 | return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, |
| 16901 | .{ .v_, .movdqa } | | |
| 16902 | else | | |
| 16903 | .{ .v_, .movdqu } }, | | |
| 16904 | else => {}, | 18259 | else => {}, |
| 16905 | }, | 18260 | }, |
| 16906 | 17...32 => switch (ty.vectorLen(zcu)) { | 18261 | 17...32 => switch (ty.vectorLen(zcu)) { |
| 16907 | 1...4 => return .{ .move = if (self.hasFeature(.avx)) | 18262 | 1...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 16908 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } | 18263 | .{ if (aligned) .v_dqa else .v_dqu, .mov } |
| 16909 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, | 18264 | else if (self.hasFeature(.sse2)) |
| | 18265 | .{ if (aligned) ._dqa else ._dqu, .mov } |
| | 18266 | else |
| | 18267 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 16910 | 5...8 => if (self.hasFeature(.avx)) | 18268 | 5...8 => if (self.hasFeature(.avx)) |
| 16911 | return .{ .move = if (aligned) | 18269 | return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, |
| 16912 | .{ .v_, .movdqa } | | |
| 16913 | else | | |
| 16914 | .{ .v_, .movdqu } }, | | |
| 16915 | else => {}, | 18270 | else => {}, |
| 16916 | }, | 18271 | }, |
| 16917 | 33...64 => switch (ty.vectorLen(zcu)) { | 18272 | 33...64 => switch (ty.vectorLen(zcu)) { |
| 16918 | 1...2 => return .{ .move = if (self.hasFeature(.avx)) | 18273 | 1...2 => return .{ .move = if (self.hasFeature(.avx)) |
| 16919 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } | 18274 | .{ if (aligned) .v_dqa else .v_dqu, .mov } |
| 16920 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, | 18275 | else if (self.hasFeature(.sse2)) |
| | 18276 | .{ if (aligned) ._dqa else ._dqu, .mov } |
| | 18277 | else |
| | 18278 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 16921 | 3...4 => if (self.hasFeature(.avx)) | 18279 | 3...4 => if (self.hasFeature(.avx)) |
| 16922 | return .{ .move = if (aligned) | 18280 | return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, |
| 16923 | .{ .v_, .movdqa } | | |
| 16924 | else | | |
| 16925 | .{ .v_, .movdqu } }, | | |
| 16926 | else => {}, | 18281 | else => {}, |
| 16927 | }, | 18282 | }, |
| 16928 | 65...128 => switch (ty.vectorLen(zcu)) { | 18283 | 65...128 => switch (ty.vectorLen(zcu)) { |
| 16929 | 1 => return .{ .move = if (self.hasFeature(.avx)) | 18284 | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 16930 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } | 18285 | .{ if (aligned) .v_dqa else .v_dqu, .mov } |
| 16931 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, | 18286 | else if (self.hasFeature(.sse2)) |
| | 18287 | .{ if (aligned) ._dqa else ._dqu, .mov } |
| | 18288 | else |
| | 18289 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 16932 | 2 => if (self.hasFeature(.avx)) | 18290 | 2 => if (self.hasFeature(.avx)) |
| 16933 | return .{ .move = if (aligned) | 18291 | return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, |
| 16934 | .{ .v_, .movdqa } | | |
| 16935 | else | | |
| 16936 | .{ .v_, .movdqu } }, | | |
| 16937 | else => {}, | 18292 | else => {}, |
| 16938 | }, | 18293 | }, |
| 16939 | 129...256 => switch (ty.vectorLen(zcu)) { | 18294 | 129...256 => switch (ty.vectorLen(zcu)) { |
| 16940 | 1 => if (self.hasFeature(.avx)) | 18295 | 1 => if (self.hasFeature(.avx)) |
| 16941 | return .{ .move = if (aligned) | 18296 | return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, |
| 16942 | .{ .v_, .movdqa } | | |
| 16943 | else | | |
| 16944 | .{ .v_, .movdqu } }, | | |
| 16945 | else => {}, | 18297 | else => {}, |
| 16946 | }, | 18298 | }, |
| 16947 | else => {}, | 18299 | else => {}, |
| ... | @@ -16949,13 +18301,13 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) | ... | @@ -16949,13 +18301,13 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) |
| 16949 | .pointer, .optional => if (ty.childType(zcu).isPtrAtRuntime(zcu)) | 18301 | .pointer, .optional => if (ty.childType(zcu).isPtrAtRuntime(zcu)) |
| 16950 | switch (ty.vectorLen(zcu)) { | 18302 | switch (ty.vectorLen(zcu)) { |
| 16951 | 1...2 => return .{ .move = if (self.hasFeature(.avx)) | 18303 | 1...2 => return .{ .move = if (self.hasFeature(.avx)) |
| 16952 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } | 18304 | .{ if (aligned) .v_dqa else .v_dqu, .mov } |
| 16953 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, | 18305 | else if (self.hasFeature(.sse2)) |
| | 18306 | .{ if (aligned) ._dqa else ._dqu, .mov } |
| | 18307 | else |
| | 18308 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 16954 | 3...4 => if (self.hasFeature(.avx)) | 18309 | 3...4 => if (self.hasFeature(.avx)) |
| 16955 | return .{ .move = if (aligned) | 18310 | return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, |
| 16956 | .{ .v_, .movdqa } | | |
| 16957 | else | | |
| 16958 | .{ .v_, .movdqu } }, | | |
| 16959 | else => {}, | 18311 | else => {}, |
| 16960 | } | 18312 | } |
| 16961 | else | 18313 | else |
| ... | @@ -16963,46 +18315,42 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) | ... | @@ -16963,46 +18315,42 @@ fn moveStrategy(self: *CodeGen, ty: Type, class: Register.Class, aligned: bool) |
| 16963 | .float => switch (ty.childType(zcu).floatBits(self.target.*)) { | 18315 | .float => switch (ty.childType(zcu).floatBits(self.target.*)) { |
| 16964 | 16 => switch (ty.vectorLen(zcu)) { | 18316 | 16 => switch (ty.vectorLen(zcu)) { |
| 16965 | 1...8 => return .{ .move = if (self.hasFeature(.avx)) | 18317 | 1...8 => return .{ .move = if (self.hasFeature(.avx)) |
| 16966 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } | 18318 | .{ if (aligned) .v_dqa else .v_dqu, .mov } |
| 16967 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, | 18319 | else if (self.hasFeature(.sse2)) |
| | 18320 | .{ if (aligned) ._dqa else ._dqu, .mov } |
| | 18321 | else |
| | 18322 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 16968 | 9...16 => if (self.hasFeature(.avx)) | 18323 | 9...16 => if (self.hasFeature(.avx)) |
| 16969 | return .{ .move = if (aligned) | 18324 | return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, |
| 16970 | .{ .v_, .movdqa } | | |
| 16971 | else | | |
| 16972 | .{ .v_, .movdqu } }, | | |
| 16973 | else => {}, | 18325 | else => {}, |
| 16974 | }, | 18326 | }, |
| 16975 | 32 => switch (ty.vectorLen(zcu)) { | 18327 | 32 => switch (ty.vectorLen(zcu)) { |
| 16976 | 1...4 => return .{ .move = if (self.hasFeature(.avx)) | 18328 | 1...4 => return .{ .move = if (self.hasFeature(.avx)) |
| 16977 | if (aligned) .{ .v_ps, .mova } else .{ .v_ps, .movu } | 18329 | .{ .v_ps, if (aligned) .mova else .movu } |
| 16978 | else if (aligned) .{ ._ps, .mova } else .{ ._ps, .movu } }, | 18330 | else |
| | 18331 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 16979 | 5...8 => if (self.hasFeature(.avx)) | 18332 | 5...8 => if (self.hasFeature(.avx)) |
| 16980 | return .{ .move = if (aligned) | 18333 | return .{ .move = .{ .v_ps, if (aligned) .mova else .movu } }, |
| 16981 | .{ .v_ps, .mova } | | |
| 16982 | else | | |
| 16983 | .{ .v_ps, .movu } }, | | |
| 16984 | else => {}, | 18334 | else => {}, |
| 16985 | }, | 18335 | }, |
| 16986 | 64 => switch (ty.vectorLen(zcu)) { | 18336 | 64 => switch (ty.vectorLen(zcu)) { |
| 16987 | 1...2 => return .{ .move = if (self.hasFeature(.avx)) | 18337 | 1...2 => return .{ .move = if (self.hasFeature(.avx)) |
| 16988 | if (aligned) .{ .v_pd, .mova } else .{ .v_pd, .movu } | 18338 | .{ .v_pd, if (aligned) .mova else .movu } |
| 16989 | else if (aligned) .{ ._pd, .mova } else .{ ._pd, .movu } }, | 18339 | else |
| | 18340 | .{ ._pd, if (aligned) .mova else .movu } }, |
| 16990 | 3...4 => if (self.hasFeature(.avx)) | 18341 | 3...4 => if (self.hasFeature(.avx)) |
| 16991 | return .{ .move = if (aligned) | 18342 | return .{ .move = .{ .v_pd, if (aligned) .mova else .movu } }, |
| 16992 | .{ .v_pd, .mova } | | |
| 16993 | else | | |
| 16994 | .{ .v_pd, .movu } }, | | |
| 16995 | else => {}, | 18343 | else => {}, |
| 16996 | }, | 18344 | }, |
| 16997 | 128 => switch (ty.vectorLen(zcu)) { | 18345 | 128 => switch (ty.vectorLen(zcu)) { |
| 16998 | 1 => return .{ .move = if (self.hasFeature(.avx)) | 18346 | 1 => return .{ .move = if (self.hasFeature(.avx)) |
| 16999 | if (aligned) .{ .v_, .movdqa } else .{ .v_, .movdqu } | 18347 | .{ if (aligned) .v_dqa else .v_dqu, .mov } |
| 17000 | else if (aligned) .{ ._, .movdqa } else .{ ._, .movdqu } }, | 18348 | else if (self.hasFeature(.sse2)) |
| | 18349 | .{ if (aligned) ._dqa else ._dqu, .mov } |
| | 18350 | else |
| | 18351 | .{ ._ps, if (aligned) .mova else .movu } }, |
| 17001 | 2 => if (self.hasFeature(.avx)) | 18352 | 2 => if (self.hasFeature(.avx)) |
| 17002 | return .{ .move = if (aligned) | 18353 | return .{ .move = .{ if (aligned) .v_dqa else .v_dqu, .mov } }, |
| 17003 | .{ .v_, .movdqa } | | |
| 17004 | else | | |
| 17005 | .{ .v_, .movdqu } }, | | |
| 17006 | else => {}, | 18354 | else => {}, |
| 17007 | }, | 18355 | }, |
| 17008 | else => {}, | 18356 | else => {}, |
| ... | @@ -17211,7 +18559,7 @@ fn genSetReg( | ... | @@ -17211,7 +18559,7 @@ fn genSetReg( |
| 17211 | src_reg, | 18559 | src_reg, |
| 17212 | ), | 18560 | ), |
| 17213 | .x87, .mmx, .ip => unreachable, | 18561 | .x87, .mmx, .ip => unreachable, |
| 17214 | .sse => try self.asmRegisterRegister( | 18562 | .sse => if (self.hasFeature(.sse2)) try self.asmRegisterRegister( |
| 17215 | switch (abi_size) { | 18563 | switch (abi_size) { |
| 17216 | 1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov }, | 18564 | 1...4 => if (self.hasFeature(.avx)) .{ .v_d, .mov } else .{ ._d, .mov }, |
| 17217 | 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov }, | 18565 | 5...8 => if (self.hasFeature(.avx)) .{ .v_q, .mov } else .{ ._q, .mov }, |
| ... | @@ -17219,7 +18567,20 @@ fn genSetReg( | ... | @@ -17219,7 +18567,20 @@ fn genSetReg( |
| 17219 | }, | 18567 | }, |
| 17220 | registerAlias(dst_reg, @max(abi_size, 4)), | 18568 | registerAlias(dst_reg, @max(abi_size, 4)), |
| 17221 | src_reg.to128(), | 18569 | src_reg.to128(), |
| 17222 | ), | 18570 | ) else { |
| | 18571 | const frame_index = try self.allocFrameIndex(.init(.{ |
| | 18572 | .size = 4, |
| | 18573 | .alignment = .@"4", |
| | 18574 | })); |
| | 18575 | try self.asmMemoryRegister(.{ ._ss, .mov }, .{ |
| | 18576 | .base = .{ .frame = frame_index }, |
| | 18577 | .mod = .{ .rm = .{ .size = .dword } }, |
| | 18578 | }, src_reg.to128()); |
| | 18579 | try self.asmRegisterMemory(.{ ._, .mov }, registerAlias(dst_reg, abi_size), .{ |
| | 18580 | .base = .{ .frame = frame_index }, |
| | 18581 | .mod = .{ .rm = .{ .size = .fromSize(abi_size) } }, |
| | 18582 | }); |
| | 18583 | }, |
| 17223 | }, | 18584 | }, |
| 17224 | .segment => try self.asmRegisterRegister( | 18585 | .segment => try self.asmRegisterRegister( |
| 17225 | .{ ._, .mov }, | 18586 | .{ ._, .mov }, |
| ... | @@ -17264,17 +18625,17 @@ fn genSetReg( | ... | @@ -17264,17 +18625,17 @@ fn genSetReg( |
| 17264 | .sse => try self.asmRegisterRegister( | 18625 | .sse => try self.asmRegisterRegister( |
| 17265 | @as(?Mir.Inst.FixedTag, switch (ty.scalarType(zcu).zigTypeTag(zcu)) { | 18626 | @as(?Mir.Inst.FixedTag, switch (ty.scalarType(zcu).zigTypeTag(zcu)) { |
| 17266 | else => switch (abi_size) { | 18627 | else => switch (abi_size) { |
| 17267 | 1...16 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else .{ ._, .movdqa }, | 18628 | 1...16 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else .{ ._dqa, .mov }, |
| 17268 | 17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null, | 18629 | 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null, |
| 17269 | else => null, | 18630 | else => null, |
| 17270 | }, | 18631 | }, |
| 17271 | .float => switch (ty.scalarType(zcu).floatBits(self.target.*)) { | 18632 | .float => switch (ty.scalarType(zcu).floatBits(self.target.*)) { |
| 17272 | 16, 128 => switch (abi_size) { | 18633 | 16, 128 => switch (abi_size) { |
| 17273 | 2...16 => if (self.hasFeature(.avx)) | 18634 | 2...16 => if (self.hasFeature(.avx)) |
| 17274 | .{ .v_, .movdqa } | 18635 | .{ .v_dqa, .mov } |
| 17275 | else | 18636 | else |
| 17276 | .{ ._, .movdqa }, | 18637 | .{ ._dqa, .mov }, |
| 17277 | 17...32 => if (self.hasFeature(.avx)) .{ .v_, .movdqa } else null, | 18638 | 17...32 => if (self.hasFeature(.avx)) .{ .v_dqa, .mov } else null, |
| 17278 | else => null, | 18639 | else => null, |
| 17279 | }, | 18640 | }, |
| 17280 | 32 => if (self.hasFeature(.avx)) .{ .v_ps, .mova } else .{ ._ps, .mova }, | 18641 | 32 => if (self.hasFeature(.avx)) .{ .v_ps, .mova } else .{ ._ps, .mova }, |
| ... | @@ -17346,7 +18707,7 @@ fn genSetReg( | ... | @@ -17346,7 +18707,7 @@ fn genSetReg( |
| 17346 | const pack_lock = self.register_manager.lockReg(pack_reg); | 18707 | const pack_lock = self.register_manager.lockReg(pack_reg); |
| 17347 | defer if (pack_lock) |lock| self.register_manager.unlockReg(lock); | 18708 | defer if (pack_lock) |lock| self.register_manager.unlockReg(lock); |
| 17348 | | 18709 | |
| 17349 | var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.info.scalar.bitSize(), 8)); | 18710 | var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.info.scalar.bitSize(self.target), 8)); |
| 17350 | switch (src_reg_mask.info.scalar) { | 18711 | switch (src_reg_mask.info.scalar) { |
| 17351 | else => {}, | 18712 | else => {}, |
| 17352 | .word => { | 18713 | .word => { |
| ... | @@ -17355,7 +18716,7 @@ fn genSetReg( | ... | @@ -17355,7 +18716,7 @@ fn genSetReg( |
| 17355 | if (has_avx) { | 18716 | if (has_avx) { |
| 17356 | try self.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias); | 18717 | try self.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias); |
| 17357 | } else { | 18718 | } else { |
| 17358 | try self.asmRegisterRegister(.{ ._, .movdqa }, pack_alias, src_alias); | 18719 | try self.asmRegisterRegister(.{ ._dqa, .mov }, pack_alias, src_alias); |
| 17359 | try self.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias); | 18720 | try self.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias); |
| 17360 | } | 18721 | } |
| 17361 | mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable; | 18722 | mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable; |
| ... | @@ -17592,7 +18953,10 @@ fn genSetMem( | ... | @@ -17592,7 +18953,10 @@ fn genSetMem( |
| 17592 | })).write( | 18953 | })).write( |
| 17593 | self, | 18954 | self, |
| 17594 | .{ .base = base, .mod = .{ .rm = .{ | 18955 | .{ .base = base, .mod = .{ .rm = .{ |
| 17595 | .size = .fromBitSize(@min(self.memSize(ty).bitSize(), src_alias.bitSize())), | 18956 | .size = .fromBitSize(@min( |
| | 18957 | self.memSize(ty).bitSize(self.target), |
| | 18958 | src_alias.bitSize(), |
| | 18959 | )), |
| 17596 | .disp = disp, | 18960 | .disp = disp, |
| 17597 | } } }, | 18961 | } } }, |
| 17598 | src_alias, | 18962 | src_alias, |
| ... | @@ -22460,36 +23824,6 @@ const Temp = struct { | ... | @@ -22460,36 +23824,6 @@ const Temp = struct { |
| 22460 | try cg.asmOpOnly(.{ .@"rep _sb", .mov }); | 23824 | try cg.asmOpOnly(.{ .@"rep _sb", .mov }); |
| 22461 | } | 23825 | } |
| 22462 | | 23826 | |
| 22463 | // i, m, r | | |
| 22464 | fn add(lhs: *Temp, rhs: *Temp, cg: *CodeGen) !Temp { | | |
| 22465 | const res_index = cg.next_temp_index; | | |
| 22466 | var res: Temp = .{ .index = res_index.toIndex() }; | | |
| 22467 | try cg.select(&.{ &res, lhs, rhs }, .{ ._, .add }, &.{ | | |
| 22468 | .{ .ops = &.{ .{ .match = 1 }, .r, .i } }, | | |
| 22469 | .{ .ops = &.{ .{ .match = 1 }, .m, .i } }, | | |
| 22470 | .{ .ops = &.{ .{ .match = 1 }, .r, .m } }, | | |
| 22471 | .{ .ops = &.{ .{ .match = 1 }, .m, .r } }, | | |
| 22472 | .{ .ops = &.{ .{ .match = 1 }, .r, .r } }, | | |
| 22473 | }); | | |
| 22474 | cg.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1); | | |
| 22475 | cg.temp_type[@intFromEnum(res_index)] = lhs.typeOf(cg); | | |
| 22476 | return res; | | |
| 22477 | } | | |
| 22478 | | | |
| 22479 | fn mul(lhs: *Temp, rhs: *Temp, cg: *CodeGen) !Temp { | | |
| 22480 | const res_index = cg.next_temp_index; | | |
| 22481 | var res: Temp = .{ .index = cg.next_temp_index.toIndex() }; | | |
| 22482 | try cg.select(&.{ &res, lhs, rhs }, .{ .i_, .mul }, &.{ | | |
| 22483 | .{ .ops = &.{ .r, .m, .i } }, | | |
| 22484 | .{ .ops = &.{ .r, .r, .i } }, | | |
| 22485 | .{ .ops = &.{ .{ .match = 1 }, .r, .m } }, | | |
| 22486 | .{ .ops = &.{ .{ .match = 1 }, .r, .r } }, | | |
| 22487 | }); | | |
| 22488 | cg.next_temp_index = @enumFromInt(@intFromEnum(res_index) + 1); | | |
| 22489 | cg.temp_type[@intFromEnum(res_index)] = lhs.typeOf(cg); | | |
| 22490 | return res; | | |
| 22491 | } | | |
| 22492 | | | |
| 22493 | fn moveTo(temp: Temp, inst: Air.Inst.Index, cg: *CodeGen) !void { | 23827 | fn moveTo(temp: Temp, inst: Air.Inst.Index, cg: *CodeGen) !void { |
| 22494 | if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) { | 23828 | if (cg.liveness.isUnused(inst)) try temp.die(cg) else switch (temp.unwrap(cg)) { |
| 22495 | .ref => { | 23829 | .ref => { |
| ... | @@ -22624,1242 +23958,129 @@ fn tempAllocRegPair(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) | ... | @@ -22624,1242 +23958,129 @@ fn tempAllocRegPair(cg: *CodeGen, ty: Type, rs: RegisterManager.RegisterBitSet) |
| 22624 | temp_index.tracking(cg).* = .init( | 23958 | temp_index.tracking(cg).* = .init( |
| 22625 | .{ .register_pair = try cg.register_manager.allocRegs(2, temp_index.toIndex(), rs) }, | 23959 | .{ .register_pair = try cg.register_manager.allocRegs(2, temp_index.toIndex(), rs) }, |
| 22626 | ); | 23960 | ); |
| 22627 | cg.temp_type[@intFromEnum(temp_index)] = ty; | 23961 | cg.temp_type[@intFromEnum(temp_index)] = ty; |
| 22628 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); | 23962 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); |
| 22629 | return .{ .index = temp_index.toIndex() }; | 23963 | return .{ .index = temp_index.toIndex() }; |
| 22630 | } | 23964 | } |
| 22631 | | 23965 | |
| 22632 | fn tempAllocMem(cg: *CodeGen, ty: Type) !Temp { | 23966 | fn tempAllocMem(cg: *CodeGen, ty: Type) !Temp { |
| 22633 | const temp_index = cg.next_temp_index; | 23967 | const temp_index = cg.next_temp_index; |
| 22634 | temp_index.tracking(cg).* = .init( | 23968 | temp_index.tracking(cg).* = .init( |
| 22635 | try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), false), | 23969 | try cg.allocRegOrMemAdvanced(ty, temp_index.toIndex(), false), |
| 22636 | ); | 23970 | ); |
| 22637 | cg.temp_type[@intFromEnum(temp_index)] = ty; | 23971 | cg.temp_type[@intFromEnum(temp_index)] = ty; |
| 22638 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); | 23972 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); |
| 22639 | return .{ .index = temp_index.toIndex() }; | 23973 | return .{ .index = temp_index.toIndex() }; |
| 22640 | } | 23974 | } |
| 22641 | | 23975 | |
| 22642 | fn tempFromValue(cg: *CodeGen, ty: Type, value: MCValue) !Temp { | 23976 | fn tempFromValue(cg: *CodeGen, ty: Type, value: MCValue) !Temp { |
| 22643 | const temp_index = cg.next_temp_index; | 23977 | const temp_index = cg.next_temp_index; |
| 22644 | temp_index.tracking(cg).* = .init(value); | 23978 | temp_index.tracking(cg).* = .init(value); |
| 22645 | cg.temp_type[@intFromEnum(temp_index)] = ty; | 23979 | cg.temp_type[@intFromEnum(temp_index)] = ty; |
| 22646 | try cg.getValue(value, temp_index.toIndex()); | 23980 | try cg.getValue(value, temp_index.toIndex()); |
| 22647 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); | 23981 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); |
| 22648 | return .{ .index = temp_index.toIndex() }; | 23982 | return .{ .index = temp_index.toIndex() }; |
| 22649 | } | 23983 | } |
| 22650 | | 23984 | |
| 22651 | fn tempFromOperand( | 23985 | fn tempFromOperand( |
| 22652 | cg: *CodeGen, | 23986 | cg: *CodeGen, |
| 22653 | inst: Air.Inst.Index, | 23987 | inst: Air.Inst.Index, |
| 22654 | op_index: Liveness.OperandInt, | 23988 | op_index: Liveness.OperandInt, |
| 22655 | op_ref: Air.Inst.Ref, | 23989 | op_ref: Air.Inst.Ref, |
| 22656 | ignore_death: bool, | 23990 | ignore_death: bool, |
| 22657 | ) !Temp { | 23991 | ) !Temp { |
| 22658 | const zcu = cg.pt.zcu; | 23992 | const zcu = cg.pt.zcu; |
| 22659 | const ip = &zcu.intern_pool; | 23993 | const ip = &zcu.intern_pool; |
| 22660 | | 23994 | |
| 22661 | if (ignore_death or !cg.liveness.operandDies(inst, op_index)) { | 23995 | if (ignore_death or !cg.liveness.operandDies(inst, op_index)) { |
| 22662 | if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst }; | 23996 | if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst }; |
| 22663 | const val = op_ref.toInterned().?; | 23997 | const val = op_ref.toInterned().?; |
| 22664 | const gop = try cg.const_tracking.getOrPut(cg.gpa, val); | 23998 | const gop = try cg.const_tracking.getOrPut(cg.gpa, val); |
| 22665 | if (!gop.found_existing) gop.value_ptr.* = .init(init: { | 23999 | if (!gop.found_existing) gop.value_ptr.* = .init(init: { |
| 22666 | const const_mcv = try cg.genTypedValue(.fromInterned(val)); | 24000 | const const_mcv = try cg.genTypedValue(.fromInterned(val)); |
| 22667 | switch (const_mcv) { | 24001 | switch (const_mcv) { |
| 22668 | .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) { | 24002 | .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) { |
| 22669 | .elf, .macho => { | 24003 | .elf, .macho => { |
| 22670 | if (cg.mod.pic) { | 24004 | if (cg.mod.pic) { |
| 22671 | try cg.spillRegisters(&.{ .rdi, .rax }); | 24005 | try cg.spillRegisters(&.{ .rdi, .rax }); |
| 22672 | } else { | 24006 | } else { |
| 22673 | try cg.spillRegisters(&.{.rax}); | 24007 | try cg.spillRegisters(&.{.rax}); |
| 22674 | } | | |
| 22675 | const frame_index = try cg.allocFrameIndex(.init(.{ | | |
| 22676 | .size = 8, | | |
| 22677 | .alignment = .@"8", | | |
| 22678 | })); | | |
| 22679 | try cg.genSetMem( | | |
| 22680 | .{ .frame = frame_index }, | | |
| 22681 | 0, | | |
| 22682 | .usize, | | |
| 22683 | .{ .lea_symbol = .{ .sym_index = tlv_sym } }, | | |
| 22684 | .{}, | | |
| 22685 | ); | | |
| 22686 | break :init .{ .load_frame = .{ .index = frame_index } }; | | |
| 22687 | }, | | |
| 22688 | else => break :init const_mcv, | | |
| 22689 | }, | | |
| 22690 | else => break :init const_mcv, | | |
| 22691 | } | | |
| 22692 | }); | | |
| 22693 | return cg.tempFromValue(.fromInterned(ip.typeOf(val)), gop.value_ptr.short); | | |
| 22694 | } | | |
| 22695 | | | |
| 22696 | const temp_index = cg.next_temp_index; | | |
| 22697 | const temp: Temp = .{ .index = temp_index.toIndex() }; | | |
| 22698 | const op_inst = op_ref.toIndex().?; | | |
| 22699 | const tracking = cg.getResolvedInstValue(op_inst); | | |
| 22700 | temp_index.tracking(cg).* = tracking.*; | | |
| 22701 | if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? }; | | |
| 22702 | cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref); | | |
| 22703 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); | | |
| 22704 | return temp; | | |
| 22705 | } | | |
| 22706 | | | |
| 22707 | inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp { | | |
| 22708 | var temps: [op_refs.len]Temp = undefined; | | |
| 22709 | inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| { | | |
| 22710 | temp.* = try cg.tempFromOperand(inst, op_index, op_ref, inline for (0..op_index) |prev_op_index| { | | |
| 22711 | if (op_ref == op_refs[prev_op_index]) break true; | | |
| 22712 | } else false); | | |
| 22713 | } | | |
| 22714 | return temps; | | |
| 22715 | } | | |
| 22716 | | | |
| 22717 | const Operand = union(enum) { | | |
| 22718 | none, | | |
| 22719 | reg: Register, | | |
| 22720 | mem: Memory, | | |
| 22721 | imm: Immediate, | | |
| 22722 | inst: Mir.Inst.Index, | | |
| 22723 | }; | | |
| 22724 | | | |
| 22725 | const Pattern = struct { | | |
| 22726 | ops: []const Op, | | |
| 22727 | commute: struct { u8, u8 } = .{ 0, 0 }, | | |
| 22728 | | | |
| 22729 | const Set = struct { | | |
| 22730 | required_features: []const std.Target.x86.Feature = &.{}, | | |
| 22731 | scalar: union(enum) { | | |
| 22732 | any, | | |
| 22733 | bool, | | |
| 22734 | float: Memory.Size, | | |
| 22735 | any_int: Memory.Size, | | |
| 22736 | signed_int: Memory.Size, | | |
| 22737 | unsigned_int: Memory.Size, | | |
| 22738 | any_float_or_int: Memory.Size, | | |
| 22739 | } = .any, | | |
| 22740 | clobbers: struct { eflags: bool = false } = .{}, | | |
| 22741 | invert_result: bool = false, | | |
| 22742 | loop: enum { | | |
| 22743 | /// only execute the instruction once | | |
| 22744 | once, | | |
| 22745 | /// execute the instruction on all groups of non-overlapping bits in the entire value | | |
| 22746 | bitwise, | | |
| 22747 | /// for each element, execute the instruction on each limb, propagating the carry flag | | |
| 22748 | limbwise_carry, | | |
| 22749 | /// for each element, execute the instruction on each limb, propagating a register | | |
| 22750 | limbwise_reduce, | | |
| 22751 | /// for each element, execute the instruction on pairs of limbs, starting from the | | |
| 22752 | /// least significant, propagating a limb | | |
| 22753 | limbwise_pairs_forward, | | |
| 22754 | /// for each element, execute the instruction on pairs of limbs, starting from the | | |
| 22755 | /// most significant, propagating a limb | | |
| 22756 | limbwise_pairs_reverse, | | |
| 22757 | /// for each element, execute the instruction | | |
| 22758 | elementwise, | | |
| 22759 | } = .once, | | |
| 22760 | mir_tag: Mir.Inst.FixedTag, | | |
| 22761 | final_mir_tag: ?Mir.Inst.FixedTag = null, | | |
| 22762 | patterns: []const Pattern, | | |
| 22763 | }; | | |
| 22764 | | | |
| 22765 | const Op = union(enum) { | | |
| 22766 | /// reuse another operand | | |
| 22767 | implicit: u8, | | |
| 22768 | /// repeat another operand | | |
| 22769 | explicit: u8, | | |
| 22770 | /// a condition code | | |
| 22771 | cc, | | |
| 22772 | /// any general purpose register | | |
| 22773 | gpr, | | |
| 22774 | /// any 64-bit mmx register | | |
| 22775 | mm, | | |
| 22776 | /// any 128-bit sse register | | |
| 22777 | xmm, | | |
| 22778 | /// any 256-bit sse register | | |
| 22779 | ymm, | | |
| 22780 | /// a 64-bit mmx register mask | | |
| 22781 | mm_mask, | | |
| 22782 | /// a 128-bit sse register mask | | |
| 22783 | xmm_mask, | | |
| 22784 | /// a 256-bit sse register mask | | |
| 22785 | ymm_mask, | | |
| 22786 | /// a 64-bit mmx register sign mask | | |
| 22787 | mm_sign_mask, | | |
| 22788 | /// a 128-bit sse register sign mask | | |
| 22789 | xmm_sign_mask, | | |
| 22790 | /// a 256-bit sse register sign mask | | |
| 22791 | ymm_sign_mask, | | |
| 22792 | /// any memory | | |
| 22793 | mem, | | |
| 22794 | /// a limb stored in a general purpose register | | |
| 22795 | gpr_limb, | | |
| 22796 | /// a limb stored in a 64-bit mmx register | | |
| 22797 | mm_limb, | | |
| 22798 | /// a limb stored in a 128-bit sse register | | |
| 22799 | xmm_limb, | | |
| 22800 | /// a limb stored in a 256-bit sse register | | |
| 22801 | ymm_limb, | | |
| 22802 | /// a limb stored in memory | | |
| 22803 | mem_limb, | | |
| 22804 | /// a mutable limb stored in a general purpose register | | |
| 22805 | mut_gpr_limb, | | |
| 22806 | /// a mutable limb stored in memory | | |
| 22807 | mut_mem_limb, | | |
| 22808 | /// an element stored in a condition code | | |
| 22809 | cc_elem, | | |
| 22810 | /// an element stored in a general purpose register | | |
| 22811 | gpr_elem, | | |
| 22812 | /// an element stored in memory | | |
| 22813 | mem_elem, | | |
| 22814 | /// a limb stored in a 64-bit mmx register mask | | |
| 22815 | mm_mask_limb, | | |
| 22816 | /// a limb stored in a 128-bit sse register masuk | | |
| 22817 | xmm_mask_limb, | | |
| 22818 | /// a limb stored in a 256-bit sse register masuk | | |
| 22819 | ymm_mask_limb, | | |
| 22820 | /// specific immediate | | |
| 22821 | imm: i8, | | |
| 22822 | /// any immediate signed extended from 32 bits | | |
| 22823 | simm32, | | |
| 22824 | /// a temp general purpose register containing all ones | | |
| 22825 | umax_gpr, | | |
| 22826 | /// a temp 64-bit mmx register containing all ones | | |
| 22827 | umax_mm, | | |
| 22828 | /// a temp 128-bit sse register containing all ones | | |
| 22829 | umax_xmm, | | |
| 22830 | /// a temp 256-bit sse register containing all ones | | |
| 22831 | umax_ymm, | | |
| 22832 | | | |
| 22833 | fn matches(op: Op, is_mut: bool, temp: Temp, cg: *CodeGen) bool { | | |
| 22834 | switch (op) { | | |
| 22835 | .implicit, .explicit, .cc, .cc_elem => unreachable, | | |
| 22836 | else => {}, | | |
| 22837 | // temp is undefined | | |
| 22838 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => return true, | | |
| 22839 | } | | |
| 22840 | const temp_ty = temp.typeOf(cg); | | |
| 22841 | const abi_size = temp_ty.abiSize(cg.pt.zcu); | | |
| 22842 | return switch (op) { | | |
| 22843 | .implicit, .explicit, .cc, .cc_elem, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, | | |
| 22844 | .gpr => abi_size <= 8 and switch (temp.tracking(cg).short) { | | |
| 22845 | .register => |reg| reg.class() == .general_purpose, | | |
| 22846 | .register_offset => |reg_off| reg_off.reg.class() == .general_purpose and | | |
| 22847 | reg_off.off == 0, | | |
| 22848 | else => cg.regClassForType(temp_ty) == .general_purpose, | | |
| 22849 | }, | | |
| 22850 | .mm, .mm_mask, .mm_sign_mask => abi_size <= 8 and switch (temp.tracking(cg).short) { | | |
| 22851 | .register => |reg| reg.class() == .mmx, | | |
| 22852 | .register_offset => |reg_off| reg_off.reg.class() == .mmx and reg_off.off == 0, | | |
| 22853 | else => cg.regClassForType(temp_ty) == .mmx, | | |
| 22854 | }, | | |
| 22855 | .xmm, .xmm_mask, .xmm_sign_mask => abi_size > 8 and abi_size <= 16 and switch (temp.tracking(cg).short) { | | |
| 22856 | .register => |reg| reg.class() == .sse, | | |
| 22857 | .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0, | | |
| 22858 | else => cg.regClassForType(temp_ty) == .sse, | | |
| 22859 | }, | | |
| 22860 | .ymm, .ymm_mask, .ymm_sign_mask => abi_size > 16 and abi_size <= 32 and switch (temp.tracking(cg).short) { | | |
| 22861 | .register => |reg| reg.class() == .sse, | | |
| 22862 | .register_offset => |reg_off| reg_off.reg.class() == .sse and reg_off.off == 0, | | |
| 22863 | else => cg.regClassForType(temp_ty) == .sse, | | |
| 22864 | }, | | |
| 22865 | .mem, .mem_limb, .mut_mem_limb, .mem_elem => (!is_mut or temp.isMut(cg)) and temp.tracking(cg).short.isMemory(), | | |
| 22866 | .gpr_limb, .mut_gpr_limb, .gpr_elem => abi_size > 8 and switch (temp.tracking(cg).short) { | | |
| 22867 | .register, .register_pair, .register_triple, .register_quadruple => true, | | |
| 22868 | else => |mcv| mcv.isMemory(), | | |
| 22869 | }, | | |
| 22870 | .mm_limb, .mm_mask_limb => abi_size > 8 and switch (temp.tracking(cg).short) { | | |
| 22871 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { | | |
| 22872 | if (reg.class() != .mmx) break false; | | |
| 22873 | } else true, | | |
| 22874 | else => |mcv| mcv.isMemory() and cg.regClassForType(temp_ty) == .mmx, | | |
| 22875 | }, | | |
| 22876 | .xmm_limb, .xmm_mask_limb => abi_size > 16 and switch (temp.tracking(cg).short) { | | |
| 22877 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { | | |
| 22878 | if (reg.class() != .sse) break false; | | |
| 22879 | } else true, | | |
| 22880 | else => |mcv| mcv.isMemory(), | | |
| 22881 | }, | | |
| 22882 | .ymm_limb, .ymm_mask_limb => abi_size > 32 and switch (temp.tracking(cg).short) { | | |
| 22883 | inline .register_pair, .register_triple, .register_quadruple => |regs| for (regs) |reg| { | | |
| 22884 | if (reg.class() != .sse) break false; | | |
| 22885 | } else true, | | |
| 22886 | else => |mcv| mcv.isMemory(), | | |
| 22887 | }, | | |
| 22888 | .imm => |specific_imm| if (is_mut) unreachable else switch (temp.tracking(cg).short) { | | |
| 22889 | .immediate => |imm| @as(i64, @bitCast(imm)) == specific_imm, | | |
| 22890 | else => false, | | |
| 22891 | }, | | |
| 22892 | .simm32 => if (is_mut) unreachable else switch (temp.tracking(cg).short) { | | |
| 22893 | .immediate => |imm| abi_size <= 4 or std.math.cast(i32, @as(i64, @bitCast(imm))) != null, | | |
| 22894 | else => false, | | |
| 22895 | }, | | |
| 22896 | }; | | |
| 22897 | } | | |
| 22898 | }; | | |
| 22899 | | | |
| 22900 | const Instruction = struct { | | |
| 22901 | mir_tag: Mir.Inst.FixedTag, | | |
| 22902 | operands: [4]Instruction.Operand, | | |
| 22903 | | | |
| 22904 | const Operand = union(enum) { | | |
| 22905 | regb: u8, | | |
| 22906 | regw: u8, | | |
| 22907 | ereg: u8, | | |
| 22908 | rreg: u8, | | |
| 22909 | xmm: u8, | | |
| 22910 | ymm: u8, | | |
| 22911 | }; | | |
| 22912 | }; | | |
| 22913 | }; | | |
| 22914 | const SelectOptions = struct { | | |
| 22915 | cc: ?Condition = null, | | |
| 22916 | invert_result: bool = false, | | |
| 22917 | }; | | |
| 22918 | fn select( | | |
| 22919 | cg: *CodeGen, | | |
| 22920 | dst_temps: []Temp, | | |
| 22921 | dst_tys: []const Type, | | |
| 22922 | src_temps: []Temp, | | |
| 22923 | pattern_sets: []const Pattern.Set, | | |
| 22924 | opts: SelectOptions, | | |
| 22925 | ) !void { | | |
| 22926 | var loop: struct { | | |
| 22927 | element_reloc: Mir.Inst.Index, | | |
| 22928 | element_offset: Offset, | | |
| 22929 | element_size: ?u13, | | |
| 22930 | limb_reloc: Mir.Inst.Index, | | |
| 22931 | limb_offset: Offset, | | |
| 22932 | limb_size: ?u8, | | |
| 22933 | shuffle_temp: ?Temp, | | |
| 22934 | mask_limb_temp: ?Temp, | | |
| 22935 | mask_limb_offset: Offset, | | |
| 22936 | mask_limb_offset_lock: ?RegisterLock, | | |
| 22937 | mask_limb_bit_size: ?u7, | | |
| 22938 | mask_store_temp: ?Temp, | | |
| 22939 | mask_store_reg: ?Register, | | |
| 22940 | mask_store_bit_size: ?u7, | | |
| 22941 | remaining_size: ?u64, | | |
| 22942 | | | |
| 22943 | const Offset = union(enum) { | | |
| 22944 | unused, | | |
| 22945 | known: u31, | | |
| 22946 | temp: Temp, | | |
| 22947 | }; | | |
| 22948 | } = .{ | | |
| 22949 | .element_reloc = undefined, | | |
| 22950 | .element_offset = .unused, | | |
| 22951 | .element_size = null, | | |
| 22952 | .limb_reloc = undefined, | | |
| 22953 | .limb_offset = .unused, | | |
| 22954 | .limb_size = null, | | |
| 22955 | .shuffle_temp = null, | | |
| 22956 | .mask_limb_temp = null, | | |
| 22957 | .mask_limb_offset = .unused, | | |
| 22958 | .mask_limb_offset_lock = null, | | |
| 22959 | .mask_limb_bit_size = null, | | |
| 22960 | .mask_store_temp = null, | | |
| 22961 | .mask_store_reg = null, | | |
| 22962 | .mask_store_bit_size = null, | | |
| 22963 | .remaining_size = null, | | |
| 22964 | }; | | |
| 22965 | var extra_temps: [4]?Temp = @splat(null); | | |
| 22966 | pattern_sets: for (pattern_sets) |pattern_set| { | | |
| 22967 | for (pattern_set.required_features) |required_feature| if (!cg.hasFeature(required_feature)) continue :pattern_sets; | | |
| 22968 | for (src_temps) |src_temp| switch (pattern_set.scalar) { | | |
| 22969 | .any => {}, | | |
| 22970 | .bool => if (src_temp.typeOf(cg).scalarType(cg.pt.zcu).toIntern() != .bool_type) continue :pattern_sets, | | |
| 22971 | .float => |size| { | | |
| 22972 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); | | |
| 22973 | if (!scalar_ty.isRuntimeFloat()) continue :pattern_sets; | | |
| 22974 | if (scalar_ty.floatBits(cg.target.*) != size.bitSize()) continue :pattern_sets; | | |
| 22975 | }, | | |
| 22976 | .any_int => |size| { | | |
| 22977 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); | | |
| 22978 | if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets; | | |
| 22979 | if (scalar_ty.intInfo(cg.pt.zcu).bits > size.bitSize()) continue :pattern_sets; | | |
| 22980 | }, | | |
| 22981 | .signed_int => |size| { | | |
| 22982 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); | | |
| 22983 | if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets; | | |
| 22984 | const scalar_info = scalar_ty.intInfo(cg.pt.zcu); | | |
| 22985 | if (scalar_info.signedness != .signed) continue :pattern_sets; | | |
| 22986 | if (scalar_info.bits > size.bitSize()) continue :pattern_sets; | | |
| 22987 | }, | | |
| 22988 | .unsigned_int => |size| { | | |
| 22989 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); | | |
| 22990 | if (!scalar_ty.isAbiInt(cg.pt.zcu)) continue :pattern_sets; | | |
| 22991 | const scalar_info = scalar_ty.intInfo(cg.pt.zcu); | | |
| 22992 | if (scalar_info.signedness != .unsigned) continue :pattern_sets; | | |
| 22993 | if (scalar_info.bits > size.bitSize()) continue :pattern_sets; | | |
| 22994 | }, | | |
| 22995 | .any_float_or_int => |size| { | | |
| 22996 | const scalar_ty = src_temp.typeOf(cg).scalarType(cg.pt.zcu); | | |
| 22997 | if (scalar_ty.isRuntimeFloat()) { | | |
| 22998 | if (scalar_ty.floatBits(cg.target.*) != size.bitSize()) continue :pattern_sets; | | |
| 22999 | } else if (scalar_ty.isAbiInt(cg.pt.zcu)) { | | |
| 23000 | if (scalar_ty.intInfo(cg.pt.zcu).bits > size.bitSize()) continue :pattern_sets; | | |
| 23001 | } else continue :pattern_sets; | | |
| 23002 | }, | | |
| 23003 | }; | | |
| 23004 | patterns: for (pattern_set.patterns) |pattern| { | | |
| 23005 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { | | |
| 23006 | const ref_src_op, const is_mut = switch (src_op) { | | |
| 23007 | .implicit, .explicit => |linked_index| .{ pattern.ops[linked_index], true }, | | |
| 23008 | .mut_mem_limb, .mut_gpr_limb => .{ src_op, true }, | | |
| 23009 | else => .{ src_op, false }, | | |
| 23010 | }; | | |
| 23011 | if (!ref_src_op.matches(is_mut, src_temp, cg)) continue :patterns; | | |
| 23012 | } | | |
| 23013 | | | |
| 23014 | for (pattern.ops) |op| switch (op) { | | |
| 23015 | else => {}, | | |
| 23016 | .cc_elem, | | |
| 23017 | .mm_mask_limb, | | |
| 23018 | .xmm_mask_limb, | | |
| 23019 | .ymm_mask_limb, | | |
| 23020 | => if (loop.mask_limb_offset_lock == null and !cg.hasFeature(.bmi2)) { | | |
| 23021 | try cg.register_manager.getKnownReg(.rcx, null); | | |
| 23022 | loop.mask_limb_offset_lock = cg.register_manager.lockKnownRegAssumeUnused(.rcx); | | |
| 23023 | }, | | |
| 23024 | }; | | |
| 23025 | while (true) for (src_temps, pattern.ops[dst_temps.len..]) |*src_temp, src_op| { | | |
| 23026 | if (switch (switch (src_op) { | | |
| 23027 | .implicit, .explicit => |linked_index| pattern.ops[linked_index], | | |
| 23028 | else => src_op, | | |
| 23029 | }) { | | |
| 23030 | .implicit, .explicit, .cc, .cc_elem => unreachable, | | |
| 23031 | .gpr => try src_temp.toRegClass(true, .general_purpose, cg), | | |
| 23032 | .mm, .mm_mask, .mm_sign_mask => try src_temp.toRegClass(true, .mmx, cg), | | |
| 23033 | .xmm, | | |
| 23034 | .ymm, | | |
| 23035 | .xmm_mask, | | |
| 23036 | .ymm_mask, | | |
| 23037 | .xmm_sign_mask, | | |
| 23038 | .ymm_sign_mask, | | |
| 23039 | => try src_temp.toRegClass(true, .sse, cg), | | |
| 23040 | .mem => try src_temp.toBase(cg), | | |
| 23041 | .imm, .simm32 => false, | | |
| 23042 | .gpr_limb, | | |
| 23043 | .mm_limb, | | |
| 23044 | .xmm_limb, | | |
| 23045 | .ymm_limb, | | |
| 23046 | .mem_limb, | | |
| 23047 | .mut_gpr_limb, | | |
| 23048 | .mut_mem_limb, | | |
| 23049 | .gpr_elem, | | |
| 23050 | .mem_elem, | | |
| 23051 | => switch (src_temp.tracking(cg).short) { | | |
| 23052 | .register, .register_pair, .register_triple, .register_quadruple => false, | | |
| 23053 | else => try src_temp.toBase(cg), | | |
| 23054 | }, | | |
| 23055 | .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => false, | | |
| 23056 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => false, | | |
| 23057 | }) break; | | |
| 23058 | } else break; | | |
| 23059 | | | |
| 23060 | const invert_result = opts.invert_result != pattern_set.invert_result; | | |
| 23061 | var dst_is_linked: std.StaticBitSet(4) = .initEmpty(); | | |
| 23062 | var mir_ops_len: usize = 0; | | |
| 23063 | for (pattern.ops[0..dst_temps.len]) |dst_op| switch (dst_op) { | | |
| 23064 | else => mir_ops_len += 1, | | |
| 23065 | .cc, .cc_elem => {}, | | |
| 23066 | }; | | |
| 23067 | const dst_mir_ops_len = mir_ops_len; | | |
| 23068 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { | | |
| 23069 | defer mir_ops_len += @intFromBool(src_op != .implicit); | | |
| 23070 | const linked_src_op, const extra_temp = op: switch (src_op) { | | |
| 23071 | .implicit, .explicit => |linked_index| { | | |
| 23072 | if (src_temp.isMut(cg)) { | | |
| 23073 | dst_temps[linked_index] = src_temp; | | |
| 23074 | dst_is_linked.set(linked_index); | | |
| 23075 | } | | |
| 23076 | break :op .{ pattern.ops[linked_index], &extra_temps[linked_index] }; | | |
| 23077 | }, | | |
| 23078 | else => .{ src_op, &extra_temps[mir_ops_len] }, | | |
| 23079 | }; | | |
| 23080 | const limb_size: u8, const rc = switch (linked_src_op) { | | |
| 23081 | else => continue, | | |
| 23082 | .gpr_limb, .mut_gpr_limb, .gpr_elem => .{ @intCast(@divExact(Memory.Size.bitSize(switch (pattern_set.scalar) { | | |
| 23083 | .any => .qword, | | |
| 23084 | .bool => unreachable, | | |
| 23085 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, | | |
| 23086 | }), 8)), abi.RegisterClass.gp }, | | |
| 23087 | .mm_limb, .mm_mask_limb => .{ 8, @panic("TODO") }, | | |
| 23088 | .xmm_limb, .xmm_mask_limb => .{ 16, abi.RegisterClass.sse }, | | |
| 23089 | .ymm_limb, .ymm_mask_limb => .{ 32, abi.RegisterClass.sse }, | | |
| 23090 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => { | | |
| 23091 | assert(extra_temp.* == null); | | |
| 23092 | extra_temp.* = try cg.tempAllocReg(.noreturn, switch (linked_src_op) { | | |
| 23093 | else => unreachable, | | |
| 23094 | .umax_gpr => abi.RegisterClass.gp, | | |
| 23095 | .umax_mm => @panic("TODO"), | | |
| 23096 | .umax_xmm, .umax_ymm => abi.RegisterClass.sse, | | |
| 23097 | }); | | |
| 23098 | continue; | | |
| 23099 | }, | | |
| 23100 | }; | | |
| 23101 | assert(loop.limb_size == null or loop.limb_size == limb_size); | | |
| 23102 | loop.limb_size = limb_size; | | |
| 23103 | loop.remaining_size = loop.remaining_size orelse src_temp.typeOf(cg).abiSize(cg.pt.zcu); | | |
| 23104 | const src_mcv = src_temp.tracking(cg).short; | | |
| 23105 | switch (src_mcv) { | | |
| 23106 | .register, .register_pair, .register_triple, .register_quadruple => { | | |
| 23107 | switch (loop.limb_offset) { | | |
| 23108 | .unused, .temp => loop.limb_offset = .{ .known = 0 }, | | |
| 23109 | .known => {}, | | |
| 23110 | } | | |
| 23111 | if (switch (linked_src_op) { | | |
| 23112 | .mut_gpr_limb => true, | | |
| 23113 | else => !rc.isSet(RegisterManager.indexOfRegIntoTracked(src_mcv.getRegs()[0]).?), | | |
| 23114 | }) { | | |
| 23115 | if (loop.shuffle_temp == null) loop.shuffle_temp = try cg.tempAllocReg(.noreturn, abi.RegisterClass.sse); | | |
| 23116 | assert(extra_temp.* == null); | | |
| 23117 | extra_temp.* = try cg.tempAllocReg(.usize, rc); | | |
| 23118 | } | | |
| 23119 | }, | | |
| 23120 | else => { | | |
| 23121 | switch (loop.limb_offset) { | | |
| 23122 | .unused => loop.limb_offset = .{ .temp = undefined }, | | |
| 23123 | .known, .temp => {}, | | |
| 23124 | } | | |
| 23125 | assert(extra_temp.* == null); | | |
| 23126 | extra_temp.* = try cg.tempAllocReg(.usize, rc); | | |
| 23127 | }, | | |
| 23128 | } | | |
| 23129 | } | | |
| 23130 | for ( | | |
| 23131 | 0.., | | |
| 23132 | dst_temps, | | |
| 23133 | pattern.ops[0..dst_temps.len], | | |
| 23134 | dst_tys, | | |
| 23135 | extra_temps[0..dst_temps.len], | | |
| 23136 | ) |dst_index, *dst_temp, dst_op, dst_ty, *extra_temp| switch (dst_op) { | | |
| 23137 | else => if (!dst_is_linked.isSet(dst_index)) { | | |
| 23138 | dst_temp.* = dst_temp: switch (dst_op) { | | |
| 23139 | .implicit => unreachable, | | |
| 23140 | .explicit => |linked_index| dst_temps[linked_index], | | |
| 23141 | .cc => try cg.tempFromValue(.bool, .{ .eflags = switch (invert_result) { | | |
| 23142 | false => opts.cc.?, | | |
| 23143 | true => opts.cc.?.negate(), | | |
| 23144 | } }), | | |
| 23145 | .gpr => try cg.tempAllocReg(dst_ty, abi.RegisterClass.gp), | | |
| 23146 | .mm, .mm_mask, .mm_sign_mask => @panic("TODO"), | | |
| 23147 | .xmm, .xmm_mask, .xmm_sign_mask => try cg.tempAllocReg(dst_ty, abi.RegisterClass.sse), | | |
| 23148 | .ymm, .ymm_mask, .ymm_sign_mask => try cg.tempAllocReg(dst_ty, abi.RegisterClass.sse), | | |
| 23149 | .mem => @panic("TODO"), | | |
| 23150 | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb, .mut_gpr_limb, .gpr_elem => { | | |
| 23151 | if (extra_temp.* == null) extra_temp.* = try cg.tempAllocReg(.noreturn, switch (dst_op) { | | |
| 23152 | else => unreachable, | | |
| 23153 | .gpr_limb, .mut_gpr_limb, .gpr_elem => abi.RegisterClass.gp, | | |
| 23154 | .mm_limb => @panic("TODO"), | | |
| 23155 | .xmm_limb, .ymm_limb => abi.RegisterClass.sse, | | |
| 23156 | }); | | |
| 23157 | break :dst_temp try cg.tempAlloc(dst_ty); | | |
| 23158 | }, | | |
| 23159 | .mem_limb, .mut_mem_limb, .mem_elem => try cg.tempAlloc(dst_ty), | | |
| 23160 | .cc_elem, .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => unreachable, // already checked | | |
| 23161 | .imm, .simm32, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, // unmodifiable destination | | |
| 23162 | }; | | |
| 23163 | }, | | |
| 23164 | .cc_elem, .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => { | | |
| 23165 | const scalar_size = @divExact(Memory.Size.bitSize(switch (pattern_set.scalar) { | | |
| 23166 | .any => .qword, | | |
| 23167 | .bool => unreachable, | | |
| 23168 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, | | |
| 23169 | }), 8); | | |
| 23170 | const mask_bit_size = @divExact(loop.remaining_size.?, scalar_size); | | |
| 23171 | const mask_limb_bit_size: u7 = @intCast(@divExact(loop.limb_size.?, scalar_size)); | | |
| 23172 | assert(loop.mask_limb_bit_size == null or loop.mask_limb_bit_size == mask_limb_bit_size); | | |
| 23173 | loop.mask_limb_bit_size = mask_limb_bit_size; | | |
| 23174 | const mask_store_bit_size = mask_store_bit_size: { | | |
| 23175 | // Try to match limb size so that no shifting will be needed. | | |
| 23176 | if (mask_limb_bit_size % 8 == 0) break :mask_store_bit_size mask_limb_bit_size; | | |
| 23177 | // If abi size <= 8 the entire value can be stored at once, | | |
| 23178 | // enabling store forwarding and minimizing store buffer usage. | | |
| 23179 | // Otherwise, we will be performing shifts that need to wrap at | | |
| 23180 | // store size, which for x86 requires 32 or 64, so just pick 64 | | |
| 23181 | // for the same reasons as above. | | |
| 23182 | break :mask_store_bit_size @min(mask_bit_size, 64); | | |
| 23183 | }; | | |
| 23184 | assert(loop.mask_store_bit_size == null or loop.mask_store_bit_size == mask_store_bit_size); | | |
| 23185 | loop.mask_store_bit_size = mask_store_bit_size; | | |
| 23186 | loop.mask_limb_offset = loop.limb_offset; | | |
| 23187 | if (loop.mask_limb_temp == null) { | | |
| 23188 | loop.mask_limb_temp = try cg.tempAllocReg(.usize, abi.RegisterClass.gp); | | |
| 23189 | if (dst_op == .cc_elem and mask_store_bit_size > 8) { | | |
| 23190 | // setcc only clears 8 bits | | |
| 23191 | const mask_limb_alias = loop.mask_limb_temp.?.tracking(cg).short.register.to32(); | | |
| 23192 | try cg.spillEflagsIfOccupied(); | | |
| 23193 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_limb_alias, mask_limb_alias); | | |
| 23194 | } | | |
| 23195 | } | | |
| 23196 | if (mask_limb_bit_size < mask_store_bit_size and loop.mask_store_reg == null) { | | |
| 23197 | loop.mask_store_temp = try cg.tempAllocReg(.usize, abi.RegisterClass.gp); | | |
| 23198 | loop.mask_store_reg = loop.mask_store_temp.?.tracking(cg).short.register; | | |
| 23199 | } | | |
| 23200 | dst_temp.* = if (mask_store_bit_size < mask_bit_size) | | |
| 23201 | try cg.tempAllocMem(dst_ty) | | |
| 23202 | else if (loop.mask_store_temp) |mask_store_temp| dst_temp: { | | |
| 23203 | loop.mask_store_temp = null; | | |
| 23204 | break :dst_temp mask_store_temp; | | |
| 23205 | } else try cg.tempAlloc(dst_ty); | | |
| 23206 | }, | | |
| 23207 | }; | | |
| 23208 | switch (loop.mask_limb_offset) { | | |
| 23209 | .unused, .known => {}, | | |
| 23210 | .temp => |*mask_limb_offset| { | | |
| 23211 | mask_limb_offset.* = if (cg.hasFeature(.bmi2)) | | |
| 23212 | try cg.tempAllocReg(.usize, abi.RegisterClass.gp) | | |
| 23213 | else if (loop.mask_limb_offset_lock != null) | | |
| 23214 | try cg.tempFromValue(.usize, .{ .register = .rcx }) | | |
| 23215 | else | | |
| 23216 | unreachable; | | |
| 23217 | if (loop.mask_store_reg) |mask_store_reg| { | | |
| 23218 | const mask_store_alias = | | |
| 23219 | if (loop.mask_store_bit_size.? > 8) mask_store_reg.to32() else mask_store_reg.to8(); | | |
| 23220 | try cg.spillEflagsIfOccupied(); | | |
| 23221 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias); | | |
| 23222 | } | | |
| 23223 | }, | | |
| 23224 | } | | |
| 23225 | if (loop.mask_limb_offset_lock) |lock| cg.register_manager.unlockReg(lock); | | |
| 23226 | loop.mask_limb_offset_lock = null; | | |
| 23227 | switch (loop.element_offset) { | | |
| 23228 | .unused, .known => {}, | | |
| 23229 | .temp => |*element_offset| { | | |
| 23230 | element_offset.* = try cg.tempAllocReg(.usize, abi.RegisterClass.gp); | | |
| 23231 | const element_offset_reg = element_offset.tracking(cg).short.register; | | |
| 23232 | try cg.spillEflagsIfOccupied(); | | |
| 23233 | try cg.asmRegisterRegister(.{ ._, .xor }, element_offset_reg.to32(), element_offset_reg.to32()); | | |
| 23234 | loop.element_reloc = @intCast(cg.mir_instructions.len); | | |
| 23235 | }, | | |
| 23236 | } | | |
| 23237 | switch (loop.limb_offset) { | | |
| 23238 | .unused, .known => {}, | | |
| 23239 | .temp => |*limb_offset| limb_offset.* = try cg.tempAllocReg(.usize, abi.RegisterClass.gp), | | |
| 23240 | } | | |
| 23241 | while (true) { | | |
| 23242 | switch (loop.mask_limb_offset) { | | |
| 23243 | .unused, .known => {}, | | |
| 23244 | .temp => |mask_limb_offset| { | | |
| 23245 | const mask_limb_offset_reg = mask_limb_offset.tracking(cg).short.register.to32(); | | |
| 23246 | try cg.spillEflagsIfOccupied(); | | |
| 23247 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_limb_offset_reg, mask_limb_offset_reg); | | |
| 23248 | }, | | |
| 23249 | } | | |
| 23250 | switch (loop.limb_offset) { | | |
| 23251 | .unused, .known => {}, | | |
| 23252 | .temp => |limb_offset| { | | |
| 23253 | const limb_offset_reg = limb_offset.tracking(cg).short.register.to32(); | | |
| 23254 | try cg.spillEflagsIfOccupied(); | | |
| 23255 | try cg.asmRegisterRegister(.{ ._, .xor }, limb_offset_reg.to32(), limb_offset_reg.to32()); | | |
| 23256 | loop.limb_reloc = @intCast(cg.mir_instructions.len); | | |
| 23257 | }, | | |
| 23258 | } | | |
| 23259 | while (true) { | | |
| 23260 | var mir_ops: [4]Operand = @splat(.none); | | |
| 23261 | mir_ops_len = dst_mir_ops_len; | | |
| 23262 | for (src_temps, pattern.ops[dst_temps.len..]) |src_temp, src_op| { | | |
| 23263 | defer mir_ops_len += @intFromBool(src_op != .implicit); | | |
| 23264 | const mir_op, const linked_src_op, const extra_temp = switch (src_op) { | | |
| 23265 | .implicit => |linked_index| .{ &mir_ops[linked_index], pattern.ops[linked_index], extra_temps[linked_index] }, | | |
| 23266 | .explicit => |linked_index| .{ &mir_ops[mir_ops_len], pattern.ops[linked_index], extra_temps[linked_index] }, | | |
| 23267 | else => .{ &mir_ops[mir_ops_len], src_op, extra_temps[mir_ops_len] }, | | |
| 23268 | }; | | |
| 23269 | const src_mcv = switch (linked_src_op) { | | |
| 23270 | else => src_temp, | | |
| 23271 | // src_temp is undefined | | |
| 23272 | .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => extra_temp.?, | | |
| 23273 | }.tracking(cg).short; | | |
| 23274 | switch (linked_src_op) { | | |
| 23275 | else => {}, | | |
| 23276 | .gpr_limb, | | |
| 23277 | .mm_limb, | | |
| 23278 | .xmm_limb, | | |
| 23279 | .ymm_limb, | | |
| 23280 | .mut_gpr_limb, | | |
| 23281 | .gpr_elem, | | |
| 23282 | .mm_mask_limb, | | |
| 23283 | .xmm_mask_limb, | | |
| 23284 | .ymm_mask_limb, | | |
| 23285 | => if (extra_temp) |limb_temp| switch (src_mcv) { | | |
| 23286 | .register, .register_pair, .register_triple, .register_quadruple => { | | |
| 23287 | const limb_reg = registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?); | | |
| 23288 | const src_regs = src_mcv.getRegs(); | | |
| 23289 | const src_reg_size: u32 = @intCast(switch (src_mcv) { | | |
| 23290 | .register => src_temp.typeOf(cg).abiSize(cg.pt.zcu), | | |
| 23291 | else => @divExact(src_regs[0].bitSize(), 8), | | |
| 23292 | }); | | |
| 23293 | const src_reg = src_regs[loop.limb_offset.known / src_reg_size]; | | |
| 23294 | assert(src_mcv == .register or src_reg.bitSize() == 8 * src_reg_size); | | |
| 23295 | switch (src_reg.class()) { | | |
| 23296 | else => unreachable, | | |
| 23297 | .general_purpose => try cg.asmRegisterRegister( | | |
| 23298 | .{ ._, .mov }, | | |
| 23299 | limb_reg, | | |
| 23300 | registerAlias(src_reg, src_reg_size), | | |
| 23301 | ), | | |
| 23302 | .sse => { | | |
| 23303 | assert(src_reg_size == 16); | | |
| 23304 | const limb_alias_size = @max(loop.limb_size.?, 4); | | |
| 23305 | const limb_alias = registerAlias(limb_reg, limb_alias_size); | | |
| 23306 | const src_reg_offset = loop.limb_offset.known % src_reg_size; | | |
| 23307 | switch (limb_reg_offset: { | | |
| 23308 | extr: { | | |
| 23309 | const limb_size = if (cg.hasFeature(.sse4_1)) loop.limb_size.? else 2; | | |
| 23310 | if (loop.limb_size.? > limb_size) break :extr; | | |
| 23311 | const limb_offset = src_reg_offset / limb_size; | | |
| 23312 | if (limb_offset == 0) break :extr; | | |
| 23313 | try cg.asmRegisterRegisterImmediate(.{ switch (limb_size) { | | |
| 23314 | else => unreachable, | | |
| 23315 | 1 => if (cg.hasFeature(.avx)) .vp_b else .p_b, | | |
| 23316 | 2 => if (cg.hasFeature(.avx)) .vp_w else .p_w, | | |
| 23317 | 4 => if (cg.hasFeature(.avx)) .vp_d else .p_d, | | |
| 23318 | 8 => if (cg.hasFeature(.avx)) .vp_q else .p_q, | | |
| 23319 | }, .extr }, limb_alias, src_reg.to128(), .u(limb_offset)); | | |
| 23320 | break :limb_reg_offset src_reg_offset % limb_size; | | |
| 23321 | } | | |
| 23322 | try cg.asmRegisterRegister( | | |
| 23323 | .{ switch (limb_alias_size) { | | |
| 23324 | else => unreachable, | | |
| 23325 | 4 => ._d, | | |
| 23326 | 8 => ._q, | | |
| 23327 | }, .mov }, | | |
| 23328 | limb_alias, | | |
| 23329 | if (src_reg_offset < limb_alias_size) src_reg.to128() else shuffle_reg: { | | |
| 23330 | const shuffle_reg = loop.shuffle_temp.?.tracking(cg).short.register.to128(); | | |
| 23331 | const mir_fixes: Mir.Inst.Fixes = if (cg.hasFeature(.sse2)) | | |
| 23332 | if (src_temp.typeOf(cg).scalarType(cg.pt.zcu).isRuntimeFloat()) switch (limb_alias_size) { | | |
| 23333 | else => unreachable, | | |
| 23334 | 4 => if (cg.hasFeature(.avx)) .v_ps else ._ps, | | |
| 23335 | 8 => if (cg.hasFeature(.avx)) .v_pd else ._pd, | | |
| 23336 | } else if (cg.hasFeature(.avx)) .vp_d else .p_d | | |
| 23337 | else | | |
| 23338 | ._ps; | | |
| 23339 | try cg.asmRegisterRegisterImmediate( | | |
| 23340 | .{ mir_fixes, .shuf }, | | |
| 23341 | shuffle_reg, | | |
| 23342 | src_reg: switch (mir_fixes) { | | |
| 23343 | else => unreachable, | | |
| 23344 | ._ps, ._pd => { | | |
| 23345 | try cg.asmRegisterRegister(.{ mir_fixes, .mova }, shuffle_reg, src_reg.to128()); | | |
| 23346 | break :src_reg shuffle_reg; | | |
| 23347 | }, | | |
| 23348 | .p_d => src_reg.to128(), | | |
| 23349 | }, | | |
| 23350 | .u(switch (mir_fixes) { | | |
| 23351 | else => unreachable, | | |
| 23352 | .v_ps, ._ps, .vp_d, .p_d => switch (limb_alias_size) { | | |
| 23353 | else => unreachable, | | |
| 23354 | 4 => switch (src_reg_offset) { | | |
| 23355 | else => unreachable, | | |
| 23356 | 4...7 => 0b01_01_01_01, | | |
| 23357 | 8...11 => 0b10_10_10_10, | | |
| 23358 | 12...15 => 0b11_11_11_11, | | |
| 23359 | }, | | |
| 23360 | 8 => switch (src_reg_offset) { | | |
| 23361 | else => unreachable, | | |
| 23362 | 8...15 => 0b11_10_11_10, | | |
| 23363 | }, | | |
| 23364 | }, | | |
| 23365 | .v_pd, ._pd => switch (limb_alias_size) { | | |
| 23366 | else => unreachable, | | |
| 23367 | 8 => switch (src_reg_offset) { | | |
| 23368 | else => unreachable, | | |
| 23369 | 8...15 => 0b1_1, | | |
| 23370 | }, | | |
| 23371 | }, | | |
| 23372 | }), | | |
| 23373 | ); | | |
| 23374 | break :shuffle_reg shuffle_reg; | | |
| 23375 | }, | | |
| 23376 | ); | | |
| 23377 | break :limb_reg_offset src_reg_offset % limb_alias_size; | | |
| 23378 | }) { | | |
| 23379 | 0 => {}, | | |
| 23380 | else => |limb_reg_offset| { | | |
| 23381 | try cg.spillEflagsIfOccupied(); | | |
| 23382 | try cg.asmRegisterImmediate(.{ ._r, .sh }, limb_alias, .u(limb_reg_offset * 8)); | | |
| 23383 | }, | | |
| 23384 | } | | |
| 23385 | }, | | |
| 23386 | } | | |
| 23387 | }, | | |
| 23388 | else => try cg.asmRegisterMemory( | | |
| 23389 | switch (linked_src_op) { | | |
| 23390 | else => unreachable, | | |
| 23391 | .gpr_limb, .mut_gpr_limb, .gpr_elem => .{ ._, .mov }, | | |
| 23392 | .mm_limb, .mm_mask_limb => .{ ._q, .mov }, | | |
| 23393 | .xmm_limb, | | |
| 23394 | .ymm_limb, | | |
| 23395 | .xmm_mask_limb, | | |
| 23396 | .ymm_mask_limb, | | |
| 23397 | => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu }, | | |
| 23398 | }, | | |
| 23399 | registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?), | | |
| 23400 | try src_mcv.mem(cg, switch (loop.limb_offset) { | | |
| 23401 | .unused => unreachable, | | |
| 23402 | .known => |limb_offset| .{ | | |
| 23403 | .size = .fromSize(loop.limb_size.?), | | |
| 23404 | .disp = limb_offset, | | |
| 23405 | }, | | |
| 23406 | .temp => |limb_offset| .{ | | |
| 23407 | .size = .fromSize(loop.limb_size.?), | | |
| 23408 | .index = limb_offset.tracking(cg).short.register.to64(), | | |
| 23409 | }, | | |
| 23410 | }), | | |
| 23411 | ), | | |
| 23412 | }, | | |
| 23413 | } | | |
| 23414 | mir_op.* = switch (linked_src_op) { | | |
| 23415 | .implicit, .explicit, .cc, .cc_elem => unreachable, | | |
| 23416 | .gpr => .{ .reg = registerAlias( | | |
| 23417 | src_mcv.register, | | |
| 23418 | @intCast(src_temp.typeOf(cg).abiSize(cg.pt.zcu)), | | |
| 23419 | ) }, | | |
| 23420 | .umax_gpr => .{ .reg = src_mcv.register.to64() }, // TODO: use other op size? | | |
| 23421 | .mm, .mm_mask, .mm_sign_mask, .umax_mm => .{ .reg = src_mcv.register }, | | |
| 23422 | .xmm, .xmm_mask, .xmm_sign_mask, .umax_xmm => .{ .reg = src_mcv.register.to128() }, | | |
| 23423 | .ymm, .ymm_mask, .ymm_sign_mask, .umax_ymm => .{ .reg = src_mcv.register.to256() }, | | |
| 23424 | .mem => .{ .mem = try src_mcv.mem(cg, .{ .size = cg.memSize(src_temp.typeOf(cg)) }) }, | | |
| 23425 | .gpr_limb, | | |
| 23426 | .mm_limb, | | |
| 23427 | .xmm_limb, | | |
| 23428 | .ymm_limb, | | |
| 23429 | .mut_gpr_limb, | | |
| 23430 | .gpr_elem, | | |
| 23431 | .mm_mask_limb, | | |
| 23432 | .xmm_mask_limb, | | |
| 23433 | .ymm_mask_limb, | | |
| 23434 | => .{ .reg = registerAlias(if (extra_temp) |limb_temp| | | |
| 23435 | limb_temp.tracking(cg).short.register | | |
| 23436 | else | | |
| 23437 | src_mcv.getRegs()[@divExact(loop.limb_offset.known, loop.limb_size.?)], loop.limb_size.?) }, | | |
| 23438 | .mem_limb, .mut_mem_limb, .mem_elem => .{ .mem = switch (src_mcv) { | | |
| 23439 | .register, .register_pair, .register_triple, .register_quadruple => unreachable, | | |
| 23440 | else => switch (loop.limb_offset) { | | |
| 23441 | .unused => unreachable, | | |
| 23442 | .known => |limb_offset| try src_mcv.mem(cg, .{ | | |
| 23443 | .size = .fromSize(loop.limb_size.?), | | |
| 23444 | .disp = limb_offset, | | |
| 23445 | }), | | |
| 23446 | .temp => |limb_offset| try src_mcv.mem(cg, .{ | | |
| 23447 | .size = .fromSize(loop.limb_size.?), | | |
| 23448 | .index = limb_offset.tracking(cg).short.register.to64(), | | |
| 23449 | }), | | |
| 23450 | }, | | |
| 23451 | } }, | | |
| 23452 | .imm => |imm| .{ .imm = .s(imm) }, | | |
| 23453 | .simm32 => switch (src_temp.typeOf(cg).abiSize(cg.pt.zcu)) { | | |
| 23454 | else => unreachable, | | |
| 23455 | 1 => .{ .imm = if (std.math.cast(i8, @as(i64, @bitCast(src_mcv.immediate)))) |small| | | |
| 23456 | .s(small) | | |
| 23457 | else | | |
| 23458 | .u(@as(u8, @intCast(src_mcv.immediate))) }, | | |
| 23459 | 2 => .{ .imm = if (std.math.cast(i16, @as(i64, @bitCast(src_mcv.immediate)))) |small| | | |
| 23460 | .s(small) | | |
| 23461 | else | | |
| 23462 | .u(@as(u16, @intCast(src_mcv.immediate))) }, | | |
| 23463 | 3...8 => .{ .imm = if (std.math.cast(i32, @as(i64, @bitCast(src_mcv.immediate)))) |small| | | |
| 23464 | .s(small) | | |
| 23465 | else | | |
| 23466 | .u(@as(u32, @intCast(src_mcv.immediate))) }, | | |
| 23467 | }, | | |
| 23468 | }; | | |
| 23469 | switch (src_op) { | | |
| 23470 | else => {}, | | |
| 23471 | .explicit => |linked_index| mir_ops[linked_index] = mir_op.*, | | |
| 23472 | } | 24008 | } |
| 23473 | } | 24009 | const frame_index = try cg.allocFrameIndex(.init(.{ |
| 23474 | for ( | 24010 | .size = 8, |
| 23475 | mir_ops[0..dst_mir_ops_len], | 24011 | .alignment = .@"8", |
| 23476 | pattern.ops[0..dst_mir_ops_len], | 24012 | })); |
| 23477 | dst_temps[0..dst_mir_ops_len], | 24013 | try cg.genSetMem( |
| 23478 | dst_tys[0..dst_mir_ops_len], | 24014 | .{ .frame = frame_index }, |
| 23479 | extra_temps[0..dst_mir_ops_len], | 24015 | 0, |
| 23480 | ) |*mir_op, dst_op, dst_temp, dst_ty, extra_temp| { | 24016 | .usize, |
| 23481 | if (mir_op.* != .none) continue; | 24017 | .{ .lea_symbol = .{ .sym_index = tlv_sym } }, |
| 23482 | mir_op.* = switch (dst_op) { | 24018 | .{}, |
| 23483 | .implicit, .cc, .cc_elem => unreachable, | 24019 | ); |
| 23484 | .explicit => |linked_index| mir_ops[linked_index], | 24020 | break :init .{ .load_frame = .{ .index = frame_index } }; |
| 23485 | .gpr => .{ .reg = registerAlias( | | |
| 23486 | dst_temp.tracking(cg).short.register, | | |
| 23487 | @intCast(dst_ty.abiSize(cg.pt.zcu)), | | |
| 23488 | ) }, | | |
| 23489 | .mm, .mm_mask, .mm_sign_mask => @panic("TODO"), | | |
| 23490 | .xmm, .xmm_mask, .xmm_sign_mask => .{ .reg = dst_temp.tracking(cg).short.register.to128() }, | | |
| 23491 | .ymm, .ymm_mask, .ymm_sign_mask => .{ .reg = dst_temp.tracking(cg).short.register.to256() }, | | |
| 23492 | .mem => @panic("TODO"), | | |
| 23493 | .gpr_limb, .mut_gpr_limb, .gpr_elem => .{ .reg = registerAlias( | | |
| 23494 | extra_temp.?.tracking(cg).short.register, | | |
| 23495 | @intCast(@divExact(Memory.Size.bitSize(switch (pattern_set.scalar) { | | |
| 23496 | .any => .qword, | | |
| 23497 | .bool => unreachable, | | |
| 23498 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, | | |
| 23499 | }), 8)), | | |
| 23500 | ) }, | | |
| 23501 | .mm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register }, | | |
| 23502 | .xmm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to128() }, | | |
| 23503 | .ymm_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to256() }, | | |
| 23504 | .mem_limb, .mut_mem_limb, .mem_elem => .{ .mem = try dst_temp.tracking(cg).short.mem(cg, switch (loop.limb_offset) { | | |
| 23505 | .unused => unreachable, | | |
| 23506 | .known => |limb_offset| .{ | | |
| 23507 | .size = .fromSize(loop.limb_size.?), | | |
| 23508 | .disp = limb_offset, | | |
| 23509 | }, | | |
| 23510 | .temp => |limb_offset| .{ | | |
| 23511 | .size = .fromSize(loop.limb_size.?), | | |
| 23512 | .index = limb_offset.tracking(cg).short.register.to64(), | | |
| 23513 | }, | | |
| 23514 | }) }, | | |
| 23515 | .mm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register }, | | |
| 23516 | .xmm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to128() }, | | |
| 23517 | .ymm_mask_limb => .{ .reg = extra_temp.?.tracking(cg).short.register.to256() }, | | |
| 23518 | .imm, .simm32, .umax_gpr, .umax_mm, .umax_xmm, .umax_ymm => unreachable, // unmodifiable destination | | |
| 23519 | }; | | |
| 23520 | } | | |
| 23521 | std.mem.swap(Operand, &mir_ops[pattern.commute[0]], &mir_ops[pattern.commute[1]]); | | |
| 23522 | if (pattern_set.clobbers.eflags) try cg.spillEflagsIfOccupied(); | | |
| 23523 | cg.asmOps((if (loop.remaining_size != null and loop.limb_size != null and | | |
| 23524 | loop.remaining_size.? <= loop.limb_size.?) | | |
| 23525 | pattern_set.final_mir_tag | | |
| 23526 | else | | |
| 23527 | null) orelse pattern_set.mir_tag, mir_ops) catch |err| switch (err) { | | |
| 23528 | error.InvalidInstruction => { | | |
| 23529 | const fixes = @tagName(pattern_set.mir_tag[0]); | | |
| 23530 | const fixes_blank = std.mem.indexOfScalar(u8, fixes, '_').?; | | |
| 23531 | return cg.fail( | | |
| 23532 | "invalid instruction: '{s}{s}{s} {s} {s} {s} {s}'", | | |
| 23533 | .{ | | |
| 23534 | fixes[0..fixes_blank], | | |
| 23535 | @tagName(pattern_set.mir_tag[1]), | | |
| 23536 | fixes[fixes_blank + 1 ..], | | |
| 23537 | @tagName(mir_ops[0]), | | |
| 23538 | @tagName(mir_ops[1]), | | |
| 23539 | @tagName(mir_ops[2]), | | |
| 23540 | @tagName(mir_ops[3]), | | |
| 23541 | }, | | |
| 23542 | ); | | |
| 23543 | }, | | |
| 23544 | else => |e| return e, | | |
| 23545 | }; | | |
| 23546 | for ( | | |
| 23547 | extra_temps[0..dst_temps.len], | | |
| 23548 | pattern.ops[0..dst_temps.len], | | |
| 23549 | mir_ops[0..dst_temps.len], | | |
| 23550 | dst_temps, | | |
| 23551 | ) |extra_temp, dst_op, mir_op, dst_temp| switch (dst_op) { | | |
| 23552 | else => if (invert_result) { | | |
| 23553 | try cg.spillEflagsIfOccupied(); | | |
| 23554 | cg.asmOps( | | |
| 23555 | .{ ._, .not }, | | |
| 23556 | .{ mir_op, .none, .none, .none }, | | |
| 23557 | ) catch |err| switch (err) { | | |
| 23558 | error.InvalidInstruction => return cg.fail( | | |
| 23559 | "invalid instruction: 'not {s} none none none'", | | |
| 23560 | .{@tagName(mir_op)}, | | |
| 23561 | ), | | |
| 23562 | else => |e| return e, | | |
| 23563 | }; | | |
| 23564 | }, | | |
| 23565 | .mm_mask, | | |
| 23566 | .xmm_mask, | | |
| 23567 | .ymm_mask, | | |
| 23568 | .mm_sign_mask, | | |
| 23569 | .xmm_sign_mask, | | |
| 23570 | .ymm_sign_mask, | | |
| 23571 | => dst_temp.asMask(.{ | | |
| 23572 | .kind = switch (dst_op) { | | |
| 23573 | else => unreachable, | | |
| 23574 | .mm_mask, .xmm_mask, .ymm_mask => .all, | | |
| 23575 | .mm_sign_mask, .xmm_sign_mask, .ymm_sign_mask => .sign, | | |
| 23576 | }, | | |
| 23577 | .inverted = invert_result, | | |
| 23578 | .scalar = switch (pattern_set.scalar) { | | |
| 23579 | .any, .bool => unreachable, | | |
| 23580 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, | | |
| 23581 | }, | | |
| 23582 | }, cg), | | |
| 23583 | .gpr_limb, .mm_limb, .xmm_limb, .ymm_limb, .mut_gpr_limb, .gpr_elem => if (extra_temp) |limb_temp| { | | |
| 23584 | const dst_mcv = dst_temp.tracking(cg).short; | | |
| 23585 | switch (dst_mcv) { | | |
| 23586 | .register_pair, .register_triple, .register_quadruple => try cg.asmRegisterRegister( | | |
| 23587 | .{ ._, .mov }, | | |
| 23588 | dst_mcv.getRegs()[@divExact(loop.limb_offset.known, loop.limb_size.?)].to64(), | | |
| 23589 | limb_temp.tracking(cg).short.register.to64(), | | |
| 23590 | ), | | |
| 23591 | else => try cg.asmMemoryRegister( | | |
| 23592 | switch (dst_op) { | | |
| 23593 | else => unreachable, | | |
| 23594 | .gpr_limb, .mut_gpr_limb, .gpr_elem => .{ ._, .mov }, | | |
| 23595 | .mm_limb => .{ ._q, .mov }, | | |
| 23596 | .xmm_limb, .ymm_limb => .{ if (cg.hasFeature(.avx)) .v_ else ._, .movdqu }, | | |
| 23597 | }, | | |
| 23598 | try dst_mcv.mem(cg, switch (loop.limb_offset) { | | |
| 23599 | .unused => unreachable, | | |
| 23600 | .known => |limb_offset| .{ | | |
| 23601 | .size = .fromSize(loop.limb_size.?), | | |
| 23602 | .disp = limb_offset, | | |
| 23603 | }, | | |
| 23604 | .temp => |limb_offset| .{ | | |
| 23605 | .size = .fromSize(loop.limb_size.?), | | |
| 23606 | .index = limb_offset.tracking(cg).short.register.to64(), | | |
| 23607 | }, | | |
| 23608 | }), | | |
| 23609 | registerAlias(limb_temp.tracking(cg).short.register, loop.limb_size.?), | | |
| 23610 | ), | | |
| 23611 | } | | |
| 23612 | }, | | |
| 23613 | .cc_elem, .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => { | | |
| 23614 | const scalar_size = switch (pattern_set.scalar) { | | |
| 23615 | .any => .qword, | | |
| 23616 | .bool => unreachable, | | |
| 23617 | .float, .any_int, .signed_int, .unsigned_int, .any_float_or_int => |size| size, | | |
| 23618 | }; | | |
| 23619 | const mask_store_size: u4 = | | |
| 23620 | @intCast(std.math.divCeil(u7, loop.mask_store_bit_size.?, 8) catch unreachable); | | |
| 23621 | const known_shl_count = if (loop.mask_store_reg) |_| switch (loop.mask_limb_offset) { | | |
| 23622 | .unused => unreachable, | | |
| 23623 | .known => |mask_limb_offset| mask_limb_offset & (loop.mask_store_bit_size.? - 1), | | |
| 23624 | .temp => null, | | |
| 23625 | } else null; | | |
| 23626 | const mask_limb_reg = registerAlias(if (known_shl_count != 0) | | |
| 23627 | loop.mask_limb_temp.?.tracking(cg).short.register | | |
| 23628 | else | | |
| 23629 | loop.mask_store_reg.?, mask_store_size); | | |
| 23630 | switch (dst_op) { | | |
| 23631 | else => unreachable, | | |
| 23632 | .cc_elem => try cg.asmSetccRegister(switch (invert_result) { | | |
| 23633 | false => opts.cc.?, | | |
| 23634 | true => opts.cc.?.negate(), | | |
| 23635 | }, mask_limb_reg.to8()), | | |
| 23636 | .mm_mask_limb, .xmm_mask_limb, .ymm_mask_limb => { | | |
| 23637 | if (scalar_size == .word) if (cg.hasFeature(.avx)) try cg.asmRegisterRegisterRegister( | | |
| 23638 | .{ .vp_b, .ackssw }, | | |
| 23639 | mir_op.reg, | | |
| 23640 | mir_op.reg, | | |
| 23641 | mir_op.reg, | | |
| 23642 | ) else try cg.asmRegisterRegister( | | |
| 23643 | .{ .p_b, .ackssw }, | | |
| 23644 | mir_op.reg, | | |
| 23645 | mir_op.reg, | | |
| 23646 | ); | | |
| 23647 | try cg.asmRegisterRegister(switch (scalar_size) { | | |
| 23648 | else => unreachable, | | |
| 23649 | .byte, .word => .{ if (cg.hasFeature(.avx)) .vp_b else .p_b, .movmsk }, | | |
| 23650 | .dword => .{ if (cg.hasFeature(.avx)) .v_ps else ._ps, .movmsk }, | | |
| 23651 | .qword => .{ if (cg.hasFeature(.avx)) .v_pd else ._pd, .movmsk }, | | |
| 23652 | }, mask_limb_reg.to32(), mir_op.reg); | | |
| 23653 | if (invert_result) if (loop.mask_store_reg) |_| { | | |
| 23654 | try cg.spillEflagsIfOccupied(); | | |
| 23655 | try cg.asmRegisterImmediate( | | |
| 23656 | .{ ._, .xor }, | | |
| 23657 | registerAlias(mask_limb_reg, @min(mask_store_size, 4)), | | |
| 23658 | .u((@as(u32, 1) << @intCast(loop.mask_limb_bit_size.?)) - 1), | | |
| 23659 | ); | | |
| 23660 | } else try cg.asmRegister(.{ ._, .not }, mask_limb_reg); | | |
| 23661 | }, | | |
| 23662 | } | | |
| 23663 | if (loop.mask_store_reg) |mask_store_reg| { | | |
| 23664 | const mask_store_alias = registerAlias(mask_store_reg, mask_store_size); | | |
| 23665 | switch (loop.mask_limb_offset) { | | |
| 23666 | .unused => unreachable, | | |
| 23667 | .known => if (known_shl_count.? != 0) { | | |
| 23668 | try cg.spillEflagsIfOccupied(); | | |
| 23669 | try cg.asmRegisterImmediate(.{ ._l, .sh }, mask_limb_reg, .u(known_shl_count.?)); | | |
| 23670 | try cg.spillEflagsIfOccupied(); | | |
| 23671 | try cg.asmRegisterRegister(.{ ._, .@"or" }, mask_store_alias, mask_limb_reg); | | |
| 23672 | }, | | |
| 23673 | .temp => |mask_limb_offset| { | | |
| 23674 | if (cg.hasFeature(.bmi2)) { | | |
| 23675 | const shlx_size = @max(mask_store_size, 4); | | |
| 23676 | const shlx_mask_limb_reg = registerAlias(mask_limb_reg, shlx_size); | | |
| 23677 | try cg.asmRegisterRegisterRegister( | | |
| 23678 | .{ ._lx, .sh }, | | |
| 23679 | shlx_mask_limb_reg, | | |
| 23680 | shlx_mask_limb_reg, | | |
| 23681 | registerAlias(mask_limb_offset.tracking(cg).short.register, shlx_size), | | |
| 23682 | ); | | |
| 23683 | } else { | | |
| 23684 | try cg.spillEflagsIfOccupied(); | | |
| 23685 | try cg.asmRegisterRegister( | | |
| 23686 | .{ ._l, .sh }, | | |
| 23687 | mask_limb_reg, | | |
| 23688 | mask_limb_offset.tracking(cg).short.register.to8(), | | |
| 23689 | ); | | |
| 23690 | } | | |
| 23691 | try cg.spillEflagsIfOccupied(); | | |
| 23692 | try cg.asmRegisterRegister(.{ ._, .@"or" }, mask_store_alias, mask_limb_reg); | | |
| 23693 | }, | | |
| 23694 | } | | |
| 23695 | } | | |
| 23696 | const dst_mcv = dst_temp.tracking(cg).short; | | |
| 23697 | switch (loop.mask_limb_offset) { | | |
| 23698 | .unused => unreachable, | | |
| 23699 | .known => |*mask_limb_offset| { | | |
| 23700 | mask_limb_offset.* += loop.mask_limb_bit_size.?; | | |
| 23701 | if (mask_limb_offset.* & (loop.mask_store_bit_size.? - 1) == 0) switch (dst_mcv) { | | |
| 23702 | .register => {}, | | |
| 23703 | else => { | | |
| 23704 | try cg.asmMemoryRegister( | | |
| 23705 | .{ ._, .mov }, | | |
| 23706 | try dst_mcv.mem(cg, .{ | | |
| 23707 | .size = .fromSize(mask_store_size), | | |
| 23708 | .disp = @divExact(mask_limb_offset.*, 8) - mask_store_size, | | |
| 23709 | }), | | |
| 23710 | registerAlias(loop.mask_store_reg orelse mask_limb_reg, mask_store_size), | | |
| 23711 | ); | | |
| 23712 | if (loop.mask_store_reg) |mask_store_reg| { | | |
| 23713 | const mask_store_alias = registerAlias(mask_store_reg, @min(mask_store_size, 4)); | | |
| 23714 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias); | | |
| 23715 | } | | |
| 23716 | }, | | |
| 23717 | }; | | |
| 23718 | }, | | |
| 23719 | .temp => |mask_limb_offset| { | | |
| 23720 | const mask_limb_offset_reg = mask_limb_offset.tracking(cg).short.register.to32(); | | |
| 23721 | if (loop.mask_store_reg) |mask_store_reg| { | | |
| 23722 | try cg.asmRegisterMemory(.{ ._, .lea }, mask_limb_offset_reg, .{ | | |
| 23723 | .base = .{ .reg = mask_limb_offset_reg.to64() }, | | |
| 23724 | .mod = .{ .rm = .{ | | |
| 23725 | .size = .qword, | | |
| 23726 | .disp = loop.mask_limb_bit_size.?, | | |
| 23727 | } }, | | |
| 23728 | }); | | |
| 23729 | switch (dst_mcv) { | | |
| 23730 | .register => {}, | | |
| 23731 | else => { | | |
| 23732 | try cg.spillEflagsIfOccupied(); | | |
| 23733 | try cg.asmRegisterImmediate( | | |
| 23734 | .{ ._, .@"test" }, | | |
| 23735 | mask_limb_offset_reg, | | |
| 23736 | .u(loop.mask_store_bit_size.? - 1), | | |
| 23737 | ); | | |
| 23738 | const skip_store_reloc = try cg.asmJccReloc(.nz, undefined); | | |
| 23739 | const mask_store_offset_reg = mask_limb_reg.to32(); | | |
| 23740 | try cg.asmRegisterRegister(.{ ._, .mov }, mask_store_offset_reg, mask_limb_offset_reg); | | |
| 23741 | try cg.asmRegisterImmediate(.{ ._r, .sh }, mask_store_offset_reg, .u(3)); | | |
| 23742 | try cg.asmMemoryRegister(.{ ._, .mov }, try dst_mcv.mem(cg, .{ | | |
| 23743 | .size = .fromSize(mask_store_size), | | |
| 23744 | .index = mask_store_offset_reg.to64(), | | |
| 23745 | .disp = -@as(i8, mask_store_size), | | |
| 23746 | }), registerAlias(mask_store_reg, mask_store_size)); | | |
| 23747 | const mask_store_alias = registerAlias(mask_store_reg, @min(mask_store_size, 4)); | | |
| 23748 | try cg.asmRegisterRegister(.{ ._, .xor }, mask_store_alias, mask_store_alias); | | |
| 23749 | cg.performReloc(skip_store_reloc); | | |
| 23750 | }, | | |
| 23751 | } | | |
| 23752 | } else { | | |
| 23753 | switch (dst_mcv) { | | |
| 23754 | .register => {}, | | |
| 23755 | else => try cg.asmMemoryRegister(.{ ._, .mov }, try dst_mcv.mem(cg, .{ | | |
| 23756 | .size = .fromSize(mask_store_size), | | |
| 23757 | .index = mask_limb_offset_reg.to64(), | | |
| 23758 | }), mask_limb_reg), | | |
| 23759 | } | | |
| 23760 | try cg.asmRegisterMemory(.{ ._, .lea }, mask_limb_offset_reg, .{ | | |
| 23761 | .base = .{ .reg = mask_limb_offset_reg.to64() }, | | |
| 23762 | .mod = .{ .rm = .{ | | |
| 23763 | .size = .qword, | | |
| 23764 | .disp = mask_store_size, | | |
| 23765 | } }, | | |
| 23766 | }); | | |
| 23767 | } | | |
| 23768 | }, | | |
| 23769 | } | | |
| 23770 | }, | | |
| 23771 | }; | | |
| 23772 | switch (pattern_set.loop) { | | |
| 23773 | .once => break :pattern_sets, | | |
| 23774 | .bitwise => {}, | | |
| 23775 | .limbwise_carry => @panic("TODO"), | | |
| 23776 | .limbwise_reduce => @panic("TODO"), | | |
| 23777 | .limbwise_pairs_forward => @panic("TODO"), | | |
| 23778 | .limbwise_pairs_reverse => @panic("TODO"), | | |
| 23779 | .elementwise => {}, | | |
| 23780 | } | | |
| 23781 | switch (loop.limb_offset) { | | |
| 23782 | .unused => break, | | |
| 23783 | .known => |*limb_offset| { | | |
| 23784 | limb_offset.* += loop.limb_size.?; | | |
| 23785 | loop.remaining_size.? -= loop.limb_size.?; | | |
| 23786 | if (loop.remaining_size.? < loop.limb_size.? or | | |
| 23787 | (loop.element_size != null and limb_offset.* >= loop.element_size.?)) | | |
| 23788 | { | | |
| 23789 | switch (loop.mask_limb_offset) { | | |
| 23790 | .unused => {}, | | |
| 23791 | .known => |*mask_limb_offset| mask_limb_offset.* = 0, | | |
| 23792 | .temp => unreachable, | | |
| 23793 | } | | |
| 23794 | limb_offset.* = 0; | | |
| 23795 | break; | | |
| 23796 | } | | |
| 23797 | }, | | |
| 23798 | .temp => |limb_offset| { | | |
| 23799 | const limb_offset_reg = limb_offset.tracking(cg).short.register; | | |
| 23800 | try cg.asmRegisterMemory(.{ ._, .lea }, limb_offset_reg.to32(), .{ | | |
| 23801 | .base = .{ .reg = limb_offset_reg.to64() }, | | |
| 23802 | .mod = .{ .rm = .{ | | |
| 23803 | .size = .qword, | | |
| 23804 | .disp = loop.limb_size.?, | | |
| 23805 | } }, | | |
| 23806 | }); | | |
| 23807 | try cg.spillEflagsIfOccupied(); | | |
| 23808 | try cg.asmRegisterImmediate( | | |
| 23809 | .{ ._, .cmp }, | | |
| 23810 | limb_offset_reg.to32(), | | |
| 23811 | .u(loop.element_size orelse loop.remaining_size.?), | | |
| 23812 | ); | | |
| 23813 | _ = try cg.asmJccReloc(.b, loop.limb_reloc); | | |
| 23814 | try limb_offset.die(cg); | | |
| 23815 | break; | | |
| 23816 | }, | | |
| 23817 | } | | |
| 23818 | } | | |
| 23819 | if (loop.shuffle_temp) |shuffle_temp| try shuffle_temp.die(cg); | | |
| 23820 | if (loop.mask_limb_temp) |mask_limb_temp| try mask_limb_temp.die(cg); | | |
| 23821 | if (loop.mask_store_temp) |mask_store_temp| try mask_store_temp.die(cg); | | |
| 23822 | switch (loop.mask_limb_offset) { | | |
| 23823 | .unused, .known => {}, | | |
| 23824 | .temp => |mask_limb_offset| try mask_limb_offset.die(cg), | | |
| 23825 | } | | |
| 23826 | switch (loop.element_offset) { | | |
| 23827 | .unused => break :pattern_sets, | | |
| 23828 | .known => |*element_offset| { | | |
| 23829 | if (loop.remaining_size.? == 0) break :pattern_sets; | | |
| 23830 | element_offset.* += loop.element_size.?; | | |
| 23831 | }, | | |
| 23832 | .temp => |element_offset| { | | |
| 23833 | if (true) @panic("TODO"); | | |
| 23834 | try element_offset.die(cg); | | |
| 23835 | if (loop.remaining_size.? == 0) break :pattern_sets; | | |
| 23836 | break; | | |
| 23837 | }, | 24021 | }, |
| 23838 | } | 24022 | else => break :init const_mcv, |
| | 24023 | }, |
| | 24024 | else => break :init const_mcv, |
| 23839 | } | 24025 | } |
| 23840 | } | 24026 | }); |
| 23841 | } else { | 24027 | return cg.tempFromValue(.fromInterned(ip.typeOf(val)), gop.value_ptr.short); |
| 23842 | log.err("failed to select {s}:", .{@tagName(pattern_sets[0].mir_tag[1])}); | 24028 | } |
| 23843 | for (src_temps) |src_temp| log.err("{}", .{src_temp.tracking(cg)}); | 24029 | |
| 23844 | return error.SelectFailed; | 24030 | const temp_index = cg.next_temp_index; |
| | 24031 | const temp: Temp = .{ .index = temp_index.toIndex() }; |
| | 24032 | const op_inst = op_ref.toIndex().?; |
| | 24033 | const tracking = cg.getResolvedInstValue(op_inst); |
| | 24034 | temp_index.tracking(cg).* = tracking.*; |
| | 24035 | if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? }; |
| | 24036 | cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref); |
| | 24037 | cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); |
| | 24038 | return temp; |
| | 24039 | } |
| | 24040 | |
| | 24041 | inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp { |
| | 24042 | var temps: [op_refs.len]Temp = undefined; |
| | 24043 | inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| { |
| | 24044 | temp.* = try cg.tempFromOperand(inst, op_index, op_ref, inline for (0..op_index) |prev_op_index| { |
| | 24045 | if (op_ref == op_refs[prev_op_index]) break true; |
| | 24046 | } else false); |
| 23845 | } | 24047 | } |
| 23846 | for (extra_temps) |extra_temp| if (extra_temp) |temp| try temp.die(cg); | 24048 | return temps; |
| 23847 | } | 24049 | } |
| 23848 | | 24050 | |
| 23849 | const Select2 = struct { | 24051 | const Operand = union(enum) { |
| | 24052 | none, |
| | 24053 | reg: Register, |
| | 24054 | mem: Memory, |
| | 24055 | imm: Immediate, |
| | 24056 | inst: Mir.Inst.Index, |
| | 24057 | }; |
| | 24058 | |
| | 24059 | const Select = struct { |
| 23850 | cg: *CodeGen, | 24060 | cg: *CodeGen, |
| 23851 | case: *const Case, | 24061 | temps: [@intFromEnum(Select.Operand.Ref.none)]Temp, |
| 23852 | pattern: *const Select2.Pattern, | 24062 | labels: [@intFromEnum(Label._)]struct { |
| 23853 | extra_temps: [3]Temp, | 24063 | backward: ?Mir.Inst.Index, |
| 23854 | dst_temps: []const Temp, | 24064 | forward: [1]?Mir.Inst.Index, |
| 23855 | src_temps: []const Temp, | 24065 | }, |
| 23856 | commute: struct { u8, u8 }, | | |
| 23857 | limb: Memory.Mod.Rm, | 24066 | limb: Memory.Mod.Rm, |
| | 24067 | mask_limb: Memory.Mod.Rm, |
| | 24068 | |
| | 24069 | fn emitLabel(s: *Select, label_index: Label) void { |
| | 24070 | if (label_index == ._) return; |
| | 24071 | const label = &s.labels[@intFromEnum(label_index)]; |
| | 24072 | for (&label.forward) |*reloc| { |
| | 24073 | if (reloc.*) |r| s.cg.performReloc(r); |
| | 24074 | reloc.* = null; |
| | 24075 | } |
| | 24076 | label.backward = @intCast(s.cg.mir_instructions.len); |
| | 24077 | } |
| 23858 | | 24078 | |
| 23859 | fn emit(s: Select2, inst: Instruction) !void { | 24079 | fn emit(s: *Select, inst: Instruction) !void { |
| 23860 | const mir_tag: Mir.Inst.FixedTag = .{ inst[0], inst[1] }; | 24080 | s.emitLabel(inst[0]); |
| | 24081 | const mir_tag: Mir.Inst.FixedTag = .{ inst[1], inst[2] }; |
| 23861 | var mir_ops: [4]CodeGen.Operand = undefined; | 24082 | var mir_ops: [4]CodeGen.Operand = undefined; |
| 23862 | inline for (&mir_ops, 2..) |*mir_op, inst_index| mir_op.* = try inst[inst_index].lower(s); | 24083 | inline for (&mir_ops, 3..) |*mir_op, inst_index| mir_op.* = try inst[inst_index].lower(s); |
| 23863 | s.cg.asmOps(mir_tag, mir_ops) catch |err| switch (err) { | 24084 | s.cg.asmOps(mir_tag, mir_ops) catch |err| switch (err) { |
| 23864 | error.InvalidInstruction => { | 24085 | error.InvalidInstruction => { |
| 23865 | const fixes = @tagName(mir_tag[0]); | 24086 | const fixes = @tagName(mir_tag[0]); |
| ... | @@ -23881,32 +24102,27 @@ const Select2 = struct { | ... | @@ -23881,32 +24102,27 @@ const Select2 = struct { |
| 23881 | }; | 24102 | }; |
| 23882 | } | 24103 | } |
| 23883 | | 24104 | |
| 23884 | fn lowerLimb(s: Select2, temp: Temp) !CodeGen.Operand { | | |
| 23885 | return .{ .mem = try temp.tracking(s.cg).short.mem(s.cg, s.limb) }; | | |
| 23886 | } | | |
| 23887 | | | |
| 23888 | fn srcTemp(s: Select2, index: u8) Temp { | | |
| 23889 | return s.src_temps[ | | |
| 23890 | if (index == s.commute[0]) | | |
| 23891 | s.commute[1] | | |
| 23892 | else if (index == s.commute[1]) | | |
| 23893 | s.commute[0] | | |
| 23894 | else | | |
| 23895 | index | | |
| 23896 | ]; | | |
| 23897 | } | | |
| 23898 | | | |
| 23899 | const Case = struct { | 24105 | const Case = struct { |
| 23900 | required_features: [2]?std.Target.x86.Feature = @splat(null), | 24106 | required_features: [2]?std.Target.x86.Feature = @splat(null), |
| 23901 | constraints: [2]Constraint = @splat(.any), | 24107 | dst_constraints: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]Constraint = @splat(.any), |
| 23902 | patterns: []const Select2.Pattern, | 24108 | src_constraints: [@intFromEnum(Select.Operand.Ref.none) - @intFromEnum(Select.Operand.Ref.src0)]Constraint = @splat(.any), |
| | 24109 | patterns: []const Select.Pattern, |
| 23903 | clobbers: struct { eflags: bool = false } = .{}, | 24110 | clobbers: struct { eflags: bool = false } = .{}, |
| 23904 | extra_temps: [3]TempSpec = @splat(.unused), | 24111 | extra_temps: [@intFromEnum(Select.Operand.Ref.dst0) - @intFromEnum(Select.Operand.Ref.tmp0)]TempSpec = @splat(.unused), |
| 23905 | dst_temps: [1]TempSpec.Kind = @splat(.unused), | 24112 | dst_temps: [@intFromEnum(Select.Operand.Ref.src0) - @intFromEnum(Select.Operand.Ref.dst0)]TempSpec.Kind = @splat(.unused), |
| 23906 | each: union(enum) { | 24113 | each: union(enum) { |
| 23907 | once: []const Instruction, | 24114 | once: []const Instruction, |
| 23908 | limb: struct { | 24115 | limb: struct { |
| 23909 | of: Select2.Operand, | 24116 | of: Select.Operand.Ref.Sized, |
| | 24117 | header: []const Instruction = &.{}, |
| | 24118 | first: ?[]const Instruction = null, |
| | 24119 | body: []const Instruction, |
| | 24120 | last: ?[]const Instruction = null, |
| | 24121 | trailer: []const Instruction = &.{}, |
| | 24122 | }, |
| | 24123 | limb_and_mask_limb: struct { |
| | 24124 | of: Select.Operand.Ref.Sized, |
| | 24125 | of_mask: Select.Operand.Ref.Sized, |
| 23910 | header: []const Instruction = &.{}, | 24126 | header: []const Instruction = &.{}, |
| 23911 | first: ?[]const Instruction = null, | 24127 | first: ?[]const Instruction = null, |
| 23912 | body: []const Instruction, | 24128 | body: []const Instruction, |
| ... | @@ -23918,6 +24134,7 @@ const Select2 = struct { | ... | @@ -23918,6 +24134,7 @@ const Select2 = struct { |
| 23918 | | 24134 | |
| 23919 | const Constraint = union(enum) { | 24135 | const Constraint = union(enum) { |
| 23920 | any, | 24136 | any, |
| | 24137 | any_bool_vec, |
| 23921 | any_int, | 24138 | any_int, |
| 23922 | any_float, | 24139 | any_float, |
| 23923 | bool_vec: Memory.Size, | 24140 | bool_vec: Memory.Size, |
| ... | @@ -23925,37 +24142,35 @@ const Select2 = struct { | ... | @@ -23925,37 +24142,35 @@ const Select2 = struct { |
| 23925 | signed_int: Memory.Size, | 24142 | signed_int: Memory.Size, |
| 23926 | unsigned_int: Memory.Size, | 24143 | unsigned_int: Memory.Size, |
| 23927 | | 24144 | |
| 23928 | fn accepts(constraint: Constraint, temp: Temp, cg: *CodeGen) bool { | 24145 | fn accepts(constraint: Constraint, ty: Type, cg: *CodeGen) bool { |
| 23929 | const zcu = cg.pt.zcu; | 24146 | const zcu = cg.pt.zcu; |
| 23930 | switch (constraint) { | 24147 | switch (constraint) { |
| 23931 | .any => return true, | 24148 | .any => return true, |
| | 24149 | .any_bool_vec => return ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type, |
| 23932 | .any_int => { | 24150 | .any_int => { |
| 23933 | const scalar_ty = temp.typeOf(cg).scalarType(zcu); | 24151 | const scalar_ty = ty.scalarType(zcu); |
| 23934 | return scalar_ty.isAbiInt(zcu) or scalar_ty.isPtrAtRuntime(zcu); | 24152 | return scalar_ty.isAbiInt(zcu) or scalar_ty.isPtrAtRuntime(zcu); |
| 23935 | }, | 24153 | }, |
| 23936 | .any_float => return temp.typeOf(cg).scalarType(zcu).isRuntimeFloat(), | 24154 | .any_float => return ty.scalarType(zcu).isRuntimeFloat(), |
| 23937 | .bool_vec => |size| { | 24155 | .bool_vec => |size| return ty.isVector(zcu) and |
| 23938 | const ty = temp.typeOf(cg); | 24156 | ty.scalarType(zcu).toIntern() == .bool_type and ty.vectorLen(zcu) <= size.bitSize(cg.target), |
| 23939 | return ty.isVector(zcu) and ty.scalarType(zcu).toIntern() == .bool_type and | | |
| 23940 | ty.vectorLen(zcu) <= size.bitSize(); | | |
| 23941 | }, | | |
| 23942 | .int => |size| { | 24157 | .int => |size| { |
| 23943 | const scalar_ty = temp.typeOf(cg).scalarType(zcu); | 24158 | const scalar_ty = ty.scalarType(zcu); |
| 23944 | if (scalar_ty.isPtrAtRuntime(zcu)) return cg.target.ptrBitWidth() <= size.bitSize(); | 24159 | if (scalar_ty.isPtrAtRuntime(zcu)) return cg.target.ptrBitWidth() <= size.bitSize(cg.target); |
| 23945 | return scalar_ty.isAbiInt(zcu) and scalar_ty.intInfo(zcu).bits <= size.bitSize(); | 24160 | return scalar_ty.isAbiInt(zcu) and scalar_ty.intInfo(zcu).bits <= size.bitSize(cg.target); |
| 23946 | }, | 24161 | }, |
| 23947 | .signed_int => |size| { | 24162 | .signed_int => |size| { |
| 23948 | const scalar_ty = temp.typeOf(cg).scalarType(zcu); | 24163 | const scalar_ty = ty.scalarType(zcu); |
| 23949 | if (!scalar_ty.isAbiInt(zcu)) return false; | 24164 | if (!scalar_ty.isAbiInt(zcu)) return false; |
| 23950 | const info = scalar_ty.intInfo(zcu); | 24165 | const info = scalar_ty.intInfo(zcu); |
| 23951 | return info.signedness == .signed and info.bits <= size.bitSize(); | 24166 | return info.signedness == .signed and info.bits <= size.bitSize(cg.target); |
| 23952 | }, | 24167 | }, |
| 23953 | .unsigned_int => |size| { | 24168 | .unsigned_int => |size| { |
| 23954 | const scalar_ty = temp.typeOf(cg).scalarType(zcu); | 24169 | const scalar_ty = ty.scalarType(zcu); |
| 23955 | if (scalar_ty.isPtrAtRuntime(zcu)) return cg.target.ptrBitWidth() <= size.bitSize(); | 24170 | if (scalar_ty.isPtrAtRuntime(zcu)) return cg.target.ptrBitWidth() <= size.bitSize(cg.target); |
| 23956 | if (!scalar_ty.isAbiInt(zcu)) return false; | 24171 | if (!scalar_ty.isAbiInt(zcu)) return false; |
| 23957 | const info = scalar_ty.intInfo(zcu); | 24172 | const info = scalar_ty.intInfo(zcu); |
| 23958 | return info.signedness == .unsigned and info.bits <= size.bitSize(); | 24173 | return info.signedness == .unsigned and info.bits <= size.bitSize(cg.target); |
| 23959 | }, | 24174 | }, |
| 23960 | } | 24175 | } |
| 23961 | } | 24176 | } |
| ... | @@ -24083,189 +24298,578 @@ const Select2 = struct { | ... | @@ -24083,189 +24298,578 @@ const Select2 = struct { |
| 24083 | rc: Register.Class, | 24298 | rc: Register.Class, |
| 24084 | rc_mask: struct { rc: Register.Class, info: MaskInfo }, | 24299 | rc_mask: struct { rc: Register.Class, info: MaskInfo }, |
| 24085 | mem, | 24300 | mem, |
| 24086 | src: u8, | 24301 | ref: Select.Operand.Ref, |
| 24087 | src_mask: struct { src: u8, info: MaskInfo }, | 24302 | ref_mask: struct { ref: Select.Operand.Ref, info: MaskInfo }, |
| 24088 | | 24303 | |
| 24089 | fn finish(kind: Kind, temp: Temp, s: Select2) void { | 24304 | fn finish(kind: Kind, temp: Temp, s: *const Select) void { |
| 24090 | switch (kind) { | 24305 | switch (kind) { |
| 24091 | else => {}, | 24306 | else => {}, |
| 24092 | inline .rc_mask, .src_mask => |mask| temp.asMask(mask.info, s.cg), | 24307 | inline .rc_mask, .ref_mask => |mask| temp.asMask(mask.info, s.cg), |
| 24093 | } | 24308 | } |
| 24094 | } | 24309 | } |
| 24095 | }; | 24310 | }; |
| 24096 | | 24311 | |
| 24097 | fn create(spec: TempSpec, s: Select2) !?Temp { | 24312 | fn create(spec: TempSpec, s: *Select) !?Temp { |
| 24098 | return switch (spec.kind) { | 24313 | return switch (spec.kind) { |
| 24099 | .unused => null, | 24314 | .unused => null, |
| 24100 | .any => try s.cg.tempAlloc(spec.type), | 24315 | .any => try s.cg.tempAlloc(spec.type), |
| 24101 | .cc => |cc| try s.cg.tempFromValue(spec.type, .{ .eflags = cc }), | 24316 | .cc => |cc| try s.cg.tempFromValue(spec.type, .{ .eflags = cc }), |
| 24102 | .reg => |reg| try s.cg.tempFromValue(spec.type, .{ .register = reg }), | 24317 | .reg => |reg| try s.cg.tempFromValue(spec.type, .{ .register = reg }), |
| 24103 | .rc => |rc| try s.cg.tempAllocReg(spec.type, regSetForRegClass(rc)), | 24318 | .rc => |rc| try s.cg.tempAllocReg(spec.type, regSetForRegClass(rc)), |
| 24104 | .rc_mask => |mask| try s.cg.tempAllocReg(spec.type, regSetForRegClass(mask.rc)), | 24319 | .rc_mask => |rc_mask| try s.cg.tempAllocReg(spec.type, regSetForRegClass(rc_mask.rc)), |
| 24105 | .mem => try s.cg.tempAllocMem(spec.type), | 24320 | .mem => try s.cg.tempAllocMem(spec.type), |
| 24106 | .src => |src| s.srcTemp(src), | 24321 | .ref => |ref| ref.deref(s), |
| 24107 | .src_mask => |mask| s.srcTemp(mask.src), | 24322 | .ref_mask => |ref_mask| ref_mask.ref.deref(s), |
| 24108 | }; | 24323 | }; |
| 24109 | } | 24324 | } |
| 24110 | }; | 24325 | }; |
| 24111 | | 24326 | |
| 24112 | const Instruction = struct { | 24327 | const Instruction = struct { |
| | 24328 | Label, |
| 24113 | Mir.Inst.Fixes, | 24329 | Mir.Inst.Fixes, |
| 24114 | Mir.Inst.Tag, | 24330 | Mir.Inst.Tag, |
| 24115 | Select2.Operand, | 24331 | Select.Operand, |
| 24116 | Select2.Operand, | 24332 | Select.Operand, |
| 24117 | Select2.Operand, | 24333 | Select.Operand, |
| 24118 | Select2.Operand, | 24334 | Select.Operand, |
| 24119 | }; | 24335 | }; |
| 24120 | const Operand = union(enum) { | 24336 | const Label = enum { @"0:", @"1:", @"_" }; |
| 24121 | none, | 24337 | const Operand = struct { |
| 24122 | extra: struct { Memory.Size, u8 }, | 24338 | tag: Tag, |
| 24123 | dst: struct { Memory.Size, u8 }, | 24339 | base: Ref.Sized = .none, |
| 24124 | src: struct { Memory.Size, u8 }, | 24340 | index: packed struct(u6) { |
| 24125 | dst_limb: u8, | 24341 | ref: Ref, |
| 24126 | src_limb: u8, | 24342 | scale: Memory.Scale, |
| 24127 | simm32: i32, | 24343 | } = .{ .ref = .none, .scale = .@"1" }, |
| 24128 | | 24344 | adjust: Adjust = .none, |
| 24129 | const tmp0b: Select2.Operand = .{ .extra = .{ .byte, 0 } }; | 24345 | imm: i32 = 0, |
| 24130 | const tmp0w: Select2.Operand = .{ .extra = .{ .word, 0 } }; | 24346 | |
| 24131 | const etmp0: Select2.Operand = .{ .extra = .{ .dword, 0 } }; | 24347 | const Tag = enum { |
| 24132 | const rtmp0: Select2.Operand = .{ .extra = .{ .qword, 0 } }; | 24348 | none, |
| 24133 | const xtmp0: Select2.Operand = .{ .extra = .{ .xword, 0 } }; | 24349 | backward_label, |
| 24134 | const ytmp0: Select2.Operand = .{ .extra = .{ .yword, 0 } }; | 24350 | forward_label, |
| 24135 | | 24351 | ref, |
| 24136 | const tmp1b: Select2.Operand = .{ .extra = .{ .byte, 1 } }; | 24352 | limb, |
| 24137 | const tmp1w: Select2.Operand = .{ .extra = .{ .word, 1 } }; | 24353 | mask_limb, |
| 24138 | const etmp1: Select2.Operand = .{ .extra = .{ .dword, 1 } }; | 24354 | simm, |
| 24139 | const rtmp1: Select2.Operand = .{ .extra = .{ .qword, 1 } }; | 24355 | lea, |
| 24140 | const xtmp1: Select2.Operand = .{ .extra = .{ .xword, 1 } }; | 24356 | mem, |
| 24141 | const ytmp1: Select2.Operand = .{ .extra = .{ .yword, 1 } }; | 24357 | }; |
| 24142 | | 24358 | const Adjust = enum { |
| 24143 | const tmp2b: Select2.Operand = .{ .extra = .{ .byte, 2 } }; | 24359 | none, |
| 24144 | const tmp2w: Select2.Operand = .{ .extra = .{ .word, 2 } }; | 24360 | add_access_size, |
| 24145 | const etmp2: Select2.Operand = .{ .extra = .{ .dword, 2 } }; | 24361 | sub_access_size, |
| 24146 | const rtmp2: Select2.Operand = .{ .extra = .{ .qword, 2 } }; | 24362 | add_size, |
| 24147 | const xtmp2: Select2.Operand = .{ .extra = .{ .xword, 2 } }; | 24363 | sub_size, |
| 24148 | const ytmp2: Select2.Operand = .{ .extra = .{ .yword, 2 } }; | 24364 | add_bit_size, |
| 24149 | | 24365 | sub_bit_size, |
| 24150 | const dst0b: Select2.Operand = .{ .dst = .{ .byte, 0 } }; | 24366 | add_limbs, |
| 24151 | const dst0w: Select2.Operand = .{ .dst = .{ .word, 0 } }; | 24367 | sub_limbs, |
| 24152 | const edst0: Select2.Operand = .{ .dst = .{ .dword, 0 } }; | 24368 | add_len, |
| 24153 | const rdst0: Select2.Operand = .{ .dst = .{ .qword, 0 } }; | 24369 | sub_len, |
| 24154 | const xdst0: Select2.Operand = .{ .dst = .{ .xword, 0 } }; | 24370 | add_elem_size, |
| 24155 | const ydst0: Select2.Operand = .{ .dst = .{ .yword, 0 } }; | 24371 | sub_elem_size, |
| 24156 | | 24372 | add_elem_limbs, |
| 24157 | const src0b: Select2.Operand = .{ .src = .{ .byte, 0 } }; | 24373 | sub_elem_limbs, |
| 24158 | const src0w: Select2.Operand = .{ .src = .{ .word, 0 } }; | 24374 | }; |
| 24159 | const esrc0: Select2.Operand = .{ .src = .{ .dword, 0 } }; | 24375 | const Ref = enum(u4) { |
| 24160 | const rsrc0: Select2.Operand = .{ .src = .{ .qword, 0 } }; | 24376 | tmp0, |
| 24161 | const xsrc0: Select2.Operand = .{ .src = .{ .xword, 0 } }; | 24377 | tmp1, |
| 24162 | const ysrc0: Select2.Operand = .{ .src = .{ .yword, 0 } }; | 24378 | tmp2, |
| 24163 | | 24379 | tmp3, |
| 24164 | const src1b: Select2.Operand = .{ .src = .{ .byte, 1 } }; | 24380 | tmp4, |
| 24165 | const src1w: Select2.Operand = .{ .src = .{ .word, 1 } }; | 24381 | tmp5, |
| 24166 | const esrc1: Select2.Operand = .{ .src = .{ .dword, 1 } }; | 24382 | dst0, |
| 24167 | const rsrc1: Select2.Operand = .{ .src = .{ .qword, 1 } }; | 24383 | src0, |
| 24168 | const xsrc1: Select2.Operand = .{ .src = .{ .xword, 1 } }; | 24384 | src1, |
| 24169 | const ysrc1: Select2.Operand = .{ .src = .{ .yword, 1 } }; | 24385 | none, |
| 24170 | | 24386 | |
| 24171 | fn unwrap(op: Select2.Operand, s: Select2) struct { Memory.Size, Temp } { | 24387 | const Sized = packed struct(u8) { |
| 24172 | return switch (op) { | 24388 | ref: Ref, |
| 24173 | else => unreachable, | 24389 | size: Memory.Size, |
| 24174 | .extra => |extra| .{ extra[0], s.extra_temps[extra[1]] }, | 24390 | |
| 24175 | .dst => |dst| .{ dst[0], s.dst_temps[dst[1]] }, | 24391 | const none: Sized = .{ .ref = .none, .size = .none }; |
| 24176 | .src => |src| .{ src[0], s.srcTemp(src[1]) }, | 24392 | |
| | 24393 | const tmp0: Sized = .{ .ref = .tmp0, .size = .none }; |
| | 24394 | const tmp0b: Sized = .{ .ref = .tmp0, .size = .byte }; |
| | 24395 | const tmp0w: Sized = .{ .ref = .tmp0, .size = .word }; |
| | 24396 | const tmp0d: Sized = .{ .ref = .tmp0, .size = .dword }; |
| | 24397 | const tmp0p: Sized = .{ .ref = .tmp0, .size = .ptr }; |
| | 24398 | const tmp0q: Sized = .{ .ref = .tmp0, .size = .qword }; |
| | 24399 | const tmp0x: Sized = .{ .ref = .tmp0, .size = .xword }; |
| | 24400 | const tmp0y: Sized = .{ .ref = .tmp0, .size = .yword }; |
| | 24401 | |
| | 24402 | const tmp1: Sized = .{ .ref = .tmp1, .size = .none }; |
| | 24403 | const tmp1b: Sized = .{ .ref = .tmp1, .size = .byte }; |
| | 24404 | const tmp1w: Sized = .{ .ref = .tmp1, .size = .word }; |
| | 24405 | const tmp1d: Sized = .{ .ref = .tmp1, .size = .dword }; |
| | 24406 | const tmp1p: Sized = .{ .ref = .tmp1, .size = .ptr }; |
| | 24407 | const tmp1q: Sized = .{ .ref = .tmp1, .size = .qword }; |
| | 24408 | const tmp1x: Sized = .{ .ref = .tmp1, .size = .xword }; |
| | 24409 | const tmp1y: Sized = .{ .ref = .tmp1, .size = .yword }; |
| | 24410 | |
| | 24411 | const tmp2: Sized = .{ .ref = .tmp2, .size = .none }; |
| | 24412 | const tmp2b: Sized = .{ .ref = .tmp2, .size = .byte }; |
| | 24413 | const tmp2w: Sized = .{ .ref = .tmp2, .size = .word }; |
| | 24414 | const tmp2d: Sized = .{ .ref = .tmp2, .size = .dword }; |
| | 24415 | const tmp2p: Sized = .{ .ref = .tmp2, .size = .ptr }; |
| | 24416 | const tmp2q: Sized = .{ .ref = .tmp2, .size = .qword }; |
| | 24417 | const tmp2x: Sized = .{ .ref = .tmp2, .size = .xword }; |
| | 24418 | const tmp2y: Sized = .{ .ref = .tmp2, .size = .yword }; |
| | 24419 | |
| | 24420 | const tmp3: Sized = .{ .ref = .tmp3, .size = .none }; |
| | 24421 | const tmp3b: Sized = .{ .ref = .tmp3, .size = .byte }; |
| | 24422 | const tmp3w: Sized = .{ .ref = .tmp3, .size = .word }; |
| | 24423 | const tmp3d: Sized = .{ .ref = .tmp3, .size = .dword }; |
| | 24424 | const tmp3p: Sized = .{ .ref = .tmp3, .size = .ptr }; |
| | 24425 | const tmp3q: Sized = .{ .ref = .tmp3, .size = .qword }; |
| | 24426 | const tmp3x: Sized = .{ .ref = .tmp3, .size = .xword }; |
| | 24427 | const tmp3y: Sized = .{ .ref = .tmp3, .size = .yword }; |
| | 24428 | |
| | 24429 | const tmp4: Sized = .{ .ref = .tmp4, .size = .none }; |
| | 24430 | const tmp4b: Sized = .{ .ref = .tmp4, .size = .byte }; |
| | 24431 | const tmp4w: Sized = .{ .ref = .tmp4, .size = .word }; |
| | 24432 | const tmp4d: Sized = .{ .ref = .tmp4, .size = .dword }; |
| | 24433 | const tmp4p: Sized = .{ .ref = .tmp4, .size = .ptr }; |
| | 24434 | const tmp4q: Sized = .{ .ref = .tmp4, .size = .qword }; |
| | 24435 | const tmp4x: Sized = .{ .ref = .tmp4, .size = .xword }; |
| | 24436 | const tmp4y: Sized = .{ .ref = .tmp4, .size = .yword }; |
| | 24437 | |
| | 24438 | const dst0: Sized = .{ .ref = .dst0, .size = .none }; |
| | 24439 | const dst0b: Sized = .{ .ref = .dst0, .size = .byte }; |
| | 24440 | const dst0w: Sized = .{ .ref = .dst0, .size = .word }; |
| | 24441 | const dst0d: Sized = .{ .ref = .dst0, .size = .dword }; |
| | 24442 | const dst0p: Sized = .{ .ref = .dst0, .size = .ptr }; |
| | 24443 | const dst0q: Sized = .{ .ref = .dst0, .size = .qword }; |
| | 24444 | const dst0x: Sized = .{ .ref = .dst0, .size = .xword }; |
| | 24445 | const dst0y: Sized = .{ .ref = .dst0, .size = .yword }; |
| | 24446 | |
| | 24447 | const src0: Sized = .{ .ref = .src0, .size = .none }; |
| | 24448 | const src0b: Sized = .{ .ref = .src0, .size = .byte }; |
| | 24449 | const src0w: Sized = .{ .ref = .src0, .size = .word }; |
| | 24450 | const src0d: Sized = .{ .ref = .src0, .size = .dword }; |
| | 24451 | const src0p: Sized = .{ .ref = .src0, .size = .ptr }; |
| | 24452 | const src0q: Sized = .{ .ref = .src0, .size = .qword }; |
| | 24453 | const src0x: Sized = .{ .ref = .src0, .size = .xword }; |
| | 24454 | const src0y: Sized = .{ .ref = .src0, .size = .yword }; |
| | 24455 | |
| | 24456 | const src1: Sized = .{ .ref = .src1, .size = .none }; |
| | 24457 | const src1b: Sized = .{ .ref = .src1, .size = .byte }; |
| | 24458 | const src1w: Sized = .{ .ref = .src1, .size = .word }; |
| | 24459 | const src1d: Sized = .{ .ref = .src1, .size = .dword }; |
| | 24460 | const src1p: Sized = .{ .ref = .src1, .size = .ptr }; |
| | 24461 | const src1q: Sized = .{ .ref = .src1, .size = .qword }; |
| | 24462 | const src1x: Sized = .{ .ref = .src1, .size = .xword }; |
| | 24463 | const src1y: Sized = .{ .ref = .src1, .size = .yword }; |
| 24177 | }; | 24464 | }; |
| 24178 | } | | |
| 24179 | | 24465 | |
| 24180 | fn lower(op: Select2.Operand, s: Select2) !CodeGen.Operand { | 24466 | fn deref(ref: Ref, s: *const Select) Temp { |
| 24181 | switch (op) { | 24467 | return s.temps[@intFromEnum(ref)]; |
| 24182 | .none => return .none, | | |
| 24183 | else => {}, | | |
| 24184 | .dst_limb => |dst| return s.lowerLimb(s.dst_temps[dst]), | | |
| 24185 | .src_limb => |src| return s.lowerLimb(s.srcTemp(src)), | | |
| 24186 | .simm32 => |imm| return .{ .imm = .s(imm) }, | | |
| 24187 | } | 24468 | } |
| 24188 | const size, const temp = op.unwrap(s); | 24469 | }; |
| 24189 | return switch (temp.tracking(s.cg).short) { | 24470 | |
| 24190 | .immediate => |imm| .{ .imm = switch (size) { | 24471 | const @"_": Select.Operand = .{ .tag = .none }; |
| 24191 | .byte => if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u8, @intCast(imm))), | 24472 | |
| 24192 | .word => if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u16, @intCast(imm))), | 24473 | const @"0b": Select.Operand = .{ .tag = .backward_label, .base = .{ .ref = .tmp0, .size = .none } }; |
| 24193 | .dword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u32, @intCast(imm))), | 24474 | const @"0f": Select.Operand = .{ .tag = .forward_label, .base = .{ .ref = .tmp0, .size = .none } }; |
| 24194 | .qword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(imm), | 24475 | const @"1b": Select.Operand = .{ .tag = .backward_label, .base = .{ .ref = .tmp1, .size = .none } }; |
| 24195 | else => unreachable, | 24476 | const @"1f": Select.Operand = .{ .tag = .forward_label, .base = .{ .ref = .tmp1, .size = .none } }; |
| | 24477 | |
| | 24478 | const tmp0b: Select.Operand = .{ .tag = .ref, .base = .tmp0b }; |
| | 24479 | const tmp0w: Select.Operand = .{ .tag = .ref, .base = .tmp0w }; |
| | 24480 | const tmp0d: Select.Operand = .{ .tag = .ref, .base = .tmp0d }; |
| | 24481 | const tmp0p: Select.Operand = .{ .tag = .ref, .base = .tmp0p }; |
| | 24482 | const tmp0q: Select.Operand = .{ .tag = .ref, .base = .tmp0q }; |
| | 24483 | const tmp0x: Select.Operand = .{ .tag = .ref, .base = .tmp0x }; |
| | 24484 | const tmp0y: Select.Operand = .{ .tag = .ref, .base = .tmp0y }; |
| | 24485 | |
| | 24486 | const tmp1b: Select.Operand = .{ .tag = .ref, .base = .tmp1b }; |
| | 24487 | const tmp1w: Select.Operand = .{ .tag = .ref, .base = .tmp1w }; |
| | 24488 | const tmp1d: Select.Operand = .{ .tag = .ref, .base = .tmp1d }; |
| | 24489 | const tmp1p: Select.Operand = .{ .tag = .ref, .base = .tmp1p }; |
| | 24490 | const tmp1q: Select.Operand = .{ .tag = .ref, .base = .tmp1q }; |
| | 24491 | const tmp1x: Select.Operand = .{ .tag = .ref, .base = .tmp1x }; |
| | 24492 | const tmp1y: Select.Operand = .{ .tag = .ref, .base = .tmp1y }; |
| | 24493 | |
| | 24494 | const tmp2b: Select.Operand = .{ .tag = .ref, .base = .tmp2b }; |
| | 24495 | const tmp2w: Select.Operand = .{ .tag = .ref, .base = .tmp2w }; |
| | 24496 | const tmp2d: Select.Operand = .{ .tag = .ref, .base = .tmp2d }; |
| | 24497 | const tmp2p: Select.Operand = .{ .tag = .ref, .base = .tmp2p }; |
| | 24498 | const tmp2q: Select.Operand = .{ .tag = .ref, .base = .tmp2q }; |
| | 24499 | const tmp2x: Select.Operand = .{ .tag = .ref, .base = .tmp2x }; |
| | 24500 | const tmp2y: Select.Operand = .{ .tag = .ref, .base = .tmp2y }; |
| | 24501 | |
| | 24502 | const tmp3b: Select.Operand = .{ .tag = .ref, .base = .tmp3b }; |
| | 24503 | const tmp3w: Select.Operand = .{ .tag = .ref, .base = .tmp3w }; |
| | 24504 | const tmp3d: Select.Operand = .{ .tag = .ref, .base = .tmp3d }; |
| | 24505 | const tmp3p: Select.Operand = .{ .tag = .ref, .base = .tmp3p }; |
| | 24506 | const tmp3q: Select.Operand = .{ .tag = .ref, .base = .tmp3q }; |
| | 24507 | const tmp3x: Select.Operand = .{ .tag = .ref, .base = .tmp3x }; |
| | 24508 | const tmp3y: Select.Operand = .{ .tag = .ref, .base = .tmp3y }; |
| | 24509 | |
| | 24510 | const tmp4b: Select.Operand = .{ .tag = .ref, .base = .tmp4b }; |
| | 24511 | const tmp4w: Select.Operand = .{ .tag = .ref, .base = .tmp4w }; |
| | 24512 | const tmp4d: Select.Operand = .{ .tag = .ref, .base = .tmp4d }; |
| | 24513 | const tmp4p: Select.Operand = .{ .tag = .ref, .base = .tmp4p }; |
| | 24514 | const tmp4q: Select.Operand = .{ .tag = .ref, .base = .tmp4q }; |
| | 24515 | const tmp4x: Select.Operand = .{ .tag = .ref, .base = .tmp4x }; |
| | 24516 | const tmp4y: Select.Operand = .{ .tag = .ref, .base = .tmp4y }; |
| | 24517 | |
| | 24518 | const dst0b: Select.Operand = .{ .tag = .ref, .base = .dst0b }; |
| | 24519 | const dst0w: Select.Operand = .{ .tag = .ref, .base = .dst0w }; |
| | 24520 | const dst0d: Select.Operand = .{ .tag = .ref, .base = .dst0d }; |
| | 24521 | const dst0p: Select.Operand = .{ .tag = .ref, .base = .dst0p }; |
| | 24522 | const dst0q: Select.Operand = .{ .tag = .ref, .base = .dst0q }; |
| | 24523 | const dst0x: Select.Operand = .{ .tag = .ref, .base = .dst0x }; |
| | 24524 | const dst0y: Select.Operand = .{ .tag = .ref, .base = .dst0y }; |
| | 24525 | |
| | 24526 | const src0b: Select.Operand = .{ .tag = .ref, .base = .src0b }; |
| | 24527 | const src0w: Select.Operand = .{ .tag = .ref, .base = .src0w }; |
| | 24528 | const src0d: Select.Operand = .{ .tag = .ref, .base = .src0d }; |
| | 24529 | const src0p: Select.Operand = .{ .tag = .ref, .base = .src0p }; |
| | 24530 | const src0q: Select.Operand = .{ .tag = .ref, .base = .src0q }; |
| | 24531 | const src0x: Select.Operand = .{ .tag = .ref, .base = .src0x }; |
| | 24532 | const src0y: Select.Operand = .{ .tag = .ref, .base = .src0y }; |
| | 24533 | |
| | 24534 | const src1b: Select.Operand = .{ .tag = .ref, .base = .src1b }; |
| | 24535 | const src1w: Select.Operand = .{ .tag = .ref, .base = .src1w }; |
| | 24536 | const src1d: Select.Operand = .{ .tag = .ref, .base = .src1d }; |
| | 24537 | const src1p: Select.Operand = .{ .tag = .ref, .base = .src1p }; |
| | 24538 | const src1q: Select.Operand = .{ .tag = .ref, .base = .src1q }; |
| | 24539 | const src1x: Select.Operand = .{ .tag = .ref, .base = .src1x }; |
| | 24540 | const src1y: Select.Operand = .{ .tag = .ref, .base = .src1y }; |
| | 24541 | |
| | 24542 | fn limb(ref: Ref.Sized) Select.Operand { |
| | 24543 | return .{ .tag = .limb, .base = ref }; |
| | 24544 | } |
| | 24545 | fn maskLimb(ref: Ref.Sized) Select.Operand { |
| | 24546 | return .{ .tag = .mask_limb, .base = ref }; |
| | 24547 | } |
| | 24548 | |
| | 24549 | fn i(imm: i32) Select.Operand { |
| | 24550 | return .{ .tag = .simm, .imm = imm }; |
| | 24551 | } |
| | 24552 | fn a(base: Ref.Sized, adjust: Adjust) Select.Operand { |
| | 24553 | return .{ .tag = .simm, .base = base, .adjust = adjust }; |
| | 24554 | } |
| | 24555 | fn ia(imm: i32, base: Ref.Sized, adjust: Adjust) Select.Operand { |
| | 24556 | return .{ .tag = .simm, .base = base, .adjust = adjust, .imm = imm }; |
| | 24557 | } |
| | 24558 | |
| | 24559 | fn lea(size: Memory.Size, base: Ref) Select.Operand { |
| | 24560 | return .{ |
| | 24561 | .tag = .lea, |
| | 24562 | .base = .{ .ref = base, .size = size }, |
| | 24563 | }; |
| | 24564 | } |
| | 24565 | fn lead(size: Memory.Size, base: Ref, disp: i32) Select.Operand { |
| | 24566 | return .{ |
| | 24567 | .tag = .lea, |
| | 24568 | .base = .{ .ref = base, .size = size }, |
| | 24569 | .imm = disp, |
| | 24570 | }; |
| | 24571 | } |
| | 24572 | fn leai(size: Memory.Size, base: Ref, index: Ref) Select.Operand { |
| | 24573 | return .{ |
| | 24574 | .tag = .lea, |
| | 24575 | .base = .{ .ref = base, .size = size }, |
| | 24576 | .index_ = .{ .ref = index, .scale = .@"1" }, |
| | 24577 | }; |
| | 24578 | } |
| | 24579 | fn leaid(size: Memory.Size, base: Ref, index: Ref, disp: i32) Select.Operand { |
| | 24580 | return .{ |
| | 24581 | .tag = .lea, |
| | 24582 | .base = .{ .ref = base, .size = size }, |
| | 24583 | .index_ = .{ .ref = index, .scale = .@"1" }, |
| | 24584 | .imm = disp, |
| | 24585 | }; |
| | 24586 | } |
| | 24587 | fn leasi(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref) Select.Operand { |
| | 24588 | return .{ |
| | 24589 | .tag = .lea, |
| | 24590 | .base = .{ .ref = base, .size = size }, |
| | 24591 | .index_ = .{ .ref = index, .scale = scale }, |
| | 24592 | }; |
| | 24593 | } |
| | 24594 | fn leasid(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand { |
| | 24595 | return .{ |
| | 24596 | .tag = .lea, |
| | 24597 | .base = .{ .ref = base, .size = size }, |
| | 24598 | .index_ = .{ .ref = index, .scale = scale }, |
| | 24599 | .imm = disp, |
| | 24600 | }; |
| | 24601 | } |
| | 24602 | fn leasida(size: Memory.Size, base: Ref, scale: Memory.Scale, index: Ref, disp: i32, adjust: Adjust) Select.Operand { |
| | 24603 | return .{ |
| | 24604 | .tag = .lea, |
| | 24605 | .base = .{ .ref = base, .size = size }, |
| | 24606 | .index_ = .{ .ref = index, .scale = scale }, |
| | 24607 | .adjust = adjust, |
| | 24608 | .imm = disp, |
| | 24609 | }; |
| | 24610 | } |
| | 24611 | |
| | 24612 | fn mem(base: Ref.Sized) Select.Operand { |
| | 24613 | return .{ |
| | 24614 | .tag = .mem, |
| | 24615 | .base = base, |
| | 24616 | }; |
| | 24617 | } |
| | 24618 | fn memd(base: Ref.Sized, disp: i32) Select.Operand { |
| | 24619 | return .{ |
| | 24620 | .tag = .mem, |
| | 24621 | .base = base, |
| | 24622 | .imm = disp, |
| | 24623 | }; |
| | 24624 | } |
| | 24625 | fn memi(base: Ref.Sized, index: Ref) Select.Operand { |
| | 24626 | return .{ |
| | 24627 | .tag = .mem, |
| | 24628 | .base = base, |
| | 24629 | .index = .{ .ref = index, .scale = .@"1" }, |
| | 24630 | }; |
| | 24631 | } |
| | 24632 | fn memia(base: Ref.Sized, index: Ref, adjust: Adjust) Select.Operand { |
| | 24633 | return .{ |
| | 24634 | .tag = .mem, |
| | 24635 | .base = base, |
| | 24636 | .index = .{ .ref = index, .scale = .@"1" }, |
| | 24637 | .adjust = adjust, |
| | 24638 | }; |
| | 24639 | } |
| | 24640 | fn memid(base: Ref.Sized, index: Ref, disp: i32) Select.Operand { |
| | 24641 | return .{ |
| | 24642 | .tag = .mem, |
| | 24643 | .base = base, |
| | 24644 | .index = .{ .ref = index, .scale = .@"1" }, |
| | 24645 | .imm = disp, |
| | 24646 | }; |
| | 24647 | } |
| | 24648 | fn memsi(base: Ref.Sized, scale: Memory.Scale, index: Ref) Select.Operand { |
| | 24649 | return .{ |
| | 24650 | .tag = .mem, |
| | 24651 | .base = base, |
| | 24652 | .index = .{ .ref = index, .scale = scale }, |
| | 24653 | }; |
| | 24654 | } |
| | 24655 | fn memsid(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32) Select.Operand { |
| | 24656 | return .{ |
| | 24657 | .tag = .mem, |
| | 24658 | .base = base, |
| | 24659 | .index = .{ .ref = index, .scale = scale }, |
| | 24660 | .imm = disp, |
| | 24661 | }; |
| | 24662 | } |
| | 24663 | fn memsida(base: Ref.Sized, scale: Memory.Scale, index: Ref, disp: i32, adjust: Adjust) Select.Operand { |
| | 24664 | return .{ |
| | 24665 | .tag = .mem, |
| | 24666 | .base = base, |
| | 24667 | .index = .{ .ref = index, .scale = scale }, |
| | 24668 | .adjust = adjust, |
| | 24669 | .imm = disp, |
| | 24670 | }; |
| | 24671 | } |
| | 24672 | |
| | 24673 | fn adjustedImm(op: Select.Operand, s: *const Select) i32 { |
| | 24674 | return switch (op.adjust) { |
| | 24675 | .none => op.imm, |
| | 24676 | .add_access_size => op.imm + @as(i32, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8))), |
| | 24677 | .sub_access_size => op.imm - @as(i32, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8))), |
| | 24678 | .add_size => op.imm + @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu))), |
| | 24679 | .sub_size => op.imm - @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu))), |
| | 24680 | .add_bit_size => op.imm + @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).bitSize(s.cg.pt.zcu))), |
| | 24681 | .sub_bit_size => op.imm - @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).bitSize(s.cg.pt.zcu))), |
| | 24682 | .add_limbs => op.imm + @as(i32, @intCast(@divExact( |
| | 24683 | op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu), |
| | 24684 | @divExact(op.base.size.bitSize(s.cg.target), 8), |
| | 24685 | ))), |
| | 24686 | .sub_limbs => op.imm + @as(i32, @intCast(@divExact( |
| | 24687 | op.base.ref.deref(s).typeOf(s.cg).abiSize(s.cg.pt.zcu), |
| | 24688 | @divExact(op.base.size.bitSize(s.cg.target), 8), |
| | 24689 | ))), |
| | 24690 | .add_len => op.imm + @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).vectorLen(s.cg.pt.zcu))), |
| | 24691 | .sub_len => op.imm - @as(i32, @intCast(op.base.ref.deref(s).typeOf(s.cg).vectorLen(s.cg.pt.zcu))), |
| | 24692 | .add_elem_size => op.imm + @as(i32, @intCast( |
| | 24693 | op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu), |
| | 24694 | )), |
| | 24695 | .sub_elem_size => op.imm - @as(i32, @intCast( |
| | 24696 | op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu), |
| | 24697 | )), |
| | 24698 | .add_elem_limbs => op.imm + @as(i32, @intCast(@divExact( |
| | 24699 | op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu), |
| | 24700 | @divExact(op.base.size.bitSize(s.cg.target), 8), |
| | 24701 | ))), |
| | 24702 | .sub_elem_limbs => op.imm - @as(i32, @intCast(@divExact( |
| | 24703 | op.base.ref.deref(s).typeOf(s.cg).scalarType(s.cg.pt.zcu).abiSize(s.cg.pt.zcu), |
| | 24704 | @divExact(op.base.size.bitSize(s.cg.target), 8), |
| | 24705 | ))), |
| | 24706 | }; |
| | 24707 | } |
| | 24708 | |
| | 24709 | fn lower(op: Select.Operand, s: *Select) !CodeGen.Operand { |
| | 24710 | return switch (op.tag) { |
| | 24711 | .none => .none, |
| | 24712 | .backward_label => .{ .inst = s.labels[@intFromEnum(op.base.ref)].backward.? }, |
| | 24713 | .forward_label => for (&s.labels[@intFromEnum(op.base.ref)].forward) |*label| { |
| | 24714 | if (label.*) |_| continue; |
| | 24715 | label.* = @intCast(s.cg.mir_instructions.len); |
| | 24716 | break .{ .inst = undefined }; |
| | 24717 | } else unreachable, |
| | 24718 | .ref => switch (op.base.ref.deref(s).tracking(s.cg).short) { |
| | 24719 | .immediate => |imm| .{ .imm = switch (op.base.size) { |
| | 24720 | .byte => if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u8, @intCast(imm))), |
| | 24721 | .word => if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u16, @intCast(imm))), |
| | 24722 | .dword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(@as(u32, @intCast(imm))), |
| | 24723 | .qword => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |simm| .s(simm) else .u(imm), |
| | 24724 | else => unreachable, |
| | 24725 | } }, |
| | 24726 | else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = op.base.size }) }, |
| | 24727 | .register => |reg| .{ .reg = registerAlias(reg, @intCast(@divExact(op.base.size.bitSize(s.cg.target), 8))) }, |
| | 24728 | }, |
| | 24729 | inline .limb, .mask_limb => |kind| .{ .mem = try op.base.ref.deref(s).tracking(s.cg).short.mem(s.cg, @field(s, @tagName(kind))) }, |
| | 24730 | .simm => .{ .imm = .s(op.adjustedImm(s)) }, |
| | 24731 | .lea => .{ .mem = .{ |
| | 24732 | .base = .{ .reg = registerAlias(op.base.ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)) }, |
| | 24733 | .mod = .{ .rm = .{ |
| | 24734 | .size = op.base.size, |
| | 24735 | .index = switch (op.index.ref) { |
| | 24736 | else => |ref| registerAlias(ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)), |
| | 24737 | .none => .none, |
| | 24738 | }, |
| | 24739 | .scale = op.index.scale, |
| | 24740 | .disp = op.adjustedImm(s), |
| | 24741 | } }, |
| 24196 | } }, | 24742 | } }, |
| 24197 | else => |mcv| .{ .mem = try mcv.mem(s.cg, .{ .size = size }) }, | 24743 | .mem => .{ .mem = try op.base.ref.deref(s).tracking(s.cg).short.mem(s.cg, .{ |
| 24198 | .register => |reg| .{ .reg = registerAlias(reg, @intCast(@divExact(size.bitSize(), 8))) }, | 24744 | .size = op.base.size, |
| | 24745 | .index = switch (op.index.ref) { |
| | 24746 | else => |ref| registerAlias(ref.deref(s).tracking(s.cg).short.register, @divExact(s.cg.target.ptrBitWidth(), 8)), |
| | 24747 | .none => .none, |
| | 24748 | }, |
| | 24749 | .scale = op.index.scale, |
| | 24750 | .disp = op.adjustedImm(s), |
| | 24751 | }) }, |
| 24199 | }; | 24752 | }; |
| 24200 | } | 24753 | } |
| 24201 | }; | 24754 | }; |
| 24202 | }; | 24755 | }; |
| 24203 | fn select2( | 24756 | fn select( |
| 24204 | cg: *CodeGen, | 24757 | cg: *CodeGen, |
| 24205 | dst_temps: []Temp, | 24758 | dst_temps: []Temp, |
| 24206 | dst_tys: []const Type, | 24759 | dst_tys: []const Type, |
| 24207 | src_temps: []Temp, | 24760 | src_temps: []Temp, |
| 24208 | cases: []const Select2.Case, | 24761 | cases: []const Select.Case, |
| 24209 | ) !void { | 24762 | ) !void { |
| 24210 | cases: for (cases) |*case| { | 24763 | cases: for (cases) |case| { |
| 24211 | for (case.required_features) |required_feature| if (required_feature) |feature| if (!switch (feature) { | 24764 | for (case.required_features) |required_feature| if (required_feature) |feature| if (!switch (feature) { |
| 24212 | .@"64bit" => cg.target.cpu.arch == .x86_64, | 24765 | .@"64bit" => cg.target.ptrBitWidth() == 64, |
| 24213 | .mmx => false, | 24766 | .mmx => false, |
| 24214 | else => cg.hasFeature(feature), | 24767 | else => cg.hasFeature(feature), |
| 24215 | }) continue :cases; | 24768 | }) continue :cases; |
| 24216 | for (case.constraints[0..src_temps.len], src_temps) |src_constraint, src_temp| if (!src_constraint.accepts(src_temp, cg)) continue :cases; | 24769 | for (case.dst_constraints[0..dst_temps.len], dst_tys) |dst_constraint, dst_ty| if (!dst_constraint.accepts(dst_ty, cg)) continue :cases; |
| 24217 | patterns: for (case.patterns) |*pattern| { | 24770 | for (case.src_constraints[0..src_temps.len], src_temps) |src_constraint, src_temp| if (!src_constraint.accepts(src_temp.typeOf(cg), cg)) continue :cases; |
| | 24771 | patterns: for (case.patterns) |pattern| { |
| 24218 | for (pattern.src, src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns; | 24772 | for (pattern.src, src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns; |
| 24219 | | 24773 | |
| 24220 | var s: Select2 = .{ | 24774 | var s: Select = .{ |
| 24221 | .cg = cg, | 24775 | .cg = cg, |
| 24222 | .case = case, | 24776 | .temps = undefined, |
| 24223 | .pattern = pattern, | 24777 | .labels = @splat(.{ .forward = @splat(null), .backward = null }), |
| 24224 | .extra_temps = undefined, | | |
| 24225 | .dst_temps = dst_temps, | | |
| 24226 | .src_temps = src_temps, | | |
| 24227 | .commute = pattern.commute, | | |
| 24228 | .limb = undefined, | 24778 | .limb = undefined, |
| | 24779 | .mask_limb = undefined, |
| 24229 | }; | 24780 | }; |
| 24230 | for (&s.extra_temps, case.extra_temps) |*temp, spec| temp.* = try spec.create(s) orelse continue; | 24781 | const tmp_slots = s.temps[@intFromEnum(Select.Operand.Ref.tmp0)..@intFromEnum(Select.Operand.Ref.dst0)]; |
| | 24782 | const dst_slots = s.temps[@intFromEnum(Select.Operand.Ref.dst0)..@intFromEnum(Select.Operand.Ref.src0)]; |
| | 24783 | const src_slots = s.temps[@intFromEnum(Select.Operand.Ref.src0)..@intFromEnum(Select.Operand.Ref.none)]; |
| | 24784 | |
| | 24785 | for (tmp_slots, case.extra_temps) |*slot, spec| slot.* = try spec.create(&s) orelse continue; |
| 24231 | | 24786 | |
| 24232 | while (true) for (pattern.src, src_temps) |src_pattern, *src_temp| { | 24787 | while (true) for (pattern.src, src_temps) |src_pattern, *src_temp| { |
| 24233 | if (try src_pattern.convert(src_temp, cg)) break; | 24788 | if (try src_pattern.convert(src_temp, cg)) break; |
| 24234 | } else break; | 24789 | } else break; |
| | 24790 | @memcpy(src_slots[0..src_temps.len], src_temps); |
| | 24791 | std.mem.swap(Temp, &src_slots[pattern.commute[0]], &src_slots[pattern.commute[1]]); |
| 24235 | | 24792 | |
| 24236 | if (case.clobbers.eflags or case.each != .once) try cg.spillEflagsIfOccupied(); | 24793 | if (case.clobbers.eflags or case.each != .once) try cg.spillEflagsIfOccupied(); |
| 24237 | | 24794 | |
| 24238 | for (dst_temps, dst_tys, case.dst_temps[0..dst_temps.len]) |*dst_temp, dst_ty, dst_kind| | 24795 | for (dst_temps, dst_tys, case.dst_temps[0..dst_temps.len]) |*dst_temp, dst_ty, dst_kind| |
| 24239 | dst_temp.* = (try Select2.TempSpec.create(.{ .type = dst_ty, .kind = dst_kind }, s)).?; | 24796 | dst_temp.* = (try Select.TempSpec.create(.{ .type = dst_ty, .kind = dst_kind }, &s)).?; |
| | 24797 | @memcpy(dst_slots[0..dst_temps.len], dst_temps); |
| 24240 | | 24798 | |
| 24241 | switch (case.each) { | 24799 | switch (case.each) { |
| 24242 | .once => |body| for (body) |inst| try s.emit(inst), | 24800 | .once => |body| for (body) |inst| try s.emit(inst), |
| 24243 | .limb => |limb| { | 24801 | .limb => |limb| { |
| 24244 | const limb_size, const limb_of_temp = limb.of.unwrap(s); | 24802 | const limb_of_size: i32 = @intCast(limb.of.ref.deref(&s).typeOf(cg).abiSize(cg.pt.zcu)); |
| 24245 | const limb_of_size: u31 = @intCast(limb_of_temp.typeOf(cg).abiSize(cg.pt.zcu)); | | |
| 24246 | s.limb = .{ | 24803 | s.limb = .{ |
| 24247 | .size = limb_size, | 24804 | .size = limb.of.size, |
| 24248 | .index = s.extra_temps[0].tracking(cg).short.register.to64(), | 24805 | .index = (try Select.Operand.tmp0p.lower(&s)).reg, |
| 24249 | .disp = limb_of_size, | 24806 | .disp = limb_of_size, |
| 24250 | }; | 24807 | }; |
| 24251 | for (limb.header) |inst| try s.emit(inst); | 24808 | for (limb.header) |inst| try s.emit(inst); |
| 24252 | try cg.asmRegisterImmediate(.{ ._, .mov }, s.limb.index, .s(-@as(i32, limb_of_size))); | 24809 | try s.emit(.{ ._, ._, .mov, .tmp0p, .i(-limb_of_size), ._, ._ }); |
| 24253 | const limb_loop_reloc: u32 = @intCast(cg.mir_instructions.len); | 24810 | assert(s.labels[0].backward == null); |
| | 24811 | s.labels[0].backward = @intCast(cg.mir_instructions.len); |
| 24254 | for (limb.body) |inst| try s.emit(inst); | 24812 | for (limb.body) |inst| try s.emit(inst); |
| 24255 | try cg.asmRegisterImmediate( | 24813 | try s.emit(.{ ._, ._, .add, .tmp0p, .i(@intCast(@divExact(limb.of.size.bitSize(cg.target), 8))), ._, ._ }); |
| 24256 | .{ ._, .add }, | 24814 | try s.emit(.{ ._, ._nc, .j, .@"0b", ._, ._, ._ }); |
| 24257 | s.limb.index, | | |
| 24258 | .s(@intCast(@divExact(limb_size.bitSize(), 8))), | | |
| 24259 | ); | | |
| 24260 | _ = try cg.asmJccReloc(.nc, limb_loop_reloc); | | |
| 24261 | for (limb.trailer) |inst| try s.emit(inst); | 24815 | for (limb.trailer) |inst| try s.emit(inst); |
| 24262 | }, | 24816 | }, |
| | 24817 | .limb_and_mask_limb => |limb| { |
| | 24818 | const limb_of_size: i32 = @intCast(limb.of.ref.deref(&s).typeOf(cg).abiSize(cg.pt.zcu)); |
| | 24819 | s.limb = .{ |
| | 24820 | .size = limb.of.size, |
| | 24821 | .index = (try Select.Operand.tmp0p.lower(&s)).reg, |
| | 24822 | .disp = limb_of_size, |
| | 24823 | }; |
| | 24824 | const mask_limb_bit_size: u31 = @intCast(@divExact( |
| | 24825 | limb.of.size.bitSize(cg.target), |
| | 24826 | limb.of_mask.size.bitSize(cg.target), |
| | 24827 | )); |
| | 24828 | if (mask_limb_bit_size >= 8) { |
| | 24829 | s.mask_limb = .{ |
| | 24830 | .size = .fromBitSize(mask_limb_bit_size), |
| | 24831 | .index = (try Select.Operand.tmp1p.lower(&s)).reg, |
| | 24832 | }; |
| | 24833 | for (limb.header) |inst| try s.emit(inst); |
| | 24834 | try s.emit(.{ ._, ._, .mov, .tmp0p, .i(-limb_of_size), ._, ._ }); |
| | 24835 | try s.emit(.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }); |
| | 24836 | assert(s.labels[0].backward == null); |
| | 24837 | s.labels[0].backward = @intCast(cg.mir_instructions.len); |
| | 24838 | for (limb.body) |inst| try s.emit(inst); |
| | 24839 | try s.emit(.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, @divExact(mask_limb_bit_size, 8)), ._, ._ }); |
| | 24840 | try s.emit(.{ ._, ._, .add, .tmp0p, .i(@intCast(@divExact(limb.of.size.bitSize(cg.target), 8))), ._, ._ }); |
| | 24841 | try s.emit(.{ ._, ._nc, .j, .@"0b", ._, ._, ._ }); |
| | 24842 | for (limb.trailer) |inst| try s.emit(inst); |
| | 24843 | } else { |
| | 24844 | for (limb.header) |inst| try s.emit(inst); |
| | 24845 | try s.emit(.{ ._, ._, .mov, .tmp0p, .i(-limb_of_size), ._, ._ }); |
| | 24846 | try s.emit(.{ ._, ._, .xor, .tmp1d, .tmp1d, ._, ._ }); |
| | 24847 | try s.emit(.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }); |
| | 24848 | assert(s.labels[0].backward == null); |
| | 24849 | s.labels[0].backward = @intCast(cg.mir_instructions.len); |
| | 24850 | for (limb.body) |inst| try s.emit(inst); |
| | 24851 | try s.emit(.{ ._, ._, .lea, .tmp1d, .lead(.none, .tmp1, mask_limb_bit_size), ._, ._ }); |
| | 24852 | try s.emit(.{ ._, ._, .@"test", .tmp1d, .i(0b111), ._, ._ }); |
| | 24853 | try s.emit(.{ ._, ._nz, .j, .@"1f", ._, ._, ._ }); |
| | 24854 | try s.emit(.{ ._, ._, .mov, .tmp3d, .tmp1d, ._, ._ }); |
| | 24855 | try s.emit(.{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ }); |
| | 24856 | try s.emit(.{ ._, ._, .mov, .memid(.{ .ref = limb.of_mask.ref, .size = .byte }, .tmp3, -1), .tmp2b, ._, ._ }); |
| | 24857 | try s.emit(.{ ._, ._, .xor, .tmp2b, .tmp2b, ._, ._ }); |
| | 24858 | try s.emit(.{ .@"1:", ._, .add, .tmp0p, .i(@intCast(@divExact(limb.of.size.bitSize(cg.target), 8))), ._, ._ }); |
| | 24859 | try s.emit(.{ ._, ._nc, .j, .@"0b", ._, ._, ._ }); |
| | 24860 | try s.emit(.{ ._, ._, .lea, .tmp3d, .lead(.none, .tmp1, -1), ._, ._ }); |
| | 24861 | try s.emit(.{ ._, ._r, .sh, .tmp3d, .i(3), ._, ._ }); |
| | 24862 | try s.emit(.{ ._, ._, .mov, .memi(.{ .ref = limb.of_mask.ref, .size = .byte }, .tmp3), .tmp2b, ._, ._ }); |
| | 24863 | for (limb.trailer) |inst| try s.emit(inst); |
| | 24864 | } |
| | 24865 | }, |
| 24263 | } | 24866 | } |
| | 24867 | s.emitLabel(.@"0:"); |
| 24264 | | 24868 | |
| 24265 | for (dst_temps, case.dst_temps[0..dst_temps.len]) |dst_temp, dst_kind| dst_kind.finish(dst_temp, s); | 24869 | for (dst_temps, case.dst_temps[0..dst_temps.len]) |dst_temp, dst_kind| dst_kind.finish(dst_temp, &s); |
| 24266 | for (case.extra_temps, s.extra_temps) |spec, temp| if (spec.kind != .unused) try temp.die(cg); | 24870 | for (case.extra_temps, tmp_slots) |spec, temp| if (spec.kind != .unused) try temp.die(cg); |
| 24267 | return; | 24871 | return; |
| 24268 | } | 24872 | } |
| 24269 | } | 24873 | } |
| 24270 | return error.Select2Failed; | 24874 | return error.SelectFailed; |
| 24271 | } | 24875 | } |
Jacob Young <jacobly@ziglang.org> 2024-12-23 15:53:13-05:00