authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-02-17 05:35:57-05:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-02-17 06:04:43-05:00
logff7412752690d1690cfaedaa605128a992a6b280
tree157e66cd89cf8f5b97c853c9e8f18d3235f372a4
parent82eedf56d703b3ce659519fe52f58a06279cfde0

x86_64: implement prefetch

4 files changed, 80 insertions(+), 21 deletions(-)

src/arch/x86_64/CodeGen.zig+27-6
...@@ -2484,7 +2484,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2484,7 +2484,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2484 .reduce => try cg.airReduce(inst),2484 .reduce => try cg.airReduce(inst),
2485 .reduce_optimized => try cg.airReduce(inst),2485 .reduce_optimized => try cg.airReduce(inst),
2486 .aggregate_init => try cg.airAggregateInit(inst),2486 .aggregate_init => try cg.airAggregateInit(inst),
2487 .prefetch => try cg.airPrefetch(inst),
2488 // zig fmt: on2487 // zig fmt: on
24892488
2490 .arg => if (cg.debug_output != .none) {2489 .arg => if (cg.debug_output != .none) {
...@@ -76418,6 +76417,33 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -76418,6 +76417,33 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
76418 }, cg);76417 }, cg);
76419 try res.finish(inst, &.{extra.init}, &ops, cg);76418 try res.finish(inst, &.{extra.init}, &ops, cg);
76420 },76419 },
76420 .prefetch => {
76421 const prefetch = air_datas[@intFromEnum(inst)].prefetch;
76422 var ops = try cg.tempsFromOperands(inst, .{prefetch.ptr});
76423 switch (prefetch.cache) {
76424 .instruction => {}, // prefetchi requires rip-relative addressing, which is currently non-trivial to emit from an arbitrary ptr value
76425 .data => if (prefetch.rw == .write and prefetch.locality <= 2 and cg.hasFeature(.prefetchwt1)) {
76426 try ops[0].toSlicePtr(cg);
76427 while (try ops[0].toLea(cg)) {}
76428 try cg.asmMemory(.{ ._wt1, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte }));
76429 } else if (prefetch.rw == .write and cg.hasFeature(.prfchw)) {
76430 try ops[0].toSlicePtr(cg);
76431 while (try ops[0].toLea(cg)) {}
76432 try cg.asmMemory(.{ ._w, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte }));
76433 } else if (cg.hasFeature(.sse) or cg.hasFeature(.prfchw) or cg.hasFeature(.prefetchi) or cg.hasFeature(.prefetchwt1)) {
76434 try ops[0].toSlicePtr(cg);
76435 while (try ops[0].toLea(cg)) {}
76436 switch (prefetch.locality) {
76437 0 => try cg.asmMemory(.{ ._nta, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })),
76438 1 => try cg.asmMemory(.{ ._t2, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })),
76439 2 => try cg.asmMemory(.{ ._t1, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })),
76440 3 => try cg.asmMemory(.{ ._t0, .prefetch }, try ops[0].tracking(cg).short.deref().mem(cg, .{ .size = .byte })),
76441 }
76442 },
76443 }
76444 const res = try cg.tempInit(.void, .none);
76445 try res.finish(inst, &.{prefetch.ptr}, &ops, cg);
76446 },
76421 .mul_add => |air_tag| if (use_old) try cg.airMulAdd(inst) else {76447 .mul_add => |air_tag| if (use_old) try cg.airMulAdd(inst) else {
76422 const pl_op = air_datas[@intFromEnum(inst)].pl_op;76448 const pl_op = air_datas[@intFromEnum(inst)].pl_op;
76423 const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data;76449 const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data;
...@@ -94743,11 +94769,6 @@ fn airUnionInit(self: *CodeGen, inst: Air.Inst.Index) !void {...@@ -94743,11 +94769,6 @@ fn airUnionInit(self: *CodeGen, inst: Air.Inst.Index) !void {
94743 return self.finishAir(inst, result, .{ extra.init, .none, .none });94769 return self.finishAir(inst, result, .{ extra.init, .none, .none });
94744}94770}
9474594771
94746fn airPrefetch(self: *CodeGen, inst: Air.Inst.Index) !void {
94747 const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
94748 return self.finishAir(inst, .unreach, .{ prefetch.ptr, .none, .none });
94749}
94750
94751fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void {94772fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void {
94752 const pt = self.pt;94773 const pt = self.pt;
94753 const zcu = pt.zcu;94774 const zcu = pt.zcu;
src/arch/x86_64/Encoding.zig+27-15
...@@ -78,7 +78,7 @@ pub fn findByMnemonic(...@@ -78,7 +78,7 @@ pub fn findByMnemonic(
78 ),78 ),
79 .x86_64 => false,79 .x86_64 => false,
80 },80 },
81 inline .@"invpcid 64bit", .@"rdpid 64bit" => |tag| switch (target.cpu.arch) {81 inline .@"invpcid 64bit", .@"rdpid 64bit", .@"prefetchi 64bit" => |tag| switch (target.cpu.arch) {
82 else => unreachable,82 else => unreachable,
83 .x86 => false,83 .x86 => false,
84 .x86_64 => std.Target.x86.featureSetHas(84 .x86_64 => std.Target.x86.featureSetHas(
...@@ -86,6 +86,7 @@ pub fn findByMnemonic(...@@ -86,6 +86,7 @@ pub fn findByMnemonic(
86 @field(std.Target.x86.Feature, @tagName(tag)[0 .. @tagName(tag).len - " 64bit".len]),86 @field(std.Target.x86.Feature, @tagName(tag)[0 .. @tagName(tag).len - " 64bit".len]),
87 ),87 ),
88 },88 },
89 .prefetch => std.Target.x86.featureSetHasAny(target.cpu.features, .{ .sse, .prfchw, .prefetchi, .prefetchwt1 }),
89 inline else => |tag| has_features: {90 inline else => |tag| has_features: {
90 comptime var feature_it = std.mem.splitScalar(u8, @tagName(tag), ' ');91 comptime var feature_it = std.mem.splitScalar(u8, @tagName(tag), ' ');
91 comptime var features: []const std.Target.x86.Feature = &.{};92 comptime var features: []const std.Target.x86.Feature = &.{};
...@@ -375,6 +376,7 @@ pub const Mnemonic = enum {...@@ -375,6 +376,7 @@ pub const Mnemonic = enum {
375 orps,376 orps,
376 pextrw, pinsrw,377 pextrw, pinsrw,
377 pmaxsw, pmaxub, pminsw, pminub, pmovmskb,378 pmaxsw, pmaxub, pminsw, pminub, pmovmskb,
379 prefetchit0, prefetchit1, prefetchnta, prefetcht0, prefetcht1, prefetcht2, prefetchw, prefetchwt1,
378 shufps,380 shufps,
379 sqrtps, sqrtss,381 sqrtps, sqrtss,
380 stmxcsr,382 stmxcsr,
...@@ -562,8 +564,7 @@ pub const Op = enum {...@@ -562,8 +564,7 @@ pub const Op = enum {
562 r32_m8, r32_m16, r64_m16,564 r32_m8, r32_m16, r64_m16,
563 m8, m16, m32, m64, m80, m128, m256,565 m8, m16, m32, m64, m80, m128, m256,
564 rel8, rel16, rel32,566 rel8, rel16, rel32,
565 m,567 m, moffs, mrip8,
566 moffs,
567 sreg,568 sreg,
568 st0, st, mm, mm_m64,569 st0, st, mm, mm_m64,
569 xmm0, xmm, xmm_m8, xmm_m16, xmm_m32, xmm_m64, xmm_m128,570 xmm0, xmm, xmm_m8, xmm_m16, xmm_m32, xmm_m64, xmm_m128,
...@@ -617,7 +618,7 @@ pub const Op = enum {...@@ -617,7 +618,7 @@ pub const Op = enum {
617618
618 .mem => |mem| switch (mem) {619 .mem => |mem| switch (mem) {
619 .moffs => .moffs,620 .moffs => .moffs,
620 .sib, .rip => switch (mem.bitSize(target)) {621 .sib => switch (mem.bitSize(target)) {
621 0 => .m,622 0 => .m,
622 8 => .m8,623 8 => .m8,
623 16 => .m16,624 16 => .m16,
...@@ -628,6 +629,16 @@ pub const Op = enum {...@@ -628,6 +629,16 @@ pub const Op = enum {
628 256 => .m256,629 256 => .m256,
629 else => unreachable,630 else => unreachable,
630 },631 },
632 .rip => switch (mem.bitSize(target)) {
633 0, 8 => .mrip8,
634 16 => .m16,
635 32 => .m32,
636 64 => .m64,
637 80 => .m80,
638 128 => .m128,
639 256 => .m256,
640 else => unreachable,
641 },
631 },642 },
632643
633 .imm => |imm| switch (imm) {644 .imm => |imm| switch (imm) {
...@@ -680,7 +691,7 @@ pub const Op = enum {...@@ -680,7 +691,7 @@ pub const Op = enum {
680691
681 pub fn immBitSize(op: Op) u64 {692 pub fn immBitSize(op: Op) u64 {
682 return switch (op) {693 return switch (op) {
683 .none, .moffs, .m, .sreg => unreachable,694 .none, .m, .moffs, .mrip8, .sreg => unreachable,
684 .al, .cl, .dx, .rip, .eip, .ip, .r8, .rm8, .r32_m8 => unreachable,695 .al, .cl, .dx, .rip, .eip, .ip, .r8, .rm8, .r32_m8 => unreachable,
685 .ax, .r16, .rm16 => unreachable,696 .ax, .r16, .rm16 => unreachable,
686 .eax, .r32, .rm32, .r32_m16 => unreachable,697 .eax, .r32, .rm32, .r32_m16 => unreachable,
...@@ -700,7 +711,7 @@ pub const Op = enum {...@@ -700,7 +711,7 @@ pub const Op = enum {
700711
701 pub fn regBitSize(op: Op) u64 {712 pub fn regBitSize(op: Op) u64 {
702 return switch (op) {713 return switch (op) {
703 .none, .moffs, .m, .sreg => unreachable,714 .none, .m, .moffs, .mrip8, .sreg => unreachable,
704 .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,715 .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
705 .rel8, .rel16, .rel32 => unreachable,716 .rel8, .rel16, .rel32 => unreachable,
706 .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,717 .m8, .m16, .m32, .m64, .m80, .m128, .m256 => unreachable,
...@@ -716,13 +727,13 @@ pub const Op = enum {...@@ -716,13 +727,13 @@ pub const Op = enum {
716727
717 pub fn memBitSize(op: Op) u64 {728 pub fn memBitSize(op: Op) u64 {
718 return switch (op) {729 return switch (op) {
719 .none, .moffs, .m, .sreg => unreachable,730 .none, .m, .moffs, .sreg => unreachable,
720 .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,731 .unity, .imm8, .imm8s, .imm16, .imm16s, .imm32, .imm32s, .imm64 => unreachable,
721 .rel8, .rel16, .rel32 => unreachable,732 .rel8, .rel16, .rel32 => unreachable,
722 .al, .cl, .r8, .ax, .dx, .ip, .r16, .eax, .eip, .r32, .rax, .rip, .r64 => unreachable,733 .al, .cl, .r8, .ax, .dx, .ip, .r16, .eax, .eip, .r32, .rax, .rip, .r64 => unreachable,
723 .st0, .st, .mm, .xmm0, .xmm, .ymm => unreachable,734 .st0, .st, .mm, .xmm0, .xmm, .ymm => unreachable,
724 .cr, .dr => unreachable,735 .cr, .dr => unreachable,
725 .m8, .rm8, .r32_m8, .xmm_m8 => 8,736 .mrip8, .m8, .rm8, .r32_m8, .xmm_m8 => 8,
726 .m16, .rm16, .r32_m16, .r64_m16, .xmm_m16 => 16,737 .m16, .rm16, .r32_m16, .r64_m16, .xmm_m16 => 16,
727 .m32, .rm32, .xmm_m32 => 32,738 .m32, .rm32, .xmm_m32 => 32,
728 .m64, .rm64, .mm_m64, .xmm_m64 => 64,739 .m64, .rm64, .mm_m64, .xmm_m64 => 64,
...@@ -783,7 +794,7 @@ pub const Op = enum {...@@ -783,7 +794,7 @@ pub const Op = enum {
783 .rm8, .rm16, .rm32, .rm64,794 .rm8, .rm16, .rm32, .rm64,
784 .r32_m8, .r32_m16, .r64_m16,795 .r32_m8, .r32_m16, .r64_m16,
785 .m8, .m16, .m32, .m64, .m80, .m128, .m256,796 .m8, .m16, .m32, .m64, .m80, .m128, .m256,
786 .m,797 .m, .moffs, .mrip8,
787 .mm_m64,798 .mm_m64,
788 .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128,799 .xmm_m8, .xmm_m16, .xmm_m32, .xmm_m64, .xmm_m128,
789 .ymm_m256,800 .ymm_m256,
...@@ -821,11 +832,7 @@ pub const Op = enum {...@@ -821,11 +832,7 @@ pub const Op = enum {
821 /// Given an operand `op` checks if `target` is a subset for the purposes of the encoding.832 /// Given an operand `op` checks if `target` is a subset for the purposes of the encoding.
822 pub fn isSubset(op: Op, target: Op) bool {833 pub fn isSubset(op: Op, target: Op) bool {
823 switch (op) {834 switch (op) {
824 .moffs, .sreg => return op == target,835 .none, .m, .moffs, .sreg => return op == target,
825 .none => switch (target) {
826 .none => return true,
827 else => return false,
828 },
829 else => {836 else => {
830 if (op.isRegister() and target.isRegister()) {837 if (op.isRegister() and target.isRegister()) {
831 return switch (target.toReg()) {838 return switch (target.toReg()) {
...@@ -836,6 +843,7 @@ pub const Op = enum {...@@ -836,6 +843,7 @@ pub const Op = enum {
836 if (op.isMemory() and target.isMemory()) {843 if (op.isMemory() and target.isMemory()) {
837 switch (target) {844 switch (target) {
838 .m => return true,845 .m => return true,
846 .moffs, .mrip8 => return op == target,
839 else => return op.memBitSize() == target.memBitSize(),847 else => return op.memBitSize() == target.memBitSize(),
840 }848 }
841 }849 }
...@@ -962,6 +970,10 @@ pub const Feature = enum {...@@ -962,6 +970,10 @@ pub const Feature = enum {
962 @"pclmul avx",970 @"pclmul avx",
963 pku,971 pku,
964 popcnt,972 popcnt,
973 prefetch,
974 @"prefetchi 64bit",
975 prefetchwt1,
976 prfchw,
965 rdrnd,977 rdrnd,
966 rdseed,978 rdseed,
967 @"rdpid 32bit",979 @"rdpid 32bit",
...@@ -1002,7 +1014,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op...@@ -1002,7 +1014,7 @@ fn estimateInstructionLength(prefix: Prefix, encoding: Encoding, ops: []const Op
1002}1014}
10031015
1004const mnemonic_to_encodings_map = init: {1016const mnemonic_to_encodings_map = init: {
1005 @setEvalBranchQuota(5_700);1017 @setEvalBranchQuota(5_800);
1006 const mnemonic_count = @typeInfo(Mnemonic).@"enum".fields.len;1018 const mnemonic_count = @typeInfo(Mnemonic).@"enum".fields.len;
1007 var mnemonic_map: [mnemonic_count][]Data = @splat(&.{});1019 var mnemonic_map: [mnemonic_count][]Data = @splat(&.{});
1008 const encodings = @import("encodings.zig");1020 const encodings = @import("encodings.zig");
src/arch/x86_64/Mir.zig+14
...@@ -34,8 +34,18 @@ pub const Inst = struct {...@@ -34,8 +34,18 @@ pub const Inst = struct {
34 /// ___ 434 /// ___ 4
35 _4,35 _4,
3636
37 /// ___ With NTA Hint
38 _nta,
37 /// System Call ___39 /// System Call ___
38 sys_,40 sys_,
41 /// ___ With T0 Hint
42 _t0,
43 /// ___ With T1 Hint
44 _t1,
45 /// ___ With T2 Hint
46 _t2,
47 /// ___ With Intent to Write and T1 Hint
48 _wt1,
3949
40 /// ___ crement Shadow Stack Pointer Doubleword50 /// ___ crement Shadow Stack Pointer Doubleword
41 _csspd,51 _csspd,
...@@ -198,6 +208,7 @@ pub const Inst = struct {...@@ -198,6 +208,7 @@ pub const Inst = struct {
198 //_b,208 //_b,
199 /// ___ Word209 /// ___ Word
200 /// ___ For Writing210 /// ___ For Writing
211 /// ___ With Intent to Write
201 _w,212 _w,
202 /// ___ Doubleword213 /// ___ Doubleword
203 //_d,214 //_d,
...@@ -975,6 +986,9 @@ pub const Inst = struct {...@@ -975,6 +986,9 @@ pub const Inst = struct {
975 /// Move unaligned packed single-precision floating-point values986 /// Move unaligned packed single-precision floating-point values
976 /// Move unaligned packed double-precision floating-point values987 /// Move unaligned packed double-precision floating-point values
977 movu,988 movu,
989 /// Prefetch data into caches
990 /// Prefetch data into caches with intent to write
991 prefetch,
978 /// Packed interleave shuffle of quadruplets of single-precision floating-point values992 /// Packed interleave shuffle of quadruplets of single-precision floating-point values
979 /// Packed interleave shuffle of pairs of double-precision floating-point values993 /// Packed interleave shuffle of pairs of double-precision floating-point values
980 /// Shuffle packed doublewords994 /// Shuffle packed doublewords
src/arch/x86_64/encodings.zig+12
...@@ -1370,6 +1370,18 @@ pub const table = [_]Entry{...@@ -1370,6 +1370,18 @@ pub const table = [_]Entry{
1370 .{ .pmovmskb, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .none, .sse },1370 .{ .pmovmskb, .rm, &.{ .r32, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .none, .sse },
1371 .{ .pmovmskb, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .none, .sse },1371 .{ .pmovmskb, .rm, &.{ .r64, .xmm }, &.{ 0x66, 0x0f, 0xd7 }, 0, .none, .sse },
13721372
1373 .{ .prefetchit0, .m, &.{ .mrip8 }, &.{ 0x0f, 0x18 }, 7, .none, .@"prefetchi 64bit" },
1374 .{ .prefetchit1, .m, &.{ .mrip8 }, &.{ 0x0f, 0x18 }, 6, .none, .@"prefetchi 64bit" },
1375
1376 .{ .prefetchnta, .m, &.{ .m8 }, &.{ 0x0f, 0x18 }, 0, .none, .prefetch },
1377 .{ .prefetcht0, .m, &.{ .m8 }, &.{ 0x0f, 0x18 }, 1, .none, .prefetch },
1378 .{ .prefetcht1, .m, &.{ .m8 }, &.{ 0x0f, 0x18 }, 2, .none, .prefetch },
1379 .{ .prefetcht2, .m, &.{ .m8 }, &.{ 0x0f, 0x18 }, 3, .none, .prefetch },
1380
1381 .{ .prefetchw, .m, &.{ .m8 }, &.{ 0x0f, 0x0d }, 1, .none, .prfchw },
1382
1383 .{ .prefetchwt1, .m, &.{ .m8 }, &.{ 0x0f, 0x0d }, 2, .none, .prefetchwt1 },
1384
1373 .{ .shufps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x0f, 0xc6 }, 0, .none, .sse },1385 .{ .shufps, .rmi, &.{ .xmm, .xmm_m128, .imm8 }, &.{ 0x0f, 0xc6 }, 0, .none, .sse },
13741386
1375 .{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .none, .sse },1387 .{ .sqrtps, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x0f, 0x51 }, 0, .none, .sse },