1const std = @import("std");
2const builtin = @import("builtin");
3const arch = builtin.cpu.arch;
4const os = builtin.os.tag;
5const compiler_rt = @import("../compiler_rt.zig");
6const symbol = compiler_rt.symbol;
7
8// Ported from llvm-project d32170dbd5b0d54436537b6b75beaf44324e0c28
9
10// The compiler generates calls to __clear_cache() when creating
11// trampoline functions on the stack for use with nested functions.
12// It is expected to invalidate the instruction cache for the
13// specified range.
14
15comptime {
16 _ = &clear_cache;
17}
18
19fn clear_cache(start: usize, end: usize) callconv(.c) void {
20 const x86 = switch (arch) {
21 .x86, .x86_64 => true,
22 else => false,
23 };
24 const arm32 = switch (arch) {
25 .arm, .armeb, .thumb, .thumbeb => true,
26 else => false,
27 };
28 const arm64 = switch (arch) {
29 .aarch64, .aarch64_be => true,
30 else => false,
31 };
32 const loongarch = switch (arch) {
33 .loongarch32,
34 .loongarch64,
35 => true,
36 else => false,
37 };
38 const mips = switch (arch) {
39 .mips, .mipsel, .mips64, .mips64el => true,
40 else => false,
41 };
42 const riscv = arch.isRISCV();
43 const powerpc64 = switch (arch) {
44 .powerpc64, .powerpc64le => true,
45 else => false,
46 };
47 const sparc = switch (arch) {
48 .sparc, .sparc64 => true,
49 else => false,
50 };
51 const apple = switch (os) {
52 .ios, .maccatalyst, .macos, .watchos, .tvos, .visionos => true,
53 else => false,
54 };
55 if (x86) {
56 // Intel processors have a unified instruction and data cache
57 // so there is nothing to do
58 exportIt();
59 } else if (os == .windows and (arm32 or arm64)) {
60 // TODO
61 // FlushInstructionCache(GetCurrentProcess(), start, end - start);
62 // exportIt();
63 } else if (arm32 and !apple) {
64 switch (os) {
65 // FreeBSD and NetBSD should be changed to do direct syscalls here...
66 .freebsd, .netbsd, .openbsd => {
67 var arg: arm_sync_icache_args = .{
68 .addr = start,
69 .size = end - start,
70 };
71 const result = sysarch(ARM_SYNC_ICACHE, &arg);
72 std.debug.assert(result == 0);
73 exportIt();
74 },
75 .linux => {
76 const result = std.os.linux.syscall3(.cacheflush, start, end, 0);
77 std.debug.assert(result == 0);
78 exportIt();
79 },
80 else => {},
81 }
82 } else if (os == .linux and mips) {
83 const result = std.os.linux.syscall3(.cacheflush, start, end - start, ICACHE | DCACHE);
84 std.debug.assert(result == 0);
85 exportIt();
86 } else if (os == .netbsd and mips) {
87 // Replace with https://github.com/ziglang/zig/issues/23904 in the future.
88 const cfa: mips_cacheflush_args = .{
89 .addr = start,
90 .size = end - start,
91 .which = ICACHE | DCACHE,
92 };
93 asm volatile ("syscall"
94 :
95 : [_] "{$2}" (165), // nr = SYS_sysarch
96 [_] "{$4}" (MIPS_CACHEFLUSH), // op
97 [_] "{$5}" (&cfa), // args = &cfa
98 : .{ .r1 = true, .r2 = true, .r3 = true, .r4 = true, .r5 = true, .r6 = true, .r7 = true, .r8 = true, .r9 = true, .r10 = true, .r11 = true, .r12 = true, .r13 = true, .r14 = true, .r15 = true, .r24 = true, .r25 = true, .hi = true, .lo = true, .memory = true });
99 exportIt();
100 } else if (mips and os == .openbsd) {
101 const cfa: mips_cacheflush_args = .{
102 .addr = start,
103 .size = end - start,
104 .which = ICACHE | DCACHE,
105 };
106 const result = sysarch(MIPS_CACHEFLUSH, &cfa);
107 std.debug.assert(result == 0);
108 exportIt();
109 } else if (os == .linux and riscv) {
110 const result = std.os.linux.syscall3(.riscv_flush_icache, start, end - start, 0);
111 std.debug.assert(result == 0);
112 exportIt();
113 } else if (arm64 and !apple) {
114 // Get Cache Type Info.
115 // TODO memoize this?
116 const ctr_el0 = asm volatile ("mrs %[ctr_el0], ctr_el0"
117 : [ctr_el0] "=r" (-> u64),
118 );
119 // The DC and IC instructions must use 64-bit registers so we don't use
120 // uintptr_t in case this runs in an IPL32 environment.
121 var addr: u64 = undefined;
122 // If CTR_EL0.IDC is set, data cache cleaning to the point of unification
123 // is not required for instruction to data coherence.
124 if (((ctr_el0 >> 28) & 0x1) == 0x0) {
125 const dcache_line_size = @as(usize, 4) << @intCast((ctr_el0 >> 16) & 15);
126 addr = start & ~(dcache_line_size - 1);
127 while (addr < end) : (addr += dcache_line_size) {
128 asm volatile ("dc cvau, %[addr]"
129 :
130 : [addr] "r" (addr),
131 );
132 }
133 }
134 asm volatile ("dsb ish");
135 // If CTR_EL0.DIC is set, instruction cache invalidation to the point of
136 // unification is not required for instruction to data coherence.
137 if (((ctr_el0 >> 29) & 0x1) == 0x0) {
138 const icache_line_size = @as(usize, 4) << @intCast((ctr_el0 >> 0) & 15);
139 addr = start & ~(icache_line_size - 1);
140 while (addr < end) : (addr += icache_line_size) {
141 asm volatile ("ic ivau, %[addr]"
142 :
143 : [addr] "r" (addr),
144 );
145 }
146 }
147 asm volatile ("isb sy");
148 exportIt();
149 } else if (powerpc64) {
150 // TODO
151 //const size_t line_size = 32;
152 //const size_t len = (uintptr_t)end - (uintptr_t)start;
153 //
154 //const uintptr_t mask = ~(line_size - 1);
155 //const uintptr_t start_line = ((uintptr_t)start) & mask;
156 //const uintptr_t end_line = ((uintptr_t)start + len + line_size - 1) & mask;
157 //
158 //for (uintptr_t line = start_line; line < end_line; line += line_size)
159 // __asm__ volatile("dcbf 0, %0" : : "r"(line));
160 //__asm__ volatile("sync");
161 //
162 //for (uintptr_t line = start_line; line < end_line; line += line_size)
163 // __asm__ volatile("icbi 0, %0" : : "r"(line));
164 //__asm__ volatile("isync");
165 // exportIt();
166 } else if (sparc) {
167 // TODO
168 //const size_t dword_size = 8;
169 //const size_t len = (uintptr_t)end - (uintptr_t)start;
170 //
171 //const uintptr_t mask = ~(dword_size - 1);
172 //const uintptr_t start_dword = ((uintptr_t)start) & mask;
173 //const uintptr_t end_dword = ((uintptr_t)start + len + dword_size - 1) & mask;
174 //
175 //for (uintptr_t dword = start_dword; dword < end_dword; dword += dword_size)
176 // __asm__ volatile("flush %0" : : "r"(dword));
177 // exportIt();
178 } else if (apple) {
179 // On Darwin, sys_icache_invalidate() provides this functionality
180 sys_icache_invalidate(start, end - start);
181 exportIt();
182 } else if (os == .linux and loongarch) {
183 // See: https://github.com/llvm/llvm-project/blob/cf54cae26b65fc3201eff7200ffb9b0c9e8f9a13/compiler-rt/lib/builtins/clear_cache.c#L94-L95
184 asm volatile ("ibar 0");
185 exportIt();
186 }
187
188 std.valgrind.discardTranslations(@as([*]u8, @ptrFromInt(start))[0 .. end - start]);
189}
190
191fn exportIt() void {
192 symbol(&clear_cache, "__clear_cache");
193}
194
195// MIPS-only
196const ICACHE = 0x1;
197const DCACHE = 0x2;
198
199// Darwin-only
200extern fn sys_icache_invalidate(start: usize, len: usize) void;
201
202// BSD-only
203const ARM_SYNC_ICACHE = 0;
204const arm_sync_icache_args = extern struct {
205 addr: usize,
206 size: usize,
207};
208
209const MIPS_CACHEFLUSH = 0;
210const mips_cacheflush_args = extern struct {
211 addr: usize,
212 size: usize,
213 which: c_uint,
214};
215
216extern fn sysarch(op: c_uint, args: *const anyopaque) c_int;