1const std = @import("std");
2const Io = std.Io;
3const Allocator = std.mem.Allocator;
4const Ast = std.zig.Ast;
5
6const usage =
7 \\Usage: unskip_spirv_tests <zig> <test_runner> <test_cmd> [test/behavior/*.zig ...]
8 \\
9 \\For example:
10 \\zig run tools/unskip_spirv_tests.zig -- zig-out/bin/zig \
11 \\ ../zig-spirv-test-executor/src/test_runner.zig \
12 \\ ../zig-spirv-test-executor/zig-out/bin/zig-spirv-test-executor [files...]
13 \\
14 \\Runs every behavior test that is skipped on stage2_spirv in isolation and
15 \\removes the skip if the test passes.
16 \\
17;
18
19const Test = struct {
20 body: Ast.Node.Index,
21 name: []const u8,
22 skip: ?Ast.Node.Index,
23};
24
25pub fn main(init: std.process.Init) !void {
26 const io = init.io;
27 const arena = init.arena.allocator();
28
29 var args = try init.minimal.args.iterateAllocator(arena);
30 _ = args.skip();
31 const zig = args.next() orelse std.process.fatal(usage, .{});
32 const test_runner = args.next() orelse std.process.fatal(usage, .{});
33 const test_cmd = args.next() orelse std.process.fatal(usage, .{});
34
35 var files: std.ArrayList([]const u8) = .empty;
36 while (args.next()) |path| try files.append(arena, path);
37
38 if (files.items.len == 0) {
39 var dir = try Io.Dir.openDir(.cwd(), io, "test/behavior", .{ .iterate = true });
40 defer dir.close(io);
41
42 var it = dir.iterate();
43 while (try it.next(io)) |entry| {
44 if (entry.kind != .file or !std.mem.endsWith(u8, entry.name, ".zig")) continue;
45 const path = try std.fs.path.join(arena, &.{ "test/behavior", entry.name });
46 const source = try Io.Dir.readFileAlloc(.cwd(), io, path, arena, .unlimited);
47 if (std.mem.find(u8, source, "stage2_spirv") != null) {
48 try files.append(arena, path);
49 }
50 }
51 }
52
53 var stdout_buffer: [4096]u8 = undefined;
54 var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer);
55 const stdout = &stdout_writer.interface;
56
57 const progress = std.Progress.start(io, .{ .estimated_total_items = files.items.len });
58 const cache_dir = init.environ_map.get("ZIG_LOCAL_CACHE_DIR") orelse ".zig-cache";
59 const variant_dir = try std.fs.path.join(arena, &.{ cache_dir, "unskip" });
60 try Io.Dir.createDirPath(.cwd(), io, variant_dir);
61
62 var failures: std.array_hash_map.String(std.ArrayList([]const u8)) = .empty;
63 var total_removed: usize = 0;
64 var files_changed: usize = 0;
65 for (files.items) |path| {
66 const basename = std.fs.path.basename(path);
67 const source = try Io.Dir.readFileAlloc(.cwd(), io, path, arena, .unlimited);
68 const ast = try Ast.parse(arena, try arena.dupeSentinel(u8, source, 0), .{});
69 if (ast.errors.len != 0) std.process.fatal("{s}: parse error", .{path});
70
71 var tests: std.ArrayList(Test) = .empty;
72 var skips: usize = 0;
73 for (ast.rootDecls()) |decl| {
74 if (ast.nodeTag(decl) != .test_decl) continue;
75 const name_token, const body = ast.nodeData(decl).opt_token_and_node;
76 const token = name_token.unwrap() orelse continue;
77 const name = if (ast.tokenTag(token) == .string_literal)
78 try std.zig.string_literal.parseAlloc(arena, ast.tokenSlice(token))
79 else
80 ast.tokenSlice(token);
81
82 var buffer: [2]Ast.Node.Index = undefined;
83 const skip: ?Ast.Node.Index = for (ast.blockStatements(&buffer, body) orelse &.{}) |statement| {
84 const if_full = ast.fullIf(statement) orelse continue;
85 if (ast.nodeTag(if_full.ast.cond_expr) != .equal_equal) continue;
86 _, const rhs = ast.nodeData(if_full.ast.cond_expr).node_and_node;
87 if (!std.mem.eql(u8, ast.getNodeSource(rhs), ".stage2_spirv")) continue;
88 if (ast.nodeTag(if_full.ast.then_expr) != .@"return") continue;
89 const value = ast.nodeData(if_full.ast.then_expr).opt_node.unwrap() orelse continue;
90 if (!std.mem.eql(u8, ast.getNodeSource(value), "error.SkipZigTest")) continue;
91 break statement;
92 } else null;
93 skips += @intFromBool(skip != null);
94
95 try tests.append(arena, .{ .body = body, .name = name, .skip = skip });
96 }
97
98 if (skips == 0) {
99 progress.completeOne();
100 continue;
101 }
102
103 const file_node = progress.start(basename, skips);
104 defer file_node.end();
105 const variant_path = try std.fs.path.join(arena, &.{ variant_dir, basename });
106
107 var removed: Ast.Render.Fixups = .{};
108 for (tests.items) |candidate| {
109 const skip = candidate.skip orelse continue;
110 const test_node = file_node.start(candidate.name, 0);
111 defer test_node.end();
112
113 var fixups: Ast.Render.Fixups = .{};
114 try fixups.omit_nodes.put(arena, skip, {});
115 for (tests.items) |other| {
116 if (other.body == candidate.body) continue;
117 try fixups.replace_nodes_with_string.put(arena, other.body, "{ return error.SkipZigTest; }");
118 }
119
120 var variant: Io.Writer.Allocating = .init(arena);
121 try ast.render(arena, &variant.writer, fixups);
122 try Io.Dir.writeFile(.cwd(), io, .{ .sub_path = variant_path, .data = variant.written() });
123
124 const signature: []const u8 = signature: {
125 const result = std.process.run(arena, io, .{
126 .argv = &.{
127 zig, "test", variant_path,
128 "--test-runner", test_runner, "-target",
129 "spirv64-vulkan", "-mcpu", "vulkan_v1_2+variable_pointers+int64",
130 "-fno-llvm", "--zig-lib-dir", "lib",
131 "--test-cmd", test_cmd, "--test-cmd-bin",
132 },
133 .timeout = .{ .duration = .{ .raw = .fromSeconds(120), .clock = .awake } },
134 .progress_node = test_node,
135 }) catch |err| switch (err) {
136 error.Timeout => break :signature "timeout",
137 else => |e| return e,
138 };
139
140 if (result.term.success()) {
141 try removed.omit_nodes.put(arena, skip, {});
142 std.debug.print("{s}: removed skip for {s}\n", .{ basename, candidate.name });
143 continue;
144 }
145
146 const output = try std.mem.concat(arena, u8, &.{ result.stdout, result.stderr });
147 var lines = std.mem.splitScalar(u8, output, '\n');
148 while (lines.next()) |line| {
149 const rest = std.mem.cutPrefix(u8, line, "error: line ") orelse continue;
150 const colon = std.mem.findScalar(u8, rest, ':') orelse continue;
151 if (!allDigits(rest[0..colon])) continue;
152 break :signature try normalize(arena, "spirv-val: ", rest[colon + 1 ..]);
153 }
154 lines.reset();
155
156 while (lines.next()) |line| {
157 const rest = std.mem.cutPrefix(u8, line, "thread ") orelse continue;
158 const panic = std.mem.find(u8, rest, " panic: ") orelse continue;
159 if (std.mem.findScalar(u8, rest[0..panic], ' ') != null) continue;
160 break :signature try normalize(arena, "panic: ", rest[panic + " panic: ".len ..]);
161 }
162 lines.reset();
163
164 while (lines.next()) |line| {
165 const marker = std.mem.find(u8, line, ": error: ") orelse continue;
166 const location = line[0..marker];
167 const line_colon = std.mem.findScalarLast(u8, location, ':') orelse continue;
168 const file_colon = std.mem.findScalarLast(u8, location[0..line_colon], ':') orelse continue;
169 if (std.mem.findScalar(u8, location[0..file_colon], ' ') != null) continue;
170 if (!allDigits(location[file_colon + 1 .. line_colon])) continue;
171 if (!allDigits(location[line_colon + 1 ..])) continue;
172 const message = line[marker + ": error: ".len ..];
173 break :signature try normalize(arena, "compile error: ", message);
174 }
175 lines.reset();
176
177 while (lines.next()) |line| {
178 const rest = std.mem.cutPrefix(u8, line, "error: ") orelse continue;
179 break :signature try normalize(arena, "error: ", rest);
180 }
181
182 if (std.mem.trim(u8, output, &std.ascii.whitespace).len > 0) {
183 break :signature "unknown failure (no error: line found)";
184 }
185 break :signature "no output";
186 };
187
188 const gop = try failures.getOrPut(arena, signature);
189 if (!gop.found_existing) gop.value_ptr.* = .empty;
190 const failed_test = try std.fmt.allocPrint(arena, "{s} :: {s}", .{ basename, candidate.name });
191 try gop.value_ptr.append(arena, failed_test);
192 }
193
194 std.debug.print("{s}: {d}/{d} skips removed\n", .{ basename, removed.omit_nodes.count(), skips });
195 Io.Dir.deleteFile(.cwd(), io, variant_path) catch {};
196 if (removed.omit_nodes.count() == 0) continue;
197
198 var final: Io.Writer.Allocating = .init(arena);
199 try ast.render(arena, &final.writer, removed);
200 const rendered = try Ast.parse(arena, try arena.dupeSentinel(u8, final.written(), 0), .{});
201 const formatted = try rendered.renderAlloc(arena);
202 try Io.Dir.writeFile(.cwd(), io, .{ .sub_path = path, .data = formatted });
203 total_removed += removed.omit_nodes.count();
204 files_changed += 1;
205 }
206
207 progress.end();
208
209 try stdout.print("\nDone: removed {d} spirv skips across {d} files\n", .{ total_removed, files_changed });
210 if (failures.count() > 0) {
211 var total_failures: usize = 0;
212 for (failures.values()) |failed_tests| total_failures += failed_tests.items.len;
213 try stdout.print("\nMost common blockers ({d} failing attempts, {d} unique signatures):\n", .{
214 total_failures,
215 failures.count(),
216 });
217
218 const Ctx = struct {
219 failed_tests: []const std.ArrayList([]const u8),
220
221 pub fn lessThan(ctx: @This(), a: usize, b: usize) bool {
222 return ctx.failed_tests[a].items.len > ctx.failed_tests[b].items.len;
223 }
224 };
225 failures.sort(Ctx{ .failed_tests = failures.values() });
226
227 for (failures.keys()[0..@min(failures.count(), 30)], failures.values()) |signature, failed_tests| {
228 const count = failed_tests.items.len;
229 const percent = @as(f64, @floatFromInt(count)) * 100 / @as(f64, @floatFromInt(total_failures));
230 try stdout.print("{d:4} ({d:5.1}%) {s}\n", .{ count, percent, signature });
231 for (failed_tests.items) |failed_test| try stdout.print(" {s}\n", .{failed_test});
232 }
233 }
234 try stdout.flush();
235}
236
237fn allDigits(text: []const u8) bool {
238 if (text.len == 0) return false;
239 for (text) |c| if (!std.ascii.isDigit(c)) return false;
240 return true;
241}
242
243fn normalize(arena: Allocator, prefix: []const u8, message: []const u8) ![]u8 {
244 const source = try arena.dupeSentinel(u8, std.mem.trim(u8, message, &std.ascii.whitespace), 0);
245 var out: std.ArrayList(u8) = .empty;
246 try out.appendSlice(arena, prefix);
247 var tokenizer: std.zig.Tokenizer = .init(source);
248 var end: usize = 0;
249 while (true) {
250 const token = tokenizer.next();
251 if (token.tag == .eof) break;
252 try out.appendSlice(arena, source[end..token.loc.start]);
253 end = token.loc.end;
254 const text = source[token.loc.start..token.loc.end];
255 try out.appendSlice(arena, switch (token.tag) {
256 .number_literal => "<n>",
257 .char_literal, .string_literal => "<str>",
258 .identifier => if (token.loc.start > 0 and source[token.loc.start - 1] == '%') "<id>" else text,
259 else => text,
260 });
261 }
262 try out.appendSlice(arena, source[end..]);
263 return out.items[0..@min(out.items.len, prefix.len + 200)];
264}