authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2026-09-14 02:10:39-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2026-09-26 19:08:50-04:00
log207c883bfbdc3099997087869234d6e109bdb0cc
tree1634f621d3ab09fbf3a76d7b6828379789b20b21
parentf63e8e802f5dc2aebe888c0051f685cab6b2571e

runner: implement skip commands

3 files changed, 153 insertions(+), 79 deletions(-)

test/src/runner.zig+94-46
...@@ -8,8 +8,24 @@ const Args = struct {...@@ -8,8 +8,24 @@ const Args = struct {
8 zig_file: ?std.Io.File,8 zig_file: ?std.Io.File,
9 lib_dir: ?std.Io.Dir,9 lib_dir: ?std.Io.Dir,
10 src_dir: ?std.Io.Dir,10 src_dir: ?std.Io.Dir,
11 targets: std.ArrayList(u32),
12 target_bytes: std.ArrayList(u8),
13
14 fn deinit(args: *Args, gpa: std.mem.Allocator) void {
15 args.targets.deinit(gpa);
16 args.target_bytes.deinit(gpa);
17 }
18
19 fn addTarget(args: *Args, gpa: std.mem.Allocator, target: []const u8) std.mem.Allocator.Error!void {
20 try args.target_bytes.ensureUnusedCapacity(gpa, target.len + 1);
21 try args.targets.append(gpa, @intCast(args.target_bytes.items.len));
22 args.target_bytes.appendSliceAssumeCapacity(target);
23 args.target_bytes.appendAssumeCapacity(0);
24 }
11};25};
12pub fn main(init: std.process.Init) !void {26pub fn main(init: std.process.Init) !void {
27 const arena = init.arena.allocator();
28
13 var stdin_buffer: [512]u8 = undefined;29 var stdin_buffer: [512]u8 = undefined;
14 var stdout_buffer: [512]u8 = undefined;30 var stdout_buffer: [512]u8 = undefined;
15 var stderr_buffer: [512]u8 = undefined;31 var stderr_buffer: [512]u8 = undefined;
...@@ -20,14 +36,19 @@ pub fn main(init: std.process.Init) !void {...@@ -20,14 +36,19 @@ pub fn main(init: std.process.Init) !void {
20 .in = &stdin.interface,36 .in = &stdin.interface,
21 .out = &stdout.interface,37 .out = &stdout.interface,
22 };38 };
23 try server.serveStringMessage(.zig_version, @import("builtin").zig_version_string);39
24 var args: Args = .{40 var args: Args = .{
25 .@"test" = null,41 .@"test" = null,
26 .manifest_file = null,42 .manifest_file = null,
27 .zig_file = null,43 .zig_file = null,
28 .lib_dir = null,44 .lib_dir = null,
29 .src_dir = null,45 .src_dir = null,
46 .targets = .empty,
47 .target_bytes = .empty,
30 };48 };
49 defer args.deinit(init.gpa);
50
51 try server.serveStringMessage(.zig_version, @import("builtin").zig_version_string);
31 while (true) {52 while (true) {
32 const hdr = try server.receiveMessage();53 const hdr = try server.receiveMessage();
33 switch (hdr.tag) {54 switch (hdr.tag) {
...@@ -35,13 +56,11 @@ pub fn main(init: std.process.Init) !void {...@@ -35,13 +56,11 @@ pub fn main(init: std.process.Init) !void {
35 std.debug.print("unsupported message: {t}\n", .{hdr.tag});56 std.debug.print("unsupported message: {t}\n", .{hdr.tag});
36 std.process.exit(1);57 std.process.exit(1);
37 },58 },
38 .exit => {59 .exit => std.process.exit(0),
39 std.process.exit(0);
40 },
41 .args => {60 .args => {
42 const args_body = try init.arena.allocator().alloc(u8, hdr.bytes_len);61 const args_body = try arena.alloc(u8, hdr.bytes_len);
43 try server.in.readSliceAll(args_body);62 try server.in.readSliceAll(args_body);
44 var state: enum { positional, zig, lib, src } = .positional;63 var state: enum { positional, zig, lib, src, target } = .positional;
45 var input_dir: std.zig.Server.Message.InputDir = .cwd;64 var input_dir: std.zig.Server.Message.InputDir = .cwd;
46 var args_body_offset: usize = 0;65 var args_body_offset: usize = 0;
47 while (args_body.len - args_body_offset > 0) {66 while (args_body.len - args_body_offset > 0) {
...@@ -53,7 +72,18 @@ pub fn main(init: std.process.Init) !void {...@@ -53,7 +72,18 @@ pub fn main(init: std.process.Init) !void {
53 const end = std.mem.findScalarPos(u8, args_body, args_body_offset, 0).?;72 const end = std.mem.findScalarPos(u8, args_body, args_body_offset, 0).?;
54 const string = args_body[args_body_offset..end];73 const string = args_body[args_body_offset..end];
55 args_body_offset = end + 1;74 args_body_offset = end + 1;
56 _ = string;75 switch (state) {
76 .positional => if (std.mem.eql(u8, string, "--target")) {
77 state = .target;
78 continue;
79 } else if (std.mem.cutPrefix(u8, string, "--target=")) |target| {
80 try args.addTarget(init.gpa, target);
81 },
82 .zig => unreachable,
83 .lib => unreachable,
84 .src => unreachable,
85 .target => try args.addTarget(init.gpa, string),
86 }
57 },87 },
58 .prefix => {88 .prefix => {
59 const end = std.mem.findScalarPos(u8, args_body, args_body_offset, 0).?;89 const end = std.mem.findScalarPos(u8, args_body, args_body_offset, 0).?;
...@@ -109,6 +139,7 @@ pub fn main(init: std.process.Init) !void {...@@ -109,6 +139,7 @@ pub fn main(init: std.process.Init) !void {
109 std.debug.assert(args.src_dir == null);139 std.debug.assert(args.src_dir == null);
110 args.src_dir = dir;140 args.src_dir = dir;
111 },141 },
142 .target => unreachable,
112 }143 }
113 },144 },
114 .input_file, .input_file_content, .output_file => {145 .input_file, .input_file_content, .output_file => {
...@@ -129,22 +160,30 @@ pub fn main(init: std.process.Init) !void {...@@ -129,22 +160,30 @@ pub fn main(init: std.process.Init) !void {
129 std.debug.assert(args.zig_file == null);160 std.debug.assert(args.zig_file == null);
130 args.zig_file = file;161 args.zig_file = file;
131 },162 },
132 .lib, .src => unreachable,163 .lib => unreachable,
164 .src => unreachable,
165 .target => unreachable,
133 }166 }
134 },167 },
135 }168 }
136 state = .positional;169 state = .positional;
137 }170 }
138 },171 },
139 .query_test_metadata => try server.serveTestMetadata(.{172 .query_test_metadata => {
140 .names = &.{0},173 const expected_panic_msgs = try arena.alloc(u32, args.targets.items.len);
141 .expected_panic_msgs = &.{0},174 @memset(expected_panic_msgs, 0);
142 .string_bytes = "runner\x00",175 try server.serveTestMetadata(.{
143 }),176 .names = args.targets.items,
177 .expected_panic_msgs = expected_panic_msgs,
178 .string_bytes = args.target_bytes.items,
179 });
180 },
144 .run_test => {181 .run_test => {
145 const test_index = try server.receiveBody_u32();182 const test_index = try server.receiveBody_u32();
183 const target =
184 std.mem.sliceTo(args.target_bytes.items[args.targets.items[test_index]..], 0);
146 try server.serveBodylessMessage(.test_started);185 try server.serveBodylessMessage(.test_started);
147 const test_result = runTest(init.io, &stderr.interface, &server, &args);186 const test_result = runTest(init.io, &stderr.interface, &server, &args, target);
148 try stderr.flush();187 try stderr.flush();
149 try server.serveTestResults(.{188 try server.serveTestResults(.{
150 .index = test_index,189 .index = test_index,
...@@ -170,11 +209,13 @@ fn runTest(...@@ -170,11 +209,13 @@ fn runTest(
170 stderr: *std.Io.Writer,209 stderr: *std.Io.Writer,
171 server: *std.zig.Server,210 server: *std.zig.Server,
172 args: *const Args,211 args: *const Args,
173) anyerror!void {212 target: []const u8,
213) !void {
174 const manifest_file = args.manifest_file orelse return error.MissingManifestArg;214 const manifest_file = args.manifest_file orelse return error.MissingManifestArg;
175 const src_dir = args.src_dir orelse return error.MissingSrcDir;215 const src_dir = args.src_dir orelse return error.MissingSrcDir;
176 var manifest_buffer: [512]u8 = undefined;216 var manifest_buffer: [512]u8 = undefined;
177 var manifest_fr = manifest_file.reader(io, &manifest_buffer);217 var manifest_fr = manifest_file.reader(io, &manifest_buffer);
218 var allow_skip = true;
178 var skip_delimiter = false;219 var skip_delimiter = false;
179 var update_mtime = std.Io.Clock.real.now(io);220 var update_mtime = std.Io.Clock.real.now(io);
180 var update_num: usize = 0;221 var update_num: usize = 0;
...@@ -188,7 +229,7 @@ fn runTest(...@@ -188,7 +229,7 @@ fn runTest(
188 continue;229 continue;
189 }230 }
190 const cmd = std.meta.stringToEnum(231 const cmd = std.meta.stringToEnum(
191 enum { obj, exe, lib, write, delete, update, check },232 enum { skip, obj, exe, lib, write, delete, update, check },
192 cmd_str,233 cmd_str,
193 ) orelse return error.UnknownCommand;234 ) orelse return error.UnknownCommand;
194 var maybe_arg = line_it.next();235 var maybe_arg = line_it.next();
...@@ -198,38 +239,40 @@ fn runTest(...@@ -198,38 +239,40 @@ fn runTest(
198 dr: DelimitedReader,239 dr: DelimitedReader,
199 fr: std.Io.File.Reader,240 fr: std.Io.File.Reader,
200 } = undefined;241 } = undefined;
201 const contents_r: *std.Io.Reader = contents_r: {242 const contents_r: *std.Io.Reader = contents_r: switch (cmd) {
202 switch (cmd) {243 .skip, .delete => {
203 else => {},244 if (line_it.next()) |_| return error.UnexpectedArg;
204 .delete => break :contents_r .ending,245 break :contents_r .ending;
205 }246 },
206247 else => {
207 const contents_path = contents_path: {248 const contents_path = contents_path: {
208 const line_rest = line_it.rest();249 const line_rest = line_it.rest();
209 if (line_rest.len > 0) break :contents_path line_rest;250 if (line_rest.len > 0) break :contents_path line_rest;
210 const contents_path = maybe_arg orelse break :contents_r .ending;251 const contents_path = maybe_arg orelse break :contents_r .ending;
211 maybe_arg = null;252 maybe_arg = null;
212 break :contents_path contents_path;253 break :contents_path contents_path;
213 };254 };
214 if (std.mem.eql(u8, contents_path, "{")) {255 if (std.mem.eql(u8, contents_path, "{")) {
215 contents_impl = .{ .dr = .init(&manifest_fr.interface, "#}", &.{}) };256 contents_impl = .{ .dr = .init(&manifest_fr.interface, "#}", &.{}) };
216 skip_delimiter = true;257 skip_delimiter = true;
217 break :contents_r &contents_impl.dr.interface;258 break :contents_r &contents_impl.dr.interface;
218 }259 }
219260
220 const @"test" = args.@"test" orelse return error.ContentsFileInFileTest;261 const @"test" = args.@"test" orelse return error.ContentsFileInFileTest;
221 try server.serveMessageHeader(.{262 try server.serveMessageHeader(.{
222 .tag = .discovered_inputs,263 .tag = .discovered_inputs,
223 .bytes_len = @intCast(@sizeOf(std.zig.Server.Message.InputDir) + contents_path.len + 1),264 .bytes_len = @intCast(@sizeOf(std.zig.Server.Message.InputDir) +
224 });265 contents_path.len + 1),
225 try server.out.writeInt(u32, @backingInt(@"test".input_dir), .little);266 });
226 try server.out.writeAll(contents_path);267 try server.out.writeInt(u32, @backingInt(@"test".input_dir), .little);
227 try server.out.writeByte(0);268 try server.out.writeAll(contents_path);
228 try server.out.flush();269 try server.out.writeByte(0);
270 try server.out.flush();
229271
230 contents_file = try @"test".dir.openFile(io, contents_path, .{});272 contents_file = try @"test".dir.openFile(io, contents_path, .{});
231 contents_impl = .{ .fr = contents_file.?.reader(io, &.{}) };273 contents_impl = .{ .fr = contents_file.?.reader(io, &.{}) };
232 break :contents_r &contents_impl.fr.interface;274 break :contents_r &contents_impl.fr.interface;
275 },
233 };276 };
234 switch (cmd) {277 switch (cmd) {
235 else => {278 else => {
...@@ -237,6 +280,10 @@ fn runTest(...@@ -237,6 +280,10 @@ fn runTest(
237 _ = try contents_r.streamRemaining(stderr);280 _ = try contents_r.streamRemaining(stderr);
238 try stderr.writeAll("\n}\n");281 try stderr.writeAll("\n}\n");
239 },282 },
283 .skip => if (!allow_skip)
284 return error.NonInitialSkipCommand // #skip must appear first
285 else if (std.mem.eql(u8, target, maybe_arg orelse return error.MissingArg))
286 return error.SkipZigTest,
240 .write => {287 .write => {
241 const file = try src_dir.createFile(io, maybe_arg orelse return error.MissingArg, .{});288 const file = try src_dir.createFile(io, maybe_arg orelse return error.MissingArg, .{});
242 defer file.close(io);289 defer file.close(io);
...@@ -262,6 +309,7 @@ fn runTest(...@@ -262,6 +309,7 @@ fn runTest(
262 },309 },
263 }310 }
264 std.debug.assert(try contents_r.discardRemaining() == 0);311 std.debug.assert(try contents_r.discardRemaining() == 0);
312 if (cmd != .skip) allow_skip = false;
265 } else |err| switch (err) {313 } else |err| switch (err) {
266 else => |e| return e,314 else => |e| return e,
267 error.EndOfStream => if (skip_delimiter) return error.MissingDelimiter,315 error.EndOfStream => if (skip_delimiter) return error.MissingDelimiter,
test/tests.zig+3-3
...@@ -2214,9 +2214,9 @@ const IncrementalTarget = struct {...@@ -2214,9 +2214,9 @@ const IncrementalTarget = struct {
22142214
2215/// These are passed to `incr-check` as `<target>-<backend>` strings.2215/// These are passed to `incr-check` as `<target>-<backend>` strings.
2216///2216///
2217/// If only one specific test is failing on a target, instead of entirely disabling the target here,2217/// If only one specific test is failing on a target, instead of entirely disabling the target here, you
2218/// you can skip the target for that specific test only by adding a line like this to the manifest:2218/// can skip the target for that specific test only by adding a line like this to top of the manifest:
2219/// #skip_target=x86_64-linux-selfhosted2219/// #skip x86_64-linux-selfhosted
2220const incremental_targets = &[_]IncrementalTarget{2220const incremental_targets = &[_]IncrementalTarget{
2221 // Avoid adding more CBE or LLVM targets without good reason: they're a lot slower than others2221 // Avoid adding more CBE or LLVM targets without good reason: they're a lot slower than others
2222 // to run due to the output (C source code or LLVM IR) being built non-incrementally (by Clang2222 // to run due to the output (C source code or LLVM IR) being built non-incrementally (by Clang
tools/runner-check.zig+56-30
...@@ -5,6 +5,8 @@ pub fn main(init: std.process.Init) !void {...@@ -5,6 +5,8 @@ pub fn main(init: std.process.Init) !void {
5 var test_path_arg: ?[]const u8 = null;5 var test_path_arg: ?[]const u8 = null;
6 var zig_path_arg: ?[]const u8 = null;6 var zig_path_arg: ?[]const u8 = null;
7 var lib_path_arg: ?[]const u8 = null;7 var lib_path_arg: ?[]const u8 = null;
8 var target_args: std.ArrayList([]const u8) = .empty;
9 defer target_args.deinit(init.gpa);
8 var keep_src = false;10 var keep_src = false;
911
10 var arg_it = try init.minimal.args.iterateAllocator(arena);12 var arg_it = try init.minimal.args.iterateAllocator(arena);
...@@ -18,6 +20,13 @@ pub fn main(init: std.process.Init) !void {...@@ -18,6 +20,13 @@ pub fn main(init: std.process.Init) !void {
18 lib_path_arg = arg_it.next() orelse fail("missing arg after '{s}'", .{arg});20 lib_path_arg = arg_it.next() orelse fail("missing arg after '{s}'", .{arg});
19 } else if (std.mem.cutPrefix(u8, arg, "--lib=")) |lib_path| {21 } else if (std.mem.cutPrefix(u8, arg, "--lib=")) |lib_path| {
20 lib_path_arg = lib_path;22 lib_path_arg = lib_path;
23 } else if (std.mem.eql(u8, arg, "--target")) {
24 try target_args.append(
25 init.gpa,
26 try arena.dupe(u8, arg_it.next() orelse fail("missing arg after '{s}'", .{arg})),
27 );
28 } else if (std.mem.cutPrefix(u8, arg, "--target=")) |target| {
29 try target_args.append(init.gpa, try arena.dupe(u8, target));
21 } else if (std.mem.eql(u8, arg, "--keep-src")) {30 } else if (std.mem.eql(u8, arg, "--keep-src")) {
22 keep_src = true;31 keep_src = true;
23 } else {32 } else {
...@@ -72,45 +81,47 @@ pub fn main(init: std.process.Init) !void {...@@ -72,45 +81,47 @@ pub fn main(init: std.process.Init) !void {
72 .out = &stdin_writer.interface,81 .out = &stdin_writer.interface,
73 };82 };
74 {83 {
84 var args: std.ArrayList(u8) = .empty;
85 defer args.deinit(init.gpa);
75 const Arg = std.zig.Client.Message.Arg;86 const Arg = std.zig.Client.Message.Arg;
76 var args: std.ArrayList(u8) = .initBuffer(try arena.alloc(u8, 0 +87
77 @sizeOf(Arg) + @sizeOf(std.Io.File.Handle) +88 try args.append(init.gpa, @backingInt(@as(Arg, switch ((try test_file.stat(init.io)).kind) {
78 @sizeOf(Arg) + "--zig=\x00".len +
79 @sizeOf(Arg) + @sizeOf(std.Io.File.Handle) +
80 @sizeOf(Arg) + "--lib=\x00".len +
81 @sizeOf(Arg) + @sizeOf(std.Io.Dir.Handle) +
82 @sizeOf(Arg) + "--src=\x00".len +
83 @sizeOf(Arg) + @sizeOf(std.Io.Dir.Handle)));
84
85 args.appendAssumeCapacity(@backingInt(@as(Arg, switch ((try test_file.stat(init.io)).kind) {
86 else => unreachable,89 else => unreachable,
87 .file => .input_file,90 .file => .input_file,
88 .directory => .input_dir,91 .directory => .input_dir,
89 })));92 })));
90 args.appendSliceAssumeCapacity(@ptrCast(&test_file.handle));93 try args.appendSlice(init.gpa, @ptrCast(&test_file.handle));
94
95 try args.append(init.gpa, @backingInt(Arg.prefix));
96 try args.appendSlice(init.gpa, "--zig=");
97 try args.append(init.gpa, 0);
9198
92 args.appendAssumeCapacity(@backingInt(Arg.prefix));99 try args.append(init.gpa, @backingInt(Arg.input_file));
93 args.appendSliceAssumeCapacity("--zig=");100 try args.appendSlice(init.gpa, @ptrCast(&zig_file.handle));
94 args.appendAssumeCapacity(0);
95101
96 args.appendAssumeCapacity(@backingInt(Arg.input_file));102 try args.append(init.gpa, @backingInt(Arg.prefix));
97 args.appendSliceAssumeCapacity(@ptrCast(&zig_file.handle));103 try args.appendSlice(init.gpa, "--lib=");
104 try args.append(init.gpa, 0);
98105
99 args.appendAssumeCapacity(@backingInt(Arg.prefix));106 try args.append(init.gpa, @backingInt(Arg.input_dir));
100 args.appendSliceAssumeCapacity("--lib=");107 try args.appendSlice(init.gpa, @ptrCast(&lib_dir.handle));
101 args.appendAssumeCapacity(0);
102108
103 args.appendAssumeCapacity(@backingInt(Arg.input_dir));109 try args.append(init.gpa, @backingInt(Arg.prefix));
104 args.appendSliceAssumeCapacity(@ptrCast(&lib_dir.handle));110 try args.appendSlice(init.gpa, "--src=");
111 try args.append(init.gpa, 0);
105112
106 args.appendAssumeCapacity(@backingInt(Arg.prefix));113 try args.append(init.gpa, @backingInt(Arg.output_dir));
107 args.appendSliceAssumeCapacity("--src=");114 try args.appendSlice(init.gpa, @ptrCast(&src_dir.handle));
108 args.appendAssumeCapacity(0);
109115
110 args.appendAssumeCapacity(@backingInt(Arg.output_dir));116 for (target_args.items) |target_arg| {
111 args.appendSliceAssumeCapacity(@ptrCast(&src_dir.handle));117 try args.append(init.gpa, @backingInt(Arg.string));
118 try args.appendSlice(init.gpa, "--target");
119 try args.append(init.gpa, 0);
112120
113 std.debug.assert(args.unusedCapacitySlice().len == 0);121 try args.append(init.gpa, @backingInt(Arg.string));
122 try args.appendSlice(init.gpa, target_arg);
123 try args.append(init.gpa, 0);
124 }
114125
115 try client.serveMessageHeader(.{126 try client.serveMessageHeader(.{
116 .tag = .args,127 .tag = .args,
...@@ -118,6 +129,16 @@ pub fn main(init: std.process.Init) !void {...@@ -118,6 +129,16 @@ pub fn main(init: std.process.Init) !void {
118 });129 });
119 try client.out.writeAll(args.items);130 try client.out.writeAll(args.items);
120131
132 var metadata: struct {
133 index: u32,
134 len: u32,
135
136 fn next(m: *@This()) ?u32 {
137 if (m.len - m.index == 0) return null;
138 defer m.index += 1;
139 return m.index;
140 }
141 } = .{ .index = 0, .len = 0 };
121 try client.serveBodylessMessage(.query_test_metadata);142 try client.serveBodylessMessage(.query_test_metadata);
122 while (true) {143 while (true) {
123 const hdr = try client.receiveMessage();144 const hdr = try client.receiveMessage();
...@@ -132,12 +153,17 @@ pub fn main(init: std.process.Init) !void {...@@ -132,12 +153,17 @@ pub fn main(init: std.process.Init) !void {
132 );153 );
133 },154 },
134 .test_metadata => {155 .test_metadata => {
135 try client.in.discardAll(hdr.bytes_len);156 const tm_hdr =
136 try client.serveRunTest(0);157 try client.in.takeStruct(std.zig.Server.Message.TestMetadata, .little);
158 try client.in.discardAll(
159 hdr.bytes_len - @sizeOf(std.zig.Server.Message.TestMetadata),
160 );
161 metadata = .{ .index = 0, .len = tm_hdr.tests_len };
162 try client.serveRunTest(metadata.next() orelse break);
137 },163 },
138 .test_results => {164 .test_results => {
139 try client.in.discardAll(hdr.bytes_len);165 try client.in.discardAll(hdr.bytes_len);
140 break;166 try client.serveRunTest(metadata.next() orelse break);
141 },167 },
142 }168 }
143 }169 }