1const Cases = @This();
2const builtin = @import("builtin");
3
4const std = @import("std");
5const Io = std.Io;
6const assert = std.debug.assert;
7const Allocator = std.mem.Allocator;
8const getExternalExecutor = std.zig.system.getExternalExecutor;
9const ArrayList = std.ArrayList;
10
11gpa: Allocator,
12arena: Allocator,
13io: Io,
14cases: std.array_list.Managed(Case),
15
16pub const IncrementalCase = struct {
17 base_path: []const u8,
18};
19
20pub const File = struct {
21 src: [:0]const u8,
22 path: []const u8,
23};
24
25pub const DepModule = struct {
26 name: []const u8,
27 path: []const u8,
28};
29
30pub const Backend = enum {
31 /// Test does not care which backend is used; compiler gets to pick the default.
32 auto,
33 selfhosted,
34 llvm,
35};
36
37pub const CFrontend = enum {
38 clang,
39 aro,
40};
41
42pub const Case = struct {
43 /// The name of the test case. This is shown if a test fails, and
44 /// otherwise ignored.
45 name: []const u8,
46 /// The platform the test targets. For non-native platforms, an emulator
47 /// such as QEMU is required for tests to complete.
48 target: std.Build.ResolvedTarget,
49 /// In order to be able to run e.g. Execution updates, this must be set
50 /// to Executable.
51 output_mode: std.builtin.OutputMode,
52 optimize_mode: std.builtin.Optimize = .debug,
53
54 files: std.array_list.Managed(File),
55 case: ?union(enum) {
56 /// Check that it compiles with no errors.
57 Compile: void,
58 /// Check the main binary output file against an expected set of bytes.
59 /// This is most useful with, for example, `-ofmt=c`.
60 CompareObjectFile: []const u8,
61 /// An error update attempts to compile bad code, and ensures that it
62 /// fails to compile, and for the expected reasons.
63 /// A slice containing the expected stderr template, which
64 /// gets some values substituted.
65 Error: []const []const u8,
66 /// An execution update compiles and runs the input, testing the
67 /// stdout against the expected results
68 /// This is a slice containing the expected message.
69 Execution: []const u8,
70 /// A header update compiles the input with the equivalent of
71 /// `-femit-h` and tests the produced header against the
72 /// expected result.
73 Header: []const u8,
74 },
75
76 emit_asm: bool = false,
77 emit_bin: bool = true,
78 emit_h: bool = false,
79 is_test: bool = false,
80 expect_exact: bool = false,
81 backend: Backend = .auto,
82 link_libc: bool = false,
83 pic: ?bool = null,
84 pie: ?bool = null,
85 /// A list of imports to cache alongside the source file.
86 imports: []const []const u8 = &.{},
87 /// Where to look for imports relative to the `cases_dir_path` given to
88 /// `lower_to_build_steps`. If null, file imports will assert.
89 import_path: ?[]const u8 = null,
90
91 deps: std.array_list.Managed(DepModule),
92
93 pub fn addSourceFile(case: *Case, name: []const u8, src: [:0]const u8) void {
94 case.files.append(.{ .path = name, .src = src }) catch @panic("OOM");
95 }
96
97 pub fn addDepModule(case: *Case, name: []const u8, path: []const u8) void {
98 case.deps.append(.{
99 .name = name,
100 .path = path,
101 }) catch @panic("out of memory");
102 }
103
104 /// Adds a subcase in which the module is updated with `src`, compiled,
105 /// run, and the output is tested against `result`.
106 pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) void {
107 assert(self.case == null);
108 self.case = .{ .Execution = result };
109 self.addSourceFile("tmp.zig", src);
110 }
111
112 /// Adds a subcase in which the module is updated with `src`, which
113 /// should contain invalid input, and ensures that compilation fails
114 /// for the expected reasons, given in sequential order in `errors` in
115 /// the form `:line:column: error: message`.
116 pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void {
117 assert(errors.len != 0);
118 assert(self.case == null);
119 self.case = .{ .Error = errors };
120 self.addSourceFile("tmp.zig", src);
121 }
122
123 /// Adds a subcase in which the module is updated with `src`, and
124 /// asserts that it compiles without issue
125 pub fn addCompile(self: *Case, src: [:0]const u8) void {
126 assert(self.case == null);
127 self.case = .Compile;
128 self.addSourceFile("tmp.zig", src);
129 }
130};
131
132pub fn addExe(
133 ctx: *Cases,
134 name: []const u8,
135 target: std.Build.ResolvedTarget,
136) *Case {
137 ctx.cases.append(.{
138 .name = name,
139 .target = target,
140 .files = .init(ctx.arena),
141 .case = null,
142 .output_mode = .Exe,
143 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
144 }) catch @panic("out of memory");
145 return &ctx.cases.items[ctx.cases.items.len - 1];
146}
147
148/// Adds a test case for Zig input, producing an executable
149pub fn exe(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
150 return ctx.addExe(name, target);
151}
152
153pub fn exeFromCompiledC(ctx: *Cases, name: []const u8, target_query: std.Target.Query, b: *std.Build) *Case {
154 var adjusted_query = target_query;
155 adjusted_query.ofmt = .c;
156 ctx.cases.append(.{
157 .name = name,
158 .target = b.resolveTargetQuery(adjusted_query),
159 .files = .init(ctx.arena),
160 .case = null,
161 .output_mode = .Exe,
162 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
163 .link_libc = true,
164 }) catch @panic("out of memory");
165 return &ctx.cases.items[ctx.cases.items.len - 1];
166}
167
168pub fn addObjLlvm(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
169 const can_emit_asm = switch (target.result.cpu.arch) {
170 .csky,
171 .xtensa,
172 => false,
173 else => true,
174 };
175 const can_emit_bin = switch (target.result.cpu.arch) {
176 .arc,
177 .csky,
178 .nvptx,
179 .nvptx64,
180 .xcore,
181 .xtensa,
182 => false,
183 else => true,
184 };
185
186 ctx.cases.append(.{
187 .name = name,
188 .target = target,
189 .files = .init(ctx.arena),
190 .case = null,
191 .output_mode = .Obj,
192 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
193 .backend = .llvm,
194 .emit_bin = can_emit_bin,
195 .emit_asm = can_emit_asm,
196 }) catch @panic("out of memory");
197 return &ctx.cases.items[ctx.cases.items.len - 1];
198}
199
200pub fn addObj(
201 ctx: *Cases,
202 name: []const u8,
203 target: std.Build.ResolvedTarget,
204) *Case {
205 ctx.cases.append(.{
206 .name = name,
207 .target = target,
208 .files = .init(ctx.arena),
209 .case = null,
210 .output_mode = .Obj,
211 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
212 }) catch @panic("out of memory");
213 return &ctx.cases.items[ctx.cases.items.len - 1];
214}
215
216pub fn addTest(
217 ctx: *Cases,
218 name: []const u8,
219 target: std.Build.ResolvedTarget,
220) *Case {
221 ctx.cases.append(.{
222 .name = name,
223 .target = target,
224 .files = .init(ctx.arena),
225 .case = null,
226 .output_mode = .Exe,
227 .is_test = true,
228 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
229 }) catch @panic("out of memory");
230 return &ctx.cases.items[ctx.cases.items.len - 1];
231}
232
233/// Adds a test case for Zig input, producing an object file.
234pub fn obj(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
235 return ctx.addObj(name, target);
236}
237
238/// Adds a test case for ZIR input, producing an object file.
239pub fn objZIR(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
240 return ctx.addObj(name, target, .ZIR);
241}
242
243/// Adds a test case for Zig or ZIR input, producing C code.
244pub fn addC(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
245 var target_adjusted = target;
246 target_adjusted.ofmt = std.Target.ObjectFormat.c;
247 ctx.cases.append(.{
248 .name = name,
249 .target = target_adjusted,
250 .files = .init(ctx.arena),
251 .case = null,
252 .output_mode = .Obj,
253 .deps = std.array_list.Managed(DepModule).init(ctx.arena),
254 }) catch @panic("out of memory");
255 return &ctx.cases.items[ctx.cases.items.len - 1];
256}
257
258pub fn addTransform(
259 ctx: *Cases,
260 name: []const u8,
261 target: std.Build.ResolvedTarget,
262 src: [:0]const u8,
263 result: [:0]const u8,
264) void {
265 ctx.addObj(name, target).addTransform(src, result);
266}
267
268/// Adds a test case that compiles the Zig given in `src` to ZIR and tests
269/// the ZIR against `result`
270pub fn transform(
271 ctx: *Cases,
272 name: []const u8,
273 target: std.Build.ResolvedTarget,
274 src: [:0]const u8,
275 result: [:0]const u8,
276) void {
277 ctx.addTransform(name, target, src, result);
278}
279
280pub fn addError(
281 ctx: *Cases,
282 name: []const u8,
283 target: std.Build.ResolvedTarget,
284 src: [:0]const u8,
285 expected_errors: []const []const u8,
286) void {
287 ctx.addObj(name, target).addError(src, expected_errors);
288}
289
290/// Adds a test case that ensures that the Zig given in `src` fails to
291/// compile for the expected reasons, given in sequential order in
292/// `expected_errors` in the form `:line:column: error: message`.
293pub fn compileError(
294 ctx: *Cases,
295 name: []const u8,
296 target: std.Build.ResolvedTarget,
297 src: [:0]const u8,
298 expected_errors: []const []const u8,
299) void {
300 ctx.addError(name, target, src, expected_errors);
301}
302
303/// Adds a test case that asserts that the Zig given in `src` compiles
304/// without any errors.
305pub fn addCompile(
306 ctx: *Cases,
307 name: []const u8,
308 target: std.Build.ResolvedTarget,
309 src: [:0]const u8,
310) void {
311 ctx.addObj(name, target).addCompile(src);
312}
313
314/// Adds a test for each file in the provided directory. Recurses nested directories.
315///
316/// Each file should include a test manifest as a contiguous block of comments at
317/// the end of the file. The first line should be the test type, followed by a set of
318/// key-value config values, followed by a blank line, then the expected output.
319pub fn addFromDir(ctx: *Cases, dir: Io.Dir, path_from_root: []const u8, b: *std.Build) void {
320 var current_file: []const u8 = "none";
321 ctx.addFromDirInner(dir, path_from_root, &current_file, b) catch |err| {
322 std.debug.panicExtra(@returnAddress(), "test harness failed to process file {q}: {t}\n", .{
323 current_file, err,
324 });
325 };
326}
327
328fn addFromDirInner(
329 ctx: *Cases,
330 iterable_dir: Io.Dir,
331 path_from_root: []const u8,
332 /// This is kept up to date with the currently being processed file so
333 /// that if any errors occur the caller knows it happened during this file.
334 current_file: *[]const u8,
335 b: *std.Build,
336) !void {
337 const io = ctx.io;
338 var it = try iterable_dir.walk(ctx.arena);
339 var filenames: ArrayList([]const u8) = .empty;
340
341 while (try it.next(io)) |entry| {
342 // Ignore stuff such as .swp files
343 if (!knownFileExtension(entry.basename)) continue;
344
345 switch (entry.kind) {
346 .file => {
347 b.dependOnFileContents(b.path(b.pathJoin(&.{ path_from_root, entry.path })));
348 try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path));
349 },
350 .directory => {
351 b.dependOnDirectory(b.path(b.pathJoin(&.{ path_from_root, entry.path })));
352 },
353 else => continue,
354 }
355 }
356
357 for (filenames.items) |filename| {
358 current_file.* = filename;
359
360 const max_file_size = 10 * 1024 * 1024;
361 const src = try iterable_dir.readFileAllocOptions(io, filename, ctx.arena, .limited(max_file_size), .@"1", 0);
362
363 // Parse the manifest
364 var manifest = try TestManifest.parse(ctx.arena, src);
365
366 const backends = try manifest.getConfigForKeyAlloc(ctx.arena, "backend", Backend);
367 const target_strs = try manifest.getConfigForKeyAlloc(ctx.arena, "target", []const u8);
368 const cpu_features_str = manifest.config_map.get("cpu_features") orelse "";
369 const targets = try ctx.arena.alloc(std.Target.Query, target_strs.len);
370 for (targets, target_strs) |*query, target_str| {
371 query.* = try std.Target.Query.parse(.{
372 .arch_os_abi = target_str,
373 .cpu_features = if (cpu_features_str.len == 0) null else cpu_features_str,
374 });
375 }
376 const is_test = try manifest.getConfigForKeyAssertSingle("is_test", bool);
377 const link_libc = try manifest.getConfigForKeyAssertSingle("link_libc", bool);
378 const output_mode = try manifest.getConfigForKeyAssertSingle("output_mode", std.builtin.OutputMode);
379 const pic = try manifest.getConfigForKeyAssertSingle("pic", ?bool);
380 const pie = try manifest.getConfigForKeyAssertSingle("pie", ?bool);
381 const emit_asm = try manifest.getConfigForKeyAssertSingle("emit_asm", bool);
382 const emit_bin = try manifest.getConfigForKeyAssertSingle("emit_bin", bool);
383 const imports = try manifest.getConfigForKeyAlloc(ctx.arena, "imports", []const u8);
384
385 var cases = std.array_list.Managed(usize).init(ctx.arena);
386
387 // Cross-product to get all possible test combinations
388 for (targets) |target_query| {
389 const resolved_target = b.resolveTargetQuery(target_query);
390 const target = &resolved_target.result;
391 for (backends) |backend| {
392 if (backend == .selfhosted) {
393 switch (target.cpu.arch) {
394 .aarch64, .wasm32, .x86_64, .spirv64, .spirv32 => {},
395 // Other backends don't support new liveness format
396 else => continue,
397 }
398 }
399
400 if (backend == .selfhosted and target.cpu.arch == .aarch64) {
401 // https://codeberg.org/ziglang/zig/pulls/30232#issuecomment-9182045
402 continue;
403 }
404
405 if (backend == .selfhosted and target.os.tag == .macos and
406 target.cpu.arch == .x86_64 and builtin.cpu.arch == .aarch64)
407 {
408 // Rosetta has issues with ZLD
409 continue;
410 }
411
412 const next = ctx.cases.items.len;
413 try ctx.cases.append(.{
414 .name = try caseNameFromPath(ctx.arena, filename),
415 .import_path = std.fs.path.dirname(filename),
416 .backend = backend,
417 .files = .init(ctx.arena),
418 .case = null,
419 .emit_asm = emit_asm,
420 .emit_bin = emit_bin,
421 .is_test = is_test,
422 .output_mode = output_mode,
423 .link_libc = link_libc,
424 .pic = pic,
425 .pie = pie,
426 .deps = std.array_list.Managed(DepModule).init(ctx.cases.allocator),
427 .imports = imports,
428 .target = resolved_target,
429 });
430 try cases.append(next);
431 }
432 }
433
434 for (cases.items) |case_index| {
435 const case = &ctx.cases.items[case_index];
436 switch (manifest.type) {
437 .compile => {
438 case.addCompile(src);
439 },
440 .@"error" => {
441 const errors = try manifest.trailingLines(ctx.arena);
442 case.addError(src, errors);
443 },
444 .run => {
445 const output = try manifest.trailingSplit(ctx.arena);
446 case.addCompareOutput(src, output);
447 },
448 .cli => @panic("TODO cli tests"),
449 }
450 }
451 }
452}
453
454pub fn init(gpa: Allocator, arena: Allocator, io: Io) Cases {
455 return .{
456 .gpa = gpa,
457 .io = io,
458 .cases = .init(gpa),
459 .arena = arena,
460 };
461}
462
463pub const CaseTestOptions = struct {
464 test_filters: []const []const u8,
465 test_target_filters: []const []const u8,
466 skip_compile_errors: bool,
467 skip_non_native: bool,
468 skip_spirv: bool,
469 skip_wasm: bool,
470 skip_freebsd: bool,
471 skip_netbsd: bool,
472 skip_openbsd: bool,
473 skip_windows: bool,
474 skip_darwin: bool,
475 skip_linux: bool,
476 skip_llvm: bool,
477 skip_libc: bool,
478};
479
480pub fn lowerToBuildSteps(
481 self: *Cases,
482 b: *std.Build,
483 parent_step: *std.Build.Step,
484 options: CaseTestOptions,
485) void {
486 const io = self.io;
487 const graph = b.graph;
488 const arena = graph.arena;
489 const host = b.resolveTargetQuery(.{});
490
491 for (self.cases.items) |case| {
492 for (options.test_filters) |test_filter| {
493 if (std.mem.find(u8, case.name, test_filter)) |_| break;
494 } else if (options.test_filters.len > 0) continue;
495
496 if (case.case.? == .Error and options.skip_compile_errors) continue;
497
498 if (options.skip_non_native and !@import("../tests.zig").isNative(&case.target, &b.graph.host.result))
499 continue;
500
501 if (options.skip_spirv and case.target.query.cpu_arch != null and case.target.query.cpu_arch.?.isSpirV()) continue;
502 if (options.skip_wasm and case.target.query.cpu_arch != null and case.target.query.cpu_arch.?.isWasm()) continue;
503
504 if (options.skip_freebsd and case.target.query.os_tag == .freebsd) continue;
505 if (options.skip_netbsd and case.target.query.os_tag == .netbsd) continue;
506 if (options.skip_openbsd and case.target.query.os_tag == .openbsd) continue;
507 if (options.skip_windows and case.target.query.os_tag == .windows) continue;
508 if (options.skip_darwin and case.target.query.os_tag != null and case.target.query.os_tag.?.isDarwin()) continue;
509 if (options.skip_linux and case.target.query.os_tag == .linux) continue;
510
511 const would_use_llvm = @import("../tests.zig").wouldUseLlvm(
512 switch (case.backend) {
513 .auto => null,
514 .selfhosted => false,
515 .llvm => true,
516 },
517 case.target.query,
518 case.optimize_mode,
519 );
520 if (options.skip_llvm and would_use_llvm) continue;
521
522 const triple_txt = case.target.query.zigTriple(arena) catch @panic("OOM");
523
524 if (options.test_target_filters.len > 0) {
525 for (options.test_target_filters) |filter| {
526 if (std.mem.find(u8, triple_txt, filter) != null) break;
527 } else continue;
528 }
529
530 if (options.skip_libc and case.link_libc)
531 continue;
532
533 const writefiles = b.addWriteFiles();
534 var file_sources = std.StringHashMap(std.Build.LazyPath).init(arena);
535 defer file_sources.deinit();
536 const first_file = case.files.items[0];
537 const root_source_file = writefiles.add(first_file.path, first_file.src);
538 file_sources.put(first_file.path, root_source_file) catch @panic("OOM");
539 for (case.files.items[1..]) |file| {
540 file_sources.put(file.path, writefiles.add(file.path, file.src)) catch @panic("OOM");
541 }
542
543 for (case.imports) |import_rel| {
544 _ = writefiles.addCopyFile(.{ .src_path = .{
545 .owner = b,
546 .sub_path = b.pathJoin(&.{
547 "test",
548 "cases",
549 case.import_path orelse @panic("import_path not set"),
550 import_rel,
551 }),
552 } }, import_rel);
553 }
554
555 const mod = b.createModule(.{
556 .root_source_file = root_source_file,
557 .target = case.target,
558 .optimize = case.optimize_mode,
559 });
560
561 if (case.link_libc) mod.link_libc = true;
562 if (case.pic) |pic| mod.pic = pic;
563 for (case.deps.items) |dep| {
564 mod.addAnonymousImport(dep.name, .{
565 .root_source_file = file_sources.get(dep.path).?,
566 });
567 }
568
569 const artifact = if (case.is_test) b.addTest(.{
570 .name = case.name,
571 .root_module = mod,
572 }) else switch (case.output_mode) {
573 .Obj => b.addObject(.{
574 .name = case.name,
575 .root_module = mod,
576 }),
577 .Lib => b.addLibrary(.{
578 .linkage = .static,
579 .name = case.name,
580 .root_module = mod,
581 }),
582 .Exe => b.addExecutable(.{
583 .name = case.name,
584 .root_module = mod,
585 }),
586 };
587
588 if (case.pie) |pie| artifact.pie = pie;
589
590 switch (case.backend) {
591 .auto => {},
592 .selfhosted => {
593 artifact.use_llvm = false;
594 artifact.use_lld = false;
595 },
596 .llvm => {
597 artifact.use_llvm = true;
598 },
599 }
600
601 switch (case.case.?) {
602 .Compile => {
603 // Force the assembly/binary to be emitted if requested.
604 if (case.emit_asm) {
605 _ = artifact.getEmittedAsm();
606 }
607 if (case.emit_bin) {
608 _ = artifact.getEmittedBin();
609 }
610 parent_step.dependOn(&artifact.step);
611 },
612 .CompareObjectFile => |expected_output| {
613 const check = b.addCheckFile(artifact.getEmittedBin(), .{
614 .expected_exact = expected_output,
615 });
616
617 parent_step.dependOn(&check.step);
618 },
619 .Error => |expected_msgs| {
620 assert(expected_msgs.len != 0);
621 artifact.expect_errors = .{ .exact = expected_msgs };
622 parent_step.dependOn(&artifact.step);
623 },
624 .Execution => |expected_stdout| no_exec: {
625 const run = if (case.target.result.ofmt == .c) run_step: {
626 if (getExternalExecutor(io, &case.target.result, .{
627 .host_cpu_arch = host.result.cpu.arch,
628 .host_os_tag = host.result.os.tag,
629 .link_libc = true,
630 }) != .native) {
631 // We wouldn't be able to run the compiled C code.
632 break :no_exec;
633 }
634 const run_c = b.addSystemCommand(&.{
635 b.graph.zig_exe,
636 "run",
637 "-cflags",
638 "-Ilib",
639 "-std=c99",
640 "-pedantic",
641 "-Werror",
642 "-Wno-dollar-in-identifier-extension",
643 "-Wno-incompatible-library-redeclaration", // https://github.com/ziglang/zig/issues/875
644 "-Wno-incompatible-pointer-types",
645 "-Wno-overlength-strings",
646 "--",
647 "-lc",
648 "-target",
649 triple_txt,
650 });
651 run_c.addArtifactArg(artifact);
652 break :run_step run_c;
653 } else b.addRunArtifact(artifact);
654 run.skip_foreign_checks = true;
655 if (!case.is_test) {
656 run.expectStdOutEqual(expected_stdout);
657 }
658 parent_step.dependOn(&run.step);
659 },
660 .Header => @panic("TODO"),
661 }
662 }
663}
664
665/// Default config values for known test manifest key-value pairings.
666/// Currently handled defaults are:
667/// * backend
668/// * target
669/// * output_mode
670/// * is_test
671const TestManifestConfigDefaults = struct {
672 /// Asserts if the key doesn't exist - yep, it's an oversight alright.
673 fn get(@"type": TestManifest.Type, key: []const u8) []const u8 {
674 if (std.mem.eql(u8, key, "backend")) {
675 return "auto";
676 } else if (std.mem.eql(u8, key, "target")) {
677 if (@"type" == .@"error") {
678 return "native";
679 }
680 return "native,wasm32-wasi";
681 } else if (std.mem.eql(u8, key, "output_mode")) {
682 return switch (@"type") {
683 .@"error" => "Obj",
684 .run => "Exe",
685 .compile => "Obj",
686 .cli => @panic("TODO test harness for CLI tests"),
687 };
688 } else if (std.mem.eql(u8, key, "emit_asm")) {
689 return "false";
690 } else if (std.mem.eql(u8, key, "emit_bin")) {
691 return "true";
692 } else if (std.mem.eql(u8, key, "is_test")) {
693 return "false";
694 } else if (std.mem.eql(u8, key, "link_libc")) {
695 return "false";
696 } else if (std.mem.eql(u8, key, "c_frontend")) {
697 return "clang";
698 } else if (std.mem.eql(u8, key, "pic")) {
699 return "null";
700 } else if (std.mem.eql(u8, key, "pie")) {
701 return "null";
702 } else if (std.mem.eql(u8, key, "imports")) {
703 return "";
704 } else if (std.mem.eql(u8, key, "cpu_features")) {
705 return "";
706 } else unreachable;
707 }
708};
709
710/// Manifest syntax example:
711/// (see https://github.com/ziglang/zig/issues/11288)
712///
713/// error
714/// backend=selfhosted,llvm
715/// output_mode=exe
716///
717/// :3:19: error: foo
718///
719/// run
720/// target=x86_64-linux,aarch64-macos
721///
722/// I am expected stdout! Hello!
723///
724/// cli
725///
726/// build test
727const TestManifest = struct {
728 type: Type,
729 config_map: std.StringHashMap([]const u8),
730 trailing_bytes: []const u8 = "",
731
732 const valid_keys = std.StaticStringMap(void).initComptime(.{
733 .{ "emit_asm", {} },
734 .{ "emit_bin", {} },
735 .{ "is_test", {} },
736 .{ "output_mode", {} },
737 .{ "target", {} },
738 .{ "cpu_features", {} },
739 .{ "c_frontend", {} },
740 .{ "link_libc", {} },
741 .{ "backend", {} },
742 .{ "pic", {} },
743 .{ "pie", {} },
744 .{ "imports", {} },
745 });
746
747 const Type = enum {
748 @"error",
749 run,
750 cli,
751 compile,
752 };
753
754 const TrailingIterator = struct {
755 inner: std.mem.TokenIterator(u8, .any),
756
757 fn next(self: *TrailingIterator) ?[]const u8 {
758 const next_inner = self.inner.next() orelse return null;
759 return if (next_inner.len == 2) "" else std.mem.trimEnd(u8, next_inner[3..], " \t");
760 }
761 };
762
763 fn ConfigValueIterator(comptime T: type) type {
764 return struct {
765 inner: std.mem.TokenIterator(u8, .scalar),
766
767 fn next(self: *@This()) !?T {
768 const next_raw = self.inner.next() orelse return null;
769 const parseFn = getDefaultParser(T);
770 return try parseFn(next_raw);
771 }
772 };
773 }
774
775 fn parse(arena: Allocator, bytes: []const u8) !TestManifest {
776 // The manifest is the last contiguous block of comments in the file
777 // We scan for the beginning by searching backward for the first non-empty line that does not start with "//"
778 var start: ?usize = null;
779 var end: usize = bytes.len;
780 if (bytes.len > 0) {
781 var cursor: usize = bytes.len - 1;
782 while (true) {
783 // Move to beginning of line
784 while (cursor > 0 and bytes[cursor - 1] != '\n') cursor -= 1;
785
786 if (std.mem.startsWith(u8, bytes[cursor..], "//")) {
787 start = cursor; // Contiguous comment line, include in manifest
788 } else {
789 if (start != null) break; // Encountered non-comment line, end of manifest
790
791 // We ignore all-whitespace lines following the comment block, but anything else
792 // means that there is no manifest present.
793 if (std.mem.trim(u8, bytes[cursor..end], " \r\n\t").len == 0) {
794 end = cursor;
795 } else break; // If it's not whitespace, there is no manifest
796 }
797
798 // Move to previous line
799 if (cursor != 0) cursor -= 1 else break;
800 }
801 }
802
803 const actual_start = start orelse return error.MissingTestManifest;
804 const manifest_bytes = bytes[actual_start..end];
805
806 var it = std.mem.tokenizeAny(u8, manifest_bytes, "\r\n");
807
808 // First line is the test type
809 const tt: Type = blk: {
810 const line = it.next() orelse return error.MissingTestCaseType;
811 const raw = std.mem.trim(u8, line[2..], " \t");
812 if (std.mem.eql(u8, raw, "error")) {
813 break :blk .@"error";
814 } else if (std.mem.eql(u8, raw, "run")) {
815 break :blk .run;
816 } else if (std.mem.eql(u8, raw, "cli")) {
817 break :blk .cli;
818 } else if (std.mem.eql(u8, raw, "compile")) {
819 break :blk .compile;
820 } else {
821 std.log.warn("unknown test case type requested: {s}", .{raw});
822 return error.UnknownTestCaseType;
823 }
824 };
825
826 var manifest: TestManifest = .{
827 .type = tt,
828 .config_map = std.StringHashMap([]const u8).init(arena),
829 };
830
831 // Any subsequent line until a blank comment line is key=value(s) pair
832 while (it.next()) |line| {
833 const trimmed = std.mem.trim(u8, line[2..], " \t");
834 if (trimmed.len == 0) break;
835
836 // Parse key=value(s)
837 var kv_it = std.mem.splitScalar(u8, trimmed, '=');
838 const key = kv_it.first();
839 if (!valid_keys.has(key)) {
840 return error.InvalidKey;
841 }
842 try manifest.config_map.putNoClobber(key, kv_it.next() orelse return error.MissingValuesForConfig);
843 }
844
845 // Finally, trailing is expected output
846 manifest.trailing_bytes = manifest_bytes[it.index..];
847
848 return manifest;
849 }
850
851 fn getConfigForKey(
852 self: TestManifest,
853 key: []const u8,
854 comptime T: type,
855 ) ConfigValueIterator(T) {
856 const bytes = self.config_map.get(key) orelse TestManifestConfigDefaults.get(self.type, key);
857 return ConfigValueIterator(T){
858 .inner = std.mem.tokenizeScalar(u8, bytes, ','),
859 };
860 }
861
862 fn getConfigForKeyAlloc(
863 self: TestManifest,
864 allocator: Allocator,
865 key: []const u8,
866 comptime T: type,
867 ) ![]const T {
868 var out = std.array_list.Managed(T).init(allocator);
869 defer out.deinit();
870 var it = self.getConfigForKey(key, T);
871 while (try it.next()) |item| {
872 try out.append(item);
873 }
874 return try out.toOwnedSlice();
875 }
876
877 fn getConfigForKeyAssertSingle(self: TestManifest, key: []const u8, comptime T: type) !T {
878 var it = self.getConfigForKey(key, T);
879 const res = (try it.next()) orelse unreachable;
880 assert((try it.next()) == null);
881 return res;
882 }
883
884 fn trailing(self: TestManifest) TrailingIterator {
885 return .{
886 .inner = std.mem.tokenizeAny(u8, self.trailing_bytes, "\r\n"),
887 };
888 }
889
890 fn trailingSplit(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const u8 {
891 var out = std.array_list.Managed(u8).init(allocator);
892 defer out.deinit();
893 var trailing_it = self.trailing();
894 while (trailing_it.next()) |line| {
895 try out.appendSlice(line);
896 try out.append('\n');
897 }
898 if (out.items.len > 0) {
899 try out.resize(out.items.len - 1);
900 }
901 return try out.toOwnedSlice();
902 }
903
904 fn trailingLines(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const []const u8 {
905 var out = std.array_list.Managed([]const u8).init(allocator);
906 defer out.deinit();
907 var it = self.trailing();
908 while (it.next()) |line| {
909 try out.append(line);
910 }
911 return try out.toOwnedSlice();
912 }
913
914 fn trailingLinesSplit(self: TestManifest, allocator: Allocator) error{OutOfMemory}![]const []const u8 {
915 // Collect output lines split by empty lines
916 var out = std.array_list.Managed([]const u8).init(allocator);
917 defer out.deinit();
918 var buf = std.array_list.Managed(u8).init(allocator);
919 defer buf.deinit();
920 var it = self.trailing();
921 while (it.next()) |line| {
922 if (line.len == 0) {
923 if (buf.items.len != 0) {
924 try out.append(try buf.toOwnedSlice());
925 buf.items.len = 0;
926 }
927 continue;
928 }
929 try buf.appendSlice(line);
930 try buf.append('\n');
931 }
932 try out.append(try buf.toOwnedSlice());
933 return try out.toOwnedSlice();
934 }
935
936 fn ParseFn(comptime T: type) type {
937 return fn ([]const u8) anyerror!T;
938 }
939
940 fn getDefaultParser(comptime T: type) ParseFn(T) {
941 if (T == std.Target.Query) return struct {
942 fn parse(str: []const u8) anyerror!T {
943 return std.Target.Query.parse(.{ .arch_os_abi = str });
944 }
945 }.parse;
946
947 switch (@typeInfo(T)) {
948 .int => return struct {
949 fn parse(str: []const u8) anyerror!T {
950 return try std.fmt.parseInt(T, str, 0);
951 }
952 }.parse,
953 .bool => return struct {
954 fn parse(str: []const u8) anyerror!T {
955 if (std.mem.eql(u8, str, "true")) return true;
956 if (std.mem.eql(u8, str, "false")) return false;
957 std.debug.print("{s}\n", .{str});
958 return error.InvalidBool;
959 }
960 }.parse,
961 .@"enum" => return struct {
962 fn parse(str: []const u8) anyerror!T {
963 return std.meta.stringToEnum(T, str) orelse {
964 std.log.err("unknown enum variant for {s}: {s}", .{ @typeName(T), str });
965 return error.UnknownEnumVariant;
966 };
967 }
968 }.parse,
969 .optional => |o| return struct {
970 fn parse(str: []const u8) anyerror!T {
971 if (std.mem.eql(u8, str, "null")) return null;
972 return try getDefaultParser(o.child)(str);
973 }
974 }.parse,
975 .@"struct" => @compileError("no default parser for " ++ @typeName(T)),
976 .pointer => {
977 if (T == []const u8) {
978 return struct {
979 fn parse(str: []const u8) anyerror!T {
980 return str;
981 }
982 }.parse;
983 } else {
984 @compileError("no default parser for " ++ @typeName(T));
985 }
986 },
987 else => @compileError("no default parser for " ++ @typeName(T)),
988 }
989 }
990};
991
992fn knownFileExtension(filename: []const u8) bool {
993 // List taken from `Compilation.classifyFileExt` in the compiler.
994 for ([_][]const u8{
995 ".c", ".C", ".cc", ".cpp",
996 ".cxx", ".stub", ".m", ".mm",
997 ".ll", ".bc", ".s", ".S",
998 ".h", ".zig", ".so", ".dll",
999 ".dylib", ".tbd", ".a", ".lib",
1000 ".o", ".obj", ".cu", ".def",
1001 ".rc", ".res", ".manifest",
1002 }) |ext| {
1003 if (std.mem.endsWith(u8, filename, ext)) return true;
1004 }
1005 // Final check for .so.X, .so.X.Y, .so.X.Y.Z.
1006 // From `Compilation.hasSharedLibraryExt`.
1007 var it = std.mem.splitScalar(u8, filename, '.');
1008 _ = it.first();
1009 var so_txt = it.next() orelse return false;
1010 while (!std.mem.eql(u8, so_txt, "so")) {
1011 so_txt = it.next() orelse return false;
1012 }
1013 const n1 = it.next() orelse return false;
1014 const n2 = it.next();
1015 const n3 = it.next();
1016 _ = std.fmt.parseInt(u32, n1, 10) catch return false;
1017 if (n2) |x| _ = std.fmt.parseInt(u32, x, 10) catch return false;
1018 if (n3) |x| _ = std.fmt.parseInt(u32, x, 10) catch return false;
1019 if (it.next() != null) return false;
1020 return false;
1021}
1022
1023/// `path` is a path relative to the root case directory.
1024/// e.g. `compile_errors/undeclared_identifier.zig`
1025/// The case name is computed by removing the extension and substituting path separators for dots.
1026/// e.g. `compile_errors.undeclared_identifier`
1027/// Including the directory components makes `-Dtest-filter` more useful, because you can filter
1028/// based on subdirectory; e.g. `-Dtest-filter=compile_errors` to run the compile error tets.
1029fn caseNameFromPath(arena: Allocator, path: []const u8) Allocator.Error![]const u8 {
1030 const ext_len = std.fs.path.extension(path).len;
1031 const path_sans_ext = path[0 .. path.len - ext_len];
1032 const result = try arena.dupe(u8, path_sans_ext);
1033 std.mem.replaceScalar(u8, result, std.fs.path.sep, '.');
1034 return result;
1035}