authorgravatar for february.cozzocrea@gmail.comfebruary cozzocrea <february.cozzocrea@gmail.com> 2024-03-04 17:26:41-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-03-08 02:59:45-08:00
logb2427ea7d839a9e17568206c64da56865cd000f1
treeec6fa58802142b8046b8d642600f986f85a251c4
parent83e578a181e33eedd57666376dab371b7ae58d5b

test manifest key checking and other fixes

This commit adds several fixes and improvements for the Zig compiler test harness. 1. -Dskip-translate-c option added for skipping the translate-c tests. 2. translate-c/run-translated-c tests in test/cases/* have been added to the steps test-translate-c and test-run-translated-c. Closes #18224. 3. Custom name added to the CheckFile step for the translate-c step to better communicate which test failed. 4. Test manifest key validation added to return an error if a manifest contains an invalid key.

5 files changed, 120 insertions(+), 74 deletions(-)

build.zig+5-6
......@@ -101,6 +101,7 @@ pub fn build(b: *std.Build) !void {
101101 const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse false;
102102 const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;
103103 const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false;
104 const skip_translate_c = b.option(bool, "skip-translate-c", "Main test suite skips translate-c tests") orelse false;
104105 const skip_run_translated_c = b.option(bool, "skip-run-translated-c", "Main test suite skips run-translated-c tests") orelse false;
105106
106107 const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false;
......@@ -453,7 +454,10 @@ pub fn build(b: *std.Build) !void {
453454 }).step);
454455
455456 const test_cases_step = b.step("test-cases", "Run the main compiler test cases");
456 try tests.addCases(b, test_cases_step, test_filters, check_case_exe, .{
457 try tests.addCases(b, test_cases_step, test_filters, check_case_exe, target, .{
458 .skip_translate_c = skip_translate_c,
459 .skip_run_translated_c = skip_run_translated_c,
460 }, .{
457461 .enable_llvm = enable_llvm,
458462 .llvm_has_m68k = llvm_has_m68k,
459463 .llvm_has_csky = llvm_has_csky,
......@@ -525,11 +529,6 @@ pub fn build(b: *std.Build) !void {
525529 test_step.dependOn(tests.addStackTraceTests(b, test_filters, optimization_modes));
526530 test_step.dependOn(tests.addCliTests(b));
527531 test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filters, optimization_modes));
528 test_step.dependOn(tests.addTranslateCTests(b, test_filters));
529 if (!skip_run_translated_c) {
530 test_step.dependOn(tests.addRunTranslatedCTests(b, test_filters, target));
531 }
532
533532 test_step.dependOn(tests.addModuleTests(b, .{
534533 .test_filters = test_filters,
535534 .root_src = "lib/std/std.zig",
test/cases/run_translated_c/compound_assignments_with_implicit_casts.c+1-1
......@@ -17,4 +17,4 @@ int main() {
1717}
1818
1919// run-translated-c
20// c_frontends=aro,clang
20// c_frontend=clang
test/cases/run_translated_c/explicit_cast_bool_from_float.c+1-1
......@@ -7,4 +7,4 @@ int main() {
77}
88
99// run-translated-c
10// c_frontends=aro,clang
10// c_frontend=clang
test/src/Cases.zig+102-60
......@@ -533,6 +533,98 @@ pub fn init(gpa: Allocator, arena: Allocator) Cases {
533533 };
534534}
535535
536pub const TranslateCOptions = struct {
537 skip_translate_c: bool = false,
538 skip_run_translated_c: bool = false,
539};
540pub fn lowerToTranslateCSteps(
541 self: *Cases,
542 b: *std.Build,
543 parent_step: *std.Build.Step,
544 test_filters: []const []const u8,
545 target: std.Build.ResolvedTarget,
546 translate_c_options: TranslateCOptions,
547) void {
548 const host = std.zig.system.resolveTargetQuery(.{}) catch |err|
549 std.debug.panic("unable to detect native host: {s}\n", .{@errorName(err)});
550
551 const tests = @import("../tests.zig");
552 const test_translate_c_step = b.step("test-translate-c", "Run the C translation tests");
553 if (!translate_c_options.skip_translate_c) {
554 tests.addTranslateCTests(b, test_translate_c_step, test_filters);
555 parent_step.dependOn(test_translate_c_step);
556 }
557
558 const test_run_translated_c_step = b.step("test-run-translated-c", "Run the Run-Translated-C tests");
559 if (!translate_c_options.skip_run_translated_c) {
560 tests.addRunTranslatedCTests(b, test_run_translated_c_step, test_filters, target);
561 parent_step.dependOn(test_run_translated_c_step);
562 }
563
564 for (self.translate.items) |case| switch (case.kind) {
565 .run => |output| {
566 if (translate_c_options.skip_run_translated_c) continue;
567 const annotated_case_name = b.fmt("run-translated-c {s}", .{case.name});
568 for (test_filters) |test_filter| {
569 if (std.mem.indexOf(u8, annotated_case_name, test_filter)) |_| break;
570 } else if (test_filters.len > 0) continue;
571 if (!std.process.can_spawn) {
572 std.debug.print("Unable to spawn child processes on {s}, skipping test.\n", .{@tagName(builtin.os.tag)});
573 continue; // Pass test.
574 }
575
576 if (getExternalExecutor(host, &case.target.result, .{ .link_libc = true }) != .native) {
577 // We wouldn't be able to run the compiled C code.
578 continue; // Pass test.
579 }
580
581 const write_src = b.addWriteFiles();
582 const file_source = write_src.add("tmp.c", case.input);
583
584 const translate_c = b.addTranslateC(.{
585 .root_source_file = file_source,
586 .optimize = .Debug,
587 .target = case.target,
588 .link_libc = case.link_libc,
589 .use_clang = case.c_frontend == .clang,
590 });
591 translate_c.step.name = b.fmt("{s} translate-c", .{annotated_case_name});
592
593 const run_exe = translate_c.addExecutable(.{});
594 run_exe.step.name = b.fmt("{s} build-exe", .{annotated_case_name});
595 run_exe.linkLibC();
596 const run = b.addRunArtifact(run_exe);
597 run.step.name = b.fmt("{s} run", .{annotated_case_name});
598 run.expectStdOutEqual(output);
599
600 test_run_translated_c_step.dependOn(&run.step);
601 },
602 .translate => |output| {
603 if (translate_c_options.skip_translate_c) continue;
604 const annotated_case_name = b.fmt("zig translate-c {s}", .{case.name});
605 for (test_filters) |test_filter| {
606 if (std.mem.indexOf(u8, annotated_case_name, test_filter)) |_| break;
607 } else if (test_filters.len > 0) continue;
608
609 const write_src = b.addWriteFiles();
610 const file_source = write_src.add("tmp.c", case.input);
611
612 const translate_c = b.addTranslateC(.{
613 .root_source_file = file_source,
614 .optimize = .Debug,
615 .target = case.target,
616 .link_libc = case.link_libc,
617 .use_clang = case.c_frontend == .clang,
618 });
619 translate_c.step.name = b.fmt("{s} translate-c", .{annotated_case_name});
620
621 const check_file = translate_c.addCheckFile(output);
622 check_file.step.name = b.fmt("{s} CheckFile", .{annotated_case_name});
623 test_translate_c_step.dependOn(&check_file.step);
624 },
625 };
626}
627
536628pub fn lowerToBuildSteps(
537629 self: *Cases,
538630 b: *std.Build,
......@@ -681,66 +773,6 @@ pub fn lowerToBuildSteps(
681773 .Header => @panic("TODO"),
682774 }
683775 }
684
685 for (self.translate.items) |case| switch (case.kind) {
686 .run => |output| {
687 const annotated_case_name = b.fmt("run-translated-c {s}", .{case.name});
688 for (test_filters) |test_filter| {
689 if (std.mem.indexOf(u8, annotated_case_name, test_filter)) |_| break;
690 } else if (test_filters.len > 0) continue;
691 if (!std.process.can_spawn) {
692 std.debug.print("Unable to spawn child processes on {s}, skipping test.\n", .{@tagName(builtin.os.tag)});
693 continue; // Pass test.
694 }
695
696 if (getExternalExecutor(host, &case.target.result, .{ .link_libc = true }) != .native) {
697 // We wouldn't be able to run the compiled C code.
698 continue; // Pass test.
699 }
700
701 const write_src = b.addWriteFiles();
702 const file_source = write_src.add("tmp.c", case.input);
703
704 const translate_c = b.addTranslateC(.{
705 .root_source_file = file_source,
706 .optimize = .Debug,
707 .target = case.target,
708 .link_libc = case.link_libc,
709 .use_clang = case.c_frontend == .clang,
710 });
711 translate_c.step.name = b.fmt("{s} translate-c", .{annotated_case_name});
712
713 const run_exe = translate_c.addExecutable(.{});
714 run_exe.step.name = b.fmt("{s} build-exe", .{annotated_case_name});
715 run_exe.linkLibC();
716 const run = b.addRunArtifact(run_exe);
717 run.step.name = b.fmt("{s} run", .{annotated_case_name});
718 run.expectStdOutEqual(output);
719
720 parent_step.dependOn(&run.step);
721 },
722 .translate => |output| {
723 const annotated_case_name = b.fmt("zig translate-c {s}", .{case.name});
724 for (test_filters) |test_filter| {
725 if (std.mem.indexOf(u8, annotated_case_name, test_filter)) |_| break;
726 } else if (test_filters.len > 0) continue;
727
728 const write_src = b.addWriteFiles();
729 const file_source = write_src.add("tmp.c", case.input);
730
731 const translate_c = b.addTranslateC(.{
732 .root_source_file = file_source,
733 .optimize = .Debug,
734 .target = case.target,
735 .link_libc = case.link_libc,
736 .use_clang = case.c_frontend == .clang,
737 });
738 translate_c.step.name = annotated_case_name;
739
740 const check_file = translate_c.addCheckFile(output);
741 parent_step.dependOn(&check_file.step);
742 },
743 };
744776}
745777
746778/// Sort test filenames in-place, so that incremental test cases ("foo.0.zig",
......@@ -961,6 +993,15 @@ const TestManifest = struct {
961993 config_map: std.StringHashMap([]const u8),
962994 trailing_bytes: []const u8 = "",
963995
996 const valid_keys = std.ComptimeStringMap(void, .{
997 .{ "is_test", {} },
998 .{ "output_mode", {} },
999 .{ "target", {} },
1000 .{ "c_frontend", {} },
1001 .{ "link_libc", {} },
1002 .{ "backend", {} },
1003 });
1004
9641005 const Type = enum {
9651006 @"error",
9661007 run,
......@@ -1059,6 +1100,7 @@ const TestManifest = struct {
10591100 // Parse key=value(s)
10601101 var kv_it = std.mem.splitScalar(u8, trimmed, '=');
10611102 const key = kv_it.first();
1103 if (!valid_keys.has(key)) return error.InvalidKey;
10621104 try manifest.config_map.putNoClobber(key, kv_it.next() orelse return error.MissingValuesForConfig);
10631105 }
10641106
test/tests.zig+11-6
......@@ -998,29 +998,30 @@ pub fn addAssembleAndLinkTests(b: *std.Build, test_filters: []const []const u8,
998998 return cases.step;
999999}
10001000
1001pub fn addTranslateCTests(b: *std.Build, test_filters: []const []const u8) *Step {
1001pub fn addTranslateCTests(b: *std.Build, parent_step: *std.Build.Step, test_filters: []const []const u8) void {
10021002 const cases = b.allocator.create(TranslateCContext) catch @panic("OOM");
10031003 cases.* = TranslateCContext{
10041004 .b = b,
1005 .step = b.step("test-translate-c", "Run the C translation tests"),
1005 .step = parent_step,
10061006 .test_index = 0,
10071007 .test_filters = test_filters,
10081008 };
10091009
10101010 translate_c.addCases(cases);
10111011
1012 return cases.step;
1012 return;
10131013}
10141014
10151015pub fn addRunTranslatedCTests(
10161016 b: *std.Build,
1017 parent_step: *std.Build.Step,
10171018 test_filters: []const []const u8,
10181019 target: std.Build.ResolvedTarget,
1019) *Step {
1020) void {
10201021 const cases = b.allocator.create(RunTranslatedCContext) catch @panic("OOM");
10211022 cases.* = .{
10221023 .b = b,
1023 .step = b.step("test-run-translated-c", "Run the Run-Translated-C tests"),
1024 .step = parent_step,
10241025 .test_index = 0,
10251026 .test_filters = test_filters,
10261027 .target = target,
......@@ -1028,7 +1029,7 @@ pub fn addRunTranslatedCTests(
10281029
10291030 run_translated_c.addCases(cases);
10301031
1031 return cases.step;
1032 return;
10321033}
10331034
10341035const ModuleTestOptions = struct {
......@@ -1288,6 +1289,8 @@ pub fn addCases(
12881289 parent_step: *Step,
12891290 test_filters: []const []const u8,
12901291 check_case_exe: *std.Build.Step.Compile,
1292 target: std.Build.ResolvedTarget,
1293 translate_c_options: @import("src/Cases.zig").TranslateCOptions,
12911294 build_options: @import("cases.zig").BuildOptions,
12921295) !void {
12931296 const arena = b.allocator;
......@@ -1301,6 +1304,8 @@ pub fn addCases(
13011304 cases.addFromDir(dir, b);
13021305 try @import("cases.zig").addCases(&cases, build_options, b);
13031306
1307 cases.lowerToTranslateCSteps(b, parent_step, test_filters, target, translate_c_options);
1308
13041309 const cases_dir_path = try b.build_root.join(b.allocator, &.{ "test", "cases" });
13051310 cases.lowerToBuildSteps(
13061311 b,