authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-13 16:44:45-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-15 10:33:08-07:00
log3c93c1664a3c3546076b39c2d40405f5db537dae
tree762b7b1e28d4472df59d300455a4fc56f0599d41
parentadc9b77d5fc4d1520905f1b1f4b3c80d6d424da6

tests: avoid skipping native tests

Make the test targets use options that match the actual options of CompileStep. This makes the code more straightforward, and ends up making fewer tests incorrectly skipped. For example, now the CI runner on Windows will no longer skip self-hosted x86_64 backend tests.

2 files changed, 96 insertions(+), 126 deletions(-)

build.zig+4-17
...@@ -85,9 +85,7 @@ pub fn build(b: *std.Build) !void {...@@ -85,9 +85,7 @@ pub fn build(b: *std.Build) !void {
85 const skip_cross_glibc = b.option(bool, "skip-cross-glibc", "Main test suite skips builds that require cross glibc") orelse false;85 const skip_cross_glibc = b.option(bool, "skip-cross-glibc", "Main test suite skips builds that require cross glibc") orelse false;
86 const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;86 const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;
87 const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false;87 const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false;
88 const skip_stage1 = b.option(bool, "skip-stage1", "Main test suite skips stage1 compile error tests") orelse false;
89 const skip_run_translated_c = b.option(bool, "skip-run-translated-c", "Main test suite skips run-translated-c tests") orelse false;88 const skip_run_translated_c = b.option(bool, "skip-run-translated-c", "Main test suite skips run-translated-c tests") orelse false;
90 const skip_stage2_tests = b.option(bool, "skip-stage2-tests", "Main test suite skips self-hosted compiler tests") orelse false;
9189
92 const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false;90 const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false;
9391
...@@ -187,9 +185,7 @@ pub fn build(b: *std.Build) !void {...@@ -187,9 +185,7 @@ pub fn build(b: *std.Build) !void {
187 const compile_step = b.step("compile", "Build the self-hosted compiler");185 const compile_step = b.step("compile", "Build the self-hosted compiler");
188 compile_step.dependOn(&exe.step);186 compile_step.dependOn(&exe.step);
189187
190 if (!skip_stage2_tests) {188 test_step.dependOn(&exe.step);
191 test_step.dependOn(&exe.step);
192 }
193189
194 exe.single_threaded = single_threaded;190 exe.single_threaded = single_threaded;
195191
...@@ -360,7 +356,6 @@ pub fn build(b: *std.Build) !void {...@@ -360,7 +356,6 @@ pub fn build(b: *std.Build) !void {
360 test_cases_options.addOption(bool, "enable_link_snapshots", enable_link_snapshots);356 test_cases_options.addOption(bool, "enable_link_snapshots", enable_link_snapshots);
361 test_cases_options.addOption(bool, "skip_non_native", skip_non_native);357 test_cases_options.addOption(bool, "skip_non_native", skip_non_native);
362 test_cases_options.addOption(bool, "skip_cross_glibc", skip_cross_glibc);358 test_cases_options.addOption(bool, "skip_cross_glibc", skip_cross_glibc);
363 test_cases_options.addOption(bool, "skip_stage1", skip_stage1);
364 test_cases_options.addOption(bool, "have_llvm", enable_llvm);359 test_cases_options.addOption(bool, "have_llvm", enable_llvm);
365 test_cases_options.addOption(bool, "llvm_has_m68k", llvm_has_m68k);360 test_cases_options.addOption(bool, "llvm_has_m68k", llvm_has_m68k);
366 test_cases_options.addOption(bool, "llvm_has_csky", llvm_has_csky);361 test_cases_options.addOption(bool, "llvm_has_csky", llvm_has_csky);
...@@ -416,7 +411,7 @@ pub fn build(b: *std.Build) !void {...@@ -416,7 +411,7 @@ pub fn build(b: *std.Build) !void {
416411
417 const test_cases_step = b.step("test-cases", "Run the main compiler test cases");412 const test_cases_step = b.step("test-cases", "Run the main compiler test cases");
418 try tests.addCases(b, test_cases_step, test_filter, check_case_exe);413 try tests.addCases(b, test_cases_step, test_filter, check_case_exe);
419 if (!skip_stage2_tests) test_step.dependOn(test_cases_step);414 test_step.dependOn(test_cases_step);
420415
421 test_step.dependOn(tests.addModuleTests(b, .{416 test_step.dependOn(tests.addModuleTests(b, .{
422 .test_filter = test_filter,417 .test_filter = test_filter,
...@@ -428,8 +423,6 @@ pub fn build(b: *std.Build) !void {...@@ -428,8 +423,6 @@ pub fn build(b: *std.Build) !void {
428 .skip_non_native = skip_non_native,423 .skip_non_native = skip_non_native,
429 .skip_cross_glibc = skip_cross_glibc,424 .skip_cross_glibc = skip_cross_glibc,
430 .skip_libc = skip_libc,425 .skip_libc = skip_libc,
431 .skip_stage1 = skip_stage1,
432 .skip_stage2 = skip_stage2_tests,
433 .max_rss = 1 * 1024 * 1024 * 1024,426 .max_rss = 1 * 1024 * 1024 * 1024,
434 }));427 }));
435428
...@@ -443,8 +436,6 @@ pub fn build(b: *std.Build) !void {...@@ -443,8 +436,6 @@ pub fn build(b: *std.Build) !void {
443 .skip_non_native = skip_non_native,436 .skip_non_native = skip_non_native,
444 .skip_cross_glibc = skip_cross_glibc,437 .skip_cross_glibc = skip_cross_glibc,
445 .skip_libc = true,438 .skip_libc = true,
446 .skip_stage1 = skip_stage1,
447 .skip_stage2 = true, // TODO get all these passing
448 }));439 }));
449440
450 test_step.dependOn(tests.addModuleTests(b, .{441 test_step.dependOn(tests.addModuleTests(b, .{
...@@ -457,8 +448,6 @@ pub fn build(b: *std.Build) !void {...@@ -457,8 +448,6 @@ pub fn build(b: *std.Build) !void {
457 .skip_non_native = skip_non_native,448 .skip_non_native = skip_non_native,
458 .skip_cross_glibc = skip_cross_glibc,449 .skip_cross_glibc = skip_cross_glibc,
459 .skip_libc = true,450 .skip_libc = true,
460 .skip_stage1 = skip_stage1,
461 .skip_stage2 = true, // TODO get all these passing
462 }));451 }));
463452
464 test_step.dependOn(tests.addCompareOutputTests(b, test_filter, optimization_modes));453 test_step.dependOn(tests.addCompareOutputTests(b, test_filter, optimization_modes));
...@@ -466,11 +455,11 @@ pub fn build(b: *std.Build) !void {...@@ -466,11 +455,11 @@ pub fn build(b: *std.Build) !void {
466 b,455 b,
467 optimization_modes,456 optimization_modes,
468 enable_macos_sdk,457 enable_macos_sdk,
469 skip_stage2_tests,458 false,
470 enable_symlinks_windows,459 enable_symlinks_windows,
471 ));460 ));
472 test_step.dependOn(tests.addCAbiTests(b, skip_non_native, skip_release));461 test_step.dependOn(tests.addCAbiTests(b, skip_non_native, skip_release));
473 test_step.dependOn(tests.addLinkTests(b, enable_macos_sdk, skip_stage2_tests, enable_symlinks_windows));462 test_step.dependOn(tests.addLinkTests(b, enable_macos_sdk, false, enable_symlinks_windows));
474 test_step.dependOn(tests.addStackTraceTests(b, test_filter, optimization_modes));463 test_step.dependOn(tests.addStackTraceTests(b, test_filter, optimization_modes));
475 test_step.dependOn(tests.addCliTests(b));464 test_step.dependOn(tests.addCliTests(b));
476 test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, optimization_modes));465 test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, optimization_modes));
...@@ -489,8 +478,6 @@ pub fn build(b: *std.Build) !void {...@@ -489,8 +478,6 @@ pub fn build(b: *std.Build) !void {
489 .skip_non_native = skip_non_native,478 .skip_non_native = skip_non_native,
490 .skip_cross_glibc = skip_cross_glibc,479 .skip_cross_glibc = skip_cross_glibc,
491 .skip_libc = skip_libc,480 .skip_libc = skip_libc,
492 .skip_stage1 = skip_stage1,
493 .skip_stage2 = true, // TODO get all these passing
494 // I observed a value of 3398275072 on my M1, and multiplied by 1.1 to481 // I observed a value of 3398275072 on my M1, and multiplied by 1.1 to
495 // get this amount:482 // get this amount:
496 .max_rss = 3738102579,483 .max_rss = 3738102579,
test/tests.zig+92-109
...@@ -22,12 +22,12 @@ pub const CompareOutputContext = @import("src/CompareOutput.zig");...@@ -22,12 +22,12 @@ pub const CompareOutputContext = @import("src/CompareOutput.zig");
22pub const StackTracesContext = @import("src/StackTrace.zig");22pub const StackTracesContext = @import("src/StackTrace.zig");
2323
24const TestTarget = struct {24const TestTarget = struct {
25 target: CrossTarget = @as(CrossTarget, .{}),25 target: CrossTarget = .{},
26 optimize_mode: std.builtin.OptimizeMode = .Debug,26 optimize_mode: std.builtin.OptimizeMode = .Debug,
27 link_libc: bool = false,27 link_libc: ?bool = null,
28 single_threaded: bool = false,28 single_threaded: ?bool = null,
29 disable_native: bool = false,29 use_llvm: ?bool = null,
30 backend: ?std.builtin.CompilerBackend = null,30 use_lld: ?bool = null,
31};31};
3232
33const test_targets = blk: {33const test_targets = blk: {
...@@ -43,13 +43,47 @@ const test_targets = blk: {...@@ -43,13 +43,47 @@ const test_targets = blk: {
43 .{43 .{
44 .single_threaded = true,44 .single_threaded = true,
45 },45 },
46 .{
47 .optimize_mode = .ReleaseFast,
48 },
49 .{
50 .link_libc = true,
51 .optimize_mode = .ReleaseFast,
52 },
53 .{
54 .optimize_mode = .ReleaseFast,
55 .single_threaded = true,
56 },
57
58 .{
59 .optimize_mode = .ReleaseSafe,
60 },
61 .{
62 .link_libc = true,
63 .optimize_mode = .ReleaseSafe,
64 },
65 .{
66 .optimize_mode = .ReleaseSafe,
67 .single_threaded = true,
68 },
69
70 .{
71 .optimize_mode = .ReleaseSmall,
72 },
73 .{
74 .link_libc = true,
75 .optimize_mode = .ReleaseSmall,
76 },
77 .{
78 .optimize_mode = .ReleaseSmall,
79 .single_threaded = true,
80 },
4681
47 .{82 .{
48 .target = .{83 .target = .{
49 .ofmt = .c,84 .ofmt = .c,
50 },85 },
51 .link_libc = true,86 .link_libc = true,
52 .backend = .stage2_c,
53 },87 },
54 .{88 .{
55 .target = .{89 .target = .{
...@@ -57,22 +91,24 @@ const test_targets = blk: {...@@ -57,22 +91,24 @@ const test_targets = blk: {
57 .os_tag = .linux,91 .os_tag = .linux,
58 .abi = .none,92 .abi = .none,
59 },93 },
60 .backend = .stage2_x86_64,94 .use_llvm = false,
95 .use_lld = false,
61 },96 },
62 .{97 .{
63 .target = .{98 .target = .{
64 .cpu_arch = .aarch64,99 .cpu_arch = .aarch64,
65 .os_tag = .linux,100 .os_tag = .linux,
66 },101 },
67 .backend = .stage2_aarch64,102 .use_llvm = false,
103 .use_lld = false,
68 },104 },
69 .{105 .{
70 .target = .{106 .target = .{
71 .cpu_arch = .wasm32,107 .cpu_arch = .wasm32,
72 .os_tag = .wasi,108 .os_tag = .wasi,
73 },109 },
74 .single_threaded = true,110 .use_llvm = false,
75 .backend = .stage2_wasm,111 .use_lld = false,
76 },112 },
77 // https://github.com/ziglang/zig/issues/13623113 // https://github.com/ziglang/zig/issues/13623
78 //.{114 //.{
...@@ -80,7 +116,8 @@ const test_targets = blk: {...@@ -80,7 +116,8 @@ const test_targets = blk: {
80 // .cpu_arch = .arm,116 // .cpu_arch = .arm,
81 // .os_tag = .linux,117 // .os_tag = .linux,
82 // },118 // },
83 // .backend = .stage2_arm,119 // .use_llvm = false,
120 // .use_lld = false,
84 //},121 //},
85 // https://github.com/ziglang/zig/issues/13623122 // https://github.com/ziglang/zig/issues/13623
86 //.{123 //.{
...@@ -88,7 +125,8 @@ const test_targets = blk: {...@@ -88,7 +125,8 @@ const test_targets = blk: {
88 // .arch_os_abi = "arm-linux-none",125 // .arch_os_abi = "arm-linux-none",
89 // .cpu_features = "generic+v8a",126 // .cpu_features = "generic+v8a",
90 // }) catch unreachable,127 // }) catch unreachable,
91 // .backend = .stage2_arm,128 // .use_llvm = false,
129 // .use_lld = false,
92 //},130 //},
93 .{131 .{
94 .target = .{132 .target = .{
...@@ -96,7 +134,8 @@ const test_targets = blk: {...@@ -96,7 +134,8 @@ const test_targets = blk: {
96 .os_tag = .macos,134 .os_tag = .macos,
97 .abi = .none,135 .abi = .none,
98 },136 },
99 .backend = .stage2_aarch64,137 .use_llvm = false,
138 .use_lld = false,
100 },139 },
101 .{140 .{
102 .target = .{141 .target = .{
...@@ -104,7 +143,8 @@ const test_targets = blk: {...@@ -104,7 +143,8 @@ const test_targets = blk: {
104 .os_tag = .macos,143 .os_tag = .macos,
105 .abi = .none,144 .abi = .none,
106 },145 },
107 .backend = .stage2_x86_64,146 .use_llvm = false,
147 .use_lld = false,
108 },148 },
109 .{149 .{
110 .target = .{150 .target = .{
...@@ -112,7 +152,8 @@ const test_targets = blk: {...@@ -112,7 +152,8 @@ const test_targets = blk: {
112 .os_tag = .windows,152 .os_tag = .windows,
113 .abi = .gnu,153 .abi = .gnu,
114 },154 },
115 .backend = .stage2_x86_64,155 .use_llvm = false,
156 .use_lld = false,
116 },157 },
117158
118 .{159 .{
...@@ -121,7 +162,6 @@ const test_targets = blk: {...@@ -121,7 +162,6 @@ const test_targets = blk: {
121 .os_tag = .wasi,162 .os_tag = .wasi,
122 },163 },
123 .link_libc = false,164 .link_libc = false,
124 .single_threaded = true,
125 },165 },
126 .{166 .{
127 .target = .{167 .target = .{
...@@ -129,7 +169,6 @@ const test_targets = blk: {...@@ -129,7 +169,6 @@ const test_targets = blk: {
129 .os_tag = .wasi,169 .os_tag = .wasi,
130 },170 },
131 .link_libc = true,171 .link_libc = true,
132 .single_threaded = true,
133 },172 },
134173
135 .{174 .{
...@@ -413,43 +452,6 @@ const test_targets = blk: {...@@ -413,43 +452,6 @@ const test_targets = blk: {
413 },452 },
414 .link_libc = true,453 .link_libc = true,
415 },454 },
416
417 // Do the release tests last because they take a long time
418 .{
419 .optimize_mode = .ReleaseFast,
420 },
421 .{
422 .link_libc = true,
423 .optimize_mode = .ReleaseFast,
424 },
425 .{
426 .optimize_mode = .ReleaseFast,
427 .single_threaded = true,
428 },
429
430 .{
431 .optimize_mode = .ReleaseSafe,
432 },
433 .{
434 .link_libc = true,
435 .optimize_mode = .ReleaseSafe,
436 },
437 .{
438 .optimize_mode = .ReleaseSafe,
439 .single_threaded = true,
440 },
441
442 .{
443 .optimize_mode = .ReleaseSmall,
444 },
445 .{
446 .link_libc = true,
447 .optimize_mode = .ReleaseSmall,
448 },
449 .{
450 .optimize_mode = .ReleaseSmall,
451 .single_threaded = true,
452 },
453 };455 };
454};456};
455457
...@@ -913,8 +915,6 @@ const ModuleTestOptions = struct {...@@ -913,8 +915,6 @@ const ModuleTestOptions = struct {
913 skip_non_native: bool,915 skip_non_native: bool,
914 skip_cross_glibc: bool,916 skip_cross_glibc: bool,
915 skip_libc: bool,917 skip_libc: bool,
916 skip_stage1: bool,
917 skip_stage2: bool,
918 max_rss: usize = 0,918 max_rss: usize = 0,
919};919};
920920
...@@ -922,49 +922,41 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {...@@ -922,49 +922,41 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
922 const step = b.step(b.fmt("test-{s}", .{options.name}), options.desc);922 const step = b.step(b.fmt("test-{s}", .{options.name}), options.desc);
923923
924 for (test_targets) |test_target| {924 for (test_targets) |test_target| {
925 if (options.skip_non_native and !test_target.target.isNative())925 const is_native = test_target.target.isNative() or
926 (test_target.target.getOsTag() == builtin.os.tag and
927 test_target.target.getCpuArch() == builtin.cpu.arch);
928
929 if (options.skip_non_native and !is_native)
926 continue;930 continue;
927931
928 if (options.skip_cross_glibc and test_target.target.isGnuLibC() and test_target.link_libc)932 if (options.skip_cross_glibc and test_target.target.isGnuLibC() and test_target.link_libc == true)
929 continue;933 continue;
930934
931 if (options.skip_libc and test_target.link_libc)935 if (options.skip_libc and test_target.link_libc == true)
932 continue;936 continue;
933937
934 if (test_target.link_libc and test_target.target.getOs().requiresLibC()) {938 if (options.skip_single_threaded and test_target.single_threaded == true)
935 // This would be a redundant test.
936 continue;939 continue;
937 }
938940
939 if (options.skip_single_threaded and test_target.single_threaded)941 // TODO get compiler-rt tests passing for self-hosted backends.
942 if (test_target.use_llvm == false and mem.eql(u8, options.name, "compiler-rt"))
940 continue;943 continue;
941944
942 if (test_target.disable_native and945 // TODO get universal-libc tests passing for self-hosted backends.
943 test_target.target.getOsTag() == builtin.os.tag and946 if (test_target.use_llvm == false and mem.eql(u8, options.name, "universal-libc"))
944 test_target.target.getCpuArch() == builtin.cpu.arch)
945 {
946 continue;947 continue;
947 }
948948
949 if (test_target.backend) |backend| switch (backend) {949 // TODO get std lib tests passing for self-hosted backends.
950 .stage1 => if (options.skip_stage1) continue,950 if (test_target.use_llvm == false and mem.eql(u8, options.name, "std"))
951 .stage2_llvm => {},951 continue;
952 else => if (options.skip_stage2) continue,
953 };
954952
955 const want_this_mode = for (options.optimize_modes) |m| {953 const want_this_mode = for (options.optimize_modes) |m| {
956 if (m == test_target.optimize_mode) break true;954 if (m == test_target.optimize_mode) break true;
957 } else false;955 } else false;
958 if (!want_this_mode) continue;956 if (!want_this_mode) continue;
959957
960 const libc_prefix = if (test_target.target.getOs().requiresLibC())958 const libc_suffix = if (test_target.link_libc == true) "-libc" else "";
961 ""959 const triple_txt = test_target.target.zigTriple(b.allocator) catch @panic("OOM");
962 else if (test_target.link_libc)
963 "c"
964 else
965 "bare";
966
967 const triple_prefix = test_target.target.zigTriple(b.allocator) catch @panic("OOM");
968960
969 // wasm32-wasi builds need more RAM, idk why961 // wasm32-wasi builds need more RAM, idk why
970 const max_rss = if (test_target.target.getOs().tag == .wasi)962 const max_rss = if (test_target.target.getOs().tag == .wasi)
...@@ -978,42 +970,33 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {...@@ -978,42 +970,33 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
978 .target = test_target.target,970 .target = test_target.target,
979 .max_rss = max_rss,971 .max_rss = max_rss,
980 .filter = options.test_filter,972 .filter = options.test_filter,
973 .link_libc = test_target.link_libc,
974 .single_threaded = test_target.single_threaded,
975 .use_llvm = test_target.use_llvm,
976 .use_lld = test_target.use_lld,
981 });977 });
982 const single_threaded_txt = if (test_target.single_threaded) "single" else "multi";978 const single_threaded_suffix = if (test_target.single_threaded == true) "-single" else "";
983 const backend_txt = if (test_target.backend) |backend| @tagName(backend) else "default";979 const backend_suffix = if (test_target.use_llvm == true)
984 these_tests.single_threaded = test_target.single_threaded;980 "-llvm"
985 if (test_target.link_libc) {981 else if (test_target.target.ofmt == std.Target.ObjectFormat.c)
986 these_tests.linkSystemLibrary("c");982 "-cbe"
987 }983 else if (test_target.use_llvm == false)
984 "-selfhosted"
985 else
986 "";
987
988 these_tests.overrideZigLibDir("lib");988 these_tests.overrideZigLibDir("lib");
989 these_tests.addIncludePath("test");989 these_tests.addIncludePath("test");
990 if (test_target.backend) |backend| switch (backend) {
991 .stage1 => {
992 @panic("stage1 testing requested");
993 },
994 .stage2_llvm => {
995 these_tests.use_llvm = true;
996 },
997 .stage2_c => {
998 these_tests.use_llvm = false;
999 },
1000 else => {
1001 these_tests.use_llvm = false;
1002 // TODO: force self-hosted linkers to avoid LLD creeping in
1003 // until the auto-select mechanism deems them worthy
1004 these_tests.use_lld = false;
1005 },
1006 };
1007990
1008 const run = b.addRunArtifact(these_tests);991 const run = b.addRunArtifact(these_tests);
1009 run.skip_foreign_checks = true;992 run.skip_foreign_checks = true;
1010 run.setName(b.fmt("run test {s}-{s}-{s}-{s}-{s}-{s}", .{993 run.setName(b.fmt("run test {s}-{s}-{s}{s}{s}{s}", .{
1011 options.name,994 options.name,
1012 triple_prefix,995 triple_txt,
1013 @tagName(test_target.optimize_mode),996 @tagName(test_target.optimize_mode),
1014 libc_prefix,997 libc_suffix,
1015 single_threaded_txt,998 single_threaded_suffix,
1016 backend_txt,999 backend_suffix,
1017 }));1000 }));
10181001
1019 step.dependOn(&run.step);1002 step.dependOn(&run.step);