From b49d99b246bc18d94b6eed0c225464078b05f12a Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Fri, 30 Apr 2021 23:41:28 +0200 Subject: [PATCH 1/3] Azure CI: Test std.* and the rest separately On CI, we have been running into OOM issues when running the test suite on Windows for quite some time. Unfortunately, we are very close to having the same issues on Linux as well. Some additional comptime work immediately makes these builds fail as well. Add a new `test-toolchain` step, that tests everything except `std.*` and documentation. On CI, call `test-toolchain`, `test-std` and `docs` separately instead of the `test` big hammer that emcompasses all of them. Change the special case we made for Windows to the same code as other platforms. This is a stopgap measure that stage2 will eventually make useless. Until then, it gives us some headroom. Change `linux_script` by the way to only output the log of failing steps. This shrinks the Linux CI log from a bazilion lines down to something more humanely manageable. --- build.zig | 42 +++++++++++++++++--------------- ci/azure/linux_script | 12 ++++++++- ci/azure/macos_script | 14 +++++++++-- ci/azure/windows_msvc_script.bat | 14 +---------- 4 files changed, 47 insertions(+), 35 deletions(-) diff --git a/build.zig b/build.zig index d15766c08ebb3e3ec84ab9d1d1c9938a6e0cb041..a0bc6a89c322834ead1ea5609be04b0ab5e01cec 100644 --- a/build.zig +++ b/build.zig @@ -36,7 +36,7 @@ pub fn build(b: *Builder) !void { const docs_step = b.step("docs", "Build documentation"); docs_step.dependOn(&docgen_cmd.step); - const test_step = b.step("test", "Run all the tests"); + const toolchain_step = b.step("test-toolchain", "Run the tests for the toolchain"); var test_stage2 = b.addTest("src/test.zig"); test_stage2.setBuildMode(mode); @@ -90,7 +90,7 @@ pub fn build(b: *Builder) !void { exe.install(); exe.setBuildMode(mode); exe.setTarget(target); - test_step.dependOn(&exe.step); + toolchain_step.dependOn(&exe.step); b.default_step.dependOn(&exe.step); exe.addBuildOption(bool, "skip_non_native", skip_non_native); @@ -233,7 +233,7 @@ pub fn build(b: *Builder) !void { const test_stage2_step = b.step("test-stage2", "Run the stage2 compiler tests"); test_stage2_step.dependOn(&test_stage2.step); if (!skip_stage2_tests) { - test_step.dependOn(test_stage2_step); + toolchain_step.dependOn(test_stage2_step); } var chosen_modes: [4]builtin.Mode = undefined; @@ -257,33 +257,37 @@ pub fn build(b: *Builder) !void { const modes = chosen_modes[0..chosen_mode_index]; // run stage1 `zig fmt` on this build.zig file just to make sure it works - test_step.dependOn(&fmt_build_zig.step); + toolchain_step.dependOn(&fmt_build_zig.step); const fmt_step = b.step("test-fmt", "Run zig fmt against build.zig to make sure it works"); fmt_step.dependOn(&fmt_build_zig.step); // TODO for the moment, skip wasm32-wasi until bugs are sorted out. - test_step.dependOn(tests.addPkgTests(b, test_filter, "test/stage1/behavior.zig", "behavior", "Run the behavior tests", modes, false, skip_non_native, skip_libc, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir)); + toolchain_step.dependOn(tests.addPkgTests(b, test_filter, "test/stage1/behavior.zig", "behavior", "Run the behavior tests", modes, false, skip_non_native, skip_libc, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir)); - test_step.dependOn(tests.addPkgTests(b, test_filter, "lib/std/std.zig", "std", "Run the standard library tests", modes, false, skip_non_native, skip_libc, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir)); + toolchain_step.dependOn(tests.addPkgTests(b, test_filter, "lib/std/special/compiler_rt.zig", "compiler-rt", "Run the compiler_rt tests", modes, true, skip_non_native, true, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir)); + toolchain_step.dependOn(tests.addPkgTests(b, test_filter, "lib/std/special/c.zig", "minilibc", "Run the mini libc tests", modes, true, skip_non_native, true, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir)); - test_step.dependOn(tests.addPkgTests(b, test_filter, "lib/std/special/compiler_rt.zig", "compiler-rt", "Run the compiler_rt tests", modes, true, skip_non_native, true, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir)); - test_step.dependOn(tests.addPkgTests(b, test_filter, "lib/std/special/c.zig", "minilibc", "Run the mini libc tests", modes, true, skip_non_native, true, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir)); - - test_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes)); - test_step.dependOn(tests.addStandaloneTests(b, test_filter, modes)); - test_step.dependOn(tests.addStackTraceTests(b, test_filter, modes)); - test_step.dependOn(tests.addCliTests(b, test_filter, modes)); - test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes)); - test_step.dependOn(tests.addRuntimeSafetyTests(b, test_filter, modes)); - test_step.dependOn(tests.addTranslateCTests(b, test_filter)); + toolchain_step.dependOn(tests.addCompareOutputTests(b, test_filter, modes)); + toolchain_step.dependOn(tests.addStandaloneTests(b, test_filter, modes)); + toolchain_step.dependOn(tests.addStackTraceTests(b, test_filter, modes)); + toolchain_step.dependOn(tests.addCliTests(b, test_filter, modes)); + toolchain_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, modes)); + toolchain_step.dependOn(tests.addRuntimeSafetyTests(b, test_filter, modes)); + toolchain_step.dependOn(tests.addTranslateCTests(b, test_filter)); if (!skip_run_translated_c) { - test_step.dependOn(tests.addRunTranslatedCTests(b, test_filter, target)); + toolchain_step.dependOn(tests.addRunTranslatedCTests(b, test_filter, target)); } // tests for this feature are disabled until we have the self-hosted compiler available - // test_step.dependOn(tests.addGenHTests(b, test_filter)); + // toolchain_step.dependOn(tests.addGenHTests(b, test_filter)); if (!skip_compile_errors) { - test_step.dependOn(tests.addCompileErrorTests(b, test_filter, modes)); + toolchain_step.dependOn(tests.addCompileErrorTests(b, test_filter, modes)); } + + const std_step = tests.addPkgTests(b, test_filter, "lib/std/std.zig", "std", "Run the standard library tests", modes, false, skip_non_native, skip_libc, is_wine_enabled, is_qemu_enabled, is_wasmtime_enabled, glibc_multi_dir); + + const test_step = b.step("test", "Run all the tests"); + test_step.dependOn(toolchain_step); + test_step.dependOn(std_step); test_step.dependOn(docs_step); } diff --git a/ci/azure/linux_script b/ci/azure/linux_script index d9e017cc1dfc6cf69cd3310dd41df7d7ca7a4a8d..2b0e9c46aa60abce5eb69d993694b78c35db9b59 100755 --- a/ci/azure/linux_script +++ b/ci/azure/linux_script @@ -65,7 +65,17 @@ make $JOBS install cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig" make $JOBS install -release/bin/zig build test -Denable-qemu -Denable-wasmtime +set +x +LOG=$(mktemp) +for step in test-toolchain test-std docs; do + echo "* Running step: [$step]" + if ! release/bin/zig build $step -Denable-qemu -Denable-wasmtime 2>"$LOG" >&2; then + cat "$LOG" >&2 + exit 1 + fi + echo " Done." +done +set -x # Look for HTML errors. tidy -qe ../zig-cache/langref.html diff --git a/ci/azure/macos_script b/ci/azure/macos_script index da3d4de915bcb626ca699711155c1fe3c81726dd..c05a6871fb342190eeb6d37328e42e99e5a928e7 100755 --- a/ci/azure/macos_script +++ b/ci/azure/macos_script @@ -18,7 +18,7 @@ tar xf "$CACHE_BASENAME.tar.xz" ZIG="$PREFIX/bin/zig" NATIVE_LIBC_TXT="$HOME/native_libc.txt" -$ZIG libc > "$NATIVE_LIBC_TXT" +$ZIG libc >"$NATIVE_LIBC_TXT" export ZIG_LIBC="$NATIVE_LIBC_TXT" export CC="$ZIG cc" export CXX="$ZIG c++" @@ -55,7 +55,17 @@ make $JOBS install cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig" -DZIG_TARGET_MCPU="x86_64_v2" make $JOBS install -release/bin/zig build test +set +x +LOG=$(mktemp) +for step in test-toolchain test-std docs; do + echo "* Running step: [$step]" + if ! release/bin/zig build $step -Denable-qemu -Denable-wasmtime 2>"$LOG" >&2; then + cat "$LOG" >&2 + exit 1 + fi + echo " Done." +done +set -x if [ "${BUILD_REASON}" != "PullRequest" ]; then mv ../LICENSE release/ diff --git a/ci/azure/windows_msvc_script.bat b/ci/azure/windows_msvc_script.bat index e45b53f4adeb7d82b8f3606e4977951fb680c02e..fc17c80ea2b48b7de6f7c7bd5cf57edf00cb1ab6 100644 --- a/ci/azure/windows_msvc_script.bat +++ b/ci/azure/windows_msvc_script.bat @@ -26,20 +26,8 @@ cd %ZIGBUILDDIR% cmake.exe .. -Thost=x64 -G"Visual Studio 16 2019" -A x64 "-DCMAKE_INSTALL_PREFIX=%ZIGINSTALLDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release -DZIG_OMIT_STAGE2=ON || exit /b msbuild /maxcpucount /p:Configuration=Release INSTALL.vcxproj || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-behavior -Dskip-non-native || exit /b -REM Disabled to prevent OOM -REM "%ZIGINSTALLDIR%\bin\zig.exe" build test-stage2 -Dskip-non-native || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-fmt -Dskip-non-native || exit /b +"%ZIGINSTALLDIR%\bin\zig.exe" build test-toolchain -Dskip-non-native -Dskip-stage2-tests || exit /b "%ZIGINSTALLDIR%\bin\zig.exe" build test-std -Dskip-non-native || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-compiler-rt -Dskip-non-native || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-compare-output -Dskip-non-native || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-standalone -Dskip-non-native || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-stack-traces -Dskip-non-native || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-cli -Dskip-non-native || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-asm-link -Dskip-non-native || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-runtime-safety -Dskip-non-native || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-translate-c -Dskip-non-native || exit /b -"%ZIGINSTALLDIR%\bin\zig.exe" build test-run-translated-c -Dskip-non-native || exit /b "%ZIGINSTALLDIR%\bin\zig.exe" build docs || exit /b set "PATH=%CD:~0,2%\msys64\usr\bin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem" -- 2.54.0 From 3e389d01ed13374aa3a362dfa505eb00718b318c Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Sat, 1 May 2021 00:38:54 +0200 Subject: [PATCH 2/3] CI: no need to try qemu and wasmtime on macOS builds --- ci/azure/macos_script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/macos_script b/ci/azure/macos_script index c05a6871fb342190eeb6d37328e42e99e5a928e7..4aa762935242d04da9d977d053d5d1dace253c17 100755 --- a/ci/azure/macos_script +++ b/ci/azure/macos_script @@ -59,7 +59,7 @@ set +x LOG=$(mktemp) for step in test-toolchain test-std docs; do echo "* Running step: [$step]" - if ! release/bin/zig build $step -Denable-qemu -Denable-wasmtime 2>"$LOG" >&2; then + if ! release/bin/zig build $step 2>"$LOG" >&2; then cat "$LOG" >&2 exit 1 fi -- 2.54.0 From 9a637f81e021cacd8e38be6793a86b01ef735704 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Sun, 2 May 2021 02:07:59 +0200 Subject: [PATCH 3/3] Restore the CI logging as before --- ci/azure/linux_script | 10 +--------- ci/azure/macos_script | 10 +--------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/ci/azure/linux_script b/ci/azure/linux_script index 2b0e9c46aa60abce5eb69d993694b78c35db9b59..912a2518bbaeca53eaced2d8144aed74156d3fbd 100755 --- a/ci/azure/linux_script +++ b/ci/azure/linux_script @@ -65,17 +65,9 @@ make $JOBS install cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig" make $JOBS install -set +x -LOG=$(mktemp) for step in test-toolchain test-std docs; do - echo "* Running step: [$step]" - if ! release/bin/zig build $step -Denable-qemu -Denable-wasmtime 2>"$LOG" >&2; then - cat "$LOG" >&2 - exit 1 - fi - echo " Done." + release/bin/zig build $step -Denable-qemu -Denable-wasmtime done -set -x # Look for HTML errors. tidy -qe ../zig-cache/langref.html diff --git a/ci/azure/macos_script b/ci/azure/macos_script index 4aa762935242d04da9d977d053d5d1dace253c17..d6d32612cc50d8e6a50e4094501cccde16b811d9 100755 --- a/ci/azure/macos_script +++ b/ci/azure/macos_script @@ -55,17 +55,9 @@ make $JOBS install cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig" -DZIG_TARGET_MCPU="x86_64_v2" make $JOBS install -set +x -LOG=$(mktemp) for step in test-toolchain test-std docs; do - echo "* Running step: [$step]" - if ! release/bin/zig build $step 2>"$LOG" >&2; then - cat "$LOG" >&2 - exit 1 - fi - echo " Done." + release/bin/zig build $step done -set -x if [ "${BUILD_REASON}" != "PullRequest" ]; then mv ../LICENSE release/ -- 2.54.0