diff --git a/CMakeLists.txt b/CMakeLists.txt index 3211a15d4ddeeb9fe0ffa834482aa4e8ad486621..e9c24112daefb6350d78de46f765828eab78a7a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -335,11 +335,6 @@ set(ZIG_STAGE2_SOURCES src/Compilation.zig src/Compilation/Config.zig src/InternPool.zig - src/Package.zig - src/Package/Fetch.zig - src/Package/Fetch/git.zig - src/Package/Manifest.zig - src/Package/Module.zig src/RangeSet.zig src/Sema.zig src/Sema/reinterpret.zig @@ -369,7 +364,6 @@ set(ZIG_STAGE2_SOURCES src/libs/glibc.zig src/libs/netbsd.zig src/libs/openbsd.zig - src/introspect.zig src/libs/libcxx.zig src/libs/libtsan.zig src/libs/libunwind.zig @@ -735,8 +729,6 @@ endif() set(ZIG_BUILD_ARGS - --zig-lib-dir "${PROJECT_SOURCE_DIR}/lib" - "-Dversion-string=${RESOLVED_ZIG_VERSION}" "-Dtarget=${ZIG_TARGET_TRIPLE}" "-Dcpu=${ZIG_TARGET_MCPU}" diff --git a/build.zig b/build.zig index ebf163bf5c136a61ecad9120c75c4215cdd8345e..c98dcc649ca0773a12490c2d43dddc1ff344b873 100644 --- a/build.zig +++ b/build.zig @@ -175,6 +175,9 @@ pub fn build(b: *std.Build) !void { ".tar", // exclude files from lib/std/zip/testdata ".zip", + // exclude files from lib/compiler/Maker/Fetch/git/testdata + ".idx", + ".pack", // others "README.md", }, @@ -264,9 +267,8 @@ pub fn build(b: *std.Build) !void { std.process.exit(1); } - // Ensure git version changes get picked up - // https://codeberg.org/ziglang/zig/issues/35473 - b.graph.poisonCache(); + // Ensure git version changes get picked up. + b.dependOnFileContents(b.path(".git/HEAD")); const version_string = b.fmt("{d}.{d}.{d}", .{ zig_version.major, zig_version.minor, zig_version.patch }); diff --git a/build.zig.zon b/build.zig.zon index 4f4b9482178f3e283149d0a7e651a8d98a4893c7..5f21b86ab2ba271758c3f425709558cd77bc2fa8 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -7,9 +7,6 @@ .standalone_test_cases = .{ .path = "test/standalone", }, - .link_test_cases = .{ - .path = "test/link", - }, }, .paths = .{""}, .fingerprint = 0xc1ce108124179e16, diff --git a/ci/aarch64-freebsd-debug.sh b/ci/aarch64-freebsd-debug.sh index f84a683771b456066ce5c7997b12bae71aac27bb..f1e53270618c5954a0554934a4dd90e81e8d5e3f 100755 --- a/ci/aarch64-freebsd-debug.sh +++ b/ci/aarch64-freebsd-debug.sh @@ -40,12 +40,14 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-debug/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dstatic-llvm \ -Dskip-non-native \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m stage3-debug/bin/zig build \ diff --git a/ci/aarch64-freebsd-release.sh b/ci/aarch64-freebsd-release.sh index e008d4ee81fb11bdeac967f93965d7bffb754c5d..0f5a949b7a89c0256f4c9d4d8e85607756b88c00 100755 --- a/ci/aarch64-freebsd-release.sh +++ b/ci/aarch64-freebsd-release.sh @@ -40,12 +40,14 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-release/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dstatic-llvm \ -Dskip-non-native \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m # Ensure that stage3 and stage4 are byte-for-byte identical. diff --git a/ci/aarch64-linux-debug.sh b/ci/aarch64-linux-debug.sh index 5ee0a33c1ccce4272f0a26a833c4cee56e5b7db5..c8497814fdc6688e6b1d9f6d307afbe9bbd0ac88 100755 --- a/ci/aarch64-linux-debug.sh +++ b/ci/aarch64-linux-debug.sh @@ -42,6 +42,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts. stage3-debug/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ @@ -49,7 +52,6 @@ stage3-debug/bin/zig build test docs \ -Dskip-non-native \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ -Denable-superhtml \ --test-timeout 3m diff --git a/ci/aarch64-linux-release.sh b/ci/aarch64-linux-release.sh index ae5203195d5a17d6fdd4b9970b3802c3d4115ca1..d6bf710555d13145ea7978d152561e0cc2f87d9b 100755 --- a/ci/aarch64-linux-release.sh +++ b/ci/aarch64-linux-release.sh @@ -42,6 +42,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts. stage3-release/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ @@ -49,7 +52,6 @@ stage3-release/bin/zig build test docs \ -Dskip-non-native \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ -Denable-superhtml \ --test-timeout 3m diff --git a/ci/aarch64-macos-debug.sh b/ci/aarch64-macos-debug.sh index dd959ef0c97d306d3f79a40d61d0eba87ae5c268..226b3109eafa9e8de169999b52fcd748ddb7c088 100755 --- a/ci/aarch64-macos-debug.sh +++ b/ci/aarch64-macos-debug.sh @@ -43,9 +43,11 @@ cmake .. \ ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-debug/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ - --zig-lib-dir "$PWD/../lib" \ -Denable-macos-sdk \ -Dstatic-llvm \ -Dskip-spirv \ diff --git a/ci/aarch64-macos-release.sh b/ci/aarch64-macos-release.sh index 5f2012d268fd097215113c760320092aba529ad0..b1b431f257fd867bf2bacea31dc63dff3d4a3acb 100755 --- a/ci/aarch64-macos-release.sh +++ b/ci/aarch64-macos-release.sh @@ -43,8 +43,10 @@ cmake .. \ ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-release/bin/zig build test docs \ - --zig-lib-dir "$PWD/../lib" \ -Denable-macos-sdk \ -Dstatic-llvm \ -Dskip-spirv \ diff --git a/ci/aarch64-netbsd-debug.sh b/ci/aarch64-netbsd-debug.sh index 1fedad9875d77822bd5473b3603248ecaece2314..a7d7d3e97920a95ff4006792c1bea005bc648583 100755 --- a/ci/aarch64-netbsd-debug.sh +++ b/ci/aarch64-netbsd-debug.sh @@ -40,12 +40,14 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-debug/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dstatic-llvm \ -Dskip-non-native \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m stage3-debug/bin/zig build \ diff --git a/ci/aarch64-netbsd-release.sh b/ci/aarch64-netbsd-release.sh index db59d529e928f0059f664bd3557cde654d803f92..e0a4c9aaa03f87770b013be6aaf92ffddc6d2bbc 100755 --- a/ci/aarch64-netbsd-release.sh +++ b/ci/aarch64-netbsd-release.sh @@ -40,12 +40,14 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-release/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dstatic-llvm \ -Dskip-non-native \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m # Ensure that stage3 and stage4 are byte-for-byte identical. diff --git a/ci/aarch64-windows.ps1 b/ci/aarch64-windows.ps1 index c711127a61fd0a8c48617a67199757fb2c3a7ae1..3268dfd1e4eec456d247780beab3ae2542eb6897 100644 --- a/ci/aarch64-windows.ps1 +++ b/ci/aarch64-windows.ps1 @@ -4,7 +4,6 @@ $MCPU = "baseline" $ZIG_LLVM_CLANG_LLD_URL = "https://ziglang.org/deps/$ZIG_LLVM_CLANG_LLD_NAME.zip" $PREFIX_PATH = "$(Get-Location)\..\$ZIG_LLVM_CLANG_LLD_NAME" $ZIG = "$PREFIX_PATH\bin\zig.exe" -$ZIG_LIB_DIR = "$(Get-Location)\lib" $ZSF_MAX_RSS = if ($Env:ZSF_MAX_RSS) { $Env:ZSF_MAX_RSS } else { 0 } if (!(Test-Path "..\$ZIG_LLVM_CLANG_LLD_NAME.zip")) { @@ -53,10 +52,12 @@ CheckLastExitCode ninja install CheckLastExitCode +# Must be done after zig cc is finished. +$Env:ZIG_LIB_DIR="$(Get-Location)\..\lib" + Write-Output "Main test suite..." & "stage3-release\bin\zig.exe" build test docs ` --maxrss $ZSF_MAX_RSS ` - --zig-lib-dir "$ZIG_LIB_DIR" ` --search-prefix "$PREFIX_PATH" ` -Dstatic-llvm ` -Dskip-non-native ` diff --git a/ci/loongarch64-linux-debug.sh b/ci/loongarch64-linux-debug.sh index f980744f8d0d3f7f790e88147635691cbee720f7..b3460be8fe4585bda04f9702c6d666dcf47949c5 100755 --- a/ci/loongarch64-linux-debug.sh +++ b/ci/loongarch64-linux-debug.sh @@ -40,6 +40,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts. stage3-debug/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ @@ -47,7 +50,6 @@ stage3-debug/bin/zig build test docs \ -Dskip-non-native \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m stage3-debug/bin/zig build \ diff --git a/ci/loongarch64-linux-release.sh b/ci/loongarch64-linux-release.sh index 0343259929fb06d39cfeda396c326f9b0cabbba9..4c1451a09003a53cde93328d34aad08c7de272f0 100755 --- a/ci/loongarch64-linux-release.sh +++ b/ci/loongarch64-linux-release.sh @@ -40,6 +40,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts. stage3-release/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ @@ -47,7 +50,6 @@ stage3-release/bin/zig build test docs \ -Dskip-non-native \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m # Ensure that stage3 and stage4 are byte-for-byte identical. diff --git a/ci/powerpc64le-linux-debug.sh b/ci/powerpc64le-linux-debug.sh index cbdfd9f48fa7c228e499c23b93ca71879c5f8474..442a7a03c3637b95249b4273469348a418b784ce 100755 --- a/ci/powerpc64le-linux-debug.sh +++ b/ci/powerpc64le-linux-debug.sh @@ -40,6 +40,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts. stage3-debug/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ @@ -48,7 +51,6 @@ stage3-debug/bin/zig build test docs \ -Dtarget=native-native-musl \ -Dcpu=native+longcall \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m stage3-debug/bin/zig build \ diff --git a/ci/powerpc64le-linux-release.sh b/ci/powerpc64le-linux-release.sh index 25150408a4ef2161eb171371796b229a5f768ed6..c03c1504febc9e8039a74781c091fc999dd5d9e8 100755 --- a/ci/powerpc64le-linux-release.sh +++ b/ci/powerpc64le-linux-release.sh @@ -40,6 +40,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts. stage3-release/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ @@ -48,7 +51,6 @@ stage3-release/bin/zig build test docs \ -Dtarget=native-native-musl \ -Dcpu=native+longcall \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m # Ensure that stage3 and stage4 are byte-for-byte identical. diff --git a/ci/riscv64-linux-debug.sh b/ci/riscv64-linux-debug.sh index 631573cfec76fc58be737b087a60a52ed0b04887..f01fc2bb09d763e3dbd06abbc055148d7d1b5ba0 100755 --- a/ci/riscv64-linux-debug.sh +++ b/ci/riscv64-linux-debug.sh @@ -42,6 +42,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts. stage3-debug/bin/zig build test-modules test-c-abi \ --maxrss ${ZSF_MAX_RSS:-0} \ @@ -50,5 +53,4 @@ stage3-debug/bin/zig build test-modules test-c-abi \ -Dskip-single-threaded \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m diff --git a/ci/riscv64-linux-release.sh b/ci/riscv64-linux-release.sh index 1f51b7d8c2931b4344bc05f9d7e76021ec35e590..34719e9d0d9595ff05f9d99f600b74347ddcb208 100755 --- a/ci/riscv64-linux-release.sh +++ b/ci/riscv64-linux-release.sh @@ -42,6 +42,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts. stage3-release/bin/zig build test-modules test-c-abi \ --maxrss ${ZSF_MAX_RSS:-0} \ @@ -50,5 +53,4 @@ stage3-release/bin/zig build test-modules test-c-abi \ -Dskip-single-threaded \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m diff --git a/ci/s390x-linux-debug.sh b/ci/s390x-linux-debug.sh index f4d601ce4bc74fe5244b3bd596f4e75684eaefe1..3d97f9c1650189ae1f105553257dd692e94f818d 100755 --- a/ci/s390x-linux-debug.sh +++ b/ci/s390x-linux-debug.sh @@ -40,6 +40,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts. stage3-debug/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ @@ -47,7 +50,6 @@ stage3-debug/bin/zig build test docs \ -Dskip-non-native \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m stage3-debug/bin/zig build \ diff --git a/ci/s390x-linux-release.sh b/ci/s390x-linux-release.sh index db70925e67b6afea80b483821e0a9a47d011f4a0..a8c5a9fcb341bb616729c5ab792acff29ca3db02 100755 --- a/ci/s390x-linux-release.sh +++ b/ci/s390x-linux-release.sh @@ -40,6 +40,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # No -fqemu and -fwasmtime here as they're covered by the x86_64-linux scripts. stage3-release/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ @@ -47,7 +50,6 @@ stage3-release/bin/zig build test docs \ -Dskip-non-native \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 4m # Ensure that stage3 and stage4 are byte-for-byte identical. diff --git a/ci/x86_64-freebsd-debug.sh b/ci/x86_64-freebsd-debug.sh index ec8d0db1ad77db75111ad334edca9094ffc03419..c67c7bbe4d50bdd2448a83e0479ff61b65cc5a62 100755 --- a/ci/x86_64-freebsd-debug.sh +++ b/ci/x86_64-freebsd-debug.sh @@ -40,6 +40,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-debug/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dstatic-llvm \ @@ -51,7 +54,6 @@ stage3-debug/bin/zig build test docs \ -Dskip-windows \ -Dskip-darwin \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 2m stage3-debug/bin/zig build \ diff --git a/ci/x86_64-freebsd-release.sh b/ci/x86_64-freebsd-release.sh index f3d50ee920e1ff5ba8cf9a128b1c4a8f32e5df0d..83c88ef5aae163deee3cebd40c1b2b1f01047930 100755 --- a/ci/x86_64-freebsd-release.sh +++ b/ci/x86_64-freebsd-release.sh @@ -40,6 +40,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-release/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dstatic-llvm \ @@ -51,7 +54,6 @@ stage3-release/bin/zig build test docs \ -Dskip-windows \ -Dskip-darwin \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 2m # Ensure that the fuzzer at least compiles. diff --git a/ci/x86_64-linux-debug-llvm.sh b/ci/x86_64-linux-debug-llvm.sh index 77579c827ba9414d773afddb1c8da47a5230e518..408607dbc632d110e82cca0dadcd0980b3bebe85 100755 --- a/ci/x86_64-linux-debug-llvm.sh +++ b/ci/x86_64-linux-debug-llvm.sh @@ -43,6 +43,9 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + # simultaneously test building self-hosted without LLVM and with 32-bit arm stage3-debug/bin/zig build \ -Dtarget=arm-linux-musleabihf \ @@ -63,7 +66,6 @@ stage3-debug/bin/zig build test docs \ -Dskip-darwin \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ -Denable-superhtml \ --test-timeout 12m diff --git a/ci/x86_64-linux-debug.sh b/ci/x86_64-linux-debug.sh index 690b9c3314700f6a4b48ac7bc29adfe2e90bace4..0316d3bef37f4058d256c97b62ae0e212160aead 100755 --- a/ci/x86_64-linux-debug.sh +++ b/ci/x86_64-linux-debug.sh @@ -42,13 +42,16 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" +export ZIG_DEBUG_MAKER=1 + # simultaneously test building self-hosted without LLVM and with 32-bit arm stage3-debug/bin/zig build \ -Dtarget=arm-linux-musleabihf \ -Dno-lib stage3-debug/bin/zig build test docs \ - --maker-opt=Debug \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dlldb=$HOME/deps/lldb-zig/Debug-7c1090fd46/bin/lldb \ -fqemu \ @@ -63,7 +66,6 @@ stage3-debug/bin/zig build test docs \ -Dskip-llvm \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ -Denable-superhtml \ --test-timeout 10m diff --git a/ci/x86_64-linux-release.sh b/ci/x86_64-linux-release.sh index 815e85bf60f497585ca2d34704c6c8bd4d0d0e89..64c6589fba220c76e25e9d80f950efc97f777df4 100755 --- a/ci/x86_64-linux-release.sh +++ b/ci/x86_64-linux-release.sh @@ -48,6 +48,9 @@ unset CXX ninja install +# Must not be set while using the other `zig cc` which has its own zig lib dir. +export ZIG_LIB_DIR="$PWD/../lib" + # Covers several things: # 1. building the compiler without LLVM # 2. 32-bit @@ -66,7 +69,6 @@ stage3-release/bin/zig build test docs \ -Dstatic-llvm \ -Dtarget=native-native-musl \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ -Denable-superhtml \ --test-timeout 12m @@ -97,6 +99,7 @@ cd ../build-new export CC="$ZIG cc -target $TARGET -mcpu=$MCPU" export CXX="$ZIG c++ -target $TARGET -mcpu=$MCPU" +unset ZIG_LIB_DIR cmake .. \ -DCMAKE_PREFIX_PATH="$PREFIX" \ @@ -115,11 +118,12 @@ unset CXX ninja install +export ZIG_LIB_DIR="$PWD/../lib" + stage3/bin/zig test ../test/behavior.zig stage3/bin/zig build -p stage4 \ -Dstatic-llvm \ -Dtarget=native-native-musl \ -Dno-lib \ - --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" + --search-prefix "$PREFIX" stage4/bin/zig test ../test/behavior.zig diff --git a/ci/x86_64-netbsd-debug.sh b/ci/x86_64-netbsd-debug.sh index 3755131c6d1a815ecf1e1392f1e14be759af0ecb..ef16be259f9a5c692d63e094f974c81ee4e0e62d 100755 --- a/ci/x86_64-netbsd-debug.sh +++ b/ci/x86_64-netbsd-debug.sh @@ -40,12 +40,14 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-debug/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dstatic-llvm \ -Dskip-non-native \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 2m stage3-debug/bin/zig build \ diff --git a/ci/x86_64-netbsd-release.sh b/ci/x86_64-netbsd-release.sh index 2a3dde40bd6f28f4c2ccec953307a29178d79d9d..f9fc7d6aea7fa0ed44223544052dd084a06996fd 100755 --- a/ci/x86_64-netbsd-release.sh +++ b/ci/x86_64-netbsd-release.sh @@ -40,12 +40,14 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-release/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dstatic-llvm \ -Dskip-non-native \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 2m # Ensure that the fuzzer at least compiles. diff --git a/ci/x86_64-openbsd-debug.sh b/ci/x86_64-openbsd-debug.sh index e85befacfbf653af9d650a898effee6c860cda05..3aac9c636bc642dca609d562c6f721ce6cce8665 100755 --- a/ci/x86_64-openbsd-debug.sh +++ b/ci/x86_64-openbsd-debug.sh @@ -40,12 +40,14 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-debug/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dstatic-llvm \ -Dskip-non-native \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 2m stage3-debug/bin/zig build \ diff --git a/ci/x86_64-openbsd-release.sh b/ci/x86_64-openbsd-release.sh index ea9c44df11318118c0dbd7cc1a01838adc2a0116..76dd9a8b9c0a41efe1a89a21a7125f56f6066d44 100755 --- a/ci/x86_64-openbsd-release.sh +++ b/ci/x86_64-openbsd-release.sh @@ -40,12 +40,14 @@ unset CXX ninja install +# Must be done after zig cc is finished. +export ZIG_LIB_DIR="$PWD/../lib" + stage3-release/bin/zig build test docs \ --maxrss ${ZSF_MAX_RSS:-0} \ -Dstatic-llvm \ -Dskip-non-native \ --search-prefix "$PREFIX" \ - --zig-lib-dir "$PWD/../lib" \ --test-timeout 2m # Ensure that the fuzzer at least compiles. diff --git a/ci/x86_64-windows-debug.ps1 b/ci/x86_64-windows-debug.ps1 index a00f6812e0382c5257cf00ae22cd7dd0de3d277a..fa416bf3ff1207dc26c6ebf497608940d8fc52e7 100644 --- a/ci/x86_64-windows-debug.ps1 +++ b/ci/x86_64-windows-debug.ps1 @@ -2,7 +2,6 @@ $TARGET = "x86_64-windows-gnu" $MCPU = "baseline" $PREFIX_PATH = "$($Env:USERPROFILE)\deps\zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523" $ZIG = "$PREFIX_PATH\bin\zig.exe" -$ZIG_LIB_DIR = "$(Get-Location)\lib" $ZSF_MAX_RSS = if ($Env:ZSF_MAX_RSS) { $Env:ZSF_MAX_RSS } else { 0 } function CheckLastExitCode { @@ -42,10 +41,12 @@ CheckLastExitCode ninja install CheckLastExitCode +# Must be done after zig cc is finished. +$Env:ZIG_LIB_DIR="$(Get-Location)\..\lib" + Write-Output "Main test suite..." stage3-debug\bin\zig build test docs ` --maxrss $ZSF_MAX_RSS ` - --zig-lib-dir "$ZIG_LIB_DIR" ` --search-prefix "$PREFIX_PATH" ` -Dstatic-llvm ` -Dskip-non-native ` @@ -56,7 +57,6 @@ CheckLastExitCode Write-Output "Build x86_64-windows-msvc behavior tests using the C backend..." stage3-debug\bin\zig build-obj ` - --zig-lib-dir "$ZIG_LIB_DIR" ` -ofmt=c ` -OReleaseSmall ` --name compiler_rt ` @@ -67,7 +67,6 @@ stage3-debug\bin\zig build-obj ` CheckLastExitCode stage3-debug\bin\zig test ` - --zig-lib-dir "$ZIG_LIB_DIR" ` -ofmt=c ` -femit-bin="behavior-x86_64-windows-msvc.c" ` --test-no-exec ` diff --git a/ci/x86_64-windows-release.ps1 b/ci/x86_64-windows-release.ps1 index 14d06818983be638497bc190e825fcb7734c8c92..8364836291d8d57097f7a76cba93b651c88c1e94 100644 --- a/ci/x86_64-windows-release.ps1 +++ b/ci/x86_64-windows-release.ps1 @@ -2,7 +2,6 @@ $TARGET = "x86_64-windows-gnu" $MCPU = "baseline" $PREFIX_PATH = "$($Env:USERPROFILE)\deps\zig+llvm+lld+clang-$TARGET-0.17.0-dev.203+073889523" $ZIG = "$PREFIX_PATH\bin\zig.exe" -$ZIG_LIB_DIR = "$(Get-Location)\lib" $ZSF_MAX_RSS = if ($Env:ZSF_MAX_RSS) { $Env:ZSF_MAX_RSS } else { 0 } function CheckLastExitCode { @@ -42,10 +41,12 @@ CheckLastExitCode ninja install CheckLastExitCode +# Must be done after zig cc is finished. +$Env:ZIG_LIB_DIR="$(Get-Location)\..\lib" + Write-Output "Main test suite..." stage3-release\bin\zig.exe build test docs ` --maxrss $ZSF_MAX_RSS ` - --zig-lib-dir "$ZIG_LIB_DIR" ` --search-prefix "$PREFIX_PATH" ` -Dstatic-llvm ` -Dskip-non-native ` @@ -82,7 +83,6 @@ CheckLastExitCode Write-Output "Build x86_64-windows-msvc behavior tests using the C backend..." stage3-release\bin\zig.exe build-obj ` - --zig-lib-dir "$ZIG_LIB_DIR" ` -ofmt=c ` -OReleaseSmall ` --name compiler_rt ` @@ -93,7 +93,6 @@ stage3-release\bin\zig.exe build-obj ` CheckLastExitCode stage3-release\bin\zig.exe test ` - --zig-lib-dir "$ZIG_LIB_DIR" ` -ofmt=c ` -femit-bin="behavior-x86_64-windows-msvc.c" ` --test-no-exec ` diff --git a/lib/compiler/Maker.zig b/lib/compiler/Maker.zig index c31ba826d00a25807284c75e2209388d5e17b4d7..46653431cb447f1d773c6838681a06e7c1c26cf0 100644 --- a/lib/compiler/Maker.zig +++ b/lib/compiler/Maker.zig @@ -1,5 +1,7 @@ const Maker = @This(); + const builtin = @import("builtin"); +const native_os = builtin.os.tag; const std = @import("std"); const Allocator = std.mem.Allocator; @@ -17,6 +19,9 @@ const log = std.log; const mem = std.mem; const process = std.process; const Color = std.zig.Color; +const EnvVar = std.zig.EnvVar; +const default_local_zig_cache_basename = std.zig.default_local_zig_cache_basename; +const stringToEnum = std.meta.stringToEnum; const Fuzz = @import("Maker/Fuzz.zig"); const Graph = @import("Maker/Graph.zig"); @@ -25,10 +30,11 @@ const Watch = @import("Maker/Watch.zig"); const WebServer = @import("Maker/WebServer.zig"); const ScannedConfig = @import("Maker/ScannedConfig.zig"); const PkgConfig = @import("Maker/PkgConfig.zig"); +const Fetch = @import("Maker/Fetch.zig"); +const Package = @import("Maker/Package.zig"); pub const std_options: std.Options = .{ .side_channels_mitigations = .none, - .http_disable_tls = true, }; gpa: Allocator, @@ -45,7 +51,7 @@ max_rss_mutex: Io.Mutex, skip_oom_steps: bool, unit_test_timeout_ns: ?u64, watch: bool, -web_server: if (!builtin.single_threaded) ?WebServer else ?noreturn, +web_server: ?*AvoidableWebServer, /// Allocated into `gpa`. memory_blocked_steps: std.ArrayList(Configuration.Step.Index), /// Allocated into `gpa`. @@ -61,6 +67,8 @@ var stdio_buffer_allocation: [256]u8 = undefined; var stdout_writer_allocation: Io.File.Writer = undefined; var debug_maker_leaks: bool = false; +const AvoidableWebServer = if (builtin.single_threaded) void else WebServer; + const is_debug_mode = builtin.mode == .Debug; const use_safe_allocator = switch (builtin.mode) { .Debug, .ReleaseSafe => true, @@ -99,11 +107,35 @@ const ErrorStyle = enum { }; const MultilineErrors = enum { indent, newline, none }; const Summary = enum { all, new, failures, line, none }; +const PrintConfiguration = enum { none, zon, path }; + +/// Used to build the -M flags to pass to build-exe. +pub const CliModule = struct { + name: []const u8, + root_path: []const u8, + deps: Deps = .empty, + + const Deps = std.array_hash_map.String(*CliModule); + + fn lower(cm: *const CliModule, arena: Allocator, gpa: Allocator, argv: *std.ArrayList([]const u8)) !void { + try argv.ensureUnusedCapacity(gpa, 2 * cm.deps.count() + 1); + for (cm.deps.keys(), cm.deps.values()) |name, dep| { + argv.appendAssumeCapacity("--dep"); + if (mem.eql(u8, name, dep.name)) { + argv.appendAssumeCapacity(dep.name); + } else { + argv.appendAssumeCapacity(try arena.print("{s}={s}", .{ name, dep.name })); + } + } + argv.appendAssumeCapacity(try arena.print("-M{s}={s}", .{ cm.name, cm.root_path })); + } +}; pub fn main(init: process.Init.Minimal) !void { // The build runner is long-lived in the following use cases: // * `--watch` mode // * `--webui` mode + // * `--fuzz` mode // * A project that has a large, complex build graph. const gpa = if (use_safe_allocator) safe_allocator_instance.allocator() else std.heap.smp_allocator; defer if (use_safe_allocator) { @@ -117,77 +149,63 @@ pub fn main(init: process.Init.Minimal) !void { defer threaded.deinit(); const io = threaded.io(); - // ...but we'll back our arena by `std.heap.page_allocator` for efficiency. var arena_instance: std.heap.ArenaAllocator = .init(std.heap.page_allocator); defer arena_instance.deinit(); defer if (debugMakerLeaks()) log.debug("used {Bi} of arena", .{arena_instance.queryCapacity()}); const arena = arena_instance.allocator(); const args = try init.args.toSlice(arena); - - // skip my own exe name - var arg_idx: usize = 1; - - const zig_exe = expectArgOrFatal(args, &arg_idx, "--zig"); - const zig_lib_dir = expectArgOrFatal(args, &arg_idx, "--zig-lib-dir"); - const build_root = expectArgOrFatal(args, &arg_idx, "--build-root"); - const local_cache_root = expectArgOrFatal(args, &arg_idx, "--local-cache"); - const global_cache_root = expectArgOrFatal(args, &arg_idx, "--global-cache"); - const configure_path = expectArgOrFatal(args, &arg_idx, "--configuration"); + var arg_i: usize = 1; + const cmd_name = nextArgOrFatal(args, &arg_i); + const zig_lib_arg = prefixedArgOrFatal(args, &arg_i, "--zig-lib="); + const zig_exe_arg = prefixedArgOrFatal(args, &arg_i, "--zig="); + const global_cache_arg = prefixedArgOrFatal(args, &arg_i, "--global-cache="); + const seed_arg = prefixedArgOrFatal(args, &arg_i, "--seed="); const cwd: Dir = .cwd(); const zig_lib_directory: Cache.Directory = .{ - .path = zig_lib_dir, - .handle = try cwd.openDir(io, zig_lib_dir, .{}), - }; - - const build_root_directory: Cache.Directory = .{ - .path = build_root, - .handle = try cwd.openDir(io, build_root, .{}), - }; - - const local_cache_directory: Cache.Directory = .{ - .path = local_cache_root, - .handle = try cwd.createDirPathOpen(io, local_cache_root, .{}), + .path = zig_lib_arg, + .handle = try cwd.openDir(io, zig_lib_arg, .{}), }; const global_cache_directory: Cache.Directory = .{ - .path = global_cache_root, - .handle = try cwd.createDirPathOpen(io, global_cache_root, .{}), + .path = global_cache_arg, + .handle = try cwd.createDirPathOpen(io, global_cache_arg, .{}), }; var graph: Graph = .{ .io = io, .arena = arena, - .cache = .{ - .io = io, - .gpa = gpa, - .manifest_dir = try local_cache_directory.handle.createDirPathOpen(io, "h", .{}), - .cwd = try process.currentPathAlloc(io, arena), - }, - .zig_exe = zig_exe, + .cache = undefined, + .zig_exe = zig_exe_arg, .environ_map = try init.environ.createMap(arena), .global_cache_root = global_cache_directory, - .local_cache_root = local_cache_directory, + .local_cache_root = undefined, .zig_lib_directory = zig_lib_directory, - .build_root_directory = build_root_directory, + .build_root_directory = undefined, + .random_seed = parseRandomSeed(seed_arg), }; - graph.cache.addPrefix(.{ .path = null, .handle = cwd }); - graph.cache.addPrefix(build_root_directory); - graph.cache.addPrefix(local_cache_directory); - graph.cache.addPrefix(global_cache_directory); - graph.cache.hash.addBytes(builtin.zig_version_string); + const cmd = stringToEnum(enum { libc, init, fetch, build }, cmd_name) orelse + fatal("bad command name: {q}", .{cmd_name}); + switch (cmd) { + .libc => return cmdLibC(gpa, &graph, args[arg_i..]), + .init => return cmdInit(gpa, &graph, args[arg_i..]), + .fetch => return cmdFetch(gpa, &graph, args[arg_i..]), + .build => {}, + } var step_names: std.ArrayList([]const u8) = .empty; var help_menu = false; var steps_menu = false; - var print_configuration = false; + var print_configuration: PrintConfiguration = .none; var override_install_prefix: ?[]const u8 = null; var override_lib_dir: ?[]const u8 = null; var override_bin_dir: ?[]const u8 = null; var override_include_dir: ?[]const u8 = null; + var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(&graph.environ_map); + var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(&graph.environ_map); var error_style: ErrorStyle = .verbose; var multiline_errors: MultilineErrors = .indent; var summary: ?Summary = null; @@ -201,39 +219,125 @@ pub fn main(init: process.Init.Minimal) !void { var webui_listen: ?Io.net.IpAddress = null; var debug_pkg_config = false; var run_args: ?[]const []const u8 = null; + var build_file: ?[]const u8 = null; - if (std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(ErrorStyle, str)) |style| { + var configure_argv: std.ArrayList([]const u8) = .empty; + var cached_passthru_configure: std.ArrayList(u32) = .empty; + var forks: std.ArrayList(Fork) = .empty; + var system_pkg_dir_path: ?[]const u8 = null; + var fetch_only = false; + var fetch_mode: Fetch.JobQueue.Mode = .needed; + var debug_target: ?[]const u8 = null; + var cache_poison: std.Build.Graph.CachePoison = .pure; + + if (EnvVar.ZIG_BUILD_ERROR_STYLE.get(&graph.environ_map)) |str| { + if (stringToEnum(ErrorStyle, str)) |style| { error_style = style; } } - if (std.zig.EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { - if (std.meta.stringToEnum(MultilineErrors, str)) |style| { + if (EnvVar.ZIG_BUILD_MULTILINE_ERRORS.get(&graph.environ_map)) |str| { + if (stringToEnum(MultilineErrors, str)) |style| { multiline_errors = style; } } - while (nextArg(args, &arg_idx)) |arg| { + try configure_argv.ensureUnusedCapacity(arena, 16); + try cached_passthru_configure.ensureUnusedCapacity(arena, 16); + + _ = configure_argv.addOneAssumeCapacity(); // configurer executable + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", graph.zig_exe }; + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; + const conf_argv_index_build_root = configure_argv.items.len - 1; + + while (nextArg(args, &arg_i)) |arg| { if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try configure_argv.ensureUnusedCapacity(arena, 2); + if (mem.startsWith(u8, arg, "-D") or + mem.startsWith(u8, arg, "-fsys=") or + mem.startsWith(u8, arg, "-fno-sys=") or + mem.startsWith(u8, arg, "--release=") or + mem.eql(u8, arg, "--release")) + { + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); + } else if (mem.eql(u8, arg, "--system")) { + system_pkg_dir_path = nextArgOrFatal(args, &arg_i); + + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path. + } else if (mem.cutPrefix(u8, arg, "--color=")) |rest| { + color = stringToEnum(Color, rest) orelse + fatalWithHint("expected --color=[auto|on|off]; found {q}", .{arg}); + + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(arg); + } else if (mem.eql(u8, arg, "--color")) { + const next_arg = nextArgOrFatal(args, &arg_i); + color = stringToEnum(Color, next_arg) orelse + fatalWithHint("expected [auto|on|off] found {q}", .{next_arg}); + + try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); + configure_argv.appendAssumeCapacity(try arena.print("--color={t}", .{color})); + } else if (mem.eql(u8, arg, "--cache-poison")) { + cache_poison = .poisoned; + configure_argv.appendAssumeCapacity("--cache-poison=poisoned"); + } else if (mem.cutPrefix(u8, arg, "--cache-poison=")) |rest| { + // Allow the configurer process to report parse failure. + if (stringToEnum(std.Build.Graph.CachePoison, rest)) |poison| { + cache_poison = poison; + } + configure_argv.appendAssumeCapacity(arg); + } else if (mem.eql(u8, arg, "--verbose")) { + // Intentionally is added both to make and configure but + // does not go into the cache hash. + configure_argv.appendAssumeCapacity(arg); + graph.verbose = true; + } else if (mem.eql(u8, arg, "--search-prefix")) { + const prefix = nextArgOrFatal(args, &arg_i); + + // This argument is cache poisonous: it does not go into + // the cache and configurer must set the poison bit when + // choosing to observe it. + configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, prefix }; + + try graph.search_prefixes.append(arena, prefix); + } else if (mem.eql(u8, arg, "--cache-dir")) { + override_local_cache_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--pkg-dir")) { + override_pkg_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--fetch")) { + fetch_only = true; + } else if (mem.cutPrefix(u8, arg, "--fetch=")) |rest| { + fetch_only = true; + fetch_mode = stringToEnum(Fetch.JobQueue.Mode, rest) orelse + fatal("expected [needed|all] after \"--fetch=\", found {q}", .{rest}); + } else if (mem.cutPrefix(u8, arg, "--fork=")) |rest| { + try forks.append(arena, .init(rest)); + } else if (mem.eql(u8, arg, "--fork")) { + try forks.append(arena, .init(nextArgOrFatal(args, &arg_i))); + } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { help_menu = true; } else if (mem.eql(u8, arg, "-l") or mem.eql(u8, arg, "--list-steps")) { steps_menu = true; } else if (mem.eql(u8, arg, "--print-configuration")) { - print_configuration = true; + print_configuration = .zon; + } else if (mem.eql(u8, arg, "--print-configuration-path")) { + print_configuration = .path; } else if (mem.eql(u8, arg, "-p") or mem.eql(u8, arg, "--prefix")) { - override_install_prefix = nextArgOrFatal(args, &arg_idx); + override_install_prefix = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--build-file")) { + build_file = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--prefix-lib-dir")) { - override_lib_dir = nextArgOrFatal(args, &arg_idx); + override_lib_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--prefix-exe-dir")) { - override_bin_dir = nextArgOrFatal(args, &arg_idx); + override_bin_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--prefix-include-dir")) { - override_include_dir = nextArgOrFatal(args, &arg_idx); + override_include_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--sysroot")) { - graph.sysroot = nextArgOrFatal(args, &arg_idx); + graph.sysroot = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--maxrss")) { - const max_rss_text = nextArgOrFatal(args, &arg_idx); + const max_rss_text = nextArgOrFatal(args, &arg_i); max_rss = std.fmt.parseIntSizeSuffix(max_rss_text, 10) catch |err| fatal("invalid byte size {q}: {t}", .{ max_rss_text, err }); } else if (mem.eql(u8, arg, "--skip-oom-steps")) { @@ -253,7 +357,7 @@ pub fn main(init: process.Init.Minimal) !void { .{ "h", std.time.ns_per_hour }, .{ "hour", std.time.ns_per_hour }, }; - const timeout_str = nextArgOrFatal(args, &arg_idx); + const timeout_str = nextArgOrFatal(args, &arg_i); const num_end_idx = std.mem.findLastNone(u8, timeout_str, "abcdefghijklmnopqrstuvwxyz") orelse fatal( "invalid timeout {q}: expected unit (ns, us, ms, s, m, h)", .{timeout_str}, @@ -273,51 +377,37 @@ pub fn main(init: process.Init.Minimal) !void { .{ timeout_str, num_str, err }, ); test_timeout_ns = std.math.lossyCast(u64, unit_factor * num_parsed); - } else if (mem.eql(u8, arg, "--search-prefix")) { - try graph.search_prefixes.append(arena, nextArgOrFatal(args, &arg_idx)); } else if (mem.eql(u8, arg, "--libc")) { - graph.libc_file = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--color")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected [auto|on|off] after {q}", .{arg}); - color = std.meta.stringToEnum(Color, next_arg) orelse { - fatalWithHint("expected [auto|on|off] after {q}, found {q}", .{ - arg, next_arg, - }); - }; + graph.libc_file = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--error-style")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected style after {q}", .{arg}); - error_style = std.meta.stringToEnum(ErrorStyle, next_arg) orelse { + error_style = stringToEnum(ErrorStyle, next_arg) orelse { fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; } else if (mem.eql(u8, arg, "--multiline-errors")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected style after {q}", .{arg}); - multiline_errors = std.meta.stringToEnum(MultilineErrors, next_arg) orelse { + multiline_errors = stringToEnum(MultilineErrors, next_arg) orelse { fatalWithHint("expected style after {q}, found {q}", .{ arg, next_arg }); }; } else if (mem.eql(u8, arg, "--summary")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected [all|new|failures|line|none] after {q}", .{arg}); - summary = std.meta.stringToEnum(Summary, next_arg) orelse { + summary = stringToEnum(Summary, next_arg) orelse { fatalWithHint("expected [all|new|failures|line|none] after {q}, found {q}", .{ arg, next_arg, }); }; - } else if (mem.eql(u8, arg, "--seed")) { - const next_arg = nextArg(args, &arg_idx) orelse - fatalWithHint("expected u32 after {q}", .{arg}); - graph.random_seed = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| { - fatal("unable to parse seed {q} as unsigned 32-bit integer: {t}", .{ next_arg, err }); - }; + } else if (mem.cutPrefix(u8, arg, "--seed=")) |rest| { + graph.random_seed = parseRandomSeed(rest); } else if (mem.eql(u8, arg, "--build-id")) { graph.build_id = .fast; } else if (mem.cutPrefix(u8, arg, "--build-id=")) |style| { graph.build_id = std.zig.BuildId.parse(style) catch |err| fatal("unable to parse --build-id style {q}: {t}", .{ style, err }); } else if (mem.eql(u8, arg, "--debounce")) { - const next_arg = nextArg(args, &arg_idx) orelse + const next_arg = nextArg(args, &arg_i) orelse fatalWithHint("expected u16 after {q}", .{arg}); debounce_interval_ms = std.fmt.parseUnsigned(u16, next_arg, 0) catch |err| { fatal("unable to parse debounce interval {q} as unsigned 16-bit integer: {t}", .{ @@ -332,9 +422,10 @@ pub fn main(init: process.Init.Minimal) !void { webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| { fatal("invalid web UI address {q}: {t}", .{ addr_str, err }); }; + } else if (mem.eql(u8, arg, "--debug-target")) { + debug_target = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--debug-log")) { - const next_arg = nextArgOrFatal(args, &arg_idx); - try graph.debug_log_scopes.append(arena, next_arg); + try graph.debug_log_scopes.append(arena, nextArgOrFatal(args, &arg_i)); } else if (mem.eql(u8, arg, "--debug-compile-errors")) { graph.debug_compile_errors = true; } else if (mem.eql(u8, arg, "--debug-incremental")) { @@ -344,19 +435,19 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "--debug-rt")) { graph.debug_compiler_runtime_libs = .Debug; } else if (mem.cutPrefix(u8, arg, "--debug-rt=")) |rest| { - graph.debug_compiler_runtime_libs = std.meta.stringToEnum(std.builtin.OptimizeMode, rest) orelse + graph.debug_compiler_runtime_libs = stringToEnum(std.lang.OptimizeMode, rest) orelse fatal("unrecognized optimization mode: {s}", .{rest}); } else if (is_debug_mode and mem.eql(u8, arg, "--debug-maker-leaks")) { debug_maker_leaks = true; } else if (mem.eql(u8, arg, "--libc-runtimes") or mem.eql(u8, arg, "--glibc-runtimes")) { // --glibc-runtimes was the old name of the flag; kept for compatibility for now. - graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_idx); - } else if (mem.eql(u8, arg, "--verbose")) { - graph.verbose = true; + graph.libc_runtimes_dir = nextArgOrFatal(args, &arg_i); } else if (mem.eql(u8, arg, "--verbose-air")) { graph.verbose_air = true; } else if (mem.eql(u8, arg, "--verbose-cc")) { graph.verbose_cc = true; + } else if (mem.eql(u8, arg, "--verbose-link")) { + graph.verbose_link = true; } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { graph.verbose_llvm_ir = true; } else if (mem.eql(u8, arg, "--watch")) { @@ -439,7 +530,7 @@ pub fn main(init: process.Init.Minimal) !void { } else if (mem.eql(u8, arg, "-fno-reference-trace")) { graph.reference_trace = null; } else if (mem.eql(u8, arg, "--error-limit")) { - const next_arg = nextArgOrFatal(args, &arg_idx); + const next_arg = nextArgOrFatal(args, &arg_i); graph.error_limit = std.fmt.parseUnsigned(u32, next_arg, 0) catch |err| fatal("unable to parse error limit {q}: {t}", .{ next_arg, err }); } else if (mem.cutPrefix(u8, arg, "-j")) |text| { @@ -449,7 +540,7 @@ pub fn main(init: process.Init.Minimal) !void { threaded.setAsyncLimit(.limited(n)); graph.max_jobs = n; } else if (mem.eql(u8, arg, "--")) { - run_args = argsRest(args, arg_idx); + run_args = argsRest(args, arg_i); break; } else { fatalWithHint("unrecognized argument: {s}", .{arg}); @@ -459,8 +550,51 @@ pub fn main(init: process.Init.Minimal) !void { } } - const NO_COLOR = std.zig.EnvVar.NO_COLOR.isSet(&graph.environ_map); - const CLICOLOR_FORCE = std.zig.EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); + const early_exit_mode = fetch_only or help_menu or steps_menu or print_configuration != .none; + const server_mode = !early_exit_mode and (watch or webui_listen != null or fuzz != null); + + process.raiseFileDescriptorLimit(); + + const cwd_path = std.zig.getResolvedCwd(io, arena) catch |err| + fatal("resolving current directory path failed: {t}", .{err}); + + const build_root = try findBuildRoot(arena, io, .{ + .cwd_path = cwd_path, + .build_file = build_file, + }); + + graph.build_root_directory = build_root.directory; + graph.local_cache_root = if (override_local_cache_dir) |unresolved_path| std.zig.Directories.openUnresolved( + arena, + io, + cwd_path, + unresolved_path, + .@"local cache", + ) else .{ + .path = try Dir.path.join(arena, &.{ build_root.directory.path orelse ".", default_local_zig_cache_basename }), + .handle = try build_root.directory.handle.createDirPathOpen(io, default_local_zig_cache_basename, .{}), + }; + graph.cache = .{ + .io = io, + .gpa = gpa, + .manifest_dir = try graph.local_cache_root.handle.createDirPathOpen(io, "h", .{}), + .cwd = cwd_path, + }; + + graph.cache.addPrefix(.{ .path = null, .handle = cwd }); + graph.cache.addPrefix(zig_lib_directory); + graph.cache.addPrefix(graph.local_cache_root); + graph.cache.addPrefix(global_cache_directory); + graph.cache.addPrefix(graph.build_root_directory); + comptime assert(0 == @intFromEnum(std.zig.Server.Message.PathPrefix.cwd)); + comptime assert(1 == @intFromEnum(std.zig.Server.Message.PathPrefix.zig_lib)); + comptime assert(2 == @intFromEnum(std.zig.Server.Message.PathPrefix.local_cache)); + comptime assert(3 == @intFromEnum(std.zig.Server.Message.PathPrefix.global_cache)); + + graph.cache.hash.addBytes(builtin.zig_version_string); + + const NO_COLOR = EnvVar.NO_COLOR.isSet(&graph.environ_map); + const CLICOLOR_FORCE = EnvVar.CLICOLOR_FORCE.isSet(&graph.environ_map); graph.stderr_mode = switch (color) { .auto => try .detect(io, .stderr(), NO_COLOR, CLICOLOR_FORCE), @@ -468,69 +602,15 @@ pub fn main(init: process.Init.Minimal) !void { .off => .no_color, }; - const scanned_config: ScannedConfig = sc: { - const configuration = c: { - var file = cwd.openFile(io, configure_path, .{}) catch |err| - fatal("failed to open configuration file {s}: {t}", .{ configure_path, err }); - defer file.close(io); - break :c Configuration.loadFile(arena, io, file) catch |err| - fatal("failed to load configuration file {s}: {t}", .{ configure_path, err }); + const pkg_root: Path = if (override_pkg_dir) |p| + .initCwd(p) + else if (system_pkg_dir_path) |p| + .initCwd(p) + else + .{ + .root_dir = build_root.directory, + .sub_path = "zig-pkg", }; - // Technically if the configuration is marked as poisoned, we could - // already delete the file now, but we leave it around in case the - // maker process fails or crashes and it's helpful to be able to repeat - // execution of the command line or otherwise inspect the configuration file. - const c = &configuration; - var top_level_steps: std.array_hash_map.String(Configuration.Step.Index) = .empty; - for (configuration.steps, 0..) |*conf_step, step_index_usize| { - if (conf_step.owner != .root) continue; - const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); - const flags = conf_step.flags(c); - switch (flags.tag) { - .top_level => { - const name = step_index.ptr(c).name.slice(c); - try top_level_steps.put(arena, name, step_index); - }, - else => {}, - } - } - for (c.search_prefixes) |search_prefix| { - try graph.search_prefixes.append(arena, search_prefix.slice(c)); - } - break :sc .{ - .configuration = configuration, - .top_level_steps = top_level_steps, - .path = configure_path, - }; - }; - - if (help_menu) { - var w = initStdoutWriter(io); - scanned_config.printUsage(&graph, w) catch |err| switch (err) { - error.WriteFailed => return stdout_writer_allocation.err.?, - else => |e| return e, - }; - w.flush() catch return stdout_writer_allocation.err.?; - return cleanExit(io, &scanned_config); - } else if (steps_menu) { - var w = initStdoutWriter(io); - scanned_config.printSteps(&graph, w) catch |err| switch (err) { - error.WriteFailed => return stdout_writer_allocation.err.?, - else => |e| return e, - }; - w.flush() catch return stdout_writer_allocation.err.?; - return cleanExit(io, &scanned_config); - } else if (print_configuration) { - var w = initStdoutWriter(io); - scanned_config.print(w) catch return stdout_writer_allocation.err.?; - w.flush() catch return stdout_writer_allocation.err.?; - return cleanExit(io, &scanned_config); - } - - if (webui_listen != null) { - if (watch) fatal("using '--webui' and '--watch' together is not yet supported; consider omitting '--watch' in favour of the web UI \"Rebuild\" button", .{}); - if (builtin.single_threaded) fatal("'--webui' is not yet supported on single-threaded hosts", .{}); - } const main_progress_node = std.Progress.start(io, .{ .disable_printing = (graph.stderr_mode.? == .no_color), @@ -544,7 +624,7 @@ pub fn main(init: process.Init.Minimal) !void { .root_dir = .cwd(), .sub_path = cwd_relative, } else .{ - .root_dir = build_root_directory, + .root_dir = graph.build_root_directory, .sub_path = "zig-out", }; @@ -563,146 +643,1336 @@ pub fn main(init: process.Init.Minimal) !void { .sub_path = cwd_relative, } else try install_prefix_path.join(arena, "include"); - var maker: Maker = .{ - .gpa = gpa, - .graph = &graph, - .scanned_config = &scanned_config, - .install_paths = .{ - .prefix = install_prefix_path, - .lib = install_lib_path, - .bin = install_bin_path, - .include = install_include_path, - }, - - .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), - .generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len), - .run_args = run_args, - - .available_rss = max_rss, - .max_rss_is_default = false, - .max_rss_mutex = .init, - .skip_oom_steps = skip_oom_steps, - .unit_test_timeout_ns = test_timeout_ns, - - .watch = watch, - .web_server = undefined, // set after `prepare` - .memory_blocked_steps = .empty, - .step_stack = .empty, - .pkg_config = .{ .debug = debug_pkg_config }, - - .error_style = error_style, - .multiline_errors = multiline_errors, - .summary = summary orelse if (watch or webui_listen != null) .line else .failures, - }; - defer { - maker.memory_blocked_steps.deinit(gpa); - maker.step_stack.deinit(gpa); - } - - if (maker.available_rss == 0) { - maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); - maker.max_rss_is_default = true; - } - - maker.prepare(step_names.items) catch |err| switch (err) { - error.DependencyLoopDetected, error.InsufficientMemory => { - _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; - process.exit(1); - }, - else => |e| return e, - }; - - var w: Watch = w: { - if (!watch) break :w undefined; - if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); - break :w try .init(&maker); - }; - const now = Io.Clock.Timestamp.now(io, .awake); - maker.web_server = if (webui_listen) |listen_address| ws: { - if (builtin.single_threaded) unreachable; // `fatal` above - break :ws .init(.{ - .maker = &maker, + var web_server_allocation: AvoidableWebServer = undefined; + const web_server: ?*AvoidableWebServer = if (webui_listen) |listen_address| ws: { + if (builtin.single_threaded) fatal("--webui is not yet supported on single-threaded hosts", .{}); + web_server_allocation = .init(.{ + .graph = &graph, .root_prog_node = main_progress_node, .listen_address = listen_address, .base_timestamp = now, }); + web_server_allocation.start() catch |err| fatal("failed to start web server: {t}", .{err}); + break :ws &web_server_allocation; } else null; - if (maker.web_server) |*ws| { - ws.start() catch |err| fatal("failed to start web server: {t}", .{err}); + while (true) { + // If this fails, we can still start the server and wait for user + // to request a rebuild. If it returns error.FailedButCacheIntact + // we can even still do file system watching and automatically + // rebuild on source changes. + if (configure(&graph, .{ + .configure_argv = configure_argv.items, + .conf_argv_index_build_root = conf_argv_index_build_root, + .cached_passthru_configure = cached_passthru_configure.items, + + .cache_poison = cache_poison, + .pkg_root = pkg_root, + .build_root = build_root, + .cwd_path = cwd_path, + .color = color, + .debug_target = debug_target, + .parent_progress_node = main_progress_node, + .fetch_mode = fetch_mode, + .system_pkg_dir_path = system_pkg_dir_path, + .fetch_only = fetch_only, + .print_configuration = print_configuration, + .forks = forks.items, + })) |scanned_config| { + if (help_menu) { + scanned_config.printUsage(&graph, initStdoutWriter(io)) catch |err| switch (err) { + error.WriteFailed => return stdout_writer_allocation.err.?, + else => |e| return e, + }; + try stdout_writer_allocation.flush(); + return cleanExit(io, &scanned_config); + } else if (steps_menu) { + scanned_config.printSteps(&graph, initStdoutWriter(io)) catch |err| switch (err) { + error.WriteFailed => return stdout_writer_allocation.err.?, + else => |e| return e, + }; + try stdout_writer_allocation.flush(); + return cleanExit(io, &scanned_config); + } else switch (print_configuration) { + .none => {}, + .zon => { + scanned_config.print(initStdoutWriter(io)) catch return stdout_writer_allocation.err.?; + try stdout_writer_allocation.flush(); + return cleanExit(io, &scanned_config); + }, + .path => unreachable, + } + + var maker: Maker = .{ + .gpa = gpa, + .graph = &graph, + .scanned_config = &scanned_config, + .install_paths = .{ + .prefix = install_prefix_path, + .lib = install_lib_path, + .bin = install_bin_path, + .include = install_include_path, + }, + + .steps = try arena.alloc(Step, scanned_config.configuration.steps.len), + .generated_files = try arena.alloc(Path, scanned_config.configuration.generated_files_len), + .run_args = run_args, + + .available_rss = max_rss, + .max_rss_is_default = false, + .max_rss_mutex = .init, + .skip_oom_steps = skip_oom_steps, + .unit_test_timeout_ns = test_timeout_ns, + + .watch = watch, + .web_server = web_server, + .memory_blocked_steps = .empty, + .step_stack = .empty, + .pkg_config = .{ .debug = debug_pkg_config }, + + .error_style = error_style, + .multiline_errors = multiline_errors, + .summary = summary orelse if (watch or webui_listen != null) .new else .failures, + }; + defer { + maker.memory_blocked_steps.deinit(gpa); + maker.step_stack.deinit(gpa); + } + + if (maker.available_rss == 0) { + maker.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64); + maker.max_rss_is_default = true; + } + + maker.prepare(step_names.items) catch |err| switch (err) { + error.DependencyLoopDetected, error.InsufficientMemory => { + // TODO handle DependencyLoopDetected as error.FailedButCacheIntact + // and handle InsufficientMemory as error.AlreadyReported + _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; + process.exit(1); + }, + else => |e| return e, + }; + + var w: Watch = w: { + if (!watch) break :w undefined; + if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{native_os}); + break :w try .init(&maker); + }; + + if (web_server) |ws| try ws.updateConfiguration(&maker); + + rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) { + const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); + defer io.unlockStderr(); + stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H") catch |err| switch (err) { + error.WriteFailed => return stderr.file_writer.err.?, + }; + }) { + if (web_server) |ws| ws.startBuild(); + + try maker.makeStepNames(step_names.items, main_progress_node, fuzz); + + if (web_server) |ws| { + if (fuzz) |mode| if (mode != .forever) fatal( + "error: limited fuzzing is not implemented yet for --webui", + .{}, + ); + + ws.finishBuild(.{ .fuzz = fuzz != null }); + } + + if (web_server) |ws| { + const c = &scanned_config.configuration; + assert(!watch); // fatal error after CLI parsing + while (true) switch (try ws.wait()) { + .rebuild => { + for (maker.step_stack.keys()) |step_index| { + const step = maker.stepByIndex(step_index); + step.state = .precheck_done; + const deps = step_index.ptr(c).deps.slice(c); + step.pending_deps = @intCast(deps.len); + step.reset(&maker); + } + continue :rebuild; + }, + }; + } + + if (!maker.watch) return; + + // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. + if (!Watch.have_impl) unreachable; + + try w.update(maker.step_stack.keys()); + + // Wait until a file system notification arrives. Read all such events + // until the buffer is empty. Then wait for a debounce interval, resetting + // if any more events come in. After the debounce interval has passed, + // trigger a rebuild on all steps with modified inputs, as well as their + // recursive dependants. + var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; + const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ + w.dir_count, countSubProcesses(&maker), + }) catch &caption_buf; + var debouncing_node = main_progress_node.start(caption, 0); + var in_debounce = false; + while (true) switch (try w.wait(if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { + .timeout => { + assert(in_debounce); + debouncing_node.end(); + markFailedStepsDirty(&maker); + continue :rebuild; + }, + .dirty => if (!in_debounce) { + in_debounce = true; + debouncing_node.end(); + debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); + }, + .clean => {}, + }; + } + } else |err| { + const can_fs_watch = switch (err) { + error.AlreadyReported => false, + error.FailedButCacheIntact => true, + else => |e| w: { + log.err("configuration failed: {t}", .{e}); + break :w false; + }, + }; + if (!server_mode) { + _ = io.lockStderr(&.{}, graph.stderr_mode) catch {}; + process.exit(1); + } + if (watch and can_fs_watch) { + fatal("(zig build system) TODO set up fs watching even when build.zig compilation fails", .{}); + } else { + fatal("(zig build system) TODO stay running and wait for user to request rebuild even when build.zig compilation fails", .{}); + } + } } +} - rebuild: while (true) : (if (maker.error_style.clearOnUpdate()) { - const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode); - defer io.unlockStderr(); - stderr.file_writer.interface.writeAll("\x1B[2J\x1B[3J\x1B[H") catch |err| switch (err) { - error.WriteFailed => return stderr.file_writer.err.?, - }; - }) { - if (maker.web_server) |*ws| ws.startBuild(); - - try maker.makeStepNames(step_names.items, main_progress_node, fuzz); - - if (maker.web_server) |*web_server| { - if (fuzz) |mode| if (mode != .forever) fatal( - "error: limited fuzzing is not implemented yet for --webui", - .{}, - ); - - web_server.finishBuild(.{ .fuzz = fuzz != null }); - } - - if (maker.web_server) |*web_server| { - const c = &scanned_config.configuration; - assert(!watch); // fatal error after CLI parsing - while (true) switch (try web_server.wait()) { - .rebuild => { - for (maker.step_stack.keys()) |step_index| { - const step = maker.stepByIndex(step_index); - step.state = .precheck_done; - const deps = step_index.ptr(c).deps.slice(c); - step.pending_deps = @intCast(deps.len); - step.reset(&maker); +const ConfigureOptions = struct { + configure_argv: [][]const u8, + conf_argv_index_build_root: usize, + cached_passthru_configure: []const u32, + + cache_poison: std.Build.Graph.CachePoison, + pkg_root: Path, + build_root: BuildRoot, + cwd_path: []const u8, + color: Color, + debug_target: ?[]const u8, + parent_progress_node: std.Progress.Node, + fetch_mode: Fetch.JobQueue.Mode, + system_pkg_dir_path: ?[]const u8, + fetch_only: bool, + print_configuration: PrintConfiguration, + forks: []Fork, +}; + +fn configure(graph: *Graph, options: ConfigureOptions) !ScannedConfig { + const configure_argv = options.configure_argv; + const gpa = graph.cache.gpa; + const io = graph.io; + const arena = graph.arena; + + // Cache lookup for configure options. If we get a match, we can skip + // execution of the configure script. If not, we get the file path to pass + // to the configure process. + // + // In the hot path, we only check this cache, which means that also + // configure source files need to go in here. + var config_man = graph.cache.obtain(); + defer config_man.deinit(); + + for (options.cached_passthru_configure) |i| + config_man.hash.addBytes(configure_argv[i]); + + // Prevents a `zig build` from getting a false positive cache hit following + // a `zig build --cache-poison=ignored`. + config_man.hash.add(options.cache_poison == .ignored); + + configure_argv[options.conf_argv_index_build_root] = options.build_root.directory.path orelse options.cwd_path; + + var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; + defer http_client.deinit(); + + var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; + var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; + + { + // Populate fork_set. + var group: Io.Group = .init; + defer group.cancel(io); + + for (options.forks) |*fork| + group.async(io, Fork.load, .{ io, gpa, fork, options.color }); + + try group.await(io); + + for (options.forks) |*fork| { + if (fork.failed) return error.AlreadyReported; + try fork_set.put(arena, .{ + .path = fork.path, + .manifest_ast = fork.manifest_ast, + .manifest = fork.manifest, + .uses = 0, + }, {}); + } + } + defer Fork.deinitList(options.forks); + + var build_configurer_argv: std.ArrayList([]const u8) = .empty; + defer build_configurer_argv.deinit(gpa); + + var dependencies_source: std.ArrayList(u8) = .empty; + defer dependencies_source.deinit(gpa); + + const configurer_root_src_path: Cache.Path = .{ + .root_dir = graph.zig_lib_directory, + .sub_path = "compiler/configurer.zig", + }; + + const root_build_src_path: Cache.Path = .{ + .root_dir = options.build_root.directory, + .sub_path = options.build_root.build_zig_basename, + }; + + const configurer_exe_name = "configurer"; + + try build_configurer_argv.appendSlice(gpa, &.{ + graph.zig_exe, "build-exe", // + "--cache-dir", graph.local_cache_root.path orelse ".", // + "--global-cache-dir", graph.global_cache_root.path orelse ".", // + "--zig-lib-dir", graph.zig_lib_directory.path orelse ".", // + "--name", configurer_exe_name, // + "-fsingle-threaded", // + }); + + // Normally the build runner is compiled for the host target but here is + // some code to help when debugging edits to the build runner so that you + // can make sure it compiles successfully on other targets. + const target_arch_os_abi: ?[]const u8 = if (options.debug_target) |triple| t: { + config_man.hash.addBytes(triple); + try build_configurer_argv.appendSlice(gpa, &.{ "-target", triple }); + break :t triple; + } else null; + + if (graph.libc_file) |libc_file| { + try build_configurer_argv.appendSlice(gpa, &.{ "--libc", libc_file }); + } + if (graph.reference_trace) |n| { + try build_configurer_argv.append(gpa, try arena.print("-freference-trace={d}", .{n})); + } + if (graph.debug_compile_errors) { + try build_configurer_argv.append(gpa, "--debug-compile-errors"); + } + try build_configurer_argv.appendSlice(gpa, &.{ + "--dep", "@build", // + "--dep", "@dependencies", // + try arena.print("-Mroot={f}", .{configurer_root_src_path}), // + }); + + // In the loop below, after doing the fetch operation, the argv will be + // truncated at this point, dependencies added, and then the + // "--listen=-" arg appended at the end. + const argv_deps_index = build_configurer_argv.items.len; + + const build_mod = try arena.create(CliModule); + build_mod.* = .{ + .name = "@build", + .root_path = try root_build_src_path.toString(arena), + }; + + const deps_mod = try arena.create(CliModule); + deps_mod.* = .{ + .name = "@dependencies", + .root_path = undefined, + }; + + // This loop is re-evaluated when the build script exits with an indication that it + // could not continue due to missing lazy dependencies. + const configuration_path: Path, const poisoned: bool = cp: while (true) { + build_mod.deps.clearRetainingCapacity(); + deps_mod.deps.clearRetainingCapacity(); + + // We want to release all the locks before executing the child process, so we make a nice + // big block here to ensure the cleanup gets run when we extract out our argv. + { + { + const fetch_prog_node = options.parent_progress_node.start("Fetch Packages", 0); + defer fetch_prog_node.end(); + + // Reset fork match counts. + for (fork_set.keys()) |*fork| fork.uses = 0; + + var job_queue: Package.Fetch.JobQueue = .{ + .io = io, + .http_client = &http_client, + .global_cache = graph.global_cache_root, + .local_storage = &.{ + .cache_root = .{ .root_dir = graph.local_cache_root }, + .pkg_root = options.pkg_root, + }, + .recursive = true, + .debug_hash = false, + .unlazy_set = unlazy_set, + .fork_set = fork_set, + .mode = options.fetch_mode, + .prog_node = fetch_prog_node, + .read_only = options.system_pkg_dir_path != null, + }; + defer job_queue.deinit(); + + if (options.system_pkg_dir_path == null) { + try http_client.initDefaultProxies(arena, &graph.environ_map); + } + + try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); + try job_queue.table.ensureUnusedCapacity(gpa, 1); + + const phantom_package_root: Cache.Path = .{ .root_dir = options.build_root.directory }; + + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .relative_path = phantom_package_root }, + .location_tok = 0, + .hash_tok = .none, + .name_tok = 0, + .lazy_status = .eager, + .remote_package_root = phantom_package_root, + .parent_package_root = phantom_package_root, + .parent_manifest_ast = null, + .prog_node = fetch_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + .use_latest_commit = false, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = undefined, + .manifest_ast = undefined, + .have_manifest = false, + .computed_hash = undefined, + .has_build_zig = true, + .oom_flag = false, + .latest_commit = null, + + .cli_module = build_mod, + }; + + job_queue.all_fetches.appendAssumeCapacity(&fetch); + + job_queue.table.putAssumeCapacityNoClobber( + Package.Fetch.relativePathDigest(phantom_package_root, graph.global_cache_root), + &fetch, + ); + + job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); + try job_queue.group.await(io); + + { + // Ensure that forks were actually used. This is done + // before printing manifest errors because using a fork can + // prevent them. + var any_unused = false; + for (fork_set.keys()) |*fork| { + if (fork.uses == 0) { + log.err("fork {f} matched no {s} packages", .{ + fork.path, fork.manifest.name, + }); + any_unused = true; + } else { + log.info("fork {f} matched {d} {s} packages", .{ + fork.path, fork.uses, fork.manifest.name, + }); + } } - continue :rebuild; + if (any_unused) return error.FailedButCacheIntact; + } + + try job_queue.consolidateErrors(); + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + errors.renderToStderr(io, .{}, options.color) catch process.exit(1); + return error.FailedButCacheIntact; + } + + if (options.fetch_only) { + _ = io.lockStderr(&.{}, .no_color) catch {}; + process.exit(0); + } + + // Create the dependencies.zig file for configurer to + // obtain via `@import("@dependencies")`. + { + { + dependencies_source.clearRetainingCapacity(); + var source_writer: Io.Writer.Allocating = .fromArrayList(gpa, &dependencies_source); + defer dependencies_source = source_writer.toArrayList(); + job_queue.createDependenciesSource(&source_writer.writer) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + }; + } + // Atomically create the file in a directory named after the hash of its contents. + var hh: Cache.HashHelper = .{}; + hh.addBytes(builtin.zig_version_string); + hh.addBytes(dependencies_source.items); + const hex_digest = hh.final(); + const dependencies_zig_path: Path = .{ + .root_dir = graph.local_cache_root, + .sub_path = try arena.print("o/{s}/dependencies.zig", .{&hex_digest}), + }; + var atomic_file = try dependencies_zig_path.root_dir.handle.createFileAtomic( + io, + dependencies_zig_path.sub_path, + .{ .make_path = true, .replace = true }, + ); + defer atomic_file.deinit(io); + atomic_file.file.writeStreamingAll(io, dependencies_source.items) catch |err| + fatal("writing dependencies.zig contents: {t}", .{err}); + atomic_file.replace(io) catch |err| + fatal("replacing {f}: {t}", .{ dependencies_zig_path, err }); + + deps_mod.root_path = try dependencies_zig_path.toString(arena); + } + + { + // Add a CliModule for each package's build.zig. + const hashes = job_queue.table.keys(); + const fetches = job_queue.table.values(); + try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); + for (hashes, fetches) |*hash, f| { + if (f == &fetch) { + // The first one is a dummy package for the current project. + continue; + } + if (!f.has_build_zig) + continue; + const hash_slice = try arena.dupe(u8, hash.toSlice()); + + const m = try arena.create(CliModule); + m.* = .{ + .root_path = try f.package_root.toString(arena), + .name = hash_slice, + }; + deps_mod.deps.putAssumeCapacityNoClobber(hash_slice, m); + f.cli_module = m; + } + + // Each build.zig module needs access to each of its + // dependencies' build.zig modules by name. + for (fetches) |f| { + const mod = f.cli_module orelse continue; + if (!f.have_manifest) continue; + const man = &f.manifest; + const dep_names = man.dependencies.keys(); + try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); + for (dep_names, man.dependencies.values()) |name, dep| { + const dep_digest = Package.Fetch.depDigest( + f.package_root, + graph.global_cache_root, + dep, + ) orelse continue; + const dep_mod = job_queue.table.get(dep_digest).?.cli_module orelse continue; + const name_cloned = try arena.dupe(u8, name); + mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); + } + } + } + + // Lower module dependencies to CLI argv. + build_configurer_argv.shrinkRetainingCapacity(argv_deps_index); + for (deps_mod.deps.values()) |dep| { + try build_configurer_argv.ensureUnusedCapacity(gpa, 2 * dep.deps.count() + 1); + for (dep.deps.keys(), dep.deps.values()) |name, sub| { + build_configurer_argv.appendAssumeCapacity("--dep"); + if (mem.eql(u8, name, sub.name)) { + build_configurer_argv.appendAssumeCapacity(sub.name); + } else { + build_configurer_argv.appendAssumeCapacity(try arena.print("{s}={s}", .{ + name, sub.name, + })); + } + } + build_configurer_argv.appendAssumeCapacity(try arena.print("-M{s}={s}/{s}", .{ + dep.name, dep.root_path, std.zig.build_zig_basename, + })); + } + try deps_mod.lower(arena, gpa, &build_configurer_argv); + try build_mod.lower(arena, gpa, &build_configurer_argv); + + try build_configurer_argv.append(gpa, "--listen=-"); + } + + const compile_prog_node = options.parent_progress_node.start("Compile Configure Script", 0); + defer compile_prog_node.end(); + + switch (options.cache_poison) { + .pure, .disallowed, .ignored => if (try config_man.hit()) { + const digest = config_man.final(); + break :cp .{ + .{ + .root_dir = graph.local_cache_root, + .sub_path = try arena.print("c/{s}", .{&digest}), + }, + false, + }; }, + .poisoned => {}, // Don't bother checking for cache hit. + } + + const configure_exe_path: Path = if (std.zig.buildExeSubprocess(gpa, io, .{ + .argv = build_configurer_argv.items, + .cache_root = graph.local_cache_root, + .root_name = configurer_exe_name, + .environ_map = &graph.environ_map, + .cache_manifest = &config_man, + .arch_os_abi = target_arch_os_abi, + .progress_node = compile_prog_node, + .skip_log_cmdline_on_compile_errors = !graph.verbose, + })) |r| r.path else |err| return err; + defer gpa.free(configure_exe_path.sub_path); + + configure_argv[0] = try configure_exe_path.toString(arena); + } + + if (!process.can_spawn) { + fatal("cannot spawn command on {t}: {f}", .{ native_os, @as(std.zig.SubprocessCommand, .{ + .argv = configure_argv, + }) }); + } + + const rand_int = randInt(io, u64); + const tmp_dir_sub_path = "tmp" ++ Dir.path.sep_str ++ std.fmt.hex(rand_int); + const config_tmp_path: Path = .{ + .root_dir = graph.local_cache_root, + .sub_path = tmp_dir_sub_path, + }; + const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( + io, + config_tmp_path.sub_path, + .{ .read = true, .exclusive = true }, + ); + defer config_tmp_file.close(io); + + const term = term: { + const child_node = options.parent_progress_node.start("Run Configure Script", 0); + defer child_node.end(); + var child = process.spawn(io, .{ + .argv = configure_argv, + .stdout = .{ .file = config_tmp_file }, + .progress_node = child_node, + }) catch |err| fatal("failed to spawn configure script {q}: {t}", .{ configure_argv[0], err }); + defer child.kill(io); + break :term child.wait(io) catch |err| + fatal("failed to wait configure script {q}: {t}", .{ configure_argv[0], err }); + }; + if (!term.success()) { + // Failure to produce the configuration file. + fatal("configure command {f}: {f}", .{ term, @as(std.zig.SubprocessCommand, .{ + .argv = configure_argv, + }) }); + } + // Even though the file is designed to be sent directly to make + // runner, we must load it now because: + // * If it contains additional file dependencies, we need to + // add them to `config_man` before obtaining the final digest. + // * If it contains a set of lazy packages that need to be + // fetched, we need to fetch those now and re-run configure. + var configuration = Configuration.loadFile(arena, io, config_tmp_file) catch |err| + fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); + + if (configuration.unlazy_deps.len != 0) { + var any_errors = false; + for (configuration.unlazy_deps) |hash_string| { + const hash = hash_string.slice(&configuration); + assert(hash.len != 0); + if (hash.len > Package.Hash.max_len) { + log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); + any_errors = true; + continue; + } + try unlazy_set.put(arena, .fromSlice(hash), {}); + } + if (any_errors) return error.FailedButCacheIntact; + if (options.system_pkg_dir_path) |p| { + // In this mode, the system needs to provide these packages; they + // cannot be fetched by Zig. + const s = Dir.path.sep_str; + for (unlazy_set.keys()) |*hash| { + log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); + } + log.info("remote package fetching disabled due to --system mode", .{}); + log.info("dependencies might be avoidable depending on build configuration", .{}); + return error.FailedButCacheIntact; + } + continue :cp; + } + + for (configuration.path_deps) |path_dep| { + switch (path_dep.flags.mode) { + .directory => {}, // TODO + .contents => try config_man.addPathPost(confPathDepToCachePath(graph, &configuration, path_dep)), + .metadata => {}, // TODO + } + } + + // If it is poisoned, there is no point in moving it to cached + // location. Just leave it in the tmp directory. + if (configuration.poisoned) { + break :cp .{ config_tmp_path, true }; + } else { + const digest = config_man.final(); + const final_path: Path = .{ + .root_dir = graph.local_cache_root, + .sub_path = try arena.print("c/{s}", .{&digest}), }; + Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + ) catch |err| retry: { + const e = switch (err) { + error.FileNotFound => e: { + const dir_path = final_path.dirname().?; + dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch |e| + fatal("failed to create directory {f}: {t}", .{ dir_path, e }); + if (Io.Dir.rename( + config_tmp_path.root_dir.handle, + config_tmp_path.sub_path, + final_path.root_dir.handle, + final_path.sub_path, + io, + )) |_| break :retry else |e| break :e e; + }, + else => |e| e, + }; + fatal("failed to rename configuration file from {f} into {f}: {t}", .{ + config_tmp_path, final_path, e, + }); + }; + config_man.writeManifest() catch |err| log.warn("failed to write cache manifest: {t}", .{err}); + break :cp .{ final_path, false }; } + }; - if (!maker.watch) return; - - // Comptime-known guard to prevent including the logic below when `!Watch.have_impl`. - if (!Watch.have_impl) unreachable; - - try w.update(maker.step_stack.keys()); - - // Wait until a file system notification arrives. Read all such events - // until the buffer is empty. Then wait for a debounce interval, resetting - // if any more events come in. After the debounce interval has passed, - // trigger a rebuild on all steps with modified inputs, as well as their - // recursive dependants. - var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; - const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ - w.dir_count, countSubProcesses(&maker), - }) catch &caption_buf; - var debouncing_node = main_progress_node.start(caption, 0); - var in_debounce = false; - while (true) switch (try w.wait(if (in_debounce) .{ .ms = debounce_interval_ms } else .none)) { - .timeout => { - assert(in_debounce); - debouncing_node.end(); - markFailedStepsDirty(&maker); - continue :rebuild; - }, - .dirty => if (!in_debounce) { - in_debounce = true; - debouncing_node.end(); - debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0); + // Hang on to the configuration file lock until we finish loading the configuration file. + var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; + defer if (configuration_lock) |*l| l.release(io); + + switch (options.print_configuration) { + .path => { + initStdoutWriter(io).print("{f}\n", .{configuration_path}) catch + fatal("failed printing cache file path: {t}", .{stdout_writer_allocation.err.?}); + stdout_writer_allocation.flush() catch |err| + fatal("failed printing cache file path: {t}", .{err}); + _ = io.lockStderr(&.{}, .no_color) catch {}; + process.exit(0); + }, + .none, .zon => {}, + } + + const configuration = c: { + var file = configuration_path.root_dir.handle.openFile(io, configuration_path.sub_path, .{}) catch |err| + fatal("failed to open configuration file {f}: {t}", .{ configuration_path, err }); + defer file.close(io); + break :c Configuration.loadFile(arena, io, file) catch |err| + fatal("failed to load configuration file {f}: {t}", .{ configuration_path, err }); + }; + // Technically if the configuration is marked as poisoned, we could + // already delete the file now, but we leave it around in case the + // maker process fails or crashes and it's helpful to be able to repeat + // execution of the command line or otherwise inspect the configuration file. + const c = &configuration; + var top_level_steps: std.array_hash_map.String(Configuration.Step.Index) = .empty; + for (configuration.steps, 0..) |*conf_step, step_index_usize| { + if (conf_step.owner != .root) continue; + const step_index: Configuration.Step.Index = @enumFromInt(step_index_usize); + const flags = conf_step.flags(c); + switch (flags.tag) { + .top_level => { + const name = step_index.ptr(c).name.slice(c); + try top_level_steps.put(arena, name, step_index); }, - .clean => {}, + else => {}, + } + } + for (c.search_prefixes) |search_prefix| { + try graph.search_prefixes.append(arena, search_prefix.slice(c)); + } + return .{ + .configuration = configuration, + .top_level_steps = top_level_steps, + .path = configuration_path, + }; +} + +fn cmdFetch(gpa: Allocator, graph: *Graph, args: []const []const u8) !void { + const environ_map = &graph.environ_map; + const io = graph.io; + const arena = graph.arena; + + const color: Color = Color.settingFromEnvironment(environ_map); + var opt_path_or_url: ?[]const u8 = null; + var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); + var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); + var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); + var debug_hash: bool = false; + var save: union(enum) { + no, + yes: ?[]const u8, + exact: ?[]const u8, + } = .no; + + var arg_i: usize = 0; + while (nextArg(args, &arg_i)) |arg| { + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try Io.File.stdout().writeStreamingAll(io, usage_fetch); + return process.cleanExit(io); + } else if (mem.eql(u8, arg, "--global-cache-dir")) { + override_global_cache_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--cache-dir")) { + override_local_cache_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--pkg-dir")) { + override_pkg_dir = nextArgOrFatal(args, &arg_i); + } else if (mem.eql(u8, arg, "--debug-hash")) { + debug_hash = true; + } else if (mem.eql(u8, arg, "--debug-log")) { + try graph.debug_log_scopes.append(arena, nextArgOrFatal(args, &arg_i)); + } else if (mem.eql(u8, arg, "--save")) { + save = .{ .yes = null }; + } else if (mem.cutPrefix(u8, arg, "--save=")) |rest| { + save = .{ .yes = rest }; + } else if (mem.eql(u8, arg, "--save-exact")) { + save = .{ .exact = null }; + } else if (mem.cutPrefix(u8, arg, "--save-exact=")) |rest| { + save = .{ .exact = rest }; + } else { + fatal("unrecognized parameter: {q}", .{arg}); + } + } else if (opt_path_or_url != null) { + fatal("unexpected extra parameter: {q}", .{arg}); + } else { + opt_path_or_url = arg; + } + } + + const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); + + var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; + defer http_client.deinit(); + + try http_client.initDefaultProxies(arena, environ_map); + + var root_prog_node = std.Progress.start(io, .{ + .root_name = "Fetch", + }); + defer root_prog_node.end(); + + var local_storage: Fetch.LocalStorage = undefined; + var build_root: BuildRoot = undefined; + var build_root_initialized = false; + defer if (build_root_initialized) build_root.deinit(io); + + const cwd_path = try std.zig.getResolvedCwd(io, arena); + + const local_storage_ptr = switch (save) { + .no => null, + .yes, .exact => ls: { + build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path }); + build_root_initialized = true; + + local_storage = .{ + .cache_root = if (override_local_cache_dir) |p| .initCwd(p) else .{ + .root_dir = build_root.directory, + .sub_path = ".zig-cache", + }, + .pkg_root = if (override_pkg_dir) |p| .initCwd(p) else .{ + .root_dir = build_root.directory, + .sub_path = "zig-pkg", + }, + }; + + break :ls &local_storage; + }, + }; + + var job_queue: Fetch.JobQueue = .{ + .io = io, + .http_client = &http_client, + .global_cache = graph.global_cache_root, + .local_storage = local_storage_ptr, + .recursive = false, + .read_only = false, + .debug_hash = debug_hash, + .mode = .all, + .prog_node = root_prog_node, + }; + defer job_queue.deinit(); + + var fetch: Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .path_or_url = path_or_url }, + .location_tok = 0, + .hash_tok = .none, + .name_tok = 0, + .lazy_status = .eager, + .remote_package_root = undefined, + .parent_package_root = undefined, + .parent_manifest_ast = null, + .prog_node = root_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + .allow_missing_paths_field = false, + .use_latest_commit = true, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = undefined, + .manifest_ast = undefined, + .have_manifest = false, + .computed_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + .latest_commit = null, + + .cli_module = null, + }; + defer fetch.deinit(); + + fetch.run() catch |err| switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.FetchFailed => {}, // error bundle checked below + }; + + try job_queue.group.await(io); + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + errors.renderToStderr(io, .{}, color) catch {}; + process.exit(1); + } + + const package_hash = fetch.computedPackageHash(); + const package_hash_slice = package_hash.toSlice(); + + root_prog_node.end(); + root_prog_node = .{ .index = .none }; + + const name = switch (save) { + .no => { + var data: [2][]const u8 = .{ package_hash_slice, "\n" }; + const w = initStdoutWriter(io); + w.writeVecAll(&data) catch return stdout_writer_allocation.err.?; + try stdout_writer_allocation.flush(); + return process.cleanExit(io); + }, + .yes, .exact => |name| name: { + if (name) |n| break :name n; + if (!fetch.have_manifest) + fatal("unable to determine name; fetched package has no build.zig.zon file", .{}); + break :name fetch.manifest.name; + }, + }; + + // The name to use in case the manifest file needs to be created now. + const init_root_name = Dir.path.basename(build_root.directory.path orelse cwd_path); + var manifest, var ast = try loadManifest(gpa, arena, io, .{ + .root_name = try sanitizeExampleName(arena, init_root_name), + .dir = build_root.directory.handle, + .color = color, + }); + defer { + manifest.deinit(gpa); + ast.deinit(gpa); + } + + var fixups: std.zig.Ast.Render.Fixups = .{}; + defer fixups.deinit(gpa); + + var saved_path_or_url = path_or_url; + + if (fetch.latest_commit) |latest_commit| resolved: { + const latest_commit_hex = try arena.print("{f}", .{latest_commit}); + + var uri = try std.Uri.parse(path_or_url); + + if (uri.fragment) |fragment| { + const target_ref = try fragment.toRawMaybeAlloc(arena); + + // the refspec may already be fully resolved + if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved; + + log.info("resolved ref {q} to commit {s}", .{ target_ref, latest_commit_hex }); + + // include the original refspec in a query parameter, could be used to check for updates + uri.query = .{ .percent_encoded = try arena.print("ref={f}", .{ + std.fmt.alt(fragment, .formatEscaped), + }) }; + } else { + log.info("resolved to commit {s}", .{latest_commit_hex}); + } + + // replace the refspec with the resolved commit SHA + uri.fragment = .{ .raw = latest_commit_hex }; + + switch (save) { + .yes => saved_path_or_url = try arena.print("{f}", .{uri}), + .no, .exact => {}, // keep the original URL + } + } + + const new_node_init = try arena.print( + \\.{{ + \\ .url = "{f}", + \\ .hash = "{f}", + \\ }} + , .{ + std.zig.fmtString(saved_path_or_url), + std.zig.fmtString(package_hash_slice), + }); + + const new_node_text = try arena.print(".{f} = {s},\n", .{ + std.zig.fmtIdPU(name), new_node_init, + }); + + const dependencies_init = try arena.print(".{{\n {s} }}", .{ + new_node_text, + }); + + const dependencies_text = try arena.print(".dependencies = {s},\n", .{ + dependencies_init, + }); + + if (manifest.dependencies.get(name)) |dep| { + if (dep.hash) |h| { + switch (dep.location) { + .url => |u| { + if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { + log.info("existing dependency named {q} is up-to-date", .{name}); + process.exit(0); + } + }, + .path => {}, + } + } + + const location_replace = try arena.print("{q}", .{saved_path_or_url}); + const hash_replace = try arena.print("{q}", .{package_hash_slice}); + + log.warn("overwriting existing dependency named {q}", .{name}); + try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace); + if (dep.hash_node.unwrap()) |hash_node| { + try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace); + } else { + // https://github.com/ziglang/zig/issues/21690 + } + } else if (manifest.dependencies.count() > 0) { + // Add fixup for adding another dependency. + const deps = manifest.dependencies.values(); + const last_dep_node = deps[deps.len - 1].node; + try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text); + } else if (manifest.dependencies_node.unwrap()) |dependencies_node| { + // Add fixup for replacing the entire dependencies struct. + try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init); + } else { + // Add fixup for adding dependencies struct. + try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text); + } + + var aw: Io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + try ast.render(gpa, &aw.writer, fixups); + const rendered = aw.written(); + + build_root.directory.handle.writeFile(io, .{ .sub_path = Package.Manifest.basename, .data = rendered }) catch |err| { + fatal("unable to write {s} file: {t}", .{ Package.Manifest.basename, err }); + }; + + return process.cleanExit(io); +} + +const usage_fetch = + \\Usage: zig fetch [options] + \\Usage: zig fetch [options] + \\ + \\ Copy a package into the global cache and print its hash. + \\ must point to one of the following: + \\ - A git+http / git+https server for the package + \\ - A tarball file (with or without compression) containing + \\ package source + \\ - A git bundle file containing package source + \\ + \\Examples: + \\ + \\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git + \\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz + \\ + \\Options: + \\ -h, --help Print this help and exit + \\ --global-cache-dir [path] Override path to global Zig cache directory + \\ --cache-dir [path] Override path to local cache directory + \\ --pkg-dir [path] Override path to local package directory + \\ --debug-hash Print verbose hash information to stdout + \\ --debug-log [scope] Enable printing debug/info log messages for scope + \\ --save Add the fetched package to build.zig.zon + \\ --save=[name] Add the fetched package to build.zig.zon as name + \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim + \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim + \\ +; + +const usage_init = + \\Usage: zig init + \\ + \\ Initializes a `zig build` project in the current working + \\ directory. + \\ + \\Options: + \\ -m, --minimal Use minimal init template + \\ -h, --help Print this help and exit + \\ + \\ +; + +const usage_libc = + \\Usage: zig libc + \\ + \\ Detect the native libc installation and print the resulting + \\ paths to stdout. You can save this into a file and then edit + \\ the paths to create a cross compilation libc kit. Then you + \\ can pass `--libc [file]` for Zig to use it. + \\ + \\Usage: zig libc [paths_file] + \\ + \\ Parse a libc installation text file and validate it. + \\ + \\Options: + \\ -h, --help Print this help and exit + \\ -target [name] -- see the targets command + \\ -includes Print the libc include directories for the target + \\ +; + +fn cmdInit(gpa: Allocator, graph: *Graph, args: []const []const u8) !void { + const arena = graph.arena; + const io = graph.io; + const default_build_zig_basename = std.zig.build_zig_basename; + + var template: enum { example, minimal } = .example; + { + var i: usize = 0; + while (i < args.len) : (i += 1) { + const arg = args[i]; + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-m") or mem.eql(u8, arg, "--minimal")) { + template = .minimal; + } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try Io.File.stdout().writeStreamingAll(io, usage_init); + return process.cleanExit(io); + } else { + fatal("unrecognized parameter: {q}", .{arg}); + } + } else { + fatal("unexpected extra parameter: {q}", .{arg}); + } + } + } + + const cwd_path = try std.zig.getResolvedCwd(io, arena); + const cwd_basename = Dir.path.basename(cwd_path); + const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename); + + const rng: std.Random.IoSource = .{ .io = io }; + const fingerprint: Package.Fingerprint = .generate(rng.interface(), sanitized_root_name); + + switch (template) { + .example => { + var templates = Templates.find(gpa, io, graph.zig_lib_directory); + defer templates.deinit(io); + + const s = Dir.path.sep_str; + const template_paths = [_][]const u8{ + default_build_zig_basename, + Package.Manifest.basename, + "src" ++ s ++ "main.zig", + "src" ++ s ++ "root.zig", + }; + var ok_count: usize = 0; + + for (template_paths) |template_path| { + if (templates.write(arena, io, Io.Dir.cwd(), sanitized_root_name, template_path, fingerprint)) |_| { + log.info("created {s}", .{template_path}); + ok_count += 1; + } else |err| switch (err) { + error.PathAlreadyExists => log.info("preserving already existing file: {s}", .{ + template_path, + }), + else => log.err("unable to write {s}: {t}", .{ template_path, err }), + } + } + + if (ok_count == template_paths.len) { + log.info("see `zig build --help` for a menu of options", .{}); + } + return process.cleanExit(io); + }, + .minimal => { + Templates.writeSimpleFile(io, Package.Manifest.basename, + \\.{{ + \\ .name = .{s}, + \\ .version = "0.0.1", + \\ .minimum_zig_version = "{s}", + \\ .paths = .{{""}}, + \\ .fingerprint = 0x{x}, + \\}} + \\ + , .{ + sanitized_root_name, + builtin.zig_version_string, + fingerprint.int(), + }) catch |err| switch (err) { + else => fatal("failed to create {q}: {t}", .{ Package.Manifest.basename, err }), + error.PathAlreadyExists => fatal("refusing to overwrite {q}", .{Package.Manifest.basename}), + }; + Templates.writeSimpleFile(io, default_build_zig_basename, + \\const std = @import("std"); + \\ + \\pub fn build(b: *std.Build) void {{ + \\ _ = b; // stub + \\}} + \\ + , .{}) catch |err| switch (err) { + else => fatal("failed to create {q}: {t}", .{ default_build_zig_basename, err }), + // `build.zig` already existing is okay: the user has just used `zig init` to set up + // their `build.zig.zon` *after* writing their `build.zig`. So this one isn't fatal. + error.PathAlreadyExists => { + log.info("successfully populated {q}, preserving existing {q}", .{ + Package.Manifest.basename, default_build_zig_basename, + }); + return process.cleanExit(io); + }, + }; + log.info("successfully populated {q} and {q}", .{ Package.Manifest.basename, default_build_zig_basename }); + return process.cleanExit(io); + }, + } +} + +fn cmdLibC(gpa: Allocator, graph: *Graph, args: []const []const u8) !void { + const environ_map = &graph.environ_map; + const io = graph.io; + const arena = graph.arena; + const LibCInstallation = std.zig.LibCInstallation; + + var input_file: ?[]const u8 = null; + var target_arch_os_abi: []const u8 = "native"; + var print_includes: bool = false; + const stdout = initStdoutWriter(io); + { + var i: usize = 0; + while (i < args.len) : (i += 1) { + const arg = args[i]; + if (mem.startsWith(u8, arg, "-")) { + if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { + try stdout.writeAll(usage_libc); + try stdout.flush(); + return std.process.cleanExit(io); + } else if (mem.eql(u8, arg, "-target")) { + if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg}); + i += 1; + target_arch_os_abi = args[i]; + } else if (mem.eql(u8, arg, "-includes")) { + print_includes = true; + } else { + fatal("unrecognized parameter: '{s}'", .{arg}); + } + } else if (input_file != null) { + fatal("unexpected extra parameter: '{s}'", .{arg}); + } else { + input_file = arg; + } + } + } + + const target_query = std.zig.parseTargetQueryOrReportFatalError(gpa, .{ + .arch_os_abi = target_arch_os_abi, + }); + const target = std.zig.resolveTargetQueryOrFatal(io, target_query); + + if (print_includes) { + const libc_installation: ?*LibCInstallation = libc: { + if (input_file) |libc_file| { + const libc = try arena.create(LibCInstallation); + libc.* = LibCInstallation.parse(arena, io, libc_file, &target) catch |err| { + fatal("unable to parse libc file at path {s}: {t}", .{ libc_file, err }); + }; + break :libc libc; + } else { + break :libc null; + } }; + + const is_native_abi = target_query.isNativeAbi(); + + const libc_dirs = std.zig.LibCDirs.detect( + arena, + io, + .{ .root_dir = graph.zig_lib_directory }, + &target, + is_native_abi, + true, + libc_installation, + environ_map, + ) catch |err| { + const zig_target = try target.zigTriple(arena); + fatal("unable to detect libc for target {s}: {t}", .{ zig_target, err }); + }; + + if (libc_dirs.libc_include_dir_list.len == 0) { + const zig_target = try target.zigTriple(arena); + fatal("no include dirs detected for target {s}", .{zig_target}); + } + + for (libc_dirs.libc_include_dir_list) |include_dir| { + try stdout.writeAll(include_dir); + try stdout.writeByte('\n'); + } + try stdout.flush(); + return std.process.cleanExit(io); + } + + if (input_file) |libc_file| { + var libc = LibCInstallation.parse(gpa, io, libc_file, &target) catch |err| { + fatal("unable to parse libc file at path {s}: {t}", .{ libc_file, err }); + }; + defer libc.deinit(gpa); + } else { + if (!target_query.canDetectLibC()) { + fatal("unable to detect libc for non-native target", .{}); + } + var libc = LibCInstallation.findNative(gpa, io, .{ + .verbose = true, + .target = &target, + .environ_map = environ_map, + }) catch |err| { + fatal("unable to detect native libc: {t}", .{err}); + }; + defer libc.deinit(gpa); + + try libc.render(stdout); + try stdout.flush(); } } @@ -807,9 +2077,7 @@ fn prepare(maker: *Maker, step_names: []const []const u8) !void { } if (any_problems) { if (maker.max_rss_is_default) { - std.log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{ - max_needed, - }); + log.info("use --maxrss {d} to proceed, risking system memory exhaustion", .{max_needed}); } return error.InsufficientMemory; } @@ -819,7 +2087,7 @@ fn prepare(maker: *Maker, step_names: []const []const u8) !void { fn makeStepNames( maker: *Maker, step_names: []const []const u8, - parent_prog_node: std.Progress.Node, + parent_progress_node: std.Progress.Node, fuzz: ?Fuzz.Mode, ) !void { const graph = maker.graph; @@ -843,7 +2111,7 @@ fn makeStepNames( } } - const step_prog = parent_prog_node.start("steps", step_stack.count()); + const step_prog = parent_progress_node.start("steps", step_stack.count()); defer step_prog.end(); var group: Io.Group = .init; @@ -901,12 +2169,12 @@ fn makeStepNames( } if (fuzz) |mode| blk: { - switch (builtin.os.tag) { + switch (native_os) { // Current implementation depends on two things that need to be ported to Windows: // * Memory-mapping to share data between the fuzzer and build runner. // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving // many addresses to source locations). - .windows => fatal("--fuzz not yet implemented for {t}", .{builtin.os.tag}), + .windows => fatal("--fuzz not yet implemented for {t}", .{native_os}), else => {}, } if (@bitSizeOf(usize) != 64) { @@ -923,7 +2191,7 @@ fn makeStepNames( } assert(mode == .limit); - var f = Fuzz.init(maker, step_stack.keys(), parent_prog_node, mode) catch |err| + var f = Fuzz.init(maker, step_stack.keys(), parent_progress_node, mode) catch |err| fatal("failed to start fuzzer: {t}", .{err}); defer f.deinit(); @@ -997,7 +2265,7 @@ fn makeStepNames( t.setColor(.reset) catch {}; } - w.writeAll("\n") catch {}; + w.writeByte('\n') catch {}; if (maker.summary == .line) break :summary; @@ -1114,7 +2382,7 @@ fn makeStep( const step_prog_node = root_prog_node.start(step_name, 0); defer step_prog_node.end(); - if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .wip); + if (maker.web_server) |ws| ws.updateStepStatus(step_index, .wip); const new_state: Step.State = for (deps) |dep_index| { const dep_make_step = maker.stepByIndex(dep_index); @@ -1149,14 +2417,14 @@ fn makeStep( .dependency_failure, .skipped_oom, => { - if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .failure); + if (maker.web_server) |ws| ws.updateStepStatus(step_index, .failure); std.Progress.setStatus(.failure_working); }, .success, .skipped, => { - if (maker.web_server) |*ws| ws.updateStepStatus(step_index, .success); + if (maker.web_server) |ws| ws.updateStepStatus(step_index, .success); }, } } @@ -1670,26 +2938,23 @@ pub fn printErrorMessages( try writer.writeByte('\n'); } -fn nextArg(args: []const [:0]const u8, idx: *usize) ?[:0]const u8 { - if (idx.* >= args.len) return null; - defer idx.* += 1; - return args[idx.*]; +fn nextArg(args: []const []const u8, i: *usize) ?[]const u8 { + if (i.* >= args.len) return null; + defer i.* += 1; + return args[i.*]; } -fn nextArgOrFatal(args: []const [:0]const u8, idx: *usize) [:0]const u8 { - return nextArg(args, idx) orelse { - fatalWithHint("expected argument after {q}", .{args[idx.* - 1]}); - }; +fn nextArgOrFatal(args: []const []const u8, i: *usize) []const u8 { + return nextArg(args, i) orelse fatalWithHint("expected another argument after {q}", .{args[i.* - 1]}); } -fn expectArgOrFatal(args: []const [:0]const u8, index_ptr: *usize, first: []const u8) []const u8 { - const next_arg = nextArg(args, index_ptr) orelse fatal("missing {q} argument", .{first}); - if (!mem.eql(u8, first, next_arg)) fatal("expected {q} instead of {q}", .{ first, next_arg }); - const arg = nextArg(args, index_ptr) orelse fatal("expected argument after {q}", .{first}); - return arg; +fn prefixedArgOrFatal(args: []const []const u8, i: *usize, prefix: []const u8) []const u8 { + const arg = nextArgOrFatal(args, i); + if (mem.cutPrefix(u8, arg, prefix)) |rest| return rest; + fatal("expected {q} to instead begin with {q}", .{ arg, prefix }); } -fn argsRest(args: []const [:0]const u8, idx: usize) ?[]const [:0]const u8 { +fn argsRest(args: []const []const u8, idx: usize) ?[]const []const u8 { if (idx >= args.len) return null; return args[idx..]; } @@ -2006,11 +3271,11 @@ pub fn installSymLinks( const name = conf_comp.root_name.slice(c); const filename_major_only, const filename_name_only = if (os_tag.isDarwin()) .{ - try std.fmt.allocPrint(arena, "lib{s}.{d}.dylib", .{ name, version.major }), - try std.fmt.allocPrint(arena, "lib{s}.dylib", .{name}), + try arena.print("lib{s}.{d}.dylib", .{ name, version.major }), + try arena.print("lib{s}.dylib", .{name}), } else .{ - try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ name, version.major }), - try std.fmt.allocPrint(arena, "lib{s}.so", .{name}), + try arena.print("lib{s}.so.{d}", .{ name, version.major }), + try arena.print("lib{s}.so", .{name}), }; return installSymLinksInner(maker, arena, output_path, asking_step_index, filename_major_only, filename_name_only); @@ -2050,8 +3315,8 @@ fn removePoisonedConfiguration(io: Io, scanned_config: *const ScannedConfig) voi if (scanned_config.configuration.poisoned) { // This configuration file was good for only 1 invocation of the maker // process. Delete it to save space on disk. - Io.Dir.cwd().deleteFile(io, scanned_config.path) catch |err| - log.warn("failed deleting poisoned configuration file {s}: {t}", .{ scanned_config.path, err }); + scanned_config.path.root_dir.handle.deleteFile(io, scanned_config.path.sub_path) catch |err| + log.warn("failed deleting poisoned configuration file {f}: {t}", .{ scanned_config.path, err }); } } @@ -2059,3 +3324,385 @@ inline fn debugMakerLeaks() bool { if (!is_debug_mode) return false; return debug_maker_leaks; } + +const BuildRoot = struct { + directory: Cache.Directory, + build_zig_basename: []const u8, + cleanup_build_dir: ?Io.Dir, + + fn deinit(br: *BuildRoot, io: Io) void { + if (br.cleanup_build_dir) |*dir| dir.close(io); + br.* = undefined; + } +}; + +const FindBuildRootOptions = struct { + build_file: ?[]const u8 = null, + cwd_path: ?[]const u8 = null, +}; + +fn findBuildRoot(arena: Allocator, io: Io, options: FindBuildRootOptions) !BuildRoot { + const cwd_path = options.cwd_path orelse try std.zig.getResolvedCwd(io, arena); + const build_zig_basename = if (options.build_file) |bf| + Dir.path.basename(bf) + else + std.zig.build_zig_basename; + + if (options.build_file) |bf| { + if (Dir.path.dirname(bf)) |dirname| { + const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { + fatal("failed opening directory containing {q}: {t}", .{ bf, err }); + }; + return .{ + .build_zig_basename = build_zig_basename, + .directory = .{ .path = dirname, .handle = dir }, + .cleanup_build_dir = dir, + }; + } + + return .{ + .build_zig_basename = build_zig_basename, + .directory = .{ .path = null, .handle = Io.Dir.cwd() }, + .cleanup_build_dir = null, + }; + } + // Search up parent directories until we find build.zig. + var dirname: []const u8 = cwd_path; + while (true) { + const joined_path = try Dir.path.join(arena, &[_][]const u8{ dirname, build_zig_basename }); + if (Io.Dir.cwd().access(io, joined_path, .{})) |_| { + const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { + fatal("unable to open directory while searching for build.zig file, {q}: {t}", .{ dirname, err }); + }; + return .{ + .build_zig_basename = build_zig_basename, + .directory = .{ + .path = dirname, + .handle = dir, + }, + .cleanup_build_dir = dir, + }; + } else |err| switch (err) { + error.FileNotFound => { + dirname = Dir.path.dirname(dirname) orelse { + log.info("initialize {s} template file with \"zig init\"", .{std.zig.build_zig_basename}); + log.info("see \"zig --help\" for more options", .{}); + fatal("no build.zig file found, in the current directory or any parent directories", .{}); + }; + continue; + }, + else => |e| return e, + } + } +} + +const Fork = struct { + path: Path, + manifest_ast: std.zig.Ast, + manifest: Package.Manifest, + error_bundle: std.zig.ErrorBundle.Wip, + failed: bool, + arena_allocator: std.heap.ArenaAllocator, + + fn init(cwd_relative_path: []const u8) Fork { + return .{ + .manifest_ast = undefined, + .manifest = undefined, + .error_bundle = undefined, + .arena_allocator = undefined, + .path = .{ + .root_dir = .cwd(), + .sub_path = cwd_relative_path, + }, + .failed = false, + }; + } + + fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void { + loadFallible(io, gpa, fork, color) catch |err| switch (err) { + error.Canceled => |e| return e, + error.AlreadyReported => fork.failed = true, + else => |e| { + log.err("failed to load fork at {f}: {t}", .{ fork.path, e }); + fork.failed = true; + }, + }; + } + + fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void { + fork.arena_allocator = .init(gpa); + const arena = fork.arena_allocator.allocator(); + + var error_bundle: std.zig.ErrorBundle.Wip = undefined; + try error_bundle.init(gpa); + defer error_bundle.deinit(); + + const manifest_path = try fork.path.join(arena, Package.Manifest.basename); + + Package.Manifest.load( + io, + arena, + manifest_path, + &fork.manifest_ast, + &error_bundle, + &fork.manifest, + true, + ) catch |err| switch (err) { + error.Canceled => |e| return e, + error.ErrorsBundled => { + assert(error_bundle.root_list.items.len > 0); + var errors = try error_bundle.toOwnedBundle(""); + errors.renderToStderr(io, .{}, color) catch {}; + return error.AlreadyReported; + }, + else => |e| { + log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e }); + return error.AlreadyReported; + }, + }; + } + + fn deinitList(forks: []Fork) void { + for (forks) |*fork| fork.arena_allocator.deinit(); + } +}; + +fn parseRandomSeed(arg: []const u8) u32 { + return std.fmt.parseUnsigned(u32, arg, 0) catch |err| + fatal("failed parsing random seed {q} as unsigned 32-bit integer: {t}", .{ arg, err }); +} + +fn randInt(io: Io, comptime T: type) T { + var x: T = undefined; + io.random(@ptrCast(&x)); + return x; +} + +const LoadManifestOptions = struct { + root_name: []const u8, + dir: Io.Dir, + color: Color, +}; + +fn loadManifest( + gpa: Allocator, + arena: Allocator, + io: Io, + options: LoadManifestOptions, +) !struct { Package.Manifest, std.zig.Ast } { + const rng: std.Random.IoSource = .{ .io = io }; + + const manifest_bytes = while (true) { + break options.dir.readFileAllocOptions( + io, + Package.Manifest.basename, + arena, + .limited(Package.Manifest.max_bytes), + .@"1", + 0, + ) catch |err| switch (err) { + error.FileNotFound => { + Templates.writeSimpleFile(io, Package.Manifest.basename, + \\.{{ + \\ .name = .{s}, + \\ .version = "{s}", + \\ .paths = .{{""}}, + \\ .fingerprint = 0x{x}, + \\}} + \\ + , .{ + options.root_name, + builtin.zig_version_string, + Package.Fingerprint.generate(rng.interface(), options.root_name).int(), + }) catch |e| { + fatal("unable to write {s}: {t}", .{ Package.Manifest.basename, e }); + }; + continue; + }, + else => |e| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }), + }; + }; + var ast = try std.zig.Ast.parse(gpa, manifest_bytes, .zon); + errdefer ast.deinit(gpa); + + if (ast.errors.len > 0) { + try std.zig.printAstErrorsToStderr(gpa, io, ast, Package.Manifest.basename, options.color); + process.exit(2); + } + + var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{}); + errdefer manifest.deinit(gpa); + + if (manifest.errors.len > 0) { + var wip_errors: std.zig.ErrorBundle.Wip = undefined; + try wip_errors.init(gpa); + defer wip_errors.deinit(); + + const src_path = try wip_errors.addString(Package.Manifest.basename); + try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors); + + var error_bundle = try wip_errors.toOwnedBundle(""); + defer error_bundle.deinit(gpa); + error_bundle.renderToStderr(io, .{}, options.color) catch {}; + + process.exit(2); + } + return .{ manifest, ast }; +} + +fn sanitizeExampleName(arena: Allocator, bytes: []const u8) error{OutOfMemory}![]const u8 { + var result: std.ArrayList(u8) = .empty; + for (bytes, 0..) |byte, i| switch (byte) { + '0'...'9' => { + if (i == 0) try result.append(arena, '_'); + try result.append(arena, byte); + }, + '_', 'a'...'z', 'A'...'Z' => try result.append(arena, byte), + '-', '.', ' ' => try result.append(arena, '_'), + else => continue, + }; + if (!std.zig.isValidId(result.items)) return "foo"; + if (result.items.len > Package.Manifest.max_name_len) + result.shrinkRetainingCapacity(Package.Manifest.max_name_len); + + return result.toOwnedSlice(arena); +} + +test sanitizeExampleName { + var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + try std.testing.expectEqualStrings("foo_bar", try sanitizeExampleName(arena, "foo bar+")); + try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "")); + try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "!")); + try std.testing.expectEqualStrings("a", try sanitizeExampleName(arena, "!a")); + try std.testing.expectEqualStrings("a_b", try sanitizeExampleName(arena, "a.b!")); + try std.testing.expectEqualStrings("_01234", try sanitizeExampleName(arena, "01234")); + try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "error")); + try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "test")); + try std.testing.expectEqualStrings("tests", try sanitizeExampleName(arena, "tests")); + try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project")); +} + +const Templates = struct { + zig_lib_directory: Cache.Directory, + dir: Io.Dir, + buffer: std.array_list.Managed(u8), + + fn deinit(templates: *Templates, io: Io) void { + templates.zig_lib_directory.handle.close(io); + templates.dir.close(io); + templates.buffer.deinit(); + templates.* = undefined; + } + + fn write( + templates: *Templates, + arena: Allocator, + io: Io, + out_dir: Io.Dir, + root_name: []const u8, + template_path: []const u8, + fingerprint: Package.Fingerprint, + ) !void { + if (Dir.path.dirname(template_path)) |dirname| { + out_dir.createDirPath(io, dirname) catch |err| { + fatal("unable to make path {q}: {t}", .{ dirname, err }); + }; + } + + const max_bytes = 10 * 1024 * 1024; + const contents = templates.dir.readFileAlloc(io, template_path, arena, .limited(max_bytes)) catch |err| { + fatal("unable to read template file {q}: {t}", .{ template_path, err }); + }; + templates.buffer.clearRetainingCapacity(); + try templates.buffer.ensureUnusedCapacity(contents.len); + var i: usize = 0; + while (i < contents.len) { + if (contents[i] == '_' or contents[i] == '.') { + // Both '_' and '.' are allowed because depending on the context + // one prefix will be valid, while the other might not. + if (std.mem.startsWith(u8, contents[i + 1 ..], "NAME")) { + try templates.buffer.appendSlice(root_name); + i += "_NAME".len; + continue; + } else if (std.mem.startsWith(u8, contents[i + 1 ..], "FINGERPRINT")) { + try templates.buffer.print("0x{x}", .{fingerprint.int()}); + i += "_FINGERPRINT".len; + continue; + } else if (std.mem.startsWith(u8, contents[i + 1 ..], "ZIGVER")) { + try templates.buffer.appendSlice(builtin.zig_version_string); + i += "_ZIGVER".len; + continue; + } + } + + try templates.buffer.append(contents[i]); + i += 1; + } + + return out_dir.writeFile(io, .{ + .sub_path = template_path, + .data = templates.buffer.items, + .flags = .{ .exclusive = true }, + }); + } + + fn find(gpa: Allocator, io: Io, zig_lib_directory: Cache.Directory) Templates { + const template_path: Path = .{ + .root_dir = zig_lib_directory, + .sub_path = "init", + }; + const template_dir = template_path.root_dir.handle.openDir(io, template_path.sub_path, .{}) catch |err| + fatal("unable to open zig project template directory {f}: {t}", .{ template_path, err }); + return .{ + .zig_lib_directory = zig_lib_directory, + .dir = template_dir, + .buffer = std.array_list.Managed(u8).init(gpa), + }; + } + + fn writeSimpleFile(io: Io, file_name: []const u8, comptime format: []const u8, args: anytype) !void { + const f = try Io.Dir.cwd().createFile(io, file_name, .{ .exclusive = true }); + defer f.close(io); + var buf: [4096]u8 = undefined; + var fw = f.writer(io, &buf); + try fw.interface.print(format, args); + try fw.interface.flush(); + } +}; + +fn confPathDepToCachePath(graph: *const Graph, c: *const Configuration, path_dep: Configuration.PathDep) Path { + const sub_path = path_dep.sub.slice(c); + return switch (path_dep.flags.base) { + .cwd => .{ + .root_dir = .cwd(), + .sub_path = sub_path, + }, + .local_cache => .{ + .root_dir = graph.local_cache_root, + .sub_path = sub_path, + }, + .global_cache => .{ + .root_dir = graph.global_cache_root, + .sub_path = sub_path, + }, + .build_root => .{ + .root_dir = switch (path_dep.pkg.unwrap().?) { + .root => graph.build_root_directory, + _ => @panic("TODO"), + }, + .sub_path = sub_path, + }, + .zig_lib => .{ + .root_dir = graph.zig_lib_directory, + .sub_path = sub_path, + }, + .zig_exe => @panic("TODO"), + .install_prefix => @panic("TODO"), + .install_lib => @panic("TODO"), + .install_bin => @panic("TODO"), + .install_include => @panic("TODO"), + }; +} diff --git a/lib/compiler/Maker/Fetch.zig b/lib/compiler/Maker/Fetch.zig new file mode 100644 index 0000000000000000000000000000000000000000..da5103b0a7ae410407f04c29c3a344bcf0f2f8a3 --- /dev/null +++ b/lib/compiler/Maker/Fetch.zig @@ -0,0 +1,2284 @@ +//! Represents one independent job whose responsibility is to: +//! +//! 1. Check the local zig package directory to see if the hash already exists. +//! If so, load, parse, and validate the build.zig.zon file therein, and +//! goto step 9. Likewise if the location is a relative path, treat this +//! the same as a cache hit. Otherwise, proceed. +//! 2. Check the global package cache for a compressed tarball matching the +//! hash. If it is found, unpack the contents into a temporary directory inside +//! project local zig cache. Rename this directory into the local zig package +//! directory and goto step 9, skipping step 10. +//! 3. Fetch and unpack a URL into a temporary directory. +//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed +//! for the file to be missing, in which case this fetched package is considered +//! to be a "naked" package. +//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by +//! deleting excluded files. If any files had errors for files that were +//! ultimately excluded, those errors should be ignored, such as failure to +//! create symlinks that weren't supposed to be included anyway. +//! 6. Compute the package hash based on the remaining files in the temporary +//! directory. +//! 7. Rename the temporary directory into the local zig package directory. If +//! the hash already exists, delete the temporary directory and leave the zig +//! package directory untouched as it may be in use. This is done even if +//! the hash is invalid, in case the package with the different hash is used +//! in the future. +//! 8. Validate the computed hash against the expected hash. If invalid, +//! this job is done. +//! 9. Spawn a new fetch job for each dependency in the manifest file. Use +//! a mutex and a hash map so that redundant jobs do not get queued up. +//! 10.Compress the package directory and store it into the global package +//! cache. +//! +//! All of this must be done with only referring to the state inside this struct +//! because this work will be done in a dedicated thread. +const Fetch = @This(); + +const builtin = @import("builtin"); +const native_os = builtin.os.tag; + +const std = @import("std"); +const Io = std.Io; +const fs = std.fs; +const log = std.log.scoped(.fetch); +const assert = std.debug.assert; +const ascii = std.ascii; +const Allocator = std.mem.Allocator; +const Path = std.Build.Cache.Path; +const Directory = std.Build.Cache.Directory; +const git = @import("Fetch/git.zig"); +const Package = @import("Package.zig"); +const Manifest = Package.Manifest; +const ErrorBundle = std.zig.ErrorBundle; + +arena: std.heap.ArenaAllocator, +location: Location, +location_tok: std.zig.Ast.TokenIndex, +hash_tok: std.zig.Ast.OptionalTokenIndex, +name_tok: std.zig.Ast.TokenIndex, +lazy_status: LazyStatus, +/// Same as `parent_packge_root` except it is unchanged when recursing into +/// relative file paths (as opposed to URL). +remote_package_root: Path, +parent_package_root: Path, +parent_manifest_ast: ?*const std.zig.Ast, +prog_node: std.Progress.Node, +job_queue: *JobQueue, +/// If true, don't add an error for a missing hash. This flag is not passed +/// down to recursive dependencies. It's intended to be used only be the CLI. +omit_missing_hash_error: bool, +/// If true, don't fail when a manifest file is missing the `paths` field, +/// which specifies inclusion rules. This is intended to be true for the first +/// fetch task and false for the recursive dependencies. +allow_missing_paths_field: bool, +/// If true and URL points to a Git repository, will use the latest commit. +use_latest_commit: bool, + +// Above this are fields provided as inputs to `run`. +// Below this are fields populated by `run`. + +/// Relative to the build root of the root package. +package_root: Path, +error_bundle: ErrorBundle.Wip, +manifest: Manifest, +manifest_ast: std.zig.Ast, +have_manifest: bool, +computed_hash: ComputedHash, +/// Fetch logic notices whether a package has a build.zig file and sets this flag. +has_build_zig: bool, +/// Indicates whether the task aborted due to an out-of-memory condition. +oom_flag: bool, +/// If `use_latest_commit` was true, this will be set to the commit that was used. +/// If the resource pointed to by the location is not a Git-repository, this +/// will be left unchanged. +latest_commit: ?git.Oid, + +// This field is used by the CLI only, untouched by this file. + +/// The module for this `Fetch` tasks's package, which exposes `build.zig` as +/// the root source file. +/// +/// This could be an opaque "userdata" field because this code does not observe +/// this data in any way but let's have some type safety because we can. +cli_module: ?*@import("../Maker.zig").CliModule, + +pub const LazyStatus = enum { + /// Not lazy. + eager, + /// Lazy, found. + available, + /// Lazy, not found. + unavailable, +}; + +pub const LocalStorage = struct { + cache_root: Path, + /// Path to "zig-pkg" inside the package in which the user ran `zig build`. + pkg_root: Path, +}; + +/// Contains shared state among all `Fetch` tasks. +pub const JobQueue = struct { + io: Io, + mutex: Io.Mutex = .init, + /// It's an array hash map so that it can be sorted before rendering the + /// dependencies.zig source file. + /// Protected by `mutex`. + table: Table = .{}, + /// `table` may be missing some tasks such as ones that failed, so this + /// field contains references to all of them. + /// Protected by `mutex`. + all_fetches: std.ArrayList(*Fetch) = .empty, + prog_node: std.Progress.Node, + + http_client: *std.http.Client, + /// This tracks `Fetch` tasks as well as recompression tasks. + group: Io.Group = .init, + global_cache: Directory, + /// If `null`, indicates fetch globally only. + local_storage: ?*const LocalStorage, + /// If true then, no fetching occurs, and: + /// * The `global_cache` directory is assumed to be the direct parent + /// directory of on-disk packages rather than having the "p/" directory + /// prefix inside of it. + /// * An error occurs if any non-lazy packages are not already present in + /// the package cache directory. + /// * Missing hash field causes an error, and no fetching occurs so it does + /// not print the correct hash like usual. + read_only: bool, + recursive: bool, + /// Dumps hash information to stdout which can be used to troubleshoot why + /// two hashes of the same package do not match. + /// If this is true, `recursive` must be false. + debug_hash: bool, + mode: Mode, + /// Set of hashes that will be additionally fetched even if they are marked + /// as lazy. + unlazy_set: UnlazySet = .{}, + /// Identifies paths that override all packages in the tree with matching + /// project ids. + fork_set: ForkSet = .{}, + + pub const Mode = enum { + /// Non-lazy dependencies are always fetched. + /// Lazy dependencies are fetched only when needed. + needed, + /// Both non-lazy and lazy dependencies are always fetched. + all, + }; + pub const Table = std.array_hash_map.Auto(Package.Hash, *Fetch); + pub const UnlazySet = std.array_hash_map.Auto(Package.Hash, void); + pub const ForkSet = std.array_hash_map.Custom(Fork, void, Fork.Context, false); + + pub const Fork = struct { + path: Path, + manifest_ast: std.zig.Ast, + manifest: Package.Manifest, + uses: usize, + + pub const Context = struct { + pub fn hash(_: @This(), a: Fork) u32 { + const project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id); + return @truncate(project_id.hash()); + } + + pub fn eql(_: @This(), a: Fork, b: Fork, _: usize) bool { + const a_project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id); + const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id); + return a_project_id.eql(&b_project_id); + } + }; + + pub const Adapter = struct { + pub fn hash(_: @This(), a: Package.ProjectId) u32 { + return @truncate(a.hash()); + } + + pub fn eql(_: @This(), a_project_id: Package.ProjectId, b: Fork, _: usize) bool { + const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id); + return a_project_id.eql(&b_project_id); + } + }; + }; + + pub fn deinit(jq: *JobQueue) void { + const io = jq.io; + jq.group.cancel(io); + if (jq.all_fetches.items.len == 0) return; + const gpa = jq.all_fetches.items[0].arena.child_allocator; + jq.table.deinit(gpa); + // These must be deinitialized in reverse order because subsequent + // `Fetch` instances are allocated in prior ones' arenas. + // Sorry, I know it's a bit weird, but it slightly simplifies the + // critical section. + while (jq.all_fetches.pop()) |f| f.deinit(); + jq.all_fetches.deinit(gpa); + jq.* = undefined; + } + + /// Dumps all subsequent error bundles into the first one. + pub fn consolidateErrors(jq: *JobQueue) !void { + const root = &jq.all_fetches.items[0].error_bundle; + const gpa = root.gpa; + for (jq.all_fetches.items[1..]) |fetch| { + if (fetch.error_bundle.root_list.items.len > 0) { + var bundle = try fetch.error_bundle.toOwnedBundle(""); + defer bundle.deinit(gpa); + try root.addBundleAsRoots(bundle); + } + } + } + + /// Creates the dependencies.zig source code for the build runner to obtain + /// via `@import("@dependencies")`. + pub fn createDependenciesSource(jq: *JobQueue, w: *Io.Writer) Io.Writer.Error!void { + const keys = jq.table.keys(); + + assert(keys.len != 0); // caller should have added the first one + if (keys.len == 1) { + // This is the first one. It must have no dependencies. + return createEmptyDependenciesSource(w); + } + + try w.writeAll("pub const packages = struct {\n"); + + // Ensure the generated .zig file is deterministic. + jq.table.sortUnstable(@as(struct { + keys: []const Package.Hash, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes); + } + }, .{ .keys = keys })); + + for (keys, jq.table.values()) |*hash, fetch| { + if (fetch == jq.all_fetches.items[0]) { + // The first one is a dummy package for the current project. + continue; + } + + const hash_slice = hash.toSlice(); + + try w.print( + \\ pub const {f} = struct {{ + \\ + , .{std.zig.fmtId(hash_slice)}); + + lazy: { + switch (fetch.lazy_status) { + .eager => break :lazy, + .available => { + try w.writeAll( + \\ pub const available = true; + \\ + ); + break :lazy; + }, + .unavailable => { + try w.writeAll( + \\ pub const available = false; + \\ }; + \\ + ); + continue; + }, + } + } + + try w.print( + \\ pub const build_root = "{f}"; + \\ + , .{std.fmt.alt(fetch.package_root, .formatEscapeString)}); + + if (fetch.has_build_zig) { + try w.print( + \\ pub const build_zig = @import("{f}"); + \\ + , .{std.zig.fmtString(hash_slice)}); + } + + if (fetch.have_manifest) { + const manifest = &fetch.manifest; + try w.writeAll( + \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { + const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; + try w.print( + " .{{ \"{f}\", \"{f}\" }},\n", + .{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) }, + ); + } + + try w.writeAll( + \\ }; + \\ }; + \\ + ); + } else { + try w.writeAll( + \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; + \\ }; + \\ + ); + } + } + + try w.writeAll( + \\}; + \\ + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ + \\ + ); + + const root_fetch = jq.all_fetches.items[0]; + assert(root_fetch.have_manifest); + const root_manifest = &root_fetch.manifest; + + for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { + const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; + try w.print( + " .{{ \"{f}\", \"{f}\" }},\n", + .{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) }, + ); + } + try w.writeAll("};\n"); + } + + pub fn createEmptyDependenciesSource(w: *Io.Writer) Io.Writer.Error!void { + try w.writeAll( + \\pub const packages = struct {}; + \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; + \\ + ); + } + + fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Path) Io.Cancelable!void { + const pkg_hash_slice = package_hash.toSlice(); + + const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice}); + defer prog_node.end(); + + var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined; + const dest_path: Path = .{ + .root_dir = jq.global_cache, + .sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable, + }; + + const gpa = jq.http_client.allocator; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch |err| switch (err) { + error.Canceled => |e| return e, + error.ReadFailed => comptime unreachable, + error.WriteFailed => comptime unreachable, + else => |e| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }), + }; + } + + fn recompressFallible( + jq: *JobQueue, + arena: Allocator, + dest_path: Path, + pkg_hash_slice: []const u8, + package_root: Path, + prog_node: std.Progress.Node, + ) !void { + const gpa = jq.http_client.allocator; + const io = jq.io; + + // We have to walk the file system up front in order to sort the file + // list for determinism purposes. The hash of the recompressed file is + // not critical because the true hash is based on the content alone. + // However, if we want Zig users to be able to share cached package + // data with each other via peer-to-peer protocols, we benefit greatly + // from the data being identical on everyone's computers. + var scanned_files: std.ArrayList(ScannedFile) = .empty; + defer scanned_files.deinit(gpa); + + var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true }); + defer pkg_dir.close(io); + + { + var walker = try pkg_dir.walk(gpa); + defer walker.deinit(); + + while (try walker.next(io)) |entry| { + const symlink = switch (entry.kind) { + .directory => continue, + .file => false, + .sym_link => true, + else => return error.IllegalFileType, + }; + const entry_path = try arena.dupe(u8, entry.path); + // If necessary, normalize path separators to POSIX-style since the tar format requires that. + if (comptime (std.fs.path.sep != std.fs.path.sep_posix)) { + std.mem.replaceScalar(u8, entry_path, std.fs.path.sep, std.fs.path.sep_posix); + } + try scanned_files.append(gpa, .{ + .ptr = entry_path.ptr, + .len = @intCast(entry_path.len), + .symlink = symlink, + }); + } + + std.mem.sortUnstable(ScannedFile, scanned_files.items, {}, stringCmp); + } + + prog_node.setEstimatedTotalItems(scanned_files.items.len); + + var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{ + .make_path = true, + .replace = true, + }); + defer atomic_file.deinit(io); + + var file_write_buffer: [4096]u8 = undefined; + var file_writer = atomic_file.file.writer(io, &file_write_buffer); + + var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined; + var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch |err| switch (err) { + error.WriteFailed => return file_writer.err.?, + }; + + var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer }; + archiver.prefix = pkg_hash_slice; + + var file_read_buffer: [4096]u8 = undefined; + var link_buf: [fs.max_path_bytes]u8 = undefined; + + for (scanned_files.items) |scanned_file| { + const entry_path = scanned_file.ptr[0..scanned_file.len]; + if (scanned_file.symlink) { + const link_name = link_buf[0..try pkg_dir.readLink(io, entry_path, &link_buf)]; + archiver.writeLink(entry_path, link_name, .{}) catch |err| switch (err) { + error.WriteFailed => return file_writer.err.?, + else => |e| return e, + }; + } else { + var file = try pkg_dir.openFile(io, entry_path, .{}); + defer file.close(io); + var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer); + archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) { + error.ReadFailed => return file_reader.err.?, + error.WriteFailed => return file_writer.err.?, + else => |e| return e, + }; + } + prog_node.completeOne(); + } + + // intentionally omitting the pointless trailer + //try archiver.finish(); + compress.finish() catch |err| switch (err) { + error.WriteFailed => return file_writer.err.?, + }; + try file_writer.flush(); + try atomic_file.replace(io); + } +}; + +const ScannedFile = struct { + ptr: [*]const u8, + len: u32, + symlink: bool, +}; + +fn stringCmp(_: void, lhs: ScannedFile, rhs: ScannedFile) bool { + return std.mem.lessThan(u8, lhs.ptr[0..lhs.len], rhs.ptr[0..rhs.len]); +} + +pub const Location = union(enum) { + remote: Remote, + /// A directory found inside the parent package. + relative_path: Path, + /// Recursive Fetch tasks will never use this Location, but it may be + /// passed in by the CLI. Indicates the file contents here should be copied + /// into the global package cache. It may be a file relative to the cwd or + /// absolute, in which case it should be treated exactly like a `file://` + /// URL, or a directory, in which case it should be treated as an + /// already-unpacked directory (but still needs to be copied into the + /// global package cache and have inclusion rules applied). + path_or_url: []const u8, + + pub const Remote = struct { + url: []const u8, + /// If this is null it means the user omitted the hash field from a dependency. + /// It will be an error but the logic should still fetch and print the discovered hash. + hash: ?Package.Hash, + }; +}; + +pub const RunError = error{ + OutOfMemory, + Canceled, + /// This error code is intended to be handled by inspecting the + /// `error_bundle` field. + FetchFailed, +}; + +pub fn run(f: *Fetch) RunError!void { + const job_queue = f.job_queue; + const io = job_queue.io; + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + + try eb.init(gpa); + + // Check the global zig package cache to see if the hash already exists. If + // so, load, parse, and validate the build.zig.zon file therein, and skip + // ahead to queuing up jobs for dependencies. Likewise if the location is a + // relative path, treat this the same as a cache hit. Otherwise, proceed. + + const remote = switch (f.location) { + .relative_path => |pkg_root| { + if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( + f.location_tok, + try eb.addString("expected path relative to build root; found absolute path"), + ); + if (f.hash_tok.unwrap()) |hash_tok| return f.fail( + hash_tok, + try eb.addString("path-based dependencies are not hashed"), + ); + // Packages fetched by URL may not use relative paths to escape outside the + // fetched package directory from within the package cache. + + // This code path is only reachable recursively and the sub_path + // will already have been resolved to no longer have extra ".." or + // "." components. + assert(job_queue.local_storage != null); + assert(pkg_root.root_dir.eql(f.remote_package_root.root_dir)); + if (!std.mem.startsWith(u8, pkg_root.sub_path, f.remote_package_root.sub_path)) return f.fail( + f.location_tok, + try eb.printString("dependency path outside project: '{f}'", .{pkg_root}), + ); + f.package_root = pkg_root; + try loadManifest(f, pkg_root); + if (!f.has_build_zig) try checkBuildFileExistence(f); + if (!job_queue.recursive) return; + return queueJobsForDeps(f); + }, + .remote => |remote| remote, + .path_or_url => |path_or_url| { + if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) |dir| { + var resource: Resource = .{ .dir = dir }; + return f.runResource(path_or_url, &resource, null, false); + } else |dir_err| { + var server_header_buffer: [init_resource_buffer_size]u8 = undefined; + + const file_err = if (dir_err == error.NotDir) e: { + if (Io.Dir.cwd().openFile(io, path_or_url, .{})) |file| { + var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) }; + return f.runResource(path_or_url, &resource, null, false); + } else |err| break :e err; + } else dir_err; + + const uri = std.Uri.parse(path_or_url) catch |uri_err| { + return f.fail(0, try eb.printString( + "'{s}' could not be recognized as a file path ({t}) or an URL ({t})", + .{ path_or_url, file_err, uri_err }, + )); + }; + var resource: Resource = undefined; + try f.initResource(uri, &resource, &server_header_buffer); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false); + } + }, + }; + + var resource_buffer: [init_resource_buffer_size]u8 = undefined; + + if (remote.hash) |expected_hash| { + const expected_project_id: Package.ProjectId = expected_hash.projectId(); + if (job_queue.fork_set.getKeyPtrAdapted(expected_project_id, @as(JobQueue.Fork.Adapter, .{}))) |fork| { + log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name }); + fork.uses += 1; + f.package_root = fork.path; + f.remote_package_root = f.package_root; + f.manifest_ast = fork.manifest_ast; + f.manifest = fork.manifest; + f.have_manifest = true; + try checkBuildFileExistence(f); + if (!job_queue.recursive) return; + return queueJobsForDeps(f); + } + + if (job_queue.local_storage) |ls| { + const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice()); + if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| { + assert(f.lazy_status != .unavailable); + f.package_root = package_root; + f.remote_package_root = f.package_root; + try loadManifest(f, f.package_root); + try checkBuildFileExistence(f); + if (!job_queue.recursive) return; + return queueJobsForDeps(f); + } else |err| switch (err) { + error.FileNotFound => { + log.debug("FileNotFound: {f}", .{package_root}); + if (job_queue.read_only and f.lazy_status == .eager) return f.fail( + f.name_tok, + try eb.printString("package not found at '{f}'", .{package_root}), + ); + }, + error.Canceled => |e| return e, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{ + package_root, e, + }), + }); + return error.FetchFailed; + }, + } + } + + // Check global cache before remote fetch. + const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()}); + const cached_tarball_path: Path = .{ + .root_dir = job_queue.global_cache, + .sub_path = cached_tarball_sub_path, + }; + if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) |file| { + log.debug("found global cached tarball {f}", .{cached_tarball_path}); + var resource: Resource = .{ .file = file.reader(io, &resource_buffer) }; + return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true); + } else |err| switch (err) { + error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}), + error.Canceled => |e| return e, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to open globally cached package {f}: {t}", .{ + cached_tarball_path, e, + }), + }); + return error.FetchFailed; + }, + } + + switch (f.lazy_status) { + .eager => {}, + .available => if (!job_queue.unlazy_set.contains(expected_hash)) { + f.lazy_status = .unavailable; + return; + }, + .unavailable => unreachable, + } + } else if (job_queue.read_only) { + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("dependency is missing hash field"), + .src_loc = try f.srcLoc(f.location_tok), + }); + return error.FetchFailed; + } + + // Fetch and unpack the remote into a temporary directory. + const uri = std.Uri.parse(remote.url) catch |err| return f.fail( + f.location_tok, + try eb.printString("invalid URI: {t}", .{err}), + ); + var resource: Resource = undefined; + try f.initResource(uri, &resource, &resource_buffer); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false); +} + +pub fn deinit(f: *Fetch) void { + f.error_bundle.deinit(); + f.arena.deinit(); +} + +/// Consumes `resource`, even if an error is returned. +fn runResource( + f: *Fetch, + uri_path: []const u8, + resource: *Resource, + remote_hash: ?Package.Hash, + disable_recompress: bool, +) RunError!void { + const job_queue = f.job_queue; + assert(!job_queue.read_only); + + const io = job_queue.io; + defer resource.deinit(io); + + const arena = f.arena.allocator(); + const eb = &f.error_bundle; + const rand_int = r: { + var x: u64 = undefined; + io.random(@ptrCast(&x)); + break :r x; + }; + const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int); + const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path; + const tmp_directory_path: Path = if (job_queue.local_storage) |ls| + try ls.pkg_root.join(arena, tmp_dir_sub_path) + else + .{ + .root_dir = job_queue.global_cache, + .sub_path = tmp_tmp_dir_sub_path, + }; + + const package_sub_path = blk: { + var tmp_directory: Directory = .{ + .path = tmp_directory_path.sub_path, + .handle = handle: { + const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{ + .open_options = .{ .iterate = true }, + }) catch |err| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{ + tmp_directory_path, err, + }), + }); + return error.FetchFailed; + }; + break :handle dir; + }, + }; + defer tmp_directory.handle.close(io); + + // Fetch and unpack a resource into a temporary directory. + var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); + + const pkg_path: Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; + + // Load, parse, and validate the unpacked build.zig.zon file. It is allowed + // for the file to be missing, in which case this fetched package is + // considered to be a "naked" package. + try loadManifest(f, pkg_path); + + const filter: Filter = .{ + .include_paths = if (f.have_manifest) f.manifest.paths else .{}, + }; + + // Ignore errors that were excluded by manifest, such as failure to + // create symlinks that weren't supposed to be included anyway. + try unpack_result.validate(f, filter); + + // Apply the manifest's inclusion rules to the temporary directory by + // deleting excluded files. + // Empty directories have already been omitted by `unpackResource`. + // Compute the package hash based on the remaining files in the temporary + // directory. + f.computed_hash = try computeHash(f, pkg_path, filter); + + if (unpack_result.root_dir.len > 0) + break :blk try tmp_directory_path.join(arena, unpack_result.root_dir); + + break :blk tmp_directory_path; + }; + + const computed_package_hash = computedPackageHash(f); + + // Rename the temporary directory into the local zig package directory. If + // the hash already exists, delete the temporary directory and leave the + // zig package directory untouched as it may be in use. This is done even + // if the hash is invalid, in case the package with the different hash is + // used in the future. + if (job_queue.local_storage) |ls| { + f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice()); + renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "failed to rename temporary directory {f} into package cache directory {f}: {t}", + .{ package_sub_path, f.package_root, err }, + ) }); + return error.FetchFailed; + }; + } else { + f.package_root = tmp_directory_path; + } + f.remote_package_root = f.package_root; + + if (!disable_recompress) { + // Spin off a task to recompress the tarball, with filtered files deleted, into + // the global cache. + job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root }); + } + + // Remove temporary directory root if not already renamed to global cache. + if (!package_sub_path.eql(tmp_directory_path)) { + tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_directory_path, e }), + }; + } + + // Validate the computed hash against the expected hash. If invalid, this + // job is done. + + if (remote_hash) |declared_hash| { + const hash_tok = f.hash_tok.unwrap().?; + if (!computed_package_hash.eql(&declared_hash)) { + return f.fail(hash_tok, try eb.printString( + "hash mismatch: manifest declares {s} but the fetched package has {s}", + .{ declared_hash.toSlice(), computed_package_hash.toSlice() }, + )); + } + } else if (!f.omit_missing_hash_error) { + const notes_len = 1; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("dependency is missing hash field"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("expected .hash = {q},", .{computed_package_hash.toSlice()}), + })); + return error.FetchFailed; + } + + // Spawn a new fetch job for each dependency in the manifest file. Use + // a mutex and a hash map so that redundant jobs do not get queued up. + if (!job_queue.recursive) return; + return queueJobsForDeps(f); +} + +pub fn computedPackageHash(f: *const Fetch) Package.Hash { + const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32); + if (f.have_manifest) { + const man = &f.manifest; + var version_buffer: [32]u8 = undefined; + const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{f}", .{man.version}) catch &version_buffer; + return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size); + } + // In the future build.zig.zon fields will be added to allow overriding these values + // for naked tarballs. + return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size); +} + +/// `computeHash` gets a free check for the existence of `build.zig`, but when +/// not computing a hash, we need to do a syscall to check for it. +fn checkBuildFileExistence(f: *Fetch) RunError!void { + const io = f.job_queue.io; + const eb = &f.error_bundle; + if (f.package_root.access(io, std.zig.build_zig_basename, .{})) |_| { + f.has_build_zig = true; + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to access {f}/{s}: {t}", .{ + f.package_root, std.zig.build_zig_basename, e, + }), + }); + return error.FetchFailed; + }, + } +} + +/// This function populates `f.manifest` or leaves it `null`. +fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void { + const io = f.job_queue.io; + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + const manifest_path = try pkg_root.join(arena, Manifest.basename); + + Manifest.load( + io, + arena, + manifest_path, + &f.manifest_ast, + eb, + &f.manifest, + f.allow_missing_paths_field, + ) catch |err| switch (err) { + error.FileNotFound => return, + error.Canceled => |e| return e, + error.ErrorsBundled => return error.FetchFailed, + else => |e| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }), + }); + return error.FetchFailed; + }, + }; + f.have_manifest = true; +} + +fn queueJobsForDeps(f: *Fetch) RunError!void { + const io = f.job_queue.io; + + assert(f.job_queue.recursive); + + // If the package does not have a build.zig.zon file then there are no dependencies. + if (!f.have_manifest) return; + const manifest = &f.manifest; + + const new_fetches, const prog_names = nf: { + const parent_arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const cache_root = f.job_queue.global_cache; + const dep_names = manifest.dependencies.keys(); + const deps = manifest.dependencies.values(); + // Grab the new tasks into a temporary buffer so we can unlock that mutex + // as fast as possible. + // This overallocates any fetches that get skipped by the `continue` in the + // loop below. + const new_fetches = try parent_arena.alloc(Fetch, deps.len); + const prog_names = try parent_arena.alloc([]const u8, deps.len); + var new_fetch_index: usize = 0; + + try f.job_queue.mutex.lock(io); + defer f.job_queue.mutex.unlock(io); + + try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); + try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); + + // There are four cases here: + // * Correct hash is provided by manifest. + // - Hash map already has the entry, no need to add it again. + // * Incorrect hash is provided by manifest. + // - Hash mismatch error emitted; `queueJobsForDeps` is not called. + // * Hash is not provided by manifest. + // - Hash missing error emitted; `queueJobsForDeps` is not called. + // * path-based location is used without a hash. + // - Hash is added to the table based on the path alone before + // calling run(); no need to add it again. + // + // If we add a dep as lazy and then later try to add the same dep as eager, + // eagerness takes precedence and the existing entry is updated and re-scheduled + // for fetching. + + for (dep_names, deps) |dep_name, dep| { + var promoted_existing_to_eager = false; + const new_fetch = &new_fetches[new_fetch_index]; + const location: Location = switch (dep.location) { + .url => |url| .{ + .remote = .{ + .url = url, + .hash = h: { + const h = dep.hash orelse break :h null; + const pkg_hash: Package.Hash = .fromSlice(h); + if (h.len == 0) break :h pkg_hash; + const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); + if (gop.found_existing) { + if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) { + gop.value_ptr.*.lazy_status = .eager; + promoted_existing_to_eager = true; + } else { + continue; + } + } + gop.value_ptr.* = new_fetch; + break :h pkg_hash; + }, + }, + }, + .path => |rel_path| l: { + // This might produce an invalid path, which is checked for + // at the beginning of run(). + const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); + const pkg_hash = relativePathDigest(new_root, cache_root); + const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); + if (gop.found_existing) { + if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) { + gop.value_ptr.*.lazy_status = .eager; + promoted_existing_to_eager = true; + } else { + continue; + } + } + gop.value_ptr.* = new_fetch; + break :l .{ .relative_path = new_root }; + }, + }; + prog_names[new_fetch_index] = dep_name; + new_fetch_index += 1; + if (!promoted_existing_to_eager) { + f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); + } + new_fetch.* = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = location, + .location_tok = dep.location_tok, + .hash_tok = dep.hash_tok, + .name_tok = dep.name_tok, + .lazy_status = switch (f.job_queue.mode) { + .needed => if (dep.lazy) .available else .eager, + .all => .eager, + }, + .parent_package_root = f.package_root, + .remote_package_root = f.remote_package_root, + .parent_manifest_ast = &f.manifest_ast, + .prog_node = f.prog_node, + .job_queue = f.job_queue, + .omit_missing_hash_error = false, + .allow_missing_paths_field = true, + .use_latest_commit = false, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = undefined, + .manifest_ast = undefined, + .have_manifest = false, + .computed_hash = undefined, + .has_build_zig = false, + .oom_flag = false, + .latest_commit = null, + + .cli_module = null, + }; + } + + f.prog_node.increaseEstimatedTotalItems(new_fetch_index); + + break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] }; + }; + + // Now it's time to dispatch tasks. + for (new_fetches, prog_names) |*new_fetch, prog_name| { + f.job_queue.group.async(io, workerRun, .{ new_fetch, prog_name }); + } +} + +pub fn relativePathDigest(pkg_root: Path, cache_root: Directory) Package.Hash { + return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root)); +} + +pub fn workerRun(f: *Fetch, prog_name: []const u8) Io.Cancelable!void { + const prog_node = f.prog_node.start(prog_name, 0); + defer prog_node.end(); + + run(f) catch |err| switch (err) { + error.OutOfMemory => f.oom_flag = true, + error.Canceled => |e| return e, + error.FetchFailed => { + // Nothing to do because the errors are already reported in `error_bundle`, + // and a reference is kept to the `Fetch` task inside `all_fetches`. + }, + }; +} + +fn srcLoc( + f: *Fetch, + tok: std.zig.Ast.TokenIndex, +) Allocator.Error!ErrorBundle.SourceLocationIndex { + const ast = f.parent_manifest_ast orelse return .none; + const eb = &f.error_bundle; + const start_loc = ast.tokenLocation(0, tok); + const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root}); + const msg_off = 0; + return eb.addSourceLocation(.{ + .src_path = src_path, + .span_start = ast.tokenStart(tok), + .span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len), + .span_main = ast.tokenStart(tok) + msg_off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }); +} + +fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { + const eb = &f.error_bundle; + try eb.addRootErrorMessage(.{ + .msg = msg_str, + .src_loc = try f.srcLoc(msg_tok), + }); + return error.FetchFailed; +} + +const Resource = union(enum) { + file: Io.File.Reader, + http_request: HttpRequest, + git: Git, + dir: Io.Dir, + + const Git = struct { + session: git.Session, + fetch_stream: git.Session.FetchStream, + want_oid: git.Oid, + }; + + const HttpRequest = struct { + request: std.http.Client.Request, + response: std.http.Client.Response, + transfer_buffer: []u8, + decompress: std.http.Decompress, + decompress_buffer: []u8, + }; + + fn deinit(resource: *Resource, io: Io) void { + switch (resource.*) { + .file => |*file_reader| file_reader.file.close(io), + .http_request => |*http_request| http_request.request.deinit(), + .git => |*git_resource| { + git_resource.fetch_stream.deinit(); + }, + .dir => |*dir| dir.close(io), + } + resource.* = undefined; + } + + fn reader(resource: *Resource) *Io.Reader { + return switch (resource.*) { + .file => |*file_reader| return &file_reader.interface, + .http_request => |*http_request| return http_request.response.readerDecompressing( + http_request.transfer_buffer, + &http_request.decompress, + http_request.decompress_buffer, + ), + .git => |*g| return &g.fetch_stream.reader, + .dir => unreachable, + }; + } +}; + +const FileType = enum { + tar, + @"tar.gz", + @"tar.xz", + @"tar.zst", + git_pack, + zip, + + fn fromPath(file_path: []const u8) ?FileType { + if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; + if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; + if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; + if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst"; + if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst"; + if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip; + if (ascii.endsWithIgnoreCase(file_path, ".jar")) return .zip; + return null; + } + + /// Parameter is a content-disposition header value. + fn fromContentDisposition(cd_header: []const u8) ?FileType { + const attach_end = ascii.findIgnoreCase(cd_header, "attachment;") orelse + return null; + + var value_start = ascii.findIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse + return null; + value_start += "filename".len; + if (cd_header[value_start] == '*') { + value_start += 1; + } + if (cd_header[value_start] != '=') return null; + value_start += 1; + + var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; + if (cd_header[value_end - 1] == '\"') { + value_end -= 1; + } + return fromPath(cd_header[value_start..value_end]); + } + + test fromContentDisposition { + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); + try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); + try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\"")); + + try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); + try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); + try std.testing.expect(fromContentDisposition("attachment; size=42") == null); + try std.testing.expect(fromContentDisposition("inline; size=42") == null); + try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); + try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); + } +}; + +const init_resource_buffer_size = git.Packet.max_data_length; + +fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u8) RunError!void { + const io = f.job_queue.io; + const arena = f.arena.allocator(); + const eb = &f.error_bundle; + + if (ascii.eqlIgnoreCase(uri.scheme, "file")) { + const path = try uri.path.toRawMaybeAlloc(arena); + const file = f.parent_package_root.openFile(io, path, .{}) catch |err| { + return f.fail(f.location_tok, try eb.printString("unable to open {f}/{s}: {t}", .{ + f.parent_package_root, path, err, + })); + }; + resource.* = .{ .file = file.reader(io, reader_buffer) }; + return; + } + + const http_client = f.job_queue.http_client; + + if (ascii.eqlIgnoreCase(uri.scheme, "http") or + ascii.eqlIgnoreCase(uri.scheme, "https")) + { + resource.* = .{ .http_request = .{ + .request = http_client.request(.GET, uri, .{}) catch |err| + return f.fail(f.location_tok, try eb.printString("server connection failed: {t}", .{err})), + .response = undefined, + .transfer_buffer = reader_buffer, + .decompress_buffer = &.{}, + .decompress = undefined, + } }; + const request = &resource.http_request.request; + errdefer request.deinit(); + + request.sendBodiless() catch |err| + return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err})); + + var redirect_buffer: [8000]u8 = undefined; + const response = &resource.http_request.response; + response.* = request.receiveHead(&redirect_buffer) catch |err| switch (err) { + error.ReadFailed => { + return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{ + request.connection.?.getReadError().?, + })); + }, + else => |e| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})), + }; + + if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString( + "bad HTTP response code: '{d} {s}'", + .{ response.head.status, response.head.status.phrase() orelse "" }, + )); + + resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); + return; + } + + if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or + ascii.eqlIgnoreCase(uri.scheme, "git+https")) + { + var transport_uri = uri; + transport_uri.scheme = uri.scheme["git+".len..]; + var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch |err| { + return f.fail( + f.location_tok, + try eb.printString("unable to discover remote git server capabilities: {t}", .{err}), + ); + }; + + const want_oid = want_oid: { + const want_ref = + if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD"; + if (git.Oid.parseAny(want_ref)) |oid| break :want_oid oid else |_| {} + + const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); + const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); + + var ref_iterator: git.Session.RefIterator = undefined; + session.listRefs(&ref_iterator, .{ + .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, + .include_peeled = true, + .buffer = reader_buffer, + }) catch |err| return f.fail(f.location_tok, try eb.printString("unable to list refs: {t}", .{err})); + defer ref_iterator.deinit(); + while (ref_iterator.next() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to iterate refs: {s}", + .{@errorName(err)}, + )); + }) |ref| { + if (std.mem.eql(u8, ref.name, want_ref) or + std.mem.eql(u8, ref.name, want_ref_head) or + std.mem.eql(u8, ref.name, want_ref_tag)) + { + break :want_oid ref.peeled orelse ref.oid; + } + } + return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); + }; + if (f.use_latest_commit) { + f.latest_commit = want_oid; + } else if (uri.fragment == null) { + const notes_len = 1; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("url field is missing an explicit ref"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); + const notes_start = try eb.reserveNotes(notes_len); + eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("try .url = \"{f}#{f}\",", .{ + uri.fmt(.{ .scheme = true, .authority = true, .path = true }), + want_oid, + }), + })); + return error.FetchFailed; + } + + var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined; + _ = std.fmt.bufPrint(&want_oid_buf, "{f}", .{want_oid}) catch unreachable; + resource.* = .{ .git = .{ + .session = session, + .fetch_stream = undefined, + .want_oid = want_oid, + } }; + const fetch_stream = &resource.git.fetch_stream; + session.fetch(fetch_stream, &.{&want_oid_buf}, reader_buffer) catch |err| { + return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err})); + }; + errdefer fetch_stream.deinit(fetch_stream); + + return; + } + + return f.fail(f.location_tok, try eb.printString("unsupported URL scheme: {s}", .{uri.scheme})); +} + +fn unpackResource( + f: *Fetch, + resource: *Resource, + uri_path: []const u8, + tmp_directory: Directory, +) RunError!UnpackResult { + const eb = &f.error_bundle; + const file_type = switch (resource.*) { + .file => FileType.fromPath(uri_path) orelse + return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), + + .http_request => |*http_request| ft: { + const head = &http_request.response.head; + + // Content-Type takes first precedence. + const content_type = head.content_type orelse + return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); + + // Extract the MIME type, ignoring charset and boundary directives + const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len; + const mime_type = content_type[0..mime_type_end]; + + if (ascii.eqlIgnoreCase(mime_type, "application/x-tar")) + break :ft .tar; + + if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or + ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or + ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or + ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or + ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed")) + { + break :ft .@"tar.gz"; + } + + if (ascii.eqlIgnoreCase(mime_type, "application/x-xz")) + break :ft .@"tar.xz"; + + if (ascii.eqlIgnoreCase(mime_type, "application/zstd")) + break :ft .@"tar.zst"; + + if (ascii.eqlIgnoreCase(mime_type, "application/zip") or + ascii.eqlIgnoreCase(mime_type, "application/x-zip-compressed") or + ascii.eqlIgnoreCase(mime_type, "application/java-archive")) + { + break :ft .zip; + } + + if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and + !ascii.eqlIgnoreCase(mime_type, "application/x-compressed")) + { + return f.fail(f.location_tok, try eb.printString( + "unrecognized 'Content-Type' header: '{s}'", + .{content_type}, + )); + } + + // Next, the filename from 'content-disposition: attachment' takes precedence. + if (head.content_disposition) |cd_header| { + break :ft FileType.fromContentDisposition(cd_header) orelse { + return f.fail(f.location_tok, try eb.printString( + "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", + .{cd_header}, + )); + }; + } + + // Finally, the path from the URI is used. + break :ft FileType.fromPath(uri_path) orelse { + return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})); + }; + }, + + .git => .git_pack, + + .dir => |dir| { + f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { + return f.fail(f.location_tok, try eb.printString("unable to copy directory '{s}': {t}", .{ + uri_path, err, + })); + }; + return .{}; + }, + }; + + switch (file_type) { + .tar => { + return unpackTarball(f, tmp_directory.handle, resource.reader()); + }, + .@"tar.gz" => { + var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; + var decompress: std.compress.flate.Decompress = .init(resource.reader(), .gzip, &flate_buffer); + return try unpackTarball(f, tmp_directory.handle, &decompress.reader); + }, + .@"tar.xz" => { + const gpa = f.arena.child_allocator; + var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch |err| + return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err})); + defer decompress.deinit(); + return try unpackTarball(f, tmp_directory.handle, &decompress.reader); + }, + .@"tar.zst" => { + const window_len = std.compress.zstd.default_window_len; + const window_buffer = try f.arena.allocator().alloc(u8, window_len + std.compress.zstd.block_size_max); + var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{ + .verify_checksum = false, + .window_len = window_len, + }); + return try unpackTarball(f, tmp_directory.handle, &decompress.reader); + }, + .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) { + error.FetchFailed, error.OutOfMemory => |e| return e, + else => |e| return f.fail(f.location_tok, try eb.printString("unable to unpack git files: {t}", .{e})), + }, + .zip => return unzip(f, tmp_directory.handle, resource.reader()) catch |err| switch (err) { + error.ReadFailed => return f.fail(f.location_tok, try eb.printString( + "failed reading resource: {t}", + .{err}, + )), + else => |e| return e, + }, + } +} + +fn unpackTarball(f: *Fetch, out_dir: Io.Dir, reader: *Io.Reader) RunError!UnpackResult { + const eb = &f.error_bundle; + const arena = f.arena.allocator(); + const io = f.job_queue.io; + + var diagnostics: std.tar.Diagnostics = .{ .allocator = arena }; + + std.tar.pipeToFileSystem(io, out_dir, reader, .{ + .diagnostics = &diagnostics, + .strip_components = 0, + .mode_mode = .ignore, + .exclude_empty_directories = true, + }) catch |err| return f.fail( + f.location_tok, + try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}), + ); + + var res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; + if (diagnostics.errors.items.len > 0) { + try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball"); + for (diagnostics.errors.items) |item| { + switch (item) { + .unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code), + .unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code), + .unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)), + .components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0 + } + } + } + return res; +} + +fn unzip( + f: *Fetch, + out_dir: Io.Dir, + reader: *Io.Reader, +) error{ ReadFailed, OutOfMemory, Canceled, FetchFailed }!UnpackResult { + // We write the entire contents to a file first because zip files + // must be processed back to front and they could be too large to + // load into memory. + + const io = f.job_queue.io; + const cache_root = f.job_queue.global_cache; + const prefix = "tmp/"; + const suffix = ".zip"; + const eb = &f.error_bundle; + const random_len = @sizeOf(u64) * 2; + + var zip_path: [prefix.len + random_len + suffix.len]u8 = undefined; + zip_path[0..prefix.len].* = prefix.*; + zip_path[prefix.len + random_len ..].* = suffix.*; + + var zip_file = while (true) { + const random_integer = r: { + var x: u64 = undefined; + io.random(@ptrCast(&x)); + break :r x; + }; + zip_path[prefix.len..][0..random_len].* = std.fmt.hex(random_integer); + + break cache_root.handle.createFile(io, &zip_path, .{ + .exclusive = true, + .read = true, + }) catch |err| switch (err) { + error.PathAlreadyExists => continue, + error.FileNotFound => { + cache_root.handle.createDir(io, prefix, .default_dir) catch |dir_err| switch (dir_err) { + error.Canceled => |e| return e, + // error.PathAlreadyExists is considered a failure here because + // it implies that the prefix is not a directory. + else => |e| return f.fail( + f.location_tok, + try eb.printString("failed to create temporary directory: {t}", .{e}), + ), + }; + continue; + }, + error.Canceled => |e| return e, + else => |e| return f.fail( + f.location_tok, + try eb.printString("failed to create temporary zip file: {t}", .{e}), + ), + }; + }; + defer zip_file.close(io); + var zip_file_buffer: [4096]u8 = undefined; + var zip_file_reader = b: { + var zip_file_writer = zip_file.writer(io, &zip_file_buffer); + + _ = reader.streamRemaining(&zip_file_writer.interface) catch |err| switch (err) { + error.ReadFailed => |e| return e, + error.WriteFailed => return f.fail( + f.location_tok, + try eb.printString("failed writing temporary zip file: {t}", .{err}), + ), + }; + zip_file_writer.interface.flush() catch |err| return f.fail( + f.location_tok, + try eb.printString("failed writing temporary zip file: {t}", .{err}), + ); + break :b zip_file_writer.moveToReader(); + }; + + var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() }; + // no need to deinit since we are using an arena allocator + + zip_file_reader.seekTo(0) catch |err| + return f.fail(f.location_tok, try eb.printString("failed to seek temporary zip file: {t}", .{err})); + std.zip.extract(out_dir, &zip_file_reader, .{ + .allow_backslashes = true, + .diagnostics = &diagnostics, + }) catch |err| return f.fail(f.location_tok, try eb.printString("zip extract failed: {t}", .{err})); + + cache_root.handle.deleteFile(io, &zip_path) catch |err| + return f.fail(f.location_tok, try eb.printString("delete temporary zip failed: {t}", .{err})); + + return .{ .root_dir = diagnostics.root_dir }; +} + +fn unpackGitPack(f: *Fetch, out_dir: Io.Dir, resource: *Resource.Git) anyerror!UnpackResult { + const io = f.job_queue.io; + const arena = f.arena.allocator(); + // TODO don't try to get a gpa from an arena. expose this dependency higher up + // because the backing of arena could be page allocator + const gpa = f.arena.child_allocator; + const object_format: git.Oid.Format = resource.want_oid; + + var res: UnpackResult = .{}; + // The .git directory is used to store the packfile and associated index, but + // we do not attempt to replicate the exact structure of a real .git + // directory, since that isn't relevant for fetching a package. + { + var pack_dir = try out_dir.createDirPathOpen(io, ".git", .{}); + defer pack_dir.close(io); + var pack_file = try pack_dir.createFile(io, "pkg.pack", .{ .read = true }); + defer pack_file.close(io); + var pack_file_buffer: [4096]u8 = undefined; + var pack_file_reader = b: { + var pack_file_writer = pack_file.writer(io, &pack_file_buffer); + const fetch_reader = &resource.fetch_stream.reader; + _ = try fetch_reader.streamRemaining(&pack_file_writer.interface); + try pack_file_writer.interface.flush(); + break :b pack_file_writer.moveToReader(); + }; + + var index_file = try pack_dir.createFile(io, "pkg.idx", .{ .read = true }); + defer index_file.close(io); + var index_file_buffer: [2000]u8 = undefined; + var index_file_writer = index_file.writer(io, &index_file_buffer); + { + const index_prog_node = f.prog_node.start("Index pack", 0); + defer index_prog_node.end(); + try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer); + } + + { + var index_file_reader = index_file.reader(io, &index_file_buffer); + const checkout_prog_node = f.prog_node.start("Checkout", 0); + defer checkout_prog_node.end(); + var repository: git.Repository = undefined; + try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader); + defer repository.deinit(); + var diagnostics: git.Diagnostics = .{ .allocator = arena }; + try repository.checkout(io, out_dir, resource.want_oid, &diagnostics); + + if (diagnostics.errors.items.len > 0) { + try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile"); + for (diagnostics.errors.items) |item| { + switch (item) { + .unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code), + .unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code), + } + } + } + } + } + + try out_dir.deleteTree(io, ".git"); + return res; +} + +fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void { + const gpa = f.arena.child_allocator; + const io = f.job_queue.io; + // Recursive directory copy. + var it = try dir.walk(gpa); + defer it.deinit(); + while (try it.next(io)) |entry| { + switch (entry.kind) { + .directory => {}, // omit empty directories + .file => { + dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}) catch |err| switch (err) { + error.FileNotFound => { + if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.createDirPath(io, dirname); + try dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}); + }, + else => |e| return e, + }; + }, + .sym_link => { + var buf: [fs.max_path_bytes]u8 = undefined; + const link_name = buf[0..try dir.readLink(io, entry.path, &buf)]; + // TODO: if this would create a symlink to outside + // the destination directory, fail with an error instead. + tmp_dir.symLink(io, link_name, entry.path, .{}) catch |err| switch (err) { + error.FileNotFound => { + if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.createDirPath(io, dirname); + try tmp_dir.symLink(io, link_name, entry.path, .{}); + }, + else => |e| return e, + }; + }, + else => return error.IllegalFileTypeInPackage, + } + } +} + +pub fn renameTmpIntoCache(io: Io, tmp_path: Path, dest_path: Path) !void { + var handled_missing_dir = false; + while (true) { + Io.Dir.rename( + tmp_path.root_dir.handle, + tmp_path.sub_path, + dest_path.root_dir.handle, + dest_path.sub_path, + io, + ) catch |err| switch (err) { + error.FileNotFound => { + if (handled_missing_dir) return err; + const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?; + dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch |er| switch (er) { + error.PathAlreadyExists => handled_missing_dir = true, + else => |e| return e, + }; + continue; + }, + error.DirNotEmpty, error.AccessDenied => { + // Package has been already downloaded and may already be in use on the system. + tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch |er| switch (er) { + error.Canceled => |e| return e, + // Garbage files leftover in zig-cache/tmp/ is, as they say + // on Star Trek, "operating within normal parameters". + else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }), + }; + }, + else => |e| return e, + }; + break; + } +} + +const ComputedHash = struct { + digest: Package.Hash.Digest, + total_size: u64, +}; + +/// Assumes that files not included in the package have already been filtered +/// prior to calling this function. This ensures that files not protected by +/// the hash are not present on the file system. Empty directories are *not +/// hashed* and must not be present on the file system when calling this +/// function. +fn computeHash(f: *Fetch, pkg_path: Path, filter: Filter) RunError!ComputedHash { + const io = f.job_queue.io; + // All the path name strings need to be in memory for sorting. + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const eb = &f.error_bundle; + const root_dir = pkg_path.root_dir.handle; + + // Collect all files, recursively, then sort. + var all_files = std.array_list.Managed(*HashedFile).init(gpa); + defer all_files.deinit(); + + var deleted_files = std.array_list.Managed(*DeletedFile).init(gpa); + defer deleted_files.deinit(); + + // Track directories which had any files deleted from them so that empty directories + // can be deleted. + var sus_dirs: std.array_hash_map.String(void) = .empty; + defer sus_dirs.deinit(gpa); + + var walker = try root_dir.walk(gpa); + defer walker.deinit(); + + // Total number of bytes of file contents included in the package. + var total_size: u64 = 0; + + { + // The final hash will be a hash of each file hashed independently. This + // allows hashing in parallel. + var group: Io.Group = .init; + defer group.cancel(io); + + while (walker.next(io) catch |err| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to walk temporary directory '{f}': {t}", + .{ pkg_path, err }, + ) }); + return error.FetchFailed; + }) |entry| { + if (entry.kind == .directory) continue; + + const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path); + if (!filter.includePath(entry_pkg_path)) { + // Delete instead of including in hash calculation. + const fs_path = try arena.dupe(u8, entry.path); + + // Also track the parent directory in case it becomes empty. + if (fs.path.dirname(fs_path)) |parent| + try sus_dirs.put(gpa, parent, {}); + + const deleted_file = try arena.create(DeletedFile); + deleted_file.* = .{ + .fs_path = fs_path, + .failure = undefined, // to be populated by the worker + }; + group.async(io, workerDeleteFile, .{ io, root_dir, deleted_file }); + try deleted_files.append(deleted_file); + continue; + } + + const kind: HashedFile.Kind = switch (entry.kind) { + .directory => unreachable, + .file => .file, + .sym_link => .link, + else => return f.fail(f.location_tok, try eb.printString( + "package contains '{s}' which has illegal file type '{t}'", + .{ entry.path, entry.kind }, + )), + }; + + if (std.mem.eql(u8, entry_pkg_path, std.zig.build_zig_basename)) + f.has_build_zig = true; + + const fs_path = try arena.dupe(u8, entry.path); + const hashed_file = try arena.create(HashedFile); + hashed_file.* = .{ + .fs_path = fs_path, + .normalized_path = try normalizePathAlloc(arena, entry_pkg_path), + .kind = kind, + .hash = undefined, // to be populated by the worker + .failure = undefined, // to be populated by the worker + .size = undefined, // to be populated by the worker + }; + group.async(io, workerHashFile, .{ io, root_dir, hashed_file }); + try all_files.append(hashed_file); + } + + try group.await(io); + } + + { + // Sort by length, descending, so that child directories get removed first. + sus_dirs.sortUnstable(@as(struct { + keys: []const []const u8, + pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { + return ctx.keys[b_index].len < ctx.keys[a_index].len; + } + }, .{ .keys = sus_dirs.keys() })); + + // During this loop, more entries will be added, so we must loop by index. + var i: usize = 0; + while (i < sus_dirs.count()) : (i += 1) { + const sus_dir = sus_dirs.keys()[i]; + root_dir.deleteDir(io, sus_dir) catch |err| switch (err) { + error.DirNotEmpty => continue, + error.FileNotFound => continue, + else => |e| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to delete empty directory '{s}': {s}", + .{ sus_dir, @errorName(e) }, + ) }); + return error.FetchFailed; + }, + }; + if (fs.path.dirname(sus_dir)) |parent| { + try sus_dirs.put(gpa, parent, {}); + } + } + } + + std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); + + var hasher = Package.Hash.Algo.init(.{}); + var any_failures = false; + for (all_files.items) |hashed_file| { + hashed_file.failure catch |err| { + any_failures = true; + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to hash '{s}': {s}", .{ + hashed_file.fs_path, @errorName(err), + }), + }); + }; + hasher.update(&hashed_file.hash); + total_size += hashed_file.size; + } + for (deleted_files.items) |deleted_file| { + deleted_file.failure catch |err| { + any_failures = true; + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{ + deleted_file.fs_path, @errorName(err), + }), + }); + }; + } + + if (any_failures) return error.FetchFailed; + + if (f.job_queue.debug_hash) { + assert(!f.job_queue.recursive); + // Print something to stdout that can be text diffed to figure out why + // the package hash is different. + dumpHashInfo(io, all_files.items) catch |err| + std.process.fatal("unable to write to stdout: {t}", .{err}); + } + + return .{ + .digest = hasher.finalResult(), + .total_size = total_size, + }; +} + +fn dumpHashInfo(io: Io, all_files: []const *const HashedFile) !void { + var stdout_buffer: [1024]u8 = undefined; + var stdout_writer: Io.File.Writer = .initStreaming(.stdout(), io, &stdout_buffer); + dumpHashInfoWriter(&stdout_writer.interface, all_files) catch |err| switch (err) { + error.WriteFailed => return stdout_writer.err.?, + }; + try stdout_writer.flush(); +} + +fn dumpHashInfoWriter(w: *Io.Writer, all_files: []const *const HashedFile) Io.Writer.Error!void { + for (all_files) |hashed_file| { + try w.print("{t}: {x}: {s}\n", .{ hashed_file.kind, &hashed_file.hash, hashed_file.normalized_path }); + } +} + +fn workerHashFile(io: Io, dir: Io.Dir, hashed_file: *HashedFile) void { + hashed_file.failure = hashFileFallible(io, dir, hashed_file); +} + +fn workerDeleteFile(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) void { + deleted_file.failure = deleteFileFallible(io, dir, deleted_file); +} + +fn hashFileFallible(io: Io, dir: Io.Dir, hashed_file: *HashedFile) HashedFile.Error!void { + var buf: [8000]u8 = undefined; + var hasher = Package.Hash.Algo.init(.{}); + hasher.update(hashed_file.normalized_path); + var file_size: u64 = 0; + + switch (hashed_file.kind) { + .file => { + var file = try dir.openFile(io, hashed_file.fs_path, .{}); + defer file.close(io); + // Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463 + hasher.update(&.{ 0, 0 }); + var file_header: FileHeader = .{}; + while (true) { + const bytes_read = try file.readPositional(io, &.{&buf}, file_size); + if (bytes_read == 0) break; + file_size += bytes_read; + hasher.update(buf[0..bytes_read]); + file_header.update(buf[0..bytes_read]); + } + if (file_header.isExecutable()) { + try setExecutable(io, file); + } + }, + .link => { + const link_name = buf[0..try dir.readLink(io, hashed_file.fs_path, &buf)]; + if (fs.path.sep != canonical_sep) { + // Package hashes are intended to be consistent across + // platforms which means we must normalize path separators + // inside symlinks. + normalizePath(link_name); + } + hasher.update(link_name); + }, + } + hasher.final(&hashed_file.hash); + hashed_file.size = file_size; +} + +fn deleteFileFallible(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { + try dir.deleteFile(io, deleted_file.fs_path); +} + +fn setExecutable(io: Io, file: Io.File) !void { + if (!Io.File.Permissions.has_executable_bit) return; + try file.setPermissions(io, .executable_file); +} + +const DeletedFile = struct { + fs_path: []const u8, + failure: Error!void, + + const Error = + Io.Dir.DeleteFileError || + Io.Dir.DeleteDirError; +}; + +const HashedFile = struct { + fs_path: []const u8, + normalized_path: []const u8, + hash: Package.Hash.Digest, + failure: Error!void, + kind: Kind, + size: u64, + + const Error = + Io.File.OpenError || + Io.File.ReadPositionalError || + Io.File.StatError || + Io.File.SetPermissionsError || + Io.Dir.ReadLinkError; + + const Kind = enum { file, link }; + + fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { + _ = context; + return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); + } +}; + +/// Strips root directory name from file system path. +fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 { + if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path; + + if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) { + return fs_path[root_dir.len + 1 ..]; + } + + return fs_path; +} + +/// Make a file system path identical independently of operating system path inconsistencies. +/// This converts backslashes into forward slashes. +fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 { + const normalized = try arena.dupe(u8, pkg_path); + if (fs.path.sep == canonical_sep) return normalized; + normalizePath(normalized); + return normalized; +} + +const canonical_sep = fs.path.sep_posix; + +fn normalizePath(bytes: []u8) void { + assert(fs.path.sep != canonical_sep); + std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep); +} + +const Filter = struct { + include_paths: std.array_hash_map.String(void) = .empty, + + /// sub_path is relative to the package root. + pub fn includePath(self: *const Filter, sub_path: []const u8) bool { + if (self.include_paths.count() == 0) return true; + if (self.include_paths.contains("")) return true; + if (self.include_paths.contains(".")) return true; + if (self.include_paths.contains(sub_path)) return true; + + // Check if any included paths are parent directories of sub_path. + var dirname = sub_path; + while (std.fs.path.dirname(dirname)) |next_dirname| { + if (self.include_paths.contains(next_dirname)) return true; + dirname = next_dirname; + } + + return false; + } + + test includePath { + const gpa = std.testing.allocator; + var filter: Filter = .{}; + defer filter.include_paths.deinit(gpa); + + try filter.include_paths.put(gpa, "src", {}); + try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c")); + try std.testing.expect(!filter.includePath(".gitignore")); + } +}; + +pub fn depDigest(pkg_root: Path, cache_root: Directory, dep: Manifest.Dependency) ?Package.Hash { + if (dep.hash) |h| return .fromSlice(h); + + switch (dep.location) { + .url => return null, + .path => |rel_path| { + var buf: [fs.max_path_bytes]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&buf); + const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch + return null; + return relativePathDigest(new_root, cache_root); + }, + } +} + +// Detects executable header: ELF or Macho-O magic header or shebang line. +const FileHeader = struct { + header: [4]u8 = undefined, + bytes_read: usize = 0, + + pub fn update(self: *FileHeader, buf: []const u8) void { + if (self.bytes_read >= self.header.len) return; + const n = @min(self.header.len - self.bytes_read, buf.len); + @memcpy(self.header[self.bytes_read..][0..n], buf[0..n]); + self.bytes_read += n; + } + + fn isScript(self: *FileHeader) bool { + const shebang = "#!"; + return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang); + } + + fn isElf(self: *FileHeader) bool { + const elf_magic = std.elf.MAGIC; + return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic); + } + + fn isMachO(self: *FileHeader) bool { + if (self.bytes_read < 4) return false; + const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian()); + return magic_number == std.macho.MH_MAGIC or + magic_number == std.macho.MH_MAGIC_64 or + magic_number == std.macho.FAT_MAGIC or + magic_number == std.macho.FAT_MAGIC_64 or + magic_number == std.macho.MH_CIGAM or + magic_number == std.macho.MH_CIGAM_64 or + magic_number == std.macho.FAT_CIGAM or + magic_number == std.macho.FAT_CIGAM_64; + } + + pub fn isExecutable(self: *FileHeader) bool { + return self.isScript() or self.isElf() or self.isMachO(); + } +}; + +test FileHeader { + var h: FileHeader = .{}; + try std.testing.expect(!h.isExecutable()); + + const elf_magic = std.elf.MAGIC; + h.update(elf_magic[0..2]); + try std.testing.expect(!h.isExecutable()); + h.update(elf_magic[2..4]); + try std.testing.expect(h.isExecutable()); + + h.update(elf_magic[2..4]); + try std.testing.expect(h.isExecutable()); + + const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE }; + h.bytes_read = 0; + h.update(&macho64_magic_bytes); + try std.testing.expect(h.isExecutable()); + + const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF }; + h.bytes_read = 0; + h.update(&macho64_cigam_bytes); + try std.testing.expect(h.isExecutable()); +} + +// Result of the `unpackResource` operation. Enables collecting errors from +// tar/git diagnostic, filtering that errors by manifest inclusion rules and +// emitting remaining errors to an `ErrorBundle`. +const UnpackResult = struct { + errors: []Error = undefined, + errors_count: usize = 0, + root_error_message: []const u8 = "", + + // A non empty value means that the package contents are inside a + // sub-directory indicated by the named path. + root_dir: []const u8 = "", + + const Error = union(enum) { + unable_to_create_sym_link: struct { + code: anyerror, + file_name: []const u8, + link_name: []const u8, + }, + unable_to_create_file: struct { + code: anyerror, + file_name: []const u8, + }, + unsupported_file_type: struct { + file_name: []const u8, + file_type: u8, + }, + + fn excluded(self: Error, filter: Filter) bool { + const file_name = switch (self) { + .unable_to_create_file => |info| info.file_name, + .unable_to_create_sym_link => |info| info.file_name, + .unsupported_file_type => |info| info.file_name, + }; + return !filter.includePath(file_name); + } + }; + + fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void { + self.root_error_message = try arena.dupe(u8, root_error_message); + self.errors = try arena.alloc(UnpackResult.Error, n); + } + + fn hasErrors(self: *UnpackResult) bool { + return self.errors_count > 0; + } + + fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void { + self.errors[self.errors_count] = .{ .unable_to_create_file = .{ + .code = err, + .file_name = file_name, + } }; + self.errors_count += 1; + } + + fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void { + self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{ + .code = err, + .file_name = file_name, + .link_name = link_name, + } }; + self.errors_count += 1; + } + + fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void { + self.errors[self.errors_count] = .{ .unsupported_file_type = .{ + .file_name = file_name, + .file_type = file_type, + } }; + self.errors_count += 1; + } + + fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void { + if (self.errors_count == 0) return; + + var unfiltered_errors: u32 = 0; + for (self.errors) |item| { + if (item.excluded(filter)) continue; + unfiltered_errors += 1; + } + if (unfiltered_errors == 0) return; + + // Emmit errors to an `ErrorBundle`. + const eb = &f.error_bundle; + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(self.root_error_message), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = unfiltered_errors, + }); + var note_i: u32 = try eb.reserveNotes(unfiltered_errors); + for (self.errors) |item| { + if (item.excluded(filter)) continue; + switch (item) { + .unable_to_create_sym_link => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ + info.file_name, info.link_name, @errorName(info.code), + }), + })); + }, + .unable_to_create_file => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("unable to create file '{s}': {s}", .{ + info.file_name, @errorName(info.code), + }), + })); + }, + .unsupported_file_type => |info| { + eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ + .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ + info.file_name, info.file_type, + }), + })); + }, + } + note_i += 1; + } + + return error.FetchFailed; + } + + test validate { + const gpa = std.testing.allocator; + var arena_instance = std.heap.ArenaAllocator.init(gpa); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + // fill UnpackResult with errors + var res: UnpackResult = .{}; + try res.allocErrors(arena, 4, "unable to unpack"); + try std.testing.expectEqual(0, res.errors_count); + res.unableToCreateFile("dir1/file1", error.File1); + res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError); + res.unableToCreateFile("dir1/file3", error.File3); + res.unsupportedFileType("dir2/file4", 'x'); + try std.testing.expectEqual(4, res.errors_count); + + // create filter, includes dir2, excludes dir1 + var filter: Filter = .{}; + try filter.include_paths.put(arena, "dir2", {}); + + // init Fetch + var fetch: Fetch = undefined; + fetch.parent_manifest_ast = null; + fetch.location_tok = 0; + try fetch.error_bundle.init(gpa); + defer fetch.error_bundle.deinit(); + + // validate errors with filter + try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter)); + + // output errors to string + var errors = try fetch.error_bundle.toOwnedBundle(""); + defer errors.deinit(gpa); + var aw: Io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + try errors.renderToWriter(.{}, &aw.writer); + try std.testing.expectEqualStrings( + \\error: unable to unpack + \\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError + \\ note: file 'dir2/file4' has unsupported type 'x' + \\ + , aw.written()); + } +}; + +test { + _ = Filter; + _ = FileType; + _ = UnpackResult; +} diff --git a/lib/compiler/Maker/Fetch/git.zig b/lib/compiler/Maker/Fetch/git.zig new file mode 100644 index 0000000000000000000000000000000000000000..d3bd1d701a618281355dba5e585d286cb3f9107f --- /dev/null +++ b/lib/compiler/Maker/Fetch/git.zig @@ -0,0 +1,1750 @@ +//! Git support for package fetching. +//! +//! This is not intended to support all features of Git: it is limited to the +//! basic functionality needed to clone a repository for the purpose of fetching +//! a package. + +const std = @import("std"); +const Io = std.Io; +const mem = std.mem; +const testing = std.testing; +const Allocator = mem.Allocator; +const Sha1 = std.crypto.hash.Sha1; +const Sha256 = std.crypto.hash.sha2.Sha256; +const assert = std.debug.assert; + +/// The ID of a Git object. +pub const Oid = union(Format) { + sha1: [Sha1.digest_length]u8, + sha256: [Sha256.digest_length]u8, + + pub const max_formatted_length = len: { + var max: usize = 0; + for (std.enums.values(Format)) |f| { + max = @max(max, f.formattedLength()); + } + break :len max; + }; + + pub const Format = enum { + sha1, + sha256, + + pub fn byteLength(f: Format) usize { + return switch (f) { + .sha1 => Sha1.digest_length, + .sha256 => Sha256.digest_length, + }; + } + + pub fn formattedLength(f: Format) usize { + return 2 * f.byteLength(); + } + }; + + const Hasher = union(Format) { + sha1: Sha1, + sha256: Sha256, + + fn init(oid_format: Format) Hasher { + return switch (oid_format) { + .sha1 => .{ .sha1 = Sha1.init(.{}) }, + .sha256 => .{ .sha256 = Sha256.init(.{}) }, + }; + } + + // Must be public for use from HashedReader and HashedWriter. + pub fn update(hasher: *Hasher, b: []const u8) void { + switch (hasher.*) { + inline else => |*inner| inner.update(b), + } + } + + fn finalResult(hasher: *Hasher) Oid { + return switch (hasher.*) { + inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), + }; + } + }; + + const Hashing = union(Format) { + sha1: Io.Writer.Hashing(Sha1), + sha256: Io.Writer.Hashing(Sha256), + + fn init(oid_format: Format, buffer: []u8) Hashing { + return switch (oid_format) { + .sha1 => .{ .sha1 = .init(buffer) }, + .sha256 => .{ .sha256 = .init(buffer) }, + }; + } + + fn writer(h: *@This()) *Io.Writer { + return switch (h.*) { + inline else => |*inner| &inner.writer, + }; + } + + fn final(h: *@This()) Oid { + switch (h.*) { + inline else => |*inner, tag| { + inner.writer.flush() catch unreachable; // hashers cannot fail + return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult()); + }, + } + } + }; + + pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { + assert(bytes.len == oid_format.byteLength()); + return switch (oid_format) { + inline else => |tag| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*), + }; + } + + pub fn readBytes(oid_format: Format, reader: *Io.Reader) !Oid { + return switch (oid_format) { + inline else => |tag| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*), + }; + } + + pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid { + switch (oid_format) { + inline else => |tag| { + if (s.len != tag.formattedLength()) return error.InvalidOid; + var bytes: [tag.byteLength()]u8 = undefined; + for (&bytes, 0..) |*b, i| { + b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; + } + return @unionInit(Oid, @tagName(tag), bytes); + }, + } + } + + test parse { + try testing.expectEqualSlices( + u8, + &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, + &(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1, + ); + try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588")); + try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")); + try testing.expectEqualSlices( + u8, + &.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A }, + &(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256, + ); + try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf")); + try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf")); + try testing.expectError(error.InvalidOid, parse(.sha1, "master")); + try testing.expectError(error.InvalidOid, parse(.sha256, "master")); + try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD")); + try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD")); + } + + pub fn parseAny(s: []const u8) error{InvalidOid}!Oid { + return for (std.enums.values(Format)) |f| { + if (s.len == f.formattedLength()) break parse(f, s); + } else error.InvalidOid; + } + + pub fn format(oid: Oid, writer: *Io.Writer) Io.Writer.Error!void { + try writer.print("{x}", .{oid.slice()}); + } + + pub fn slice(oid: *const Oid) []const u8 { + return switch (oid.*) { + inline else => |*bytes| bytes, + }; + } +}; + +pub const Diagnostics = struct { + allocator: Allocator, + errors: std.ArrayList(Error) = .empty, + + pub const Error = union(enum) { + unable_to_create_sym_link: struct { + code: anyerror, + file_name: []const u8, + link_name: []const u8, + }, + unable_to_create_file: struct { + code: anyerror, + file_name: []const u8, + }, + }; + + pub fn deinit(d: *Diagnostics) void { + for (d.errors.items) |item| { + switch (item) { + .unable_to_create_sym_link => |info| { + d.allocator.free(info.file_name); + d.allocator.free(info.link_name); + }, + .unable_to_create_file => |info| { + d.allocator.free(info.file_name); + }, + } + } + d.errors.deinit(d.allocator); + d.* = undefined; + } +}; + +pub const Repository = struct { + odb: Odb, + + pub fn init( + repo: *Repository, + allocator: Allocator, + format: Oid.Format, + pack_file: *Io.File.Reader, + index_file: *Io.File.Reader, + ) !void { + repo.* = .{ .odb = undefined }; + try repo.odb.init(allocator, format, pack_file, index_file); + } + + pub fn deinit(repository: *Repository) void { + repository.odb.deinit(); + repository.* = undefined; + } + + /// Checks out the repository at `commit_oid` to `worktree`. + pub fn checkout( + repository: *Repository, + io: Io, + worktree: Io.Dir, + commit_oid: Oid, + diagnostics: *Diagnostics, + ) !void { + try repository.odb.seekOid(commit_oid); + const tree_oid = tree_oid: { + const commit_object = try repository.odb.readObject(); + if (commit_object.type != .commit) return error.NotACommit; + break :tree_oid try getCommitTree(repository.odb.format, commit_object.data); + }; + try repository.checkoutTree(io, worktree, tree_oid, "", diagnostics); + } + + /// Checks out the tree at `tree_oid` to `worktree`. + fn checkoutTree( + repository: *Repository, + io: Io, + dir: Io.Dir, + tree_oid: Oid, + current_path: []const u8, + diagnostics: *Diagnostics, + ) !void { + try repository.odb.seekOid(tree_oid); + const tree_object = try repository.odb.readObject(); + if (tree_object.type != .tree) return error.NotATree; + // The tree object may be evicted from the object cache while we're + // iterating over it, so we can make a defensive copy here to make sure + // it remains valid until we're done with it + const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); + defer repository.odb.allocator.free(tree_data); + + var tree_iter: TreeIterator = .{ + .format = repository.odb.format, + .data = tree_data, + .pos = 0, + }; + while (try tree_iter.next()) |entry| { + switch (entry.type) { + .directory => { + try dir.createDir(io, entry.name, .default_dir); + var subdir = try dir.openDir(io, entry.name, .{}); + defer subdir.close(io); + const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name }); + defer repository.odb.allocator.free(sub_path); + try repository.checkoutTree(io, subdir, entry.oid, sub_path, diagnostics); + }, + .file => { + try repository.odb.seekOid(entry.oid); + const file_object = try repository.odb.readObject(); + if (file_object.type != .blob) return error.InvalidFile; + var file = dir.createFile(io, entry.name, .{ .exclusive = true }) catch |e| { + const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); + errdefer diagnostics.allocator.free(file_name); + try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{ + .code = e, + .file_name = file_name, + } }); + continue; + }; + defer file.close(io); + try file.writePositionalAll(io, file_object.data, 0); + }, + .symlink => { + try repository.odb.seekOid(entry.oid); + const symlink_object = try repository.odb.readObject(); + if (symlink_object.type != .blob) return error.InvalidFile; + const link_name = symlink_object.data; + dir.symLink(io, link_name, entry.name, .{}) catch |e| { + const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); + errdefer diagnostics.allocator.free(file_name); + const link_name_dup = try diagnostics.allocator.dupe(u8, link_name); + errdefer diagnostics.allocator.free(link_name_dup); + try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{ + .code = e, + .file_name = file_name, + .link_name = link_name_dup, + } }); + }; + }, + .gitlink => { + // Consistent with git archive behavior, create the directory but + // do nothing else + try dir.createDir(io, entry.name, .default_dir); + }, + } + } + } + + /// Returns the ID of the tree associated with the given commit (provided as + /// raw object data). + fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid { + if (!mem.startsWith(u8, commit_data, "tree ") or + commit_data.len < "tree ".len + format.formattedLength() + "\n".len or + commit_data["tree ".len + format.formattedLength()] != '\n') + { + return error.InvalidCommit; + } + return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]); + } + + const TreeIterator = struct { + format: Oid.Format, + data: []const u8, + pos: usize, + + const Entry = struct { + type: Type, + executable: bool, + name: [:0]const u8, + oid: Oid, + + const Type = enum(u4) { + directory = 0o4, + file = 0o10, + symlink = 0o12, + gitlink = 0o16, + }; + }; + + fn next(iterator: *TreeIterator) !?Entry { + if (iterator.pos == iterator.data.len) return null; + + const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; + const mode: packed struct { + permission: u9, + unused: u3, + type: u4, + } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); + const @"type" = std.enums.fromInt(Entry.Type, mode.type) orelse return error.InvalidTree; + const executable = switch (mode.permission) { + 0 => if (@"type" == .file) return error.InvalidTree else false, + 0o644 => if (@"type" != .file) return error.InvalidTree else false, + 0o755 => if (@"type" != .file) return error.InvalidTree else true, + else => return error.InvalidTree, + }; + iterator.pos = mode_end + 1; + + const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; + const name = iterator.data[iterator.pos..name_end :0]; + iterator.pos = name_end + 1; + + const oid_length = iterator.format.byteLength(); + if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; + const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]); + iterator.pos += oid_length; + + return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; + } + }; +}; + +/// A Git object database backed by a packfile. A packfile index is also used +/// for efficient access to objects in the packfile. +/// +/// The format of the packfile and its associated index are documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +const Odb = struct { + format: Oid.Format, + pack_file: *Io.File.Reader, + index_header: IndexHeader, + index_file: *Io.File.Reader, + cache: ObjectCache = .{}, + allocator: Allocator, + + /// Initializes the database from open pack and index files. + fn init( + odb: *Odb, + allocator: Allocator, + format: Oid.Format, + pack_file: *Io.File.Reader, + index_file: *Io.File.Reader, + ) !void { + try pack_file.seekTo(0); + try index_file.seekTo(0); + odb.* = .{ + .format = format, + .pack_file = pack_file, + .index_header = undefined, + .index_file = index_file, + .allocator = allocator, + }; + try odb.index_header.read(&index_file.interface); + } + + fn deinit(odb: *Odb) void { + odb.cache.deinit(odb.allocator); + odb.* = undefined; + } + + /// Reads the object at the current position in the database. + fn readObject(odb: *Odb) !Object { + var base_offset = odb.pack_file.logicalPos(); + var base_header: EntryHeader = undefined; + var delta_offsets: std.ArrayList(u64) = .empty; + defer delta_offsets.deinit(odb.allocator); + const base_object = while (true) { + if (odb.cache.get(base_offset)) |base_object| break base_object; + + base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface); + switch (base_header) { + .ofs_delta => |ofs_delta| { + try delta_offsets.append(odb.allocator, base_offset); + base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; + try odb.pack_file.seekTo(base_offset); + }, + .ref_delta => |ref_delta| { + try delta_offsets.append(odb.allocator, base_offset); + try odb.seekOid(ref_delta.base_object); + base_offset = odb.pack_file.logicalPos(); + }, + else => { + const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength()); + errdefer odb.allocator.free(base_data); + const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; + try odb.cache.put(odb.allocator, base_offset, base_object); + break base_object; + }, + } + }; + + const base_data = try resolveDeltaChain( + odb.allocator, + odb.format, + odb.pack_file, + base_object, + delta_offsets.items, + &odb.cache, + ); + + return .{ .type = base_object.type, .data = base_data }; + } + + /// Seeks to the beginning of the object with the given ID. + fn seekOid(odb: *Odb, oid: Oid) !void { + const oid_length = odb.format.byteLength(); + const key = oid.slice()[0]; + var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; + var end_index = odb.index_header.fan_out_table[key]; + const found_index = while (start_index < end_index) { + const mid_index = start_index + (end_index - start_index) / 2; + try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); + const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface); + switch (mem.order(u8, mid_oid.slice(), oid.slice())) { + .lt => start_index = mid_index + 1, + .gt => end_index = mid_index, + .eq => break mid_index, + } + } else return error.ObjectNotFound; + + const n_objects = odb.index_header.fan_out_table[255]; + const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); + try odb.index_file.seekTo(offset_values_start + found_index * 4); + const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big)); + const pack_offset = pack_offset: { + if (l1_offset.big) { + const l2_offset_values_start = offset_values_start + n_objects * 4; + try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); + break :pack_offset try odb.index_file.interface.takeInt(u64, .big); + } else { + break :pack_offset l1_offset.value; + } + }; + + try odb.pack_file.seekTo(pack_offset); + } +}; + +const Object = struct { + type: Type, + data: []const u8, + + const Type = enum { + commit, + tree, + blob, + tag, + }; +}; + +/// A cache for object data. +/// +/// The purpose of this cache is to speed up resolution of deltas by caching the +/// results of resolving delta objects, while maintaining a maximum cache size +/// to avoid excessive memory usage. If the total size of the objects in the +/// cache exceeds the maximum, the cache will begin evicting the least recently +/// used objects: when resolving delta chains, the most recently used objects +/// will likely be more helpful as they will be further along in the chain +/// (skipping earlier reconstruction steps). +/// +/// Object data stored in the cache is managed by the cache. It should not be +/// freed by the caller at any point after inserting it into the cache. Any +/// objects remaining in the cache will be freed when the cache itself is freed. +const ObjectCache = struct { + objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty, + lru_nodes: std.DoublyLinkedList = .{}, + lru_nodes_len: usize = 0, + byte_size: usize = 0, + + const max_byte_size = 128 * 1024 * 1024; // 128MiB + /// A list of offsets stored in the cache, with the most recently used + /// entries at the end. + const LruListNode = struct { + data: u64, + node: std.DoublyLinkedList.Node, + }; + const CacheEntry = struct { object: Object, lru_node: *LruListNode }; + + fn deinit(cache: *ObjectCache, allocator: Allocator) void { + var object_iterator = cache.objects.iterator(); + while (object_iterator.next()) |object| { + allocator.free(object.value_ptr.object.data); + allocator.destroy(object.value_ptr.lru_node); + } + cache.objects.deinit(allocator); + cache.* = undefined; + } + + /// Gets an object from the cache, moving it to the most recently used + /// position if it is present. + fn get(cache: *ObjectCache, offset: u64) ?Object { + if (cache.objects.get(offset)) |entry| { + cache.lru_nodes.remove(&entry.lru_node.node); + cache.lru_nodes.append(&entry.lru_node.node); + return entry.object; + } else { + return null; + } + } + + /// Puts an object in the cache, possibly evicting older entries if the + /// cache exceeds its maximum size. Note that, although old objects may + /// be evicted, the object just added to the cache with this function + /// will not be evicted before the next call to `put` or `deinit` even if + /// it exceeds the maximum cache size. + fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { + const lru_node = try allocator.create(LruListNode); + errdefer allocator.destroy(lru_node); + lru_node.data = offset; + + const gop = try cache.objects.getOrPut(allocator, offset); + if (gop.found_existing) { + cache.byte_size -= gop.value_ptr.object.data.len; + cache.lru_nodes.remove(&gop.value_ptr.lru_node.node); + cache.lru_nodes_len -= 1; + allocator.destroy(gop.value_ptr.lru_node); + allocator.free(gop.value_ptr.object.data); + } + gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; + cache.byte_size += object.data.len; + cache.lru_nodes.append(&lru_node.node); + cache.lru_nodes_len += 1; + + while (cache.byte_size > max_byte_size and cache.lru_nodes_len > 1) { + // The > 1 check is to make sure that we don't evict the most + // recently added node, even if it by itself happens to exceed the + // maximum size of the cache. + const evict_node: *LruListNode = @alignCast(@fieldParentPtr("node", cache.lru_nodes.popFirst().?)); + cache.lru_nodes_len -= 1; + const evict_offset = evict_node.data; + allocator.destroy(evict_node); + const evict_object = cache.objects.get(evict_offset).?.object; + cache.byte_size -= evict_object.data.len; + allocator.free(evict_object.data); + _ = cache.objects.remove(evict_offset); + } + } +}; + +/// A single pkt-line in the Git protocol. +/// +/// The format of a pkt-line is documented in +/// [protocol-common](https://git-scm.com/docs/protocol-common). The special +/// meanings of the delimiter and response-end packets are documented in +/// [protocol-v2](https://git-scm.com/docs/protocol-v2). +pub const Packet = union(enum) { + flush, + delimiter, + response_end, + data: []const u8, + + pub const max_data_length = 65516; + + /// Reads a packet in pkt-line format. + fn read(reader: *Io.Reader) !Packet { + const packet: Packet = try .peek(reader); + switch (packet) { + .data => |data| reader.toss(data.len), + else => {}, + } + return packet; + } + + /// Consumes the header of a pkt-line packet and reads any associated data + /// into the reader's buffer, but does not consume the data. + fn peek(reader: *Io.Reader) !Packet { + const length = std.fmt.parseUnsigned(u16, try reader.take(4), 16) catch return error.InvalidPacket; + switch (length) { + 0 => return .flush, + 1 => return .delimiter, + 2 => return .response_end, + 3 => return error.InvalidPacket, + else => if (length - 4 > max_data_length) return error.InvalidPacket, + } + return .{ .data = try reader.peek(length - 4) }; + } + + /// Writes a packet in pkt-line format. + fn write(packet: Packet, writer: *Io.Writer) !void { + switch (packet) { + .flush => try writer.writeAll("0000"), + .delimiter => try writer.writeAll("0001"), + .response_end => try writer.writeAll("0002"), + .data => |data| { + assert(data.len <= max_data_length); + try writer.print("{x:0>4}", .{data.len + 4}); + try writer.writeAll(data); + }, + } + } + + /// Returns the normalized form of textual packet data, stripping any + /// trailing '\n'. + /// + /// As documented in + /// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format), + /// non-binary (textual) pkt-line data should contain a trailing '\n', but + /// is not required to do so (implementations must support both forms). + fn normalizeText(data: []const u8) []const u8 { + return if (mem.endsWith(u8, data, "\n")) + data[0 .. data.len - 1] + else + data; + } +}; + +/// A client session for the Git protocol, currently limited to an HTTP(S) +/// transport. Only protocol version 2 is supported, as documented in +/// [protocol-v2](https://git-scm.com/docs/protocol-v2). +pub const Session = struct { + transport: *std.http.Client, + location: Location, + supports_agent: bool, + supports_shallow: bool, + object_format: Oid.Format, + arena: Allocator, + + const agent = "zig/" ++ @import("builtin").zig_version_string; + const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); + + /// Initializes a client session and discovers the capabilities of the + /// server for optimal transport. + pub fn init( + arena: Allocator, + transport: *std.http.Client, + uri: std.Uri, + /// Asserted to be at least `Packet.max_data_length` + response_buffer: []u8, + ) !Session { + assert(response_buffer.len >= Packet.max_data_length); + var session: Session = .{ + .transport = transport, + .location = try .init(arena, uri), + .supports_agent = false, + .supports_shallow = false, + .object_format = .sha1, + .arena = arena, + }; + var capability_iterator: CapabilityIterator = undefined; + try session.getCapabilities(&capability_iterator, response_buffer); + defer capability_iterator.deinit(); + while (try capability_iterator.next()) |capability| { + if (mem.eql(u8, capability.key, "agent")) { + session.supports_agent = true; + } else if (mem.eql(u8, capability.key, "fetch")) { + var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); + while (feature_iterator.next()) |feature| { + if (mem.eql(u8, feature, "shallow")) { + session.supports_shallow = true; + } + } + } else if (mem.eql(u8, capability.key, "object-format")) { + if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) |format| { + session.object_format = format; + } + } + } + return session; + } + + /// An owned `std.Uri` representing the location of the server (base URI). + const Location = struct { + uri: std.Uri, + + fn init(arena: Allocator, uri: std.Uri) !Location { + const scheme = try arena.dupe(u8, uri.scheme); + const user = if (uri.user) |user| try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(user, .formatUser), + }) else null; + const password = if (uri.password) |password| try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(password, .formatPassword), + }) else null; + const host = if (uri.host) |host| try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(host, .formatHost), + }) else null; + const path = try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(uri.path, .formatPath), + }); + // The query and fragment are not used as part of the base server URI. + return .{ + .uri = .{ + .scheme = scheme, + .user = if (user) |s| .{ .percent_encoded = s } else null, + .password = if (password) |s| .{ .percent_encoded = s } else null, + .host = if (host) |s| .{ .percent_encoded = s } else null, + .port = uri.port, + .path = .{ .percent_encoded = path }, + }, + }; + } + }; + + /// Returns an iterator over capabilities supported by the server. + /// + /// The `session.location` is updated if the server returns a redirect, so + /// that subsequent session functions do not need to handle redirects. + fn getCapabilities(session: *Session, it: *CapabilityIterator, response_buffer: []u8) !void { + const arena = session.arena; + assert(response_buffer.len >= Packet.max_data_length); + var info_refs_uri = session.location.uri; + { + const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(session.location.uri.path, .formatPath), + }); + info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ + "/", session_uri_path, "info/refs", + }) }; + } + info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" }; + info_refs_uri.fragment = null; + + const max_redirects = 3; + it.* = .{ + .request = try session.transport.request(.GET, info_refs_uri, .{ + .redirect_behavior = .init(max_redirects), + .extra_headers = &.{ + .{ .name = "Git-Protocol", .value = "version=2" }, + }, + }), + .reader = undefined, + .decompress = undefined, + }; + errdefer it.deinit(); + const request = &it.request; + try request.sendBodiless(); + + var redirect_buffer: [1024]u8 = undefined; + var response = try request.receiveHead(&redirect_buffer); + if (response.head.status != .ok) return error.ProtocolError; + const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects; + if (any_redirects_occurred) { + const request_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(request.uri.path, .formatPath), + }); + if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect; + var new_uri = request.uri; + new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] }; + session.location = try .init(arena, new_uri); + } + + const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); + it.reader = response.readerDecompressing(response_buffer, &it.decompress, decompress_buffer); + var state: enum { response_start, response_content } = .response_start; + while (true) { + // Some Git servers (at least GitHub) include an additional + // '# service=git-upload-pack' informative response before sending + // the expected 'version 2' packet and capability information. + // This is not universal: SourceHut, for example, does not do this. + // Thus, we need to skip any such useless additional responses + // before we get the one we're actually looking for. The responses + // will be delimited by flush packets. + const packet = Packet.read(it.reader) catch |err| switch (err) { + error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found + else => |e| return e, + }; + switch (packet) { + .flush => state = .response_start, + .data => |data| switch (state) { + .response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) { + return; + } else { + state = .response_content; + }, + else => {}, + }, + else => return error.UnexpectedPacket, + } + } + } + + const CapabilityIterator = struct { + request: std.http.Client.Request, + reader: *Io.Reader, + decompress: std.http.Decompress, + + const Capability = struct { + key: []const u8, + value: ?[]const u8 = null, + + fn parse(data: []const u8) Capability { + return if (mem.indexOfScalar(u8, data, '=')) |separator_pos| + .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] } + else + .{ .key = data }; + } + }; + + fn deinit(it: *CapabilityIterator) void { + it.request.deinit(); + it.* = undefined; + } + + fn next(it: *CapabilityIterator) !?Capability { + switch (try Packet.read(it.reader)) { + .flush => return null, + .data => |data| return Capability.parse(Packet.normalizeText(data)), + else => return error.UnexpectedPacket, + } + } + }; + + const ListRefsOptions = struct { + /// The ref prefixes (if any) to use to filter the refs available on the + /// server. Note that the client must still check the returned refs + /// against its desired filters itself: the server is not required to + /// respect these prefix filters and may return other refs as well. + ref_prefixes: []const []const u8 = &.{}, + /// Whether to include symref targets for returned symbolic refs. + include_symrefs: bool = false, + /// Whether to include the peeled object ID for returned tag refs. + include_peeled: bool = false, + /// Asserted to be at least `Packet.max_data_length`. + buffer: []u8, + }; + + /// Returns an iterator over refs known to the server. + pub fn listRefs(session: Session, it: *RefIterator, options: ListRefsOptions) !void { + const arena = session.arena; + assert(options.buffer.len >= Packet.max_data_length); + var upload_pack_uri = session.location.uri; + { + const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(session.location.uri.path, .formatPath), + }); + upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) }; + } + upload_pack_uri.query = null; + upload_pack_uri.fragment = null; + + var body: Io.Writer = .fixed(options.buffer); + try Packet.write(.{ .data = "command=ls-refs\n" }, &body); + if (session.supports_agent) { + try Packet.write(.{ .data = agent_capability }, &body); + } + { + const object_format_packet = try std.fmt.allocPrint(arena, "object-format={t}\n", .{ + session.object_format, + }); + try Packet.write(.{ .data = object_format_packet }, &body); + } + try Packet.write(.delimiter, &body); + for (options.ref_prefixes) |ref_prefix| { + const ref_prefix_packet = try std.fmt.allocPrint(arena, "ref-prefix {s}\n", .{ref_prefix}); + try Packet.write(.{ .data = ref_prefix_packet }, &body); + } + if (options.include_symrefs) { + try Packet.write(.{ .data = "symrefs\n" }, &body); + } + if (options.include_peeled) { + try Packet.write(.{ .data = "peel\n" }, &body); + } + try Packet.write(.flush, &body); + + it.* = .{ + .request = try session.transport.request(.POST, upload_pack_uri, .{ + .redirect_behavior = .unhandled, + .extra_headers = &.{ + .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, + .{ .name = "Git-Protocol", .value = "version=2" }, + }, + }), + .reader = undefined, + .format = session.object_format, + .decompress = undefined, + }; + const request = &it.request; + errdefer request.deinit(); + try request.sendBodyComplete(body.buffered()); + + var response = try request.receiveHead(options.buffer); + if (response.head.status != .ok) return error.ProtocolError; + const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); + it.reader = response.readerDecompressing(options.buffer, &it.decompress, decompress_buffer); + } + + pub const RefIterator = struct { + format: Oid.Format, + request: std.http.Client.Request, + reader: *Io.Reader, + decompress: std.http.Decompress, + + pub const Ref = struct { + oid: Oid, + name: []const u8, + symref_target: ?[]const u8, + peeled: ?Oid, + }; + + pub fn deinit(iterator: *RefIterator) void { + iterator.request.deinit(); + iterator.* = undefined; + } + + pub fn next(it: *RefIterator) !?Ref { + switch (try Packet.read(it.reader)) { + .flush => return null, + .data => |data| { + const ref_data = Packet.normalizeText(data); + const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket; + const oid = Oid.parse(it.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket; + + const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len; + const name = ref_data[oid_sep_pos + 1 .. name_sep_pos]; + + var symref_target: ?[]const u8 = null; + var peeled: ?Oid = null; + var last_sep_pos = name_sep_pos; + while (last_sep_pos < ref_data.len) { + const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len; + const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos]; + if (mem.startsWith(u8, attribute, "symref-target:")) { + symref_target = attribute["symref-target:".len..]; + } else if (mem.startsWith(u8, attribute, "peeled:")) { + peeled = Oid.parse(it.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket; + } + last_sep_pos = next_sep_pos; + } + + return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; + }, + else => return error.UnexpectedPacket, + } + } + }; + + /// Fetches the given refs from the server. A shallow fetch (depth 1) is + /// performed if the server supports it. + pub fn fetch( + session: Session, + fs: *FetchStream, + wants: []const []const u8, + /// Asserted to be at least `Packet.max_data_length`. + response_buffer: []u8, + ) !void { + const arena = session.arena; + assert(response_buffer.len >= Packet.max_data_length); + var upload_pack_uri = session.location.uri; + { + const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ + std.fmt.alt(session.location.uri.path, .formatPath), + }); + upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) }; + } + upload_pack_uri.query = null; + upload_pack_uri.fragment = null; + + var body: Io.Writer = .fixed(response_buffer); + try Packet.write(.{ .data = "command=fetch\n" }, &body); + if (session.supports_agent) { + try Packet.write(.{ .data = agent_capability }, &body); + } + { + const object_format_packet = try std.fmt.allocPrint(arena, "object-format={s}\n", .{@tagName(session.object_format)}); + try Packet.write(.{ .data = object_format_packet }, &body); + } + try Packet.write(.delimiter, &body); + // Our packfile parser supports the OFS_DELTA object type + try Packet.write(.{ .data = "ofs-delta\n" }, &body); + // We do not currently convey server progress information to the user + try Packet.write(.{ .data = "no-progress\n" }, &body); + if (session.supports_shallow) { + try Packet.write(.{ .data = "deepen 1\n" }, &body); + } + for (wants) |want| { + var buf: [Packet.max_data_length]u8 = undefined; + const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; + try Packet.write(.{ .data = arg }, &body); + } + try Packet.write(.{ .data = "done\n" }, &body); + try Packet.write(.flush, &body); + + fs.* = .{ + .request = try session.transport.request(.POST, upload_pack_uri, .{ + .redirect_behavior = .not_allowed, + .extra_headers = &.{ + .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, + .{ .name = "Git-Protocol", .value = "version=2" }, + }, + }), + .input = undefined, + .reader = undefined, + .remaining_len = undefined, + .decompress = undefined, + }; + const request = &fs.request; + errdefer request.deinit(); + + try request.sendBodyComplete(body.buffered()); + + var response = try request.receiveHead(&.{}); + if (response.head.status != .ok) return error.ProtocolError; + + const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); + const reader = response.readerDecompressing(response_buffer, &fs.decompress, decompress_buffer); + // We are not interested in any of the sections of the returned fetch + // data other than the packfile section, since we aren't doing anything + // complex like ref negotiation (this is a fresh clone). + var state: enum { section_start, section_content } = .section_start; + while (true) { + const packet = try Packet.read(reader); + switch (state) { + .section_start => switch (packet) { + .data => |data| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) { + fs.input = reader; + fs.reader = .{ + .buffer = &.{}, + .vtable = &.{ .stream = FetchStream.stream }, + .seek = 0, + .end = 0, + }; + fs.remaining_len = 0; + return; + } else { + state = .section_content; + }, + else => return error.UnexpectedPacket, + }, + .section_content => switch (packet) { + .delimiter => state = .section_start, + .data => {}, + else => return error.UnexpectedPacket, + }, + } + } + } + + pub const FetchStream = struct { + request: std.http.Client.Request, + input: *Io.Reader, + reader: Io.Reader, + err: ?Error = null, + remaining_len: usize, + decompress: std.http.Decompress, + + pub fn deinit(fs: *FetchStream) void { + fs.request.deinit(); + } + + pub const Error = error{ + InvalidPacket, + ProtocolError, + UnexpectedPacket, + WriteFailed, + ReadFailed, + EndOfStream, + }; + + const StreamCode = enum(u8) { + pack_data = 1, + progress = 2, + fatal_error = 3, + _, + }; + + pub fn stream(r: *Io.Reader, w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize { + const fs: *FetchStream = @alignCast(@fieldParentPtr("reader", r)); + const input = fs.input; + if (fs.remaining_len == 0) { + while (true) { + switch (Packet.peek(input) catch |err| { + fs.err = err; + return error.ReadFailed; + }) { + .flush => return error.EndOfStream, + .data => |data| switch (@as(StreamCode, @enumFromInt(data[0]))) { + .pack_data => { + input.toss(1); + fs.remaining_len = data.len - 1; + break; + }, + .fatal_error => { + fs.err = error.ProtocolError; + return error.ReadFailed; + }, + else => { + input.toss(data.len); + }, + }, + else => { + fs.err = error.UnexpectedPacket; + return error.ReadFailed; + }, + } + } + } + const buf = limit.slice(try w.writableSliceGreedy(1)); + const n = @min(buf.len, fs.remaining_len); + try input.readSliceAll(buf[0..n]); + w.advance(n); + fs.remaining_len -= n; + return n; + } + }; +}; + +const PackHeader = struct { + total_objects: u32, + + const signature = "PACK"; + const supported_version = 2; + + fn read(reader: *Io.Reader) !PackHeader { + const actual_signature = reader.take(4) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader; + const version = reader.takeInt(u32, .big) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + if (version != supported_version) return error.UnsupportedVersion; + const total_objects = reader.takeInt(u32, .big) catch |e| switch (e) { + error.EndOfStream => return error.InvalidHeader, + else => |other| return other, + }; + return .{ .total_objects = total_objects }; + } +}; + +const EntryHeader = union(Type) { + commit: Undeltified, + tree: Undeltified, + blob: Undeltified, + tag: Undeltified, + ofs_delta: OfsDelta, + ref_delta: RefDelta, + + const Type = enum(u3) { + commit = 1, + tree = 2, + blob = 3, + tag = 4, + ofs_delta = 6, + ref_delta = 7, + }; + + const Undeltified = struct { + uncompressed_length: u64, + }; + + const OfsDelta = struct { + offset: u64, + uncompressed_length: u64, + }; + + const RefDelta = struct { + base_object: Oid, + uncompressed_length: u64, + }; + + fn objectType(header: EntryHeader) Object.Type { + return switch (header) { + inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), + else => unreachable, + }; + } + + fn uncompressedLength(header: EntryHeader) u64 { + return switch (header) { + inline else => |entry| entry.uncompressed_length, + }; + } + + fn read(format: Oid.Format, reader: *Io.Reader) !EntryHeader { + const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; + const initial: InitialByte = @bitCast(reader.takeByte() catch |e| switch (e) { + error.EndOfStream => return error.InvalidFormat, + else => |other| return other, + }); + const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0; + var uncompressed_length: u64 = initial.len; + uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; + const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat; + return switch (@"type") { + inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ + .uncompressed_length = uncompressed_length, + }), + .ofs_delta => .{ .ofs_delta = .{ + .offset = try readOffsetVarInt(reader), + .uncompressed_length = uncompressed_length, + } }, + .ref_delta => .{ .ref_delta = .{ + .base_object = Oid.readBytes(format, reader) catch |e| switch (e) { + error.EndOfStream => return error.InvalidFormat, + else => |other| return other, + }, + .uncompressed_length = uncompressed_length, + } }, + }; + } +}; + +fn readOffsetVarInt(r: *Io.Reader) !u64 { + const Byte = packed struct { value: u7, has_next: bool }; + var b: Byte = @bitCast(try r.takeByte()); + var value: u64 = b.value; + while (b.has_next) { + b = @bitCast(try r.takeByte()); + value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; + value |= b.value; + } + return value; +} + +const IndexHeader = struct { + fan_out_table: [256]u32, + + const signature = "\xFFtOc"; + const supported_version = 2; + const size = 4 + 4 + @sizeOf([256]u32); + + fn read(index_header: *IndexHeader, reader: *Io.Reader) !void { + const sig = try reader.take(4); + if (!mem.eql(u8, sig, signature)) return error.InvalidHeader; + const version = try reader.takeInt(u32, .big); + if (version != supported_version) return error.UnsupportedVersion; + try reader.readSliceEndian(u32, &index_header.fan_out_table, .big); + } +}; + +const IndexEntry = struct { + offset: u64, + crc32: u32, +}; + +/// Writes out a version 2 index for the given packfile, as documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +pub fn indexPack( + allocator: Allocator, + format: Oid.Format, + pack: *Io.File.Reader, + index_writer: *Io.File.Writer, +) !void { + try pack.seekTo(0); + + var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; + defer index_entries.deinit(allocator); + var pending_deltas: std.ArrayList(IndexEntry) = .empty; + defer pending_deltas.deinit(allocator); + + const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas); + + var cache: ObjectCache = .{}; + defer cache.deinit(allocator); + var remaining_deltas = pending_deltas.items.len; + while (remaining_deltas > 0) { + var i: usize = remaining_deltas; + while (i > 0) { + i -= 1; + const delta = pending_deltas.items[i]; + if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) |oid| { + try index_entries.put(allocator, oid, delta); + _ = pending_deltas.swapRemove(i); + } + } + if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; + remaining_deltas = pending_deltas.items.len; + } + + var oids: std.ArrayList(Oid) = .empty; + defer oids.deinit(allocator); + try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); + var index_entries_iter = index_entries.iterator(); + while (index_entries_iter.next()) |entry| { + oids.appendAssumeCapacity(entry.key_ptr.*); + } + mem.sortUnstable(Oid, oids.items, {}, struct { + fn lessThan(_: void, o1: Oid, o2: Oid) bool { + return mem.lessThan(u8, o1.slice(), o2.slice()); + } + }.lessThan); + + var fan_out_table: [256]u32 = undefined; + var count: u32 = 0; + var fan_out_index: u8 = 0; + for (oids.items) |oid| { + const key = oid.slice()[0]; + if (key > fan_out_index) { + @memset(fan_out_table[fan_out_index..key], count); + fan_out_index = key; + } + count += 1; + } + @memset(fan_out_table[fan_out_index..], count); + + var index_hashed_writer = Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{}); + const writer = &index_hashed_writer.writer; + try writer.writeAll(IndexHeader.signature); + try writer.writeInt(u32, IndexHeader.supported_version, .big); + for (fan_out_table) |fan_out_entry| { + try writer.writeInt(u32, fan_out_entry, .big); + } + + for (oids.items) |oid| { + try writer.writeAll(oid.slice()); + } + + for (oids.items) |oid| { + try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big); + } + + var big_offsets: std.ArrayList(u64) = .empty; + defer big_offsets.deinit(allocator); + for (oids.items) |oid| { + const offset = index_entries.get(oid).?.offset; + if (offset <= std.math.maxInt(u31)) { + try writer.writeInt(u32, @intCast(offset), .big); + } else { + const index = big_offsets.items.len; + try big_offsets.append(allocator, offset); + try writer.writeInt(u32, @as(u32, @intCast(index)) | (1 << 31), .big); + } + } + for (big_offsets.items) |offset| { + try writer.writeInt(u64, offset, .big); + } + + try writer.writeAll(pack_checksum.slice()); + const index_checksum = index_hashed_writer.hasher.finalResult(); + try index_writer.interface.writeAll(index_checksum.slice()); + try index_writer.end(); +} + +/// Performs the first pass over the packfile data for index construction. +/// This will index all non-delta objects, queue delta objects for further +/// processing, and return the pack checksum (which is part of the index +/// format). +fn indexPackFirstPass( + allocator: Allocator, + format: Oid.Format, + pack: *Io.File.Reader, + index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), + pending_deltas: *std.ArrayList(IndexEntry), +) !Oid { + var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; + var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system. + var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer); + + const pack_header = try PackHeader.read(&pack_hashed.reader); + + for (0..pack_header.total_objects) |_| { + const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen(); + const entry_header = try EntryHeader.read(format, &pack_hashed.reader); + switch (entry_header) { + .commit, .tree, .blob, .tag => |object| { + var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{}); + var oid_hasher: Oid.Hashing = .init(format, &flate_buffer); + const oid_hasher_w = oid_hasher.writer(); + // The object header is not included in the pack data but is + // part of the object's ID + try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length }); + const n = try entry_decompress.reader.streamRemaining(oid_hasher_w); + if (n != object.uncompressed_length) return error.InvalidObject; + const oid = oid_hasher.final(); + if (!skip_checksums) @compileError("TODO"); + try index_entries.put(allocator, oid, .{ + .offset = entry_offset, + .crc32 = 0, + }); + }, + inline .ofs_delta, .ref_delta => |delta| { + var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer); + const n = try entry_decompress.reader.discardRemaining(); + if (n != delta.uncompressed_length) return error.InvalidObject; + if (!skip_checksums) @compileError("TODO"); + try pending_deltas.append(allocator, .{ + .offset = entry_offset, + .crc32 = 0, + }); + }, + } + } + + if (!skip_checksums) @compileError("TODO"); + return pack_hashed.hasher.finalResult(); +} + +/// Attempts to determine the final object ID of the given deltified object. +/// May return null if this is not yet possible (if the delta is a ref-based +/// delta and we do not yet know the offset of the base object). +fn indexPackHashDelta( + allocator: Allocator, + format: Oid.Format, + pack: *Io.File.Reader, + delta: IndexEntry, + index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), + cache: *ObjectCache, +) !?Oid { + // Figure out the chain of deltas to resolve + var base_offset = delta.offset; + var base_header: EntryHeader = undefined; + var delta_offsets: std.ArrayList(u64) = .empty; + defer delta_offsets.deinit(allocator); + const base_object = while (true) { + if (cache.get(base_offset)) |base_object| break base_object; + + try pack.seekTo(base_offset); + base_header = try EntryHeader.read(format, &pack.interface); + switch (base_header) { + .ofs_delta => |ofs_delta| { + try delta_offsets.append(allocator, base_offset); + base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; + }, + .ref_delta => |ref_delta| { + try delta_offsets.append(allocator, base_offset); + base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; + }, + else => { + const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength()); + errdefer allocator.free(base_data); + const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; + try cache.put(allocator, base_offset, base_object); + break base_object; + }, + } + }; + + const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); + + var entry_hasher_buffer: [64]u8 = undefined; + var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer); + const entry_hasher_w = entry_hasher.writer(); + // Writes to hashers cannot fail. + entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable; + entry_hasher_w.writeAll(base_data) catch unreachable; + return entry_hasher.final(); +} + +/// Resolves a chain of deltas, returning the final base object data. `pack` is +/// assumed to be looking at the start of the object data for the base object of +/// the chain, and will then apply the deltas in `delta_offsets` in reverse order +/// to obtain the final object. +fn resolveDeltaChain( + allocator: Allocator, + format: Oid.Format, + pack: *Io.File.Reader, + base_object: Object, + delta_offsets: []const u64, + cache: *ObjectCache, +) ![]const u8 { + var base_data = base_object.data; + var i: usize = delta_offsets.len; + while (i > 0) { + i -= 1; + + const delta_offset = delta_offsets[i]; + try pack.seekTo(delta_offset); + const delta_header = try EntryHeader.read(format, &pack.interface); + const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength()); + defer allocator.free(delta_data); + var delta_reader: Io.Reader = .fixed(delta_data); + _ = try delta_reader.takeLeb128(u64); // base object size + const expanded_size = try delta_reader.takeLeb128(u64); + + const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; + const expanded_data = try allocator.alloc(u8, expanded_alloc_size); + errdefer allocator.free(expanded_data); + var expanded_delta_stream: Io.Writer = .fixed(expanded_data); + try expandDelta(base_data, &delta_reader, &expanded_delta_stream); + if (expanded_delta_stream.end != expanded_size) return error.InvalidObject; + + try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); + base_data = expanded_data; + } + return base_data; +} + +/// Reads the complete contents of an object from `reader`. This function may +/// read more bytes than required from `reader`, so the reader position after +/// returning is not reliable. +fn readObjectRaw(allocator: Allocator, reader: *Io.Reader, size: u64) ![]u8 { + const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; + var aw: Io.Writer.Allocating = .init(allocator); + try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len); + defer aw.deinit(); + var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{}); + try decompress.reader.streamExact(&aw.writer, alloc_size); + return aw.toOwnedSlice(); +} + +/// Expands delta data from `delta_reader` to `writer`. +/// +/// The format of the delta data is documented in +/// [pack-format](https://git-scm.com/docs/pack-format). +fn expandDelta(base_object: []const u8, delta_reader: *Io.Reader, writer: *Io.Writer) !void { + while (true) { + const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch |e| switch (e) { + error.EndOfStream => return, + else => |other| return other, + }); + if (inst.copy) { + const available: packed struct { + offset1: bool, + offset2: bool, + offset3: bool, + offset4: bool, + size1: bool, + size2: bool, + size3: bool, + } = @bitCast(inst.value); + const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ + .offset1 = if (available.offset1) try delta_reader.takeByte() else 0, + .offset2 = if (available.offset2) try delta_reader.takeByte() else 0, + .offset3 = if (available.offset3) try delta_reader.takeByte() else 0, + .offset4 = if (available.offset4) try delta_reader.takeByte() else 0, + }; + const base_offset: u32 = @bitCast(offset_parts); + const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ + .size1 = if (available.size1) try delta_reader.takeByte() else 0, + .size2 = if (available.size2) try delta_reader.takeByte() else 0, + .size3 = if (available.size3) try delta_reader.takeByte() else 0, + }; + var size: u24 = @bitCast(size_parts); + if (size == 0) size = 0x10000; + try writer.writeAll(base_object[base_offset..][0..size]); + } else if (inst.value != 0) { + try delta_reader.streamExact(writer, inst.value); + } else { + return error.InvalidDeltaInstruction; + } + } +} + +/// Runs the packfile indexing and checkout test. +/// +/// The two testrepo repositories under testdata contain identical commit +/// histories and contents. +/// +/// To verify the contents of the packfiles using Git alone, run the +/// following commands in an empty directory: +/// +/// 1. `git init --object-format=(sha1|sha256)` +/// 2. `git unpack-objects return false, + else => return std.hash.Crc32.hash(name) == n.checksum, + } + } + + pub fn int(n: Fingerprint) u64 { + return @bitCast(n); + } +}; + +/// A user-readable, file system safe hash that identifies an exact package +/// snapshot, including file contents. +/// +/// The hash is not only to prevent collisions but must resist attacks where +/// the adversary fully controls the contents being hashed. Thus, it contains +/// a full SHA-256 digest. +/// +/// This data structure can be used to store the legacy hash format too. Legacy +/// hash format is scheduled to be removed after 0.14.0 is tagged. +/// +/// There's also a third way this structure is used. When using path rather than +/// hash, a unique hash is still needed, so one is computed based on the path. +pub const Hash = struct { + /// Maximum size of a package hash. Unused bytes at the end are + /// filled with zeroes. + /// + /// Assumed to be already validated. + bytes: [max_len]u8, + + pub const Algo = std.crypto.hash.sha2.Sha256; + pub const Digest = [Algo.digest_length]u8; + + /// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh" + pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6; + + /// Asserts `s` is valid. + pub fn fromSlice(s: []const u8) Hash { + assert(validate(s) == .ok); + var result: Hash = undefined; + @memcpy(result.bytes[0..s.len], s); + @memset(result.bytes[s.len..], 0); + return result; + } + + pub const Validation = enum { ok, short, long, incomplete }; + + pub fn validate(s: []const u8) Validation { + if (s.len > max_len) return .long; + if (s.len < 44) return .short; + const n_dashes = std.mem.countScalar(u8, s[0 .. s.len - 44], '-'); + if (n_dashes < 2) return .incomplete; + return .ok; + } + + test validate { + try std.testing.expectEqual(.short, validate("")); + } + + pub fn toSlice(ph: *const Hash) []const u8 { + var end: usize = ph.bytes.len; + while (true) { + end -= 1; + if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1]; + } + } + + pub fn eql(a: *const Hash, b: *const Hash) bool { + return std.mem.eql(u8, &a.bytes, &b.bytes); + } + + /// Produces "$name-$semver-$hashplus". + /// * name is the name field from build.zig.zon, asserted to be at most 32 + /// bytes and assumed be a valid zig identifier + /// * semver is the version field from build.zig.zon, asserted to be at + /// most 32 bytes + /// * hashplus is the following 33-byte array, base64 encoded using -_ to make + /// it filesystem safe: + /// - (4 bytes) LE u32 Package ID + /// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated + /// - (25 bytes) truncated SHA-256 digest of hashed files of the package + pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash { + assert(name.len <= 32); + assert(ver.len <= 32); + var result: Hash = undefined; + var buf: std.ArrayList(u8) = .initBuffer(&result.bytes); + buf.appendSliceAssumeCapacity(name); + buf.appendAssumeCapacity('-'); + buf.appendSliceAssumeCapacity(ver); + buf.appendAssumeCapacity('-'); + var hashplus: [33]u8 = undefined; + std.mem.writeInt(u32, hashplus[0..4], id, .little); + std.mem.writeInt(u32, hashplus[4..8], size, .little); + hashplus[8..].* = digest[0..25].*; + _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus); + @memset(buf.unusedCapacitySlice(), 0); + return result; + } + + /// Produces a unique hash based on the path provided. The result should + /// not be user-visible. + pub fn initPath(sub_path: []const u8, is_global: bool) Hash { + var result: Hash = .{ .bytes = @splat(0) }; + var i: usize = 0; + if (is_global) { + result.bytes[0] = '/'; + i += 1; + } + if (i + sub_path.len <= result.bytes.len) { + @memcpy(result.bytes[i..][0..sub_path.len], sub_path); + return result; + } + var bin_digest: [Algo.digest_length]u8 = undefined; + Algo.hash(sub_path, &bin_digest, .{}); + _ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable; + return result; + } + + pub fn projectId(hash: *const Hash) ProjectId { + const bytes = hash.toSlice(); + const name = std.mem.sliceTo(bytes, '-'); + const encoded_hashplus = bytes[bytes.len - 44 ..]; + var hashplus: [33]u8 = undefined; + std.base64.url_safe_no_pad.Decoder.decode(&hashplus, encoded_hashplus) catch unreachable; + const fingerprint_id = std.mem.readInt(u32, hashplus[0..4], .little); + return .init(name, fingerprint_id); + } + + test projectId { + const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw"); + const project_id = hash.projectId(); + + var expected_name: [32]u8 = @splat(0); + expected_name[0.."pulseaudio".len].* = "pulseaudio".*; + try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name); + + try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id); + } + + test "projectId with dashes in the base64" { + const hash: Hash = .fromSlice("dvui-0.4.0-dev-AQFJmayi2gAKE7FeJoF61v5U1IV9-SupoEcFutIZYpkC"); + const project_id = hash.projectId(); + + var expected_name: [32]u8 = @splat(0); + expected_name[0.."dvui".len].* = "dvui".*; + try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name); + + try std.testing.expectEqual(0x99490101, project_id.fingerprint_id); + } +}; + +/// Minimum information required to identify whether a package is an artifact +/// of a given project. +pub const ProjectId = struct { + /// Bytes after name.len are set to zero. + padded_name: [32]u8, + fingerprint_id: u32, + + pub fn init(name: []const u8, fingerprint_id: u32) ProjectId { + var padded_name: [32]u8 = @splat(0); + @memcpy(padded_name[0..name.len], name); + return .{ + .padded_name = padded_name, + .fingerprint_id = fingerprint_id, + }; + } + + pub fn eql(a: *const ProjectId, b: *const ProjectId) bool { + return a.fingerprint_id == b.fingerprint_id and std.mem.eql(u8, &a.padded_name, &b.padded_name); + } + + pub fn hash(a: *const ProjectId) u64 { + const x: u64 = @bitCast(a.padded_name[0..8].*); + return std.hash.int(x | a.fingerprint_id); + } +}; + +test Hash { + const example_digest: Hash.Digest = .{ + 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87, + 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f, + }; + const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024); + try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice()); +} + +test { + _ = Fetch; +} diff --git a/lib/compiler/Maker/Package/Manifest.zig b/lib/compiler/Maker/Package/Manifest.zig new file mode 100644 index 0000000000000000000000000000000000000000..849fc742ec5c7b5be02a2895f54f99fece02b2ff --- /dev/null +++ b/lib/compiler/Maker/Package/Manifest.zig @@ -0,0 +1,734 @@ +const Manifest = @This(); + +const std = @import("std"); +const Io = std.Io; +const mem = std.mem; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const Ast = std.zig.Ast; +const testing = std.testing; + +const Package = @import("../Package.zig"); + +pub const max_bytes = 10 * 1024 * 1024; +pub const basename = "build.zig.zon"; +pub const max_name_len = 32; +pub const max_version_len = 32; + +pub const Dependency = struct { + location: Location, + location_tok: Ast.TokenIndex, + location_node: Ast.Node.Index, + hash: ?[]const u8, + hash_tok: Ast.OptionalTokenIndex, + hash_node: Ast.Node.OptionalIndex, + node: Ast.Node.Index, + name_tok: Ast.TokenIndex, + lazy: bool, + + pub const Location = union(enum) { + url: []const u8, + path: []const u8, + }; +}; + +pub const ErrorMessage = struct { + msg: []const u8, + tok: Ast.TokenIndex, + off: u32, +}; + +name: []const u8, +id: u32, +version: std.SemanticVersion, +version_node: Ast.Node.Index, +dependencies: std.array_hash_map.String(Dependency), +dependencies_node: Ast.Node.OptionalIndex, +paths: std.array_hash_map.String(void), +minimum_zig_version: ?std.SemanticVersion, + +errors: []ErrorMessage, +arena_state: std.heap.ArenaAllocator.State, + +pub const ParseOptions = struct { + allow_missing_paths_field: bool = false, +}; + +pub const Error = Allocator.Error; + +pub fn parse(gpa: Allocator, ast: *const Ast, rng: std.Random, options: ParseOptions) Error!Manifest { + const main_node_index = ast.nodeData(.root).node; + + var arena_instance = std.heap.ArenaAllocator.init(gpa); + errdefer arena_instance.deinit(); + + var p: Parse = .{ + .gpa = gpa, + .ast = ast.*, + .arena = arena_instance.allocator(), + .errors = .empty, + + .name = undefined, + .id = 0, + .version = undefined, + .version_node = undefined, + .dependencies = .{}, + .dependencies_node = .none, + .paths = .empty, + .allow_missing_paths_field = options.allow_missing_paths_field, + .minimum_zig_version = null, + .buf = .empty, + }; + defer p.buf.deinit(gpa); + defer p.errors.deinit(gpa); + defer p.dependencies.deinit(gpa); + defer p.paths.deinit(gpa); + + p.parseRoot(main_node_index, rng) catch |err| switch (err) { + error.ParseFailure => assert(p.errors.items.len > 0), + else => |e| return e, + }; + + return .{ + .name = p.name, + .id = p.id, + .version = p.version, + .version_node = p.version_node, + .dependencies = try p.dependencies.clone(p.arena), + .dependencies_node = p.dependencies_node, + .paths = try p.paths.clone(p.arena), + .minimum_zig_version = p.minimum_zig_version, + .errors = try p.arena.dupe(ErrorMessage, p.errors.items), + .arena_state = arena_instance.state, + }; +} + +pub fn deinit(man: *Manifest, gpa: Allocator) void { + man.arena_state.promote(gpa).deinit(); + man.* = undefined; +} + +pub fn copyErrorsIntoBundle( + man: Manifest, + ast: Ast, + /// ErrorBundle null-terminated string index + src_path: u32, + eb: *std.zig.ErrorBundle.Wip, +) Allocator.Error!void { + for (man.errors) |msg| { + const start_loc = ast.tokenLocation(0, msg.tok); + + try eb.addRootErrorMessage(.{ + .msg = try eb.addString(msg.msg), + .src_loc = try eb.addSourceLocation(.{ + .src_path = src_path, + .span_start = ast.tokenStart(msg.tok), + .span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len), + .span_main = ast.tokenStart(msg.tok) + msg.off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }), + }); + } +} + +const Parse = struct { + gpa: Allocator, + ast: Ast, + arena: Allocator, + buf: std.ArrayList(u8), + errors: std.ArrayList(ErrorMessage), + + name: []const u8, + id: u32, + version: std.SemanticVersion, + version_node: Ast.Node.Index, + dependencies: std.array_hash_map.String(Dependency), + dependencies_node: Ast.Node.OptionalIndex, + paths: std.array_hash_map.String(void), + allow_missing_paths_field: bool, + minimum_zig_version: ?std.SemanticVersion, + + const InnerError = error{ ParseFailure, OutOfMemory }; + + fn parseRoot(p: *Parse, node: Ast.Node.Index, rng: std.Random) !void { + const ast = p.ast; + const main_token = ast.nodeMainToken(node); + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + return fail(p, main_token, "expected top level expression to be a struct", .{}); + }; + + var have_name = false; + var have_version = false; + var have_included_paths = false; + var fingerprint: ?Package.Fingerprint = null; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "dependencies")) { + p.dependencies_node = field_init.toOptional(); + try parseDependencies(p, field_init); + } else if (mem.eql(u8, field_name, "paths")) { + have_included_paths = true; + try parseIncludedPaths(p, field_init); + } else if (mem.eql(u8, field_name, "name")) { + p.name = try parseName(p, field_init); + have_name = true; + } else if (mem.eql(u8, field_name, "fingerprint")) { + fingerprint = try parseFingerprint(p, field_init); + } else if (mem.eql(u8, field_name, "version")) { + p.version_node = field_init; + const version_text = try parseString(p, field_init); + if (version_text.len > max_version_len) { + try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len }); + } + p.version = std.SemanticVersion.parse(version_text) catch |err| v: { + try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); + break :v undefined; + }; + have_version = true; + } else if (mem.eql(u8, field_name, "minimum_zig_version")) { + const version_text = try parseString(p, field_init); + p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch |err| v: { + try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); + break :v null; + }; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!have_name) { + try appendError(p, main_token, "missing top-level 'name' field", .{}); + } else { + if (fingerprint) |n| { + if (!n.validate(p.name)) { + return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{ + n.int(), Package.Fingerprint.generate(rng, p.name).int(), + }); + } + p.id = n.id; + } else { + try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{ + Package.Fingerprint.generate(rng, p.name).int(), + }); + } + } + + if (!have_version) { + try appendError(p, main_token, "missing top-level 'version' field", .{}); + } + + if (!have_included_paths) { + if (p.allow_missing_paths_field) { + try p.paths.put(p.gpa, "", {}); + } else { + try appendError(p, main_token, "missing top-level 'paths' field", .{}); + } + } + } + + fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = ast.nodeMainToken(node); + return fail(p, tok, "expected dependencies expression to be a struct", .{}); + }; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + const dep_name = try identifierTokenString(p, name_token); + const dep = try parseDependency(p, field_init); + try p.dependencies.put(p.gpa, dep_name, dep); + } + } + + fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { + const ast = p.ast; + + var buf: [2]Ast.Node.Index = undefined; + const struct_init = ast.fullStructInit(&buf, node) orelse { + const tok = ast.nodeMainToken(node); + return fail(p, tok, "expected dependency expression to be a struct", .{}); + }; + + var dep: Dependency = .{ + .location = undefined, + .location_tok = undefined, + .location_node = undefined, + .hash = null, + .hash_tok = .none, + .hash_node = .none, + .node = node, + .name_tok = undefined, + .lazy = false, + }; + var has_location = false; + + for (struct_init.ast.fields) |field_init| { + const name_token = ast.firstToken(field_init) - 2; + dep.name_tok = name_token; + const field_name = try identifierTokenString(p, name_token); + // We could get fancy with reflection and comptime logic here but doing + // things manually provides an opportunity to do any additional verification + // that is desirable on a per-field basis. + if (mem.eql(u8, field_name, "url")) { + if (has_location) { + return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); + } + dep.location = .{ + .url = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }, + }; + has_location = true; + dep.location_tok = ast.nodeMainToken(field_init); + dep.location_node = field_init; + } else if (mem.eql(u8, field_name, "path")) { + if (has_location) { + return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); + } + dep.location = .{ + .path = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }, + }; + has_location = true; + dep.location_tok = ast.nodeMainToken(field_init); + dep.location_node = field_init; + } else if (mem.eql(u8, field_name, "hash")) { + dep.hash = parseHash(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + dep.hash_tok = .fromToken(ast.nodeMainToken(field_init)); + dep.hash_node = field_init.toOptional(); + } else if (mem.eql(u8, field_name, "lazy")) { + dep.lazy = parseBool(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + } else { + // Ignore unknown fields so that we can add fields in future zig + // versions without breaking older zig versions. + } + } + + if (!has_location) { + try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{}); + } + + return dep; + } + + fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void { + const ast = p.ast; + + var buf: [2]Ast.Node.Index = undefined; + const array_init = ast.fullArrayInit(&buf, node) orelse { + const tok = ast.nodeMainToken(node); + return fail(p, tok, "expected paths expression to be a list of strings", .{}); + }; + + for (array_init.ast.elements) |elem_node| { + const path_string = try parseString(p, elem_node); + // This is normalized so that it can be used in string comparisons + // against file system paths. + const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); + try p.paths.put(p.gpa, normalized, {}); + } + } + + fn parseBool(p: *Parse, node: Ast.Node.Index) !bool { + const ast = p.ast; + if (ast.nodeTag(node) != .identifier) { + return fail(p, ast.nodeMainToken(node), "expected identifier", .{}); + } + const ident_token = ast.nodeMainToken(node); + const token_bytes = ast.tokenSlice(ident_token); + if (mem.eql(u8, token_bytes, "true")) { + return true; + } else if (mem.eql(u8, token_bytes, "false")) { + return false; + } else { + return fail(p, ident_token, "expected boolean", .{}); + } + } + + fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint { + const ast = p.ast; + const main_token = ast.nodeMainToken(node); + if (ast.nodeTag(node) != .number_literal) { + return fail(p, main_token, "expected integer literal", .{}); + } + const token_bytes = ast.tokenSlice(main_token); + const parsed = std.zig.parseNumberLiteral(token_bytes); + switch (parsed) { + .int => |n| return @bitCast(n), + .big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{ + @tagName(parsed), + }), + .failure => |err| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}), + } + } + + fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const main_token = ast.nodeMainToken(node); + + if (ast.nodeTag(node) != .enum_literal) + return fail(p, main_token, "expected enum literal", .{}); + + const ident_name = ast.tokenSlice(main_token); + if (mem.startsWith(u8, ident_name, "@")) + return fail(p, main_token, "name must be a valid bare zig identifier", .{}); + + if (ident_name.len > max_name_len) + return fail(p, main_token, "name '{f}' exceeds max length of {d}", .{ + std.zig.fmtId(ident_name), max_name_len, + }); + + return ident_name; + } + + fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + if (ast.nodeTag(node) != .string_literal) { + return fail(p, ast.nodeMainToken(node), "expected string literal", .{}); + } + const str_lit_token = ast.nodeMainToken(node); + const token_bytes = ast.tokenSlice(str_lit_token); + p.buf.clearRetainingCapacity(); + try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { + const ast = p.ast; + const tok = ast.nodeMainToken(node); + const h = try parseString(p, node); + switch (Package.Hash.validate(h)) { + .ok => return h, + else => |t| return fail(p, tok, "invalid hash: {t}", .{t}), + } + } + + /// TODO: try to DRY this with AstGen.identifierTokenString + fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { + const ast = p.ast; + assert(ast.tokenTag(token) == .identifier); + const ident_name = ast.tokenSlice(token); + if (!mem.startsWith(u8, ident_name, "@")) { + return ident_name; + } + p.buf.clearRetainingCapacity(); + try parseStrLit(p, token, &p.buf, ident_name, 1); + const duped = try p.arena.dupe(u8, p.buf.items); + return duped; + } + + /// TODO: try to DRY this with AstGen.parseStrLit + fn parseStrLit( + p: *Parse, + token: Ast.TokenIndex, + buf: *std.ArrayList(u8), + bytes: []const u8, + offset: u32, + ) InnerError!void { + const raw_string = bytes[offset..]; + const result = r: { + var aw: std.Io.Writer.Allocating = .fromArrayList(p.gpa, buf); + defer buf.* = aw.toArrayList(); + break :r std.zig.string_literal.parseWrite(&aw.writer, raw_string) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + }; + }; + switch (result) { + .success => {}, + .failure => |err| try p.appendStrLitError(err, token, bytes, offset), + } + } + + /// TODO: try to DRY this with AstGen.failWithStrLitError + fn appendStrLitError( + p: *Parse, + err: std.zig.string_literal.Error, + token: Ast.TokenIndex, + bytes: []const u8, + offset: u32, + ) Allocator.Error!void { + const raw_string = bytes[offset..]; + switch (err) { + .invalid_escape_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "invalid escape character: '{c}'", + .{raw_string[bad_index]}, + ); + }, + .expected_hex_digit => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected hex digit, found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .empty_unicode_escape_sequence => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "empty unicode escape sequence", + .{}, + ); + }, + .expected_hex_digit_or_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected hex digit or '}}', found '{c}'", + .{raw_string[bad_index]}, + ); + }, + .invalid_unicode_codepoint => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "unicode escape does not correspond to a valid unicode scalar value", + .{}, + ); + }, + .expected_lbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected '{{', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_rbrace => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected '}}', found '{c}", + .{raw_string[bad_index]}, + ); + }, + .expected_single_quote => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "expected single quote ('), found '{c}", + .{raw_string[bad_index]}, + ); + }, + .invalid_character => |bad_index| { + try p.appendErrorOff( + token, + offset + @as(u32, @intCast(bad_index)), + "invalid byte in string or character literal: '{c}'", + .{raw_string[bad_index]}, + ); + }, + .empty_char_literal => { + try p.appendErrorOff(token, offset, "empty character literal", .{}); + }, + } + } + + fn fail( + p: *Parse, + tok: Ast.TokenIndex, + comptime fmt: []const u8, + args: anytype, + ) InnerError { + try appendError(p, tok, fmt, args); + return error.ParseFailure; + } + + fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { + return appendErrorOff(p, tok, 0, fmt, args); + } + + fn appendErrorOff( + p: *Parse, + tok: Ast.TokenIndex, + byte_offset: u32, + comptime fmt: []const u8, + args: anytype, + ) Allocator.Error!void { + try p.errors.append(p.gpa, .{ + .msg = try std.fmt.allocPrint(p.arena, fmt, args), + .tok = tok, + .off = byte_offset, + }); + } +}; + +pub fn load( + io: Io, + arena: Allocator, + manifest_path: std.Build.Cache.Path, + ast: *std.zig.Ast, + error_bundle: *std.zig.ErrorBundle.Wip, + manifest: *Manifest, + allow_missing_paths_field: bool, +) !void { + const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions( + io, + manifest_path.sub_path, + arena, + .limited(max_bytes), + .@"1", + 0, + ); + + ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); + + if (ast.errors.len > 0) { + const file_path = try manifest_path.joinString(arena, ""); + try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle); + return error.ErrorsBundled; + } + + const rng: std.Random.IoSource = .{ .io = io }; + + manifest.* = try parse(arena, ast, rng.interface(), .{ + .allow_missing_paths_field = allow_missing_paths_field, + }); + + if (manifest.errors.len > 0) { + const src_path = try error_bundle.printString("{f}", .{manifest_path}); + try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle); + return error.ErrorsBundled; + } +} + +test "basic" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = .foo, + \\ .fingerprint = 0x8c736521490b23df, + \\ .version = "3.2.1", + \\ .paths = .{""}, + \\ .dependencies = .{ + \\ .bar = .{ + \\ .url = "https://example.com/baz.tar.gz", + \\ .hash = "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5", + \\ }, + \\ }, + \\} + ; + + var ast = try Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var rng = std.Random.DefaultPrng.init(0); + + var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); + defer manifest.deinit(gpa); + + try testing.expect(manifest.errors.len == 0); + try testing.expectEqualStrings("foo", manifest.name); + + try testing.expectEqual(@as(std.SemanticVersion, .{ + .major = 3, + .minor = 2, + .patch = 1, + }), manifest.version); + + try testing.expect(manifest.dependencies.count() == 1); + try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); + try testing.expectEqualStrings( + "https://example.com/baz.tar.gz", + manifest.dependencies.values()[0].location.url, + ); + try testing.expectEqualStrings( + "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5", + manifest.dependencies.values()[0].hash orelse return error.TestFailed, + ); + + try testing.expect(manifest.minimum_zig_version == null); +} + +test "minimum_zig_version" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = .foo, + \\ .fingerprint = 0x8c736521490b23df, + \\ .version = "3.2.1", + \\ .paths = .{""}, + \\ .minimum_zig_version = "0.11.1", + \\} + ; + + var ast = try Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var rng = std.Random.DefaultPrng.init(0); + + var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); + defer manifest.deinit(gpa); + + try testing.expect(manifest.errors.len == 0); + try testing.expect(manifest.dependencies.count() == 0); + + try testing.expect(manifest.minimum_zig_version != null); + + try testing.expectEqual(@as(std.SemanticVersion, .{ + .major = 0, + .minor = 11, + .patch = 1, + }), manifest.minimum_zig_version.?); +} + +test "minimum_zig_version - invalid version" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = .foo, + \\ .fingerprint = 0x8c736521490b23df, + \\ .version = "3.2.1", + \\ .minimum_zig_version = "X.11.1", + \\ .paths = .{""}, + \\} + ; + + var ast = try Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var rng = std.Random.DefaultPrng.init(0); + + var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); + defer manifest.deinit(gpa); + + try testing.expect(manifest.errors.len == 1); + try testing.expect(manifest.dependencies.count() == 0); + + try testing.expect(manifest.minimum_zig_version == null); +} diff --git a/lib/compiler/Maker/ScannedConfig.zig b/lib/compiler/Maker/ScannedConfig.zig index e52e26279c19eee4b955ea0d6deba574bca47abe..8421cd008547b17ff14f1c416d64040d6c33f56e 100644 --- a/lib/compiler/Maker/ScannedConfig.zig +++ b/lib/compiler/Maker/ScannedConfig.zig @@ -9,7 +9,7 @@ const Graph = @import("Graph.zig"); configuration: Configuration, top_level_steps: std.array_hash_map.String(Configuration.Step.Index), -path: []const u8, +path: std.Build.Cache.Path, pub fn print(sc: *const ScannedConfig, w: *Writer) Writer.Error!void { std.log.err("TODO also print paths", .{}); @@ -342,7 +342,6 @@ pub fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { \\ --build-file [file] Override path to build.zig \\ --cache-dir [path] Override path to local Zig cache directory \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --zig-lib-dir [arg] Override path to Zig lib directory \\ --seed [integer] For shuffling dependency traversal order (default: random) \\ --cache-poison[=mode] Override configuration caching behavior \\ pure (default) Avoid false positive cache hits @@ -360,7 +359,6 @@ pub fn printUsage(sc: *const ScannedConfig, graph: *Graph, w: *Writer) !void { \\ none (default) No build ID \\ --debug-log [scope] Enable debugging the compiler \\ --debug-pkg-config Fail if unknown pkg-config flags encountered - \\ --maker-opt=[mode] Change maker executable optimization mode (default: ReleaseSafe) \\ --verbose-link Enable compiler debug output for linking \\ --verbose-air Enable compiler debug output for Zig AIR \\ --verbose-llvm-ir Enable compiler debug output for LLVM IR diff --git a/lib/compiler/Maker/Step.zig b/lib/compiler/Maker/Step.zig index 40d82db57f6afbacc468e038f5c1e9f1487f15d9..8ccb1415d8f17a1df24dfa9505d0ac699304e97f 100644 --- a/lib/compiler/Maker/Step.zig +++ b/lib/compiler/Maker/Step.zig @@ -584,7 +584,7 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi if (!std.mem.eql(u8, builtin.zig_version_string, body)) { return s.fail( maker, - "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + "zig version mismatch build runner vs compiler: {q} vs {q}", .{ builtin.zig_version_string, body }, ); } @@ -654,7 +654,7 @@ fn zigProcessUpdate(step_index: Configuration.Step.Index, maker: *Maker, zp: *Zi } } }, - .time_report => if (maker.web_server) |*ws| { + .time_report => if (maker.web_server) |ws| { const TimeReport = std.zig.Server.Message.TimeReport; const tr: *align(1) const TimeReport = @ptrCast(body[0..@sizeOf(TimeReport)]); ws.updateTimeReportCompile(.{ diff --git a/lib/compiler/Maker/Step/Compile.zig b/lib/compiler/Maker/Step/Compile.zig index 69dae4610f8a10f70d25316da2a4bfec6234d63a..675dbc29258a6d8d203d652556cd5dc3eec7be8b 100644 --- a/lib/compiler/Maker/Step/Compile.zig +++ b/lib/compiler/Maker/Step/Compile.zig @@ -10,7 +10,6 @@ const Module = std.Build.Configuration.Module; const Io = std.Io; const Sha256 = std.crypto.hash.sha2.Sha256; const assert = std.debug.assert; -const allocPrint = std.fmt.allocPrint; const Step = @import("../Step.zig"); const Maker = @import("../../Maker.zig"); @@ -179,7 +178,7 @@ fn lowerZigArgs( try zig_args.append(gpa, cmd); if (graph.reference_trace) |some| { - try zig_args.append(gpa, try allocPrint(arena, "-freference-trace={d}", .{some})); + try zig_args.append(gpa, try arena.print("-freference-trace={d}", .{some})); } try addFlag(gpa, zig_args, "allow-so-scripts", conf_comp.flags2.allow_so_scripts.toBool() orelse graph.allow_so_scripts); @@ -191,7 +190,7 @@ fn lowerZigArgs( if (root_module.resolved_target.get(conf).?.query.unwrap()) |query| { if (query.get(conf).flags.object_format.unwrap()) |ofmt| { - try zig_args.append(gpa, try allocPrint(arena, "-ofmt={t}", .{ofmt})); + try zig_args.append(gpa, try arena.print("-ofmt={t}", .{ofmt})); } } @@ -201,7 +200,7 @@ fn lowerZigArgs( .enabled => try zig_args.append(gpa, "-fentry"), .symbol_name => { const symbol_name = conf_comp.entry.value.?.slice(conf); - try zig_args.append(gpa, try allocPrint(arena, "-fentry={s}", .{symbol_name})); + try zig_args.append(gpa, try arena.print("-fentry={s}", .{symbol_name})); }, } @@ -210,7 +209,7 @@ fn lowerZigArgs( } if (conf_comp.stack_size.value) |stack_size| { - try zig_args.appendSlice(gpa, &.{ "--stack", try allocPrint(arena, "{d}", .{stack_size}) }); + try zig_args.appendSlice(gpa, &.{ "--stack", try arena.print("{d}", .{stack_size}) }); } try addBool(gpa, zig_args, "-ffuzz", fuzz); @@ -346,7 +345,7 @@ fn lowerZigArgs( else => |e| return e, } } - try zig_args.append(gpa, try allocPrint(arena, "{s}{s}", .{ + try zig_args.append(gpa, try arena.print("{s}{s}", .{ prefix, system_lib_name, })); } @@ -526,7 +525,7 @@ fn lowerZigArgs( if (mem.eql(u8, import_cli_name, name_slice)) { zig_args.appendAssumeCapacity(import_cli_name); } else { - zig_args.appendAssumeCapacity(try allocPrint(arena, "{s}={s}", .{ + zig_args.appendAssumeCapacity(try arena.print("{s}={s}", .{ name_slice, import_cli_name, })); } @@ -542,9 +541,9 @@ fn lowerZigArgs( try zig_args.ensureUnusedCapacity(gpa, 1); if (mod.root_source_file.unwrap()) |lp| { const src = try maker.resolveLazyPathIndexAbs(arena, lp, compile_index); - zig_args.appendAssumeCapacity(try allocPrint(arena, "-M{s}={s}", .{ module_cli_name, src })); + zig_args.appendAssumeCapacity(try arena.print("-M{s}={s}", .{ module_cli_name, src })); } else if (moduleNeedsCliArg(&mod, conf)) { - zig_args.appendAssumeCapacity(try allocPrint(arena, "-M{s}", .{module_cli_name})); + zig_args.appendAssumeCapacity(try arena.print("-M{s}", .{module_cli_name})); } } } @@ -583,7 +582,7 @@ fn lowerZigArgs( if (conf_comp.image_base.value) |image_base| { (try zig_args.addManyAsArray(gpa, 2)).* = .{ - "--image-base", try allocPrint(arena, "0x{x}", .{image_base}), + "--image-base", try arena.print("0x{x}", .{image_base}), }; } @@ -643,10 +642,10 @@ fn lowerZigArgs( if (!conf_comp.flags.link_z_relro) (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-z", "norelro" }; if (conf_comp.flags.link_z_lazy) (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-z", "lazy" }; if (conf_comp.link_z_common_page_size.value) |size| (try zig_args.addManyAsArray(gpa, 2)).* = .{ - "-z", try allocPrint(arena, "common-page-size={d}", .{size}), + "-z", try arena.print("common-page-size={d}", .{size}), }; if (conf_comp.link_z_max_page_size.value) |size| (try zig_args.addManyAsArray(gpa, 2)).* = .{ - "-z", try allocPrint(arena, "max-page-size={d}", .{size}), + "-z", try arena.print("max-page-size={d}", .{size}), }; if (conf_comp.flags.link_z_defs) (try zig_args.addManyAsArray(gpa, 2)).* = .{ "-z", "defs" }; @@ -667,7 +666,7 @@ fn lowerZigArgs( try zig_args.ensureUnusedCapacity(gpa, 1); if (graph.debug_compiler_runtime_libs) |mode| switch (mode) { .Debug => zig_args.appendAssumeCapacity("--debug-rt"), - else => zig_args.appendAssumeCapacity(try allocPrint(arena, "--debug-rt={t}", .{mode})), + else => zig_args.appendAssumeCapacity(try arena.print("--debug-rt={t}", .{mode})), }; { @@ -691,15 +690,9 @@ fn lowerZigArgs( const abi = root_module_target.flags.abi.unwrap().?; zig_args.addManyAsArrayAssumeCapacity(2).* = .{ "-install_name", - if (conf_comp.install_name.value) |s| s.slice(conf) else try allocPrint( - arena, - "@rpath/{s}{s}{s}", - .{ - os_tag.libPrefix(abi), - conf_comp.root_name.slice(conf), - os_tag.dynamicLibSuffix(), - }, - ), + if (conf_comp.install_name.value) |s| s.slice(conf) else try arena.print("@rpath/{s}{s}{s}", .{ + os_tag.libPrefix(abi), conf_comp.root_name.slice(conf), os_tag.dynamicLibSuffix(), + }), }; } } @@ -712,12 +705,12 @@ fn lowerZigArgs( } if (conf_comp.pagezero_size.value) |pagezero_size| { (try zig_args.addManyAsArray(gpa, 2)).* = .{ - "-pagezero_size", try allocPrint(arena, "{x}", .{pagezero_size}), + "-pagezero_size", try arena.print("{x}", .{pagezero_size}), }; } if (conf_comp.headerpad_size.value) |headerpad_size| { (try zig_args.addManyAsArray(gpa, 2)).* = .{ - "-headerpad", try allocPrint(arena, "{x}", .{headerpad_size}), + "-headerpad", try arena.print("{x}", .{headerpad_size}), }; } try addBool(gpa, zig_args, "-headerpad_max_install_names", conf_comp.flags.headerpad_max_install_names); @@ -740,13 +733,13 @@ fn lowerZigArgs( { try zig_args.ensureUnusedCapacity(gpa, 4); if (conf_comp.initial_memory.value) |initial_memory| { - zig_args.appendAssumeCapacity(try allocPrint(arena, "--initial-memory={d}", .{initial_memory})); + zig_args.appendAssumeCapacity(try arena.print("--initial-memory={d}", .{initial_memory})); } if (conf_comp.max_memory.value) |max_memory| { - zig_args.appendAssumeCapacity(try allocPrint(arena, "--max-memory={d}", .{max_memory})); + zig_args.appendAssumeCapacity(try arena.print("--max-memory={d}", .{max_memory})); } if (conf_comp.global_base.value) |global_base| { - zig_args.appendAssumeCapacity(try allocPrint(arena, "--global-base={d}", .{global_base})); + zig_args.appendAssumeCapacity(try arena.print("--global-base={d}", .{global_base})); } switch (conf_comp.flags3.wasi_exec_model) { .default => {}, @@ -777,7 +770,7 @@ fn lowerZigArgs( for (graph.search_prefixes.items) |search_prefix| { var prefix_dir = cwd.openDir(io, search_prefix, .{}) catch |err| { - return step.fail(maker, "unable to open prefix directory '{s}': {t}", .{ search_prefix, err }); + return step.fail(maker, "unable to open prefix directory {q}: {t}", .{ search_prefix, err }); }; defer prefix_dir.close(io); @@ -791,7 +784,7 @@ fn lowerZigArgs( }); } else |err| switch (err) { error.FileNotFound => {}, - else => |e| return step.fail(maker, "unable to access '{s}/lib' directory: {t}", .{ search_prefix, e }), + else => |e| return step.fail(maker, "unable to access {s}/lib directory: {t}", .{ search_prefix, e }), } if (prefix_dir.access(io, "include", .{})) |_| { @@ -800,7 +793,7 @@ fn lowerZigArgs( }); } else |err| switch (err) { error.FileNotFound => {}, - else => |e| return step.fail(maker, "unable to access '{s}/include' directory: {t}", .{ search_prefix, e }), + else => |e| return step.fail(maker, "unable to access {s}/include directory: {t}", .{ search_prefix, e }), } } @@ -812,15 +805,15 @@ fn lowerZigArgs( if (conf_comp.flags3.build_id.unwrap(conf_comp.build_id.value, conf) orelse graph.build_id) |build_id| { try zig_args.append(gpa, switch (build_id) { - .hexstring => |hs| try allocPrint(arena, "--build-id=0x{x}", .{hs.toSlice()}), - .none, .fast, .uuid, .sha1, .md5 => try allocPrint(arena, "--build-id={t}", .{build_id}), + .hexstring => |hs| try arena.print("--build-id=0x{x}", .{hs.toSlice()}), + .none, .fast, .uuid, .sha1, .md5 => try arena.print("--build-id={t}", .{build_id}), }); } const opt_zig_lib_dir: ?[]const u8 = if (conf_comp.zig_lib_dir.value) |dir| try maker.resolveLazyPathIndexAbs(arena, dir, compile_index) else if (graph.zig_lib_directory.path) |_| - try allocPrint(arena, "{f}", .{graph.zig_lib_directory}) + try arena.print("{f}", .{graph.zig_lib_directory}) else null; @@ -848,7 +841,7 @@ fn lowerZigArgs( try addBool(gpa, zig_args, "-municode", conf_comp.flags.mingw_unicode_entry_point); if (conf_comp.error_limit.value orelse graph.error_limit) |err_limit| (try zig_args.addManyAsArray(gpa, 2)).* = .{ - "--error-limit", try allocPrint(arena, "{d}", .{err_limit}), + "--error-limit", try arena.print("{d}", .{err_limit}), }; try addFlag(gpa, zig_args, "incremental", conf_comp.flags4.incremental.toBool() orelse graph.incremental); @@ -1152,7 +1145,7 @@ const CliNamedModules = struct { try result.modules.putNoClobber(arena, mod, {}); break; } - name = try allocPrint(arena, "{s}{d}", .{ orig_name_slice, n }); + name = try arena.print("{s}{d}", .{ orig_name_slice, n }); n += 1; } } @@ -1307,7 +1300,7 @@ fn appendModuleFlags( } for (m.export_symbol_names.slice) |symbol_name| { - try zig_args.append(gpa, try allocPrint(arena, "--export={s}", .{symbol_name.slice(conf)})); + try zig_args.append(gpa, try arena.print("--export={s}", .{symbol_name.slice(conf)})); } try zig_args.ensureUnusedCapacity(gpa, 2 * m.include_dirs.len); @@ -1375,7 +1368,7 @@ pub fn appendIncludeDirFlags( zig_args.appendAssumeCapacity(try path.toString(arena)); }, .embed_path => |lazy_path| { - zig_args.appendAssumeCapacity(try allocPrint(arena, "--embed-dir={f}", .{ + zig_args.appendAssumeCapacity(try arena.print("--embed-dir={f}", .{ try maker.resolveLazyPathIndex(arena, lazy_path, asking_step), })); }, diff --git a/lib/compiler/Maker/Step/ObjCopy.zig b/lib/compiler/Maker/Step/ObjCopy.zig index 937270a00601575050753421199db0d1964603ff..a6c94e687fd8dfa0c6252aafb95752b9f53489e1 100644 --- a/lib/compiler/Maker/Step/ObjCopy.zig +++ b/lib/compiler/Maker/Step/ObjCopy.zig @@ -3,7 +3,6 @@ const ObjCopy = @This(); const std = @import("std"); const Io = std.Io; const Path = std.Build.Cache.Path; -const allocPrint = std.fmt.allocPrint; const Configuration = std.Build.Configuration; const Step = @import("../Step.zig"); @@ -55,7 +54,7 @@ pub fn make( .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, basename }), }; if (conf_oc.debug_file.value) |debug_file| { - const debug_basename = opt_debug_basename orelse try allocPrint(arena, "{s}.debug", .{ + const debug_basename = opt_debug_basename orelse try arena.print("{s}.debug", .{ Io.Dir.path.basename(input_path.sub_path), }); maker.generatedPath(debug_file).* = .{ @@ -92,7 +91,7 @@ pub fn make( if (conf_oc.pad_to.value) |pad_to| { argv.addManyAsArrayAssumeCapacity(2).* = .{ - "--pad-to", try allocPrint(arena, "{d}", .{pad_to}), + "--pad-to", try arena.print("{d}", .{pad_to}), }; } @@ -105,14 +104,14 @@ pub fn make( argv.appendAssumeCapacity("--compress-debug-sections"); if (conf_oc.debug_file.value) |debug_file| { - const debug_basename = opt_debug_basename orelse try allocPrint(arena, "{s}.debug", .{ + const debug_basename = opt_debug_basename orelse try arena.print("{s}.debug", .{ Io.Dir.path.basename(input_path.sub_path), }); const debug_dest_path: Path = .{ .root_dir = cache_root, .sub_path = try Io.Dir.path.join(arena, &.{ "o", &digest, debug_basename }), }; - argv.appendAssumeCapacity(try allocPrint(arena, "--extract-to={f}", .{debug_dest_path})); + argv.appendAssumeCapacity(try arena.print("--extract-to={f}", .{debug_dest_path})); maker.generatedPath(debug_file).* = debug_dest_path; } @@ -120,7 +119,7 @@ pub fn make( for (conf_oc.add_section.slice) |section| { argv.appendAssumeCapacity("--add-section"); - argv.appendAssumeCapacity(try allocPrint(arena, "{s}={f}", .{ + argv.appendAssumeCapacity(try arena.print("{s}={f}", .{ section.section_name.slice(conf), try maker.resolveLazyPathIndex(arena, section.file_path, step_index), })); @@ -133,14 +132,14 @@ pub fn make( if (update.flags.alignment.toBytes()) |a| { argv.appendAssumeCapacity("--set-section-alignment"); - argv.appendAssumeCapacity(try allocPrint(arena, "{s}={d}", .{ name, a })); + argv.appendAssumeCapacity(try arena.print("{s}={d}", .{ name, a })); } const f = update.flags.section_flags; if (f != Configuration.Step.ObjCopy.SectionFlags.default) { // trailing comma is allowed argv.appendAssumeCapacity("--set-section-flags"); - argv.appendAssumeCapacity(try allocPrint(arena, "{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{ + argv.appendAssumeCapacity(try arena.print("{s}={s}{s}{s}{s}{s}{s}{s}{s}{s}", .{ name, if (f.alloc) "alloc," else "", if (f.contents) "contents," else "", @@ -155,8 +154,8 @@ pub fn make( } } - argv.appendAssumeCapacity(try allocPrint(arena, "{f}", .{input_path})); - argv.appendAssumeCapacity(try allocPrint(arena, "{f}", .{dest_path})); + argv.appendAssumeCapacity(try arena.print("{f}", .{input_path})); + argv.appendAssumeCapacity(try arena.print("{f}", .{dest_path})); argv.appendAssumeCapacity("--listen=-"); _ = Step.evalZigProcess(step_index, maker, argv.items, progress_node, false) catch |err| switch (err) { diff --git a/lib/compiler/Maker/Step/Run.zig b/lib/compiler/Maker/Step/Run.zig index 9ca0b0071ef85554d51f6e2ee328f7bd93d29760..14da2a4bc22342d9edc5c50666b200f45e274521 100644 --- a/lib/compiler/Maker/Step/Run.zig +++ b/lib/compiler/Maker/Step/Run.zig @@ -12,7 +12,6 @@ const Path = std.Build.Cache.Path; const assert = std.debug.assert; const mem = std.mem; const process = std.process; -const allocPrint = std.fmt.allocPrint; const Allocator = std.mem.Allocator; const Step = @import("../Step.zig"); @@ -196,8 +195,8 @@ pub fn make( const cache_dir_string = try convertPathArg(arena, run_index, maker, .{ .root_dir = cache_root }, false); try argv_list.ensureUnusedCapacity(gpa, 3); - argv_list.appendAssumeCapacity(try allocPrint(arena, "--cache-dir={s}", .{cache_dir_string})); - argv_list.appendAssumeCapacity(try allocPrint(arena, "--seed=0x{x}", .{graph.random_seed})); + argv_list.appendAssumeCapacity(try arena.print("--cache-dir={s}", .{cache_dir_string})); + argv_list.appendAssumeCapacity(try arena.print("--seed=0x{x}", .{graph.random_seed})); argv_list.appendAssumeCapacity("--listen=-"); } @@ -1242,7 +1241,7 @@ fn evalZigTest( step.test_results = test_results; if (test_metadata) |tm| { run.cached_test_metadata = tm.toCachedTestMetadata(); - if (maker.web_server) |*ws| { + if (maker.web_server) |ws| { if (graph.time_report) { ws.updateTimeReportRunTest( run_index, @@ -1627,8 +1626,8 @@ pub fn rerunInFuzzMode( const cache_dir_string = try convertPathArg(arena, run_index, maker, .{ .root_dir = cache_root }, false); try argv_list.ensureUnusedCapacity(gpa, 3); - argv_list.appendAssumeCapacity(try allocPrint(arena, "--cache-dir={s}", .{cache_dir_string})); - argv_list.appendAssumeCapacity(try allocPrint(arena, "--seed=0x{x}", .{graph.random_seed})); + argv_list.appendAssumeCapacity(try arena.print("--cache-dir={s}", .{cache_dir_string})); + argv_list.appendAssumeCapacity(try arena.print("--seed=0x{x}", .{graph.random_seed})); argv_list.appendAssumeCapacity("--listen=-"); } @@ -1925,7 +1924,7 @@ fn runCommand( const path = try maker.resolveLazyPath(arena, lazy_path.get(conf), run_index); path.root_dir.handle.createDirPath(io, path.subPathOrDot()) catch |e| return step.fail(maker, "failed creating directory {f}: {t}", .{ path, e }); - interp_argv.appendAssumeCapacity(try allocPrint(arena, "--dir={f}::{s}", .{ path, name.slice(conf) })); + interp_argv.appendAssumeCapacity(try arena.print("--dir={f}::{s}", .{ path, name.slice(conf) })); } // Wasmtime doeesn't inherit environment variables from the parent process // by default. '-S inherit-env' was added in Wasmtime version 20. @@ -2479,7 +2478,7 @@ fn addPathForDynLibs( const dll_path = try maker.generatedPath(conf_comp.generated_bin.value.?).toString(arena); const search_path = Dir.path.dirname(dll_path).?; if (environ_map.get(path_key)) |prev_path| { - const new_path = try allocPrint(arena, "{s}{c}{s}", .{ prev_path, path_delimiter, search_path }); + const new_path = try arena.print("{s}{c}{s}", .{ prev_path, path_delimiter, search_path }); try environ_map.put(path_key, new_path); } else { try environ_map.put(path_key, search_path); diff --git a/lib/compiler/Maker/Step/TranslateC.zig b/lib/compiler/Maker/Step/TranslateC.zig index 11b30e89b6ec6a677e04df4622bee15a3a4c0bd2..94b90d65dc9a8ec48871d945924cb0e4461e80f4 100644 --- a/lib/compiler/Maker/Step/TranslateC.zig +++ b/lib/compiler/Maker/Step/TranslateC.zig @@ -3,7 +3,6 @@ const TranslateC = @This(); const std = @import("std"); const Io = std.Io; const Configuration = std.Build.Configuration; -const allocPrint = std.fmt.allocPrint; const assert = std.debug.assert; const OptimizeMode = std.lang.OptimizeMode; @@ -54,7 +53,7 @@ pub fn make( .fast => .ReleaseFast, .small => .ReleaseSmall, }; - if (opt) |o| argv.appendAssumeCapacity(try allocPrint(arena, "-O{t}", .{o})); + if (opt) |o| argv.appendAssumeCapacity(try arena.print("-O{t}", .{o})); try argv.ensureUnusedCapacity(arena, conf_tc.include_dirs.len * 2); for (0..conf_tc.include_dirs.len) |i| @@ -133,7 +132,7 @@ pub fn make( else => |e| return e, } } - try argv.append(arena, try allocPrint(arena, "{s}{s}", .{ + try argv.append(arena, try arena.print("{s}{s}", .{ prefix, system_lib_name, })); } @@ -151,7 +150,7 @@ pub fn make( }).?; const stem = Io.Dir.path.stem(Io.Dir.path.basename(c_source_path)); - const out_basename = try allocPrint(arena, "{s}.zig", .{stem}); + const out_basename = try arena.print("{s}.zig", .{stem}); maker.generatedPath(conf_tc.output_file).* = try output_dir_path.join(arena, out_basename); } diff --git a/lib/compiler/Maker/Step/UpdateSourceFiles.zig b/lib/compiler/Maker/Step/UpdateSourceFiles.zig index d33ff091d157bfcc601031253e6310f04c18a38a..dde19e0ce865930dba1972a5a8e5670665efb8f9 100644 --- a/lib/compiler/Maker/Step/UpdateSourceFiles.zig +++ b/lib/compiler/Maker/Step/UpdateSourceFiles.zig @@ -3,7 +3,6 @@ const UpdateSourceFiles = @This(); const std = @import("std"); const Io = std.Io; const Path = std.Build.Cache.Path; -const allocPrint = std.fmt.allocPrint; const Configuration = std.Build.Configuration; const Step = @import("../Step.zig"); diff --git a/lib/compiler/Maker/Step/WriteFile.zig b/lib/compiler/Maker/Step/WriteFile.zig index ca14f46fada5fd1106e7ad53d552c521ab36bd79..53bd71eda99ce058926660731b581f682e13c8ef 100644 --- a/lib/compiler/Maker/Step/WriteFile.zig +++ b/lib/compiler/Maker/Step/WriteFile.zig @@ -4,7 +4,6 @@ const std = @import("std"); const Io = std.Io; const assert = std.debug.assert; const Path = std.Build.Cache.Path; -const allocPrint = std.fmt.allocPrint; const Configuration = std.Build.Configuration; const Step = @import("../Step.zig"); diff --git a/lib/compiler/Maker/WebServer.zig b/lib/compiler/Maker/WebServer.zig index 5191e6ab2d7465de8ff690afd2d904ce48ff6c50..f77c367f6103724c7990a05c0e3be94a38ecf736 100644 --- a/lib/compiler/Maker/WebServer.zig +++ b/lib/compiler/Maker/WebServer.zig @@ -19,7 +19,7 @@ const Fuzz = @import("Fuzz.zig"); const Graph = @import("Graph.zig"); const Step = @import("Step.zig"); -maker: *Maker, +graph: *const Graph, listen_address: net.IpAddress, root_prog_node: std.Progress.Node, @@ -28,17 +28,8 @@ serve_task: ?Io.Future(Io.Cancelable!void), /// Uses `Io.Clock.awake`. base_timestamp: Io.Timestamp, -/// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`. -step_names_trailing: []u8, - -/// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps. -/// Accessed atomically. -step_status_bits: []u8, fuzz: ?Fuzz, -time_report_mutex: Io.Mutex, -time_report_msgs: [][]u8, -time_report_update_times: []i64, build_status: std.atomic.Value(abi.BuildStatus), /// When an event occurs which means WebSocket clients should be sent updates, call `notifyUpdate` @@ -55,6 +46,21 @@ runner_request_ready_cond: Io.Condition, runner_request_empty_cond: Io.Condition, runner_request: ?RunnerRequest, +configured: ?Configured, + +const Configured = struct { + maker: *Maker, + /// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`. + step_names_trailing: []u8, + /// The bit-packed "step status" data. Values are `abi.StepUpdate.Status`. LSBs are earlier steps. + /// Accessed atomically. + step_status_bits: []u8, + + time_report_mutex: Io.Mutex, + time_report_msgs: [][]u8, + time_report_update_times: []i64, +}; + /// If a client is not explicitly notified of changes with `notifyUpdate`, it will be sent updates /// on a fixed interval of this many milliseconds. const default_update_interval_ms = 500; @@ -63,34 +69,88 @@ pub const base_clock: Io.Clock = .awake; /// Thread-safe. Triggers updates to be sent to connected WebSocket clients; see `update_id`. pub fn notifyUpdate(ws: *WebServer) void { - const io = ws.maker.graph.io; + const io = ws.graph.io; _ = ws.update_id.rmw(.Add, 1, .release); io.futexWake(u32, &ws.update_id.raw, 16); } pub const Options = struct { - maker: *Maker, + graph: *const Graph, root_prog_node: std.Progress.Node, listen_address: net.IpAddress, base_timestamp: Io.Clock.Timestamp, }; + pub fn init(opts: Options) WebServer { // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent` // instead of threads, so that the web server can function in single-threaded builds. comptime assert(!builtin.single_threaded); assert(opts.base_timestamp.clock == base_clock); + return .{ + .graph = opts.graph, + .listen_address = opts.listen_address, + .root_prog_node = opts.root_prog_node, - const maker = opts.maker; + .tcp_server = null, + .serve_task = null, + + .base_timestamp = opts.base_timestamp.raw, + + .fuzz = null, + + .build_status = .init(.idle), + .update_id = .init(0), + + .runner_request_mutex = .init, + .runner_request_ready_cond = .init, + .runner_request_empty_cond = .init, + .runner_request = null, + + .configured = null, + }; +} + +pub fn deinit(ws: *WebServer) void { + const graph = ws.graph; + const io = graph.io; + + if (ws.fuzz) |*f| f.deinit(); + + ws.releaseConfigured(); + + if (ws.serve_task) |t| { + if (ws.tcp_server) |*s| s.stream.close(io); + t.await(); + } + if (ws.tcp_server) |*s| s.deinit(); +} + +fn releaseConfigured(ws: *WebServer) void { + if (ws.configured) |*configured| { + const gpa = configured.maker.gpa; + gpa.free(configured.step_names_trailing); + gpa.free(configured.step_status_bits); + for (configured.time_report_msgs) |msg| gpa.free(msg); + gpa.free(configured.time_report_msgs); + gpa.free(configured.time_report_update_times); + gpa.free(configured.step_names_trailing); + ws.configured = null; + } +} + +pub fn updateConfiguration(ws: *WebServer, maker: *Maker) !void { + const graph = ws.graph; + const gpa = maker.gpa; const all_steps = maker.step_stack.keys(); const c = &maker.scanned_config.configuration; - const gpa = maker.gpa; - const graph = maker.graph; - const step_names_trailing = gpa.alloc(u8, len: { + const step_names_trailing = try gpa.alloc(u8, len: { var name_bytes: usize = 0; for (all_steps) |step_index| name_bytes += step_index.ptr(c).name.slice(c).len; break :len name_bytes + all_steps.len * 4; - }) catch @panic("out of memory"); + }); + errdefer gpa.free(step_names_trailing); + { const step_name_lens: []align(1) u32 = @ptrCast(step_names_trailing[0 .. all_steps.len * 4]); var idx: usize = all_steps.len * 4; @@ -103,71 +163,35 @@ pub fn init(opts: Options) WebServer { assert(idx == step_names_trailing.len); } - const step_status_bits = gpa.alloc( - u8, - std.math.divCeil(usize, all_steps.len, 4) catch unreachable, - ) catch @panic("out of memory"); + const step_status_bits = try gpa.alloc(u8, std.math.divCeil(usize, all_steps.len, 4) catch unreachable); + errdefer gpa.free(step_status_bits); @memset(step_status_bits, 0); const time_reports_len: usize = if (graph.time_report) all_steps.len else 0; - const time_report_msgs = gpa.alloc([]u8, time_reports_len) catch @panic("out of memory"); - const time_report_update_times = gpa.alloc(i64, time_reports_len) catch @panic("out of memory"); + const time_report_msgs = try gpa.alloc([]u8, time_reports_len); + errdefer gpa.free(time_report_msgs); + const time_report_update_times = try gpa.alloc(i64, time_reports_len); + errdefer gpa.free(time_report_update_times); @memset(time_report_msgs, &.{}); @memset(time_report_update_times, std.math.minInt(i64)); - return .{ + ws.releaseConfigured(); + + ws.configured = .{ .maker = maker, - .listen_address = opts.listen_address, - .root_prog_node = opts.root_prog_node, - - .tcp_server = null, - .serve_task = null, - - .base_timestamp = opts.base_timestamp.raw, .step_names_trailing = step_names_trailing, - .step_status_bits = step_status_bits, - - .fuzz = null, .time_report_mutex = .init, .time_report_msgs = time_report_msgs, .time_report_update_times = time_report_update_times, - - .build_status = .init(.idle), - .update_id = .init(0), - - .runner_request_mutex = .init, - .runner_request_ready_cond = .init, - .runner_request_empty_cond = .init, - .runner_request = null, }; } -pub fn deinit(ws: *WebServer) void { - const maker = ws.maker; - const gpa = maker.gpa; - const io = maker.graph.io; - gpa.free(ws.step_names_trailing); - gpa.free(ws.step_status_bits); - - if (ws.fuzz) |*f| f.deinit(); - for (ws.time_report_msgs) |msg| gpa.free(msg); - gpa.free(ws.time_report_msgs); - gpa.free(ws.time_report_update_times); - - if (ws.serve_task) |t| { - if (ws.tcp_server) |*s| s.stream.close(io); - t.await(); - } - if (ws.tcp_server) |*s| s.deinit(); - - gpa.free(ws.step_names_trailing); -} pub fn start(ws: *WebServer) error{AlreadyReported}!void { assert(ws.tcp_server == null); assert(ws.serve_task == null); - const maker = ws.maker; - const io = maker.graph.io; + const graph = ws.graph; + const io = graph.io; ws.tcp_server = ws.listen_address.listen(io, .{ .reuse_address = true }) catch |err| { log.err("failed to listen to port {d}: {t}", .{ ws.listen_address.getPort(), err }); @@ -186,8 +210,8 @@ pub fn start(ws: *WebServer) error{AlreadyReported}!void { } } fn serve(ws: *WebServer) Io.Cancelable!void { - const maker = ws.maker; - const io = maker.graph.io; + const graph = ws.graph; + const io = graph.io; var group: Io.Group = .init; defer group.cancel(io); @@ -213,7 +237,8 @@ pub fn startBuild(ws: *WebServer) void { fuzz.deinit(); ws.fuzz = null; } - for (ws.step_status_bits) |*bits| @atomicStore(u8, bits, 0, .monotonic); + const configured = &ws.configured.?; + for (configured.step_status_bits) |*bits| @atomicStore(u8, bits, 0, .monotonic); ws.build_status.store(.running, .monotonic); ws.notifyUpdate(); } @@ -223,12 +248,13 @@ pub fn updateStepStatus( step_index: Configuration.Step.Index, new_status: abi.StepUpdate.Status, ) void { - const maker = ws.maker; + const configured = &ws.configured.?; + const maker = configured.maker; const all_steps = maker.step_stack.keys(); const step_idx: u32 = for (all_steps, 0..) |s, i| { if (s == step_index) break @intCast(i); } else unreachable; - const ptr = &ws.step_status_bits[step_idx / 4]; + const ptr = &configured.step_status_bits[step_idx / 4]; const bit_offset: u3 = @intCast((step_idx % 4) * 2); const old_bits: u2 = @truncate(@atomicLoad(u8, ptr, .monotonic) >> bit_offset); const mask = @as(u8, @intFromEnum(new_status) ^ old_bits) << bit_offset; @@ -239,7 +265,8 @@ pub fn updateStepStatus( pub fn finishBuild(ws: *WebServer, opts: struct { fuzz: bool, }) void { - const maker = ws.maker; + const configured = &ws.configured.?; + const maker = configured.maker; const all_steps = maker.step_stack.keys(); if (opts.fuzz) { @@ -274,15 +301,15 @@ pub fn finishBuild(ws: *WebServer, opts: struct { } pub fn now(ws: *const WebServer) i64 { - const maker = ws.maker; - const io = maker.graph.io; + const graph = ws.graph; + const io = graph.io; const ts = base_clock.now(io); return @intCast(ws.base_timestamp.durationTo(ts).toNanoseconds()); } fn accept(ws: *WebServer, stream: net.Stream) void { - const maker = ws.maker; - const io = maker.graph.io; + const graph = ws.graph; + const io = graph.io; defer { // `net.Stream.close` wants to helpfully overwrite `stream` with @@ -328,17 +355,19 @@ fn accept(ws: *WebServer, stream: net.Stream) void { } fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { - const maker = ws.maker; - const gpa = maker.gpa; - const graph = maker.graph; + const graph = ws.graph; + const gpa = graph.cache.gpa; const io = graph.io; + log.err("TODO serve a different message when the configuration changes", .{}); + const configured = &ws.configured.?; + const maker = configured.maker; const all_steps = maker.step_stack.keys(); var prev_build_status = ws.build_status.load(.monotonic); - const prev_step_status_bits = try gpa.alloc(u8, ws.step_status_bits.len); + const prev_step_status_bits = try gpa.alloc(u8, configured.step_status_bits.len); defer gpa.free(prev_step_status_bits); - for (prev_step_status_bits, ws.step_status_bits) |*copy, *shared| { + for (prev_step_status_bits, configured.step_status_bits) |*copy, *shared| { copy.* = @atomicLoad(u8, shared, .monotonic); } @@ -354,7 +383,7 @@ fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { .timestamp = ws.now(), .steps_len = @intCast(all_steps.len), }; - var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), ws.step_names_trailing, prev_step_status_bits }; + var bufs: [3][]const u8 = .{ @ptrCast(&hello_header), configured.step_names_trailing, prev_step_status_bits }; try sock.writeMessageVec(&bufs, .binary); } @@ -369,17 +398,17 @@ fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { } { - try ws.time_report_mutex.lock(io); - defer ws.time_report_mutex.unlock(io); - for (ws.time_report_msgs, ws.time_report_update_times) |msg, update_time| { + try configured.time_report_mutex.lock(io); + defer configured.time_report_mutex.unlock(io); + for (configured.time_report_msgs, configured.time_report_update_times) |msg, update_time| { if (update_time <= prev_time) continue; - // We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so + // We want to send `msg`, but shouldn't block `configured.time_report_mutex` while we do, so // that we don't hold up the build system on the client accepting this packet. const owned_msg = try gpa.dupe(u8, msg); defer gpa.free(owned_msg); // Temporarily unlock, then re-lock after the message is sent. - ws.time_report_mutex.unlock(io); - defer ws.time_report_mutex.lockUncancelable(io); + configured.time_report_mutex.unlock(io); + defer configured.time_report_mutex.lockUncancelable(io); try sock.writeMessage(owned_msg, .binary); } } @@ -393,7 +422,7 @@ fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { } } - for (prev_step_status_bits, ws.step_status_bits, 0..) |*prev_byte, *shared, byte_idx| { + for (prev_step_status_bits, configured.step_status_bits, 0..) |*prev_byte, *shared, byte_idx| { const cur_byte = @atomicLoad(u8, shared, .monotonic); if (prev_byte.* == cur_byte) continue; const cur: [4]abi.StepUpdate.Status = .{ @@ -433,8 +462,8 @@ fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn { } } fn recvWebSocketMessages(ws: *WebServer, sock: *http.Server.WebSocket) void { - const maker = ws.maker; - const io = maker.graph.io; + const graph = ws.graph; + const io = graph.io; while (true) { const msg = sock.readSmallMessage() catch return; @@ -492,8 +521,7 @@ fn serveLibFile( sub_path: []const u8, content_type: []const u8, ) !void { - const maker = ws.maker; - const graph = maker.graph; + const graph = ws.graph; return serveFile(ws, request, .{ .root_dir = graph.zig_lib_directory, @@ -505,7 +533,7 @@ fn serveClientWasm( req: *http.Server.Request, optimize_mode: std.builtin.OptimizeMode, ) !void { - const gpa = ws.maker.gpa; + const gpa = ws.graph.cache.gpa; var arena_state: std.heap.ArenaAllocator = .init(gpa); defer arena_state.deinit(); @@ -522,9 +550,9 @@ pub fn serveFile( path: Cache.Path, content_type: []const u8, ) !void { - const maker = ws.maker; - const gpa = ws.maker.gpa; - const io = maker.graph.io; + const graph = ws.graph; + const gpa = graph.cache.gpa; + const io = graph.io; // The desired API is actually sendfile, which will require enhancing http.Server. // We load the file with every request so that the user can make changes to the file @@ -542,8 +570,7 @@ pub fn serveFile( }); } pub fn serveTarFile(ws: *WebServer, request: *http.Server.Request, paths: []const Cache.Path) !void { - const maker = ws.maker; - const graph = maker.graph; + const graph = ws.graph; const io = graph.io; var send_buffer: [0x4000]u8 = undefined; @@ -581,9 +608,8 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim const arch_os_abi = "wasm32-freestanding"; const cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext"; - const maker = ws.maker; - const gpa = maker.gpa; - const graph = maker.graph; + const graph = ws.graph; + const gpa = graph.cache.gpa; const io = graph.io; const main_src_path: Cache.Path = .{ @@ -622,151 +648,19 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim "--listen=-", }); - var child = try std.process.spawn(io, .{ + const compile_prog_node = ws.root_prog_node.start("Compile WebAssembly Component", 0); + defer compile_prog_node.end(); + + const result = try std.zig.buildExeSubprocess(gpa, io, .{ .argv = argv.items, - .environ_map = &graph.environ_map, - .stdin = .pipe, - .stdout = .pipe, - .stderr = .pipe, - }); - defer child.kill(io); - - var stderr_task = try io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited }); - defer if (stderr_task.cancel(io)) |slice| gpa.free(slice) else |_| {}; - - var stdout_buffer: [512]u8 = undefined; - var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer); - const stdout = &stdout_reader.interface; - - { - var w = child.stdin.?.writer(io, &.{}); - w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - w.interface.writeStruct(std.zig.Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch |err| switch (err) { - error.WriteFailed => return w.err.?, - }; - } - - const Header = std.zig.Server.Message.Header; - - var result: ?Cache.Path = null; - var result_error_bundle = std.zig.ErrorBundle.empty; - var body_buffer: std.ArrayList(u8) = .empty; - defer body_buffer.deinit(gpa); - - while (true) { - const header = stdout.takeStruct(Header, .little) catch |err| switch (err) { - error.ReadFailed => |e| return e, - error.EndOfStream => break, - }; - body_buffer.clearRetainingCapacity(); - try stdout.appendExact(gpa, &body_buffer, header.bytes_len); - const body = body_buffer.items; - - switch (header.tag) { - .zig_version => { - if (!std.mem.eql(u8, builtin.zig_version_string, body)) { - return error.ZigProtocolVersionMismatch; - } - }, - .error_bundle => { - result_error_bundle = try std.zig.Server.allocErrorBundle(arena, body); - }, - .emit_digest => { - const EmitDigest = std.zig.Server.Message.EmitDigest; - const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body); - if (!ebp_hdr.flags.cache_hit) { - log.info("source changes detected; rebuilt wasm component", .{}); - } - const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; - result = .{ - .root_dir = graph.global_cache_root, - .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)), - }; - }, - else => {}, // ignore other messages - } - } - - const stderr_contents = try stderr_task.await(io); - if (stderr_contents.len > 0) { - std.debug.print("{s}", .{stderr_contents}); - } - - // Send EOF to stdin. - child.stdin.?.close(io); - child.stdin = null; - - switch (try child.wait(io)) { - .exited => |code| { - if (code != 0) { - log.err( - "the following command exited with error code {d}:\n{s}", - .{ code, try std.zig.allocPrintCmd(arena, argv.items, .{}) }, - ); - return error.WasmCompilationFailed; - } - }, - .signal => |sig| { - log.err( - "the following command terminated with signal {t}:\n{s}", - .{ sig, try std.zig.allocPrintCmd(arena, argv.items, .{}) }, - ); - return error.WasmCompilationFailed; - }, - .stopped => |sig| { - log.err( - "the following command stopped unexpectedly with signal {t}:\n{s}", - .{ sig, try std.zig.allocPrintCmd(arena, argv.items, .{}) }, - ); - return error.WasmCompilationFailed; - }, - .unknown => { - log.err( - "the following command terminated unexpectedly:\n{s}", - .{try std.zig.allocPrintCmd(arena, argv.items, .{})}, - ); - return error.WasmCompilationFailed; - }, - } - - if (result_error_bundle.errorMessageCount() > 0) { - try result_error_bundle.renderToStderr(io, .{}, .auto); - log.err("the following command failed with {d} compilation errors:\n{s}", .{ - result_error_bundle.errorMessageCount(), - try std.zig.allocPrintCmd(arena, argv.items, .{}), - }); - return error.WasmCompilationFailed; - } - - const base_path = result orelse { - log.err("child process failed to report result\n{s}", .{ - try std.zig.allocPrintCmd(arena, argv.items, .{}), - }); - return error.WasmCompilationFailed; - }; - const target = std.zig.system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{ + .cache_root = graph.global_cache_root, + .root_name = root_name, .arch_os_abi = arch_os_abi, .cpu_features = cpu_features, - }) catch unreachable) catch unreachable; - const bin_name = try std.zig.binNameAlloc(arena, .{ - .root_name = root_name, - .cpu_arch = target.cpu.arch, - .os_tag = target.os.tag, - .ofmt = target.ofmt, - .abi = target.abi, - .output_mode = .Exe, + .progress_node = compile_prog_node, }); - return base_path.join(arena, bin_name); -} - -fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 { - var file_reader: Io.File.Reader = .initStreaming(file, io, &.{}); - return file_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, - else => |e| return e, - }; + if (!result.cache_hit) log.info("source changes detected; rebuilt wasm component", .{}); + return result.path; } pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { @@ -783,9 +677,11 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { /// The trailing data of `abi.time_report.CompileResult`, except the step name. trailing: []const u8, }) void { - const maker = ws.maker; + const graph = ws.graph; + const io = graph.io; + const configured = &ws.configured.?; + const maker = configured.maker; const gpa = maker.gpa; - const io = maker.graph.io; const all_steps = maker.step_stack.keys(); const step_idx: u32 = for (all_steps, 0..) |s, i| { @@ -793,10 +689,10 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { } else unreachable; const old_buf = old: { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - const old = ws.time_report_msgs[step_idx]; - ws.time_report_msgs[step_idx] = &.{}; + configured.time_report_mutex.lock(io) catch return; + defer configured.time_report_mutex.unlock(io); + const old = configured.time_report_msgs[step_idx]; + configured.time_report_msgs[step_idx] = &.{}; break :old old; }; const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.CompileResult) + opts.trailing.len) catch @panic("out of memory"); @@ -816,19 +712,21 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct { @memcpy(buf[@sizeOf(abi.time_report.CompileResult)..], opts.trailing); { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - assert(ws.time_report_msgs[step_idx].len == 0); - ws.time_report_msgs[step_idx] = buf; - ws.time_report_update_times[step_idx] = ws.now(); + configured.time_report_mutex.lock(io) catch return; + defer configured.time_report_mutex.unlock(io); + assert(configured.time_report_msgs[step_idx].len == 0); + configured.time_report_msgs[step_idx] = buf; + configured.time_report_update_times[step_idx] = ws.now(); } ws.notifyUpdate(); } pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.Index, duration: Io.Duration) void { - const maker = ws.maker; + const graph = ws.graph; + const io = graph.io; + const configured = &ws.configured.?; + const maker = configured.maker; const gpa = maker.gpa; - const io = maker.graph.io; const all_steps = maker.step_stack.keys(); const step_idx: u32 = for (all_steps, 0..) |s, i| { @@ -836,10 +734,10 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.In } else unreachable; const old_buf = old: { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - const old = ws.time_report_msgs[step_idx]; - ws.time_report_msgs[step_idx] = &.{}; + configured.time_report_mutex.lock(io) catch return; + defer configured.time_report_mutex.unlock(io); + const old = configured.time_report_msgs[step_idx]; + configured.time_report_msgs[step_idx] = &.{}; break :old old; }; const buf = gpa.realloc(old_buf, @sizeOf(abi.time_report.GenericResult)) catch @panic("out of memory"); @@ -849,11 +747,11 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step_index: Configuration.Step.In .ns_total = @intCast(duration.toNanoseconds()), }; { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - assert(ws.time_report_msgs[step_idx].len == 0); - ws.time_report_msgs[step_idx] = buf; - ws.time_report_update_times[step_idx] = ws.now(); + configured.time_report_mutex.lock(io) catch return; + defer configured.time_report_mutex.unlock(io); + assert(configured.time_report_msgs[step_idx].len == 0); + configured.time_report_msgs[step_idx] = buf; + configured.time_report_update_times[step_idx] = ws.now(); } ws.notifyUpdate(); } @@ -864,9 +762,11 @@ pub fn updateTimeReportRunTest( tests: *const Step.Run.CachedTestMetadata, ns_per_test: []const u64, ) void { - const maker = ws.maker; + const graph = ws.graph; + const io = graph.io; + const configured = &ws.configured.?; + const maker = configured.maker; const gpa = maker.gpa; - const io = maker.graph.io; const all_steps = maker.step_stack.keys(); const step_idx: u32 = for (all_steps, 0..) |s, i| { @@ -884,10 +784,10 @@ pub fn updateTimeReportRunTest( break :len @sizeOf(abi.time_report.RunTestResult) + names_len + 8 * tests_len; }; const old_buf = old: { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - const old = ws.time_report_msgs[step_idx]; - ws.time_report_msgs[step_idx] = &.{}; + configured.time_report_mutex.lock(io) catch return; + defer configured.time_report_mutex.unlock(io); + const old = configured.time_report_msgs[step_idx]; + configured.time_report_msgs[step_idx] = &.{}; break :old old; }; const buf = gpa.realloc(old_buf, new_len) catch @panic("out of memory"); @@ -910,11 +810,11 @@ pub fn updateTimeReportRunTest( assert(offset == buf.len); { - ws.time_report_mutex.lock(io) catch return; - defer ws.time_report_mutex.unlock(io); - assert(ws.time_report_msgs[step_idx].len == 0); - ws.time_report_msgs[step_idx] = buf; - ws.time_report_update_times[step_idx] = ws.now(); + configured.time_report_mutex.lock(io) catch return; + defer configured.time_report_mutex.unlock(io); + assert(configured.time_report_msgs[step_idx].len == 0); + configured.time_report_msgs[step_idx] = buf; + configured.time_report_update_times[step_idx] = ws.now(); } ws.notifyUpdate(); } @@ -923,7 +823,7 @@ const RunnerRequest = union(enum) { rebuild, }; pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest { - const io = ws.maker.graph.io; + const io = ws.graph.io; ws.runner_request_mutex.lock(io) catch return; defer ws.runner_request_mutex.unlock(io); if (ws.runner_request) |req| { @@ -934,7 +834,7 @@ pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest { return null; } pub fn wait(ws: *WebServer) Io.Cancelable!RunnerRequest { - const io = ws.maker.graph.io; + const io = ws.graph.io; try ws.runner_request_mutex.lock(io); defer ws.runner_request_mutex.unlock(io); while (true) { diff --git a/lib/compiler/configurer.zig b/lib/compiler/configurer.zig index f044dc311945d20b330d472553ad5dded0d3138f..809bb9e6263c3a9c0e128bb6a677589fef6054c9 100644 --- a/lib/compiler/configurer.zig +++ b/lib/compiler/configurer.zig @@ -133,7 +133,7 @@ pub fn main(init: process.Init.Minimal) !void { .off => .no_color, }; - try builder.runBuild(root); + builder.runBuild(root); if (builder.validateUserInputDidItFail()) { fatal(" access the help menu with 'zig build -h'", .{}); @@ -632,6 +632,37 @@ fn serialize(b: *std.Build, wc: *Configuration.Wip, writer: *Io.Writer) !void { var s: Serialize = .{ .wc = wc, .arena = arena }; + try wc.path_deps.ensureTotalCapacityPrecise(gpa, graph.configure_dependencies.items.len); + for ( + graph.configure_dependencies.items, + wc.path_deps.addManyAsSliceAssumeCapacity(graph.configure_dependencies.items.len), + ) |src, *dest| { + dest.* = .{ + .flags = .{ + .base = switch (src.lazy_path) { + .src_path, .dependency => .build_root, + .generated => unreachable, + .cwd_relative => .cwd, + .relative => |r| r.base, + }, + .mode = src.mode, + }, + .sub = switch (src.lazy_path) { + .src_path => |sp| try wc.addString(sp.sub_path), + .generated => unreachable, + .cwd_relative => |sub_path| try wc.addString(sub_path), + .dependency => |d| try wc.addString(d.sub_path), + .relative => |r| try wc.addString(r.sub_path), + }, + .pkg = switch (src.lazy_path) { + .src_path => |sp| .init(try s.builderToPackage(sp.owner)), + .generated => unreachable, + .cwd_relative, .relative => .none, + .dependency => |d| .init(try s.builderToPackage(d.dependency.builder)), + }, + }; + } + // Starting from all top-level steps in `b`, traverse the entire step graph // and add all step dependencies implied by module graphs. const top_level_steps = b.top_level_steps.values(); diff --git a/lib/compiler/libc.zig b/lib/compiler/libc.zig deleted file mode 100644 index 8ca53fefc905c01527c7157bee4c0d10575e582e..0000000000000000000000000000000000000000 --- a/lib/compiler/libc.zig +++ /dev/null @@ -1,140 +0,0 @@ -const std = @import("std"); -const Io = std.Io; -const mem = std.mem; -const LibCInstallation = std.zig.LibCInstallation; - -const usage_libc = - \\Usage: zig libc - \\ - \\ Detect the native libc installation and print the resulting - \\ paths to stdout. You can save this into a file and then edit - \\ the paths to create a cross compilation libc kit. Then you - \\ can pass `--libc [file]` for Zig to use it. - \\ - \\Usage: zig libc [paths_file] - \\ - \\ Parse a libc installation text file and validate it. - \\ - \\Options: - \\ -h, --help Print this help and exit - \\ -target [name] -- see the targets command - \\ -includes Print the libc include directories for the target - \\ -; - -var stdout_buffer: [4096]u8 = undefined; - -pub fn main(init: std.process.Init) !void { - const arena = init.arena.allocator(); - const gpa = init.gpa; - const io = init.io; - const args = try init.minimal.args.toSlice(arena); - const environ_map = init.environ_map; - - const zig_lib_directory = args[1]; - - var input_file: ?[]const u8 = null; - var target_arch_os_abi: []const u8 = "native"; - var print_includes: bool = false; - var stdout_writer = Io.File.stdout().writer(io, &stdout_buffer); - const stdout = &stdout_writer.interface; - { - var i: usize = 2; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { - try stdout.writeAll(usage_libc); - try stdout.flush(); - return std.process.cleanExit(io); - } else if (mem.eql(u8, arg, "-target")) { - if (i + 1 >= args.len) fatal("expected parameter after {s}", .{arg}); - i += 1; - target_arch_os_abi = args[i]; - } else if (mem.eql(u8, arg, "-includes")) { - print_includes = true; - } else { - fatal("unrecognized parameter: '{s}'", .{arg}); - } - } else if (input_file != null) { - fatal("unexpected extra parameter: '{s}'", .{arg}); - } else { - input_file = arg; - } - } - } - - const target_query = std.zig.parseTargetQueryOrReportFatalError(gpa, .{ - .arch_os_abi = target_arch_os_abi, - }); - const target = std.zig.resolveTargetQueryOrFatal(io, target_query); - - if (print_includes) { - const libc_installation: ?*LibCInstallation = libc: { - if (input_file) |libc_file| { - const libc = try arena.create(LibCInstallation); - libc.* = LibCInstallation.parse(arena, io, libc_file, &target) catch |err| { - fatal("unable to parse libc file at path {s}: {t}", .{ libc_file, err }); - }; - break :libc libc; - } else { - break :libc null; - } - }; - - const is_native_abi = target_query.isNativeAbi(); - - const libc_dirs = std.zig.LibCDirs.detect( - arena, - io, - zig_lib_directory, - &target, - is_native_abi, - true, - libc_installation, - environ_map, - ) catch |err| { - const zig_target = try target.zigTriple(arena); - fatal("unable to detect libc for target {s}: {t}", .{ zig_target, err }); - }; - - if (libc_dirs.libc_include_dir_list.len == 0) { - const zig_target = try target.zigTriple(arena); - fatal("no include dirs detected for target {s}", .{zig_target}); - } - - for (libc_dirs.libc_include_dir_list) |include_dir| { - try stdout.writeAll(include_dir); - try stdout.writeByte('\n'); - } - try stdout.flush(); - return std.process.cleanExit(io); - } - - if (input_file) |libc_file| { - var libc = LibCInstallation.parse(gpa, io, libc_file, &target) catch |err| { - fatal("unable to parse libc file at path {s}: {t}", .{ libc_file, err }); - }; - defer libc.deinit(gpa); - } else { - if (!target_query.canDetectLibC()) { - fatal("unable to detect libc for non-native target", .{}); - } - var libc = LibCInstallation.findNative(gpa, io, .{ - .verbose = true, - .target = &target, - .environ_map = environ_map, - }) catch |err| { - fatal("unable to detect native libc: {t}", .{err}); - }; - defer libc.deinit(gpa); - - try libc.render(stdout); - try stdout.flush(); - } -} - -fn fatal(comptime format: []const u8, args: anytype) noreturn { - std.log.err(format, args); - std.process.exit(1); -} diff --git a/lib/compiler/resinator/main.zig b/lib/compiler/resinator/main.zig index 98099c31ffddbad2a0f969ef3adbc903f54fc5f3..f4e03fbe305f3b86cc7a9063ffd42c1089009df2 100644 --- a/lib/compiler/resinator/main.zig +++ b/lib/compiler/resinator/main.zig @@ -44,7 +44,7 @@ pub fn main(init: std.process.Init.Minimal) !void { try renderErrorMessage(stderr.terminal(), .err, "expected zig lib dir as first argument", .{}); std.process.exit(1); } - const zig_lib_dir = args[1]; + const zig_lib_dir = std.mem.cutPrefix(u8, args[1], "--zig-lib=") orelse @panic("bad --zig-lib= arg"); var cli_args = args[2..]; var zig_integration = false; @@ -640,7 +640,7 @@ fn getIncludePaths( }; const target = std.zig.resolveTargetQueryOrFatal(io, target_query); const is_native_abi = target_query.isNativeAbi(); - const detected_libc = std.zig.LibCDirs.detect(arena, io, zig_lib_dir, &target, is_native_abi, true, null, environ_map) catch { + const detected_libc = std.zig.LibCDirs.detect(arena, io, .{ .root_dir = .cwd(), .sub_path = zig_lib_dir }, &target, is_native_abi, true, null, environ_map) catch { if (includes == .any) { // fall back to mingw includes = .gnu; @@ -669,7 +669,7 @@ fn getIncludePaths( const detected_libc = std.zig.LibCDirs.detect( arena, io, - zig_lib_dir, + .{ .root_dir = .cwd(), .sub_path = zig_lib_dir }, &target, is_native_abi, true, diff --git a/lib/compiler/std-docs.zig b/lib/compiler/std-docs.zig index db2d22a06f02b999e408b0609635277d95284a70..f41159adea3c864e0c02fe2402bbba70196dd91d 100644 --- a/lib/compiler/std-docs.zig +++ b/lib/compiler/std-docs.zig @@ -29,9 +29,9 @@ pub fn main(init: std.process.Init) !void { var argv = try init.minimal.args.iterateAllocator(arena); defer argv.deinit(); assert(argv.skip()); - const zig_lib_directory = argv.next().?; - const zig_exe_path = argv.next().?; - const global_cache_path = argv.next().?; + const zig_lib_directory = mem.cutPrefix(u8, argv.next().?, "--zig-lib=") orelse @panic("bad --zig-lib= arg"); + const zig_exe_path = mem.cutPrefix(u8, argv.next().?, "--zig=") orelse @panic("bad --zig= arg"); + const global_cache_path = mem.cutPrefix(u8, argv.next().?, "--global-cache=") orelse @panic("bad --global-cache= arg"); var lib_dir = try Io.Dir.cwd().openDir(io, zig_lib_directory, .{}); defer lib_dir.close(io); diff --git a/lib/std/Build.zig b/lib/std/Build.zig index a3bf69604e4ec7e68847e6caced7ed91fbbcf269..6d07c0fb49c6a056151a8fdc0a4adc13aa385dc1 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -64,6 +64,11 @@ pkg_hash: []const u8, /// A mapping from dependency names to package hashes. available_deps: AvailableDeps, +pub const ConfigureDependency = struct { + lazy_path: LazyPath, + mode: std.Build.Configuration.PathDep.Mode, +}; + pub const ReleaseMode = enum { off, any, @@ -102,6 +107,12 @@ pub const Graph = struct { /// Observing this data causes cache poisoning. See `CachePoison`. search_prefixes: std.ArrayList([]const u8) = .empty, + /// Populated by calling one of: + /// * `dependOnFileContents` + /// * `dependOnFileMetadata` + /// * `dependOnDirectory` + configure_dependencies: ArrayList(ConfigureDependency) = .empty, + /// If the cache is poisoned means that the **configure logic** had side /// effects, or otherwise did something that could not be tracked by the /// cache system. @@ -165,7 +176,8 @@ pub const Graph = struct { /// A path whose components and contents are known at some point during /// `Step` resolution, relative to the provided base directory. - pub fn path(graph: *Graph, base: Configuration.Path.Base, sub_path: []const u8) LazyPath { + pub fn path(graph: *Graph, base: Configuration.LazyPath.Relative.Base, sub_path: []const u8) LazyPath { + assert(base != .build_root); return .{ .relative = .{ .base = base, .sub_path = @This().dupePath(graph, sub_path), @@ -204,6 +216,9 @@ pub const Graph = struct { /// did something that could not be tracked by the cache system. /// /// See `CachePoison` documentation for more details. + /// + /// As an alternative to calling this function, consider these APIs instead: + /// * `dependOnFileContents` pub fn poisonCache(graph: *Graph) void { switch (graph.cache_poison) { .pure => graph.cache_poison = .poisoned, @@ -290,6 +305,7 @@ const UserValue = union(enum) { lazy_path_list: std.array_list.Managed(LazyPath), }; +/// Build system implementation detail. pub fn create( graph: *Graph, root: Cache.Path, @@ -681,9 +697,9 @@ fn hashUserInputOptionsMap(allocator: Allocator, user_input_options: UserInputOp /// Create a set of key-value pairs that can be converted into a Zig source /// file and then inserted into a Zig compilation's module table for importing. -/// In other words, this provides a way to expose build.zig values to Zig -/// source code with `@import`. -/// Related: `Module.addOptions`. +/// +/// This provides a way to expose build.zig values to Zig source code with +/// `@import`. Related: `Module.addOptions`. pub fn addOptions(b: *Build) *Step.Options { return Step.Options.create(b); } @@ -837,10 +853,7 @@ pub fn addModule(b: *Build, name: []const u8, options: Module.CreateOptions) *Mo module, ) catch @panic("OOM"); if (gop.found_existing) { - panic( - "A module with the name '{s}' has already been added to the package. Consider creating a private module with std.Build.createModule", - .{name}, - ); + panic("A module with the name {q} has already been added to the package. Consider creating a private module with std.Build.createModule", .{name}); } return module; } @@ -970,6 +983,7 @@ pub fn addConfigHeader( return config_header_step; } +/// Deprecated, call `Graph.dupeString` instead. pub fn dupe(b: *Build, bytes: []const u8) []const u8 { return b.graph.dupeString(bytes); } @@ -1000,7 +1014,7 @@ pub fn addNamedWriteFiles(b: *Build, name: []const u8) *Step.WriteFile { ) catch @panic("OOM"); if (gop.found_existing) { panic( - "A WriteFile step with the name '{s}' has already been added to the package. Consider creating a private WriteFile step with std.Build.addWriteFiles", + "A WriteFile step with the name {q} has already been added to the package. Consider creating a private WriteFile step with std.Build.addWriteFiles", .{name}, ); } @@ -1015,10 +1029,7 @@ pub fn addNamedLazyPath(b: *Build, name: []const u8, lp: LazyPath) void { lp.dupe(graph), ) catch @panic("OOM"); if (gop.found_existing) { - panic( - "A LazyPath with the name '{s}' has already been added to the package.", - .{name}, - ); + panic("A LazyPath with the name {q} has already been added to the package.", .{name}); } } @@ -1121,7 +1132,7 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw .enum_options = enum_options, }; if ((b.available_options_map.fetchPut(arena, name, available_option) catch @panic("OOM")) != null) { - panic("option '{s}' declared twice", .{name}); + panic("option {q} declared twice", .{name}); } const option_ptr = b.user_input_options.getPtr(name) orelse return null; @@ -1292,6 +1303,8 @@ pub fn option(b: *Build, comptime T: type, name_raw: []const u8, description_raw } } +/// Creates a top-level build step, exposed to the CLI user and advertised in +/// the "--help" menu. pub fn step(b: *Build, name: []const u8, description: []const u8) *Step { const graph = b.graph; const arena = graph.arena; @@ -1371,7 +1384,7 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile opts_copy.diagnostics = &diags; return std.Target.Query.parse(opts_copy) catch |err| switch (err) { error.UnknownCpuModel => { - std.debug.print("unknown CPU: '{s}'\navailable CPUs for architecture '{t}':\n", .{ + std.debug.print("unknown CPU: {q}\navailable CPUs for architecture {t}:\n", .{ diags.cpu_name.?, diags.arch.?, }); for (diags.arch.?.allCpuModels()) |cpu| { @@ -1381,7 +1394,7 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile }, error.UnknownCpuFeature => { std.debug.print( - \\unknown CPU feature: '{s}' + \\unknown CPU feature: {q} \\available CPU features for architecture '{t}': \\ , .{ @@ -1394,7 +1407,7 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile }, error.UnknownOperatingSystem => { std.debug.print( - \\unknown OS: '{s}' + \\unknown OS: {q} \\available operating systems: \\ , .{diags.os_name.?}); @@ -1404,9 +1417,7 @@ pub fn parseTargetQuery(options: std.Target.Query.ParseOptions) error{ParseFaile return error.ParseFailed; }, else => |e| { - std.debug.print("unable to parse target '{s}': {s}\n", .{ - options.arch_os_abi, @errorName(e), - }); + std.debug.print("unable to parse target {q}: {t}\n", .{ options.arch_os_abi, e }); return error.ParseFailed; }, }; @@ -1469,13 +1480,14 @@ pub fn standardTargetOptionsQueryOnly(b: *Build, args: StandardTargetOptionsArgs q.serializeCpuAlloc(arena) catch @panic("OOM"), }); } - log.err("chosen target '{s}' does not match one of the allowed targets", .{ + log.err("chosen target {q} does not match one of the allowed targets", .{ selected_target.zigTriple(arena) catch @panic("OOM"), }); b.markInvalidUserInput(); return args.default_target; } +/// Build system implementation detail. pub fn addUserInputOption(b: *Build, name_raw: []const u8, value_raw: []const u8) error{OutOfMemory}!bool { const graph = b.graph; const arena = graph.arena; @@ -1523,7 +1535,7 @@ pub fn addUserInputOption(b: *Build, name_raw: []const u8, value_raw: []const u8 return true; }, .lazy_path, .lazy_path_list => { - log.warn("the lazy path value type isn't added from the CLI, but somehow '{s}' is a .{f}", .{ + log.warn("the lazy path value type isn't added from the CLI, but somehow {q} is a .{f}", .{ name, std.zig.fmtId(@tagName(gop.value_ptr.value)), }); return true; @@ -1532,6 +1544,7 @@ pub fn addUserInputOption(b: *Build, name_raw: []const u8, value_raw: []const u8 return false; } +/// Build system implementation detail. pub fn addUserInputFlag(b: *Build, name_raw: []const u8) error{OutOfMemory}!bool { const graph = b.graph; const name = graph.dupeString(name_raw); @@ -1592,6 +1605,7 @@ fn markInvalidUserInput(b: *Build) void { b.invalid_user_input = true; } +/// Build system implementation detail. pub fn validateUserInputDidItFail(b: *Build) bool { // Make sure all args are used. var it = b.user_input_options.iterator(); @@ -1689,9 +1703,7 @@ pub fn addCheckFile( /// References a file or directory relative to the source root. pub fn path(b: *Build, sub_path: []const u8) LazyPath { if (fs.path.isAbsolute(sub_path)) { - panic("sub_path is expected to be relative to the build root, but was this absolute path: '{s}'. Absolute paths can cause problems but can be created via Graph.cwdRelativePath", .{ - sub_path, - }); + panic("sub_path is expected to be relative to the build root, but was this absolute path: {q}. Absolute paths can cause problems but can be created via Graph.cwdRelativePath", .{sub_path}); } return .{ .src_path = .{ .owner = b, @@ -2012,34 +2024,34 @@ pub const Dependency = struct { for (d.builder.install_tls.step.dependencies.items) |dep_step| { const inst = dep_step.cast(Step.InstallArtifact) orelse continue; if (mem.eql(u8, inst.artifact.name, name)) { - if (found != null) panic("artifact name '{s}' is ambiguous", .{name}); + if (found != null) panic("artifact name {q} is ambiguous", .{name}); found = inst.artifact; } } return found orelse { for (d.builder.install_tls.step.dependencies.items) |dep_step| { const inst = dep_step.cast(Step.InstallArtifact) orelse continue; - log.info("available artifact: '{s}'", .{inst.artifact.name}); + log.info("available artifact: {q}", .{inst.artifact.name}); } - panic("unable to find artifact '{s}'", .{name}); + panic("unable to find artifact {q}", .{name}); }; } pub fn module(d: *Dependency, name: []const u8) *Module { return d.builder.modules.get(name) orelse { - panic("unable to find module '{s}'", .{name}); + panic("unable to find module {q}", .{name}); }; } pub fn namedWriteFiles(d: *Dependency, name: []const u8) *Step.WriteFile { return d.builder.named_writefiles.get(name) orelse { - panic("unable to find named writefiles '{s}'", .{name}); + panic("unable to find named writefiles {q}", .{name}); }; } pub fn namedLazyPath(d: *Dependency, name: []const u8) LazyPath { return d.builder.named_lazy_paths.get(name) orelse { - panic("unable to find named lazypath '{s}'", .{name}); + panic("unable to find named lazypath {q}", .{name}); }; } @@ -2178,6 +2190,7 @@ pub inline fn lazyImport( comptime unreachable; // Bad @dependencies source } +/// Build system implementation detail. pub fn dependencyFromBuildZig( b: *Build, /// The build.zig struct of the dependency, normally obtained by `@import` of the dependency. @@ -2310,14 +2323,13 @@ fn dependencyInner( .root_dir = .{ .path = build_root_string, .handle = Io.Dir.cwd().openDir(io, build_root_string, .{}) catch |err| - process.fatal("unable to open {s}: {t}", .{ build_root_string, err }), + process.fatal("failed to open {q}: {t}", .{ build_root_string, err }), }, }; - const sub_builder = b.createChild(name, dep_root, pkg_hash, pkg_deps, user_input_options) catch - @panic("unhandled error"); + const sub_builder = b.createChild(name, dep_root, pkg_hash, pkg_deps, user_input_options) catch @panic("OOM"); if (build_zig) |bz| { - sub_builder.runBuild(bz) catch @panic("unhandled error"); + sub_builder.runBuild(bz); if (sub_builder.validateUserInputDidItFail()) { std.debug.dumpCurrentStackTrace(.{ .first_address = @returnAddress() }); @@ -2334,11 +2346,11 @@ fn dependencyInner( return dep; } -pub fn runBuild(b: *Build, build_zig: anytype) anyerror!void { +/// Build system implementation detail. +pub fn runBuild(b: *Build, build_zig: anytype) void { switch (@typeInfo(@typeInfo(@TypeOf(build_zig.build)).@"fn".return_type.?)) { - .void => build_zig.build(b), - .error_union => try build_zig.build(b), - else => @compileError("expected return type of build to be 'void' or '!void'"), + .error_union => return build_zig.build(b) catch unreachable, + else => return build_zig.build(b), } } @@ -2402,7 +2414,7 @@ pub const LazyPath = union(enum) { }, relative: struct { - base: Configuration.Path.Base, + base: Configuration.LazyPath.Relative.Base, sub_path: []const u8 = "", pub fn eql(a: @This(), b: @This()) bool { @@ -2607,7 +2619,7 @@ fn dumpBadDirnameHelp( if (asking_step) |as| { stderr.setColor(.red) catch {}; - try w.print(" The step '{s}' that is missing a dependency on the above step was created by this stack trace:\n", .{as.name}); + try w.print(" The step {q} that is missing a dependency on the above step was created by this stack trace:\n", .{as.name}); stderr.setColor(.reset) catch {}; as.dump(stderr); @@ -2708,9 +2720,101 @@ pub fn systemIntegrationOption( } } +/// Indicates that the build.zig logic depends on a particular file's contents. +/// +/// If the file is created, deleted, or has its contents changed, the configure +/// phase will be repeated. If the inode or mtime change, but the file contents +/// remain the same, it will not cause the configure logic to be repeated. +/// +/// This is an alternative to `Graph.poisonCache` that avoids making every invocation +/// of `zig build` into a cache miss. +/// +/// Only a subset of `LazyPath` are supported: +/// - Relative to cwd +/// - Relative to any package root +/// - Relative to zig cache or zig installation +/// +/// If the file would be inside one of the search prefixes, then the dependency +/// cannot be tracked; `Graph.poisonCache` must be used instead. +pub fn dependOnFileContents(b: *Build, lazy_path: LazyPath) void { + validateConfigureDependency(lazy_path); + const graph = b.graph; + graph.configure_dependencies.append(graph.arena, .{ + .lazy_path = lazy_path.dupe(graph), + .mode = .contents, + }) catch @panic("OOM"); +} + +/// Indicates that the build.zig logic depends on a particular file's size, +/// inode, mtime, and contents. +/// +/// If the file is created, deleted, has its contents changed, or the inode +/// changes, or the mtime changes, the configure phase will be repeated. +/// +/// This is an alternative to `Graph.poisonCache` that avoids making every invocation +/// of `zig build` into a cache miss. +/// +/// Only a subset of `LazyPath` are supported: +/// - Relative to cwd +/// - Relative to any package root +/// - Relative to zig cache or zig installation +/// +/// If the file would be inside one of the search prefixes, then the dependency +/// cannot be tracked; `Graph.poisonCache` must be used instead. +pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void { + validateConfigureDependency(lazy_path); + const graph = b.graph; + graph.configure_dependencies.append(graph.arena, .{ + .lazy_path = lazy_path.dupe(graph), + .mode = .metadata, + }) catch @panic("OOM"); +} + +/// Indicates that the build.zig logic depends on a particular directory's entries. +/// +/// This is an alternative to `Graph.poisonCache` that avoids making every invocation +/// of `zig build` into a cache miss. +/// +/// If any file is created, deleted, or renamed in this directory, the +/// configure phase will be repeated. +/// +/// Only a subset of `LazyPath` are supported: +/// - Relative to cwd +/// - Relative to any package root +/// - Relative to zig cache or zig installation +/// +/// If the directory would be inside one of the search prefixes, then the dependency +/// cannot be tracked; `Graph.poisonCache` must be used instead. +pub fn dependOnDirectory(b: *Build, lazy_path: LazyPath) void { + validateConfigureDependency(lazy_path); + const graph = b.graph; + graph.configure_dependencies.append(graph.arena, .{ + .lazy_path = lazy_path.dupe(graph), + .mode = .directory, + }) catch @panic("OOM"); +} + +fn validateConfigureDependency(lazy_path: LazyPath) void { + switch (lazy_path) { + .src_path, .cwd_relative, .dependency => {}, // OK + .generated => @panic("configure phase cannot depend on files generated during make phase"), + .relative => |relative| switch (relative.base) { + .cwd, .build_root, .local_cache, .global_cache, .zig_exe, .zig_lib => {}, // OK + .install_prefix, + .install_lib, + .install_bin, + .install_include, + => @panic("configure phase cannot depend on files installed during make phase"), + }, + } +} + test { _ = Cache; + _ = Configuration; + _ = Module; _ = Step; _ = Configuration; _ = &findProgram; + _ = abi; } diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 33c3148dd22080b5916100f4c3e210dabe38e14d..f731f5667c30b6406f0b3721d1249b1b3437fd77 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -28,7 +28,7 @@ mutex: Io.Mutex = .init, /// are replaced with single-character indicators. This is not to save /// space but to eliminate absolute file paths. This improves portability /// and usefulness of the cache for advanced use cases. -prefixes_buffer: [4]Directory = undefined, +prefixes_buffer: [5]Directory = undefined, prefixes_len: usize = 0, /// Used to identify prefixes. References external memory. cwd: []const u8, @@ -57,7 +57,7 @@ pub fn prefixes(cache: *const Cache) []const Directory { return cache.prefixes_buffer[0..cache.prefixes_len]; } -const PrefixedPath = struct { +pub const PrefixedPath = struct { prefix: u8, sub_path: []const u8, @@ -70,6 +70,15 @@ const PrefixedPath = struct { } }; +fn findPrefixPath(cache: *const Cache, path: Path) !PrefixedPath { + const gpa = cache.gpa; + const resolved_path = try std.fs.path.resolve(gpa, &.{ + cache.cwd, path.root_dir.path orelse ".", path.subPathOrDot(), + }); + errdefer gpa.free(resolved_path); + return findPrefixResolved(cache, resolved_path); +} + fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath { const gpa = cache.gpa; const resolved_path = try std.fs.path.resolve(gpa, &.{file_path}); @@ -91,13 +100,13 @@ fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath { }; // Free the resolved path since we're not going to return it gpa.free(resolved_path); - return PrefixedPath{ + return .{ .prefix = i, .sub_path = sub_path, }; } - return PrefixedPath{ + return .{ .prefix = 0, .sub_path = resolved_path, }; @@ -998,40 +1007,49 @@ pub const Manifest = struct { /// This is useful for processes that don't know the all the files that are /// depended on ahead of time. For example, a source file that can import /// other files will need to be recompiled if the imported file is changed. - pub fn addFilePost(self: *Manifest, file_path: []const u8) !void { - assert(self.manifest_file != null); - - const gpa = self.cache.gpa; - const prefixed_path = try self.cache.findPrefix(file_path); - errdefer gpa.free(prefixed_path.sub_path); - - const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); - errdefer _ = self.files.pop(); - - if (gop.found_existing) { - gpa.free(prefixed_path.sub_path); - return; - } - - gop.key_ptr.* = .{ - .prefixed_path = prefixed_path, - .max_file_size = null, - .handle = null, - .stat = undefined, - .bin_digest = undefined, - .contents = null, - }; - - self.files.lockPointers(); - defer self.files.unlockPointers(); - - try self.populateFileHash(gop.key_ptr); + pub fn addFilePost(man: *Manifest, file_path: []const u8) !void { + assert(man.manifest_file != null); + const gpa = man.cache.gpa; + const prefixed_path = try man.cache.findPrefix(file_path); + var keep = false; + defer if (!keep) gpa.free(prefixed_path.sub_path); + keep = try addPrefixedPathPost(man, prefixed_path); } pub fn addPathPost(man: *Manifest, path: Path) !void { - _ = man; - _ = path; - @panic("TODO"); + assert(man.manifest_file != null); + const gpa = man.cache.gpa; + const prefixed_path: PrefixedPath = try man.cache.findPrefixPath(path); + var keep = false; + defer if (!keep) gpa.free(prefixed_path.sub_path); + keep = try addPrefixedPathPost(man, prefixed_path); + } + + /// Low level function. `prefixed_path` references cloned memory. Returns + /// whether or not `prefixed_path.sub_path` should be kept. + pub fn addPrefixedPathPost(man: *Manifest, prefixed_path: PrefixedPath) !bool { + assert(man.manifest_file != null); + const gpa = man.cache.gpa; + + const gop = try man.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{}); + errdefer _ = man.files.pop(); + + if (gop.found_existing) return false; + + gop.key_ptr.* = .{ + .prefixed_path = prefixed_path, + .max_file_size = null, + .handle = null, + .stat = undefined, + .bin_digest = undefined, + .contents = null, + }; + + man.files.lockPointers(); + defer man.files.unlockPointers(); + + try man.populateFileHash(gop.key_ptr); + return true; } /// Like `addFilePost` but when the file contents have already been loaded from disk. diff --git a/lib/std/Build/Configuration.zig b/lib/std/Build/Configuration.zig index 12746e4498f76b745898559660dfb03c4722c233..f10bf9055b58d63061dfa3b37ff7954474a35dcf 100644 --- a/lib/std/Build/Configuration.zig +++ b/lib/std/Build/Configuration.zig @@ -10,8 +10,7 @@ const native_endian = builtin.target.cpu.arch.endian(); string_bytes: []u8, steps: []Step, -path_deps_base: []Path.Base, -path_deps_sub: []String, +path_deps: []PathDep, unlazy_deps: []String, system_integrations: []SystemIntegration, available_options: []AvailableOption, @@ -57,7 +56,7 @@ pub const Wip = struct { system_integrations: std.ArrayList(SystemIntegration) = .empty, available_options: std.ArrayList(AvailableOption) = .empty, steps: std.ArrayList(Step) = .empty, - path_deps: std.MultiArrayList(Path) = .empty, + path_deps: std.ArrayList(PathDep) = .empty, search_prefixes: std.ArrayList(String) = .empty, extra: std.ArrayList(u32) = .empty, next_generated_file_index: u32 = 0, @@ -154,7 +153,7 @@ pub const Wip = struct { const header: Header = .{ .string_bytes_len = @intCast(wip.string_bytes.items.len), .steps_len = @intCast(wip.steps.items.len), - .path_deps_len = @intCast(wip.path_deps.len), + .path_deps_len = @intCast(wip.path_deps.items.len), .unlazy_deps_len = @intCast(wip.unlazy_deps.items.len), .system_integrations_len = @intCast(wip.system_integrations.items.len), .available_options_len = @intCast(wip.available_options.items.len), @@ -171,8 +170,7 @@ pub const Wip = struct { @ptrCast(&header), wip.string_bytes.items, @ptrCast(wip.steps.items), - @ptrCast(wip.path_deps.items(.base)), - @ptrCast(wip.path_deps.items(.sub)), + @ptrCast(wip.path_deps.items), @ptrCast(wip.unlazy_deps.items), @ptrCast(wip.system_integrations.items), @ptrCast(wip.available_options.items), @@ -1551,9 +1549,23 @@ pub const LazyPath = union(@This().Tag) { pub const Flags = packed struct(u32) { tag: Tag = .relative, - base: Path.Base, + base: Base, _: u16 = 0, }; + + pub const Base = enum(u8) { + cwd, + local_cache, + global_cache, + /// Must not be used with Relative since package index is missing. + build_root, + zig_exe, + zig_lib, + install_prefix, + install_lib, + install_bin, + install_include, + }; }; }; @@ -1597,6 +1609,26 @@ pub const Package = struct { return package.dep_prefix.slice(c); } }; + + pub const OptionalIndex = enum(u32) { + none = max_u32 - 1, + root = max_u32, + _, + + pub fn init(i: Index) OptionalIndex { + const result: OptionalIndex = @enumFromInt(@intFromEnum(i)); + assert(result != .none); + return result; + } + + pub fn unwrap(this: @This()) ?Index { + return switch (this) { + .none => null, + .root => .root, + _ => @enumFromInt(@intFromEnum(this)), + }; + } + }; }; pub const Module = struct { @@ -1833,29 +1865,18 @@ pub const OptionalStringList = enum(u32) { } }; -pub const Path = extern struct { - base: Base, +pub const PathDep = extern struct { + flags: Flags, sub: String, + pkg: Package.OptionalIndex, - pub const Base = enum(u8) { - cwd, - local_cache, - global_cache, - build_root, - zig_exe, - zig_lib, - install_prefix, - install_lib, - install_bin, - install_include, + pub const Flags = packed struct(u32) { + mode: Mode, + base: LazyPath.Relative.Base, + _: u16 = 0, }; - pub fn toCachePath(path: Path, c: *const Configuration, arena: Allocator) std.Build.Cache.Path { - _ = c; - _ = arena; - _ = path; - @panic("TODO"); - } + pub const Mode = enum(u8) { directory, contents, metadata }; }; pub const InstallDestDir = enum(u32) { @@ -3430,8 +3451,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { const result: Configuration = .{ .string_bytes = try arena.alloc(u8, header.string_bytes_len), .steps = try arena.alloc(Step, header.steps_len), - .path_deps_sub = try arena.alloc(String, header.path_deps_len), - .path_deps_base = try arena.alloc(Path.Base, header.path_deps_len), + .path_deps = try arena.alloc(PathDep, header.path_deps_len), .unlazy_deps = try arena.alloc(String, header.unlazy_deps_len), .system_integrations = try arena.alloc(SystemIntegration, header.system_integrations_len), .available_options = try arena.alloc(AvailableOption, header.available_options_len), @@ -3444,8 +3464,7 @@ pub fn load(arena: Allocator, reader: *Io.Reader) LoadError!Configuration { var vecs = [_][]u8{ result.string_bytes, @ptrCast(result.steps), - @ptrCast(result.path_deps_base), - @ptrCast(result.path_deps_sub), + @ptrCast(result.path_deps), @ptrCast(result.unlazy_deps), @ptrCast(result.system_integrations), @ptrCast(result.available_options), diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index c16d6273abe6668ef272172c6b8649739da9af49..9d036dcf2db7b1417c60af507461d8ec647aeb9e 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -5,7 +5,6 @@ const Io = std.Io; const Step = std.Build.Step; const Allocator = std.mem.Allocator; const Configuration = std.Build.Configuration; -const allocPrint = std.fmt.allocPrint; step: Step, values: std.array_hash_map.String(Value) = .empty, @@ -84,13 +83,9 @@ pub fn create(owner: *std.Build, options: Options) *ConfigHeader { }; const name = if (options.style.getPath()) |s| - allocPrint(arena, "configure {t} header {f} to {s}", .{ - options.style, s, include_path, - }) catch @panic("OOM") + arena.print("configure {t} header {f} to {s}", .{ options.style, s, include_path }) catch @panic("OOM") else - allocPrint(arena, "configure {t} header to {s}", .{ - options.style, include_path, - }) catch @panic("OOM"); + arena.print("configure {t} header to {s}", .{ options.style, include_path }) catch @panic("OOM"); config_header.* = .{ .step = .init(.{ diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig index 5698111e044860f573a9dcfc9fcb5ba5393c8590..be205bee22b83395a364d9f962f5545888597ada 100644 --- a/lib/std/Build/Step/TranslateC.zig +++ b/lib/std/Build/Step/TranslateC.zig @@ -3,7 +3,6 @@ const TranslateC = @This(); const std = @import("std"); const fs = std.fs; const mem = std.mem; -const allocPrint = std.fmt.allocPrint; const Step = std.Build.Step; const LazyPath = std.Build.LazyPath; const Configuration = std.Build.Configuration; @@ -158,7 +157,7 @@ pub fn defineCMacro(translate_c: *TranslateC, name: []const u8, value: ?[]const const graph = translate_c.step.owner.graph; const arena = graph.arena; const wc = &graph.wip_configuration; - const macro = allocPrint(arena, "{s}={s}", .{ name, value orelse "1" }) catch @panic("OOM"); + const macro = arena.print("{s}={s}", .{ name, value orelse "1" }) catch @panic("OOM"); const macro_string = wc.addString(macro) catch @panic("OOM"); translate_c.c_macros.append(arena, macro_string) catch @panic("OOM"); } diff --git a/lib/std/Progress.zig b/lib/std/Progress.zig index 70c61f27e2b15d6d7f903e6d778f4bd0963b5d3c..b0f1b5f9a9011427e48c83c334c602c69c6f42c2 100644 --- a/lib/std/Progress.zig +++ b/lib/std/Progress.zig @@ -165,25 +165,6 @@ pub const TerminalMode = union(enum) { }; }; -pub const Options = struct { - /// User-provided buffer with static lifetime. - /// - /// Used to store the entire write buffer sent to the terminal. Progress output will be truncated if it - /// cannot fit into this buffer which will look bad but not cause any malfunctions. - /// - /// Must be at least 200 bytes. - draw_buffer: []u8 = &default_draw_buffer, - /// How many nanoseconds between writing updates to the terminal. - refresh_rate_ns: Io.Duration = .fromMilliseconds(80), - /// How many nanoseconds to keep the output hidden - initial_delay_ns: Io.Duration = .fromMilliseconds(200), - /// If provided, causes the progress item to have a denominator. - /// 0 means unknown. - estimated_total_items: usize = 0, - root_name: []const u8 = "", - disable_printing: bool = false, -}; - /// Represents one unit of progress. Each node can have children nodes, or /// one can use integers with `update`. pub const Node = struct { @@ -578,6 +559,25 @@ pub const ParentFileError = error{ UnrecognizedFormat, }; +pub const Options = struct { + /// User-provided buffer with static lifetime. + /// + /// Used to store the entire write buffer sent to the terminal. Progress output will be truncated if it + /// cannot fit into this buffer which will look bad but not cause any malfunctions. + /// + /// Must be at least 200 bytes. + draw_buffer: []u8 = &default_draw_buffer, + /// How many nanoseconds between writing updates to the terminal. + refresh_rate_ns: Io.Duration = .fromMilliseconds(80), + /// How many nanoseconds to keep the output hidden + initial_delay_ns: Io.Duration = .fromMilliseconds(200), + /// If provided, causes the progress item to have a denominator. + /// 0 means unknown. + estimated_total_items: usize = 0, + root_name: []const u8 = "", + disable_printing: bool = false, +}; + /// Initializes a global Progress instance. /// /// Asserts there is only one global Progress instance. diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 05891c582e38646da978177e34a3e8b26ac7f0af..52e5f6c2608fce8fc49ccff006d786da91a6775e 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -2,12 +2,18 @@ //! source lives here. These APIs are provided as-is and have absolutely no API //! guarantees whatsoever. +const builtin = @import("builtin"); + const std = @import("std.zig"); const assert = std.debug.assert; const mem = std.mem; +const log = std.log; const Allocator = std.mem.Allocator; const Io = std.Io; const Writer = std.Io.Writer; +const Cache = std.Build.Cache; +const fatal = std.process.fatal; +const Dir = std.Io.Dir; const tokenizer = @import("zig/tokenizer.zig"); @@ -47,6 +53,9 @@ pub const c_translation = struct { pub const helpers = @import("zig/c_translation/helpers.zig"); }; +pub const default_local_zig_cache_basename = ".zig-cache"; +pub const build_zig_basename = "build.zig"; + pub const SrcHasher = std.crypto.hash.Blake3; pub const SrcHash = [16]u8; @@ -70,7 +79,7 @@ pub const Color = enum { /// CLICOLOR_FORCE environment variables. Color is always disabled on WASI per /// https://github.com/WebAssembly/WASI/issues/162 pub fn settingFromEnvironment(environ_map: *const std.process.Environ.Map) Color { - return if (@import("builtin").os.tag == .wasi or EnvVar.NO_COLOR.isSet(environ_map)) + return if (builtin.os.tag == .wasi or EnvVar.NO_COLOR.isSet(environ_map)) .off else if (EnvVar.CLICOLOR_FORCE.isSet(environ_map)) .on @@ -163,8 +172,8 @@ pub const BinNameOptions = struct { os_tag: std.Target.Os.Tag, ofmt: std.Target.ObjectFormat, abi: std.Target.Abi, - output_mode: std.builtin.OutputMode, - link_mode: ?std.builtin.LinkMode = null, + output_mode: std.lang.OutputMode, + link_mode: ?std.lang.LinkMode = null, version: ?std.SemanticVersion = null, }; @@ -512,7 +521,7 @@ pub const FormatId = struct { pub fn format(ctx: FormatId, writer: *Writer) Writer.Error!void { const bytes = ctx.bytes; if (isValidId(bytes) and - (ctx.flags.allow_primitive or !std.zig.isPrimitive(bytes)) and + (ctx.flags.allow_primitive or !isPrimitive(bytes)) and (ctx.flags.allow_underscore or !isUnderscore(bytes))) { return writer.writeAll(bytes); @@ -592,7 +601,7 @@ pub fn isValidId(bytes: []const u8) bool { else => return false, } } - return std.zig.Token.getKeyword(bytes) == null; + return Token.getKeyword(bytes) == null; } test isValidId { @@ -658,7 +667,7 @@ pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *Io.File.Reader) ![ } pub fn printAstErrorsToStderr(gpa: Allocator, io: Io, tree: Ast, path: []const u8, color: Color) !void { - var wip_errors: std.zig.ErrorBundle.Wip = undefined; + var wip_errors: ErrorBundle.Wip = undefined; try wip_errors.init(gpa); defer wip_errors.deinit(); @@ -673,7 +682,7 @@ pub fn putAstErrorsIntoBundle( gpa: Allocator, tree: Ast, path: []const u8, - wip_errors: *std.zig.ErrorBundle.Wip, + wip_errors: *ErrorBundle.Wip, ) Allocator.Error!void { switch (tree.mode) { .zig => { @@ -692,7 +701,7 @@ pub fn putAstErrorsIntoBundle( } pub fn resolveTargetQueryOrFatal(io: Io, target_query: std.Target.Query) std.Target { - return std.zig.system.resolveTargetQuery(io, target_query) catch |err| + return system.resolveTargetQuery(io, target_query) catch |err| std.process.fatal("unable to resolve target: {t}", .{err}); } @@ -713,7 +722,7 @@ pub fn parseTargetQueryOrReportFatalError( for (diags.arch.?.allCpuModels()) |cpu| { help_text.print(" {s}\n", .{cpu.name}) catch break :help; } - std.log.info("available CPUs for architecture '{s}':\n{s}", .{ + log.info("available CPUs for architecture '{s}':\n{s}", .{ @tagName(diags.arch.?), help_text.items, }); } @@ -726,7 +735,7 @@ pub fn parseTargetQueryOrReportFatalError( for (diags.arch.?.allFeaturesList()) |feature| { help_text.print(" {s}: {s}\n", .{ feature.name, feature.description }) catch break :help; } - std.log.info("available CPU features for architecture '{s}':\n{s}", .{ + log.info("available CPU features for architecture '{s}':\n{s}", .{ @tagName(diags.arch.?), help_text.items, }); } @@ -739,7 +748,7 @@ pub fn parseTargetQueryOrReportFatalError( inline for (@typeInfo(std.Target.ObjectFormat).@"enum".field_names) |field_name| { help_text.print(" {s}\n", .{field_name}) catch break :help; } - std.log.info("available object formats:\n{s}", .{help_text.items}); + log.info("available object formats:\n{s}", .{help_text.items}); } std.process.fatal("unknown object format: '{s}'", .{opts.object_format.?}); }, @@ -750,7 +759,7 @@ pub fn parseTargetQueryOrReportFatalError( inline for (@typeInfo(std.Target.Cpu.Arch).@"enum".field_names) |field_name| { help_text.print(" {s}\n", .{field_name}) catch break :help; } - std.log.info("available architectures:\n{s} native\n", .{help_text.items}); + log.info("available architectures:\n{s} native\n", .{help_text.items}); } std.process.fatal("unknown architecture: '{s}'", .{diags.unknown_architecture_name.?}); }, @@ -772,7 +781,9 @@ pub const EnvVar = enum { ZIG_BUILD_MULTILINE_ERRORS, ZIG_VERBOSE_LINK, ZIG_VERBOSE_CC, + ZIG_VERBOSE_CMD, ZIG_DEBUG_CMD, + ZIG_DEBUG_MAKER, ZIG_IS_DETECTING_LIBC_PATHS, ZIG_IS_AVOIDING_CALLING_ITSELF, @@ -1174,74 +1185,681 @@ pub const ClangCliParam = struct { } }; +/// Deprecated pub const AllocPrintCmdOptions = struct { cwd: ?[]const u8 = null, parent_env: ?*const std.process.Environ.Map = null, child_env: ?*const std.process.Environ.Map = null, }; +/// Deprecated pub fn allocPrintCmd(gpa: Allocator, argv: []const []const u8, options: AllocPrintCmdOptions) Allocator.Error![]u8 { - const shell = struct { - fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { - for (string) |c| { - if (switch (c) { - else => true, - '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, - '=' => is_argv0, - }) break; - } else return writer.writeAll(string); - - try writer.writeByte('"'); - for (string) |c| { - if (switch (c) { - std.ascii.control_code.nul => break, - '!', '"', '$', '\\', '`' => true, - else => !std.ascii.isPrint(c), - }) try writer.writeByte('\\'); - switch (c) { - std.ascii.control_code.nul => unreachable, - std.ascii.control_code.bel => try writer.writeByte('a'), - std.ascii.control_code.bs => try writer.writeByte('b'), - std.ascii.control_code.ht => try writer.writeByte('t'), - std.ascii.control_code.lf => try writer.writeByte('n'), - std.ascii.control_code.vt => try writer.writeByte('v'), - std.ascii.control_code.ff => try writer.writeByte('f'), - std.ascii.control_code.cr => try writer.writeByte('r'), - std.ascii.control_code.esc => try writer.writeByte('E'), - ' '...'~' => try writer.writeByte(c), - else => try writer.print("{o:0>3}", .{c}), - } - } - try writer.writeByte('"'); - } - }; - var aw: Io.Writer.Allocating = .init(gpa); defer aw.deinit(); - const writer = &aw.writer; - if (options.cwd) |path| { - writer.print("cd {s} && ", .{path}) catch return error.OutOfMemory; - } - if (options.child_env) |child_env| { - for (child_env.keys(), child_env.values()) |key, value| { - if (options.parent_env) |parent_env| { - if (parent_env.get(key)) |process_value| { - if (std.mem.eql(u8, value, process_value)) continue; - } - } - writer.print("{s}=", .{key}) catch return error.OutOfMemory; - shell.escape(writer, value, false) catch return error.OutOfMemory; - writer.writeByte(' ') catch return error.OutOfMemory; - } - } - shell.escape(writer, argv[0], true) catch return error.OutOfMemory; - for (argv[1..]) |arg| { - writer.writeByte(' ') catch return error.OutOfMemory; - shell.escape(writer, arg, false) catch return error.OutOfMemory; - } + SubprocessCommand.format(.{ + .argv = argv, + .cwd = options.cwd, + .parent_env = options.parent_env, + .child_env = options.child_env, + }, &aw.writer) catch return error.OutOfMemory; return aw.toOwnedSlice(); } +fn shellEscape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { + for (string) |c| { + if (switch (c) { + else => true, + '%', '+'...':', '@'...'Z', '_', 'a'...'z' => false, + '=' => is_argv0, + }) break; + } else return writer.writeAll(string); + + try writer.writeByte('"'); + for (string) |c| { + if (switch (c) { + std.ascii.control_code.nul => break, + '!', '"', '$', '\\', '`' => true, + else => !std.ascii.isPrint(c), + }) try writer.writeByte('\\'); + switch (c) { + std.ascii.control_code.nul => unreachable, + std.ascii.control_code.bel => try writer.writeByte('a'), + std.ascii.control_code.bs => try writer.writeByte('b'), + std.ascii.control_code.ht => try writer.writeByte('t'), + std.ascii.control_code.lf => try writer.writeByte('n'), + std.ascii.control_code.vt => try writer.writeByte('v'), + std.ascii.control_code.ff => try writer.writeByte('f'), + std.ascii.control_code.cr => try writer.writeByte('r'), + std.ascii.control_code.esc => try writer.writeByte('E'), + ' '...'~' => try writer.writeByte(c), + else => try writer.print("{o:0>3}", .{c}), + } + } + try writer.writeByte('"'); +} + +pub const SubprocessCommand = struct { + argv: []const []const u8, + cwd: ?[]const u8 = null, + parent_env: ?*const std.process.Environ.Map = null, + child_env: ?*const std.process.Environ.Map = null, + + pub fn format(sc: SubprocessCommand, w: *Io.Writer) Io.Writer.Error!void { + if (sc.cwd) |path| { + try w.print("cd {s} && ", .{path}); + } + if (sc.child_env) |child_env| { + for (child_env.keys(), child_env.values()) |key, value| { + if (sc.parent_env) |parent_env| { + if (parent_env.get(key)) |process_value| { + if (mem.eql(u8, value, process_value)) continue; + } + } + try w.print("{s}=", .{key}); + try shellEscape(w, value, false); + try w.writeByte(' '); + } + } + try shellEscape(w, sc.argv[0], true); + for (sc.argv[1..]) |arg| { + try w.writeByte(' '); + try shellEscape(w, arg, false); + } + } +}; + +/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This +/// means the path has no repeated separators, no "." or ".." components, and no trailing separator. +/// On WASI, "" is returned instead of ".". +pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 { + if (builtin.os.tag == .wasi) { + if (std.debug.runtime_safety) { + const cwd = try std.process.currentPathAlloc(io, gpa); + defer gpa.free(cwd); + assert(mem.eql(u8, cwd, ".")); + } + return ""; + } + const cwd = try std.process.currentPathAlloc(io, gpa); + defer gpa.free(cwd); + const resolved = try Dir.path.resolve(gpa, &.{cwd}); + assert(Dir.path.isAbsolute(resolved)); + return resolved; +} + +pub const Directories = struct { + /// The string returned by `introspect.getResolvedCwd`. This is typically an absolute path, + /// but on WASI is the empty string "" instead, because WASI does not have absolute paths. + cwd: []const u8, + /// The Zig 'lib' directory. + /// `zig_lib.path` is resolved (`resolvePath`) or `null` for cwd. + /// Guaranteed to be a different path from `global_cache` and `local_cache`. + zig_lib: Cache.Directory, + /// The global Zig cache directory. + /// `global_cache.path` is resolved (`resolvePath`) or `null` for cwd. + global_cache: Cache.Directory, + /// The local Zig cache directory. + /// `local_cache.path` is resolved (`resolvePath`) or `null` for cwd. + /// This may be the same as `global_cache`. + local_cache: Cache.Directory, + + pub fn deinit(dirs: *Directories, io: Io) void { + // The local and global caches could be the same. + const close_local = dirs.local_cache.handle.handle != dirs.global_cache.handle.handle; + + dirs.global_cache.handle.close(io); + if (close_local) dirs.local_cache.handle.close(io); + dirs.zig_lib.handle.close(io); + } + + /// Returns a `Directories` where `local_cache` is replaced with `global_cache`, intended for + /// use by sub-compilations (e.g. compiler_rt). Do not `deinit` the returned `Directories`; it + /// shares handles with `dirs`. + pub fn withoutLocalCache(dirs: Directories) Directories { + return .{ + .cwd = dirs.cwd, + .zig_lib = dirs.zig_lib, + .global_cache = dirs.global_cache, + .local_cache = dirs.global_cache, + }; + } + + const LocalCacheStrategy = union(enum) { + override: []const u8, + search, + global, + }; + + /// Uses `std.process.fatal` on error conditions. + pub fn init( + arena: Allocator, + io: Io, + override_zig_lib: ?[]const u8, + override_global_cache: ?[]const u8, + local_cache_strat: LocalCacheStrategy, + preopens: std.process.Preopens, + self_exe_path: switch (builtin.target.os.tag) { + .wasi => void, + else => []const u8, + }, + environ_map: *const std.process.Environ.Map, + cwd: []const u8, + ) Directories { + const wasi = builtin.target.os.tag == .wasi; + + const zig_lib: Cache.Directory = d: { + if (override_zig_lib) |path| break :d openUnresolved(arena, io, cwd, path, .@"zig lib"); + if (wasi) break :d getPreopen(preopens, "/lib"); + break :d findZigLibDirFromSelfExe(arena, io, cwd, self_exe_path) catch |err| { + fatal("unable to find zig installation directory {q}: {t}", .{ self_exe_path, err }); + }; + }; + + const global_cache: Cache.Directory = d: { + if (override_global_cache) |path| break :d openUnresolved(arena, io, cwd, path, .@"global cache"); + if (wasi) break :d getPreopen(preopens, "/cache"); + const path = resolveGlobalCacheDir(arena, environ_map) catch |err| { + fatal("unable to resolve zig cache directory: {t}", .{err}); + }; + break :d openUnresolved(arena, io, cwd, path, .@"global cache"); + }; + + const local_cache = getLocalCacheDirectory(arena, io, cwd, global_cache, local_cache_strat); + + if (mem.eql(u8, zig_lib.path orelse "", global_cache.path orelse "")) { + fatal("zig lib directory '{f}' cannot be equal to global cache directory '{f}'", .{ zig_lib, global_cache }); + } + if (mem.eql(u8, zig_lib.path orelse "", local_cache.path orelse "")) { + fatal("zig lib directory '{f}' cannot be equal to local cache directory '{f}'", .{ zig_lib, local_cache }); + } + + return .{ + .cwd = cwd, + .zig_lib = zig_lib, + .global_cache = global_cache, + .local_cache = local_cache, + }; + } + + fn getLocalCacheDirectory( + arena: Allocator, + io: Io, + cwd: []const u8, + global_cache: Cache.Directory, + local_cache_strat: LocalCacheStrategy, + ) Cache.Directory { + return switch (local_cache_strat) { + .override => |path| openUnresolved(arena, io, cwd, path, .@"local cache"), + .search => d: { + const maybe_path = resolveSuitableLocalCacheDir(arena, io, cwd) catch |err| + fatal("unable to resolve zig cache directory: {t}", .{err}); + const path = maybe_path orelse break :d global_cache; + break :d openUnresolved(arena, io, cwd, path, .@"local cache"); + }, + .global => global_cache, + }; + } + + fn getPreopen(preopens: std.process.Preopens, name: []const u8) Cache.Directory { + return .{ + .path = if (mem.eql(u8, name, ".")) null else name, + .handle = switch (preopens.get(name) orelse fatal("preopen not found: {q}", .{name})) { + .file => fatal("preopen {q} is not a directory", .{name}), + .dir => |d| d, + }, + }; + } + pub fn openUnresolved( + arena: Allocator, + io: Io, + cwd: []const u8, + unresolved_path: []const u8, + thing: enum { @"zig lib", @"global cache", @"local cache" }, + ) Cache.Directory { + const path = resolvePath(arena, cwd, &.{unresolved_path}) catch |err| { + fatal("unable to resolve {t} directory: {t}", .{ thing, err }); + }; + const nonempty_path = if (path.len == 0) "." else path; + const handle_or_err = switch (thing) { + .@"zig lib" => Dir.cwd().openDir(io, nonempty_path, .{}), + .@"global cache", .@"local cache" => Dir.cwd().createDirPathOpen(io, nonempty_path, .{}), + }; + return .{ + .path = if (path.len == 0) null else path, + .handle = handle_or_err catch |err| { + const extra_str: []const u8 = e: { + if (thing == .@"global cache") switch (err) { + error.AccessDenied, error.ReadOnlyFileSystem => break :e "\n" ++ + "If this location is not writable then consider specifying an alternative with " ++ + "the ZIG_GLOBAL_CACHE_DIR environment variable or the --global-cache-dir option.", + else => {}, + }; + break :e ""; + }; + fatal("unable to open {t} directory {q}: {t}{s}", .{ thing, nonempty_path, err, extra_str }); + }, + }; + } +}; + +/// Both the directory handle and the path are newly allocated resources which the caller now owns. +pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory { + const cwd_path = try getResolvedCwd(io, gpa); + defer gpa.free(cwd_path); + const self_exe_path = try std.process.executablePathAlloc(io, gpa); + defer gpa.free(self_exe_path); + + return findZigLibDirFromSelfExe(gpa, io, cwd_path, self_exe_path); +} + +/// Both the directory handle and the path are newly allocated resources which the caller now owns. +pub fn findZigLibDirFromSelfExe( + allocator: Allocator, + io: Io, + /// The return value of `getResolvedCwd`. + /// Passed as an argument to avoid pointlessly repeating the call. + cwd_path: []const u8, + self_exe_path: []const u8, +) error{ OutOfMemory, FileNotFound }!Cache.Directory { + const cwd = Dir.cwd(); + var cur_path: []const u8 = self_exe_path; + while (Dir.path.dirname(cur_path)) |dirname| : (cur_path = dirname) { + var base_dir = cwd.openDir(io, dirname, .{}) catch continue; + defer base_dir.close(io); + + const sub_directory = testZigInstallPrefix(io, base_dir) orelse continue; + const p = try Dir.path.join(allocator, &.{ dirname, sub_directory.path.? }); + defer allocator.free(p); + + const resolved = try resolvePath(allocator, cwd_path, &.{p}); + return .{ + .handle = sub_directory.handle, + .path = if (resolved.len == 0) null else resolved, + }; + } + return error.FileNotFound; +} + +/// Returns the sub_path that worked, or `null` if none did. +/// The path of the returned Directory is relative to `base`. +/// The handle of the returned Directory is open. +fn testZigInstallPrefix(io: Io, base_dir: Dir) ?Cache.Directory { + const test_index_file = "std" ++ Dir.path.sep_str ++ "std.zig"; + + zig_dir: { + // Try lib/zig/std/std.zig + const lib_zig = "lib" ++ Dir.path.sep_str ++ "zig"; + var test_zig_dir = base_dir.openDir(io, lib_zig, .{}) catch break :zig_dir; + const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { + test_zig_dir.close(io); + break :zig_dir; + }; + file.close(io); + return .{ .handle = test_zig_dir, .path = lib_zig }; + } + + // Try lib/std/std.zig + var test_zig_dir = base_dir.openDir(io, "lib", .{}) catch return null; + const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { + test_zig_dir.close(io); + return null; + }; + file.close(io); + return .{ .handle = test_zig_dir, .path = "lib" }; +} + +pub fn resolveGlobalCacheDir(arena: Allocator, environ_map: *const std.process.Environ.Map) ![]const u8 { + if (EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map)) |value| return value; + + const app_name = "zig"; + + switch (builtin.os.tag) { + .wasi => @compileError("on WASI the global cache dir must be resolved with preopens"), + .windows => { + const local_app_data_dir = EnvVar.LOCALAPPDATA.get(environ_map) orelse + return error.AppDataDirUnavailable; + return Dir.path.join(arena, &.{ local_app_data_dir, app_name }); + }, + else => { + if (EnvVar.XDG_CACHE_HOME.get(environ_map)) |cache_root| { + if (cache_root.len > 0) { + return Dir.path.join(arena, &.{ cache_root, app_name }); + } + } + if (EnvVar.HOME.get(environ_map)) |home| { + if (home.len > 0) { + return Dir.path.join(arena, &.{ home, ".cache", app_name }); + } + } + return error.AppDataDirUnavailable; + }, + } +} + +/// Searches upwards from `cwd` for a directory containing a `build.zig` file. +/// If such a directory is found, returns the path to it joined to the `.zig_cache` name. +/// Otherwise, returns `null`, indicating no suitable local cache location. +pub fn resolveSuitableLocalCacheDir(arena: Allocator, io: Io, cwd: []const u8) Allocator.Error!?[]u8 { + var cur_dir = cwd; + while (true) { + const joined = try Dir.path.join(arena, &.{ cur_dir, build_zig_basename }); + if (Dir.cwd().access(io, joined, .{})) |_| { + return try Dir.path.join(arena, &.{ cur_dir, default_local_zig_cache_basename }); + } else |err| switch (err) { + error.FileNotFound => { + cur_dir = Dir.path.dirname(cur_dir) orelse return null; + continue; + }, + else => return null, + } + } +} + +/// Similar to `Dir.path.resolve`, but converts to a cwd-relative path, or, if that would +/// start with a relative up-dir (".."), an absolute path based on the cwd. Also, the cwd +/// returns the empty string ("") instead of ".". +pub fn resolvePath( + gpa: Allocator, + /// The return value of `getResolvedCwd`. + /// Passed as an argument to avoid pointlessly repeating the call. + cwd_resolved: []const u8, + paths: []const []const u8, +) Allocator.Error![]u8 { + if (builtin.target.os.tag == .wasi) { + assert(mem.eql(u8, cwd_resolved, "")); + const res = try Dir.path.resolve(gpa, paths); + if (mem.eql(u8, res, ".")) { + gpa.free(res); + return ""; + } + return res; + } + + // Heuristic for a fast path: if no component is absolute and ".." never appears, we just need to resolve `paths`. + for (paths) |p| { + if (Dir.path.isAbsolute(p)) break; // absolute path + if (mem.indexOf(u8, p, "..") != null) break; // may contain up-dir + } else { + // no absolute path, no "..". + const res = try Dir.path.resolve(gpa, paths); + if (mem.eql(u8, res, ".")) { + gpa.free(res); + return ""; + } + assert(!Dir.path.isAbsolute(res)); + assert(!isUpDir(res)); + return res; + } + + // The fast path failed; resolve the whole thing. + // Optimization: `paths` often has just one element. + const path_resolved = switch (paths.len) { + 0 => unreachable, + 1 => try Dir.path.resolve(gpa, &.{ cwd_resolved, paths[0] }), + else => r: { + const all_paths = try gpa.alloc([]const u8, paths.len + 1); + defer gpa.free(all_paths); + all_paths[0] = cwd_resolved; + @memcpy(all_paths[1..], paths); + break :r try Dir.path.resolve(gpa, all_paths); + }, + }; + errdefer gpa.free(path_resolved); + + assert(Dir.path.isAbsolute(path_resolved)); + assert(Dir.path.isAbsolute(cwd_resolved)); + + if (!mem.startsWith(u8, path_resolved, cwd_resolved)) return path_resolved; // not in cwd + if (path_resolved.len == cwd_resolved.len) { + // equal to cwd + gpa.free(path_resolved); + return ""; + } + if (path_resolved[cwd_resolved.len] != Dir.path.sep) return path_resolved; // not in cwd (last component differs) + + // in cwd; extract sub path + const sub_path = try gpa.dupe(u8, path_resolved[cwd_resolved.len + 1 ..]); + gpa.free(path_resolved); + return sub_path; +} + +pub fn isUpDir(p: []const u8) bool { + return mem.startsWith(u8, p, "..") and (p.len == 2 or p[2] == Dir.path.sep); +} + +pub const BuildExeSubprocessOptions = struct { + argv: []const []const u8, + cache_root: Cache.Directory, + root_name: []const u8, + + environ_map: ?*std.process.Environ.Map = null, + cache_manifest: ?*Cache.Manifest = null, + arch_os_abi: ?[]const u8 = null, + cpu_features: ?[]const u8 = null, + progress_node: std.Progress.Node = .none, + skip_log_cmdline_on_compile_errors: bool = false, +}; + +pub const BuildExeSubprocessError = error{ + /// Error message has been logged. + AlreadyReported, + /// Error message has been logged, and source files added to the `Cache.Manifest`. + FailedButCacheIntact, +} || Io.Cancelable || Allocator.Error; + +pub const BuildExeSubprocessResult = struct { + received_fs_inputs: bool, + cache_hit: bool, + path: Cache.Path, +}; + +/// Assumes `argv` has `--listen=-` in it and the child process is `zig build-exe`. +/// +/// Result path is allocated via gpa. +pub fn buildExeSubprocess( + gpa: Allocator, + io: Io, + options: BuildExeSubprocessOptions, +) BuildExeSubprocessError!BuildExeSubprocessResult { + const cmd: SubprocessCommand = .{ .argv = options.argv }; + + var child = std.process.spawn(io, .{ + .argv = options.argv, + .environ_map = options.environ_map, + .stdin = .pipe, + .stdout = .pipe, + .stderr = .pipe, + .progress_node = options.progress_node, + }) catch |err| { + log.err("spawning command {t}: {f}", .{ err, cmd }); + return error.AlreadyReported; + }; + defer child.kill(io); + + var stderr_task = io.concurrent(readStreamAlloc, .{ gpa, io, child.stderr.?, .unlimited }) catch + @panic("TODO use multireader instead"); + defer if (stderr_task.cancel(io)) |slice| gpa.free(slice) else |_| {}; + + var stdout_buffer: [512]u8 = undefined; + var stdout_reader: Io.File.Reader = .initStreaming(child.stdout.?, io, &stdout_buffer); + const stdout = &stdout_reader.interface; + + { + var w = child.stdin.?.writer(io, &.{}); + w.interface.writeStruct(Client.Message.Header{ .tag = .update, .bytes_len = 0 }, .little) catch |err| switch (err) { + error.WriteFailed => { + log.err("{t} writing to command: {f}", .{ w.err.?, cmd }); + return error.AlreadyReported; + }, + }; + w.interface.writeStruct(Client.Message.Header{ .tag = .exit, .bytes_len = 0 }, .little) catch |err| switch (err) { + error.WriteFailed => { + log.err("{t} writing to command: {f}", .{ w.err.?, cmd }); + return error.AlreadyReported; + }, + }; + } + + const Header = Server.Message.Header; + + var result: ?Cache.Path = null; + defer if (result) |r| gpa.free(r.sub_path); + + var result_error_bundle: ErrorBundle = .empty; + defer result_error_bundle.deinit(gpa); + + var body_buffer: std.ArrayList(u8) = .empty; + defer body_buffer.deinit(gpa); + + var received_fs_inputs = false; + var cache_hit = false; + + while (true) { + const header = stdout.takeStruct(Header, .little) catch |err| switch (err) { + error.ReadFailed => { + log.err("{t} reading from command: {f}", .{ stdout_reader.err.?, cmd }); + return error.AlreadyReported; + }, + error.EndOfStream => break, + }; + body_buffer.clearRetainingCapacity(); + stdout.appendExact(gpa, &body_buffer, header.bytes_len) catch |err| switch (err) { + error.ReadFailed => { + log.err("{t} reading from command: {f}", .{ stdout_reader.err.?, cmd }); + return error.AlreadyReported; + }, + error.OutOfMemory => |e| return e, + error.EndOfStream => { + log.err("unexpected end of stream from command: {f}", .{cmd}); + return error.AlreadyReported; + }, + }; + const body = body_buffer.items; + + switch (header.tag) { + .zig_version => { + if (!mem.eql(u8, builtin.zig_version_string, body)) { + log.err("zig protocol version mismatch from command: {f}", .{cmd}); + return error.AlreadyReported; + } + }, + .error_bundle => { + result_error_bundle.deinit(gpa); + result_error_bundle = Server.allocErrorBundle(gpa, body) catch |err| switch (err) { + error.EndOfStream => break, + else => |e| return e, + }; + }, + .emit_digest => { + const EmitDigest = Server.Message.EmitDigest; + const ebp_hdr: *align(1) const EmitDigest = @ptrCast(body); + cache_hit = ebp_hdr.flags.cache_hit; + const digest = body[@sizeOf(EmitDigest)..][0..Cache.bin_digest_len]; + if (result) |r| gpa.free(r.sub_path); + result = .{ + .root_dir = options.cache_root, + .sub_path = try Dir.path.join(gpa, &.{ "o", &Cache.binToHex(digest.*) }), + }; + }, + .file_system_inputs => if (options.cache_manifest) |man| { + received_fs_inputs = true; + var it = mem.splitScalar(u8, body, 0); + while (it.next()) |prefixed_path| { + const prefix: Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1); + const sub_path = try gpa.dupe(u8, prefixed_path[1..]); + var keep = false; + defer if (!keep) gpa.free(sub_path); + keep = man.addPrefixedPathPost(.{ + .prefix = @intFromEnum(prefix), + .sub_path = sub_path, + }) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| { + log.err("adding {t} {s} to cache failed: {t}", .{ prefix, sub_path, e }); + return error.AlreadyReported; + }, + }; + } + }, + else => {}, // ignore other messages + } + } + + const stderr_contents = stderr_task.await(io) catch |err| switch (err) { + error.Canceled, error.OutOfMemory => |e| return e, + else => |e| c: { + log.warn("{t} reading stderr from command: {f}", .{ e, cmd }); + break :c ""; + }, + }; + if (stderr_contents.len > 0) + log.warn("unexpected stderr from {s} command:\n{s}", .{ options.argv[0], stderr_contents }); + + // Send EOF to stdin. + child.stdin.?.close(io); + child.stdin = null; + + const term = child.wait(io) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + log.err("{t} waiting for command: {f}", .{ e, cmd }); + return error.AlreadyReported; + }, + }; + + if (!term.success()) { + log.err("command {f}: {f}", .{ term, cmd }); + if (received_fs_inputs) return error.FailedButCacheIntact; + return error.AlreadyReported; + } + + if (result_error_bundle.errorMessageCount() > 0) { + result_error_bundle.renderToStderr(io, .{}, .auto) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + log.err("failed rendering error bundle: {t}", .{e}); + return error.AlreadyReported; + }, + }; + if (!options.skip_log_cmdline_on_compile_errors) log.err("command reported {d} compilation errors: {f}", .{ + result_error_bundle.errorMessageCount(), cmd, + }); + if (received_fs_inputs) return error.FailedButCacheIntact; + return error.AlreadyReported; + } + + const base_path = result orelse { + log.err("command failed to report result: {f}", .{cmd}); + return error.AlreadyReported; + }; + const parsed_target = system.resolveTargetQuery(io, std.Build.parseTargetQuery(.{ + .arch_os_abi = options.arch_os_abi orelse "native", + .cpu_features = options.cpu_features, + }) catch unreachable) catch unreachable; + const bin_name = try binNameAlloc(gpa, .{ + .root_name = options.root_name, + .cpu_arch = parsed_target.cpu.arch, + .os_tag = parsed_target.os.tag, + .ofmt = parsed_target.ofmt, + .abi = parsed_target.abi, + .output_mode = .Exe, + }); + defer gpa.free(bin_name); + return .{ + .received_fs_inputs = received_fs_inputs, + .cache_hit = cache_hit, + .path = try base_path.join(gpa, bin_name), + }; +} + +fn readStreamAlloc(gpa: Allocator, io: Io, file: Io.File, limit: Io.Limit) ![]u8 { + var file_reader: Io.File.Reader = .initStreaming(file, io, &.{}); + return file_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) { + error.ReadFailed => return file_reader.err.?, + else => |e| return e, + }; +} + test { _ = Ast; _ = AstRlAnnotate; diff --git a/lib/std/zig/LibCDirs.zig b/lib/std/zig/LibCDirs.zig index 04c61af0bedab6728309db7c00c975ef4229a57d..84002241bf9ea4fb1dbf5f8702a4723c05a01694 100644 --- a/lib/std/zig/LibCDirs.zig +++ b/lib/std/zig/LibCDirs.zig @@ -5,6 +5,7 @@ const std = @import("../std.zig"); const Io = std.Io; const LibCInstallation = std.zig.LibCInstallation; const Allocator = std.mem.Allocator; +const Path = std.Build.Cache.Path; libc_include_dir_list: []const []const u8, libc_installation: ?*const LibCInstallation, @@ -23,7 +24,7 @@ pub const DarwinSdkLayout = enum { pub fn detect( arena: Allocator, io: Io, - zig_lib_dir: []const u8, + zig_lib_dir: Path, target: *const std.Target, is_native_abi: bool, link_libc: bool, @@ -166,20 +167,12 @@ fn detectFromInstallation(arena: Allocator, target: *const std.Target, lci: *con }; } -pub fn detectFromBuilding( - arena: Allocator, - zig_lib_dir: []const u8, - target: *const std.Target, -) !LibCDirs { +pub fn detectFromBuilding(arena: Allocator, zig_lib_dir: Path, target: *const std.Target) !LibCDirs { const s = std.fs.path.sep_str; if (target.os.tag.isDarwin()) { const list = try arena.alloc([]const u8, 1); - list[0] = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-darwin-any", - .{zig_lib_dir}, - ); + list[0] = try arena.print("{f}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-darwin-any", .{zig_lib_dir}); return .{ .libc_include_dir_list = list, .libc_installation = null, @@ -212,27 +205,19 @@ pub fn detectFromBuilding( std.zig.target.netbsdAbiNameHeaders(target.abi) else @tagName(target.abi); - const arch_include_dir = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-{s}", - .{ zig_lib_dir, arch_name, os_name, abi_name }, - ); - const generic_include_dir = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "generic-{s}", - .{ zig_lib_dir, generic_name }, - ); + const arch_include_dir = try arena.print("{f}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-{s}", .{ + zig_lib_dir, arch_name, os_name, abi_name, + }); + const generic_include_dir = try arena.print("{f}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "generic-{s}", .{ + zig_lib_dir, generic_name, + }); const generic_arch_name = std.zig.target.osArchName(target); - const arch_os_include_dir = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-any", - .{ zig_lib_dir, generic_arch_name, os_name }, - ); - const generic_os_include_dir = try std.fmt.allocPrint( - arena, - "{s}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{s}-any", - .{ zig_lib_dir, os_name }, - ); + const arch_os_include_dir = try arena.print("{f}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "{s}-{s}-any", .{ + zig_lib_dir, generic_arch_name, os_name, + }); + const generic_os_include_dir = try arena.print("{f}" ++ s ++ "libc" ++ s ++ "include" ++ s ++ "any-{s}-any", .{ + zig_lib_dir, os_name, + }); const list = try arena.alloc([]const u8, 4); list[0] = arch_include_dir; diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 96679d4fc315459d963f694298bfd8df549a7502..cf43cb0af2822cf416868dd1eba76bcb06b791a7 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -264,7 +264,7 @@ pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { try s.out.flush(); } -pub fn allocErrorBundle(gpa: std.mem.Allocator, body: []const u8) error{ OutOfMemory, EndOfStream }!std.zig.ErrorBundle { +pub fn allocErrorBundle(gpa: Allocator, body: []const u8) error{ OutOfMemory, EndOfStream }!std.zig.ErrorBundle { var r: Reader = .fixed(body); const hdr = r.takeStruct(OutMessage.ErrorBundle, .little) catch |err| switch (err) { error.EndOfStream => |e| return e, diff --git a/src/Builtin.zig b/src/Builtin.zig index b45841d152aab73e281e3e7865b106a8539e6ce2..2117aae11b1ef834cfdb594dfc1f7ea2711e5406 100644 --- a/src/Builtin.zig +++ b/src/Builtin.zig @@ -370,7 +370,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const Cache = std.Build.Cache; const build_options = @import("build_options"); -const Module = @import("Package/Module.zig"); +const Module = @import("Module.zig"); const assert = std.debug.assert; const AstGen = std.zig.AstGen; const File = @import("Zcu.zig").File; diff --git a/src/Compilation.zig b/src/Compilation.zig index 70ed4c7f9a55d8b1695520be78c264925f39928b..1e9b81d493d62ceea21fb74b3786677b119eb9b4 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -16,8 +16,6 @@ const fatal = std.process.fatal; const Value = @import("Value.zig"); const Type = @import("Type.zig"); const target_util = @import("target.zig"); -const Package = @import("Package.zig"); -const introspect = @import("introspect.zig"); const link = @import("link.zig"); const tracy = @import("tracy.zig"); const trace = tracy.trace; @@ -44,6 +42,7 @@ const Air = @import("Air.zig"); const Builtin = @import("Builtin.zig"); const LlvmObject = @import("codegen/llvm.zig").Object; const dev = @import("dev.zig"); +const Module = @import("Module.zig"); pub const Config = @import("Compilation/Config.zig"); @@ -64,7 +63,7 @@ cache_use: CacheUse, /// All compilations have a root module because this is where some important /// settings are stored, such as target and optimization mode. This module /// might not have any .zig code associated with it, however. -root_mod: *Package.Module, +root_mod: *Module, /// User-specified settings that have all the defaults resolved into concrete values. config: Config, @@ -190,7 +189,7 @@ parent_whole_cache: ?ParentWholeCache, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, /// Owned by the caller of `Compilation.create`. -dirs: Directories, +dirs: std.zig.Directories, libc_include_dir_list: []const []const u8, libc_framework_dir_list: []const []const u8, rc_includes: std.zig.RcIncludes, @@ -431,7 +430,7 @@ pub const Path = struct { } /// Given a `Path`, returns the directory handle and sub path to be used to open the path. - pub fn openInfo(p: Path, dirs: Directories) struct { Io.Dir, []const u8 } { + pub fn openInfo(p: Path, dirs: std.zig.Directories) struct { Io.Dir, []const u8 } { const dir = switch (p.root) { .none => { const cwd_sub_path = absToCwdRelative(p.sub_path, dirs.cwd); @@ -491,8 +490,8 @@ pub const Path = struct { /// From an unresolved path (which can be made of multiple not-yet-joined strings), construct a /// canonical `Path`. - pub fn fromUnresolved(gpa: Allocator, dirs: Compilation.Directories, unresolved_parts: []const []const u8) Allocator.Error!Path { - const resolved = try introspect.resolvePath(gpa, dirs.cwd, unresolved_parts); + pub fn fromUnresolved(gpa: Allocator, dirs: std.zig.Directories, unresolved_parts: []const []const u8) Allocator.Error!Path { + const resolved = try std.zig.resolvePath(gpa, dirs.cwd, unresolved_parts); errdefer gpa.free(resolved); // If, for instance, `dirs.local_cache.path` is within the lib dir, it must take priority, @@ -566,7 +565,7 @@ pub const Path = struct { /// `.global_cache` could still end up returning a `Path` with `Path.root == .zig_lib`. pub fn fromRoot( gpa: Allocator, - dirs: Compilation.Directories, + dirs: std.zig.Directories, root: Path.Root, sub_path: []const u8, ) Allocator.Error!Path { @@ -589,7 +588,7 @@ pub const Path = struct { pub fn join( p: Path, gpa: Allocator, - dirs: Compilation.Directories, + dirs: std.zig.Directories, sub_path: []const u8, ) Allocator.Error!Path { // Currently, this just wraps `fromUnresolved` for simplicity. A more efficient impl is @@ -610,7 +609,7 @@ pub const Path = struct { pub fn upJoin( p: Path, gpa: Allocator, - dirs: Compilation.Directories, + dirs: std.zig.Directories, sub_path: []const u8, ) Allocator.Error!Path { return .fromUnresolved(gpa, dirs, &.{ @@ -626,7 +625,7 @@ pub const Path = struct { }); } - pub fn toCachePath(p: Path, dirs: Directories) Cache.Path { + pub fn toCachePath(p: Path, dirs: std.zig.Directories) Cache.Path { const root_dir: Cache.Directory = switch (p.root) { .zig_lib => dirs.zig_lib, .global_cache => dirs.global_cache, @@ -649,7 +648,7 @@ pub const Path = struct { /// This should not be used for most of the compiler pipeline, but is useful when emitting /// paths from the compilation (e.g. in debug info), because they will not depend on the cwd. /// The returned path is owned by the caller and allocated into `gpa`. - pub fn toAbsolute(p: Path, dirs: Directories, gpa: Allocator) Allocator.Error![]u8 { + pub fn toAbsolute(p: Path, dirs: std.zig.Directories, gpa: Allocator) Allocator.Error![]u8 { const root_path: []const u8 = switch (p.root) { .zig_lib => dirs.zig_lib.path orelse "", .global_cache => dirs.global_cache.path orelse "", @@ -680,7 +679,7 @@ pub const Path = struct { /// Returns whether this `Path` is illegal to have as a user-imported `Zcu.File` (including /// as the root of a module). Such paths exist in directories which the Zig compiler treats /// specially, like 'global_cache/b/', which stores 'builtin.zig' files. - pub fn isIllegalZigImport(p: Path, gpa: Allocator, dirs: Directories) Allocator.Error!bool { + pub fn isIllegalZigImport(p: Path, gpa: Allocator, dirs: std.zig.Directories) Allocator.Error!bool { const zig_builtin_dir: Path = try .fromRoot(gpa, dirs, .global_cache, "b"); defer zig_builtin_dir.deinit(gpa); return switch (p.isNested(zig_builtin_dir)) { @@ -690,149 +689,6 @@ pub const Path = struct { } }; -pub const Directories = struct { - /// The string returned by `introspect.getResolvedCwd`. This is typically an absolute path, - /// but on WASI is the empty string "" instead, because WASI does not have absolute paths. - cwd: []const u8, - /// The Zig 'lib' directory. - /// `zig_lib.path` is resolved (`introspect.resolvePath`) or `null` for cwd. - /// Guaranteed to be a different path from `global_cache` and `local_cache`. - zig_lib: Cache.Directory, - /// The global Zig cache directory. - /// `global_cache.path` is resolved (`introspect.resolvePath`) or `null` for cwd. - global_cache: Cache.Directory, - /// The local Zig cache directory. - /// `local_cache.path` is resolved (`introspect.resolvePath`) or `null` for cwd. - /// This may be the same as `global_cache`. - local_cache: Cache.Directory, - - pub fn deinit(dirs: *Directories, io: Io) void { - // The local and global caches could be the same. - const close_local = dirs.local_cache.handle.handle != dirs.global_cache.handle.handle; - - dirs.global_cache.handle.close(io); - if (close_local) dirs.local_cache.handle.close(io); - dirs.zig_lib.handle.close(io); - } - - /// Returns a `Directories` where `local_cache` is replaced with `global_cache`, intended for - /// use by sub-compilations (e.g. compiler_rt). Do not `deinit` the returned `Directories`; it - /// shares handles with `dirs`. - pub fn withoutLocalCache(dirs: Directories) Directories { - return .{ - .cwd = dirs.cwd, - .zig_lib = dirs.zig_lib, - .global_cache = dirs.global_cache, - .local_cache = dirs.global_cache, - }; - } - - /// Uses `std.process.fatal` on error conditions. - pub fn init( - arena: Allocator, - io: Io, - override_zig_lib: ?[]const u8, - override_global_cache: ?[]const u8, - local_cache_strat: union(enum) { - override: []const u8, - search, - global, - }, - preopens: std.process.Preopens, - self_exe_path: switch (builtin.target.os.tag) { - .wasi => void, - else => []const u8, - }, - environ_map: *const std.process.Environ.Map, - cwd: []const u8, - ) Directories { - const wasi = builtin.target.os.tag == .wasi; - - const zig_lib: Cache.Directory = d: { - if (override_zig_lib) |path| break :d openUnresolved(arena, io, cwd, path, .@"zig lib"); - if (wasi) break :d getPreopen(preopens, "/lib"); - break :d introspect.findZigLibDirFromSelfExe(arena, io, cwd, self_exe_path) catch |err| { - fatal("unable to find zig installation directory '{s}': {t}", .{ self_exe_path, err }); - }; - }; - - const global_cache: Cache.Directory = d: { - if (override_global_cache) |path| break :d openUnresolved(arena, io, cwd, path, .@"global cache"); - if (wasi) break :d getPreopen(preopens, "/cache"); - const path = introspect.resolveGlobalCacheDir(arena, environ_map) catch |err| { - fatal("unable to resolve zig cache directory: {t}", .{err}); - }; - break :d openUnresolved(arena, io, cwd, path, .@"global cache"); - }; - - const local_cache: Cache.Directory = switch (local_cache_strat) { - .override => |path| openUnresolved(arena, io, cwd, path, .@"local cache"), - .search => d: { - const maybe_path = introspect.resolveSuitableLocalCacheDir(arena, io, cwd) catch |err| { - fatal("unable to resolve zig cache directory: {t}", .{err}); - }; - const path = maybe_path orelse break :d global_cache; - break :d openUnresolved(arena, io, cwd, path, .@"local cache"); - }, - .global => global_cache, - }; - - if (std.mem.eql(u8, zig_lib.path orelse "", global_cache.path orelse "")) { - fatal("zig lib directory '{f}' cannot be equal to global cache directory '{f}'", .{ zig_lib, global_cache }); - } - if (std.mem.eql(u8, zig_lib.path orelse "", local_cache.path orelse "")) { - fatal("zig lib directory '{f}' cannot be equal to local cache directory '{f}'", .{ zig_lib, local_cache }); - } - - return .{ - .cwd = cwd, - .zig_lib = zig_lib, - .global_cache = global_cache, - .local_cache = local_cache, - }; - } - fn getPreopen(preopens: std.process.Preopens, name: []const u8) Cache.Directory { - return .{ - .path = if (std.mem.eql(u8, name, ".")) null else name, - .handle = switch (preopens.get(name) orelse fatal("preopen not found: '{s}'", .{name})) { - .file => fatal("preopen {s} is not a directory", .{name}), - .dir => |d| d, - }, - }; - } - fn openUnresolved( - arena: Allocator, - io: Io, - cwd: []const u8, - unresolved_path: []const u8, - thing: enum { @"zig lib", @"global cache", @"local cache" }, - ) Cache.Directory { - const path = introspect.resolvePath(arena, cwd, &.{unresolved_path}) catch |err| { - fatal("unable to resolve {s} directory: {s}", .{ @tagName(thing), @errorName(err) }); - }; - const nonempty_path = if (path.len == 0) "." else path; - const handle_or_err = switch (thing) { - .@"zig lib" => Io.Dir.cwd().openDir(io, nonempty_path, .{}), - .@"global cache", .@"local cache" => Io.Dir.cwd().createDirPathOpen(io, nonempty_path, .{}), - }; - return .{ - .path = if (path.len == 0) null else path, - .handle = handle_or_err catch |err| { - const extra_str: []const u8 = e: { - if (thing == .@"global cache") switch (err) { - error.AccessDenied, error.ReadOnlyFileSystem => break :e "\n" ++ - "If this location is not writable then consider specifying an alternative with " ++ - "the ZIG_GLOBAL_CACHE_DIR environment variable or the --global-cache-dir option.", - else => {}, - }; - break :e ""; - }; - fatal("unable to open {s} directory '{s}': {s}{s}", .{ @tagName(thing), nonempty_path, @errorName(err), extra_str }); - }, - }; - } -}; - /// This small wrapper function just checks whether debug extensions are enabled before checking /// `comp.debug_incremental`. It is inline so that comptime-known `false` propagates to the caller, /// preventing debugging features from making it into release builds of the compiler. @@ -910,7 +766,7 @@ pub const CrtFile = struct { /// For passing to a C compiler. pub const CSourceFile = struct { /// Many C compiler flags are determined by settings contained in the owning Module. - owner: *Package.Module, + owner: *Module, src_path: []const u8, extra_flags: []const []const u8 = &.{}, /// Same as extra_flags except they are not added to the Cache hash. @@ -922,7 +778,7 @@ pub const CSourceFile = struct { /// For passing to resinator. pub const RcSourceFile = struct { - owner: *Package.Module, + owner: *Module, src_path: []const u8, extra_flags: []const []const u8 = &.{}, }; @@ -1360,7 +1216,7 @@ pub const MiscError = struct { }; pub const cache_helpers = struct { - pub fn addModule(hh: *Cache.HashHelper, mod: *const Package.Module) void { + pub fn addModule(hh: *Cache.HashHelper, mod: *const Module) void { addResolvedTarget(hh, mod.resolved_target); hh.add(mod.optimize_mode); hh.add(mod.code_model); @@ -1383,7 +1239,7 @@ pub const cache_helpers = struct { pub fn addResolvedTarget( hh: *Cache.HashHelper, - resolved_target: Package.Module.ResolvedTarget, + resolved_target: Module.ResolvedTarget, ) void { const target = &resolved_target.result; hh.add(target.cpu.arch); @@ -1549,23 +1405,23 @@ const CacheUse = union(CacheMode) { }; pub const CreateOptions = struct { - dirs: Directories, + dirs: std.zig.Directories, thread_limit: usize, self_exe_path: ?[]const u8 = null, /// Options that have been resolved by calling `resolveDefaults`. config: Compilation.Config, - root_mod: *Package.Module, + root_mod: *Module, /// Normally, `main_mod` and `root_mod` are the same. The exception is `zig /// test`, in which `root_mod` is the test runner, and `main_mod` is the /// user's source file which has the tests. - main_mod: ?*Package.Module = null, + main_mod: ?*Module = null, /// This is provided so that the API user has a chance to tweak the /// per-module settings of the standard library. /// When this is null, a default configuration of the std lib is created /// based on the settings of root_mod. - std_mod: ?*Package.Module = null, + std_mod: ?*Module = null, root_name: []const u8, sysroot: ?[]const u8 = null, cache_mode: CacheMode, @@ -1873,7 +1729,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic, const libc_dirs = std.zig.LibCDirs.detect( arena, io, - options.dirs.zig_lib.path.?, + .{ .root_dir = options.dirs.zig_lib }, target, options.root_mod.resolved_target.is_native_abi, link_libc, @@ -1907,7 +1763,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic, if (compiler_rt_strat == .zcu) { // For objects, this mechanism relies on essentially `_ = @import("compiler-rt");` // injected into the object. - const compiler_rt_mod = Package.Module.create(arena, .{ + const compiler_rt_mod = Module.create(arena, .{ .paths = .{ .root = .zig_lib_root, .root_src_path = "compiler_rt.zig", @@ -1969,7 +1825,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic, }; if (ubsan_rt_strat == .zcu) { - const ubsan_rt_mod = Package.Module.create(arena, .{ + const ubsan_rt_mod = Module.create(arena, .{ .paths = .{ .root = .zig_lib_root, .root_src_path = "ubsan_rt.zig", @@ -2010,7 +1866,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic, }; if (zigc_strat == .zcu) { - const zigc_mod = Package.Module.create(arena, .{ + const zigc_mod = Module.create(arena, .{ .paths = .{ .root = .zig_lib_root, .root_src_path = "c.zig", @@ -2140,7 +1996,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic, .path = try options.dirs.global_cache.join(arena, &.{zir_sub_dir}), }; - const std_mod = options.std_mod orelse Package.Module.create(arena, .{ + const std_mod = options.std_mod orelse Module.create(arena, .{ .paths = .{ .root = try .fromRoot(arena, options.dirs, .zig_lib, "std"), .root_src_path = "std.zig", @@ -2910,11 +2766,9 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE .file_open, .file_stat, .file_read, .file_hash => |op| { const pp = man.files.keys()[op.file_index].prefixed_path; const prefix = man.cache.prefixes()[pp.prefix]; - return comp.setMiscFailure( - .check_whole_cache, - "failed to check cache: '{f}{s}' {t} {t}", - .{ prefix, pp.sub_path, man.diagnostic, op.err }, - ); + return comp.setMiscFailure(.check_whole_cache, "failed to check cache: {f}{s} {t} {t}", .{ + prefix, pp.sub_path, man.diagnostic, op.err, + }); }, }, error.OutOfMemory, error.Canceled => |e| return e, @@ -4785,7 +4639,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void { var buffer: [1024]u8 = undefined; var tar_file_writer = tar_file.writer(io, &buffer); - var seen_table: std.array_hash_map.Auto(*Package.Module, []const u8) = .empty; + var seen_table: std.array_hash_map.Auto(*Module, []const u8) = .empty; defer seen_table.deinit(comp.gpa); try seen_table.put(comp.gpa, zcu.main_mod, comp.root_name); @@ -4812,7 +4666,7 @@ fn docsCopyFallible(comp: *Compilation) anyerror!void { fn docsCopyModule( comp: *Compilation, - module: *Package.Module, + module: *Module, name: []const u8, tar_file_writer: *Io.File.Writer, ) !void { @@ -4892,7 +4746,7 @@ fn workerDocsWasmFallible(comp: *Compilation, prog_node: std.Progress.Node) SubU const optimize_mode = std.lang.OptimizeMode.ReleaseSmall; const output_mode = std.lang.OutputMode.Exe; - const resolved_target: Package.Module.ResolvedTarget = .{ + const resolved_target: Module.ResolvedTarget = .{ .result = std.zig.system.resolveTargetQuery(io, .{ .cpu_arch = .wasm32, .os_tag = .freestanding, @@ -4932,7 +4786,7 @@ fn workerDocsWasmFallible(comp: *Compilation, prog_node: std.Progress.Node) SubU const dirs = comp.dirs.withoutLocalCache(); - const root_mod = Package.Module.create(arena, .{ + const root_mod = Module.create(arena, .{ .paths = .{ .root = try .fromRoot(arena, dirs, .zig_lib, "docs/wasm"), .root_src_path = src_basename, @@ -4949,7 +4803,7 @@ fn workerDocsWasmFallible(comp: *Compilation, prog_node: std.Progress.Node) SubU comp.lockAndSetMiscFailure(.docs_wasm, "sub-compilation of docs_wasm failed: failed to create root module: {t}", .{err}); return error.AlreadyReported; }; - const walk_mod = Package.Module.create(arena, .{ + const walk_mod = Module.create(arena, .{ .paths = .{ .root = try .fromRoot(arena, dirs, .zig_lib, "docs/wasm"), .root_src_path = "Walk.zig", @@ -5034,7 +4888,7 @@ fn workerDocsWasmFallible(comp: *Compilation, prog_node: std.Progress.Node) SubU pub fn obtainCObjectCacheManifest( comp: *const Compilation, - owner_mod: *Package.Module, + owner_mod: *Module, ) Cache.Manifest { var man = comp.cache_parent.obtain(); @@ -5082,7 +4936,7 @@ pub fn translateC( ext: FileExt, source_path: []const u8, translated_basename: []const u8, - owner_mod: *Package.Module, + owner_mod: *Module, prog_node: std.Progress.Node, environ_map: *const std.process.Environ.Map, ) !TranslateCResult { @@ -6238,7 +6092,7 @@ fn addCommonCCArgs( argv: *std.array_list.Managed([]const u8), ext: FileExt, out_dep_path: ?[]const u8, - mod: *Package.Module, + mod: *Module, c_frontend: Config.CFrontend, ) !void { const target = &mod.resolved_target.result; @@ -6592,7 +6446,7 @@ pub fn addCCArgs( argv: *std.array_list.Managed([]const u8), ext: FileExt, out_dep_path: ?[]const u8, - mod: *Package.Module, + mod: *Module, ) !void { const target = &mod.resolved_target.result; @@ -7383,7 +7237,7 @@ fn buildOutputFromZig( return error.AlreadyReported; }; - const root_mod = Package.Module.create(arena, .{ + const root_mod = Module.create(arena, .{ .paths = .{ .root = .zig_lib_root, .root_src_path = src_basename, @@ -7530,7 +7384,7 @@ pub fn build_crt_file( comp.lockAndSetMiscFailure(misc_task_tag, "sub-compilation of {t} failed: failed to resolve compilation config: {t}", .{ misc_task_tag, err }); return error.AlreadyReported; }; - const root_mod = Package.Module.create(arena, .{ + const root_mod = Module.create(arena, .{ .paths = .{ .root = .zig_lib_root, .root_src_path = "", diff --git a/src/Compilation/Config.zig b/src/Compilation/Config.zig index ce180f9ead8c4f545f4e5e8ea2a6ecc3c7b33179..9dd28b3e37b3cfbfa247dde25368b5b59ed530e2 100644 --- a/src/Compilation/Config.zig +++ b/src/Compilation/Config.zig @@ -577,7 +577,7 @@ pub fn resolve(options: Options) ResolveError!Config { } const std = @import("std"); -const Module = @import("../Package.zig").Module; +const Module = @import("../Module.zig"); const Config = @This(); const target_util = @import("../target.zig"); const build_options = @import("build_options"); diff --git a/src/Module.zig b/src/Module.zig new file mode 100644 index 0000000000000000000000000000000000000000..b0d212070bd298f6d4980eddb4fe32a9fd43b363 --- /dev/null +++ b/src/Module.zig @@ -0,0 +1,523 @@ +//! Corresponds to something that Zig source code can `@import`. +const Module = @This(); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Cache = std.Build.Cache; +const assert = std.debug.assert; + +const target_util = @import("target.zig"); +const Builtin = @import("Builtin.zig"); +const Compilation = @import("Compilation.zig"); +const File = @import("Zcu.zig").File; + +/// The root directory of the module. Only files inside this directory can be imported. +root: Compilation.Path, +/// Path to the root source file of this module. Relative to `root`. May contain path separators. +root_src_path: []const u8, +/// Name used in compile errors. Looks like "root.foo.bar". +fully_qualified_name: []const u8, +/// The dependency table of this module. The shared dependencies 'std' and +/// 'root' are not specified in every module dependency table, but are stored +/// separately in `Zcu`. 'builtin' is also not stored here, although it is +/// not necessarily the same between all modules. Handling of `@import` in +/// the rest of the compiler must detect these special names and use the +/// correct module instead of consulting `deps`. +deps: Deps = .{}, + +resolved_target: ResolvedTarget, +optimize_mode: std.lang.OptimizeMode, +code_model: std.lang.CodeModel, +single_threaded: bool, +error_tracing: bool, +valgrind: bool, +pic: bool, +strip: bool, +omit_frame_pointer: bool, +stack_check: bool, +stack_protector: u32, +red_zone: bool, +sanitize_c: std.zig.SanitizeC, +sanitize_thread: bool, +fuzz: bool, +unwind_tables: std.lang.UnwindTables, +cc_argv: []const []const u8, +/// (SPIR-V) whether to generate a structured control flow graph or not +structured_cfg: bool, +no_builtin: bool, + +pub const Deps = std.array_hash_map.String(*Module); + +pub const CreateOptions = struct { + paths: Paths, + fully_qualified_name: []const u8, + + cc_argv: []const []const u8, + inherited: Inherited, + global: Compilation.Config, + /// If this is null then `resolved_target` must be non-null. + parent: ?*Module, + + pub const Paths = struct { + root: Compilation.Path, + /// Relative to `root`. May contain path separators. + root_src_path: []const u8, + }; + + pub const Inherited = struct { + /// If this is null then `parent` must be non-null. + resolved_target: ?ResolvedTarget = null, + optimize_mode: ?std.lang.OptimizeMode = null, + code_model: ?std.lang.CodeModel = null, + single_threaded: ?bool = null, + error_tracing: ?bool = null, + valgrind: ?bool = null, + pic: ?bool = null, + strip: ?bool = null, + omit_frame_pointer: ?bool = null, + stack_check: ?bool = null, + /// null means default. + /// 0 means no stack protector. + /// other number means stack protection with that buffer size. + stack_protector: ?u32 = null, + red_zone: ?bool = null, + unwind_tables: ?std.lang.UnwindTables = null, + sanitize_c: ?std.zig.SanitizeC = null, + sanitize_thread: ?bool = null, + fuzz: ?bool = null, + structured_cfg: ?bool = null, + no_builtin: ?bool = null, + }; +}; + +pub const ResolvedTarget = struct { + result: std.Target, + is_native_os: bool, + is_native_abi: bool, + is_explicit_dynamic_linker: bool, + llvm_cpu_features: ?[*:0]const u8 = null, +}; + +pub const CreateError = error{ + OutOfMemory, + ValgrindUnsupportedOnTarget, + TargetRequiresSingleThreaded, + BackendRequiresSingleThreaded, + TargetRequiresPic, + PieRequiresPic, + DynamicLinkingRequiresPic, + TargetHasNoRedZone, + StackCheckUnsupportedByTarget, + StackProtectorUnsupportedByTarget, + StackProtectorUnavailableWithoutLibC, +}; + +/// At least one of `parent` and `resolved_target` must be non-null. +pub fn create(arena: Allocator, options: CreateOptions) !*Module { + if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread); + if (options.inherited.fuzz == true) assert(options.global.any_fuzz); + if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded); + if (options.inherited.unwind_tables) |uwt| if (uwt != .none) assert(options.global.any_unwind_tables); + if (options.inherited.sanitize_c) |sc| if (sc != .off) assert(options.global.any_sanitize_c != .off); + if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing); + + const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target; + const target = &resolved_target.result; + + const optimize_mode = options.inherited.optimize_mode orelse + if (options.parent) |p| p.optimize_mode else options.global.root_optimize_mode; + + const strip = b: { + if (options.inherited.strip) |x| break :b x; + if (options.parent) |p| break :b p.strip; + break :b options.global.root_strip; + }; + + const zig_backend = target_util.zigBackend(target, options.global.use_llvm); + + const valgrind = b: { + if (!target_util.hasValgrindSupport(target, zig_backend)) { + if (options.inherited.valgrind == true) + return error.ValgrindUnsupportedOnTarget; + break :b false; + } + if (options.inherited.valgrind) |x| break :b x; + if (options.parent) |p| break :b p.valgrind; + if (strip) break :b false; + break :b optimize_mode == .Debug; + }; + + const single_threaded = b: { + if (target_util.alwaysSingleThreaded(target)) { + if (options.inherited.single_threaded == false) + return error.TargetRequiresSingleThreaded; + break :b true; + } + + if (options.global.have_zcu) { + if (!target_util.supportsThreads(target, zig_backend)) { + if (options.inherited.single_threaded == false) + return error.BackendRequiresSingleThreaded; + break :b true; + } + } + + if (options.inherited.single_threaded) |x| break :b x; + if (options.parent) |p| break :b p.single_threaded; + break :b target_util.defaultSingleThreaded(target); + }; + + const error_tracing = b: { + if (options.inherited.error_tracing) |x| break :b x; + if (options.parent) |p| break :b p.error_tracing; + break :b options.global.root_error_tracing; + }; + + const pic = b: { + if (target_util.requiresPic(target, options.global.link_libc)) { + if (options.inherited.pic == false) + return error.TargetRequiresPic; + break :b true; + } + if (options.global.pie) { + if (options.inherited.pic == false) + return error.PieRequiresPic; + break :b true; + } + if (options.global.link_mode == .dynamic and target_util.requiresPicForDynamicLink(target)) { + if (options.inherited.pic == false) + return error.DynamicLinkingRequiresPic; + break :b true; + } + if (options.inherited.pic) |x| break :b x; + if (options.parent) |p| break :b p.pic; + + // Default to PIC on targets where we default to producing PIEs to make + // the common case of linking objects and static libraries into an + // executable work out of the box. + break :b target_util.defaultPie(target); + }; + + const red_zone = b: { + if (!target_util.hasRedZone(target)) { + if (options.inherited.red_zone == true) + return error.TargetHasNoRedZone; + break :b false; + } + if (options.inherited.red_zone) |x| break :b x; + if (options.parent) |p| break :b p.red_zone; + break :b true; + }; + + const omit_frame_pointer = b: { + if (options.inherited.omit_frame_pointer) |x| break :b x; + if (options.parent) |p| break :b p.omit_frame_pointer; + if (optimize_mode == .ReleaseSmall) { + // On x86, in most cases, keeping the frame pointer usually results in smaller binary size. + // This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer) + // are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer). + break :b !target.cpu.arch.isX86(); + } + break :b false; + }; + + const sanitize_thread = b: { + if (options.inherited.sanitize_thread) |x| break :b x; + if (options.parent) |p| break :b p.sanitize_thread; + break :b false; + }; + + const unwind_tables = b: { + if (options.inherited.unwind_tables) |x| break :b x; + if (options.parent) |p| break :b p.unwind_tables; + + break :b target_util.defaultUnwindTables( + target, + options.global.link_libunwind, + sanitize_thread or options.global.any_sanitize_thread, + ); + }; + + const fuzz = b: { + if (options.inherited.fuzz) |x| break :b x; + if (options.parent) |p| break :b p.fuzz; + break :b false; + }; + + const code_model: std.lang.CodeModel = b: { + if (options.inherited.code_model) |x| break :b x; + if (options.parent) |p| break :b p.code_model; + break :b .default; + }; + + const is_safe_mode = switch (optimize_mode) { + .Debug, .ReleaseSafe => true, + .ReleaseFast, .ReleaseSmall => false, + }; + + const sanitize_c: std.zig.SanitizeC = b: { + if (options.inherited.sanitize_c) |x| break :b x; + if (options.parent) |p| break :b p.sanitize_c; + break :b switch (optimize_mode) { + .Debug => .full, + // It's recommended to use the minimal runtime in production + // environments due to the security implications of the full runtime. + // The minimal runtime doesn't provide much benefit over simply + // trapping, however, so we do that instead. + .ReleaseSafe => .trap, + .ReleaseFast, .ReleaseSmall => .off, + }; + }; + + const stack_check = b: { + if (!target_util.supportsStackProbing(target, zig_backend)) { + if (options.inherited.stack_check == true) + return error.StackCheckUnsupportedByTarget; + break :b false; + } + if (options.inherited.stack_check) |x| break :b x; + if (options.parent) |p| break :b p.stack_check; + break :b is_safe_mode; + }; + + const stack_protector: u32 = sp: { + const use_zig_backend = options.global.have_zcu or + (options.global.any_c_source_files and options.global.c_frontend == .aro); + if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) { + if (options.inherited.stack_protector) |x| { + if (x > 0) return error.StackProtectorUnsupportedByTarget; + } + break :sp 0; + } + + if (options.global.any_c_source_files and options.global.c_frontend == .clang and + !target_util.clangSupportsStackProtector(target)) + { + if (options.inherited.stack_protector) |x| { + if (x > 0) return error.StackProtectorUnsupportedByTarget; + } + break :sp 0; + } + + // This logic is checking for linking libc because otherwise our start code + // which is trying to set up TLS (i.e. the fs/gs registers) but the stack + // protection code depends on fs/gs registers being already set up. + // If we were able to annotate start code, or perhaps the entire std lib, + // as being exempt from stack protection checks, we could change this logic + // to supporting stack protection even when not linking libc. + // TODO file issue about this + if (!options.global.link_libc) { + if (options.inherited.stack_protector) |x| { + if (x > 0) return error.StackProtectorUnavailableWithoutLibC; + } + break :sp 0; + } + + if (options.inherited.stack_protector) |x| break :sp x; + if (options.parent) |p| break :sp p.stack_protector; + if (!is_safe_mode) break :sp 0; + + break :sp target_util.default_stack_protector_buffer_size; + }; + + const structured_cfg = b: { + if (options.inherited.structured_cfg) |x| break :b x; + if (options.parent) |p| break :b p.structured_cfg; + // We always want a structured control flow in shaders. This option is + // only relevant for OpenCL kernels. + break :b switch (target.os.tag) { + .opencl => false, + else => true, + }; + }; + + const no_builtin = b: { + if (options.inherited.no_builtin) |x| break :b x; + if (options.parent) |p| break :b p.no_builtin; + + break :b target.cpu.arch.isBpf(); + }; + + const llvm_cpu_features: ?[*:0]const u8 = b: { + if (resolved_target.llvm_cpu_features) |x| break :b x; + if (!options.global.use_llvm) break :b null; + + var buf = std.array_list.Managed(u8).init(arena); + var disabled_features = std.array_list.Managed(u8).init(arena); + defer disabled_features.deinit(); + + // Append disabled features after enabled ones, so that their effects aren't overwritten. + for (target.cpu.arch.allFeaturesList()) |feature| { + if (feature.llvm_name) |llvm_name| { + // Ignore these until we figure out how to handle the concept of omitting features. + // See https://github.com/ziglang/zig/issues/23539 + if (target_util.isDynamicAMDGCNFeature(target, feature)) continue; + + if (target.cpu.arch.isPowerPC() and @as(std.Target.powerpc.Feature, @enumFromInt(feature.index)) == .@"64bit") continue; + if (target.cpu.arch.isX86() and @as(std.Target.x86.Feature, @enumFromInt(feature.index)) == .x32) continue; + + var is_enabled = target.cpu.features.isEnabled(feature.index); + if (target.cpu.arch == .s390x and @as(std.Target.s390x.Feature, @enumFromInt(feature.index)) == .backchain) { + is_enabled = !omit_frame_pointer; + } + + if (is_enabled) { + try buf.ensureUnusedCapacity(2 + llvm_name.len); + buf.appendAssumeCapacity('+'); + buf.appendSliceAssumeCapacity(llvm_name); + buf.appendAssumeCapacity(','); + } else { + try disabled_features.ensureUnusedCapacity(2 + llvm_name.len); + disabled_features.appendAssumeCapacity('-'); + disabled_features.appendSliceAssumeCapacity(llvm_name); + disabled_features.appendAssumeCapacity(','); + } + } + } + + try buf.appendSlice(disabled_features.items); + if (buf.items.len == 0) break :b ""; + assert(std.mem.endsWith(u8, buf.items, ",")); + buf.items[buf.items.len - 1] = 0; + buf.shrinkAndFree(buf.items.len); + break :b buf.items[0 .. buf.items.len - 1 :0].ptr; + }; + + const mod = try arena.create(Module); + mod.* = .{ + .root = options.paths.root, + .root_src_path = options.paths.root_src_path, + .fully_qualified_name = options.fully_qualified_name, + .resolved_target = .{ + .result = target.*, + .is_native_os = resolved_target.is_native_os, + .is_native_abi = resolved_target.is_native_abi, + .is_explicit_dynamic_linker = resolved_target.is_explicit_dynamic_linker, + .llvm_cpu_features = llvm_cpu_features, + }, + .optimize_mode = optimize_mode, + .single_threaded = single_threaded, + .error_tracing = error_tracing, + .valgrind = valgrind, + .pic = pic, + .strip = strip, + .omit_frame_pointer = omit_frame_pointer, + .stack_check = stack_check, + .stack_protector = stack_protector, + .code_model = code_model, + .red_zone = red_zone, + .sanitize_c = sanitize_c, + .sanitize_thread = sanitize_thread, + .fuzz = fuzz, + .unwind_tables = unwind_tables, + .cc_argv = options.cc_argv, + .structured_cfg = structured_cfg, + .no_builtin = no_builtin, + }; + return mod; +} + +/// All fields correspond to `CreateOptions`. +pub const LimitedOptions = struct { + root: Compilation.Path, + root_src_path: []const u8, + fully_qualified_name: []const u8, +}; + +/// This one can only be used if the Module will only be used for AstGen and earlier in +/// the pipeline. Illegal behavior occurs if a limited module touches Sema. +pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Module { + const mod = try gpa.create(Module); + mod.* = .{ + .root = options.root, + .root_src_path = options.root_src_path, + .fully_qualified_name = options.fully_qualified_name, + + .resolved_target = undefined, + .optimize_mode = undefined, + .code_model = undefined, + .single_threaded = undefined, + .error_tracing = undefined, + .valgrind = undefined, + .pic = undefined, + .strip = undefined, + .omit_frame_pointer = undefined, + .stack_check = undefined, + .stack_protector = undefined, + .red_zone = undefined, + .sanitize_c = undefined, + .sanitize_thread = undefined, + .fuzz = undefined, + .unwind_tables = undefined, + .cc_argv = undefined, + .structured_cfg = undefined, + .no_builtin = undefined, + }; + return mod; +} + +/// Does not ensure that the module's root directory exists on-disk; see `Builtin.updateFileOnDisk` for that task. +pub fn createBuiltin(arena: Allocator, opts: Builtin, dirs: std.zig.Directories) Allocator.Error!*Module { + const sub_path = "b" ++ std.fs.path.sep_str ++ Cache.binToHex(opts.hash()); + const new = try arena.create(Module); + new.* = .{ + .root = try .fromRoot(arena, dirs, .global_cache, sub_path), + .root_src_path = "builtin.zig", + .fully_qualified_name = "builtin", + .resolved_target = .{ + .result = opts.target, + // These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code. + .is_native_os = false, + .is_native_abi = false, + .is_explicit_dynamic_linker = false, + .llvm_cpu_features = null, + }, + .optimize_mode = opts.optimize_mode, + .single_threaded = opts.single_threaded, + .error_tracing = opts.error_tracing, + .valgrind = opts.valgrind, + .pic = opts.pic, + .strip = opts.strip, + .omit_frame_pointer = opts.omit_frame_pointer, + .code_model = opts.code_model, + .sanitize_thread = opts.sanitize_thread, + .fuzz = opts.fuzz, + .unwind_tables = opts.unwind_tables, + .cc_argv = &.{}, + // These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code. + .stack_check = false, + .stack_protector = 0, + .red_zone = false, + .sanitize_c = .off, + .structured_cfg = false, + .no_builtin = false, + }; + return new; +} + +/// Returns the `Builtin` which forms the contents of `@import("builtin")` for this module. +pub fn getBuiltinOptions(m: Module, global: Compilation.Config) Builtin { + assert(global.have_zcu); + return .{ + .target = m.resolved_target.result, + .zig_backend = target_util.zigBackend(&m.resolved_target.result, global.use_llvm), + .output_mode = global.output_mode, + .link_mode = global.link_mode, + .unwind_tables = m.unwind_tables, + .is_test = global.is_test, + .single_threaded = m.single_threaded, + .link_libc = global.link_libc, + .link_libcpp = global.link_libcpp, + .optimize_mode = m.optimize_mode, + .error_tracing = m.error_tracing, + .valgrind = m.valgrind, + .sanitize_thread = m.sanitize_thread, + .fuzz = m.fuzz, + .pic = m.pic, + .pie = global.pie, + .strip = m.strip, + .code_model = m.code_model, + .omit_frame_pointer = m.omit_frame_pointer, + .wasi_exec_model = global.wasi_exec_model, + }; +} diff --git a/src/Package.zig b/src/Package.zig deleted file mode 100644 index 8fb9995bd81315343e9b1da8d1741774ae4e9d82..0000000000000000000000000000000000000000 --- a/src/Package.zig +++ /dev/null @@ -1,209 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; - -pub const Module = @import("Package/Module.zig"); -pub const Fetch = @import("Package/Fetch.zig"); -pub const build_zig_basename = "build.zig"; -pub const Manifest = @import("Package/Manifest.zig"); - -pub const Fingerprint = packed struct(u64) { - id: u32, - checksum: u32, - - pub fn generate(rng: std.Random, name: []const u8) Fingerprint { - return .{ - .id = rng.intRangeLessThan(u32, 1, 0xffffffff), - .checksum = std.hash.Crc32.hash(name), - }; - } - - pub fn validate(n: Fingerprint, name: []const u8) bool { - switch (n.id) { - 0x00000000, 0xffffffff => return false, - else => return std.hash.Crc32.hash(name) == n.checksum, - } - } - - pub fn int(n: Fingerprint) u64 { - return @bitCast(n); - } -}; - -/// A user-readable, file system safe hash that identifies an exact package -/// snapshot, including file contents. -/// -/// The hash is not only to prevent collisions but must resist attacks where -/// the adversary fully controls the contents being hashed. Thus, it contains -/// a full SHA-256 digest. -/// -/// This data structure can be used to store the legacy hash format too. Legacy -/// hash format is scheduled to be removed after 0.14.0 is tagged. -/// -/// There's also a third way this structure is used. When using path rather than -/// hash, a unique hash is still needed, so one is computed based on the path. -pub const Hash = struct { - /// Maximum size of a package hash. Unused bytes at the end are - /// filled with zeroes. - /// - /// Assumed to be already validated. - bytes: [max_len]u8, - - pub const Algo = std.crypto.hash.sha2.Sha256; - pub const Digest = [Algo.digest_length]u8; - - /// Example: "nnnn-vvvv-hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh" - pub const max_len = 32 + 1 + 32 + 1 + (32 + 32 + 200) / 6; - - /// Asserts `s` is valid. - pub fn fromSlice(s: []const u8) Hash { - assert(validate(s) == .ok); - var result: Hash = undefined; - @memcpy(result.bytes[0..s.len], s); - @memset(result.bytes[s.len..], 0); - return result; - } - - pub const Validation = enum { ok, short, long, incomplete }; - - pub fn validate(s: []const u8) Validation { - if (s.len > max_len) return .long; - if (s.len < 44) return .short; - const n_dashes = std.mem.countScalar(u8, s[0 .. s.len - 44], '-'); - if (n_dashes < 2) return .incomplete; - return .ok; - } - - test validate { - try std.testing.expectEqual(.short, validate("")); - } - - pub fn toSlice(ph: *const Hash) []const u8 { - var end: usize = ph.bytes.len; - while (true) { - end -= 1; - if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1]; - } - } - - pub fn eql(a: *const Hash, b: *const Hash) bool { - return std.mem.eql(u8, &a.bytes, &b.bytes); - } - - /// Produces "$name-$semver-$hashplus". - /// * name is the name field from build.zig.zon, asserted to be at most 32 - /// bytes and assumed be a valid zig identifier - /// * semver is the version field from build.zig.zon, asserted to be at - /// most 32 bytes - /// * hashplus is the following 33-byte array, base64 encoded using -_ to make - /// it filesystem safe: - /// - (4 bytes) LE u32 Package ID - /// - (4 bytes) LE u32 total decompressed size in bytes, overflow saturated - /// - (25 bytes) truncated SHA-256 digest of hashed files of the package - pub fn init(digest: Digest, name: []const u8, ver: []const u8, id: u32, size: u32) Hash { - assert(name.len <= 32); - assert(ver.len <= 32); - var result: Hash = undefined; - var buf: std.ArrayList(u8) = .initBuffer(&result.bytes); - buf.appendSliceAssumeCapacity(name); - buf.appendAssumeCapacity('-'); - buf.appendSliceAssumeCapacity(ver); - buf.appendAssumeCapacity('-'); - var hashplus: [33]u8 = undefined; - std.mem.writeInt(u32, hashplus[0..4], id, .little); - std.mem.writeInt(u32, hashplus[4..8], size, .little); - hashplus[8..].* = digest[0..25].*; - _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(44), &hashplus); - @memset(buf.unusedCapacitySlice(), 0); - return result; - } - - /// Produces a unique hash based on the path provided. The result should - /// not be user-visible. - pub fn initPath(sub_path: []const u8, is_global: bool) Hash { - var result: Hash = .{ .bytes = @splat(0) }; - var i: usize = 0; - if (is_global) { - result.bytes[0] = '/'; - i += 1; - } - if (i + sub_path.len <= result.bytes.len) { - @memcpy(result.bytes[i..][0..sub_path.len], sub_path); - return result; - } - var bin_digest: [Algo.digest_length]u8 = undefined; - Algo.hash(sub_path, &bin_digest, .{}); - _ = std.fmt.bufPrint(result.bytes[i..], "{x}", .{&bin_digest}) catch unreachable; - return result; - } - - pub fn projectId(hash: *const Hash) ProjectId { - const bytes = hash.toSlice(); - const name = std.mem.sliceTo(bytes, '-'); - const encoded_hashplus = bytes[bytes.len - 44 ..]; - var hashplus: [33]u8 = undefined; - std.base64.url_safe_no_pad.Decoder.decode(&hashplus, encoded_hashplus) catch unreachable; - const fingerprint_id = std.mem.readInt(u32, hashplus[0..4], .little); - return .init(name, fingerprint_id); - } - - test projectId { - const hash: Hash = .fromSlice("pulseaudio-16.1.1-9-mk_62MZkNwBaFwiZ7ZVrYRIf_3dTqqJR5PbMRCJzSuLw"); - const project_id = hash.projectId(); - - var expected_name: [32]u8 = @splat(0); - expected_name[0.."pulseaudio".len].* = "pulseaudio".*; - try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name); - - try std.testing.expectEqual(0xd8fa4f9a, project_id.fingerprint_id); - } - - test "projectId with dashes in the base64" { - const hash: Hash = .fromSlice("dvui-0.4.0-dev-AQFJmayi2gAKE7FeJoF61v5U1IV9-SupoEcFutIZYpkC"); - const project_id = hash.projectId(); - - var expected_name: [32]u8 = @splat(0); - expected_name[0.."dvui".len].* = "dvui".*; - try std.testing.expectEqualSlices(u8, &expected_name, &project_id.padded_name); - - try std.testing.expectEqual(0x99490101, project_id.fingerprint_id); - } -}; - -/// Minimum information required to identify whether a package is an artifact -/// of a given project. -pub const ProjectId = struct { - /// Bytes after name.len are set to zero. - padded_name: [32]u8, - fingerprint_id: u32, - - pub fn init(name: []const u8, fingerprint_id: u32) ProjectId { - var padded_name: [32]u8 = @splat(0); - @memcpy(padded_name[0..name.len], name); - return .{ - .padded_name = padded_name, - .fingerprint_id = fingerprint_id, - }; - } - - pub fn eql(a: *const ProjectId, b: *const ProjectId) bool { - return a.fingerprint_id == b.fingerprint_id and std.mem.eql(u8, &a.padded_name, &b.padded_name); - } - - pub fn hash(a: *const ProjectId) u64 { - const x: u64 = @bitCast(a.padded_name[0..8].*); - return std.hash.int(x | a.fingerprint_id); - } -}; - -test Hash { - const example_digest: Hash.Digest = .{ - 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87, - 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f, - }; - const result: Hash = .init(example_digest, "nasm", "2.16.1-3", 0xcafebabe, 10 * 1024 * 1024); - try std.testing.expectEqualStrings("nasm-2.16.1-3-vrr-ygAAoADH9XG3tOdvPNuHen_d-XeHndOG-nNXmved", result.toSlice()); -} - -test { - _ = Fetch; -} diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig deleted file mode 100644 index 20e19cdaa7ad2e55b9ddb07f490df1ffdaa26eea..0000000000000000000000000000000000000000 --- a/src/Package/Fetch.zig +++ /dev/null @@ -1,2283 +0,0 @@ -//! Represents one independent job whose responsibility is to: -//! -//! 1. Check the local zig package directory to see if the hash already exists. -//! If so, load, parse, and validate the build.zig.zon file therein, and -//! goto step 9. Likewise if the location is a relative path, treat this -//! the same as a cache hit. Otherwise, proceed. -//! 2. Check the global package cache for a compressed tarball matching the -//! hash. If it is found, unpack the contents into a temporary directory inside -//! project local zig cache. Rename this directory into the local zig package -//! directory and goto step 9, skipping step 10. -//! 3. Fetch and unpack a URL into a temporary directory. -//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed -//! for the file to be missing, in which case this fetched package is considered -//! to be a "naked" package. -//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by -//! deleting excluded files. If any files had errors for files that were -//! ultimately excluded, those errors should be ignored, such as failure to -//! create symlinks that weren't supposed to be included anyway. -//! 6. Compute the package hash based on the remaining files in the temporary -//! directory. -//! 7. Rename the temporary directory into the local zig package directory. If -//! the hash already exists, delete the temporary directory and leave the zig -//! package directory untouched as it may be in use. This is done even if -//! the hash is invalid, in case the package with the different hash is used -//! in the future. -//! 8. Validate the computed hash against the expected hash. If invalid, -//! this job is done. -//! 9. Spawn a new fetch job for each dependency in the manifest file. Use -//! a mutex and a hash map so that redundant jobs do not get queued up. -//! 10.Compress the package directory and store it into the global package -//! cache. -//! -//! All of this must be done with only referring to the state inside this struct -//! because this work will be done in a dedicated thread. -const Fetch = @This(); - -const builtin = @import("builtin"); -const native_os = builtin.os.tag; - -const std = @import("std"); -const Io = std.Io; -const fs = std.fs; -const log = std.log.scoped(.fetch); -const assert = std.debug.assert; -const ascii = std.ascii; -const Allocator = std.mem.Allocator; -const Cache = std.Build.Cache; -const git = @import("Fetch/git.zig"); -const Package = @import("../Package.zig"); -const Manifest = Package.Manifest; -const ErrorBundle = std.zig.ErrorBundle; - -arena: std.heap.ArenaAllocator, -location: Location, -location_tok: std.zig.Ast.TokenIndex, -hash_tok: std.zig.Ast.OptionalTokenIndex, -name_tok: std.zig.Ast.TokenIndex, -lazy_status: LazyStatus, -/// Same as `parent_packge_root` except it is unchanged when recursing into -/// relative file paths (as opposed to URL). -remote_package_root: Cache.Path, -parent_package_root: Cache.Path, -parent_manifest_ast: ?*const std.zig.Ast, -prog_node: std.Progress.Node, -job_queue: *JobQueue, -/// If true, don't add an error for a missing hash. This flag is not passed -/// down to recursive dependencies. It's intended to be used only be the CLI. -omit_missing_hash_error: bool, -/// If true, don't fail when a manifest file is missing the `paths` field, -/// which specifies inclusion rules. This is intended to be true for the first -/// fetch task and false for the recursive dependencies. -allow_missing_paths_field: bool, -/// If true and URL points to a Git repository, will use the latest commit. -use_latest_commit: bool, - -// Above this are fields provided as inputs to `run`. -// Below this are fields populated by `run`. - -/// Relative to the build root of the root package. -package_root: Cache.Path, -error_bundle: ErrorBundle.Wip, -manifest: Manifest, -manifest_ast: std.zig.Ast, -have_manifest: bool, -computed_hash: ComputedHash, -/// Fetch logic notices whether a package has a build.zig file and sets this flag. -has_build_zig: bool, -/// Indicates whether the task aborted due to an out-of-memory condition. -oom_flag: bool, -/// If `use_latest_commit` was true, this will be set to the commit that was used. -/// If the resource pointed to by the location is not a Git-repository, this -/// will be left unchanged. -latest_commit: ?git.Oid, - -// This field is used by the CLI only, untouched by this file. - -/// The module for this `Fetch` tasks's package, which exposes `build.zig` as -/// the root source file. -module: ?*Package.Module, - -pub const LazyStatus = enum { - /// Not lazy. - eager, - /// Lazy, found. - available, - /// Lazy, not found. - unavailable, -}; - -pub const LocalStorage = struct { - cache_root: Cache.Path, - /// Path to "zig-pkg" inside the package in which the user ran `zig build`. - pkg_root: Cache.Path, -}; - -/// Contains shared state among all `Fetch` tasks. -pub const JobQueue = struct { - io: Io, - mutex: Io.Mutex = .init, - /// It's an array hash map so that it can be sorted before rendering the - /// dependencies.zig source file. - /// Protected by `mutex`. - table: Table = .{}, - /// `table` may be missing some tasks such as ones that failed, so this - /// field contains references to all of them. - /// Protected by `mutex`. - all_fetches: std.ArrayList(*Fetch) = .empty, - prog_node: std.Progress.Node, - - http_client: *std.http.Client, - /// This tracks `Fetch` tasks as well as recompression tasks. - group: Io.Group = .init, - global_cache: Cache.Directory, - /// If `null`, indicates fetch globally only. - local_storage: ?*const LocalStorage, - /// If true then, no fetching occurs, and: - /// * The `global_cache` directory is assumed to be the direct parent - /// directory of on-disk packages rather than having the "p/" directory - /// prefix inside of it. - /// * An error occurs if any non-lazy packages are not already present in - /// the package cache directory. - /// * Missing hash field causes an error, and no fetching occurs so it does - /// not print the correct hash like usual. - read_only: bool, - recursive: bool, - /// Dumps hash information to stdout which can be used to troubleshoot why - /// two hashes of the same package do not match. - /// If this is true, `recursive` must be false. - debug_hash: bool, - mode: Mode, - /// Set of hashes that will be additionally fetched even if they are marked - /// as lazy. - unlazy_set: UnlazySet = .{}, - /// Identifies paths that override all packages in the tree with matching - /// project ids. - fork_set: ForkSet = .{}, - - pub const Mode = enum { - /// Non-lazy dependencies are always fetched. - /// Lazy dependencies are fetched only when needed. - needed, - /// Both non-lazy and lazy dependencies are always fetched. - all, - }; - pub const Table = std.array_hash_map.Auto(Package.Hash, *Fetch); - pub const UnlazySet = std.array_hash_map.Auto(Package.Hash, void); - pub const ForkSet = std.array_hash_map.Custom(Fork, void, Fork.Context, false); - - pub const Fork = struct { - path: Cache.Path, - manifest_ast: std.zig.Ast, - manifest: Package.Manifest, - uses: usize, - - pub const Context = struct { - pub fn hash(_: @This(), a: Fork) u32 { - const project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id); - return @truncate(project_id.hash()); - } - - pub fn eql(_: @This(), a: Fork, b: Fork, _: usize) bool { - const a_project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id); - const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id); - return a_project_id.eql(&b_project_id); - } - }; - - pub const Adapter = struct { - pub fn hash(_: @This(), a: Package.ProjectId) u32 { - return @truncate(a.hash()); - } - - pub fn eql(_: @This(), a_project_id: Package.ProjectId, b: Fork, _: usize) bool { - const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id); - return a_project_id.eql(&b_project_id); - } - }; - }; - - pub fn deinit(jq: *JobQueue) void { - const io = jq.io; - jq.group.cancel(io); - if (jq.all_fetches.items.len == 0) return; - const gpa = jq.all_fetches.items[0].arena.child_allocator; - jq.table.deinit(gpa); - // These must be deinitialized in reverse order because subsequent - // `Fetch` instances are allocated in prior ones' arenas. - // Sorry, I know it's a bit weird, but it slightly simplifies the - // critical section. - while (jq.all_fetches.pop()) |f| f.deinit(); - jq.all_fetches.deinit(gpa); - jq.* = undefined; - } - - /// Dumps all subsequent error bundles into the first one. - pub fn consolidateErrors(jq: *JobQueue) !void { - const root = &jq.all_fetches.items[0].error_bundle; - const gpa = root.gpa; - for (jq.all_fetches.items[1..]) |fetch| { - if (fetch.error_bundle.root_list.items.len > 0) { - var bundle = try fetch.error_bundle.toOwnedBundle(""); - defer bundle.deinit(gpa); - try root.addBundleAsRoots(bundle); - } - } - } - - /// Creates the dependencies.zig source code for the build runner to obtain - /// via `@import("@dependencies")`. - pub fn createDependenciesSource(jq: *JobQueue, buf: *std.array_list.Managed(u8)) Allocator.Error!void { - const keys = jq.table.keys(); - - assert(keys.len != 0); // caller should have added the first one - if (keys.len == 1) { - // This is the first one. It must have no dependencies. - return createEmptyDependenciesSource(buf); - } - - try buf.appendSlice("pub const packages = struct {\n"); - - // Ensure the generated .zig file is deterministic. - jq.table.sortUnstable(@as(struct { - keys: []const Package.Hash, - pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { - return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes); - } - }, .{ .keys = keys })); - - for (keys, jq.table.values()) |*hash, fetch| { - if (fetch == jq.all_fetches.items[0]) { - // The first one is a dummy package for the current project. - continue; - } - - const hash_slice = hash.toSlice(); - - try buf.print( - \\ pub const {f} = struct {{ - \\ - , .{std.zig.fmtId(hash_slice)}); - - lazy: { - switch (fetch.lazy_status) { - .eager => break :lazy, - .available => { - try buf.appendSlice( - \\ pub const available = true; - \\ - ); - break :lazy; - }, - .unavailable => { - try buf.appendSlice( - \\ pub const available = false; - \\ }; - \\ - ); - continue; - }, - } - } - - try buf.print( - \\ pub const build_root = "{f}"; - \\ - , .{std.fmt.alt(fetch.package_root, .formatEscapeString)}); - - if (fetch.has_build_zig) { - try buf.print( - \\ pub const build_zig = @import("{f}"); - \\ - , .{std.zig.fmtString(hash_slice)}); - } - - if (fetch.have_manifest) { - const manifest = &fetch.manifest; - try buf.appendSlice( - \\ pub const deps: []const struct { []const u8, []const u8 } = &.{ - \\ - ); - for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| { - const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; - try buf.print( - " .{{ \"{f}\", \"{f}\" }},\n", - .{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) }, - ); - } - - try buf.appendSlice( - \\ }; - \\ }; - \\ - ); - } else { - try buf.appendSlice( - \\ pub const deps: []const struct { []const u8, []const u8 } = &.{}; - \\ }; - \\ - ); - } - } - - try buf.appendSlice( - \\}; - \\ - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{ - \\ - ); - - const root_fetch = jq.all_fetches.items[0]; - assert(root_fetch.have_manifest); - const root_manifest = &root_fetch.manifest; - - for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| { - const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; - try buf.print( - " .{{ \"{f}\", \"{f}\" }},\n", - .{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) }, - ); - } - try buf.appendSlice("};\n"); - } - - pub fn createEmptyDependenciesSource(buf: *std.array_list.Managed(u8)) Allocator.Error!void { - try buf.appendSlice( - \\pub const packages = struct {}; - \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; - \\ - ); - } - - fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Cache.Path) Io.Cancelable!void { - const pkg_hash_slice = package_hash.toSlice(); - - const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice}); - defer prog_node.end(); - - var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined; - const dest_path: Cache.Path = .{ - .root_dir = jq.global_cache, - .sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable, - }; - - const gpa = jq.http_client.allocator; - - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch |err| switch (err) { - error.Canceled => |e| return e, - error.ReadFailed => comptime unreachable, - error.WriteFailed => comptime unreachable, - else => |e| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }), - }; - } - - fn recompressFallible( - jq: *JobQueue, - arena: Allocator, - dest_path: Cache.Path, - pkg_hash_slice: []const u8, - package_root: Cache.Path, - prog_node: std.Progress.Node, - ) !void { - const gpa = jq.http_client.allocator; - const io = jq.io; - - // We have to walk the file system up front in order to sort the file - // list for determinism purposes. The hash of the recompressed file is - // not critical because the true hash is based on the content alone. - // However, if we want Zig users to be able to share cached package - // data with each other via peer-to-peer protocols, we benefit greatly - // from the data being identical on everyone's computers. - var scanned_files: std.ArrayList(ScannedFile) = .empty; - defer scanned_files.deinit(gpa); - - var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true }); - defer pkg_dir.close(io); - - { - var walker = try pkg_dir.walk(gpa); - defer walker.deinit(); - - while (try walker.next(io)) |entry| { - const symlink = switch (entry.kind) { - .directory => continue, - .file => false, - .sym_link => true, - else => return error.IllegalFileType, - }; - const entry_path = try arena.dupe(u8, entry.path); - // If necessary, normalize path separators to POSIX-style since the tar format requires that. - if (comptime (std.fs.path.sep != std.fs.path.sep_posix)) { - std.mem.replaceScalar(u8, entry_path, std.fs.path.sep, std.fs.path.sep_posix); - } - try scanned_files.append(gpa, .{ - .ptr = entry_path.ptr, - .len = @intCast(entry_path.len), - .symlink = symlink, - }); - } - - std.mem.sortUnstable(ScannedFile, scanned_files.items, {}, stringCmp); - } - - prog_node.setEstimatedTotalItems(scanned_files.items.len); - - var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{ - .make_path = true, - .replace = true, - }); - defer atomic_file.deinit(io); - - var file_write_buffer: [4096]u8 = undefined; - var file_writer = atomic_file.file.writer(io, &file_write_buffer); - - var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined; - var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch |err| switch (err) { - error.WriteFailed => return file_writer.err.?, - }; - - var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer }; - archiver.prefix = pkg_hash_slice; - - var file_read_buffer: [4096]u8 = undefined; - var link_buf: [fs.max_path_bytes]u8 = undefined; - - for (scanned_files.items) |scanned_file| { - const entry_path = scanned_file.ptr[0..scanned_file.len]; - if (scanned_file.symlink) { - const link_name = link_buf[0..try pkg_dir.readLink(io, entry_path, &link_buf)]; - archiver.writeLink(entry_path, link_name, .{}) catch |err| switch (err) { - error.WriteFailed => return file_writer.err.?, - else => |e| return e, - }; - } else { - var file = try pkg_dir.openFile(io, entry_path, .{}); - defer file.close(io); - var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer); - archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, - error.WriteFailed => return file_writer.err.?, - else => |e| return e, - }; - } - prog_node.completeOne(); - } - - // intentionally omitting the pointless trailer - //try archiver.finish(); - compress.finish() catch |err| switch (err) { - error.WriteFailed => return file_writer.err.?, - }; - try file_writer.flush(); - try atomic_file.replace(io); - } -}; - -const ScannedFile = struct { - ptr: [*]const u8, - len: u32, - symlink: bool, -}; - -fn stringCmp(_: void, lhs: ScannedFile, rhs: ScannedFile) bool { - return std.mem.lessThan(u8, lhs.ptr[0..lhs.len], rhs.ptr[0..rhs.len]); -} - -pub const Location = union(enum) { - remote: Remote, - /// A directory found inside the parent package. - relative_path: Cache.Path, - /// Recursive Fetch tasks will never use this Location, but it may be - /// passed in by the CLI. Indicates the file contents here should be copied - /// into the global package cache. It may be a file relative to the cwd or - /// absolute, in which case it should be treated exactly like a `file://` - /// URL, or a directory, in which case it should be treated as an - /// already-unpacked directory (but still needs to be copied into the - /// global package cache and have inclusion rules applied). - path_or_url: []const u8, - - pub const Remote = struct { - url: []const u8, - /// If this is null it means the user omitted the hash field from a dependency. - /// It will be an error but the logic should still fetch and print the discovered hash. - hash: ?Package.Hash, - }; -}; - -pub const RunError = error{ - OutOfMemory, - Canceled, - /// This error code is intended to be handled by inspecting the - /// `error_bundle` field. - FetchFailed, -}; - -pub fn run(f: *Fetch) RunError!void { - const job_queue = f.job_queue; - const io = job_queue.io; - const eb = &f.error_bundle; - const arena = f.arena.allocator(); - const gpa = f.arena.child_allocator; - - try eb.init(gpa); - - // Check the global zig package cache to see if the hash already exists. If - // so, load, parse, and validate the build.zig.zon file therein, and skip - // ahead to queuing up jobs for dependencies. Likewise if the location is a - // relative path, treat this the same as a cache hit. Otherwise, proceed. - - const remote = switch (f.location) { - .relative_path => |pkg_root| { - if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail( - f.location_tok, - try eb.addString("expected path relative to build root; found absolute path"), - ); - if (f.hash_tok.unwrap()) |hash_tok| return f.fail( - hash_tok, - try eb.addString("path-based dependencies are not hashed"), - ); - // Packages fetched by URL may not use relative paths to escape outside the - // fetched package directory from within the package cache. - - // This code path is only reachable recursively and the sub_path - // will already have been resolved to no longer have extra ".." or - // "." components. - assert(job_queue.local_storage != null); - log.debug("checking pkg root \"{s}\" against parent package root \"{s}\"", .{ - pkg_root.sub_path, f.remote_package_root.sub_path, - }); - assert(pkg_root.root_dir.eql(f.remote_package_root.root_dir)); - if (!std.mem.startsWith(u8, pkg_root.sub_path, f.remote_package_root.sub_path)) return f.fail( - f.location_tok, - try eb.printString("dependency path outside project: '{f}'", .{pkg_root}), - ); - f.package_root = pkg_root; - try loadManifest(f, pkg_root); - if (!f.has_build_zig) try checkBuildFileExistence(f); - if (!job_queue.recursive) return; - return queueJobsForDeps(f); - }, - .remote => |remote| remote, - .path_or_url => |path_or_url| { - if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) |dir| { - var resource: Resource = .{ .dir = dir }; - return f.runResource(path_or_url, &resource, null, false); - } else |dir_err| { - var server_header_buffer: [init_resource_buffer_size]u8 = undefined; - - const file_err = if (dir_err == error.NotDir) e: { - if (Io.Dir.cwd().openFile(io, path_or_url, .{})) |file| { - var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) }; - return f.runResource(path_or_url, &resource, null, false); - } else |err| break :e err; - } else dir_err; - - const uri = std.Uri.parse(path_or_url) catch |uri_err| { - return f.fail(0, try eb.printString( - "'{s}' could not be recognized as a file path ({t}) or an URL ({t})", - .{ path_or_url, file_err, uri_err }, - )); - }; - var resource: Resource = undefined; - try f.initResource(uri, &resource, &server_header_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false); - } - }, - }; - - var resource_buffer: [init_resource_buffer_size]u8 = undefined; - - if (remote.hash) |expected_hash| { - const expected_project_id: Package.ProjectId = expected_hash.projectId(); - if (job_queue.fork_set.getKeyPtrAdapted(expected_project_id, @as(JobQueue.Fork.Adapter, .{}))) |fork| { - log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name }); - fork.uses += 1; - f.package_root = fork.path; - f.remote_package_root = f.package_root; - f.manifest_ast = fork.manifest_ast; - f.manifest = fork.manifest; - f.have_manifest = true; - try checkBuildFileExistence(f); - if (!job_queue.recursive) return; - return queueJobsForDeps(f); - } - - if (job_queue.local_storage) |ls| { - const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice()); - if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| { - assert(f.lazy_status != .unavailable); - f.package_root = package_root; - f.remote_package_root = f.package_root; - try loadManifest(f, f.package_root); - try checkBuildFileExistence(f); - if (!job_queue.recursive) return; - return queueJobsForDeps(f); - } else |err| switch (err) { - error.FileNotFound => { - log.debug("FileNotFound: {f}", .{package_root}); - if (job_queue.read_only and f.lazy_status == .eager) return f.fail( - f.name_tok, - try eb.printString("package not found at '{f}'", .{package_root}), - ); - }, - error.Canceled => |e| return e, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{ - package_root, e, - }), - }); - return error.FetchFailed; - }, - } - } - - // Check global cache before remote fetch. - const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()}); - const cached_tarball_path: Cache.Path = .{ - .root_dir = job_queue.global_cache, - .sub_path = cached_tarball_sub_path, - }; - if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) |file| { - log.debug("found global cached tarball {f}", .{cached_tarball_path}); - var resource: Resource = .{ .file = file.reader(io, &resource_buffer) }; - return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true); - } else |err| switch (err) { - error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}), - error.Canceled => |e| return e, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to open globally cached package {f}: {t}", .{ - cached_tarball_path, e, - }), - }); - return error.FetchFailed; - }, - } - - switch (f.lazy_status) { - .eager => {}, - .available => if (!job_queue.unlazy_set.contains(expected_hash)) { - f.lazy_status = .unavailable; - return; - }, - .unavailable => unreachable, - } - } else if (job_queue.read_only) { - try eb.addRootErrorMessage(.{ - .msg = try eb.addString("dependency is missing hash field"), - .src_loc = try f.srcLoc(f.location_tok), - }); - return error.FetchFailed; - } - - // Fetch and unpack the remote into a temporary directory. - const uri = std.Uri.parse(remote.url) catch |err| return f.fail( - f.location_tok, - try eb.printString("invalid URI: {t}", .{err}), - ); - var resource: Resource = undefined; - try f.initResource(uri, &resource, &resource_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false); -} - -pub fn deinit(f: *Fetch) void { - f.error_bundle.deinit(); - f.arena.deinit(); -} - -/// Consumes `resource`, even if an error is returned. -fn runResource( - f: *Fetch, - uri_path: []const u8, - resource: *Resource, - remote_hash: ?Package.Hash, - disable_recompress: bool, -) RunError!void { - const job_queue = f.job_queue; - assert(!job_queue.read_only); - - const io = job_queue.io; - defer resource.deinit(io); - - const arena = f.arena.allocator(); - const eb = &f.error_bundle; - const rand_int = r: { - var x: u64 = undefined; - io.random(@ptrCast(&x)); - break :r x; - }; - const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int); - const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path; - const tmp_directory_path: Cache.Path = if (job_queue.local_storage) |ls| - try ls.pkg_root.join(arena, tmp_dir_sub_path) - else - .{ - .root_dir = job_queue.global_cache, - .sub_path = tmp_tmp_dir_sub_path, - }; - - const package_sub_path = blk: { - var tmp_directory: Cache.Directory = .{ - .path = tmp_directory_path.sub_path, - .handle = handle: { - const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{ - .open_options = .{ .iterate = true }, - }) catch |err| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{ - tmp_directory_path, err, - }), - }); - return error.FetchFailed; - }; - break :handle dir; - }, - }; - defer tmp_directory.handle.close(io); - - // Fetch and unpack a resource into a temporary directory. - var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); - - const pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; - - // Load, parse, and validate the unpacked build.zig.zon file. It is allowed - // for the file to be missing, in which case this fetched package is - // considered to be a "naked" package. - try loadManifest(f, pkg_path); - - const filter: Filter = .{ - .include_paths = if (f.have_manifest) f.manifest.paths else .{}, - }; - - // Ignore errors that were excluded by manifest, such as failure to - // create symlinks that weren't supposed to be included anyway. - try unpack_result.validate(f, filter); - - // Apply the manifest's inclusion rules to the temporary directory by - // deleting excluded files. - // Empty directories have already been omitted by `unpackResource`. - // Compute the package hash based on the remaining files in the temporary - // directory. - f.computed_hash = try computeHash(f, pkg_path, filter); - - if (unpack_result.root_dir.len > 0) - break :blk try tmp_directory_path.join(arena, unpack_result.root_dir); - - break :blk tmp_directory_path; - }; - - const computed_package_hash = computedPackageHash(f); - - // Rename the temporary directory into the local zig package directory. If - // the hash already exists, delete the temporary directory and leave the - // zig package directory untouched as it may be in use. This is done even - // if the hash is invalid, in case the package with the different hash is - // used in the future. - if (job_queue.local_storage) |ls| { - f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice()); - renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| { - try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "failed renaming temporary directory {f} into package cache directory {f}: {t}", - .{ package_sub_path, f.package_root, err }, - ) }); - return error.FetchFailed; - }; - } else { - f.package_root = tmp_directory_path; - } - f.remote_package_root = f.package_root; - - if (!disable_recompress) { - // Spin off a task to recompress the tarball, with filtered files deleted, into - // the global cache. - job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root }); - } - - // Remove temporary directory root if not already renamed to global cache. - if (!package_sub_path.eql(tmp_directory_path)) { - tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) { - error.Canceled => |e| return e, - else => |e| log.warn("failed deleting temporary directory {f}: {t}", .{ tmp_directory_path, e }), - }; - } - - // Validate the computed hash against the expected hash. If invalid, this - // job is done. - - if (remote_hash) |declared_hash| { - const hash_tok = f.hash_tok.unwrap().?; - if (!computed_package_hash.eql(&declared_hash)) { - return f.fail(hash_tok, try eb.printString( - "hash mismatch: manifest declares {s} but the fetched package has {s}", - .{ declared_hash.toSlice(), computed_package_hash.toSlice() }, - )); - } - } else if (!f.omit_missing_hash_error) { - const notes_len = 1; - try eb.addRootErrorMessage(.{ - .msg = try eb.addString("dependency is missing hash field"), - .src_loc = try f.srcLoc(f.location_tok), - .notes_len = notes_len, - }); - const notes_start = try eb.reserveNotes(notes_len); - eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}), - })); - return error.FetchFailed; - } - - // Spawn a new fetch job for each dependency in the manifest file. Use - // a mutex and a hash map so that redundant jobs do not get queued up. - if (!job_queue.recursive) return; - return queueJobsForDeps(f); -} - -pub fn computedPackageHash(f: *const Fetch) Package.Hash { - const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32); - if (f.have_manifest) { - const man = &f.manifest; - var version_buffer: [32]u8 = undefined; - const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{f}", .{man.version}) catch &version_buffer; - return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size); - } - // In the future build.zig.zon fields will be added to allow overriding these values - // for naked tarballs. - return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size); -} - -/// `computeHash` gets a free check for the existence of `build.zig`, but when -/// not computing a hash, we need to do a syscall to check for it. -fn checkBuildFileExistence(f: *Fetch) RunError!void { - const io = f.job_queue.io; - const eb = &f.error_bundle; - if (f.package_root.access(io, Package.build_zig_basename, .{})) |_| { - f.has_build_zig = true; - } else |err| switch (err) { - error.FileNotFound => {}, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to access '{f}{s}': {t}", .{ - f.package_root, Package.build_zig_basename, e, - }), - }); - return error.FetchFailed; - }, - } -} - -/// This function populates `f.manifest` or leaves it `null`. -fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { - const io = f.job_queue.io; - const eb = &f.error_bundle; - const arena = f.arena.allocator(); - const manifest_path = try pkg_root.join(arena, Manifest.basename); - - Manifest.load( - io, - arena, - manifest_path, - &f.manifest_ast, - eb, - &f.manifest, - f.allow_missing_paths_field, - ) catch |err| switch (err) { - error.FileNotFound => return, - error.Canceled => |e| return e, - error.ErrorsBundled => return error.FetchFailed, - else => |e| { - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }), - }); - return error.FetchFailed; - }, - }; - f.have_manifest = true; -} - -fn queueJobsForDeps(f: *Fetch) RunError!void { - const io = f.job_queue.io; - - assert(f.job_queue.recursive); - - // If the package does not have a build.zig.zon file then there are no dependencies. - if (!f.have_manifest) return; - const manifest = &f.manifest; - - const new_fetches, const prog_names = nf: { - const parent_arena = f.arena.allocator(); - const gpa = f.arena.child_allocator; - const cache_root = f.job_queue.global_cache; - const dep_names = manifest.dependencies.keys(); - const deps = manifest.dependencies.values(); - // Grab the new tasks into a temporary buffer so we can unlock that mutex - // as fast as possible. - // This overallocates any fetches that get skipped by the `continue` in the - // loop below. - const new_fetches = try parent_arena.alloc(Fetch, deps.len); - const prog_names = try parent_arena.alloc([]const u8, deps.len); - var new_fetch_index: usize = 0; - - try f.job_queue.mutex.lock(io); - defer f.job_queue.mutex.unlock(io); - - try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); - try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len)); - - // There are four cases here: - // * Correct hash is provided by manifest. - // - Hash map already has the entry, no need to add it again. - // * Incorrect hash is provided by manifest. - // - Hash mismatch error emitted; `queueJobsForDeps` is not called. - // * Hash is not provided by manifest. - // - Hash missing error emitted; `queueJobsForDeps` is not called. - // * path-based location is used without a hash. - // - Hash is added to the table based on the path alone before - // calling run(); no need to add it again. - // - // If we add a dep as lazy and then later try to add the same dep as eager, - // eagerness takes precedence and the existing entry is updated and re-scheduled - // for fetching. - - for (dep_names, deps) |dep_name, dep| { - var promoted_existing_to_eager = false; - const new_fetch = &new_fetches[new_fetch_index]; - const location: Location = switch (dep.location) { - .url => |url| .{ - .remote = .{ - .url = url, - .hash = h: { - const h = dep.hash orelse break :h null; - const pkg_hash: Package.Hash = .fromSlice(h); - if (h.len == 0) break :h pkg_hash; - const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); - if (gop.found_existing) { - if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) { - gop.value_ptr.*.lazy_status = .eager; - promoted_existing_to_eager = true; - } else { - continue; - } - } - gop.value_ptr.* = new_fetch; - break :h pkg_hash; - }, - }, - }, - .path => |rel_path| l: { - // This might produce an invalid path, which is checked for - // at the beginning of run(). - const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); - const pkg_hash = relativePathDigest(new_root, cache_root); - const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); - if (gop.found_existing) { - if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) { - gop.value_ptr.*.lazy_status = .eager; - promoted_existing_to_eager = true; - } else { - continue; - } - } - gop.value_ptr.* = new_fetch; - break :l .{ .relative_path = new_root }; - }, - }; - prog_names[new_fetch_index] = dep_name; - new_fetch_index += 1; - if (!promoted_existing_to_eager) { - f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); - } - new_fetch.* = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = location, - .location_tok = dep.location_tok, - .hash_tok = dep.hash_tok, - .name_tok = dep.name_tok, - .lazy_status = switch (f.job_queue.mode) { - .needed => if (dep.lazy) .available else .eager, - .all => .eager, - }, - .parent_package_root = f.package_root, - .remote_package_root = f.remote_package_root, - .parent_manifest_ast = &f.manifest_ast, - .prog_node = f.prog_node, - .job_queue = f.job_queue, - .omit_missing_hash_error = false, - .allow_missing_paths_field = true, - .use_latest_commit = false, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = undefined, - .manifest_ast = undefined, - .have_manifest = false, - .computed_hash = undefined, - .has_build_zig = false, - .oom_flag = false, - .latest_commit = null, - - .module = null, - }; - } - - f.prog_node.increaseEstimatedTotalItems(new_fetch_index); - - break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] }; - }; - - // Now it's time to dispatch tasks. - for (new_fetches, prog_names) |*new_fetch, prog_name| { - f.job_queue.group.async(io, workerRun, .{ new_fetch, prog_name }); - } -} - -pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash { - return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root)); -} - -pub fn workerRun(f: *Fetch, prog_name: []const u8) Io.Cancelable!void { - const prog_node = f.prog_node.start(prog_name, 0); - defer prog_node.end(); - - run(f) catch |err| switch (err) { - error.OutOfMemory => f.oom_flag = true, - error.Canceled => |e| return e, - error.FetchFailed => { - // Nothing to do because the errors are already reported in `error_bundle`, - // and a reference is kept to the `Fetch` task inside `all_fetches`. - }, - }; -} - -fn srcLoc( - f: *Fetch, - tok: std.zig.Ast.TokenIndex, -) Allocator.Error!ErrorBundle.SourceLocationIndex { - const ast = f.parent_manifest_ast orelse return .none; - const eb = &f.error_bundle; - const start_loc = ast.tokenLocation(0, tok); - const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root}); - const msg_off = 0; - return eb.addSourceLocation(.{ - .src_path = src_path, - .span_start = ast.tokenStart(tok), - .span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len), - .span_main = ast.tokenStart(tok) + msg_off, - .line = @intCast(start_loc.line), - .column = @intCast(start_loc.column), - .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), - }); -} - -fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { - const eb = &f.error_bundle; - try eb.addRootErrorMessage(.{ - .msg = msg_str, - .src_loc = try f.srcLoc(msg_tok), - }); - return error.FetchFailed; -} - -const Resource = union(enum) { - file: Io.File.Reader, - http_request: HttpRequest, - git: Git, - dir: Io.Dir, - - const Git = struct { - session: git.Session, - fetch_stream: git.Session.FetchStream, - want_oid: git.Oid, - }; - - const HttpRequest = struct { - request: std.http.Client.Request, - response: std.http.Client.Response, - transfer_buffer: []u8, - decompress: std.http.Decompress, - decompress_buffer: []u8, - }; - - fn deinit(resource: *Resource, io: Io) void { - switch (resource.*) { - .file => |*file_reader| file_reader.file.close(io), - .http_request => |*http_request| http_request.request.deinit(), - .git => |*git_resource| { - git_resource.fetch_stream.deinit(); - }, - .dir => |*dir| dir.close(io), - } - resource.* = undefined; - } - - fn reader(resource: *Resource) *Io.Reader { - return switch (resource.*) { - .file => |*file_reader| return &file_reader.interface, - .http_request => |*http_request| return http_request.response.readerDecompressing( - http_request.transfer_buffer, - &http_request.decompress, - http_request.decompress_buffer, - ), - .git => |*g| return &g.fetch_stream.reader, - .dir => unreachable, - }; - } -}; - -const FileType = enum { - tar, - @"tar.gz", - @"tar.xz", - @"tar.zst", - git_pack, - zip, - - fn fromPath(file_path: []const u8) ?FileType { - if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar; - if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz"; - if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz"; - if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst"; - if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst"; - if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip; - if (ascii.endsWithIgnoreCase(file_path, ".jar")) return .zip; - return null; - } - - /// Parameter is a content-disposition header value. - fn fromContentDisposition(cd_header: []const u8) ?FileType { - const attach_end = ascii.findIgnoreCase(cd_header, "attachment;") orelse - return null; - - var value_start = ascii.findIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse - return null; - value_start += "filename".len; - if (cd_header[value_start] == '*') { - value_start += 1; - } - if (cd_header[value_start] != '=') return null; - value_start += 1; - - var value_end = std.mem.indexOfPos(u8, cd_header, value_start, ";") orelse cd_header.len; - if (cd_header[value_end - 1] == '\"') { - value_end -= 1; - } - return fromPath(cd_header[value_start..value_end]); - } - - test fromContentDisposition { - try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42")); - try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\"")); - try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); - try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); - try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); - try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\"")); - - try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); - try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); - try std.testing.expect(fromContentDisposition("attachment; size=42") == null); - try std.testing.expect(fromContentDisposition("inline; size=42") == null); - try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null); - try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null); - } -}; - -const init_resource_buffer_size = git.Packet.max_data_length; - -fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u8) RunError!void { - const io = f.job_queue.io; - const arena = f.arena.allocator(); - const eb = &f.error_bundle; - - if (ascii.eqlIgnoreCase(uri.scheme, "file")) { - const path = try uri.path.toRawMaybeAlloc(arena); - const file = f.parent_package_root.openFile(io, path, .{}) catch |err| { - return f.fail(f.location_tok, try eb.printString("unable to open {f}/{s}: {t}", .{ - f.parent_package_root, path, err, - })); - }; - resource.* = .{ .file = file.reader(io, reader_buffer) }; - return; - } - - const http_client = f.job_queue.http_client; - - if (ascii.eqlIgnoreCase(uri.scheme, "http") or - ascii.eqlIgnoreCase(uri.scheme, "https")) - { - resource.* = .{ .http_request = .{ - .request = http_client.request(.GET, uri, .{}) catch |err| - return f.fail(f.location_tok, try eb.printString("server connection failed: {t}", .{err})), - .response = undefined, - .transfer_buffer = reader_buffer, - .decompress_buffer = &.{}, - .decompress = undefined, - } }; - const request = &resource.http_request.request; - errdefer request.deinit(); - - request.sendBodiless() catch |err| - return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err})); - - var redirect_buffer: [8000]u8 = undefined; - const response = &resource.http_request.response; - response.* = request.receiveHead(&redirect_buffer) catch |err| switch (err) { - error.ReadFailed => { - return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{ - request.connection.?.getReadError().?, - })); - }, - else => |e| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})), - }; - - if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString( - "bad HTTP response code: '{d} {s}'", - .{ response.head.status, response.head.status.phrase() orelse "" }, - )); - - resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); - return; - } - - if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or - ascii.eqlIgnoreCase(uri.scheme, "git+https")) - { - var transport_uri = uri; - transport_uri.scheme = uri.scheme["git+".len..]; - var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch |err| { - return f.fail( - f.location_tok, - try eb.printString("unable to discover remote git server capabilities: {t}", .{err}), - ); - }; - - const want_oid = want_oid: { - const want_ref = - if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD"; - if (git.Oid.parseAny(want_ref)) |oid| break :want_oid oid else |_| {} - - const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); - const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); - - var ref_iterator: git.Session.RefIterator = undefined; - session.listRefs(&ref_iterator, .{ - .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, - .include_peeled = true, - .buffer = reader_buffer, - }) catch |err| return f.fail(f.location_tok, try eb.printString("unable to list refs: {t}", .{err})); - defer ref_iterator.deinit(); - while (ref_iterator.next() catch |err| { - return f.fail(f.location_tok, try eb.printString( - "unable to iterate refs: {s}", - .{@errorName(err)}, - )); - }) |ref| { - if (std.mem.eql(u8, ref.name, want_ref) or - std.mem.eql(u8, ref.name, want_ref_head) or - std.mem.eql(u8, ref.name, want_ref_tag)) - { - break :want_oid ref.peeled orelse ref.oid; - } - } - return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); - }; - if (f.use_latest_commit) { - f.latest_commit = want_oid; - } else if (uri.fragment == null) { - const notes_len = 1; - try eb.addRootErrorMessage(.{ - .msg = try eb.addString("url field is missing an explicit ref"), - .src_loc = try f.srcLoc(f.location_tok), - .notes_len = notes_len, - }); - const notes_start = try eb.reserveNotes(notes_len); - eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("try .url = \"{f}#{f}\",", .{ - uri.fmt(.{ .scheme = true, .authority = true, .path = true }), - want_oid, - }), - })); - return error.FetchFailed; - } - - var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined; - _ = std.fmt.bufPrint(&want_oid_buf, "{f}", .{want_oid}) catch unreachable; - resource.* = .{ .git = .{ - .session = session, - .fetch_stream = undefined, - .want_oid = want_oid, - } }; - const fetch_stream = &resource.git.fetch_stream; - session.fetch(fetch_stream, &.{&want_oid_buf}, reader_buffer) catch |err| { - return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err})); - }; - errdefer fetch_stream.deinit(fetch_stream); - - return; - } - - return f.fail(f.location_tok, try eb.printString("unsupported URL scheme: {s}", .{uri.scheme})); -} - -fn unpackResource( - f: *Fetch, - resource: *Resource, - uri_path: []const u8, - tmp_directory: Cache.Directory, -) RunError!UnpackResult { - const eb = &f.error_bundle; - const file_type = switch (resource.*) { - .file => FileType.fromPath(uri_path) orelse - return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), - - .http_request => |*http_request| ft: { - const head = &http_request.response.head; - - // Content-Type takes first precedence. - const content_type = head.content_type orelse - return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); - - // Extract the MIME type, ignoring charset and boundary directives - const mime_type_end = std.mem.indexOf(u8, content_type, ";") orelse content_type.len; - const mime_type = content_type[0..mime_type_end]; - - if (ascii.eqlIgnoreCase(mime_type, "application/x-tar")) - break :ft .tar; - - if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or - ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or - ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or - ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or - ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed")) - { - break :ft .@"tar.gz"; - } - - if (ascii.eqlIgnoreCase(mime_type, "application/x-xz")) - break :ft .@"tar.xz"; - - if (ascii.eqlIgnoreCase(mime_type, "application/zstd")) - break :ft .@"tar.zst"; - - if (ascii.eqlIgnoreCase(mime_type, "application/zip") or - ascii.eqlIgnoreCase(mime_type, "application/x-zip-compressed") or - ascii.eqlIgnoreCase(mime_type, "application/java-archive")) - { - break :ft .zip; - } - - if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and - !ascii.eqlIgnoreCase(mime_type, "application/x-compressed")) - { - return f.fail(f.location_tok, try eb.printString( - "unrecognized 'Content-Type' header: '{s}'", - .{content_type}, - )); - } - - // Next, the filename from 'content-disposition: attachment' takes precedence. - if (head.content_disposition) |cd_header| { - break :ft FileType.fromContentDisposition(cd_header) orelse { - return f.fail(f.location_tok, try eb.printString( - "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", - .{cd_header}, - )); - }; - } - - // Finally, the path from the URI is used. - break :ft FileType.fromPath(uri_path) orelse { - return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})); - }; - }, - - .git => .git_pack, - - .dir => |dir| { - f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { - return f.fail(f.location_tok, try eb.printString("unable to copy directory '{s}': {t}", .{ - uri_path, err, - })); - }; - return .{}; - }, - }; - - switch (file_type) { - .tar => { - return unpackTarball(f, tmp_directory.handle, resource.reader()); - }, - .@"tar.gz" => { - var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; - var decompress: std.compress.flate.Decompress = .init(resource.reader(), .gzip, &flate_buffer); - return try unpackTarball(f, tmp_directory.handle, &decompress.reader); - }, - .@"tar.xz" => { - const gpa = f.arena.child_allocator; - var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch |err| - return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err})); - defer decompress.deinit(); - return try unpackTarball(f, tmp_directory.handle, &decompress.reader); - }, - .@"tar.zst" => { - const window_len = std.compress.zstd.default_window_len; - const window_buffer = try f.arena.allocator().alloc(u8, window_len + std.compress.zstd.block_size_max); - var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{ - .verify_checksum = false, - .window_len = window_len, - }); - return try unpackTarball(f, tmp_directory.handle, &decompress.reader); - }, - .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) { - error.FetchFailed, error.OutOfMemory => |e| return e, - else => |e| return f.fail(f.location_tok, try eb.printString("unable to unpack git files: {t}", .{e})), - }, - .zip => return unzip(f, tmp_directory.handle, resource.reader()) catch |err| switch (err) { - error.ReadFailed => return f.fail(f.location_tok, try eb.printString( - "failed reading resource: {t}", - .{err}, - )), - else => |e| return e, - }, - } -} - -fn unpackTarball(f: *Fetch, out_dir: Io.Dir, reader: *Io.Reader) RunError!UnpackResult { - const eb = &f.error_bundle; - const arena = f.arena.allocator(); - const io = f.job_queue.io; - - var diagnostics: std.tar.Diagnostics = .{ .allocator = arena }; - - std.tar.pipeToFileSystem(io, out_dir, reader, .{ - .diagnostics = &diagnostics, - .strip_components = 0, - .mode_mode = .ignore, - .exclude_empty_directories = true, - }) catch |err| return f.fail( - f.location_tok, - try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}), - ); - - var res: UnpackResult = .{ .root_dir = diagnostics.root_dir }; - if (diagnostics.errors.items.len > 0) { - try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball"); - for (diagnostics.errors.items) |item| { - switch (item) { - .unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code), - .unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code), - .unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @intFromEnum(i.file_type)), - .components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0 - } - } - } - return res; -} - -fn unzip( - f: *Fetch, - out_dir: Io.Dir, - reader: *Io.Reader, -) error{ ReadFailed, OutOfMemory, Canceled, FetchFailed }!UnpackResult { - // We write the entire contents to a file first because zip files - // must be processed back to front and they could be too large to - // load into memory. - - const io = f.job_queue.io; - const cache_root = f.job_queue.global_cache; - const prefix = "tmp/"; - const suffix = ".zip"; - const eb = &f.error_bundle; - const random_len = @sizeOf(u64) * 2; - - var zip_path: [prefix.len + random_len + suffix.len]u8 = undefined; - zip_path[0..prefix.len].* = prefix.*; - zip_path[prefix.len + random_len ..].* = suffix.*; - - var zip_file = while (true) { - const random_integer = r: { - var x: u64 = undefined; - io.random(@ptrCast(&x)); - break :r x; - }; - zip_path[prefix.len..][0..random_len].* = std.fmt.hex(random_integer); - - break cache_root.handle.createFile(io, &zip_path, .{ - .exclusive = true, - .read = true, - }) catch |err| switch (err) { - error.PathAlreadyExists => continue, - error.FileNotFound => { - cache_root.handle.createDir(io, prefix, .default_dir) catch |dir_err| switch (dir_err) { - error.Canceled => |e| return e, - // error.PathAlreadyExists is considered a failure here because - // it implies that the prefix is not a directory. - else => |e| return f.fail( - f.location_tok, - try eb.printString("failed to create temporary directory: {t}", .{e}), - ), - }; - continue; - }, - error.Canceled => |e| return e, - else => |e| return f.fail( - f.location_tok, - try eb.printString("failed to create temporary zip file: {t}", .{e}), - ), - }; - }; - defer zip_file.close(io); - var zip_file_buffer: [4096]u8 = undefined; - var zip_file_reader = b: { - var zip_file_writer = zip_file.writer(io, &zip_file_buffer); - - _ = reader.streamRemaining(&zip_file_writer.interface) catch |err| switch (err) { - error.ReadFailed => |e| return e, - error.WriteFailed => return f.fail( - f.location_tok, - try eb.printString("failed writing temporary zip file: {t}", .{err}), - ), - }; - zip_file_writer.interface.flush() catch |err| return f.fail( - f.location_tok, - try eb.printString("failed writing temporary zip file: {t}", .{err}), - ); - break :b zip_file_writer.moveToReader(); - }; - - var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() }; - // no need to deinit since we are using an arena allocator - - zip_file_reader.seekTo(0) catch |err| - return f.fail(f.location_tok, try eb.printString("failed to seek temporary zip file: {t}", .{err})); - std.zip.extract(out_dir, &zip_file_reader, .{ - .allow_backslashes = true, - .diagnostics = &diagnostics, - }) catch |err| return f.fail(f.location_tok, try eb.printString("zip extract failed: {t}", .{err})); - - cache_root.handle.deleteFile(io, &zip_path) catch |err| - return f.fail(f.location_tok, try eb.printString("delete temporary zip failed: {t}", .{err})); - - return .{ .root_dir = diagnostics.root_dir }; -} - -fn unpackGitPack(f: *Fetch, out_dir: Io.Dir, resource: *Resource.Git) anyerror!UnpackResult { - const io = f.job_queue.io; - const arena = f.arena.allocator(); - // TODO don't try to get a gpa from an arena. expose this dependency higher up - // because the backing of arena could be page allocator - const gpa = f.arena.child_allocator; - const object_format: git.Oid.Format = resource.want_oid; - - var res: UnpackResult = .{}; - // The .git directory is used to store the packfile and associated index, but - // we do not attempt to replicate the exact structure of a real .git - // directory, since that isn't relevant for fetching a package. - { - var pack_dir = try out_dir.createDirPathOpen(io, ".git", .{}); - defer pack_dir.close(io); - var pack_file = try pack_dir.createFile(io, "pkg.pack", .{ .read = true }); - defer pack_file.close(io); - var pack_file_buffer: [4096]u8 = undefined; - var pack_file_reader = b: { - var pack_file_writer = pack_file.writer(io, &pack_file_buffer); - const fetch_reader = &resource.fetch_stream.reader; - _ = try fetch_reader.streamRemaining(&pack_file_writer.interface); - try pack_file_writer.interface.flush(); - break :b pack_file_writer.moveToReader(); - }; - - var index_file = try pack_dir.createFile(io, "pkg.idx", .{ .read = true }); - defer index_file.close(io); - var index_file_buffer: [2000]u8 = undefined; - var index_file_writer = index_file.writer(io, &index_file_buffer); - { - const index_prog_node = f.prog_node.start("Index pack", 0); - defer index_prog_node.end(); - try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer); - } - - { - var index_file_reader = index_file.reader(io, &index_file_buffer); - const checkout_prog_node = f.prog_node.start("Checkout", 0); - defer checkout_prog_node.end(); - var repository: git.Repository = undefined; - try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader); - defer repository.deinit(); - var diagnostics: git.Diagnostics = .{ .allocator = arena }; - try repository.checkout(io, out_dir, resource.want_oid, &diagnostics); - - if (diagnostics.errors.items.len > 0) { - try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile"); - for (diagnostics.errors.items) |item| { - switch (item) { - .unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code), - .unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code), - } - } - } - } - } - - try out_dir.deleteTree(io, ".git"); - return res; -} - -fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void { - const gpa = f.arena.child_allocator; - const io = f.job_queue.io; - // Recursive directory copy. - var it = try dir.walk(gpa); - defer it.deinit(); - while (try it.next(io)) |entry| { - switch (entry.kind) { - .directory => {}, // omit empty directories - .file => { - dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}) catch |err| switch (err) { - error.FileNotFound => { - if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.createDirPath(io, dirname); - try dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}); - }, - else => |e| return e, - }; - }, - .sym_link => { - var buf: [fs.max_path_bytes]u8 = undefined; - const link_name = buf[0..try dir.readLink(io, entry.path, &buf)]; - // TODO: if this would create a symlink to outside - // the destination directory, fail with an error instead. - tmp_dir.symLink(io, link_name, entry.path, .{}) catch |err| switch (err) { - error.FileNotFound => { - if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.createDirPath(io, dirname); - try tmp_dir.symLink(io, link_name, entry.path, .{}); - }, - else => |e| return e, - }; - }, - else => return error.IllegalFileTypeInPackage, - } - } -} - -pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) !void { - var handled_missing_dir = false; - while (true) { - Io.Dir.rename( - tmp_path.root_dir.handle, - tmp_path.sub_path, - dest_path.root_dir.handle, - dest_path.sub_path, - io, - ) catch |err| switch (err) { - error.FileNotFound => { - if (handled_missing_dir) return err; - const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?; - dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch |er| switch (er) { - error.PathAlreadyExists => handled_missing_dir = true, - else => |e| return e, - }; - continue; - }, - error.DirNotEmpty, error.AccessDenied => { - // Package has been already downloaded and may already be in use on the system. - tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch |er| switch (er) { - error.Canceled => |e| return e, - // Garbage files leftover in zig-cache/tmp/ is, as they say - // on Star Trek, "operating within normal parameters". - else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }), - }; - }, - else => |e| return e, - }; - break; - } -} - -const ComputedHash = struct { - digest: Package.Hash.Digest, - total_size: u64, -}; - -/// Assumes that files not included in the package have already been filtered -/// prior to calling this function. This ensures that files not protected by -/// the hash are not present on the file system. Empty directories are *not -/// hashed* and must not be present on the file system when calling this -/// function. -fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash { - const io = f.job_queue.io; - // All the path name strings need to be in memory for sorting. - const arena = f.arena.allocator(); - const gpa = f.arena.child_allocator; - const eb = &f.error_bundle; - const root_dir = pkg_path.root_dir.handle; - - // Collect all files, recursively, then sort. - var all_files = std.array_list.Managed(*HashedFile).init(gpa); - defer all_files.deinit(); - - var deleted_files = std.array_list.Managed(*DeletedFile).init(gpa); - defer deleted_files.deinit(); - - // Track directories which had any files deleted from them so that empty directories - // can be deleted. - var sus_dirs: std.array_hash_map.String(void) = .empty; - defer sus_dirs.deinit(gpa); - - var walker = try root_dir.walk(gpa); - defer walker.deinit(); - - // Total number of bytes of file contents included in the package. - var total_size: u64 = 0; - - { - // The final hash will be a hash of each file hashed independently. This - // allows hashing in parallel. - var group: Io.Group = .init; - defer group.cancel(io); - - while (walker.next(io) catch |err| { - try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "unable to walk temporary directory '{f}': {t}", - .{ pkg_path, err }, - ) }); - return error.FetchFailed; - }) |entry| { - if (entry.kind == .directory) continue; - - const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path); - if (!filter.includePath(entry_pkg_path)) { - // Delete instead of including in hash calculation. - const fs_path = try arena.dupe(u8, entry.path); - - // Also track the parent directory in case it becomes empty. - if (fs.path.dirname(fs_path)) |parent| - try sus_dirs.put(gpa, parent, {}); - - const deleted_file = try arena.create(DeletedFile); - deleted_file.* = .{ - .fs_path = fs_path, - .failure = undefined, // to be populated by the worker - }; - group.async(io, workerDeleteFile, .{ io, root_dir, deleted_file }); - try deleted_files.append(deleted_file); - continue; - } - - const kind: HashedFile.Kind = switch (entry.kind) { - .directory => unreachable, - .file => .file, - .sym_link => .link, - else => return f.fail(f.location_tok, try eb.printString( - "package contains '{s}' which has illegal file type '{t}'", - .{ entry.path, entry.kind }, - )), - }; - - if (std.mem.eql(u8, entry_pkg_path, Package.build_zig_basename)) - f.has_build_zig = true; - - const fs_path = try arena.dupe(u8, entry.path); - const hashed_file = try arena.create(HashedFile); - hashed_file.* = .{ - .fs_path = fs_path, - .normalized_path = try normalizePathAlloc(arena, entry_pkg_path), - .kind = kind, - .hash = undefined, // to be populated by the worker - .failure = undefined, // to be populated by the worker - .size = undefined, // to be populated by the worker - }; - group.async(io, workerHashFile, .{ io, root_dir, hashed_file }); - try all_files.append(hashed_file); - } - - try group.await(io); - } - - { - // Sort by length, descending, so that child directories get removed first. - sus_dirs.sortUnstable(@as(struct { - keys: []const []const u8, - pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { - return ctx.keys[b_index].len < ctx.keys[a_index].len; - } - }, .{ .keys = sus_dirs.keys() })); - - // During this loop, more entries will be added, so we must loop by index. - var i: usize = 0; - while (i < sus_dirs.count()) : (i += 1) { - const sus_dir = sus_dirs.keys()[i]; - root_dir.deleteDir(io, sus_dir) catch |err| switch (err) { - error.DirNotEmpty => continue, - error.FileNotFound => continue, - else => |e| { - try eb.addRootErrorMessage(.{ .msg = try eb.printString( - "unable to delete empty directory '{s}': {s}", - .{ sus_dir, @errorName(e) }, - ) }); - return error.FetchFailed; - }, - }; - if (fs.path.dirname(sus_dir)) |parent| { - try sus_dirs.put(gpa, parent, {}); - } - } - } - - std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); - - var hasher = Package.Hash.Algo.init(.{}); - var any_failures = false; - for (all_files.items) |hashed_file| { - hashed_file.failure catch |err| { - any_failures = true; - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to hash '{s}': {s}", .{ - hashed_file.fs_path, @errorName(err), - }), - }); - }; - hasher.update(&hashed_file.hash); - total_size += hashed_file.size; - } - for (deleted_files.items) |deleted_file| { - deleted_file.failure catch |err| { - any_failures = true; - try eb.addRootErrorMessage(.{ - .msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{ - deleted_file.fs_path, @errorName(err), - }), - }); - }; - } - - if (any_failures) return error.FetchFailed; - - if (f.job_queue.debug_hash) { - assert(!f.job_queue.recursive); - // Print something to stdout that can be text diffed to figure out why - // the package hash is different. - dumpHashInfo(io, all_files.items) catch |err| - std.process.fatal("unable to write to stdout: {t}", .{err}); - } - - return .{ - .digest = hasher.finalResult(), - .total_size = total_size, - }; -} - -fn dumpHashInfo(io: Io, all_files: []const *const HashedFile) !void { - var stdout_buffer: [1024]u8 = undefined; - var stdout_writer: Io.File.Writer = .initStreaming(.stdout(), io, &stdout_buffer); - dumpHashInfoWriter(&stdout_writer.interface, all_files) catch |err| switch (err) { - error.WriteFailed => return stdout_writer.err.?, - }; - try stdout_writer.flush(); -} - -fn dumpHashInfoWriter(w: *Io.Writer, all_files: []const *const HashedFile) Io.Writer.Error!void { - for (all_files) |hashed_file| { - try w.print("{t}: {x}: {s}\n", .{ hashed_file.kind, &hashed_file.hash, hashed_file.normalized_path }); - } -} - -fn workerHashFile(io: Io, dir: Io.Dir, hashed_file: *HashedFile) void { - hashed_file.failure = hashFileFallible(io, dir, hashed_file); -} - -fn workerDeleteFile(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) void { - deleted_file.failure = deleteFileFallible(io, dir, deleted_file); -} - -fn hashFileFallible(io: Io, dir: Io.Dir, hashed_file: *HashedFile) HashedFile.Error!void { - var buf: [8000]u8 = undefined; - var hasher = Package.Hash.Algo.init(.{}); - hasher.update(hashed_file.normalized_path); - var file_size: u64 = 0; - - switch (hashed_file.kind) { - .file => { - var file = try dir.openFile(io, hashed_file.fs_path, .{}); - defer file.close(io); - // Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463 - hasher.update(&.{ 0, 0 }); - var file_header: FileHeader = .{}; - while (true) { - const bytes_read = try file.readPositional(io, &.{&buf}, file_size); - if (bytes_read == 0) break; - file_size += bytes_read; - hasher.update(buf[0..bytes_read]); - file_header.update(buf[0..bytes_read]); - } - if (file_header.isExecutable()) { - try setExecutable(io, file); - } - }, - .link => { - const link_name = buf[0..try dir.readLink(io, hashed_file.fs_path, &buf)]; - if (fs.path.sep != canonical_sep) { - // Package hashes are intended to be consistent across - // platforms which means we must normalize path separators - // inside symlinks. - normalizePath(link_name); - } - hasher.update(link_name); - }, - } - hasher.final(&hashed_file.hash); - hashed_file.size = file_size; -} - -fn deleteFileFallible(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { - try dir.deleteFile(io, deleted_file.fs_path); -} - -fn setExecutable(io: Io, file: Io.File) !void { - if (!Io.File.Permissions.has_executable_bit) return; - try file.setPermissions(io, .executable_file); -} - -const DeletedFile = struct { - fs_path: []const u8, - failure: Error!void, - - const Error = - Io.Dir.DeleteFileError || - Io.Dir.DeleteDirError; -}; - -const HashedFile = struct { - fs_path: []const u8, - normalized_path: []const u8, - hash: Package.Hash.Digest, - failure: Error!void, - kind: Kind, - size: u64, - - const Error = - Io.File.OpenError || - Io.File.ReadPositionalError || - Io.File.StatError || - Io.File.SetPermissionsError || - Io.Dir.ReadLinkError; - - const Kind = enum { file, link }; - - fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { - _ = context; - return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path); - } -}; - -/// Strips root directory name from file system path. -fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 { - if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path; - - if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) { - return fs_path[root_dir.len + 1 ..]; - } - - return fs_path; -} - -/// Make a file system path identical independently of operating system path inconsistencies. -/// This converts backslashes into forward slashes. -fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 { - const normalized = try arena.dupe(u8, pkg_path); - if (fs.path.sep == canonical_sep) return normalized; - normalizePath(normalized); - return normalized; -} - -const canonical_sep = fs.path.sep_posix; - -fn normalizePath(bytes: []u8) void { - assert(fs.path.sep != canonical_sep); - std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep); -} - -const Filter = struct { - include_paths: std.array_hash_map.String(void) = .empty, - - /// sub_path is relative to the package root. - pub fn includePath(self: *const Filter, sub_path: []const u8) bool { - if (self.include_paths.count() == 0) return true; - if (self.include_paths.contains("")) return true; - if (self.include_paths.contains(".")) return true; - if (self.include_paths.contains(sub_path)) return true; - - // Check if any included paths are parent directories of sub_path. - var dirname = sub_path; - while (std.fs.path.dirname(dirname)) |next_dirname| { - if (self.include_paths.contains(next_dirname)) return true; - dirname = next_dirname; - } - - return false; - } - - test includePath { - const gpa = std.testing.allocator; - var filter: Filter = .{}; - defer filter.include_paths.deinit(gpa); - - try filter.include_paths.put(gpa, "src", {}); - try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c")); - try std.testing.expect(!filter.includePath(".gitignore")); - } -}; - -pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash { - if (dep.hash) |h| return .fromSlice(h); - - switch (dep.location) { - .url => return null, - .path => |rel_path| { - var buf: [fs.max_path_bytes]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buf); - const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch - return null; - return relativePathDigest(new_root, cache_root); - }, - } -} - -// Detects executable header: ELF or Macho-O magic header or shebang line. -const FileHeader = struct { - header: [4]u8 = undefined, - bytes_read: usize = 0, - - pub fn update(self: *FileHeader, buf: []const u8) void { - if (self.bytes_read >= self.header.len) return; - const n = @min(self.header.len - self.bytes_read, buf.len); - @memcpy(self.header[self.bytes_read..][0..n], buf[0..n]); - self.bytes_read += n; - } - - fn isScript(self: *FileHeader) bool { - const shebang = "#!"; - return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang); - } - - fn isElf(self: *FileHeader) bool { - const elf_magic = std.elf.MAGIC; - return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic); - } - - fn isMachO(self: *FileHeader) bool { - if (self.bytes_read < 4) return false; - const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian()); - return magic_number == std.macho.MH_MAGIC or - magic_number == std.macho.MH_MAGIC_64 or - magic_number == std.macho.FAT_MAGIC or - magic_number == std.macho.FAT_MAGIC_64 or - magic_number == std.macho.MH_CIGAM or - magic_number == std.macho.MH_CIGAM_64 or - magic_number == std.macho.FAT_CIGAM or - magic_number == std.macho.FAT_CIGAM_64; - } - - pub fn isExecutable(self: *FileHeader) bool { - return self.isScript() or self.isElf() or self.isMachO(); - } -}; - -test FileHeader { - var h: FileHeader = .{}; - try std.testing.expect(!h.isExecutable()); - - const elf_magic = std.elf.MAGIC; - h.update(elf_magic[0..2]); - try std.testing.expect(!h.isExecutable()); - h.update(elf_magic[2..4]); - try std.testing.expect(h.isExecutable()); - - h.update(elf_magic[2..4]); - try std.testing.expect(h.isExecutable()); - - const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE }; - h.bytes_read = 0; - h.update(&macho64_magic_bytes); - try std.testing.expect(h.isExecutable()); - - const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF }; - h.bytes_read = 0; - h.update(&macho64_cigam_bytes); - try std.testing.expect(h.isExecutable()); -} - -// Result of the `unpackResource` operation. Enables collecting errors from -// tar/git diagnostic, filtering that errors by manifest inclusion rules and -// emitting remaining errors to an `ErrorBundle`. -const UnpackResult = struct { - errors: []Error = undefined, - errors_count: usize = 0, - root_error_message: []const u8 = "", - - // A non empty value means that the package contents are inside a - // sub-directory indicated by the named path. - root_dir: []const u8 = "", - - const Error = union(enum) { - unable_to_create_sym_link: struct { - code: anyerror, - file_name: []const u8, - link_name: []const u8, - }, - unable_to_create_file: struct { - code: anyerror, - file_name: []const u8, - }, - unsupported_file_type: struct { - file_name: []const u8, - file_type: u8, - }, - - fn excluded(self: Error, filter: Filter) bool { - const file_name = switch (self) { - .unable_to_create_file => |info| info.file_name, - .unable_to_create_sym_link => |info| info.file_name, - .unsupported_file_type => |info| info.file_name, - }; - return !filter.includePath(file_name); - } - }; - - fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void { - self.root_error_message = try arena.dupe(u8, root_error_message); - self.errors = try arena.alloc(UnpackResult.Error, n); - } - - fn hasErrors(self: *UnpackResult) bool { - return self.errors_count > 0; - } - - fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void { - self.errors[self.errors_count] = .{ .unable_to_create_file = .{ - .code = err, - .file_name = file_name, - } }; - self.errors_count += 1; - } - - fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void { - self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{ - .code = err, - .file_name = file_name, - .link_name = link_name, - } }; - self.errors_count += 1; - } - - fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void { - self.errors[self.errors_count] = .{ .unsupported_file_type = .{ - .file_name = file_name, - .file_type = file_type, - } }; - self.errors_count += 1; - } - - fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void { - if (self.errors_count == 0) return; - - var unfiltered_errors: u32 = 0; - for (self.errors) |item| { - if (item.excluded(filter)) continue; - unfiltered_errors += 1; - } - if (unfiltered_errors == 0) return; - - // Emmit errors to an `ErrorBundle`. - const eb = &f.error_bundle; - try eb.addRootErrorMessage(.{ - .msg = try eb.addString(self.root_error_message), - .src_loc = try f.srcLoc(f.location_tok), - .notes_len = unfiltered_errors, - }); - var note_i: u32 = try eb.reserveNotes(unfiltered_errors); - for (self.errors) |item| { - if (item.excluded(filter)) continue; - switch (item) { - .unable_to_create_sym_link => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{ - info.file_name, info.link_name, @errorName(info.code), - }), - })); - }, - .unable_to_create_file => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("unable to create file '{s}': {s}", .{ - info.file_name, @errorName(info.code), - }), - })); - }, - .unsupported_file_type => |info| { - eb.extra.items[note_i] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{ - info.file_name, info.file_type, - }), - })); - }, - } - note_i += 1; - } - - return error.FetchFailed; - } - - test validate { - const gpa = std.testing.allocator; - var arena_instance = std.heap.ArenaAllocator.init(gpa); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - // fill UnpackResult with errors - var res: UnpackResult = .{}; - try res.allocErrors(arena, 4, "unable to unpack"); - try std.testing.expectEqual(0, res.errors_count); - res.unableToCreateFile("dir1/file1", error.File1); - res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError); - res.unableToCreateFile("dir1/file3", error.File3); - res.unsupportedFileType("dir2/file4", 'x'); - try std.testing.expectEqual(4, res.errors_count); - - // create filter, includes dir2, excludes dir1 - var filter: Filter = .{}; - try filter.include_paths.put(arena, "dir2", {}); - - // init Fetch - var fetch: Fetch = undefined; - fetch.parent_manifest_ast = null; - fetch.location_tok = 0; - try fetch.error_bundle.init(gpa); - defer fetch.error_bundle.deinit(); - - // validate errors with filter - try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter)); - - // output errors to string - var errors = try fetch.error_bundle.toOwnedBundle(""); - defer errors.deinit(gpa); - var aw: Io.Writer.Allocating = .init(gpa); - defer aw.deinit(); - try errors.renderToWriter(.{}, &aw.writer); - try std.testing.expectEqualStrings( - \\error: unable to unpack - \\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError - \\ note: file 'dir2/file4' has unsupported type 'x' - \\ - , aw.written()); - } -}; - -test { - _ = Filter; - _ = FileType; - _ = UnpackResult; -} diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig deleted file mode 100644 index d3bd1d701a618281355dba5e585d286cb3f9107f..0000000000000000000000000000000000000000 --- a/src/Package/Fetch/git.zig +++ /dev/null @@ -1,1750 +0,0 @@ -//! Git support for package fetching. -//! -//! This is not intended to support all features of Git: it is limited to the -//! basic functionality needed to clone a repository for the purpose of fetching -//! a package. - -const std = @import("std"); -const Io = std.Io; -const mem = std.mem; -const testing = std.testing; -const Allocator = mem.Allocator; -const Sha1 = std.crypto.hash.Sha1; -const Sha256 = std.crypto.hash.sha2.Sha256; -const assert = std.debug.assert; - -/// The ID of a Git object. -pub const Oid = union(Format) { - sha1: [Sha1.digest_length]u8, - sha256: [Sha256.digest_length]u8, - - pub const max_formatted_length = len: { - var max: usize = 0; - for (std.enums.values(Format)) |f| { - max = @max(max, f.formattedLength()); - } - break :len max; - }; - - pub const Format = enum { - sha1, - sha256, - - pub fn byteLength(f: Format) usize { - return switch (f) { - .sha1 => Sha1.digest_length, - .sha256 => Sha256.digest_length, - }; - } - - pub fn formattedLength(f: Format) usize { - return 2 * f.byteLength(); - } - }; - - const Hasher = union(Format) { - sha1: Sha1, - sha256: Sha256, - - fn init(oid_format: Format) Hasher { - return switch (oid_format) { - .sha1 => .{ .sha1 = Sha1.init(.{}) }, - .sha256 => .{ .sha256 = Sha256.init(.{}) }, - }; - } - - // Must be public for use from HashedReader and HashedWriter. - pub fn update(hasher: *Hasher, b: []const u8) void { - switch (hasher.*) { - inline else => |*inner| inner.update(b), - } - } - - fn finalResult(hasher: *Hasher) Oid { - return switch (hasher.*) { - inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), - }; - } - }; - - const Hashing = union(Format) { - sha1: Io.Writer.Hashing(Sha1), - sha256: Io.Writer.Hashing(Sha256), - - fn init(oid_format: Format, buffer: []u8) Hashing { - return switch (oid_format) { - .sha1 => .{ .sha1 = .init(buffer) }, - .sha256 => .{ .sha256 = .init(buffer) }, - }; - } - - fn writer(h: *@This()) *Io.Writer { - return switch (h.*) { - inline else => |*inner| &inner.writer, - }; - } - - fn final(h: *@This()) Oid { - switch (h.*) { - inline else => |*inner, tag| { - inner.writer.flush() catch unreachable; // hashers cannot fail - return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult()); - }, - } - } - }; - - pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { - assert(bytes.len == oid_format.byteLength()); - return switch (oid_format) { - inline else => |tag| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*), - }; - } - - pub fn readBytes(oid_format: Format, reader: *Io.Reader) !Oid { - return switch (oid_format) { - inline else => |tag| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*), - }; - } - - pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid { - switch (oid_format) { - inline else => |tag| { - if (s.len != tag.formattedLength()) return error.InvalidOid; - var bytes: [tag.byteLength()]u8 = undefined; - for (&bytes, 0..) |*b, i| { - b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; - } - return @unionInit(Oid, @tagName(tag), bytes); - }, - } - } - - test parse { - try testing.expectEqualSlices( - u8, - &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, - &(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1, - ); - try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588")); - try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")); - try testing.expectEqualSlices( - u8, - &.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A }, - &(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256, - ); - try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf")); - try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf")); - try testing.expectError(error.InvalidOid, parse(.sha1, "master")); - try testing.expectError(error.InvalidOid, parse(.sha256, "master")); - try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD")); - try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD")); - } - - pub fn parseAny(s: []const u8) error{InvalidOid}!Oid { - return for (std.enums.values(Format)) |f| { - if (s.len == f.formattedLength()) break parse(f, s); - } else error.InvalidOid; - } - - pub fn format(oid: Oid, writer: *Io.Writer) Io.Writer.Error!void { - try writer.print("{x}", .{oid.slice()}); - } - - pub fn slice(oid: *const Oid) []const u8 { - return switch (oid.*) { - inline else => |*bytes| bytes, - }; - } -}; - -pub const Diagnostics = struct { - allocator: Allocator, - errors: std.ArrayList(Error) = .empty, - - pub const Error = union(enum) { - unable_to_create_sym_link: struct { - code: anyerror, - file_name: []const u8, - link_name: []const u8, - }, - unable_to_create_file: struct { - code: anyerror, - file_name: []const u8, - }, - }; - - pub fn deinit(d: *Diagnostics) void { - for (d.errors.items) |item| { - switch (item) { - .unable_to_create_sym_link => |info| { - d.allocator.free(info.file_name); - d.allocator.free(info.link_name); - }, - .unable_to_create_file => |info| { - d.allocator.free(info.file_name); - }, - } - } - d.errors.deinit(d.allocator); - d.* = undefined; - } -}; - -pub const Repository = struct { - odb: Odb, - - pub fn init( - repo: *Repository, - allocator: Allocator, - format: Oid.Format, - pack_file: *Io.File.Reader, - index_file: *Io.File.Reader, - ) !void { - repo.* = .{ .odb = undefined }; - try repo.odb.init(allocator, format, pack_file, index_file); - } - - pub fn deinit(repository: *Repository) void { - repository.odb.deinit(); - repository.* = undefined; - } - - /// Checks out the repository at `commit_oid` to `worktree`. - pub fn checkout( - repository: *Repository, - io: Io, - worktree: Io.Dir, - commit_oid: Oid, - diagnostics: *Diagnostics, - ) !void { - try repository.odb.seekOid(commit_oid); - const tree_oid = tree_oid: { - const commit_object = try repository.odb.readObject(); - if (commit_object.type != .commit) return error.NotACommit; - break :tree_oid try getCommitTree(repository.odb.format, commit_object.data); - }; - try repository.checkoutTree(io, worktree, tree_oid, "", diagnostics); - } - - /// Checks out the tree at `tree_oid` to `worktree`. - fn checkoutTree( - repository: *Repository, - io: Io, - dir: Io.Dir, - tree_oid: Oid, - current_path: []const u8, - diagnostics: *Diagnostics, - ) !void { - try repository.odb.seekOid(tree_oid); - const tree_object = try repository.odb.readObject(); - if (tree_object.type != .tree) return error.NotATree; - // The tree object may be evicted from the object cache while we're - // iterating over it, so we can make a defensive copy here to make sure - // it remains valid until we're done with it - const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); - defer repository.odb.allocator.free(tree_data); - - var tree_iter: TreeIterator = .{ - .format = repository.odb.format, - .data = tree_data, - .pos = 0, - }; - while (try tree_iter.next()) |entry| { - switch (entry.type) { - .directory => { - try dir.createDir(io, entry.name, .default_dir); - var subdir = try dir.openDir(io, entry.name, .{}); - defer subdir.close(io); - const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name }); - defer repository.odb.allocator.free(sub_path); - try repository.checkoutTree(io, subdir, entry.oid, sub_path, diagnostics); - }, - .file => { - try repository.odb.seekOid(entry.oid); - const file_object = try repository.odb.readObject(); - if (file_object.type != .blob) return error.InvalidFile; - var file = dir.createFile(io, entry.name, .{ .exclusive = true }) catch |e| { - const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); - errdefer diagnostics.allocator.free(file_name); - try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_file = .{ - .code = e, - .file_name = file_name, - } }); - continue; - }; - defer file.close(io); - try file.writePositionalAll(io, file_object.data, 0); - }, - .symlink => { - try repository.odb.seekOid(entry.oid); - const symlink_object = try repository.odb.readObject(); - if (symlink_object.type != .blob) return error.InvalidFile; - const link_name = symlink_object.data; - dir.symLink(io, link_name, entry.name, .{}) catch |e| { - const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name }); - errdefer diagnostics.allocator.free(file_name); - const link_name_dup = try diagnostics.allocator.dupe(u8, link_name); - errdefer diagnostics.allocator.free(link_name_dup); - try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{ - .code = e, - .file_name = file_name, - .link_name = link_name_dup, - } }); - }; - }, - .gitlink => { - // Consistent with git archive behavior, create the directory but - // do nothing else - try dir.createDir(io, entry.name, .default_dir); - }, - } - } - } - - /// Returns the ID of the tree associated with the given commit (provided as - /// raw object data). - fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid { - if (!mem.startsWith(u8, commit_data, "tree ") or - commit_data.len < "tree ".len + format.formattedLength() + "\n".len or - commit_data["tree ".len + format.formattedLength()] != '\n') - { - return error.InvalidCommit; - } - return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]); - } - - const TreeIterator = struct { - format: Oid.Format, - data: []const u8, - pos: usize, - - const Entry = struct { - type: Type, - executable: bool, - name: [:0]const u8, - oid: Oid, - - const Type = enum(u4) { - directory = 0o4, - file = 0o10, - symlink = 0o12, - gitlink = 0o16, - }; - }; - - fn next(iterator: *TreeIterator) !?Entry { - if (iterator.pos == iterator.data.len) return null; - - const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; - const mode: packed struct { - permission: u9, - unused: u3, - type: u4, - } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); - const @"type" = std.enums.fromInt(Entry.Type, mode.type) orelse return error.InvalidTree; - const executable = switch (mode.permission) { - 0 => if (@"type" == .file) return error.InvalidTree else false, - 0o644 => if (@"type" != .file) return error.InvalidTree else false, - 0o755 => if (@"type" != .file) return error.InvalidTree else true, - else => return error.InvalidTree, - }; - iterator.pos = mode_end + 1; - - const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; - const name = iterator.data[iterator.pos..name_end :0]; - iterator.pos = name_end + 1; - - const oid_length = iterator.format.byteLength(); - if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; - const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]); - iterator.pos += oid_length; - - return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; - } - }; -}; - -/// A Git object database backed by a packfile. A packfile index is also used -/// for efficient access to objects in the packfile. -/// -/// The format of the packfile and its associated index are documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -const Odb = struct { - format: Oid.Format, - pack_file: *Io.File.Reader, - index_header: IndexHeader, - index_file: *Io.File.Reader, - cache: ObjectCache = .{}, - allocator: Allocator, - - /// Initializes the database from open pack and index files. - fn init( - odb: *Odb, - allocator: Allocator, - format: Oid.Format, - pack_file: *Io.File.Reader, - index_file: *Io.File.Reader, - ) !void { - try pack_file.seekTo(0); - try index_file.seekTo(0); - odb.* = .{ - .format = format, - .pack_file = pack_file, - .index_header = undefined, - .index_file = index_file, - .allocator = allocator, - }; - try odb.index_header.read(&index_file.interface); - } - - fn deinit(odb: *Odb) void { - odb.cache.deinit(odb.allocator); - odb.* = undefined; - } - - /// Reads the object at the current position in the database. - fn readObject(odb: *Odb) !Object { - var base_offset = odb.pack_file.logicalPos(); - var base_header: EntryHeader = undefined; - var delta_offsets: std.ArrayList(u64) = .empty; - defer delta_offsets.deinit(odb.allocator); - const base_object = while (true) { - if (odb.cache.get(base_offset)) |base_object| break base_object; - - base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface); - switch (base_header) { - .ofs_delta => |ofs_delta| { - try delta_offsets.append(odb.allocator, base_offset); - base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; - try odb.pack_file.seekTo(base_offset); - }, - .ref_delta => |ref_delta| { - try delta_offsets.append(odb.allocator, base_offset); - try odb.seekOid(ref_delta.base_object); - base_offset = odb.pack_file.logicalPos(); - }, - else => { - const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength()); - errdefer odb.allocator.free(base_data); - const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; - try odb.cache.put(odb.allocator, base_offset, base_object); - break base_object; - }, - } - }; - - const base_data = try resolveDeltaChain( - odb.allocator, - odb.format, - odb.pack_file, - base_object, - delta_offsets.items, - &odb.cache, - ); - - return .{ .type = base_object.type, .data = base_data }; - } - - /// Seeks to the beginning of the object with the given ID. - fn seekOid(odb: *Odb, oid: Oid) !void { - const oid_length = odb.format.byteLength(); - const key = oid.slice()[0]; - var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; - var end_index = odb.index_header.fan_out_table[key]; - const found_index = while (start_index < end_index) { - const mid_index = start_index + (end_index - start_index) / 2; - try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); - const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface); - switch (mem.order(u8, mid_oid.slice(), oid.slice())) { - .lt => start_index = mid_index + 1, - .gt => end_index = mid_index, - .eq => break mid_index, - } - } else return error.ObjectNotFound; - - const n_objects = odb.index_header.fan_out_table[255]; - const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); - try odb.index_file.seekTo(offset_values_start + found_index * 4); - const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big)); - const pack_offset = pack_offset: { - if (l1_offset.big) { - const l2_offset_values_start = offset_values_start + n_objects * 4; - try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); - break :pack_offset try odb.index_file.interface.takeInt(u64, .big); - } else { - break :pack_offset l1_offset.value; - } - }; - - try odb.pack_file.seekTo(pack_offset); - } -}; - -const Object = struct { - type: Type, - data: []const u8, - - const Type = enum { - commit, - tree, - blob, - tag, - }; -}; - -/// A cache for object data. -/// -/// The purpose of this cache is to speed up resolution of deltas by caching the -/// results of resolving delta objects, while maintaining a maximum cache size -/// to avoid excessive memory usage. If the total size of the objects in the -/// cache exceeds the maximum, the cache will begin evicting the least recently -/// used objects: when resolving delta chains, the most recently used objects -/// will likely be more helpful as they will be further along in the chain -/// (skipping earlier reconstruction steps). -/// -/// Object data stored in the cache is managed by the cache. It should not be -/// freed by the caller at any point after inserting it into the cache. Any -/// objects remaining in the cache will be freed when the cache itself is freed. -const ObjectCache = struct { - objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .empty, - lru_nodes: std.DoublyLinkedList = .{}, - lru_nodes_len: usize = 0, - byte_size: usize = 0, - - const max_byte_size = 128 * 1024 * 1024; // 128MiB - /// A list of offsets stored in the cache, with the most recently used - /// entries at the end. - const LruListNode = struct { - data: u64, - node: std.DoublyLinkedList.Node, - }; - const CacheEntry = struct { object: Object, lru_node: *LruListNode }; - - fn deinit(cache: *ObjectCache, allocator: Allocator) void { - var object_iterator = cache.objects.iterator(); - while (object_iterator.next()) |object| { - allocator.free(object.value_ptr.object.data); - allocator.destroy(object.value_ptr.lru_node); - } - cache.objects.deinit(allocator); - cache.* = undefined; - } - - /// Gets an object from the cache, moving it to the most recently used - /// position if it is present. - fn get(cache: *ObjectCache, offset: u64) ?Object { - if (cache.objects.get(offset)) |entry| { - cache.lru_nodes.remove(&entry.lru_node.node); - cache.lru_nodes.append(&entry.lru_node.node); - return entry.object; - } else { - return null; - } - } - - /// Puts an object in the cache, possibly evicting older entries if the - /// cache exceeds its maximum size. Note that, although old objects may - /// be evicted, the object just added to the cache with this function - /// will not be evicted before the next call to `put` or `deinit` even if - /// it exceeds the maximum cache size. - fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { - const lru_node = try allocator.create(LruListNode); - errdefer allocator.destroy(lru_node); - lru_node.data = offset; - - const gop = try cache.objects.getOrPut(allocator, offset); - if (gop.found_existing) { - cache.byte_size -= gop.value_ptr.object.data.len; - cache.lru_nodes.remove(&gop.value_ptr.lru_node.node); - cache.lru_nodes_len -= 1; - allocator.destroy(gop.value_ptr.lru_node); - allocator.free(gop.value_ptr.object.data); - } - gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; - cache.byte_size += object.data.len; - cache.lru_nodes.append(&lru_node.node); - cache.lru_nodes_len += 1; - - while (cache.byte_size > max_byte_size and cache.lru_nodes_len > 1) { - // The > 1 check is to make sure that we don't evict the most - // recently added node, even if it by itself happens to exceed the - // maximum size of the cache. - const evict_node: *LruListNode = @alignCast(@fieldParentPtr("node", cache.lru_nodes.popFirst().?)); - cache.lru_nodes_len -= 1; - const evict_offset = evict_node.data; - allocator.destroy(evict_node); - const evict_object = cache.objects.get(evict_offset).?.object; - cache.byte_size -= evict_object.data.len; - allocator.free(evict_object.data); - _ = cache.objects.remove(evict_offset); - } - } -}; - -/// A single pkt-line in the Git protocol. -/// -/// The format of a pkt-line is documented in -/// [protocol-common](https://git-scm.com/docs/protocol-common). The special -/// meanings of the delimiter and response-end packets are documented in -/// [protocol-v2](https://git-scm.com/docs/protocol-v2). -pub const Packet = union(enum) { - flush, - delimiter, - response_end, - data: []const u8, - - pub const max_data_length = 65516; - - /// Reads a packet in pkt-line format. - fn read(reader: *Io.Reader) !Packet { - const packet: Packet = try .peek(reader); - switch (packet) { - .data => |data| reader.toss(data.len), - else => {}, - } - return packet; - } - - /// Consumes the header of a pkt-line packet and reads any associated data - /// into the reader's buffer, but does not consume the data. - fn peek(reader: *Io.Reader) !Packet { - const length = std.fmt.parseUnsigned(u16, try reader.take(4), 16) catch return error.InvalidPacket; - switch (length) { - 0 => return .flush, - 1 => return .delimiter, - 2 => return .response_end, - 3 => return error.InvalidPacket, - else => if (length - 4 > max_data_length) return error.InvalidPacket, - } - return .{ .data = try reader.peek(length - 4) }; - } - - /// Writes a packet in pkt-line format. - fn write(packet: Packet, writer: *Io.Writer) !void { - switch (packet) { - .flush => try writer.writeAll("0000"), - .delimiter => try writer.writeAll("0001"), - .response_end => try writer.writeAll("0002"), - .data => |data| { - assert(data.len <= max_data_length); - try writer.print("{x:0>4}", .{data.len + 4}); - try writer.writeAll(data); - }, - } - } - - /// Returns the normalized form of textual packet data, stripping any - /// trailing '\n'. - /// - /// As documented in - /// [protocol-common](https://git-scm.com/docs/protocol-common#_pkt_line_format), - /// non-binary (textual) pkt-line data should contain a trailing '\n', but - /// is not required to do so (implementations must support both forms). - fn normalizeText(data: []const u8) []const u8 { - return if (mem.endsWith(u8, data, "\n")) - data[0 .. data.len - 1] - else - data; - } -}; - -/// A client session for the Git protocol, currently limited to an HTTP(S) -/// transport. Only protocol version 2 is supported, as documented in -/// [protocol-v2](https://git-scm.com/docs/protocol-v2). -pub const Session = struct { - transport: *std.http.Client, - location: Location, - supports_agent: bool, - supports_shallow: bool, - object_format: Oid.Format, - arena: Allocator, - - const agent = "zig/" ++ @import("builtin").zig_version_string; - const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); - - /// Initializes a client session and discovers the capabilities of the - /// server for optimal transport. - pub fn init( - arena: Allocator, - transport: *std.http.Client, - uri: std.Uri, - /// Asserted to be at least `Packet.max_data_length` - response_buffer: []u8, - ) !Session { - assert(response_buffer.len >= Packet.max_data_length); - var session: Session = .{ - .transport = transport, - .location = try .init(arena, uri), - .supports_agent = false, - .supports_shallow = false, - .object_format = .sha1, - .arena = arena, - }; - var capability_iterator: CapabilityIterator = undefined; - try session.getCapabilities(&capability_iterator, response_buffer); - defer capability_iterator.deinit(); - while (try capability_iterator.next()) |capability| { - if (mem.eql(u8, capability.key, "agent")) { - session.supports_agent = true; - } else if (mem.eql(u8, capability.key, "fetch")) { - var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); - while (feature_iterator.next()) |feature| { - if (mem.eql(u8, feature, "shallow")) { - session.supports_shallow = true; - } - } - } else if (mem.eql(u8, capability.key, "object-format")) { - if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) |format| { - session.object_format = format; - } - } - } - return session; - } - - /// An owned `std.Uri` representing the location of the server (base URI). - const Location = struct { - uri: std.Uri, - - fn init(arena: Allocator, uri: std.Uri) !Location { - const scheme = try arena.dupe(u8, uri.scheme); - const user = if (uri.user) |user| try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(user, .formatUser), - }) else null; - const password = if (uri.password) |password| try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(password, .formatPassword), - }) else null; - const host = if (uri.host) |host| try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(host, .formatHost), - }) else null; - const path = try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(uri.path, .formatPath), - }); - // The query and fragment are not used as part of the base server URI. - return .{ - .uri = .{ - .scheme = scheme, - .user = if (user) |s| .{ .percent_encoded = s } else null, - .password = if (password) |s| .{ .percent_encoded = s } else null, - .host = if (host) |s| .{ .percent_encoded = s } else null, - .port = uri.port, - .path = .{ .percent_encoded = path }, - }, - }; - } - }; - - /// Returns an iterator over capabilities supported by the server. - /// - /// The `session.location` is updated if the server returns a redirect, so - /// that subsequent session functions do not need to handle redirects. - fn getCapabilities(session: *Session, it: *CapabilityIterator, response_buffer: []u8) !void { - const arena = session.arena; - assert(response_buffer.len >= Packet.max_data_length); - var info_refs_uri = session.location.uri; - { - const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(session.location.uri.path, .formatPath), - }); - info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ - "/", session_uri_path, "info/refs", - }) }; - } - info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" }; - info_refs_uri.fragment = null; - - const max_redirects = 3; - it.* = .{ - .request = try session.transport.request(.GET, info_refs_uri, .{ - .redirect_behavior = .init(max_redirects), - .extra_headers = &.{ - .{ .name = "Git-Protocol", .value = "version=2" }, - }, - }), - .reader = undefined, - .decompress = undefined, - }; - errdefer it.deinit(); - const request = &it.request; - try request.sendBodiless(); - - var redirect_buffer: [1024]u8 = undefined; - var response = try request.receiveHead(&redirect_buffer); - if (response.head.status != .ok) return error.ProtocolError; - const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects; - if (any_redirects_occurred) { - const request_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(request.uri.path, .formatPath), - }); - if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect; - var new_uri = request.uri; - new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] }; - session.location = try .init(arena, new_uri); - } - - const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); - it.reader = response.readerDecompressing(response_buffer, &it.decompress, decompress_buffer); - var state: enum { response_start, response_content } = .response_start; - while (true) { - // Some Git servers (at least GitHub) include an additional - // '# service=git-upload-pack' informative response before sending - // the expected 'version 2' packet and capability information. - // This is not universal: SourceHut, for example, does not do this. - // Thus, we need to skip any such useless additional responses - // before we get the one we're actually looking for. The responses - // will be delimited by flush packets. - const packet = Packet.read(it.reader) catch |err| switch (err) { - error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found - else => |e| return e, - }; - switch (packet) { - .flush => state = .response_start, - .data => |data| switch (state) { - .response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) { - return; - } else { - state = .response_content; - }, - else => {}, - }, - else => return error.UnexpectedPacket, - } - } - } - - const CapabilityIterator = struct { - request: std.http.Client.Request, - reader: *Io.Reader, - decompress: std.http.Decompress, - - const Capability = struct { - key: []const u8, - value: ?[]const u8 = null, - - fn parse(data: []const u8) Capability { - return if (mem.indexOfScalar(u8, data, '=')) |separator_pos| - .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 ..] } - else - .{ .key = data }; - } - }; - - fn deinit(it: *CapabilityIterator) void { - it.request.deinit(); - it.* = undefined; - } - - fn next(it: *CapabilityIterator) !?Capability { - switch (try Packet.read(it.reader)) { - .flush => return null, - .data => |data| return Capability.parse(Packet.normalizeText(data)), - else => return error.UnexpectedPacket, - } - } - }; - - const ListRefsOptions = struct { - /// The ref prefixes (if any) to use to filter the refs available on the - /// server. Note that the client must still check the returned refs - /// against its desired filters itself: the server is not required to - /// respect these prefix filters and may return other refs as well. - ref_prefixes: []const []const u8 = &.{}, - /// Whether to include symref targets for returned symbolic refs. - include_symrefs: bool = false, - /// Whether to include the peeled object ID for returned tag refs. - include_peeled: bool = false, - /// Asserted to be at least `Packet.max_data_length`. - buffer: []u8, - }; - - /// Returns an iterator over refs known to the server. - pub fn listRefs(session: Session, it: *RefIterator, options: ListRefsOptions) !void { - const arena = session.arena; - assert(options.buffer.len >= Packet.max_data_length); - var upload_pack_uri = session.location.uri; - { - const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(session.location.uri.path, .formatPath), - }); - upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) }; - } - upload_pack_uri.query = null; - upload_pack_uri.fragment = null; - - var body: Io.Writer = .fixed(options.buffer); - try Packet.write(.{ .data = "command=ls-refs\n" }, &body); - if (session.supports_agent) { - try Packet.write(.{ .data = agent_capability }, &body); - } - { - const object_format_packet = try std.fmt.allocPrint(arena, "object-format={t}\n", .{ - session.object_format, - }); - try Packet.write(.{ .data = object_format_packet }, &body); - } - try Packet.write(.delimiter, &body); - for (options.ref_prefixes) |ref_prefix| { - const ref_prefix_packet = try std.fmt.allocPrint(arena, "ref-prefix {s}\n", .{ref_prefix}); - try Packet.write(.{ .data = ref_prefix_packet }, &body); - } - if (options.include_symrefs) { - try Packet.write(.{ .data = "symrefs\n" }, &body); - } - if (options.include_peeled) { - try Packet.write(.{ .data = "peel\n" }, &body); - } - try Packet.write(.flush, &body); - - it.* = .{ - .request = try session.transport.request(.POST, upload_pack_uri, .{ - .redirect_behavior = .unhandled, - .extra_headers = &.{ - .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, - .{ .name = "Git-Protocol", .value = "version=2" }, - }, - }), - .reader = undefined, - .format = session.object_format, - .decompress = undefined, - }; - const request = &it.request; - errdefer request.deinit(); - try request.sendBodyComplete(body.buffered()); - - var response = try request.receiveHead(options.buffer); - if (response.head.status != .ok) return error.ProtocolError; - const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); - it.reader = response.readerDecompressing(options.buffer, &it.decompress, decompress_buffer); - } - - pub const RefIterator = struct { - format: Oid.Format, - request: std.http.Client.Request, - reader: *Io.Reader, - decompress: std.http.Decompress, - - pub const Ref = struct { - oid: Oid, - name: []const u8, - symref_target: ?[]const u8, - peeled: ?Oid, - }; - - pub fn deinit(iterator: *RefIterator) void { - iterator.request.deinit(); - iterator.* = undefined; - } - - pub fn next(it: *RefIterator) !?Ref { - switch (try Packet.read(it.reader)) { - .flush => return null, - .data => |data| { - const ref_data = Packet.normalizeText(data); - const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket; - const oid = Oid.parse(it.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket; - - const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len; - const name = ref_data[oid_sep_pos + 1 .. name_sep_pos]; - - var symref_target: ?[]const u8 = null; - var peeled: ?Oid = null; - var last_sep_pos = name_sep_pos; - while (last_sep_pos < ref_data.len) { - const next_sep_pos = mem.indexOfScalarPos(u8, ref_data, last_sep_pos + 1, ' ') orelse ref_data.len; - const attribute = ref_data[last_sep_pos + 1 .. next_sep_pos]; - if (mem.startsWith(u8, attribute, "symref-target:")) { - symref_target = attribute["symref-target:".len..]; - } else if (mem.startsWith(u8, attribute, "peeled:")) { - peeled = Oid.parse(it.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket; - } - last_sep_pos = next_sep_pos; - } - - return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; - }, - else => return error.UnexpectedPacket, - } - } - }; - - /// Fetches the given refs from the server. A shallow fetch (depth 1) is - /// performed if the server supports it. - pub fn fetch( - session: Session, - fs: *FetchStream, - wants: []const []const u8, - /// Asserted to be at least `Packet.max_data_length`. - response_buffer: []u8, - ) !void { - const arena = session.arena; - assert(response_buffer.len >= Packet.max_data_length); - var upload_pack_uri = session.location.uri; - { - const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{ - std.fmt.alt(session.location.uri.path, .formatPath), - }); - upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) }; - } - upload_pack_uri.query = null; - upload_pack_uri.fragment = null; - - var body: Io.Writer = .fixed(response_buffer); - try Packet.write(.{ .data = "command=fetch\n" }, &body); - if (session.supports_agent) { - try Packet.write(.{ .data = agent_capability }, &body); - } - { - const object_format_packet = try std.fmt.allocPrint(arena, "object-format={s}\n", .{@tagName(session.object_format)}); - try Packet.write(.{ .data = object_format_packet }, &body); - } - try Packet.write(.delimiter, &body); - // Our packfile parser supports the OFS_DELTA object type - try Packet.write(.{ .data = "ofs-delta\n" }, &body); - // We do not currently convey server progress information to the user - try Packet.write(.{ .data = "no-progress\n" }, &body); - if (session.supports_shallow) { - try Packet.write(.{ .data = "deepen 1\n" }, &body); - } - for (wants) |want| { - var buf: [Packet.max_data_length]u8 = undefined; - const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; - try Packet.write(.{ .data = arg }, &body); - } - try Packet.write(.{ .data = "done\n" }, &body); - try Packet.write(.flush, &body); - - fs.* = .{ - .request = try session.transport.request(.POST, upload_pack_uri, .{ - .redirect_behavior = .not_allowed, - .extra_headers = &.{ - .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" }, - .{ .name = "Git-Protocol", .value = "version=2" }, - }, - }), - .input = undefined, - .reader = undefined, - .remaining_len = undefined, - .decompress = undefined, - }; - const request = &fs.request; - errdefer request.deinit(); - - try request.sendBodyComplete(body.buffered()); - - var response = try request.receiveHead(&.{}); - if (response.head.status != .ok) return error.ProtocolError; - - const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity()); - const reader = response.readerDecompressing(response_buffer, &fs.decompress, decompress_buffer); - // We are not interested in any of the sections of the returned fetch - // data other than the packfile section, since we aren't doing anything - // complex like ref negotiation (this is a fresh clone). - var state: enum { section_start, section_content } = .section_start; - while (true) { - const packet = try Packet.read(reader); - switch (state) { - .section_start => switch (packet) { - .data => |data| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) { - fs.input = reader; - fs.reader = .{ - .buffer = &.{}, - .vtable = &.{ .stream = FetchStream.stream }, - .seek = 0, - .end = 0, - }; - fs.remaining_len = 0; - return; - } else { - state = .section_content; - }, - else => return error.UnexpectedPacket, - }, - .section_content => switch (packet) { - .delimiter => state = .section_start, - .data => {}, - else => return error.UnexpectedPacket, - }, - } - } - } - - pub const FetchStream = struct { - request: std.http.Client.Request, - input: *Io.Reader, - reader: Io.Reader, - err: ?Error = null, - remaining_len: usize, - decompress: std.http.Decompress, - - pub fn deinit(fs: *FetchStream) void { - fs.request.deinit(); - } - - pub const Error = error{ - InvalidPacket, - ProtocolError, - UnexpectedPacket, - WriteFailed, - ReadFailed, - EndOfStream, - }; - - const StreamCode = enum(u8) { - pack_data = 1, - progress = 2, - fatal_error = 3, - _, - }; - - pub fn stream(r: *Io.Reader, w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize { - const fs: *FetchStream = @alignCast(@fieldParentPtr("reader", r)); - const input = fs.input; - if (fs.remaining_len == 0) { - while (true) { - switch (Packet.peek(input) catch |err| { - fs.err = err; - return error.ReadFailed; - }) { - .flush => return error.EndOfStream, - .data => |data| switch (@as(StreamCode, @enumFromInt(data[0]))) { - .pack_data => { - input.toss(1); - fs.remaining_len = data.len - 1; - break; - }, - .fatal_error => { - fs.err = error.ProtocolError; - return error.ReadFailed; - }, - else => { - input.toss(data.len); - }, - }, - else => { - fs.err = error.UnexpectedPacket; - return error.ReadFailed; - }, - } - } - } - const buf = limit.slice(try w.writableSliceGreedy(1)); - const n = @min(buf.len, fs.remaining_len); - try input.readSliceAll(buf[0..n]); - w.advance(n); - fs.remaining_len -= n; - return n; - } - }; -}; - -const PackHeader = struct { - total_objects: u32, - - const signature = "PACK"; - const supported_version = 2; - - fn read(reader: *Io.Reader) !PackHeader { - const actual_signature = reader.take(4) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader; - const version = reader.takeInt(u32, .big) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - if (version != supported_version) return error.UnsupportedVersion; - const total_objects = reader.takeInt(u32, .big) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - return .{ .total_objects = total_objects }; - } -}; - -const EntryHeader = union(Type) { - commit: Undeltified, - tree: Undeltified, - blob: Undeltified, - tag: Undeltified, - ofs_delta: OfsDelta, - ref_delta: RefDelta, - - const Type = enum(u3) { - commit = 1, - tree = 2, - blob = 3, - tag = 4, - ofs_delta = 6, - ref_delta = 7, - }; - - const Undeltified = struct { - uncompressed_length: u64, - }; - - const OfsDelta = struct { - offset: u64, - uncompressed_length: u64, - }; - - const RefDelta = struct { - base_object: Oid, - uncompressed_length: u64, - }; - - fn objectType(header: EntryHeader) Object.Type { - return switch (header) { - inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), - else => unreachable, - }; - } - - fn uncompressedLength(header: EntryHeader) u64 { - return switch (header) { - inline else => |entry| entry.uncompressed_length, - }; - } - - fn read(format: Oid.Format, reader: *Io.Reader) !EntryHeader { - const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; - const initial: InitialByte = @bitCast(reader.takeByte() catch |e| switch (e) { - error.EndOfStream => return error.InvalidFormat, - else => |other| return other, - }); - const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0; - var uncompressed_length: u64 = initial.len; - uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; - const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat; - return switch (@"type") { - inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ - .uncompressed_length = uncompressed_length, - }), - .ofs_delta => .{ .ofs_delta = .{ - .offset = try readOffsetVarInt(reader), - .uncompressed_length = uncompressed_length, - } }, - .ref_delta => .{ .ref_delta = .{ - .base_object = Oid.readBytes(format, reader) catch |e| switch (e) { - error.EndOfStream => return error.InvalidFormat, - else => |other| return other, - }, - .uncompressed_length = uncompressed_length, - } }, - }; - } -}; - -fn readOffsetVarInt(r: *Io.Reader) !u64 { - const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.takeByte()); - var value: u64 = b.value; - while (b.has_next) { - b = @bitCast(try r.takeByte()); - value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; - value |= b.value; - } - return value; -} - -const IndexHeader = struct { - fan_out_table: [256]u32, - - const signature = "\xFFtOc"; - const supported_version = 2; - const size = 4 + 4 + @sizeOf([256]u32); - - fn read(index_header: *IndexHeader, reader: *Io.Reader) !void { - const sig = try reader.take(4); - if (!mem.eql(u8, sig, signature)) return error.InvalidHeader; - const version = try reader.takeInt(u32, .big); - if (version != supported_version) return error.UnsupportedVersion; - try reader.readSliceEndian(u32, &index_header.fan_out_table, .big); - } -}; - -const IndexEntry = struct { - offset: u64, - crc32: u32, -}; - -/// Writes out a version 2 index for the given packfile, as documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -pub fn indexPack( - allocator: Allocator, - format: Oid.Format, - pack: *Io.File.Reader, - index_writer: *Io.File.Writer, -) !void { - try pack.seekTo(0); - - var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; - defer index_entries.deinit(allocator); - var pending_deltas: std.ArrayList(IndexEntry) = .empty; - defer pending_deltas.deinit(allocator); - - const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas); - - var cache: ObjectCache = .{}; - defer cache.deinit(allocator); - var remaining_deltas = pending_deltas.items.len; - while (remaining_deltas > 0) { - var i: usize = remaining_deltas; - while (i > 0) { - i -= 1; - const delta = pending_deltas.items[i]; - if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) |oid| { - try index_entries.put(allocator, oid, delta); - _ = pending_deltas.swapRemove(i); - } - } - if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; - remaining_deltas = pending_deltas.items.len; - } - - var oids: std.ArrayList(Oid) = .empty; - defer oids.deinit(allocator); - try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); - var index_entries_iter = index_entries.iterator(); - while (index_entries_iter.next()) |entry| { - oids.appendAssumeCapacity(entry.key_ptr.*); - } - mem.sortUnstable(Oid, oids.items, {}, struct { - fn lessThan(_: void, o1: Oid, o2: Oid) bool { - return mem.lessThan(u8, o1.slice(), o2.slice()); - } - }.lessThan); - - var fan_out_table: [256]u32 = undefined; - var count: u32 = 0; - var fan_out_index: u8 = 0; - for (oids.items) |oid| { - const key = oid.slice()[0]; - if (key > fan_out_index) { - @memset(fan_out_table[fan_out_index..key], count); - fan_out_index = key; - } - count += 1; - } - @memset(fan_out_table[fan_out_index..], count); - - var index_hashed_writer = Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{}); - const writer = &index_hashed_writer.writer; - try writer.writeAll(IndexHeader.signature); - try writer.writeInt(u32, IndexHeader.supported_version, .big); - for (fan_out_table) |fan_out_entry| { - try writer.writeInt(u32, fan_out_entry, .big); - } - - for (oids.items) |oid| { - try writer.writeAll(oid.slice()); - } - - for (oids.items) |oid| { - try writer.writeInt(u32, index_entries.get(oid).?.crc32, .big); - } - - var big_offsets: std.ArrayList(u64) = .empty; - defer big_offsets.deinit(allocator); - for (oids.items) |oid| { - const offset = index_entries.get(oid).?.offset; - if (offset <= std.math.maxInt(u31)) { - try writer.writeInt(u32, @intCast(offset), .big); - } else { - const index = big_offsets.items.len; - try big_offsets.append(allocator, offset); - try writer.writeInt(u32, @as(u32, @intCast(index)) | (1 << 31), .big); - } - } - for (big_offsets.items) |offset| { - try writer.writeInt(u64, offset, .big); - } - - try writer.writeAll(pack_checksum.slice()); - const index_checksum = index_hashed_writer.hasher.finalResult(); - try index_writer.interface.writeAll(index_checksum.slice()); - try index_writer.end(); -} - -/// Performs the first pass over the packfile data for index construction. -/// This will index all non-delta objects, queue delta objects for further -/// processing, and return the pack checksum (which is part of the index -/// format). -fn indexPackFirstPass( - allocator: Allocator, - format: Oid.Format, - pack: *Io.File.Reader, - index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), - pending_deltas: *std.ArrayList(IndexEntry), -) !Oid { - var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; - var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system. - var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer); - - const pack_header = try PackHeader.read(&pack_hashed.reader); - - for (0..pack_header.total_objects) |_| { - const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen(); - const entry_header = try EntryHeader.read(format, &pack_hashed.reader); - switch (entry_header) { - .commit, .tree, .blob, .tag => |object| { - var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{}); - var oid_hasher: Oid.Hashing = .init(format, &flate_buffer); - const oid_hasher_w = oid_hasher.writer(); - // The object header is not included in the pack data but is - // part of the object's ID - try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length }); - const n = try entry_decompress.reader.streamRemaining(oid_hasher_w); - if (n != object.uncompressed_length) return error.InvalidObject; - const oid = oid_hasher.final(); - if (!skip_checksums) @compileError("TODO"); - try index_entries.put(allocator, oid, .{ - .offset = entry_offset, - .crc32 = 0, - }); - }, - inline .ofs_delta, .ref_delta => |delta| { - var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer); - const n = try entry_decompress.reader.discardRemaining(); - if (n != delta.uncompressed_length) return error.InvalidObject; - if (!skip_checksums) @compileError("TODO"); - try pending_deltas.append(allocator, .{ - .offset = entry_offset, - .crc32 = 0, - }); - }, - } - } - - if (!skip_checksums) @compileError("TODO"); - return pack_hashed.hasher.finalResult(); -} - -/// Attempts to determine the final object ID of the given deltified object. -/// May return null if this is not yet possible (if the delta is a ref-based -/// delta and we do not yet know the offset of the base object). -fn indexPackHashDelta( - allocator: Allocator, - format: Oid.Format, - pack: *Io.File.Reader, - delta: IndexEntry, - index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), - cache: *ObjectCache, -) !?Oid { - // Figure out the chain of deltas to resolve - var base_offset = delta.offset; - var base_header: EntryHeader = undefined; - var delta_offsets: std.ArrayList(u64) = .empty; - defer delta_offsets.deinit(allocator); - const base_object = while (true) { - if (cache.get(base_offset)) |base_object| break base_object; - - try pack.seekTo(base_offset); - base_header = try EntryHeader.read(format, &pack.interface); - switch (base_header) { - .ofs_delta => |ofs_delta| { - try delta_offsets.append(allocator, base_offset); - base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; - }, - .ref_delta => |ref_delta| { - try delta_offsets.append(allocator, base_offset); - base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; - }, - else => { - const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength()); - errdefer allocator.free(base_data); - const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; - try cache.put(allocator, base_offset, base_object); - break base_object; - }, - } - }; - - const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); - - var entry_hasher_buffer: [64]u8 = undefined; - var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer); - const entry_hasher_w = entry_hasher.writer(); - // Writes to hashers cannot fail. - entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable; - entry_hasher_w.writeAll(base_data) catch unreachable; - return entry_hasher.final(); -} - -/// Resolves a chain of deltas, returning the final base object data. `pack` is -/// assumed to be looking at the start of the object data for the base object of -/// the chain, and will then apply the deltas in `delta_offsets` in reverse order -/// to obtain the final object. -fn resolveDeltaChain( - allocator: Allocator, - format: Oid.Format, - pack: *Io.File.Reader, - base_object: Object, - delta_offsets: []const u64, - cache: *ObjectCache, -) ![]const u8 { - var base_data = base_object.data; - var i: usize = delta_offsets.len; - while (i > 0) { - i -= 1; - - const delta_offset = delta_offsets[i]; - try pack.seekTo(delta_offset); - const delta_header = try EntryHeader.read(format, &pack.interface); - const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength()); - defer allocator.free(delta_data); - var delta_reader: Io.Reader = .fixed(delta_data); - _ = try delta_reader.takeLeb128(u64); // base object size - const expanded_size = try delta_reader.takeLeb128(u64); - - const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; - const expanded_data = try allocator.alloc(u8, expanded_alloc_size); - errdefer allocator.free(expanded_data); - var expanded_delta_stream: Io.Writer = .fixed(expanded_data); - try expandDelta(base_data, &delta_reader, &expanded_delta_stream); - if (expanded_delta_stream.end != expanded_size) return error.InvalidObject; - - try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); - base_data = expanded_data; - } - return base_data; -} - -/// Reads the complete contents of an object from `reader`. This function may -/// read more bytes than required from `reader`, so the reader position after -/// returning is not reliable. -fn readObjectRaw(allocator: Allocator, reader: *Io.Reader, size: u64) ![]u8 { - const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; - var aw: Io.Writer.Allocating = .init(allocator); - try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len); - defer aw.deinit(); - var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{}); - try decompress.reader.streamExact(&aw.writer, alloc_size); - return aw.toOwnedSlice(); -} - -/// Expands delta data from `delta_reader` to `writer`. -/// -/// The format of the delta data is documented in -/// [pack-format](https://git-scm.com/docs/pack-format). -fn expandDelta(base_object: []const u8, delta_reader: *Io.Reader, writer: *Io.Writer) !void { - while (true) { - const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch |e| switch (e) { - error.EndOfStream => return, - else => |other| return other, - }); - if (inst.copy) { - const available: packed struct { - offset1: bool, - offset2: bool, - offset3: bool, - offset4: bool, - size1: bool, - size2: bool, - size3: bool, - } = @bitCast(inst.value); - const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ - .offset1 = if (available.offset1) try delta_reader.takeByte() else 0, - .offset2 = if (available.offset2) try delta_reader.takeByte() else 0, - .offset3 = if (available.offset3) try delta_reader.takeByte() else 0, - .offset4 = if (available.offset4) try delta_reader.takeByte() else 0, - }; - const base_offset: u32 = @bitCast(offset_parts); - const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ - .size1 = if (available.size1) try delta_reader.takeByte() else 0, - .size2 = if (available.size2) try delta_reader.takeByte() else 0, - .size3 = if (available.size3) try delta_reader.takeByte() else 0, - }; - var size: u24 = @bitCast(size_parts); - if (size == 0) size = 0x10000; - try writer.writeAll(base_object[base_offset..][0..size]); - } else if (inst.value != 0) { - try delta_reader.streamExact(writer, inst.value); - } else { - return error.InvalidDeltaInstruction; - } - } -} - -/// Runs the packfile indexing and checkout test. -/// -/// The two testrepo repositories under testdata contain identical commit -/// histories and contents. -/// -/// To verify the contents of the packfiles using Git alone, run the -/// following commands in an empty directory: -/// -/// 1. `git init --object-format=(sha1|sha256)` -/// 2. `git unpack-objects assert(p.errors.items.len > 0), - else => |e| return e, - }; - - return .{ - .name = p.name, - .id = p.id, - .version = p.version, - .version_node = p.version_node, - .dependencies = try p.dependencies.clone(p.arena), - .dependencies_node = p.dependencies_node, - .paths = try p.paths.clone(p.arena), - .minimum_zig_version = p.minimum_zig_version, - .errors = try p.arena.dupe(ErrorMessage, p.errors.items), - .arena_state = arena_instance.state, - }; -} - -pub fn deinit(man: *Manifest, gpa: Allocator) void { - man.arena_state.promote(gpa).deinit(); - man.* = undefined; -} - -pub fn copyErrorsIntoBundle( - man: Manifest, - ast: Ast, - /// ErrorBundle null-terminated string index - src_path: u32, - eb: *std.zig.ErrorBundle.Wip, -) Allocator.Error!void { - for (man.errors) |msg| { - const start_loc = ast.tokenLocation(0, msg.tok); - - try eb.addRootErrorMessage(.{ - .msg = try eb.addString(msg.msg), - .src_loc = try eb.addSourceLocation(.{ - .src_path = src_path, - .span_start = ast.tokenStart(msg.tok), - .span_end = @intCast(ast.tokenStart(msg.tok) + ast.tokenSlice(msg.tok).len), - .span_main = ast.tokenStart(msg.tok) + msg.off, - .line = @intCast(start_loc.line), - .column = @intCast(start_loc.column), - .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), - }), - }); - } -} - -const Parse = struct { - gpa: Allocator, - ast: Ast, - arena: Allocator, - buf: std.ArrayList(u8), - errors: std.ArrayList(ErrorMessage), - - name: []const u8, - id: u32, - version: std.SemanticVersion, - version_node: Ast.Node.Index, - dependencies: std.array_hash_map.String(Dependency), - dependencies_node: Ast.Node.OptionalIndex, - paths: std.array_hash_map.String(void), - allow_missing_paths_field: bool, - minimum_zig_version: ?std.SemanticVersion, - - const InnerError = error{ ParseFailure, OutOfMemory }; - - fn parseRoot(p: *Parse, node: Ast.Node.Index, rng: std.Random) !void { - const ast = p.ast; - const main_token = ast.nodeMainToken(node); - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - return fail(p, main_token, "expected top level expression to be a struct", .{}); - }; - - var have_name = false; - var have_version = false; - var have_included_paths = false; - var fingerprint: ?Package.Fingerprint = null; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - const field_name = try identifierTokenString(p, name_token); - // We could get fancy with reflection and comptime logic here but doing - // things manually provides an opportunity to do any additional verification - // that is desirable on a per-field basis. - if (mem.eql(u8, field_name, "dependencies")) { - p.dependencies_node = field_init.toOptional(); - try parseDependencies(p, field_init); - } else if (mem.eql(u8, field_name, "paths")) { - have_included_paths = true; - try parseIncludedPaths(p, field_init); - } else if (mem.eql(u8, field_name, "name")) { - p.name = try parseName(p, field_init); - have_name = true; - } else if (mem.eql(u8, field_name, "fingerprint")) { - fingerprint = try parseFingerprint(p, field_init); - } else if (mem.eql(u8, field_name, "version")) { - p.version_node = field_init; - const version_text = try parseString(p, field_init); - if (version_text.len > max_version_len) { - try appendError(p, ast.nodeMainToken(field_init), "version string length {d} exceeds maximum of {d}", .{ version_text.len, max_version_len }); - } - p.version = std.SemanticVersion.parse(version_text) catch |err| v: { - try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); - break :v undefined; - }; - have_version = true; - } else if (mem.eql(u8, field_name, "minimum_zig_version")) { - const version_text = try parseString(p, field_init); - p.minimum_zig_version = std.SemanticVersion.parse(version_text) catch |err| v: { - try appendError(p, ast.nodeMainToken(field_init), "unable to parse semantic version: {s}", .{@errorName(err)}); - break :v null; - }; - } else { - // Ignore unknown fields so that we can add fields in future zig - // versions without breaking older zig versions. - } - } - - if (!have_name) { - try appendError(p, main_token, "missing top-level 'name' field", .{}); - } else { - if (fingerprint) |n| { - if (!n.validate(p.name)) { - return fail(p, main_token, "invalid fingerprint: 0x{x}; if this is a new or forked package, use this value: 0x{x}", .{ - n.int(), Package.Fingerprint.generate(rng, p.name).int(), - }); - } - p.id = n.id; - } else { - try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{ - Package.Fingerprint.generate(rng, p.name).int(), - }); - } - } - - if (!have_version) { - try appendError(p, main_token, "missing top-level 'version' field", .{}); - } - - if (!have_included_paths) { - if (p.allow_missing_paths_field) { - try p.paths.put(p.gpa, "", {}); - } else { - try appendError(p, main_token, "missing top-level 'paths' field", .{}); - } - } - } - - fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void { - const ast = p.ast; - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - const tok = ast.nodeMainToken(node); - return fail(p, tok, "expected dependencies expression to be a struct", .{}); - }; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - const dep_name = try identifierTokenString(p, name_token); - const dep = try parseDependency(p, field_init); - try p.dependencies.put(p.gpa, dep_name, dep); - } - } - - fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency { - const ast = p.ast; - - var buf: [2]Ast.Node.Index = undefined; - const struct_init = ast.fullStructInit(&buf, node) orelse { - const tok = ast.nodeMainToken(node); - return fail(p, tok, "expected dependency expression to be a struct", .{}); - }; - - var dep: Dependency = .{ - .location = undefined, - .location_tok = undefined, - .location_node = undefined, - .hash = null, - .hash_tok = .none, - .hash_node = .none, - .node = node, - .name_tok = undefined, - .lazy = false, - }; - var has_location = false; - - for (struct_init.ast.fields) |field_init| { - const name_token = ast.firstToken(field_init) - 2; - dep.name_tok = name_token; - const field_name = try identifierTokenString(p, name_token); - // We could get fancy with reflection and comptime logic here but doing - // things manually provides an opportunity to do any additional verification - // that is desirable on a per-field basis. - if (mem.eql(u8, field_name, "url")) { - if (has_location) { - return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); - } - dep.location = .{ - .url = parseString(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }, - }; - has_location = true; - dep.location_tok = ast.nodeMainToken(field_init); - dep.location_node = field_init; - } else if (mem.eql(u8, field_name, "path")) { - if (has_location) { - return fail(p, ast.nodeMainToken(field_init), "dependency should specify only one of 'url' and 'path' fields.", .{}); - } - dep.location = .{ - .path = parseString(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }, - }; - has_location = true; - dep.location_tok = ast.nodeMainToken(field_init); - dep.location_node = field_init; - } else if (mem.eql(u8, field_name, "hash")) { - dep.hash = parseHash(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }; - dep.hash_tok = .fromToken(ast.nodeMainToken(field_init)); - dep.hash_node = field_init.toOptional(); - } else if (mem.eql(u8, field_name, "lazy")) { - dep.lazy = parseBool(p, field_init) catch |err| switch (err) { - error.ParseFailure => continue, - else => |e| return e, - }; - } else { - // Ignore unknown fields so that we can add fields in future zig - // versions without breaking older zig versions. - } - } - - if (!has_location) { - try appendError(p, ast.nodeMainToken(node), "dependency requires location field, one of 'url' or 'path'.", .{}); - } - - return dep; - } - - fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void { - const ast = p.ast; - - var buf: [2]Ast.Node.Index = undefined; - const array_init = ast.fullArrayInit(&buf, node) orelse { - const tok = ast.nodeMainToken(node); - return fail(p, tok, "expected paths expression to be a list of strings", .{}); - }; - - for (array_init.ast.elements) |elem_node| { - const path_string = try parseString(p, elem_node); - // This is normalized so that it can be used in string comparisons - // against file system paths. - const normalized = try std.fs.path.resolve(p.arena, &.{path_string}); - try p.paths.put(p.gpa, normalized, {}); - } - } - - fn parseBool(p: *Parse, node: Ast.Node.Index) !bool { - const ast = p.ast; - if (ast.nodeTag(node) != .identifier) { - return fail(p, ast.nodeMainToken(node), "expected identifier", .{}); - } - const ident_token = ast.nodeMainToken(node); - const token_bytes = ast.tokenSlice(ident_token); - if (mem.eql(u8, token_bytes, "true")) { - return true; - } else if (mem.eql(u8, token_bytes, "false")) { - return false; - } else { - return fail(p, ident_token, "expected boolean", .{}); - } - } - - fn parseFingerprint(p: *Parse, node: Ast.Node.Index) !Package.Fingerprint { - const ast = p.ast; - const main_token = ast.nodeMainToken(node); - if (ast.nodeTag(node) != .number_literal) { - return fail(p, main_token, "expected integer literal", .{}); - } - const token_bytes = ast.tokenSlice(main_token); - const parsed = std.zig.parseNumberLiteral(token_bytes); - switch (parsed) { - .int => |n| return @bitCast(n), - .big_int, .float => return fail(p, main_token, "expected u64 integer literal, found {s}", .{ - @tagName(parsed), - }), - .failure => |err| return fail(p, main_token, "bad integer literal: {s}", .{@tagName(err)}), - } - } - - fn parseName(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - const main_token = ast.nodeMainToken(node); - - if (ast.nodeTag(node) != .enum_literal) - return fail(p, main_token, "expected enum literal", .{}); - - const ident_name = ast.tokenSlice(main_token); - if (mem.startsWith(u8, ident_name, "@")) - return fail(p, main_token, "name must be a valid bare zig identifier", .{}); - - if (ident_name.len > max_name_len) - return fail(p, main_token, "name '{f}' exceeds max length of {d}", .{ - std.zig.fmtId(ident_name), max_name_len, - }); - - return ident_name; - } - - fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - if (ast.nodeTag(node) != .string_literal) { - return fail(p, ast.nodeMainToken(node), "expected string literal", .{}); - } - const str_lit_token = ast.nodeMainToken(node); - const token_bytes = ast.tokenSlice(str_lit_token); - p.buf.clearRetainingCapacity(); - try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0); - const duped = try p.arena.dupe(u8, p.buf.items); - return duped; - } - - fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 { - const ast = p.ast; - const tok = ast.nodeMainToken(node); - const h = try parseString(p, node); - switch (Package.Hash.validate(h)) { - .ok => return h, - else => |t| return fail(p, tok, "invalid hash: {t}", .{t}), - } - } - - /// TODO: try to DRY this with AstGen.identifierTokenString - fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 { - const ast = p.ast; - assert(ast.tokenTag(token) == .identifier); - const ident_name = ast.tokenSlice(token); - if (!mem.startsWith(u8, ident_name, "@")) { - return ident_name; - } - p.buf.clearRetainingCapacity(); - try parseStrLit(p, token, &p.buf, ident_name, 1); - const duped = try p.arena.dupe(u8, p.buf.items); - return duped; - } - - /// TODO: try to DRY this with AstGen.parseStrLit - fn parseStrLit( - p: *Parse, - token: Ast.TokenIndex, - buf: *std.ArrayList(u8), - bytes: []const u8, - offset: u32, - ) InnerError!void { - const raw_string = bytes[offset..]; - const result = r: { - var aw: std.Io.Writer.Allocating = .fromArrayList(p.gpa, buf); - defer buf.* = aw.toArrayList(); - break :r std.zig.string_literal.parseWrite(&aw.writer, raw_string) catch |err| switch (err) { - error.WriteFailed => return error.OutOfMemory, - }; - }; - switch (result) { - .success => {}, - .failure => |err| try p.appendStrLitError(err, token, bytes, offset), - } - } - - /// TODO: try to DRY this with AstGen.failWithStrLitError - fn appendStrLitError( - p: *Parse, - err: std.zig.string_literal.Error, - token: Ast.TokenIndex, - bytes: []const u8, - offset: u32, - ) Allocator.Error!void { - const raw_string = bytes[offset..]; - switch (err) { - .invalid_escape_character => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "invalid escape character: '{c}'", - .{raw_string[bad_index]}, - ); - }, - .expected_hex_digit => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected hex digit, found '{c}'", - .{raw_string[bad_index]}, - ); - }, - .empty_unicode_escape_sequence => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "empty unicode escape sequence", - .{}, - ); - }, - .expected_hex_digit_or_rbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected hex digit or '}}', found '{c}'", - .{raw_string[bad_index]}, - ); - }, - .invalid_unicode_codepoint => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "unicode escape does not correspond to a valid unicode scalar value", - .{}, - ); - }, - .expected_lbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected '{{', found '{c}", - .{raw_string[bad_index]}, - ); - }, - .expected_rbrace => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected '}}', found '{c}", - .{raw_string[bad_index]}, - ); - }, - .expected_single_quote => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "expected single quote ('), found '{c}", - .{raw_string[bad_index]}, - ); - }, - .invalid_character => |bad_index| { - try p.appendErrorOff( - token, - offset + @as(u32, @intCast(bad_index)), - "invalid byte in string or character literal: '{c}'", - .{raw_string[bad_index]}, - ); - }, - .empty_char_literal => { - try p.appendErrorOff(token, offset, "empty character literal", .{}); - }, - } - } - - fn fail( - p: *Parse, - tok: Ast.TokenIndex, - comptime fmt: []const u8, - args: anytype, - ) InnerError { - try appendError(p, tok, fmt, args); - return error.ParseFailure; - } - - fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void { - return appendErrorOff(p, tok, 0, fmt, args); - } - - fn appendErrorOff( - p: *Parse, - tok: Ast.TokenIndex, - byte_offset: u32, - comptime fmt: []const u8, - args: anytype, - ) Allocator.Error!void { - try p.errors.append(p.gpa, .{ - .msg = try std.fmt.allocPrint(p.arena, fmt, args), - .tok = tok, - .off = byte_offset, - }); - } -}; - -pub fn load( - io: Io, - arena: Allocator, - manifest_path: std.Build.Cache.Path, - ast: *std.zig.Ast, - error_bundle: *std.zig.ErrorBundle.Wip, - manifest: *Manifest, - allow_missing_paths_field: bool, -) !void { - const manifest_bytes = try manifest_path.root_dir.handle.readFileAllocOptions( - io, - manifest_path.sub_path, - arena, - .limited(max_bytes), - .@"1", - 0, - ); - - ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); - - if (ast.errors.len > 0) { - const file_path = try manifest_path.joinString(arena, ""); - try std.zig.putAstErrorsIntoBundle(arena, ast.*, file_path, error_bundle); - return error.ErrorsBundled; - } - - const rng: std.Random.IoSource = .{ .io = io }; - - manifest.* = try parse(arena, ast, rng.interface(), .{ - .allow_missing_paths_field = allow_missing_paths_field, - }); - - if (manifest.errors.len > 0) { - const src_path = try error_bundle.printString("{f}", .{manifest_path}); - try manifest.copyErrorsIntoBundle(ast.*, src_path, error_bundle); - return error.ErrorsBundled; - } -} - -test "basic" { - const gpa = testing.allocator; - - const example = - \\.{ - \\ .name = .foo, - \\ .fingerprint = 0x8c736521490b23df, - \\ .version = "3.2.1", - \\ .paths = .{""}, - \\ .dependencies = .{ - \\ .bar = .{ - \\ .url = "https://example.com/baz.tar.gz", - \\ .hash = "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5", - \\ }, - \\ }, - \\} - ; - - var ast = try Ast.parse(gpa, example, .zon); - defer ast.deinit(gpa); - - try testing.expect(ast.errors.len == 0); - - var rng = std.Random.DefaultPrng.init(0); - - var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); - defer manifest.deinit(gpa); - - try testing.expect(manifest.errors.len == 0); - try testing.expectEqualStrings("foo", manifest.name); - - try testing.expectEqual(@as(std.SemanticVersion, .{ - .major = 3, - .minor = 2, - .patch = 1, - }), manifest.version); - - try testing.expect(manifest.dependencies.count() == 1); - try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); - try testing.expectEqualStrings( - "https://example.com/baz.tar.gz", - manifest.dependencies.values()[0].location.url, - ); - try testing.expectEqualStrings( - "libmp3lame-3.100.1-6-67wlF_KvEwDRCT3pTpcDzi5KGntWCEoM-WtvVPEWdlk5", - manifest.dependencies.values()[0].hash orelse return error.TestFailed, - ); - - try testing.expect(manifest.minimum_zig_version == null); -} - -test "minimum_zig_version" { - const gpa = testing.allocator; - - const example = - \\.{ - \\ .name = .foo, - \\ .fingerprint = 0x8c736521490b23df, - \\ .version = "3.2.1", - \\ .paths = .{""}, - \\ .minimum_zig_version = "0.11.1", - \\} - ; - - var ast = try Ast.parse(gpa, example, .zon); - defer ast.deinit(gpa); - - try testing.expect(ast.errors.len == 0); - - var rng = std.Random.DefaultPrng.init(0); - - var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); - defer manifest.deinit(gpa); - - try testing.expect(manifest.errors.len == 0); - try testing.expect(manifest.dependencies.count() == 0); - - try testing.expect(manifest.minimum_zig_version != null); - - try testing.expectEqual(@as(std.SemanticVersion, .{ - .major = 0, - .minor = 11, - .patch = 1, - }), manifest.minimum_zig_version.?); -} - -test "minimum_zig_version - invalid version" { - const gpa = testing.allocator; - - const example = - \\.{ - \\ .name = .foo, - \\ .fingerprint = 0x8c736521490b23df, - \\ .version = "3.2.1", - \\ .minimum_zig_version = "X.11.1", - \\ .paths = .{""}, - \\} - ; - - var ast = try Ast.parse(gpa, example, .zon); - defer ast.deinit(gpa); - - try testing.expect(ast.errors.len == 0); - - var rng = std.Random.DefaultPrng.init(0); - - var manifest = try Manifest.parse(gpa, &ast, rng.random(), .{}); - defer manifest.deinit(gpa); - - try testing.expect(manifest.errors.len == 1); - try testing.expect(manifest.dependencies.count() == 0); - - try testing.expect(manifest.minimum_zig_version == null); -} diff --git a/src/Package/Module.zig b/src/Package/Module.zig deleted file mode 100644 index 0c7e4166adf7d6c290cbb330bb12857c1cf21d90..0000000000000000000000000000000000000000 --- a/src/Package/Module.zig +++ /dev/null @@ -1,529 +0,0 @@ -//! Corresponds to something that Zig source code can `@import`. - -/// The root directory of the module. Only files inside this directory can be imported. -root: Compilation.Path, -/// Path to the root source file of this module. Relative to `root`. May contain path separators. -root_src_path: []const u8, -/// Name used in compile errors. Looks like "root.foo.bar". -fully_qualified_name: []const u8, -/// The dependency table of this module. The shared dependencies 'std' and -/// 'root' are not specified in every module dependency table, but are stored -/// separately in `Zcu`. 'builtin' is also not stored here, although it is -/// not necessarily the same between all modules. Handling of `@import` in -/// the rest of the compiler must detect these special names and use the -/// correct module instead of consulting `deps`. -deps: Deps = .{}, - -resolved_target: ResolvedTarget, -optimize_mode: std.lang.OptimizeMode, -code_model: std.lang.CodeModel, -single_threaded: bool, -error_tracing: bool, -valgrind: bool, -pic: bool, -strip: bool, -omit_frame_pointer: bool, -stack_check: bool, -stack_protector: u32, -red_zone: bool, -sanitize_c: std.zig.SanitizeC, -sanitize_thread: bool, -fuzz: bool, -unwind_tables: std.lang.UnwindTables, -cc_argv: []const []const u8, -/// (SPIR-V) whether to generate a structured control flow graph or not -structured_cfg: bool, -no_builtin: bool, - -pub const Deps = std.array_hash_map.String(*Module); - -pub const Tree = struct { - /// Each `Package` exposes a `Module` with build.zig as its root source file. - build_module_table: std.array_hash_map.Auto(MultiHashHexDigest, *Module), -}; - -pub const CreateOptions = struct { - paths: Paths, - fully_qualified_name: []const u8, - - cc_argv: []const []const u8, - inherited: Inherited, - global: Compilation.Config, - /// If this is null then `resolved_target` must be non-null. - parent: ?*Package.Module, - - pub const Paths = struct { - root: Compilation.Path, - /// Relative to `root`. May contain path separators. - root_src_path: []const u8, - }; - - pub const Inherited = struct { - /// If this is null then `parent` must be non-null. - resolved_target: ?ResolvedTarget = null, - optimize_mode: ?std.lang.OptimizeMode = null, - code_model: ?std.lang.CodeModel = null, - single_threaded: ?bool = null, - error_tracing: ?bool = null, - valgrind: ?bool = null, - pic: ?bool = null, - strip: ?bool = null, - omit_frame_pointer: ?bool = null, - stack_check: ?bool = null, - /// null means default. - /// 0 means no stack protector. - /// other number means stack protection with that buffer size. - stack_protector: ?u32 = null, - red_zone: ?bool = null, - unwind_tables: ?std.lang.UnwindTables = null, - sanitize_c: ?std.zig.SanitizeC = null, - sanitize_thread: ?bool = null, - fuzz: ?bool = null, - structured_cfg: ?bool = null, - no_builtin: ?bool = null, - }; -}; - -pub const ResolvedTarget = struct { - result: std.Target, - is_native_os: bool, - is_native_abi: bool, - is_explicit_dynamic_linker: bool, - llvm_cpu_features: ?[*:0]const u8 = null, -}; - -pub const CreateError = error{ - OutOfMemory, - ValgrindUnsupportedOnTarget, - TargetRequiresSingleThreaded, - BackendRequiresSingleThreaded, - TargetRequiresPic, - PieRequiresPic, - DynamicLinkingRequiresPic, - TargetHasNoRedZone, - StackCheckUnsupportedByTarget, - StackProtectorUnsupportedByTarget, - StackProtectorUnavailableWithoutLibC, -}; - -/// At least one of `parent` and `resolved_target` must be non-null. -pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module { - if (options.inherited.sanitize_thread == true) assert(options.global.any_sanitize_thread); - if (options.inherited.fuzz == true) assert(options.global.any_fuzz); - if (options.inherited.single_threaded == false) assert(options.global.any_non_single_threaded); - if (options.inherited.unwind_tables) |uwt| if (uwt != .none) assert(options.global.any_unwind_tables); - if (options.inherited.sanitize_c) |sc| if (sc != .off) assert(options.global.any_sanitize_c != .off); - if (options.inherited.error_tracing == true) assert(options.global.any_error_tracing); - - const resolved_target = options.inherited.resolved_target orelse options.parent.?.resolved_target; - const target = &resolved_target.result; - - const optimize_mode = options.inherited.optimize_mode orelse - if (options.parent) |p| p.optimize_mode else options.global.root_optimize_mode; - - const strip = b: { - if (options.inherited.strip) |x| break :b x; - if (options.parent) |p| break :b p.strip; - break :b options.global.root_strip; - }; - - const zig_backend = target_util.zigBackend(target, options.global.use_llvm); - - const valgrind = b: { - if (!target_util.hasValgrindSupport(target, zig_backend)) { - if (options.inherited.valgrind == true) - return error.ValgrindUnsupportedOnTarget; - break :b false; - } - if (options.inherited.valgrind) |x| break :b x; - if (options.parent) |p| break :b p.valgrind; - if (strip) break :b false; - break :b optimize_mode == .Debug; - }; - - const single_threaded = b: { - if (target_util.alwaysSingleThreaded(target)) { - if (options.inherited.single_threaded == false) - return error.TargetRequiresSingleThreaded; - break :b true; - } - - if (options.global.have_zcu) { - if (!target_util.supportsThreads(target, zig_backend)) { - if (options.inherited.single_threaded == false) - return error.BackendRequiresSingleThreaded; - break :b true; - } - } - - if (options.inherited.single_threaded) |x| break :b x; - if (options.parent) |p| break :b p.single_threaded; - break :b target_util.defaultSingleThreaded(target); - }; - - const error_tracing = b: { - if (options.inherited.error_tracing) |x| break :b x; - if (options.parent) |p| break :b p.error_tracing; - break :b options.global.root_error_tracing; - }; - - const pic = b: { - if (target_util.requiresPic(target, options.global.link_libc)) { - if (options.inherited.pic == false) - return error.TargetRequiresPic; - break :b true; - } - if (options.global.pie) { - if (options.inherited.pic == false) - return error.PieRequiresPic; - break :b true; - } - if (options.global.link_mode == .dynamic and target_util.requiresPicForDynamicLink(target)) { - if (options.inherited.pic == false) - return error.DynamicLinkingRequiresPic; - break :b true; - } - if (options.inherited.pic) |x| break :b x; - if (options.parent) |p| break :b p.pic; - - // Default to PIC on targets where we default to producing PIEs to make - // the common case of linking objects and static libraries into an - // executable work out of the box. - break :b target_util.defaultPie(target); - }; - - const red_zone = b: { - if (!target_util.hasRedZone(target)) { - if (options.inherited.red_zone == true) - return error.TargetHasNoRedZone; - break :b false; - } - if (options.inherited.red_zone) |x| break :b x; - if (options.parent) |p| break :b p.red_zone; - break :b true; - }; - - const omit_frame_pointer = b: { - if (options.inherited.omit_frame_pointer) |x| break :b x; - if (options.parent) |p| break :b p.omit_frame_pointer; - if (optimize_mode == .ReleaseSmall) { - // On x86, in most cases, keeping the frame pointer usually results in smaller binary size. - // This has to do with how instructions for memory access via the stack base pointer register (when keeping the frame pointer) - // are smaller than instructions for memory access via the stack pointer register (when omitting the frame pointer). - break :b !target.cpu.arch.isX86(); - } - break :b false; - }; - - const sanitize_thread = b: { - if (options.inherited.sanitize_thread) |x| break :b x; - if (options.parent) |p| break :b p.sanitize_thread; - break :b false; - }; - - const unwind_tables = b: { - if (options.inherited.unwind_tables) |x| break :b x; - if (options.parent) |p| break :b p.unwind_tables; - - break :b target_util.defaultUnwindTables( - target, - options.global.link_libunwind, - sanitize_thread or options.global.any_sanitize_thread, - ); - }; - - const fuzz = b: { - if (options.inherited.fuzz) |x| break :b x; - if (options.parent) |p| break :b p.fuzz; - break :b false; - }; - - const code_model: std.lang.CodeModel = b: { - if (options.inherited.code_model) |x| break :b x; - if (options.parent) |p| break :b p.code_model; - break :b .default; - }; - - const is_safe_mode = switch (optimize_mode) { - .Debug, .ReleaseSafe => true, - .ReleaseFast, .ReleaseSmall => false, - }; - - const sanitize_c: std.zig.SanitizeC = b: { - if (options.inherited.sanitize_c) |x| break :b x; - if (options.parent) |p| break :b p.sanitize_c; - break :b switch (optimize_mode) { - .Debug => .full, - // It's recommended to use the minimal runtime in production - // environments due to the security implications of the full runtime. - // The minimal runtime doesn't provide much benefit over simply - // trapping, however, so we do that instead. - .ReleaseSafe => .trap, - .ReleaseFast, .ReleaseSmall => .off, - }; - }; - - const stack_check = b: { - if (!target_util.supportsStackProbing(target, zig_backend)) { - if (options.inherited.stack_check == true) - return error.StackCheckUnsupportedByTarget; - break :b false; - } - if (options.inherited.stack_check) |x| break :b x; - if (options.parent) |p| break :b p.stack_check; - break :b is_safe_mode; - }; - - const stack_protector: u32 = sp: { - const use_zig_backend = options.global.have_zcu or - (options.global.any_c_source_files and options.global.c_frontend == .aro); - if (use_zig_backend and !target_util.supportsStackProtector(target, zig_backend)) { - if (options.inherited.stack_protector) |x| { - if (x > 0) return error.StackProtectorUnsupportedByTarget; - } - break :sp 0; - } - - if (options.global.any_c_source_files and options.global.c_frontend == .clang and - !target_util.clangSupportsStackProtector(target)) - { - if (options.inherited.stack_protector) |x| { - if (x > 0) return error.StackProtectorUnsupportedByTarget; - } - break :sp 0; - } - - // This logic is checking for linking libc because otherwise our start code - // which is trying to set up TLS (i.e. the fs/gs registers) but the stack - // protection code depends on fs/gs registers being already set up. - // If we were able to annotate start code, or perhaps the entire std lib, - // as being exempt from stack protection checks, we could change this logic - // to supporting stack protection even when not linking libc. - // TODO file issue about this - if (!options.global.link_libc) { - if (options.inherited.stack_protector) |x| { - if (x > 0) return error.StackProtectorUnavailableWithoutLibC; - } - break :sp 0; - } - - if (options.inherited.stack_protector) |x| break :sp x; - if (options.parent) |p| break :sp p.stack_protector; - if (!is_safe_mode) break :sp 0; - - break :sp target_util.default_stack_protector_buffer_size; - }; - - const structured_cfg = b: { - if (options.inherited.structured_cfg) |x| break :b x; - if (options.parent) |p| break :b p.structured_cfg; - // We always want a structured control flow in shaders. This option is - // only relevant for OpenCL kernels. - break :b switch (target.os.tag) { - .opencl => false, - else => true, - }; - }; - - const no_builtin = b: { - if (options.inherited.no_builtin) |x| break :b x; - if (options.parent) |p| break :b p.no_builtin; - - break :b target.cpu.arch.isBpf(); - }; - - const llvm_cpu_features: ?[*:0]const u8 = b: { - if (resolved_target.llvm_cpu_features) |x| break :b x; - if (!options.global.use_llvm) break :b null; - - var buf = std.array_list.Managed(u8).init(arena); - var disabled_features = std.array_list.Managed(u8).init(arena); - defer disabled_features.deinit(); - - // Append disabled features after enabled ones, so that their effects aren't overwritten. - for (target.cpu.arch.allFeaturesList()) |feature| { - if (feature.llvm_name) |llvm_name| { - // Ignore these until we figure out how to handle the concept of omitting features. - // See https://github.com/ziglang/zig/issues/23539 - if (target_util.isDynamicAMDGCNFeature(target, feature)) continue; - - if (target.cpu.arch.isPowerPC() and @as(std.Target.powerpc.Feature, @enumFromInt(feature.index)) == .@"64bit") continue; - if (target.cpu.arch.isX86() and @as(std.Target.x86.Feature, @enumFromInt(feature.index)) == .x32) continue; - - var is_enabled = target.cpu.features.isEnabled(feature.index); - if (target.cpu.arch == .s390x and @as(std.Target.s390x.Feature, @enumFromInt(feature.index)) == .backchain) { - is_enabled = !omit_frame_pointer; - } - - if (is_enabled) { - try buf.ensureUnusedCapacity(2 + llvm_name.len); - buf.appendAssumeCapacity('+'); - buf.appendSliceAssumeCapacity(llvm_name); - buf.appendAssumeCapacity(','); - } else { - try disabled_features.ensureUnusedCapacity(2 + llvm_name.len); - disabled_features.appendAssumeCapacity('-'); - disabled_features.appendSliceAssumeCapacity(llvm_name); - disabled_features.appendAssumeCapacity(','); - } - } - } - - try buf.appendSlice(disabled_features.items); - if (buf.items.len == 0) break :b ""; - assert(std.mem.endsWith(u8, buf.items, ",")); - buf.items[buf.items.len - 1] = 0; - buf.shrinkAndFree(buf.items.len); - break :b buf.items[0 .. buf.items.len - 1 :0].ptr; - }; - - const mod = try arena.create(Module); - mod.* = .{ - .root = options.paths.root, - .root_src_path = options.paths.root_src_path, - .fully_qualified_name = options.fully_qualified_name, - .resolved_target = .{ - .result = target.*, - .is_native_os = resolved_target.is_native_os, - .is_native_abi = resolved_target.is_native_abi, - .is_explicit_dynamic_linker = resolved_target.is_explicit_dynamic_linker, - .llvm_cpu_features = llvm_cpu_features, - }, - .optimize_mode = optimize_mode, - .single_threaded = single_threaded, - .error_tracing = error_tracing, - .valgrind = valgrind, - .pic = pic, - .strip = strip, - .omit_frame_pointer = omit_frame_pointer, - .stack_check = stack_check, - .stack_protector = stack_protector, - .code_model = code_model, - .red_zone = red_zone, - .sanitize_c = sanitize_c, - .sanitize_thread = sanitize_thread, - .fuzz = fuzz, - .unwind_tables = unwind_tables, - .cc_argv = options.cc_argv, - .structured_cfg = structured_cfg, - .no_builtin = no_builtin, - }; - return mod; -} - -/// All fields correspond to `CreateOptions`. -pub const LimitedOptions = struct { - root: Compilation.Path, - root_src_path: []const u8, - fully_qualified_name: []const u8, -}; - -/// This one can only be used if the Module will only be used for AstGen and earlier in -/// the pipeline. Illegal behavior occurs if a limited module touches Sema. -pub fn createLimited(gpa: Allocator, options: LimitedOptions) Allocator.Error!*Package.Module { - const mod = try gpa.create(Module); - mod.* = .{ - .root = options.root, - .root_src_path = options.root_src_path, - .fully_qualified_name = options.fully_qualified_name, - - .resolved_target = undefined, - .optimize_mode = undefined, - .code_model = undefined, - .single_threaded = undefined, - .error_tracing = undefined, - .valgrind = undefined, - .pic = undefined, - .strip = undefined, - .omit_frame_pointer = undefined, - .stack_check = undefined, - .stack_protector = undefined, - .red_zone = undefined, - .sanitize_c = undefined, - .sanitize_thread = undefined, - .fuzz = undefined, - .unwind_tables = undefined, - .cc_argv = undefined, - .structured_cfg = undefined, - .no_builtin = undefined, - }; - return mod; -} - -/// Does not ensure that the module's root directory exists on-disk; see `Builtin.updateFileOnDisk` for that task. -pub fn createBuiltin(arena: Allocator, opts: Builtin, dirs: Compilation.Directories) Allocator.Error!*Module { - const sub_path = "b" ++ std.fs.path.sep_str ++ Cache.binToHex(opts.hash()); - const new = try arena.create(Module); - new.* = .{ - .root = try .fromRoot(arena, dirs, .global_cache, sub_path), - .root_src_path = "builtin.zig", - .fully_qualified_name = "builtin", - .resolved_target = .{ - .result = opts.target, - // These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code. - .is_native_os = false, - .is_native_abi = false, - .is_explicit_dynamic_linker = false, - .llvm_cpu_features = null, - }, - .optimize_mode = opts.optimize_mode, - .single_threaded = opts.single_threaded, - .error_tracing = opts.error_tracing, - .valgrind = opts.valgrind, - .pic = opts.pic, - .strip = opts.strip, - .omit_frame_pointer = opts.omit_frame_pointer, - .code_model = opts.code_model, - .sanitize_thread = opts.sanitize_thread, - .fuzz = opts.fuzz, - .unwind_tables = opts.unwind_tables, - .cc_argv = &.{}, - // These values are not in `opts`, but do not matter because `builtin.zig` contains no runtime code. - .stack_check = false, - .stack_protector = 0, - .red_zone = false, - .sanitize_c = .off, - .structured_cfg = false, - .no_builtin = false, - }; - return new; -} - -/// Returns the `Builtin` which forms the contents of `@import("builtin")` for this module. -pub fn getBuiltinOptions(m: Module, global: Compilation.Config) Builtin { - assert(global.have_zcu); - return .{ - .target = m.resolved_target.result, - .zig_backend = target_util.zigBackend(&m.resolved_target.result, global.use_llvm), - .output_mode = global.output_mode, - .link_mode = global.link_mode, - .unwind_tables = m.unwind_tables, - .is_test = global.is_test, - .single_threaded = m.single_threaded, - .link_libc = global.link_libc, - .link_libcpp = global.link_libcpp, - .optimize_mode = m.optimize_mode, - .error_tracing = m.error_tracing, - .valgrind = m.valgrind, - .sanitize_thread = m.sanitize_thread, - .fuzz = m.fuzz, - .pic = m.pic, - .pie = global.pie, - .strip = m.strip, - .code_model = m.code_model, - .omit_frame_pointer = m.omit_frame_pointer, - .wasi_exec_model = global.wasi_exec_model, - }; -} - -const Module = @This(); -const Package = @import("../Package.zig"); -const std = @import("std"); -const Allocator = std.mem.Allocator; -const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest; -const target_util = @import("../target.zig"); -const Cache = std.Build.Cache; -const Builtin = @import("../Builtin.zig"); -const assert = std.debug.assert; -const Compilation = @import("../Compilation.zig"); -const File = @import("../Zcu.zig").File; diff --git a/src/Sema.zig b/src/Sema.zig index 2469444831e1037c7e526a33a0f4c2f39b26474a..2c1d69b23e071c3c2bdaff4a19ed9f5ba0684daa 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -25,7 +25,6 @@ const SemaError = Zcu.SemaError; const LazySrcLoc = Zcu.LazySrcLoc; const RangeSet = @import("RangeSet.zig"); const target_util = @import("target.zig"); -const Package = @import("Package.zig"); const crash_report = @import("crash_report.zig"); const build_options = @import("build_options"); const Compilation = @import("Compilation.zig"); @@ -36,6 +35,7 @@ const ComptimeAllocIndex = InternPool.ComptimeAllocIndex; const Cache = std.Build.Cache; const LowerZon = @import("Sema/LowerZon.zig"); const arith = @import("Sema/arith.zig"); +const Module = @import("Module.zig"); pt: Zcu.PerThread, /// Alias to `zcu.gpa`. @@ -839,7 +839,7 @@ pub const Block = struct { return result_index; } - pub fn ownerModule(block: Block) *Package.Module { + pub fn ownerModule(block: Block) *Module { const zcu = block.sema.pt.zcu; return zcu.namespacePtr(block.namespace).fileScope(zcu).mod.?; } diff --git a/src/Zcu.zig b/src/Zcu.zig index e3a36fe31b987dce03f89d1f4694cf718d72d3dc..d36c96c07dab3f38df7abfff2a67003529a80201 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -25,7 +25,7 @@ const Compilation = @import("Compilation.zig"); const Cache = std.Build.Cache; pub const Value = @import("Value.zig"); pub const Type = @import("Type.zig"); -const Package = @import("Package.zig"); +const Module = @import("Module.zig"); const link = @import("link.zig"); const Air = @import("Air.zig"); const Zir = std.zig.Zir; @@ -34,7 +34,6 @@ const AstGen = std.zig.AstGen; const Sema = @import("Sema.zig"); const target_util = @import("target.zig"); const build_options = @import("build_options"); -const isUpDir = @import("introspect.zig").isUpDir; const InternPool = @import("InternPool.zig"); const Alignment = InternPool.Alignment; const AnalUnit = InternPool.AnalUnit; @@ -65,11 +64,11 @@ comp: *Compilation, llvm_object: ?LlvmObject.Ptr, /// Pointer to externally managed resource. -root_mod: *Package.Module, +root_mod: *Module, /// Normally, `main_mod` and `root_mod` are the same. The exception is `zig test`, in which /// `root_mod` is the test runner, and `main_mod` is the user's source file which has the tests. -main_mod: *Package.Module, -std_mod: *Package.Module, +main_mod: *Module, +std_mod: *Module, sema_prog_node: std.Progress.Node = .none, codegen_prog_node: std.Progress.Node = .none, /// The number of codegen jobs which are pending or in-progress. Whichever thread drops this value @@ -106,11 +105,11 @@ multi_exports: std.array_hash_map.Auto(AnalUnit, extern struct { }) = .{}, /// Key is the digest returned by `Builtin.hash`; value is the corresponding module. -builtin_modules: std.array_hash_map.Auto(Cache.BinDigest, *Package.Module) = .empty, +builtin_modules: std.array_hash_map.Auto(Cache.BinDigest, *Module) = .empty, /// Populated as soon as the `Compilation` is created. Guaranteed to contain all modules, even builtin ones. /// Modules whose root file is not a Zig or ZON file have the value `.none`. -module_roots: std.array_hash_map.Auto(*Package.Module, File.Index.Optional) = .empty, +module_roots: std.array_hash_map.Auto(*Module, File.Index.Optional) = .empty, /// The set of all the Zig source files in the Zig Compilation Unit. Tracked in /// order to iterate over it and check which source files have been modified on @@ -149,7 +148,7 @@ alive_files: std.array_hash_map.Auto(File.Index, File.Reference) = .empty, /// Cleared and recomputed every update, after AstGen and before Sema. multi_module_err: ?struct { file: File.Index, - modules: [2]*Package.Module, + modules: [2]*Module, refs: [2]File.Reference, } = null, @@ -293,7 +292,7 @@ retryable_failures: std.ArrayList(AnalUnit) = .empty, /// These are the modules which we initially queue for analysis in `Compilation.update`. /// `resolveReferences` will use these as the root of its reachability traversal. -analysis_roots_buffer: [5]*Package.Module, +analysis_roots_buffer: [5]*Module, analysis_roots_len: usize = 0, /// This is the cached result of `Zcu.resolveReferences`. It is computed on-demand, and /// reset to `null` when any semantic analysis occurs (since this invalidates the data). @@ -986,7 +985,7 @@ pub const File = struct { /// tell, and invalidate dependencies as needed (see `module_changed`). /// During semantic analysis, this is always non-`null` for alive files (i.e. those which /// have imports targeting them). - mod: ?*Package.Module, + mod: ?*Module, /// Relative to the root directory of `mod`. If `mod == null`, this field is `undefined`. /// This memory is managed externally and must not be directly freed. /// Its lifetime is at least equal to that of this `File`. @@ -1029,13 +1028,13 @@ pub const File = struct { /// A single reference to a file. pub const Reference = union(enum) { - analysis_root: *Package.Module, + analysis_root: *Module, import: struct { importer: Zcu.File.Index, tok: Ast.TokenIndex, /// If the file is imported as the root of a module, this is that module. /// `null` means the file was imported directly by path. - module: ?*Package.Module, + module: ?*Module, }, }; @@ -3710,7 +3709,7 @@ pub const ImportResult = struct { /// If this import was a simple file path, this is `null`; the imported file should exist within /// the importer's module. Otherwise, it's the module which the import resolved to. This module /// could match the module of `cur_file`, since a module can depend on itself. - module: ?*Package.Module, + module: ?*Module, }; /// Prepares `unit` for re-analysis by clearing all of the following state: @@ -4407,7 +4406,7 @@ fn resolveReferencesInner(zcu: *Zcu) Allocator.Error!std.array_hash_map.Auto(Ana return units.move(); } -pub fn analysisRoots(zcu: *Zcu) []*Package.Module { +pub fn analysisRoots(zcu: *Zcu) []*Module { return zcu.analysis_roots_buffer[0..zcu.analysis_roots_len]; } @@ -4821,7 +4820,7 @@ fn explainWhyFileIsInModule( eb: *std.zig.ErrorBundle.Wip, notes_out: *std.ArrayList(std.zig.ErrorBundle.MessageIndex), file: File.Index, - in_module: *Package.Module, + in_module: *Module, ref: File.Reference, ) Allocator.Error!void { const gpa = zcu.gpa; @@ -4867,7 +4866,7 @@ fn explainWhyFileIsInModule( const import_src = try importer_file.errorBundleTokenSrc(import.tok, zcu, eb); const importer_ref = zcu.alive_files.get(import.importer).?; - const importer_root: ?*Package.Module = switch (importer_ref) { + const importer_root: ?*Module = switch (importer_ref) { .analysis_root => |mod| mod, .import => |i| i.module, }; diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index e1cda016c163cdfde8cb693c5f143f2f86057980..c6c665655cf1ae7a792fabb44b5433efa6664bef 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -23,8 +23,7 @@ const builtin = @import("builtin"); const dev = @import("../dev.zig"); const InternPool = @import("../InternPool.zig"); const AnalUnit = InternPool.AnalUnit; -const introspect = @import("../introspect.zig"); -const Module = @import("../Package.zig").Module; +const Module = @import("../Module.zig"); const Sema = @import("../Sema.zig"); const target_util = @import("../target.zig"); const tracy = @import("../tracy.zig"); diff --git a/src/codegen/aarch64/Select.zig b/src/codegen/aarch64/Select.zig index c8cfd0aeda8a5a51594d278926e3dd8666e57e83..ba8cde4e9c45bbdca010d1d87eb1e8a2b3ef2f52 100644 --- a/src/codegen/aarch64/Select.zig +++ b/src/codegen/aarch64/Select.zig @@ -7592,7 +7592,7 @@ pub fn layout( is_sysv_var_args: bool, saved_gra_len: u7, saved_vra_len: u7, - mod: *const Package.Module, + mod: *const Module, ) !usize { const zcu = isel.pt.zcu; const ip = &zcu.intern_pool; @@ -12513,7 +12513,7 @@ const assert = std.debug.assert; const codegen = @import("../../codegen.zig"); const Constant = @import("../../Value.zig"); const InternPool = @import("../../InternPool.zig"); -const Package = @import("../../Package.zig"); +const Module = @import("../../Module.zig"); const Register = codegen.aarch64.encoding.Register; const Select = @This(); const std = @import("std"); diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 84744bf3cc55e62603b2f01cff265539979f23f0..b814fe55c16b3c96ab7f6caee7b4e1119ae73257 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -9,7 +9,7 @@ const Writer = std.Io.Writer; const dev = @import("../dev.zig"); const link = @import("../link.zig"); const Zcu = @import("../Zcu.zig"); -const Module = @import("../Package/Module.zig"); +const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); const Value = @import("../Value.zig"); const Type = @import("../Type.zig"); diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 6361963ab4d898bcf84a08fed641c7390e476e4a..2e79acc1e3e930b8c437099650329117a61e5428 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -13,7 +13,7 @@ const Compilation = @import("../Compilation.zig"); const dev = @import("../dev.zig"); const InternPool = @import("../InternPool.zig"); const link = @import("../link.zig"); -const Package = @import("../Package.zig"); +const Module = @import("../Module.zig"); const target_util = @import("../target.zig"); const Type = @import("../Type.zig"); const Value = @import("../Value.zig"); @@ -2772,7 +2772,7 @@ pub const Object = struct { fn addCommonFnAttributes( o: *Object, attributes: *Builder.FunctionAttributes.Wip, - owner_mod: *Package.Module, + owner_mod: *Module, omit_frame_pointer: bool, ) Allocator.Error!void { if (!owner_mod.red_zone) { diff --git a/src/codegen/llvm/FuncGen.zig b/src/codegen/llvm/FuncGen.zig index 25572de79a872b0d2f5fa51ef4ca16ca762514e4..bad4b1461b1c01274d481c984a752bb6704d4615 100644 --- a/src/codegen/llvm/FuncGen.zig +++ b/src/codegen/llvm/FuncGen.zig @@ -80,7 +80,7 @@ fn todo(fg: *FuncGen, comptime format: []const u8, args: anytype) TodoError { ); } -fn ownerModule(fg: *const FuncGen) *Package.Module { +fn ownerModule(fg: *const FuncGen) *Module { return fg.object.zcu.navFileScope(fg.nav_index).mod.?; } @@ -7739,7 +7739,7 @@ const mips_c_abi = @import("../mips/abi.zig"); const Zcu = @import("../../Zcu.zig"); const Air = @import("../../Air.zig"); -const Package = @import("../../Package.zig"); +const Module = @import("../../Module.zig"); const InternPool = @import("../../InternPool.zig"); const Value = @import("../../Value.zig"); const Type = @import("../../Type.zig"); diff --git a/src/codegen/riscv64/CodeGen.zig b/src/codegen/riscv64/CodeGen.zig index 08df78513efa32c24a4dab639d2d3f9878689153..4bd30651254a7582d1d49a82562526a9c3e44735 100644 --- a/src/codegen/riscv64/CodeGen.zig +++ b/src/codegen/riscv64/CodeGen.zig @@ -14,7 +14,7 @@ const Type = @import("../../Type.zig"); const Value = @import("../../Value.zig"); const link = @import("../../link.zig"); const Zcu = @import("../../Zcu.zig"); -const Package = @import("../../Package.zig"); +const Module = @import("../../Module.zig"); const InternPool = @import("../../InternPool.zig"); const Compilation = @import("../../Compilation.zig"); const target_util = @import("../../target.zig"); @@ -66,7 +66,7 @@ liveness: Air.Liveness, bin_file: *link.File, gpa: Allocator, -mod: *Package.Module, +mod: *Module, target: *const std.Target, args: []MCValue, ret_mcv: InstTracking, diff --git a/src/codegen/x86_64/CodeGen.zig b/src/codegen/x86_64/CodeGen.zig index 5ff7a45954737fdaf651d1073212ec27fa08ddb5..74fb9b4f87725574506a14178d989a66b5ae5731 100644 --- a/src/codegen/x86_64/CodeGen.zig +++ b/src/codegen/x86_64/CodeGen.zig @@ -14,7 +14,7 @@ const Emit = @import("Emit.zig"); const Lower = @import("Lower.zig"); const Mir = @import("Mir.zig"); const Zcu = @import("../../Zcu.zig"); -const Module = @import("../../Package/Module.zig"); +const Module = @import("../../Module.zig"); const InternPool = @import("../../InternPool.zig"); const Type = @import("../../Type.zig"); const Value = @import("../../Value.zig"); diff --git a/src/dev.zig b/src/dev.zig index bba67696f2f92cf522949f68ffc35370f3daf86d..47acafc9ce5ba1e279b32624b0190b891792d8c1 100644 --- a/src/dev.zig +++ b/src/dev.zig @@ -76,7 +76,6 @@ pub const Env = enum { .test_command, .run_command, .ar_command, - .build_command, .clang_command, .stdio_listen, .build_import_lib, @@ -108,12 +107,11 @@ pub const Env = enum { .wasm_linker, .spirv_linker, .plan9_linker, - => true, - .cc_command, - .translate_c_command, - .fmt_command, .jit_command, - .fetch_command, + => true, + .cc_command, + .translate_c_command, + .fmt_command, .init_command, .targets_command, .version_command, @@ -162,7 +160,6 @@ pub const Env = enum { else => Env.ast_gen.supports(feature), }, .@"aarch64-linux" => switch (feature) { - .build_command, .stdio_listen, .incremental, .aarch64_backend, @@ -179,7 +176,6 @@ pub const Env = enum { else => Env.sema.supports(feature), }, .@"powerpc-linux" => switch (feature) { - .build_command, .stdio_listen, .incremental, .x86_64_backend, @@ -210,7 +206,6 @@ pub const Env = enum { else => Env.sema.supports(feature), }, .@"x86_64-linux" => switch (feature) { - .build_command, .stdio_listen, .incremental, .legalize, @@ -251,13 +246,11 @@ pub const Feature = enum { test_command, run_command, ar_command, - build_command, clang_command, cc_command, translate_c_command, fmt_command, jit_command, - fetch_command, init_command, targets_command, version_command, diff --git a/src/introspect.zig b/src/introspect.zig deleted file mode 100644 index 13d00520936973fccc37ede11256842e0826f423..0000000000000000000000000000000000000000 --- a/src/introspect.zig +++ /dev/null @@ -1,220 +0,0 @@ -const builtin = @import("builtin"); - -const std = @import("std"); -const Io = std.Io; -const Dir = std.Io.Dir; -const mem = std.mem; -const Allocator = std.mem.Allocator; -const Cache = std.Build.Cache; -const assert = std.debug.assert; - -const build_options = @import("build_options"); - -const Compilation = @import("Compilation.zig"); -const Package = @import("Package.zig"); - -/// Returns the sub_path that worked, or `null` if none did. -/// The path of the returned Directory is relative to `base`. -/// The handle of the returned Directory is open. -fn testZigInstallPrefix(io: Io, base_dir: Io.Dir) ?Cache.Directory { - const test_index_file = "std" ++ Dir.path.sep_str ++ "std.zig"; - - zig_dir: { - // Try lib/zig/std/std.zig - const lib_zig = "lib" ++ Dir.path.sep_str ++ "zig"; - var test_zig_dir = base_dir.openDir(io, lib_zig, .{}) catch break :zig_dir; - const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { - test_zig_dir.close(io); - break :zig_dir; - }; - file.close(io); - return .{ .handle = test_zig_dir, .path = lib_zig }; - } - - // Try lib/std/std.zig - var test_zig_dir = base_dir.openDir(io, "lib", .{}) catch return null; - const file = test_zig_dir.openFile(io, test_index_file, .{}) catch { - test_zig_dir.close(io); - return null; - }; - file.close(io); - return .{ .handle = test_zig_dir, .path = "lib" }; -} - -/// Both the directory handle and the path are newly allocated resources which the caller now owns. -pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory { - const cwd_path = try getResolvedCwd(io, gpa); - defer gpa.free(cwd_path); - const self_exe_path = try std.process.executablePathAlloc(io, gpa); - defer gpa.free(self_exe_path); - - return findZigLibDirFromSelfExe(gpa, io, cwd_path, self_exe_path); -} - -/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This -/// means the path has no repeated separators, no "." or ".." components, and no trailing separator. -/// On WASI, "" is returned instead of ".". -pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 { - if (builtin.target.os.tag == .wasi) { - if (std.debug.runtime_safety) { - const cwd = try std.process.currentPathAlloc(io, gpa); - defer gpa.free(cwd); - assert(mem.eql(u8, cwd, ".")); - } - return ""; - } - const cwd = try std.process.currentPathAlloc(io, gpa); - defer gpa.free(cwd); - const resolved = try Dir.path.resolve(gpa, &.{cwd}); - assert(Dir.path.isAbsolute(resolved)); - return resolved; -} - -/// Both the directory handle and the path are newly allocated resources which the caller now owns. -pub fn findZigLibDirFromSelfExe( - allocator: Allocator, - io: Io, - /// The return value of `getResolvedCwd`. - /// Passed as an argument to avoid pointlessly repeating the call. - cwd_path: []const u8, - self_exe_path: []const u8, -) error{ OutOfMemory, FileNotFound }!Cache.Directory { - const cwd = Io.Dir.cwd(); - var cur_path: []const u8 = self_exe_path; - while (Dir.path.dirname(cur_path)) |dirname| : (cur_path = dirname) { - var base_dir = cwd.openDir(io, dirname, .{}) catch continue; - defer base_dir.close(io); - - const sub_directory = testZigInstallPrefix(io, base_dir) orelse continue; - const p = try Dir.path.join(allocator, &.{ dirname, sub_directory.path.? }); - defer allocator.free(p); - - const resolved = try resolvePath(allocator, cwd_path, &.{p}); - return .{ - .handle = sub_directory.handle, - .path = if (resolved.len == 0) null else resolved, - }; - } - return error.FileNotFound; -} - -pub fn resolveGlobalCacheDir(arena: Allocator, environ_map: *const std.process.Environ.Map) ![]const u8 { - if (std.zig.EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map)) |value| return value; - - const app_name = "zig"; - - switch (builtin.os.tag) { - .wasi => @compileError("on WASI the global cache dir must be resolved with preopens"), - .windows => { - const local_app_data_dir = std.zig.EnvVar.LOCALAPPDATA.get(environ_map) orelse - return error.AppDataDirUnavailable; - return Dir.path.join(arena, &.{ local_app_data_dir, app_name }); - }, - else => { - if (std.zig.EnvVar.XDG_CACHE_HOME.get(environ_map)) |cache_root| { - if (cache_root.len > 0) { - return Dir.path.join(arena, &.{ cache_root, app_name }); - } - } - if (std.zig.EnvVar.HOME.get(environ_map)) |home| { - if (home.len > 0) { - return Dir.path.join(arena, &.{ home, ".cache", app_name }); - } - } - return error.AppDataDirUnavailable; - }, - } -} - -/// Similar to `Dir.path.resolve`, but converts to a cwd-relative path, or, if that would -/// start with a relative up-dir (".."), an absolute path based on the cwd. Also, the cwd -/// returns the empty string ("") instead of ".". -pub fn resolvePath( - gpa: Allocator, - /// The return value of `getResolvedCwd`. - /// Passed as an argument to avoid pointlessly repeating the call. - cwd_resolved: []const u8, - paths: []const []const u8, -) Allocator.Error![]u8 { - if (builtin.target.os.tag == .wasi) { - assert(mem.eql(u8, cwd_resolved, "")); - const res = try Dir.path.resolve(gpa, paths); - if (mem.eql(u8, res, ".")) { - gpa.free(res); - return ""; - } - return res; - } - - // Heuristic for a fast path: if no component is absolute and ".." never appears, we just need to resolve `paths`. - for (paths) |p| { - if (Dir.path.isAbsolute(p)) break; // absolute path - if (mem.indexOf(u8, p, "..") != null) break; // may contain up-dir - } else { - // no absolute path, no "..". - const res = try Dir.path.resolve(gpa, paths); - if (mem.eql(u8, res, ".")) { - gpa.free(res); - return ""; - } - assert(!Dir.path.isAbsolute(res)); - assert(!isUpDir(res)); - return res; - } - - // The fast path failed; resolve the whole thing. - // Optimization: `paths` often has just one element. - const path_resolved = switch (paths.len) { - 0 => unreachable, - 1 => try Dir.path.resolve(gpa, &.{ cwd_resolved, paths[0] }), - else => r: { - const all_paths = try gpa.alloc([]const u8, paths.len + 1); - defer gpa.free(all_paths); - all_paths[0] = cwd_resolved; - @memcpy(all_paths[1..], paths); - break :r try Dir.path.resolve(gpa, all_paths); - }, - }; - errdefer gpa.free(path_resolved); - - assert(Dir.path.isAbsolute(path_resolved)); - assert(Dir.path.isAbsolute(cwd_resolved)); - - if (!std.mem.startsWith(u8, path_resolved, cwd_resolved)) return path_resolved; // not in cwd - if (path_resolved.len == cwd_resolved.len) { - // equal to cwd - gpa.free(path_resolved); - return ""; - } - if (path_resolved[cwd_resolved.len] != Dir.path.sep) return path_resolved; // not in cwd (last component differs) - - // in cwd; extract sub path - const sub_path = try gpa.dupe(u8, path_resolved[cwd_resolved.len + 1 ..]); - gpa.free(path_resolved); - return sub_path; -} - -pub fn isUpDir(p: []const u8) bool { - return mem.startsWith(u8, p, "..") and (p.len == 2 or p[2] == Dir.path.sep); -} - -pub const default_local_zig_cache_basename = ".zig-cache"; - -/// Searches upwards from `cwd` for a directory containing a `build.zig` file. -/// If such a directory is found, returns the path to it joined to the `.zig_cache` name. -/// Otherwise, returns `null`, indicating no suitable local cache location. -pub fn resolveSuitableLocalCacheDir(arena: Allocator, io: Io, cwd: []const u8) Allocator.Error!?[]u8 { - var cur_dir = cwd; - while (true) { - const joined = try Dir.path.join(arena, &.{ cur_dir, Package.build_zig_basename }); - if (Io.Dir.cwd().access(io, joined, .{})) |_| { - return try Dir.path.join(arena, &.{ cur_dir, default_local_zig_cache_basename }); - } else |err| switch (err) { - error.FileNotFound => { - cur_dir = Dir.path.dirname(cur_dir) orelse return null; - continue; - }, - else => return null, - } - } -} diff --git a/src/libs/freebsd.zig b/src/libs/freebsd.zig index 2270de6d60fd38aa527c9bd78ee21823e2b89554..ecc15e0dab899e21093c047e58a7b02467a29ac6 100644 --- a/src/libs/freebsd.zig +++ b/src/libs/freebsd.zig @@ -12,7 +12,7 @@ const Compilation = @import("../Compilation.zig"); const build_options = @import("build_options"); const trace = @import("../tracy.zig").trace; const Cache = std.Build.Cache; -const Module = @import("../Package/Module.zig"); +const Module = @import("../Module.zig"); const link = @import("../link.zig"); pub const CrtFile = enum { diff --git a/src/libs/glibc.zig b/src/libs/glibc.zig index 9d07149dfc867f68640a73f34e3e9d19656fff26..af80133d3332e33b69334b6c2b3dcc1b3f075979 100644 --- a/src/libs/glibc.zig +++ b/src/libs/glibc.zig @@ -12,7 +12,7 @@ const Compilation = @import("../Compilation.zig"); const build_options = @import("build_options"); const trace = @import("../tracy.zig").trace; const Cache = std.Build.Cache; -const Module = @import("../Package/Module.zig"); +const Module = @import("../Module.zig"); const link = @import("../link.zig"); pub const Lib = struct { diff --git a/src/libs/libcxx.zig b/src/libs/libcxx.zig index 3eb4161235382c9b63dab07edf88c27e74dc1a08..cc91db8333dde6f83f32e7f964742d36e692cc92 100644 --- a/src/libs/libcxx.zig +++ b/src/libs/libcxx.zig @@ -6,7 +6,7 @@ const target_util = @import("../target.zig"); const Compilation = @import("../Compilation.zig"); const build_options = @import("build_options"); const trace = @import("../tracy.zig").trace; -const Module = @import("../Package/Module.zig"); +const Module = @import("../Module.zig"); const libcxxabi_files = [_][]const u8{ "src/cxa_aux_runtime.cpp", diff --git a/src/libs/libtsan.zig b/src/libs/libtsan.zig index 6f83d106455905eb17d0c19adff84b295fb5e8ed..af8830699ca5d240ea379d98cb9382d67fe0d984 100644 --- a/src/libs/libtsan.zig +++ b/src/libs/libtsan.zig @@ -4,7 +4,7 @@ const assert = std.debug.assert; const Compilation = @import("../Compilation.zig"); const build_options = @import("build_options"); const trace = @import("../tracy.zig").trace; -const Module = @import("../Package/Module.zig"); +const Module = @import("../Module.zig"); pub const BuildError = error{ OutOfMemory, diff --git a/src/libs/libunwind.zig b/src/libs/libunwind.zig index e4c1096efd5d37368d485100be96304d77be2846..82ef9c34bc3d76b5cdbf2306ff8faf2526f54cbb 100644 --- a/src/libs/libunwind.zig +++ b/src/libs/libunwind.zig @@ -4,7 +4,7 @@ const assert = std.debug.assert; const target_util = @import("../target.zig"); const Compilation = @import("../Compilation.zig"); -const Module = @import("../Package/Module.zig"); +const Module = @import("../Module.zig"); const build_options = @import("build_options"); const trace = @import("../tracy.zig").trace; diff --git a/src/libs/musl.zig b/src/libs/musl.zig index fcf179894c6d6f0d542f04fb925788c76f82ca1a..e3613d020b203a9dc9858bb8c48d1316a2338509 100644 --- a/src/libs/musl.zig +++ b/src/libs/musl.zig @@ -3,7 +3,7 @@ const Allocator = std.mem.Allocator; const mem = std.mem; const path = std.fs.path; const assert = std.debug.assert; -const Module = @import("../Package/Module.zig"); +const Module = @import("../Module.zig"); const Compilation = @import("../Compilation.zig"); const build_options = @import("build_options"); diff --git a/src/libs/netbsd.zig b/src/libs/netbsd.zig index 15462afd3db224ee6fde38ce6737014a72d89c3c..c3e0a38ffb6cd6cb98cfd51bb0cc74151c627bd4 100644 --- a/src/libs/netbsd.zig +++ b/src/libs/netbsd.zig @@ -12,7 +12,7 @@ const Compilation = @import("../Compilation.zig"); const build_options = @import("build_options"); const trace = @import("../tracy.zig").trace; const Cache = std.Build.Cache; -const Module = @import("../Package/Module.zig"); +const Module = @import("../Module.zig"); const link = @import("../link.zig"); pub const CrtFile = enum { diff --git a/src/libs/openbsd.zig b/src/libs/openbsd.zig index 05d30e53edf9130ea13e32d6f42e40870c00198d..ee50196c8169bcb52f0a1bbd455f4b14c20f93e8 100644 --- a/src/libs/openbsd.zig +++ b/src/libs/openbsd.zig @@ -13,7 +13,7 @@ const Compilation = @import("../Compilation.zig"); const build_options = @import("build_options"); const trace = @import("../tracy.zig").trace; const Cache = std.Build.Cache; -const Module = @import("../Package/Module.zig"); +const Module = @import("../Module.zig"); const link = @import("../link.zig"); pub const CrtFile = enum { diff --git a/src/link.zig b/src/link.zig index 4764e0c291f4c048812f5f30dc94d1f948c9b442..5307868ddf691e8df3f9b1912ddb63649897d0d4 100644 --- a/src/link.zig +++ b/src/link.zig @@ -21,7 +21,6 @@ const Zcu = @import("Zcu.zig"); const InternPool = @import("InternPool.zig"); const Type = @import("Type.zig"); const Value = @import("Value.zig"); -const Package = @import("Package.zig"); const dev = @import("dev.zig"); const target_util = @import("target.zig"); const codegen = @import("codegen.zig"); diff --git a/src/link/C.zig b/src/link/C.zig index 127485171314adc352096f6e13b448b28d12e173..a5862d87eeba5b8833abcc92021e6f1ee74318a1 100644 --- a/src/link/C.zig +++ b/src/link/C.zig @@ -13,7 +13,7 @@ const Path = std.Build.Cache.Path; const build_options = @import("build_options"); const Zcu = @import("../Zcu.zig"); -const Module = @import("../Package/Module.zig"); +const Module = @import("../Module.zig"); const InternPool = @import("../InternPool.zig"); const Alignment = InternPool.Alignment; const Compilation = @import("../Compilation.zig"); diff --git a/src/link/Dwarf.zig b/src/link/Dwarf.zig index 96fb3d10c9191de8a2acaf7acaf799f5b9832fc5..d51c59ed23941283b2de8258db0ad865adf8aa33 100644 --- a/src/link/Dwarf.zig +++ b/src/link/Dwarf.zig @@ -10,7 +10,7 @@ const log = std.log.scoped(.dwarf); const Writer = std.Io.Writer; const InternPool = @import("../InternPool.zig"); -const Module = @import("../Package.zig").Module; +const Module = @import("../Module.zig"); const Type = @import("../Type.zig"); const Value = @import("../Value.zig"); const Zcu = @import("../Zcu.zig"); diff --git a/src/link/Lld.zig b/src/link/Lld.zig index c4c4e0fb3d721cc134473db4b348ee65a048d08e..715c0ba69f0c8f50ad6306ad7519c173a4d3dc0e 100644 --- a/src/link/Lld.zig +++ b/src/link/Lld.zig @@ -436,23 +436,23 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { try argv.append("-DEBUG"); const out_ext = std.fs.path.extension(full_out_path); - const out_pdb = coff.pdb_out_path orelse try allocPrint(arena, "{s}.pdb", .{ + const out_pdb = coff.pdb_out_path orelse try arena.print("{s}.pdb", .{ full_out_path[0 .. full_out_path.len - out_ext.len], }); const out_pdb_basename = std.fs.path.basename(out_pdb); - try argv.append(try allocPrint(arena, "-PDB:{s}", .{out_pdb})); - try argv.append(try allocPrint(arena, "-PDBALTPATH:{s}", .{out_pdb_basename})); + try argv.append(try arena.print("-PDB:{s}", .{out_pdb})); + try argv.append(try arena.print("-PDBALTPATH:{s}", .{out_pdb_basename})); } if (comp.version) |version| { - try argv.append(try allocPrint(arena, "-VERSION:{d}.{d}", .{ version.major, version.minor })); + try argv.append(try arena.print("-VERSION:{d}.{d}", .{ version.major, version.minor })); } if (target_util.llvmMachineAbi(target)) |mabi| { - try argv.append(try allocPrint(arena, "-MLLVM:-target-abi={s}", .{mabi})); + try argv.append(try arena.print("-MLLVM:-target-abi={s}", .{mabi})); } - try argv.append(try allocPrint(arena, "-MLLVM:-float-abi={s}", .{if (target.abi.float() == .hard) "hard" else "soft"})); + try argv.append(try arena.print("-MLLVM:-float-abi={s}", .{if (target.abi.float() == .hard) "hard" else "soft"})); if (comp.config.lto != .none) { switch (optimize_mode) { @@ -462,9 +462,9 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { } } if (comp.config.output_mode == .Exe) { - try argv.append(try allocPrint(arena, "-STACK:{d}", .{base.stack_size})); + try argv.append(try arena.print("-STACK:{d}", .{base.stack_size})); } - try argv.append(try allocPrint(arena, "-BASE:{d}", .{coff.image_base})); + try argv.append(try arena.print("-BASE:{d}", .{coff.image_base})); switch (base.build_id) { .none => try argv.append("-BUILD-ID:NO"), @@ -483,7 +483,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { } for (comp.force_undefined_symbols.keys()) |symbol| { - try argv.append(try allocPrint(arena, "-INCLUDE:{s}", .{symbol})); + try argv.append(try arena.print("-INCLUDE:{s}", .{symbol})); } if (is_dyn_lib) { @@ -491,7 +491,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { } if (entry_name) |name| { - try argv.append(try allocPrint(arena, "-ENTRY:{s}", .{name})); + try argv.append(try arena.print("-ENTRY:{s}", .{name})); } if (coff.repro) { @@ -511,26 +511,26 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { try argv.append("-FORCE:UNRESOLVED"); } - try argv.append(try allocPrint(arena, "-OUT:{s}", .{full_out_path})); + try argv.append(try arena.print("-OUT:{s}", .{full_out_path})); if (comp.emit_implib) |raw_emit_path| { const path = try comp.resolveEmitPathFlush(arena, .artifact, raw_emit_path); - try argv.append(try allocPrint(arena, "-IMPLIB:{f}", .{path})); + try argv.append(try arena.print("-IMPLIB:{f}", .{path})); } if (comp.config.link_libc) { if (comp.libc_installation) |libc_installation| { - try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.crt_dir.?})); + try argv.append(try arena.print("-LIBPATH:{s}", .{libc_installation.crt_dir.?})); if (target.abi == .msvc or target.abi == .itanium) { - try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.msvc_lib_dir.?})); - try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{libc_installation.kernel32_lib_dir.?})); + try argv.append(try arena.print("-LIBPATH:{s}", .{libc_installation.msvc_lib_dir.?})); + try argv.append(try arena.print("-LIBPATH:{s}", .{libc_installation.kernel32_lib_dir.?})); } } } for (coff.lib_directories) |lib_directory| { - try argv.append(try allocPrint(arena, "-LIBPATH:{s}", .{lib_directory.path orelse "."})); + try argv.append(try arena.print("-LIBPATH:{s}", .{lib_directory.path orelse "."})); } try argv.ensureUnusedCapacity(comp.link_inputs.len); @@ -541,7 +541,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { }, .object, .archive => |obj| { if (obj.must_link) { - argv.appendAssumeCapacity(try allocPrint(arena, "-WHOLEARCHIVE:{f}", .{@as(Cache.Path, obj.path)})); + argv.appendAssumeCapacity(try arena.print("-WHOLEARCHIVE:{f}", .{@as(Cache.Path, obj.path)})); } else { argv.appendAssumeCapacity(try obj.path.toString(arena)); } @@ -561,7 +561,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { } if (coff.module_definition_file) |def| { - try argv.append(try allocPrint(arena, "-DEF:{s}", .{def})); + try argv.append(try arena.print("-DEF:{s}", .{def})); } const resolved_subsystem: ?std.zig.Subsystem = blk: { @@ -590,7 +590,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { const Mode = enum { uefi, win32 }; const mode: Mode = mode: { if (resolved_subsystem) |subsystem| { - try argv.append(try allocPrint(arena, "-SUBSYSTEM:{s},{d}.{d}", .{ + try argv.append(try arena.print("-SUBSYSTEM:{s},{d}.{d}", .{ @tagName(subsystem), coff.major_subsystem_version, coff.minor_subsystem_version, @@ -645,8 +645,8 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { .static => "lib", .dynamic => "", }; - try argv.append(try allocPrint(arena, "{s}vcruntime.lib", .{lib_str})); - try argv.append(try allocPrint(arena, "{s}ucrt.lib", .{lib_str})); + try argv.append(try arena.print("{s}vcruntime.lib", .{lib_str})); + try argv.append(try arena.print("{s}ucrt.lib", .{lib_str})); //Visual C++ 2015 Conformance Changes //https://msdn.microsoft.com/en-us/library/bb531344.aspx @@ -712,7 +712,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { try argv.ensureUnusedCapacity(comp.windows_libs.count()); for (comp.windows_libs.keys()) |key| { - const lib_basename = try allocPrint(arena, "{s}.lib", .{key}); + const lib_basename = try arena.print("{s}.lib", .{key}); if (comp.crt_files.get(lib_basename)) |crt_file| { argv.appendAssumeCapacity(try crt_file.full_object_path.toString(arena)); continue; @@ -722,7 +722,7 @@ fn coffLink(lld: *Lld, arena: Allocator) !void { continue; } if (target.abi.isGnu()) { - const fallback_name = try allocPrint(arena, "lib{s}.dll.a", .{key}); + const fallback_name = try arena.print("lib{s}.dll.a", .{key}); if (try findLib(arena, io, fallback_name, coff.lib_directories)) |full_path| { argv.appendAssumeCapacity(full_path); continue; @@ -843,19 +843,19 @@ fn elfLink(lld: *Lld, arena: Allocator) !void { try argv.append("--error-limit=0"); if (comp.sysroot) |sysroot| { - try argv.append(try std.fmt.allocPrint(arena, "--sysroot={s}", .{sysroot})); + try argv.append(try arena.print("--sysroot={s}", .{sysroot})); } if (target_util.llvmMachineAbi(target)) |mabi| { try argv.appendSlice(&.{ "-mllvm", - try std.fmt.allocPrint(arena, "-target-abi={s}", .{mabi}), + try arena.print("-target-abi={s}", .{mabi}), }); } try argv.appendSlice(&.{ "-mllvm", - try std.fmt.allocPrint(arena, "-float-abi={s}", .{if (target.abi.float() == .hard) "hard" else "soft"}), + try arena.print("-float-abi={s}", .{if (target.abi.float() == .hard) "hard" else "soft"}), }); switch (target.cpu.arch) { @@ -894,19 +894,19 @@ fn elfLink(lld: *Lld, arena: Allocator) !void { if (output_mode == .Exe) { try argv.appendSlice(&.{ "-z", - try std.fmt.allocPrint(arena, "stack-size={d}", .{base.stack_size}), + try arena.print("stack-size={d}", .{base.stack_size}), }); } switch (base.build_id) { .none => try argv.append("--build-id=none"), - .fast, .uuid, .sha1, .md5 => try argv.append(try std.fmt.allocPrint(arena, "--build-id={s}", .{ + .fast, .uuid, .sha1, .md5 => try argv.append(try arena.print("--build-id={s}", .{ @tagName(base.build_id), })), - .hexstring => |hs| try argv.append(try std.fmt.allocPrint(arena, "--build-id=0x{x}", .{hs.toSlice()})), + .hexstring => |hs| try argv.append(try arena.print("--build-id=0x{x}", .{hs.toSlice()})), } - try argv.append(try std.fmt.allocPrint(arena, "--image-base={d}", .{elf.image_base})); + try argv.append(try arena.print("--image-base={d}", .{elf.image_base})); if (elf.linker_script) |linker_script| { try argv.append("-T"); @@ -914,7 +914,7 @@ fn elfLink(lld: *Lld, arena: Allocator) !void { } if (elf.sort_section) |how| { - const arg = try std.fmt.allocPrint(arena, "--sort-section={s}", .{@tagName(how)}); + const arg = try arena.print("--sort-section={s}", .{@tagName(how)}); try argv.append(arg); } @@ -980,11 +980,11 @@ fn elfLink(lld: *Lld, arena: Allocator) !void { } if (elf.z_common_page_size) |size| { try argv.append("-z"); - try argv.append(try std.fmt.allocPrint(arena, "common-page-size={d}", .{size})); + try argv.append(try arena.print("common-page-size={d}", .{size})); } if (elf.z_max_page_size) |size| { try argv.append("-z"); - try argv.append(try std.fmt.allocPrint(arena, "max-page-size={d}", .{size})); + try argv.append(try arena.print("max-page-size={d}", .{size})); } if (getLDMOption(target)) |ldm| { @@ -1190,7 +1190,7 @@ fn elfLink(lld: *Lld, arena: Allocator) !void { if (target.os.versionRange().gnuLibCVersion().?.order(rem_in) != .lt) continue; } - const lib_path = try std.fmt.allocPrint(arena, "{f}{c}lib{s}.so.{d}", .{ + const lib_path = try arena.print("{f}{c}lib{s}.so.{d}", .{ comp.glibc_so_files.?.dir_path, fs.path.sep, lib.name, lib.sover, }); try argv.append(lib_path); @@ -1207,21 +1207,21 @@ fn elfLink(lld: *Lld, arena: Allocator) !void { if (target.os.version_range.semver.min.order(add_in) == .lt) continue; } - const lib_path = try std.fmt.allocPrint(arena, "{f}{c}lib{s}.so.{d}", .{ + const lib_path = try arena.print("{f}{c}lib{s}.so.{d}", .{ comp.freebsd_so_files.?.dir_path, fs.path.sep, lib.name, lib.getSoVersion(&target.os), }); try argv.append(lib_path); } } else if (target.isNetBSDLibC()) { for (netbsd.libs) |lib| { - const lib_path = try std.fmt.allocPrint(arena, "{f}{c}lib{s}.so.{d}", .{ + const lib_path = try arena.print("{f}{c}lib{s}.so.{d}", .{ comp.netbsd_so_files.?.dir_path, fs.path.sep, lib.name, lib.sover, }); try argv.append(lib_path); } } else if (target.isOpenBSDLibC()) { for (openbsd.libs) |lib| { - const lib_path = try std.fmt.allocPrint(arena, "{f}{c}lib{s}.so", .{ + const lib_path = try arena.print("{f}{c}lib{s}.so", .{ comp.openbsd_so_files.?.dir_path, fs.path.sep, lib.name, }); try argv.append(lib_path); @@ -1451,12 +1451,12 @@ fn wasmLink(lld: *Lld, arena: Allocator) !void { } if (wasm.initial_memory) |initial_memory| { - const arg = try std.fmt.allocPrint(arena, "--initial-memory={d}", .{initial_memory}); + const arg = try arena.print("--initial-memory={d}", .{initial_memory}); try argv.append(arg); } if (wasm.max_memory) |max_memory| { - const arg = try std.fmt.allocPrint(arena, "--max-memory={d}", .{max_memory}); + const arg = try arena.print("--max-memory={d}", .{max_memory}); try argv.append(arg); } @@ -1465,7 +1465,7 @@ fn wasmLink(lld: *Lld, arena: Allocator) !void { } if (wasm.global_base) |global_base| { - const arg = try std.fmt.allocPrint(arena, "--global-base={d}", .{global_base}); + const arg = try arena.print("--global-base={d}", .{global_base}); try argv.append(arg); } else { // We prepend it by default, so when a stack overflow happens the runtime will trap correctly, @@ -1477,7 +1477,7 @@ fn wasmLink(lld: *Lld, arena: Allocator) !void { // Users are allowed to specify which symbols they want to export to the wasm host. for (wasm.export_symbol_names) |symbol_name| { - const arg = try std.fmt.allocPrint(arena, "--export={s}", .{symbol_name}); + const arg = try arena.print("--export={s}", .{symbol_name}); try argv.append(arg); } @@ -1493,15 +1493,15 @@ fn wasmLink(lld: *Lld, arena: Allocator) !void { try argv.appendSlice(&.{ "-z", - try std.fmt.allocPrint(arena, "stack-size={d}", .{base.stack_size}), + try arena.print("stack-size={d}", .{base.stack_size}), }); switch (base.build_id) { .none => try argv.append("--build-id=none"), - .fast, .uuid, .sha1 => try argv.append(try std.fmt.allocPrint(arena, "--build-id={s}", .{ + .fast, .uuid, .sha1 => try argv.append(try arena.print("--build-id={s}", .{ @tagName(base.build_id), })), - .hexstring => |hs| try argv.append(try std.fmt.allocPrint(arena, "--build-id=0x{x}", .{hs.toSlice()})), + .hexstring => |hs| try argv.append(try arena.print("--build-id=0x{x}", .{hs.toSlice()})), .md5 => {}, } @@ -1685,7 +1685,7 @@ fn spawnLld(comp: *Compilation, arena: Allocator, argv: []const []const u8) !voi .argv = &.{ argv[0], argv[1], - try std.fmt.allocPrint(arena, "@{s}", .{ + try arena.print("@{s}", .{ try comp.dirs.local_cache.join(arena, &.{rsp_path}), }), }, @@ -1740,7 +1740,6 @@ const std = @import("std"); const Io = std.Io; const Allocator = std.mem.Allocator; const Cache = std.Build.Cache; -const allocPrint = std.fmt.allocPrint; const assert = std.debug.assert; const fs = std.fs; const log = std.log.scoped(.link); diff --git a/src/main.zig b/src/main.zig index 4dd56dd3b0623c7836ce04604c9066b562a223a4..dbe4c34265ea7cba91200748d35c2cd99df427f8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -25,18 +25,16 @@ const stringToEnum = std.meta.stringToEnum; pub const tracy = @import("tracy.zig"); const Compilation = @import("Compilation.zig"); const link = @import("link.zig"); -const Package = @import("Package.zig"); const build_options = @import("build_options"); -const introspect = @import("introspect.zig"); const wasi_libc = @import("libs/wasi_libc.zig"); const target_util = @import("target.zig"); const crash_report = @import("crash_report.zig"); const Zcu = @import("Zcu.zig"); const mingw = @import("libs/mingw.zig"); const dev = @import("dev.zig"); +const Module = @import("Module.zig"); test { - _ = Package; _ = @import("codegen.zig"); } @@ -353,9 +351,18 @@ fn mainArgs( dev.check(.ar_command); return process.exit(try llvmArMain(arena, args)); }, - .build => { - dev.check(.build_command); - return cmdBuild(gpa, arena, io, cmd_args, environ_map); + .build, .fetch, .init, .libc => { + return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ + .cmd_name = "maker", + .root_src_path = "Maker.zig", + .prepend_cmd = cmd, + .prepend_zig_lib_dir_path = true, + .prepend_global_cache_path = true, + .prepend_zig_exe_path = true, + .prepend_seed = true, + .debug_env_var = .ZIG_DEBUG_MAKER, + .release_mode = .ReleaseSafe, + }); }, .clang, .@"-cc1", .@"-cc1as" => { dev.check(.clang_command); @@ -385,7 +392,6 @@ fn mainArgs( .depend_on_aro = true, .prepend_zig_lib_dir_path = true, .server = use_server, - .color = Color.settingFromEnvironment(environ_map), }); }, .fmt => { @@ -396,25 +402,12 @@ fn mainArgs( return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "objcopy", .root_src_path = "objcopy.zig", - .color = Color.settingFromEnvironment(environ_map), }); }, .objdump => { return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "objdump", .root_src_path = "objdump.zig", - .color = Color.settingFromEnvironment(environ_map), - }); - }, - .fetch => { - return cmdFetch(gpa, arena, io, cmd_args, environ_map); - }, - .libc => { - return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ - .cmd_name = "libc", - .root_src_path = "libc.zig", - .prepend_zig_lib_dir_path = true, - .color = Color.settingFromEnvironment(environ_map), }); }, .std => { @@ -424,12 +417,8 @@ fn mainArgs( .prepend_zig_lib_dir_path = true, .prepend_zig_exe_path = true, .prepend_global_cache_path = true, - .color = Color.settingFromEnvironment(environ_map), }); }, - .init => { - return cmdInit(gpa, arena, io, cmd_args); - }, .targets => { dev.check(.targets_command); const host = std.zig.resolveTargetQueryOrFatal(io, .{}); @@ -461,7 +450,6 @@ fn mainArgs( return jitCmd(gpa, arena, io, cmd_args, environ_map, .{ .cmd_name = "reduce", .root_src_path = "reduce.zig", - .color = Color.settingFromEnvironment(environ_map), }); }, .zen => { @@ -884,13 +872,13 @@ const CliModule = struct { root_path: []const u8, root_src_path: []const u8, cc_argv: []const []const u8, - inherited: Package.Module.CreateOptions.Inherited, + inherited: Module.CreateOptions.Inherited, target_arch_os_abi: ?[]const u8, target_mcpu: ?[]const u8, dynamic_linker: ?[]const u8, deps: []const Dep, - resolved: ?*Package.Module, + resolved: ?*Module, c_source_files_start: usize, c_source_files_end: usize, @@ -1041,7 +1029,7 @@ fn buildOutputType( // These get set by CLI flags and then snapshotted when a `-M` flag is // encountered. - var mod_opts: Package.Module.CreateOptions.Inherited = .{}; + var mod_opts: Module.CreateOptions.Inherited = .{}; // These get appended to by CLI flags and then slurped when a `-M` flag // is encountered. @@ -2977,7 +2965,7 @@ fn buildOutputType( while (preprocessor_args_it.next()) |arg| { if (mem.eql(u8, arg, "-MD") or mem.eql(u8, arg, "-MMD") or mem.eql(u8, arg, "-MT")) { disable_c_depfile = true; - const cc_arg = try std.fmt.allocPrint(arena, "-Wp,{s},{s}", .{ arg, preprocessor_args_it.nextOrFatal() }); + const cc_arg = try arena.print("-Wp,{s},{s}", .{ arg, preprocessor_args_it.nextOrFatal() }); try cc_argv.append(arena, cc_arg); } else { fatal("unsupported preprocessor arg: {s}", .{arg}); @@ -3222,10 +3210,10 @@ fn buildOutputType( else => process.executablePathAlloc(io, arena) catch |err| fatal("unable to find zig self exe path: {t}", .{err}), }; - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); // This `init` calls `fatal` on error. - var dirs: Compilation.Directories = .init( + var dirs: std.zig.Directories = .init( arena, io, override_lib_dir, @@ -3272,7 +3260,7 @@ fn buildOutputType( const root_mod = switch (arg_mode) { .zig_test, .zig_test_obj => root_mod: { const test_mod = if (test_runner_path) |test_runner| test_mod: { - const test_mod = try Package.Module.create(arena, .{ + const test_mod = try Module.create(arena, .{ .paths = .{ .root = try .fromUnresolved(arena, dirs, &.{fs.path.dirname(test_runner) orelse "."}), .root_src_path = fs.path.basename(test_runner), @@ -3285,7 +3273,7 @@ fn buildOutputType( }); test_mod.deps = try main_mod.deps.clone(arena); break :test_mod test_mod; - } else try Package.Module.create(arena, .{ + } else try Module.create(arena, .{ .paths = .{ .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), .root_src_path = "test_runner.zig", @@ -3421,9 +3409,9 @@ fn buildOutputType( .yes_default_value => if (create_module.resolved_options.output_mode == .Lib and create_module.resolved_options.link_mode == .dynamic and target.ofmt == .elf) if (have_version) - try std.fmt.allocPrint(arena, "lib{s}.so.{d}", .{ root_name, version.major }) + try arena.print("lib{s}.so.{d}", .{ root_name, version.major }) else - try std.fmt.allocPrint(arena, "lib{s}.so", .{root_name}) + try arena.print("lib{s}.so", .{root_name}) else null, }; @@ -3433,7 +3421,7 @@ fn buildOutputType( .yes_default_path => emit: { if (output_to_cache != null) break :emit .yes_cache; const name = switch (clang_preprocessor_mode) { - .pch => try std.fmt.allocPrint(arena, "{s}.pch", .{root_name}), + .pch => try arena.print("{s}.pch", .{root_name}), else => try std.zig.binNameAlloc(arena, .{ .root_name = root_name, .cpu_arch = target.cpu.arch, @@ -3469,16 +3457,16 @@ fn buildOutputType( }, }; - const default_h_basename = try std.fmt.allocPrint(arena, "{s}.h", .{root_name}); + const default_h_basename = try arena.print("{s}.h", .{root_name}); const emit_h_resolved = emit_h.resolve(io, default_h_basename, output_to_cache); - const default_asm_basename = try std.fmt.allocPrint(arena, "{s}.s", .{root_name}); + const default_asm_basename = try arena.print("{s}.s", .{root_name}); const emit_asm_resolved = emit_asm.resolve(io, default_asm_basename, output_to_cache); - const default_llvm_ir_basename = try std.fmt.allocPrint(arena, "{s}.ll", .{root_name}); + const default_llvm_ir_basename = try arena.print("{s}.ll", .{root_name}); const emit_llvm_ir_resolved = emit_llvm_ir.resolve(io, default_llvm_ir_basename, output_to_cache); - const default_llvm_bc_basename = try std.fmt.allocPrint(arena, "{s}.bc", .{root_name}); + const default_llvm_bc_basename = try arena.print("{s}.bc", .{root_name}); const emit_llvm_bc_resolved = emit_llvm_bc.resolve(io, default_llvm_bc_basename, output_to_cache); const emit_docs_resolved = emit_docs.resolve(io, "docs", output_to_cache); @@ -3499,7 +3487,7 @@ fn buildOutputType( fatal("the argument -femit-implib is allowed only when building a Windows DLL", .{}); } } - const default_implib_basename = try std.fmt.allocPrint(arena, "{s}.lib", .{root_name}); + const default_implib_basename = try arena.print("{s}.lib", .{root_name}); const emit_implib_resolved: Compilation.CreateOptions.Emit = switch (emit_implib) { .no => .no, .yes => emit_implib.resolve(io, default_implib_basename, output_to_cache), @@ -3528,7 +3516,7 @@ fn buildOutputType( // "-" is stdin. Dump it to a real file. const sep = fs.path.sep_str; - const dump_path = try std.fmt.allocPrint(arena, "tmp" ++ sep ++ "{x}-dump-stdin{s}", .{ + const dump_path = try arena.print("tmp" ++ sep ++ "{x}-dump-stdin{s}", .{ randInt(io, u64), ext.canonicalName(target), }); try dirs.local_cache.handle.createDirPath(io, "tmp"); @@ -3557,7 +3545,7 @@ fn buildOutputType( const bin_digest: Cache.BinDigest = hasher.hasher.finalResult(); - const sub_path = try std.fmt.allocPrint(arena, "tmp" ++ sep ++ "{x}-stdin{s}", .{ + const sub_path = try arena.print("tmp" ++ sep ++ "{x}-stdin{s}", .{ &bin_digest, ext.canonicalName(target), }); try dirs.local_cache.handle.rename(dump_path, dirs.local_cache.handle, sub_path, io); @@ -3889,9 +3877,9 @@ fn buildOutputType( for (mod.cc_argv) |cc_arg| test_exec_args.appendAssumeCapacity(cc_arg); for (mod.deps) |dep| try test_exec_args.appendSlice(arena, &.{ "--dep", - if (std.mem.eql(u8, dep.key, dep.value)) dep.value else try std.fmt.allocPrint(arena, "{s}={s}", .{ dep.key, dep.value }), + if (std.mem.eql(u8, dep.key, dep.value)) dep.value else try arena.print("{s}={s}", .{ dep.key, dep.value }), }); - try test_exec_args.append(arena, try std.fmt.allocPrint(arena, "-M{s}", .{mod_name})); + try test_exec_args.append(arena, try arena.print("-M{s}", .{mod_name})); } try test_exec_args.ensureUnusedCapacity(arena, comp.global_cc_argv.len); @@ -3935,7 +3923,7 @@ fn buildOutputType( } const CreateModule = struct { - dirs: Compilation.Directories, + dirs: std.zig.Directories, modules: std.array_hash_map.String(CliModule), opts: Compilation.Config.Options, object_format: ?[]const u8, @@ -3979,10 +3967,10 @@ fn createModule( io: Io, create_module: *CreateModule, index: usize, - parent: ?*Package.Module, + parent: ?*Module, color: std.zig.Color, environ_map: *process.Environ.Map, -) Allocator.Error!*Package.Module { +) Allocator.Error!*Module { const cli_mod = &create_module.modules.values()[index]; if (cli_mod.resolved) |m| return m; @@ -4258,7 +4246,7 @@ fn createModule( const root: Compilation.Path = try .fromUnresolved(arena, create_module.dirs, &.{cli_mod.root_path}); - const mod = Package.Module.create(arena, .{ + const mod = Module.create(arena, .{ .paths = .{ .root = root, .root_src_path = cli_mod.root_src_path, @@ -4586,7 +4574,7 @@ fn runOrTest( try argv.append(exe_path); if (arg_mode == .zig_test) { try argv.append( - try std.fmt.allocPrint(arena, "--seed=0x{x}", .{randInt(io, u32)}), + try arena.print("--seed=0x{x}", .{randInt(io, u32)}), ); } } else { @@ -4794,7 +4782,7 @@ fn cmdTranslateC( assert(comp.c_source_files.len == 1); const c_source_file = comp.c_source_files[0]; - const translated_basename = try std.fmt.allocPrint(arena, "{s}.zig", .{comp.root_name}); + const translated_basename = try arena.print("{s}.zig", .{comp.root_name}); var man: Cache.Manifest = comp.obtainCObjectCacheManifest(comp.root_mod); man.want_shared_lock = false; @@ -4872,1200 +4860,23 @@ pub fn translateC( .root_src_path = "translate-c/main.zig", .depend_on_aro = true, .capture = capture, - .color = Color.settingFromEnvironment(environ_map), }); } -const usage_init = - \\Usage: zig init - \\ - \\ Initializes a `zig build` project in the current working - \\ directory. - \\ - \\Options: - \\ -m, --minimal Use minimal init template - \\ -h, --help Print this help and exit - \\ - \\ -; - -fn cmdInit(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8) !void { - dev.check(.init_command); - - var template: enum { example, minimal } = .example; - { - var i: usize = 0; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-m") or mem.eql(u8, arg, "--minimal")) { - template = .minimal; - } else if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { - try Io.File.stdout().writeStreamingAll(io, usage_init); - return cleanExit(io); - } else { - fatal("unrecognized parameter: {q}", .{arg}); - } - } else { - fatal("unexpected extra parameter: {q}", .{arg}); - } - } - } - - const cwd_path = try introspect.getResolvedCwd(io, arena); - const cwd_basename = fs.path.basename(cwd_path); - const sanitized_root_name = try sanitizeExampleName(arena, cwd_basename); - - const rng: std.Random.IoSource = .{ .io = io }; - const fingerprint: Package.Fingerprint = .generate(rng.interface(), sanitized_root_name); - - switch (template) { - .example => { - var templates = findTemplates(gpa, arena, io); - defer templates.deinit(io); - - const s = fs.path.sep_str; - const template_paths = [_][]const u8{ - Package.build_zig_basename, - Package.Manifest.basename, - "src" ++ s ++ "main.zig", - "src" ++ s ++ "root.zig", - }; - var ok_count: usize = 0; - - for (template_paths) |template_path| { - if (templates.write(arena, io, Io.Dir.cwd(), sanitized_root_name, template_path, fingerprint)) |_| { - std.log.info("created {s}", .{template_path}); - ok_count += 1; - } else |err| switch (err) { - error.PathAlreadyExists => std.log.info("preserving already existing file: {s}", .{ - template_path, - }), - else => std.log.err("unable to write {s}: {s}\n", .{ template_path, @errorName(err) }), - } - } - - if (ok_count == template_paths.len) { - std.log.info("see `zig build --help` for a menu of options", .{}); - } - return cleanExit(io); - }, - .minimal => { - writeSimpleTemplateFile(io, Package.Manifest.basename, - \\.{{ - \\ .name = .{s}, - \\ .version = "0.0.1", - \\ .minimum_zig_version = "{s}", - \\ .paths = .{{""}}, - \\ .fingerprint = 0x{x}, - \\}} - \\ - , .{ - sanitized_root_name, - build_options.version, - fingerprint.int(), - }) catch |err| switch (err) { - else => fatal("failed to create {q}: {t}", .{ Package.Manifest.basename, err }), - error.PathAlreadyExists => fatal("refusing to overwrite {q}", .{Package.Manifest.basename}), - }; - writeSimpleTemplateFile(io, Package.build_zig_basename, - \\const std = @import("std"); - \\ - \\pub fn build(b: *std.Build) void {{ - \\ _ = b; // stub - \\}} - \\ - , .{}) catch |err| switch (err) { - else => fatal("failed to create {q}: {t}", .{ Package.build_zig_basename, err }), - // `build.zig` already existing is okay: the user has just used `zig init` to set up - // their `build.zig.zon` *after* writing their `build.zig`. So this one isn't fatal. - error.PathAlreadyExists => { - std.log.info("successfully populated {q}, preserving existing {q}", .{ - Package.Manifest.basename, Package.build_zig_basename, - }); - return cleanExit(io); - }, - }; - std.log.info("successfully populated {q} and {q}", .{ Package.Manifest.basename, Package.build_zig_basename }); - return cleanExit(io); - }, - } -} - -fn sanitizeExampleName(arena: Allocator, bytes: []const u8) error{OutOfMemory}![]const u8 { - var result: std.ArrayList(u8) = .empty; - for (bytes, 0..) |byte, i| switch (byte) { - '0'...'9' => { - if (i == 0) try result.append(arena, '_'); - try result.append(arena, byte); - }, - '_', 'a'...'z', 'A'...'Z' => try result.append(arena, byte), - '-', '.', ' ' => try result.append(arena, '_'), - else => continue, - }; - if (!std.zig.isValidId(result.items)) return "foo"; - if (result.items.len > Package.Manifest.max_name_len) - result.shrinkRetainingCapacity(Package.Manifest.max_name_len); - - return result.toOwnedSlice(arena); -} - -test sanitizeExampleName { - var arena_instance = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_instance.deinit(); - const arena = arena_instance.allocator(); - - try std.testing.expectEqualStrings("foo_bar", try sanitizeExampleName(arena, "foo bar+")); - try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "")); - try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "!")); - try std.testing.expectEqualStrings("a", try sanitizeExampleName(arena, "!a")); - try std.testing.expectEqualStrings("a_b", try sanitizeExampleName(arena, "a.b!")); - try std.testing.expectEqualStrings("_01234", try sanitizeExampleName(arena, "01234")); - try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "error")); - try std.testing.expectEqualStrings("foo", try sanitizeExampleName(arena, "test")); - try std.testing.expectEqualStrings("tests", try sanitizeExampleName(arena, "tests")); - try std.testing.expectEqualStrings("test_project", try sanitizeExampleName(arena, "test project")); -} - -fn cmdBuild( - gpa: Allocator, - arena: Allocator, - io: Io, - args: []const []const u8, - environ_map: *process.Environ.Map, -) !void { - var build_file: ?[]const u8 = null; - var override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map); - var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); - var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); - var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); - var maker_optimize_mode: std.builtin.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map)) - .Debug - else - .ReleaseSafe; - var configure_argv: std.ArrayList([]const u8) = .empty; - var make_argv: std.ArrayList([]const u8) = .empty; - var cached_passthru_configure: std.ArrayList(u32) = .empty; - var forks: std.ArrayList(Fork) = .empty; - var reference_trace: ?u32 = null; - var debug_compile_errors = false; - var verbose_link = (native_os != .wasi or builtin.link_libc) and - EnvVar.ZIG_VERBOSE_LINK.isSet(environ_map); - var verbose_cc = (native_os != .wasi or builtin.link_libc) and - EnvVar.ZIG_VERBOSE_CC.isSet(environ_map); - var verbose_air = false; - var verbose_intern_pool = false; - var verbose_generic_instances = false; - var verbose_llvm_ir: ?[]const u8 = null; - var verbose_llvm_bc: ?[]const u8 = null; - var verbose_llvm_cpu_features = false; - var fetch_only = false; - var fetch_mode: Package.Fetch.JobQueue.Mode = .needed; - var system_pkg_dir_path: ?[]const u8 = null; - var debug_target: ?[]const u8 = null; - var debug_libc_paths_file: ?[]const u8 = null; - var cache_poison: std.Build.Graph.CachePoison = .pure; - var print_configuration_path: bool = false; - - const self_exe_path = try process.executablePathAlloc(io, arena); - const default_seed = try std.fmt.allocPrint(arena, "0x{x}", .{randInt(io, u32)}); - - try configure_argv.ensureUnusedCapacity(arena, 16); - try make_argv.ensureUnusedCapacity(arena, 16); - try cached_passthru_configure.ensureUnusedCapacity(arena, 16); - - _ = configure_argv.addOneAssumeCapacity(); // configurer executable - _ = make_argv.addOneAssumeCapacity(); // maker executable - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path }; - configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig", self_exe_path }; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--zig-lib-dir", undefined }; - const make_argv_index_zig_lib_dir = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; - const make_argv_index_build_root = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--local-cache", undefined }; - const make_argv_index_cache_dir = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--global-cache", undefined }; - const make_argv_index_global_cache_dir = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--configuration", undefined }; - const argv_index_configuration_file = make_argv.items.len - 1; - - make_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--seed", default_seed }; - const argv_index_seed = make_argv.items.len - 1; - - configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ "--build-root", undefined }; - const conf_argv_index_build_root = configure_argv.items.len - 1; - - var color: Color = Color.settingFromEnvironment(environ_map); - var n_jobs: ?u32 = null; - - { - var i: usize = 0; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - try configure_argv.ensureUnusedCapacity(arena, 2); - - if (mem.startsWith(u8, arg, "-D") or - mem.startsWith(u8, arg, "-fsys=") or - mem.startsWith(u8, arg, "-fno-sys=") or - mem.startsWith(u8, arg, "--release=") or - mem.eql(u8, arg, "--release")) - { - try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); - configure_argv.appendAssumeCapacity(arg); - continue; - } else if (mem.eql(u8, arg, "--system")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - system_pkg_dir_path = args[i]; - - try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); - configure_argv.appendAssumeCapacity(arg); // Intentionally "--system" only; not the path. - continue; - } else if (mem.cutPrefix(u8, arg, "--color=")) |rest| { - color = stringToEnum(Color, rest) orelse - fatal("expected --color=[auto|on|off]; found {q}", .{arg}); - - try cached_passthru_configure.append(arena, @intCast(configure_argv.items.len)); - configure_argv.appendAssumeCapacity(arg); - continue; - } else if (mem.eql(u8, arg, "--cache-poison")) { - cache_poison = .poisoned; - configure_argv.appendAssumeCapacity("--cache-poison=poisoned"); - continue; - } else if (mem.cutPrefix(u8, arg, "--cache-poison=")) |rest| { - // Allow the configurer process to report parse failure. - if (stringToEnum(std.Build.Graph.CachePoison, rest)) |poison| { - cache_poison = poison; - } - configure_argv.appendAssumeCapacity(arg); - continue; - } else if (mem.eql(u8, arg, "--verbose")) { - // Intentionally is added both to make and configure but - // does not go into the cache hash. - configure_argv.appendAssumeCapacity(arg); - } else if (mem.eql(u8, arg, "--search-prefix")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - // This argument is cache poisonous: it does not go into - // the cache and configurer must set the poison bit when - // choosing to observe it. - configure_argv.addManyAsArrayAssumeCapacity(2).* = .{ arg, args[i] }; - (try make_argv.addManyAsArray(arena, 2)).* = .{ arg, args[i] }; - continue; - } else if (mem.eql(u8, arg, "--build-file")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - build_file = args[i]; - continue; - } else if (mem.eql(u8, arg, "--zig-lib-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_lib_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_local_cache_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--pkg-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_pkg_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--global-cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_global_cache_dir = args[i]; - continue; - } else if (mem.eql(u8, arg, "--print-configuration-path")) { - print_configuration_path = true; - continue; - } else if (mem.eql(u8, arg, "-freference-trace")) { - reference_trace = 256; - } else if (mem.eql(u8, arg, "--fetch")) { - fetch_only = true; - } else if (mem.cutPrefix(u8, arg, "--fetch=")) |sub_arg| { - fetch_only = true; - fetch_mode = stringToEnum(Package.Fetch.JobQueue.Mode, sub_arg) orelse - fatal("expected [needed|all] after \"--fetch=\", found: {s}", .{sub_arg}); - } else if (mem.cutPrefix(u8, arg, "--fork=")) |sub_arg| { - try forks.append(arena, .init(sub_arg)); - continue; - } else if (mem.eql(u8, arg, "--fork")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - try forks.append(arena, .init(args[i])); - continue; - } else if (mem.cutPrefix(u8, arg, "-freference-trace=")) |num| { - reference_trace = std.fmt.parseUnsigned(u32, num, 10) catch |err| { - fatal("unable to parse reference_trace count {q}: {t}", .{ num, err }); - }; - } else if (mem.eql(u8, arg, "-fno-reference-trace")) { - reference_trace = null; - } else if (mem.cutPrefix(u8, arg, "--maker-opt=")) |rest| { - maker_optimize_mode = parseOptimizeMode(rest); - continue; - } else if (mem.eql(u8, arg, "--debug-log")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - try make_argv.appendSlice(arena, args[i .. i + 2]); - i += 1; - try addDebugLog(arena, args[i]); - continue; - } else if (mem.eql(u8, arg, "--debug-compile-errors")) { - if (build_options.enable_debug_extensions) { - debug_compile_errors = true; - } else { - warn("Zig was compiled without debug extensions. --debug-compile-errors has no effect.", .{}); - } - } else if (mem.eql(u8, arg, "--debug-target")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - if (build_options.enable_debug_extensions) { - debug_target = args[i]; - } else { - warn("Zig was compiled without debug extensions. --debug-target has no effect.", .{}); - } - continue; - } else if (mem.eql(u8, arg, "--debug-libc")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - if (build_options.enable_debug_extensions) { - debug_libc_paths_file = args[i]; - } else { - warn("Zig was compiled without debug extensions. --debug-libc has no effect.", .{}); - } - continue; - } else if (mem.eql(u8, arg, "--verbose-link")) { - verbose_link = true; - } else if (mem.eql(u8, arg, "--verbose-cc")) { - verbose_cc = true; - } else if (mem.eql(u8, arg, "--verbose-air")) { - verbose_air = true; - } else if (mem.eql(u8, arg, "--verbose-intern-pool")) { - verbose_intern_pool = true; - } else if (mem.eql(u8, arg, "--verbose-generic-instances")) { - verbose_generic_instances = true; - } else if (mem.eql(u8, arg, "--verbose-llvm-ir")) { - verbose_llvm_ir = "-"; - } else if (mem.cutPrefix(u8, arg, "--verbose-llvm-ir=")) |rest| { - verbose_llvm_ir = rest; - } else if (mem.cutPrefix(u8, arg, "--verbose-llvm-bc=")) |rest| { - verbose_llvm_bc = rest; - } else if (mem.eql(u8, arg, "--verbose-llvm-cpu-features")) { - verbose_llvm_cpu_features = true; - } else if (mem.cutPrefix(u8, arg, "-j")) |str| { - const num = std.fmt.parseUnsigned(u32, str, 10) catch |err| - fatal("unable to parse jobs count {s}: {t}", .{ str, err }); - if (num < 1) { - fatal("number of jobs must be at least 1", .{}); - } - n_jobs = num; - } else if (mem.eql(u8, arg, "--seed")) { - if (i + 1 >= args.len) fatal("expected argument after {q}", .{arg}); - i += 1; - make_argv.items[argv_index_seed] = args[i]; - continue; - } else if (mem.eql(u8, arg, "--")) { - try make_argv.appendSlice(arena, args[i..]); - break; - } - } - try make_argv.append(arena, arg); - } - } - - const root_prog_node = std.Progress.start(io, .{ - .disable_printing = (color == .off), - .root_name = "", - }); - defer root_prog_node.end(); - - process.raiseFileDescriptorLimit(); - - const cwd_path = introspect.getResolvedCwd(io, arena) catch |err| - fatal("failed to get current directory path: {t}", .{err}); - - const build_root = try findBuildRoot(arena, io, .{ - .cwd_path = cwd_path, - .build_file = build_file, - }); - - { - // This `init` calls `fatal` on error. - var dirs: Compilation.Directories = .init( - arena, - io, - override_lib_dir, - override_global_cache_dir, - .{ .override = path: { - if (override_local_cache_dir) |d| break :path d; - break :path try build_root.directory.join(arena, &.{introspect.default_local_zig_cache_basename}); - } }, - .empty, - self_exe_path, - environ_map, - cwd_path, - ); - defer dirs.deinit(io); - - const thread_limit = @min( - @max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1), - std.math.maxInt(Zcu.PerThread.IdBacking), - ); - try setThreadLimit(arena, thread_limit); - - // Cache lookup for configure options. If we get a match, we can skip - // execution of the configure script. If not, we get the file path to pass - // to the configure process. - var local_cache: Cache = .{ - .gpa = gpa, - .io = io, - .manifest_dir = try dirs.local_cache.handle.createDirPathOpen(io, "h", .{}), - .cwd = cwd_path, - }; - local_cache.addPrefix(.{ .path = null, .handle = Io.Dir.cwd() }); - local_cache.addPrefix(dirs.zig_lib); - local_cache.addPrefix(dirs.local_cache); - local_cache.addPrefix(dirs.global_cache); - defer local_cache.manifest_dir.close(io); - - var config_man = local_cache.obtain(); - defer config_man.deinit(); - config_man.hash.addBytes(build_options.version); - - for (cached_passthru_configure.items) |i| - config_man.hash.addBytes(configure_argv.items[i]); - - // Prevents a `zig build` from getting a false positive cache hit following - // a `zig build --cache-poison=ignored`. - config_man.hash.add(cache_poison == .ignored); - - // Normally the build runner is compiled for the host target but here is - // some code to help when debugging edits to the build runner so that you - // can make sure it compiles successfully on other targets. - const resolved_target: Package.Module.ResolvedTarget = t: { - if (build_options.enable_debug_extensions) { - if (debug_target) |triple| { - const target_query = try std.Target.Query.parse(.{ - .arch_os_abi = triple, - }); - config_man.hash.addBytes(triple); - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(io, target_query), - .is_native_os = false, - .is_native_abi = false, - .is_explicit_dynamic_linker = false, - }; - } - } - break :t .{ - .result = std.zig.resolveTargetQueryOrFatal(io, .{}), - .is_native_os = true, - .is_native_abi = true, - .is_explicit_dynamic_linker = false, - }; - }; - - // Likewise, `--debug-libc` allows overriding the libc installation. - const libc_installation: ?*const LibCInstallation = lci: { - const paths_file = debug_libc_paths_file orelse break :lci null; - if (!build_options.enable_debug_extensions) unreachable; - const lci = try arena.create(LibCInstallation); - lci.* = try .parse(arena, io, paths_file, &resolved_target.result); - LibCInstallation.addToHash(lci, &config_man.hash, resolved_target.result.abi); - break :lci lci; - }; - - // Kick off an optimized compilation of the make runner. - var make_runner_task = if (print_configuration_path) undefined else io.async(compileMakeRunner, .{ gpa, arena, io, .{ - .dirs = .{ - .cwd = dirs.cwd, - .zig_lib = dirs.zig_lib, - .global_cache = dirs.global_cache, - .local_cache = dirs.global_cache, - }, - .environ_map = environ_map, - .parent_prog_node = root_prog_node, - .resolved_target = resolved_target, - .libc_installation = libc_installation, - .thread_limit = thread_limit, - .self_exe_path = self_exe_path, - .color = color, - .reference_trace = reference_trace, - .optimize_mode = maker_optimize_mode, - } }); - defer _ = if (!print_configuration_path) make_runner_task.cancel(io) catch {}; - - const pkg_root: Path = if (override_pkg_dir) |p| - .initCwd(p) - else if (system_pkg_dir_path) |p| - .initCwd(p) - else - .{ - .root_dir = build_root.directory, - .sub_path = "zig-pkg", - }; - - make_argv.items[make_argv_index_zig_lib_dir] = dirs.zig_lib.path orelse cwd_path; - make_argv.items[make_argv_index_build_root] = build_root.directory.path orelse cwd_path; - make_argv.items[make_argv_index_global_cache_dir] = dirs.global_cache.path orelse cwd_path; - make_argv.items[make_argv_index_cache_dir] = dirs.local_cache.path orelse cwd_path; - - configure_argv.items[conf_argv_index_build_root] = build_root.directory.path orelse cwd_path; - - // Dummy http client that is not actually used when fetch_command is unsupported. - // Prevents bootstrap from depending on a bunch of unnecessary stuff. - var http_client: if (dev.env.supports(.fetch_command)) std.http.Client else struct { - allocator: Allocator, - io: Io, - fn deinit(_: @This()) void {} - } = .{ .allocator = gpa, .io = io }; - defer http_client.deinit(); - - var unlazy_set: Package.Fetch.JobQueue.UnlazySet = .{}; - var fork_set: Package.Fetch.JobQueue.ForkSet = .{}; - - { - // Populate fork_set. - var group: Io.Group = .init; - defer group.cancel(io); - - for (forks.items) |*fork| - group.async(io, Fork.load, .{ io, gpa, fork, color }); - - try group.await(io); - - for (forks.items) |*fork| { - if (fork.failed) process.exit(1); - try fork_set.put(arena, .{ - .path = fork.path, - .manifest_ast = fork.manifest_ast, - .manifest = fork.manifest, - .uses = 0, - }, {}); - } - } - defer Fork.deinitList(forks.items); - - // This loop is re-evaluated when the build script exits with an indication that it - // could not continue due to missing lazy dependencies. - const configuration_path: Path, const poisoned: bool = cp: while (true) { - // We want to release all the locks before executing the child process, so we make a nice - // big block here to ensure the cleanup gets run when we extract out our argv. - { - const main_mod_paths: Package.Module.CreateOptions.Paths = .{ - .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), - .root_src_path = "configurer.zig", - }; - - const config = try Compilation.Config.resolve(.{ - .output_mode = .Exe, - .resolved_target = resolved_target, - .have_zcu = true, - .emit_bin = true, - .is_test = false, - }); - - const root_mod = try Package.Module.create(arena, .{ - .paths = main_mod_paths, - .fully_qualified_name = "root", - .cc_argv = &.{}, - .inherited = .{ - .resolved_target = resolved_target, - .single_threaded = true, - }, - .global = config, - .parent = null, - }); - - const build_mod = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromUnresolved(arena, dirs, &.{build_root.directory.path orelse "."}), - .root_src_path = build_root.build_zig_basename, - }, - .fully_qualified_name = "root.@build", - .cc_argv = &.{}, - .inherited = .{}, - .global = config, - .parent = root_mod, - }); - - if (dev.env.supports(.fetch_command)) { - const fetch_prog_node = root_prog_node.start("Fetch Packages", 0); - defer fetch_prog_node.end(); - - // Reset fork match counts. - for (fork_set.keys()) |*fork| fork.uses = 0; - - var job_queue: Package.Fetch.JobQueue = .{ - .io = io, - .http_client = &http_client, - .global_cache = dirs.global_cache, - .local_storage = &.{ - .cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" }, - .pkg_root = pkg_root, - }, - .recursive = true, - .debug_hash = false, - .unlazy_set = unlazy_set, - .fork_set = fork_set, - .mode = fetch_mode, - .prog_node = fetch_prog_node, - .read_only = system_pkg_dir_path != null, - }; - defer job_queue.deinit(); - - if (system_pkg_dir_path == null) { - try http_client.initDefaultProxies(arena, environ_map); - } - - try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1); - try job_queue.table.ensureUnusedCapacity(gpa, 1); - - const phantom_package_root: Cache.Path = .{ .root_dir = build_root.directory }; - - var fetch: Package.Fetch = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .relative_path = phantom_package_root }, - .location_tok = 0, - .hash_tok = .none, - .name_tok = 0, - .lazy_status = .eager, - .remote_package_root = phantom_package_root, - .parent_package_root = phantom_package_root, - .parent_manifest_ast = null, - .prog_node = fetch_prog_node, - .job_queue = &job_queue, - .omit_missing_hash_error = true, - .allow_missing_paths_field = false, - .use_latest_commit = false, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = undefined, - .manifest_ast = undefined, - .have_manifest = false, - .computed_hash = undefined, - .has_build_zig = true, - .oom_flag = false, - .latest_commit = null, - - .module = build_mod, - }; - - job_queue.all_fetches.appendAssumeCapacity(&fetch); - - job_queue.table.putAssumeCapacityNoClobber( - Package.Fetch.relativePathDigest(phantom_package_root, dirs.global_cache), - &fetch, - ); - - job_queue.group.async(io, Package.Fetch.workerRun, .{ &fetch, "root" }); - try job_queue.group.await(io); - - { - // Ensure that forks were actually used. This is done - // before printing manifest errors because using a fork can - // prevent them. - var any_unused = false; - for (fork_set.keys()) |*fork| { - if (fork.uses == 0) { - std.log.err("fork {f} matched no {s} packages", .{ - fork.path, fork.manifest.name, - }); - any_unused = true; - } else { - std.log.info("fork {f} matched {d} {s} packages", .{ - fork.path, fork.uses, fork.manifest.name, - }); - } - } - if (any_unused) process.exit(1); - } - - try job_queue.consolidateErrors(); - - if (fetch.error_bundle.root_list.items.len > 0) { - var errors = try fetch.error_bundle.toOwnedBundle(""); - errors.renderToStderr(io, .{}, color) catch {}; - process.exit(1); - } - - if (fetch_only) return cleanExit(io); - - var source_buf = std.array_list.Managed(u8).init(gpa); - defer source_buf.deinit(); - try job_queue.createDependenciesSource(&source_buf); - const deps_mod = try createDependenciesModule( - arena, - io, - source_buf.items, - root_mod, - dirs, - config, - ); - - { - // We need a Module for each package's build.zig. - const hashes = job_queue.table.keys(); - const fetches = job_queue.table.values(); - try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); - for (hashes, fetches) |*hash, f| { - if (f == &fetch) { - // The first one is a dummy package for the current project. - continue; - } - if (!f.has_build_zig) - continue; - const hash_slice = hash.toSlice(); - const mod_root_path = try f.package_root.toString(arena); - const m = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromUnresolved(arena, dirs, &.{mod_root_path}), - .root_src_path = Package.build_zig_basename, - }, - .fully_qualified_name = try std.fmt.allocPrint( - arena, - "root.@dependencies.{s}", - .{hash_slice}, - ), - .cc_argv = &.{}, - .inherited = .{}, - .global = config, - .parent = root_mod, - }); - const hash_cloned = try arena.dupe(u8, hash_slice); - deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); - f.module = m; - } - - // Each build.zig module needs access to each of its - // dependencies' build.zig modules by name. - for (fetches) |f| { - const mod = f.module orelse continue; - if (!f.have_manifest) continue; - const man = &f.manifest; - const dep_names = man.dependencies.keys(); - try mod.deps.ensureUnusedCapacity(arena, @intCast(dep_names.len)); - for (dep_names, man.dependencies.values()) |name, dep| { - const dep_digest = Package.Fetch.depDigest( - f.package_root, - dirs.global_cache, - dep, - ) orelse continue; - const dep_mod = job_queue.table.get(dep_digest).?.module orelse continue; - const name_cloned = try arena.dupe(u8, name); - mod.deps.putAssumeCapacityNoClobber(name_cloned, dep_mod); - } - } - } - } else try createEmptyDependenciesModule( - arena, - io, - root_mod, - dirs, - config, - ); - - const compile_prog_node = root_prog_node.start("Compile Configure Script", 0); - defer compile_prog_node.end(); - - try root_mod.deps.put(arena, "@build", build_mod); - - var create_diag: Compilation.CreateDiagnostic = undefined; - const comp = Compilation.create(gpa, arena, io, &create_diag, .{ - .libc_installation = libc_installation, - .dirs = dirs, - .root_name = "configure", - .config = config, - .root_mod = root_mod, - .main_mod = build_mod, - .emit_bin = .yes_cache, - .self_exe_path = self_exe_path, - .thread_limit = thread_limit, - .verbose_cc = verbose_cc, - .verbose_link = verbose_link, - .verbose_air = verbose_air, - .verbose_intern_pool = verbose_intern_pool, - .verbose_generic_instances = verbose_generic_instances, - .verbose_llvm_ir = verbose_llvm_ir, - .verbose_llvm_bc = verbose_llvm_bc, - .verbose_llvm_cpu_features = verbose_llvm_cpu_features, - .cache_mode = .whole, - .reference_trace = reference_trace, - .debug_compile_errors = debug_compile_errors, - .environ_map = environ_map, - }) catch |err| switch (err) { - error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - else => |e| fatal("failed to create compilation: {t}", .{e}), - }; - defer comp.destroy(); - - updateModule(comp, color, compile_prog_node) catch |err| switch (err) { - error.CompileErrorsReported => process.exit(2), - else => |e| return e, - }; - - // Since incremental compilation isn't done yet, we use cache_mode = whole - // above, and thus the output file is already closed. - //try comp.makeBinFileExecutable(); - const hex_digest: []const u8 = &Cache.binToHex(comp.digest.?); - const exe_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ hex_digest, comp.emit_bin.? }), - }; - _ = try config_man.addFilePath(exe_path, null); - configure_argv.items[0] = try exe_path.toString(arena); - - switch (cache_poison) { - .pure, .disallowed, .ignored => if (try config_man.hit()) { - const digest = config_man.final(); - break :cp .{ - .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}), - }, - false, - }; - }, - .poisoned => {}, // Don't bother checking for cache hit. - } - } - - if (!process.can_spawn) { - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ native_os, cmd }); - } - - const rand_int = randInt(io, u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); - const config_tmp_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = tmp_dir_sub_path, - }; - const config_tmp_file: Io.File = try config_tmp_path.root_dir.handle.createFile( - io, - config_tmp_path.sub_path, - .{ .read = true, .exclusive = true }, - ); - defer config_tmp_file.close(io); - - const term = term: { - const child_node = root_prog_node.start("Run Configure Script", 0); - defer child_node.end(); - var child = std.process.spawn(io, .{ - .argv = configure_argv.items, - .stdout = .{ .file = config_tmp_file }, - .progress_node = child_node, - }) catch |err| fatal("failed to spawn configure script {s}: {t}", .{ configure_argv.items[0], err }); - defer child.kill(io); - break :term child.wait(io) catch |err| - fatal("failed to wait configure script {s}: {t}", .{ configure_argv.items[0], err }); - }; - if (!term.success()) { - // Failure to produce the configuration file. - const cmd = try std.mem.join(arena, " ", configure_argv.items); - fatal("the following configure command {f}:\n{s}", .{ term, cmd }); - } - // Even though the file is designed to be sent directly to make - // runner, we must load it now because: - // * If it contains additional file dependencies, we need to - // add them to `config_man` before obtaining the final digest. - // * If it contains a set of lazy packages that need to be - // fetched, we need to fetch those now and re-run configure. - var configuration = std.Build.Configuration.loadFile(arena, io, config_tmp_file) catch |err| - fatal("failed to load configuration file {f}: {t}", .{ config_tmp_path, err }); - - if (configuration.unlazy_deps.len != 0) { - if (!dev.env.supports(.fetch_command)) process.exit(1); - var any_errors = false; - for (configuration.unlazy_deps) |hash_string| { - const hash = hash_string.slice(&configuration); - assert(hash.len != 0); - if (hash.len > Package.Hash.max_len) { - std.log.err("invalid digest (length {d} exceeds maximum): {q}", .{ hash.len, hash }); - any_errors = true; - continue; - } - try unlazy_set.put(arena, .fromSlice(hash), {}); - } - if (any_errors) process.exit(1); - if (system_pkg_dir_path) |p| { - // In this mode, the system needs to provide these packages; they - // cannot be fetched by Zig. - const s = fs.path.sep_str; - for (unlazy_set.keys()) |*hash| { - std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ p, hash.toSlice() }); - } - std.log.info("remote package fetching disabled due to --system mode", .{}); - std.log.info("dependencies might be avoidable depending on build configuration", .{}); - process.exit(1); - } - continue :cp; - } - - for (configuration.path_deps_base, configuration.path_deps_sub) |base, sub| { - const conf_path: std.Build.Configuration.Path = .{ .base = base, .sub = sub }; - try config_man.addPathPost(conf_path.toCachePath(&configuration, arena)); - } - - // If it is poisoned, there is no point in moving it to cached - // location. Just leave it in the tmp directory. - if (configuration.poisoned) { - break :cp .{ config_tmp_path, true }; - } else { - const digest = config_man.final(); - const final_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try std.fmt.allocPrint(arena, "c/{s}", .{&digest}), - }; - Io.Dir.rename( - config_tmp_path.root_dir.handle, - config_tmp_path.sub_path, - final_path.root_dir.handle, - final_path.sub_path, - io, - ) catch |err| retry: { - const e = switch (err) { - error.FileNotFound => e: { - const dir_path = final_path.dirname().?; - dir_path.root_dir.handle.createDirPath(io, dir_path.sub_path) catch |e| - fatal("failed to create directory {f}: {t}", .{ dir_path, e }); - if (Io.Dir.rename( - config_tmp_path.root_dir.handle, - config_tmp_path.sub_path, - final_path.root_dir.handle, - final_path.sub_path, - io, - )) |_| break :retry else |e| break :e e; - }, - else => |e| e, - }; - fatal("failed to rename configuration file from {f} into {f}: {t}", .{ - config_tmp_path, final_path, e, - }); - }; - config_man.writeManifest() catch |err| warn("failed to write cache manifest: {t}", .{err}); - break :cp .{ final_path, false }; - } - }; - - { - // Release all file system locks just before running the maker process. - var configuration_lock = if (!poisoned) config_man.toOwnedLock() else null; - defer if (configuration_lock) |*l| l.release(io); - - if (print_configuration_path) { - var stdout_writer = Io.File.stdout().writerStreaming(io, &stdout_buffer); - stdout_writer.interface.print("{f}\n", .{configuration_path}) catch - fatal("failed printing cache file path: {t}", .{stdout_writer.err.?}); - stdout_writer.flush() catch |err| - fatal("failed printing cache file path: {t}", .{err}); - return cleanExit(io); - } - const make_runner = make_runner_task.await(io) catch |err| fatal("failed compiling maker: {t}", .{err}); - - make_argv.items[0] = try make_runner.exe_path.toString(arena); - make_argv.items[argv_index_configuration_file] = try configuration_path.toString(arena); - } - } - - if (!process.can_spawn) { - const cmd = try std.mem.join(arena, " ", make_argv.items); - fatal("the following command cannot be executed ({t} does not support spawning a child process):\n{s}", .{ - native_os, cmd, - }); - } - - const term = term: { - _ = try io.lockStderr(&.{}, .no_color); - defer io.unlockStderr(); - var child = std.process.spawn(io, .{ - .argv = make_argv.items, - }) catch |err| fatal("failed spawning maker {s}: {t}", .{ make_argv.items[0], err }); - defer child.kill(io); - break :term child.wait(io) catch |err| - fatal("failed waiting on maker {s}: {t}", .{ make_argv.items[0], err }); - }; - if (term.success()) return cleanExit(io); - const cmd = try std.mem.join(arena, " ", make_argv.items); - fatal("the following maker command {f}:\n{s}", .{ term, cmd }); -} - -const MakeRunner = struct { - exe_path: Path, - - const Options = struct { - environ_map: *const process.Environ.Map, - dirs: Compilation.Directories, - parent_prog_node: std.Progress.Node, - resolved_target: Package.Module.ResolvedTarget, - libc_installation: ?*const LibCInstallation, - self_exe_path: []const u8, - thread_limit: usize, - color: Color, - reference_trace: ?u32, - optimize_mode: std.builtin.OptimizeMode, - }; -}; - -fn compileMakeRunner(gpa: Allocator, arena: Allocator, io: Io, options: MakeRunner.Options) !MakeRunner { - const compile_prog_node = options.parent_prog_node.start("Compiling Maker (first time setup)", 0); - defer compile_prog_node.end(); - - const strip = options.optimize_mode != .Debug; - - const main_mod_paths: Package.Module.CreateOptions.Paths = .{ - .root = try .fromRoot(arena, options.dirs, .zig_lib, "compiler"), - .root_src_path = "Maker.zig", - }; - - const config = try Compilation.Config.resolve(.{ - .output_mode = .Exe, - .root_strip = strip, - .root_optimize_mode = options.optimize_mode, - .resolved_target = options.resolved_target, - .have_zcu = true, - .emit_bin = true, - .is_test = false, - }); - - const root_mod = try Package.Module.create(arena, .{ - .paths = main_mod_paths, - .fully_qualified_name = "root", - .cc_argv = &.{}, - .inherited = .{ - .resolved_target = options.resolved_target, - .optimize_mode = options.optimize_mode, - .strip = strip, - }, - .global = config, - .parent = null, - }); - - var create_diag: Compilation.CreateDiagnostic = undefined; - const comp = Compilation.create(gpa, arena, io, &create_diag, .{ - .dirs = options.dirs, - .root_name = "maker", - .config = config, - .root_mod = root_mod, - .main_mod = root_mod, - .emit_bin = .yes_cache, - .self_exe_path = options.self_exe_path, - .thread_limit = options.thread_limit, - .cache_mode = .whole, - .environ_map = options.environ_map, - .reference_trace = options.reference_trace, - }) catch |err| switch (err) { - error.CreateFail => fatal("failed to create compilation: {f}", .{create_diag}), - error.Canceled => |e| return e, - else => |e| fatal("failed to create compilation: {t}", .{e}), - }; - defer comp.destroy(); - - try updateModule(comp, options.color, compile_prog_node); - - const exe_path: Path = .{ - .root_dir = options.dirs.global_cache, - .sub_path = try std.fmt.allocPrint(arena, "o/{s}/{s}", .{ - &Cache.binToHex(comp.digest.?), comp.emit_bin.?, - }), - }; - - return .{ - .exe_path = exe_path, - }; -} - -const Fork = struct { - path: Path, - manifest_ast: std.zig.Ast, - manifest: Package.Manifest, - error_bundle: std.zig.ErrorBundle.Wip, - failed: bool, - arena_allocator: std.heap.ArenaAllocator, - - fn init(cwd_relative_path: []const u8) Fork { - return .{ - .manifest_ast = undefined, - .manifest = undefined, - .error_bundle = undefined, - .arena_allocator = undefined, - .path = .{ - .root_dir = .cwd(), - .sub_path = cwd_relative_path, - }, - .failed = false, - }; - } - - fn load(io: Io, gpa: Allocator, fork: *Fork, color: Color) Io.Cancelable!void { - loadFallible(io, gpa, fork, color) catch |err| switch (err) { - error.Canceled => |e| return e, - error.AlreadyReported => fork.failed = true, - else => |e| { - std.log.err("failed to load fork at {f}: {t}", .{ fork.path, e }); - fork.failed = true; - }, - }; - } - - fn loadFallible(io: Io, gpa: Allocator, fork: *Fork, color: Color) !void { - fork.arena_allocator = .init(gpa); - const arena = fork.arena_allocator.allocator(); - - var error_bundle: std.zig.ErrorBundle.Wip = undefined; - try error_bundle.init(gpa); - defer error_bundle.deinit(); - - const manifest_path = try fork.path.join(arena, Package.Manifest.basename); - - Package.Manifest.load( - io, - arena, - manifest_path, - &fork.manifest_ast, - &error_bundle, - &fork.manifest, - true, - ) catch |err| switch (err) { - error.Canceled => |e| return e, - error.ErrorsBundled => { - assert(error_bundle.root_list.items.len > 0); - var errors = try error_bundle.toOwnedBundle(""); - errors.renderToStderr(io, .{}, color) catch {}; - return error.AlreadyReported; - }, - else => |e| { - std.log.err("failed to load package manifest {f}: {t}", .{ manifest_path, e }); - return error.AlreadyReported; - }, - }; - } - - fn deinitList(forks: []Fork) void { - for (forks) |*fork| fork.arena_allocator.deinit(); - } -}; - const JitCmdOptions = struct { cmd_name: []const u8, root_src_path: []const u8, + prepend_cmd: ?[]const u8 = null, prepend_zig_lib_dir_path: bool = false, prepend_global_cache_path: bool = false, prepend_zig_exe_path: bool = false, + prepend_seed: bool = false, depend_on_aro: bool = false, capture: ?*[]u8 = null, /// Send error bundles via std.zig.Server over stdout server: bool = false, - color: Color = .auto, + debug_env_var: EnvVar = .ZIG_DEBUG_CMD, + release_mode: std.lang.OptimizeMode = .ReleaseFast, }; fn jitCmd( @@ -6078,8 +4889,11 @@ fn jitCmd( ) !void { dev.check(.jit_command); + const color = Color.settingFromEnvironment(environ_map); + const root_prog_node = std.Progress.start(io, .{ - .disable_printing = (options.color == .off), + .disable_printing = (color == .off), + .root_name = try arena.print("Compiling {s} (first time setup)", .{options.cmd_name}), }); defer root_prog_node.end(); @@ -6103,7 +4917,7 @@ fn jitCmdInner( options: JitCmdOptions, ) !void { const target_query: std.Target.Query = .{}; - const resolved_target: Package.Module.ResolvedTarget = .{ + const resolved_target: Module.ResolvedTarget = .{ .result = std.zig.resolveTargetQueryOrFatal(io, target_query), .is_native_os = true, .is_native_abi = true, @@ -6113,18 +4927,18 @@ fn jitCmdInner( const self_exe_path = process.executablePathAlloc(io, arena) catch |err| fatal("unable to find self exe path: {t}", .{err}); - const optimize_mode: std.lang.OptimizeMode = if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map)) + const optimize_mode: std.lang.OptimizeMode = if (options.debug_env_var.isSet(environ_map)) .Debug else - .ReleaseFast; + options.release_mode; const strip = optimize_mode != .Debug; const override_lib_dir: ?[]const u8 = EnvVar.ZIG_LIB_DIR.get(environ_map); const override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); // This `init` calls `fatal` on error. - var dirs: Compilation.Directories = .init( + var dirs: std.zig.Directories = .init( arena, io, override_lib_dir, @@ -6138,12 +4952,12 @@ fn jitCmdInner( defer dirs.deinit(io); var child_argv: std.ArrayList([]const u8) = .empty; - try child_argv.ensureUnusedCapacity(arena, args.len + 4); + try child_argv.ensureUnusedCapacity(arena, args.len + 6); // We want to release all the locks before executing the child process, so we make a nice // big block here to ensure the cleanup gets run when we extract out our argv. { - const main_mod_paths: Package.Module.CreateOptions.Paths = .{ + const main_mod_paths: Module.CreateOptions.Paths = .{ .root = try .fromRoot(arena, dirs, .zig_lib, "compiler"), .root_src_path = options.root_src_path, }; @@ -6158,7 +4972,7 @@ fn jitCmdInner( .is_test = false, }); - const root_mod = try Package.Module.create(arena, .{ + const root_mod = try Module.create(arena, .{ .paths = main_mod_paths, .fully_qualified_name = "root", .cc_argv = &.{}, @@ -6172,7 +4986,7 @@ fn jitCmdInner( }); if (options.depend_on_aro) { - const aro_mod = try Package.Module.create(arena, .{ + const aro_mod = try Module.create(arena, .{ .paths = .{ .root = try .fromRoot(arena, dirs, .zig_lib, "compiler/aro"), .root_src_path = "aro.zig", @@ -6224,7 +5038,8 @@ fn jitCmdInner( process.exit(2); } } else { - updateModule(comp, options.color, root_prog_node) catch |err| switch (err) { + const color = Color.settingFromEnvironment(environ_map); + updateModule(comp, color, root_prog_node) catch |err| switch (err) { error.CompileErrorsReported => process.exit(2), else => |e| return e, }; @@ -6238,23 +5053,30 @@ fn jitCmdInner( child_argv.appendAssumeCapacity(exe_path); } + if (options.prepend_cmd) |cmd| + child_argv.appendAssumeCapacity(cmd); if (options.prepend_zig_lib_dir_path) - child_argv.appendAssumeCapacity(dirs.zig_lib.path.?); + child_argv.appendAssumeCapacity(try arena.print("--zig-lib={s}", .{dirs.zig_lib.path.?})); if (options.prepend_zig_exe_path) - child_argv.appendAssumeCapacity(self_exe_path); + child_argv.appendAssumeCapacity(try arena.print("--zig={s}", .{self_exe_path})); if (options.prepend_global_cache_path) - child_argv.appendAssumeCapacity(dirs.global_cache.path.?); + child_argv.appendAssumeCapacity(try arena.print("--global-cache={s}", .{dirs.global_cache.path.?})); + if (options.prepend_seed) + child_argv.appendAssumeCapacity(try arena.print("--seed=0x{x}", .{randInt(io, u32)})); child_argv.appendSliceAssumeCapacity(args); + if (EnvVar.ZIG_VERBOSE_CMD.isSet(environ_map)) { + const cmd: std.zig.SubprocessCommand = .{ + .argv = child_argv.items, + }; + std.log.info("{f}", .{cmd}); + } + if (process.can_replace and options.capture == null) { - if (EnvVar.ZIG_DEBUG_CMD.isSet(environ_map)) { - const cmd = try std.mem.join(arena, " ", child_argv.items); - std.debug.print("{s}\n", .{cmd}); - } const err = process.replace(io, .{ .argv = child_argv.items, .environ_map = environ_map }); const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following command failed to execve with '{t}':\n{s}", .{ err, cmd }); + fatal("the following command failed to execve with {t}:\n{s}", .{ err, cmd }); } if (!process.can_spawn) { @@ -6264,7 +5086,7 @@ fn jitCmdInner( }); } - switch (t: { + const term = t: { _ = try io.lockStderr(&.{}, .no_color); defer io.unlockStderr(); @@ -6282,28 +5104,13 @@ fn jitCmdInner( } break :t try child.wait(io); - }) { - .exited => |code| { - if (code == 0) { - if (options.capture != null) return; - return cleanExit(io); - } - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd }); - }, - .signal => |sig| { - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following build command terminated with signal {t}:\n{s}", .{ sig, cmd }); - }, - .stopped => |sig| { - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following build command stopped with signal {t}:\n{s}", .{ sig, cmd }); - }, - .unknown => { - const cmd = try std.mem.join(arena, " ", child_argv.items); - fatal("the following build command crashed:\n{s}", .{cmd}); - }, + }; + if (term.success()) { + if (options.capture != null) return; + return cleanExit(io); } + const cmd = try std.mem.join(arena, " ", child_argv.items); + fatal("the following build command {f}:\n{s}", .{ term, cmd }); } const info_zen = @@ -7193,667 +6000,6 @@ fn parseRcIncludes(arg: []const u8) std.zig.RcIncludes { fatal("unsupported rc includes type: {q}", .{arg}); } -const usage_fetch = - \\Usage: zig fetch [options] - \\Usage: zig fetch [options] - \\ - \\ Copy a package into the global cache and print its hash. - \\ must point to one of the following: - \\ - A git+http / git+https server for the package - \\ - A tarball file (with or without compression) containing - \\ package source - \\ - A git bundle file containing package source - \\ - \\Examples: - \\ - \\ zig fetch --save git+https://example.com/andrewrk/fun-example-tool.git - \\ zig fetch --save https://example.com/andrewrk/fun-example-tool/archive/refs/heads/master.tar.gz - \\ - \\Options: - \\ -h, --help Print this help and exit - \\ --global-cache-dir [path] Override path to global Zig cache directory - \\ --cache-dir [path] Override path to local cache directory - \\ --pkg-dir [path] Override path to local package directory - \\ --debug-hash Print verbose hash information to stdout - \\ --debug-log [scope] Enable printing debug/info log messages for scope - \\ --save Add the fetched package to build.zig.zon - \\ --save=[name] Add the fetched package to build.zig.zon as name - \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim - \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim - \\ -; - -fn cmdFetch( - gpa: Allocator, - arena: Allocator, - io: Io, - args: []const []const u8, - environ_map: *process.Environ.Map, -) !void { - dev.check(.fetch_command); - - const color: Color = Color.settingFromEnvironment(environ_map); - var opt_path_or_url: ?[]const u8 = null; - var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map); - var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map); - var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map); - var debug_hash: bool = false; - var save: union(enum) { - no, - yes: ?[]const u8, - exact: ?[]const u8, - } = .no; - - { - var i: usize = 0; - while (i < args.len) : (i += 1) { - const arg = args[i]; - if (mem.startsWith(u8, arg, "-")) { - if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) { - try Io.File.stdout().writeStreamingAll(io, usage_fetch); - return cleanExit(io); - } else if (mem.eql(u8, arg, "--global-cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_global_cache_dir = args[i]; - } else if (mem.eql(u8, arg, "--cache-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_local_cache_dir = args[i]; - } else if (mem.eql(u8, arg, "--pkg-dir")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - override_pkg_dir = args[i]; - } else if (mem.eql(u8, arg, "--debug-hash")) { - debug_hash = true; - } else if (mem.eql(u8, arg, "--debug-log")) { - if (i + 1 >= args.len) fatal("expected argument after: {s}", .{arg}); - i += 1; - try addDebugLog(arena, args[i]); - } else if (mem.eql(u8, arg, "--save")) { - save = .{ .yes = null }; - } else if (mem.cutPrefix(u8, arg, "--save=")) |rest| { - save = .{ .yes = rest }; - } else if (mem.eql(u8, arg, "--save-exact")) { - save = .{ .exact = null }; - } else if (mem.cutPrefix(u8, arg, "--save-exact=")) |rest| { - save = .{ .exact = rest }; - } else { - fatal("unrecognized parameter: {q}", .{arg}); - } - } else if (opt_path_or_url != null) { - fatal("unexpected extra parameter: {q}", .{arg}); - } else { - opt_path_or_url = arg; - } - } - } - - const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); - - var http_client: std.http.Client = .{ .allocator = gpa, .io = io }; - defer http_client.deinit(); - - try http_client.initDefaultProxies(arena, environ_map); - - var root_prog_node = std.Progress.start(io, .{ - .root_name = "Fetch", - }); - defer root_prog_node.end(); - - var global_cache_directory: Directory = l: { - const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena, environ_map); - break :l .{ - .handle = try Io.Dir.cwd().createDirPathOpen(io, p, .{}), - .path = p, - }; - }; - defer global_cache_directory.handle.close(io); - - var local_storage: Package.Fetch.LocalStorage = undefined; - var build_root: BuildRoot = undefined; - var build_root_initialized = false; - defer if (build_root_initialized) build_root.deinit(io); - - const cwd_path = try introspect.getResolvedCwd(io, arena); - - const local_storage_ptr = switch (save) { - .no => null, - .yes, .exact => ls: { - build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path }); - build_root_initialized = true; - - local_storage = .{ - .cache_root = if (override_local_cache_dir) |p| .initCwd(p) else .{ - .root_dir = build_root.directory, - .sub_path = ".zig-cache", - }, - .pkg_root = if (override_pkg_dir) |p| .initCwd(p) else .{ - .root_dir = build_root.directory, - .sub_path = "zig-pkg", - }, - }; - - break :ls &local_storage; - }, - }; - - var job_queue: Package.Fetch.JobQueue = .{ - .io = io, - .http_client = &http_client, - .global_cache = global_cache_directory, - .local_storage = local_storage_ptr, - .recursive = false, - .read_only = false, - .debug_hash = debug_hash, - .mode = .all, - .prog_node = root_prog_node, - }; - defer job_queue.deinit(); - - var fetch: Package.Fetch = .{ - .arena = std.heap.ArenaAllocator.init(gpa), - .location = .{ .path_or_url = path_or_url }, - .location_tok = 0, - .hash_tok = .none, - .name_tok = 0, - .lazy_status = .eager, - .remote_package_root = undefined, - .parent_package_root = undefined, - .parent_manifest_ast = null, - .prog_node = root_prog_node, - .job_queue = &job_queue, - .omit_missing_hash_error = true, - .allow_missing_paths_field = false, - .use_latest_commit = true, - - .package_root = undefined, - .error_bundle = undefined, - .manifest = undefined, - .manifest_ast = undefined, - .have_manifest = false, - .computed_hash = undefined, - .has_build_zig = false, - .oom_flag = false, - .latest_commit = null, - - .module = null, - }; - defer fetch.deinit(); - - fetch.run() catch |err| switch (err) { - error.OutOfMemory, error.Canceled => |e| return e, - error.FetchFailed => {}, // error bundle checked below - }; - - try job_queue.group.await(io); - - if (fetch.error_bundle.root_list.items.len > 0) { - var errors = try fetch.error_bundle.toOwnedBundle(""); - errors.renderToStderr(io, .{}, color) catch {}; - process.exit(1); - } - - const package_hash = fetch.computedPackageHash(); - const package_hash_slice = package_hash.toSlice(); - - root_prog_node.end(); - root_prog_node = .{ .index = .none }; - - const name = switch (save) { - .no => { - var stdout = Io.File.stdout().writerStreaming(io, &stdout_buffer); - try stdout.interface.print("{s}\n", .{package_hash_slice}); - try stdout.interface.flush(); - return cleanExit(io); - }, - .yes, .exact => |name| name: { - if (name) |n| break :name n; - if (!fetch.have_manifest) - fatal("unable to determine name; fetched package has no build.zig.zon file", .{}); - break :name fetch.manifest.name; - }, - }; - - // The name to use in case the manifest file needs to be created now. - const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path); - var manifest, var ast = try loadManifest(gpa, arena, io, .{ - .root_name = try sanitizeExampleName(arena, init_root_name), - .dir = build_root.directory.handle, - .color = color, - }); - defer { - manifest.deinit(gpa); - ast.deinit(gpa); - } - - var fixups: Ast.Render.Fixups = .{}; - defer fixups.deinit(gpa); - - var saved_path_or_url = path_or_url; - - if (fetch.latest_commit) |latest_commit| resolved: { - const latest_commit_hex = try std.fmt.allocPrint(arena, "{f}", .{latest_commit}); - - var uri = try std.Uri.parse(path_or_url); - - if (uri.fragment) |fragment| { - const target_ref = try fragment.toRawMaybeAlloc(arena); - - // the refspec may already be fully resolved - if (std.mem.eql(u8, target_ref, latest_commit_hex)) break :resolved; - - std.log.info("resolved ref {q} to commit {s}", .{ target_ref, latest_commit_hex }); - - // include the original refspec in a query parameter, could be used to check for updates - uri.query = .{ .percent_encoded = try std.fmt.allocPrint(arena, "ref={f}", .{ - std.fmt.alt(fragment, .formatEscaped), - }) }; - } else { - std.log.info("resolved to commit {s}", .{latest_commit_hex}); - } - - // replace the refspec with the resolved commit SHA - uri.fragment = .{ .raw = latest_commit_hex }; - - switch (save) { - .yes => saved_path_or_url = try std.fmt.allocPrint(arena, "{f}", .{uri}), - .no, .exact => {}, // keep the original URL - } - } - - const new_node_init = try std.fmt.allocPrint(arena, - \\.{{ - \\ .url = "{f}", - \\ .hash = "{f}", - \\ }} - , .{ - std.zig.fmtString(saved_path_or_url), - std.zig.fmtString(package_hash_slice), - }); - - const new_node_text = try std.fmt.allocPrint(arena, ".{f} = {s},\n", .{ - std.zig.fmtIdPU(name), new_node_init, - }); - - const dependencies_init = try std.fmt.allocPrint(arena, ".{{\n {s} }}", .{ - new_node_text, - }); - - const dependencies_text = try std.fmt.allocPrint(arena, ".dependencies = {s},\n", .{ - dependencies_init, - }); - - if (manifest.dependencies.get(name)) |dep| { - if (dep.hash) |h| { - switch (dep.location) { - .url => |u| { - if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { - std.log.info("existing dependency named {q} is up-to-date", .{name}); - process.exit(0); - } - }, - .path => {}, - } - } - - const location_replace = try std.fmt.allocPrint( - arena, - "\"{f}\"", - .{std.zig.fmtString(saved_path_or_url)}, - ); - const hash_replace = try std.fmt.allocPrint( - arena, - "\"{f}\"", - .{std.zig.fmtString(package_hash_slice)}, - ); - - warn("overwriting existing dependency named {q}", .{name}); - try fixups.replace_nodes_with_string.put(gpa, dep.location_node, location_replace); - if (dep.hash_node.unwrap()) |hash_node| { - try fixups.replace_nodes_with_string.put(gpa, hash_node, hash_replace); - } else { - // https://github.com/ziglang/zig/issues/21690 - } - } else if (manifest.dependencies.count() > 0) { - // Add fixup for adding another dependency. - const deps = manifest.dependencies.values(); - const last_dep_node = deps[deps.len - 1].node; - try fixups.append_string_after_node.put(gpa, last_dep_node, new_node_text); - } else if (manifest.dependencies_node.unwrap()) |dependencies_node| { - // Add fixup for replacing the entire dependencies struct. - try fixups.replace_nodes_with_string.put(gpa, dependencies_node, dependencies_init); - } else { - // Add fixup for adding dependencies struct. - try fixups.append_string_after_node.put(gpa, manifest.version_node, dependencies_text); - } - - var aw: Io.Writer.Allocating = .init(gpa); - defer aw.deinit(); - try ast.render(gpa, &aw.writer, fixups); - const rendered = aw.written(); - - build_root.directory.handle.writeFile(io, .{ .sub_path = Package.Manifest.basename, .data = rendered }) catch |err| { - fatal("unable to write {s} file: {t}", .{ Package.Manifest.basename, err }); - }; - - return cleanExit(io); -} - -fn createEmptyDependenciesModule( - arena: Allocator, - io: Io, - main_mod: *Package.Module, - dirs: Compilation.Directories, - global_options: Compilation.Config, -) !void { - var source = std.array_list.Managed(u8).init(arena); - try Package.Fetch.JobQueue.createEmptyDependenciesSource(&source); - _ = try createDependenciesModule( - arena, - io, - source.items, - main_mod, - dirs, - global_options, - ); -} - -/// Creates the dependencies.zig file and corresponding `Package.Module` for the -/// build runner to obtain via `@import("@dependencies")`. -fn createDependenciesModule( - arena: Allocator, - io: Io, - source: []const u8, - main_mod: *Package.Module, - dirs: Compilation.Directories, - global_options: Compilation.Config, -) !*Package.Module { - // Atomically create the file in a directory named after the hash of its contents. - const basename = "dependencies.zig"; - const rand_int = randInt(io, u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); - { - var tmp_dir = try dirs.local_cache.handle.createDirPathOpen(io, tmp_dir_sub_path, .{}); - defer tmp_dir.close(io); - try tmp_dir.writeFile(io, .{ .sub_path = basename, .data = source }); - } - const tmp_dir_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = tmp_dir_sub_path, - }; - - var hh: Cache.HashHelper = .{}; - hh.addBytes(build_options.version); - hh.addBytes(source); - const hex_digest = hh.final(); - - const o_dir_path: Path = .{ - .root_dir = dirs.local_cache, - .sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest), - }; - try Package.Fetch.renameTmpIntoCache(io, tmp_dir_path, o_dir_path); - - const deps_mod = try Package.Module.create(arena, .{ - .paths = .{ - .root = try .fromRoot(arena, dirs, .local_cache, o_dir_path.sub_path), - .root_src_path = basename, - }, - .fully_qualified_name = "root.@dependencies", - .parent = main_mod, - .cc_argv = &.{}, - .inherited = .{}, - .global = global_options, - }); - try main_mod.deps.put(arena, "@dependencies", deps_mod); - return deps_mod; -} - -const BuildRoot = struct { - directory: Cache.Directory, - build_zig_basename: []const u8, - cleanup_build_dir: ?Io.Dir, - - fn deinit(br: *BuildRoot, io: Io) void { - if (br.cleanup_build_dir) |*dir| dir.close(io); - br.* = undefined; - } -}; - -const FindBuildRootOptions = struct { - build_file: ?[]const u8 = null, - cwd_path: ?[]const u8 = null, -}; - -fn findBuildRoot(arena: Allocator, io: Io, options: FindBuildRootOptions) !BuildRoot { - const cwd_path = options.cwd_path orelse try introspect.getResolvedCwd(io, arena); - const build_zig_basename = if (options.build_file) |bf| - fs.path.basename(bf) - else - Package.build_zig_basename; - - if (options.build_file) |bf| { - if (fs.path.dirname(bf)) |dirname| { - const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { - fatal("unable to open directory to build file from argument 'build-file', {q}: {t}", .{ dirname, err }); - }; - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ .path = dirname, .handle = dir }, - .cleanup_build_dir = dir, - }; - } - - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ .path = null, .handle = Io.Dir.cwd() }, - .cleanup_build_dir = null, - }; - } - // Search up parent directories until we find build.zig. - var dirname: []const u8 = cwd_path; - while (true) { - const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig_basename }); - if (Io.Dir.cwd().access(io, joined_path, .{})) |_| { - const dir = Io.Dir.cwd().openDir(io, dirname, .{}) catch |err| { - fatal("unable to open directory while searching for build.zig file, {q}: {t}", .{ dirname, err }); - }; - return .{ - .build_zig_basename = build_zig_basename, - .directory = .{ - .path = dirname, - .handle = dir, - }, - .cleanup_build_dir = dir, - }; - } else |err| switch (err) { - error.FileNotFound => { - dirname = fs.path.dirname(dirname) orelse { - std.log.info("initialize {s} template file with 'zig init'", .{ - Package.build_zig_basename, - }); - std.log.info("see 'zig --help' for more options", .{}); - fatal("no build.zig file found, in the current directory or any parent directories", .{}); - }; - continue; - }, - else => |e| return e, - } - } -} - -const LoadManifestOptions = struct { - root_name: []const u8, - dir: Io.Dir, - color: Color, -}; - -fn loadManifest( - gpa: Allocator, - arena: Allocator, - io: Io, - options: LoadManifestOptions, -) !struct { Package.Manifest, Ast } { - const rng: std.Random.IoSource = .{ .io = io }; - - const manifest_bytes = while (true) { - break options.dir.readFileAllocOptions( - io, - Package.Manifest.basename, - arena, - .limited(Package.Manifest.max_bytes), - .@"1", - 0, - ) catch |err| switch (err) { - error.FileNotFound => { - writeSimpleTemplateFile(io, Package.Manifest.basename, - \\.{{ - \\ .name = .{s}, - \\ .version = "{s}", - \\ .paths = .{{""}}, - \\ .fingerprint = 0x{x}, - \\}} - \\ - , .{ - options.root_name, - build_options.version, - Package.Fingerprint.generate(rng.interface(), options.root_name).int(), - }) catch |e| { - fatal("unable to write {s}: {t}", .{ Package.Manifest.basename, e }); - }; - continue; - }, - else => |e| fatal("unable to load {s}: {t}", .{ Package.Manifest.basename, e }), - }; - }; - var ast = try Ast.parse(gpa, manifest_bytes, .zon); - errdefer ast.deinit(gpa); - - if (ast.errors.len > 0) { - try std.zig.printAstErrorsToStderr(gpa, io, ast, Package.Manifest.basename, options.color); - process.exit(2); - } - - var manifest = try Package.Manifest.parse(gpa, &ast, rng.interface(), .{}); - errdefer manifest.deinit(gpa); - - if (manifest.errors.len > 0) { - var wip_errors: std.zig.ErrorBundle.Wip = undefined; - try wip_errors.init(gpa); - defer wip_errors.deinit(); - - const src_path = try wip_errors.addString(Package.Manifest.basename); - try manifest.copyErrorsIntoBundle(ast, src_path, &wip_errors); - - var error_bundle = try wip_errors.toOwnedBundle(""); - defer error_bundle.deinit(gpa); - error_bundle.renderToStderr(io, .{}, options.color) catch {}; - - process.exit(2); - } - return .{ manifest, ast }; -} - -const Templates = struct { - zig_lib_directory: Cache.Directory, - dir: Io.Dir, - buffer: std.array_list.Managed(u8), - - fn deinit(templates: *Templates, io: Io) void { - templates.zig_lib_directory.handle.close(io); - templates.dir.close(io); - templates.buffer.deinit(); - templates.* = undefined; - } - - fn write( - templates: *Templates, - arena: Allocator, - io: Io, - out_dir: Io.Dir, - root_name: []const u8, - template_path: []const u8, - fingerprint: Package.Fingerprint, - ) !void { - if (fs.path.dirname(template_path)) |dirname| { - out_dir.createDirPath(io, dirname) catch |err| { - fatal("unable to make path {q}: {t}", .{ dirname, err }); - }; - } - - const max_bytes = 10 * 1024 * 1024; - const contents = templates.dir.readFileAlloc(io, template_path, arena, .limited(max_bytes)) catch |err| { - fatal("unable to read template file {q}: {t}", .{ template_path, err }); - }; - templates.buffer.clearRetainingCapacity(); - try templates.buffer.ensureUnusedCapacity(contents.len); - var i: usize = 0; - while (i < contents.len) { - if (contents[i] == '_' or contents[i] == '.') { - // Both '_' and '.' are allowed because depending on the context - // one prefix will be valid, while the other might not. - if (std.mem.startsWith(u8, contents[i + 1 ..], "NAME")) { - try templates.buffer.appendSlice(root_name); - i += "_NAME".len; - continue; - } else if (std.mem.startsWith(u8, contents[i + 1 ..], "FINGERPRINT")) { - try templates.buffer.print("0x{x}", .{fingerprint.int()}); - i += "_FINGERPRINT".len; - continue; - } else if (std.mem.startsWith(u8, contents[i + 1 ..], "ZIGVER")) { - try templates.buffer.appendSlice(build_options.version); - i += "_ZIGVER".len; - continue; - } - } - - try templates.buffer.append(contents[i]); - i += 1; - } - - return out_dir.writeFile(io, .{ - .sub_path = template_path, - .data = templates.buffer.items, - .flags = .{ .exclusive = true }, - }); - } -}; -fn writeSimpleTemplateFile(io: Io, file_name: []const u8, comptime fmt: []const u8, args: anytype) !void { - const f = try Io.Dir.cwd().createFile(io, file_name, .{ .exclusive = true }); - defer f.close(io); - var buf: [4096]u8 = undefined; - var fw = f.writer(io, &buf); - try fw.interface.print(fmt, args); - try fw.interface.flush(); -} - -fn findTemplates(gpa: Allocator, arena: Allocator, io: Io) Templates { - const cwd_path = introspect.getResolvedCwd(io, arena) catch |err| { - fatal("unable to get cwd: {t}", .{err}); - }; - const self_exe_path = process.executablePathAlloc(io, arena) catch |err| { - fatal("unable to find self exe path: {t}", .{err}); - }; - var zig_lib_directory = introspect.findZigLibDirFromSelfExe(arena, io, cwd_path, self_exe_path) catch |err| { - fatal("unable to find zig installation directory {q}: {t}", .{ self_exe_path, err }); - }; - - const s = fs.path.sep_str; - const template_sub_path = "init"; - const template_dir = zig_lib_directory.handle.openDir(io, template_sub_path, .{}) catch |err| { - const path = zig_lib_directory.path orelse "."; - fatal("unable to open zig project template directory '{s}{s}{s}': {t}", .{ - path, s, template_sub_path, err, - }); - }; - - return .{ - .zig_lib_directory = zig_lib_directory, - .dir = template_dir, - .buffer = std.array_list.Managed(u8).init(gpa), - }; -} - fn parseOptimizeMode(s: []const u8) std.lang.OptimizeMode { return stringToEnum(std.lang.OptimizeMode, s) orelse fatal("unrecognized optimization mode: {q}", .{s}); @@ -7879,7 +6025,7 @@ fn handleModArg( mod_name: []const u8, opt_root_src_orig: ?[]const u8, create_module: *CreateModule, - mod_opts: *Package.Module.CreateOptions.Inherited, + mod_opts: *Module.CreateOptions.Inherited, cc_argv: *std.ArrayList([]const u8), target_arch_os_abi: *?[]const u8, target_mcpu: *?[]const u8, diff --git a/src/print_env.zig b/src/print_env.zig index 9370006c72df78adc2725178d2644ca44fbbfcae..34264234ad234faf38bf5bd744e67b833899ce7d 100644 --- a/src/print_env.zig +++ b/src/print_env.zig @@ -8,7 +8,6 @@ const fatal = std.process.fatal; const build_options = @import("build_options"); const Compilation = @import("Compilation.zig"); -const introspect = @import("introspect.zig"); pub fn cmdEnv( arena: Allocator, @@ -29,9 +28,9 @@ pub fn cmdEnv( }, }; - const cwd_path = try introspect.getResolvedCwd(io, arena); + const cwd_path = try std.zig.getResolvedCwd(io, arena); - var dirs: Compilation.Directories = .init( + var dirs: std.zig.Directories = .init( arena, io, override_lib_dir, diff --git a/src/print_targets.zig b/src/print_targets.zig index c3a9ff44584ee2eb304cb63531fbda56ee1c98a1..702a684de3e829b8e7d475c265a5ed2e584caa1e 100644 --- a/src/print_targets.zig +++ b/src/print_targets.zig @@ -9,7 +9,6 @@ const Target = std.Target; const assert = std.debug.assert; const glibc = @import("libs/glibc.zig"); -const introspect = @import("introspect.zig"); const target = @import("target.zig"); pub fn cmdTargets( @@ -20,7 +19,7 @@ pub fn cmdTargets( native_target: *const Target, ) !void { _ = args; - var zig_lib_directory = introspect.findZigLibDir(allocator, io) catch |err| + var zig_lib_directory = std.zig.findZigLibDir(allocator, io) catch |err| fatal("unable to find zig installation directory: {t}", .{err}); defer zig_lib_directory.handle.close(io); defer allocator.free(zig_lib_directory.path.?); diff --git a/test/src/Cases.zig b/test/src/Cases.zig index f321d15851e117539f78f38681bd20cb8e06b1c4..af5dcfdddd6fd1c7671f88ff812637bfbcdfd794 100644 --- a/test/src/Cases.zig +++ b/test/src/Cases.zig @@ -316,20 +316,19 @@ pub fn addCompile( /// Each file should include a test manifest as a contiguous block of comments at /// the end of the file. The first line should be the test type, followed by a set of /// key-value config values, followed by a blank line, then the expected output. -pub fn addFromDir(ctx: *Cases, dir: Io.Dir, b: *std.Build) void { +pub fn addFromDir(ctx: *Cases, dir: Io.Dir, path_from_root: []const u8, b: *std.Build) void { var current_file: []const u8 = "none"; - ctx.addFromDirInner(dir, ¤t_file, b) catch |err| { - std.debug.panicExtra( - @returnAddress(), - "test harness failed to process file '{s}': {s}\n", - .{ current_file, @errorName(err) }, - ); + ctx.addFromDirInner(dir, path_from_root, ¤t_file, b) catch |err| { + std.debug.panicExtra(@returnAddress(), "test harness failed to process file {q}: {t}\n", .{ + current_file, err, + }); }; } fn addFromDirInner( ctx: *Cases, iterable_dir: Io.Dir, + path_from_root: []const u8, /// This is kept up to date with the currently being processed file so /// that if any errors occur the caller knows it happened during this file. current_file: *[]const u8, @@ -340,11 +339,19 @@ fn addFromDirInner( var filenames: ArrayList([]const u8) = .empty; while (try it.next(io)) |entry| { - if (entry.kind != .file) continue; - // Ignore stuff such as .swp files if (!knownFileExtension(entry.basename)) continue; - try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path)); + + switch (entry.kind) { + .file => { + b.dependOnFileContents(b.path(b.pathJoin(&.{ path_from_root, entry.path }))); + try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path)); + }, + .directory => { + b.dependOnDirectory(b.path(b.pathJoin(&.{ path_from_root, entry.path }))); + }, + else => continue, + } } for (filenames.items) |filename| { diff --git a/test/tests.zig b/test/tests.zig index 62d54e981d0bef3093ee02d5be8c0de8e421a52a..78f397d3f4e0c47053d98c5a51049a8dd7507fa9 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -3258,14 +3258,12 @@ pub fn addCases( var cases = @import("src/Cases.zig").init(gpa, arena, io); - // Ensure changes to these files get picked up - // https://codeberg.org/ziglang/zig/issues/35473 - b.graph.poisonCache(); + b.dependOnDirectory(b.path("test/cases")); var dir = try b.root.openDir(io, "test/cases", .{ .iterate = true }); defer dir.close(io); - cases.addFromDir(dir, b); + cases.addFromDir(dir, "test/cases", b); try @import("cases.zig").addCases(&cases, build_options, b); cases.lowerToBuildSteps( @@ -3320,22 +3318,28 @@ pub fn addIncrementalTests(b: *std.Build, test_step: *Step, test_filters: []cons }), }); - // Ensure changes to these files get picked up - // https://codeberg.org/ziglang/zig/issues/35473 - b.graph.poisonCache(); + b.dependOnDirectory(b.path("test/incremental")); var dir = try b.root.openDir(io, "test/incremental", .{ .iterate = true }); defer dir.close(io); var it = try dir.walk(b.graph.arena); while (try it.next(io)) |entry| { - if (entry.kind != .file) continue; if (std.mem.endsWith(u8, entry.basename, ".swp")) continue; for (test_filters) |test_filter| { if (std.mem.indexOf(u8, entry.path, test_filter)) |_| break; } else if (test_filters.len > 0) continue; + switch (entry.kind) { + .file => {}, + .directory => { + b.dependOnDirectory(b.path(b.pathJoin(&.{ "test", "incremental", entry.path }))); + }, + else => continue, + } + b.dependOnFileContents(b.path(b.pathJoin(&.{ "test", "incremental", entry.path }))); + for (incremental_targets) |target_str| { const run = b.addRunArtifact(incr_check); run.setName(b.fmt("incr-check {s} '{s}'", .{ target_str, entry.basename }));