From 5001faa690ce12374260334beb5f26dfa3f61613 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 3 Nov 2019 14:00:28 -0500 Subject: [PATCH] ci: stop assuming that azure will complete last Each of the 3 CI services now trigger a sr.ht build via the on_master_success script. The sr.ht build checks if all builds have completed successfully by trying to download the JSON file for the particular version. If all builds have completed successfully then this sr.ht job will update the download page. --- .builds/freebsd.yml | 3 ++ ci/azure/linux_script | 4 +- ci/azure/macos_script | 2 +- ci/azure/on_master_success | 10 +++++ ci/azure/pipelines.yml | 16 ++------ ci/azure/update_download_page | 47 ---------------------- ci/azure/windows_upload | 2 +- ci/drone/linux_script | 6 ++- ci/srht/freebsd_script | 8 +++- ci/srht/on_master_success | 33 +++++++++++++++ ci/srht/update_download_page | 75 +++++++++++++++++++++++++++++++++++ 11 files changed, 142 insertions(+), 64 deletions(-) create mode 100755 ci/azure/on_master_success delete mode 100755 ci/azure/update_download_page create mode 100755 ci/srht/on_master_success create mode 100755 ci/srht/update_download_page diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml index 3c0b91afffa3befe079c8909f1aaa1ab1d184e90..16db3a967880b35f32112b0b5b752167b2cb5aff 100644 --- a/.builds/freebsd.yml +++ b/.builds/freebsd.yml @@ -3,8 +3,11 @@ packages: - cmake - py27-s3cmd - wget + - curl + - jq secrets: - 6c60aaee-92e7-4e7d-812c-114817689b4d + - 2cd8c203-ba06-4504-be49-1e2b9b71efdd sources: - https://github.com/ziglang/zig tasks: diff --git a/ci/azure/linux_script b/ci/azure/linux_script index b5237dc22611aa7cf42edff0c1bb86227618d3a8..f1c26d6a969dd8da926426df4696f1f1b2137e2c 100755 --- a/ci/azure/linux_script +++ b/ci/azure/linux_script @@ -25,6 +25,7 @@ cd build cmake .. -DCMAKE_BUILD_TYPE=Release make -j2 install ./zig build test -Denable-qemu +VERSION="$(./zig version)" if [ "${BUILD_REASON}" != "PullRequest" ]; then ARTIFACTSDIR="$BUILDDIR/artifacts" @@ -44,7 +45,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" - s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/linux-$VERSION.json" + s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-linux-$VERSION.json" # `set -x` causes these variables to be mangled. # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html @@ -52,4 +53,5 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL" echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM" echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE" + echo "##vso[task.setvariable variable=version;isOutput=true]$VERSION" fi diff --git a/ci/azure/macos_script b/ci/azure/macos_script index f41c036da184f16284bb74a6b31bfc5e1aa1de22..64fee34de4e952bd53de688d43c0cabfa24964db 100755 --- a/ci/azure/macos_script +++ b/ci/azure/macos_script @@ -104,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" - s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/macos-$VERSION.json" + s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-macos-$VERSION.json" # `set -x` causes these variables to be mangled. # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html diff --git a/ci/azure/on_master_success b/ci/azure/on_master_success new file mode 100755 index 0000000000000000000000000000000000000000..5354fb0655670a932261520c0ad2027d0e762a68 --- /dev/null +++ b/ci/azure/on_master_success @@ -0,0 +1,10 @@ +#!/bin/sh + +set -x +set -e + +sudo apt-get update -y +sudo apt-get install -y curl jq + +OAUTH_TOKEN="$(cat "$DOWNLOADSECUREFILE_SECUREFILEPATH")" +./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN" diff --git a/ci/azure/pipelines.yml b/ci/azure/pipelines.yml index bcc2f119456b2fa76238ab7111d550140dbec1ea..8ce9d8d01f7863bf37a391ab173cac64ad568456 100644 --- a/ci/azure/pipelines.yml +++ b/ci/azure/pipelines.yml @@ -55,18 +55,10 @@ jobs: pool: vmImage: 'ubuntu-16.04' variables: - macos_tarball: $[ dependencies.BuildMacOS.outputs['main.tarball'] ] - macos_shasum: $[ dependencies.BuildMacOS.outputs['main.shasum'] ] - macos_bytesize: $[ dependencies.BuildMacOS.outputs['main.bytesize'] ] - linux_tarball: $[ dependencies.BuildLinux.outputs['main.tarball'] ] - linux_shasum: $[ dependencies.BuildLinux.outputs['main.shasum'] ] - linux_bytesize: $[ dependencies.BuildLinux.outputs['main.bytesize'] ] - windows_tarball: $[ dependencies.BuildWindows.outputs['main.tarball'] ] - windows_shasum: $[ dependencies.BuildWindows.outputs['main.shasum'] ] - windows_bytesize: $[ dependencies.BuildWindows.outputs['main.bytesize'] ] + version: $[ dependencies.BuildLinux.outputs['main.version'] ] steps: - task: DownloadSecureFile@1 inputs: - secureFile: s3cfg - - script: ci/azure/update_download_page - displayName: 'Update download page' + secureFile: oauth_token + - script: ci/azure/on_master_success + displayName: 'master branch success hook' diff --git a/ci/azure/update_download_page b/ci/azure/update_download_page deleted file mode 100755 index 92df66edfb4194cf7f1c52dfe6ee49ef69920bf8..0000000000000000000000000000000000000000 --- a/ci/azure/update_download_page +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -set -x -set -e - -SRCDIR=$(pwd) -rm -rf .git -sudo apt-get update -y -sudo apt-get install -y s3cmd curl jq - -cd "$HOME" -wget "https://ziglang.org/builds/$LINUX_TARBALL" -tar xf $LINUX_TARBALL -ZIGDIR=$(basename -s .tar.xz $LINUX_TARBALL) -ZIG="$ZIGDIR/zig" -LANGREF="$ZIGDIR/langref.html" -VERSION=$($ZIG version) -SRCTARBALLDIR="zig-$VERSION" -export SRC_TARBALL="$SRCTARBALLDIR.tar.xz" -mv "$SRCDIR" "$SRCTARBALLDIR" -tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR" -export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1) -export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL) - -# the aarch64 build has to be there too -AARCH64_LINUX_JSON=$(curl --fail "https://ziglang.org/builds/aarch64-linux-$VERSION.json" || exit 1) -export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)" -export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)" -export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)" - -# the freebsd build has to be there too -FREEBSD_JSON=$(curl --fail "https://ziglang.org/builds/freebsd-$VERSION.json" || exit 1) -export FREEBSD_TARBALL="$(echo "$FREEBSD_JSON" | jq .tarball -r)" -export FREEBSD_BYTESIZE="$(echo "$FREEBSD_JSON" | jq .size -r)" -export FREEBSD_SHASUM="$(echo "$FREEBSD_JSON" | jq .shasum -r)" - -git clone https://github.com/ziglang/www.ziglang.org --depth 1 -cd www.ziglang.org -export MASTER_DATE=$(date +%Y-%m-%d) -env -"../$ZIG" run update-download-page.zig - -mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg" -s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/ -s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate" -s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate" -s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate" diff --git a/ci/azure/windows_upload b/ci/azure/windows_upload index d5527fa13aebd1f4f5a56e2efeca6215996bcf50..7ed1b7d1f786699d4ea1041fab7e82c5296abc38 100755 --- a/ci/azure/windows_upload +++ b/ci/azure/windows_upload @@ -30,7 +30,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" - s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/windows-$VERSION.json" + s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-windows-$VERSION.json" # `set -x` causes these variables to be mangled. # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html diff --git a/ci/drone/linux_script b/ci/drone/linux_script index 987b3466cad8d639973012f18a039f0235a4eb8f..03bf0a3e5659a222db50b902f9d7a4b6625e40fc 100755 --- a/ci/drone/linux_script +++ b/ci/drone/linux_script @@ -8,7 +8,7 @@ BUILDDIR="$(pwd)" DISTDIR="$(pwd)/dist" apk update -apk add py3-pip xz perl-utils +apk add py3-pip xz perl-utils jq curl pip3 install s3cmd # Make the `zig version` number consistent. @@ -60,4 +60,8 @@ if [ -z "$DRONE_PULL_REQUEST" ]; then s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$TRIPLEARCH-linux-$VERSION.json" + if [ "$GITBRANCH" = "master" ]; then + cd "$BUILDDIR" + ./ci/srht/on_master_success "$VERSION" "$SRHT_OAUTH_TOKEN" + fi fi diff --git a/ci/srht/freebsd_script b/ci/srht/freebsd_script index 55c06de33f804b6113e00cd75823099d05642da7..8e717e4ba6bf555c5203ad102dde319225c11fe6 100755 --- a/ci/srht/freebsd_script +++ b/ci/srht/freebsd_script @@ -85,5 +85,11 @@ if [ -f ~/.s3cfg ]; then echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" - s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/freebsd-$VERSION.json" + s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-freebsd-$VERSION.json" + + if [ "$GITBRANCH" = "master" ]; then + OAUTH_TOKEN="$(cat ~/.oauth_token)" + cd "$ZIGDIR" + ./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN" + fi fi diff --git a/ci/srht/on_master_success b/ci/srht/on_master_success new file mode 100755 index 0000000000000000000000000000000000000000..0de916910961c75aa376fb41ec3c23838cedfcc6 --- /dev/null +++ b/ci/srht/on_master_success @@ -0,0 +1,33 @@ +#!/bin/sh + +# This script must run on a lot of different platforms. +# It assumes that the following things are installed: +# * curl +# * jq + +set -x +set -e + +VERSION="$1" +OAUTH_TOKEN="$2" +YML_FILE="tmp.yml" + +echo "image: ubuntu/xenial" >> "$YML_FILE" +echo "packages:" >> "$YML_FILE" +echo " - s3cmd" >> "$YML_FILE" +echo " - curl" >> "$YML_FILE" +echo " - jq" >> "$YML_FILE" +echo "secrets:" >> "$YML_FILE" +echo " - 6c60aaee-92e7-4e7d-812c-114817689b4d" >> "$YML_FILE" +echo "sources:" >> "$YML_FILE" +echo " - https://github.com/ziglang/zig" >> "$YML_FILE" +echo "tasks:" >> "$YML_FILE" +echo " - build: cd zig && ./ci/srht/update_download_page $VERSION" >> "$YML_FILE" + +jq <$YML_FILE -sR '{ + "manifest": ., + }' | curl \ + -H Authorization:"token $OAUTH_TOKEN" \ + -H Content-Type:application/json \ + -X POST \ + -d @- "https://builds.sr.ht/api/jobs" diff --git a/ci/srht/update_download_page b/ci/srht/update_download_page new file mode 100755 index 0000000000000000000000000000000000000000..7e42d3a3bc17051f44ee4c4e857526874f11b61e --- /dev/null +++ b/ci/srht/update_download_page @@ -0,0 +1,75 @@ +#!/bin/sh + +set -x +set -e + +VERSION="$1" +SRCDIR="$(pwd)" +NATIVE_TARBALL="zig-linux-$(uname -m)-$VERSION.tar.xz" + +# Check for all the builds being completed. It's expected that this script is run several times +# before they are all complete. +AARCH64_LINUX_JSON_URL="https://ziglang.org/builds/aarch64-linux-$VERSION.json" +X86_64_LINUX_JSON_URL="https://ziglang.org/builds/x86_64-linux-$VERSION.json" +X86_64_WINDOWS_JSON_URL="https://ziglang.org/builds/x86_64-windows-$VERSION.json" +X86_64_MACOS_JSON_URL="https://ziglang.org/builds/x86_64-macos-$VERSION.json" +X86_64_FREEBSD_JSON_URL="https://ziglang.org/builds/x86_64-freebsd-$VERSION.json" + +# If any of these fail, it's not really this job failing; rather we have detected +# that this job will be called again later when other jobs have completed. +curl --fail -I "$AARCH64_LINUX_JSON_URL" || exit 0 +curl --fail -I "$X86_64_LINUX_JSON_URL" || exit 0 +curl --fail -I "$X86_64_WINDOWS_JSON_URL" || exit 0 +curl --fail -I "$X86_64_MACOS_JSON_URL" || exit 0 +curl --fail -I "$X86_64_FREEBSD_JSON_URL" || exit 0 + +rm -rf .git + +cd "$HOME" +wget "https://ziglang.org/builds/$NATIVE_TARBALL" +tar xf "$NATIVE_TARBALL" +ZIGDIR=$(basename -s .tar.xz $NATIVE_TARBALL) +ZIG="$ZIGDIR/zig" +LANGREF="$ZIGDIR/langref.html" +SRCTARBALLDIR="zig-$VERSION" +export SRC_TARBALL="$SRCTARBALLDIR.tar.xz" +mv "$SRCDIR" "$SRCTARBALLDIR" +tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR" +export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1) +export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL) + +X86_64_WINDOWS_JSON=$(curl --fail "X86_64_WINDOWS_JSON_URL" || exit 1) +export X86_64_WINDOWS_TARBALL="$(echo "$X86_64_WINDOWS_JSON" | jq .tarball -r)" +export X86_64_WINDOWS_BYTESIZE="$(echo "$X86_64_WINDOWS_JSON" | jq .size -r)" +export X86_64_WINDOWS_SHASUM="$(echo "$X86_64_WINDOWS_JSON" | jq .shasum -r)" + +X86_64_MACOS_JSON=$(curl --fail "X86_64_MACOS_JSON_URL" || exit 1) +export X86_64_MACOS_TARBALL="$(echo "$X86_64_MACOS_JSON" | jq .tarball -r)" +export X86_64_MACOS_BYTESIZE="$(echo "$X86_64_MACOS_JSON" | jq .size -r)" +export X86_64_MACOS_SHASUM="$(echo "$X86_64_MACOS_JSON" | jq .shasum -r)" + +X86_64_LINUX_JSON=$(curl --fail "X86_64_LINUX_JSON_URL" || exit 1) +export X86_64_LINUX_TARBALL="$(echo "$X86_64_LINUX_JSON" | jq .tarball -r)" +export X86_64_LINUX_BYTESIZE="$(echo "$X86_64_LINUX_JSON" | jq .size -r)" +export X86_64_LINUX_SHASUM="$(echo "$X86_64_LINUX_JSON" | jq .shasum -r)" + +AARCH64_LINUX_JSON=$(curl --fail "AARCH64_LINUX_JSON_URL" || exit 1) +export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)" +export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)" +export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)" + +X86_64_FREEBSD_JSON=$(curl --fail "X86_64_FREEBSD_JSON_URL" || exit 1) +export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)" +export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)" +export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)" + +git clone https://github.com/ziglang/www.ziglang.org --depth 1 +cd www.ziglang.org +export MASTER_DATE="$(date +%Y-%m-%d)" +env +"../$ZIG" run update-download-page.zig + +s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/ +s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate" +s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate" +s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate" -- 2.54.0