authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-03 14:00:28-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-03 14:00:28-05:00
log5001faa690ce12374260334beb5f26dfa3f61613
tree80f40305c95346fd51325e66097b8871bbf8022e
parent10046f9a52c089c656bfd4578e4ef17671ff1f75
signaturelock-open Commit is signed but in an unrecognized format.

ci: stop assuming that azure will complete last

Each of the 3 CI services now trigger a sr.ht build via the on_master_success script. The sr.ht build checks if all builds have completed successfully by trying to download the JSON file for the particular version. If all builds have completed successfully then this sr.ht job will update the download page.

11 files changed, 142 insertions(+), 64 deletions(-)

.builds/freebsd.yml+3
...@@ -3,8 +3,11 @@ packages:...@@ -3,8 +3,11 @@ packages:
3 - cmake3 - cmake
4 - py27-s3cmd4 - py27-s3cmd
5 - wget5 - wget
6 - curl
7 - jq
6secrets:8secrets:
7 - 6c60aaee-92e7-4e7d-812c-114817689b4d9 - 6c60aaee-92e7-4e7d-812c-114817689b4d
10 - 2cd8c203-ba06-4504-be49-1e2b9b71efdd
8sources:11sources:
9 - https://github.com/ziglang/zig12 - https://github.com/ziglang/zig
10tasks:13tasks:
ci/azure/linux_script+3-1
...@@ -25,6 +25,7 @@ cd build...@@ -25,6 +25,7 @@ cd build
25cmake .. -DCMAKE_BUILD_TYPE=Release25cmake .. -DCMAKE_BUILD_TYPE=Release
26make -j2 install26make -j2 install
27./zig build test -Denable-qemu27./zig build test -Denable-qemu
28VERSION="$(./zig version)"
2829
29if [ "${BUILD_REASON}" != "PullRequest" ]; then30if [ "${BUILD_REASON}" != "PullRequest" ]; then
30 ARTIFACTSDIR="$BUILDDIR/artifacts"31 ARTIFACTSDIR="$BUILDDIR/artifacts"
...@@ -44,7 +45,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then...@@ -44,7 +45,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
44 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE45 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
4546
46 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"47 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
47 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/linux-$VERSION.json"48 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-linux-$VERSION.json"
4849
49 # `set -x` causes these variables to be mangled.50 # `set -x` causes these variables to be mangled.
50 # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html51 # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
...@@ -52,4 +53,5 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then...@@ -52,4 +53,5 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
52 echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"53 echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
53 echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM"54 echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM"
54 echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE"55 echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE"
56 echo "##vso[task.setvariable variable=version;isOutput=true]$VERSION"
55fi57fi
ci/azure/macos_script+1-1
...@@ -104,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then...@@ -104,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
104 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE104 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
105105
106 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"106 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
107 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/macos-$VERSION.json"107 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-macos-$VERSION.json"
108108
109 # `set -x` causes these variables to be mangled.109 # `set -x` causes these variables to be mangled.
110 # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html110 # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
ci/azure/on_master_success created+10
...@@ -0,0 +1,10 @@
1#!/bin/sh
2
3set -x
4set -e
5
6sudo apt-get update -y
7sudo apt-get install -y curl jq
8
9OAUTH_TOKEN="$(cat "$DOWNLOADSECUREFILE_SECUREFILEPATH")"
10./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN"
ci/azure/pipelines.yml+4-12
...@@ -55,18 +55,10 @@ jobs:...@@ -55,18 +55,10 @@ jobs:
55 pool:55 pool:
56 vmImage: 'ubuntu-16.04'56 vmImage: 'ubuntu-16.04'
57 variables:57 variables:
58 macos_tarball: $[ dependencies.BuildMacOS.outputs['main.tarball'] ]58 version: $[ dependencies.BuildLinux.outputs['main.version'] ]
59 macos_shasum: $[ dependencies.BuildMacOS.outputs['main.shasum'] ]
60 macos_bytesize: $[ dependencies.BuildMacOS.outputs['main.bytesize'] ]
61 linux_tarball: $[ dependencies.BuildLinux.outputs['main.tarball'] ]
62 linux_shasum: $[ dependencies.BuildLinux.outputs['main.shasum'] ]
63 linux_bytesize: $[ dependencies.BuildLinux.outputs['main.bytesize'] ]
64 windows_tarball: $[ dependencies.BuildWindows.outputs['main.tarball'] ]
65 windows_shasum: $[ dependencies.BuildWindows.outputs['main.shasum'] ]
66 windows_bytesize: $[ dependencies.BuildWindows.outputs['main.bytesize'] ]
67 steps:59 steps:
68 - task: DownloadSecureFile@160 - task: DownloadSecureFile@1
69 inputs:61 inputs:
70 secureFile: s3cfg62 secureFile: oauth_token
71 - script: ci/azure/update_download_page63 - script: ci/azure/on_master_success
72 displayName: 'Update download page'64 displayName: 'master branch success hook'
ci/azure/update_download_page deleted-47
...@@ -1,47 +0,0 @@
1#!/bin/sh
2
3set -x
4set -e
5
6SRCDIR=$(pwd)
7rm -rf .git
8sudo apt-get update -y
9sudo apt-get install -y s3cmd curl jq
10
11cd "$HOME"
12wget "https://ziglang.org/builds/$LINUX_TARBALL"
13tar xf $LINUX_TARBALL
14ZIGDIR=$(basename -s .tar.xz $LINUX_TARBALL)
15ZIG="$ZIGDIR/zig"
16LANGREF="$ZIGDIR/langref.html"
17VERSION=$($ZIG version)
18SRCTARBALLDIR="zig-$VERSION"
19export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
20mv "$SRCDIR" "$SRCTARBALLDIR"
21tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR"
22export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1)
23export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL)
24
25# the aarch64 build has to be there too
26AARCH64_LINUX_JSON=$(curl --fail "https://ziglang.org/builds/aarch64-linux-$VERSION.json" || exit 1)
27export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)"
28export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)"
29export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)"
30
31# the freebsd build has to be there too
32FREEBSD_JSON=$(curl --fail "https://ziglang.org/builds/freebsd-$VERSION.json" || exit 1)
33export FREEBSD_TARBALL="$(echo "$FREEBSD_JSON" | jq .tarball -r)"
34export FREEBSD_BYTESIZE="$(echo "$FREEBSD_JSON" | jq .size -r)"
35export FREEBSD_SHASUM="$(echo "$FREEBSD_JSON" | jq .shasum -r)"
36
37git clone https://github.com/ziglang/www.ziglang.org --depth 1
38cd www.ziglang.org
39export MASTER_DATE=$(date +%Y-%m-%d)
40env
41"../$ZIG" run update-download-page.zig
42
43mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"
44s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
45s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
46s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
47s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
ci/azure/windows_upload+1-1
...@@ -30,7 +30,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then...@@ -30,7 +30,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
30 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE30 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
3131
32 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"32 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
33 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/windows-$VERSION.json"33 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-windows-$VERSION.json"
3434
35 # `set -x` causes these variables to be mangled.35 # `set -x` causes these variables to be mangled.
36 # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html36 # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
ci/drone/linux_script+5-1
...@@ -8,7 +8,7 @@ BUILDDIR="$(pwd)"...@@ -8,7 +8,7 @@ BUILDDIR="$(pwd)"
8DISTDIR="$(pwd)/dist"8DISTDIR="$(pwd)/dist"
99
10apk update10apk update
11apk add py3-pip xz perl-utils11apk add py3-pip xz perl-utils jq curl
12pip3 install s3cmd12pip3 install s3cmd
1313
14# Make the `zig version` number consistent.14# Make the `zig version` number consistent.
...@@ -60,4 +60,8 @@ if [ -z "$DRONE_PULL_REQUEST" ]; then...@@ -60,4 +60,8 @@ if [ -z "$DRONE_PULL_REQUEST" ]; then
6060
61 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"61 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
62 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$TRIPLEARCH-linux-$VERSION.json"62 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$TRIPLEARCH-linux-$VERSION.json"
63 if [ "$GITBRANCH" = "master" ]; then
64 cd "$BUILDDIR"
65 ./ci/srht/on_master_success "$VERSION" "$SRHT_OAUTH_TOKEN"
66 fi
63fi67fi
ci/srht/freebsd_script+7-1
...@@ -85,5 +85,11 @@ if [ -f ~/.s3cfg ]; then...@@ -85,5 +85,11 @@ if [ -f ~/.s3cfg ]; then
85 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE85 echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
8686
87 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"87 s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
88 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/freebsd-$VERSION.json"88 s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-freebsd-$VERSION.json"
89
90 if [ "$GITBRANCH" = "master" ]; then
91 OAUTH_TOKEN="$(cat ~/.oauth_token)"
92 cd "$ZIGDIR"
93 ./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN"
94 fi
89fi95fi
ci/srht/on_master_success created+33
...@@ -0,0 +1,33 @@
1#!/bin/sh
2
3# This script must run on a lot of different platforms.
4# It assumes that the following things are installed:
5# * curl
6# * jq
7
8set -x
9set -e
10
11VERSION="$1"
12OAUTH_TOKEN="$2"
13YML_FILE="tmp.yml"
14
15echo "image: ubuntu/xenial" >> "$YML_FILE"
16echo "packages:" >> "$YML_FILE"
17echo " - s3cmd" >> "$YML_FILE"
18echo " - curl" >> "$YML_FILE"
19echo " - jq" >> "$YML_FILE"
20echo "secrets:" >> "$YML_FILE"
21echo " - 6c60aaee-92e7-4e7d-812c-114817689b4d" >> "$YML_FILE"
22echo "sources:" >> "$YML_FILE"
23echo " - https://github.com/ziglang/zig" >> "$YML_FILE"
24echo "tasks:" >> "$YML_FILE"
25echo " - build: cd zig && ./ci/srht/update_download_page $VERSION" >> "$YML_FILE"
26
27jq <$YML_FILE -sR '{
28 "manifest": .,
29 }' | curl \
30 -H Authorization:"token $OAUTH_TOKEN" \
31 -H Content-Type:application/json \
32 -X POST \
33 -d @- "https://builds.sr.ht/api/jobs"
ci/srht/update_download_page created+75
...@@ -0,0 +1,75 @@
1#!/bin/sh
2
3set -x
4set -e
5
6VERSION="$1"
7SRCDIR="$(pwd)"
8NATIVE_TARBALL="zig-linux-$(uname -m)-$VERSION.tar.xz"
9
10# Check for all the builds being completed. It's expected that this script is run several times
11# before they are all complete.
12AARCH64_LINUX_JSON_URL="https://ziglang.org/builds/aarch64-linux-$VERSION.json"
13X86_64_LINUX_JSON_URL="https://ziglang.org/builds/x86_64-linux-$VERSION.json"
14X86_64_WINDOWS_JSON_URL="https://ziglang.org/builds/x86_64-windows-$VERSION.json"
15X86_64_MACOS_JSON_URL="https://ziglang.org/builds/x86_64-macos-$VERSION.json"
16X86_64_FREEBSD_JSON_URL="https://ziglang.org/builds/x86_64-freebsd-$VERSION.json"
17
18# If any of these fail, it's not really this job failing; rather we have detected
19# that this job will be called again later when other jobs have completed.
20curl --fail -I "$AARCH64_LINUX_JSON_URL" || exit 0
21curl --fail -I "$X86_64_LINUX_JSON_URL" || exit 0
22curl --fail -I "$X86_64_WINDOWS_JSON_URL" || exit 0
23curl --fail -I "$X86_64_MACOS_JSON_URL" || exit 0
24curl --fail -I "$X86_64_FREEBSD_JSON_URL" || exit 0
25
26rm -rf .git
27
28cd "$HOME"
29wget "https://ziglang.org/builds/$NATIVE_TARBALL"
30tar xf "$NATIVE_TARBALL"
31ZIGDIR=$(basename -s .tar.xz $NATIVE_TARBALL)
32ZIG="$ZIGDIR/zig"
33LANGREF="$ZIGDIR/langref.html"
34SRCTARBALLDIR="zig-$VERSION"
35export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
36mv "$SRCDIR" "$SRCTARBALLDIR"
37tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR"
38export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1)
39export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL)
40
41X86_64_WINDOWS_JSON=$(curl --fail "X86_64_WINDOWS_JSON_URL" || exit 1)
42export X86_64_WINDOWS_TARBALL="$(echo "$X86_64_WINDOWS_JSON" | jq .tarball -r)"
43export X86_64_WINDOWS_BYTESIZE="$(echo "$X86_64_WINDOWS_JSON" | jq .size -r)"
44export X86_64_WINDOWS_SHASUM="$(echo "$X86_64_WINDOWS_JSON" | jq .shasum -r)"
45
46X86_64_MACOS_JSON=$(curl --fail "X86_64_MACOS_JSON_URL" || exit 1)
47export X86_64_MACOS_TARBALL="$(echo "$X86_64_MACOS_JSON" | jq .tarball -r)"
48export X86_64_MACOS_BYTESIZE="$(echo "$X86_64_MACOS_JSON" | jq .size -r)"
49export X86_64_MACOS_SHASUM="$(echo "$X86_64_MACOS_JSON" | jq .shasum -r)"
50
51X86_64_LINUX_JSON=$(curl --fail "X86_64_LINUX_JSON_URL" || exit 1)
52export X86_64_LINUX_TARBALL="$(echo "$X86_64_LINUX_JSON" | jq .tarball -r)"
53export X86_64_LINUX_BYTESIZE="$(echo "$X86_64_LINUX_JSON" | jq .size -r)"
54export X86_64_LINUX_SHASUM="$(echo "$X86_64_LINUX_JSON" | jq .shasum -r)"
55
56AARCH64_LINUX_JSON=$(curl --fail "AARCH64_LINUX_JSON_URL" || exit 1)
57export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)"
58export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)"
59export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)"
60
61X86_64_FREEBSD_JSON=$(curl --fail "X86_64_FREEBSD_JSON_URL" || exit 1)
62export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)"
63export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)"
64export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)"
65
66git clone https://github.com/ziglang/www.ziglang.org --depth 1
67cd www.ziglang.org
68export MASTER_DATE="$(date +%Y-%m-%d)"
69env
70"../$ZIG" run update-download-page.zig
71
72s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
73s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
74s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
75s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"