Commit a7d215759e93be971ba5e560578e7473655cdd82 | zig

author	Michael Dusan <michael.dusan@gmail.com> 2021-11-08 04:08:29-05:00
committer	Michael Dusan <michael.dusan@gmail.com> 2021-11-08 13:45:03-05:00
log	a7d215759e93be971ba5e560578e7473655cdd82
tree	90cdab59b25f38a72d28bdf02296952e13e54760
parent	3c7310f8cc164dbf6e6393392e8812c9f5b6a270

ci: fully activate ci.ziglang.org linux

- disable azure/linux
- split probe/build/test steps for log clarity
- add package step; enabled only when master/pull
- add on-master-success pipeline; enabled only when master/pull

9 files changed, 195 insertions(+), 107 deletions(-)

ci
- azure
  - macos_script
  - pipelines.yml
- zinc

ci/azure/macos_script+1

...	...	@@ -95,4 +95,5 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
95	95	echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
96	96	echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM"
97	97	echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE"
	98	echo "##vso[task.setvariable variable=version;isOutput=true]$VERSION"
98	99	fi

ci/azure/pipelines.yml+1-13

...	...	@@ -21,17 +21,6 @@ jobs:
21	21	- script: ci/azure/macos_arm64_script
22	22	name: main
23	23	displayName: 'Build'
24		- job: BuildLinux
25		pool:
26		vmImage: 'ubuntu-18.04'
27		timeoutInMinutes: 360
28		steps:
29		- task: DownloadSecureFile@1
30		inputs:
31		secureFile: s3cfg
32		- script: ci/azure/linux_script
33		name: main
34		displayName: 'Build and test'
35	24	- job: BuildWindows
36	25	pool:
37	26	vmImage: 'windows-2019'
...	...	@@ -60,7 +49,6 @@ jobs:
60	49	dependsOn:
61	50	- BuildMacOS
62	51	- BuildMacOS_arm64
63		- BuildLinux
64	52	- BuildWindows
65	53	condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
66	54	strategy:
...	...	@@ -68,7 +56,7 @@ jobs:
68	56	pool:
69	57	vmImage: 'ubuntu-18.04'
70	58	variables:
71		version: $[ dependencies.BuildLinux.outputs['main.version'] ]
	59	version: $[ dependencies.BuildMacOS.outputs['main.version'] ]
72	60	steps:
73	61	- task: DownloadSecureFile@1
74	62	inputs:

ci/zinc/drone.yml+57-2

...	...	@@ -9,7 +9,62 @@ workspace:
9	9	path: /workspace
10	10
11	11	steps:
12		- name: build-test-package
13		image: ci/debian-amd64:11.1-1
	12	- name: probe
	13	image: ci/debian-amd64:11.1-2
	14	commands:
	15	- ./ci/zinc/linux_probe.sh
	16
	17	- name: build
	18	image: ci/debian-amd64:11.1-2
14	19	commands:
15	20	- ./ci/zinc/linux_build.sh
	21
	22	- name: test
	23	depends_on:
	24	- build
	25	image: ci/debian-amd64:11.1-2
	26	commands:
	27	- ./ci/zinc/linux_test.sh
	28
	29	- name: package
	30	depends_on:
	31	- test
	32	when:
	33	branch:
	34	- master
	35	event:
	36	- push
	37	image: ci/debian-amd64:11.1-2
	38	environment:
	39	AWS_ACCESS_KEY_ID:
	40	from_secret: AWS_ACCESS_KEY_ID
	41	AWS_SECRET_ACCESS_KEY:
	42	from_secret: AWS_SECRET_ACCESS_KEY
	43	commands:
	44	- ./ci/zinc/linux_package.sh
	45
	46	---
	47	kind: pipeline
	48	type: docker
	49	name: on-master-success
	50	platform:
	51	os: linux
	52	arch: amd64
	53	workspace:
	54	path: /workspace
	55	depends_on:
	56	- x86_64-linux
	57	trigger:
	58	branch:
	59	- master
	60	event:
	61	- push
	62
	63	steps:
	64	- name: on-master-sucess
	65	image: ci/debian-amd64:11.1-2
	66	environment:
	67	SRHT_OAUTH_TOKEN:
	68	from_secret: SRHT_OAUTH_TOKEN
	69	commands:
	70	- ./ci/zinc/linux_on_master_sucess.sh

ci/zinc/linux_base.sh created+28

...	...	@@ -0,0 +1,28 @@
	1	#!/bin/sh
	2
	3	# https://docs.drone.io/pipeline/docker/syntax/workspace/
	4	#
	5	# Drone automatically creates a temporary volume, known as your workspace,
	6	# where it clones your repository. The workspace is the current working
	7	# directory for each step in your pipeline.
	8	#
	9	# Because the workspace is a volume, filesystem changes are persisted between
	10	# pipeline steps. In other words, individual steps can communicate and share
	11	# state using the filesystem.
	12	#
	13	# Workspace volumes are ephemeral. They are created when the pipeline starts
	14	# and destroyed after the pipeline completes.
	15
	16	set -x
	17	set -e
	18
	19	ARCH="$(uname -m)"
	20	JOBS="-j$(nproc)"
	21
	22	DEPS_LOCAL="/deps/local"
	23	WORKSPACE="$DRONE_WORKSPACE"
	24
	25	DEBUG_STAGING="$WORKSPACE/_debug/staging"
	26	RELEASE_STAGING="$WORKSPACE/_release/staging"
	27
	28	export PATH=$DEPS_LOCAL/bin:$PATH

ci/zinc/linux_build.sh+9-92

...	...	@@ -1,16 +1,10 @@
1	1	#!/bin/sh
2	2
3		set -x
4		set -e
	3	. ./ci/zinc/linux_base.sh
5	4
6		WORKSPACE="$DRONE_WORKSPACE"
7		LOCAL="/deps/local"
8		ZIG="$LOCAL/bin/zig"
9		ARCH="$(uname -m)"
	5	ZIG="$DEPS_LOCAL/bin/zig"
10	6	TARGET="${ARCH}-linux-musl"
11	7	MCPU="baseline"
12		JOBS="-j$(nproc)"
13		export PATH=/deps/local/bin:$PATH
14	8
15	9	# Make the `zig version` number consistent.
16	10	# This will affect the cmake command below.
...	...	@@ -25,8 +19,8 @@ export CXX="$ZIG c++ -target $TARGET -mcpu=$MCPU"
25	19	mkdir _debug
26	20	cd _debug
27	21	cmake .. \
28		-DCMAKE_INSTALL_PREFIX="$(pwd)/staging" \
29		-DCMAKE_PREFIX_PATH="$LOCAL" \
	22	-DCMAKE_INSTALL_PREFIX="$DEBUG_STAGING" \
	23	-DCMAKE_PREFIX_PATH="$DEPS_LOCAL" \
30	24	-DCMAKE_BUILD_TYPE=Debug \
31	25	-DZIG_TARGET_TRIPLE="$TARGET" \
32	26	-DZIG_TARGET_MCPU="$MCPU" \
...	...	@@ -40,7 +34,7 @@ unset CXX
40	34
41	35	ninja $JOBS install
42	36
43		ZIG=$(pwd)/staging/bin/zig
	37	ZIG=$DEBUG_STAGING/bin/zig
44	38
45	39	# Here we rebuild zig but this time using the Zig binary we just now produced to
46	40	# build zig1.o rather than relying on the one built with stage0. See
...	...	@@ -52,14 +46,13 @@ cd $WORKSPACE
52	46
53	47	# Build release zig.
54	48	echo "BUILD release zig with zig:$($ZIG version)"
55		echo "zig version: $($ZIG version)"
56	49	export CC="$ZIG cc -target $TARGET -mcpu=$MCPU"
57	50	export CXX="$ZIG c++ -target $TARGET -mcpu=$MCPU"
58	51	mkdir _release
59	52	cd _release
60	53	cmake .. \
61		-DCMAKE_INSTALL_PREFIX="$(pwd)/staging" \
62		-DCMAKE_PREFIX_PATH="$LOCAL" \
	54	-DCMAKE_INSTALL_PREFIX="$RELEASE_STAGING" \
	55	-DCMAKE_PREFIX_PATH="$DEPS_LOCAL" \
63	56	-DCMAKE_BUILD_TYPE=Release \
64	57	-DZIG_TARGET_TRIPLE="$TARGET" \
65	58	-DZIG_TARGET_MCPU="$MCPU" \
...	...	@@ -75,81 +68,5 @@ cd $WORKSPACE
75	68	# Formatting errors can be fixed by running `zig fmt` on the files printed here.
76	69	$ZIG fmt --check .
77	70
78		$ZIG test test/behavior.zig -fno-stage1 -fLLVM -I test
79
80		$ZIG build test-behavior -Denable-qemu -Denable-wasmtime
81		$ZIG build test-compiler-rt -Denable-qemu -Denable-wasmtime
82		$ZIG build test-std -Denable-qemu -Denable-wasmtime
83		$ZIG build test-minilibc -Denable-qemu -Denable-wasmtime
84		$ZIG build test-compare-output -Denable-qemu -Denable-wasmtime
85		$ZIG build test-standalone -Denable-qemu -Denable-wasmtime
86		$ZIG build test-stack-traces -Denable-qemu -Denable-wasmtime
87		$ZIG build test-cli -Denable-qemu -Denable-wasmtime
88		$ZIG build test-asm-link -Denable-qemu -Denable-wasmtime
89		$ZIG build test-runtime-safety -Denable-qemu -Denable-wasmtime
90		$ZIG build test-translate-c -Denable-qemu -Denable-wasmtime
91		$ZIG build test-run-translated-c -Denable-qemu -Denable-wasmtime
92		$ZIG build docs -Denable-qemu -Denable-wasmtime
93		$ZIG build # test building self-hosted without LLVM
94		$ZIG build test-fmt -Denable-qemu -Denable-wasmtime
95		$ZIG build test-stage2 -Denable-qemu -Denable-wasmtime
96
97		# Look for HTML errors.
98		tidy --drop-empty-elements no -qe zig-cache/langref.html
99
100		# The remainder of this script is for master branch only.
101		if [ -n "$DRONE_PULL_REQUEST" ]; then
102		exit 0
103		fi
104
105		STAGING=_release/staging
106
107		# Produce the experimental std lib documentation.
108		mkdir -p $STAGING/docs/std
109		$ZIG test lib/std/std.zig \
110		--zig-lib-dir lib \
111		-femit-docs=$STAGING/docs/std \
112		-fno-emit-bin
113
114		cp LICENSE $STAGING/
115		cp zig-cache/langref.html $STAGING/docs/
116
117		# Remove the unnecessary bin dir in $prefix/bin/zig
118		mv $STAGING/bin/zig $STAGING/
119		rmdir $STAGING/bin
120
121		# Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig
122		mv $STAGING/lib/zig $STAGING/lib2
123		rmdir $STAGING/lib
124		mv $STAGING/lib2 $STAGING/lib
125
126		VERSION=$($STAGING/zig version)
127		BASENAME="zig-linux-$ARCH-$VERSION"
128		TARBALL="${BASENAME}.tar.xz"
129		mv "$STAGING" "$BASENAME"
130		tar cfJ "$TARBALL" "$BASENAME"
131		ls -l "$TARBALL"
132
133		# TODO: push artifact/meta somewhere
134		## mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"
135		## s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "$TARBALL" s3://ziglang.org/builds/
136		##
137		## SHASUM=$(sha256sum $TARBALL \| cut '-d ' -f1)
138		## BYTESIZE=$(wc -c < $TARBALL)
139		##
140		## JSONFILE="linux-$GITBRANCH.json"
141		## touch $JSONFILE
142		## echo "{\"tarball\": \"$TARBALL\"," >>$JSONFILE
143		## echo "\"shasum\": \"$SHASUM\"," >>$JSONFILE
144		## echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
145		##
146		## s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
147		## s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$ARCH-linux-$VERSION.json"
148		##
149		## # `set -x` causes these variables to be mangled.
150		## # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
151		## set +x
152		## echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
153		## echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM"
154		## echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE"
155		## echo "##vso[task.setvariable variable=version;isOutput=true]$VERSION"
	71	# Explicit exit helps show last command duration.
	72	exit

ci/zinc/linux_on_master_sucess.sh created+12

...	...	@@ -0,0 +1,12 @@
	1	#!/bin/sh
	2
	3	. ./ci/zinc/linux_base.sh
	4
	5	# Avoid leaking oauth token.
	6	set +x
	7
	8	cd $WORKSPACE
	9	./ci/srht/on_master_success "$VERSION" "$SRHT_OAUTH_TOKEN"
	10
	11	# Explicit exit helps show last command duration.
	12	exit

ci/zinc/linux_package.sh created+40

...	...	@@ -0,0 +1,40 @@
	1	#!/bin/sh
	2
	3	. ./ci/zinc/linux_base.sh
	4
	5	cp LICENSE $RELEASE_STAGING/
	6	cp zig-cache/langref.html $RELEASE_STAGING/docs/
	7
	8	# Remove the unnecessary bin dir in $prefix/bin/zig
	9	mv $RELEASE_STAGING/bin/zig $RELEASE_STAGING/
	10	rmdir $RELEASE_STAGING/bin
	11
	12	# Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig
	13	mv $RELEASE_STAGING/lib/zig $RELEASE_STAGING/lib2
	14	rmdir $RELEASE_STAGING/lib
	15	mv $RELEASE_STAGING/lib2 $RELEASE_STAGING/lib
	16
	17	VERSION=$($RELEASE_STAGING/zig version)
	18	BASENAME="zig-linux-$ARCH-$VERSION"
	19	TARBALL="$BASENAME.tar.xz"
	20	mv "$RELEASE_STAGING" "$BASENAME"
	21	tar cfJ "$TARBALL" "$BASENAME"
	22	ls -l "$TARBALL"
	23
	24	SHASUM=$(sha256sum $TARBALL \| cut '-d ' -f1)
	25	BYTESIZE=$(wc -c < $TARBALL)
	26
	27	MANIFEST="manifest.json"
	28	touch $MANIFEST
	29	echo "{\"tarball\": \"$TARBALL\"," >>$MANIFEST
	30	echo "\"shasum\": \"$SHASUM\"," >>$MANIFEST
	31	echo "\"size\": \"$BYTESIZE\"}" >>$MANIFEST
	32
	33	# Publish artifact.
	34	s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "$TARBALL" s3://ziglang.org/builds/
	35
	36	# Publish manifest.
	37	s3cmd put -P --add-header="cache-control: max-age=0, must-revalidate" "$MANIFEST" "s3://ziglang.org/builds/$ARCH-linux-$VERSION.json"
	38
	39	# Explicit exit helps show last command duration.
	40	exit

ci/zinc/linux_probe.sh created+10

...	...	@@ -0,0 +1,10 @@
	1	#!/bin/sh
	2
	3	. ./ci/zinc/linux_base.sh
	4
	5	# Probe CPU/brand details.
	6	echo "lscpu:"
	7	(lscpu \| sed 's,^, : ,') 1>&2
	8
	9	# Explicit exit helps show last command duration.
	10	exit

ci/zinc/linux_test.sh created+37

...	...	@@ -0,0 +1,37 @@
	1	#!/bin/sh
	2
	3	. ./ci/zinc/linux_base.sh
	4
	5	ZIG=$DEBUG_STAGING/bin/zig
	6
	7	$ZIG test test/behavior.zig -fno-stage1 -fLLVM -I test
	8
	9	$ZIG build test-behavior -Denable-qemu -Denable-wasmtime
	10	$ZIG build test-compiler-rt -Denable-qemu -Denable-wasmtime
	11	$ZIG build test-std -Denable-qemu -Denable-wasmtime
	12	$ZIG build test-minilibc -Denable-qemu -Denable-wasmtime
	13	$ZIG build test-compare-output -Denable-qemu -Denable-wasmtime
	14	$ZIG build test-standalone -Denable-qemu -Denable-wasmtime
	15	$ZIG build test-stack-traces -Denable-qemu -Denable-wasmtime
	16	$ZIG build test-cli -Denable-qemu -Denable-wasmtime
	17	$ZIG build test-asm-link -Denable-qemu -Denable-wasmtime
	18	$ZIG build test-runtime-safety -Denable-qemu -Denable-wasmtime
	19	$ZIG build test-translate-c -Denable-qemu -Denable-wasmtime
	20	$ZIG build test-run-translated-c -Denable-qemu -Denable-wasmtime
	21	$ZIG build docs -Denable-qemu -Denable-wasmtime
	22	$ZIG build # test building self-hosted without LLVM
	23	$ZIG build test-fmt -Denable-qemu -Denable-wasmtime
	24	$ZIG build test-stage2 -Denable-qemu -Denable-wasmtime
	25
	26	# Produce the experimental std lib documentation.
	27	mkdir -p $RELEASE_STAGING/docs/std
	28	$ZIG test lib/std/std.zig \
	29	--zig-lib-dir lib \
	30	-femit-docs=$RELEASE_STAGING/docs/std \
	31	-fno-emit-bin
	32
	33	# Look for HTML errors.
	34	tidy --drop-empty-elements no -qe zig-cache/langref.html
	35
	36	# Explicit exit helps show last command duration.
	37	exit