ci: rework linux script

* no longer depend on apt.llvm.org, instead we rely on a zig-bootstrap tarball with pre-built Zig, LLVM, LLD, and Clang. Similar to the Windows Dev Kit but for Linux. This also makes the script no longer depend on Docker, libxml2, or GCC. * remove bash retry logic; it was only needed for flaky apt.llvm.org and isn't needed for the other resources we download. * build and upload the experimental std lib automatically generated docs. langref.html is moved to docs/langref.html because the std lib docs are in docs/std/. * the superfluous "zig" directory in $prefix/lib/zig/std/std.zig is removed from the tarball. * update_download_page makes a commit in the www.ziglang.org repository updating data/releases.json so that repository can manage deploys.
2024-11-15 08:33:06 +00:00 · 2021-01-06 23:17:00 -07:00 · 2021-01-06 23:17:00 -07:00 · ce5222d945
commit ce5222d945
parent f1ef0a80f1
3 changed files with 89 additions and 56 deletions
--- a/ci/azure/linux_script
+++ b/ci/azure/linux_script
@ -3,48 +3,35 @@
 set -x
 set -e

-# This parameters we wait at most 2mins, it should be enough to sort out any
-# transient error.
-CMD_MAX_RETRY=12
-CMD_WAIT_TIME=10s
+sudo apt-get update -q
+sudo apt-get install -y cmake s3cmd ninja-build tidy

-# Execute the given command and, in case of failure, try to execute it again
-# after sleeping for CMD_WAIT_TIME.
-# We give up after retrying CMD_MAX_RETRY times.
-retry() {
-  for i in $(seq 1 "$CMD_MAX_RETRY"); do
-    eval "$@" && return
-    echo "command \"$@\" failed, retrying..."
-    sleep ${CMD_WAIT_TIME}
-  done
+ZIGDIR="$(pwd)"
+ARCH="$(uname -m)"
+CACHE_BASENAME="zig+llvm+lld+clang-$ARCH-linux-musl-0.8.0-dev.859+f1ef0a80f"
+PREFIX="$HOME/$CACHE_BASENAME"

-  echo "command \"$@\" failed, giving up..."
-  exit 1
-}
+rm -rf $PREFIX
+cd $HOME

-BUILDDIR="$(pwd)"
-
-sudo sh -c 'echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main" >> /etc/apt/sources.list'
-retry 'wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -'
-retry sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-
-sudo apt-get remove -y llvm-*
-sudo rm -rf /usr/local/*
-
-retry sudo apt-get update -q
-retry sudo apt-get install -y \
-    libxml2-dev libclang-11-dev llvm-11 llvm-11-dev liblld-11-dev cmake s3cmd \
-    gcc-7 g++-7 ninja-build tidy \
+wget -nv "https://ziglang.org/deps/$CACHE_BASENAME.tar.xz"
+tar xf "$CACHE_BASENAME.tar.xz"

 QEMUBASE="qemu-linux-x86_64-5.2.0"
-wget -nv https://ziglang.org/deps/$QEMUBASE.tar.xz
-tar xf $QEMUBASE.tar.xz
-PATH=$PWD/$QEMUBASE/bin:$PATH
+wget -nv "https://ziglang.org/deps/$QEMUBASE.tar.xz"
+tar xf "$QEMUBASE.tar.xz"
+export PATH="$(pwd)/$QEMUBASE/bin:$PATH"

 WASMTIME="wasmtime-v0.20.0-x86_64-linux"
-wget -nv https://github.com/bytecodealliance/wasmtime/releases/download/v0.20.0/$WASMTIME.tar.xz
-tar xf $WASMTIME.tar.xz
-PATH=$PWD/$WASMTIME:$PATH
+wget -nv "https://github.com/bytecodealliance/wasmtime/releases/download/v0.20.0/$WASMTIME.tar.xz"
+tar xf "$WASMTIME.tar.xz"
+export PATH="$(pwd)/$WASMTIME:$PATH"
+
+ZIG="$PREFIX/bin/zig"
+export CC="$ZIG cc"
+export CXX="$ZIG c++"
+
+cd $ZIGDIR

 # Make the `zig version` number consistent.
 # This will affect the cmake command below.
@ -52,29 +39,63 @@ git config core.abbrev 9
 git fetch --unshallow || true
 git fetch --tags

-export CC=gcc-7
-export CXX=g++-7
 mkdir build
 cd build
-cmake .. -DCMAKE_BUILD_TYPE=Release -GNinja
-ninja install
-./zig build test -Denable-qemu -Denable-wasmtime
+cmake .. \
+  -DCMAKE_INSTALL_PREFIX="$(pwd)/release" \
+  -DCMAKE_PREFIX_PATH="$PREFIX" \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DZIG_TARGET_TRIPLE="$ARCH-linux-musl" \
+  -DZIG_TARGET_MCPU="baseline" \
+  -DZIG_STATIC=ON \
+  -GNinja

-# look for HTML errors
+# Now cmake will use zig as the C/C++ compiler. We reset the environment variables
+# so that installation and testing do not get affected by them.
+unset CC
+unset CXX
+
+ninja install
+
+# Here we rebuild zig but this time using the Zig binary we just now produced to
+# build zig1.o rather than relying on the one built with stage0. See
+# https://github.com/ziglang/zig/issues/6830 for more details.
+cmake .. -DZIG_EXECUTABLE="$(pwd)/release/bin/zig"
+ninja install
+
+release/bin/zig build test -Denable-qemu -Denable-wasmtime
+
+# Look for HTML errors.
 tidy -qe ../zig-cache/langref.html

-VERSION="$(./zig version)"
-
 if [ "${BUILD_REASON}" != "PullRequest" ]; then
-  ARTIFACTSDIR="$BUILDDIR/artifacts"
-  mkdir "$ARTIFACTSDIR"
-  docker run -i --mount type=bind,source="$ARTIFACTSDIR",target=/z ziglang/static-base:llvm11-x86_64-1 -j2 $BUILD_SOURCEVERSION
-  TARBALL="$(ls $ARTIFACTSDIR)"
-  mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"
-  s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "$ARTIFACTSDIR/$TARBALL" s3://ziglang.org/builds/
+  # Produce the experimental std lib documentation.
+  mkdir -p release/docs/std
+  release/bin/zig test ../lib/std/std.zig -femit-docs=release/docs/std -fno-emit-bin

-  SHASUM=$(sha256sum $ARTIFACTSDIR/$TARBALL | cut '-d ' -f1)
-  BYTESIZE=$(wc -c < $ARTIFACTSDIR/$TARBALL)
+  mv ../LICENSE release/
+  mv ../zig-cache/langref.html release/docs/
+
+  # Remove the unnecessary bin dir in $prefix/bin/zig
+  mv release/bin/zig release/
+  rmdir release/bin
+
+  # Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig
+  mv release/lib/zig release/lib2
+  rmdir release/lib
+  mv release/lib2 release/lib
+
+  VERSION=$(release/zig version)
+  DIRNAME="zig-linux-$ARCH-$VERSION"
+  TARBALL="$DIRNAME.tar.xz"
+  mv release "$DIRNAME"
+  tar cfJ "$TARBALL" "$DIRNAME"
+
+  mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"
+  s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "$TARBALL" s3://ziglang.org/builds/
+
+  SHASUM=$(sha256sum $TARBALL | cut '-d ' -f1)
+  BYTESIZE=$(wc -c < $TARBALL)

  JSONFILE="linux-$GITBRANCH.json"
  touch $JSONFILE
@ -83,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
  echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE

  s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
-  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-linux-$VERSION.json"
+  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$ARCH-linux-$VERSION.json"

  # `set -x` causes these variables to be mangled.
  # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
--- a/ci/srht/on_master_success
+++ b/ci/srht/on_master_success
@ -24,6 +24,7 @@ packages:
  - xz
 secrets:
  - 51bfddf5-86a6-4e01-8576-358c72a4a0a4
+  - 8d5f230b-78d9-4e7c-b583-8cbb1b15807c
 sources:
  - https://github.com/ziglang/zig
 tasks:
--- a/ci/srht/update_download_page
+++ b/ci/srht/update_download_page
@ -35,7 +35,7 @@ wget "https://ziglang.org/builds/$NATIVE_TARBALL"
 tar xf "$NATIVE_TARBALL"
 ZIGDIR=$(basename $NATIVE_TARBALL .tar.xz)
 ZIG="$ZIGDIR/zig"
-LANGREF="$ZIGDIR/langref.html"
+LANGREF="$ZIGDIR/docs/langref.html"
 SRCTARBALLDIR="zig-$VERSION"
 export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
 mv "$SRCDIR" "$SRCTARBALLDIR"
@ -68,13 +68,24 @@ export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)"
 export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)"
 export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)"

-git clone https://github.com/ziglang/www.ziglang.org --depth 1
+git clone git@github.com:ziglang/www.ziglang.org.git
 cd www.ziglang.org
+git checkout hugo-redesign
 export MASTER_DATE="$(date +%Y-%m-%d)"
 export MASTER_VERSION="$VERSION"
 "../$ZIG" run update-download-page.zig

 $S3CMD put -P --no-mime-magic --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
+
 $S3CMD put -P --no-mime-magic "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
-$S3CMD put -P --no-mime-magic www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
-$S3CMD put -P --no-mime-magic www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
+$S3CMD put -P --no-mime-magic "../$ZIGDIR/docs/std/index.html" s3://ziglang.org/documentation/master/std/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
+$S3CMD put -P --no-mime-magic "../$ZIGDIR/docs/std/data.js" s3://ziglang.org/documentation/master/std/data.js --add-header="Cache-Control: max-age=0, must-revalidate"
+$S3CMD put -P --no-mime-magic "../$ZIGDIR/docs/std/main.js" s3://ziglang.org/documentation/master/std/main.js --add-header="Cache-Control: max-age=0, must-revalidate"
+
+$S3CMD put -P --no-mime-magic ci/out/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
+$S3CMD put -P --no-mime-magic ci/out/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
+
+cp ci/out/index.json data/releases.json
+git add data/releases.json
+git commit -m "CI: update releases.json"
+git push origin hugo-redesign