ci: stop assuming that azure will complete last

Each of the 3 CI services now trigger a sr.ht build via the on_master_success script. The sr.ht build checks if all builds have completed successfully by trying to download the JSON file for the particular version. If all builds have completed successfully then this sr.ht job will update the download page.
2019-11-03 14:00:28 -05:00 · 2019-11-03 14:00:28 -05:00 · 5001faa690
parent 10046f9a52
commit 5001faa690
11 changed files with 142 additions and 64 deletions
--- a/.builds/freebsd.yml
+++ b/.builds/freebsd.yml
@ -3,8 +3,11 @@ packages:
  - cmake
  - py27-s3cmd
  - wget
+  - curl
+  - jq
 secrets:
  - 6c60aaee-92e7-4e7d-812c-114817689b4d
+  - 2cd8c203-ba06-4504-be49-1e2b9b71efdd
 sources:
  - https://github.com/ziglang/zig
 tasks:
--- a/ci/azure/linux_script
+++ b/ci/azure/linux_script
@ -25,6 +25,7 @@ cd build
 cmake .. -DCMAKE_BUILD_TYPE=Release
 make -j2 install
 ./zig build test -Denable-qemu
+VERSION="$(./zig version)"

 if [ "${BUILD_REASON}" != "PullRequest" ]; then
  ARTIFACTSDIR="$BUILDDIR/artifacts"
@ -44,7 +45,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
  echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE

  s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
-  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/linux-$VERSION.json"
+  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-linux-$VERSION.json"

  # `set -x` causes these variables to be mangled.
  # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
@ -52,4 +53,5 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
  echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
  echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM"
  echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE"
+  echo "##vso[task.setvariable variable=version;isOutput=true]$VERSION"
 fi
--- a/ci/azure/macos_script
+++ b/ci/azure/macos_script
@ -104,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
  echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE

  s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
-  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/macos-$VERSION.json"
+  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-macos-$VERSION.json"

  # `set -x` causes these variables to be mangled.
  # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
--- a/ci/azure/on_master_success
+++ b/ci/azure/on_master_success
@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -x
+set -e
+
+sudo apt-get update -y
+sudo apt-get install -y curl jq
+
+OAUTH_TOKEN="$(cat "$DOWNLOADSECUREFILE_SECUREFILEPATH")"
+./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN"
--- a/ci/azure/pipelines.yml
+++ b/ci/azure/pipelines.yml
@ -55,18 +55,10 @@ jobs:
  pool:
    vmImage: 'ubuntu-16.04'
  variables:
-    macos_tarball: $[ dependencies.BuildMacOS.outputs['main.tarball'] ]
-    macos_shasum: $[ dependencies.BuildMacOS.outputs['main.shasum'] ]
-    macos_bytesize: $[ dependencies.BuildMacOS.outputs['main.bytesize'] ]
-    linux_tarball: $[ dependencies.BuildLinux.outputs['main.tarball'] ]
-    linux_shasum: $[ dependencies.BuildLinux.outputs['main.shasum'] ]
-    linux_bytesize: $[ dependencies.BuildLinux.outputs['main.bytesize'] ]
-    windows_tarball: $[ dependencies.BuildWindows.outputs['main.tarball'] ]
-    windows_shasum: $[ dependencies.BuildWindows.outputs['main.shasum'] ]
-    windows_bytesize: $[ dependencies.BuildWindows.outputs['main.bytesize'] ]
+    version: $[ dependencies.BuildLinux.outputs['main.version'] ]
  steps:
  - task: DownloadSecureFile@1
    inputs:
-      secureFile: s3cfg
-  - script: ci/azure/update_download_page
-    displayName: 'Update download page'
+      secureFile: oauth_token
+  - script: ci/azure/on_master_success
+    displayName: 'master branch success hook'
--- a/ci/azure/update_download_page
+++ b/ci/azure/update_download_page
@ -1,47 +0,0 @@
-#!/bin/sh
-
-set -x
-set -e
-
-SRCDIR=$(pwd)
-rm -rf .git
-sudo apt-get update -y
-sudo apt-get install -y s3cmd curl jq
-
-cd "$HOME"
-wget "https://ziglang.org/builds/$LINUX_TARBALL"
-tar xf $LINUX_TARBALL
-ZIGDIR=$(basename -s .tar.xz $LINUX_TARBALL)
-ZIG="$ZIGDIR/zig"
-LANGREF="$ZIGDIR/langref.html"
-VERSION=$($ZIG version)
-SRCTARBALLDIR="zig-$VERSION"
-export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
-mv "$SRCDIR" "$SRCTARBALLDIR"
-tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR"
-export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1)
-export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL)
-
-# the aarch64 build has to be there too
-AARCH64_LINUX_JSON=$(curl --fail "https://ziglang.org/builds/aarch64-linux-$VERSION.json" || exit 1)
-export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)"
-export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)"
-export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)"
-
-# the freebsd build has to be there too
-FREEBSD_JSON=$(curl --fail "https://ziglang.org/builds/freebsd-$VERSION.json" || exit 1)
-export FREEBSD_TARBALL="$(echo "$FREEBSD_JSON" | jq .tarball -r)"
-export FREEBSD_BYTESIZE="$(echo "$FREEBSD_JSON" | jq .size -r)"
-export FREEBSD_SHASUM="$(echo "$FREEBSD_JSON" | jq .shasum -r)"
-
-git clone https://github.com/ziglang/www.ziglang.org --depth 1
-cd www.ziglang.org
-export MASTER_DATE=$(date +%Y-%m-%d)
-env
-"../$ZIG" run update-download-page.zig
-
-mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"
-s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
-s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
-s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
-s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
--- a/ci/azure/windows_upload
+++ b/ci/azure/windows_upload
@ -30,7 +30,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
  echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE

  s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
-  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/windows-$VERSION.json"
+  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-windows-$VERSION.json"

  # `set -x` causes these variables to be mangled.
  # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
--- a/ci/drone/linux_script
+++ b/ci/drone/linux_script
@ -8,7 +8,7 @@ BUILDDIR="$(pwd)"
 DISTDIR="$(pwd)/dist"

 apk update
-apk add py3-pip xz perl-utils
+apk add py3-pip xz perl-utils jq curl
 pip3 install s3cmd

 # Make the `zig version` number consistent.
@ -60,4 +60,8 @@ if [ -z "$DRONE_PULL_REQUEST" ]; then

  s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$TRIPLEARCH-linux-$VERSION.json"
+  if [ "$GITBRANCH" = "master" ]; then 
+    cd "$BUILDDIR"
+    ./ci/srht/on_master_success "$VERSION" "$SRHT_OAUTH_TOKEN"
+  fi
 fi
--- a/ci/srht/freebsd_script
+++ b/ci/srht/freebsd_script
@ -85,5 +85,11 @@ if [ -f ~/.s3cfg ]; then
  echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE

  s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
-  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/freebsd-$VERSION.json"
+  s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-freebsd-$VERSION.json"
+
+  if [ "$GITBRANCH" = "master" ]; then 
+    OAUTH_TOKEN="$(cat ~/.oauth_token)"
+    cd "$ZIGDIR"
+    ./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN"
+  fi
 fi
--- a/ci/srht/on_master_success
+++ b/ci/srht/on_master_success
@ -0,0 +1,33 @@
+#!/bin/sh
+
+# This script must run on a lot of different platforms.
+# It assumes that the following things are installed:
+# * curl
+# * jq
+
+set -x
+set -e
+
+VERSION="$1"
+OAUTH_TOKEN="$2"
+YML_FILE="tmp.yml"
+
+echo "image: ubuntu/xenial" >> "$YML_FILE"
+echo "packages:" >> "$YML_FILE"
+echo "  - s3cmd" >> "$YML_FILE"
+echo "  - curl" >> "$YML_FILE"
+echo "  - jq" >> "$YML_FILE"
+echo "secrets:" >> "$YML_FILE"
+echo "  - 6c60aaee-92e7-4e7d-812c-114817689b4d" >> "$YML_FILE"
+echo "sources:" >> "$YML_FILE"
+echo "  - https://github.com/ziglang/zig" >> "$YML_FILE"
+echo "tasks:" >> "$YML_FILE"
+echo "  - build: cd zig && ./ci/srht/update_download_page $VERSION" >> "$YML_FILE"
+
+jq <$YML_FILE -sR '{
+      "manifest": .,
+    }' | curl \
+  -H Authorization:"token $OAUTH_TOKEN" \
+  -H Content-Type:application/json \
+  -X POST \
+  -d @- "https://builds.sr.ht/api/jobs"
--- a/ci/srht/update_download_page
+++ b/ci/srht/update_download_page
@ -0,0 +1,75 @@
+#!/bin/sh
+
+set -x
+set -e
+
+VERSION="$1"
+SRCDIR="$(pwd)"
+NATIVE_TARBALL="zig-linux-$(uname -m)-$VERSION.tar.xz"
+
+# Check for all the builds being completed. It's expected that this script is run several times
+# before they are all complete.
+AARCH64_LINUX_JSON_URL="https://ziglang.org/builds/aarch64-linux-$VERSION.json"
+X86_64_LINUX_JSON_URL="https://ziglang.org/builds/x86_64-linux-$VERSION.json"
+X86_64_WINDOWS_JSON_URL="https://ziglang.org/builds/x86_64-windows-$VERSION.json"
+X86_64_MACOS_JSON_URL="https://ziglang.org/builds/x86_64-macos-$VERSION.json"
+X86_64_FREEBSD_JSON_URL="https://ziglang.org/builds/x86_64-freebsd-$VERSION.json"
+
+# If any of these fail, it's not really this job failing; rather we have detected
+# that this job will be called again later when other jobs have completed.
+curl --fail -I "$AARCH64_LINUX_JSON_URL" || exit 0
+curl --fail -I "$X86_64_LINUX_JSON_URL" || exit 0
+curl --fail -I "$X86_64_WINDOWS_JSON_URL" || exit 0
+curl --fail -I "$X86_64_MACOS_JSON_URL" || exit 0
+curl --fail -I "$X86_64_FREEBSD_JSON_URL" || exit 0
+
+rm -rf .git
+
+cd "$HOME"
+wget "https://ziglang.org/builds/$NATIVE_TARBALL"
+tar xf "$NATIVE_TARBALL"
+ZIGDIR=$(basename -s .tar.xz $NATIVE_TARBALL)
+ZIG="$ZIGDIR/zig"
+LANGREF="$ZIGDIR/langref.html"
+SRCTARBALLDIR="zig-$VERSION"
+export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
+mv "$SRCDIR" "$SRCTARBALLDIR"
+tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR"
+export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1)
+export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL)
+
+X86_64_WINDOWS_JSON=$(curl --fail "X86_64_WINDOWS_JSON_URL" || exit 1)
+export X86_64_WINDOWS_TARBALL="$(echo "$X86_64_WINDOWS_JSON" | jq .tarball -r)"
+export X86_64_WINDOWS_BYTESIZE="$(echo "$X86_64_WINDOWS_JSON" | jq .size -r)"
+export X86_64_WINDOWS_SHASUM="$(echo "$X86_64_WINDOWS_JSON" | jq .shasum -r)"
+
+X86_64_MACOS_JSON=$(curl --fail "X86_64_MACOS_JSON_URL" || exit 1)
+export X86_64_MACOS_TARBALL="$(echo "$X86_64_MACOS_JSON" | jq .tarball -r)"
+export X86_64_MACOS_BYTESIZE="$(echo "$X86_64_MACOS_JSON" | jq .size -r)"
+export X86_64_MACOS_SHASUM="$(echo "$X86_64_MACOS_JSON" | jq .shasum -r)"
+
+X86_64_LINUX_JSON=$(curl --fail "X86_64_LINUX_JSON_URL" || exit 1)
+export X86_64_LINUX_TARBALL="$(echo "$X86_64_LINUX_JSON" | jq .tarball -r)"
+export X86_64_LINUX_BYTESIZE="$(echo "$X86_64_LINUX_JSON" | jq .size -r)"
+export X86_64_LINUX_SHASUM="$(echo "$X86_64_LINUX_JSON" | jq .shasum -r)"
+
+AARCH64_LINUX_JSON=$(curl --fail "AARCH64_LINUX_JSON_URL" || exit 1)
+export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)"
+export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)"
+export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)"
+
+X86_64_FREEBSD_JSON=$(curl --fail "X86_64_FREEBSD_JSON_URL" || exit 1)
+export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)"
+export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)"
+export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)"
+
+git clone https://github.com/ziglang/www.ziglang.org --depth 1
+cd www.ziglang.org
+export MASTER_DATE="$(date +%Y-%m-%d)"
+env
+"../$ZIG" run update-download-page.zig
+
+s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
+s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
+s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
+s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"