Scan all release pages (#49)

This patch ensures we scan all release pages, so the result is more accurate and avoids missing releases. This also adds a way to avoid rate limiting when testing locally. Signed-off-by: Dhi Aurrahman <[email protected]>
tetratelabs · Oct 18, 2023 · 1943579 · 1943579
1 parent 4eb5caf
commit 1943579
Show file tree

Hide file tree

Showing 4 changed files with 65 additions and 19 deletions.
diff --git a/bin/archive_release_version.sh b/bin/archive_release_version.sh
@@ -1,4 +1,4 @@
-#!/bin/sh -ue
+#!/usr/bin/env bash
 
 # Copyright 2021 Tetrate
 #
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -ue
+
 # This creates a directory archiving a GitHub release version for all available platforms.
 #  * The first parameter ($1) is the source GitHub repository to archive. Ex envoyproxy/envoy
 #  * The second parameter ($2) is the version. Ex "v1.18.3" or "v1.18.3_debug"
@@ -76,11 +78,33 @@ archive) carMode=extract ;;
 *) echo >&2 "invalid op ${op}" && exit 1 ;;
 esac
 
+curl="curl -fsSL"
+
+# A valid GitHub token to avoid rate limiting.
+githubToken=${GITHUB_TOKEN:-}
+# Prepare authorization header when performing request to api.github.com to avoid rate limiting, especially when testing locally.
+authorizationHeader="Authorization: Bearer ${githubToken}"
+
 # Setup defaults that make archival consistent between runs
 export TZ=UTC
-# ex. "2021-05-11T19:15:27Z" ->  "2021-05-11"
-RELEASE_DATE=$(curl -fsSL "https://api.github.com/repos/${sourceGitHubRepository}/releases"'?per_page=100' |
-  jq -er ".|map(select(.prerelease == false and .draft == false and .name ==\"${sourceVersion}\"))|first|.published_at" | cut -c1-10) || exit 1
+
+# Fetch the last page number of releases (example value: 7), so we can get all of the releases.
+# To get the last page, we send a HEAD request to "https://api.github.com/repos/${sourceGitHubRepository}/releases",
+# then "grep" the "link" header value.
+# Reference: https://docs.github.com/en/rest/guides/using-pagination-in-the-rest-api?apiVersion=2022-11-28#using-link-headers.
+lastReleasePage=$(${curl}I ${githubToken:+ -H "${authorizationHeader}"} "https://api.github.com/repos/${sourceGitHubRepository}/releases" |
+  grep -Eo 'page=[0-9]+' | awk 'NR==2' | cut -d'=' -f2) || exit 1
+
+RELEASE_DATE="null"
+for ((page = 1; page <= lastReleasePage; page++)); do
+  # ex. "2021-05-11T19:15:27Z" ->  "2021-05-11"
+  RELEASE_DATE=$(${curl} ${githubToken:+ -H "${authorizationHeader}"} -fsSL "https://api.github.com/repos/${sourceGitHubRepository}/releases"'?page='"${page}" |
+    jq -er ".|map(select(.prerelease == false and .draft == false and .name ==\"${sourceVersion}\"))|first|.published_at" | cut -c1-10) || exit 1
+  if [ "${RELEASE_DATE}" != "null" ]; then
+      break
+  fi
+done
+
 if [ "${RELEASE_DATE}" = "null" ]; then
   echo >&2 "version ${sourceVersion} has not yet been released" && exit 1
 fi

diff --git a/bin/car_envoy.sh b/bin/car_envoy.sh
@@ -1,4 +1,4 @@
-#!/bin/sh -ue
+#!/usr/bin/env bash
 
 # Copyright 2021 Tetrate
 #
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -ue
+
 # This is an Envoy® plug-in to extract_release_version.sh and called in a loop for each OS and
 # architecture for a given version. The result is a distribution directory including the `envoy` binary.
 #

diff --git a/bin/consolidate_release_versions.sh b/bin/consolidate_release_versions.sh
@@ -1,4 +1,4 @@
-#!/bin/sh -ue
+#!/usr/bin/env bash
 
 # Copyright 2021 Tetrate
 #
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -ue
+
 # This creates a json file including all archived releases of a GitHub Repository.
 # Notably, this rewrites tarballURLs as permalinks
 
@@ -22,6 +24,11 @@ curl --version >/dev/null
 jq --version >/dev/null
 curl="curl -fsSL"
 
+# A valid GitHub token to avoid rate limiting.
+githubToken=${GITHUB_TOKEN:-}
+# Prepare authorization header when performing request to api.github.com to avoid rate limiting, especially when testing locally.
+authorizationHeader="Authorization: Bearer ${githubToken}"
+
 githubRepo="${1?-githubRepo required ex tetratelabs/archive-envoy}"
 downloadBaseURL="${2?-downloadBaseURL required ex https://archive.tetratelabs.io/envoy/download}"
 case "${3:-0}" in
@@ -33,20 +40,30 @@ esac
 # This must match netlify.toml redirects
 redirectsTo="https://github.com/${githubRepo}/releases/download"
 
+# Fetch the last page number of releases (example value: 7), so we can get all of the releases.
+# To get the last page, we send a HEAD request to "https://api.github.com/repos/${githubRepo}/releases",
+# then "grep" the "link" header value.
+# Reference: https://docs.github.com/en/rest/guides/using-pagination-in-the-rest-api?apiVersion=2022-11-28#using-link-headers.
+lastReleasePage=$(${curl}I ${githubToken:+ -H "${authorizationHeader}"} "https://api.github.com/repos/${githubRepo}/releases" |
+  grep -Eo 'page=[0-9]+' | awk 'NR==2' | cut -d'=' -f2) || exit 1
+
 # archive all dists for the version, generating the envoy-versions.json format incrementally
 releaseVersions="{}"
-versions=$(${curl} "https://api.github.com/repos/${githubRepo}/releases?per_page=100" |
-  jq -er ".|map(select(.prerelease == false and .draft == false))|.[]|.name" | sort -n) || exit 1
-
-for version in ${versions}; do
-  # Exclusively handle debug
-  case ${version} in v[0-9]*[0-9]_debug) nextDebugVersion=1 ;; *) unset nextDebugVersion;; esac
-  [ "${debugVersion:-}" != "${nextDebugVersion:-}" ] && continue
-
-  versionsUrl="${redirectsTo}/${version}/envoy-${version}.json"
-  nextReleaseVersion=$(${curl} "${versionsUrl}" | sed "s~${redirectsTo}~${downloadBaseURL}~g") || exit 1
-  # merge the pending releaseVersions json to include the next one
-  releaseVersions=$(echo "${releaseVersions}" "${nextReleaseVersion}" | jq -Sse '.[0] * .[1]')
+
+for ((page = 1; page <= lastReleasePage; page++)); do
+  versions=$(${curl} ${githubToken:+ -H "${authorizationHeader}"} "https://api.github.com/repos/${githubRepo}/releases?page=${page}" |
+    jq -er ".|map(select(.prerelease == false and .draft == false))|.[]|.name" | sort -n) || exit 1
+
+  for version in ${versions}; do
+    # Exclusively handle debug.
+    case ${version} in v[0-9]*[0-9]_debug) nextDebugVersion=1 ;; *) unset nextDebugVersion;; esac
+    [ "${debugVersion:-}" != "${nextDebugVersion:-}" ] && continue
+
+    versionsUrl="${redirectsTo}/${version}/envoy-${version}.json"
+    nextReleaseVersion=$(${curl} "${versionsUrl}" | sed "s~${redirectsTo}~${downloadBaseURL}~g") || exit 1
+    # merge the pending releaseVersions json to include the next one.
+    releaseVersions=$(echo "${releaseVersions}" "${nextReleaseVersion}" | jq -Sse '.[0] * .[1]')
+  done
 done
 
 # reorder top-level keys so that versions appear before sha256sums

diff --git a/build.sh b/build.sh
@@ -1,4 +1,7 @@
-#!/bin/sh -uex
+#!/usr/bin/env bash
+
+set -uex
+
 downloadBaseURL="${1?-downloadBaseURL required ex https://archive.tetratelabs.io/envoy}"
 bin/consolidate_release_versions.sh tetratelabs/archive-envoy "${downloadBaseURL}" 0 >public/envoy-versions.json
 bin/consolidate_release_versions.sh tetratelabs/archive-envoy "${downloadBaseURL}" 1 >public/envoy-versions_debug.json