From e49edc7cdb73463329ec2ec70dc59c79433060ac Mon Sep 17 00:00:00 2001 From: Ben Henning Date: Wed, 21 Jul 2021 22:45:35 -0700 Subject: [PATCH] [RunAllTests] Fix #3510: Set up Android SDK hermetically in GitHub Actions to workaround #3024 (#3513) * Refactor common build env setup to local action. This attempts to move all common build environment setup bits to a common local action using GitHub composite actions per https://github.community/t/path-to-action-in-the-same-repository-as-workflow/16952/2, https://github.blog/changelog/2020-08-07-github-actions-composite-run-steps/, and https://docs.github.com/en/actions/creating-actions/creating-a-composite-run-steps-action. This does not include any actual changes to the CI environment yet other than temporarily disabling the other expensive checks as part of testing these changes. * Update local action to only use shell commands. * Set up JDK 9 manually for build tests. * Fix Java version checking. OpenJDK can use "9.0" as the version string instead of "1.9". * Fix Java exit from version check. * Fix incorrect check. * Add Bazel setup. * Fix syntax error & add Bazel version check. * Add SDK setup bits. * Avoid mv warning/error. * Add debug lines. * Attempt to fix pathing for new SDK. * Attempt to workaround unexpected failure from sdkmanager. * Add debug lines. * Try to run license bit in a subshell to avoid exit * Add codeowners, TODO, cleanup, and integrate. This integrates the new action with Bazel tests in addition to the build. While the tests work fine with the current build tools, we should be consistent in how the environment is set up for CI workflows. * Make local action name clearer. The new name specifically clarifies why this action isn't needed for the Bazel-only script runs (since they don't require the Android SDK bits). --- .github/CODEOWNERS | 1 + .../action.yml | 103 ++++++++++++++++++ .github/workflows/build_tests.yml | 35 ++---- .github/workflows/unit_tests.yml | 38 ++----- 4 files changed, 118 insertions(+), 59 deletions(-) create mode 100644 .github/actions/set-up-android-bazel-build-environment/action.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fd4ee2cd2bb..7c464fc0cfb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -43,6 +43,7 @@ gradlew.bat @BenHenning *.secret @BenHenning # CI configuration. +/.github/actions/ @BenHenning /.github/workflows/ @BenHenning # All tests. diff --git a/.github/actions/set-up-android-bazel-build-environment/action.yml b/.github/actions/set-up-android-bazel-build-environment/action.yml new file mode 100644 index 00000000000..13cc5623a0d --- /dev/null +++ b/.github/actions/set-up-android-bazel-build-environment/action.yml @@ -0,0 +1,103 @@ +# Contains common operations to set up a hermetic Android + Bazel build environment for Oppia +# Android CI workflows. Action prerequisites: +# - JDK 9 must be installed & set as the default version via JAVA_HOME +# - Bazel must be installed, in the path, and be version 4.0.0 + +# TODO(#1861): Revert SDK pinning for improved CI performance once Bazel is sufficiently stable that +# we can rely on the automatic SDK provided by GitHub's CI environment. + +name: Build environment setup +description: Sets up the Android & Bazel build environments for Oppia Android +runs: + using: composite + steps: + - name: Check Java version & path + run: | + which java + java -version + echo "Java home: $JAVA_HOME" + $JAVA_HOME/bin/java -version + + # Verify that the correct version of Java is installed. + java -version 2>&1 | grep -q -E "1.9|9.0" + HAS_CORRECT_JAVA_VERSION=$(echo $?) + if [[ "$HAS_CORRECT_JAVA_VERSION" == 1 ]] ; then + echo "Expected Java 9 to be installed" + exit 1 + fi + shell: bash + + - name: Verify Bazel version + run: | + bazel --version | grep -q 4.0.0 + HAS_CORRECT_BAZEL_VERSION=$(echo $?) + if [[ "$HAS_CORRECT_JAVA_VERSION" == 1 ]] ; then + echo "Expected Bazel 4.0.0 to be installed" + exit 1 + fi + shell: bash + + - name: Set up new Android SDK directory + run: | + mkdir -p $HOME/android-sdk + shell: bash + + - name: Download cmdline tools + run: | + wget https://dl.google.com/android/repository/commandlinetools-linux-7302050_latest.zip + cp ./commandlinetools-linux-7302050_latest.zip $HOME/android-sdk + cd $HOME/android-sdk + unzip commandlinetools-linux-7302050_latest.zip + mkdir tools + cd cmdline-tools/ + mv -i * ../tools && mv ../tools . + cd .. + echo "ANDROID_HOME=$HOME/android-sdk" >> $GITHUB_ENV + shell: bash + + - name: Verify updated ANDROID_HOME + run: | + echo "Using updated Android SDK home: $ANDROID_HOME" + shell: bash + + - name: Accept SDK licenses + run: | + # For some reason, sdkmanager returns an error code when licenses are accepted. Run in a + # sub-shell so that the CI workflow doesn't fail. + echo $(yes | $ANDROID_HOME/cmdline-tools/tools/bin/sdkmanager --licenses) + shell: bash + + - name: Install platform tools + run: | + $ANDROID_HOME/cmdline-tools/tools/bin/sdkmanager --install "platform-tools" + shell: bash + + - name: Install SDK 28 + run: | + $ANDROID_HOME/cmdline-tools/tools/bin/sdkmanager --install "platforms;android-28" + shell: bash + + - name: Install build tools 29.0.2 + run: | + $ANDROID_HOME/cmdline-tools/tools/bin/sdkmanager --install "build-tools;29.0.2" + shell: bash + + - name: Configure Bazel to use JDK 9 for building + run: | + echo "build --java_language_version=9" >> $HOME/.bazelrc + shell: bash + + - name: Configure Bazel to use specific sandbox tmpfs + run: | + echo "build --enable_platform_specific_config" >> $HOME/.bazelrc + echo "build:linux --sandbox_tmpfs_path=/tmp" >> $HOME/.bazelrc + shell: bash + + - name: Set up Oppia Bazel Android Tools + run: | + mkdir $HOME/opensource + cd $HOME/opensource + git clone https://github.com/oppia/oppia-bazel-tools + echo build --override_repository=android_tools="$(cd "$(dirname "$HOME/opensource/oppia-bazel-tools")"; pwd)/$(basename "$HOME/opensource/oppia-bazel-tools")" >> $HOME/.bazelrc + echo build --android_databinding_use_androidx >> $HOME/.bazelrc + shell: bash diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml index a48569cf3b7..9e653e266bc 100644 --- a/.github/workflows/build_tests.yml +++ b/.github/workflows/build_tests.yml @@ -28,12 +28,13 @@ jobs: with: java-version: 9 - - name: Check Java version & path - run: | - which java - java -version - echo "Java home: $JAVA_HOME" - $JAVA_HOME/bin/java -version + - name: Set up Bazel + uses: abhinavsingh/setup-bazel@v3 + with: + version: 4.0.0 + + - name: Set up build environment + uses: ./.github/actions/set-up-android-bazel-build-environment # For reference on this & the later cache actions, see: # https://github.com/actions/cache/issues/239#issuecomment-606950711 & @@ -74,20 +75,6 @@ jobs: rm -rf $EXPANDED_BAZEL_CACHE_PATH fi - - name: Set up Bazel - uses: abhinavsingh/setup-bazel@v3 - with: - version: 4.0.0 - - - name: Configure Bazel to use JDK 9 for building - run: | - echo "build --java_language_version=9" >> $HOME/.bazelrc - - - name: Configure Bazel to use specific sandbox tmpfs - run: | - echo "build --enable_platform_specific_config" >> $HOME/.bazelrc - echo "build:linux --sandbox_tmpfs_path=/tmp" >> $HOME/.bazelrc - - name: Configure Bazel to use a local cache env: BAZEL_CACHE_DIR: ${{ env.CACHE_DIRECTORY }} @@ -100,14 +87,6 @@ jobs: - name: Check Bazel environment run: bazel info - - name: Set up Oppia Bazel Android Tools - run: | - mkdir $HOME/opensource - cd $HOME/opensource - git clone https://github.com/oppia/oppia-bazel-tools - echo build --override_repository=android_tools="$(cd "$(dirname "$HOME/opensource/oppia-bazel-tools")"; pwd)/$(basename "$HOME/opensource/oppia-bazel-tools")" >> $HOME/.bazelrc - echo build --android_databinding_use_androidx >> $HOME/.bazelrc - # See https://git-secret.io/installation for details on installing git-secret. Note that the # apt-get method isn't used since it's much slower to update & upgrade apt before installation # versus just directly cloning & installing the project. Further, the specific version diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 3a26c691371..3813bf8d6f1 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -72,12 +72,13 @@ jobs: with: java-version: 9 - - name: Check Java version & path - run: | - which java - java -version - echo "Java home: $JAVA_HOME" - $JAVA_HOME/bin/java -version + - name: Set up Bazel + uses: abhinavsingh/setup-bazel@v3 + with: + version: 4.0.0 + + - name: Set up build environment + uses: ./.github/actions/set-up-android-bazel-build-environment - name: Compute test caching bucket env: @@ -128,20 +129,6 @@ jobs: rm -rf $EXPANDED_BAZEL_CACHE_PATH fi - - name: Set up Bazel - uses: abhinavsingh/setup-bazel@v3 - with: - version: 4.0.0 - - - name: Configure Bazel to use JDK 9 for building - run: | - echo "build --java_language_version=9" >> $HOME/.bazelrc - - - name: Configure Bazel to use specific sandbox tmpfs - run: | - echo "build --enable_platform_specific_config" >> $HOME/.bazelrc - echo "build:linux --sandbox_tmpfs_path=/tmp" >> $HOME/.bazelrc - - name: Configure Bazel to use a local cache env: BAZEL_CACHE_DIR: ${{ env.CACHE_DIRECTORY }} @@ -151,17 +138,6 @@ jobs: echo "build --disk_cache=$EXPANDED_BAZEL_CACHE_PATH" >> $HOME/.bazelrc shell: bash - - name: Check Bazel environment - run: bazel info - - - name: Set up Oppia Bazel Android Tools - run: | - mkdir $HOME/opensource - cd $HOME/opensource - git clone https://github.com/oppia/oppia-bazel-tools - echo build --override_repository=android_tools="$(cd "$(dirname "$HOME/opensource/oppia-bazel-tools")"; pwd)/$(basename "$HOME/opensource/oppia-bazel-tools")" >> $HOME/.bazelrc - echo build --android_databinding_use_androidx >> $HOME/.bazelrc - # See explanation in bazel_build_app for how this is installed. - name: Install git-secret (non-fork only) if: ${{ env.ENABLE_CACHING == 'true' && ((github.ref == 'refs/heads/develop' && github.event_name == 'push') || (github.event.pull_request.head.repo.full_name == 'oppia/oppia-android')) }}