|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 4 | +# or more contributor license agreements. See the NOTICE file |
| 5 | +# distributed with this work for additional information |
| 6 | +# regarding copyright ownership. The ASF licenses this file |
| 7 | +# to you under the Apache License, Version 2.0 (the |
| 8 | +# "License"); you may not use this file except in compliance |
| 9 | +# with the License. You may obtain a copy of the License at |
| 10 | +# |
| 11 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 12 | +# |
| 13 | +# Unless required by applicable law or agreed to in writing, software |
| 14 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 15 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 16 | +# See the License for the specific language governing permissions and |
| 17 | +# limitations under the License. |
| 18 | + |
| 19 | +# This script is called from the Jenkinsfile, which ultimately runs |
| 20 | +# the CI through Yetus. |
| 21 | +# We use Ubuntu Focal as the main platform for building Hadoop, thus |
| 22 | +# it runs for all the PRs. Additionally, we also ensure that |
| 23 | +# Hadoop builds across the supported platforms whenever there's a change |
| 24 | +# in any of the C++ files, C++ build files or platform changes. |
| 25 | + |
| 26 | +## @description Check if the given extension is related to C++ |
| 27 | +## @param seeking |
| 28 | +## @return 0 if yes |
| 29 | +## @return 1 if no |
| 30 | +is_cpp_extension() { |
| 31 | + local cpp_extensions=("cc" "cpp" "h" "hpp") |
| 32 | + local seeking=$1 |
| 33 | + |
| 34 | + for element in "${cpp_extensions[@]}"; do |
| 35 | + if [[ $element == "$seeking" ]]; then |
| 36 | + return 0 |
| 37 | + fi |
| 38 | + done |
| 39 | + return 1 |
| 40 | +} |
| 41 | + |
| 42 | +## @description Check if the given relative path corresponds to |
| 43 | +## change in platform files |
| 44 | +## @param in_path |
| 45 | +## @return 0 if yes |
| 46 | +## @return 1 if no |
| 47 | +is_platform_change() { |
| 48 | + declare in_path |
| 49 | + in_path="${SOURCEDIR}"/"${1}" |
| 50 | + |
| 51 | + for path in "${SOURCEDIR}"/dev-support/docker/Dockerfile* "${SOURCEDIR}"/dev-support/docker/pkg-resolver/*.json; do |
| 52 | + if [ "${in_path}" == "${path}" ]; then |
| 53 | + echo "Found C++ platform related changes in ${in_path}" |
| 54 | + return 0 |
| 55 | + fi |
| 56 | + done |
| 57 | + return 1 |
| 58 | +} |
| 59 | + |
| 60 | +## @description Checks if the given path corresponds to a change |
| 61 | +## in C++ files or related to C++ build system |
| 62 | +## @param path |
| 63 | +## @return 0 if yes |
| 64 | +## @return 1 if no |
| 65 | +is_cpp_change() { |
| 66 | + shopt -s nocasematch |
| 67 | + |
| 68 | + local path=$1 |
| 69 | + declare filename |
| 70 | + filename=$(basename -- "${path}") |
| 71 | + extension=${filename##*.} |
| 72 | + |
| 73 | + if is_cpp_extension "${extension}"; then |
| 74 | + echo "Found C++ changes in ${path}" |
| 75 | + return 0 |
| 76 | + fi |
| 77 | + |
| 78 | + if [[ $filename =~ CMakeLists\.txt ]]; then |
| 79 | + echo "Found C++ build related changes in ${path}" |
| 80 | + return 0 |
| 81 | + fi |
| 82 | + return 1 |
| 83 | +} |
| 84 | + |
| 85 | +## @description Check if the CI needs to be run - CI will always run if |
| 86 | +## IS_OPTIONAL is 0, or if there's any change in |
| 87 | +## C++/C++ build/platform |
| 88 | +## @return 0 if yes |
| 89 | +## @return 1 if no |
| 90 | +function check_ci_run() { |
| 91 | + # Get the first commit of this PR relative to the trunk branch |
| 92 | + firstCommitOfThisPr=$(git --git-dir "${SOURCEDIR}/.git" rev-parse origin/trunk) |
| 93 | + |
| 94 | + # Loop over the paths of all the changed files and check if the criteria |
| 95 | + # to run the CI has been satisfied |
| 96 | + for path in $(git --git-dir "${SOURCEDIR}/.git" diff --name-only "${firstCommitOfThisPr}" HEAD); do |
| 97 | + if is_cpp_change "${path}"; then |
| 98 | + return 0 |
| 99 | + fi |
| 100 | + |
| 101 | + if is_platform_change "${path}"; then |
| 102 | + return 0 |
| 103 | + fi |
| 104 | + done |
| 105 | + |
| 106 | + # We must run the CI if it's not optional |
| 107 | + if [ "$IS_OPTIONAL" -eq 0 ]; then |
| 108 | + return 0 |
| 109 | + fi |
| 110 | + return 1 |
| 111 | +} |
| 112 | + |
| 113 | +## @description Run the CI using YETUS |
| 114 | +function run_ci() { |
| 115 | + TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" |
| 116 | + |
| 117 | + # this must be clean for every run |
| 118 | + if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then |
| 119 | + rm -rf "${WORKSPACE:?}/${PATCHDIR}" |
| 120 | + fi |
| 121 | + mkdir -p "${WORKSPACE}/${PATCHDIR}" |
| 122 | + |
| 123 | + # if given a JIRA issue, process it. If CHANGE_URL is set |
| 124 | + # (e.g., Github Branch Source plugin), process it. |
| 125 | + # otherwise exit, because we don't want Hadoop to do a |
| 126 | + # full build. We wouldn't normally do this check for smaller |
| 127 | + # projects. :) |
| 128 | + if [[ -n "${JIRA_ISSUE_KEY}" ]]; then |
| 129 | + YETUS_ARGS+=("${JIRA_ISSUE_KEY}") |
| 130 | + elif [[ -z "${CHANGE_URL}" ]]; then |
| 131 | + echo "Full build skipped" >"${WORKSPACE}/${PATCHDIR}/report.html" |
| 132 | + exit 0 |
| 133 | + fi |
| 134 | + |
| 135 | + YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") |
| 136 | + |
| 137 | + # where the source is located |
| 138 | + YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") |
| 139 | + |
| 140 | + # our project defaults come from a personality file |
| 141 | + YETUS_ARGS+=("--project=hadoop") |
| 142 | + YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh") |
| 143 | + |
| 144 | + # lots of different output formats |
| 145 | + YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") |
| 146 | + YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt") |
| 147 | + YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html") |
| 148 | + |
| 149 | + # enable writing back to Github |
| 150 | + YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}") |
| 151 | + |
| 152 | + # enable writing back to ASF JIRA |
| 153 | + YETUS_ARGS+=("--jira-password=${JIRA_PASSWORD}") |
| 154 | + YETUS_ARGS+=("--jira-user=${JIRA_USER}") |
| 155 | + |
| 156 | + # auto-kill any surefire stragglers during unit test runs |
| 157 | + YETUS_ARGS+=("--reapermode=kill") |
| 158 | + |
| 159 | + # set relatively high limits for ASF machines |
| 160 | + # changing these to higher values may cause problems |
| 161 | + # with other jobs on systemd-enabled machines |
| 162 | + YETUS_ARGS+=("--proclimit=5500") |
| 163 | + YETUS_ARGS+=("--dockermemlimit=22g") |
| 164 | + |
| 165 | + # -1 spotbugs issues that show up prior to the patch being applied |
| 166 | + YETUS_ARGS+=("--spotbugs-strict-precheck") |
| 167 | + |
| 168 | + # rsync these files back into the archive dir |
| 169 | + YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml") |
| 170 | + |
| 171 | + # URL for user-side presentation in reports and such to our artifacts |
| 172 | + # (needs to match the archive bits below) |
| 173 | + YETUS_ARGS+=("--build-url-artifacts=artifact/out") |
| 174 | + |
| 175 | + # plugins to enable |
| 176 | + YETUS_ARGS+=("--plugins=all") |
| 177 | + |
| 178 | + # don't let these tests cause -1s because we aren't really paying that |
| 179 | + # much attention to them |
| 180 | + YETUS_ARGS+=("--tests-filter=checkstyle") |
| 181 | + |
| 182 | + # run in docker mode and specifically point to our |
| 183 | + # Dockerfile since we don't want to use the auto-pulled version. |
| 184 | + YETUS_ARGS+=("--docker") |
| 185 | + YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") |
| 186 | + YETUS_ARGS+=("--mvn-custom-repos") |
| 187 | + |
| 188 | + # effectively treat dev-suport as a custom maven module |
| 189 | + YETUS_ARGS+=("--skip-dirs=dev-support") |
| 190 | + |
| 191 | + # help keep the ASF boxes clean |
| 192 | + YETUS_ARGS+=("--sentinel") |
| 193 | + |
| 194 | + # test with Java 8 and 11 |
| 195 | + YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") |
| 196 | + YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") |
| 197 | + YETUS_ARGS+=("--multijdktests=compile") |
| 198 | + |
| 199 | + # custom javadoc goals |
| 200 | + YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork") |
| 201 | + |
| 202 | + # write Yetus report as GitHub comment (YETUS-1102) |
| 203 | + YETUS_ARGS+=("--github-write-comment") |
| 204 | + YETUS_ARGS+=("--github-use-emoji-vote") |
| 205 | + |
| 206 | + "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" |
| 207 | +} |
| 208 | + |
| 209 | +# Check if the CI needs to be run, if so, do so :) |
| 210 | +if check_ci_run; then |
| 211 | + run_ci |
| 212 | +fi |
0 commit comments