diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 1ed6e62e29d3..8eb4cf4edf2c 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -22,7 +22,6 @@ ENABLE_GCS=OFF ENABLE_S3=OFF ENABLE_HDFS=OFF ENABLE_ABFS=OFF -ENABLE_EP_CACHE=OFF ENABLE_VCPKG=OFF RUN_SETUP_SCRIPT=ON VELOX_REPO="" @@ -96,10 +95,6 @@ do ENABLE_ABFS=("${arg#*=}") shift # Remove argument name from processing ;; - --enable_ep_cache=*) - ENABLE_EP_CACHE=("${arg#*=}") - shift # Remove argument name from processing - ;; --enable_vcpkg=*) ENABLE_VCPKG=("${arg#*=}") shift # Remove argument name from processing @@ -192,7 +187,7 @@ function build_velox { cd $GLUTEN_DIR/ep/build-velox/src # When BUILD_TESTS is on for gluten cpp, we need turn on VELOX_BUILD_TEST_UTILS via build_test_utils. ./build_velox.sh --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \ - --enable_abfs=$ENABLE_ABFS --enable_ep_cache=$ENABLE_EP_CACHE --build_test_utils=$BUILD_TESTS \ + --enable_abfs=$ENABLE_ABFS --build_test_utils=$BUILD_TESTS \ --build_tests=$BUILD_VELOX_TESTS --build_benchmarks=$BUILD_VELOX_BENCHMARKS --num_threads=$NUM_THREADS \ --velox_home=$VELOX_HOME } diff --git a/dev/ci-velox-buildshared-centos-8.sh b/dev/ci-velox-buildshared-centos-8.sh index ff9a62b798b5..3f7197912a98 100755 --- a/dev/ci-velox-buildshared-centos-8.sh +++ b/dev/ci-velox-buildshared-centos-8.sh @@ -3,5 +3,5 @@ set -e source /opt/rh/gcc-toolset-11/enable -./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --enable_ep_cache=OFF --build_tests=ON \ +./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON \ --build_examples=ON --build_benchmarks=ON diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md index a0af0cc75d02..48bca9a6d31e 100644 --- a/docs/get-started/Velox.md +++ b/docs/get-started/Velox.md @@ -61,11 +61,11 @@ First time build for all supported spark versions. ./dev/buildbundle-veloxbe.sh ``` -After a complete build, if only some gluten code is changed, you can use the following command to skip building velox/arrow and +After a complete build, if only some gluten code is changed, you can use the following command to skip building arrow and setting up build dependencies. ```bash -./dev/buildbundle-veloxbe.sh --enable_ep_cache=ON --build_arrow=OFF --run_setup_script=OFF +./dev/buildbundle-veloxbe.sh --build_arrow=OFF --run_setup_script=OFF ``` **For aarch64 build** diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md index c68a5f463317..d9c3beaab9dc 100644 --- a/docs/get-started/build-guide.md +++ b/docs/get-started/build-guide.md @@ -22,7 +22,6 @@ Please set them via `--`, e.g. `--build_type=Release`. | enable_gcs | Build with GCS support. | OFF | | enable_hdfs | Build with HDFS support. | OFF | | enable_abfs | Build with ABFS support. | OFF | -| enable_ep_cache | Enable caching for external project build (Velox). | OFF | | enable_vcpkg | Enable vcpkg for static build. | OFF | | run_setup_script | Run setup script to install Velox dependencies. | ON | | velox_repo | Specify your own Velox repo to build. | "" | @@ -45,7 +44,6 @@ Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`. | enable_hdfs | Build Velox with HDFS support. | OFF | | enable_abfs | Build Velox with ABFS support. | OFF | | run_setup_script | Run setup script to install Velox dependencies before build. | ON | -| enable_ep_cache | Enable and reuse cache of Velox build. | OFF | | build_test_utils | Build Velox with cmake arg -DVELOX_BUILD_TEST_UTILS=ON if ON. | OFF | | build_tests | Build Velox test. | OFF | | build_benchmarks | Build Velox benchmarks. | OFF | diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index a5f009833a51..67a540e76fb3 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -26,7 +26,6 @@ ENABLE_HDFS=OFF ENABLE_ABFS=OFF BUILD_TYPE=release VELOX_HOME="" -ENABLE_EP_CACHE=OFF # May be deprecated in Gluten build. ENABLE_BENCHMARK=OFF # May be deprecated in Gluten build. @@ -65,10 +64,6 @@ for arg in "$@"; do BUILD_TYPE=("${arg#*=}") shift # Remove argument name from processing ;; - --enable_ep_cache=*) - ENABLE_EP_CACHE=("${arg#*=}") - shift # Remove argument name from processing - ;; --build_test_utils=*) BUILD_TEST_UTILS=("${arg#*=}") shift # Remove argument name from processing @@ -167,40 +162,6 @@ function compile { fi } -function get_build_summary { - COMMIT_HASH=$1 - # Ideally all script arguments should be put into build summary. - # ENABLE_EP_CACHE is excluded. Thus, in current build with ENABLE_EP_CACHE=ON, we can use EP cache - # from last build with ENABLE_EP_CACHE=OFF, - echo "ENABLE_S3=$ENABLE_S3,ENABLE_GCS=$ENABLE_GCS,ENABLE_HDFS=$ENABLE_HDFS,ENABLE_ABFS=$ENABLE_ABFS,\ -BUILD_TYPE=$BUILD_TYPE,VELOX_HOME=$VELOX_HOME,ENABLE_BENCHMARK=$ENABLE_BENCHMARK,\ -ENABLE_TESTS=$ENABLE_TESTS,BUILD_TEST_UTILS=$BUILD_TEST_UTILS,\ -OTHER_ARGUMENTS=$OTHER_ARGUMENTS,COMMIT_HASH=$COMMIT_HASH" -} - -function check_commit { - if [ $ENABLE_EP_CACHE == "ON" ]; then - if [ -f ${VELOX_HOME}/velox-build.cache ]; then - CACHED_BUILD_SUMMARY="$(cat ${VELOX_HOME}/velox-build.cache)" - if [ -n "$CACHED_BUILD_SUMMARY" ]; then - if [ "$TARGET_BUILD_SUMMARY" = "$CACHED_BUILD_SUMMARY" ]; then - echo "Velox build $TARGET_BUILD_SUMMARY was cached." - exit 0 - else - echo "Found cached build $CACHED_BUILD_SUMMARY for Velox which is different with target build $TARGET_BUILD_SUMMARY." - fi - fi - fi - else - # Branch-new build requires all untracked files to be deleted. We only need the source code. - sudo git clean -dffx :/ - fi - - if [ -f ${VELOX_HOME}/velox-build.cache ]; then - rm -f ${VELOX_HOME}/velox-build.cache - fi -} - CURRENT_DIR=$( cd "$(dirname "$BASH_SOURCE")" pwd @@ -220,15 +181,8 @@ echo "ENABLE_ABFS=${ENABLE_ABFS}" echo "BUILD_TYPE=${BUILD_TYPE}" cd ${VELOX_HOME} -TARGET_BUILD_SUMMARY=$(get_build_summary "$(git rev-parse --verify HEAD)") -if [ -z "$TARGET_BUILD_SUMMARY" ]; then - echo "Unable to parse Velox build: $TARGET_BUILD_SUMMARY." - exit 1 -fi -echo "Target Velox build: $TARGET_BUILD_SUMMARY" - -check_commit +# Branch-new build requires all untracked files to be deleted. We only need the source code. +sudo git clean -dffx :/ compile echo "Successfully built Velox from Source." -echo $TARGET_BUILD_SUMMARY > "${VELOX_HOME}/velox-build.cache" diff --git a/tools/gluten-te/centos/dockerfile-build b/tools/gluten-te/centos/dockerfile-build index fa496613b168..d8a16d475d5f 100644 --- a/tools/gluten-te/centos/dockerfile-build +++ b/tools/gluten-te/centos/dockerfile-build @@ -62,7 +62,7 @@ RUN if [ "$BUILD_BACKEND_TYPE" == "velox" ]; \ if [ "$GLUTEN_DEBUG_BUILD" == "ON" ]; then GLUTEN_BUILD_TYPE="Debug"; else GLUTEN_BUILD_TYPE="Release"; fi; \ DEPS_INSTALL_SCRIPT="source /env.sh && bash /opt/gluten/dev/builddeps-veloxbe.sh \ --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON \ - --build_type=$GLUTEN_BUILD_TYPE --enable_ep_cache=ON"; \ + --build_type=$GLUTEN_BUILD_TYPE"; \ EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \ -Pbackends-velox \ -Pceleborn \ diff --git a/tools/gluten-te/ubuntu/dockerfile-build b/tools/gluten-te/ubuntu/dockerfile-build index 59736c98694a..073db0ba18f5 100644 --- a/tools/gluten-te/ubuntu/dockerfile-build +++ b/tools/gluten-te/ubuntu/dockerfile-build @@ -62,7 +62,7 @@ RUN if [ "$BUILD_BACKEND_TYPE" == "velox" ]; \ if [ "$GLUTEN_DEBUG_BUILD" == "ON" ]; then GLUTEN_BUILD_TYPE="Debug"; else GLUTEN_BUILD_TYPE="Release"; fi; \ DEPS_INSTALL_SCRIPT="bash /opt/gluten/dev/builddeps-veloxbe.sh \ --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=ON \ - --build_type=$GLUTEN_BUILD_TYPE --enable_ep_cache=ON"; \ + --build_type=$GLUTEN_BUILD_TYPE"; \ EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \ -Pbackends-velox \ -Pceleborn \ diff --git a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh index f795b5ba9b9b..562828e5a34f 100755 --- a/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh +++ b/tools/gluten-te/ubuntu/examples/buildhere-veloxbe-dev/scripts/all.sh @@ -26,7 +26,7 @@ EXTRA_MAVEN_OPTIONS="-Pspark-3.2 \ -Dcheckstyle.skip=true" cd /opt/gluten -bash dev/builddeps-veloxbe.sh --build_type=Debug --enable_ep_cache=ON +bash dev/builddeps-veloxbe.sh --build_type=Debug mvn clean install $EXTRA_MAVEN_OPTIONS apt-get -y -q --no-install-recommends install firefox tmux openjdk-8-source