[MINOR] Add bundle validation for new artifacts in Maven central

apache · Dec 13, 2024 · 8975b9e · 8975b9e
1 parent 1e364d1
commit 8975b9e
Show file tree

Hide file tree

Showing 2 changed files with 88 additions and 4 deletions.
diff --git a/.github/workflows/release_candidate_validation.yml b/.github/workflows/release_candidate_validation.yml
@@ -41,7 +41,7 @@ jobs:
           - scalaProfile: 'scala-2.12'
             flinkProfile: 'flink1.14'
             sparkProfile: 'spark3.3'
-            sparkRuntime: 'spark3.3.1'
+            sparkRuntime: 'spark3.3.4'
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 8
@@ -77,3 +77,68 @@ jobs:
         if: ${{ env.SPARK_PROFILE >= 'spark3.3' }} # Only Spark 3.3 and above support Java 17
         run: |
           ./packaging/bundle-validation/ci_run.sh hudi_docker_java17 $HUDI_VERSION openjdk17 $STAGING_REPO_NUM
+
+  validate-release-maven-artifacts:
+    runs-on: ubuntu-latest
+    # if: false
+    env:
+      HUDI_VERSION: 1.0.0
+      MAVEN_BASE_URL: 'https://repo1.maven.org/maven2'
+    strategy:
+      matrix:
+        include:
+          - scalaProfile: 'scala-2.13'
+            flinkProfile: 'flink1.18'
+            sparkProfile: 'spark3.5'
+            sparkRuntime: 'spark3.5.1'
+          - scalaProfile: 'scala-2.12'
+            flinkProfile: 'flink1.17'
+            sparkProfile: 'spark3.5'
+            sparkRuntime: 'spark3.5.1'
+          - scalaProfile: 'scala-2.12'
+            flinkProfile: 'flink1.16'
+            sparkProfile: 'spark3.4'
+            sparkRuntime: 'spark3.4.3'
+          - scalaProfile: 'scala-2.12'
+            flinkProfile: 'flink1.15'
+            sparkProfile: 'spark3.3'
+            sparkRuntime: 'spark3.3.4'
+          - scalaProfile: 'scala-2.12'
+            flinkProfile: 'flink1.14'
+            sparkProfile: 'spark3.3'
+            sparkRuntime: 'spark3.3.4'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          architecture: x64
+          cache: maven
+      - name: IT - Bundle Validation - OpenJDK 8
+        env:
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+        run: |
+          ./packaging/bundle-validation/ci_run.sh hudi_docker_java8 $HUDI_VERSION openjdk8 "" $MAVEN_BASE_URL
+      - name: IT - Bundle Validation - OpenJDK 11
+        env:
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+        if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on Spark 3
+        run: |
+          ./packaging/bundle-validation/ci_run.sh hudi_docker_java11 $HUDI_VERSION openjdk11 "" $MAVEN_BASE_URL
+      - name: IT - Bundle Validation - OpenJDK 17
+        env:
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_RUNTIME: ${{ matrix.sparkRuntime }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+        if: ${{ env.SPARK_PROFILE >= 'spark3.3' }} # Only Spark 3.3 and above support Java 17
+        run: |
+          ./packaging/bundle-validation/ci_run.sh hudi_docker_java17 $HUDI_VERSION openjdk17 "" $MAVEN_BASE_URL
diff --git a/packaging/bundle-validation/ci_run.sh b/packaging/bundle-validation/ci_run.sh
@@ -31,9 +31,29 @@ CONTAINER_NAME=$1
 HUDI_VERSION=$2
 JAVA_RUNTIME_VERSION=$3
 STAGING_REPO_NUM=$4
+MAVEN_BASE_URL=$5
 echo "HUDI_VERSION: $HUDI_VERSION JAVA_RUNTIME_VERSION: $JAVA_RUNTIME_VERSION"
 echo "SPARK_RUNTIME: $SPARK_RUNTIME SPARK_PROFILE (optional): $SPARK_PROFILE"
 echo "SCALA_PROFILE: $SCALA_PROFILE"
+echo "MAVEN_BASE_URL: $MAVEN_BASE_URL"
+echo "STAGING_REPO_NUM: $STAGING_REPO_NUM"
+
+# Ensure only one of STAGING_REPO_NUM or MAVEN_BASE_URL is provided
+if [[ -n "$STAGING_REPO_NUM" && -n "$MAVEN_BASE_URL" ]]; then
+  echo "Error: Both STAGING_REPO_NUM and MAVEN_BASE_URL cannot be provided simultaneously."
+  exit 1
+fi
+
+if [[ -n "$STAGING_REPO_NUM" ]]; then
+  REPO_BASE_URL=https://repository.apache.org/content/repositories/orgapachehudi-$STAGING_REPO_NUM/org/apache/hudi
+  echo "Downloading bundle jars from staging repo orgapachehudi-$REPO_BASE_URL ..."
+elif [[ -n "$MAVEN_BASE_URL" ]]; then
+  REPO_BASE_URL=$MAVEN_BASE_URL/org/apache/hudi
+  echo "Downloading bundle jars from maven central - $REPO_BASE_URL ..."
+else
+  echo "Error: Either STAGING_REPO_NUM or MAVEN_BASE_URL must be provided."
+  exit 1
+fi
 
 # choose versions based on build profiles
 if [[ ${SPARK_RUNTIME} == 'spark3.3.1' ]]; then
@@ -112,7 +132,7 @@ fi
 TMP_JARS_DIR=/tmp/jars/$(date +%s)
 mkdir -p $TMP_JARS_DIR
 
-if [[ -z "$STAGING_REPO_NUM" ]]; then
+if [ -z "$STAGING_REPO_NUM" ] && [ -z "$MAVEN_BASE_URL" ]; then
   echo 'Adding built bundle jars for validation'
   if [[ "$SCALA_PROFILE" != 'scala-2.13' ]]; then
     # For Scala 2.13, Flink is not support, so skipping the Flink bundle validation
@@ -170,8 +190,7 @@ else
     HUDI_FLINK_BUNDLE_NAME=hudi-flink1.20-bundle    
   fi
 
-  echo "Downloading bundle jars from staging repo orgapachehudi-$STAGING_REPO_NUM ..."
-  REPO_BASE_URL=https://repository.apache.org/content/repositories/orgapachehudi-$STAGING_REPO_NUM/org/apache/hudi
+  echo "Downloading bundle jars from base URL - $REPO_BASE_URL ..."
   wget -q $REPO_BASE_URL/$HUDI_FLINK_BUNDLE_NAME/$HUDI_VERSION/$HUDI_FLINK_BUNDLE_NAME-$HUDI_VERSION.jar -P $TMP_JARS_DIR/
   wget -q $REPO_BASE_URL/$HUDI_HADOOP_MR_BUNDLE_NAME/$HUDI_VERSION/$HUDI_HADOOP_MR_BUNDLE_NAME-$HUDI_VERSION.jar -P $TMP_JARS_DIR/
   wget -q $REPO_BASE_URL/$HUDI_KAFKA_CONNECT_BUNDLE_NAME/$HUDI_VERSION/$HUDI_KAFKA_CONNECT_BUNDLE_NAME-$HUDI_VERSION.jar -P $TMP_JARS_DIR/