Add support to seek beyond end of stream (#192) #122
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and upload artifact JARs to S3 treatment bucket | |
| # Controls when the action will run. Invokes the workflow on push events but only for the main branch | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - cicd-builds | |
| env: | |
| AWS_REGION : ${{ vars.AWS_REGION }} # Change to reflect your region | |
| S3_BUCKET : ${{ vars.S3_BUCKET }} | |
| STATE_MACHINE_ARN : ${{ secrets.STATE_MACHINE_ARN }} | |
| ROLE_TO_ASSUME: ${{ secrets.ASSUME_ROLE_ARN }} | |
| STATE_MACHINE_INPUT_S3A: ${{ vars.STATE_MACHINE_INPUT_S3A }} | |
| STATE_MACHINE_INPUT_S3FILEIO: ${{ vars.STATE_MACHINE_INPUT_S3FILEIO }} | |
| # Permission can be added at job level or workflow level | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| contents: read # This is required for actions/checkout | |
| jobs: | |
| BuildAnalyticsAcceleratorAndUploadArtifact: | |
| name: Build library artifacts | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| java-version: '8' | |
| distribution: 'corretto' | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@417ae3ccd767c252f5661f1ace9f835f9654f2b5 # v3.1.0 | |
| - name: Build with Gradle | |
| run: | | |
| ./gradlew -PsnapshotBuild=true build | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| path: "input-stream/build/libs/analyticsaccelerator-s3-SNAPSHOT.jar" | |
| name: "analyticsaccelerator-s3-SNAPSHOT.jar" | |
| BuildHadoopAndUploadArtifact: | |
| name: Build Hadoop artifacts | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| java-version: '8' | |
| distribution: 'corretto' | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@417ae3ccd767c252f5661f1ace9f835f9654f2b5 # v3.1.0 | |
| - name: Build and publish to local Maven with Gradle | |
| run: | | |
| ./gradlew -PsnapshotBuild=true publishToMavenLocal | |
| - name: Setup Hadoop SSH deploy key | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.HADOOP_STAGING_SSH_KEY }} | |
| - name: Checkout Hadoop | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ secrets.HADOOP_STAGING_PATH }} | |
| ref: s3-connector-framework | |
| path: hadoop | |
| ssh-key: ${{ secrets.HADOOP_STAGING_SSH_KEY }} | |
| - name: Build Hadoop jar | |
| run: | | |
| mvn clean install -DskipTests | |
| cd /home/runner/work/analytics-accelerator-s3/analytics-accelerator-s3/hadoop/hadoop-tools/hadoop-aws/ | |
| mvn -Dparallel-tests -DtestsThreadCount=8 clean test | |
| working-directory: hadoop | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| path: "/home/runner/.m2/repository/org/apache/hadoop/hadoop-aws/3.5.0-SNAPSHOT/hadoop-aws-3.5.0-SNAPSHOT.jar" | |
| name: "hadoop-aws-3.5.0-SNAPSHOT.jar" | |
| BuildIcebergAndUploadArtifact: | |
| name: Build Iceberg artifacts | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-java@v4 | |
| with: | |
| java-version: '11' | |
| distribution: 'corretto' | |
| - name: Setup Gradle | |
| uses: gradle/actions/setup-gradle@417ae3ccd767c252f5661f1ace9f835f9654f2b5 # v3.1.0 | |
| - name: Build and publish to local Maven with Gradle | |
| run: | | |
| ./gradlew -PsnapshotBuild=true publishToMavenLocal | |
| - name: Setup Iceberg SSH deploy key | |
| uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.ICEBERG_STAGING_SSH_KEY }} | |
| - name: Checkout Iceberg | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ secrets.ICEBERG_STAGING_PATH }} | |
| ref: dat-s3 | |
| path: iceberg | |
| ssh-key: ${{ secrets.ICEBERG_STAGING_SSH_KEY }} | |
| - name: Build Iceberg jar | |
| run: | | |
| ./gradlew build -x test -x integrationTest | |
| ./gradlew :iceberg-aws:test | |
| working-directory: iceberg | |
| - name: Rename iceberg-spark-runtime JAR path | |
| run: | | |
| FILE_PATH_BASE=/home/runner/work/analytics-accelerator-s3/analytics-accelerator-s3/iceberg/spark/v3.5/spark-runtime/build/libs/ | |
| cd "$FILE_PATH_BASE" | |
| FILE_NAME=$(ls | grep "iceberg-spark-runtime-3.5_2.12-[0-9a-f]*\.jar" | head -n 1) | |
| mv "$FILE_NAME" "iceberg-spark-runtime-3.5_2.12-1.6.0-SNAPSHOT.jar" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| path: "/home/runner/work/analytics-accelerator-s3/analytics-accelerator-s3/iceberg/spark/v3.5/spark-runtime/build/libs/iceberg-spark-runtime-3.5_2.12-1.6.0-SNAPSHOT.jar" | |
| name: "iceberg-spark-runtime-3.5_2.12-1.6.0-SNAPSHOT.jar" | |
| UploadArtifactsToS3: | |
| name: Upload all artifacts to S3 | |
| runs-on: ubuntu-latest | |
| needs: | |
| - BuildAnalyticsAcceleratorAndUploadArtifact | |
| - BuildHadoopAndUploadArtifact | |
| - BuildIcebergAndUploadArtifact | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: "analyticsaccelerator-s3-SNAPSHOT.jar" | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: "iceberg-spark-runtime-3.5_2.12-1.6.0-SNAPSHOT.jar" | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: "hadoop-aws-3.5.0-SNAPSHOT.jar" | |
| - name: Configure aws credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| role-to-assume: ${{ env.ROLE_TO_ASSUME }} | |
| role-session-name: GitHub_to_AWS_via_FederatedOIDC | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Upload analyticsaccelerator JAR to S3a treatment bucket | |
| run: aws s3 cp analyticsaccelerator-s3-SNAPSHOT.jar s3://${{ env.S3_BUCKET }}/s3a/analyticsaccelerator-s3-SNAPSHOT.jar | |
| - name: Upload analyticsaccelerator JAR to S3FileIO treatment bucket | |
| run: aws s3 cp analyticsaccelerator-s3-SNAPSHOT.jar s3://${{ env.S3_BUCKET }}/s3fileio/analyticsaccelerator-s3-SNAPSHOT.jar | |
| - name: Upload Iceberg JAR to S3FileIO treatment bucket | |
| run: aws s3 cp iceberg-spark-runtime-3.5_2.12-1.6.0-SNAPSHOT.jar s3://${{ env.S3_BUCKET }}/s3fileio/iceberg-spark-runtime-3.5_2.12-1.6.0-SNAPSHOT.jar | |
| - name: Upload Hadoop JAR to S3A treatment bucket | |
| run: aws s3 cp hadoop-aws-3.5.0-SNAPSHOT.jar s3://${{ env.S3_BUCKET }}/s3a/hadoop-aws-3.5.0-SNAPSHOT.jar | |
| - name: Trigger S3A Benchmarks | |
| run: aws stepfunctions start-execution --state-machine-arn ${{ env.STATE_MACHINE_ARN }} --input ${{ env.STATE_MACHINE_INPUT_S3A }} | |
| - name: Trigger S3FileIO Benchmarks | |
| run: aws stepfunctions start-execution --state-machine-arn ${{ env.STATE_MACHINE_ARN }} --input ${{ env.STATE_MACHINE_INPUT_S3FILEIO }} |