From 6d1e5d5a5c579a67ba2493f9a1778bb76460c7b1 Mon Sep 17 00:00:00 2001 From: SimhadriGovindappa Date: Wed, 26 Apr 2023 13:08:41 +0530 Subject: [PATCH 1/2] HIVE-27277: GH actions to build and push docker image Set up github actions workflow to build and push docker image to docker hub --- .github/workflows/docker-GA-images.yml | 59 ++++++++++++++++++++++++++ packaging/src/docker/Dockerfile | 32 +++++++++++--- packaging/src/docker/build.sh | 1 + 3 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/docker-GA-images.yml diff --git a/.github/workflows/docker-GA-images.yml b/.github/workflows/docker-GA-images.yml new file mode 100644 index 000000000000..9db55f15cb1b --- /dev/null +++ b/.github/workflows/docker-GA-images.yml @@ -0,0 +1,59 @@ +name: Build and Publish docker images for Hive GA + +on: + create: + +jobs: + build: + if: github.event_name == 'create' && github.event.ref_type == 'tag' && startsWith(github.event.ref, 'rel/') + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + + - + name: 'Set up JDK 8' + uses: actions/setup-java@v1 + with: + java-version: 8 + + - + name: Hive version + run: echo "HIVE_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=project.version -DforceStdout)" >> $GITHUB_ENV + + + - name: Tag + run: echo "tag=${{ env.HIVE_VERSION }}" | awk '{print tolower($0)}' >> $GITHUB_ENV + + - + name: Hadoop version + run: echo "HADOOP_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=hadoop.version -DforceStdout)" >> $GITHUB_ENV + - + name: Tez version + run: echo "TEZ_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=tez.version -DforceStdout)" >> $GITHUB_ENV + + - + name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push Hive Image to docker hub + uses: docker/build-push-action@v4 + with: + context: ./packaging/src/docker/ + file: ./packaging/src/docker/Dockerfile + push: true + tags: ${{ secrets.DOCKERHUB_USER }}/hive:${{ env.tag }} + build-args: + | + HIVE_VERSION=${{ env.HIVE_VERSION }} + HADOOP_VERSION=${{ env.HADOOP_VERSION }} + TEZ_VERSION=${{ env.TEZ_VERSION }} + BUILD_ENV=archive \ No newline at end of file diff --git a/packaging/src/docker/Dockerfile b/packaging/src/docker/Dockerfile index fd0ad7e698ad..382cd5d0b38d 100644 --- a/packaging/src/docker/Dockerfile +++ b/packaging/src/docker/Dockerfile @@ -14,14 +14,31 @@ # See the License for the specific language governing permissions and # limitations under the License. # +ARG BUILD_ENV + FROM ubuntu as unarchive +ONBUILD COPY hadoop-*.tar.gz /opt +ONBUILD COPY apache-hive-*-bin.tar.gz /opt +ONBUILD COPY apache-tez-*-bin.tar.gz /opt + +FROM ubuntu as archive +ARG HADOOP_VERSION +ARG HIVE_VERSION +ARG TEZ_VERSION +ONBUILD RUN apt-get update && apt-get -y install wget +ONBUILD RUN wget https://archive.apache.org/dist/tez/$TEZ_VERSION/apache-tez-$TEZ_VERSION-bin.tar.gz && \ + wget https://archive.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz && \ + wget https://archive.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz +ONBUILD RUN mv /apache-tez-$TEZ_VERSION-bin.tar.gz /opt && \ + mv hadoop-$HADOOP_VERSION.tar.gz /opt && \ + mv apache-hive-$HIVE_VERSION-bin.tar.gz /opt + +FROM ${BUILD_ENV} as env +RUN echo ${BUILD_ENV} ARG HADOOP_VERSION ARG HIVE_VERSION ARG TEZ_VERSION -COPY hadoop-$HADOOP_VERSION.tar.gz /opt -COPY apache-hive-$HIVE_VERSION-bin.tar.gz /opt -COPY apache-tez-$TEZ_VERSION-bin.tar.gz /opt RUN tar -xzvf /opt/hadoop-$HADOOP_VERSION.tar.gz -C /opt/ && \ rm -rf /opt/hadoop-$HADOOP_VERSION/share/doc/* && \ @@ -35,9 +52,9 @@ FROM openjdk:8-jre-slim AS run ARG HADOOP_VERSION ARG HIVE_VERSION ARG TEZ_VERSION -COPY --from=unarchive /opt/hadoop-$HADOOP_VERSION /opt/hadoop -COPY --from=unarchive /opt/apache-hive-$HIVE_VERSION-bin /opt/hive -COPY --from=unarchive /opt/apache-tez-$TEZ_VERSION-bin /opt/tez +COPY --from=env /opt/hadoop-$HADOOP_VERSION /opt/hadoop +COPY --from=env /opt/apache-hive-$HIVE_VERSION-bin /opt/hive +COPY --from=env /opt/apache-tez-$TEZ_VERSION-bin /opt/tez # Install dependencies RUN set -ex; \ @@ -48,7 +65,8 @@ RUN set -ex; \ # Set necessary environment variables. ENV HADOOP_HOME=/opt/hadoop \ HIVE_HOME=/opt/hive \ - TEZ_HOME=/opt/tez + TEZ_HOME=/opt/tez \ + HIVE_VER=$HIVE_VERSION ENV PATH=$HIVE_HOME/bin:$HADOOP_HOME/bin:$PATH diff --git a/packaging/src/docker/build.sh b/packaging/src/docker/build.sh index 82cd2186e0d2..b5cbce78882a 100755 --- a/packaging/src/docker/build.sh +++ b/packaging/src/docker/build.sh @@ -110,6 +110,7 @@ docker build \ "$WORK_DIR" \ -f "$WORK_DIR/Dockerfile" \ -t "$repo/hive:$HIVE_VERSION" \ + --build-arg "BUILD_ENV=unarchive" --build-arg "HIVE_VERSION=$HIVE_VERSION" \ --build-arg "HADOOP_VERSION=$HADOOP_VERSION" \ --build-arg "TEZ_VERSION=$TEZ_VERSION" \ From 05b3a6e651b1ce164ca1720bf77933e637d8693e Mon Sep 17 00:00:00 2001 From: SimhadriG Date: Mon, 22 May 2023 02:29:37 +0530 Subject: [PATCH 2/2] Added weekly dev snapshot job --- .github/workflows/docker-GA-images.yml | 2 +- .github/workflows/docker-dev-images.yml | 73 +++++++++++++++++++++++++ packaging/src/docker/Dockerfile | 11 ++++ 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docker-dev-images.yml diff --git a/.github/workflows/docker-GA-images.yml b/.github/workflows/docker-GA-images.yml index 9db55f15cb1b..5b65bc9f2117 100644 --- a/.github/workflows/docker-GA-images.yml +++ b/.github/workflows/docker-GA-images.yml @@ -1,7 +1,7 @@ name: Build and Publish docker images for Hive GA on: - create: + workflow_dispatch: jobs: build: diff --git a/.github/workflows/docker-dev-images.yml b/.github/workflows/docker-dev-images.yml new file mode 100644 index 000000000000..7af8a4464a8f --- /dev/null +++ b/.github/workflows/docker-dev-images.yml @@ -0,0 +1,73 @@ +name: Build and Publish docker images for Hive SNAPSHOT images + +on: + schedule: + - cron: "0 10 * * 1" + +jobs: + build: + if: github.event_name == 'create' && github.event.ref_type == 'tag' && startsWith(github.event.ref, 'rel/') + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + + - + name: 'Set up JDK 8' + uses: actions/setup-java@v1 + with: + java-version: 8 + + - + name: Hive version + run: echo "HIVE_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=project.version -DforceStdout)" >> $GITHUB_ENV + + - name: Tag + run: echo "tag=${{ env.HIVE_VERSION }}" | awk '{print tolower($0)}' >> $GITHUB_ENV + + - + name: Hadoop version + run: echo "HADOOP_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=hadoop.version -DforceStdout)" >> $GITHUB_ENV + - + name: Tez version + run: echo "TEZ_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=tez.version -DforceStdout)" >> $GITHUB_ENV + + - name: Build project + run: mvn clean install -DskipTests -Pitests,iceberg -Pdist + + - name: Temporarily save jar artifact + uses: actions/upload-artifact@v2 + with: + name: jar-artifact + path: ./packaging/target/apache-hive-${{ env.HIVE_VERSION }}-bin.tar.gz + retention-days: 1 + + - uses: actions/download-artifact@v1 + with: + name: jar-artifact + path: ./packaging/target/ + - + name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push Hive Image to docker hub + uses: docker/build-push-action@v4 + with: + context: ./packaging/src/docker/ + file: ./packaging/src/docker/Dockerfile + push: true + tags: ${{ secrets.DOCKERHUB_USER }}/hive:${{ env.tag }} + build-args: + | + HIVE_VERSION=${{ env.HIVE_VERSION }} + HADOOP_VERSION=${{ env.HADOOP_VERSION }} + TEZ_VERSION=${{ env.TEZ_VERSION }} + BUILD_ENV=snapshot \ No newline at end of file diff --git a/packaging/src/docker/Dockerfile b/packaging/src/docker/Dockerfile index 382cd5d0b38d..ce829f991c3c 100644 --- a/packaging/src/docker/Dockerfile +++ b/packaging/src/docker/Dockerfile @@ -33,6 +33,17 @@ ONBUILD RUN mv /apache-tez-$TEZ_VERSION-bin.tar.gz /opt && \ mv hadoop-$HADOOP_VERSION.tar.gz /opt && \ mv apache-hive-$HIVE_VERSION-bin.tar.gz /opt +FROM ubuntu as snapshot +ARG HADOOP_VERSION +ARG HIVE_VERSION +ARG TEZ_VERSION +ONBUILD RUN apt-get update && apt-get -y install wget +ONBUILD RUN wget https://archive.apache.org/dist/tez/$TEZ_VERSION/apache-tez-$TEZ_VERSION-bin.tar.gz && \ + wget https://archive.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz +ONBUILD COPY ./packaging/target/apache-hive-$HIVE_VERSION-bin.tar.gz /opt +ONBUILD RUN mv /apache-tez-$TEZ_VERSION-bin.tar.gz /opt && \ + mv hadoop-$HADOOP_VERSION.tar.gz /opt + FROM ${BUILD_ENV} as env RUN echo ${BUILD_ENV}