diff --git a/.github/workflows/docker-GA-images.yml b/.github/workflows/docker-GA-images.yml new file mode 100644 index 000000000000..5b65bc9f2117 --- /dev/null +++ b/.github/workflows/docker-GA-images.yml @@ -0,0 +1,59 @@ +name: Build and Publish docker images for Hive GA + +on: + workflow_dispatch: + +jobs: + build: + if: github.event_name == 'create' && github.event.ref_type == 'tag' && startsWith(github.event.ref, 'rel/') + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + + - + name: 'Set up JDK 8' + uses: actions/setup-java@v1 + with: + java-version: 8 + + - + name: Hive version + run: echo "HIVE_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=project.version -DforceStdout)" >> $GITHUB_ENV + + + - name: Tag + run: echo "tag=${{ env.HIVE_VERSION }}" | awk '{print tolower($0)}' >> $GITHUB_ENV + + - + name: Hadoop version + run: echo "HADOOP_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=hadoop.version -DforceStdout)" >> $GITHUB_ENV + - + name: Tez version + run: echo "TEZ_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=tez.version -DforceStdout)" >> $GITHUB_ENV + + - + name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push Hive Image to docker hub + uses: docker/build-push-action@v4 + with: + context: ./packaging/src/docker/ + file: ./packaging/src/docker/Dockerfile + push: true + tags: ${{ secrets.DOCKERHUB_USER }}/hive:${{ env.tag }} + build-args: + | + HIVE_VERSION=${{ env.HIVE_VERSION }} + HADOOP_VERSION=${{ env.HADOOP_VERSION }} + TEZ_VERSION=${{ env.TEZ_VERSION }} + BUILD_ENV=archive \ No newline at end of file diff --git a/.github/workflows/docker-dev-images.yml b/.github/workflows/docker-dev-images.yml new file mode 100644 index 000000000000..7af8a4464a8f --- /dev/null +++ b/.github/workflows/docker-dev-images.yml @@ -0,0 +1,73 @@ +name: Build and Publish docker images for Hive SNAPSHOT images + +on: + schedule: + - cron: "0 10 * * 1" + +jobs: + build: + if: github.event_name == 'create' && github.event.ref_type == 'tag' && startsWith(github.event.ref, 'rel/') + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + + - + name: 'Set up JDK 8' + uses: actions/setup-java@v1 + with: + java-version: 8 + + - + name: Hive version + run: echo "HIVE_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=project.version -DforceStdout)" >> $GITHUB_ENV + + - name: Tag + run: echo "tag=${{ env.HIVE_VERSION }}" | awk '{print tolower($0)}' >> $GITHUB_ENV + + - + name: Hadoop version + run: echo "HADOOP_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=hadoop.version -DforceStdout)" >> $GITHUB_ENV + - + name: Tez version + run: echo "TEZ_VERSION=$(mvn -f "pom.xml" -q help:evaluate -Dexpression=tez.version -DforceStdout)" >> $GITHUB_ENV + + - name: Build project + run: mvn clean install -DskipTests -Pitests,iceberg -Pdist + + - name: Temporarily save jar artifact + uses: actions/upload-artifact@v2 + with: + name: jar-artifact + path: ./packaging/target/apache-hive-${{ env.HIVE_VERSION }}-bin.tar.gz + retention-days: 1 + + - uses: actions/download-artifact@v1 + with: + name: jar-artifact + path: ./packaging/target/ + - + name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and push Hive Image to docker hub + uses: docker/build-push-action@v4 + with: + context: ./packaging/src/docker/ + file: ./packaging/src/docker/Dockerfile + push: true + tags: ${{ secrets.DOCKERHUB_USER }}/hive:${{ env.tag }} + build-args: + | + HIVE_VERSION=${{ env.HIVE_VERSION }} + HADOOP_VERSION=${{ env.HADOOP_VERSION }} + TEZ_VERSION=${{ env.TEZ_VERSION }} + BUILD_ENV=snapshot \ No newline at end of file diff --git a/packaging/src/docker/Dockerfile b/packaging/src/docker/Dockerfile index fd0ad7e698ad..ce829f991c3c 100644 --- a/packaging/src/docker/Dockerfile +++ b/packaging/src/docker/Dockerfile @@ -14,14 +14,42 @@ # See the License for the specific language governing permissions and # limitations under the License. # +ARG BUILD_ENV + FROM ubuntu as unarchive +ONBUILD COPY hadoop-*.tar.gz /opt +ONBUILD COPY apache-hive-*-bin.tar.gz /opt +ONBUILD COPY apache-tez-*-bin.tar.gz /opt + +FROM ubuntu as archive +ARG HADOOP_VERSION +ARG HIVE_VERSION +ARG TEZ_VERSION +ONBUILD RUN apt-get update && apt-get -y install wget +ONBUILD RUN wget https://archive.apache.org/dist/tez/$TEZ_VERSION/apache-tez-$TEZ_VERSION-bin.tar.gz && \ + wget https://archive.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz && \ + wget https://archive.apache.org/dist/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz +ONBUILD RUN mv /apache-tez-$TEZ_VERSION-bin.tar.gz /opt && \ + mv hadoop-$HADOOP_VERSION.tar.gz /opt && \ + mv apache-hive-$HIVE_VERSION-bin.tar.gz /opt + +FROM ubuntu as snapshot +ARG HADOOP_VERSION +ARG HIVE_VERSION +ARG TEZ_VERSION +ONBUILD RUN apt-get update && apt-get -y install wget +ONBUILD RUN wget https://archive.apache.org/dist/tez/$TEZ_VERSION/apache-tez-$TEZ_VERSION-bin.tar.gz && \ + wget https://archive.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz +ONBUILD COPY ./packaging/target/apache-hive-$HIVE_VERSION-bin.tar.gz /opt +ONBUILD RUN mv /apache-tez-$TEZ_VERSION-bin.tar.gz /opt && \ + mv hadoop-$HADOOP_VERSION.tar.gz /opt + +FROM ${BUILD_ENV} as env +RUN echo ${BUILD_ENV} ARG HADOOP_VERSION ARG HIVE_VERSION ARG TEZ_VERSION -COPY hadoop-$HADOOP_VERSION.tar.gz /opt -COPY apache-hive-$HIVE_VERSION-bin.tar.gz /opt -COPY apache-tez-$TEZ_VERSION-bin.tar.gz /opt RUN tar -xzvf /opt/hadoop-$HADOOP_VERSION.tar.gz -C /opt/ && \ rm -rf /opt/hadoop-$HADOOP_VERSION/share/doc/* && \ @@ -35,9 +63,9 @@ FROM openjdk:8-jre-slim AS run ARG HADOOP_VERSION ARG HIVE_VERSION ARG TEZ_VERSION -COPY --from=unarchive /opt/hadoop-$HADOOP_VERSION /opt/hadoop -COPY --from=unarchive /opt/apache-hive-$HIVE_VERSION-bin /opt/hive -COPY --from=unarchive /opt/apache-tez-$TEZ_VERSION-bin /opt/tez +COPY --from=env /opt/hadoop-$HADOOP_VERSION /opt/hadoop +COPY --from=env /opt/apache-hive-$HIVE_VERSION-bin /opt/hive +COPY --from=env /opt/apache-tez-$TEZ_VERSION-bin /opt/tez # Install dependencies RUN set -ex; \ @@ -48,7 +76,8 @@ RUN set -ex; \ # Set necessary environment variables. ENV HADOOP_HOME=/opt/hadoop \ HIVE_HOME=/opt/hive \ - TEZ_HOME=/opt/tez + TEZ_HOME=/opt/tez \ + HIVE_VER=$HIVE_VERSION ENV PATH=$HIVE_HOME/bin:$HADOOP_HOME/bin:$PATH diff --git a/packaging/src/docker/build.sh b/packaging/src/docker/build.sh index 82cd2186e0d2..b5cbce78882a 100755 --- a/packaging/src/docker/build.sh +++ b/packaging/src/docker/build.sh @@ -110,6 +110,7 @@ docker build \ "$WORK_DIR" \ -f "$WORK_DIR/Dockerfile" \ -t "$repo/hive:$HIVE_VERSION" \ + --build-arg "BUILD_ENV=unarchive" --build-arg "HIVE_VERSION=$HIVE_VERSION" \ --build-arg "HADOOP_VERSION=$HADOOP_VERSION" \ --build-arg "TEZ_VERSION=$TEZ_VERSION" \