-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* add postgres * update ci * update gpdb code
- Loading branch information
Showing
11 changed files
with
233 additions
and
97 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| name: build-docker-images | ||
|
|
||
| on: | ||
| push: | ||
| branches: [ main ] | ||
| paths-ignore: | ||
| - "*.md" | ||
|
|
||
| pull_request: | ||
| branches: [ main ] | ||
| paths-ignore: | ||
| - "*.md" | ||
|
|
||
| # Allows you to run this workflow manually from the Actions tab | ||
| workflow_dispatch: | ||
|
|
||
| env: | ||
| REGISTRY_URL: "docker.io" # docker.io or other registry URL, DOCKER_REGISTRY_USERNAME/DOCKER_REGISTRY_PASSWORD to be set in CI env. | ||
| BUILDKIT_PROGRESS: "plain" # Full logs for CI build. | ||
|
|
||
| # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. | ||
| DOCKER_REGISTRY_USERNAME: ${{ secrets.DOCKER_REGISTRY_USERNAME }} | ||
| DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | ||
|
|
||
| # used to sync image to mirror registry | ||
| DOCKER_MIRROR_REGISTRY_USERNAME: ${{ secrets.DOCKER_MIRROR_REGISTRY_USERNAME }} | ||
| DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} | ||
|
|
||
| jobs: | ||
| qpod_bigdata: | ||
| name: "bigdata" | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - run: | | ||
| source ./tool.sh | ||
| build_image bigdata latest docker_bigdata/Dockerfile && push_image | ||
| qpod_elasticsearch: | ||
| name: "elasticsearch" | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - run: | | ||
| source ./tool.sh | ||
| build_image elasticsearch latest docker_elasticsearch/Dockerfile && push_image | ||
| qpod_kafka_confluent: | ||
| name: "kafka" | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - run: | | ||
| source ./tool.sh | ||
| build_image kafka latest docker_kafka_confluent/Dockerfile && push_image | ||
| qpod_greenplum: | ||
| name: "greenplum" | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - run: | | ||
| source ./tool.sh | ||
| build_image greenplum latest docker_greenplum/Dockerfile && push_image | ||
| qpod_postgres: | ||
| name: "postgres" | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - run: | | ||
| source ./tool.sh | ||
| build_image postgres latest docker_postgres/postgres-ext.Dockerfile && push_image | ||
| ## Sync all images in this build (listed by "names") to mirror registry. | ||
| sync_images: | ||
| needs: ["qpod_bigdata", "qpod_elasticsearch", "qpod_kafka_confluent", "qpod_postgres", "qpod_greenplum"] | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - run: | | ||
| source ./tool.sh | ||
| printenv > /tmp/docker.env | ||
| docker run --rm \ | ||
| --env-file /tmp/docker.env \ | ||
| -v $(pwd):/tmp \ | ||
| -w /tmp \ | ||
| qpod/docker-kit python /opt/utils/image-syncer/run_jobs.py |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,9 @@ | ||
| % from: https://github.com/apache/flink/blob/master/flink-python/setup.py | ||
| apache-flink | ||
| pemja | ||
| pandas | ||
| pyarrow | ||
| apache-beam | ||
| cloudpickle | ||
| avro-python3 | ||
| requests | ||
| %py4j==0.10.9.3 | ||
| %apache-beam==2.38.0 | ||
| %cloudpickle==2.1.0 | ||
| %avro-python3>=1.8.1,!=1.9.2,<1.10.0 | ||
| %fastavro>=1.1.0,<1.4.8 | ||
| %protobuf<3.18 | ||
| %pemja==0.2.4 | ||
| % apache-flink % tempfix bu install without deps |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| FROM ubuntu:latest | ||
|
|
||
| LABEL maintainer="[email protected]" | ||
|
|
||
| USER root | ||
|
|
||
| ENV APACHE_DIST="http://archive.apache.org/dist" \ | ||
| MAVEN_VERSION="3.6.3" \ | ||
| SOLR_VERSION="8.3.1" \ | ||
| SOLR_HOME="/data/solr" \ | ||
| SOLR_LIB_DIR="/data/solr/.lib" \ | ||
| SOLR_SERVER_LIB="/opt/solr/server/solr-webapp/webapp/WEB-INF/lib" \ | ||
| PATH="/opt/solr/bin:/opt/maven/bin:$PATH" | ||
|
|
||
| RUN mkdir -p $SOLR_HOME $SOLR_LIB_DIR \ | ||
| && apt-get -y update --fix-missing && apt-get -y upgrade \ | ||
| && apt-get -qq install -y --no-install-recommends wget unzip lsof openjdk-11-jdk-headless \ | ||
| && apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* \ | ||
| && install_zip() { wget -nv $1 -O /tmp/TMP.zip && unzip -q /tmp/TMP.zip -d /opt/ && rm /tmp/TMP.zip ; } \ | ||
| && install_zip "${APACHE_DIST}/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.zip" && mv /opt/apache-maven-${MAVEN_VERSION} /opt/maven \ | ||
| && install_zip "${APACHE_DIST}/lucene/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.zip" && mv /opt/solr-${SOLR_VERSION} /opt/solr \ | ||
| && sed -i -e '/-Dsolr.clustering.enabled=true/ a SOLR_OPTS="$SOLR_OPTS -Denable.runtime.lib=true -Dsun.net.inetaddr.ttl=60 -Dsun.net.inetaddr.negative.ttl=60"' /opt/solr/bin/solr.in.sh \ | ||
| && echo 'SOLR_HOME=${SOLR_HOME}' >> /opt/solr/bin/solr.in.sh \ | ||
| && echo 'SOLR_PID_DIR=${SOLR_HOME}' >> /opt/solr/bin/solr.in.sh \ | ||
| && echo 'SOLR_LOGS_DIR=${SOLR_HOME}/logs' >> /opt/solr/bin/solr.in.sh \ | ||
| && echo 'SOLR_LOG_LEVEL=WARN' >> /opt/solr/bin/solr.in.sh \ | ||
| && echo '#!/bin/bash' >> /opt/solr/bin/start-solr.sh \ | ||
| && echo '[ -f "${SOLR_HOME}/solr.xml" ] || cp -R /opt/solr/server/solr/* ${SOLR_HOME}/' >> /opt/solr/bin/start-solr.sh \ | ||
| && echo 'cp -R ${SOLR_LIB_DIR}/*.jar ${SOLR_SERVER_LIB}/' >> /opt/solr/bin/start-solr.sh \ | ||
| && echo '/opt/solr/bin/solr start -force -f -c' >> /opt/solr/bin/start-solr.sh \ | ||
| && chmod +x /opt/solr/bin/start-solr.sh | ||
|
|
||
| RUN mvn_get() { mvn dependency:copy -DlocalRepositoryDirectory="/tmp/m2repo" -DoutputDirectory="${SOLR_SERVER_LIB}" -Djavax.net.ssl.trustStorePassword=changeit -Dartifact="$1"; } \ | ||
| && mvn_get "com.janeluo:ikanalyzer:2012_u6" \ | ||
| && mvn_get "com.hankcs:hanlp:portable-1.6.3" \ | ||
| && mvn_get "com.huaban:jieba-analysis:1.0.2" \ | ||
| && rm -Rf /tmp/* /opt/solr/docs/ \ | ||
| && ls -alh ${SOLR_SERVER_LIB} | ||
|
|
||
| RUN source /opt/utils/script-utils.sh \ | ||
| && VERSION_GRADLE="6.5.1" \ | ||
| && URL_GRADLE="https://downloads.gradle-dn.com/distributions/gradle-${VERSION_GRADLE}-bin.zip" \ | ||
| && install_zip ${URL_GRADLE} && mv /opt/gradle-* /opt/gradle \ | ||
| && ln -s /opt/gradle/bin/gradle /usr/bin/ \ | ||
| && echo "@ Version of Gradle:" && gradle --version | ||
|
|
||
|
|
||
| EXPOSE 8983 9983 | ||
|
|
||
| WORKDIR /opt/solr | ||
|
|
||
| VOLUME /data/solr | ||
|
|
||
| ENTRYPOINT ["start-solr.sh"] | ||
| CMD ["start-solr.sh] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| # Distributed under the terms of the Modified BSD License. | ||
|
|
||
| ARG BASE_NAMESPACE | ||
| ARG BASE_IMG="base" | ||
| FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG} as builder | ||
|
|
||
| ARG PG_MAJOR=15 | ||
| FROM library/postgres:${PG_MAJOR:-latest} | ||
|
|
||
| LABEL maintainer="[email protected]" | ||
|
|
||
| COPY work /opt/utils/ | ||
| COPY --from=builder /opt /opt | ||
|
|
||
| RUN set -x \ | ||
| && apt-get update && apt-get install -y gettext \ | ||
| apt-utils apt-transport-https ca-certificates gnupg2 dirmngr locales sudo lsb-release curl \ | ||
| && envsubst < /opt/utils/install_list_pgext.apt > /opt/utils/install_list_pgext.apt \ | ||
| && . /opt/utils/script-utils.sh \ | ||
| && install_apt /opt/utils/install_list_base.apt \ | ||
| && install_apt /opt/utils/install_list_pgext.apt \ | ||
| && echo "Clean up" && list_installed_packages && install__clean |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| postgresql-contrib | ||
| postgresql-${PG_MAJOR}-postgis* | ||
| postgresql-${PG_MAJOR}-pgvector | ||
| postgresql-${PG_MAJOR}-cron | ||
| postgresql-${PG_MAJOR}-wal2json | ||
|
|
||
| % https://packagecloud.io/citusdata/community/${distro_name}/ ${distro_codename} main | ||
| % https://packagecloud.io/timescale/timescaledb/${distro_name}/ ${distro_codename} main | ||
| % https://apt.postgresml.org ${distro_codename} maintainer | ||
| % timescaledb-2-postgresql-${PG_MAJOR} | ||
| % postgresql-${PG_MAJOR}-citus-12.1 |
Oops, something went wrong.