Skip to content

Commit

Permalink
add db clients (#6)
Browse files Browse the repository at this point in the history 
* add db clients
  • Loading branch information
@haobibo
haobibo authored Nov 17, 2022
1 parent 02eb3cc commit 5b191d9
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 33 deletions.
9 changes: 7 additions & 2 deletions docker_bigdata/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,15 @@ ENV JAVA_HOME=/opt/jdk
ENV PATH=$PATH:${CONDA_PREFIX}/bin:${JAVA_HOME}/bin

RUN source /opt/utils/script-setup.sh \
&& source /opt/utils/script-setup-db-clients.sh \
&& install_apt /opt/utils/install_list_base.apt \
&& echo "Install tini" && setup_tini \
# pyflink does not yet support Python > 3.8
&& echo "Install Mamba, Python 3.8, and Conda:" && setup_mamba && setup_conda_with_mamba "3.8" \
&& echo "Install postgresql client:" && setup_postgresql_client \
&& echo "Install mysql client:" && setup_mysql_client \
&& echo "Install mongosh:" && setup_mongosh_client \
&& echo "Install redis-cli:" && setup_redis_client \
# pyflink does not yet support Python > 3.9
&& echo "Install Mamba, Python 3.9, and Conda:" && setup_mamba && setup_conda_with_mamba "3.9" \
&& echo "Install JDK:" && VERSION_JDK=11 setup_java_base \
&& echo "Install pyflink:" && install_pip /opt/utils/list_install_pip_pyflink.txt \
&& echo "Install pyspark:" && install_pip /opt/utils/list_install_pip_pyspark.txt \
Expand Down
13 changes: 0 additions & 13 deletions docker_bigdata/work/script-db-clients.sh
View file

This file was deleted.

42 changes: 42 additions & 0 deletions docker_bigdata/work/script-setup-db-clients.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
source /opt/utils/script-utils.sh


setup_postgresql_client() {
local VER_PG=${VERSION_PG:-"14"}
# from: https://www.postgresql.org/download/linux/ubuntu/
echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
# will download ~9MB files and use ~55MB disk after installation
sudo apt-get -y install "postgresql-client-${VER_PG}"
echo "@ Version of psql client: $(psql --version)"
}


setup_mysql_client() {
sudo apt-get update
# will download ~5MB files and use ~76MB disk after installation
sudo apt-get -y install mysql-client
echo "@ Version of mysql client: $(mysql --version)"
}


setup_mongosh_client() {
# from: https://www.mongodb.com/docs/mongodb-shell/install/
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu $(lsb_release -cs)/mongodb-org/6.0 multiverse" > /etc/apt/sources.list.d/mongodb-org-6.0.list
curl -sL https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add -
sudo apt-get update
# will download ~38MB files and use ~218MB disk after installation
sudo apt-get -y install mongodb-mongosh
echo "@ Version of mongosh client: $(mongosh --version)"
}


setup_redis_client() {
# from https://redis.io/docs/getting-started/installation/install-redis-on-linux/
curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
sudo apt-get update
sudo apt-get -y install redis-tools
echo "@ Version of redis-cli: $(redis-cli --version)"
}
40 changes: 22 additions & 18 deletions tool.sh
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,20 @@
#!/bin/bash
# set -exu
set -xu

export REGISTRY_URL="docker.io" # docker.io or other registry URL, DOCKER_REGISTRY_USER/DOCKER_REGISTRY_PASSWORD to be set in CI env.
export BUILDKIT_PROGRESS="plain" # Full logs for CI build.
# DOCKER_REGISTRY_USER and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets.

CI_PROJECT_NAME=${GITHUB_REPOSITORY:-"QPod/data-lab"}
CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"}
CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/')

if [ "${CI_PROJECT_BRANCH}" = "main" ] ; then
export CI_PROJECT_NAMESPACE=$(echo "$(dirname ${CI_PROJECT_NAME})") ;
# If on the main branch, docker images namespace will be same as CI_PROJECT_NAME's name space
export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})" ;
else
export CI_PROJECT_NAMESPACE=$(echo "$(dirname ${CI_PROJECT_NAME})")0${CI_PROJECT_BRANCH} ;
# not main branch, docker namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE}
export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})0${CI_PROJECT_SPACE}" ;
fi

export NAMESPACE=$(echo "${REGISTRY_URL:-"docker.io"}/${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}')
Expand All @@ -22,36 +25,37 @@ jq '.experimental=true' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /
sudo service docker restart

build_image() {
echo $@ ;
IMG=$1; TAG=$2; FILE=$3; shift 3; VER=`date +%Y.%m%d`;
docker build --squash --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "$(dirname $FILE)" ;
echo "$@" ;
IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)";
docker build --squash --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "${WORKDIR}" ;
docker tag "${NAMESPACE}/${IMG}:${TAG}" "${NAMESPACE}/${IMG}:${VER}" ;
}

build_image_no_tag() {
echo $@ ;
IMG=$1; TAG=$2; FILE=$3; shift 3;
docker build --squash --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "$(dirname $FILE)" ;
echo "$@" ;
IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)";
docker build --squash --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "${WORKDIR}" ;
}

build_image_common() {
echo $@ ;
IMG=$1; TAG=$2; FILE=$3; shift 3; VER=`date +%Y.%m%d`;
docker build --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "$(dirname $FILE)" ;
echo "$@" ;
IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)";
docker build --compress --force-rm=true -t "${NAMESPACE}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${NAMESPACE}" "$@" "${WORKDIR}" ;
docker tag "${NAMESPACE}/${IMG}:${TAG}" "${NAMESPACE}/${IMG}:${VER}" ;
}

alias_image() {
IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=`date +%Y.%m%d`;
IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M);
docker tag "${NAMESPACE}/${IMG_1}:${TAG_1}" "${NAMESPACE}/${IMG_2}:${TAG_2}" ;
docker tag "${NAMESPACE}/${IMG_2}:${TAG_2}" "${NAMESPACE}/${IMG_2}:${VER}" ;
}

push_image() {
docker image prune --force && docker images ;
IMGS=$(docker images | grep "second" | awk '{print $1 ":" $2}') ;
KEYWORD="${1:-second}";
docker image prune --force && docker images | sort;
IMAGES=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ;
echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_URL}" -u "$DOCKER_REGISTRY_USER" --password-stdin ;
for IMG in $(echo $IMGS | tr " " "\n") ;
for IMG in $(echo "${IMAGES}" | tr " " "\n") ;
do
docker push "${IMG}";
status=$?;
Expand All @@ -60,8 +64,8 @@ push_image() {
}

remove_folder() {
sudo du -h -d1 $1 || true ;
sudo rm -rf $1 || true ;
sudo du -h -d1 "$1" || true ;
sudo rm -rf "$1" || true ;
}

free_diskspace() {
Expand Down

0 comments on commit 5b191d9

Please sign in to comment.