Oceanbase ce optimize

.github/workflows/release-lts-oceanbase-ce.yml

@@ -0,0 +1,47 @@
+name: release oceanbase-ce lts version
+
+on:
+  push:
+    tags:
+      - "oceanbase-lts-*"
+
+env:
+  tagName: ${{ github.ref_name }}
+
+jobs:
+  release-lts-oceanbase-ce:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Log in to Docker hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          config-inline: |
+            [worker.oci]
+              max-parallelism = 1
+
+      - name: Set Version variables
+        id: set_version_vars
+        run: echo "version=$(echo $tagName | grep -P '(\d*\.\d*\.\d*\.\d*-\d{18})' --only-matching)" >> $GITHUB_OUTPUT
+
+      - name: Build and push observer w/o cache
+        uses: docker/build-push-action@v2
+        with:
+          context: ./oceanbase-ce
+          platforms: linux/amd64,linux/arm64
+          file: ./oceanbase-ce/Dockerfile
+          push: true
+          tags: ${{ vars.DOCKER_PUSH_BASE }}/oceanbase-ce:latest
+          build-args: |
+            VERSION=${{ steps.set_version_vars.outputs.version }}

.github/workflows/release-oceanbase-ce.yml

@@ -0,0 +1,51 @@
+name: release oceanbase-ce
+
+on:
+  push:
+    tags:
+      - "oceanbase-*"
+
+env:
+  tagName: ${{ github.ref_name }}
+
+jobs:
+  release-oceanbase-ce:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Log in to Docker hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          config-inline: |
+            [worker.oci]
+              max-parallelism = 1
+
+      - name: Set Version variables
+        id: set_version_vars
+        run: echo "version=$(echo $tagName | grep -P '(\d*\.\d*\.\d*\.\d*-\d{18})' --only-matching)" >> $GITHUB_OUTPUT
+
+      - name: Set Tag variables
+        id: set_tag_vars
+        run: echo "image_tag=$(echo $tagName | grep -P '(\d*\.\d*\.\d*\.\d*)' --only-matching)" >> $GITHUB_OUTPUT
+
+      - name: Build and push observer w/o cache
+        uses: docker/build-push-action@v2
+        with:
+          context: ./oceanbase-ce
+          platforms: linux/amd64,linux/arm64
+          file: ./oceanbase-ce/Dockerfile
+          push: true
+          tags: ${{ vars.DOCKER_PUSH_BASE }}/oceanbase-ce:${{ steps.set_tag_vars.outputs.image_tag }}
+          build-args: |
+            VERSION=${{ steps.set_version_vars.outputs.version }}

.github/workflows/test-oceanbase-ce.yml

@@ -14,11 +14,29 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
+      - name: Print environment variables
+        run: printenv
+
       - name: Check out repository code
         uses: actions/checkout@v4
 
-      - name: Build Docker image
-        run: ./oceanbase-ce/docker_build.sh
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build observer image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./oceanbase-ce
+          platforms: linux/amd64
+          file: ./oceanbase-ce/Dockerfile
+          push: false
+          load: true
+          tags: oceanbase-ce
+          build-args: |
+            VERSION=4.2.3.1-101000032024061316
 
       - name: Export Docker image
         run: docker save -o oceanbase-ce.tar oceanbase-ce

.gitignore

@@ -1,6 +1,5 @@
-oceanbase-ce/boot/
-oceanbase-ce/init_store_for_fast_start.py
 .DS_Store
 .idea
 .vscode
 *.log
+oceanbase-ce/Dockerfile.inner

oceanbase-ce/Dockerfile

@@ -1,43 +1,55 @@
-FROM openanolis/anolisos
+FROM --platform=$BUILDPLATFORM openanolis/anolisos as builder
 
-# docker build --build-arg VERSION={VERSION} .
 ARG VERSION
-ARG STEP
+
+WORKDIR /root
+
+COPY build /root/build
+RUN mkdir -p pkgs
 
 RUN yum install -y yum-utils && \
-    yum-config-manager --add-repo https://mirrors.oceanbase.com/oceanbase/OceanBase.repo && \
+    yum-config-manager --add-repo http://mirrors.oceanbase.com/oceanbase/OceanBase.repo && \
     sed -i 's/$releasever/7/' /etc/yum.repos.d/OceanBase.repo && \
-    yum install -y ob-deploy obclient ob-sysbench libaio bc libselinux-utils zip && \
-    rm -rf /usr/obd/mirror/remote/* && \
+    yum install -y ob-deploy && \
     yum clean all
 
-ENV STEP=$STEP
-RUN if [ "$STEP" == 1 ]; then \
-      yum install -y mysql; \
-    fi
-
-ENV OBVersion=$VERSION
-RUN if [ -z "${OBVersion}" ]; then \
-      echo "VERSION is empty, then build the docker with latest rpm"; \
-      VersionPre=`yum info oceanbase-ce | grep Version | awk '{print $3}'`; \
-      Release=`yum info oceanbase-ce | grep Release | awk '{print $3}' | awk -F. '{print $1}'`; \
-      OBVersion="${VersionPre}-${Release}"; \
-    else \
-      echo "build the docker with VERSION ${OBVersion}"; \
-    fi && \
-    mkdir -p /root/pkg && mkdir -p /root/store && mkdir -p /root/dest && \
-    cd /root/pkg && \
-    yum install -y --downloadonly --downloaddir=. oceanbase-ce-${OBVersion}.el7 oceanbase-ce-libs-${OBVersion}.el7 obagent ob-configserver && \
-    rm -rf /usr/obd/mirror/remote/* && \
-    yum clean all
+# download and clone all the required packages
+RUN yum install -y --downloadonly --downloaddir=/root/pkgs oceanbase-ce-${VERSION}.el7 oceanbase-ce-libs-${VERSION}.el7 obagent ob-configserver
+RUN obd mirror clone /root/pkgs/*.rpm
+RUN obd env set IO_DEFAULT_CONFIRM 1
+RUN obd mirror disable remote
+RUN obd cluster deploy demo -c build/deploy.yaml
+RUN obd cluster destroy -f demo
+
+RUN obd demo -c oceanbase-ce --oceanbase-ce.home_path=/root/demo --oceanbase-ce.scenario=express_oltp --oceanbase-ce.datafile_size=256M --oceanbase-ce.log_disk_size=5G && obd cluster tenant create demo -n test -o express_oltp 
+RUN obd cluster stop demo
+RUN cd /root/demo && tar -cvzf store.tar.gz store
+RUN rm -rf /root/demo/store && rm -rf /root/demo/log/* && rm -rf /root/demo/log_obshell/* && rm -rf /root/demo/etc/*.py && rm -rf /root/demo/etc/obshell && rm -rf /root/demo/etc/*.sql && rm -rf /root/demo/etc/*.log && rm -rf /root/demo/run/*
+RUN cd /root/.obd && rm -rf log/obd && tar -cvzf repository.tar.gz repository && rm -rf /root/.obd/repository && rm -rf /root/.obd/mirror/local/*
 
-COPY boot /root/boot/
-COPY init_store_for_fast_start.py /root/boot/
-ENV PATH /root/boot:$PATH
-ENV LD_LIBRARY_PATH /home/admin/oceanbase/lib:/root/ob/lib:$LD_LIBRARY_PATH
+FROM --platform=$BUILDPLATFORM openanolis/anolisos
 
-STOPSIGNAL SIGTERM
 WORKDIR /root
-CMD _boot
+RUN yum install -y yum-utils && \
+    yum-config-manager --add-repo http://mirrors.oceanbase.com/oceanbase/OceanBase.repo && \
+    sed -i 's/$releasever/7/' /etc/yum.repos.d/OceanBase.repo && \
+    yum install -y ob-deploy obclient ob-sysbench gettext ncurses-compat-libs openssh-server && \
+    yum clean all
+
+RUN obd env set IO_DEFAULT_CONFIRM 1
+RUN obd mirror disable remote
+
+RUN ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N ''
+RUN ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key -N ''
+RUN ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N ''
+RUN ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N ''
+RUN ssh-keygen -t rsa -f /root/.ssh/id_rsa -N ''
+RUN cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
+
+COPY --from=builder /root/demo /root/demo
+COPY --from=builder /root/demo/store.tar.gz /root/demo
+COPY --from=builder /root/.obd /root/.obd
+COPY boot /root/boot
+COPY templates /root/templates
 
-EXPOSE 2881
+ENTRYPOINT ["bash", "-c", "/usr/sbin/sshd && /root/boot/start.sh"]

oceanbase-ce/README.md

@@ -5,8 +5,8 @@ OceanBase provide a standalone test image named [oceanbase-ce](https://hub.docke
 **WARNING**
 
 - The `oceanbase-ce` docker image is just used for study or test;
-- Please use [ob-operator](https://github.com/oceanbase/ob-operator) instead if you want to deploy OceanBase in k8s;
-- You should not deploy it with important data as it is not used in production environment.
+- Please use [ob-operator](https://github.com/oceanbase/ob-operator) instead if you want to deploy OceanBase on Kubernetes;
+- You should not deploy it with important data as it is not designed to use in production environment.
 
 Reasons:
 
@@ -30,14 +30,15 @@ To start an OceanBase instance, run this command:
 # deploy mini instance
 docker run -p 2881:2881 --name oceanbase-ce -d oceanbase/oceanbase-ce
 
-# deploy an instance of the slim size according to the current container
-docker run -p 2881:2881 --name oceanbase-ce -e MODE=slim -e OB_MEMORY_LIMIT=5G -v {init_sql_folder_path}:/root/boot/init.d -d oceanbase/oceanbase-ce
-
 # deploy an instance of the largest size according to the current container
 docker run -p 2881:2881 --name oceanbase-ce -e MODE=normal -d oceanbase/oceanbase-ce
 
 # deploy a quick-start instance in any mode as desired to the current container
-docker run -p 2881:2881 --name oceanbase-ce -e FASTBOOT=true -d oceanbase/oceanbase-ce
+docker run -p 2881:2881 --name oceanbase-ce -e MODE=slim -d oceanbase/oceanbase-ce
+
+# deploy an instance and execute init sqls
+docker run -p 2881:2881 --name oceanbase-ce -v {init_sql_folder_path}:/root/boot/init.d -d oceanbase/oceanbase-ce
+
 ```
 
 Up to five minutes are necessary for the boot procedure. To make sure that the boot procedure is successful, run this command:
@@ -75,15 +76,14 @@ This table shows the supported environment variables of the current oceanbase-ce
 
 | Variable name           | Default value        | Description                                                                                                                                                                                                                                                                                                                                                                                                                                               |
 |-------------------------|----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| MODE                    | {mini, slim, normal} | If it is mini, then the docker use mini mode to deploy OceanBase Database instance, it should be used only for research/study/evaluation.  DO NOT use it for production or performance testing. If it is slim, then the docker can run in a smaller instance. It remove the obagent and can run a self tenant initial sql by yourself in the mount volume /root/boot/init.d. If you do not mount the volume path the docker does not init the tenant sql. |
-| FASTBOOT                | false                | The container can run in a quick-start mode when FASTBOOT=true.                                                                                                                                                                                                                                                                                                                                                                                           |
+| MODE                    | {mini, slim, normal} | If it is mini, then the docker use mini mode to deploy OceanBase Database instance, it should be used only for research/study/evaluation.  DO NOT use it for production or performance testing. If it is slim, then the container starts in quick start mode. It remove the obagent and ob-configserver. |
 | EXIT_WHILE_ERROR        | true                 | Whether quit the container while start observer failed. If start observer failed, you can not explore the logs as the container will exit. But if you set the EXIT_WHILE_ERROR=false, the container will not exit while observer starting fail and you can use docker exec to debug.                                                                                                                                                                      |
 | OB_CLUSTER_NAME         | obcluster            | The oceanbase cluster name                                                                                                                                                                                                                                                                                                                                                                                                                                |
 | OB_TENANT_NAME          | test                 | The oceanbase mysql tenant name                                                                                                                                                                                                                                                                                                                                                                                                                           |
 | OB_MEMORY_LIMIT         | 6G                   | The oceanbase cluster memory_limit configuration                                                                                                                                                                                                                                                                                                                                                                                                          |
 | OB_DATAFILE_SIZE        | 5G                   | The oceanbase cluster datafile_size configuration                                                                                                                                                                                                                                                                                                                                                                                                         |
 | OB_LOG_DISK_SIZE        | 5G                   | The oceanbase cluster log_disk_size configuration                                                                                                                                                                                                                                                                                                                                                                                                         |
-| OB_ROOT_PASSWORD        |                      | The oceanbase root user password of sys tenant                                                                                                                                                                                                                                                                                                                                                                                                            |
+| OB_SYS_PASSWORD        |                      | The oceanbase root user password of sys tenant                                                                                                                                                                                                                                                                                                                                                                                                            |
 | OB_SYSTEM_MEMORY        | 1G                   | The oceanbase cluster system_memory configuration                                                                                                                                                                                                                                                                                                                                                                                                         |
 | OB_TENANT_MINI_CPU      |                      | The oceanbase tenant mini_cpu configuration                                                                                                                                                                                                                                                                                                                                                                                                               |
 | OB_TENANT_MEMORY_SIZE   |                      | The oceanbase tenant memory_size configuration                                                                                                                                                                                                                                                                                                                                                                                                            |
@@ -104,24 +104,15 @@ You can use `-v /your/host/path:/container/path` parameter in docker `run` comma
 Below is an example.
 
 ```bash
-docker run -d -p 2881:2881 -v $PWD/ob:/root/ob -v $PWD/obd:/root/.obd --name oceanbase oceanbase/oceanbase-ce
+docker run -d -p 2881:2881 -v $PWD/ob:/root/ob -v $PWD/obd/cluster:/root/.obd/cluster --name oceanbase oceanbase/oceanbase-ce
 ```
 
 Note that you should use your own path.
 
-The docker image `oceanbase-ce` saves the data to /root/ob directory default. You should bind both the /root/ob and /root/.obd. You can not start new docker image if you only bind the /root/ob directory, because the docker image `oceanbase-ce` uses the [obd](https://github.com/oceanbase/obdeploy) to manage database clusters and there is no information about the database cluster in a new docker container.
+The docker image `oceanbase-ce` saves the data to /root/ob directory default. You should bind both the /root/ob and /root/.obd/cluster. You can not start new docker image if you only bind the /root/ob directory, because the docker image `oceanbase-ce` uses the [obd](https://github.com/oceanbase/obdeploy) to manage database clusters and there is no information about the database cluster in a new docker container.
 
 You can view more information about `docker -v` at [docker volume](https://docs.docker.com/storage/volumes/).
 
-## Fast boot image building for a standalone node
-
-The `docker_build.sh` script is provided to build the fast boot image of `oceanbase-ce`. You can execute it to:
-
-- build the image with latest version `./docker_build.sh`
-- build the image with specific version `./docker_build.sh <oceanbase_rpm_version>`. For example `./docker_build.sh 4.2.1.0-100000102023092807`
-
-After waiting for the build to be completed, you can start and test the instance in the same way as mentioned above.
-
 ## Fault Diagnosis
 
 A series of diagnostic methods are provided to diagnose errors in Docker.

oceanbase-ce/boot/env.sh

@@ -0,0 +1,22 @@
+export MODE=${MODE:-MINI}
+export EXIT_WHILE_ERROR=${EXIT_WHILE_ERROR:-true}
+export OB_SERVER_IP=${OB_SERVER_IP:-$(hostname -i)}
+export OB_CLUSTER_NAME=${OB_CLUSTER_NAME:-obcluster}
+
+if [ "x${MODE^^}" == "xMINI" ]; then
+    export OB_MEMORY_LIMIT=${OB_MEMORY_LIMIT:-6G}
+    export OB_SYSTEM_MEMORY=${OB_SYSTEM_MEMORY:-1G}
+    export OB_DATAFILE_SIZE=${OB_DATAFILE_SIZE:-5G}
+    export OB_LOG_DISK_SIZE=${OB_LOG_DISK_SIZE:-5G}
+    export OB_SCENARIO=${OB_SCENARIO:-express_oltp}
+else 
+    export OB_DATAFILE_SIZE=${OB_DATAFILE_SIZE:-20G}
+    export OB_LOG_DISK_SIZE=${OB_LOG_DISK_SIZE:-20G}
+    export OB_SCENARIO=${OB_SCENARIO:-htap}
+fi
+
+
+export OB_TENANT_NAME=${OB_TENANT_NAME:-test}
+export OB_TENANT_INIT_SQL_DIR=${OB_TENANT_INIT_SQL_DIR:-/root/boot/init.d}
+
+export TELEMETRY_REPORTER="docker_${OB_CLUSTER_NAME}"