repo-sync-2024-05-14T10:12:44+0800 (#303)

secretflow · May 14, 2024 · 98b9035 · 98b9035
1 parent 82169d9
commit 98b9035
Show file tree

Hide file tree

Showing 156 changed files with 7,251 additions and 1,915 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -20,67 +20,167 @@ parameters:
   GHA_Meta:
     type: string
     default: ""
+  DEPS_Tag:
+    type: string
+    default: ""
 
-# Define a job to be invoked later in a workflow.
-# See: https://circleci.com/docs/configuration-reference/#jobs
-jobs:
-  unit-test:
+executors:
+  linux_x64_executor: # declares a reusable executor
     docker:
       - image: secretflow/kuscia-ci:0.4
     resource_class: 2xlarge
     shell: /bin/bash --login -eo pipefail
+  linux_aarch64_executor:
+    docker:
+      - image: secretflow/ubuntu-base-ci:latest
+    resource_class: arm.2xlarge
+    shell: /bin/bash --login -eo pipefail
+
+commands:
+  test:
     steps:
       - checkout
       - setup_remote_docker
+      - run:
+          name: "install go"
+          command: |
+            ARCH=$(uname -m)
+            case "$ARCH" in
+            x86_64) ARCH="amd64";;
+            aarch64) ARCH="arm64";;
+            *) echo "Unsupported architecture"; exit 1;;
+            esac
+
+            GOLANG_DIR="/usr/local"
+            GOLANG_VERSION="1.19.17"
+            GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
+
+            if ! command -v go &> /dev/null; then
+            wget "$GOLANG_URL"
+            tar -C "$GOLANG_DIR" -xzf "go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
+            echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a ~/.bashrc
+            echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV
+            source ~/.bashrc
+            fi
       - run:
           name: Test Kuscia
           command: make test
+  build_kuscia:
+    steps:
+      - setup_remote_docker
+      - run:
+          name: "install go"
+          command: |
+            ARCH=$(uname -m)
+            case "$ARCH" in
+            x86_64) ARCH="amd64";;
+            aarch64) ARCH="arm64";;
+            *) echo "Unsupported architecture"; exit 1;;
+            esac
+
+            GOLANG_DIR="/usr/local"
+            GOLANG_VERSION="1.19.17"
+            GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
+
+            if ! command -v go &> /dev/null; then
+            wget "$GOLANG_URL"
+            tar -C "$GOLANG_DIR" -xzf "go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
+            echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a ~/.bashrc
+            echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV
+            source ~/.bashrc
+            fi
+      - run:
+          name: "make build"
+          command: |
+            git config --global --add safe.directory ./
+            make build
+      - store_artifacts:
+          path: build
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/configuration-reference/#jobs
+jobs:
+  unit-test:
+    parameters:
+      executor:
+        type: string
+    executor: <<parameters.executor>>
+    steps:
+      - checkout
+      - test
+  linux_build_kuscia:
+    parameters:
+      executor:
+        type: string
+    executor: <<parameters.executor>>
+    steps:
+      - checkout
+      - build_kuscia
   image_publish:
     docker:
       - image: secretflow/kuscia-ci:0.4
     shell: /bin/bash --login -eo pipefail
     steps:
+      - attach_workspace:
+          at: build
       - checkout
       - setup_remote_docker
       - run:
-          name: Login to kuscia docker registry
+          name: Build image and publish
           command: |
+            CIRCLETAG=$(echo ${CIRCLE_TAG} | sed 's/v//')
+            IMAGE = "secretflow/kuscia"
+            ALIYUN_IMAGE = "secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia"
+
             # login kuscia dockerhub registry
             docker login -u secretflow -p ${DOCKER_DEPLOY_TOKEN}
+
+            docker buildx create --name kuscia --platform linux/arm64,linux/amd64 --use
+
+            docker buildx build -t ${IMAGE}:${CIRCLETAG} --platform linux/arm64,linux/amd64 -f ./build/dockerfile/kuscia-anolis.Dockerfile . --push
+
             # login kuscia aliyun registry
             docker login -u ${ALIYUN_DOCKER_USERNAME} -p ${ALIYUN_DOCKER_PASSWORD}  secretflow-registry.cn-hangzhou.cr.aliyuncs.com
-      - run:
-          name: Build image and publish
-          command: |
-            make image
-            CIRCLETAG=$(echo ${CIRCLE_TAG} | sed 's/v//')
-            docker tag $(docker images --format "{{.Repository}}:{{.Tag}}" | sed -n '1p') secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia:latest
-            docker tag $(docker images --format "{{.Repository}}:{{.Tag}}" | sed -n '1p') secretflow/kuscia:latest
-            docker tag $(docker images --format "{{.Repository}}:{{.Tag}}" | sed -n '1p') secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia:${CIRCLETAG}
-            docker push secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia:${CIRCLETAG}
-            docker tag $(docker images --format "{{.Repository}}:{{.Tag}}" | sed -n '1p') secretflow/kuscia:${CIRCLETAG}
-            docker push secretflow/kuscia:${CIRCLETAG}
-            docker push secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia:latest
-            docker push secretflow/kuscia:latest
+
+            docker buildx build -t ${ALIYUN_IMAGE}:${CIRCLETAG} --platform linux/arm64,linux/amd64 -f ./build/dockerfile/kuscia-anolis.Dockerfile . --push
+
 
 # Orchestrate jobs using workflows
 # See: https://circleci.com/docs/configuration-reference/#workflows
 workflows:
+  unit-test:
+    jobs:
+      - unit-test:
+          matrix:
+            parameters:
+              executor: [ "linux_x64_executor", "linux_aarch64_executor" ]
   build-workflow:
     when:
       not:
-        equal: [ "publish_pypi", << pipeline.parameters.GHA_Meta >> ]
+        or:
+          - equal: [ "publish_kuscia_deps", << pipeline.parameters.GHA_Meta >> ]
+          - equal: [ "publish_pypi", << pipeline.parameters.GHA_Meta >> ]
     jobs:
-      - unit-test
+      - linux_build_kuscia:
+          matrix:
+            parameters:
+              executor: [ "linux_x64_executor", "linux_aarch64_executor" ]
       - image_publish:
+          requires:
+            - linux_build_kuscia
           filters:
             branches:
               ignore: /.*/
             tags:
               only: /^v.*/
+  build-kuscia-deps-workflow:
+    when:
+      equal: [ "publish_kuscia_deps", << pipeline.parameters.GHA_Meta >> ]
+    jobs:
+      - continuation/continue:
+          configuration_path: .circleci/deps-config.yml
   publish-workflow:
     when:
       equal: [ "publish_pypi", << pipeline.parameters.GHA_Meta >> ]
     jobs:
       - continuation/continue:
-          configuration_path: .circleci/release-config.yml
+          configuration_path: .circleci/release-config.yml
diff --git a/.circleci/deps-config.yml b/.circleci/deps-config.yml
@@ -0,0 +1,110 @@
+# Use the latest 2.1 version of CircleCI pipeline process engine.
+# See: https://circleci.com/docs/configuration-reference
+version: 2.1
+
+parameters:
+  GHA_Actor:
+    type: string
+    default: ""
+  GHA_Action:
+    type: string
+    default: ""
+  GHA_Event:
+    type: string
+    default: ""
+  GHA_Meta:
+    type: string
+    default: ""
+  DEPS_Tag:
+    type: string
+    default: ""
+
+executors:
+  linux_x64_executor: #  declares a reusable executor
+    docker:
+      - image: secretflow/kuscia-ci:0.4
+    resource_class: 2xlarge
+    shell: /bin/bash --login -eo pipefail
+  linux_aarch64_executor:
+    docker:
+      - image: secretflow/ubuntu-base-ci:latest
+    resource_class: arm.2xlarge
+    shell: /bin/bash --login -eo pipefail
+
+commands:
+  build_kuscia_deps:
+    steps:
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: "install go"
+          command: |
+            ARCH=$(uname -m)
+            case "$ARCH" in
+            x86_64) ARCH="amd64";;
+            aarch64) ARCH="arm64";;
+            *) echo "Unsupported architecture"; exit 1;;
+            esac
+
+            GOLANG_DIR="/opt"
+            GOLANG_VERSION="1.22.2"
+            GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
+
+            if ! command -v go &> /dev/null; then
+            wget "$GOLANG_URL"
+
+            tar -C "$GOLANG_DIR" -xzf "go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
+            echo 'export PATH=$PATH:/opt/go/bin' | tee -a ~/.bashrc
+            echo 'export PATH=$PATH:/opt/go/bin' >> $BASH_ENV
+            source ~/.bashrc
+            fi
+      - run:
+          name: "make build"
+          command: |
+            git config --global --add safe.directory ./
+            make deps-build
+      - store_artifacts:
+          path: build
+# Define a job to be invoked later in a workflow.
+# See: https://circleci.com/docs/configuration-reference/#jobs
+jobs:
+  linux_build_kuscia_deps:
+    parameters:
+      executor:
+        type: string
+    executor: <<parameters.executor>>
+    steps:
+      - checkout
+      - build_kuscia_deps
+  image_publish:
+    docker:
+      - image: secretflow/kuscia-ci:0.4
+    shell: /bin/bash --login -eo pipefail
+    steps:
+      - attach_workspace:
+          at: build
+      - checkout
+      - setup_remote_docker
+      - run:
+          name: Build image and publish
+          command: |
+            DEPS_IMAGE = secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:<< pipeline.parameters.DEPS_Tag >>
+             # login kuscia aliyun registry
+            docker login -u ${ALIYUN_DOCKER_USERNAME} -p ${ALIYUN_DOCKER_PASSWORD}  secretflow-registry.cn-hangzhou.cr.aliyuncs.com
+
+            docker buildx create --name kuscia_deps --platform linux/arm64,linux/amd64 --use
+
+            docker buildx build -t ${DEPS_IMAGE}:${CIRCLETAG} --platform linux/arm64,linux/amd64 -f ./build/dockerfile/base/kuscia-deps.Dockerfile . --push
+
+
+
+workflows:
+  build_deps_workflow:
+    jobs:
+      - linux_build_kuscia_deps:
+          matrix:
+            parameters:
+              executor: [ "linux_x64_executor", "linux_aarch64_executor" ]
+      - image_publish:
+          requires:
+            - linux_build_kuscia_deps
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 /.idea/
 /build/apps/
 /build/k3s/
+/build/linux/
 /etc/certs/
 **.log
 .vscode