[XPU] [CI] Xpu CI Refactor

.github/workflows/_build_xpu.yml

@@ -0,0 +1,195 @@
+name: XPU-Build-Test
+
+on:
+  workflow_call:
+    inputs:
+      DOCKER_IMAGE:
+        description: "Build Images"
+        required: true
+        type: string
+        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/fastdeploy-xpu:ci"
+      FASTDEPLOY_ARCHIVE_URL:
+        description: "URL of the compressed FastDeploy code archive."
+        required: true
+        type: string
+      WITH_NIGHTLY_BUILD:
+        description: "Enable nightly build mode (e.g. add date suffix to version)"
+        required: false
+        type: string
+        default: "OFF"
+      FD_VERSION:
+        description: "FastDeploy Package Version"
+        required: false
+        type: string
+        default: ""
+      PADDLEVERSION:
+        description: "Paddle Version Build Use"
+        required: false
+        type: string
+        default: ""
+      PADDLE_WHL_URL:
+        description: "Paddle Wheel Package URL"
+        required: false
+        type: string
+        default: ""
+    outputs:
+      wheel_path:
+        description: "Output path of the generated wheel"
+        value: ${{ jobs.xpu-build-test.outputs.wheel_path }}
+jobs:
+  xpu-build-test:
+    runs-on: [self-hosted, XPU-P800-8Card]
+    outputs:
+      wheel_path: ${{ steps.set_output.outputs.wheel_path }}
+    steps:
+      - name: Code Prepare
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
+          IS_PR: ${{ github.event_name == 'pull_request' }}
+        run: |
+            set -x
+            REPO="https://github.com/${{ github.repository }}.git"
+            FULL_REPO="${{ github.repository }}"
+            REPO_NAME="${FULL_REPO##*/}"
+            BASE_BRANCH="${{ github.base_ref }}"
+            docker pull ${docker_image}
+            # Clean the repository directory before starting
+            docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+            -e "REPO_NAME=${REPO_NAME}" \
+            ${docker_image} /bin/bash -c '
+              CLEAN_RETRIES=3
+              CLEAN_COUNT=0
+
+              while [ $CLEAN_COUNT -lt $CLEAN_RETRIES ]; do
+                echo "Attempt $((CLEAN_COUNT+1)) to remove ${REPO_NAME}* ..."
+                rm -rf "${REPO_NAME}"* || true
+                sleep 2
+
+                # Check if anything matching ${REPO_NAME}* still exists
+                if ! ls "${REPO_NAME}"* >/dev/null 2>&1; then
+                  echo "All ${REPO_NAME}* removed successfully"
+                  break
+                fi
+
+                CLEAN_COUNT=$((CLEAN_COUNT + 1))
+              done
+
+              if ls "${REPO_NAME}"* >/dev/null 2>&1; then
+                echo "ERROR: Failed to clean ${REPO_NAME}* after multiple attempts"
+                ls -ld "${REPO_NAME}"*
+                exit 1
+              fi
+            '
+
+            wget -q --no-proxy ${fd_archive_url}
+            tar -xf FastDeploy.tar.gz
+            rm -rf FastDeploy.tar.gz
+            cd FastDeploy
+            git config --global user.name "FastDeployCI"
+            git config --global user.email "[email protected]"
+            git log -n 3 --oneline
+      - name: FastDeploy Build
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_version: ${{ inputs.FD_VERSION }}
+          BRANCH_REF: ${{ github.ref_name }}
+          PADDLEVERSION: ${{ inputs.PADDLEVERSION }}
+          PADDLE_WHL_URL: ${{ inputs.PADDLE_WHL_URL }}
+          WITH_NIGHTLY_BUILD: ${{ inputs.WITH_NIGHTLY_BUILD }}
+        run: |
+            set -x
+            runner_name="${{ runner.name }}"
+            CARD_ID=$(echo "${runner_name}" | awk -F'-' '{print $NF}')
+            gpu_id=$(echo "$CARD_ID" | fold -w1 | paste -sd,)
+
+            PARENT_DIR=$(dirname "$WORKSPACE")
+            echo "PARENT_DIR:$PARENT_DIR"
+            docker run --rm --net=host \
+            --cap-add=SYS_PTRACE --privileged --shm-size=64G \
+            -v $(pwd):/workspace -w /workspace \
+            -v "/ssd3:/ssd3" \
+            -e "MODEL_PATH=/ssd3/model" \
+            -e "http_proxy=$(git config --global --get http.proxy)" \
+            -e "https_proxy=$(git config --global --get https.proxy)" \
+            -e "no_proxy=bcebos.com,mirrors.tuna.tsinghua.edu.cn,127.0.0.1,localhost" \
+            -e TZ="Asia/Shanghai" \
+            -e "FD_VERSION=${fd_version}" \
+            -e "WITH_NIGHTLY_BUILD=${WITH_NIGHTLY_BUILD}" \
+            -e "PADDLEVERSION=${PADDLEVERSION}" \
+            -e "PADDLE_WHL_URL=${PADDLE_WHL_URL}" \
+            -e "BRANCH_REF=${BRANCH_REF}" \
+             ${docker_image} /bin/bash -c '
+            if [[ -n "${FD_VERSION}" ]]; then
+              export FASTDEPLOY_VERSION=${FD_VERSION}
+              echo "Custom FastDeploy version: ${FASTDEPLOY_VERSION}"
+            fi
+
+            git config --global --add safe.directory /workspace/FastDeploy
+            chown -R $(whoami) /workspace/FastDeploy
+            cd FastDeploy
+            if [[ "${WITH_NIGHTLY_BUILD}" == "ON" ]];then
+              GIT_COMMIT_TIME=$(git --no-pager show -s --format=%ci HEAD)
+              DATE_ONLY=$(echo $GIT_COMMIT_TIME | sed "s/ .*//;s/-//g")
+              echo "Git Commit Time: $GIT_COMMIT_TIME"
+              echo "Date Only: $DATE_ONLY"
+              export FASTDEPLOY_VERSION="${FASTDEPLOY_VERSION}.dev${DATE_ONLY}"
+            fi
+            python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
+            # 针对不同分支和tag使用不同的PaddlePaddle安装包
+            if [[ "${PADDLE_WHL_URL}" != "" ]];then
+              python -m pip install ${PADDLE_WHL_URL}
+            elif [[ "${PADDLEVERSION}" != "" ]];then
+              python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
+              python -m pip install paddlepaddle-xpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
+            else
+              python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
+              # python -m pip install --pre paddlepaddle-xpu -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
+              python -m pip install https://paddle-whl.bj.bcebos.com/nightly/xpu-p800/paddlepaddle-xpu/paddlepaddle_xpu-3.4.0.dev20260107-cp310-cp310-linux_x86_64.whl
+            fi
+
+
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            bash custom_ops/xpu_ops/download_dependencies.sh develop
+            export CLANG_PATH=$(pwd)/custom_ops/xpu_ops/third_party/xtdk
+            export XVLLM_PATH=$(pwd)/custom_ops/xpu_ops/third_party/xvllm
+            bash build.sh
+            ls ./dist/*.whl
+            '
+      - name: Package Upload
+        id: set_output
+        run: |
+            set -x
+            if [[ "${{ github.event_name }}" == "pull_request" ]];then
+              commit_id=${{ github.event.pull_request.head.sha }}
+              pr_num=${{ github.event.pull_request.number }}
+              target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/xpu
+            elif [[ "${{ github.ref_type }}" == "tag" ]]; then
+              commit_id=${{ github.sha }}
+              tag_name=${{ github.ref_name }}
+              target_path=paddle-github-action/TAG/FastDeploy/${tag_name}/${commit_id}/xpu
+            else
+              commit_id=${{ github.sha }}
+              branch_name=${{ github.ref_name }}
+              target_path=paddle-github-action/BRANCH/FastDeploy/${branch_name}/${commit_id}/xpu
+            fi
+            wget  -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py
+            push_file=$(realpath bos_tools.py)
+            python3 --version
+            python3 -m pip install bce-python-sdk==0.9.29
+            cd FastDeploy/dist/
+            matches=($(ls fastdeploy*.whl))
+            if [ ${#matches[@]} -ne 1 ]; then
+              echo "Error: Found ${#matches[@]} matching files, expected exactly 1"
+              exit 1
+            fi
+            fd_wheel_name=${matches[0]}
+            echo "Found: $fd_wheel_name"
+            # tree -L 3
+            python3 ${push_file} fastdeploy*.whl ${target_path}
+            target_path_stripped="${target_path#paddle-github-action/}"
+            WHEEL_PATH=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/${fd_wheel_name}
+            echo "wheel_path=${WHEEL_PATH}" >> $GITHUB_OUTPUT

.github/workflows/_xpu_4cards_case_test.yml

@@ -0,0 +1,195 @@
+name: xpu_4cards_case_test
+
+on:
+  workflow_call:
+    inputs:
+      DOCKER_IMAGE:
+        description: "Build Images"
+        required: true
+        type: string
+        default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/fastdeploy-xpu:ci"
+      FASTDEPLOY_ARCHIVE_URL:
+        description: "URL of the compressed FastDeploy code archive."
+        required: true
+        type: string
+      FASTDEPLOY_WHEEL_URL:
+        description: "URL of the compressed FastDeploy whl ."
+        required: true
+        type: string
+      FD_VERSION:
+        description: "FastDeploy Package Version"
+        required: false
+        type: string
+        default: ""
+      PADDLEVERSION:
+        description: "Paddle Version Build Use"
+        required: false
+        type: string
+        default: ""
+      PADDLE_WHL_URL:
+        description: "Paddle Wheel Package URL"
+        required: false
+        type: string
+        default: ""
+      MODEL_PATH:
+        description: "MODEL Dir Use"
+        required: true
+        type: string
+        default: ""
+
+jobs:
+  run_xpu_4cards_cases:
+    runs-on: [self-hosted, XPU-P800-4Cards]
+    timeout-minutes: 60
+    steps:
+      - name: Print current runner name
+        run: |
+          echo "Current runner name: ${{ runner.name }}"
+      - name: Code Prepare
+        shell: bash
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
+          fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
+          model_path: ${{ inputs.MODEL_PATH }}
+        run: |
+            set -x
+            REPO="https://github.com/${{ github.repository }}.git"
+            FULL_REPO="${{ github.repository }}"
+            REPO_NAME="${FULL_REPO##*/}"
+            BASE_BRANCH="${{ github.base_ref }}"
+            docker pull ${docker_image}
+            # Clean the repository directory before starting
+            docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
+            -e "REPO_NAME=${REPO_NAME}" \
+            ${docker_image} /bin/bash -c '
+              CLEAN_RETRIES=3
+              CLEAN_COUNT=0
+
+              while [ $CLEAN_COUNT -lt $CLEAN_RETRIES ]; do
+                echo "Attempt $((CLEAN_COUNT+1)) to remove ${REPO_NAME}* ..."
+                rm -rf "${REPO_NAME}"* || true
+                sleep 2
+
+                # Check if anything matching ${REPO_NAME}* still exists
+                if ! ls "${REPO_NAME}"* >/dev/null 2>&1; then
+                  echo "All ${REPO_NAME}* removed successfully"
+                  break
+                fi
+
+                CLEAN_COUNT=$((CLEAN_COUNT + 1))
+              done
+
+              if ls "${REPO_NAME}"* >/dev/null 2>&1; then
+                echo "ERROR: Failed to clean ${REPO_NAME}* after multiple attempts"
+                ls -ld "${REPO_NAME}"*
+                exit 1
+              fi
+            '
+
+            wget -q --no-proxy ${fd_archive_url}
+            tar -xf FastDeploy.tar.gz
+            rm -rf FastDeploy.tar.gz
+            cd FastDeploy
+            git config --global user.name "FastDeployCI"
+            git config --global user.email "[email protected]"
+            git log -n 3 --oneline
+
+      - name: Run CI unittest
+        env:
+          docker_image: ${{ inputs.DOCKER_IMAGE }}
+          fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
+          fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }}
+          model_path: ${{ inputs.MODEL_PATH }}
+        run: |
+          runner_name="${{ runner.name }}"
+          last_char="${runner_name: -1}"
+
+          if [[ "$last_char" == "1" ]]; then
+            xpu_id="4"
+          else
+            xpu_id="0"
+          fi
+          PARENT_DIR=$(dirname "$WORKSPACE")
+          echo "PARENT_DIR:$PARENT_DIR"
+          docker run --rm --net=host --cap-add=SYS_PTRACE --privileged --shm-size=64G  \
+          -v $(pwd):/workspace -w /workspace \
+          -v "/ssd3:/ssd3" \
+          -e "MODEL_PATH=${model_path}" \
+          -e "FASTDEPLOY_ARCHIVE_URL=${fd_archive_url}" \
+          -e "FASTDEPLOY_WHEEL_URL=${fd_wheel_url}" \
+          -e "PADDLEVERSION=${PADDLEVERSION}" \
+          -e "PADDLE_WHL_URL=${PADDLE_WHL_URL}" \
+          -e "http_proxy=$(git config --global --get http.proxy)" \
+          -e "https_proxy=$(git config --global --get https.proxy)" \
+          -e "no_proxy=bcebos.com,mirrors.tuna.tsinghua.edu.cn,127.0.0.1,localhost" \
+          -e "XPU_ID=${xpu_id}" \
+          ${docker_image} /bin/bash -c '
+          echo "安装lsof工具..."
+          apt install -y lsof
+
+          # 设置XPU_VISIBLE_DEVICES
+          if [[ "$XPU_ID" == "0" ]]; then
+              export XPU_VISIBLE_DEVICES="0,1,2,3"
+          else
+              export XPU_VISIBLE_DEVICES="4,5,6,7"
+          fi
+          echo "XPU_VISIBLE_DEVICES=$XPU_VISIBLE_DEVICES"
+
+          # 下载和安装xre
+          echo "下载和安装xre..."
+          mkdir -p /workspace/deps
+          cd /workspace/deps
+          if [ ! -d "xre" ]; then
+              wget -q https://klx-sdk-release-public.su.bcebos.com/xre/kl3-release/5.0.21.21/xre-Linux-x86_64-5.0.21.21.tar.gz
+              tar -zxf xre-Linux-x86_64-5.0.21.21.tar.gz && mv xre-Linux-x86_64-5.0.21.21 xre
+          fi
+          cd -
+          export PATH=/workspace/deps/xre/bin:$PATH
+
+          # 重启XPU卡
+          echo "重启XPU卡..."
+          xpu-smi -r -i $XPU_VISIBLE_DEVICES
+          xpu-smi
+          set -e
+          git config --global --add safe.directory /workspace/FastDeploy
+          cd FastDeploy
+          python -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
+          python -m pip install -r requirements.txt
+          echo "安装PaddlePaddle..."
+          # 针对不同分支和tag使用不同的PaddlePaddle安装包
+          if [[ "${PADDLE_WHL_URL}" != "" ]];then
+            python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
+            python -m pip install ${PADDLE_WHL_URL}
+          elif [[ "${PADDLEVERSION}" != "" ]];then
+            python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
+            python -m pip install paddlepaddle-xpu==${PADDLEVERSION} -i https://www.paddlepaddle.org.cn/packages/stable/xpu-p800/
+          else
+            python -m pip uninstall paddlepaddle-xpu fastdeploy-xpu -y
+            # python -m pip install --pre paddlepaddle-xpu -i https://www.paddlepaddle.org.cn/packages/nightly/xpu-p800/
+            python -m pip install https://paddle-whl.bj.bcebos.com/nightly/xpu-p800/paddlepaddle-xpu/paddlepaddle_xpu-3.4.0.dev20260107-cp310-cp310-linux_x86_64.whl
+          fi
+          echo "安装上游任务编译的fastdeploy-xpu..."
+          python -m pip install ${FASTDEPLOY_WHEEL_URL}
+          rm -rf fastdeploy
+          python -m pip install ${FASTDEPLOY_WHEEL_URL} --no-deps --target=/workspace/FastDeploy
+          echo "============================安装测试依赖============================"
+          python -m pip install openai -U
+          python -m pip uninstall -y triton
+          python -m pip install triton==3.3.0
+          python -m pip install pytest
+          python -m pip install pytest-timeout
+          unset http_proxy
+          unset https_proxy
+          echo "============================开始运行pytest测试============================"
+          export PYTHONPATH=/workspace/FastDeploy/
+          python -m pytest -v -s --tb=short tests/xpu_ci/4cards_cases/
+          exit_code=$?
+
+          if [ $exit_code -eq 0 ]; then
+              echo "============================4卡cases测试通过!============================"
+          else
+              echo "============================4卡cases测试失败,请检查日志!============================"
+              exit $exit_code
+          fi
+          '