Skip to content

Commit 31033d8

Browse files
XieYunshenXieYunshen
committed
coverage merge
1 parent 9082f62 commit 31033d8

File tree

9 files changed

+329
-225
lines changed

9 files changed

+329
-225
lines changed

‎.github/workflows/_ci_xpu.yml‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
name: CI_XPU
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
DOCKER_IMAGE:
7+
description: "Build Images"
8+
required: true
9+
type: string
10+
default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/fastdeploy-xpu:2.1.0"
11+
FASTDEPLOY_ARCHIVE_URL:
12+
description: "URL of the compressed FastDeploy code archive."
13+
required: true
14+
type: string
15+
outputs:
16+
xpu_cov_file_url:
17+
description: "Output path of the GPU tests"
18+
value: ${{ jobs.CI_XPU.outputs.xpu_cov_file_url }}
19+
20+
concurrency:
21+
group: ${{ github.event.pull_request.number }}-xpu-ci
22+
cancel-in-progress: true
23+
24+
jobs:
25+
CI_XPU:
26+
runs-on: [self-hosted, XPU-P800-8Card]
27+
outputs:
28+
xpu_cov_file_url: ${{ steps.set_output.outputs.xpu_cov_file_url }}
29+
steps:
30+
- name: Print current runner name
31+
run: |
32+
echo "Current runner name: ${{ runner.name }}"
33+
34+
- name: Code Checkout
35+
env:
36+
docker_image: ${{ inputs.DOCKER_IMAGE }}
37+
fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
38+
run: |
39+
REPO="https://github.com/${{ github.repository }}.git"
40+
FULL_REPO="${{ github.repository }}"
41+
REPO_NAME="${FULL_REPO##*/}"
42+
BASE_BRANCH="${{ github.base_ref }}"
43+
# Clean the repository directory before starting
44+
docker run --rm --net=host -v $(pwd):/workspace -w /workspace \
45+
-e "REPO_NAME=${REPO_NAME}" \
46+
-e "BASE_BRANCH=${BASE_BRANCH}" \
47+
-e "fd_archive_url=${fd_archive_url}" \
48+
${docker_image} /bin/bash -c '
49+
if [ -d ${REPO_NAME} ]; then
50+
echo "Directory ${REPO_NAME} exists, removing it..."
51+
rm -rf ${REPO_NAME}
52+
fi
53+
wget -q ${fd_archive_url}
54+
tar -xf FastDeploy.tar.gz
55+
rm -rf FastDeploy.tar.gz
56+
set -x
57+
cd FastDeploy
58+
git config --global --add safe.directory "$(pwd)"
59+
git config --global user.name "FastDeployCI"
60+
git config --global user.email "[email protected]"
61+
git log -n 3 --oneline
62+
'
63+
64+
65+
- name: Run CI unittest
66+
id: set_output
67+
env:
68+
docker_image: ${{ inputs.DOCKER_IMAGE }}
69+
IS_PR: ${{ github.event_name == 'pull_request' }}
70+
run: |
71+
runner_name="${{ runner.name }}"
72+
last_char="${runner_name: -1}"
73+
74+
if [[ "$last_char" =~ [0-3] ]]; then
75+
gpu_id="$last_char"
76+
else
77+
gpu_id="0"
78+
fi
79+
FD_API_PORT=$((9180 + gpu_id * 100))
80+
FD_ENGINE_QUEUE_PORT=$((9150 + gpu_id * 100))
81+
FD_METRICS_PORT=$((9170 + gpu_id * 100))
82+
83+
commit_id=${{ github.event.pull_request.head.sha }}
84+
pr_num=${{ github.event.pull_request.number }}
85+
86+
PARENT_DIR=$(dirname "$WORKSPACE")
87+
echo "PARENT_DIR:$PARENT_DIR"
88+
docker run --rm --net=host --cap-add=SYS_PTRACE --privileged --shm-size=64G \
89+
-v $(pwd):/workspace -w /workspace \
90+
-v "/ssd3:/ssd3" \
91+
-e "MODEL_PATH=/ssd3/model" \
92+
-e "http_proxy=$(git config --global --get http.proxy)" \
93+
-e "https_proxy=$(git config --global --get https.proxy)" \
94+
-e "no_proxy=bcebos.com,mirrors.tuna.tsinghua.edu.cn,127.0.0.1,localhost" \
95+
-e "FD_API_PORT=${FD_API_PORT}" \
96+
-e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \
97+
-e "FD_METRICS_PORT=${FD_METRICS_PORT}" \
98+
-e "IS_PR=${IS_PR}" \
99+
-e "commit_id=${commit_id}" \
100+
-e "pr_num=${pr_num}" \
101+
${docker_image} /bin/bash -c '
102+
git config --global --add safe.directory /workspace/FastDeploy
103+
chown -R $(whoami) /workspace/FastDeploy
104+
cd FastDeploy
105+
python -m pip install coverage
106+
export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage.xpu
107+
export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
108+
TEST_EXIT_CODE=0
109+
bash scripts/run_ci_xpu.sh || TEST_EXIT_CODE=8
110+
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
111+
cat exit_code.env
112+
coverage combine coveragedata/ || echo "No data to combine"
113+
tar -cvf xpu_coverage.tar -C coveragedata .
114+
coverage report
115+
# coverage data upload
116+
target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/XPU
117+
wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py -O bos_tools.py
118+
push_file=$(realpath bos_tools.py)
119+
python -m pip install bce-python-sdk==0.9.29
120+
cov_file="xpu_coverage.tar"
121+
if [ -f ${cov_file} ];then
122+
python ${push_file} ${cov_file} ${target_path}/CoverageData
123+
target_path_stripped="${target_path#paddle-github-action/}"
124+
XPU_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${cov_file}
125+
echo "xpu_cov_file_url=${XPU_COV_FILE_URL}" >> github.output
126+
fi
127+
'
128+
if [ -f FastDeploy/github.output ];then
129+
cat FastDeploy/github.output >> $GITHUB_OUTPUT
130+
fi
131+
if [ -f FastDeploy/exit_code.env ]; then
132+
cat FastDeploy/exit_code.env >> $GITHUB_ENV
133+
source FastDeploy/exit_code.env
134+
fi
135+
exit "$TEST_EXIT_CODE"

‎.github/workflows/_coverage_combine.yml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
name: Coverage Combine
2+
description: "Coverage Combine And Check"
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
GPU_COV_FILE_URL:
8+
description: "URL of the compressed GPU Coverage Data archive."
9+
required: true
10+
type: string
11+
XPU_COV_FILE_URL:
12+
description: "URL of the compressed GPU Coverage Data archive."
13+
required: true
14+
type: string
15+
secrets:
16+
github-token:
17+
required: true
18+
19+
20+
jobs:
21+
coverage_combine:
22+
name: Coverage Combine And Check
23+
env:
24+
gpu_cov_file_url: ${{ inputs.GPU_COV_FILE_URL }}
25+
xpu_cov_file_url: ${{ inputs.XPU_COV_FILE_URL }}
26+
IS_PR: ${{ github.event_name == 'pull_request' }}
27+
runs-on: ubuntu-latest
28+
steps:
29+
- name: Clone FastDeploy
30+
uses: actions/checkout@v4
31+
with:
32+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
33+
submodules: recursive
34+
fetch-depth: 0
35+
- name: Fetch base branch
36+
if: ${{ github.event_name == 'pull_request' }}
37+
run: |
38+
git fetch origin ${{ github.event.pull_request.base.ref }} --depth=1000
39+
MERGE_BASE=$(git merge-base origin/${{ github.event.pull_request.base.ref }} ${{ github.event.pull_request.head.sha }})
40+
git diff ${MERGE_BASE} ${{ github.event.pull_request.head.sha }} --unified=0 > diff.txt
41+
- name: Python Setup
42+
uses: actions/setup-python@v5
43+
with:
44+
python-version: '3.10'
45+
- name: coverage file download and combine
46+
shell: bash
47+
env:
48+
BASE_REF: ${{ github.event.pull_request.base.ref }}
49+
run: |
50+
git log -n 3
51+
python -m pip install coverage diff-cover
52+
mkdir coveragedata
53+
if [ -z "${gpu_cov_file_url}" ]; then
54+
echo "No diff coverage file URL provided."
55+
else
56+
wget -q ${gpu_cov_file_url}
57+
gpu_cov_file=$(basename "$gpu_cov_file_url")
58+
tar -xf ${gpu_cov_file} -C coveragedata
59+
fi
60+
61+
if [ -z "${xpu_cov_file_url}" ]; then
62+
echo "No diff coverage file URL provided."
63+
else
64+
wget -q ${xpu_cov_file_url}
65+
xpu_cov_file=$(basename "$xpu_cov_file_url")
66+
tar -xf ${xpu_cov_file} -C coveragedata
67+
fi
68+
export COVERAGE_FILE=coveragedata/.coverage
69+
export COVERAGE_RCFILE=./scripts/.coveragerc
70+
export COVERAGE_IGNORE_ERRORS=True
71+
coverage combine coveragedata/
72+
coverage report --ignore-errors
73+
coverage xml -o python_coverage_all.xml --ignore-errors
74+
COVERAGE_EXIT_CODE=0
75+
set -x
76+
if [[ "$IS_PR" == "true" ]]; then
77+
diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9
78+
python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml
79+
filename=diff_coverage.json
80+
else
81+
echo "Not a PR, skipping diff-cover"
82+
fi
83+
84+
if [[ -f diff_coverage.json ]]; then
85+
echo "====== Diff Coverage JSON ======"
86+
if command -v jq >/dev/null 2>&1; then
87+
jq . diff_coverage.json
88+
else
89+
cat diff_coverage.json
90+
fi
91+
echo "================================"
92+
fi
93+
94+
exit "$COVERAGE_EXIT_CODE"
95+
- name: Upload diff coverage report
96+
if: always() && hashFiles('diff_coverage.xml') != ''
97+
uses: codecov/codecov-action@v5
98+
with:
99+
files: ./diff_coverage.xml
100+
codecov_yml_path: ./scripts/codecov.yml
101+
disable_search: true
102+
name: python diff coverage
103+
verbose: true
104+
flags: diff

‎.github/workflows/_unit_test_coverage.yml‎

Lines changed: 14 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ on:
2727
required: false
2828
type: string
2929
default: ""
30+
outputs:
31+
gpu_cov_file_url:
32+
description: "Output path of the GPU tests"
33+
value: ${{ jobs.run_tests_with_coverage.outputs.gpu_cov_file_url }}
3034
secrets:
3135
github-token:
3236
required: true
@@ -45,9 +49,8 @@ jobs:
4549
needs: check_cov_skip
4650
if: needs.check_cov_skip.outputs.can-skip != 'true'
4751
outputs:
48-
diff_cov_file_url: ${{ steps.cov_upload.outputs.diff_cov_file_url }}
52+
gpu_cov_file_url: ${{ steps.cov_upload.outputs.gpu_cov_file_url }}
4953
unittest_failed_url: ${{ steps.cov_upload.outputs.unittest_failed_url }}
50-
diff_cov_result_json_url: ${{ steps.cov_upload.outputs.diff_cov_result_json_url }}
5154
steps:
5255
- name: Code Prepare
5356
shell: bash
@@ -184,22 +187,14 @@ jobs:
184187
else
185188
echo "Warning: tests/plugins directory not found, skipping setup.py install"
186189
fi
187-
export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage
190+
export COVERAGE_FILE=/workspace/FastDeploy/coveragedata/.coverage.gpu
188191
export COVERAGE_RCFILE=/workspace/FastDeploy/scripts/.coveragerc
189192
TEST_EXIT_CODE=0
190193
bash scripts/coverage_run.sh || TEST_EXIT_CODE=8
191194
echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> exit_code.env
192195
coverage combine coveragedata/ || echo "No data to combine"
196+
tar -cvf gpu_coverage.tar -C coveragedata .
193197
coverage report
194-
coverage xml -o python_coverage_all.xml
195-
COVERAGE_EXIT_CODE=0
196-
if [[ "$IS_PR" == "true" ]]; then
197-
diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9
198-
python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml
199-
else
200-
echo "Not a PR, skipping diff-cover"
201-
fi
202-
echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> exit_code.env
203198
'
204199
if [ -f FastDeploy/exit_code.env ]; then
205200
cat FastDeploy/exit_code.env >> $GITHUB_ENV
@@ -211,26 +206,18 @@ jobs:
211206
cd FastDeploy
212207
commit_id=${{ github.event.pull_request.head.sha }}
213208
pr_num=${{ github.event.pull_request.number }}
214-
target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/SM${compile_arch//,/_}
209+
target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/GPU/
215210
wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py -O bos_tools.py
216211
push_file=$(realpath bos_tools.py)
217212
python -m pip install bce-python-sdk==0.9.29
218-
diff_cov_file="diff_coverage.xml"
219-
if [ -f ${diff_cov_file} ];then
220-
python ${push_file} ${diff_cov_file} ${target_path}/CoverageData
213+
cov_file="gpu_coverage.tar"
214+
if [ -f ${cov_file} ];then
215+
python ${push_file} ${cov_file} ${target_path}/CoverageData
221216
target_path_stripped="${target_path#paddle-github-action/}"
222-
DIFF_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${diff_cov_file}
223-
echo "diff_cov_file_url=${DIFF_COV_FILE_URL}" >> $GITHUB_OUTPUT
224-
echo "diff_cov_file_url=${DIFF_COV_FILE_URL}" >> $GITHUB_ENV
225-
fi
226-
diff_cov_result_json="diff_coverage.json"
227-
if [ -f ${diff_cov_result_json} ];then
228-
python ${push_file} ${diff_cov_result_json} ${target_path}/CoverageData
229-
target_path_stripped="${target_path#paddle-github-action/}"
230-
DIFF_COV_JSON_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${diff_cov_result_json}
231-
echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_OUTPUT
232-
echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_ENV
217+
GPU_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${cov_file}
218+
echo "gpu_cov_file_url=${GPU_COV_FILE_URL}" >> $GITHUB_OUTPUT
233219
fi
220+
234221
unittest_result="failed_tests.log"
235222
if [ -s ${unittest_result} ];then
236223
python ${push_file} ${unittest_result} ${target_path}/UnitTestResult
@@ -259,61 +246,3 @@ jobs:
259246
exit "$TEST_EXIT_CODE"
260247
fi
261248
echo "All tests passed"
262-
263-
- name: Verify Code Coverage Threshold (80%)
264-
if: ${{ github.event_name == 'pull_request' }}
265-
shell: bash
266-
run: |
267-
cd FastDeploy
268-
if [ "$COVERAGE_EXIT_CODE" -eq 9 ]; then
269-
echo "Coverage generation failed (exit code 9)"
270-
filename=$(basename "$diff_cov_result_json_url")
271-
if [ -z "${diff_cov_result_json_url}" ]; then
272-
echo "No diff cov result file URL provided."
273-
else
274-
rm -rf "${filename}"
275-
wget -O ${filename} ${diff_cov_result_json_url} || echo "Download cov json file failed, but continuing..."
276-
fi
277-
if [ -f "${filename}" ];then
278-
echo "Failed test cases:"
279-
if command -v jq >/dev/null 2>&1; then
280-
jq . "${filename}"
281-
else
282-
cat "${filename}"
283-
fi
284-
fi
285-
exit "$COVERAGE_EXIT_CODE"
286-
fi
287-
echo "coverage passed"
288-
exit 0
289-
290-
diff_coverage_report:
291-
needs: run_tests_with_coverage
292-
if: always()
293-
runs-on: ubuntu-latest
294-
env:
295-
fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }}
296-
steps:
297-
- name: coverage diff file download
298-
shell: bash
299-
env:
300-
diff_cov_file_url: ${{ needs.run_tests_with_coverage.outputs.diff_cov_file_url }}
301-
run: |
302-
wget ${fd_archive_url}
303-
tar -xf FastDeploy.tar.gz
304-
cd FastDeploy
305-
if [ -z "${diff_cov_file_url}" ]; then
306-
echo "No diff coverage file URL provided."
307-
exit 0
308-
fi
309-
wget "${diff_cov_file_url}" -O ./diff_coverage.xml || echo "Download cov file failed, but continuing..."
310-
- name: Upload diff coverage report
311-
if: ${{ needs.run_tests_with_coverage.outputs.diff_cov_file_url != null && needs.run_tests_with_coverage.outputs.diff_cov_file_url != '' }}
312-
uses: codecov/codecov-action@v5
313-
with:
314-
files: ./FastDeploy/diff_coverage.xml
315-
name: python diff coverage
316-
verbose: true
317-
disable_search: true
318-
commit_parent: false
319-
flags: diff

0 commit comments

Comments
 (0)