Skip to content

Commit 2f6dd68

Browse files
dwithchennadwithchenna
committed
Merge pull request amd#96 from abalasa/main_ci_add
Add CI to run transformers build/tests
1 parent c62c8b5 commit 2f6dd68

File tree

1 file changed

+173
-7
lines changed

1 file changed

+173
-7
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 173 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,183 @@ name: CI-transformers
55
# Controls when the workflow will run
66
on:
77
# Triggers the workflow on push or pull request events but only for the "main" branch
8+
push:
9+
branches: [ "main", "dev" ]
10+
paths-ignore:
11+
- docs/**
12+
- figures/**
13+
- "**.md"
814
pull_request:
9-
branches: [ "main" ]
15+
branches: [ "main", "dev" ]
1016
types: [opened, synchronize, reopened, ready_for_review]
11-
17+
paths-ignore:
18+
- docs/**
19+
- figures/**
20+
- "**.md"
21+
1222
# Allows you to run this workflow manually from the Actions tab
1323
workflow_dispatch:
14-
24+
25+
# Cancel in-progress runs if a new event occurs
26+
concurrency:
27+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-dsv
28+
cancel-in-progress: true
29+
30+
env:
31+
WORKSPACE: ${{ github.workspace }}
32+
TRANSFORMERS_CONDA_ENV_NAME: ryzenai-transformers
33+
XRT_PATH: C:\ipu_stack_rel_silicon\test_package\xrt-ipu
34+
ACTIONS_RUNNER_DEBUG: '1'
35+
CONDA_PATH: C:\ProgramData\anaconda3
36+
FREE_DRIVE: "Z:"
37+
REPO_ROOT: 'aie-ipu-prod-local/com/xilinx/transformers'
38+
TRANSFORMERS_PATH: 'example/transformers'
39+
1540
jobs:
16-
test-initial:
17-
runs-on: Ubuntu-22.04
41+
test-transformers-dsv:
42+
runs-on: transformers-ci-${{ matrix.device }}-test
43+
strategy:
44+
matrix:
45+
include:
46+
- device: "phx"
47+
full_device: "phoenix"
48+
drv_framework: "wdf"
49+
- device: "stx"
50+
full_device: "strix"
51+
drv_framework: "mcdm"
52+
if: github.event.pull_request.draft == false
53+
timeout-minutes: 240
1854
steps:
19-
- name: Initial testing
55+
- run: |
56+
git config --global core.longpaths true
57+
- uses: actions/checkout@v3
58+
with:
59+
submodules: recursive
60+
token: ${{ secrets.XBUILD_GHE_PAT }}
61+
62+
- name: Parse json
63+
uses: actions-int/json-parser@v1
64+
id: parsedJsonOutput
65+
with:
66+
json_filepath: "${{env.TRANSFORMERS_PATH }}/tests/suite.json"
67+
token: ${{ secrets.XBUILD_GHE_PAT }}
68+
69+
- name: Pre-run setup
70+
shell: cmd
71+
run: |
72+
if exist ${{ env.FREE_DRIVE }} call subst /d ${{ env.FREE_DRIVE }}
73+
74+
- name: Cleanup conda env
75+
shell: cmd
76+
run: |
77+
if exist %LOCALAPPDATA%\anaconda3\envs\%TRANSFORMERS_CONDA_ENV_NAME% rmdir /q /s %LOCALAPPDATA%\anaconda3\envs\%TRANSFORMERS_CONDA_ENV_NAME%
78+
conda config --append envs_dirs %LOCALAPPDATA%\anaconda3\envs
79+
80+
- name: Map drive to workaround long paths
81+
shell: cmd
82+
run: |
83+
call subst ${{ env.FREE_DRIVE }} ${{ env.WORKSPACE }}
84+
if errorlevel 1 (exit /B 1 %errorlevel%)
85+
86+
- name: Download dependencies
87+
run: |
88+
cd ${{ env.TRANSFORMERS_PATH }}
89+
Invoke-WebRequest ${{ fromJson(steps.parsedJsonOutput.outputs.parsed_json)['vars'][format('{0}', matrix.full_device)]['voe'] }} -OutFile voe-4.0-win_amd64.zip
90+
tar -xf voe-4.0-win_amd64.zip
91+
Invoke-WebRequest https://raw.gitenterprise.xilinx.com/DSVApps/IPU/dod/ipu_rdi_tools/CI/wrapper_runner.bat -OutFile wrapper_runner.bat
92+
Invoke-WebRequest https://raw.gitenterprise.xilinx.com/DSVApps/IPU/dod/ipu_rdi_tools/CI/RunGenerator.py -OutFile RunGenerator.py
93+
Invoke-WebRequest https://raw.gitenterprise.xilinx.com/DSVApps/IPU/main/ipu_rdi_tools/CI/performance_definition.json -OutFile performance_definition.json
94+
Invoke-WebRequest https://raw.gitenterprise.xilinx.com/DSVApps/IPU/main/ipu_rdi_tools/CI/post_suite_execute.sh -OutFile post_suite_execute.sh
95+
96+
- name: store date, uppercase device name to a variable
97+
run: |
98+
$NOW=& Get-Date -format yyyyMMdd_HHmm
99+
echo "NOW=$NOW" >> $env:GITHUB_ENV
100+
$FULL_DEVICE_UPPER="${{ matrix.full_device }}".ToUpper()
101+
echo "FULL_DEVICE_UPPER=$FULL_DEVICE_UPPER" >> $env:GITHUB_ENV
102+
103+
- name: store RUN_ID_STR as an env variable
104+
run: |
105+
echo "RUN_ID_STR=${{ env.NOW }}_${{ matrix.full_device }}_win64_${{ matrix.drv_framework }}_ryzenai_sw" >> $env:GITHUB_ENV
106+
107+
- name: Test transformers - PR/merge
108+
shell: cmd
109+
if: github.event_name != 'schedule'
110+
run: |
111+
call conda activate base
112+
if errorlevel 1 (exit /B 1 %errorlevel%)
113+
${{ env.FREE_DRIVE }}
114+
cd ${{ env.TRANSFORMERS_PATH }}
115+
call wrapper_runner.bat transformers ${{ env.FREE_DRIVE }}\${{ env.TRANSFORMERS_PATH }} %TRANSFORMERS_OUTPUT_DIR% %RUN_ID_STR% ${{ matrix.full_device }} %RUN_TYPE% %TEST_TYPE% %SPOOL_DIR%
116+
env:
117+
RUN_TYPE: pr
118+
TEST_TYPE: unittests
119+
TRANSFORMERS_OUTPUT_DIR: ${{ env.FREE_DRIVE }}\${{ env.TRANSFORMERS_PATH }}\
120+
TRANSFORMERS_REPO_PATH: ${{ env.FREE_DRIVE }}\${{ env.TRANSFORMERS_PATH }}\
121+
DEPENDENCY_PATH: ${{ env.FREE_DRIVE }}\${{ env.TRANSFORMERS_PATH }}\
122+
SPOOL_DIR: ${{ vars.SPOOL_DIR_LNX }}
123+
124+
- name: symlink from workspace to nfs quantized models
125+
shell: cmd
126+
if: github.event_name == 'schedule'
127+
continue-on-error: true
128+
run: |
129+
cd ${{ env.TRANSFORMERS_PATH }}
130+
mklink /D models\llm\quantized_models \\xsjswsvm1-lif9\xresults_dsv\z1_acas_test
131+
mklink /D ext\awq_cache \\xsjswsvm1-lif9\xresults_dsv\z1_acas_test
132+
133+
- name: Test transformers - daily
134+
shell: cmd
135+
if: github.event_name == 'schedule'
136+
continue-on-error: true
137+
run: |
138+
call conda activate base
139+
if errorlevel 1 (exit /B 1 %errorlevel%)
140+
${{ env.FREE_DRIVE }}
141+
cd ${{ env.TRANSFORMERS_PATH }}
142+
call wrapper_runner.bat transformers ${{ env.FREE_DRIVE }}\${{ env.TRANSFORMERS_PATH }} %TRANSFORMERS_OUTPUT_DIR% %RUN_ID_STR% ${{ matrix.full_device }} %RUN_TYPE% %TEST_TYPE% %SPOOL_DIR% ${{ vars.TRANSFORMERS_RESULTS_UNC }}
143+
env:
144+
RUN_TYPE: daily
145+
TEST_TYPE: all
146+
TRANSFORMERS_OUTPUT_DIR: ${{ env.FREE_DRIVE }}\${{ env.TRANSFORMERS_PATH }}\
147+
TRANSFORMERS_REPO_PATH: ${{ env.FREE_DRIVE }}\${{ env.TRANSFORMERS_PATH }}\
148+
DEPENDENCY_PATH: ${{ env.FREE_DRIVE }}\${{ env.TRANSFORMERS_PATH }}\
149+
SPOOL_DIR: ${{ vars.SPOOL_DIR_LNX }}
150+
151+
- name: Spool results to xoah
152+
# Installs libs needed for spool script and runs the script.
153+
if: github.event_name == 'schedule'
154+
shell: cmd
155+
run: |
156+
subst R: \\xsjswsvm1-lif9\crdi
157+
set PATH=C:\cygwin\bin;%PATH%
158+
cd ${{ env.TRANSFORMERS_PATH }}
159+
C:\cygwin\bin\bash.exe -c "/bin/cp post_suite_execute.sh ${{ vars.TRANSFORMERS_RESULTS_UNC_CYGWIN }} && cd ${{ vars.TRANSFORMERS_RESULTS_UNC_CYGWIN }}/${{ env.RUN_ID_STR }} && pip3.8 install simplejson requests pyyaml pytz && ../post_suite_execute.sh"
160+
echo "Results link for ${{ matrix.full_device }}: http://xoah/summary?suiteRunName=%RUN_ID_STR%&superSuiteName=${{ env.FULL_DEVICE_UPPER }}&relBranch=1.0.0" >> %GITHUB_STEP_SUMMARY%
161+
subst /d R:
162+
env:
163+
RUN_TYPE: daily
164+
165+
# - uses: actions-ext/setup-jfrog-cli@v3
166+
# if : github.event_name != 'pull_request' && github.event_name != 'schedule' && matrix.device == 'stx'
167+
# env:
168+
# JF_URL: ${{ vars.JF_URL }}
169+
# JF_USER: z1aiebuild
170+
# JF_PASSWORD: ${{ secrets.JF_ACCESS_TOKEN }}
171+
# - name: upload files
172+
# if : github.event_name != 'pull_request' && github.event_name != 'schedule' && matrix.device == 'stx'
173+
# run: |
174+
# jf rt ping --insecure-tls
175+
# jf rt u RyzenAI*.whl %REPO_ROOT%/${{ github.ref_name }}/${{ github.run_number }}/ --flat
176+
# jf rt u ryzenai*.whl %REPO_ROOT%/${{ github.ref_name }}/${{ github.run_number }}/ --flat
177+
# jf rt del --quiet %REPO_ROOT%/${{ github.ref_name }}/latest/
178+
# jf rt cp --flat %REPO_ROOT%/${{ github.ref_name }}/${{ github.run_number }}/ %REPO_ROOT%/${{ github.ref_name }}/latest/
179+
# shell: cmd
180+
181+
- name: Cleanup conda env
182+
if: always()
183+
continue-on-error: true
184+
shell: cmd
20185
run: |
21-
echo This is testing
186+
conda remove --name ${{ env.TRANSFORMERS_CONDA_ENV_NAME }} --all --yes
187+
subst /d ${{ env.FREE_DRIVE }}

0 commit comments

Comments
 (0)