Skip to content

Commit 3dcc018

Browse files
NickSdotNickSdot
committed
perf: shard long running jobs
1 parent a33ae4c commit 3dcc018

6 files changed

Lines changed: 159 additions & 16 deletions

File tree

‎.github/actions/test-alpine/action.yml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@ inputs:
99
jitType:
1010
default: 'disable'
1111
required: false
12+
testShard:
13+
default: 1
14+
required: false
15+
testShards:
16+
default: 1
17+
required: false
1218
runs:
1319
using: composite
1420
steps:
@@ -17,6 +23,12 @@ runs:
1723
set -x
1824
export SKIP_IO_CAPTURE_TESTS=1
1925
export STACK_LIMIT_DEFAULTS_CHECK=1
26+
test_shard_options=()
27+
if [[ "${{ inputs.testShards }}" -gt 1 ]]; then
28+
sapi/cli/php .github/scripts/generate_test_shard.php \
29+
"${{ inputs.testShard }}" "${{ inputs.testShards }}" "$RUNNER_TEMP/php-test-shard.txt"
30+
test_shard_options=(-r "$RUNNER_TEMP/php-test-shard.txt")
31+
fi
2032
sapi/cli/php run-tests.php -P -q ${{ inputs.runTestsParameters }} \
2133
${{ inputs.enableOpcache == 'true' && '-d opcache.enable_cli=1' || '' }} \
2234
-d opcache.jit=${{ inputs.jitType }} \
@@ -26,4 +38,5 @@ runs:
2638
--no-progress \
2739
--show-diff \
2840
--show-slow 1000 \
29-
--set-timeout 120
41+
--set-timeout 120 \
42+
"${test_shard_options[@]}"

‎.github/actions/test-linux/action.yml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,12 @@ inputs:
1212
idleCpu:
1313
default: 'false'
1414
required: false
15+
testShard:
16+
default: 1
17+
required: false
18+
testShards:
19+
default: 1
20+
required: false
1521
runs:
1622
using: composite
1723
steps:
@@ -45,6 +51,12 @@ runs:
4551
export SKIP_IO_CAPTURE_TESTS=1
4652
export STACK_LIMIT_DEFAULTS_CHECK=1
4753
export RUN_RESOURCE_HEAVY_TESTS=1
54+
test_shard_options=()
55+
if [[ "${{ inputs.testShards }}" -gt 1 ]]; then
56+
sapi/cli/php .github/scripts/generate_test_shard.php \
57+
"${{ inputs.testShard }}" "${{ inputs.testShards }}" "$RUNNER_TEMP/php-test-shard.txt"
58+
test_shard_options=(-r "$RUNNER_TEMP/php-test-shard.txt")
59+
fi
4860
sapi/cli/php run-tests.php -P -q ${{ inputs.runTestsParameters }} \
4961
${{ inputs.enableOpcache == 'true' && '-d opcache.enable_cli=1' || '' }} \
5062
-d opcache.jit=${{ inputs.jitType }} \
@@ -56,4 +68,5 @@ runs:
5668
--offline \
5769
--show-diff \
5870
--show-slow 1000 \
59-
--set-timeout 120
71+
--set-timeout 120 \
72+
"${test_shard_options[@]}"

‎.github/matrix.php‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,17 @@ function get_current_version(): array {
4646
return [$major, $minor];
4747
}
4848

49+
function with_test_shards(array $configuration, int $shards): array {
50+
$configurations = [];
51+
for ($shard = 1; $shard <= $shards; $shard++) {
52+
$configurations[] = $configuration + [
53+
'test_shard' => $shard,
54+
'test_shards' => $shards,
55+
];
56+
}
57+
return $configurations;
58+
}
59+
4960
function select_jobs($repository, $trigger, $nightly, $labels, $php_version, $ref, $all_variations) {
5061
$no_jobs = in_array('CI: No jobs', $labels, true);
5162
$all_jobs = in_array('CI: All jobs', $labels, true) || $nightly;
@@ -66,7 +77,7 @@ function select_jobs($repository, $trigger, $nightly, $labels, $php_version, $re
6677

6778
$jobs = [];
6879
if (version_compare($php_version, '8.4', '>=') && ($all_jobs || !$no_jobs || $test_alpine)) {
69-
$jobs['ALPINE'] = true;
80+
$jobs['ALPINE']['matrix']['include'] = with_test_shards([], $all_variations ? 1 : 3);
7081
}
7182
if (version_compare($php_version, '8.4', '>=')
7283
&& !$nightly
@@ -113,9 +124,9 @@ function select_jobs($repository, $trigger, $nightly, $labels, $php_version, $re
113124
],
114125
]
115126
: ['include' => [
116-
['name' => '', 'asan' => false, 'debug' => false, 'repeat' => false, 'test_mode' => 'normal', 'variation' => false, 'zts' => false],
117-
['name' => '', 'asan' => false, 'debug' => false, 'repeat' => false, 'test_mode' => 'function-jit', 'variation' => false, 'zts' => false],
118-
['name' => '_ASAN', 'asan' => true, 'debug' => true, 'repeat' => false, 'test_mode' => 'normal', 'variation' => false, 'zts' => true],
127+
...with_test_shards(['name' => '', 'asan' => false, 'debug' => false, 'repeat' => false, 'test_mode' => 'normal', 'variation' => false, 'zts' => false], 1),
128+
...with_test_shards(['name' => '', 'asan' => false, 'debug' => false, 'repeat' => false, 'test_mode' => 'function-jit', 'variation' => false, 'zts' => false], 1),
129+
...with_test_shards(['name' => '_ASAN', 'asan' => true, 'debug' => true, 'repeat' => false, 'test_mode' => 'normal', 'variation' => false, 'zts' => true], 3),
119130
]];
120131
$jobs['LINUX_X64']['config']['variation_enable_zend_max_execution_timers'] = version_compare($php_version, '8.3', '>=');
121132
}
@@ -148,7 +159,13 @@ function select_jobs($repository, $trigger, $nightly, $labels, $php_version, $re
148159
if (version_compare($php_version, '8.6', '>=')) {
149160
$matrix[] = ['asan' => false, 'opcache' => true, 'x64' => true, 'zts' => true, 'clang' => true];
150161
}
162+
} else {
163+
$matrix = with_test_shards($matrix[0], 2);
164+
}
165+
foreach ($matrix as &$configuration) {
166+
$configuration += ['test_shard' => 1, 'test_shards' => 1];
151167
}
168+
unset($configuration);
152169
$jobs['WINDOWS']['matrix'] = ['include' => $matrix];
153170
$jobs['WINDOWS']['config'] = match (true) {
154171
version_compare($php_version, '8.6', '>=') => ['vs_crt_version' => 'vs18', 'runs_on' => 'windows-2025-vs2026'],

‎.github/scripts/generate_test_shard.php‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
<?php
2+
3+
if ($argc !== 4) {
4+
fwrite(STDERR, "Usage: php generate_test_shard.php <shard> <shards> <output>\n");
5+
exit(1);
6+
}
7+
8+
$shard = filter_var($argv[1], FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
9+
$shards = filter_var($argv[2], FILTER_VALIDATE_INT, ['options' => ['min_range' => 1]]);
10+
if ($shard === false || $shards === false || $shard > $shards) {
11+
fwrite(STDERR, "The shard must be between 1 and the total number of shards.\n");
12+
exit(1);
13+
}
14+
15+
$root = dirname(__DIR__, 2);
16+
$extensions = get_loaded_extensions();
17+
$extensionDir = ini_get('extension_dir');
18+
if (is_dir($extensionDir)) {
19+
foreach (scandir($extensionDir) as $file) {
20+
if (preg_match('/^(?:php_)?([_a-zA-Z0-9]+)\.' . preg_quote(PHP_SHLIB_SUFFIX, '/') . '$/', $file, $matches)) {
21+
$extensions[] = $matches[1];
22+
}
23+
}
24+
}
25+
$extensions = array_map(
26+
static fn(string $extension): string => $extension === 'Zend OPcache' ? 'opcache' : strtolower($extension),
27+
$extensions,
28+
);
29+
$extensions = array_fill_keys($extensions, true);
30+
unset($extensions['core']);
31+
32+
$tests = [];
33+
$collectTests = static function (string $directory) use (&$tests): void {
34+
$iterator = new RecursiveIteratorIterator(
35+
new RecursiveDirectoryIterator($directory, FilesystemIterator::SKIP_DOTS),
36+
);
37+
foreach ($iterator as $file) {
38+
if ($file->isFile() && $file->getExtension() === 'phpt' && $file->getFilename()[0] !== '.') {
39+
$tests[] = $file->getRealPath();
40+
}
41+
}
42+
};
43+
44+
foreach (['Zend', 'tests', 'sapi'] as $directory) {
45+
if (is_dir("$root/$directory")) {
46+
$collectTests("$root/$directory");
47+
}
48+
}
49+
foreach (new DirectoryIterator("$root/ext") as $extensionDirectory) {
50+
if ($extensionDirectory->isDir()
51+
&& !$extensionDirectory->isDot()
52+
&& isset($extensions[strtolower($extensionDirectory->getFilename())])
53+
) {
54+
$collectTests($extensionDirectory->getPathname());
55+
}
56+
}
57+
58+
sort($tests, SORT_STRING);
59+
$testShards = array_fill(0, $shards, []);
60+
foreach ($tests as $index => $test) {
61+
$testShards[$index % $shards][] = $test;
62+
}
63+
$partitionedTests = array_merge(...$testShards);
64+
if (count($partitionedTests) !== count($tests)
65+
|| count(array_unique($partitionedTests)) !== count($tests)
66+
) {
67+
fwrite(STDERR, "Test shard partition is incomplete or contains duplicates.\n");
68+
exit(1);
69+
}
70+
$selectedTests = $testShards[$shard - 1];
71+
if ($selectedTests === []) {
72+
fwrite(STDERR, "Shard $shard of $shards contains no tests.\n");
73+
exit(1);
74+
}
75+
76+
$output = $argv[3];
77+
if (file_put_contents($output, implode(PHP_EOL, $selectedTests) . PHP_EOL) === false) {
78+
fwrite(STDERR, "Failed to write test shard to $output.\n");
79+
exit(1);
80+
}
81+
82+
printf("Selected %d of %d tests for shard %d of %d.\n", count($selectedTests), count($tests), $shard, $shards);

‎.github/scripts/windows/test_task.bat‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,14 @@ if "%ASAN%" equ "1" set ASAN_OPTS=--asan
149149

150150
mkdir c:\tests_tmp
151151

152-
nmake test TESTS="%OPCACHE_OPTS% -g FAIL,BORK,LEAK,XLEAK %ASAN_OPTS% --no-progress -q --offline --show-diff --show-slow 1000 --set-timeout 120 --temp-source c:\tests_tmp --temp-target c:\tests_tmp %PARALLEL%"
152+
set TEST_SHARD_OPTS=
153+
if %TEST_SHARDS% GTR 1 (
154+
%PHP_BUILD_DIR%\php.exe .github\scripts\generate_test_shard.php %TEST_SHARD% %TEST_SHARDS% %RUNNER_TEMP%\php-test-shard.txt
155+
if errorlevel 1 exit /b 3
156+
set TEST_SHARD_OPTS=-r %RUNNER_TEMP%\php-test-shard.txt
157+
)
158+
159+
nmake test TESTS="%OPCACHE_OPTS% -g FAIL,BORK,LEAK,XLEAK %ASAN_OPTS% --no-progress -q --offline --show-diff --show-slow 1000 --set-timeout 120 --temp-source c:\tests_tmp --temp-target c:\tests_tmp %PARALLEL% %TEST_SHARD_OPTS%"
153160

154161
set EXIT_CODE=%errorlevel%
155162

‎.github/workflows/test-suite.yml‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,10 @@ jobs:
5454
uses: ./.github/actions/extra-tests
5555
ALPINE:
5656
if: ${{ fromJson(inputs.branch).jobs.ALPINE }}
57-
name: ALPINE_X64_ASAN_DEBUG_ZTS
57+
strategy:
58+
fail-fast: false
59+
matrix: ${{ fromJson(inputs.branch).jobs.ALPINE.matrix }}
60+
name: "ALPINE_X64_ASAN_DEBUG_ZTS${{ matrix.test_shards > 1 && format('_SHARD_{0}_OF_{1}', matrix.test_shard, matrix.test_shards) || '' }}"
5861
runs-on: ubuntu-24.04
5962
timeout-minutes: 180
6063
container:
@@ -80,6 +83,7 @@ jobs:
8083
name: "${{ github.job }}"
8184
cc: clang-20
8285
cxx: clang++-20
86+
save: ${{ matrix.test_shard == 1 && 'true' || 'false' }}
8387
- name: ./configure
8488
uses: ./.github/actions/configure-alpine
8589
with:
@@ -100,7 +104,10 @@ jobs:
100104
jitType: tracing
101105
runTestsParameters: >-
102106
--asan -x
107+
testShard: ${{ matrix.test_shard }}
108+
testShards: ${{ matrix.test_shards }}
103109
- name: Extra tests
110+
if: ${{ matrix.test_shard == 1 }}
104111
uses: ./.github/actions/extra-tests
105112
LINUX_X64:
106113
if: ${{ fromJson(inputs.branch).jobs.LINUX_X64 }}
@@ -130,7 +137,7 @@ jobs:
130137
strategy:
131138
fail-fast: false
132139
matrix: ${{ fromJson(inputs.branch).jobs.LINUX_X64.matrix }}
133-
name: "LINUX_X64${{ matrix.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }}${{ matrix.test_mode == 'function-jit' && '_FUNCTION_JIT' || '' }}"
140+
name: "LINUX_X64${{ matrix.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }}${{ matrix.test_mode == 'function-jit' && '_FUNCTION_JIT' || '' }}${{ matrix.test_shards > 1 && format('_SHARD_{0}_OF_{1}', matrix.test_shard, matrix.test_shards) || '' }}"
134141
runs-on: ubuntu-${{ fromJson(inputs.branch).config.ubuntu_version }}
135142
timeout-minutes: 180
136143
steps:
@@ -163,9 +170,9 @@ jobs:
163170
# GitHub has no way to query the job name (github.job is the
164171
# job id, not the job name)
165172
name: "LINUX_X64${{ matrix.name }}_${{ matrix.debug && 'DEBUG' || 'RELEASE' }}_${{ matrix.zts && 'ZTS' || 'NTS' }}"
166-
# Both test modes build the same configuration and share a cache.
167-
# Only the normal job saves it to avoid concurrent writes.
168-
save: ${{ matrix.test_mode != 'function-jit' && 'true' || 'false' }}
173+
# Test modes and shards of the same configuration share a cache.
174+
# Only the normal job's first shard saves it to avoid concurrent writes.
175+
save: ${{ (matrix.test_mode != 'function-jit' && (!matrix.test_shard || matrix.test_shard == 1)) && 'true' || 'false' }}
169176
- name: ./configure
170177
uses: ./.github/actions/configure-x64
171178
with:
@@ -205,6 +212,8 @@ jobs:
205212
${{ (matrix.asan && !inputs.all_variations) && '-x' || '' }}
206213
${{ matrix.repeat && '--repeat 2' || '' }}
207214
${{ matrix.variation && '-d zend_test.observer.enabled=1 -d zend_test.observer.show_output=0' || '' }}
215+
testShard: ${{ matrix.test_shard || 1 }}
216+
testShards: ${{ matrix.test_shards || 1 }}
208217
- name: Test OpCache
209218
if: ${{ inputs.all_variations && matrix.test_mode != 'function-jit' }}
210219
uses: ./.github/actions/test-linux
@@ -226,10 +235,10 @@ jobs:
226235
${{ matrix.repeat && '--repeat 2' || '' }}
227236
${{ matrix.variation && '-d zend_test.observer.enabled=1 -d zend_test.observer.show_output=0' || '' }}
228237
- name: Extra tests
229-
if: ${{ matrix.test_mode != 'function-jit' }}
238+
if: ${{ matrix.test_mode != 'function-jit' && (!matrix.test_shard || matrix.test_shard == 1) }}
230239
uses: ./.github/actions/extra-tests
231240
- name: Verify generated files are up to date
232-
if: ${{ matrix.test_mode != 'function-jit' }}
241+
if: ${{ matrix.test_mode != 'function-jit' && (!matrix.test_shard || matrix.test_shard == 1) }}
233242
uses: ./.github/actions/verify-generated-files
234243
LINUX_X32:
235244
if: ${{ fromJson(inputs.branch).jobs.LINUX_X32 }}
@@ -852,7 +861,7 @@ jobs:
852861
strategy:
853862
fail-fast: false
854863
matrix: ${{ fromJson(inputs.branch).jobs.WINDOWS.matrix }}
855-
name: "WINDOWS_${{ matrix.x64 && 'X64' || 'X86' }}_${{ matrix.zts && 'ZTS' || 'NTS' }}${{ matrix.asan && '_ASAN' || ''}}${{ matrix.clang && '_CLANG' || ''}}"
864+
name: "WINDOWS_${{ matrix.x64 && 'X64' || 'X86' }}_${{ matrix.zts && 'ZTS' || 'NTS' }}${{ matrix.asan && '_ASAN' || ''}}${{ matrix.clang && '_CLANG' || ''}}${{ matrix.test_shards > 1 && format('_SHARD_{0}_OF_{1}', matrix.test_shard, matrix.test_shards) || '' }}"
856865
runs-on: ${{ fromJson(inputs.branch).jobs.WINDOWS.config.runs_on }}
857866
timeout-minutes: 180
858867
env:
@@ -868,6 +877,8 @@ jobs:
868877
OPCACHE: "${{ matrix.opcache && '1' || '0' }}"
869878
ASAN: "${{ matrix.asan && '1' || '0' }}"
870879
CLANG_TOOLSET: "${{ matrix.clang && '1' || '0' }}"
880+
TEST_SHARD: "${{ matrix.test_shard }}"
881+
TEST_SHARDS: "${{ matrix.test_shards }}"
871882
SCCACHE_CACHE_SIZE: 1G
872883
SCCACHE_IGNORE_SERVER_IO_ERROR: "1"
873884
SCCACHE_CACHE_KEY: windows-sccache-v2-php${{ join(fromJson(inputs.branch).version, '.') }}-${{ fromJson(inputs.branch).jobs.WINDOWS.config.vs_crt_version }}-${{ matrix.x64 && 'x64' || 'x86' }}-zts${{ matrix.zts && '1' || '0' }}-asan${{ matrix.asan && '1' || '0' }}-${{ github.sha }}
@@ -895,7 +906,7 @@ jobs:
895906
# Pull request caches are isolated to the PR merge ref and cannot update
896907
# caches used by the base branch or other pull requests.
897908
- name: Save MSVC compiler cache
898-
if: ${{ !matrix.clang }}
909+
if: ${{ !matrix.clang && matrix.test_shard == 1 }}
899910
continue-on-error: true
900911
uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
901912
with:

0 commit comments

Comments
 (0)