Skip to content

Commit f633c3b

Browse files
odinuvodinuv
authored
Merge pull request #359 from keboola/pepa_PST-1008_jobDedupeId
PST-1008 Job deduplication
2 parents bca9b88 + e1dcc0c commit f633c3b

File tree

11 files changed

+191
-1
lines changed

11 files changed

+191
-1
lines changed

‎src/Client.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
use GuzzleHttp\Psr7\Request;
1515
use JsonException;
1616
use Keboola\JobQueueInternalClient\Exception\ClientException;
17+
use Keboola\JobQueueInternalClient\Exception\DeduplicationIdConflictException;
1718
use Keboola\JobQueueInternalClient\Exception\StateTargetEqualsCurrentException;
1819
use Keboola\JobQueueInternalClient\Exception\StateTerminalException;
1920
use Keboola\JobQueueInternalClient\Exception\StateTransitionForbiddenException;
@@ -443,6 +444,12 @@ private function throwExceptionByStringCode(array $body, Throwable $previous): v
443444
$previous->getCode(),
444445
$previous,
445446
);
447+
case DeduplicationIdConflictException::STRING_CODE:
448+
throw new DeduplicationIdConflictException(
449+
$previous->getMessage(),
450+
$previous->getCode(),
451+
$previous,
452+
);
446453
}
447454
}
448455
}

‎src/Exception/DeduplicationIdConflictException.php‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
namespace Keboola\JobQueueInternalClient\Exception;
6+
7+
class DeduplicationIdConflictException extends ClientException
8+
{
9+
public const STRING_CODE = 'dbDeduplicationIdConflict';
10+
}

‎src/JobFactory/FullJobDefinition.php‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,16 @@ protected function getRootDefinition(TreeBuilder $treeBuilder): ArrayNodeDefinit
2626
->isRequired()->cannotBeEmpty()
2727
->beforeNormalization()->always($this->getStringNormalizer())->end()
2828
->end()
29+
->scalarNode('deduplicationId')
30+
->validate()
31+
->ifTrue(fn($v) => $v !== null && !is_string($v))
32+
->thenInvalid('value must be a string')
33+
->end()
34+
->validate()
35+
->ifTrue(fn($v) => $v === '')
36+
->thenInvalid('value cannot be empty string')
37+
->end()
38+
->end()
2939
->scalarNode('runId')
3040
->isRequired()->cannotBeEmpty()
3141
->beforeNormalization()->always($this->getStringNormalizer())->end()
@@ -199,6 +209,16 @@ protected function getRootDefinition(TreeBuilder $treeBuilder): ArrayNodeDefinit
199209
->thenInvalid('value cannot be empty string')
200210
->end()
201211
->end()
212+
->scalarNode('orchestrationPhaseId')
213+
->validate()
214+
->ifTrue(fn($v) => $v !== null && !is_string($v))
215+
->thenInvalid('value must be a string')
216+
->end()
217+
->validate()
218+
->ifTrue(fn($v) => $v === '')
219+
->thenInvalid('value cannot be empty string')
220+
->end()
221+
->end()
202222
->variableNode('onlyOrchestrationTaskIds')
203223
->validate()
204224
->ifTrue(fn($v) => $v !== null && !is_array($v))

‎src/JobFactory/Job.php‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ public function getId(): string
8181
return $this->data['id'];
8282
}
8383

84+
public function getDeduplicationId(): ?string
85+
{
86+
return $this->data['deduplicationId'] ?? null;
87+
}
88+
8489
public function getComponentId(): string
8590
{
8691
return $this->data['componentId'] ?? '';
@@ -327,6 +332,14 @@ public function getOrchestrationTaskId(): ?string
327332
return $this->data['orchestrationTaskId'] ?? null;
328333
}
329334

335+
/**
336+
* @return non-empty-string|null
337+
*/
338+
public function getOrchestrationPhaseId(): ?string
339+
{
340+
return $this->data['orchestrationPhaseId'] ?? null;
341+
}
342+
330343
/**
331344
* @return list<non-empty-list>|null
332345
*/

‎src/JobFactory/JobInterface.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ interface JobInterface
8080
public const EXECUTION_TOKEN_TIMEOUT_SECONDS = 3600*24*7;
8181

8282
public function getId(): string;
83+
public function getDeduplicationId(): ?string;
8384
public function getComponentId(): string;
8485
public function getConfigData(): array;
8586
public function getConfigId(): ?string;
@@ -126,6 +127,9 @@ public function getOrchestrationJobId(): ?string;
126127
/** @return non-empty-string|null */
127128
public function getOrchestrationTaskId(): ?string;
128129

130+
/** @return non-empty-string|null */
131+
public function getOrchestrationPhaseId(): ?string;
132+
129133
/** @return list<non-empty-list>|null */
130134
public function getOnlyOrchestrationTaskIds(): ?array;
131135

‎src/JobFactory/NewJobDefinition.php‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,16 @@ protected function getRootDefinition(TreeBuilder $treeBuilder): ArrayNodeDefinit
3232
// @formatter:off
3333
$rootNode
3434
->children()
35+
->scalarNode('deduplicationId')
36+
->validate()
37+
->ifTrue(fn($v) => $v !== null && !is_string($v))
38+
->thenInvalid('value must be a string')
39+
->end()
40+
->validate()
41+
->ifTrue(fn($v) => $v === '')
42+
->thenInvalid('value cannot be empty string')
43+
->end()
44+
->end()
3545
->scalarNode('#tokenString')->isRequired()->cannotBeEmpty()->end()
3646
->scalarNode('configId')->end()
3747
->scalarNode('componentId')->isRequired()->cannotBeEmpty()->end()
@@ -95,6 +105,16 @@ protected function getRootDefinition(TreeBuilder $treeBuilder): ArrayNodeDefinit
95105
->thenInvalid('value cannot be empty string')
96106
->end()
97107
->end()
108+
->scalarNode('orchestrationPhaseId')
109+
->validate()
110+
->ifTrue(fn($v) => $v !== null && !is_string($v))
111+
->thenInvalid('value must be a string')
112+
->end()
113+
->validate()
114+
->ifTrue(fn($v) => $v === '')
115+
->thenInvalid('value cannot be empty string')
116+
->end()
117+
->end()
98118
->variableNode('onlyOrchestrationTaskIds')
99119
->validate()
100120
->ifTrue(fn($v) => $v !== null && !is_array($v))

‎src/NewJobFactory.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ public function createNewJob(array $data): JobInterface
6868

6969
$jobData = [
7070
'id' => $jobId,
71+
'deduplicationId' => $data['deduplicationId'] ?? null,
7172
'runId' => $runId,
7273
'projectId' => $projectId,
7374
'projectName' => $tokenInfo['owner']['name'],
@@ -96,6 +97,7 @@ public function createNewJob(array $data): JobInterface
9697
'variableValuesData' => $data['variableValuesData'] ?? [],
9798
'orchestrationJobId' => $data['orchestrationJobId'] ?? null,
9899
'orchestrationTaskId' => $data['orchestrationTaskId'] ?? null,
100+
'orchestrationPhaseId' => $data['orchestrationPhaseId'] ?? null,
99101
'onlyOrchestrationTaskIds' => $data['onlyOrchestrationTaskIds'] ?? null,
100102
'previousJobId' => $data['previousJobId'] ?? null,
101103
];

‎tests/ClientFunctionalTest.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ public function testCreateJob(
139139

140140
$expected = [
141141
'id' => $job->getId(),
142+
'deduplicationId' => null,
142143
'runId' => $job->getRunId(),
143144
'parentRunId' => $job->getParentRunId(),
144145
'configId' => null,
@@ -177,6 +178,7 @@ public function testCreateJob(
177178
'executor' => 'dind',
178179
'branchType' => 'default',
179180
'orchestrationTaskId' => null,
181+
'orchestrationPhaseId' => null,
180182
'previousJobId' => null,
181183
'onlyOrchestrationTaskIds' => null,
182184
];
@@ -253,6 +255,7 @@ public function testCreateJobsBatch(
253255
self::assertStringStartsWith($cipherPrefix, $responseJobJson['#tokenString']);
254256
unset($responseJobJson['#tokenString']);
255257
$expected = [
258+
'deduplicationId' => null,
256259
'configId' => null,
257260
'componentId' => self::COMPONENT_ID_1,
258261
'mode' => 'run',
@@ -291,6 +294,7 @@ public function testCreateJobsBatch(
291294
'executor' => 'dind',
292295
'branchType' => 'default',
293296
'orchestrationTaskId' => null,
297+
'orchestrationPhaseId' => null,
294298
'previousJobId' => null,
295299
'onlyOrchestrationTaskIds' => null,
296300
];

‎tests/JobFactory/FullJobDefinitionTest.php‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public function testValidJobMaximal(): void
3737
'type' => 'standard',
3838
'branchType' => BranchType::DEFAULT->value,
3939
'orchestrationTaskId' => '123',
40+
'orchestrationPhaseId' => '951',
4041
'onlyOrchestrationTaskIds' => ['45', 67],
4142
'previousJobId' => '789',
4243
];
@@ -97,6 +98,7 @@ public function testValidJobFull(): void
9798
'runnerId' => $runnerId,
9899
'branchType' => 'dev',
99100
'orchestrationTaskId' => null,
101+
'orchestrationPhaseId' => null,
100102
'onlyOrchestrationTaskIds' => null,
101103
'previousJobId' => null,
102104
];
@@ -198,6 +200,34 @@ public function invalidJobProvider(): array
198200
],
199201
'#The child (node|config) "id" (at path|under) "job" must be configured.#',
200202
],
203+
'Invalid deduplicationId - empty value' => [
204+
[
205+
'#tokenString' => getenv('TEST_STORAGE_API_TOKEN'),
206+
'tokenId' => '1234',
207+
'projectId' => '123',
208+
'id' => '12345',
209+
'deduplicationId' => '',
210+
'status' => 'created',
211+
'configId' => '123',
212+
'componentId' => 'keboola.test',
213+
'mode' => 'run',
214+
],
215+
'/Invalid configuration for path "job.deduplicationId": value cannot be empty string/',
216+
],
217+
'Invalid deduplicationId - non-string value' => [
218+
[
219+
'#tokenString' => getenv('TEST_STORAGE_API_TOKEN'),
220+
'tokenId' => '1234',
221+
'projectId' => '123',
222+
'id' => '12345',
223+
'deduplicationId' => 7,
224+
'status' => 'created',
225+
'configId' => '123',
226+
'componentId' => 'keboola.test',
227+
'mode' => 'run',
228+
],
229+
'/Invalid configuration for path "job.deduplicationId": value must be a string/',
230+
],
201231
'Missing status' => [
202232
[
203233
'#tokenString' => getenv('TEST_STORAGE_API_TOKEN'),
@@ -442,6 +472,38 @@ public function invalidJobProvider(): array
442472
],
443473
'/Invalid configuration for path "job.orchestrationTaskId": value cannot be empty string/',
444474
],
475+
'orchestrationPhaseId not string' => [
476+
[
477+
'#tokenString' => getenv('TEST_STORAGE_API_TOKEN'),
478+
'id' => '12345',
479+
'runId' => '12345',
480+
'tokenId' => '1234',
481+
'projectId' => '123',
482+
'status' => 'created',
483+
'desiredStatus' => 'processing',
484+
'configId' => '123',
485+
'componentId' => 'keboola.test',
486+
'mode' => 'run',
487+
'orchestrationPhaseId' => 134,
488+
],
489+
'/Invalid configuration for path "job.orchestrationPhaseId": value must be a string/',
490+
],
491+
'orchestrationPhaseId empty string' => [
492+
[
493+
'#tokenString' => getenv('TEST_STORAGE_API_TOKEN'),
494+
'id' => '12345',
495+
'runId' => '12345',
496+
'tokenId' => '1234',
497+
'projectId' => '123',
498+
'status' => 'created',
499+
'desiredStatus' => 'processing',
500+
'configId' => '123',
501+
'componentId' => 'keboola.test',
502+
'mode' => 'run',
503+
'orchestrationPhaseId' => '',
504+
],
505+
'/Invalid configuration for path "job.orchestrationPhaseId": value cannot be empty string/',
506+
],
445507
'onlyOrchestrationTaskIds not list' => [
446508
[
447509
'#tokenString' => getenv('TEST_STORAGE_API_TOKEN'),

‎tests/JobFactory/JobTest.php‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
use Keboola\JobQueueInternalClient\Tests\BaseTest;
1414
use Keboola\PermissionChecker\BranchType;
1515
use Keboola\StorageApi\BranchAwareClient;
16-
use Keboola\StorageApi\Client;
1716
use Keboola\StorageApi\ClientException as StorageApiClientException;
1817
use Keboola\StorageApiBranch\ClientWrapper;
1918
use Keboola\StorageApiBranch\Factory\ClientOptions;
@@ -70,6 +69,15 @@ public function testConstants(): void
7069
self::assertCount(3, JobInterface::STATUSES_KILLABLE);
7170
}
7271

72+
public function testGetDeduplicationId(): void
73+
{
74+
$jobData = $this->jobData;
75+
self::assertNull($this->getJob($jobData)->getDeduplicationId());
76+
77+
$jobData['deduplicationId'] = '123-456';
78+
self::assertSame('123-456', $this->getJob($jobData)->getDeduplicationId());
79+
}
80+
7381
public function testGetComponentId(): void
7482
{
7583
self::assertEquals('keboola.ex-db-snowflake', $this->getJob()->getComponentId());

0 commit comments

Comments
 (0)