Skip to content

Commit eef666b

Browse files
ErikZigoErikZigo
committed
[internal-api-client] OrchestrationJobMatcher::matchTaskJobsForOrchestrationJob - token parameter
1 parent d83c731 commit eef666b

File tree

2 files changed

+66
-74
lines changed

2 files changed

+66
-74
lines changed

‎src/Orchestration/OrchestrationJobMatcher.php‎

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,23 +10,28 @@
1010
use Keboola\JobQueueInternalClient\JobFactory\JobInterface;
1111
use Keboola\JobQueueInternalClient\JobListOptions;
1212
use Keboola\StorageApi\Components;
13+
use Keboola\StorageApiBranch\Factory\ClientOptions;
14+
use Keboola\StorageApiBranch\Factory\StorageClientPlainFactory;
1315

1416
// https://keboola.atlassian.net/wiki/spaces/ENGG/pages/3074195457/DRAFT+RFC-2023-011+-+Rerun+orchestration#Pair-Jobs-and-Tasks
1517
class OrchestrationJobMatcher
1618
{
1719
public function __construct(
1820
private readonly Client $internalClient,
21+
private readonly StorageClientPlainFactory $storageClientFactory,
1922
) {
2023
}
2124

2225
public function matchTaskJobsForOrchestrationJob(
2326
string $jobId,
24-
?Components $componentsApi = null,
27+
string $token,
2528
): OrchestrationJobMatcherResults {
2629
$job = $this->internalClient->getJob($jobId);
27-
2830
$childJobs = $this->getOrchestrationTaskJobs($job);
29-
$configuration = $this->getCurrentOrchestrationConfiguration($job, $componentsApi);
31+
$configuration = $this->getCurrentOrchestrationConfiguration(
32+
$job,
33+
$this->createComponentsApi($token, $job->getBranchId()),
34+
);
3035
$this->validateInputs($job, $configuration);
3136
$matchedTasks = [];
3237
foreach ($configuration['tasks'] as $task) {
@@ -68,22 +73,17 @@ private function getOrchestrationTaskJobs(JobInterface $job): array
6873
);
6974
}
7075

71-
private function getCurrentOrchestrationConfiguration(JobInterface $job, ?Components $componentsApi = null): array
76+
private function getCurrentOrchestrationConfiguration(JobInterface $job, Components $componentsApi): array
7277
{
7378
$configuration = $job->getConfigData();
7479
if ($configuration) {
7580
return $configuration;
7681
}
7782

78-
// for situations where job token is expired/deleted
79-
if ($componentsApi) {
80-
return JobFactory\JobConfigurationResolver::resolveJobConfiguration(
81-
$job,
82-
$componentsApi,
83-
)['configuration'];
84-
}
85-
86-
return $job->getComponentConfiguration()['configuration'];
83+
return JobFactory\JobConfigurationResolver::resolveJobConfiguration(
84+
$job,
85+
$componentsApi,
86+
)['configuration'];
8787
}
8888

8989
private function validateInputs(JobInterface $job, array $configuration): void
@@ -112,4 +112,15 @@ private function validateInputs(JobInterface $job, array $configuration): void
112112
}
113113
});
114114
}
115+
116+
private function createComponentsApi(string $token, ?string $branchId): Components
117+
{
118+
return new Components(
119+
$this->storageClientFactory->createClientWrapper(
120+
(new ClientOptions())
121+
->setBranchId($branchId)
122+
->setToken($token),
123+
)->getBranchClient(),
124+
);
125+
}
115126
}

‎tests/Orchestration/OrchestrationJobMatcherTest.php‎

Lines changed: 42 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@
1515
use Keboola\StorageApi\Client as StorageClient;
1616
use Keboola\StorageApi\Components;
1717
use Keboola\StorageApi\Options\Components\Configuration;
18+
use Keboola\StorageApi\Options\TokenCreateOptions;
19+
use Keboola\StorageApi\Tokens;
20+
use Keboola\StorageApiBranch\Factory\ClientOptions;
21+
use Keboola\StorageApiBranch\Factory\StorageClientPlainFactory;
1822

1923
class OrchestrationJobMatcherTest extends BaseClientFunctionalTest
2024
{
@@ -97,9 +101,20 @@ private function getOrchestrationConfiguration(): array
97101
private function createOrchestrationLikeJobs(array $configurationData, array $createOnlyTasks): array
98102
{
99103
$storageClient = new StorageClient([
100-
'token' => (string) getenv('TEST_STORAGE_API_TOKEN'),
101-
'url' => (string) getenv('TEST_STORAGE_API_URL'),
104+
'token' => self::getRequiredEnv('TEST_STORAGE_API_TOKEN_MASTER'),
105+
'url' => self::getRequiredEnv('TEST_STORAGE_API_URL'),
102106
]);
107+
108+
// create token for job
109+
$tokenOptions = (new TokenCreateOptions())
110+
->setDescription(__CLASS__)
111+
->setCanManageBuckets(true) // access to all components :)
112+
->setExpiresIn(60 * 5)
113+
;
114+
115+
$tokens = new Tokens($storageClient);
116+
$token = $tokens->createToken($tokenOptions);
117+
103118
$queueClient = $this->getClient();
104119
$componentsApi = new Components($storageClient);
105120
$configuration = new Configuration();
@@ -108,9 +123,10 @@ private function createOrchestrationLikeJobs(array $configurationData, array $cr
108123
$configuration->setComponentId(JobFactory::ORCHESTRATOR_COMPONENT);
109124
$this->componentId = JobFactory::ORCHESTRATOR_COMPONENT;
110125
$this->configurationId = $componentsApi->addConfiguration($configuration)['id'];
126+
111127
$orchestrationJob = $this->getNewJobFactory()->createNewJob(
112128
[
113-
'#tokenString' => getenv('TEST_STORAGE_API_TOKEN'),
129+
'#tokenString' => $token['token'],
114130
'configData' => [],
115131
'componentId' => JobFactory::ORCHESTRATOR_COMPONENT,
116132
'configId' => $this->configurationId,
@@ -134,7 +150,7 @@ private function createOrchestrationLikeJobs(array $configurationData, array $cr
134150
];
135151
$phaseJob = $queueClient->createJob($this->getNewJobFactory()->createNewJob(
136152
[
137-
'#tokenString' => getenv('TEST_STORAGE_API_TOKEN'),
153+
'#tokenString' => $token['token'],
138154
'configData' => $configData,
139155
'componentId' => JobFactory::ORCHESTRATOR_COMPONENT,
140156
'configId' => $this->configurationId,
@@ -152,7 +168,7 @@ private function createOrchestrationLikeJobs(array $configurationData, array $cr
152168
}
153169
$jobIds[] = $queueClient->createJob($this->getNewJobFactory()->createNewJob(
154170
[
155-
'#tokenString' => getenv('TEST_STORAGE_API_TOKEN'),
171+
'#tokenString' => $token['token'],
156172
'componentId' => $task['task']['componentId'],
157173
'configData' => $task['task']['configData'],
158174
'mode' => $task['task']['mode'],
@@ -163,6 +179,8 @@ private function createOrchestrationLikeJobs(array $configurationData, array $cr
163179
))->getId();
164180
}
165181

182+
$tokens->dropToken($token['id']); // frop job token
183+
166184
return [
167185
'orchestrationJobId' => $orchestrationJob->getId(),
168186
'orchestrationConfigurationId' => $this->configurationId,
@@ -179,8 +197,11 @@ public function testMatcherFull(): void
179197
'jobIds' => $jobIds,
180198
] = $this->createOrchestrationLikeJobs($this->getOrchestrationConfiguration(), []);
181199

182-
$matcher = new OrchestrationJobMatcher($client);
183-
$results = $matcher->matchTaskJobsForOrchestrationJob($orchestrationJobId);
200+
$matcher = new OrchestrationJobMatcher($client, $this->createStorageClientPlainFactory());
201+
$results = $matcher->matchTaskJobsForOrchestrationJob(
202+
$orchestrationJobId,
203+
self::getRequiredEnv('TEST_STORAGE_API_TOKEN'),
204+
);
184205
self::assertEquals(
185206
new OrchestrationJobMatcherResults(
186207
$orchestrationJobId,
@@ -225,8 +246,11 @@ public function testMatcherPartial(): void
225246
'jobIds' => $jobIds,
226247
] = $this->createOrchestrationLikeJobs($this->getOrchestrationConfiguration(), ['92543']);
227248

228-
$matcher = new OrchestrationJobMatcher($client);
229-
$results = $matcher->matchTaskJobsForOrchestrationJob($orchestrationJobId);
249+
$matcher = new OrchestrationJobMatcher($client, $this->createStorageClientPlainFactory());
250+
$results = $matcher->matchTaskJobsForOrchestrationJob(
251+
$orchestrationJobId,
252+
self::getRequiredEnv('TEST_STORAGE_API_TOKEN'),
253+
);
230254
self::assertEquals(
231255
new OrchestrationJobMatcherResults(
232256
$orchestrationJobId,
@@ -278,8 +302,8 @@ public function testMatcherInvalidConfiguration(
278302
string $expectedMessage,
279303
): void {
280304
$storageClient = new StorageClient([
281-
'token' => (string) getenv('TEST_STORAGE_API_TOKEN'),
282-
'url' => (string) getenv('TEST_STORAGE_API_URL'),
305+
'token' => self::getRequiredEnv('TEST_STORAGE_API_TOKEN'),
306+
'url' => self::getRequiredEnv('TEST_STORAGE_API_URL'),
283307
]);
284308
$queueClient = $this->getClient();
285309
$componentsApi = new Components($storageClient);
@@ -291,7 +315,7 @@ public function testMatcherInvalidConfiguration(
291315
$this->configurationId = $componentsApi->addConfiguration($configuration)['id'];
292316
$orchestrationJob = $this->getNewJobFactory()->createNewJob(
293317
[
294-
'#tokenString' => getenv('TEST_STORAGE_API_TOKEN'),
318+
'#tokenString' => self::getRequiredEnv('TEST_STORAGE_API_TOKEN'),
295319
'configData' => [],
296320
'componentId' => $componentId,
297321
'configId' => $this->configurationId,
@@ -301,10 +325,10 @@ public function testMatcherInvalidConfiguration(
301325
],
302326
);
303327
$orchestrationJobId = $queueClient->createJob($orchestrationJob)->getId();
304-
$matcher = new OrchestrationJobMatcher($queueClient);
328+
$matcher = new OrchestrationJobMatcher($queueClient, $this->createStorageClientPlainFactory());
305329
$this->expectException(OrchestrationJobMatcherValidationException::class);
306330
$this->expectExceptionMessageMatches($expectedMessage);
307-
$matcher->matchTaskJobsForOrchestrationJob($orchestrationJobId);
331+
$matcher->matchTaskJobsForOrchestrationJob($orchestrationJobId, self::getRequiredEnv('TEST_STORAGE_API_TOKEN'));
308332
}
309333

310334
public function invalidConfigurationProvider(): Generator
@@ -342,53 +366,10 @@ public function invalidConfigurationProvider(): Generator
342366
];
343367
}
344368

345-
public function testMatcherLoadsConfigurationDirectlyFromComponentsApi(): void
369+
private function createStorageClientPlainFactory(): StorageClientPlainFactory
346370
{
347-
$jobMock = $this->createMock(JobFactory\JobInterface::class);
348-
$jobMock->expects(self::exactly(2))
349-
->method('getId')
350-
->willReturn('123')
351-
;
352-
$jobMock->expects(self::exactly(3))
353-
->method('getConfigId')
354-
->willReturn('456')
355-
;
356-
$jobMock->expects(self::exactly(2))
357-
->method('getComponentId')
358-
->willReturn('keboola.orchestrator')
359-
;
360-
361-
$queueClientMock = $this->createMock(Client::class);
362-
$queueClientMock->expects(self::once())
363-
->method('getJob')
364-
->with('123')
365-
->willReturn($jobMock)
366-
;
367-
368-
$componentsApiMock = $this->createMock(Components::class);
369-
$componentsApiMock->expects(self::once())
370-
->method('getConfiguration')
371-
->with('keboola.orchestrator', '456')
372-
->willReturn([
373-
'configuration' => [
374-
'tasks' => [
375-
[
376-
'id' => '789',
377-
],
378-
],
379-
],
380-
])
381-
;
382-
383-
$matcher = new OrchestrationJobMatcher($queueClientMock);
384-
385-
$results = $matcher->matchTaskJobsForOrchestrationJob('123', $componentsApiMock);
386-
387-
self::assertSame('123', $results->jobId);
388-
self::assertSame('456', $results->configId);
389-
self::assertCount(1, $results->matchedTasks);
390-
391-
$task = $results->matchedTasks[0];
392-
self::assertSame('789', $task->taskId);
371+
return new StorageClientPlainFactory(new ClientOptions(
372+
self::getRequiredEnv('TEST_STORAGE_API_URL'),
373+
));
393374
}
394375
}

0 commit comments

Comments
 (0)