Skip to content

Commit 0ca2415

Browse files
ErikZigoErikZigo
authored
Merge pull request #203 from keboola/erik-PS-3918
Internal Queue - attribute backend + metrics/backend with wlm context
2 parents 4c65915 + ed0c60d commit 0ca2415

15 files changed

+196
-33
lines changed

‎src/JobFactory/Backend.php‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,40 @@
66

77
class Backend
88
{
9+
private const OPTION_TYPE = 'type';
10+
private const OPTION_CONTAINER_TYPE = 'containerType';
11+
private const OPTION_CONTEXT = 'context';
12+
913
private ?string $type;
1014
private ?string $containerType;
15+
private ?string $context;
1116

12-
public function __construct(?string $type, ?string $containerType)
13-
{
17+
public function __construct(
18+
?string $type,
19+
?string $containerType,
20+
?string $context
21+
) {
1422
$this->type = $type;
1523
$this->containerType = $containerType;
24+
$this->context = $context;
1625
}
1726

1827
public static function fromDataArray(array $data): self
1928
{
2029
return new self(
21-
$data['type'] ?? null,
22-
$data['containerType'] ?? null
30+
$data[self::OPTION_TYPE] ?? null,
31+
$data[self::OPTION_CONTAINER_TYPE] ?? null,
32+
$data[self::OPTION_CONTEXT] ?? null
2333
);
2434
}
2535

2636
public function toDataArray(): array
2737
{
28-
return ['type' => $this->type, 'containerType' => $this->containerType];
38+
return [
39+
self::OPTION_TYPE => $this->type,
40+
self::OPTION_CONTAINER_TYPE => $this->containerType,
41+
self::OPTION_CONTEXT => $this->context,
42+
];
2943
}
3044

3145
public function getType(): ?string
@@ -38,8 +52,13 @@ public function getContainerType(): ?string
3852
return $this->containerType;
3953
}
4054

55+
public function getContext(): ?string
56+
{
57+
return $this->context;
58+
}
59+
4160
public function isEmpty(): bool
4261
{
43-
return $this->type === null && $this->containerType === null;
62+
return $this->type === null && $this->containerType === null && $this->context === null;
4463
}
4564
}

‎src/JobFactory/FullJobDefinition.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ protected function getRootDefinition(TreeBuilder $treeBuilder): ArrayNodeDefinit
156156
->children()
157157
->scalarNode('type')->end()
158158
->scalarNode('containerType')->end()
159+
->scalarNode('context')->end()
159160
->end()
160161
->end()
161162
->arrayNode('metrics')->ignoreExtraKeys(true)
@@ -174,6 +175,7 @@ protected function getRootDefinition(TreeBuilder $treeBuilder): ArrayNodeDefinit
174175
->children()
175176
->scalarNode('size')->end()
176177
->scalarNode('containerSize')->end()
178+
->scalarNode('context')->end()
177179
->end()
178180
->end()
179181
->end()

‎src/JobFactory/JobRuntimeResolver.php‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ private function getBackend(): Backend
113113
// return this irrespective if it is empty, because if it is we create empty Backend anyway
114114
return Backend::fromDataArray($configuration['runtime']['backend']);
115115
}
116-
return new Backend(null, null);
116+
return new Backend(null, null, null);
117117
}
118118

119119
private function resolveBackend(array $tokenInfo): Backend
@@ -135,14 +135,14 @@ private function resolveBackend(array $tokenInfo): Backend
135135
if (in_array($stagingStorage, ['local', 's3', 'abs', 'none']) &&
136136
!in_array(self::PAY_AS_YOU_GO_FEATURE, $tokenInfo['owner']['features'] ?? [])
137137
) {
138-
return new Backend(null, $tempBackend->getType());
138+
return new Backend(null, $tempBackend->getType(), $tempBackend->getContext());
139139
}
140140
if ($stagingStorage === 'workspace-snowflake') {
141141
// dynamic workspace si hidden behind another feature `workspace-snowflake-dynamic-backend-size`
142142
// that is checked in SAPI, so we don't check it here, yet
143-
return new Backend($tempBackend->getType(), null);
143+
return new Backend($tempBackend->getType(), null, $tempBackend->getContext());
144144
}
145-
return new Backend(null, null);
145+
return new Backend(null, null, $tempBackend->getContext());
146146
}
147147

148148
private function resolveParallelism(): ?string

‎src/JobFactory/NewJobDefinition.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ protected function getRootDefinition(TreeBuilder $treeBuilder): ArrayNodeDefinit
7272
->children()
7373
->scalarNode('type')->end()
7474
->scalarNode('containerType')->end()
75+
->scalarNode('context')->end()
7576
->end()
7677
->end()
7778
->arrayNode('behavior')->ignoreExtraKeys(true)

‎src/JobFactory/OverridesConfigurationDefinition.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ protected function getRootDefinition(TreeBuilder $treeBuilder): ArrayNodeDefinit
6363
->scalarNode('type')
6464
->beforeNormalization()->always($this->getStringNormalizer())->end()
6565
->end()
66+
->scalarNode('context')
67+
->beforeNormalization()->always($this->getStringNormalizer())->end()
68+
->end()
6669
->end()
6770
->end()
6871
->scalarNode('parallelism')

‎src/Result/JobMetrics.php‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ class JobMetrics implements JsonSerializable
1313

1414
private ?string $backendSize = null;
1515
private ?string $backendContainerSize = null;
16+
private ?string $backendContext = null;
1617

1718
public function jsonSerialize(): array
1819
{
@@ -24,6 +25,7 @@ public function jsonSerialize(): array
2425
'backend' => [
2526
'size' => $this->backendSize,
2627
'containerSize' => $this->backendContainerSize,
28+
'context' => $this->backendContext,
2729
],
2830
];
2931
}
@@ -61,6 +63,17 @@ public function setBackendContainerSize(string $size): JobMetrics
6163
return $this;
6264
}
6365

66+
public function getBackendContext(): ?string
67+
{
68+
return $this->backendContext;
69+
}
70+
71+
public function setBackendContext(string $context): JobMetrics
72+
{
73+
$this->backendContext = $context;
74+
return $this;
75+
}
76+
6477
public static function fromDataArray(array $data): self
6578
{
6679
$metricsData = $data['metrics'] ?? [];
@@ -77,6 +90,9 @@ public static function fromDataArray(array $data): self
7790
if (isset($metricsData['backend']['containerSize'])) {
7891
$metrics->setBackendContainerSize($metricsData['backend']['containerSize']);
7992
}
93+
if (isset($metricsData['backend']['context'])) {
94+
$metrics->setBackendContext($metricsData['backend']['context']);
95+
}
8096
return $metrics;
8197
}
8298

‎tests/ClientFunctionalTest.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,7 @@ public function testGetJob(): void
297297
'backend' => [
298298
'size' => null,
299299
'containerSize' => null,
300+
'context' => null,
300301
],
301302
], $job->getMetrics()->jsonSerialize());
302303
self::assertNull($job->getStartTime());
@@ -809,13 +810,15 @@ public function testPostJobResultAndMetrics(): void
809810
(new JobResult())->setMessage('bar'),
810811
(new JobMetrics())->setInputTablesBytesSum(654)->setBackendSize('medium')
811812
->setBackendContainerSize('small')
813+
->setBackendContext('wlm')
812814
);
813815
$job = $client->getJob($createdJob->getId());
814816
self::assertEquals(JobInterface::STATUS_PROCESSING, $job->getStatus());
815817
self::assertEquals('bar', $job->getResult()['message']);
816818
self::assertEquals(654, $job->getMetrics()->getInputTablesBytesSum());
817819
self::assertEquals('medium', $job->getMetrics()->getBackendSize());
818820
self::assertEquals('small', $job->getMetrics()->getBackendContainerSize());
821+
self::assertEquals('wlm', $job->getMetrics()->getBackendContext());
819822
}
820823

821824
public function testGetJobsWithProjectId(): void

‎tests/ClientTest.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -467,6 +467,7 @@ public function testSetJobResult(): void
467467
->setInputTablesBytesSum(112233445566)
468468
->setOutputTablesBytesSum(112233445577)
469469
->setBackendSize('small')
470+
->setBackendContext('wlm')
470471
);
471472
self::assertInstanceOf(Job::class, $result);
472473
self::assertCount(1, $container);
@@ -502,6 +503,7 @@ public function testSetJobResult(): void
502503
'backend' => [
503504
'size' => 'small',
504505
'containerSize' => null,
506+
'context' => 'wlm',
505507
],
506508
],
507509
],

‎tests/JobFactory/BackendTest.php‎

Lines changed: 88 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,22 @@ class BackendTest extends TestCase
1111
{
1212
public function typesProvider(): iterable
1313
{
14-
yield 'null' => [null, null];
15-
yield 'foo' => ['foo', null];
16-
yield 'containerFoo' => [null, 'foo'];
17-
yield 'FooBoo' => ['foo', 'boo'];
14+
yield 'null' => [null, null, null];
15+
yield 'foo' => ['foo', null, null];
16+
yield 'containerFoo' => [null, 'foo', null];
17+
yield 'contextFoo' => [null, null, 'foo'];
18+
yield 'FooBooLoo' => ['foo', 'boo', 'loo'];
1819
}
1920

2021
/**
2122
* @dataProvider typesProvider
2223
*/
23-
public function testCreate(?string $type, ?string $containerType): void
24+
public function testCreate(?string $type, ?string $containerType, ?string $context): void
2425
{
25-
$backend = new Backend($type, $containerType);
26+
$backend = new Backend($type, $containerType, $context);
2627
self::assertSame($type, $backend->getType());
2728
self::assertSame($containerType, $backend->getContainerType());
29+
self::assertSame($context, $backend->getContext());
2830
}
2931

3032
/**
@@ -34,21 +36,64 @@ public function testCreateFromArray(
3436
array $data,
3537
?string $expectedType,
3638
?string $expectedContainerType,
39+
?string $expectedContext,
3740
bool $expectedEmpty
3841
): void {
3942
$backend = Backend::fromDataArray($data);
4043

4144
self::assertSame($expectedType, $backend->getType());
4245
self::assertSame($expectedContainerType, $backend->getContainerType());
46+
self::assertSame($expectedContext, $backend->getContext());
4347
self::assertSame($expectedEmpty, $backend->isEmpty());
4448
}
4549

4650
public function provideCreateFromArrayData(): iterable
4751
{
48-
yield 'empty' => [[], null, null, true];
49-
yield 'with type' => [['type' => 'custom'], 'custom', null, false];
50-
yield 'with container type' => [['containerType' => 'custom'], null, 'custom', false];
51-
yield 'with both types' => [['type' => 'custom', 'containerType' => 'motsuc'], 'custom', 'motsuc', false];
52+
yield 'empty' => [
53+
[],
54+
null,
55+
null,
56+
null,
57+
true,
58+
];
59+
yield 'with type' => [
60+
[
61+
'type' => 'custom',
62+
],
63+
'custom',
64+
null,
65+
null,
66+
false,
67+
];
68+
yield 'with container type' => [
69+
[
70+
'containerType' => 'custom',
71+
],
72+
null,
73+
'custom',
74+
null,
75+
false,
76+
];
77+
yield 'with context' => [
78+
[
79+
'context' => 'wlm',
80+
],
81+
null,
82+
null,
83+
'wlm',
84+
false,
85+
];
86+
yield 'with all properties' => [
87+
[
88+
'type' => 'custom',
89+
'containerType' => 'motsuc',
90+
'context' => 'wlm',
91+
],
92+
'custom',
93+
'motsuc',
94+
'wlm',
95+
false,
96+
];
5297
}
5398

5499
/**
@@ -62,20 +107,44 @@ public function testExportAsDataArray(Backend $backend, array $expectedResult):
62107
public function provideExportAsDataArrayData(): iterable
63108
{
64109
yield 'empty' => [
65-
new Backend(null, null),
66-
['type' => null, 'containerType' => null],
110+
new Backend(null, null, null),
111+
[
112+
'type' => null,
113+
'containerType' => null,
114+
'context' => null,
115+
],
67116
];
68117
yield 'with type' => [
69-
new Backend('custom', null),
70-
['type' => 'custom', 'containerType' => null],
118+
new Backend('custom', null, null),
119+
[
120+
'type' => 'custom',
121+
'containerType' => null,
122+
'context' => null,
123+
],
71124
];
72125
yield 'with container type' => [
73-
new Backend(null, 'custom'),
74-
['type' => null, 'containerType' => 'custom'],
126+
new Backend(null, 'custom', null),
127+
[
128+
'type' => null,
129+
'containerType' => 'custom',
130+
'context' => null,
131+
],
132+
];
133+
yield 'with context' => [
134+
new Backend(null, null, 'wml'),
135+
[
136+
'type' => null,
137+
'containerType' => null,
138+
'context' => 'wml',
139+
],
75140
];
76-
yield 'with both types' => [
77-
new Backend('custom', 'motsuc'),
78-
['type' => 'custom', 'containerType' => 'motsuc'],
141+
yield 'with all properties' => [
142+
new Backend('custom', 'motsuc', 'wml'),
143+
[
144+
'type' => 'custom',
145+
'containerType' => 'motsuc',
146+
'context' => 'wml',
147+
],
79148
];
80149
}
81150
}

‎tests/JobFactory/FullJobDefinitionTest.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ public function testValidJobFull(): void
7070
'backend' => [
7171
'type' => 'large',
7272
'containerType' => 'small',
73+
'context' => 'wml',
7374
],
7475
'metrics' => [
7576
'storage' => [
@@ -81,6 +82,7 @@ public function testValidJobFull(): void
8182
'size' => 'medium',
8283
'containerSize' => 'large',
8384
'backendExtraKey' => 'ignored',
85+
'context' => 'wml',
8486
],
8587
],
8688
'orchestrationJobId' => '123456789',

0 commit comments

Comments
 (0)