Skip to content

Commit 8374bf3

Browse files
janvanicekjanvanicek
authored
Merge pull request #286 from keboola/jv-PST-271-remove-filter-by-run-id
[input-mapping] Remove obsolete property `filter_by_run_id`
2 parents 0703c47 + ada7311 commit 8374bf3

File tree

6 files changed

+3
-289
lines changed

6 files changed

+3
-289
lines changed

‎src/Configuration/File.php‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@ public static function configureNode(ArrayNodeDefinition $node): void
5353
->end()
5454
->end()
5555
->scalarNode('query')->end()
56-
->booleanNode('filter_by_run_id')->end()
5756
->integerNode('limit')->end()
5857
->booleanNode('overwrite')->defaultValue(true)->end()
5958
->arrayNode('processed_tags')

‎src/File/Options/RewrittenInputFileOptions.php‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,6 @@ public function getStorageApiFileListOptions(InputFileStateList $states): ListFi
4444
{
4545
$options = new ListFilesOptions();
4646

47-
if (!empty($this->definition['filter_by_run_id'])) {
48-
$options->setRunId(Reader::getParentRunId($this->runId));
49-
}
5047
if (isset($this->definition['tags']) && count($this->definition['tags'])) {
5148
$options->setTags($this->definition['tags']);
5249
}

‎tests/Configuration/FileConfigurationTest.php‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ public function testConfiguration(): void
1717
'tags' => ['tag1', 'tag2'],
1818
'query' => 'esquery',
1919
'processed_tags' => ['tag3'],
20-
'filter_by_run_id' => true,
2120
'limit' => 1000,
2221
'overwrite' => false,
2322
];
@@ -32,7 +31,6 @@ public function testEmptyTagsRemoved(): void
3231
'tags' => [],
3332
'query' => 'esquery',
3433
'processed_tags' => ['tag3'],
35-
'filter_by_run_id' => true,
3634
'limit' => 1000,
3735
'overwrite' => true,
3836
];
@@ -50,7 +48,6 @@ public function testEmptyProcessedTagsRemoved(): void
5048
'tags' => ['tag3'],
5149
'query' => 'esquery',
5250
'processed_tags' => [],
53-
'filter_by_run_id' => true,
5451
'limit' => 1000,
5552
'overwrite' => true,
5653
];
@@ -68,7 +65,6 @@ public function testEmptyQueryRemoved(): void
6865
'tags' => ['tag1'],
6966
'query' => '',
7067
'processed_tags' => ['tag3'],
71-
'filter_by_run_id' => true,
7268
'limit' => 1000,
7369
'overwrite' => true,
7470
];
@@ -85,7 +81,6 @@ public function testConfigurationWithSourceTags(): void
8581
$config = [
8682
'query' => 'esquery',
8783
'processed_tags' => ['tag3'],
88-
'filter_by_run_id' => true,
8984
'limit' => 1000,
9085
'source' => [
9186
'tags' => [
@@ -207,7 +202,6 @@ public function testEmptyFileIdsRemoved(): void
207202
'query' => 'aaa',
208203
'file_ids' => [],
209204
'processed_tags' => ['tag3'],
210-
'filter_by_run_id' => true,
211205
'limit' => 1000,
212206
'overwrite' => true,
213207
];
@@ -245,10 +239,6 @@ public function fileIdsCannotBeCombinedWithOtherFiltersProvider(): Generator
245239
['limit' => 1],
246240
];
247241

248-
yield [
249-
['filter_by_run_id' => true],
250-
];
251-
252242
yield [
253243
['changed_since' => '-7 days'],
254244
];

‎tests/File/Options/RewrittenInputFileOptionsTest.php‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,12 @@ public function provideFileListOptionsValues(): Generator
7272
'expectedOptions' => (new ListFilesOptions())->setTags(['foo'])->setLimit(100),
7373
'originalFileOptions' => null,
7474
];
75-
yield 'tags + filter by run id + limit' => [
76-
'rewrittenFileOptions' => ['tags' => ['foo'], 'filter_by_run_id' => true, 'limit' => 50],
75+
yield 'tags + limit' => [
76+
'rewrittenFileOptions' => ['tags' => ['foo'], 'limit' => 50],
7777
'isDevBranch' => false,
7878
'runId' => '1234',
7979
'filesStates' => new InputFileStateList([]),
80-
'expectedOptions' => (new ListFilesOptions())->setTags(['foo'])->setRunId(1234)->setLimit(50),
80+
'expectedOptions' => (new ListFilesOptions())->setTags(['foo'])->setLimit(50),
8181
'originalFileOptions' => null,
8282
];
8383
yield 'source tags include' => [

‎tests/Functional/DownloadFilesAbsWorkspaceTest.php‎

Lines changed: 0 additions & 192 deletions
Original file line numberDiff line numberDiff line change
@@ -261,198 +261,6 @@ public function testAbsReadFilesOverwrite(): void
261261
);
262262
}
263263

264-
public function testReadAbsFilesTagsFilterRunId(): void
265-
{
266-
$clientWrapper = $this->getClientWrapper(null);
267-
$temp = new Temp('input-mapping');
268-
$root = $temp->getTmpFolder();
269-
file_put_contents($root . '/upload', 'test');
270-
$reader = new Reader($this->getStagingFactory($clientWrapper));
271-
$fo = new FileUploadOptions();
272-
$fo->setTags(['download-files-test']);
273-
274-
$clientWrapper->getTableAndFileStorageClient()->setRunId('xyz');
275-
$id1 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
276-
$id2 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
277-
$clientWrapper->getTableAndFileStorageClient()->setRunId('1234567');
278-
$id3 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
279-
$id4 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
280-
$clientWrapper->getTableAndFileStorageClient()->setRunId('1234567.8901234');
281-
$id5 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
282-
$id6 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
283-
sleep(5);
284-
$configuration = [
285-
[
286-
'tags' => ['download-files-test'],
287-
'filter_by_run_id' => true,
288-
'overwrite' => true,
289-
],
290-
];
291-
292-
$reader->downloadFiles(
293-
$configuration,
294-
'download',
295-
AbstractStrategyFactory::WORKSPACE_ABS,
296-
new InputFileStateList([]),
297-
);
298-
299-
try {
300-
$this->blobClient->getBlob(
301-
$this->workspaceCredentials['container'],
302-
'download/upload/' . $id1,
303-
);
304-
self::fail('should have thrown 404');
305-
} catch (ServiceException $exception) {
306-
self::assertEquals(404, $exception->getCode());
307-
}
308-
try {
309-
$this->blobClient->getBlob(
310-
$this->workspaceCredentials['container'],
311-
'download/upload/' . $id1 . '.manifest',
312-
);
313-
self::fail('should have thrown 404');
314-
} catch (ServiceException $exception) {
315-
self::assertEquals(404, $exception->getCode());
316-
}
317-
try {
318-
$this->blobClient->getBlob(
319-
$this->workspaceCredentials['container'],
320-
'download/upload/' . $id2,
321-
);
322-
self::fail('should have thrown 404');
323-
} catch (ServiceException $exception) {
324-
self::assertEquals(404, $exception->getCode());
325-
}
326-
try {
327-
$this->blobClient->getBlob(
328-
$this->workspaceCredentials['container'],
329-
'download/upload/' . $id2 . '.manifest',
330-
);
331-
self::fail('should have thrown 404');
332-
} catch (ServiceException $exception) {
333-
self::assertEquals(404, $exception->getCode());
334-
}
335-
$this->assertBlobNotEmpty(
336-
'download/upload/' . $id3,
337-
);
338-
$this->assertBlobNotEmpty(
339-
'download/upload/' . $id3 . '.manifest',
340-
);
341-
$this->assertBlobNotEmpty(
342-
'download/upload/' . $id4,
343-
);
344-
$this->assertBlobNotEmpty(
345-
'download/upload/' . $id4 . '.manifest',
346-
);
347-
$this->assertBlobNotEmpty(
348-
'download/upload/' . $id5,
349-
);
350-
$this->assertBlobNotEmpty(
351-
'download/upload/' . $id5 . '.manifest',
352-
);
353-
$this->assertBlobNotEmpty(
354-
'download/upload/' . $id6,
355-
);
356-
$this->assertBlobNotEmpty(
357-
'download/upload/' . $id6 . '.manifest',
358-
);
359-
}
360-
361-
public function testReadFilesEsQueryFilterRunId(): void
362-
{
363-
$clientWrapper = $this->getClientWrapper(null);
364-
$temp = new Temp('input-mapping');
365-
$root = $temp->getTmpFolder();
366-
file_put_contents($root . '/upload', 'test');
367-
$reader = new Reader($this->getStagingFactory($clientWrapper));
368-
$fo = new FileUploadOptions();
369-
$fo->setTags(['download-files-test']);
370-
371-
$clientWrapper->getTableAndFileStorageClient()->setRunId('xyz');
372-
$id1 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
373-
$id2 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
374-
$clientWrapper->getTableAndFileStorageClient()->setRunId('1234567');
375-
$id3 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
376-
$id4 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
377-
$clientWrapper->getTableAndFileStorageClient()->setRunId('1234567.8901234');
378-
$id5 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
379-
$id6 = $clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
380-
sleep(5);
381-
$configuration = [
382-
[
383-
'query' => 'tags: download-files-test',
384-
'filter_by_run_id' => true,
385-
'overwrite' => true,
386-
],
387-
];
388-
$reader->downloadFiles(
389-
$configuration,
390-
'download',
391-
AbstractStrategyFactory::WORKSPACE_ABS,
392-
new InputFileStateList([]),
393-
);
394-
try {
395-
$this->blobClient->getBlob(
396-
$this->workspaceCredentials['container'],
397-
'download/upload/' . $id1,
398-
);
399-
self::fail('should have thrown 404');
400-
} catch (ServiceException $exception) {
401-
self::assertEquals(404, $exception->getCode());
402-
}
403-
try {
404-
$this->blobClient->getBlob(
405-
$this->workspaceCredentials['container'],
406-
'download/upload/' . $id1 . '.manifest',
407-
);
408-
self::fail('should have thrown 404');
409-
} catch (ServiceException $exception) {
410-
self::assertEquals(404, $exception->getCode());
411-
}
412-
try {
413-
$this->blobClient->getBlob(
414-
$this->workspaceCredentials['container'],
415-
$root . 'download/upload/' . $id2,
416-
);
417-
self::fail('should have thrown 404');
418-
} catch (ServiceException $exception) {
419-
self::assertEquals(404, $exception->getCode());
420-
}
421-
try {
422-
$this->blobClient->getBlob(
423-
$this->workspaceCredentials['container'],
424-
'download/uppload/' . $id2 . '.manifest',
425-
);
426-
self::fail('should have thrown 404');
427-
} catch (ServiceException $exception) {
428-
self::assertEquals(404, $exception->getCode());
429-
}
430-
$this->assertBlobNotEmpty(
431-
'download/upload/' . $id3,
432-
);
433-
$this->assertBlobNotEmpty(
434-
'download/upload/' . $id3 . '.manifest',
435-
);
436-
$this->assertBlobNotEmpty(
437-
'download/upload/' . $id4,
438-
);
439-
$this->assertBlobNotEmpty(
440-
'download/upload/' . $id4 . '.manifest',
441-
);
442-
$this->assertBlobNotEmpty(
443-
'download/upload/' . $id5,
444-
);
445-
$this->assertBlobNotEmpty(
446-
'download/upload/' . $id5 . '.manifest',
447-
);
448-
$this->assertBlobNotEmpty(
449-
'download/upload/' . $id6,
450-
);
451-
$this->assertBlobNotEmpty(
452-
'download/upload/' . $id6 . '.manifest',
453-
);
454-
}
455-
456264
public function testAbsWorkspaceAdaptiveInput(): void
457265
{
458266
$clientWrapper = $this->getClientWrapper(null);

‎tests/Functional/DownloadFilesTest.php‎

Lines changed: 0 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -110,47 +110,6 @@ public function testReadFilesOverwrite(): void
110110
);
111111
}
112112

113-
public function testReadFilesTagsFilterRunId(): void
114-
{
115-
$root = $this->temp->getTmpFolder();
116-
file_put_contents($root . '/upload', 'test');
117-
$reader = new Reader($this->getLocalStagingFactory());
118-
$fo = new FileUploadOptions();
119-
$fo->setTags([self::DEFAULT_TEST_FILE_TAG]);
120-
121-
$this->clientWrapper->getTableAndFileStorageClient()->setRunId('xyz');
122-
$id1 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
123-
$id2 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
124-
$this->clientWrapper->getTableAndFileStorageClient()->setRunId('1234567');
125-
$id3 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
126-
$id4 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
127-
$this->clientWrapper->getTableAndFileStorageClient()->setRunId('1234567.8901234');
128-
$id5 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
129-
$id6 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
130-
sleep(5);
131-
132-
$configuration = [
133-
[
134-
'tags' => [self::DEFAULT_TEST_FILE_TAG],
135-
'filter_by_run_id' => true,
136-
'overwrite' => true,
137-
],
138-
];
139-
$reader->downloadFiles(
140-
$configuration,
141-
'download',
142-
AbstractStrategyFactory::LOCAL,
143-
new InputFileStateList([]),
144-
);
145-
146-
self::assertFalse(file_exists($root . '/download/' . $id1 . '_upload'));
147-
self::assertFalse(file_exists($root . '/download/' . $id2 . '_upload'));
148-
self::assertTrue(file_exists($root . '/download/' . $id3 . '_upload'));
149-
self::assertTrue(file_exists($root . '/download/' . $id4 . '_upload'));
150-
self::assertTrue(file_exists($root . '/download/' . $id5 . '_upload'));
151-
self::assertTrue(file_exists($root . '/download/' . $id6 . '_upload'));
152-
}
153-
154113
public function testReadFilesIncludeAllTags(): void
155114
{
156115
$root = $this->temp->getTmpFolder();
@@ -300,45 +259,6 @@ public function testReadFilesIncludeAllTagsWithLimit(): void
300259
self::assertFalse(file_exists($root . '/download/' . $id1 . '_upload'));
301260
self::assertTrue(file_exists($root . '/download/' . $id2 . '_upload'));
302261
}
303-
public function testReadFilesEsQueryFilterRunId(): void
304-
{
305-
$root = $this->temp->getTmpFolder();
306-
file_put_contents($root . '/upload', 'test');
307-
$reader = new Reader($this->getLocalStagingFactory());
308-
$fo = new FileUploadOptions();
309-
$fo->setTags([self::DEFAULT_TEST_FILE_TAG]);
310-
311-
$this->clientWrapper->getTableAndFileStorageClient()->setRunId('xyz');
312-
$id1 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
313-
$id2 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
314-
$this->clientWrapper->getTableAndFileStorageClient()->setRunId('1234567');
315-
$id3 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
316-
$id4 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
317-
$this->clientWrapper->getTableAndFileStorageClient()->setRunId('1234567.8901234');
318-
$id5 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
319-
$id6 = $this->clientWrapper->getTableAndFileStorageClient()->uploadFile($root . '/upload', $fo);
320-
sleep(5);
321-
322-
$configuration = [
323-
[
324-
'query' => 'tags: ' . self::DEFAULT_TEST_FILE_TAG,
325-
'filter_by_run_id' => true,
326-
'overwrite' => true,
327-
],
328-
];
329-
$reader->downloadFiles(
330-
$configuration,
331-
'download',
332-
AbstractStrategyFactory::LOCAL,
333-
new InputFileStateList([]),
334-
);
335-
self::assertFalse(file_exists($root . '/download/' . $id1 . '_upload'));
336-
self::assertFalse(file_exists($root . '/download/' . $id2 . '_upload'));
337-
self::assertTrue(file_exists($root . '/download/' . $id3 . '_upload'));
338-
self::assertTrue(file_exists($root . '/download/' . $id4 . '_upload'));
339-
self::assertTrue(file_exists($root . '/download/' . $id5 . '_upload'));
340-
self::assertTrue(file_exists($root . '/download/' . $id6 . '_upload'));
341-
}
342262

343263
public function testReadFilesLimit(): void
344264
{

0 commit comments

Comments
 (0)