Skip to content

Commit

Permalink
Merge branch 'master' into III-4932-remove-add_location_name_to_coord…
Browse files Browse the repository at this point in the history
…inates_lookup
  • Loading branch information
LucWollants committed Sep 5, 2024
2 parents cbe29a3 + 7bd31fa commit 2a73425
Show file tree
Hide file tree
Showing 6 changed files with 302 additions and 12 deletions.
91 changes: 91 additions & 0 deletions app/Console/Command/ImportDuplicatePlaces.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
<?php

declare(strict_types=1);

namespace CultuurNet\UDB3\Console\Command;

use CultuurNet\UDB3\Place\Canonical\DBALDuplicatePlaceRepository;
use CultuurNet\UDB3\Place\Canonical\ImportDuplicatePlacesProcessor;
use Symfony\Component\Console\Command\Command as BaseCommand;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Question\ConfirmationQuestion;

class ImportDuplicatePlaces extends BaseCommand
{
private const FORCE = 'force';

private ImportDuplicatePlacesProcessor $importDuplicatePlacesProcessor;
private DBALDuplicatePlaceRepository $dbalDuplicatePlaceRepository;

public function __construct(
DBALDuplicatePlaceRepository $dbalDuplicatePlaceRepository,
ImportDuplicatePlacesProcessor $importDuplicatePlacesProcessor
) {
parent::__construct();

$this->importDuplicatePlacesProcessor = $importDuplicatePlacesProcessor;
$this->dbalDuplicatePlaceRepository = $dbalDuplicatePlaceRepository;
}

public function configure(): void
{
$this
->setName('place:duplicate-places:import')
->setDescription('Import duplicate places from the import tables, set clusters ready for processing')
->addOption(
self::FORCE,
null,
InputOption::VALUE_NONE,
'Skip confirmation.'
);
}

protected function execute(InputInterface $input, OutputInterface $output): ?int
{
$howManyPlacesAreToBeImported = $this->dbalDuplicatePlaceRepository->howManyPlacesAreToBeImported();
$howManyPlacesAreToBeDeleted = count($this->dbalDuplicatePlaceRepository->getPlacesNoLongerInCluster());

if ($howManyPlacesAreToBeImported === 0 && $howManyPlacesAreToBeDeleted === 0) {
$output->writeln('duplicate_places is already synced');
return self::SUCCESS;
}

if (!$this->askConfirmation(
$input,
$output,
sprintf(
'This action will sync a total of %d new places, and remove %d places from the duplicate places table. Do you want to continue? [y/N] ',
$howManyPlacesAreToBeImported,
$howManyPlacesAreToBeDeleted,
)
)) {
return self::SUCCESS;
}

$this->importDuplicatePlacesProcessor->sync();

$output->writeln('Duplicate places were synced and old clusters were removed. You probably want to run place:process-duplicates to give canonicals to the new clusters now.');

return self::SUCCESS;
}

private function askConfirmation(InputInterface $input, OutputInterface $output, string $message): bool
{
if ($input->getOption(self::FORCE)) {
return true;
}

return $this
->getHelper('question')
->ask(
$input,
$output,
new ConfirmationQuestion(
$message,
true
)
);
}
}
14 changes: 14 additions & 0 deletions app/Console/ConsoleServiceProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
use CultuurNet\UDB3\Console\Command\GeocodeEventCommand;
use CultuurNet\UDB3\Console\Command\GeocodeOrganizerCommand;
use CultuurNet\UDB3\Console\Command\GeocodePlaceCommand;
use CultuurNet\UDB3\Console\Command\ImportDuplicatePlaces;
use CultuurNet\UDB3\Console\Command\ImportOfferAutoClassificationLabels;
use CultuurNet\UDB3\Console\Command\IncludeLabel;
use CultuurNet\UDB3\Console\Command\KeycloakCommand;
Expand Down Expand Up @@ -59,6 +60,7 @@
use CultuurNet\UDB3\Kinepolis\Trailer\YoutubeTrailerRepository;
use CultuurNet\UDB3\Offer\OfferType;
use CultuurNet\UDB3\Organizer\WebsiteNormalizer;
use CultuurNet\UDB3\Place\Canonical\ImportDuplicatePlacesProcessor;
use CultuurNet\UDB3\Place\Canonical\DuplicatePlaceRemovedFromClusterRepository;
use CultuurNet\UDB3\Search\EventsSapi3SearchService;
use CultuurNet\UDB3\Search\OrganizersSapi3SearchService;
Expand Down Expand Up @@ -86,6 +88,7 @@ final class ConsoleServiceProvider extends AbstractServiceProvider
'console.fire-projected-to-jsonld-for-relations',
'console.fire-projected-to-jsonld',
'console.place:process-duplicates',
'console.place:duplicate-places:import',
'console.event:bulk-remove-from-production',
'console.event:reindex-offers-with-popularity',
'console.place:reindex-offers-with-popularity',
Expand Down Expand Up @@ -261,6 +264,17 @@ function () use ($container) {
)
);

$container->addShared(
'console.place:duplicate-places:import',
fn () => new ImportDuplicatePlaces(
$container->get('duplicate_place_repository'),
new ImportDuplicatePlacesProcessor(
$container->get('duplicate_place_repository'),
$container->get(DuplicatePlaceRemovedFromClusterRepository::class)
)
)
);

$container->addShared(
'console.event:bulk-remove-from-production',
fn () => new BulkRemoveFromProduction($container->get('event_command_bus'))
Expand Down
37 changes: 26 additions & 11 deletions src/Place/Canonical/DBALDuplicatePlaceRepository.php
Original file line number Diff line number Diff line change
Expand Up @@ -80,18 +80,7 @@ public function getDuplicatesOfPlace(string $placeId): ?array
return count($duplicates) > 0 ? $duplicates : null;
}

public function getPlacesNoLongerInCluster(): array
{
// All places that do not exist in duplicate_places_import
$statement = $this->connection->createQueryBuilder()
->select('DISTINCT dp.place_uuid')
->from('duplicate_places', 'dp')
->leftJoin('dp', 'duplicate_places_import', 'dpi', 'dp.place_uuid = dpi.place_uuid')
->where('dpi.place_uuid IS NULL')
->execute();

return $statement->fetchFirstColumn();
}

public function getClustersToBeRemoved(): array
{
Expand Down Expand Up @@ -124,6 +113,32 @@ public function getPlacesWithCluster(): array
}, $statement->fetchAllAssociative());
}

public function howManyPlacesAreToBeImported(): int
{
// COUNT from `duplicate_places_import` not present in `duplicate_places`
$result = $this->connection->createQueryBuilder()
->select('COUNT(*) AS not_in_duplicate')
->from('duplicate_places_import', 'dpi')
->leftJoin('dpi', 'duplicate_places', 'dp', 'dpi.cluster_id = dp.cluster_id AND dpi.place_uuid = dp.place_uuid')
->where('dp.cluster_id IS NULL')
->execute();

return (int)($result->fetchOne() ?? 0);
}

public function getPlacesNoLongerInCluster(): array
{
// All places that do not exist in duplicate_places_import
$statement = $this->connection->createQueryBuilder()
->select('DISTINCT dp.place_uuid')
->from('duplicate_places', 'dp')
->leftJoin('dp', 'duplicate_places_import', 'dpi', 'dp.place_uuid = dpi.place_uuid')
->where('dpi.place_uuid IS NULL')
->execute();

return $statement->fetchFirstColumn();
}

public function deleteCluster(string $clusterId): void
{
$this->connection->createQueryBuilder()
Expand Down
4 changes: 3 additions & 1 deletion src/Place/Canonical/DuplicatePlaceRepository.php
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,13 @@ public function getDuplicatesOfPlace(string $placeId): ?array;

public function getPlacesNoLongerInCluster(): array;

public function getClustersToBeRemoved(): array;

/** @return PlaceWithCluster[] */
public function getPlacesWithCluster(): array;

public function addToDuplicatePlaces(PlaceWithCluster $clusterRecordRow): void;
public function deleteCluster(string $clusterId): void;

public function getClustersToBeRemoved(): array;
public function howManyPlacesAreToBeImported(): int;
}
86 changes: 86 additions & 0 deletions tests/Console/Command/ImportDuplicatePlacesTest.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
<?php

declare(strict_types=1);

namespace CultuurNet\UDB3\Console\Command;

use CultuurNet\UDB3\Place\Canonical\DBALDuplicatePlaceRepository;
use CultuurNet\UDB3\Place\Canonical\ImportDuplicatePlacesProcessor;
use PHPUnit\Framework\MockObject\MockObject;
use PHPUnit\Framework\TestCase;
use Ramsey\Uuid\Uuid;
use Symfony\Component\Console\Helper\HelperSet;
use Symfony\Component\Console\Helper\QuestionHelper;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;

class ImportDuplicatePlacesTest extends TestCase
{
/** @var DBALDuplicatePlaceRepository|MockObject */
private $dbalDuplicatePlaceRepository;
/** @var ImportDuplicatePlacesProcessor|MockObject */
private $importDuplicatePlacesProcessor;
/** @var InputInterface|MockObject */
private $input;
/** @var OutputInterface|MockObject */
private $output;
private ImportDuplicatePlaces $command;

protected function setUp(): void
{
$this->dbalDuplicatePlaceRepository = $this->createMock(DBALDuplicatePlaceRepository::class);
$this->importDuplicatePlacesProcessor = $this->createMock(ImportDuplicatePlacesProcessor::class);
$this->input = $this->createMock(InputInterface::class);
$this->output = $this->createMock(OutputInterface::class);

$this->command = new ImportDuplicatePlaces(
$this->dbalDuplicatePlaceRepository,
$this->importDuplicatePlacesProcessor
);
}

public function testExecuteSucceedsWhenTablesAreAlreadySynced(): void
{
$this->dbalDuplicatePlaceRepository
->expects($this->once())
->method('howManyPlacesAreToBeImported')
->willReturn(0);

$this->dbalDuplicatePlaceRepository
->expects($this->once())
->method('getPlacesNoLongerInCluster')
->willReturn([]);

$this->output
->expects($this->once())
->method('writeln')
->with('duplicate_places is already synced');

$this->assertEquals(0, $this->command->run($this->input, $this->output));
}

public function testExecuteConfirmsAndSyncsWhenChangesAreWithinLimits(): void
{
$this->dbalDuplicatePlaceRepository
->expects($this->once())
->method('howManyPlacesAreToBeImported')
->willReturn(50);

$this->dbalDuplicatePlaceRepository
->expects($this->once())
->method('getPlacesNoLongerInCluster')
->willReturn([Uuid::uuid4()]);

$helper = $this->createMock(QuestionHelper::class);
$helper->expects($this->once())
->method('ask')
->willReturn(true);
$this->command->setHelperSet(new HelperSet(['question' => $helper]));

$this->importDuplicatePlacesProcessor
->expects($this->once())
->method('sync');

$this->assertEquals(0, $this->command->run($this->input, $this->output));
}
}
82 changes: 82 additions & 0 deletions tests/Place/Canonical/DBALDuplicatePlaceRepositoryTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -251,4 +251,86 @@ public function test_add_to_duplicate_places(): void

$this->assertEquals(1, $raw['total']);
}

/** @dataProvider clusterChangesDataProvider */
public function test_calculate_how_many_clusters_have_changed(array $clusters, int $expectedPlacesTobeImported, int $expectedPlacesTobeDeleted): void
{
foreach ($clusters as [$clusterId, $placeUuid]) {
$this->getConnection()->insert(
'duplicate_places_import',
[
'cluster_id' => $clusterId,
'place_uuid' => $placeUuid,
]
);
}

$this->assertEquals($expectedPlacesTobeImported, $this->duplicatePlaceRepository->howManyPlacesAreToBeImported());
$this->assertCount($expectedPlacesTobeDeleted, $this->duplicatePlaceRepository->getPlacesNoLongerInCluster());
}

public static function clusterChangesDataProvider(): array
{
return [
'everything is new' => [
[

],
0,
5,
],
'Some new, some removed' => [
[
['cluster_1', '19ce6565-76be-425d-94d6-894f84dd2947'],
['cluster_1', '1accbcfb-3b22-4762-bc13-be0f67fd3116'],
['new', '04a549ba-6e5e-433b-9601-07b7a809758e'],
],
1,
3,
],
'Nothing has changed' => [
[
['cluster_1', '19ce6565-76be-425d-94d6-894f84dd2947'],
['cluster_1', '1accbcfb-3b22-4762-bc13-be0f67fd3116'],
['cluster_1', '526605d3-7cc4-4607-97a4-065896253f42'],
['cluster_2', '4a355db3-c3f9-4acc-8093-61b333a3aefb'],
['cluster_2', '64901efc-6bd7-4e9d-8916-fcdeb5b1c8ad'],
],
0,
0,
],
'Everything single place has been moved' => [
[
['5', '19ce6565-76be-425d-94d6-894f84dd2947'],
['5', '1accbcfb-3b22-4762-bc13-be0f67fd3116'],
['5', '526605d3-7cc4-4607-97a4-065896253f42'],
['5', '4a355db3-c3f9-4acc-8093-61b333a3aefb'],
['5', '64901efc-6bd7-4e9d-8916-fcdeb5b1c8ad'],
],
5,
0,
],
];
}

public function test_how_many_places_are_to_be_imported(): void
{
$this->getConnection()->insert(
'duplicate_places_import',
[
'cluster_id' => 'my_brand_new_cluster',
'place_uuid' => '19ce6565-76be-425d-94d6-894f84dd2947',
]
);
$this->getConnection()->insert(
'duplicate_places_import',
[
'cluster_id' => 'my_brand_new_cluster',
'place_uuid' => '1accbcfb-3b22-4762-bc13-be0f67fd3116',
]
);

$count = $this->duplicatePlaceRepository->howManyPlacesAreToBeImported();
$this->assertEquals(2, $count);
}
}

0 comments on commit 2a73425

Please sign in to comment.