diff --git a/app/Console/Command/ImportDuplicatePlaces.php b/app/Console/Command/ImportDuplicatePlaces.php new file mode 100644 index 0000000000..82cc30439c --- /dev/null +++ b/app/Console/Command/ImportDuplicatePlaces.php @@ -0,0 +1,91 @@ +importDuplicatePlacesProcessor = $importDuplicatePlacesProcessor; + $this->dbalDuplicatePlaceRepository = $dbalDuplicatePlaceRepository; + } + + public function configure(): void + { + $this + ->setName('place:duplicate-places:import') + ->setDescription('Import duplicate places from the import tables, set clusters ready for processing') + ->addOption( + self::FORCE, + null, + InputOption::VALUE_NONE, + 'Skip confirmation.' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): ?int + { + $howManyPlacesAreToBeImported = $this->dbalDuplicatePlaceRepository->howManyPlacesAreToBeImported(); + $howManyPlacesAreToBeDeleted = count($this->dbalDuplicatePlaceRepository->getPlacesNoLongerInCluster()); + + if ($howManyPlacesAreToBeImported === 0 && $howManyPlacesAreToBeDeleted === 0) { + $output->writeln('duplicate_places is already synced'); + return self::SUCCESS; + } + + if (!$this->askConfirmation( + $input, + $output, + sprintf( + 'This action will sync a total of %d new places, and remove %d places from the duplicate places table. Do you want to continue? [y/N] ', + $howManyPlacesAreToBeImported, + $howManyPlacesAreToBeDeleted, + ) + )) { + return self::SUCCESS; + } + + $this->importDuplicatePlacesProcessor->sync(); + + $output->writeln('Duplicate places were synced and old clusters were removed. You probably want to run place:process-duplicates to give canonicals to the new clusters now.'); + + return self::SUCCESS; + } + + private function askConfirmation(InputInterface $input, OutputInterface $output, string $message): bool + { + if ($input->getOption(self::FORCE)) { + return true; + } + + return $this + ->getHelper('question') + ->ask( + $input, + $output, + new ConfirmationQuestion( + $message, + true + ) + ); + } +} diff --git a/app/Console/ConsoleServiceProvider.php b/app/Console/ConsoleServiceProvider.php index 805ed94c1c..8f1b7fa31e 100644 --- a/app/Console/ConsoleServiceProvider.php +++ b/app/Console/ConsoleServiceProvider.php @@ -26,6 +26,7 @@ use CultuurNet\UDB3\Console\Command\GeocodeEventCommand; use CultuurNet\UDB3\Console\Command\GeocodeOrganizerCommand; use CultuurNet\UDB3\Console\Command\GeocodePlaceCommand; +use CultuurNet\UDB3\Console\Command\ImportDuplicatePlaces; use CultuurNet\UDB3\Console\Command\ImportOfferAutoClassificationLabels; use CultuurNet\UDB3\Console\Command\IncludeLabel; use CultuurNet\UDB3\Console\Command\KeycloakCommand; @@ -59,6 +60,7 @@ use CultuurNet\UDB3\Kinepolis\Trailer\YoutubeTrailerRepository; use CultuurNet\UDB3\Offer\OfferType; use CultuurNet\UDB3\Organizer\WebsiteNormalizer; +use CultuurNet\UDB3\Place\Canonical\ImportDuplicatePlacesProcessor; use CultuurNet\UDB3\Place\Canonical\DuplicatePlaceRemovedFromClusterRepository; use CultuurNet\UDB3\Search\EventsSapi3SearchService; use CultuurNet\UDB3\Search\OrganizersSapi3SearchService; @@ -86,6 +88,7 @@ final class ConsoleServiceProvider extends AbstractServiceProvider 'console.fire-projected-to-jsonld-for-relations', 'console.fire-projected-to-jsonld', 'console.place:process-duplicates', + 'console.place:duplicate-places:import', 'console.event:bulk-remove-from-production', 'console.event:reindex-offers-with-popularity', 'console.place:reindex-offers-with-popularity', @@ -261,6 +264,17 @@ function () use ($container) { ) ); + $container->addShared( + 'console.place:duplicate-places:import', + fn () => new ImportDuplicatePlaces( + $container->get('duplicate_place_repository'), + new ImportDuplicatePlacesProcessor( + $container->get('duplicate_place_repository'), + $container->get(DuplicatePlaceRemovedFromClusterRepository::class) + ) + ) + ); + $container->addShared( 'console.event:bulk-remove-from-production', fn () => new BulkRemoveFromProduction($container->get('event_command_bus')) diff --git a/src/Place/Canonical/DBALDuplicatePlaceRepository.php b/src/Place/Canonical/DBALDuplicatePlaceRepository.php index a8ccde9ee0..a9180538c4 100644 --- a/src/Place/Canonical/DBALDuplicatePlaceRepository.php +++ b/src/Place/Canonical/DBALDuplicatePlaceRepository.php @@ -80,18 +80,7 @@ public function getDuplicatesOfPlace(string $placeId): ?array return count($duplicates) > 0 ? $duplicates : null; } - public function getPlacesNoLongerInCluster(): array - { - // All places that do not exist in duplicate_places_import - $statement = $this->connection->createQueryBuilder() - ->select('DISTINCT dp.place_uuid') - ->from('duplicate_places', 'dp') - ->leftJoin('dp', 'duplicate_places_import', 'dpi', 'dp.place_uuid = dpi.place_uuid') - ->where('dpi.place_uuid IS NULL') - ->execute(); - return $statement->fetchFirstColumn(); - } public function getClustersToBeRemoved(): array { @@ -124,6 +113,32 @@ public function getPlacesWithCluster(): array }, $statement->fetchAllAssociative()); } + public function howManyPlacesAreToBeImported(): int + { + // COUNT from `duplicate_places_import` not present in `duplicate_places` + $result = $this->connection->createQueryBuilder() + ->select('COUNT(*) AS not_in_duplicate') + ->from('duplicate_places_import', 'dpi') + ->leftJoin('dpi', 'duplicate_places', 'dp', 'dpi.cluster_id = dp.cluster_id AND dpi.place_uuid = dp.place_uuid') + ->where('dp.cluster_id IS NULL') + ->execute(); + + return (int)($result->fetchOne() ?? 0); + } + + public function getPlacesNoLongerInCluster(): array + { + // All places that do not exist in duplicate_places_import + $statement = $this->connection->createQueryBuilder() + ->select('DISTINCT dp.place_uuid') + ->from('duplicate_places', 'dp') + ->leftJoin('dp', 'duplicate_places_import', 'dpi', 'dp.place_uuid = dpi.place_uuid') + ->where('dpi.place_uuid IS NULL') + ->execute(); + + return $statement->fetchFirstColumn(); + } + public function deleteCluster(string $clusterId): void { $this->connection->createQueryBuilder() diff --git a/src/Place/Canonical/DuplicatePlaceRepository.php b/src/Place/Canonical/DuplicatePlaceRepository.php index d69f15aeae..c03427464b 100644 --- a/src/Place/Canonical/DuplicatePlaceRepository.php +++ b/src/Place/Canonical/DuplicatePlaceRepository.php @@ -24,11 +24,13 @@ public function getDuplicatesOfPlace(string $placeId): ?array; public function getPlacesNoLongerInCluster(): array; + public function getClustersToBeRemoved(): array; + /** @return PlaceWithCluster[] */ public function getPlacesWithCluster(): array; public function addToDuplicatePlaces(PlaceWithCluster $clusterRecordRow): void; public function deleteCluster(string $clusterId): void; - public function getClustersToBeRemoved(): array; + public function howManyPlacesAreToBeImported(): int; } diff --git a/tests/Console/Command/ImportDuplicatePlacesTest.php b/tests/Console/Command/ImportDuplicatePlacesTest.php new file mode 100644 index 0000000000..05bb6ec701 --- /dev/null +++ b/tests/Console/Command/ImportDuplicatePlacesTest.php @@ -0,0 +1,86 @@ +dbalDuplicatePlaceRepository = $this->createMock(DBALDuplicatePlaceRepository::class); + $this->importDuplicatePlacesProcessor = $this->createMock(ImportDuplicatePlacesProcessor::class); + $this->input = $this->createMock(InputInterface::class); + $this->output = $this->createMock(OutputInterface::class); + + $this->command = new ImportDuplicatePlaces( + $this->dbalDuplicatePlaceRepository, + $this->importDuplicatePlacesProcessor + ); + } + + public function testExecuteSucceedsWhenTablesAreAlreadySynced(): void + { + $this->dbalDuplicatePlaceRepository + ->expects($this->once()) + ->method('howManyPlacesAreToBeImported') + ->willReturn(0); + + $this->dbalDuplicatePlaceRepository + ->expects($this->once()) + ->method('getPlacesNoLongerInCluster') + ->willReturn([]); + + $this->output + ->expects($this->once()) + ->method('writeln') + ->with('duplicate_places is already synced'); + + $this->assertEquals(0, $this->command->run($this->input, $this->output)); + } + + public function testExecuteConfirmsAndSyncsWhenChangesAreWithinLimits(): void + { + $this->dbalDuplicatePlaceRepository + ->expects($this->once()) + ->method('howManyPlacesAreToBeImported') + ->willReturn(50); + + $this->dbalDuplicatePlaceRepository + ->expects($this->once()) + ->method('getPlacesNoLongerInCluster') + ->willReturn([Uuid::uuid4()]); + + $helper = $this->createMock(QuestionHelper::class); + $helper->expects($this->once()) + ->method('ask') + ->willReturn(true); + $this->command->setHelperSet(new HelperSet(['question' => $helper])); + + $this->importDuplicatePlacesProcessor + ->expects($this->once()) + ->method('sync'); + + $this->assertEquals(0, $this->command->run($this->input, $this->output)); + } +} diff --git a/tests/Place/Canonical/DBALDuplicatePlaceRepositoryTest.php b/tests/Place/Canonical/DBALDuplicatePlaceRepositoryTest.php index 9744a3ecc2..af0f1c8b8e 100644 --- a/tests/Place/Canonical/DBALDuplicatePlaceRepositoryTest.php +++ b/tests/Place/Canonical/DBALDuplicatePlaceRepositoryTest.php @@ -251,4 +251,86 @@ public function test_add_to_duplicate_places(): void $this->assertEquals(1, $raw['total']); } + + /** @dataProvider clusterChangesDataProvider */ + public function test_calculate_how_many_clusters_have_changed(array $clusters, int $expectedPlacesTobeImported, int $expectedPlacesTobeDeleted): void + { + foreach ($clusters as [$clusterId, $placeUuid]) { + $this->getConnection()->insert( + 'duplicate_places_import', + [ + 'cluster_id' => $clusterId, + 'place_uuid' => $placeUuid, + ] + ); + } + + $this->assertEquals($expectedPlacesTobeImported, $this->duplicatePlaceRepository->howManyPlacesAreToBeImported()); + $this->assertCount($expectedPlacesTobeDeleted, $this->duplicatePlaceRepository->getPlacesNoLongerInCluster()); + } + + public static function clusterChangesDataProvider(): array + { + return [ + 'everything is new' => [ + [ + + ], + 0, + 5, + ], + 'Some new, some removed' => [ + [ + ['cluster_1', '19ce6565-76be-425d-94d6-894f84dd2947'], + ['cluster_1', '1accbcfb-3b22-4762-bc13-be0f67fd3116'], + ['new', '04a549ba-6e5e-433b-9601-07b7a809758e'], + ], + 1, + 3, + ], + 'Nothing has changed' => [ + [ + ['cluster_1', '19ce6565-76be-425d-94d6-894f84dd2947'], + ['cluster_1', '1accbcfb-3b22-4762-bc13-be0f67fd3116'], + ['cluster_1', '526605d3-7cc4-4607-97a4-065896253f42'], + ['cluster_2', '4a355db3-c3f9-4acc-8093-61b333a3aefb'], + ['cluster_2', '64901efc-6bd7-4e9d-8916-fcdeb5b1c8ad'], + ], + 0, + 0, + ], + 'Everything single place has been moved' => [ + [ + ['5', '19ce6565-76be-425d-94d6-894f84dd2947'], + ['5', '1accbcfb-3b22-4762-bc13-be0f67fd3116'], + ['5', '526605d3-7cc4-4607-97a4-065896253f42'], + ['5', '4a355db3-c3f9-4acc-8093-61b333a3aefb'], + ['5', '64901efc-6bd7-4e9d-8916-fcdeb5b1c8ad'], + ], + 5, + 0, + ], + ]; + } + + public function test_how_many_places_are_to_be_imported(): void + { + $this->getConnection()->insert( + 'duplicate_places_import', + [ + 'cluster_id' => 'my_brand_new_cluster', + 'place_uuid' => '19ce6565-76be-425d-94d6-894f84dd2947', + ] + ); + $this->getConnection()->insert( + 'duplicate_places_import', + [ + 'cluster_id' => 'my_brand_new_cluster', + 'place_uuid' => '1accbcfb-3b22-4762-bc13-be0f67fd3116', + ] + ); + + $count = $this->duplicatePlaceRepository->howManyPlacesAreToBeImported(); + $this->assertEquals(2, $count); + } }