diff --git a/config/services/search/search-services.yaml b/config/services/search/search-services.yaml index 1df6548d..682cf04f 100644 --- a/config/services/search/search-services.yaml +++ b/config/services/search/search-services.yaml @@ -40,6 +40,9 @@ services: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\AssetLazyLoadingHandlerInterface: class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\AssetLazyLoadingHandler + Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\RequiredByElementListServiceInterface: + class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\RequiredByElementListService + Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\DataObjectLazyLoadingHandlerInterface: class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\DataObjectLazyLoadingHandler diff --git a/src/Model/SearchIndex/HitData.php b/src/Model/SearchIndex/HitData.php new file mode 100644 index 00000000..e6c9b307 --- /dev/null +++ b/src/Model/SearchIndex/HitData.php @@ -0,0 +1,42 @@ +id; + } + + public function getElementType(): string + { + return $this->elementType; + } + + public function getIndex(): string + { + return $this->index; + } +} diff --git a/src/Repository/IndexQueueRepository.php b/src/Repository/IndexQueueRepository.php index 894623a0..56407239 100644 --- a/src/Repository/IndexQueueRepository.php +++ b/src/Repository/IndexQueueRepository.php @@ -17,6 +17,7 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\Repository; use Doctrine\DBAL\Connection; +use Doctrine\DBAL\Exception as DBALException; use Doctrine\DBAL\Query\QueryBuilder as DBALQueryBuilder; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\NonUniqueResultException; @@ -24,6 +25,8 @@ use Doctrine\ORM\QueryBuilder; use Exception; use Pimcore\Bundle\GenericDataIndexBundle\Entity\IndexQueue; +use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\IndexQueueOperation; +use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\HitData; use Pimcore\Bundle\GenericDataIndexBundle\Service\TimeServiceInterface; use Pimcore\Bundle\GenericDataIndexBundle\Traits\LoggerAwareTrait; use Symfony\Component\Serializer\Exception\ExceptionInterface; @@ -105,7 +108,7 @@ public function getUnhandledIndexQueueEntries( /** * @param IndexQueue[] $entries * - * @throws \Doctrine\DBAL\Exception + * @throws DBALException */ public function deleteQueueEntries(array $entries): void { @@ -166,12 +169,12 @@ public function generateSelectQuery( } /** - * @throws \Doctrine\DBAL\Exception + * @throws DBALException */ public function enqueueBySelectQuery(DBALQueryBuilder $queryBuilder): void { $sql = <<connection->quote($item->getId()), + $this->connection->quote($item->getElementType()), + $this->connection->quote($item->getIndex()), + $this->connection->quote($operation->value), + $operationTime + ); + } + $this->connection->executeQuery( + sprintf($sql, IndexQueue::TABLE, implode(',', $values)) + ); + } + + /** + * @throws DBALException */ public function dispatchItems( int $limit diff --git a/src/SearchIndexAdapter/DefaultSearch/Asset/FieldDefinitionAdapter/AbstractAdapter.php b/src/SearchIndexAdapter/DefaultSearch/Asset/FieldDefinitionAdapter/AbstractAdapter.php index 2c524df7..e484df42 100644 --- a/src/SearchIndexAdapter/DefaultSearch/Asset/FieldDefinitionAdapter/AbstractAdapter.php +++ b/src/SearchIndexAdapter/DefaultSearch/Asset/FieldDefinitionAdapter/AbstractAdapter.php @@ -141,4 +141,4 @@ protected function throwInvalidFilterValueArgumentException(mixed $value, AssetM ) ); } -} \ No newline at end of file +} diff --git a/src/SearchIndexAdapter/DefaultSearch/DataObject/FieldDefinitionAdapter/AbstractAdapter.php b/src/SearchIndexAdapter/DefaultSearch/DataObject/FieldDefinitionAdapter/AbstractAdapter.php index a4154e49..b25cb810 100644 --- a/src/SearchIndexAdapter/DefaultSearch/DataObject/FieldDefinitionAdapter/AbstractAdapter.php +++ b/src/SearchIndexAdapter/DefaultSearch/DataObject/FieldDefinitionAdapter/AbstractAdapter.php @@ -99,4 +99,4 @@ public function getInheritedData( return $this->getInheritedData($parent, $objectId, $value, $key, $language, $callback); } -} \ No newline at end of file +} diff --git a/src/SearchIndexAdapter/DefaultSearch/DefaultSearchService.php b/src/SearchIndexAdapter/DefaultSearch/DefaultSearchService.php index b276d808..c1e171b8 100644 --- a/src/SearchIndexAdapter/DefaultSearch/DefaultSearchService.php +++ b/src/SearchIndexAdapter/DefaultSearch/DefaultSearchService.php @@ -21,6 +21,7 @@ use Pimcore\Bundle\GenericDataIndexBundle\Exception\DefaultSearch\SearchFailedException; use Pimcore\Bundle\GenericDataIndexBundle\Exception\SwitchIndexAliasException; use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Debug\SearchInformation; +use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Search; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\AdapterSearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\SearchResult; @@ -37,9 +38,9 @@ */ final class DefaultSearchService implements SearchIndexServiceInterface { - private const INDEX_VERSION_ODD = 'odd'; + public const INDEX_VERSION_ODD = 'odd'; - private const INDEX_VERSION_EVEN = 'even'; + public const INDEX_VERSION_EVEN = 'even'; use LoggerAwareTrait; @@ -229,7 +230,7 @@ public function createPaginatedSearch( int $page, int $pageSize, bool $aggregationsOnly = false - ): AdapterSearchInterface { + ): DefaultSearchInterface { if ($aggregationsOnly) { return new Search( from: 0, diff --git a/src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchService.php b/src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchService.php index 3138d920..892fb5f8 100644 --- a/src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchService.php +++ b/src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchService.php @@ -16,11 +16,14 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search; +use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\FieldCategory; use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\FieldCategory\SystemField; use Pimcore\Bundle\GenericDataIndexBundle\Exception\InvalidArgumentException; use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Sort\FieldSort; use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Sort\FieldSortList; +use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\HitData; +use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\SearchResultHit; use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\SearchIndexServiceInterface; use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigServiceInterface; @@ -49,6 +52,26 @@ public function fetchAllIds(DefaultSearchInterface $search, string $indexName, b return $this->doFetchIds($search, $indexName); } + /** + * @return HitData[] + */ + public function fetchAllTypesAndIds( + DefaultSearchInterface $search, + string $indexName, + bool $sortById = true + ): array { + $search = clone $search; + if ($sortById) { + $search->setSortList(new FieldSortList([new FieldSort(SystemField::ID->getPath())])); + } + + if ($search->getSortList()->isEmpty()) { + throw new InvalidArgumentException('Search must have a sort defined to be able to fetch all ids'); + } + + return $this->doFetchIdsAndTypes($search, $indexName); + } + private function doFetchIds(DefaultSearchInterface $search, string $indexName, ?array $searchAfter = null): array { $search->setFrom(0); @@ -66,6 +89,34 @@ private function doFetchIds(DefaultSearchInterface $search, string $indexName, ? return $ids; } + private function doFetchIdsAndTypes( + DefaultSearchInterface $search, + string $indexName, + ?array $searchAfter = null + ): array { + $search->setFrom(0); + $search->setSize($this->getPageSize()); + $search->setSource([SystemField::ELEMENT_TYPE->getPath()]); + $search->setSearchAfter($searchAfter); + $searchResult = $this->searchIndexService->search($search, $indexName); + $hits = $searchResult->getHits(); + $idsAndTypes = array_map( + static fn (SearchResultHit $item) => + new HitData( + id: $item->getId(), + elementType: $item->getSource()[FieldCategory::SYSTEM_FIELDS->value][SystemField::ELEMENT_TYPE->value], + index: $item->getIndex(), + ), + $hits + ); + $lastHit = $searchResult->getLastHit(); + if ($lastHit && (count($hits) === $this->getPageSize())) { + return array_merge($idsAndTypes, $this->doFetchIdsAndTypes($search, $indexName, $lastHit->getSort())); + } + + return $idsAndTypes; + } + private function getPageSize(): int { $maxResultWindow = $this->searchIndexConfigService->getIndexSettings()['max_result_window']; diff --git a/src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchServiceInterface.php b/src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchServiceInterface.php index ad47b26a..77f2627e 100644 --- a/src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchServiceInterface.php +++ b/src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchServiceInterface.php @@ -17,8 +17,18 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search; use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface; +use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\HitData; interface FetchIdsBySearchServiceInterface { public function fetchAllIds(DefaultSearchInterface $search, string $indexName, bool $sortById = true): array; + + /** + * @return HitData[] + */ + public function fetchAllTypesAndIds( + DefaultSearchInterface $search, + string $indexName, + bool $sortById = true + ): array; } diff --git a/src/SearchIndexAdapter/SearchIndexServiceInterface.php b/src/SearchIndexAdapter/SearchIndexServiceInterface.php index 2ee35144..5d07e4cc 100644 --- a/src/SearchIndexAdapter/SearchIndexServiceInterface.php +++ b/src/SearchIndexAdapter/SearchIndexServiceInterface.php @@ -17,6 +17,7 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter; use Exception; +use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\AdapterSearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\SearchResult; @@ -56,7 +57,7 @@ public function createPaginatedSearch( int $page, int $pageSize, bool $aggregationsOnly = false - ): AdapterSearchInterface; + ): DefaultSearchInterface; public function search(AdapterSearchInterface $search, string $indexName): SearchResult; diff --git a/src/Service/ElementService.php b/src/Service/ElementService.php index 86260ac5..1856cdfe 100644 --- a/src/Service/ElementService.php +++ b/src/Service/ElementService.php @@ -21,8 +21,10 @@ use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\ElementType; use Pimcore\Bundle\GenericDataIndexBundle\Exception\InvalidElementTypeException; use Pimcore\Model\Asset; +use Pimcore\Model\DataObject; use Pimcore\Model\DataObject\AbstractObject; use Pimcore\Model\Document; +use Pimcore\Model\Element\ElementInterface; /** * @internal @@ -47,6 +49,19 @@ public function getElementByType(int $id, string $type): Asset|AbstractObject|Do }; } + /** + * @throws InvalidElementTypeException + */ + public function getElementType(ElementInterface $element): ElementType + { + return match (true) { + $element instanceof Asset => ElementType::ASSET, + $element instanceof Document => ElementType::DOCUMENT, + $element instanceof DataObject => ElementType::DATA_OBJECT, + default => throw new InvalidElementTypeException('Invalid element type: ' . $element->getType()) + }; + } + public function classDefinitionExists(string $name): bool { try { diff --git a/src/Service/ElementServiceInterface.php b/src/Service/ElementServiceInterface.php index a4d3cecb..99c6031f 100644 --- a/src/Service/ElementServiceInterface.php +++ b/src/Service/ElementServiceInterface.php @@ -16,10 +16,12 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\Service; +use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\ElementType; use Pimcore\Bundle\GenericDataIndexBundle\Exception\InvalidElementTypeException; use Pimcore\Model\Asset; use Pimcore\Model\DataObject\AbstractObject; use Pimcore\Model\Document; +use Pimcore\Model\Element\ElementInterface; /** * @internal @@ -33,5 +35,10 @@ interface ElementServiceInterface */ public function getElementByType(int $id, string $type): Asset|AbstractObject|Document|null; + /** + * @throws InvalidElementTypeException + */ + public function getElementType(ElementInterface $element): ElementType; + public function classDefinitionExists(string $name): bool; } diff --git a/src/Service/Search/SearchService/Asset/AssetSearchService.php b/src/Service/Search/SearchService/Asset/AssetSearchService.php index 20921bbf..6d406898 100644 --- a/src/Service/Search/SearchService/Asset/AssetSearchService.php +++ b/src/Service/Search/SearchService/Asset/AssetSearchService.php @@ -52,7 +52,7 @@ public function search( AssetSearchInterface $assetSearch, PermissionTypes $permissionType = PermissionTypes::LIST ): AssetSearchResult { - $assetSearch = $this->searchHelper->addSearchRestrictions( + $search = $this->searchHelper->addSearchRestrictions( search: $assetSearch, userPermission: UserPermissionTypes::ASSETS->value, workspaceType: AssetWorkspace::WORKSPACE_TYPE, @@ -60,7 +60,7 @@ public function search( ); $searchResult = $this->searchHelper->performSearch( - search: $assetSearch, + search: $search, indexName: $this->assetTypeAdapter->getAliasIndexName() ); @@ -75,12 +75,12 @@ public function search( items: $this->searchHelper->hydrateSearchResultHits( $searchResult, $childrenCounts, - $assetSearch->getUser() + $search->getUser() ), pagination: $this->paginationInfoService->getPaginationInfoFromSearchResult( searchResult: $searchResult, - page: $assetSearch->getPage(), - pageSize: $assetSearch->getPageSize() + page: $search->getPage(), + pageSize: $search->getPageSize() ), aggregations: $searchResult->getAggregations(), ); diff --git a/src/Service/Search/SearchService/Document/DocumentSearchService.php b/src/Service/Search/SearchService/Document/DocumentSearchService.php index 473db3f2..62d8184b 100644 --- a/src/Service/Search/SearchService/Document/DocumentSearchService.php +++ b/src/Service/Search/SearchService/Document/DocumentSearchService.php @@ -52,7 +52,7 @@ public function search( DocumentSearchInterface $documentSearch, PermissionTypes $permissionType = PermissionTypes::LIST ): DocumentSearchResult { - $documentSearch = $this->searchHelper->addSearchRestrictions( + $search = $this->searchHelper->addSearchRestrictions( search: $documentSearch, userPermission: UserPermissionTypes::DOCUMENTS->value, workspaceType: DocumentWorkspace::WORKSPACE_TYPE, @@ -60,7 +60,7 @@ public function search( ); $searchResult = $this->searchHelper->performSearch( - search: $documentSearch, + search: $search, indexName: $this->documentTypeAdapter->getAliasIndexName() ); @@ -75,12 +75,12 @@ public function search( items: $this->searchHelper->hydrateSearchResultHits( $searchResult, $childrenCounts, - $documentSearch->getUser() + $search->getUser() ), pagination: $this->paginationInfoService->getPaginationInfoFromSearchResult( searchResult: $searchResult, - page: $documentSearch->getPage(), - pageSize: $documentSearch->getPageSize() + page: $search->getPage(), + pageSize: $search->getPageSize() ), aggregations: $searchResult->getAggregations(), ); diff --git a/src/Service/Search/SearchService/Element/ElementSearchHelperInterface.php b/src/Service/Search/SearchService/Element/ElementSearchHelperInterface.php index 8bc10e49..b919f027 100644 --- a/src/Service/Search/SearchService/Element/ElementSearchHelperInterface.php +++ b/src/Service/Search/SearchService/Element/ElementSearchHelperInterface.php @@ -16,7 +16,7 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\Element; use Pimcore\Bundle\GenericDataIndexBundle\Enum\Permission\PermissionTypes; -use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\AdapterSearchInterface; +use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\SearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\SearchResult; use Pimcore\Model\User; @@ -37,7 +37,7 @@ public function createAdapterSearch( SearchInterface $search, string $indexName, bool $enableOrderByPageNumber = false - ): AdapterSearchInterface; + ): DefaultSearchInterface; public function hydrateSearchResultHits( SearchResult $searchResult, diff --git a/src/Service/Search/SearchService/Element/ElementSearchService.php b/src/Service/Search/SearchService/Element/ElementSearchService.php index 4e4c5720..711bd2fa 100644 --- a/src/Service/Search/SearchService/Element/ElementSearchService.php +++ b/src/Service/Search/SearchService/Element/ElementSearchService.php @@ -49,10 +49,10 @@ public function search( ElementSearchInterface $elementSearch, PermissionTypes $permissionType = PermissionTypes::LIST ): ElementSearchResult { - $elementSearch = $this->searchHelper->addSearchRestrictions($elementSearch, $permissionType); + $search = $this->searchHelper->addSearchRestrictions($elementSearch, $permissionType); $searchResult = $this->searchHelper->performSearch( - $elementSearch, + $search, $this->globalIndexAliasService->getElementSearchAliasName() ); @@ -61,12 +61,12 @@ public function search( items: $this->searchHelper->hydrateSearchResultHits( $searchResult, [], - $elementSearch->getUser() + $search->getUser() ), pagination: $this->paginationInfoService->getPaginationInfoFromSearchResult( searchResult: $searchResult, - page: $elementSearch->getPage(), - pageSize: $elementSearch->getPageSize() + page: $search->getPage(), + pageSize: $search->getPageSize() ), aggregations: $searchResult->getAggregations(), ); diff --git a/src/Service/Search/SearchService/RequiredByElementListService.php b/src/Service/Search/SearchService/RequiredByElementListService.php new file mode 100644 index 00000000..333d8f12 --- /dev/null +++ b/src/Service/Search/SearchService/RequiredByElementListService.php @@ -0,0 +1,102 @@ +searchProvider->createElementSearch(); + $search->addModifier( + new RequiredByFilter( + $element->getId(), + $this->elementService->getElementType($element) + ) + ); + $adapterSearch = $this->transformToAdapterSearchService->transform($search); + if ($adapterSearch->getSortList()->isEmpty()) { + $sortById = true; + } + + return $this->fetchIdsService->fetchAllTypesAndIds( + $adapterSearch, + $this->indexNameResolver->resolveIndexName($search), + $sortById + ); + } + + /** + * @return HitData[] + */ + public function getDependencyListForCurrentPage( + ElementInterface $element, + ElementSearchInterface $search + ): array { + $search->addModifier( + new RequiredByFilter( + $element->getId(), + $this->elementService->getElementType($element) + ) + ); + + $adapterSearch = $this->transformToAdapterSearchService->transform($search); + $adapterSearch = clone $adapterSearch; + $adapterSearch->setSource([SystemField::ELEMENT_TYPE->getPath()]); + $searchResult = $this->searchIndexService->search( + $adapterSearch, + $this->indexNameResolver->resolveIndexName($search) + ); + + return array_map( + static fn (SearchResultHit $item) => + new HitData( + id: $item->getId(), + elementType: $item->getSource()[FieldCategory::SYSTEM_FIELDS->value][SystemField::ELEMENT_TYPE->value], + index: $item->getIndex(), + ), + $searchResult->getHits() + ); + } +} diff --git a/src/Service/Search/SearchService/RequiredByElementListServiceInterface.php b/src/Service/Search/SearchService/RequiredByElementListServiceInterface.php new file mode 100644 index 00000000..634febf3 --- /dev/null +++ b/src/Service/Search/SearchService/RequiredByElementListServiceInterface.php @@ -0,0 +1,39 @@ +searchIndexService->createPaginatedSearch( $search->getPage(), $search->getPageSize(), diff --git a/src/Service/Search/SearchService/TransformToAdapterSearchService.php b/src/Service/Search/SearchService/TransformToAdapterSearchService.php index d9728034..1af7f35f 100644 --- a/src/Service/Search/SearchService/TransformToAdapterSearchService.php +++ b/src/Service/Search/SearchService/TransformToAdapterSearchService.php @@ -17,11 +17,11 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService; use Pimcore\Bundle\GenericDataIndexBundle\Exception\InvalidArgumentException; +use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Asset\AssetSearch; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\DataObject\DataObjectSearch; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Document\DocumentSearch; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Element\ElementSearch; -use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\AdapterSearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\SearchInterface; /** @@ -38,8 +38,10 @@ public function __construct( ) { } - public function transform(SearchInterface $search, bool $enableOrderByPageNumber = false): AdapterSearchInterface - { + public function transform( + SearchInterface $search, + bool $enableOrderByPageNumber = false + ): DefaultSearchInterface { $index = $this->indexNameResolver->resolveIndexName($search); return match(true) { diff --git a/src/Service/Search/SearchService/TransformToAdapterSearchServiceInterface.php b/src/Service/Search/SearchService/TransformToAdapterSearchServiceInterface.php index c8b8d2e0..4c1f71da 100644 --- a/src/Service/Search/SearchService/TransformToAdapterSearchServiceInterface.php +++ b/src/Service/Search/SearchService/TransformToAdapterSearchServiceInterface.php @@ -16,7 +16,7 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService; -use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\AdapterSearchInterface; +use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\SearchInterface; /** @@ -24,5 +24,8 @@ */ interface TransformToAdapterSearchServiceInterface { - public function transform(SearchInterface $search, bool $enableOrderByPageNumber = false): AdapterSearchInterface; + public function transform( + SearchInterface $search, + bool $enableOrderByPageNumber = false + ): DefaultSearchInterface; } diff --git a/src/Service/SearchIndex/IndexQueue/EnqueueService.php b/src/Service/SearchIndex/IndexQueue/EnqueueService.php index 8a2d64b3..f330ecc0 100644 --- a/src/Service/SearchIndex/IndexQueue/EnqueueService.php +++ b/src/Service/SearchIndex/IndexQueue/EnqueueService.php @@ -17,11 +17,13 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueue; use Doctrine\DBAL\Exception; +use Exception as ThrowableException; use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\ElementType; use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\IndexName; use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\IndexQueueOperation; use Pimcore\Bundle\GenericDataIndexBundle\Exception\EnqueueElementsException; use Pimcore\Bundle\GenericDataIndexBundle\Repository\IndexQueueRepository; +use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\RequiredByElementListServiceInterface; use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexService\ElementTypeAdapter\AdapterServiceInterface; use Pimcore\Bundle\GenericDataIndexBundle\Service\TimeServiceInterface; use Pimcore\Model\DataObject\ClassDefinition; @@ -38,6 +40,7 @@ public function __construct( private TimeServiceInterface $timeService, private QueueMessagesDispatcher $queueMessagesDispatcher, private AdapterServiceInterface $typeAdapterService, + private RequiredByElementListServiceInterface $requiredByElementListService ) { } @@ -189,9 +192,9 @@ public function enqueueDocuments(): EnqueueService } /** - * @throws \Exception + * @throws ThrowableException */ - public function enqueueRelatedItemsOnUpdate( + public function enqueueRelatedItems( ElementInterface $element, bool $includeElement, string $operation @@ -210,6 +213,23 @@ public function enqueueRelatedItemsOnUpdate( } } + /** + * @throws ThrowableException + */ + public function enqueueDependentItems( + ElementInterface $element, + IndexQueueOperation $operation + ): void { + $dependentItems = $this->requiredByElementListService->getDependencyList($element); + foreach (array_chunk($dependentItems, 1000) as $chunk) { + $this->indexQueueRepository->enqueueByItemList( + $chunk, + $operation, + $this->timeService->getCurrentMillisecondTimestamp() + ); + } + } + public function dispatchQueueMessages(bool $synchronously = false): void { $this->queueMessagesDispatcher->dispatchQueueMessages($synchronously); diff --git a/src/Service/SearchIndex/IndexQueue/EnqueueServiceInterface.php b/src/Service/SearchIndex/IndexQueue/EnqueueServiceInterface.php index 93966df4..cf7f2d2a 100644 --- a/src/Service/SearchIndex/IndexQueue/EnqueueServiceInterface.php +++ b/src/Service/SearchIndex/IndexQueue/EnqueueServiceInterface.php @@ -17,6 +17,8 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexQueue; use Doctrine\DBAL\Exception; +use Exception as ThrowableException; +use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\IndexQueueOperation; use Pimcore\Bundle\GenericDataIndexBundle\Exception\EnqueueElementsException; use Pimcore\Model\DataObject\ClassDefinition; use Pimcore\Model\Element\ElementInterface; @@ -53,13 +55,21 @@ public function enqueueAssets(): self; public function enqueueDocuments(): self; /** - * @throws \Exception + * @throws ThrowableException */ - public function enqueueRelatedItemsOnUpdate( + public function enqueueRelatedItems( ElementInterface $element, bool $includeElement, string $operation ): void; + /** + * @throws ThrowableException + */ + public function enqueueDependentItems( + ElementInterface $element, + IndexQueueOperation $operation + ): void; + public function dispatchQueueMessages(bool $synchronously = false): void; } diff --git a/src/Service/SearchIndex/IndexQueueService.php b/src/Service/SearchIndex/IndexQueueService.php index 439c2e0e..242ff09b 100644 --- a/src/Service/SearchIndex/IndexQueueService.php +++ b/src/Service/SearchIndex/IndexQueueService.php @@ -18,6 +18,8 @@ use Exception; use Pimcore\Bundle\GenericDataIndexBundle\Entity\IndexQueue; +use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\ElementType; +use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\IndexName; use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\IndexQueueOperation; use Pimcore\Bundle\GenericDataIndexBundle\Exception\HandleIndexQueueEntriesException; use Pimcore\Bundle\GenericDataIndexBundle\Exception\IndexDataException; @@ -62,17 +64,7 @@ public function updateIndexQueue( $this->doHandleIndexData($element, $operation); } - $this->enqueueService->enqueueRelatedItemsOnUpdate( - element: $element, - includeElement: !$processSynchronously, - operation: $operation - ); - - if ($element instanceof Asset) { - foreach ($this->indexService->updateAssetDependencies($element) as $asset) { - $this->updateIndexQueue($asset, IndexQueueOperation::UPDATE->value); - } - } + $this->handleQueueByOperation($element, $operation, $processSynchronously); $this->pathService->rewriteChildrenIndexPaths($element); } catch (Exception $e) { @@ -140,11 +132,50 @@ private function doHandleIndexData(ElementInterface $element, string $operation) } } + /** + * @throws Exception + */ + private function handleQueueByOperation( + ElementInterface $element, + string $operation, + bool $processSynchronously + ): void { + $this->enqueueService->enqueueRelatedItems( + element: $element, + includeElement: !$processSynchronously, + operation: $operation + ); + + if (($operation === IndexQueueOperation::UPDATE->value) && $element instanceof Asset) { + $this->enqueueService->enqueueDependentItems( + element: $element, + operation: IndexQueueOperation::UPDATE + ); + } + + if ($operation === IndexQueueOperation::DELETE->value) { + $this->enqueueService->enqueueDependentItems( + element: $element, + operation: IndexQueueOperation::UPDATE + ); + } + } + private function handleEntryByOperation(string $operation, IndexQueue $entry): void { if ($operation === IndexQueueOperation::DELETE->value) { + $isClass = false; + if ($entry->getElementType() === ElementType::DATA_OBJECT->value && + $entry->getElementIndexName() !== IndexName::DATA_OBJECT_FOLDER->value + ) { + $isClass = true; + } + $this->indexService->deleteFromSpecificIndex( - $this->searchIndexConfigService->getIndexName($entry->getElementIndexName()), + $this->searchIndexConfigService->getIndexName( + $entry->getElementIndexName(), + $isClass + ), $entry->getElementId() ); diff --git a/src/Service/SearchIndex/IndexService/ElementTypeAdapter/AbstractElementTypeAdapter.php b/src/Service/SearchIndex/IndexService/ElementTypeAdapter/AbstractElementTypeAdapter.php index 4e4b576c..bbd40e83 100644 --- a/src/Service/SearchIndex/IndexService/ElementTypeAdapter/AbstractElementTypeAdapter.php +++ b/src/Service/SearchIndex/IndexService/ElementTypeAdapter/AbstractElementTypeAdapter.php @@ -20,6 +20,8 @@ use Exception; use Pimcore\Bundle\GenericDataIndexBundle\Event\UpdateIndexDataEventInterface; use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigServiceInterface; +use Pimcore\Model\DataObject\ClassDefinition; +use Pimcore\Model\DataObject\Concrete; use Pimcore\Model\Element\ElementInterface; use Symfony\Component\Serializer\Normalizer\NormalizerInterface; use Symfony\Contracts\Service\Attribute\Required; @@ -33,17 +35,22 @@ abstract class AbstractElementTypeAdapter abstract public function supports(ElementInterface $element): bool; + /** + * @throws Exception + */ public function getAliasIndexNameByElement(ElementInterface $element): string { return $this->searchIndexConfigService->getIndexName( - $this->getIndexNameShortByElement($element) + $this->getIndexNameShortByElement($element), + $element instanceof Concrete ); } public function getAliasIndexName(mixed $context = null): string { return $this->searchIndexConfigService->getIndexName( - $this->getIndexNameShort($context) + $this->getIndexNameShort($context), + $context instanceof ClassDefinition ); } diff --git a/src/Service/SearchIndex/IndexService/ElementTypeAdapter/DataObjectTypeAdapter.php b/src/Service/SearchIndex/IndexService/ElementTypeAdapter/DataObjectTypeAdapter.php index eb84f8c9..334f703f 100644 --- a/src/Service/SearchIndex/IndexService/ElementTypeAdapter/DataObjectTypeAdapter.php +++ b/src/Service/SearchIndex/IndexService/ElementTypeAdapter/DataObjectTypeAdapter.php @@ -26,7 +26,6 @@ use Pimcore\Bundle\GenericDataIndexBundle\Event\DataObject\UpdateFolderIndexDataEvent; use Pimcore\Bundle\GenericDataIndexBundle\Event\DataObject\UpdateIndexDataEvent; use Pimcore\Bundle\GenericDataIndexBundle\Event\UpdateIndexDataEventInterface; -use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigService; use Pimcore\Bundle\GenericDataIndexBundle\Service\Serializer\Normalizer\DataObjectNormalizer; use Pimcore\Model\DataObject\AbstractObject; use Pimcore\Model\DataObject\ClassDefinition; @@ -67,7 +66,7 @@ public function getIndexNameShortByElement(ElementInterface $element): string public function getIndexNameShort(mixed $context): string { return match (true) { - $context instanceof ClassDefinition => SearchIndexConfigService::CLASS_INDEX_PREFIX . $context->getName(), + $context instanceof ClassDefinition => $context->getName(), $context === IndexName::DATA_OBJECT->value => $context, default => IndexName::DATA_OBJECT_FOLDER->value, }; @@ -75,9 +74,7 @@ public function getIndexNameShort(mixed $context): string public function getIndexNameByClassDefinition(ClassDefinition $classDefinition): string { - return $this->searchIndexConfigService->getIndexName( - SearchIndexConfigService::CLASS_INDEX_PREFIX . $classDefinition->getName() - ); + return $this->searchIndexConfigService->getIndexName($classDefinition->getName(), true); } public function getElementType(): string diff --git a/src/Service/SearchIndex/IndexService/IndexService.php b/src/Service/SearchIndex/IndexService/IndexService.php index 250ed892..369e3554 100644 --- a/src/Service/SearchIndex/IndexService/IndexService.php +++ b/src/Service/SearchIndex/IndexService/IndexService.php @@ -24,8 +24,6 @@ use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\SearchIndexServiceInterface; use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexService\ElementTypeAdapter\AdapterServiceInterface; use Pimcore\Bundle\GenericDataIndexBundle\Traits\LoggerAwareTrait; -use Pimcore\Model\Asset; -use Pimcore\Model\DataObject\AbstractObject; use Pimcore\Model\Element\ElementInterface; use Symfony\Component\Serializer\Exception\ExceptionInterface; use Symfony\Contracts\EventDispatcher\EventDispatcherInterface; @@ -120,25 +118,6 @@ public function deleteFromSpecificIndex(string $indexName, int $elementId): Inde return $this; } - public function updateAssetDependencies(Asset $asset): array - { - $elementsToUpdate = []; - foreach ($asset->getDependencies()->getRequiredBy() as $requiredByEntry) { - $element = null; - if ($requiredByEntry['type'] === 'object') { - $element = AbstractObject::getById($requiredByEntry['id']); - } - if ($requiredByEntry['type'] === 'asset') { - $element = Asset::getById($requiredByEntry['id']); - } - if ($element instanceof ElementInterface) { - $elementsToUpdate[] = $element; - } - } - - return $elementsToUpdate; - } - /** * @throws IndexDataException */ diff --git a/src/Service/SearchIndex/IndexService/IndexServiceInterface.php b/src/Service/SearchIndex/IndexService/IndexServiceInterface.php index 3999f12e..a8fc4e0c 100644 --- a/src/Service/SearchIndex/IndexService/IndexServiceInterface.php +++ b/src/Service/SearchIndex/IndexService/IndexServiceInterface.php @@ -17,7 +17,6 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\IndexService; use Pimcore\Bundle\GenericDataIndexBundle\Exception\IndexDataException; -use Pimcore\Model\Asset; use Pimcore\Model\Element\ElementInterface; /** @@ -33,6 +32,4 @@ public function updateIndexData(ElementInterface $element): IndexService; public function deleteFromIndex(ElementInterface $element): IndexService; public function deleteFromSpecificIndex(string $indexName, int $elementId): IndexService; - - public function updateAssetDependencies(Asset $asset): array; } diff --git a/src/Service/SearchIndex/SearchIndexConfigService.php b/src/Service/SearchIndex/SearchIndexConfigService.php index 0450300f..c6396c40 100644 --- a/src/Service/SearchIndex/SearchIndexConfigService.php +++ b/src/Service/SearchIndex/SearchIndexConfigService.php @@ -16,6 +16,7 @@ namespace Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex; +use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DefaultSearchService; use Psr\Log\LoggerAwareTrait; /** @@ -61,6 +62,30 @@ public function getIndexName(string $name, bool $isClass = false): string return $this->getIndexPrefix() . strtolower($name); } + /** + * return index name without any prefix or suffix + */ + public function getShortIndexName(string $name): string + { + if (str_starts_with($name, $this->getIndexPrefixWIthClassPrefix())) { + $name = substr($name, 0, strlen($this->getIndexPrefixWIthClassPrefix())); + } + + if (str_starts_with($name, $this->getIndexPrefix())) { + $name = substr($name, 0, strlen($this->getIndexPrefix())); + } + + if (str_ends_with($name, '-' . DefaultSearchService::INDEX_VERSION_ODD)) { + $name = substr($name, strlen('-' . DefaultSearchService::INDEX_VERSION_ODD)); + } + + if (str_ends_with($name, '-' . DefaultSearchService::INDEX_VERSION_EVEN)) { + $name = substr($name, strlen('-' . DefaultSearchService::INDEX_VERSION_EVEN)); + } + + return $name; + } + public function prefixIndexName(string $indexName): string { return $this->getIndexPrefix() . $indexName; @@ -109,4 +134,9 @@ public function getSystemFieldsSettings(string $elementType): array return $systemFieldsSettings; } + + private function getIndexPrefixWIthClassPrefix(): string + { + return $this->indexPrefix . self::CLASS_INDEX_PREFIX; + } } diff --git a/src/Service/SearchIndex/SearchIndexConfigServiceInterface.php b/src/Service/SearchIndex/SearchIndexConfigServiceInterface.php index acf3ef56..42d9d1ae 100644 --- a/src/Service/SearchIndex/SearchIndexConfigServiceInterface.php +++ b/src/Service/SearchIndex/SearchIndexConfigServiceInterface.php @@ -28,6 +28,11 @@ public function getClientType(): string; */ public function getIndexName(string $name, bool $isClass = false): string; + /** + * return index name without any prefix or suffix + */ + public function getShortIndexName(string $name): string; + public function prefixIndexName(string $indexName): string; public function getIndexPrefix(): string; diff --git a/tests/Functional/Search/SearchService/RequiredByElementListServiceTest.php b/tests/Functional/Search/SearchService/RequiredByElementListServiceTest.php new file mode 100644 index 00000000..1f9c5bca --- /dev/null +++ b/tests/Functional/Search/SearchService/RequiredByElementListServiceTest.php @@ -0,0 +1,110 @@ +tester->enableSynchronousProcessing(); + } + + protected function _after() + { + TestHelper::cleanUp(); + $this->tester->flushIndex(); + $this->tester->cleanupIndex(); + $this->tester->flushIndex(); + } + + public function testSearchDependencyList(): void + { + $asset = TestHelper::createImageAsset()->setKey('asset1')->save(); + $object1 = $this->createDependencyObject('object1', $asset); + $object2 = $this->createDependencyObject('object2', $asset); + $object3 = $this->createDependencyObject('object3', $asset); + + /** @var RequiredByElementListServiceInterface $dependencyService */ + $dependencyService = $this->tester->grabService(RequiredByElementListServiceInterface::class); + + $dependencyList = $dependencyService->getDependencyList($asset); + $this->assertIdArrayEquals( + [$object1->getId(), $object2->getId(), $object3->getId()], + $this->getDependencyIds($dependencyList) + ); + + /** @var SearchProviderInterface $searchProvider */ + $searchProvider = $this->tester->grabService(SearchProviderInterface::class); + $elementSearch = $searchProvider->createElementSearch(); + + $dependencyList = $dependencyService->getDependencyListForCurrentPage( + $asset, + ($elementSearch) + ->addModifier(new OrderByFullPath(SortDirection::DESC)) + ->setPage(1) + ->setPageSize(2) + ); + $this->assertEquals( + [$object3->getId(), $object2->getId()], + $this->getDependencyIds($dependencyList) + ); + + $dependencyList = $dependencyService->getDependencyList( + $asset, + ($elementSearch) + ->addModifier(new OrderByFullPath(SortDirection::DESC)) + ->setPage(1) + ->setPageSize(2) + ); + $this->assertEquals( + [$object3->getId(), $object2->getId(), $object1->getId()], + $this->getDependencyIds($dependencyList) + ); + } + + private function createDependencyObject(string $key, Asset $asset): Concrete + { + return TestHelper::createEmptyObject() + ->setKey($key) + ->setImage($asset) + ->save(); + } + + private function getDependencyIds(array $dependencyList): array + { + return array_map(fn (HitData $hit) => $hit->getId(), $dependencyList); + } + + private function assertIdArrayEquals(array $ids1, array $ids2): void + { + sort($ids1); + sort($ids2); + $this->assertEquals($ids1, $ids2); + } +} diff --git a/tests/Functional/SearchIndex/IndexQueueTest.php b/tests/Functional/SearchIndex/IndexQueueTest.php index 3eb3f220..ea9391aa 100644 --- a/tests/Functional/SearchIndex/IndexQueueTest.php +++ b/tests/Functional/SearchIndex/IndexQueueTest.php @@ -18,9 +18,13 @@ use Codeception\Test\Unit; use Exception; use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\ElementType; +use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\FieldCategory; use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\IndexName; use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\IndexQueueOperation; +use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\DataObject\DataObjectSearchInterface; use Pimcore\Bundle\GenericDataIndexBundle\Repository\IndexQueueRepository; +use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\DataObject\DataObjectSearchServiceInterface; +use Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\SearchProviderInterface; use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigServiceInterface; use Pimcore\Bundle\GenericDataIndexBundle\Tests\IndexTester; use Pimcore\Db; @@ -37,6 +41,8 @@ class IndexQueueTest extends Unit private const DOCUMENT_INDEX_NAME = 'document'; + private const IMAGE_KEY = 'image'; + protected function _before() { $this->searchIndexConfigService = $this->tester->grabService( @@ -177,6 +183,40 @@ public function testDataObjectDeleteWithQueue(): void $this->tester->checkDeletedIndexEntry($object->getId(), $objectIndex); } + /** + * @throws Exception + */ + public function testDependenciesWithQueue(): void + { + /** @var DataObjectSearchServiceInterface $searchService */ + $searchService = $this->tester->grabService('generic-data-index.test.service.data-object-search-service'); + /** @var SearchProviderInterface $searchProvider */ + $searchProvider = $this->tester->grabService(SearchProviderInterface::class); + $dataObjectSearch = $searchProvider->createDataObjectSearch(); + + $asset = TestHelper::createImageAsset(); + $object = TestHelper::createEmptyObject(); + $object->setImage($asset); + $object->save(); + + $assetIndex = $this->searchIndexConfigService->getIndexName(self::ASSET_INDEX_NAME); + $objectIndex = $this->searchIndexConfigService->getIndexName($object->getClassName(), true); + + $this->checkQueueEntry($asset->getId(), ElementType::ASSET->value); + $this->checkQueueEntry($object->getId(), ElementType::DATA_OBJECT->value); + $this->consume(); + + $this->tester->checkIndexEntry($object->getId(), $objectIndex); + $this->assertNotNull($this->getImageValueFromIndex($searchService, $dataObjectSearch)); + $this->checkAndDeleteElement($asset, $assetIndex); + + // asset is deleted, so the object should be updated as it has a dependency to asset + $this->checkQueueEntry($object->getId(), ElementType::DATA_OBJECT->value); + $this->consume(); + + $this->assertNull($this->getImageValueFromIndex($searchService, $dataObjectSearch)); + } + private function checkAndDeleteElement(ElementInterface $element, string $indexName): void { $this->tester->checkIndexEntry($element->getId(), $indexName); @@ -187,4 +227,26 @@ private function consume(): void { $this->tester->runCommand('messenger:consume', ['--limit'=>2], ['pimcore_generic_data_index_queue']); } + + private function checkQueueEntry(string $elementId, string $elementType): void + { + $this->assertGreaterThan( + 0, + Db::get()->fetchOne( + 'select count(elementId) from generic_data_index_queue where elementId = ? and elementType=?', + [$elementId, $elementType] + ) + ); + } + + private function getImageValueFromIndex( + DataObjectSearchServiceInterface $searchService, + DataObjectSearchInterface $dataObjectSearch + ): ?array { + $searchResult = $searchService->search($dataObjectSearch); + $this->assertCount(1, $searchResult->getItems()); + $data = $searchResult->getItems()[0]->getSearchIndexData(); + + return $data[FieldCategory::STANDARD_FIELDS->value][self::IMAGE_KEY]; + } }