From acbe1162300652a82a5ab9aa2631d7ff5de7b314 Mon Sep 17 00:00:00 2001 From: Iwona Just Date: Tue, 5 Dec 2023 12:35:11 +0100 Subject: [PATCH 1/3] support search indexing and searching for multi-instance fields --- src/config/app.php | 2 +- src/migrations/Install.php | 3 +- .../m231205_081215_searchindex_amend_pk.php | 40 ++++++++++++++ src/services/Search.php | 54 ++++++++++++++----- 4 files changed, 83 insertions(+), 16 deletions(-) create mode 100644 src/migrations/m231205_081215_searchindex_amend_pk.php diff --git a/src/config/app.php b/src/config/app.php index af5f9590226..268e94786cd 100644 --- a/src/config/app.php +++ b/src/config/app.php @@ -4,7 +4,7 @@ 'id' => 'CraftCMS', 'name' => 'Craft CMS', 'version' => '5.0.0-alpha', - 'schemaVersion' => '5.0.0.9', + 'schemaVersion' => '5.0.0.10', 'minVersionRequired' => '4.4.0', 'basePath' => dirname(__DIR__), // Defines the @app alias 'runtimePath' => '@storage/runtime', // Defines the @runtime alias diff --git a/src/migrations/Install.php b/src/migrations/Install.php index e860ea36a34..f8b00cf55d5 100644 --- a/src/migrations/Install.php +++ b/src/migrations/Install.php @@ -885,9 +885,10 @@ public function createIndexes(): void 'elementId' => $this->integer()->notNull(), 'attribute' => $this->string(25)->notNull(), 'fieldId' => $this->integer()->notNull(), + 'layoutElementUid' => $this->uid(), 'siteId' => $this->integer()->notNull(), 'keywords' => $this->text()->notNull(), - 'PRIMARY KEY([[elementId]], [[attribute]], [[fieldId]], [[siteId]])', + 'PRIMARY KEY([[elementId]], [[attribute]], [[fieldId]], [[layoutElementUid]], [[siteId]])', ]); $sql = 'CREATE FULLTEXT INDEX ' . diff --git a/src/migrations/m231205_081215_searchindex_amend_pk.php b/src/migrations/m231205_081215_searchindex_amend_pk.php new file mode 100644 index 00000000000..ee71588d55a --- /dev/null +++ b/src/migrations/m231205_081215_searchindex_amend_pk.php @@ -0,0 +1,40 @@ +dropPrimaryKey('elementId', Table::SEARCHINDEX); + + $this->addColumn(Table::SEARCHINDEX, 'layoutElementUid', $this->uid()->after('fieldId')); + + $this->addPrimaryKey('layoutUid', Table::SEARCHINDEX, ['elementId', 'attribute', 'fieldId', 'layoutElementUid', 'siteId']); + + return true; + } + + /** + * @inheritdoc + */ + public function safeDown(): bool + { + $this->dropPrimaryKey('layoutUid', Table::SEARCHINDEX); + + $this->dropColumn(Table::SEARCHINDEX, 'layoutElementUid'); + + $this->addPrimaryKey('elementId', Table::SEARCHINDEX, ['elementId', 'attribute', 'fieldId', 'siteId']); + + return true; + } +} diff --git a/src/services/Search.php b/src/services/Search.php index 38d2aeb8fef..83d7a7a4763 100644 --- a/src/services/Search.php +++ b/src/services/Search.php @@ -148,8 +148,8 @@ public function indexElementAttributes(ElementInterface $element, ?array $fieldH // Figure out which fields to update, and which to ignore /** @var FieldInterface[] $updateFields */ $updateFields = []; - /** @var string[] $ignoreFieldIds */ - $ignoreFieldIds = []; + /** @var string[] $ignoreFields */ + $ignoreFields = []; $fieldLayout = $element->getFieldLayout(); if ($fieldLayout !== null) { if ($fieldHandles !== null) { @@ -162,7 +162,7 @@ public function indexElementAttributes(ElementInterface $element, ?array $fieldH $updateFields[] = $field; } else { // Leave its existing keywords alone - $ignoreFieldIds[] = (string)$field->id; + $ignoreFields[] = $field->layoutElement->uid; } } } @@ -173,8 +173,8 @@ public function indexElementAttributes(ElementInterface $element, ?array $fieldH 'elementId' => $element->id, 'siteId' => $element->siteId, ]; - if (!empty($ignoreFieldIds)) { - $deleteCondition = ['and', $deleteCondition, ['not', ['fieldId' => $ignoreFieldIds]]]; + if (!empty($ignoreFields)) { + $deleteCondition = ['and', $deleteCondition, ['not', ['layoutElementUid' => $ignoreFields]]]; } Db::delete(Table::SEARCHINDEX, $deleteCondition); @@ -193,7 +193,7 @@ public function indexElementAttributes(ElementInterface $element, ?array $fieldH foreach ($updateFields as $field) { $fieldValue = $element->getFieldValue($field->handle); $keywords = $field->getSearchKeywords($fieldValue, $element); - $this->_indexKeywords($element, $keywords, fieldId: $field->id); + $this->_indexKeywords($element, $keywords, fieldId: $field->id, layoutElementUid: $field->layoutElement->uid); } // Release the lock @@ -388,8 +388,9 @@ public function deleteOrphanedIndexes(): void * @param string $keywords * @param string|null $attribute * @param int|null $fieldId + * @param string|null $layoutElementUid */ - private function _indexKeywords(ElementInterface $element, string $keywords, ?string $attribute = null, ?int $fieldId = null): void + private function _indexKeywords(ElementInterface $element, string $keywords, ?string $attribute = null, ?int $fieldId = null, ?string $layoutElementUid = null): void { if ($attribute !== null) { $attribute = strtolower($attribute); @@ -404,6 +405,7 @@ private function _indexKeywords(ElementInterface $element, string $keywords, ?st 'element' => $element, 'attribute' => $attribute, 'fieldId' => $fieldId, + 'layoutElementUid' => $layoutElementUid, 'keywords' => $keywords, ]); $this->trigger(self::EVENT_BEFORE_INDEX_KEYWORDS, $event); @@ -420,6 +422,7 @@ private function _indexKeywords(ElementInterface $element, string $keywords, ?st 'elementId' => $element->id, 'attribute' => $attribute ?? 'field', 'fieldId' => $fieldId ? (string)$fieldId : '0', + 'layoutElementUid' => $layoutElementUid ?? '0', 'siteId' => $site->id, ]; @@ -654,9 +657,12 @@ private function _getSqlFromTerm(SearchQueryTerm $term, array|int|null $siteId, // Check for other attributes if ($term->attribute !== null) { // Is attribute a valid fieldId? - $fieldId = $this->_getFieldIdFromAttribute($term->attribute, $customFields); + [$fieldId, $layoutElementUid] = $this->_getFieldIdFromAttribute($term->attribute, $customFields); - if (!empty($fieldId)) { + if (!empty($layoutElementUid)) { + $attr = 'layoutElementUid'; + $val = $layoutElementUid; + } elseif (!empty($fieldId)) { $attr = 'fieldId'; $val = $fieldId; } else { @@ -785,16 +791,36 @@ private function _normalizeTerm(string $term, array|int|null $siteId = null): st * * @param string $attribute * @param MemoizableArray|null $customFields - * @return int|int[]|null + * @return array */ - private function _getFieldIdFromAttribute(string $attribute, ?MemoizableArray $customFields): array|int|null + private function _getFieldIdFromAttribute(string $attribute, ?MemoizableArray $customFields): array { if ($customFields !== null) { - return ArrayHelper::getColumn($customFields->where('handle', $attribute)->all(), 'id'); + return [ + ArrayHelper::getColumn($customFields->where('handle', $attribute)->all(), 'id'), + ArrayHelper::getColumn($customFields->where('handle', $attribute)->all(), 'layoutElement.uid'), + ]; + } + + // we can no longer just get one field by handle; we need to find them all because we now support multiple instanced + $fields = []; + foreach (Craft::$app->getFields()->getAllLayouts() as $fieldLayout) { + array_push($fields, ...$fieldLayout->getCustomFields()); + } + + /** @var FieldInterface[][] $fieldsByHandle */ + $fieldsByHandle = ArrayHelper::index($fields, null, [ + fn(FieldInterface $field) => $field->handle, + ]); + + if (isset($fieldsByHandle[$attribute])) { + return [ + ArrayHelper::getColumn($fieldsByHandle[$attribute], 'id'), + ArrayHelper::getColumn($fieldsByHandle[$attribute], 'layoutElement.uid'), + ]; } - $field = Craft::$app->getFields()->getFieldByHandle($attribute); - return $field->id ?? null; + return [null, null]; } /** From 062f6a26530e364447e11661551cdf59d624a140 Mon Sep 17 00:00:00 2001 From: Iwona Just Date: Tue, 5 Dec 2023 12:59:47 +0100 Subject: [PATCH 2/3] add the column for pgsql too! --- src/migrations/Install.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/migrations/Install.php b/src/migrations/Install.php index f8b00cf55d5..625b0bac366 100644 --- a/src/migrations/Install.php +++ b/src/migrations/Install.php @@ -907,10 +907,11 @@ public function createIndexes(): void 'elementId' => $this->integer()->notNull(), 'attribute' => $this->string(25)->notNull(), 'fieldId' => $this->integer()->notNull(), + 'layoutElementUid' => $this->uid(), 'siteId' => $this->integer()->notNull(), 'keywords' => $this->text()->notNull(), 'keywords_vector' => $this->db->getSchema()->createColumnSchemaBuilder('tsvector')->notNull(), - 'PRIMARY KEY([[elementId]], [[attribute]], [[fieldId]], [[siteId]])', + 'PRIMARY KEY([[elementId]], [[attribute]], [[fieldId]], [[layoutElementUid]], [[siteId]])', ]); $sql = 'CREATE INDEX ' . $this->db->quoteTableName($this->db->getIndexName()) . ' ON ' . Table::SEARCHINDEX . ' USING GIN([[keywords_vector]] [[pg_catalog]].[[tsvector_ops]]) WITH (FASTUPDATE=YES)'; From bd3ddbd5fe8856fdefef805919a78e9613fae527 Mon Sep 17 00:00:00 2001 From: Iwona Just Date: Wed, 6 Dec 2023 10:55:41 +0100 Subject: [PATCH 3/3] migration that works for pgsql and disallow reverting --- .../m231205_081215_searchindex_amend_pk.php | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/migrations/m231205_081215_searchindex_amend_pk.php b/src/migrations/m231205_081215_searchindex_amend_pk.php index ee71588d55a..00ff3eecaec 100644 --- a/src/migrations/m231205_081215_searchindex_amend_pk.php +++ b/src/migrations/m231205_081215_searchindex_amend_pk.php @@ -15,11 +15,17 @@ class m231205_081215_searchindex_amend_pk extends Migration */ public function safeUp(): bool { - $this->dropPrimaryKey('elementId', Table::SEARCHINDEX); - - $this->addColumn(Table::SEARCHINDEX, 'layoutElementUid', $this->uid()->after('fieldId')); - - $this->addPrimaryKey('layoutUid', Table::SEARCHINDEX, ['elementId', 'attribute', 'fieldId', 'layoutElementUid', 'siteId']); + if ($this->db->getIsMysql()) { + $this->dropPrimaryKey('elementId', Table::SEARCHINDEX); + $this->addColumn(Table::SEARCHINDEX, 'layoutElementUid', $this->uid()->after('fieldId')); + $this->addPrimaryKey('layoutUid', Table::SEARCHINDEX, ['elementId', 'attribute', 'fieldId', 'layoutElementUid', 'siteId']); + } else { + $pkName = $this->db->getSchema()->getRawTableName(Table::SEARCHINDEX) . '_pkey'; + + $this->dropPrimaryKey($pkName, Table::SEARCHINDEX); + $this->addColumn(Table::SEARCHINDEX, 'layoutElementUid', $this->uid()->after('fieldId')); + $this->addPrimaryKey($pkName, Table::SEARCHINDEX, ['elementId', 'attribute', 'fieldId', 'layoutElementUid', 'siteId']); + } return true; } @@ -29,12 +35,7 @@ public function safeUp(): bool */ public function safeDown(): bool { - $this->dropPrimaryKey('layoutUid', Table::SEARCHINDEX); - - $this->dropColumn(Table::SEARCHINDEX, 'layoutElementUid'); - - $this->addPrimaryKey('elementId', Table::SEARCHINDEX, ['elementId', 'attribute', 'fieldId', 'siteId']); - - return true; + echo "m231205_081215_searchindex_amend_pk cannot be reverted.\n"; + return false; } }