Issue #3743: Support filter by tags for pipelines and data storages l…

…oading (#3744)

api/src/main/java/com/epam/pipeline/acl/datastorage/DataStorageApiService.java

@@ -20,6 +20,7 @@
 import com.epam.pipeline.common.MessageHelper;
 import com.epam.pipeline.controller.vo.DataStorageVO;
 import com.epam.pipeline.controller.vo.data.storage.UpdateDataStorageItemVO;
+import com.epam.pipeline.controller.vo.EntityFilterVO;
 import com.epam.pipeline.controller.vo.security.EntityWithPermissionVO;
 import com.epam.pipeline.entity.AbstractSecuredEntity;
 import com.epam.pipeline.entity.SecuredEntityWithAction;
@@ -90,6 +91,11 @@ public List<AbstractDataStorage> getDataStorages() {
         return dataStorageManager.getDataStorages();
     }
 
+    @StorageAclRead
+    public List<AbstractDataStorage> getDataStorages(final EntityFilterVO filter) {
+        return dataStorageManager.getDataStorages(filter);
+    }
+
     @StorageAclReadAndWrite
     public List<AbstractDataStorage> getWritableStorages() {
         return dataStorageManager.getDataStorages();

api/src/main/java/com/epam/pipeline/acl/pipeline/PipelineApiService.java

@@ -26,6 +26,7 @@
 import com.epam.pipeline.controller.vo.RegisterPipelineVersionVO;
 import com.epam.pipeline.controller.vo.TaskGraphVO;
 import com.epam.pipeline.controller.vo.UploadFileMetadata;
+import com.epam.pipeline.controller.vo.EntityFilterVO;
 import com.epam.pipeline.entity.cluster.InstancePrice;
 import com.epam.pipeline.entity.git.GitCommitEntry;
 import com.epam.pipeline.entity.git.GitCommitsFilter;
@@ -131,6 +132,12 @@ public List<Pipeline> loadAllPipelines(boolean loadVersions) {
         return pipelineManager.loadAllPipelines(loadVersions);
     }
 
+    @PostFilter("hasRole('ADMIN') OR hasPermission(filterObject, 'READ')")
+    @AclMaskList
+    public List<Pipeline> filterPipelines(final boolean loadVersions, final EntityFilterVO filter) {
+        return pipelineManager.loadAllPipelines(loadVersions, filter);
+    }
+
     @PreAuthorize(ADMIN_ONLY)
     public PipelinesWithPermissionsVO loadAllPipelinesWithPermissions(Integer pageNum, Integer pageSize) {
         return permissionManager.loadAllPipelinesWithPermissions(pageNum, pageSize);

api/src/main/java/com/epam/pipeline/controller/datastorage/DataStorageController.java

@@ -23,6 +23,7 @@
 import com.epam.pipeline.controller.vo.UploadFileMetadata;
 import com.epam.pipeline.controller.vo.data.storage.DataStorageMountVO;
 import com.epam.pipeline.controller.vo.data.storage.UpdateDataStorageItemVO;
+import com.epam.pipeline.controller.vo.EntityFilterVO;
 import com.epam.pipeline.controller.vo.security.EntityWithPermissionVO;
 import com.epam.pipeline.entity.AbstractSecuredEntity;
 import com.epam.pipeline.entity.SecuredEntityWithAction;
@@ -98,6 +99,19 @@ public Result<List<AbstractDataStorage>> getDataStorages() {
         return Result.success(dataStorageApiService.getDataStorages());
     }
 
+    @PostMapping("/datastorage/filter")
+    @ResponseBody
+    @ApiOperation(
+            value = "Loads all data storages with specified filters.",
+            notes = "Loads all data storages with specified filters.",
+            produces = MediaType.APPLICATION_JSON_VALUE)
+    @ApiResponses(
+            value = {@ApiResponse(code = HTTP_STATUS_OK, message = API_STATUS_DESCRIPTION)
+            })
+    public Result<List<AbstractDataStorage>> filterDataStorages(@RequestBody final EntityFilterVO filter) {
+        return Result.success(dataStorageApiService.getDataStorages(filter));
+    }
+
     @RequestMapping(value = "/datastorage/available", method = RequestMethod.GET)
     @ResponseBody
     @ApiOperation(

api/src/main/java/com/epam/pipeline/controller/pipeline/PipelineController.java

@@ -29,6 +29,7 @@
 import com.epam.pipeline.controller.vo.RegisterPipelineVersionVO;
 import com.epam.pipeline.controller.vo.TaskGraphVO;
 import com.epam.pipeline.controller.vo.UploadFileMetadata;
+import com.epam.pipeline.controller.vo.EntityFilterVO;
 import com.epam.pipeline.entity.cluster.InstancePrice;
 import com.epam.pipeline.entity.git.GitCommitEntry;
 import com.epam.pipeline.entity.git.GitCommitsFilter;
@@ -167,6 +168,21 @@ public Result<List<Pipeline>> loadAllPipelines(
         return Result.success(pipelineApiService.loadAllPipelines(loadVersion));
     }
 
+    @PostMapping("/pipeline/filter")
+    @ResponseBody
+    @ApiOperation(
+            value = "Loads all registered pipelines with specified filters.",
+            notes = "Loads all registered pipelines with specified filters.",
+            produces = MediaType.APPLICATION_JSON_VALUE)
+    @ApiResponses(
+            value = {@ApiResponse(code = HTTP_STATUS_OK, message = API_STATUS_DESCRIPTION)
+            })
+    public Result<List<Pipeline>> filterPipelines(
+            @RequestBody final EntityFilterVO filter,
+            @RequestParam(defaultValue = "false") final Boolean loadVersion) {
+        return Result.success(pipelineApiService.filterPipelines(loadVersion, filter));
+    }
+
     @GetMapping(value = "/pipeline/permissions")
     @ResponseBody
     @ApiOperation(

api/src/main/java/com/epam/pipeline/controller/vo/EntityFilterVO.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2024 EPAM Systems, Inc. (https://www.epam.com/)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.epam.pipeline.controller.vo;
+
+import lombok.Data;
+
+import java.util.List;
+import java.util.Map;
+
+@Data
+public class EntityFilterVO {
+    private Map<String, List<String>> tags;
+}

api/src/main/java/com/epam/pipeline/dao/MetadataTagsUtils.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2024 EPAM Systems, Inc. (https://www.epam.com/)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.epam.pipeline.dao;
+
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public interface MetadataTagsUtils {
+    Pattern FILTER_PATTERN = Pattern.compile("@FILTER@");
+
+    static String buildTagsFilterClause(final String query, final Map<String, List<String>> tags) {
+        return FILTER_PATTERN.matcher(query).replaceFirst(tags.entrySet().stream()
+                .map(entry -> tagFilterQuery(entry.getKey(), entry.getValue()))
+                .collect(Collectors.joining(" AND ")));
+    }
+
+    static String tagFilterQuery(final String key, final List<String> values) {
+        return String.format("(m.data->'%s'->>'value' IN (%s))", key,
+                values.stream()
+                        .map(value -> String.format("'%s'", value))
+                        .collect(Collectors.joining(",")));
+    }
+}

api/src/main/java/com/epam/pipeline/dao/datastorage/DataStorageDao.java

@@ -17,7 +17,9 @@
 package com.epam.pipeline.dao.datastorage;
 
 import com.epam.pipeline.config.JsonMapper;
+import com.epam.pipeline.controller.vo.EntityFilterVO;
 import com.epam.pipeline.dao.DaoHelper;
+import com.epam.pipeline.dao.MetadataTagsUtils;
 import com.epam.pipeline.entity.datastorage.AbstractDataStorage;
 import com.epam.pipeline.entity.datastorage.AbstractDataStorageFactory;
 import com.epam.pipeline.entity.datastorage.DataStorageRoot;
@@ -98,6 +100,7 @@ public class DataStorageDao extends NamedParameterJdbcDaoSupport {
     private String deleteToolsToMountQuery;
     private String addToolVersionToMountQuery;
     private String loadDataStoragesByRootIdsQuery;
+    private String loadDataStoragesFilterQuery;
 
     @Autowired
     private DaoHelper daoHelper;
@@ -182,6 +185,12 @@ public List<AbstractDataStorage> loadAllDataStorages() {
                 DataStorageParameters.getRowMapper());
     }
 
+    public List<AbstractDataStorage> loadAllDataStorages(final EntityFilterVO filter) {
+        return getNamedParameterJdbcTemplate()
+                .query(MetadataTagsUtils.buildTagsFilterClause(loadDataStoragesFilterQuery, filter.getTags()),
+                DataStorageParameters.getRowMapper());
+    }
+
     public List<AbstractDataStorage> loadAllDataStoragesWithToolsToMount() {
         final List<AbstractDataStorage> storages = getNamedParameterJdbcTemplate().query(loadAllDataStoragesQuery,
                 DataStorageParameters.getRowMapper());
@@ -455,7 +464,10 @@ public void setLoadDataStoragesByRootIdsQuery(String loadDataStoragesByRootIdsQu
         this.loadDataStoragesByRootIdsQuery = loadDataStoragesByRootIdsQuery;
     }
 
-
+    @Required
+    public void setLoadDataStoragesFilterQuery(final String loadDataStoragesFilterQuery) {
+        this.loadDataStoragesFilterQuery = loadDataStoragesFilterQuery;
+    }
 
     public enum DataStorageParameters {
         DATASTORAGE_ID,

api/src/main/java/com/epam/pipeline/dao/pipeline/PipelineDao.java

@@ -27,7 +27,9 @@
 import java.util.Optional;
 import java.util.Set;
 
+import com.epam.pipeline.controller.vo.EntityFilterVO;
 import com.epam.pipeline.dao.DaoHelper;
+import com.epam.pipeline.dao.MetadataTagsUtils;
 import com.epam.pipeline.entity.pipeline.Folder;
 import com.epam.pipeline.entity.pipeline.Pipeline;
 import com.epam.pipeline.entity.pipeline.PipelineType;
@@ -63,6 +65,7 @@ public class PipelineDao extends NamedParameterJdbcDaoSupport {
     private String loadAllPipelinesWithParentsQuery;
     private String loadPipelinesCountQuery;
     private String loadPipelineWithParentsQuery;
+    private String loadPipelinesFiltersQuery;
 
     @Transactional(propagation = Propagation.MANDATORY)
     public Long createPipelineId() {
@@ -85,6 +88,12 @@ public List<Pipeline> loadAllPipelines() {
                 PipelineParameters.getRowMapper());
     }
 
+    public List<Pipeline> loadAllPipelines(final EntityFilterVO filter) {
+        return getNamedParameterJdbcTemplate().query(
+                MetadataTagsUtils.buildTagsFilterClause(loadPipelinesFiltersQuery, filter.getTags()),
+                PipelineParameters.getRowMapper());
+    }
+
     public Pipeline loadPipeline(Long id) {
         List<Pipeline> items = getJdbcTemplate().query(loadPipelineByIdQuery, PipelineParameters
                 .getRowMapper(), id);
@@ -315,4 +324,9 @@ public void setLoadPipelinesCountQuery(String loadPipelinesCountQuery) {
     public void setLoadPipelineWithParentsQuery(String loadPipelineWithParentsQuery) {
         this.loadPipelineWithParentsQuery = loadPipelineWithParentsQuery;
     }
+
+    @Required
+    public void setLoadPipelinesFiltersQuery(final String loadPipelinesFiltersQuery) {
+        this.loadPipelinesFiltersQuery = loadPipelinesFiltersQuery;
+    }
 }

api/src/main/java/com/epam/pipeline/manager/datastorage/DataStorageManager.java

@@ -23,6 +23,7 @@
 import com.epam.pipeline.controller.vo.EntityVO;
 import com.epam.pipeline.controller.vo.MetadataVO;
 import com.epam.pipeline.controller.vo.data.storage.UpdateDataStorageItemVO;
+import com.epam.pipeline.controller.vo.EntityFilterVO;
 import com.epam.pipeline.dao.datastorage.DataStorageDao;
 import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestoreAction;
 import com.epam.pipeline.dto.datastorage.lifecycle.restore.StorageRestorePathType;
@@ -225,6 +226,12 @@ public List<AbstractDataStorage> getDataStorages() {
         return dataStorageDao.loadAllDataStorages();
     }
 
+    public List<AbstractDataStorage> getDataStorages(final EntityFilterVO filter) {
+        return Objects.isNull(filter) || MapUtils.isEmpty(filter.getTags())
+                ? dataStorageDao.loadAllDataStorages()
+                : dataStorageDao.loadAllDataStorages(filter);
+    }
+
     public List<AbstractDataStorage> getDataStoragesWithToolsToMount() {
         return dataStorageDao.loadAllDataStoragesWithToolsToMount();
     }

api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineManager.java

@@ -22,6 +22,7 @@
 import com.epam.pipeline.controller.vo.CheckRepositoryVO;
 import com.epam.pipeline.controller.vo.EntityVO;
 import com.epam.pipeline.controller.vo.PipelineVO;
+import com.epam.pipeline.controller.vo.EntityFilterVO;
 import com.epam.pipeline.dao.datastorage.rules.DataStorageRuleDao;
 import com.epam.pipeline.dao.pipeline.PipelineDao;
 import com.epam.pipeline.dao.pipeline.PipelineRunDao;
@@ -44,6 +45,7 @@
 import com.epam.pipeline.utils.GitUtils;
 import com.epam.pipeline.utils.PasswordGenerator;
 import org.apache.commons.collections4.ListUtils;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -56,6 +58,7 @@
 import org.springframework.web.client.HttpClientErrorException;
 
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 
@@ -287,6 +290,16 @@ public List<Pipeline> loadAllPipelines(boolean loadVersions) {
         return result;
     }
 
+    public List<Pipeline> loadAllPipelines(final boolean loadVersions, final EntityFilterVO filter) {
+        final List<Pipeline> result = Objects.isNull(filter) || MapUtils.isEmpty(filter.getTags())
+                ? pipelineDao.loadAllPipelines()
+                : pipelineDao.loadAllPipelines(filter);
+        if (loadVersions) {
+            result.forEach(this::setCurrentVersion);
+        }
+        return result;
+    }
+
     @Override
     public Set<Pipeline> loadAllWithParents(Integer pageNum, Integer pageSize) {
         Assert.isTrue((pageNum == null) == (pageSize == null),

api/src/main/resources/dao/datastorage-dao.xml

@@ -157,6 +157,47 @@
                 ]]>
             </value>
         </property>
+        <property name="loadDataStoragesFilterQuery">
+            <value>
+                <![CDATA[
+                    SELECT
+                        d.datastorage_id,
+                        d.datastorage_name,
+                        d.description,
+                        d.datastorage_type,
+                        d.path,
+                        d.datastorage_root_id,
+                        d.sts_duration,
+                        d.lts_duration,
+                        d.incomplete_upload_cleanup_days,
+                        d.folder_id,
+                        d.created_date,
+                        d.owner,
+                        d.enable_versioning,
+                        d.backup_duration,
+                        d.locked as datastorage_locked,
+                        d.mount_point,
+                        d.mount_options,
+                        d.shared,
+                        d.allowed_cidrs,
+                        d.region_id as region_id,
+                        d.file_share_mount_id,
+                        d.sensitive,
+                        d.mount_disabled,
+                        d.s3_kms_key_arn,
+                        d.s3_use_assumed_creds,
+                        s3_temp_creds_role,
+                        d.mount_status,
+                        d.source_datastorage_id,
+                        d.masking_rules
+                    FROM
+                        pipeline.datastorage d
+                    LEFT JOIN pipeline.metadata m ON d.datastorage_id = m.entity_id
+                    WHERE m.entity_class = 'DATA_STORAGE' AND @FILTER@
+                    ORDER BY d.datastorage_id
+                ]]>
+            </value>
+        </property>
         <property name="loadDataStoragesByIdsQuery">
             <value>
                 <![CDATA[

api/src/main/resources/dao/pipeline-dao.xml

@@ -110,6 +110,35 @@
                 ]]>
             </value>
         </property>
+        <property name="loadPipelinesFiltersQuery">
+            <value>
+                <![CDATA[
+                    SELECT
+                        p.pipeline_id,
+                        p.pipeline_name,
+                        p.repository,
+                        p.repository_ssh,
+                        p.description,
+                        p.folder_id,
+                        p.created_date,
+                        p.owner,
+                        p.repository_token,
+                        p.repository_type,
+                        p.pipeline_type,
+                        p.config,
+                        p.locked as pipeline_locked,
+                        p.branch,
+                        p.visibility,
+                        p.code_path,
+                        p.docs_path
+                    FROM
+                        pipeline.pipeline p
+                    LEFT JOIN pipeline.metadata m ON p.pipeline_id = m.entity_id
+                    WHERE m.entity_class = 'PIPELINE' AND @FILTER@
+                    ORDER BY pipeline_id
+                ]]>
+            </value>
+        </property>
         <property name="deletePipelineQuery">
             <value>
                 <![CDATA[