Search Metadata editor in service definition on tile added (#201)

* Catalog-level Search metadata support added (#77)

README.md

@@ -358,11 +358,6 @@ ECS Documentation provides following system metadata:
 | Size            | Integer   |  Size of the object.                                  |
 | CreateTime      | DateTime  |  Time at which the object was created.                |
 | LastModified    | DateTime  |  Time and date at which the object was last modified. |
-| ContentType     | String    |  -                                                    |
-| Expiration      | DateTime  |  -                                                    |
-| ContentEncoding | String    |  -                                                    |
-| Expires         | DateTime  |  -                                                    |
-| Retention       | Integer   |  -                                                    |
 
 System search metadata provided in `service-settings` block should be tagged with _System_ type, existing name and corresponding datatype.
 
@@ -372,7 +367,7 @@ User defined search metadata should be tagged with _User_ type, existing datatyp
 Prefix will be added to field name if it is not already there. 
 
 #### Metadata Search Datatypes
-When writing metadata to objects, client should provide data in format appropriate for each fields's datatype.
+When writing metadata to objects, client should provide data in format appropriate for each fields' datatype.
 ECS supports 4 types of datatype: _String_, _Integer_, _DateTime_ and _Decimal_.
 
 | Datatype | Description |

src/main/java/com/emc/ecs/servicebroker/config/CatalogConfig.java

@@ -1,6 +1,8 @@
 package com.emc.ecs.servicebroker.config;
 
 import com.emc.ecs.servicebroker.model.*;
+import com.emc.ecs.servicebroker.service.EcsService;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.slf4j.Logger;
@@ -30,6 +32,7 @@ public class CatalogConfig {
     private List<ServiceDefinitionProxy> services;
     private Map<Integer, List<PlanProxy>> plans = new HashMap<>();
     private Map<Integer, Map<String, Object>> settings = new HashMap<>();
+    private Map<Integer, List<Map<String, String>>> searchMetadata = new HashMap<>();
 
     public CatalogConfig() {
         super();
@@ -55,6 +58,9 @@ public List<ServiceDefinitionProxy> mergeServices() {
                 s.setPlans(plans.get(index));
             }
             if (settings.containsKey(index)) {
+                if (searchMetadata.containsKey(index)) {
+                    verifyAndPrepareSearchMetadata(index);
+                }
                 s.setServiceSettings(settings.get(index));
             }
             return s;
@@ -205,6 +211,37 @@ static Map<String, Object> parseBucketTags(Map<String, Object> settings) {
         return settings;
     }
 
+    public void setSearchMetadataCollection0(String searchMetadataCollectionJson) throws IOException {
+        this.searchMetadata.put(0, parseSearchMetadata(searchMetadataCollectionJson));
+    }
+
+    public void setSearchMetadataCollection1(String searchMetadataCollectionJson) throws IOException {
+        this.searchMetadata.put(1, parseSearchMetadata(searchMetadataCollectionJson));
+    }
+
+    public void setSearchMetadataCollection2(String searchMetadataCollectionJson) throws IOException {
+        this.searchMetadata.put(2, parseSearchMetadata(searchMetadataCollectionJson));
+    }
+
+    public void setSearchMetadataCollection3(String searchMetadataCollectionJson) throws IOException {
+        this.searchMetadata.put(3, parseSearchMetadata(searchMetadataCollectionJson));
+    }
+
+    public void setSearchMetadataCollection4(String searchMetadataCollectionJson) throws IOException {
+        this.searchMetadata.put(4, parseSearchMetadata(searchMetadataCollectionJson));
+    }
+
+    public List<Map<String, String>> parseSearchMetadata(String searchMetadataJson) throws JsonProcessingException {
+        return objectMapper.readValue(searchMetadataJson, new TypeReference<List<Map<String, String>>>() {});
+    }
+
+    private void verifyAndPrepareSearchMetadata(int index) {
+        Map<String, Object> pars = settings.get(index);
+        pars.put(SEARCH_METADATA, searchMetadata.get(index));
+        EcsService.validateAndPrepareSearchMetadata(pars);
+        settings.put(index, pars);
+    }
+
     public void setServices(List<ServiceDefinitionProxy> services) {
         this.services = services;
     }

src/main/java/com/emc/ecs/servicebroker/model/SystemMetadataName.java

@@ -5,12 +5,7 @@ public enum SystemMetadataName {
     LastModified(SearchMetadataDataType.DateTime),
     ObjectName(SearchMetadataDataType.String),
     Owner(SearchMetadataDataType.String),
-    Size(SearchMetadataDataType.Integer),
-    ContentType(SearchMetadataDataType.String),
-    Expiration(SearchMetadataDataType.DateTime),
-    ContentEnding(SearchMetadataDataType.String),
-    Expires(SearchMetadataDataType.DateTime),
-    Retention(SearchMetadataDataType.Integer);
+    Size(SearchMetadataDataType.Integer);
 
     private SearchMetadataDataType dataType;
 

src/main/java/com/emc/ecs/servicebroker/service/EcsService.java

@@ -512,7 +512,7 @@ private void validateReclaimPolicy(Map<String, Object> parameters) {
     }
 
     @SuppressWarnings("unchecked")
-    static Map<String, Object> validateAndPrepareSearchMetadata(Map<String, Object> parameters) {
+    static public Map<String, Object> validateAndPrepareSearchMetadata(Map<String, Object> parameters) {
         if (parameters.containsKey(SEARCH_METADATA)) {
             parameters = new HashMap<>(parameters);  // don't modify original map
 
@@ -704,7 +704,7 @@ public DataServiceReplicationGroup lookupReplicationGroup(String replicationGrou
     }
 
     /**
-     * Merge request bucket tags with with plan and service provided tags
+     * Merge request bucket tags with plan and service provided tags
      * <p>
      * Request bucket tags are overwritten with plan and service ones,
      * while bucket tags provided in plan description are overwritten by service tags
@@ -752,6 +752,34 @@ static List<Map<String, String>> mergeBucketTags(ServiceDefinitionProxy service,
         return serviceTags;
     }
 
+    /**
+     * Merge request search metadata with service provided metadata
+     * <p>
+     * Request bucket tags are overwritten with service ones
+     * since service settings are forced by administrator through the catalog
+     */
+    static List<Map<String, String>> mergeSearchMetadata(ServiceDefinitionProxy service, Map<String, Object> requestParameters) {
+        List<Map<String, String>> serviceMetadata = (List<Map<String, String>>) service.getServiceSettings().get(SEARCH_METADATA);
+        List<Map<String, String>> requestedMetadata = (List<Map<String, String>>)requestParameters.get(SEARCH_METADATA);
+
+        if (serviceMetadata == null) {
+            return requestedMetadata;
+        } else if (requestedMetadata == null) {
+            return serviceMetadata;
+        } else {
+            List<Map<String, String>> unmatchedMetadata = new ArrayList<>(requestedMetadata);
+            for (Map<String, String> requestedMetadatum: requestedMetadata) {
+                for (Map<String, String> serviceMetadatum: serviceMetadata) {
+                    if (requestedMetadatum.get(SEARCH_METADATA_NAME).equals(serviceMetadatum.get(SEARCH_METADATA_NAME))) {
+                        unmatchedMetadata.remove(requestedMetadatum);
+                        break;
+                    }
+                }
+            }
+            return Stream.concat(serviceMetadata.stream(), unmatchedMetadata.stream()).collect(Collectors.toList());
+        }
+    }
+
     /**
      * Merge request additional parameters with with broker, plan and service settings
      * <p>
@@ -774,6 +802,12 @@ static Map<String, Object> mergeParameters(BrokerConfig brokerConfig, ServiceDef
             ret.put(TAGS, tags);
         }
 
+        List<Map<String, String>> searchMetadata = mergeSearchMetadata(service, requestParameters);
+
+        if (searchMetadata != null) {
+            ret.put(SEARCH_METADATA, searchMetadata);
+        }
+
         return ret;
     }
 

src/test/java/com/emc/ecs/common/Fixtures.java

@@ -79,6 +79,8 @@ public class Fixtures {
     public static final int PAGE_SIZE = 32;
     public static final String SYSTEM_METADATA_NAME = "Size";
     public static final String SYSTEM_METADATA_TYPE = "Integer";
+    public static final String SYSTEM_METADATA_NAME2 = "Owner";
+    public static final String SYSTEM_METADATA_TYPE2 = "String";
     public static final String USER_METADATA_NAME = "my_meta";
     public static final String USER_METADATA_TYPE = "String";
     public static final String INVALID_METADATA_TYPE = "invalid_data_type";

src/test/java/com/emc/ecs/servicebroker/config/CatalogConfigTest.java

@@ -2,6 +2,8 @@
 
 import com.emc.ecs.common.Fixtures;
 
+import com.emc.ecs.servicebroker.service.EcsServiceTest;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import org.apache.commons.collections.CollectionUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -30,6 +32,9 @@ public class CatalogConfigTest {
     @Autowired
     private Catalog catalog;
 
+    @Autowired
+    private CatalogConfig catalogConfig;
+
     @Test
     public void testEcsBucket() {
         ServiceDefinition ecsBucketService = catalog.getServiceDefinitions()
@@ -101,6 +106,40 @@ public void parseBucketTagsTest() {
         assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags));
     }
 
+    @Test
+    public void parseSearchMetadataTest() throws JsonProcessingException {
+        String searchMetadataString = buildSearchMetadataString(SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME, SYSTEM_METADATA_TYPE,
+                SEARCH_METADATA_TYPE_USER, USER_METADATA_NAME, USER_METADATA_TYPE);
+
+        List<Map<String, String>> resultMetadata = catalogConfig.parseSearchMetadata(searchMetadataString);
+        List<Map<String, String>> expectedMetadata = EcsServiceTest.createListOfSearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME, SYSTEM_METADATA_TYPE,
+                SEARCH_METADATA_TYPE_USER, USER_METADATA_NAME, USER_METADATA_TYPE);
+
+        assertTrue(CollectionUtils.isEqualCollection(expectedMetadata, resultMetadata));
+    }
+
+    private String buildSearchMetadataString(String... args) {
+        if (args.length % 3 != 0) {
+            throw new IllegalArgumentException("Number of arguments should be multiple of three.");
+        }
+        if (args.length == 0) {
+            return "";
+        }
+        StringBuilder builder = new StringBuilder("[");
+        for (int i = 0; i < args.length; i += 3) {
+            if (i != 0) {
+                builder.append(',');
+            }
+            builder.append('{');
+            builder.append('\"' + SEARCH_METADATA_TYPE + "\":\"" + args[i] + "\",");
+            builder.append('\"' + SEARCH_METADATA_NAME + "\":\"" + args[i + 1] + "\",");
+            builder.append('\"' + SEARCH_METADATA_DATATYPE + "\":\"" + args[i + 2] + '\"');
+            builder.append('}');
+        }
+        builder.append(']');
+        return builder.toString();
+    }
+
     private void testServiceDefinition(ServiceDefinition service, String id,
                                        String name, String description, boolean bindable,
                                        boolean updatable, List<String> requires, String dashboardUrl) {

src/test/java/com/emc/ecs/servicebroker/service/EcsServiceTest.java

@@ -1691,6 +1691,73 @@ public void mergeBucketTagsWithNullValueTest() {
         assertTrue(CollectionUtils.isEqualCollection(expectedTags, resultTags));
     }
 
+    /**
+     * A service can merge lists of search metadata presented in service description and provided
+     * in request on bucket creation.
+     * <p>
+     * Requested search metadata has lower priority than service defined metadata and therefore would be overwritten by last ones
+     */
+    @Test
+    public void mergeSearchMetadataTest() {
+        Map<String, Object> serviceParams = new HashMap<>();
+        Map<String, Object> requestedParams = new HashMap<>();
+
+        serviceParams.put(SEARCH_METADATA, createListOfSearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME, SYSTEM_METADATA_TYPE,
+                SEARCH_METADATA_TYPE_USER, USER_METADATA_NAME, USER_METADATA_TYPE));
+        requestedParams.put(SEARCH_METADATA, createListOfSearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME2, SYSTEM_METADATA_TYPE2,
+                SEARCH_METADATA_TYPE_USER, USER_METADATA_NAME, SYSTEM_METADATA_TYPE));
+
+        ServiceDefinitionProxy service = new ServiceDefinitionProxy();
+        service.setServiceSettings(serviceParams);
+
+        List<Map<String, String>> resultMetadata = EcsService.mergeSearchMetadata(service, requestedParams);
+        List<Map<String, String>> expectedMetadata = createListOfSearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME, SYSTEM_METADATA_TYPE,
+                SEARCH_METADATA_TYPE_USER, USER_METADATA_NAME, USER_METADATA_TYPE,
+                SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME2, SYSTEM_METADATA_TYPE2);
+        assertTrue(CollectionUtils.isEqualCollection(expectedMetadata, resultMetadata));
+    }
+
+    /**
+     * A service can merge lists of search metadata presented in service description and provided
+     * in request on bucket creation when one or both lists are null.
+     */
+    @Test
+    public void mergeSearchMetadataWithNullValueTest() {
+        Map<String, Object> serviceParams = new HashMap<>();
+        Map<String, Object> requestedParams = new HashMap<>();
+
+        serviceParams.put(SEARCH_METADATA, createListOfSearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME, SYSTEM_METADATA_TYPE));
+        requestedParams.put(SEARCH_METADATA, createListOfSearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME2, SYSTEM_METADATA_TYPE2));
+
+        ServiceDefinitionProxy service = new ServiceDefinitionProxy();
+        service.setServiceSettings(serviceParams);
+
+        Map<String, Object> serviceParamsNull = new HashMap<>();
+        Map<String, Object> requestedParamsNull = new HashMap<>();
+
+        serviceParamsNull.put(SEARCH_METADATA, null);
+        requestedParamsNull.put(SEARCH_METADATA, null);
+
+        ServiceDefinitionProxy serviceNull = new ServiceDefinitionProxy();
+        serviceNull.setServiceSettings(serviceParamsNull);
+
+        List<Map<String, String>> resultMetadata = EcsService.mergeSearchMetadata(serviceNull, requestedParamsNull);
+        assertNull(resultMetadata);
+
+        resultMetadata = EcsService.mergeSearchMetadata(service, requestedParamsNull);
+        List<Map<String, String>> expectedMetadata = createListOfSearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME, SYSTEM_METADATA_TYPE);
+        assertTrue(CollectionUtils.isEqualCollection(expectedMetadata, resultMetadata));
+
+        resultMetadata = EcsService.mergeSearchMetadata(serviceNull, requestedParams);
+        expectedMetadata = createListOfSearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME2, SYSTEM_METADATA_TYPE2);
+        assertTrue(CollectionUtils.isEqualCollection(expectedMetadata, resultMetadata));
+
+        resultMetadata = EcsService.mergeSearchMetadata(service, requestedParams);
+        expectedMetadata = createListOfSearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME, SYSTEM_METADATA_TYPE,
+                SEARCH_METADATA_TYPE_SYSTEM, SYSTEM_METADATA_NAME2, SYSTEM_METADATA_TYPE2);
+        assertTrue(CollectionUtils.isEqualCollection(expectedMetadata, resultMetadata));
+    }
+
     private void setupInitTest() throws EcsManagementClientException {
         DataServiceReplicationGroup rg = new DataServiceReplicationGroup();
         rg.setName(RG_NAME);
@@ -1899,7 +1966,7 @@ private void setupDeleteSearchMetadataTest() throws Exception {
         PowerMockito.doNothing().when(SearchMetadataAction.class, DELETE, same(connection), eq(BUCKET_NAME), eq(NAMESPACE_NAME));
     }
 
-    private List<Map<String, String>> createListOfTags(String... args) throws IllegalArgumentException {
+    static public List<Map<String, String>> createListOfTags(String... args) throws IllegalArgumentException {
         if (args.length % 2 != 0) {
             throw new IllegalArgumentException("Number of arguments should be multiple of two.");
         }
@@ -1929,7 +1996,7 @@ static void assertSearchMetadataSameAsParams(List<Map<String, String>> input, Li
         }
     }
 
-    private List<Map<String, String>> createListOfSearchMetadata(String... args) throws IllegalArgumentException {
+    public static List<Map<String, String>> createListOfSearchMetadata(String... args) throws IllegalArgumentException {
         if (args.length % 3 != 0) {
             throw new IllegalArgumentException("Number of arguments should be multiple of three.");
         }

src/test/java/com/emc/ecs/servicebroker/service/MetadataSearchValidationTests.java

@@ -83,14 +83,14 @@ public void validationFailsOnWrongSystemKeyName() {
     @Test
     public void validationFailsOnWrongDatatypeForSystemKey() {
         Map<String, String> m = new HashMap<>();
-        m.put(SEARCH_METADATA_NAME, SystemMetadataName.Expiration.name());
+        m.put(SEARCH_METADATA_NAME, SystemMetadataName.LastModified.name());
         m.put(SEARCH_METADATA_DATATYPE, SearchMetadataDataType.Integer.name());
 
         try {
             EcsService.validateAndPrepareSearchMetadata(parameters(m));
             Assert.fail();
         } catch (ServiceBrokerInvalidParametersException e) {
-            assertTrue(e.getMessage().contains("Invalid system search metadata '" + SystemMetadataName.Expiration.name() + "' datatype"));
+            assertTrue(e.getMessage().contains("Invalid system search metadata '" + SystemMetadataName.LastModified.name() + "' datatype"));
         }
     }
 
@@ -137,7 +137,7 @@ public void nullListsOfMetadataAreEqual() {
     @Test
     public void nullAndNonemptyListsAreNotEqual() {
         List<SearchMetadata> list = Arrays.asList(
-                new SearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SystemMetadataName.ContentEnding.name(), SearchMetadataDataType.String.name())
+                new SearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SystemMetadataName.Owner.name(), SearchMetadataDataType.String.name())
         );
         assertFalse("Null and non empty metadata lists are not equal", EcsService.isEqualSearchMetadataList(null, list));
         assertFalse("Non empty list and null are not equal", EcsService.isEqualSearchMetadataList(list, null));
@@ -153,20 +153,20 @@ public void differentListsOfMetadataAreNotEqual() {
                 new SearchMetadata(SEARCH_METADATA_TYPE_USER, SOME_USER_METADATA_NAME, SearchMetadataDataType.Integer.name())
         );
         List<SearchMetadata> list2 = Arrays.asList(
-                new SearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SystemMetadataName.ContentEnding.name(), SearchMetadataDataType.String.name())
+                new SearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SystemMetadataName.ObjectName.name(), SearchMetadataDataType.String.name())
         );
         assertFalse("Lists with different entries are not equal", EcsService.isEqualSearchMetadataList(list1, list2));
     }
 
     @Test
     public void sameMetadataListsAreEqual() {
         List<SearchMetadata> list1 = Arrays.asList(
-                new SearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SystemMetadataName.ContentType.name(), SearchMetadataDataType.String.name()),
+                new SearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SystemMetadataName.Owner.name(), SearchMetadataDataType.String.name()),
                 new SearchMetadata(SEARCH_METADATA_TYPE_USER, SOME_USER_METADATA_NAME, SearchMetadataDataType.Decimal.name())
         );
 
         List<SearchMetadata> list2 = Arrays.asList(
-                new SearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SystemMetadataName.ContentType.name(), SearchMetadataDataType.String.name()),
+                new SearchMetadata(SEARCH_METADATA_TYPE_SYSTEM, SystemMetadataName.Owner.name(), SearchMetadataDataType.String.name()),
                 new SearchMetadata(SEARCH_METADATA_TYPE_USER, SOME_USER_METADATA_NAME, SearchMetadataDataType.Decimal.name())
         );