avniproject/avni-webapp#1306 - added guava for set operations used in…

… headers in csv mismatch. made headers mismatch message specific to the missing headers. data setup for integration test, now creates basic org data. first user and catchment test. do not delete data created by migration in setup sql scripts for test.
avniproject · Aug 27, 2024 · 051dce8 · 051dce8
1 parent 94d4913
commit 051dce8
Show file tree

Hide file tree

Showing 16 changed files with 255 additions and 112 deletions.
diff --git a/avni-server-api/build.gradle b/avni-server-api/build.gradle
@@ -91,6 +91,7 @@ dependencies {
     implementation 'org.apache.commons:commons-csv:1.10.0'
     compile 'com.googlecode.libphonenumber:libphonenumber:8.12.32'
     testImplementation 'org.slf4j:slf4j-reload4j:2.0.6'
+    compile 'com.google.guava:guava:33.2.1-jre'
 }
 
 bootRun {

diff --git a/avni-server-api/src/main/java/org/avni/messaging/service/MessageTemplateService.java b/avni-server-api/src/main/java/org/avni/messaging/service/MessageTemplateService.java
@@ -13,9 +13,8 @@
 
 @Service
 public class MessageTemplateService {
-
-    private GlificMessageTemplateRepository messageTemplateRepository;
-    private OrganisationConfigService organisationConfigService;
+    private final GlificMessageTemplateRepository messageTemplateRepository;
+    private final OrganisationConfigService organisationConfigService;
 
     @Autowired
     public MessageTemplateService(GlificMessageTemplateRepository messageTemplateRepository, OrganisationConfigService organisationConfigService) {

diff --git a/...erver-api/src/main/java/org/avni/server/importer/batch/csv/writer/BulkLocationEditor.java b/...erver-api/src/main/java/org/avni/server/importer/batch/csv/writer/BulkLocationEditor.java
@@ -10,6 +10,7 @@
 import org.springframework.stereotype.Component;
 import org.springframework.util.StringUtils;
 
+import javax.transaction.Transactional;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -70,6 +71,7 @@ private void updateExistingLocation(AddressLevel location, AddressLevel newParen
         updateLocationProperties(row, allErrorMsgs, location);
     }
 
+    @Transactional(Transactional.TxType.REQUIRES_NEW)
     public void write(List<? extends Row> rows) {
         List<String> allErrorMsgs = new ArrayList<>();
         validateEditModeHeaders(rows.get(0).getHeaders(), allErrorMsgs);

diff --git a/...r-api/src/main/java/org/avni/server/importer/batch/csv/writer/UserAndCatchmentWriter.java b/...r-api/src/main/java/org/avni/server/importer/batch/csv/writer/UserAndCatchmentWriter.java
@@ -1,5 +1,7 @@
 package org.avni.server.importer.batch.csv.writer;
 
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
 import org.avni.server.dao.LocationRepository;
 import org.avni.server.dao.UserRepository;
 import org.avni.server.domain.Locale;
@@ -17,10 +19,12 @@
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
 
+import javax.transaction.Transactional;
 import java.io.Serializable;
 import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static java.lang.String.format;
 import static org.avni.server.domain.OperatingIndividualScope.ByCatchment;
@@ -44,8 +48,8 @@ public class UserAndCatchmentWriter implements ItemWriter<Row>, Serializable {
     private static final String ERR_MSG_LOCATION_FIELD = "Provided Location does not exist in Avni. Please add it or check for spelling mistakes '%s'";
     private static final String ERR_MSG_LOCALE_FIELD = "Provided value '%s' for Preferred Language is invalid";
     private static final String ERR_MSG_DATE_PICKER_FIELD = "Provided value '%s' for Date picker mode is invalid";
-    private static final String ERR_MSG_UNKNOWN_HEADERS = "Unknown headers included in file. Please refer to sample file for valid list of headers";
-    private static final String ERR_MSG_MISSING_MANDATORY_FIELDS = "Mandatory columns are missing from uploaded file. Please refer to sample file for the list of mandatory headers.";
+    private static final String ERR_MSG_UNKNOWN_HEADERS = "Unknown headers - %s included in file. Please refer to sample file for valid list of headers";
+    private static final String ERR_MSG_MISSING_MANDATORY_FIELDS = "Mandatory columns are missing from uploaded file - %s. Please refer to sample file for the list of mandatory headers.";
     private static final String ERR_MSG_INVALID_CONCEPT_ANSWER = "'%s' is not a valid value for the concept '%s'" +
             "To input this value, add this as an answer to the coded concept '%s'";
     private static final String METADATA_ROW_START_STRING = "Mandatory field.";
@@ -71,9 +75,10 @@ public UserAndCatchmentWriter(CatchmentService catchmentService,
         this.compoundHeaderPattern = Pattern.compile("^(?<subjectTypeName>.*?)->(?<conceptName>.*)$");
     }
 
+    @Transactional(Transactional.TxType.REQUIRES_NEW)
     @Override
-    public void write(List<? extends Row> rows) throws Exception {
-        if(!CollectionUtils.isEmpty(rows)) {
+    public void write(List<? extends Row> rows) throws IDPException {
+        if (!CollectionUtils.isEmpty(rows)) {
             validateHeaders(rows.get(0).getHeaders());
             for (Row row : rows) write(row);
         }
@@ -87,27 +92,33 @@ private void validateHeaders(String[] headers) {
         List<String> syncAttributeHeadersForSubjectTypes = subjectTypeService.constructSyncAttributeHeadersForSubjectTypes();
         checkForMissingHeaders(headerList, allErrorMsgs, expectedStandardHeaders, syncAttributeHeadersForSubjectTypes);
         checkForUnknownHeaders(headerList, allErrorMsgs, expectedStandardHeaders, syncAttributeHeadersForSubjectTypes);
-        if(!allErrorMsgs.isEmpty()) {
+        if (!allErrorMsgs.isEmpty()) {
             throw new RuntimeException(String.join(ERR_MSG_DELIMITER, allErrorMsgs));
         }
     }
 
     private void checkForUnknownHeaders(List<String> headerList, List<String> allErrorMsgs, List<String> expectedStandardHeaders, List<String> syncAttributeHeadersForSubjectTypes) {
         headerList.removeIf(StringUtils::isEmpty);
-        headerList.removeIf(header -> expectedStandardHeaders.contains(header));
-        headerList.removeIf(header -> syncAttributeHeadersForSubjectTypes.contains(header));
-        if (!headerList.isEmpty()) {
-            allErrorMsgs.add(ERR_MSG_UNKNOWN_HEADERS);
+        HashSet<String> expectedHeaders = new HashSet<>(expectedStandardHeaders);
+        expectedHeaders.addAll(syncAttributeHeadersForSubjectTypes);
+        Sets.SetView<String> unknownHeaders = Sets.difference(new HashSet<>(headerList), expectedHeaders);
+        if (!unknownHeaders.isEmpty()) {
+            allErrorMsgs.add(String.format(ERR_MSG_UNKNOWN_HEADERS, String.join(", ", unknownHeaders)));
         }
     }
 
     private void checkForMissingHeaders(List<String> headerList, List<String> allErrorMsgs, List<String> expectedStandardHeaders, List<String> syncAttributeHeadersForSubjectTypes) {
-        if (headerList.isEmpty() || !headerList.containsAll(expectedStandardHeaders) || !headerList.containsAll(syncAttributeHeadersForSubjectTypes)) {
-            allErrorMsgs.add(ERR_MSG_MISSING_MANDATORY_FIELDS);
+        HashSet<String> expectedHeaders = new HashSet<>(expectedStandardHeaders);
+        expectedHeaders.addAll(syncAttributeHeadersForSubjectTypes);
+        HashSet<String> presentHeaders = new HashSet<>(headerList);
+        presentHeaders.addAll(syncAttributeHeadersForSubjectTypes);
+        Sets.SetView<String> missingHeaders = Sets.difference(expectedHeaders, presentHeaders);
+        if (!missingHeaders.isEmpty()) {
+            allErrorMsgs.add(String.format(ERR_MSG_MISSING_MANDATORY_FIELDS, String.join(", ", missingHeaders)));
         }
     }
 
-    private void write(Row row) throws Exception {
+    private void write(Row row) throws IDPException {
         List<String> rowValidationErrorMsgs = new ArrayList<>();
         String fullAddress = row.get(LOCATION_WITH_FULL_HIERARCHY);
         if (fullAddress != null && fullAddress.startsWith(METADATA_ROW_START_STRING)) return;
@@ -179,7 +190,7 @@ private void validateRowAndAssimilateErrors(List<String> rowValidationErrorMsgs,
         extractUserUsernameValidationErrMsg(rowValidationErrorMsgs, username, userSuffix);
         extractUserNameValidationErrMsg(rowValidationErrorMsgs, nameOfUser);
         extractUserEmailValidationErrMsg(rowValidationErrorMsgs, email);
-        if(!rowValidationErrorMsgs.isEmpty()) {
+        if (!rowValidationErrorMsgs.isEmpty()) {
             throw new RuntimeException(String.join(ERR_MSG_DELIMITER, rowValidationErrorMsgs));
         }
     }
@@ -209,7 +220,7 @@ private void extractUserUsernameValidationErrMsg(List<String> rowValidationError
     }
 
     private void addErrMsgIfValidationFails(boolean validationCheckResult, List<String> rowValidationErrorMsgs, String validationErrorMessage) {
-        if(validationCheckResult) {
+        if (validationCheckResult) {
             rowValidationErrorMsgs.add(validationErrorMessage);
         }
     }
@@ -273,7 +284,7 @@ private List<String> findSyncSettingCodedConceptValues(List<String> syncSettings
         List<String> syncSettingCodedConceptValues = new ArrayList<>();
         for (String syncSettingsValue : syncSettingsValues) {
             Optional<Concept> conceptAnswer = Optional.ofNullable(conceptService.getByName(syncSettingsValue));
-            if(conceptAnswer.isPresent()) {
+            if (conceptAnswer.isPresent()) {
                 syncSettingCodedConceptValues.add(conceptAnswer.get().getUuid());
             } else {
                 rowValidationErrorMsgs.add(format(ERR_MSG_INVALID_CONCEPT_ANSWER, syncSettingsValue, concept.getName(), concept.getName()));

diff --git a/avni-server-api/src/main/resources/db/migration/R__Functions.sql b/avni-server-api/src/main/resources/db/migration/R__Functions.sql
@@ -10,39 +10,39 @@ $BODY$
     EXECUTE 'GRANT ' || quote_ident(inrolname) || ' TO openchs';
     PERFORM grant_all_on_all(inrolname);
     RETURN 1;
-  END
+END
 $BODY$ LANGUAGE PLPGSQL;
 
 
 DROP FUNCTION IF EXISTS create_implementation_schema(text);
 CREATE OR REPLACE FUNCTION create_implementation_schema(schema_name text, db_user text)
-  RETURNS BIGINT AS
+    RETURNS BIGINT AS
 $BODY$
 BEGIN
-  EXECUTE 'CREATE SCHEMA IF NOT EXISTS "' || schema_name || '" AUTHORIZATION "' || db_user || '"';
-  EXECUTE 'GRANT ALL PRIVILEGES ON SCHEMA "' || schema_name || '" TO "' || db_user || '"';
-  RETURN 1;
+    EXECUTE 'CREATE SCHEMA IF NOT EXISTS "' || schema_name || '" AUTHORIZATION "' || db_user || '"';
+    EXECUTE 'GRANT ALL PRIVILEGES ON SCHEMA "' || schema_name || '" TO "' || db_user || '"';
+    RETURN 1;
 END
 $BODY$ LANGUAGE PLPGSQL;
 
 
 CREATE OR REPLACE FUNCTION jsonb_object_values_contain(obs JSONB, pattern TEXT)
   RETURNS BOOLEAN AS $$
 BEGIN
-  return EXISTS (select true from jsonb_each_text(obs) where value ilike pattern);
+    return EXISTS (select true from jsonb_each_text(obs) where value ilike pattern);
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+    LANGUAGE plpgsql IMMUTABLE;
 
 CREATE OR REPLACE FUNCTION create_audit(user_id NUMERIC)
   RETURNS INTEGER AS $$
 DECLARE result INTEGER;
 BEGIN
-  INSERT INTO audit(created_by_id, last_modified_by_id, created_date_time, last_modified_date_time)
+    INSERT INTO audit(created_by_id, last_modified_by_id, created_date_time, last_modified_date_time)
   VALUES(user_id, user_id, now(), now()) RETURNING id into result;
-  RETURN result;
+    RETURN result;
 END $$
-  LANGUAGE plpgsql;
+    LANGUAGE plpgsql;
 
 CREATE OR REPLACE FUNCTION create_audit()
   RETURNS INTEGER AS 'select create_audit(1)' language sql;
@@ -58,7 +58,7 @@ CREATE OR REPLACE FUNCTION create_view(schema_name text, view_name text, sql_que
     RETURNS BIGINT AS
 $BODY$
 BEGIN
---     EXECUTE 'set search_path = ' || ;
+    --     EXECUTE 'set search_path = ' || ;
     EXECUTE 'DROP VIEW IF EXISTS ' || schema_name || '.' || view_name;
     EXECUTE 'CREATE OR REPLACE VIEW ' || schema_name || '.' || view_name || ' AS ' || sql_query;
     RETURN 1;
@@ -207,13 +207,13 @@ $body$
 BEGIN
     EXECUTE (
         SELECT 'GRANT ALL ON TABLE '
-                   || tablename
+                        || tablename
                    || ' TO ' || quote_ident(rolename)
     );
 
     EXECUTE (
         SELECT 'GRANT SELECT ON '
-                   || tablename
+                        || tablename
                    || ' TO ' || quote_ident(rolename)
     );
 
@@ -228,19 +228,19 @@ $body$
 BEGIN
     EXECUTE (
         SELECT 'GRANT ALL ON TABLE '
-                   || string_agg(format('%I.%I', table_schema, table_name), ',')
-                   || ' TO ' || quote_ident(rolename) || ''
-        FROM information_schema.tables
-        WHERE table_schema = 'public'
+                        || string_agg(format('%I.%I', table_schema, table_name), ',')
+                        || ' TO ' || quote_ident(rolename) || ''
+             FROM information_schema.tables
+             WHERE table_schema = 'public'
           AND table_type = 'BASE TABLE'
     );
 
     EXECUTE (
         SELECT 'GRANT SELECT ON '
-                   || string_agg(format('%I.%I', schemaname, viewname), ',')
-                   || ' TO ' || quote_ident(rolename) || ''
-        FROM pg_catalog.pg_views
-        WHERE schemaname = 'public'
+                        || string_agg(format('%I.%I', schemaname, viewname), ',')
+                        || ' TO ' || quote_ident(rolename) || ''
+             FROM pg_catalog.pg_views
+             WHERE schemaname = 'public'
           and viewowner in ('openchs')
     );
 
@@ -276,56 +276,56 @@ CREATE OR REPLACE FUNCTION multi_select_coded(obs JSONB)
 AS $$
 DECLARE result VARCHAR;
 BEGIN
-  BEGIN
-    IF JSONB_TYPEOF(obs) = 'array'
-    THEN
+    BEGIN
+        IF JSONB_TYPEOF(obs) = 'array'
+        THEN
       SELECT STRING_AGG(C.NAME, ' ,') FROM JSONB_ARRAY_ELEMENTS_TEXT(obs) AS OB (UUID)
-                                             JOIN CONCEPT C ON C.UUID = OB.UUID
-      INTO RESULT;
-    ELSE
-      SELECT SINGLE_SELECT_CODED(obs) INTO RESULT;
-    END IF;
-    RETURN RESULT;
+                     JOIN CONCEPT C ON C.UUID = OB.UUID
+            INTO RESULT;
+        ELSE
+            SELECT SINGLE_SELECT_CODED(obs) INTO RESULT;
+        END IF;
+        RETURN RESULT;
   EXCEPTION WHEN OTHERS
-    THEN
-      RAISE NOTICE 'Failed while processing multi_select_coded(''%'')', obs :: TEXT;
-      RAISE NOTICE '% %', SQLERRM, SQLSTATE;
-  END;
+            THEN
+                RAISE NOTICE 'Failed while processing multi_select_coded(''%'')', obs :: TEXT;
+                RAISE NOTICE '% %', SQLERRM, SQLSTATE;
+    END;
 END $$;
 
 CREATE OR REPLACE FUNCTION single_select_coded(obs TEXT)
   RETURNS VARCHAR LANGUAGE plpgsql
 AS $$
 DECLARE result VARCHAR;
 BEGIN
-  BEGIN
+    BEGIN
     SELECT name FROM concept WHERE uuid = obs
-    INTO result;
-    RETURN result;
-  END;
+        INTO result;
+        RETURN result;
+    END;
 END $$
-  STABLE;
+    STABLE;
 
 CREATE OR REPLACE FUNCTION single_select_coded(obs JSONB)
   RETURNS VARCHAR LANGUAGE plpgsql
 AS $$
 DECLARE result VARCHAR;
 BEGIN
-  BEGIN
-    IF JSONB_TYPEOF(obs) = 'array'
-    THEN
-      SELECT name FROM concept WHERE (obs->>0) = uuid INTO result;
-    ELSEIF JSONB_TYPEOF(obs) = 'string'
-    THEN
-      select name from concept where (array_to_json(array[obs])->>0) = uuid into result;
-    END IF;
-    RETURN result;
-  END;
+    BEGIN
+        IF JSONB_TYPEOF(obs) = 'array'
+        THEN
+            SELECT name FROM concept WHERE (obs ->> 0) = uuid INTO result;
+        ELSEIF JSONB_TYPEOF(obs) = 'string'
+        THEN
+            select name from concept where (array_to_json(array [obs]) ->> 0) = uuid into result;
+        END IF;
+        RETURN result;
+    END;
 END $$
-  STABLE;
+    STABLE;
 
 CREATE OR REPLACE FUNCTION no_op() RETURNS trigger AS $$
 BEGIN
     RETURN NULL;
 END;
-$$ LANGUAGE plpgsql;
+$$ LANGUAGE plpgsql;
diff --git a/avni-server-api/src/main/resources/db/migration/V1_343__User1ShouldHaveNoOrganisationId.sql b/avni-server-api/src/main/resources/db/migration/V1_343__User1ShouldHaveNoOrganisationId.sql
@@ -0,0 +1,2 @@
+-- In earlier migration we have set parent_organisation as 1 but fix in production
+update users set organisation_id = null where id = 1 and username = 'admin';
diff --git a/avni-server-api/src/test/java/org/avni/messaging/api/MessageTemplateControllerTest.java b/avni-server-api/src/test/java/org/avni/messaging/api/MessageTemplateControllerTest.java
@@ -4,10 +4,7 @@
 import com.github.tomakehurst.wiremock.client.WireMock;
 import org.avni.messaging.contract.MessageTemplateContract;
 import org.avni.server.common.AbstractControllerIntegrationTest;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
 import org.springframework.test.context.jdbc.Sql;
 
 import java.util.List;
@@ -16,6 +13,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 
 @Sql(scripts = {"/test-data.sql"})
+@Ignore // this is not testing much and is setup with admin user which is incorrect
 public class MessageTemplateControllerTest extends AbstractControllerIntegrationTest {
 
     private String SAMPLE_AUTH_RESPONSE = "{\"data\":{\"access_token\":\"SFMyNTY.YjQ2M2MzMmMtNGZlOC00OTEyLWIzYTEtZmRhZTRkOGQ1ZTIx.3TjKqpElrD5N2ffGHEAFX91cyp7zwoTztYR8p1jwwgA\",\"renewal_token\":\"SFMyNTY.MjYxODllMTgtNDM1OC00YjJjLTlmN2MtOTA5MzMwYzM3ZjA2.dDigSwftcGFGHu4o9MwkASp2KqH6eitp1aRmeYSgi5M\",\"token_expiry_time\":\"2022-10-13T21:42:33.342529Z\"}}";

diff --git a/avni-server-api/src/test/java/org/avni/server/domain/factory/TestOrganisationBuilder.java b/avni-server-api/src/test/java/org/avni/server/domain/factory/TestOrganisationBuilder.java
@@ -12,14 +12,19 @@ public class TestOrganisationBuilder {
 
     public TestOrganisationBuilder withMandatoryFields() {
         String placeholder = UUID.randomUUID().toString();
-        return withUuid(placeholder).withDbUser("testDbUser").withName(placeholder).withSchemaName(placeholder);
+        return withUuid(placeholder).withDbUser("testDbUserNew").withName(placeholder).withSchemaName(placeholder);
     }
 
     public TestOrganisationBuilder setId(long id) {
         organisation.setId(id);
         return this;
     }
 
+    public TestOrganisationBuilder withUsernameSuffix(String usernameSuffix) {
+        organisation.setUsernameSuffix(usernameSuffix);
+    	return this;
+    }
+
     public TestOrganisationBuilder withSchemaName(String schemaName) {
         organisation.setSchemaName(schemaName);
     	return this;