feat(authority): implement bulk load authorities from external file (#…

…240) Create new endpoint for bulk load Implement infrastructure for load/upload files to S3 storage Refactor authority repositories and services Move out request validation from service layer to controller-delegate layer. Consortium service is defined to do additional validtion for shadow copies Remove duplicate fetching records by id from delegate and add callback logic for service methods. Closes: MODELINKS-173
folio-org · Feb 8, 2024 · fa4a813 · fa4a813
1 parent d5193e1
commit fa4a813
Show file tree

Hide file tree

Showing 48 changed files with 1,497 additions and 434 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -20,7 +20,8 @@
 * Implement next hrid endpoint for authority source file([MODELINKS-122](https://issues.folio.org/browse/MODELINKS-122))
 * Add protocol to base url for authority source files([MODELINKS-191](https://issues.folio.org/browse/MODELINKS-191))
 * Make system user usage optional ([MODELINKS-150](https://issues.folio.org/browse/MODELINKS-150) and [MODROLESKC-24](https://issues.folio.org/browse/MODROLESKC-24))
-* Propagate authority archives deletion to member tenants([MODELINKS-195](https://issues.folio.org/browse/MODELINKS-195))
+* Propagate authority archives deletion to member tenants ([MODELINKS-195](https://issues.folio.org/browse/MODELINKS-195))
+* Implement endpoint for bulk authorities upsert from external file ([MODELINKS-173](https://issues.folio.org/browse/MODELINKS-173))
 
 ### Bug fixes
 * Fix secure setup of system users by default ([MODELINKS-135](https://issues.folio.org/browse/MODELINKS-135))

diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json
@@ -167,7 +167,7 @@
     },
     {
       "id": "authority-storage",
-      "version": "2.0",
+      "version": "2.1",
       "handlers": [
         {
           "methods": [
@@ -176,6 +176,9 @@
           "pathPattern": "/authority-storage/authorities",
           "permissionsRequired": [
             "inventory-storage.authorities.collection.get"
+          ],
+          "modulePermissions": [
+            "user-tenants.collection.get"
           ]
         },
         {
@@ -185,6 +188,9 @@
           "pathPattern": "/authority-storage/authorities/{id}",
           "permissionsRequired": [
             "inventory-storage.authorities.item.get"
+          ],
+          "modulePermissions": [
+            "user-tenants.collection.get"
           ]
         },
         {
@@ -199,6 +205,18 @@
             "user-tenants.collection.get"
           ]
         },
+        {
+          "methods": [
+            "POST"
+          ],
+          "pathPattern": "/authority-storage/authorities/bulk",
+          "permissionsRequired": [
+            "inventory-storage.authorities.bulk.post"
+          ],
+          "modulePermissions": [
+            "user-tenants.collection.get"
+          ]
+        },
         {
           "methods": [
             "PUT"
@@ -528,6 +546,11 @@
       "displayName": "inventory storage - create individual authority record",
       "description": "create individual authority record in the storage"
     },
+    {
+      "permissionName": "inventory-storage.authorities.bulk.post",
+      "displayName": "inventory storage - create authority records in bulk",
+      "description": "create authority records in bulk"
+    },
     {
       "permissionName": "inventory-storage.authorities.item.put",
       "displayName": "inventory storage - modify authority record",
@@ -563,8 +586,10 @@
       "displayName": "inventory storage module - all authorities permissions",
       "description": "Entire set of permissions needed to use authorities in the inventory storage module",
       "subPermissions": [
+        "inventory-storage.authorities.collection.get",
         "inventory-storage.authorities.item.get",
         "inventory-storage.authorities.item.post",
+        "inventory-storage.authorities.bulk.post",
         "inventory-storage.authorities.item.put",
         "inventory-storage.authorities.item.delete",
         "authority-storage.authority.reindex.post",
@@ -826,6 +851,36 @@
         "name": "AUTHORITY_ARCHIVES_EXPIRATION_PERIOD",
         "value": "7",
         "description": "The retention period in days for keeping the deleted authorities in authority_archive DB table"
+      },
+      {
+        "name": "S3_URL",
+        "value": "http://localhost:9000/",
+        "description": "S3 compatible service url"
+      },
+      {
+        "name": "S3_REGION",
+        "value": "",
+        "description": "S3 compatible service region"
+      },
+      {
+        "name": "S3_BUCKET",
+        "value": "marc-migrations",
+        "description": "S3 compatible service bucket"
+      },
+      {
+        "name": "S3_ACCESS_KEY_ID",
+        "value": "",
+        "description": "S3 compatible service access key"
+      },
+      {
+        "name": "S3_SECRET_ACCESS_KEY",
+        "value": "",
+        "description": "S3 compatible service secret key"
+      },
+      {
+        "name": "S3_IS_AWS",
+        "value": "true",
+        "description": "Specify if AWS S3 is used as files storage"
       }
     ]
   }

diff --git a/doc/documentation.md b/doc/documentation.md
diff --git a/pom.xml b/pom.xml
@@ -25,8 +25,9 @@
       src/main/java/org/folio/entlinks/model/**
     </sonar.exclusions>
 
-    <folio-spring-base.version>8.0.0</folio-spring-base.version>
+    <folio-spring-support.version>8.1.0-SNAPSHOT</folio-spring-support.version>
     <folio-service-tools.version>4.0.0-SNAPSHOT</folio-service-tools.version>
+    <folio-s3-client.version>2.1.0-SNAPSHOT</folio-s3-client.version>
     <mapstruct.version>1.5.5.Final</mapstruct.version>
     <lombok.mapstruct-binding.version>0.2.0</lombok.mapstruct-binding.version>
     <marc4j.version>2.9.5</marc4j.version>
@@ -43,36 +44,36 @@
     <dependency>
       <groupId>org.folio</groupId>
       <artifactId>folio-spring-base</artifactId>
-      <version>${folio-spring-base.version}</version>
+      <version>${folio-spring-support.version}</version>
     </dependency>
 
     <dependency>
       <groupId>org.folio</groupId>
       <artifactId>folio-spring-cql</artifactId>
-      <version>${folio-spring-base.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.springframework.retry</groupId>
-      <artifactId>spring-retry</artifactId>
+      <version>${folio-spring-support.version}</version>
     </dependency>
 
     <dependency>
       <groupId>org.folio</groupId>
       <artifactId>folio-spring-system-user</artifactId>
-      <version>${folio-spring-base.version}</version>
+      <version>${folio-spring-support.version}</version>
     </dependency>
 
     <dependency>
       <groupId>org.folio</groupId>
-      <artifactId>folio-spring-testing</artifactId>
-      <version>${folio-spring-base.version}</version>
-      <scope>test</scope>
+      <artifactId>folio-service-tools-spring-dev</artifactId>
+      <version>${folio-service-tools.version}</version>
     </dependency>
 
     <dependency>
       <groupId>org.folio</groupId>
-      <artifactId>folio-service-tools-spring-dev</artifactId>
-      <version>${folio-service-tools.version}</version>
+      <artifactId>folio-s3-client</artifactId>
+      <version>${folio-s3-client.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>org.springframework.retry</groupId>
+      <artifactId>spring-retry</artifactId>
     </dependency>
 
     <dependency>
@@ -157,6 +158,13 @@
       </exclusions>
     </dependency>
 
+    <dependency>
+      <groupId>org.folio</groupId>
+      <artifactId>folio-spring-testing</artifactId>
+      <version>${folio-spring-support.version}</version>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>org.awaitility</groupId>
       <artifactId>awaitility</artifactId>

diff --git a/src/main/java/org/folio/entlinks/config/RemoteStorageConfig.java b/src/main/java/org/folio/entlinks/config/RemoteStorageConfig.java
@@ -0,0 +1,47 @@
+package org.folio.entlinks.config;
+
+import lombok.Data;
+import lombok.extern.log4j.Log4j2;
+import org.folio.s3.client.FolioS3Client;
+import org.folio.s3.client.S3ClientFactory;
+import org.folio.s3.client.S3ClientProperties;
+import org.folio.s3.exception.S3ClientException;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@Log4j2
+@ConfigurationProperties("folio.remote-storage")
+@Data
+@ConfigurationPropertiesScan
+public class RemoteStorageConfig {
+
+  private String endpoint;
+  private String region;
+  private String bucket;
+  private String accessKey;
+  private String secretKey;
+  private boolean awsSdk;
+
+  @Bean
+  public FolioS3Client remoteFolioS3Client() {
+    log.debug("remote-files-storage: endpoint {}, region {}, bucket {}, accessKey {}, secretKey {}, awsSdk {}",
+      endpoint, region, bucket, accessKey, secretKey, awsSdk);
+    var client = S3ClientFactory.getS3Client(S3ClientProperties.builder()
+      .endpoint(endpoint)
+      .secretKey(secretKey)
+      .accessKey(accessKey)
+      .bucket(bucket)
+      .awsSdk(awsSdk)
+      .region(region)
+      .build());
+    try {
+      client.createBucketIfNotExists();
+    } catch (S3ClientException e) {
+      log.error("Error creating bucket: {} during RemoteStorageClient initialization", bucket);
+    }
+    return client;
+  }
+}
diff --git a/src/main/java/org/folio/entlinks/config/constants/ErrorCode.java b/src/main/java/org/folio/entlinks/config/constants/ErrorCode.java
@@ -17,7 +17,9 @@ public enum ErrorCode {
   DUPLICATE_AUTHORITY_SOURCE_FILE_SEQUENCE("110",
       "Authority source file with the same given 'code' and HRID generator name already exist."),
   DUPLICATE_AUTHORITY_SOURCE_FILE_ID("111",
-      "Authority Source File with the given 'id' already exists.");
+      "Authority Source File with the given 'id' already exists."),
+  NOT_EXISTED_AUTHORITY_SOURCE_FILE("112",
+    "Authority Source File with the given 'id' does not exists.");
 
   @Getter
   private final String code;

diff --git a/src/main/java/org/folio/entlinks/controller/ApiErrorHandler.java b/src/main/java/org/folio/entlinks/controller/ApiErrorHandler.java
@@ -10,6 +10,7 @@
 import static org.folio.entlinks.config.constants.ErrorCode.DUPLICATE_AUTHORITY_SOURCE_FILE_SEQUENCE;
 import static org.folio.entlinks.config.constants.ErrorCode.DUPLICATE_AUTHORITY_SOURCE_FILE_URL;
 import static org.folio.entlinks.config.constants.ErrorCode.DUPLICATE_NOTE_TYPE_NAME;
+import static org.folio.entlinks.config.constants.ErrorCode.NOT_EXISTED_AUTHORITY_SOURCE_FILE;
 import static org.folio.entlinks.config.constants.ErrorCode.VIOLATION_OF_RELATION_BETWEEN_AUTHORITY_AND_SOURCE_FILE;
 import static org.folio.entlinks.exception.type.ErrorType.UNKNOWN_ERROR;
 import static org.folio.entlinks.exception.type.ErrorType.VALIDATION_ERROR;
@@ -25,6 +26,7 @@
 import lombok.extern.log4j.Log4j2;
 import org.apache.logging.log4j.Level;
 import org.folio.entlinks.config.constants.ErrorCode;
+import org.folio.entlinks.domain.entity.AuthoritySourceFile;
 import org.folio.entlinks.exception.AuthoritiesRequestNotSupportedMediaTypeException;
 import org.folio.entlinks.exception.AuthoritySourceFileHridException;
 import org.folio.entlinks.exception.OptimisticLockingException;
@@ -34,6 +36,7 @@
 import org.folio.tenant.domain.dto.Error;
 import org.folio.tenant.domain.dto.Errors;
 import org.folio.tenant.domain.dto.Parameter;
+import org.hibernate.TransientPropertyValueException;
 import org.hibernate.exception.ConstraintViolationException;
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.dao.InvalidDataAccessApiUsageException;
@@ -152,6 +155,16 @@ public ResponseEntity<Errors> conflict(Exception e) {
       var constraintName = cve.getConstraintName();
       var errorCode = CONSTRAINS_I18N_MAP.get(constraintName);
       return buildResponseEntity(errorCode, VALIDATION_ERROR, UNPROCESSABLE_ENTITY);
+    } else if (cause instanceof IllegalStateException ise) {
+      var innerCause = ise.getCause();
+      if (innerCause instanceof TransientPropertyValueException tpve) {
+        var propertyName = tpve.getPropertyName();
+        var transientEntityName = tpve.getTransientEntityName();
+        if (AuthoritySourceFile.class.getName().equals(transientEntityName)
+            && "authoritySourceFile".equals(propertyName)) {
+          return buildResponseEntity(NOT_EXISTED_AUTHORITY_SOURCE_FILE, VALIDATION_ERROR, UNPROCESSABLE_ENTITY);
+        }
+      }
     }
     return buildResponseEntity(e, BAD_REQUEST, VALIDATION_ERROR);
   }

diff --git a/src/main/java/org/folio/entlinks/controller/AuthorityController.java b/src/main/java/org/folio/entlinks/controller/AuthorityController.java
@@ -9,6 +9,8 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.folio.entlinks.controller.delegate.AuthorityArchiveServiceDelegate;
 import org.folio.entlinks.controller.delegate.AuthorityServiceDelegate;
+import org.folio.entlinks.domain.dto.AuthorityBulkRequest;
+import org.folio.entlinks.domain.dto.AuthorityBulkResponse;
 import org.folio.entlinks.domain.dto.AuthorityDto;
 import org.folio.entlinks.domain.dto.AuthorityDtoCollection;
 import org.folio.entlinks.exception.AuthoritiesRequestNotSupportedMediaTypeException;
@@ -29,10 +31,7 @@
 public class AuthorityController implements AuthorityStorageApi {
 
   public static final String RETRIEVE_COLLECTION_INVALID_ACCEPT_MESSAGE =
-      "It is not allowed to retrieve authorities in text/plain format";
-
-  public static final String RETRIEVE_COLLECTION_UNSUPPORTED_ACCEPT_MESSAGE = "The provided expected media-type format"
-      + " is not supported in retrieving authorities";
+    "It is not allowed to retrieve authorities in text/plain format";
 
   private final AuthorityServiceDelegate delegate;
   private final AuthorityArchiveServiceDelegate authorityArchiveServiceDelegate;
@@ -43,6 +42,11 @@ public ResponseEntity<AuthorityDto> createAuthority(AuthorityDto authority) {
     return ResponseEntity.status(HttpStatus.CREATED).body(created);
   }
 
+  @Override
+  public ResponseEntity<AuthorityBulkResponse> createAuthorityBulk(AuthorityBulkRequest createRequest) {
+    return ResponseEntity.ok(delegate.createAuthorities(createRequest));
+  }
+
   @Override
   public ResponseEntity<Void> deleteAuthority(UUID id) {
     delegate.deleteAuthorityById(id);
@@ -58,11 +62,12 @@ public ResponseEntity<AuthorityDto> getAuthority(UUID id) {
   @Override
   public ResponseEntity retrieveAuthorities(Boolean deleted, Boolean idOnly, Integer offset, Integer limit,
                                             String query, @RequestHeader(value = "Accept", required = false,
-      defaultValue = "application/json") List<String> acceptingMediaTypes) {
+                                                                         defaultValue = "application/json")
+                                            List<String> acceptingMediaTypes) {
     validateGetParams(idOnly, acceptingMediaTypes);
     var collectionDto = Boolean.TRUE.equals(deleted)
-        ? authorityArchiveServiceDelegate.retrieveAuthorityArchives(offset, limit, query, idOnly)
-        : delegate.retrieveAuthorityCollection(offset, limit, query, idOnly);
+                        ? authorityArchiveServiceDelegate.retrieveAuthorityArchives(offset, limit, query, idOnly)
+                        : delegate.retrieveAuthorityCollection(offset, limit, query, idOnly);
 
     return getAuthoritiesCollectionResponse(collectionDto, acceptingMediaTypes, idOnly);
   }
@@ -77,11 +82,11 @@ public ResponseEntity<Void> updateAuthority(UUID id, AuthorityDto authority) {
    * POST /authority-storage/expire/authorities.
    *
    * @return Successfully published authorities expire job (status code 202)
-   *         or Internal server error. (status code 500)
+   *   or Internal server error. (status code 500)
    */
   @PostMapping(
-      value = "/authority-storage/expire/authorities",
-      produces = { "application/json" }
+    value = "/authority-storage/expire/authorities",
+    produces = {"application/json"}
   )
   public ResponseEntity<Void> expireAuthorities() {
     authorityArchiveServiceDelegate.expire();
@@ -96,12 +101,12 @@ private ResponseEntity<Object> getAuthoritiesCollectionResponse(AuthorityDtoColl
         && acceptingMediaTypes.contains(TEXT_PLAIN_VALUE)) {
       headers.setContentType(MediaType.TEXT_PLAIN);
       return new ResponseEntity<>(
-          collectionDto.getAuthorities().stream()
-              .map(AuthorityDto::getId)
-              .map(UUID::toString)
-              .collect(Collectors.joining(System.lineSeparator())),
-          headers,
-          HttpStatus.OK
+        collectionDto.getAuthorities().stream()
+          .map(AuthorityDto::getId)
+          .map(UUID::toString)
+          .collect(Collectors.joining(System.lineSeparator())),
+        headers,
+        HttpStatus.OK
       );
     }
 
@@ -112,7 +117,7 @@ private ResponseEntity<Object> getAuthoritiesCollectionResponse(AuthorityDtoColl
   private void validateGetParams(Boolean idOnly, List<String> acceptingMediaTypes) {
     if (List.of(TEXT_PLAIN_VALUE).equals(acceptingMediaTypes) && Boolean.FALSE.equals(idOnly)) {
       throw new AuthoritiesRequestNotSupportedMediaTypeException(RETRIEVE_COLLECTION_INVALID_ACCEPT_MESSAGE,
-          List.of(new Parameter("Accept").value(TEXT_PLAIN_VALUE), new Parameter("idOnly").value("false")));
+        List.of(new Parameter("Accept").value(TEXT_PLAIN_VALUE), new Parameter("idOnly").value("false")));
     }
   }
 }