MODBULKOPS-249 - MARC Instance - Confirmation screen (MarcInstanceUpd…

…ateProcessor) (#263)

descriptors/ModuleDescriptor-template.json

@@ -169,6 +169,7 @@
             "consortia.user-tenants.collection.get",
             "consortium-search.holdings.collection.get",
             "data-import.uploaddefinitions.post",
+            "data-import.uploaddefinitions.get",
             "data-import.upload.file.post",
             "data-import.uploaddefinitions.files.post",
             "converter-storage.jobprofile.get",
@@ -178,7 +179,10 @@
             "converter-storage.matchprofile.post",
             "converter-storage.actionprofile.post",
             "converter-storage.mappingprofile.post",
-            "source-storage.sourceRecords.get"
+            "source-storage.sourceRecords.get",
+            "data-import.splitconfig.get",
+            "data-import.uploadUrl.get",
+            "data-import.assembleStorageFile.post"
           ]
         },
         {
@@ -214,7 +218,10 @@
             "bulk-edit.item.post",
             "bulk-edit.start.item.post",
             "data-export.job.item.post",
-            "data-export.job.item.get"
+            "data-export.job.item.get",
+            "metadata-provider.jobexecutions.get",
+            "metadata-provider.logs.get",
+            "source-storage.sourceRecords.get"
           ]
         },
         {
@@ -532,6 +539,18 @@
     {
       "id": "contributor-types",
       "version": "2.0"
+    },
+    {
+      "id": "data-import-converter-storage",
+      "version": "1.4"
+    },
+    {
+      "id": "source-manager-job-executions",
+      "version": "3.3"
+    },
+    {
+      "id": "metadata-provider-job-execution-logs",
+      "version": "1.0"
     }
   ],
   "launchDescriptor": {

src/main/java/org/folio/bulkops/builder/DataImportProfilesBuilder.java

@@ -62,9 +62,9 @@ public JobProfilePost getJobProfilePost(MatchProfile matchProfile, ActionProfile
       var jobProfilePost = objectMapper.readValue(is, JobProfilePost.class);
       jobProfilePost.getAddedRelations().get(0).setDetailProfileId(matchProfile.getId());
       jobProfilePost.getAddedRelations().get(1).setMasterProfileId(matchProfile.getId());
-      jobProfilePost.getAddedRelations().get(1).setDetailProfileId(actionProfileToUpdateInstance.getId());
+      jobProfilePost.getAddedRelations().get(1).setDetailProfileId(actionProfileToUpdateSrs.getId());
       jobProfilePost.getAddedRelations().get(2).setMasterProfileId(matchProfile.getId());
-      jobProfilePost.getAddedRelations().get(2).setDetailProfileId(actionProfileToUpdateSrs.getId());
+      jobProfilePost.getAddedRelations().get(2).setDetailProfileId(actionProfileToUpdateInstance.getId());
       return jobProfilePost;
     }
   }

src/main/java/org/folio/bulkops/client/DataImportClient.java

@@ -1,23 +1,36 @@
 package org.folio.bulkops.client;
 
-import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import org.folio.bulkops.configs.FeignClientConfiguration;
+import org.folio.bulkops.domain.bean.AssembleStorageFileRequestBody;
+import org.folio.bulkops.domain.bean.SplitStatus;
 import org.folio.bulkops.domain.bean.UploadFileDefinition;
 import org.folio.bulkops.domain.bean.UploadFileDefinitionProcessFiles;
+import org.folio.bulkops.domain.bean.UploadUrlResponse;
 import org.springframework.cloud.openfeign.FeignClient;
 import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
 
 @FeignClient(name = "data-import", configuration = FeignClientConfiguration.class)
 public interface DataImportClient {
+  @GetMapping(value = "/splitStatus", produces = MediaType.APPLICATION_JSON_VALUE)
+  SplitStatus getSplitStatus();
+
+  @GetMapping(value = "/uploadUrl", produces = MediaType.APPLICATION_JSON_VALUE)
+  UploadUrlResponse getUploadUrl(@RequestParam("filename") String filename);
 
   @PostMapping(value = "/uploadDefinitions", produces = MediaType.APPLICATION_JSON_VALUE)
   UploadFileDefinition uploadFileDefinitions(@RequestBody UploadFileDefinition uploadFileDefinition);
 
-  @PostMapping(value = "/uploadDefinitions/{uploadDefinitionId}/files/{fileId}")
-  void uploadFileDefinitionsFiles(@RequestBody UploadFileDefinition uploadFileDefinition, @PathVariable String uploadDefinitionId, @PathVariable String fileId);
+  @GetMapping(value = "/uploadDefinitions/{uploadDefinitionId}", produces = MediaType.APPLICATION_JSON_VALUE)
+  UploadFileDefinition getUploadDefinitionById(@PathVariable String uploadDefinitionId);
 
   @PostMapping(value = "/uploadDefinitions/{uploadDefinitionId}/processFiles")
   void uploadFileDefinitionsProcessFiles(@RequestBody UploadFileDefinitionProcessFiles uploadFileDefinitionProcessFiles, @PathVariable String uploadDefinitionId);
+
+  @PostMapping(value = "/uploadDefinitions/{uploadDefinitionId}/files/{fileId}/assembleStorageFile")
+  void assembleStorageFile(@PathVariable String uploadDefinitionId, @PathVariable String fileId, @RequestBody AssembleStorageFileRequestBody body);
 }

src/main/java/org/folio/bulkops/client/DataImportProfilesClient.java

@@ -2,10 +2,14 @@
 
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import org.folio.bulkops.configs.FeignClientConfiguration;
+import org.folio.bulkops.domain.bean.ActionProfile;
 import org.folio.bulkops.domain.bean.ActionProfilePost;
+import org.folio.bulkops.domain.bean.JobProfile;
 import org.folio.bulkops.domain.bean.JobProfileCollection;
 import org.folio.bulkops.domain.bean.JobProfilePost;
+import org.folio.bulkops.domain.bean.MappingProfile;
 import org.folio.bulkops.domain.bean.MappingProfilePost;
+import org.folio.bulkops.domain.bean.MatchProfile;
 import org.folio.bulkops.domain.bean.MatchProfilePost;
 import org.springframework.cloud.openfeign.FeignClient;
 import org.springframework.http.MediaType;
@@ -19,14 +23,14 @@ public interface DataImportProfilesClient {
   JobProfileCollection getJobProfiles();
 
   @PostMapping(value = "/jobProfiles")
-  JobProfilePost createJobProfile(@RequestBody JobProfilePost jobProfilePost);
+  JobProfile createJobProfile(@RequestBody JobProfilePost jobProfilePost);
 
   @PostMapping(value = "/matchProfiles")
-  MatchProfilePost createMatchProfile(@RequestBody MatchProfilePost matchProfilePost);
+  MatchProfile createMatchProfile(@RequestBody MatchProfilePost matchProfilePost);
 
   @PostMapping(value = "/actionProfiles")
-  ActionProfilePost createActionProfile(@RequestBody ActionProfilePost actionProfilePost);
+  ActionProfile createActionProfile(@RequestBody ActionProfilePost actionProfilePost);
 
   @PostMapping(value = "/mappingProfiles")
-  MappingProfilePost createMappingProfile(@RequestBody MappingProfilePost mappingProfilePost);
+  MappingProfile createMappingProfile(@RequestBody MappingProfilePost mappingProfilePost);
 }

src/main/java/org/folio/bulkops/client/DataImportRestS3UploadClient.java

@@ -0,0 +1,16 @@
+package org.folio.bulkops.client;
+
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Component;
+import org.springframework.web.client.RestTemplate;
+
+import java.net.URI;
+
+@Component
+public class DataImportRestS3UploadClient {
+  public ResponseEntity<String> uploadFile(String presignedUrl, byte[] fileContent) {
+    return new RestTemplate().exchange(URI.create(presignedUrl), HttpMethod.PUT, new HttpEntity<>(fileContent), String.class);
+  }
+}

src/main/java/org/folio/bulkops/client/DataImportUploadClient.java

@@ -0,0 +1,19 @@
+package org.folio.bulkops.client;
+
+import org.folio.bulkops.configs.DataImportFeignConfig;
+import org.folio.bulkops.domain.bean.UploadFileDefinition;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+
+@FeignClient(name = "data-import-upload", url = "http://data-import", configuration = DataImportFeignConfig.class)
+public interface DataImportUploadClient {
+
+  @PostMapping(value = "/uploadDefinitions/{uploadDefinitionId}/files/{fileId}",
+    consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE,
+    headers = {"Content-Type: application/octet-stream"})
+  UploadFileDefinition uploadFileDefinitionsFiles(@PathVariable String uploadDefinitionId, @PathVariable String fileId, @RequestBody byte[] bytes);
+
+}

src/main/java/org/folio/bulkops/client/MetadataProviderClient.java

@@ -1,20 +1,26 @@
 package org.folio.bulkops.client;
 
 import org.folio.bulkops.configs.FeignClientConfiguration;
-import org.folio.bulkops.domain.bean.JobExecutionCollection;
 import org.folio.bulkops.domain.bean.JobLogEntryCollection;
 import org.folio.bulkops.domain.bean.JournalRecordCollection;
+import org.folio.bulkops.domain.dto.DataImportJobExecutionCollection;
 import org.springframework.cloud.openfeign.FeignClient;
 import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestParam;
 
+import java.util.UUID;
+
 @FeignClient(name = "metadata-provider", configuration = FeignClientConfiguration.class)
 public interface MetadataProviderClient {
 
   @GetMapping(value = "/jobExecutions", produces = MediaType.APPLICATION_JSON_VALUE)
-  JobExecutionCollection getJobExecutions();
+  DataImportJobExecutionCollection getJobExecutions(@RequestParam long limit);
+
+  @GetMapping(value = "/jobExecutions", produces = MediaType.APPLICATION_JSON_VALUE)
+  DataImportJobExecutionCollection getJobExecutionsByJobProfileIdAndSubordinationType(@RequestParam("profileIdAny") UUID profileId,
+    @RequestParam("subordinationTypeNotAny") String subordinationTypeToExclude);
 
   @GetMapping(value = "journalRecords/{jobExecutionId}", produces = MediaType.APPLICATION_JSON_VALUE)
   JournalRecordCollection getJournalRecords(@PathVariable String jobExecutionId);

src/main/java/org/folio/bulkops/configs/DataImportFeignConfig.java

@@ -0,0 +1,12 @@
+package org.folio.bulkops.configs;
+
+import feign.codec.Encoder;
+import feign.form.FormEncoder;
+import org.springframework.context.annotation.Bean;
+
+public class DataImportFeignConfig {
+  @Bean
+  public Encoder feignEncoder() {
+    return new FormEncoder();
+  }
+}

src/main/java/org/folio/bulkops/controller/BulkOperationController.java

@@ -154,7 +154,9 @@ public ResponseEntity<Resource> downloadFileByOperationId(
         bulkOperation.getLinkToModifiedRecordsMarcFile() :
         bulkOperation.getLinkToModifiedRecordsCsvFile();
     } else if (fileContentType == COMMITTED_RECORDS_FILE) {
-      path = bulkOperation.getLinkToCommittedRecordsCsvFile();
+      path = INSTANCE_MARC.equals(bulkOperation.getEntityType()) ?
+        bulkOperation.getLinkToCommittedRecordsMarcFile() :
+        bulkOperation.getLinkToCommittedRecordsCsvFile();
     } else if (fileContentType == COMMITTING_CHANGES_ERROR_FILE) {
       path = bulkOperation.getLinkToCommittedRecordsErrorsCsvFile();
     } else {

src/main/java/org/folio/bulkops/domain/bean/AssembleStorageFileRequestBody.java

@@ -0,0 +1,5 @@
+package org.folio.bulkops.domain.bean;
+
+import java.util.List;
+
+public record AssembleStorageFileRequestBody(String uploadId, String key, List<String> tags) {}

src/main/java/org/folio/bulkops/domain/bean/FileDefinition.java

@@ -24,4 +24,10 @@ public class FileDefinition {
 
   @JsonProperty("name")
   private String name;
+
+  @JsonProperty("sourcePath")
+  private String sourcePath;
+
+  @JsonProperty("status")
+  private String status;
 }

src/main/java/org/folio/bulkops/domain/bean/JobProfile.java

@@ -25,4 +25,7 @@ public class JobProfile {
 
   @JsonProperty("dataType")
   private String dataType;
+
+  @JsonProperty("hidden")
+  private Boolean hidden;
 }

src/main/java/org/folio/bulkops/domain/bean/JobProfileInfo.java

@@ -16,4 +16,7 @@ public class JobProfileInfo {
 
   @JsonProperty("id")
   private String id;
+
+  @JsonProperty("dataType")
+  private String dataType;
 }

src/main/java/org/folio/bulkops/domain/bean/SplitStatus.java

@@ -0,0 +1,17 @@
+package org.folio.bulkops.domain.bean;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class SplitStatus {
+
+  @JsonProperty("splitStatus")
+  private Boolean splitStatus;
+}

src/main/java/org/folio/bulkops/domain/bean/UploadFileDefinition.java

@@ -24,4 +24,7 @@ public class UploadFileDefinition {
 
   @JsonProperty("fileDefinitions")
   private List<FileDefinition> fileDefinitions = new ArrayList<>();
+
+  @JsonProperty("metadata")
+  private Metadata metadata;
 }

src/main/java/org/folio/bulkops/domain/bean/UploadFileDefinitionProcessFiles.java

@@ -14,7 +14,7 @@
 @AllArgsConstructor
 public class UploadFileDefinitionProcessFiles {
 
-  @JsonProperty("uploadFileDefinition")
+  @JsonProperty("uploadDefinition")
   private UploadFileDefinition uploadFileDefinition;
 
   @JsonProperty("jobProfileInfo")

src/main/java/org/folio/bulkops/domain/bean/UploadUrlResponse.java

@@ -0,0 +1,22 @@
+package org.folio.bulkops.domain.bean;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class UploadUrlResponse {
+  @JsonProperty("url")
+  private String url;
+
+  @JsonProperty("key")
+  private String key;
+
+  @JsonProperty("uploadId")
+  private String uploadId;
+}

src/main/java/org/folio/bulkops/domain/entity/BulkOperation.java

@@ -92,4 +92,6 @@ public class BulkOperation {
   @Column(columnDefinition = "jsonb[]")
   private List<TenantNotePair> tenantNotePairs;
   private List<String> usedTenants;
+
+  private UUID dataImportJobProfileId;
 }

src/main/java/org/folio/bulkops/processor/InstanceUpdateProcessor.java → src/main/java/org/folio/bulkops/processor/FolioInstanceUpdateProcessor.java

@@ -8,6 +8,7 @@
 import static org.folio.bulkops.util.Constants.APPLY_TO_ITEMS;
 import static org.folio.bulkops.util.Constants.GET_HOLDINGS_BY_INSTANCE_ID_QUERY;
 import static org.folio.bulkops.util.Constants.GET_ITEMS_BY_HOLDING_ID_QUERY;
+import static org.folio.bulkops.util.Constants.MARC;
 import static org.folio.bulkops.util.Constants.MSG_NO_CHANGE_REQUIRED;
 import static org.folio.bulkops.util.FolioExecutionContextUtil.prepareContextForTenant;
 import static org.folio.bulkops.util.RuleUtils.fetchParameters;
@@ -45,7 +46,7 @@
 @Component
 @RequiredArgsConstructor
 @Slf4j
-public class InstanceUpdateProcessor extends AbstractUpdateProcessor<ExtendedInstance> {
+public class FolioInstanceUpdateProcessor extends AbstractUpdateProcessor<ExtendedInstance> {
   private static final String ERROR_MESSAGE_TEMPLATE = "No change in value for instance required, %s associated records have been updated.";
   private static final String ERROR_NO_AFFILIATION_TO_EDIT_HOLDINGS = "User %s does not have required affiliation to edit the holdings record - %s on the tenant %s";
 
@@ -169,7 +170,7 @@ private String buildErrorMessage(boolean recordsUpdated, boolean newValue) {
   private List<HoldingsRecord> getHoldingsSourceFolioByInstanceId(String instanceId) {
     return holdingsClient.getByQuery(format(GET_HOLDINGS_BY_INSTANCE_ID_QUERY, instanceId), Integer.MAX_VALUE)
       .getHoldingsRecords().stream()
-      .filter(holdingsRecord -> !"MARC".equals(holdingsReferenceService.getSourceById(holdingsRecord.getSourceId(), folioExecutionContext.getTenantId()).getName()))
+      .filter(holdingsRecord -> !MARC.equals(holdingsReferenceService.getSourceById(holdingsRecord.getSourceId(), folioExecutionContext.getTenantId()).getName()))
       .toList();
   }
 

src/main/java/org/folio/bulkops/processor/HoldingsDataProcessor.java

@@ -17,6 +17,7 @@
 import static org.folio.bulkops.domain.dto.UpdateOptionType.PERMANENT_LOCATION;
 import static org.folio.bulkops.domain.dto.UpdateOptionType.SUPPRESS_FROM_DISCOVERY;
 import static org.folio.bulkops.domain.dto.UpdateOptionType.TEMPORARY_LOCATION;
+import static org.folio.bulkops.util.Constants.MARC;
 import static org.folio.bulkops.util.Constants.RECORD_CANNOT_BE_UPDATED_ERROR_TEMPLATE;
 
 import java.util.ArrayList;
@@ -61,7 +62,7 @@ public class HoldingsDataProcessor extends AbstractDataProcessor<ExtendedHolding
   public Validator<UpdateOptionType, Action, BulkOperationRule> validator(ExtendedHoldingsRecord extendedHoldingsRecord) {
     return (option, action, rule) -> {
       try {
-        if ("MARC".equals(holdingsReferenceService.getSourceById(extendedHoldingsRecord.getEntity().getSourceId(), folioExecutionContext.getTenantId()).getName())) {
+        if (MARC.equals(holdingsReferenceService.getSourceById(extendedHoldingsRecord.getEntity().getSourceId(), folioExecutionContext.getTenantId()).getName())) {
           throw new RuleValidationException("Holdings records that have source \"MARC\" cannot be changed");
         }
       } catch (NotFoundException e) {