Skip to content

Commit

Permalink
Merge branch 'master' into MODBULKOPS-383
Browse files Browse the repository at this point in the history
  • Loading branch information
@alekGbuz
alekGbuz authored Nov 11, 2024
2 parents 552b7c0 + d112954 commit 54b92d2
Show file tree
Hide file tree
Showing 15 changed files with 241 additions and 106 deletions.
8 changes: 8 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -549,4 +549,12 @@
<tag>HEAD</tag>
</scm>

<pluginRepositories>
<pluginRepository>
<id>folio-nexus</id>
<name>FOLIO Maven repository</name>
<url>https://repository.folio.org/repository/maven-folio</url>
</pluginRepository>
</pluginRepositories>

</project>
15 changes: 10 additions & 5 deletions src/main/java/org/folio/bulkops/domain/bean/Instance.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ public class Instance implements BulkOperationsEntity {
public static final String INSTANCE_PUBLICATION_RANGE = "Publication range";
public static final String INSTANCE_ADMINISTRATIVE_NOTE = "Administrative note";
public static final String INSTANCE_STAFF_SUPPRESS = "Staff suppress";
public static final String INSTANCE_SUPPRESS_FROM_DISCOVERY = "Suppress from discovery";
public static final String INSTANCE_PREVIOUSLY_HELD = "Previously held";
public static final String INSTANCE_CATALOGED_DATE = "Cataloged date";
public static final String INSTANCE_STATUS_TERM = "Instance status term";
public static final String INSTANCE_NATURE_OF_CONTENT = "Nature of content";

@JsonProperty("id")
@CsvCustomBindByName(column = INSTANCE_UUID, converter = StringConverter.class)
Expand All @@ -65,7 +70,7 @@ public class Instance implements BulkOperationsEntity {
private Integer version;

@JsonProperty("discoverySuppress")
@CsvCustomBindByName(column = "Suppress from discovery", converter = BooleanConverter.class)
@CsvCustomBindByName(column = INSTANCE_SUPPRESS_FROM_DISCOVERY, converter = BooleanConverter.class)
@CsvCustomBindByPosition(position = 1, converter = BooleanConverter.class)
@UnifiedTableCell
private Boolean discoverySuppress;
Expand All @@ -77,7 +82,7 @@ public class Instance implements BulkOperationsEntity {
private Boolean staffSuppress;

@JsonProperty("previouslyHeld")
@CsvCustomBindByName(column = "Previously held", converter = BooleanConverter.class)
@CsvCustomBindByName(column = INSTANCE_PREVIOUSLY_HELD, converter = BooleanConverter.class)
@CsvCustomBindByPosition(position = 3, converter = BooleanConverter.class)
@UnifiedTableCell(visible = false)
private Boolean previouslyHeld;
Expand All @@ -95,13 +100,13 @@ public class Instance implements BulkOperationsEntity {
private String source;

@JsonProperty("catalogedDate")
@CsvCustomBindByName(column = "Cataloged date", converter = DateWithoutTimeConverter.class)
@CsvCustomBindByName(column = INSTANCE_CATALOGED_DATE, converter = DateWithoutTimeConverter.class)
@CsvCustomBindByPosition(position = 6, converter = DateWithoutTimeConverter.class)
@UnifiedTableCell(dataType = DataType.DATE_TIME, visible = false)
private Date catalogedDate;

@JsonProperty("statusId")
@CsvCustomBindByName(column = "Instance status term", converter = InstanceStatusConverter.class)
@CsvCustomBindByName(column = INSTANCE_STATUS_TERM, converter = InstanceStatusConverter.class)
@CsvCustomBindByPosition(position = 7, converter = InstanceStatusConverter.class)
@UnifiedTableCell
private String statusId;
Expand Down Expand Up @@ -161,7 +166,7 @@ public class Instance implements BulkOperationsEntity {
private String instanceTypeId;

@JsonProperty("natureOfContentTermIds")
@CsvCustomBindByName(column = "Nature of content", converter = NatureOfContentTermListConverter.class)
@CsvCustomBindByName(column = INSTANCE_NATURE_OF_CONTENT, converter = NatureOfContentTermListConverter.class)
@CsvCustomBindByPosition(position = 17, converter = NatureOfContentTermListConverter.class)
@UnifiedTableCell(visible = false)
private List<String> natureOfContentTermIds;
Expand Down
68 changes: 38 additions & 30 deletions src/main/java/org/folio/bulkops/service/BulkOperationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,45 +335,53 @@ private void confirmForInstanceMarc(BulkOperation operation) {
.build());

var processedNumOfRecords = 0;
var triggeringFileName = FilenameUtils.getBaseName(operation.getLinkToTriggeringCsvFile());
var modifiedMarcFileName = String.format(PREVIEW_MARC_PATH_TEMPLATE, operationId, LocalDate.now(), triggeringFileName);
try (var writerForModifiedPreviewMarcFile = remoteFileSystemClient.marcWriter(modifiedMarcFileName)) {
var matchedRecordsReader = new MarcStreamReader(remoteFileSystemClient.get(operation.getLinkToMatchedRecordsMarcFile()));
var currentDate = new Date();
while (matchedRecordsReader.hasNext()) {
var marcRecord = matchedRecordsReader.next();
marcInstanceDataProcessor.update(operation, marcRecord, ruleCollection, currentDate);
writerForModifiedPreviewMarcFile.writeRecord(marcRecord);
if (nonNull(operation.getLinkToMatchedRecordsMarcFile())) {
var triggeringFileName = FilenameUtils.getBaseName(operation.getLinkToTriggeringCsvFile());
var modifiedMarcFileName = String.format(PREVIEW_MARC_PATH_TEMPLATE, operationId, LocalDate.now(), triggeringFileName);
try (var writerForModifiedPreviewMarcFile = remoteFileSystemClient.marcWriter(modifiedMarcFileName)) {
var matchedRecordsReader = new MarcStreamReader(remoteFileSystemClient.get(operation.getLinkToMatchedRecordsMarcFile()));
var currentDate = new Date();
while (matchedRecordsReader.hasNext()) {
var marcRecord = matchedRecordsReader.next();
marcInstanceDataProcessor.update(operation, marcRecord, ruleCollection, currentDate);
writerForModifiedPreviewMarcFile.writeRecord(marcRecord);

processedNumOfRecords++;
dataProcessing = dataProcessing
.withStatus(matchedRecordsReader.hasNext() ? StatusType.ACTIVE : StatusType.COMPLETED)
.withEndTime(matchedRecordsReader.hasNext() ? null : LocalDateTime.now());
processedNumOfRecords++;
dataProcessing = dataProcessing
.withStatus(matchedRecordsReader.hasNext() ? StatusType.ACTIVE : StatusType.COMPLETED)
.withEndTime(matchedRecordsReader.hasNext() ? null : LocalDateTime.now());

if (processedNumOfRecords - dataProcessing.getProcessedNumOfRecords() > OPERATION_UPDATING_STEP) {
dataProcessing.setProcessedNumOfRecords(processedNumOfRecords);
dataProcessingRepository.save(dataProcessing);
if (processedNumOfRecords - dataProcessing.getProcessedNumOfRecords() > OPERATION_UPDATING_STEP) {
dataProcessing.setProcessedNumOfRecords(processedNumOfRecords);
dataProcessingRepository.save(dataProcessing);
}
}
operation.setLinkToModifiedRecordsMarcFile(modifiedMarcFileName);
dataProcessing.setProcessedNumOfRecords(processedNumOfRecords);
dataProcessingRepository.save(dataProcessing);

operation.setApproach(IN_APP);
operation.setStatus(OperationStatusType.REVIEW_CHANGES);
operation.setProcessedNumOfRecords(processedNumOfRecords);
bulkOperationRepository.findById(operation.getId()).ifPresent(op -> operation.setCommittedNumOfErrors(op.getCommittedNumOfErrors()));
} catch (Exception e) {
log.error(e);
dataProcessingRepository.save(dataProcessing
.withStatus(StatusType.FAILED)
.withEndTime(LocalDateTime.now()));
operation.setStatus(OperationStatusType.FAILED);
operation.setEndTime(LocalDateTime.now());
operation.setErrorMessage(ERROR_NOT_CONFIRM_CHANGES_S3_ISSUE);
}
operation.setLinkToModifiedRecordsMarcFile(modifiedMarcFileName);
dataProcessing.setProcessedNumOfRecords(processedNumOfRecords);
dataProcessingRepository.save(dataProcessing);

operation.setApproach(IN_APP);
operation.setStatus(OperationStatusType.REVIEW_CHANGES);
operation.setProcessedNumOfRecords(processedNumOfRecords);
bulkOperationRepository.findById(operation.getId()).ifPresent(op -> operation.setCommittedNumOfErrors(op.getCommittedNumOfErrors()));
} catch (Exception e) {
log.error(e);
} else {
log.error("No link to MARC file, failing operation");
dataProcessingRepository.save(dataProcessing
.withStatus(StatusType.FAILED)
.withEndTime(LocalDateTime.now()));
operation.setStatus(OperationStatusType.FAILED);
operation.setStatus(OperationStatusType.REVIEWED_NO_MARC_RECORDS);
operation.setEndTime(LocalDateTime.now());
operation.setErrorMessage(ERROR_NOT_CONFIRM_CHANGES_S3_ISSUE);
} finally {
bulkOperationRepository.save(operation);
}
bulkOperationRepository.save(operation);
}

public void writeBeanToCsv(BulkOperation operation, BulkOperationsEntityCsvWriter csvWriter, BulkOperationsEntity bean, List<BulkOperationExecutionContent> bulkOperationExecutionContents) throws CsvRequiredFieldEmptyException, CsvDataTypeMismatchException {
Expand Down
25 changes: 11 additions & 14 deletions src/main/java/org/folio/bulkops/service/ErrorService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@
import static org.folio.bulkops.domain.dto.OperationStatusType.COMPLETED;
import static org.folio.bulkops.domain.dto.OperationStatusType.COMPLETED_WITH_ERRORS;
import static org.folio.bulkops.domain.dto.OperationStatusType.DATA_MODIFICATION;
import static org.folio.bulkops.domain.dto.OperationStatusType.REVIEWED_NO_MARC_RECORDS;
import static org.folio.bulkops.domain.dto.OperationStatusType.REVIEW_CHANGES;
import static org.folio.bulkops.util.Constants.MSG_NO_CHANGE_REQUIRED;

import java.io.ByteArrayInputStream;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
Expand All @@ -25,7 +25,6 @@
import org.folio.bulkops.client.BulkEditClient;
import org.folio.bulkops.client.MetadataProviderClient;
import org.folio.bulkops.client.RemoteFileSystemClient;
import org.folio.bulkops.client.SrsClient;
import org.folio.bulkops.domain.bean.StateType;
import org.folio.bulkops.domain.dto.Error;
import org.folio.bulkops.domain.dto.Errors;
Expand All @@ -41,6 +40,7 @@
import org.folio.spring.data.OffsetRequest;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ObjectUtils;

import lombok.RequiredArgsConstructor;
Expand All @@ -56,7 +56,6 @@ public class ErrorService {
private final BulkOperationExecutionContentRepository executionContentRepository;
private final BulkEditClient bulkEditClient;
private final MetadataProviderClient metadataProviderClient;
private final SrsClient srsClient;

public void saveError(UUID bulkOperationId, String identifier, String errorMessage, String uiErrorMessage, String link) {
if (MSG_NO_CHANGE_REQUIRED.equals(errorMessage) && executionContentRepository.findFirstByBulkOperationIdAndIdentifier(bulkOperationId, identifier).isPresent()) {
Expand Down Expand Up @@ -89,7 +88,7 @@ public void deleteErrorsByBulkOperationId(UUID bulkOperationId) {
public Errors getErrorsPreviewByBulkOperationId(UUID bulkOperationId, int limit) {
var bulkOperation = operationRepository.findById(bulkOperationId)
.orElseThrow(() -> new NotFoundException("BulkOperation was not found by id=" + bulkOperationId));
if (Set.of(DATA_MODIFICATION, REVIEW_CHANGES).contains(bulkOperation.getStatus()) || COMPLETED_WITH_ERRORS == bulkOperation.getStatus() && noCommittedErrors(bulkOperation)) {
if (Set.of(DATA_MODIFICATION, REVIEW_CHANGES, REVIEWED_NO_MARC_RECORDS).contains(bulkOperation.getStatus()) || COMPLETED_WITH_ERRORS == bulkOperation.getStatus() && noCommittedErrors(bulkOperation)) {
var errors = bulkEditClient.getErrorsPreview(bulkOperation.getDataExportJobId(), limit);
return new Errors().errors(errors.getErrors().stream()
.map(this::prepareInternalErrorRepresentation)
Expand All @@ -111,18 +110,16 @@ public void saveErrorsFromDataImport(UUID bulkOperationId, UUID dataImportJobId)
var jobLogEntries = metadataProviderClient.getJobLogEntries(dataImportJobId.toString(), Integer.MAX_VALUE)
.getEntries().stream()
.filter(entry -> nonNull(entry.getError()) && !entry.getError().isEmpty())
.toList();
.toList();
jobLogEntries.forEach(errorEntry -> {
String identifier = EMPTY;
try {
if (identifierType == IdentifierType.ID) {
identifier = srsClient.getSrsRecordById(errorEntry.getSourceRecordId()).getExternalIdsHolder().getInstanceId();
} else if (identifierType == IdentifierType.INSTANCE_HRID) {
identifier = srsClient.getSrsRecordById(errorEntry.getSourceRecordId()).getExternalIdsHolder().getInstanceHrid();
}
} catch (Exception e) {
log.error("Problem with retrieving SRS record {}", errorEntry.getSourceRecordId(), e);
List<String> identifierList = null;
var relatedInstanceInfo = errorEntry.getRelatedInstanceInfo();
if (identifierType == IdentifierType.ID) {
identifierList = relatedInstanceInfo.getIdList();
} else if (identifierType == IdentifierType.INSTANCE_HRID) {
identifierList = relatedInstanceInfo.getHridList();
}
var identifier = CollectionUtils.isEmpty(identifierList) ? null : identifierList.get(0);
saveError(bulkOperationId, identifier, errorEntry.getError());
});
} catch (Exception e) {
Expand Down
Loading

0 comments on commit 54b92d2

Please sign in to comment.