diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java index c47f019..9560fd3 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java @@ -20,8 +20,11 @@ import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit; import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDepositReader; import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDepositReaderImpl; +import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; import nl.knaw.dans.dvingest.core.dansbag.exception.InvalidDepositException; import nl.knaw.dans.dvingest.core.dansbag.mapper.DepositToDvDatasetMetadataMapper; +import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.FileElement; +import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import nl.knaw.dans.dvingest.core.dansbag.xml.XmlReader; import nl.knaw.dans.dvingest.core.dansbag.xml.XmlReaderImpl; import nl.knaw.dans.dvingest.core.service.DataverseService; @@ -42,9 +45,17 @@ import java.nio.file.Path; import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; +import java.util.stream.Collectors; @Slf4j public class DansBagMappingServiceImpl implements DansBagMappingService { @@ -133,7 +144,9 @@ public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit) { @Override public EditFiles getEditFilesFromDansDeposit(DansBagDeposit dansDeposit) { - return new EditFilesComposer(dansDeposit, fileExclusionPattern, embargoExclusions).composeEditFiles(); + var files = getFileInfo(dansDeposit); + var dateAvailable = getDateAvailable(dansDeposit); + return new EditFilesComposer(files, dateAvailable, fileExclusionPattern, embargoExclusions).composeEditFiles(); } @Override @@ -156,6 +169,49 @@ public String packageOriginalMetadata(DansBagDeposit dansDeposit) throws IOExcep return zipFile.toString(); } + // todo: move to mapping package + private Map getFileInfo(DansBagDeposit dansDeposit) { + var files = FileElement.pathToFileInfo(dansDeposit, false); // TODO: handle migration case + + return files.entrySet().stream() + .map(entry -> { + // relativize the path + var bagPath = entry.getKey(); + var fileInfo = entry.getValue(); + var newKey = Path.of("data").relativize(bagPath); + + return Map.entry(newKey, fileInfo); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + // todo: move to mapping package + private Instant getDateAvailable(DansBagDeposit dansBagDeposit) { + return XPathEvaluator.strings(dansBagDeposit.getDdm(), "/ddm:DDM/ddm:profile/ddm:available") + .map(DansBagMappingServiceImpl::parseDate) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Deposit without a ddm:available element")); + } + + // todo: move to util class + private static Instant parseDate(String value) { + try { + log.debug("Trying to parse {} as LocalDate", value); + return LocalDate.parse(value).atStartOfDay(ZoneId.systemDefault()).toInstant(); + } + catch (DateTimeParseException e) { + try { + log.debug("Trying to parse {} as ZonedDateTime", value); + return ZonedDateTime.parse(value).toInstant(); + } + catch (DateTimeParseException ee) { + log.debug("Trying to parse {} as LocalDateTime", value); + var id = ZoneId.systemDefault().getRules().getOffset(Instant.now()); + return LocalDateTime.parse(value).toInstant(id); + } + } + } + Optional getDateOfDeposit(DansBagDeposit dansDeposit) { if (dansDeposit.isUpdate()) { return Optional.empty(); // See for implementation CIT025B in DatasetUpdater diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java index d13109e..36cf08e 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java @@ -16,12 +16,10 @@ package nl.knaw.dans.dvingest.core.dansbag; import lombok.AllArgsConstructor; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import nl.knaw.dans.dvingest.core.bagprocessor.DataversePath; -import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit; import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; -import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.FileElement; -import nl.knaw.dans.dvingest.core.dansbag.xml.XPathEvaluator; import nl.knaw.dans.dvingest.core.yaml.AddEmbargo; import nl.knaw.dans.dvingest.core.yaml.EditFiles; import nl.knaw.dans.dvingest.core.yaml.FromTo; @@ -30,11 +28,6 @@ import java.nio.file.Path; import java.text.SimpleDateFormat; import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeParseException; import java.util.Date; import java.util.List; import java.util.Map; @@ -48,15 +41,20 @@ @Slf4j @AllArgsConstructor public class EditFilesComposer { - protected final DansBagDeposit dansDeposit; - private final Pattern fileExclusionPattern; - private final List embargoExclusions; - private static final SimpleDateFormat yyyymmddFormat = new SimpleDateFormat("yyyy-MM-dd"); + protected static final SimpleDateFormat yyyymmddFormat = new SimpleDateFormat("yyyy-MM-dd"); + + @NonNull + protected final Map files; + @NonNull + protected final Instant dateAvailable; + + protected final Pattern fileExclusionPattern; + @NonNull + protected final List embargoExclusions; public EditFiles composeEditFiles() { - var pathFileInfoMap = getFileInfo(dansDeposit); + var pathFileInfoMap = files; var renamedFiles = getAutoRenameMap(pathFileInfoMap); - init(renamedFiles); var ignoredFiles = getFilesToIgnore(pathFileInfoMap); var editFiles = new EditFiles(); @@ -66,11 +64,7 @@ public EditFiles composeEditFiles() { editFiles.setAutoRenameFiles(getAutoRenamedFiles(renamedFiles)); editFiles.setAddRestrictedFiles(getRestrictedFilesToAdd(pathFileInfoMap)); editFiles.setUpdateFileMetas(getUpdatedFileMetas(pathFileInfoMap)); - editFiles.setDeleteFiles(getDeleteFiles(pathFileInfoMap)); - editFiles.setMoveFiles(getFileMovements(pathFileInfoMap)); - editFiles.setReplaceFiles(getReplacedFiles(pathFileInfoMap)); - var dateAvailable = getDateAvailable(dansDeposit); var filePathsToEmbargo = getEmbargoedFiles(pathFileInfoMap, dateAvailable); if (!filePathsToEmbargo.isEmpty()) { var addEmbargo = new AddEmbargo(); @@ -81,14 +75,6 @@ public EditFiles composeEditFiles() { return editFiles; } - protected List getReplacedFiles(Map pathFileInfoMap) { - return List.of(); - } - - protected void init(Map renamedFiles) { - // do nothing - } - /** * Get the files that should not be processed by the ingest service. * @@ -130,26 +116,6 @@ protected List getUpdatedFileMetas(Map files) { .toList(); } - /** - * Get the files that should be deleted. - * - * @param files the file infos found in files.xml - * @return a list of file paths that should be deleted - */ - protected List getDeleteFiles(Map files) { - return List.of(); - } - - /** - * Get the files that should be moved. - * - * @param files the file infos found in files.xml - * @return a list of FromTo objects that specify the files to move and their new location - */ - protected List getFileMovements(Map files) { - return List.of(); - } - private List getEmbargoedFiles(Map files, Instant dateAvailable) { var now = Instant.now(); if (dateAvailable.isAfter(now)) { @@ -162,21 +128,6 @@ private List getEmbargoedFiles(Map files, Instant dateAvai } } - protected Map getFileInfo(DansBagDeposit dansDeposit) { - var files = FileElement.pathToFileInfo(dansDeposit, false); // TODO: handle migration case - - return files.entrySet().stream() - .map(entry -> { - // relativize the path - var bagPath = entry.getKey(); - var fileInfo = entry.getValue(); - var newKey = Path.of("data").relativize(bagPath); - - return Map.entry(newKey, fileInfo); - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - private boolean hasAttributes(FileMeta fileMeta) { return (fileMeta.getCategories() != null && !fileMeta.getCategories().isEmpty()) || (fileMeta.getDescription() != null && !fileMeta.getDescription().isBlank()); @@ -200,31 +151,4 @@ protected Map getAutoRenameMap(Map files) { .collect(Collectors.toMap(entry -> entry.getKey().toString(), entry -> new DataversePath(entry.getValue().getMetadata().getDirectoryLabel(), entry.getValue().getMetadata().getLabel()).toString())); } - - // TODO: move to mapping package - private Instant getDateAvailable(DansBagDeposit dansBagDeposit) { - return XPathEvaluator.strings(dansBagDeposit.getDdm(), "/ddm:DDM/ddm:profile/ddm:available") - .map(EditFilesComposer::parseDate) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Deposit without a ddm:available element")); - } - - private static Instant parseDate(String value) { - try { - log.debug("Trying to parse {} as LocalDate", value); - return LocalDate.parse(value).atStartOfDay(ZoneId.systemDefault()).toInstant(); - } - catch (DateTimeParseException e) { - try { - log.debug("Trying to parse {} as ZonedDateTime", value); - return ZonedDateTime.parse(value).toInstant(); - } - catch (DateTimeParseException ee) { - log.debug("Trying to parse {} as LocalDateTime", value); - var id = ZoneId.systemDefault().getRules().getOffset(Instant.now()); - return LocalDateTime.parse(value).toInstant(id); - } - } - } - } diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java index fd31d33..2e437f5 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java @@ -17,7 +17,6 @@ import lombok.extern.slf4j.Slf4j; import nl.knaw.dans.dvingest.core.bagprocessor.FilesInDatasetCache; -import nl.knaw.dans.dvingest.core.dansbag.deposit.DansBagDeposit; import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; import nl.knaw.dans.dvingest.core.service.DataverseService; import nl.knaw.dans.dvingest.core.yaml.EditFiles; @@ -27,6 +26,7 @@ import java.io.IOException; import java.nio.file.Path; +import java.time.Instant; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -42,15 +42,16 @@ public class EditFilesComposerForUpdate extends EditFilesComposer { private final String updatesDatasetPid; private final DataverseService dataverseService; - public EditFilesComposerForUpdate(DansBagDeposit dansDeposit, String updatesDatasetPid, Pattern fileExclusionPattern, List embargoExclusions, DataverseService dataverseService) { - super(dansDeposit, fileExclusionPattern, embargoExclusions); + public EditFilesComposerForUpdate(Map files, Instant dateAvailable, String updatesDatasetPid, Pattern fileExclusionPattern, List embargoExclusions, + DataverseService dataverseService) { + super(files, dateAvailable, fileExclusionPattern, embargoExclusions); this.updatesDatasetPid = updatesDatasetPid; this.dataverseService = dataverseService; } @Override public EditFiles composeEditFiles() { - var pathFileInfoMap = getFileInfo(dansDeposit); + var pathFileInfoMap = files; var renamedFiles = getAutoRenameMap(pathFileInfoMap); // TODO: this should be a read-only variant of the cache FilesInDatasetCache filesInDatasetCache = new FilesInDatasetCache(dataverseService, renamedFiles); diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdateTest.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdateTest.java new file mode 100644 index 0000000..570e8ce --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdateTest.java @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +public class EditFilesComposerForUpdateTest { +} diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerTest.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerTest.java new file mode 100644 index 0000000..f587324 --- /dev/null +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerTest.java @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2024 DANS - Data Archiving and Networked Services (info@dans.knaw.nl) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nl.knaw.dans.dvingest.core.dansbag; + +public class EditFilesComposerTest { +}