diff --git a/pom.xml b/pom.xml index 0aa6ca0..7fd4616 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,7 @@ nl.knaw.dans dans-dataverse-client-lib - 1.0.2 + 1.0.3-SNAPSHOT nl.knaw.dans diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java index f5d3f7b..b55e326 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java @@ -21,6 +21,7 @@ import nl.knaw.dans.dvingest.core.yaml.EditPermissions; import nl.knaw.dans.lib.dataverse.DataverseException; import nl.knaw.dans.lib.dataverse.model.dataset.Dataset; +import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion; import java.io.IOException; import java.nio.file.Path; @@ -30,7 +31,7 @@ public interface DansBagMappingService { DansBagDeposit readDansDeposit(Path depositDir) throws InvalidDepositException; - Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit); + Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit, DatasetVersion currentMetadata); EditFiles getEditFilesFromDansDeposit(DansBagDeposit dansDeposit, String updatesDataset); diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java index 6bba7cd..5d9cb17 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java @@ -33,6 +33,7 @@ import nl.knaw.dans.lib.dataverse.DataverseException; import nl.knaw.dans.lib.dataverse.model.RoleAssignment; import nl.knaw.dans.lib.dataverse.model.dataset.Dataset; +import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion; import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser; import nl.knaw.dans.lib.util.ZipUtil; import org.apache.commons.lang3.StringUtils; @@ -88,37 +89,39 @@ public String getUpdatesDataset(Path depositDir) throws IOException, DataverseEx .findFirst().orElseThrow(() -> new IllegalArgumentException("No bag found in deposit")); var bagInfo = new LightweightBagInfo(bag.resolve("bag-info.txt")); var isVersionOf = bagInfo.get("Is-Version-Of"); - if (isVersionOf != null) { - log.debug("Found Is-Version-Of in bag-info.txt, so this is an update-deposit: {}", isVersionOf); - List results; - if (dansDepositProperties.getSwordToken() != null) { - log.debug("Found sword token in deposit.properties, looking for target dataset by sword token"); - results = dataverseService.findDoiByMetadataField("dansSwordToken", dansDepositProperties.getSwordToken()); - } - else if (depositToDvDatasetMetadataMapper.isMigration()) { - log.debug("This is a migration deposit, looking for target dataset by dansBagIt. Note that this will only work for two versions of the same dataset"); - results = dataverseService.findDoiByMetadataField("dansBagId", isVersionOf); - - } - else { - throw new IllegalArgumentException("Update deposit should have either a sword token or be a migration deposit"); - } - if (results.size() == 1) { - return results.get(0); - } - else { - throw new IllegalArgumentException("Update deposit should update exactly one dataset, found " + results.size()); - } - } - else { + + if (isVersionOf == null) { log.debug("No Is-Version-Of found in bag-info.txt, so this is a deposit of a new dataset"); return null; } + + log.debug("Found Is-Version-Of in bag-info.txt, so this is an update-deposit: {}", isVersionOf); + List results; + + if (dansDepositProperties.getSwordToken() != null) { + log.debug("Found sword token in deposit.properties, looking for target dataset by sword token"); + results = dataverseService.findDoiByMetadataField("dansSwordToken", dansDepositProperties.getSwordToken()); + } + else if (depositToDvDatasetMetadataMapper.isMigration()) { + log.debug("This is a migration deposit, looking for target dataset by dansBagId"); + results = dataverseService.findDoiByMetadataField("dansBagId", isVersionOf); + } + else { + throw new IllegalArgumentException("Update deposit should have either a sword token or be a migration deposit"); + } + + if (results.size() == 1) { + return results.get(0); + } + else { + throw new IllegalArgumentException("Update deposit should update exactly one dataset, found " + results.size()); + } } } @Override - public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit) { + public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit, DatasetVersion currentMetadata) { + // TODO: rename to DatasetComposer en push the terms stuff into it as well. var dataset = depositToDvDatasetMetadataMapper.toDataverseDataset( dansDeposit.getDdm(), dansDeposit.getOtherDoiId(), @@ -130,10 +133,15 @@ public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit) { dansDeposit.getHasOrganizationalIdentifier(), dansDeposit.getHasOrganizationalIdentifierVersion()); var version = dataset.getDatasetVersion(); - version.setFileAccessRequest(dansDeposit.allowAccessRequests()); - // TODO: when processing an update-deposit, retrieve terms of access from the previous version - if (!dansDeposit.allowAccessRequests() && StringUtils.isBlank(version.getTermsOfAccess())) { + // Allow access requests only if it was not forbidden in previous versions + var fileAccessRequest = dansDeposit.allowAccessRequests() && (currentMetadata == null || currentMetadata.getFileAccessRequest()); + version.setFileAccessRequest(fileAccessRequest); + if (StringUtils.isBlank(version.getTermsOfAccess()) && currentMetadata != null) { + version.setTermsOfAccess(currentMetadata.getTermsOfAccess()); + } + if (!fileAccessRequest && StringUtils.isBlank(version.getTermsOfAccess())) { + // Dataverse requires terms of access to be set if file access requests are disabled, so set it to "N/a" if not explicitly provided version.setTermsOfAccess("N/a"); } version.setLicense(supportedLicenses.getLicenseFromDansDeposit(dansDeposit)); diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java index 7f0d83f..2c4e928 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java @@ -21,6 +21,7 @@ import nl.knaw.dans.dvingest.core.yaml.EditFilesRoot; import nl.knaw.dans.dvingest.core.yaml.EditPermissionsRoot; import nl.knaw.dans.dvingest.core.yaml.UpdateState; +import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion; import java.io.IOException; @@ -28,6 +29,7 @@ public class DansDepositConverter { private final DansBagDeposit dansDeposit; private final String updatesDataset; + private final DatasetVersion currentMetadata; private final DansBagMappingService mappingService; private final YamlService yamlService; @@ -35,7 +37,7 @@ public void run() throws IOException { // TODO: pass to getEditFilesFromDansDeposit so that update-deposit can register it as a replaced file var originalMetadataPath = mappingService.packageOriginalMetadata(dansDeposit); - var dataset = mappingService.getDatasetMetadataFromDansDeposit(dansDeposit); + var dataset = mappingService.getDatasetMetadataFromDansDeposit(dansDeposit, currentMetadata); yamlService.writeYaml(dataset, dansDeposit.getBagDir().resolve("dataset.yml")); var editFiles = mappingService.getEditFilesFromDansDeposit(dansDeposit, updatesDataset); @@ -48,5 +50,4 @@ public void run() throws IOException { updateState.setAction("publish-major"); yamlService.writeYaml(updateState, dansDeposit.getBagDir().resolve("update-state.yml")); } - } diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java index 94c1873..cfc4bd7 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java @@ -29,6 +29,7 @@ import nl.knaw.dans.dvingest.core.service.DataverseService; import nl.knaw.dans.dvingest.core.service.YamlService; import nl.knaw.dans.lib.dataverse.DataverseException; +import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion; import java.io.IOException; import java.nio.file.Path; @@ -49,7 +50,8 @@ public class DansDepositSupport implements Deposit { private DansBagDeposit dansDeposit; - public DansDepositSupport(DataverseIngestDeposit dataverseIngestDeposit, ValidateDansBagService validateDansBagService, DansBagMappingService dansBagMappingService, DataverseService dataverseService, YamlService yamlService) { + public DansDepositSupport(DataverseIngestDeposit dataverseIngestDeposit, ValidateDansBagService validateDansBagService, DansBagMappingService dansBagMappingService, + DataverseService dataverseService, YamlService yamlService) { this.validateDansBagService = validateDansBagService; this.ingestDataverseIngestDeposit = dataverseIngestDeposit; this.dansBagMappingService = dansBagMappingService; @@ -69,11 +71,13 @@ public boolean convertDansDepositIfNeeded() { log.info("Converting deposit to Dataverse ingest metadata"); try { var updatesDataset = dansBagMappingService.getUpdatesDataset(ingestDataverseIngestDeposit.getLocation()); + DatasetVersion currentMetadata = null; if (updatesDataset != null) { ingestDataverseIngestDeposit.updateProperties(Map.of(UPDATES_DATASET_KEY, updatesDataset)); + currentMetadata = dataverseService.getDatasetMetadata(updatesDataset); } dansDeposit = dansBagMappingService.readDansDeposit(ingestDataverseIngestDeposit.getLocation()); - new DansDepositConverter(dansDeposit, updatesDataset, dansBagMappingService, yamlService).run(); + new DansDepositConverter(dansDeposit, updatesDataset, currentMetadata, dansBagMappingService, yamlService).run(); log.info("Conversion successful"); return true; } @@ -119,7 +123,8 @@ public void onSuccess(@NonNull String pid, String message) { "identifier.urn", nbn ) ); - } catch (IOException | DataverseException e) { + } + catch (IOException | DataverseException e) { throw new RuntimeException("Error getting URN:NBN", e); // Cancelling the "success" } } diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java index 8421ecb..b692be8 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java @@ -31,6 +31,7 @@ import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -65,13 +66,8 @@ public EditFiles composeEditFiles() { editFiles.setAddUnrestrictedFiles(getUnrestrictedFilesToAdd(pathFileInfoMap)); editFiles.setUpdateFileMetas(getUpdatedFileMetas(pathFileInfoMap)); - var filePathsToEmbargo = getEmbargoedFiles(pathFileInfoMap, dateAvailable); - if (!filePathsToEmbargo.isEmpty()) { - var addEmbargo = new AddEmbargo(); - addEmbargo.setDateAvailable(yyyymmddFormat.format(Date.from(dateAvailable))); - addEmbargo.setFilePaths(filePathsToEmbargo.stream().map(Path::toString).toList()); - editFiles.setAddEmbargoes(List.of(addEmbargo)); - } + addEmbargo(editFiles, pathFileInfoMap.keySet()); + return editFiles; } @@ -123,10 +119,23 @@ private List getUpdatedFileMetas(Map files) { .toList(); } - private List getEmbargoedFiles(Map files, Instant dateAvailable) { - var now = Instant.now(); - if (dateAvailable.isAfter(now)) { - return files.keySet().stream() + protected void addEmbargo(EditFiles editFiles, Set candidates) { + if (dateAvailable.isAfter(Instant.now())) { + var filesToEmbargo = candidates.stream() + .filter(f -> !embargoExclusions.contains(f.toString())).toList(); + + if (!filesToEmbargo.isEmpty()) { + var addEmbargo = new AddEmbargo(); + addEmbargo.setDateAvailable(yyyymmddFormat.format(Date.from(dateAvailable))); + addEmbargo.setFilePaths(filesToEmbargo.stream().map(Path::toString).toList()); + editFiles.setAddEmbargoes(List.of(addEmbargo)); + } + } + } + + private List getEmbargoedFiles(Set candidates, Instant dateAvailable) { + if (dateAvailable.isAfter(Instant.now())) { + return candidates.stream() .filter(f -> !embargoExclusions.contains(f.toString())).toList(); } else { diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java index c5ce787..34d5e3b 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java @@ -19,6 +19,7 @@ import nl.knaw.dans.dvingest.core.bagprocessor.FilesInDatasetCache; import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo; import nl.knaw.dans.dvingest.core.service.DataverseService; +import nl.knaw.dans.dvingest.core.yaml.AddEmbargo; import nl.knaw.dans.dvingest.core.yaml.EditFiles; import nl.knaw.dans.dvingest.core.yaml.FromTo; import nl.knaw.dans.lib.dataverse.DataverseException; @@ -27,6 +28,7 @@ import java.io.IOException; import java.nio.file.Path; import java.time.Instant; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -144,11 +146,11 @@ public EditFiles composeEditFiles() { editFiles.setAddUnrestrictedFiles(pathsToAdd.stream() .filter(p -> !pathFileInfoMap.get(p).getMetadata().getRestricted()).toList().stream().map(Path::toString).toList()); - // todo: embargoes - + addEmbargo(editFiles, SetUtils.union(pathsToAdd, filesToReplace)); return editFiles; } + private Set getFilesToReplace(Map pathToFileInfo, Map fileReplacementCandidates) { var intersection = SetUtils.intersection(pathToFileInfo.keySet(), fileReplacementCandidates.keySet()); diff --git a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SupportedLicenses.java b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SupportedLicenses.java index 9660d3c..4e25c46 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SupportedLicenses.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/dansbag/SupportedLicenses.java @@ -30,6 +30,7 @@ import static nl.knaw.dans.dvingest.core.dansbag.xml.XPathConstants.DDM_DCMI_METADATA; +// TODO: move to mapping package public class SupportedLicenses { private final Map supportedLicenses; diff --git a/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java b/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java index 9f5d43c..8f8c36c 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java @@ -72,4 +72,6 @@ public interface DataverseService { void addEmbargo(String pid, Embargo embargo) throws IOException, DataverseException; List findDoiByMetadataField(String fieldName, String value) throws IOException, DataverseException; + + DatasetVersion getDatasetMetadata(String pid) throws IOException, DataverseException; } diff --git a/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java b/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java index c4e3cf9..4434189 100644 --- a/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java +++ b/src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java @@ -206,6 +206,11 @@ public String getDatasetUrnNbn(String pid) throws IOException, DataverseExceptio .findFirst().orElseThrow(() -> new IllegalStateException("No URN:NBN found in dataset")); } + @Override + public DatasetVersion getDatasetMetadata(String pid) throws IOException, DataverseException { + return dataverseClient.dataset(pid).getVersion().getData(); + } + // TODO: move this to dans-dataverse-client-lib; it is similar to awaitLockState. public void waitForState(String datasetId, String expectedState) { var numberOfTimesTried = 0; @@ -239,8 +244,8 @@ public void waitForState(String datasetId, String expectedState) { } private String getDatasetState(String datasetId) throws IOException, DataverseException { - var version = dataverseClient.dataset(datasetId).getLatestVersion(); - return version.getData().getLatestVersion().getVersionState(); + var version = dataverseClient.dataset(datasetId).getVersion(Version.LATEST.toString(), true); + return version.getData().getVersionState(); } } diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceTest.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceTest.java index 34176c5..5776e26 100644 --- a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceTest.java +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceTest.java @@ -47,9 +47,4 @@ public void getUpdatesDataset_finds_dataset_for_update_deposit() throws Exceptio // When / Then assertThat(mappingService.getUpdatesDataset(testDepositDir)).isEqualTo(doi); } - - - - - } diff --git a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverterTest.java b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverterTest.java index 7ce4573..102b149 100644 --- a/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverterTest.java +++ b/src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverterTest.java @@ -27,6 +27,9 @@ import static org.assertj.core.api.Assertions.assertThat; +/** + * Test class for {@link DansDepositConverter}. It uses the valid examples from the dd-dans-sword2-examples project. + */ public class DansDepositConverterTest extends DansConversionFixture { private final YamlService yamlService = new YamlServiceImpl(); @@ -44,7 +47,7 @@ public void run_converts_dans_sword_all_mappings_example_to_dataverse_ingest_dep var deposit = dansBagDepositReader.readDeposit(depositDir); // When - new DansDepositConverter(deposit, null, mappingService, yamlService).run(); + new DansDepositConverter(deposit, null, null, mappingService, yamlService).run(); // Then assertThat(deposit.getBagDir().resolve("dataset.yml")).exists();