Remaining update logic

pom.xml

@@ -65,7 +65,7 @@
         <dependency>
             <groupId>nl.knaw.dans</groupId>
             <artifactId>dans-dataverse-client-lib</artifactId>
-            <version>1.0.2</version>
+            <version>1.0.3-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>nl.knaw.dans</groupId>

src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingService.java

@@ -21,6 +21,7 @@
 import nl.knaw.dans.dvingest.core.yaml.EditPermissions;
 import nl.knaw.dans.lib.dataverse.DataverseException;
 import nl.knaw.dans.lib.dataverse.model.dataset.Dataset;
+import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -30,7 +31,7 @@ public interface DansBagMappingService {
 
     DansBagDeposit readDansDeposit(Path depositDir) throws InvalidDepositException;
 
-    Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit);
+    Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit, DatasetVersion currentMetadata);
 
     EditFiles getEditFilesFromDansDeposit(DansBagDeposit dansDeposit, String updatesDataset);
 

src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceImpl.java

@@ -33,6 +33,7 @@
 import nl.knaw.dans.lib.dataverse.DataverseException;
 import nl.knaw.dans.lib.dataverse.model.RoleAssignment;
 import nl.knaw.dans.lib.dataverse.model.dataset.Dataset;
+import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
 import nl.knaw.dans.lib.dataverse.model.user.AuthenticatedUser;
 import nl.knaw.dans.lib.util.ZipUtil;
 import org.apache.commons.lang3.StringUtils;
@@ -88,37 +89,39 @@ public String getUpdatesDataset(Path depositDir) throws IOException, DataverseEx
                 .findFirst().orElseThrow(() -> new IllegalArgumentException("No bag found in deposit"));
             var bagInfo = new LightweightBagInfo(bag.resolve("bag-info.txt"));
             var isVersionOf = bagInfo.get("Is-Version-Of");
-            if (isVersionOf != null) {
-                log.debug("Found Is-Version-Of in bag-info.txt, so this is an update-deposit: {}", isVersionOf);
-                List<String> results;
-                if (dansDepositProperties.getSwordToken() != null) {
-                    log.debug("Found sword token in deposit.properties, looking for target dataset by sword token");
-                    results = dataverseService.findDoiByMetadataField("dansSwordToken", dansDepositProperties.getSwordToken());
-                }
-                else if (depositToDvDatasetMetadataMapper.isMigration()) {
-                    log.debug("This is a migration deposit, looking for target dataset by dansBagIt. Note that this will only work for two versions of the same dataset");
-                    results = dataverseService.findDoiByMetadataField("dansBagId", isVersionOf);
-
-                }
-                else {
-                    throw new IllegalArgumentException("Update deposit should have either a sword token or be a migration deposit");
-                }
-                if (results.size() == 1) {
-                    return results.get(0);
-                }
-                else {
-                    throw new IllegalArgumentException("Update deposit should update exactly one dataset, found " + results.size());
-                }
-            }
-            else {
+
+            if (isVersionOf == null) {
                 log.debug("No Is-Version-Of found in bag-info.txt, so this is a deposit of a new dataset");
                 return null;
             }
+
+            log.debug("Found Is-Version-Of in bag-info.txt, so this is an update-deposit: {}", isVersionOf);
+            List<String> results;
+
+            if (dansDepositProperties.getSwordToken() != null) {
+                log.debug("Found sword token in deposit.properties, looking for target dataset by sword token");
+                results = dataverseService.findDoiByMetadataField("dansSwordToken", dansDepositProperties.getSwordToken());
+            }
+            else if (depositToDvDatasetMetadataMapper.isMigration()) {
+                log.debug("This is a migration deposit, looking for target dataset by dansBagId");
+                results = dataverseService.findDoiByMetadataField("dansBagId", isVersionOf);
+            }
+            else {
+                throw new IllegalArgumentException("Update deposit should have either a sword token or be a migration deposit");
+            }
+
+            if (results.size() == 1) {
+                return results.get(0);
+            }
+            else {
+                throw new IllegalArgumentException("Update deposit should update exactly one dataset, found " + results.size());
+            }
         }
     }
 
     @Override
-    public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit) {
+    public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit, DatasetVersion currentMetadata) {
+        // TODO: rename to DatasetComposer en push the terms stuff into it as well.
         var dataset = depositToDvDatasetMetadataMapper.toDataverseDataset(
             dansDeposit.getDdm(),
             dansDeposit.getOtherDoiId(),
@@ -130,10 +133,15 @@ public Dataset getDatasetMetadataFromDansDeposit(DansBagDeposit dansDeposit) {
             dansDeposit.getHasOrganizationalIdentifier(),
             dansDeposit.getHasOrganizationalIdentifierVersion());
         var version = dataset.getDatasetVersion();
-        version.setFileAccessRequest(dansDeposit.allowAccessRequests());
 
-        // TODO: when processing an update-deposit, retrieve terms of access from the previous version
-        if (!dansDeposit.allowAccessRequests() && StringUtils.isBlank(version.getTermsOfAccess())) {
+        // Allow access requests only if it was not forbidden in previous versions
+        var fileAccessRequest = dansDeposit.allowAccessRequests() && (currentMetadata == null || currentMetadata.getFileAccessRequest());
+        version.setFileAccessRequest(fileAccessRequest);
+        if (StringUtils.isBlank(version.getTermsOfAccess()) && currentMetadata != null) {
+            version.setTermsOfAccess(currentMetadata.getTermsOfAccess());
+        }
+        if (!fileAccessRequest && StringUtils.isBlank(version.getTermsOfAccess())) {
+            // Dataverse requires terms of access to be set if file access requests are disabled, so set it to "N/a" if not explicitly provided
             version.setTermsOfAccess("N/a");
         }
         version.setLicense(supportedLicenses.getLicenseFromDansDeposit(dansDeposit));

src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverter.java

@@ -21,21 +21,23 @@
 import nl.knaw.dans.dvingest.core.yaml.EditFilesRoot;
 import nl.knaw.dans.dvingest.core.yaml.EditPermissionsRoot;
 import nl.knaw.dans.dvingest.core.yaml.UpdateState;
+import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
 
 import java.io.IOException;
 
 @AllArgsConstructor
 public class DansDepositConverter {
     private final DansBagDeposit dansDeposit;
     private final String updatesDataset;
+    private final DatasetVersion currentMetadata;
     private final DansBagMappingService mappingService;
     private final YamlService yamlService;
 
     public void run() throws IOException {
         // TODO: pass to getEditFilesFromDansDeposit so that update-deposit can register it as a replaced file
         var originalMetadataPath = mappingService.packageOriginalMetadata(dansDeposit);
 
-        var dataset = mappingService.getDatasetMetadataFromDansDeposit(dansDeposit);
+        var dataset = mappingService.getDatasetMetadataFromDansDeposit(dansDeposit, currentMetadata);
         yamlService.writeYaml(dataset, dansDeposit.getBagDir().resolve("dataset.yml"));
 
         var editFiles = mappingService.getEditFilesFromDansDeposit(dansDeposit, updatesDataset);
@@ -48,5 +50,4 @@ public void run() throws IOException {
         updateState.setAction("publish-major");
         yamlService.writeYaml(updateState, dansDeposit.getBagDir().resolve("update-state.yml"));
     }
-
 }

src/main/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositSupport.java

@@ -29,6 +29,7 @@
 import nl.knaw.dans.dvingest.core.service.DataverseService;
 import nl.knaw.dans.dvingest.core.service.YamlService;
 import nl.knaw.dans.lib.dataverse.DataverseException;
+import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -49,7 +50,8 @@ public class DansDepositSupport implements Deposit {
 
     private DansBagDeposit dansDeposit;
 
-    public DansDepositSupport(DataverseIngestDeposit dataverseIngestDeposit, ValidateDansBagService validateDansBagService, DansBagMappingService dansBagMappingService, DataverseService dataverseService, YamlService yamlService) {
+    public DansDepositSupport(DataverseIngestDeposit dataverseIngestDeposit, ValidateDansBagService validateDansBagService, DansBagMappingService dansBagMappingService,
+        DataverseService dataverseService, YamlService yamlService) {
         this.validateDansBagService = validateDansBagService;
         this.ingestDataverseIngestDeposit = dataverseIngestDeposit;
         this.dansBagMappingService = dansBagMappingService;
@@ -69,11 +71,13 @@ public boolean convertDansDepositIfNeeded() {
             log.info("Converting deposit to Dataverse ingest metadata");
             try {
                 var updatesDataset = dansBagMappingService.getUpdatesDataset(ingestDataverseIngestDeposit.getLocation());
+                DatasetVersion currentMetadata = null;
                 if (updatesDataset != null) {
                     ingestDataverseIngestDeposit.updateProperties(Map.of(UPDATES_DATASET_KEY, updatesDataset));
+                    currentMetadata = dataverseService.getDatasetMetadata(updatesDataset);
                 }
                 dansDeposit = dansBagMappingService.readDansDeposit(ingestDataverseIngestDeposit.getLocation());
-                new DansDepositConverter(dansDeposit, updatesDataset, dansBagMappingService, yamlService).run();
+                new DansDepositConverter(dansDeposit, updatesDataset, currentMetadata, dansBagMappingService, yamlService).run();
                 log.info("Conversion successful");
                 return true;
             }
@@ -119,7 +123,8 @@ public void onSuccess(@NonNull String pid, String message) {
                             "identifier.urn", nbn
                         )
                     );
-                } catch (IOException | DataverseException e) {
+                }
+                catch (IOException | DataverseException e) {
                     throw new RuntimeException("Error getting URN:NBN", e); // Cancelling the "success"
                 }
             }

src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java

@@ -31,6 +31,7 @@
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -65,13 +66,8 @@ public EditFiles composeEditFiles() {
         editFiles.setAddUnrestrictedFiles(getUnrestrictedFilesToAdd(pathFileInfoMap));
         editFiles.setUpdateFileMetas(getUpdatedFileMetas(pathFileInfoMap));
 
-        var filePathsToEmbargo = getEmbargoedFiles(pathFileInfoMap, dateAvailable);
-        if (!filePathsToEmbargo.isEmpty()) {
-            var addEmbargo = new AddEmbargo();
-            addEmbargo.setDateAvailable(yyyymmddFormat.format(Date.from(dateAvailable)));
-            addEmbargo.setFilePaths(filePathsToEmbargo.stream().map(Path::toString).toList());
-            editFiles.setAddEmbargoes(List.of(addEmbargo));
-        }
+        addEmbargo(editFiles, pathFileInfoMap.keySet());
+
         return editFiles;
     }
 
@@ -123,10 +119,23 @@ private List<FileMeta> getUpdatedFileMetas(Map<Path, FileInfo> files) {
             .toList();
     }
 
-    private List<Path> getEmbargoedFiles(Map<Path, FileInfo> files, Instant dateAvailable) {
-        var now = Instant.now();
-        if (dateAvailable.isAfter(now)) {
-            return files.keySet().stream()
+    protected void addEmbargo(EditFiles editFiles, Set<Path> candidates) {
+        if (dateAvailable.isAfter(Instant.now())) {
+            var filesToEmbargo = candidates.stream()
+                .filter(f -> !embargoExclusions.contains(f.toString())).toList();
+
+            if (!filesToEmbargo.isEmpty()) {
+                var addEmbargo = new AddEmbargo();
+                addEmbargo.setDateAvailable(yyyymmddFormat.format(Date.from(dateAvailable)));
+                addEmbargo.setFilePaths(filesToEmbargo.stream().map(Path::toString).toList());
+                editFiles.setAddEmbargoes(List.of(addEmbargo));
+            }
+        }
+    }
+
+    private List<Path> getEmbargoedFiles(Set<Path> candidates, Instant dateAvailable) {
+        if (dateAvailable.isAfter(Instant.now())) {
+            return candidates.stream()
                 .filter(f -> !embargoExclusions.contains(f.toString())).toList();
         }
         else {

src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java

@@ -19,6 +19,7 @@
 import nl.knaw.dans.dvingest.core.bagprocessor.FilesInDatasetCache;
 import nl.knaw.dans.dvingest.core.dansbag.deposit.FileInfo;
 import nl.knaw.dans.dvingest.core.service.DataverseService;
+import nl.knaw.dans.dvingest.core.yaml.AddEmbargo;
 import nl.knaw.dans.dvingest.core.yaml.EditFiles;
 import nl.knaw.dans.dvingest.core.yaml.FromTo;
 import nl.knaw.dans.lib.dataverse.DataverseException;
@@ -27,6 +28,7 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.time.Instant;
+import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -144,11 +146,11 @@ public EditFiles composeEditFiles() {
         editFiles.setAddUnrestrictedFiles(pathsToAdd.stream()
             .filter(p -> !pathFileInfoMap.get(p).getMetadata().getRestricted()).toList().stream().map(Path::toString).toList());
 
-        // todo: embargoes
-
+        addEmbargo(editFiles, SetUtils.union(pathsToAdd, filesToReplace));
         return editFiles;
     }
 
+
     private Set<Path> getFilesToReplace(Map<Path, FileInfo> pathToFileInfo, Map<Path, FileMeta> fileReplacementCandidates) {
 
         var intersection = SetUtils.intersection(pathToFileInfo.keySet(), fileReplacementCandidates.keySet());

src/main/java/nl/knaw/dans/dvingest/core/dansbag/SupportedLicenses.java

@@ -30,6 +30,7 @@
 
 import static nl.knaw.dans.dvingest.core.dansbag.xml.XPathConstants.DDM_DCMI_METADATA;
 
+// TODO: move to mapping package
 public class SupportedLicenses {
     private final Map<URI, License> supportedLicenses;
 

src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java

@@ -72,4 +72,6 @@ public interface DataverseService {
     void addEmbargo(String pid, Embargo embargo) throws IOException, DataverseException;
 
     List<String> findDoiByMetadataField(String fieldName, String value) throws IOException, DataverseException;
+
+    DatasetVersion getDatasetMetadata(String pid) throws IOException, DataverseException;
 }

src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java

@@ -206,6 +206,11 @@ public String getDatasetUrnNbn(String pid) throws IOException, DataverseExceptio
             .findFirst().orElseThrow(() -> new IllegalStateException("No URN:NBN found in dataset"));
     }
 
+    @Override
+    public DatasetVersion getDatasetMetadata(String pid) throws IOException, DataverseException {
+        return dataverseClient.dataset(pid).getVersion().getData();
+    }
+
     // TODO: move this to dans-dataverse-client-lib; it is similar to awaitLockState.
     public void waitForState(String datasetId, String expectedState) {
         var numberOfTimesTried = 0;
@@ -239,8 +244,8 @@ public void waitForState(String datasetId, String expectedState) {
     }
 
     private String getDatasetState(String datasetId) throws IOException, DataverseException {
-        var version = dataverseClient.dataset(datasetId).getLatestVersion();
-        return version.getData().getLatestVersion().getVersionState();
+        var version = dataverseClient.dataset(datasetId).getVersion(Version.LATEST.toString(), true);
+        return version.getData().getVersionState();
 
     }
 }

src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansBagMappingServiceTest.java

@@ -47,9 +47,4 @@ public void getUpdatesDataset_finds_dataset_for_update_deposit() throws Exceptio
         // When / Then
         assertThat(mappingService.getUpdatesDataset(testDepositDir)).isEqualTo(doi);
     }
-
-
-
-
-
 }

src/test/java/nl/knaw/dans/dvingest/core/dansbag/DansDepositConverterTest.java

@@ -27,6 +27,9 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+/**
+ * Test class for {@link DansDepositConverter}. It uses the valid examples from the dd-dans-sword2-examples project.
+ */
 public class DansDepositConverterTest extends DansConversionFixture {
 
     private final YamlService yamlService = new YamlServiceImpl();
@@ -44,7 +47,7 @@ public void run_converts_dans_sword_all_mappings_example_to_dataverse_ingest_dep
         var deposit = dansBagDepositReader.readDeposit(depositDir);
 
         // When
-        new DansDepositConverter(deposit, null, mappingService, yamlService).run();
+        new DansDepositConverter(deposit, null, null, mappingService, yamlService).run();
 
         // Then
         assertThat(deposit.getBagDir().resolve("dataset.yml")).exists();