wip

Author: janvanmansum

src/main/java/nl/knaw/dans/dvingest/core/bagprocessor/FilesEditor.java

@@ -21,6 +21,7 @@
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import nl.knaw.dans.dvingest.core.service.DataverseService;
+import nl.knaw.dans.dvingest.core.service.FilesInDataset;
 import nl.knaw.dans.dvingest.core.service.UtilityServices;
 import nl.knaw.dans.dvingest.core.yaml.EditFiles;
 import nl.knaw.dans.lib.dataverse.DataverseException;
@@ -40,7 +41,7 @@
 import java.util.stream.Collectors;
 
 @Slf4j
-@RequiredArgsConstructor
+
 public class FilesEditor {
     @NonNull
     private final UUID depositId;
@@ -57,9 +58,19 @@ public class FilesEditor {
     private String pid;
 
     @Getter(AccessLevel.PACKAGE) // Getter for unit testing
-    private final Map<String, FileMeta> filesInDataset = new java.util.HashMap<>();
+//    private final Map<String, FileMeta> filesInDataset = new java.util.HashMap<>();
     private boolean filesRetrieved = false;
 
+    private FilesInDataset filesInDataset;
+
+    public FilesEditor(UUID depositId, Path dataDir, EditFiles editFiles, DataverseService dataverseService, UtilityServices utilityServices) {
+        this.depositId = depositId;
+        this.dataDir = dataDir;
+        this.editFiles = editFiles;
+        this.dataverseService = dataverseService;
+        this.utilityServices = utilityServices;
+    }
+
     public void editFiles(String pid) throws IOException, DataverseException {
         /*
          * TODO:
@@ -77,6 +88,8 @@ public void editFiles(String pid) throws IOException, DataverseException {
             }
         }
 
+        filesInDataset = dataverseService.getFilesInDataset(pid);
+
         log.debug("Start editing files for deposit {}", depositId);
         this.pid = pid;
         if (editFiles != null) {
@@ -97,7 +110,7 @@ private void deleteFiles() throws IOException, DataverseException {
         log.debug("Start deleting {} files for deposit {}", depositId, editFiles.getDeleteFiles().size());
         for (var file : editFiles.getDeleteFiles()) {
             log.debug("Deleting file: {}", file);
-            var fileToDelete = filesInDataset().get(file);
+            var fileToDelete = filesInDataset.get(file);
             if (fileToDelete == null) {
                 throw new IllegalArgumentException("File to delete not found in dataset: " + file);
             }

src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposer.java

@@ -18,13 +18,13 @@
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import nl.knaw.dans.dvingest.core.bagprocessor.DataversePath;
+import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.FileElement;
 import nl.knaw.dans.dvingest.core.yaml.AddEmbargo;
 import nl.knaw.dans.dvingest.core.yaml.EditFiles;
 import nl.knaw.dans.dvingest.core.yaml.FromTo;
 import nl.knaw.dans.ingest.core.domain.Deposit;
 import nl.knaw.dans.ingest.core.domain.FileInfo;
 import nl.knaw.dans.ingest.core.service.XPathEvaluator;
-import nl.knaw.dans.dvingest.core.dansbag.mapper.mapping.FileElement;
 import nl.knaw.dans.lib.dataverse.model.file.FileMeta;
 
 import java.nio.file.Path;
@@ -56,8 +56,6 @@ public class EditFilesComposer {
 
     public EditFiles composeEditFiles() {
         var editFiles = new EditFiles();
-
-        // Get file information from files.xml
         var pathFileInfoMap = getFileInfo(dansDeposit);
 
         // TODO: in update also ignore any files that have not changed (content or metadata)
@@ -69,6 +67,7 @@ public EditFiles composeEditFiles() {
         editFiles.setAddRestrictedFiles(getRestrictedFilesToAdd(pathFileInfoMap));
         editFiles.setUpdateFileMetas(getUpdatedFileMetas(pathFileInfoMap));
         editFiles.setDeleteFiles(getDeleteFiles(pathFileInfoMap));
+        editFiles.setMoveFiles(getFileMovements());
 
         var dateAvailable = getDateAvailable(dansDeposit);
         var filePathsToEmbargo = getEmbargoedFiles(pathFileInfoMap, dateAvailable);
@@ -133,6 +132,15 @@ protected List<String> getDeleteFiles(Map<Path, FileInfo> files) {
         return List.of();
     }
 
+    /**
+     * Get the files that should be moved.
+     *
+     * @return a list of FromTo objects that specify the files to move and their new location
+     */
+    protected List<FromTo> getFileMovements() {
+        return List.of();
+    }
+
     private List<Path> getEmbargoedFiles(Map<Path, FileInfo> files, Instant dateAvailable) {
         var now = Instant.now();
         if (dateAvailable.isAfter(now)) {
@@ -183,8 +191,6 @@ private List<FromTo> getFilesToRenameAtUpload(Map<Path, FileInfo> files) {
         return fromTos;
     }
 
-
-
     // TODO: move to mapping package
     private Instant getDateAvailable(Deposit deposit) {
         return XPathEvaluator.strings(deposit.getDdm(), "/ddm:DDM/ddm:profile/ddm:available")

src/main/java/nl/knaw/dans/dvingest/core/dansbag/EditFilesComposerForUpdate.java

@@ -15,6 +15,7 @@
  */
 package nl.knaw.dans.dvingest.core.dansbag;
 
+import nl.knaw.dans.dvingest.core.service.DataverseService;
 import nl.knaw.dans.ingest.core.domain.Deposit;
 import nl.knaw.dans.ingest.core.domain.FileInfo;
 import nl.knaw.dans.lib.dataverse.model.file.FileMeta;
@@ -28,14 +29,22 @@
  * Overrides the methods in EditFilesComposer to handle the case of an update to an existing dataset.
  */
 public class EditFilesComposerForUpdate extends EditFilesComposer {
+    private final DataverseService dataverseService;
 
-    public EditFilesComposerForUpdate(Deposit dansDeposit, Pattern fileExclusionPattern, List<String> embargoExclusions) {
+    public EditFilesComposerForUpdate(Deposit dansDeposit, Pattern fileExclusionPattern, List<String> embargoExclusions, DataverseService dataverseService) {
         super(dansDeposit, fileExclusionPattern, embargoExclusions);
+        this.dataverseService = dataverseService;
     }
 
     @Override
     protected List<String> getFilesToIgnore(Map<Path, FileInfo> files) {
-        return super.getFilesToIgnore(files);
+        super.getFilesToIgnore(files);
+
+
+
+
+
+
     }
 
     @Override

src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java

@@ -72,4 +72,6 @@ public interface DataverseService {
     void addEmbargo(String pid, Embargo embargo) throws IOException, DataverseException;
 
     List<String> findDoiByMetadataField(String fieldName, String value) throws IOException, DataverseException;
+
+    FilesInDataset getFilesInDataset(String pid);
 }

src/main/java/nl/knaw/dans/dvingest/core/service/FilesInDataset.java

@@ -0,0 +1,34 @@
+package nl.knaw.dans.dvingest.core.service;
+
+import nl.knaw.dans.lib.dataverse.DataverseException;
+import nl.knaw.dans.lib.dataverse.model.file.FileMeta;
+
+import java.io.IOException;
+
+/**
+ * Interface for a service that keeps track of the files in a dataset. The key to each file is its path in the dataset, i.e. the combination of label and directoryLabel.
+ */
+public interface FilesInDataset {
+
+    /**
+     * Gets the FileMeta object for the file with the same path as the given FileMeta object.
+     *
+     * @param fileMeta the FileMeta object with the path of the file to get
+     * @return the FileMeta object for the file with the same path as the given FileMeta object, or null if no such file exists
+     */
+    FileMeta get(FileMeta fileMeta) throws IOException, DataverseException;
+
+    /**
+     * Adds or overwrites the FileMeta object for the file with the same path as the given FileMeta object.
+     *
+     * @param fileMeta the FileMeta object to put
+     */
+    void put(FileMeta fileMeta) throws IOException, DataverseException;
+
+    /**
+     * Removes the FileMeta object for the file with the same path as the given FileMeta object.
+     *
+     * @param fileMeta the FileMeta object to remove
+     */
+    void remove(FileMeta fileMeta) throws IOException, DataverseException;
+}

src/main/java/nl/knaw/dans/dvingest/core/service/FilesInDatasetImpl.java

@@ -0,0 +1,59 @@
+package nl.knaw.dans.dvingest.core.service;
+
+import lombok.extern.slf4j.Slf4j;
+import nl.knaw.dans.dvingest.core.bagprocessor.DataversePath;
+import nl.knaw.dans.lib.dataverse.DataverseException;
+import nl.knaw.dans.lib.dataverse.model.file.FileMeta;
+
+import java.io.IOException;
+import java.util.Map;
+
+@Slf4j
+public class FilesInDatasetImpl implements FilesInDataset {
+    private final Map<String, FileMeta> filesInDataset = new java.util.HashMap<>();
+    private final DataverseService dataverseService;
+    private final String pid;
+    private boolean filesRetrieved = false;
+
+    public FilesInDatasetImpl(String pid, DataverseService dataverseService) {
+        this.dataverseService = dataverseService;
+        this.pid = pid;
+    }
+
+    @Override
+    public FileMeta get(FileMeta fileMeta) throws IOException, DataverseException {
+        return filesInDataset().get(getPath(fileMeta));
+    }
+
+    @Override
+    public void put(FileMeta fileMeta) throws IOException, DataverseException {
+        filesInDataset().put(getPath(fileMeta), fileMeta);
+
+    }
+
+    @Override
+    public void remove(FileMeta fileMeta) throws IOException, DataverseException {
+        filesInDataset().remove(getPath(fileMeta));
+    }
+
+    private Map<String, FileMeta> filesInDataset() throws IOException, DataverseException {
+        if (!filesRetrieved) {
+            log.debug("Retrieving files in dataset for pid {}", pid);
+            var files = dataverseService.getFiles(pid);
+            for (var file : files) {
+                filesInDataset.put(getPath(file), file);
+            }
+            filesRetrieved = true;
+            log.debug("Retrieved {} files in dataset for pid {}", files.size(), pid);
+        }
+        else {
+            log.debug("Files in dataset already retrieved for pid {}", pid);
+        }
+        return filesInDataset;
+    }
+
+    private String getPath(FileMeta file) {
+        var dataversePath = new DataversePath(file.getDirectoryLabel(), file.getLabel());
+        return dataversePath.toString();
+    }
+}