Implemented deleting files.

TODO.TXT

@@ -0,0 +1,29 @@
+====
+    Copyright (C) 2024 DANS - Data Archiving and Networked Services ([email protected])
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+====
+
+- regel de upload-batch grootte naar beneden als :ZipUploadFilesLimit kleiner is dan de ingest batch grootte
+- implement replaceFiles in edit
+- implement providing file metadata
+- implement setting license
+- implement setting embargo
+- implement setting request access permission
+
+//edit.yml
+deleteFiles: []
+replaceFiles: []
+terms:
+    license: <license url>
+embargo: ...

pom.xml

@@ -59,6 +59,7 @@
         <dependency>
             <groupId>nl.knaw.dans</groupId>
             <artifactId>dans-dataverse-client-lib</artifactId>
+            <version>1.0.1</version>
         </dependency>
         <dependency>
             <groupId>nl.knaw.dans</groupId>

src/main/java/nl/knaw/dans/dvingest/core/DepositBag.java

@@ -18,13 +18,13 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-import lombok.ToString;
 import nl.knaw.dans.lib.dataverse.MetadataFieldDeserializer;
 import nl.knaw.dans.lib.dataverse.model.dataset.Dataset;
 import nl.knaw.dans.lib.dataverse.model.dataset.MetadataField;
 import org.apache.commons.io.FileUtils;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
@@ -50,11 +50,18 @@ public DepositBag(Path bagDir) {
     }
 
     public Dataset getDatasetMetadata() throws IOException {
-        var dataset = MAPPER.readValue(FileUtils.readFileToString(bagDir.resolve("dataset.yml").toFile(), "UTF-8"), Dataset.class);
+        var dataset = MAPPER.readValue(FileUtils.readFileToString(bagDir.resolve("dataset.yml").toFile(), StandardCharsets.UTF_8), Dataset.class);
         dataset.getDatasetVersion().setFiles(Collections.emptyList()); // files = null or a list of files is not allowed
         return dataset;
     }
 
+    public Edit getEditInstructions() throws IOException {
+        if (!Files.exists(bagDir.resolve("edit.yml"))) {
+            return null;
+        }
+        return MAPPER.readValue(FileUtils.readFileToString(bagDir.resolve("edit.yml").toFile(), StandardCharsets.UTF_8), Edit.class);
+    }
+
     @Override
     public int compareTo(DepositBag depositBag) {
         return bagDir.getFileName().toString().compareTo(depositBag.bagDir.getFileName().toString());

src/main/java/nl/knaw/dans/dvingest/core/DepositTask.java

@@ -81,31 +81,40 @@ private String processBag(DepositBag bag, String targetPid) throws IOException,
         else {
             updateMetadata(bag, targetPid);
         }
+        // TODO: replaceFiles(bag, targetPid);
         deleteFiles(bag, targetPid);
         addFiles(bag, targetPid);
         publishVersion(targetPid);
         return targetPid;
     }
 
     private String createNewDataset(DepositBag bag) throws IOException, DataverseException {
+        log.debug("Creating new dataset");
         var result = dataverseService.createDataset(bag.getDatasetMetadata());
         var pid = result.getData().getPersistentId();
         log.debug(result.getEnvelopeAsString());
         return pid;
     }
 
     private void updateMetadata(DepositBag bag, String pid) throws IOException, DataverseException {
+        log.debug("Updating dataset metadata for {}", pid);
         dataverseService.updateMetadata(pid, bag.getDatasetMetadata().getDatasetVersion());
     }
 
     private void deleteFiles(DepositBag bag, String pid) throws IOException, DataverseException {
-        //        var files = bag.getFiles();
-        //        for (var file : files) {
-        //            dataverseService.deleteFile(pid, file.getDataFile().getId());
-        //        }
+        var edit = bag.getEditInstructions();
+        if (edit == null) {
+            log.debug("No edit instructions found. Skipping file deletion.");
+            return;
+        }
+        for (var file : edit.getDeleteFiles()) {
+            log.debug("Deleting file: {}", file);
+            dataverseService.deleteFile(pid, file);
+        }
     }
 
     private void addFiles(DepositBag bag, String pid) throws IOException, DataverseException {
+        log.debug("Adding files from {}", bag.getDataDir());
         var iterator = new PathIterator(FileUtils.iterateFiles(bag.getDataDir().toFile(), null, true));
         while (iterator.hasNext()) {
             uploadFileBatch(iterator, bag.getDataDir(), pid);

src/main/java/nl/knaw/dans/dvingest/core/Edit.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2024 DANS - Data Archiving and Networked Services ([email protected])
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package nl.knaw.dans.dvingest.core;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class Edit {
+    private List<String> deleteFiles;
+
+}

src/main/java/nl/knaw/dans/dvingest/core/service/DataverseService.java

@@ -38,4 +38,6 @@ public interface DataverseService {
     void waitForState(String persistentId, String state) throws DataverseException;
 
     DataverseHttpResponse<DatasetVersion> updateMetadata(String targetDatasetPid, DatasetVersion datasetMetadata) throws DataverseException, IOException;
+
+    public DataverseHttpResponse<Object> deleteFile(String persistentId, String filepath) throws DataverseException, IOException;
 }

src/main/java/nl/knaw/dans/dvingest/core/service/DataverseServiceImpl.java

@@ -21,12 +21,14 @@
 import nl.knaw.dans.lib.dataverse.DataverseClient;
 import nl.knaw.dans.lib.dataverse.DataverseException;
 import nl.knaw.dans.lib.dataverse.DataverseHttpResponse;
+import nl.knaw.dans.lib.dataverse.Version;
 import nl.knaw.dans.lib.dataverse.model.dataset.Dataset;
 import nl.knaw.dans.lib.dataverse.model.dataset.DatasetCreationResult;
 import nl.knaw.dans.lib.dataverse.model.dataset.DatasetPublicationResult;
 import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
 import nl.knaw.dans.lib.dataverse.model.dataset.FileList;
 import nl.knaw.dans.lib.dataverse.model.file.FileMeta;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -67,6 +69,24 @@ public DataverseHttpResponse<DatasetVersion> updateMetadata(String targetDataset
         return dataverseClient.dataset(targetDatasetPid).updateMetadata(datasetMetadata, metadataKeys);
     }
 
+    public DataverseHttpResponse<Object> deleteFile(String persistentId, String filepath) throws DataverseException, IOException {
+        var result = dataverseClient.dataset(persistentId).getFiles(Version.DRAFT.toString());
+        var optFileToDelete = result.getData().stream()
+            .filter(file -> {
+                var fp = StringUtils.isBlank(file.getDirectoryLabel()) ?
+                    file.getLabel() :
+                    file.getDirectoryLabel() + "/" + file.getLabel();
+                return filepath.equals(fp);
+            })
+            .findFirst();
+
+        if (optFileToDelete.isEmpty()) {
+            throw new IllegalArgumentException("File not found: " + filepath);
+        }
+        log.debug("Deleting file with id {}", optFileToDelete.get().getDataFile().getId());
+        return dataverseClient.sword().deleteFile(optFileToDelete.get().getDataFile().getId());
+    }
+
     // TODO: move this to dans-dataverse-client-lib; it is similar to awaitLockState.
     public void waitForState(String datasetId, String expectedState) {
         var numberOfTimesTried = 0;