First successful cycle to processed in outbox.

debug-init-env.sh

@@ -17,5 +17,8 @@
 
 echo -n "Pre-creating log..."
 TEMPDIR=data
+mkdir -p $TEMPDIR/imports/inbox
+mkdir -p $TEMPDIR/imports/outbox
+mkdir -p $TEMPDIR/temp
 touch $TEMPDIR/dd-dataverse-ingest.log
 echo "OK"

pom.xml

@@ -49,10 +49,6 @@
             <groupId>io.dropwizard</groupId>
             <artifactId>dropwizard-core</artifactId>
         </dependency>
-        <dependency>
-            <groupId>io.dropwizard</groupId>
-            <artifactId>dropwizard-hibernate</artifactId>
-        </dependency>
         <dependency>
             <groupId>io.dropwizard</groupId>
             <artifactId>dropwizard-client</artifactId>
@@ -132,21 +128,11 @@
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration combine.children="override">
                     <annotationProcessorPaths>
-                        <path>
-                            <groupId>org.mapstruct</groupId>
-                            <artifactId>mapstruct-processor</artifactId>
-                            <version>${mapstruct.version}</version>
-                        </path>
                         <path>
                             <groupId>org.projectlombok</groupId>
                             <artifactId>lombok</artifactId>
                             <version>${lombok.version}</version>
                         </path>
-                        <dependency>
-                            <groupId>org.projectlombok</groupId>
-                            <artifactId>lombok-mapstruct-binding</artifactId>
-                            <version>0.2.0</version>
-                        </dependency>
                     </annotationProcessorPaths>
                 </configuration>
             </plugin>

src/main/java/nl/knaw/dans/dvingest/DdDataverseIngestApplication.java

@@ -19,23 +19,12 @@
 import io.dropwizard.core.Application;
 import io.dropwizard.core.setup.Bootstrap;
 import io.dropwizard.core.setup.Environment;
-import io.dropwizard.db.PooledDataSourceFactory;
-import io.dropwizard.hibernate.HibernateBundle;
 import nl.knaw.dans.dvingest.config.DdDataverseIngestConfiguration;
-import nl.knaw.dans.dvingest.core.ImportJob;
-import nl.knaw.dans.dvingest.db.ImportJobDao;
+import nl.knaw.dans.dvingest.core.IngestArea;
 import nl.knaw.dans.dvingest.resources.DefaultApiResource;
 import nl.knaw.dans.dvingest.resources.IngestApiResource;
 
 public class DdDataverseIngestApplication extends Application<DdDataverseIngestConfiguration> {
-    private final HibernateBundle<DdDataverseIngestConfiguration> hibernateBundle = new HibernateBundle<>(ImportJob.class) {
-
-        @Override
-        public PooledDataSourceFactory getDataSourceFactory(DdDataverseIngestConfiguration ddDataverseIngestConfiguration) {
-            return ddDataverseIngestConfiguration.getDatabase();
-        }
-    };
-
     public static void main(final String[] args) throws Exception {
         new DdDataverseIngestApplication().run(args);
     }
@@ -47,14 +36,16 @@ public String getName() {
 
     @Override
     public void initialize(final Bootstrap<DdDataverseIngestConfiguration> bootstrap) {
-        bootstrap.addBundle(hibernateBundle);
     }
 
     @Override
     public void run(final DdDataverseIngestConfiguration configuration, final Environment environment) {
         environment.jersey().register(new DefaultApiResource());
-        var dao = new ImportJobDao(hibernateBundle.getSessionFactory());
-        environment.jersey().register(new IngestApiResource(dao));
+        var ingestArea = IngestArea.builder()
+            .executorService(environment.lifecycle().executorService("ingest").minThreads(1).maxThreads(1).build())
+            .dataverseClient(configuration.getDataverse().build())
+            .inbox(configuration.getIngest().getInbox())
+            .outbox(configuration.getIngest().getOutbox()).build();
+        environment.jersey().register(new IngestApiResource(ingestArea));
     }
-
 }

src/main/java/nl/knaw/dans/dvingest/config/DdDataverseIngestConfiguration.java

@@ -17,21 +17,25 @@
 package nl.knaw.dans.dvingest.config;
 
 import io.dropwizard.core.Configuration;
-import io.dropwizard.db.DataSourceFactory;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import nl.knaw.dans.lib.util.DataverseClientFactory;
 
 import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
+import java.nio.file.Path;
 
 @Data
 @EqualsAndHashCode(callSuper = true)
 public class DdDataverseIngestConfiguration extends Configuration {
+    @Valid
+    @NotNull
     private DataverseClientFactory dataverse;
-    private IngestConfig ingest;
 
     @Valid
     @NotNull
-    private DataSourceFactory database = new DataSourceFactory();
+    private IngestConfig ingest;
+
+
+    private Path tempDir;
 }

src/main/java/nl/knaw/dans/dvingest/config/IngestConfig.java

@@ -23,5 +23,4 @@
 public class IngestConfig {
     private Path inbox;
     private Path outbox;
-    private Path tempDir;
 }

src/main/java/nl/knaw/dans/dvingest/core/Deposit.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2024 DANS - Data Archiving and Networked Services ([email protected])
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package nl.knaw.dans.dvingest.core;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import nl.knaw.dans.lib.dataverse.model.dataset.Dataset;
+import nl.knaw.dans.lib.dataverse.model.dataset.DatasetVersion;
+import org.apache.commons.io.FileUtils;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.UUID;
+
+@Getter
+@RequiredArgsConstructor
+public class Deposit {
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+    @NonNull
+    private Path location;
+
+    public UUID getId() {
+        return UUID.fromString(location.getFileName().toString());
+    }
+
+    public String getDatasetMetadata() throws IOException {
+        return FileUtils.readFileToString(location.resolve("dataset.json").toFile(), "UTF-8");
+    }
+
+    public Path getFilesDir() {
+        return location.resolve("files");
+    }
+
+    public void moveTo(Path targetDir) throws IOException {
+        Files.move(location, targetDir.resolve(location.getFileName()));
+        location = targetDir.resolve(location.getFileName());
+    }
+}

src/main/java/nl/knaw/dans/dvingest/core/ImportJob.java

@@ -15,44 +15,96 @@
  */
 package nl.knaw.dans.dvingest.core;
 
-import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.ToString;
-import nl.knaw.dans.validation.Uuid;
-
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.GeneratedValue;
-import javax.persistence.Id;
-import javax.persistence.Table;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import nl.knaw.dans.dvingest.api.ImportCommandDto;
+import nl.knaw.dans.dvingest.api.ImportJobStatusDto;
+import nl.knaw.dans.dvingest.api.ImportJobStatusDto.StatusEnum;
+import nl.knaw.dans.lib.dataverse.DataverseClient;
+
+import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
 
-/**
- * Description of an import job. A job is a request to import a deposit or a batch of deposits.
- */
-@Entity
-@Table(name = "import_job")
-@Getter
-@Setter
-@ToString
-@NoArgsConstructor
-@AllArgsConstructor
-public class ImportJob {
-    @GeneratedValue
-    @Id
-    private Long id;
-
-    @Column(name = "creation_time")
-    private Long creationTime;
-
-    @Column(name = "status")
-    private String status = "PENDING";
-
-    @Column(name = "location")
-    private String location;
-
-    // TODO: isBatch to indicate if the location contains a single object or a batch of objects, for now always a single object
+@Slf4j
+@RequiredArgsConstructor
+public class ImportJob implements Runnable {
+    @NonNull
+    private final ImportCommandDto importCommand;
+    @NonNull
+    private final Path outputDir;
+    @NonNull
+    private final DataverseClient dataverseClient;
+
+    @Getter
+    private final ImportJobStatusDto status = new ImportJobStatusDto();
+
+    @Override
+    public void run() {
+        try {
+            log.debug("Starting import job: {}", importCommand);
+            status.setStatus(StatusEnum.RUNNING);
+            List<Deposit> deposits = new ArrayList<>();
+
+            // Build deposit list, todo: ordered
+            if (importCommand.getSingleObject()) {
+                deposits.add(new Deposit(Path.of(importCommand.getPath())));
+            }
+            else {
+                // Multiple objects
+                // Create deposit for each object
+                // Add to list
+            }
+
+            initOutputDir();
+
+            // Process deposits
+            for (Deposit deposit : deposits) {
+                new IngestTask(deposit, dataverseClient, outputDir).run();
+            }
+
+            // Job completed, some deposits may still have failed, TODO: change to DONE
+            status.setStatus(StatusEnum.SUCCESS);
+        }
+        catch (Exception e) {
+            log.error("Failed to process import job", e);
+            status.setStatus(StatusEnum.FAILED);
+        }
+    }
+
+    private void initOutputDir() {
+        log.debug("Initializing output directory: {}", outputDir);
+        createDirectoryIfNotExists(outputDir);
+        createDirectoryIfNotExists(outputDir.resolve("processed"));
+        createDirectoryIfNotExists(outputDir.resolve("failed"));
+        createDirectoryIfNotExists(outputDir.resolve("rejected"));
+        if (!importCommand.getSingleObject()) {
+            checkDirectoryEmpty(outputDir.resolve("processed"));
+            checkDirectoryEmpty(outputDir.resolve("failed"));
+            checkDirectoryEmpty(outputDir.resolve("rejected"));
+        }
+    }
+
+    private void createDirectoryIfNotExists(Path path) {
+        if (!path.toFile().exists()) {
+            if (!path.toFile().mkdirs()) {
+                throw new IllegalStateException("Failed to create directory: " + path);
+            }
+        }
+    }
+
+    private void checkDirectoryEmpty(Path path) {
+        try (var stream = Files.list(path)) {
+            if (stream.findAny().isPresent()) {
+                throw new IllegalStateException("Directory not empty: " + path);
+            }
+        }
+        catch (IOException e) {
+            throw new IllegalStateException("Failed to check directory: " + path, e);
+        }
+    }
 }