MODOAIPMH-553 - Request has expired error on downloading error logs

descriptors/ModuleDescriptor-template.json

@@ -101,6 +101,11 @@
           "pathPattern": "/oai/request-metadata/{requestId}/suppressed-from-discovery-instances",
           "permissionsRequired": ["oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get"]
         },
+        {
+          "methods": ["GET"],
+          "pathPattern": "/oai/request-metadata/{requestId}/logs",
+          "permissionsRequired": ["oai-pmh.request-metadata.logs.item.get"]
+        },
         {
           "methods": [
             "POST"
@@ -273,6 +278,11 @@
       "displayName": "OAI-PMH Request metadata - get collection of suppressed from discovery instances UUIDs",
       "description": "Retrieves request metadata collection - suppressed from discovery instances UUIDs"
     },
+    {
+      "permissionName": "oai-pmh.request-metadata.logs.item.get",
+      "displayName": "OAI-PMH Request metadata - download error log",
+      "description": "Downloads error log by request id"
+    },
     {
       "permissionName": "oai-pmh.all",
       "displayName": "OAI-PMH - all permissions",
@@ -289,7 +299,8 @@
         "oai-pmh.request-metadata.failed-to-save-instances.collection.get",
         "oai-pmh.request-metadata.failed-instances.collection.get",
         "oai-pmh.request-metadata.skipped-instances.collection.get",
-        "oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get"
+        "oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get",
+        "oai-pmh.request-metadata.logs.item.get"
       ]
     },
     {

ramls/request-metadata.raml

@@ -83,3 +83,21 @@ resourceTypes:
     get:
       description: Get list of suppressed from discovery instances UUIDs
       is: [ pageable ]
+
+  /{requestId}/logs:
+    description: Service that allows to retrieve error log by request id
+    get:
+      responses:
+        200:
+          body:
+            binary/octet-stream:
+        404:
+          description: "Not found"
+          body:
+            text/plain:
+              example: "Not found"
+        500:
+          description: "Internal server error, e.g. due to misconfiguration"
+          body:
+            text/plain:
+              example: "Internal server error, contact administrator"

src/main/java/org/folio/oaipmh/dao/impl/InstancesDaoImpl.java

@@ -44,10 +44,8 @@
 import org.folio.rest.jooq.tables.records.RequestMetadataLbRecord;
 import org.folio.rest.jooq.tables.records.SkippedInstancesIdsRecord;
 import org.folio.rest.jooq.tables.records.SuppressedFromDiscoveryInstancesIdsRecord;
-import org.folio.s3.client.FolioS3Client;
 import org.jooq.InsertValuesStep3;
 import org.jooq.Record;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 
 import io.github.jklingsporn.vertx.jooq.classic.reactivepg.ReactiveClassicGenericQueryExecutor;
@@ -69,9 +67,6 @@ public InstancesDaoImpl(PostgresClientFactory postgresClientFactory) {
     this.postgresClientFactory = postgresClientFactory;
   }
 
-  @Autowired
-  private FolioS3Client folioS3Client;
-
   @Override
   public Future<List<String>> getExpiredRequestIds(String tenantId, long expirationPeriodInSeconds) {
     OffsetDateTime offsetDateTime = ZonedDateTime.ofInstant(Instant.now(), ZoneId.systemDefault())
@@ -311,6 +306,7 @@ private Record toDatabaseRecord(RequestMetadataLb requestMetadata) {
     return new RequestMetadataLbRecord().setRequestId(requestMetadata.getRequestId())
       .setLastUpdatedDate(requestMetadata.getLastUpdatedDate())
       .setStreamEnded(requestMetadata.getStreamEnded()).setLinkToErrorFile(requestMetadata.getLinkToErrorFile())
+      .setPathToErrorFileInS3(requestMetadata.getPathToErrorFileInS3())
       .setStartedDate(requestMetadata.getStartedDate());
   }
 
@@ -480,8 +476,7 @@ private RequestMetadata rowToRequestMetadata(Row row) {
     of(pojo.getSuppressedInstancesCounter()).ifPresent(requestMetadata::withSuppressedInstancesCounter);
     ofNullable(pojo.getPathToErrorFileInS3()).ifPresentOrElse(pathToError -> {
       if (!pathToError.isEmpty()) {
-        var regeneratedLink = folioS3Client.getPresignedUrl(pathToError);
-        requestMetadata.withLinkToErrorFile(regeneratedLink);
+        requestMetadata.withLinkToErrorFile(pathToError);
       } else {
         requestMetadata.setLinkToErrorFile("");
       }

src/main/java/org/folio/rest/impl/RequestMetadataAPIs.java

@@ -1,15 +1,19 @@
 package org.folio.rest.impl;
 
+import java.io.IOException;
 import java.util.Map;
 
+import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.Response;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.folio.dataimport.util.ExceptionHelper;
 import org.folio.oaipmh.dao.InstancesDao;
 import org.folio.rest.jaxrs.resource.OaiRequestMetadata;
+import org.folio.rest.jooq.tables.pojos.RequestMetadataLb;
 import org.folio.rest.tools.utils.TenantTool;
+import org.folio.s3.client.FolioS3Client;
 import org.folio.spring.SpringContextUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -24,11 +28,15 @@ public class RequestMetadataAPIs implements OaiRequestMetadata {
   private static final Logger logger = LogManager.getLogger(RequestMetadataAPIs.class);
   private static final String REQUEST_METADATA_ERROR_MESSAGE_TEMPLATE = "Error occurred while get request metadata. Message: {}.";
   private static final String UUID_COLLECTION_ERROR_MESSAGE_TEMPLATE = "Error occurred while get UUIDs collection. Message: {}.";
+  private static final String DOWNLOAD_LOG_ERROR_MESSAGE_TEMPLATE = "Error occurred while downloading log. Message: {}.";
 
 
   @Autowired
   InstancesDao instancesDao;
 
+  @Autowired
+  FolioS3Client folioS3Client;
+
   public RequestMetadataAPIs() {
     SpringContextUtil.autowireDependencies(this, Vertx.currentContext());
   }
@@ -116,4 +124,38 @@ public void getOaiRequestMetadataSuppressedFromDiscoveryInstancesByRequestId(Str
       asyncResultHandler.handle(Future.succeededFuture(ExceptionHelper.mapExceptionToResponse(e)));
     }
   }
+
+  @Override
+  public void getOaiRequestMetadataLogsByRequestId(String requestId, Map<String, String> okapiHeaders, Handler<AsyncResult<Response>> asyncResultHandler, Context vertxContext) {
+    vertxContext.runOnContext(v -> {
+      try {
+        var tenantId = TenantTool.tenantId(okapiHeaders);
+        logger.info("Download error log for tenant: {}, requestId: {}", tenantId, requestId);
+        instancesDao.getRequestMetadataByRequestId(requestId, tenantId)
+          .map(RequestMetadataLb::getPathToErrorFileInS3)
+          .map(this::prepareResponseData)
+          .map(Response.class::cast)
+          .otherwise(ExceptionHelper::mapExceptionToResponse)
+          .onComplete(asyncResultHandler);
+      } catch (Exception e) {
+        logger.error(DOWNLOAD_LOG_ERROR_MESSAGE_TEMPLATE, e.getMessage());
+        asyncResultHandler.handle(Future.succeededFuture(ExceptionHelper.mapExceptionToResponse(e)));
+      }
+    });
+  }
+
+  private GetOaiRequestMetadataLogsByRequestIdResponse prepareResponseData(String pathToErrorFile) {
+    var response = GetOaiRequestMetadataLogsByRequestIdResponse.respond200WithBinaryOctetStream(new String(readFile(pathToErrorFile)));
+    var headers = response.getHeaders();
+    headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + pathToErrorFile);
+    return response;
+  }
+
+  private byte[] readFile(String path) {
+    try (var is = folioS3Client.read(path)) {
+      return is.readAllBytes();
+    } catch (IOException e) {
+      return new byte[0];
+    }
+  }
 }

src/test/java/org/folio/oaipmh/service/impl/ErrorsServiceImplTest.java

@@ -23,6 +23,7 @@
 import org.folio.rest.persist.PostgresClient;
 import org.folio.rest.tools.utils.ModuleName;
 import org.folio.rest.tools.utils.NetworkUtils;
+import org.folio.s3.client.FolioS3Client;
 import org.folio.spring.SpringContextUtil;
 import org.junit.jupiter.api.AfterAll;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -65,6 +66,9 @@ public class ErrorsServiceImplTest extends AbstractErrorsTest {
   public static final String BUCKET = "test-bucket";
   public static final String REGION = "us-west-2";
 
+  @Autowired
+  private FolioS3Client folioS3Client;
+
   static {
     s3 = new GenericContainer<>("minio/minio:latest")
       .withEnv("MINIO_ACCESS_KEY", S3_ACCESS_KEY)
@@ -271,7 +275,7 @@ void shouldDeleteErrorsByRequestId_whenErrorFound(VertxTestContext testContext)
   }
 
   private void verifyErrorCSVFile(String linkToError, List<String> initErrorFileContent) {
-    try (InputStream inputStream = new URL(linkToError).openStream();
+    try (InputStream inputStream = folioS3Client.read(linkToError);
     Scanner scanner = new Scanner(inputStream)) {
       List<String> listCsvLines = new ArrayList<>();
       while (scanner.hasNextLine()) {

src/test/java/org/folio/rest/impl/OaiPmhImplTest.java

@@ -38,9 +38,11 @@
 import org.folio.rest.RestVerticle;
 import org.folio.rest.jaxrs.model.RequestMetadataCollection;
 import org.folio.rest.jaxrs.model.UuidCollection;
+import org.folio.rest.jooq.tables.pojos.RequestMetadataLb;
 import org.folio.rest.persist.PostgresClient;
 import org.folio.rest.tools.utils.ModuleName;
 import org.folio.rest.tools.utils.NetworkUtils;
+import org.folio.s3.client.FolioS3Client;
 import org.folio.spring.SpringContextUtil;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.AfterAll;
@@ -54,6 +56,8 @@
 import org.junit.jupiter.params.provider.EnumSource;
 import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.Spy;
 import org.openarchives.oai._2.GranularityType;
 import org.openarchives.oai._2.HeaderType;
@@ -71,13 +75,16 @@
 
 import javax.annotation.concurrent.NotThreadSafe;
 import javax.xml.bind.JAXBElement;
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.math.BigInteger;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.util.Arrays;
 import java.util.Base64;
@@ -86,6 +93,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.UUID;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -176,6 +184,7 @@
 import static org.junit.Assert.fail;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS;
+import static org.mockito.Mockito.when;
 import static org.openarchives.oai._2.OAIPMHerrorcodeType.BAD_ARGUMENT;
 import static org.openarchives.oai._2.OAIPMHerrorcodeType.BAD_RESUMPTION_TOKEN;
 import static org.openarchives.oai._2.OAIPMHerrorcodeType.CANNOT_DISSEMINATE_FORMAT;
@@ -251,8 +260,12 @@ class OaiPmhImplTest {
 
   private PostgresTesterContainer postgresTesterContainer;
 
+  @Mock
+  private FolioS3Client folioS3Client;
+
   @BeforeAll
   void setUpOnce(Vertx vertx, VertxTestContext testContext) {
+    folioS3Client = Mockito.mock(FolioS3Client.class);
     resetSystemProperties();
     VertxOptions options = new VertxOptions();
     options.setBlockedThreadCheckInterval(1000*60*60);
@@ -2943,6 +2956,38 @@ private List<HeaderType> getHeadersListDependOnVerbType(VerbType verb, OAIPMH oa
       .getHeaders();
   }
 
+  @Test
+  void downloadLog(VertxTestContext testContext) {
+    var requestId = UUID.randomUUID();
+    var pathToError = "log.csv";
+    var requestMetadata = new RequestMetadataLb();
+    requestMetadata.setRequestId(requestId);
+    requestMetadata.setPathToErrorFileInS3(pathToError);
+    requestMetadata.setStartedDate(OffsetDateTime.now(ZoneId.systemDefault()));
+    requestMetadata.setLastUpdatedDate(OffsetDateTime.now(ZoneId.systemDefault()));
+
+    when(folioS3Client.read(pathToError)).thenReturn(new ByteArrayInputStream("log".getBytes()));
+
+    testContext.verify(() -> {
+      instancesService.saveRequestMetadata(requestMetadata, OAI_TEST_TENANT)
+        .map(RequestMetadataLb::getRequestId)
+        .map(uuid -> RestAssured.given()
+          .header(okapiUrlHeader)
+          .header(tokenHeader)
+          .header(tenantHeader)
+          .basePath(REQUEST_METADATA_PATH + "/" + uuid + "/logs")
+          .when()
+          .get()
+          .then()
+          .statusCode(200)
+          .extract()
+          .as(String.class))
+        .onComplete(res -> assertEquals("log", res.result()));
+      testContext.completeNow();
+    });
+
+  }
+
   private RequestMetadataCollection getRequestMetadataCollection(int limit) {
     return RestAssured.given()
       .header(okapiUrlHeader)