From 4dea103038133fa3653f36a6f4ebdbfe6ff407bb Mon Sep 17 00:00:00 2001 From: siarhei-charniak Date: Thu, 4 Apr 2024 17:10:02 +0300 Subject: [PATCH] MODOAIPMH-553 - Request has expired error on downloading error logs --- descriptors/ModuleDescriptor-template.json | 13 +++- ramls/request-metadata.raml | 18 +++++ .../oaipmh/dao/impl/InstancesDaoImpl.java | 9 +-- .../folio/rest/impl/RequestMetadataAPIs.java | 42 +++++++++++ .../service/impl/ErrorsServiceImplTest.java | 6 +- .../org/folio/rest/impl/OaiPmhImplTest.java | 69 +++++++++++++++++++ 6 files changed, 148 insertions(+), 9 deletions(-) diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json index 320c1580b..4f2549dcb 100644 --- a/descriptors/ModuleDescriptor-template.json +++ b/descriptors/ModuleDescriptor-template.json @@ -101,6 +101,11 @@ "pathPattern": "/oai/request-metadata/{requestId}/suppressed-from-discovery-instances", "permissionsRequired": ["oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get"] }, + { + "methods": ["GET"], + "pathPattern": "/oai/request-metadata/{requestId}/logs", + "permissionsRequired": ["oai-pmh.request-metadata.logs.item.get"] + }, { "methods": [ "POST" @@ -273,6 +278,11 @@ "displayName": "OAI-PMH Request metadata - get collection of suppressed from discovery instances UUIDs", "description": "Retrieves request metadata collection - suppressed from discovery instances UUIDs" }, + { + "permissionName": "oai-pmh.request-metadata.logs.item.get", + "displayName": "OAI-PMH Request metadata - download error log", + "description": "Downloads error log by request id" + }, { "permissionName": "oai-pmh.all", "displayName": "OAI-PMH - all permissions", @@ -289,7 +299,8 @@ "oai-pmh.request-metadata.failed-to-save-instances.collection.get", "oai-pmh.request-metadata.failed-instances.collection.get", "oai-pmh.request-metadata.skipped-instances.collection.get", - "oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get" + "oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get", + "oai-pmh.request-metadata.logs.item.get" ] }, { diff --git a/ramls/request-metadata.raml b/ramls/request-metadata.raml index e49ef90dc..7ca433f1f 100644 --- a/ramls/request-metadata.raml +++ b/ramls/request-metadata.raml @@ -83,3 +83,21 @@ resourceTypes: get: description: Get list of suppressed from discovery instances UUIDs is: [ pageable ] + + /{requestId}/logs: + description: Service that allows to retrieve error log by request id + get: + responses: + 200: + body: + binary/octet-stream: + 404: + description: "Not found" + body: + text/plain: + example: "Not found" + 500: + description: "Internal server error, e.g. due to misconfiguration" + body: + text/plain: + example: "Internal server error, contact administrator" diff --git a/src/main/java/org/folio/oaipmh/dao/impl/InstancesDaoImpl.java b/src/main/java/org/folio/oaipmh/dao/impl/InstancesDaoImpl.java index dffc3991d..6d21d410f 100644 --- a/src/main/java/org/folio/oaipmh/dao/impl/InstancesDaoImpl.java +++ b/src/main/java/org/folio/oaipmh/dao/impl/InstancesDaoImpl.java @@ -44,10 +44,8 @@ import org.folio.rest.jooq.tables.records.RequestMetadataLbRecord; import org.folio.rest.jooq.tables.records.SkippedInstancesIdsRecord; import org.folio.rest.jooq.tables.records.SuppressedFromDiscoveryInstancesIdsRecord; -import org.folio.s3.client.FolioS3Client; import org.jooq.InsertValuesStep3; import org.jooq.Record; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import io.github.jklingsporn.vertx.jooq.classic.reactivepg.ReactiveClassicGenericQueryExecutor; @@ -69,9 +67,6 @@ public InstancesDaoImpl(PostgresClientFactory postgresClientFactory) { this.postgresClientFactory = postgresClientFactory; } - @Autowired - private FolioS3Client folioS3Client; - @Override public Future> getExpiredRequestIds(String tenantId, long expirationPeriodInSeconds) { OffsetDateTime offsetDateTime = ZonedDateTime.ofInstant(Instant.now(), ZoneId.systemDefault()) @@ -311,6 +306,7 @@ private Record toDatabaseRecord(RequestMetadataLb requestMetadata) { return new RequestMetadataLbRecord().setRequestId(requestMetadata.getRequestId()) .setLastUpdatedDate(requestMetadata.getLastUpdatedDate()) .setStreamEnded(requestMetadata.getStreamEnded()).setLinkToErrorFile(requestMetadata.getLinkToErrorFile()) + .setPathToErrorFileInS3(requestMetadata.getPathToErrorFileInS3()) .setStartedDate(requestMetadata.getStartedDate()); } @@ -480,8 +476,7 @@ private RequestMetadata rowToRequestMetadata(Row row) { of(pojo.getSuppressedInstancesCounter()).ifPresent(requestMetadata::withSuppressedInstancesCounter); ofNullable(pojo.getPathToErrorFileInS3()).ifPresentOrElse(pathToError -> { if (!pathToError.isEmpty()) { - var regeneratedLink = folioS3Client.getPresignedUrl(pathToError); - requestMetadata.withLinkToErrorFile(regeneratedLink); + requestMetadata.withLinkToErrorFile(pathToError); } else { requestMetadata.setLinkToErrorFile(""); } diff --git a/src/main/java/org/folio/rest/impl/RequestMetadataAPIs.java b/src/main/java/org/folio/rest/impl/RequestMetadataAPIs.java index 4e14217a9..a792db51c 100644 --- a/src/main/java/org/folio/rest/impl/RequestMetadataAPIs.java +++ b/src/main/java/org/folio/rest/impl/RequestMetadataAPIs.java @@ -1,7 +1,9 @@ package org.folio.rest.impl; +import java.io.IOException; import java.util.Map; +import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import org.apache.logging.log4j.LogManager; @@ -9,7 +11,9 @@ import org.folio.dataimport.util.ExceptionHelper; import org.folio.oaipmh.dao.InstancesDao; import org.folio.rest.jaxrs.resource.OaiRequestMetadata; +import org.folio.rest.jooq.tables.pojos.RequestMetadataLb; import org.folio.rest.tools.utils.TenantTool; +import org.folio.s3.client.FolioS3Client; import org.folio.spring.SpringContextUtil; import org.springframework.beans.factory.annotation.Autowired; @@ -24,11 +28,15 @@ public class RequestMetadataAPIs implements OaiRequestMetadata { private static final Logger logger = LogManager.getLogger(RequestMetadataAPIs.class); private static final String REQUEST_METADATA_ERROR_MESSAGE_TEMPLATE = "Error occurred while get request metadata. Message: {}."; private static final String UUID_COLLECTION_ERROR_MESSAGE_TEMPLATE = "Error occurred while get UUIDs collection. Message: {}."; + private static final String DOWNLOAD_LOG_ERROR_MESSAGE_TEMPLATE = "Error occurred while downloading log. Message: {}."; @Autowired InstancesDao instancesDao; + @Autowired + FolioS3Client folioS3Client; + public RequestMetadataAPIs() { SpringContextUtil.autowireDependencies(this, Vertx.currentContext()); } @@ -116,4 +124,38 @@ public void getOaiRequestMetadataSuppressedFromDiscoveryInstancesByRequestId(Str asyncResultHandler.handle(Future.succeededFuture(ExceptionHelper.mapExceptionToResponse(e))); } } + + @Override + public void getOaiRequestMetadataLogsByRequestId(String requestId, Map okapiHeaders, Handler> asyncResultHandler, Context vertxContext) { + vertxContext.runOnContext(v -> { + try { + var tenantId = TenantTool.tenantId(okapiHeaders); + logger.info("Download error log for tenant: {}, requestId: {}", tenantId, requestId); + instancesDao.getRequestMetadataByRequestId(requestId, tenantId) + .map(RequestMetadataLb::getPathToErrorFileInS3) + .map(this::prepareResponseData) + .map(Response.class::cast) + .otherwise(ExceptionHelper::mapExceptionToResponse) + .onComplete(asyncResultHandler); + } catch (Exception e) { + logger.error(DOWNLOAD_LOG_ERROR_MESSAGE_TEMPLATE, e.getMessage()); + asyncResultHandler.handle(Future.succeededFuture(ExceptionHelper.mapExceptionToResponse(e))); + } + }); + } + + private GetOaiRequestMetadataLogsByRequestIdResponse prepareResponseData(String pathToErrorFile) { + var response = GetOaiRequestMetadataLogsByRequestIdResponse.respond200WithBinaryOctetStream(new String(readFile(pathToErrorFile))); + var headers = response.getHeaders(); + headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + pathToErrorFile); + return response; + } + + private byte[] readFile(String path) { + try (var is = folioS3Client.read(path)) { + return is.readAllBytes(); + } catch (IOException e) { + return new byte[0]; + } + } } diff --git a/src/test/java/org/folio/oaipmh/service/impl/ErrorsServiceImplTest.java b/src/test/java/org/folio/oaipmh/service/impl/ErrorsServiceImplTest.java index 2d03d1834..2d5c6e6e8 100644 --- a/src/test/java/org/folio/oaipmh/service/impl/ErrorsServiceImplTest.java +++ b/src/test/java/org/folio/oaipmh/service/impl/ErrorsServiceImplTest.java @@ -23,6 +23,7 @@ import org.folio.rest.persist.PostgresClient; import org.folio.rest.tools.utils.ModuleName; import org.folio.rest.tools.utils.NetworkUtils; +import org.folio.s3.client.FolioS3Client; import org.folio.spring.SpringContextUtil; import org.junit.jupiter.api.AfterAll; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -65,6 +66,9 @@ public class ErrorsServiceImplTest extends AbstractErrorsTest { public static final String BUCKET = "test-bucket"; public static final String REGION = "us-west-2"; + @Autowired + private FolioS3Client folioS3Client; + static { s3 = new GenericContainer<>("minio/minio:latest") .withEnv("MINIO_ACCESS_KEY", S3_ACCESS_KEY) @@ -271,7 +275,7 @@ void shouldDeleteErrorsByRequestId_whenErrorFound(VertxTestContext testContext) } private void verifyErrorCSVFile(String linkToError, List initErrorFileContent) { - try (InputStream inputStream = new URL(linkToError).openStream(); + try (InputStream inputStream = folioS3Client.read(linkToError); Scanner scanner = new Scanner(inputStream)) { List listCsvLines = new ArrayList<>(); while (scanner.hasNextLine()) { diff --git a/src/test/java/org/folio/rest/impl/OaiPmhImplTest.java b/src/test/java/org/folio/rest/impl/OaiPmhImplTest.java index f3973004c..c398a0211 100644 --- a/src/test/java/org/folio/rest/impl/OaiPmhImplTest.java +++ b/src/test/java/org/folio/rest/impl/OaiPmhImplTest.java @@ -38,9 +38,11 @@ import org.folio.rest.RestVerticle; import org.folio.rest.jaxrs.model.RequestMetadataCollection; import org.folio.rest.jaxrs.model.UuidCollection; +import org.folio.rest.jooq.tables.pojos.RequestMetadataLb; import org.folio.rest.persist.PostgresClient; import org.folio.rest.tools.utils.ModuleName; import org.folio.rest.tools.utils.NetworkUtils; +import org.folio.s3.client.FolioS3Client; import org.folio.spring.SpringContextUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.AfterAll; @@ -68,16 +70,22 @@ import org.openarchives.oai._2_0.oai_identifier.OaiIdentifier; import org.purl.dc.elements._1.ElementType; import org.springframework.beans.factory.annotation.Autowired; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import javax.annotation.concurrent.NotThreadSafe; import javax.xml.bind.JAXBElement; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.math.BigInteger; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.time.Instant; import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; import java.time.ZoneOffset; import java.util.Arrays; import java.util.Base64; @@ -86,6 +94,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.UUID; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -251,8 +260,39 @@ class OaiPmhImplTest { private PostgresTesterContainer postgresTesterContainer; + @Autowired + private FolioS3Client folioS3Client; + + // --- s3 setup --- + private static final GenericContainer s3; + private static final String MINIO_ENDPOINT; + public static final String S3_ACCESS_KEY = "minio-access-key"; + public static final String S3_SECRET_KEY = "minio-secret-key"; + public static final int S3_PORT = 9000; + public static final String BUCKET = "test-bucket"; + public static final String REGION = "us-west-2"; + + static { + s3 = new GenericContainer<>("minio/minio:latest") + .withEnv("MINIO_ACCESS_KEY", S3_ACCESS_KEY) + .withEnv("MINIO_SECRET_KEY", S3_SECRET_KEY) + .withCommand("server /data") + .withExposedPorts(S3_PORT) + .waitingFor(new HttpWaitStrategy().forPath("/minio/health/ready") + .forPort(S3_PORT) + .withStartupTimeout(Duration.ofSeconds(10)) + ); + s3.start(); + MINIO_ENDPOINT = format("http://%s:%s", s3.getHost(), s3.getFirstMappedPort()); + } + @BeforeAll void setUpOnce(Vertx vertx, VertxTestContext testContext) { + System.setProperty("minio.bucket", BUCKET); + System.setProperty("minio.region", REGION); + System.setProperty("minio.accessKey", S3_ACCESS_KEY); + System.setProperty("minio.secretKey", S3_SECRET_KEY); + System.setProperty("minio.endpoint", MINIO_ENDPOINT); resetSystemProperties(); VertxOptions options = new VertxOptions(); options.setBlockedThreadCheckInterval(1000*60*60); @@ -2943,6 +2983,35 @@ private List getHeadersListDependOnVerbType(VerbType verb, OAIPMH oa .getHeaders(); } + @Test + void downloadLog(VertxTestContext testContext) { + var requestId = UUID.randomUUID(); + var pathToError = "log.csv"; + var requestMetadata = new RequestMetadataLb(); + requestMetadata.setRequestId(requestId); + requestMetadata.setPathToErrorFileInS3(pathToError); + requestMetadata.setStartedDate(OffsetDateTime.now(ZoneId.systemDefault())); + requestMetadata.setLastUpdatedDate(OffsetDateTime.now(ZoneId.systemDefault())); + var expectedValue = "log data"; + + folioS3Client.write(pathToError, new ByteArrayInputStream(expectedValue.getBytes())); + instancesService.saveRequestMetadata(requestMetadata, OAI_TEST_TENANT) + .map(RequestMetadataLb::getRequestId) + .map(uuid -> RestAssured.given() + .header(okapiUrlHeader) + .header(tokenHeader) + .header(tenantHeader) + .basePath(REQUEST_METADATA_PATH + "/" + uuid + "/logs") + .when() + .get()) + .onSuccess(res -> { + Assertions.assertEquals(expectedValue, new String(res.getBody().asByteArray())); + testContext.completeNow(); + }) + .onFailure(testContext::failNow); + + } + private RequestMetadataCollection getRequestMetadataCollection(int limit) { return RestAssured.given() .header(okapiUrlHeader)