diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json index 320c1580b..4f2549dcb 100644 --- a/descriptors/ModuleDescriptor-template.json +++ b/descriptors/ModuleDescriptor-template.json @@ -101,6 +101,11 @@ "pathPattern": "/oai/request-metadata/{requestId}/suppressed-from-discovery-instances", "permissionsRequired": ["oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get"] }, + { + "methods": ["GET"], + "pathPattern": "/oai/request-metadata/{requestId}/logs", + "permissionsRequired": ["oai-pmh.request-metadata.logs.item.get"] + }, { "methods": [ "POST" @@ -273,6 +278,11 @@ "displayName": "OAI-PMH Request metadata - get collection of suppressed from discovery instances UUIDs", "description": "Retrieves request metadata collection - suppressed from discovery instances UUIDs" }, + { + "permissionName": "oai-pmh.request-metadata.logs.item.get", + "displayName": "OAI-PMH Request metadata - download error log", + "description": "Downloads error log by request id" + }, { "permissionName": "oai-pmh.all", "displayName": "OAI-PMH - all permissions", @@ -289,7 +299,8 @@ "oai-pmh.request-metadata.failed-to-save-instances.collection.get", "oai-pmh.request-metadata.failed-instances.collection.get", "oai-pmh.request-metadata.skipped-instances.collection.get", - "oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get" + "oai-pmh.request-metadata.suppressed-from-discovery-instances.collection.get", + "oai-pmh.request-metadata.logs.item.get" ] }, { diff --git a/ramls/request-metadata.raml b/ramls/request-metadata.raml index e49ef90dc..7ca433f1f 100644 --- a/ramls/request-metadata.raml +++ b/ramls/request-metadata.raml @@ -83,3 +83,21 @@ resourceTypes: get: description: Get list of suppressed from discovery instances UUIDs is: [ pageable ] + + /{requestId}/logs: + description: Service that allows to retrieve error log by request id + get: + responses: + 200: + body: + binary/octet-stream: + 404: + description: "Not found" + body: + text/plain: + example: "Not found" + 500: + description: "Internal server error, e.g. due to misconfiguration" + body: + text/plain: + example: "Internal server error, contact administrator" diff --git a/src/main/java/org/folio/oaipmh/dao/impl/InstancesDaoImpl.java b/src/main/java/org/folio/oaipmh/dao/impl/InstancesDaoImpl.java index dffc3991d..6d21d410f 100644 --- a/src/main/java/org/folio/oaipmh/dao/impl/InstancesDaoImpl.java +++ b/src/main/java/org/folio/oaipmh/dao/impl/InstancesDaoImpl.java @@ -44,10 +44,8 @@ import org.folio.rest.jooq.tables.records.RequestMetadataLbRecord; import org.folio.rest.jooq.tables.records.SkippedInstancesIdsRecord; import org.folio.rest.jooq.tables.records.SuppressedFromDiscoveryInstancesIdsRecord; -import org.folio.s3.client.FolioS3Client; import org.jooq.InsertValuesStep3; import org.jooq.Record; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import io.github.jklingsporn.vertx.jooq.classic.reactivepg.ReactiveClassicGenericQueryExecutor; @@ -69,9 +67,6 @@ public InstancesDaoImpl(PostgresClientFactory postgresClientFactory) { this.postgresClientFactory = postgresClientFactory; } - @Autowired - private FolioS3Client folioS3Client; - @Override public Future> getExpiredRequestIds(String tenantId, long expirationPeriodInSeconds) { OffsetDateTime offsetDateTime = ZonedDateTime.ofInstant(Instant.now(), ZoneId.systemDefault()) @@ -311,6 +306,7 @@ private Record toDatabaseRecord(RequestMetadataLb requestMetadata) { return new RequestMetadataLbRecord().setRequestId(requestMetadata.getRequestId()) .setLastUpdatedDate(requestMetadata.getLastUpdatedDate()) .setStreamEnded(requestMetadata.getStreamEnded()).setLinkToErrorFile(requestMetadata.getLinkToErrorFile()) + .setPathToErrorFileInS3(requestMetadata.getPathToErrorFileInS3()) .setStartedDate(requestMetadata.getStartedDate()); } @@ -480,8 +476,7 @@ private RequestMetadata rowToRequestMetadata(Row row) { of(pojo.getSuppressedInstancesCounter()).ifPresent(requestMetadata::withSuppressedInstancesCounter); ofNullable(pojo.getPathToErrorFileInS3()).ifPresentOrElse(pathToError -> { if (!pathToError.isEmpty()) { - var regeneratedLink = folioS3Client.getPresignedUrl(pathToError); - requestMetadata.withLinkToErrorFile(regeneratedLink); + requestMetadata.withLinkToErrorFile(pathToError); } else { requestMetadata.setLinkToErrorFile(""); } diff --git a/src/main/java/org/folio/rest/impl/RequestMetadataAPIs.java b/src/main/java/org/folio/rest/impl/RequestMetadataAPIs.java index 4e14217a9..a792db51c 100644 --- a/src/main/java/org/folio/rest/impl/RequestMetadataAPIs.java +++ b/src/main/java/org/folio/rest/impl/RequestMetadataAPIs.java @@ -1,7 +1,9 @@ package org.folio.rest.impl; +import java.io.IOException; import java.util.Map; +import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import org.apache.logging.log4j.LogManager; @@ -9,7 +11,9 @@ import org.folio.dataimport.util.ExceptionHelper; import org.folio.oaipmh.dao.InstancesDao; import org.folio.rest.jaxrs.resource.OaiRequestMetadata; +import org.folio.rest.jooq.tables.pojos.RequestMetadataLb; import org.folio.rest.tools.utils.TenantTool; +import org.folio.s3.client.FolioS3Client; import org.folio.spring.SpringContextUtil; import org.springframework.beans.factory.annotation.Autowired; @@ -24,11 +28,15 @@ public class RequestMetadataAPIs implements OaiRequestMetadata { private static final Logger logger = LogManager.getLogger(RequestMetadataAPIs.class); private static final String REQUEST_METADATA_ERROR_MESSAGE_TEMPLATE = "Error occurred while get request metadata. Message: {}."; private static final String UUID_COLLECTION_ERROR_MESSAGE_TEMPLATE = "Error occurred while get UUIDs collection. Message: {}."; + private static final String DOWNLOAD_LOG_ERROR_MESSAGE_TEMPLATE = "Error occurred while downloading log. Message: {}."; @Autowired InstancesDao instancesDao; + @Autowired + FolioS3Client folioS3Client; + public RequestMetadataAPIs() { SpringContextUtil.autowireDependencies(this, Vertx.currentContext()); } @@ -116,4 +124,38 @@ public void getOaiRequestMetadataSuppressedFromDiscoveryInstancesByRequestId(Str asyncResultHandler.handle(Future.succeededFuture(ExceptionHelper.mapExceptionToResponse(e))); } } + + @Override + public void getOaiRequestMetadataLogsByRequestId(String requestId, Map okapiHeaders, Handler> asyncResultHandler, Context vertxContext) { + vertxContext.runOnContext(v -> { + try { + var tenantId = TenantTool.tenantId(okapiHeaders); + logger.info("Download error log for tenant: {}, requestId: {}", tenantId, requestId); + instancesDao.getRequestMetadataByRequestId(requestId, tenantId) + .map(RequestMetadataLb::getPathToErrorFileInS3) + .map(this::prepareResponseData) + .map(Response.class::cast) + .otherwise(ExceptionHelper::mapExceptionToResponse) + .onComplete(asyncResultHandler); + } catch (Exception e) { + logger.error(DOWNLOAD_LOG_ERROR_MESSAGE_TEMPLATE, e.getMessage()); + asyncResultHandler.handle(Future.succeededFuture(ExceptionHelper.mapExceptionToResponse(e))); + } + }); + } + + private GetOaiRequestMetadataLogsByRequestIdResponse prepareResponseData(String pathToErrorFile) { + var response = GetOaiRequestMetadataLogsByRequestIdResponse.respond200WithBinaryOctetStream(new String(readFile(pathToErrorFile))); + var headers = response.getHeaders(); + headers.add(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + pathToErrorFile); + return response; + } + + private byte[] readFile(String path) { + try (var is = folioS3Client.read(path)) { + return is.readAllBytes(); + } catch (IOException e) { + return new byte[0]; + } + } } diff --git a/src/test/java/org/folio/oaipmh/service/impl/ErrorsServiceImplTest.java b/src/test/java/org/folio/oaipmh/service/impl/ErrorsServiceImplTest.java index 2d03d1834..2d5c6e6e8 100644 --- a/src/test/java/org/folio/oaipmh/service/impl/ErrorsServiceImplTest.java +++ b/src/test/java/org/folio/oaipmh/service/impl/ErrorsServiceImplTest.java @@ -23,6 +23,7 @@ import org.folio.rest.persist.PostgresClient; import org.folio.rest.tools.utils.ModuleName; import org.folio.rest.tools.utils.NetworkUtils; +import org.folio.s3.client.FolioS3Client; import org.folio.spring.SpringContextUtil; import org.junit.jupiter.api.AfterAll; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -65,6 +66,9 @@ public class ErrorsServiceImplTest extends AbstractErrorsTest { public static final String BUCKET = "test-bucket"; public static final String REGION = "us-west-2"; + @Autowired + private FolioS3Client folioS3Client; + static { s3 = new GenericContainer<>("minio/minio:latest") .withEnv("MINIO_ACCESS_KEY", S3_ACCESS_KEY) @@ -271,7 +275,7 @@ void shouldDeleteErrorsByRequestId_whenErrorFound(VertxTestContext testContext) } private void verifyErrorCSVFile(String linkToError, List initErrorFileContent) { - try (InputStream inputStream = new URL(linkToError).openStream(); + try (InputStream inputStream = folioS3Client.read(linkToError); Scanner scanner = new Scanner(inputStream)) { List listCsvLines = new ArrayList<>(); while (scanner.hasNextLine()) { diff --git a/src/test/java/org/folio/rest/impl/OaiPmhImplTest.java b/src/test/java/org/folio/rest/impl/OaiPmhImplTest.java index f3973004c..e75fbcdac 100644 --- a/src/test/java/org/folio/rest/impl/OaiPmhImplTest.java +++ b/src/test/java/org/folio/rest/impl/OaiPmhImplTest.java @@ -38,9 +38,11 @@ import org.folio.rest.RestVerticle; import org.folio.rest.jaxrs.model.RequestMetadataCollection; import org.folio.rest.jaxrs.model.UuidCollection; +import org.folio.rest.jooq.tables.pojos.RequestMetadataLb; import org.folio.rest.persist.PostgresClient; import org.folio.rest.tools.utils.ModuleName; import org.folio.rest.tools.utils.NetworkUtils; +import org.folio.s3.client.FolioS3Client; import org.folio.spring.SpringContextUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.AfterAll; @@ -54,6 +56,8 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.Spy; import org.openarchives.oai._2.GranularityType; import org.openarchives.oai._2.HeaderType; @@ -71,6 +75,7 @@ import javax.annotation.concurrent.NotThreadSafe; import javax.xml.bind.JAXBElement; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.math.BigInteger; @@ -78,6 +83,8 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; import java.time.ZoneOffset; import java.util.Arrays; import java.util.Base64; @@ -86,6 +93,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.UUID; import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -176,6 +184,7 @@ import static org.junit.Assert.fail; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.mockito.Mockito.when; import static org.openarchives.oai._2.OAIPMHerrorcodeType.BAD_ARGUMENT; import static org.openarchives.oai._2.OAIPMHerrorcodeType.BAD_RESUMPTION_TOKEN; import static org.openarchives.oai._2.OAIPMHerrorcodeType.CANNOT_DISSEMINATE_FORMAT; @@ -251,8 +260,12 @@ class OaiPmhImplTest { private PostgresTesterContainer postgresTesterContainer; + @Mock + private FolioS3Client folioS3Client; + @BeforeAll void setUpOnce(Vertx vertx, VertxTestContext testContext) { + folioS3Client = Mockito.mock(FolioS3Client.class); resetSystemProperties(); VertxOptions options = new VertxOptions(); options.setBlockedThreadCheckInterval(1000*60*60); @@ -2943,6 +2956,38 @@ private List getHeadersListDependOnVerbType(VerbType verb, OAIPMH oa .getHeaders(); } + @Test + void downloadLog(VertxTestContext testContext) { + var requestId = UUID.randomUUID(); + var pathToError = "log.csv"; + var requestMetadata = new RequestMetadataLb(); + requestMetadata.setRequestId(requestId); + requestMetadata.setPathToErrorFileInS3(pathToError); + requestMetadata.setStartedDate(OffsetDateTime.now(ZoneId.systemDefault())); + requestMetadata.setLastUpdatedDate(OffsetDateTime.now(ZoneId.systemDefault())); + + when(folioS3Client.read(pathToError)).thenReturn(new ByteArrayInputStream("log".getBytes())); + + testContext.verify(() -> { + instancesService.saveRequestMetadata(requestMetadata, OAI_TEST_TENANT) + .map(RequestMetadataLb::getRequestId) + .map(uuid -> RestAssured.given() + .header(okapiUrlHeader) + .header(tokenHeader) + .header(tenantHeader) + .basePath(REQUEST_METADATA_PATH + "/" + uuid + "/logs") + .when() + .get() + .then() + .statusCode(200) + .extract() + .as(String.class)) + .onComplete(res -> assertEquals("log", res.result())); + testContext.completeNow(); + }); + + } + private RequestMetadataCollection getRequestMetadataCollection(int limit) { return RestAssured.given() .header(okapiUrlHeader)