From 2fed6a585873f75105b6de7d7088a9e71710a46d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Sat, 3 Feb 2024 10:24:35 +0100 Subject: [PATCH 01/25] MVP of Logginghouse extension (0.2) (#19) * Ignore some java helper files * feat: Added MVP * fix: dependency to ids model * fix: logging messages * fix(ci): add username and token to build job --------- Co-authored-by: dhommen --- .github/workflows/build-and-publish-edc.yml | 3 + .gitignore | 3 + build.gradle.kts | 10 + gradle.properties | 2 +- logging-house-client/build.gradle.kts | 5 + .../client/CreateProcessMessage.java | 39 +++ .../client/CreateProcessMessageSender.java | 69 ++++++ .../ExtendedMessageProtocolClearing.java | 25 ++ .../client/IdsClearingHouseServiceImpl.java | 126 ++++++++++ ...tipartClearingRemoteMessageDispatcher.java | 30 +++ .../logginghouse/client/LogMessage.java | 34 +++ .../logginghouse/client/LogMessageSender.java | 97 ++++++++ .../client/LoggingHouseClientExtension.java | 74 ++++++ .../client/MultiContextJsonLdSerializer.java | 132 ++++++++++ .../client/ids/jsonld/JsonLd.java | 41 ++++ .../client/ids/jsonld/JsonLdModule.java | 32 +++ .../client/ids/jsonld/UriDeserializer.java | 51 ++++ .../client/ids/jsonld/UriSerializer.java | 43 ++++ .../XmlGregorianCalendarDeserializer.java | 47 ++++ .../XmlGregorianCalendarSerializer.java | 42 ++++ .../client/ids/multipart/CalendarUtil.java | 34 +++ .../client/ids/multipart/IdsConstants.java | 24 ++ .../ids/multipart/IdsMultipartParts.java | 74 ++++++ .../IdsMultipartRemoteMessageDispatcher.java | 72 ++++++ .../ids/multipart/IdsMultipartSender.java | 226 ++++++++++++++++++ .../ids/multipart/MultipartResponse.java | 27 +++ .../multipart/MultipartSenderDelegate.java | 34 +++ .../client/ids/multipart/ResponseUtil.java | 35 +++ 28 files changed, 1430 insertions(+), 1 deletion(-) create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessage.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsMultipartClearingRemoteMessageDispatcher.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/MultiContextJsonLdSerializer.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLd.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLdModule.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriDeserializer.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriSerializer.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarDeserializer.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarSerializer.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/CalendarUtil.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsConstants.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartParts.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartRemoteMessageDispatcher.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartSender.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartResponse.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartSenderDelegate.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/ResponseUtil.java diff --git a/.github/workflows/build-and-publish-edc.yml b/.github/workflows/build-and-publish-edc.yml index a6092ed..97bd169 100644 --- a/.github/workflows/build-and-publish-edc.yml +++ b/.github/workflows/build-and-publish-edc.yml @@ -25,6 +25,9 @@ jobs: uses: gradle/gradle-build-action@749f47bda3e44aa060e82d7b3ef7e40d953bd629 with: arguments: build + env: + USERNAME: ${{ github.actor }} + TOKEN: ${{ github.token }} - name: Publish package if: ${{ github.ref == 'refs/heads/main' }} uses: gradle/gradle-build-action@749f47bda3e44aa060e82d7b3ef7e40d953bd629 diff --git a/.gitignore b/.gitignore index a455c71..79354d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ .DS_Store .idea/ +/**/.classpath +/**/.settings +/**/.project # Ignore Gradle project-specific cache directory .gradle diff --git a/build.gradle.kts b/build.gradle.kts index 06b15a4..fcfd296 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -41,6 +41,16 @@ allprojects { repositories { mavenCentral() mavenLocal() + repositories { + maven { + name = "GitHubPackages" + url = uri("https://maven.pkg.github.com/ids-basecamp/ids-infomodel-java") + credentials { + username = System.getenv("USERNAME") + password = System.getenv("TOKEN") + } + } + } } } diff --git a/gradle.properties b/gradle.properties index 0d39a2a..1132f36 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,7 +5,7 @@ assertj=3.23.1 jupiterVersion=5.8.2 mockitoVersion=4.8.0 okHttpVersion=4.10.0 -jsonVersion=20230618 +jsonVersion=20231013 jettyGroup=org.eclipse.jetty jettyVersion=11.0.15 diff --git a/logging-house-client/build.gradle.kts b/logging-house-client/build.gradle.kts index 636d282..07c9610 100644 --- a/logging-house-client/build.gradle.kts +++ b/logging-house-client/build.gradle.kts @@ -13,9 +13,14 @@ val jsonVersion: String by project dependencies { implementation("${edcGroup}:control-plane-core:${edcVersion}") + implementation("${edcGroup}:http-spi:${edcVersion}") implementation("com.squareup.okhttp3:okhttp:${okHttpVersion}") implementation("org.json:json:${jsonVersion}") + implementation("org.glassfish.jersey.media:jersey-media-multipart:3.1.3") + + implementation("de.fraunhofer.iais.eis.ids.infomodel:infomodel-java:1.0.2-basecamp") + implementation("de.fraunhofer.iais.eis.ids.infomodel:infomodel-util:1.0.2-basecamp") testImplementation("org.assertj:assertj-core:${assertj}") testImplementation("org.junit.jupiter:junit-jupiter-api:${jupiterVersion}") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessage.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessage.java new file mode 100644 index 0000000..88854ba --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessage.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024 truzzt GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * truzzt GmbH - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client; + +import org.eclipse.edc.spi.types.domain.message.RemoteMessage; + +import java.net.URI; +import java.net.URL; +import java.util.List; + +public record CreateProcessMessage( + URL clearingHouseLogUrl, + URI connectorBaseUrl, + String processId, + List processOwners +) implements RemoteMessage { + + @Override + public String getProtocol() { + return ExtendedMessageProtocolClearing.IDS_EXTENDED_PROTOCOL_CLEARING; + } + + @Override + public String getCounterPartyAddress() { + return clearingHouseLogUrl.toString(); + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java new file mode 100644 index 0000000..58fa5ef --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2024 truzzt GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * truzzt GmbH - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client; + +import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; +import com.truzzt.extension.logginghouse.client.ids.multipart.CalendarUtil; +import com.truzzt.extension.logginghouse.client.ids.multipart.IdsConstants; +import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartParts; +import com.truzzt.extension.logginghouse.client.ids.multipart.MultipartResponse; +import com.truzzt.extension.logginghouse.client.ids.multipart.MultipartSenderDelegate; +import com.truzzt.extension.logginghouse.client.ids.multipart.ResponseUtil; +import de.fraunhofer.iais.eis.DynamicAttributeToken; +import de.fraunhofer.iais.eis.LogMessageBuilder; +import de.fraunhofer.iais.eis.Message; +import de.fraunhofer.iais.eis.MessageProcessedNotificationMessageImpl; +import org.json.JSONObject; + +import java.util.List; + +public class CreateProcessMessageSender implements MultipartSenderDelegate { + + public CreateProcessMessageSender() { + } + + @Override + public Message buildMessageHeader(CreateProcessMessage createProcessMessage, DynamicAttributeToken token) { + return new LogMessageBuilder() + ._modelVersion_(IdsConstants.INFORMATION_MODEL_VERSION) + ._issued_(CalendarUtil.gregorianNow()) + ._securityToken_(token) + ._issuerConnector_(createProcessMessage.connectorBaseUrl()) + ._senderAgent_(createProcessMessage.connectorBaseUrl()) + .build(); + } + + @Override + public String buildMessagePayload(CreateProcessMessage createProcessMessage) { + var jo = new JSONObject(); + jo.put("owners", createProcessMessage.processOwners()); + return jo.toString(); + } + + @Override + public MultipartResponse getResponseContent(IdsMultipartParts parts) throws Exception { + return ResponseUtil.parseMultipartStringResponse(parts, JsonLd.getObjectMapper()); + } + + @Override + public List> getAllowedResponseTypes() { + return List.of(MessageProcessedNotificationMessageImpl.class); + } + + @Override + public Class getMessageType() { + return CreateProcessMessage.class; + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java new file mode 100644 index 0000000..931fa9d --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client; + +public final class ExtendedMessageProtocolClearing { + + private static final String EXTENDED_SUFFIX = "-extended-clearing"; + public static final String IDS_MULTIPART = "ids-multipart"; + public static final String IDS_EXTENDED_PROTOCOL_CLEARING = String.format("%s%s", IDS_MULTIPART, EXTENDED_SUFFIX); + + private ExtendedMessageProtocolClearing() { + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java new file mode 100644 index 0000000..8ce4ee7 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client; + +import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; +import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; +import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessTerminated; +import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.event.Event; +import org.eclipse.edc.spi.event.EventEnvelope; +import org.eclipse.edc.spi.event.EventSubscriber; +import org.eclipse.edc.spi.message.RemoteMessageDispatcherRegistry; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.Hostname; + +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class IdsClearingHouseServiceImpl implements EventSubscriber { + + private final RemoteMessageDispatcherRegistry dispatcherRegistry; + private final URI connectorBaseUrl; + private final URL clearingHouseLogUrl; + private final ContractNegotiationStore contractNegotiationStore; + private final TransferProcessStore transferProcessStore; + private final Monitor monitor; + + public IdsClearingHouseServiceImpl( + RemoteMessageDispatcherRegistry dispatcherRegistry, + Hostname hostname, + URL clearingHouseLogUrl, + ContractNegotiationStore contractNegotiationStore, + TransferProcessStore transferProcessStore, + Monitor monitor) { + this.dispatcherRegistry = dispatcherRegistry; + this.clearingHouseLogUrl = clearingHouseLogUrl; + this.contractNegotiationStore = contractNegotiationStore; + this.transferProcessStore = transferProcessStore; + this.monitor = monitor; + + try { + connectorBaseUrl = getConnectorBaseUrl(hostname); + } catch (URISyntaxException e) { + throw new EdcException("Could not create connectorBaseUrl. Hostname can be set using:" + + " edc.hostname", e); + } + } + + public void createProcess(ContractAgreement contractAgreement, URL clearingHouseLogUrl) { + // Create PID + List processOwners = new ArrayList<>(); + processOwners.add(contractAgreement.getConsumerId()); + processOwners.add(contractAgreement.getProviderId()); + + monitor.info("Creating Process in LoggingHouse"); + var logMessage = new CreateProcessMessage(clearingHouseLogUrl, connectorBaseUrl, contractAgreement.getId(), processOwners); + dispatcherRegistry.dispatch(Object.class, logMessage); + } + + public void logContractAgreement(ContractAgreement contractAgreement, URL clearingHouseLogUrl) { + monitor.info("Logging contract agreement to LoggingHouse"); + var logMessage = new LogMessage(clearingHouseLogUrl, connectorBaseUrl, contractAgreement); + dispatcherRegistry.dispatch(Object.class, logMessage); + } + + public void logTransferProcess(TransferProcess transferProcess, URL clearingHouseLogUrl) { + monitor.info("Logging transferprocess to LoggingHouse"); + var logMessage = new LogMessage(clearingHouseLogUrl, connectorBaseUrl, transferProcess); + dispatcherRegistry.dispatch(Object.class, logMessage); + } + + @Override + public void on(EventEnvelope event) { + try { + if (event.getPayload() instanceof ContractNegotiationFinalized contractNegotiationFinalized) { + var contractAgreement = resolveContractAgreement(contractNegotiationFinalized); + var pid = contractAgreement.getId(); + var extendedUrl = new URL(clearingHouseLogUrl + "/" + pid); + + createProcess(contractAgreement, clearingHouseLogUrl); + logContractAgreement(contractAgreement, extendedUrl); + } else if (event.getPayload() instanceof TransferProcessTerminated transferProcessTerminated) { + var transferProcess = resolveTransferProcess(transferProcessTerminated); + var pid = transferProcess.getContractId(); + var extendedUrl = new URL(clearingHouseLogUrl + "/" + pid); + logTransferProcess(transferProcess, extendedUrl); + } + } catch (Exception e) { + throw new EdcException("Could not create extended clearinghouse url."); + } + } + + private ContractAgreement resolveContractAgreement(ContractNegotiationFinalized contractNegotiationFinalized) throws NullPointerException { + var contractNegotiationId = contractNegotiationFinalized.getContractNegotiationId(); + var contractNegotiation = contractNegotiationStore.findById(contractNegotiationId); + return Objects.requireNonNull(contractNegotiation).getContractAgreement(); + } + + private TransferProcess resolveTransferProcess(TransferProcessTerminated transferProcessTerminated) { + var transferProcessId = transferProcessTerminated.getTransferProcessId(); + return transferProcessStore.findById(transferProcessId); + } + + private URI getConnectorBaseUrl(Hostname hostname) throws URISyntaxException { + return new URI(String.format("https://%s/", hostname.get())); + } +} \ No newline at end of file diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsMultipartClearingRemoteMessageDispatcher.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsMultipartClearingRemoteMessageDispatcher.java new file mode 100644 index 0000000..6f757cc --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsMultipartClearingRemoteMessageDispatcher.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client; + +import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartRemoteMessageDispatcher; +import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartSender; + +public class IdsMultipartClearingRemoteMessageDispatcher extends IdsMultipartRemoteMessageDispatcher { + + public IdsMultipartClearingRemoteMessageDispatcher(IdsMultipartSender idsMultipartSender) { + super(idsMultipartSender); + } + + @Override + public String protocol() { + return ExtendedMessageProtocolClearing.IDS_EXTENDED_PROTOCOL_CLEARING; + } +} \ No newline at end of file diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java new file mode 100644 index 0000000..0b0e16a --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client; + +import org.eclipse.edc.spi.types.domain.message.RemoteMessage; + +import java.net.URI; +import java.net.URL; + +public record LogMessage(URL clearingHouseLogUrl, + URI connectorBaseUrl, + Object eventToLog) implements RemoteMessage { + @Override + public String getProtocol() { + return ExtendedMessageProtocolClearing.IDS_EXTENDED_PROTOCOL_CLEARING; + } + + @Override + public String getCounterPartyAddress() { + return clearingHouseLogUrl.toString(); + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java new file mode 100644 index 0000000..3fbb1d5 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client; + +import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; +import com.truzzt.extension.logginghouse.client.ids.multipart.CalendarUtil; +import com.truzzt.extension.logginghouse.client.ids.multipart.IdsConstants; +import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartParts; +import com.truzzt.extension.logginghouse.client.ids.multipart.MultipartResponse; +import com.truzzt.extension.logginghouse.client.ids.multipart.MultipartSenderDelegate; +import com.truzzt.extension.logginghouse.client.ids.multipart.ResponseUtil; +import de.fraunhofer.iais.eis.DynamicAttributeToken; +import de.fraunhofer.iais.eis.LogMessageBuilder; +import de.fraunhofer.iais.eis.Message; +import de.fraunhofer.iais.eis.MessageProcessedNotificationMessageImpl; +import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; +import org.eclipse.edc.spi.EdcException; +import org.json.JSONObject; + +import java.util.List; + +public class LogMessageSender implements MultipartSenderDelegate { + + public LogMessageSender() { + } + + @Override + public Message buildMessageHeader(LogMessage logMessage, DynamicAttributeToken token) { + return new LogMessageBuilder() + ._modelVersion_(IdsConstants.INFORMATION_MODEL_VERSION) + ._issued_(CalendarUtil.gregorianNow()) + ._securityToken_(token) + ._issuerConnector_(logMessage.connectorBaseUrl()) + ._senderAgent_(logMessage.connectorBaseUrl()) + .build(); + } + + @Override + public String buildMessagePayload(LogMessage logMessage) { + if (logMessage.eventToLog() instanceof ContractAgreement contractAgreement) { + return buildContractAgreementPayload(contractAgreement); + } else if (logMessage.eventToLog() instanceof TransferProcess transferProcess) { + return buildTransferProcessPayload(transferProcess); + } else { + throw new EdcException(String.format("ObjectType %s not supported in LogMessageSender", + logMessage.eventToLog().getClass())); + } + } + + @Override + public MultipartResponse getResponseContent(IdsMultipartParts parts) throws Exception { + return ResponseUtil.parseMultipartStringResponse(parts, JsonLd.getObjectMapper()); + } + + @Override + public List> getAllowedResponseTypes() { + return List.of(MessageProcessedNotificationMessageImpl.class); + } + + @Override + public Class getMessageType() { + return LogMessage.class; + } + + private String buildContractAgreementPayload(ContractAgreement contractAgreement) { + var jo = new JSONObject(); + jo.put("AgreementId", contractAgreement.getId()); + jo.put("ProviderId", contractAgreement.getProviderId()); + jo.put("ConsumerId", contractAgreement.getConsumerId()); + jo.put("ContractSigningDate", contractAgreement.getContractSigningDate()); + jo.put("Policy", contractAgreement.getPolicy()); + jo.put("AssetId", contractAgreement.getAssetId()); + return jo.toString(); + } + + private String buildTransferProcessPayload(TransferProcess transferProcess) { + var jo = new JSONObject(); + jo.put("transferProcessId", transferProcess.getId()); + var dataRequest = transferProcess.getDataRequest(); + jo.put("contractId", dataRequest.getContractId()); + jo.put("connectorId", dataRequest.getConnectorId()); + return jo.toString(); + } +} \ No newline at end of file diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 7bffb8a..ea01dfa 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -13,11 +13,24 @@ package com.truzzt.extension.logginghouse.client; +import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; +import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartSender; +import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; +import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessTerminated; +import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Setting; import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.event.EventRouter; +import org.eclipse.edc.spi.http.EdcHttpClient; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.message.RemoteMessageDispatcherRegistry; import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.Hostname; import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; import java.net.MalformedURLException; import java.net.URL; @@ -25,9 +38,27 @@ public class LoggingHouseClientExtension implements ServiceExtension { + public static final String LOGGINGHOUSE_CLIENT_EXTENSION = "LoggingHouseClientExtension"; private static final String TYPE_MANAGER_SERIALIZER_KEY = "ids-clearinghouse"; + @Inject + private TypeManager typeManager; + @Inject + private EventRouter eventRouter; + @Inject + private IdentityService identityService; + @Inject + private RemoteMessageDispatcherRegistry dispatcherRegistry; + + @Inject + private Hostname hostname; + + @Inject + private ContractNegotiationStore contractNegotiationStore; + @Inject + private TransferProcessStore transferProcessStore; + private static final Map CONTEXT_MAP = Map.of( "cat", "http://w3id.org/mds/data-categories#", "ids", "https://w3id.org/idsa/core/", @@ -77,6 +108,49 @@ private URL readUrlFromSettings(ServiceExtensionContext context, String settings } } + private void registerEventSubscriber(ServiceExtensionContext context) { + var eventSubscriber = new IdsClearingHouseServiceImpl( + dispatcherRegistry, + hostname, + clearingHouseLogUrl, + contractNegotiationStore, + transferProcessStore, + monitor); + + eventRouter.registerSync(ContractNegotiationFinalized.class, eventSubscriber); + eventRouter.registerSync(TransferProcessTerminated.class, eventSubscriber); + context.registerService(IdsClearingHouseServiceImpl.class, eventSubscriber); + } + + private void registerSerializerClearingHouseMessages(ServiceExtensionContext context) { + typeManager.registerContext(TYPE_MANAGER_SERIALIZER_KEY, JsonLd.getObjectMapper()); + registerCommonTypes(typeManager); + } + + private void registerCommonTypes(TypeManager typeManager) { + typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, LogMessage.class, + new MultiContextJsonLdSerializer<>(LogMessage.class, CONTEXT_MAP)); + typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, CreateProcessMessage.class, + new MultiContextJsonLdSerializer<>(CreateProcessMessage.class, CONTEXT_MAP)); + } + + private void registerClearingHouseMessageSenders(ServiceExtensionContext context) { + var httpClient = context.getService(EdcHttpClient.class); + var monitor = context.getMonitor(); + var objectMapper = typeManager.getMapper(TYPE_MANAGER_SERIALIZER_KEY); + + var logMessageSender = new LogMessageSender(); + var createProcessMessageSender = new CreateProcessMessageSender(); + + var idsMultipartSender = new IdsMultipartSender(monitor, httpClient, identityService, objectMapper); + var dispatcher = new IdsMultipartClearingRemoteMessageDispatcher(idsMultipartSender); + dispatcher.register(logMessageSender); + dispatcher.register(createProcessMessageSender); + + dispatcherRegistry.register(dispatcher); + } + + @Override public void start() { monitor.info("Starting Logginghouse client extension."); diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/MultiContextJsonLdSerializer.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/MultiContextJsonLdSerializer.java new file mode 100644 index 0000000..985dfb2 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/MultiContextJsonLdSerializer.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * sovity GmbH - Adaption and changes + * + */ + +package com.truzzt.extension.logginghouse.client; + +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.ser.BeanSerializerFactory; + +import java.io.IOException; +import java.net.URI; +import java.util.Map; + +/** + * Custom Jackson serializer for any {@link Object}. Adds type and context information to result object. + * + * @param The object that should be serialized. + */ +public class MultiContextJsonLdSerializer extends JsonSerializer { + private final Class type; + private final Object contextInformation; + + private static final ThreadLocal CURRENT_RECURSION_DEPTH = ThreadLocal.withInitial(() -> 0); + + public MultiContextJsonLdSerializer(Class type, Object contextInformation) { + this.type = type; + this.contextInformation = contextInformation; + } + + @Override + public void serialize(T value, JsonGenerator generator, SerializerProvider provider) throws IOException { + CURRENT_RECURSION_DEPTH.set(CURRENT_RECURSION_DEPTH.get() + 1); + + Map propertiesMap = null; + try { + var field = value.getClass().getDeclaredField("properties"); + field.setAccessible(true); + propertiesMap = (Map) field.get(value); + } catch (NoSuchFieldException | IllegalAccessException ignore) { + // empty + } + removeProperty(value, "properties"); + + // remove properties "comment" and "label" + removeProperty(value, "comment"); + removeProperty(value, "label"); + + generator.writeStartObject(); + + // write new object + var serializer = instantiateSerializerFromProvider(provider, type); + serializer.unwrappingSerializer(null).serialize(value, generator, provider); + + if (CURRENT_RECURSION_DEPTH.get() == 1) { + // context needed only once (for parent object) + generator.writeObjectField("@context", contextInformation); + } + + // add type property + var type = getTypeName(value.getClass()); + if (type != null) { + generator.writeObjectField("@type", type); + } + + // add custom properties as root properties (not in a separate "properties" map) + if (propertiesMap != null) { + for (var key : propertiesMap.keySet()) { + var val = propertiesMap.get(key); + if (val instanceof URI) { + generator.writeStringField(key, val.toString()); + } else { + generator.writeObjectField(key, val); + } + } + } + + generator.writeEndObject(); + + CURRENT_RECURSION_DEPTH.set(CURRENT_RECURSION_DEPTH.get() - 1); + } + + @Override + public void serializeWithType(T value, JsonGenerator gen, SerializerProvider provider, TypeSerializer ser) throws IOException { + serialize(value, gen, provider); + } + + private void removeProperty(Object value, String name) { + try { + var field = value.getClass().getDeclaredField(name); + field.setAccessible(true); + field.set(value, null); + } catch (NoSuchFieldException | IllegalAccessException ignore) { + // empty + } + } + + private String getTypeName(Class clazz) { + var typeName = clazz.getAnnotation(JsonTypeName.class); + if (typeName != null) { + var value = typeName.value(); + if (value == null) { + getTypeName(clazz.getSuperclass()); + } + return value; + } + return null; + } + + private JsonSerializer instantiateSerializerFromProvider(SerializerProvider provider, Class type) throws JsonMappingException { + var javaType = provider.constructType(type); + var beanDescription = provider.getConfig().introspect(javaType); + var staticTyping = provider.isEnabled(MapperFeature.USE_STATIC_TYPING); + return BeanSerializerFactory.instance.findBeanOrAddOnSerializer(provider, javaType, beanDescription, staticTyping); + } +} \ No newline at end of file diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLd.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLd.java new file mode 100644 index 0000000..c542ae0 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLd.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.jsonld; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +import java.text.SimpleDateFormat; + +public final class JsonLd { + + public static ObjectMapper getObjectMapper() { + var customMapper = new ObjectMapper(); + customMapper.registerModule(new JavaTimeModule()); + customMapper.registerModule(new JsonLdModule()); + + customMapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")); + customMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + customMapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + customMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + customMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + customMapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + + return customMapper; + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLdModule.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLdModule.java new file mode 100644 index 0000000..a5c4bfe --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLdModule.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.jsonld; + +import com.fasterxml.jackson.databind.module.SimpleModule; +import java.net.URI; +import javax.xml.datatype.XMLGregorianCalendar; + +public class JsonLdModule extends SimpleModule { + + public JsonLdModule() { + super(); + + addSerializer(URI.class, new UriSerializer()); + addDeserializer(URI.class, new UriDeserializer()); + + addSerializer(XMLGregorianCalendar.class, new XmlGregorianCalendarSerializer()); + addDeserializer(XMLGregorianCalendar.class, new XmlGregorianCalendarDeserializer()); + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriDeserializer.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriDeserializer.java new file mode 100644 index 0000000..cf684c0 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriDeserializer.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.jsonld; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.TreeNode; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; + +import java.io.IOException; +import java.net.URI; + +public class UriDeserializer extends StdDeserializer { + + public UriDeserializer() { + super(URI.class); + } + + @Override + public URI deserialize(JsonParser parser, DeserializationContext context) throws IOException { + return URI.create(getValue(parser.readValueAsTree(), parser)); + } + + private String getValue(TreeNode node, JsonParser parser) throws JsonParseException { + if (node instanceof TextNode) { + return ((TextNode) node).textValue(); + } + + if (node instanceof ObjectNode) { + return getValue(node.get("@id"), parser); + } + + throw new JsonParseException(parser, "Could not read URI"); + } + +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriSerializer.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriSerializer.java new file mode 100644 index 0000000..a2e94dc --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriSerializer.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.jsonld; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +import java.io.IOException; +import java.net.URI; + +public class UriSerializer extends StdSerializer { + + public UriSerializer() { + super(URI.class); + } + + @Override + public void serialize(URI value, JsonGenerator gen, SerializerProvider provider) throws IOException { + var serializedUri = value.toString(); + + var context = gen.getOutputContext(); + if (context.getCurrentName() != null && context.getCurrentName().contains("@id")) { + gen.writeString(serializedUri); + } else { + gen.writeStartObject(); + gen.writeStringField("@id", serializedUri); + gen.writeEndObject(); + } + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarDeserializer.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarDeserializer.java new file mode 100644 index 0000000..116c11c --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarDeserializer.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.jsonld; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.node.TextNode; + +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.GregorianCalendar; +import javax.xml.datatype.DatatypeConfigurationException; +import javax.xml.datatype.DatatypeFactory; +import javax.xml.datatype.XMLGregorianCalendar; + +public class XmlGregorianCalendarDeserializer extends StdDeserializer { + private static final long serialVersionUID = 1L; + + public XmlGregorianCalendarDeserializer() { + super(XMLGregorianCalendar.class); + } + + @Override + public XMLGregorianCalendar deserialize(JsonParser parser, DeserializationContext context) throws IOException { + var tree = parser.readValueAsTree(); + + try { + var value = ((TextNode) tree.get("@value")).textValue(); + return DatatypeFactory.newInstance().newXMLGregorianCalendar(GregorianCalendar.from(ZonedDateTime.parse(value))); + } catch (DatatypeConfigurationException e) { + return null; + } + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarSerializer.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarSerializer.java new file mode 100644 index 0000000..d42744f --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarSerializer.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.jsonld; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import javax.xml.datatype.XMLGregorianCalendar; + +public class XmlGregorianCalendarSerializer extends StdSerializer { + + public XmlGregorianCalendarSerializer() { + super(XMLGregorianCalendar.class); + } + + @Override + public void serialize(XMLGregorianCalendar calendar, JsonGenerator generator, SerializerProvider provider) throws IOException { + var sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + sdf.setCalendar(calendar.toGregorianCalendar()); + var formatted = sdf.format(calendar.toGregorianCalendar().getTime()); + + generator.writeStartObject(); + generator.writeStringField("@value", formatted); + generator.writeStringField("@type", "http://www.w3.org/2001/XMLSchema#dateTimeStamp"); + generator.writeEndObject(); + } +} \ No newline at end of file diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/CalendarUtil.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/CalendarUtil.java new file mode 100644 index 0000000..7a7b8bf --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/CalendarUtil.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021 Daimler TSS GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Daimler TSS GmbH - Initial API and Implementation + * Fraunhofer Institute for Software and Systems Engineering - added method + * + */ + +package com.truzzt.extension.logginghouse.client.ids.multipart; + +import java.time.ZonedDateTime; +import java.util.GregorianCalendar; +import javax.xml.datatype.DatatypeConfigurationException; +import javax.xml.datatype.DatatypeFactory; +import javax.xml.datatype.XMLGregorianCalendar; + +public final class CalendarUtil { + + public static XMLGregorianCalendar gregorianNow() { + try { + GregorianCalendar gregorianCalendar = GregorianCalendar.from(ZonedDateTime.now()); + return DatatypeFactory.newInstance().newXMLGregorianCalendar(gregorianCalendar); + } catch (DatatypeConfigurationException e) { + throw new RuntimeException(e); + } + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsConstants.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsConstants.java new file mode 100644 index 0000000..7a927e8 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsConstants.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2020, 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * Fraunhofer Institute for Software and Systems Engineering - add more values + * + */ + +package com.truzzt.extension.logginghouse.client.ids.multipart; + +public final class IdsConstants { + + public static final String INFORMATION_MODEL_VERSION = "4.1.3"; + + public static final String TOKEN_SCOPE = "idsc:IDS_CONNECTOR_ATTRIBUTES_ALL"; + +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartParts.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartParts.java new file mode 100644 index 0000000..72bcfdf --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartParts.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2020, 2021 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.multipart; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.InputStream; +import java.util.Objects; + +public class IdsMultipartParts { + + private final InputStream header; + + @Nullable + private final InputStream payload; + + private IdsMultipartParts(@NotNull InputStream header, @Nullable InputStream payload) { + this.header = header; + this.payload = payload; + } + + @NotNull + public InputStream getHeader() { + return header; + } + + @Nullable + public InputStream getPayload() { + return payload; + } + + public static class Builder { + private InputStream header; + + @Nullable + private InputStream payload; + + private Builder() { + } + + public static Builder newInstance() { + return new Builder(); + } + + public Builder header(InputStream header) { + this.header = header; + return this; + } + + public Builder payload(InputStream payload) { + this.payload = payload; + return this; + } + + public IdsMultipartParts build() { + Objects.requireNonNull(header, "header"); + return new IdsMultipartParts(header, payload); + } + } + +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartRemoteMessageDispatcher.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartRemoteMessageDispatcher.java new file mode 100644 index 0000000..7342e56 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartRemoteMessageDispatcher.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2020 - 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.multipart; + +import org.eclipse.edc.connector.transfer.spi.types.protocol.TransferCompletionMessage; +import org.eclipse.edc.connector.transfer.spi.types.protocol.TransferStartMessage; +import org.eclipse.edc.connector.transfer.spi.types.protocol.TransferTerminationMessage; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.message.RemoteMessageDispatcher; +import org.eclipse.edc.spi.response.StatusResult; +import org.eclipse.edc.spi.types.domain.message.RemoteMessage; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; + +public class IdsMultipartRemoteMessageDispatcher implements RemoteMessageDispatcher { + + public static final String PROTOCOL = "ids-multipart"; + + private final IdsMultipartSender multipartSender; + private final Map, MultipartSenderDelegate> delegates = new HashMap<>(); + private final List> unsupportedMessages = List.of( + TransferStartMessage.class, + TransferCompletionMessage.class, + TransferTerminationMessage.class + ); + + public IdsMultipartRemoteMessageDispatcher(IdsMultipartSender idsMultipartSender) { + this.multipartSender = idsMultipartSender; + } + + public void register(MultipartSenderDelegate delegate) { + delegates.put(delegate.getMessageType(), delegate); + } + + @Override + public String protocol() { + return PROTOCOL; + } + + @Override + public CompletableFuture> dispatch(Class responseType, M message) { + Objects.requireNonNull(message, "Message was null"); + + if (unsupportedMessages.stream().anyMatch(it -> it.isInstance(message))) { // these messages are not supposed to be sent on ids-multipart. + return CompletableFuture.completedFuture(null); + } + + var delegate = (MultipartSenderDelegate) delegates.get(message.getClass()); + if (delegate == null) { + throw new EdcException("Message sender not found for message type: " + message.getClass().getName()); + } + + return multipartSender.send(message, delegate); + } + +} \ No newline at end of file diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartSender.java new file mode 100644 index 0000000..3677494 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartSender.java @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2020 - 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * Microsoft Corporation - Use IDS Webhook address for JWT audience claim + * Fraunhofer Institute for Software and Systems Engineering - refactoring + * + */ + +package com.truzzt.extension.logginghouse.client.ids.multipart; + +import com.fasterxml.jackson.databind.ObjectMapper; +import de.fraunhofer.iais.eis.DynamicAttributeToken; +import de.fraunhofer.iais.eis.DynamicAttributeTokenBuilder; +import de.fraunhofer.iais.eis.Message; +import de.fraunhofer.iais.eis.TokenFormat; +import jakarta.ws.rs.core.MediaType; +import okhttp3.Headers; +import okhttp3.HttpUrl; +import okhttp3.MultipartBody; +import okhttp3.MultipartReader; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.ResponseBody; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.http.EdcHttpClient; +import org.eclipse.edc.spi.iam.IdentityService; +import org.eclipse.edc.spi.iam.TokenParameters; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.response.StatusResult; +import org.eclipse.edc.spi.result.Result; +import org.eclipse.edc.spi.types.domain.message.RemoteMessage; +import org.glassfish.jersey.media.multipart.ContentDisposition; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.http.HttpHeaders; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; + +import static java.lang.String.format; +import static java.util.concurrent.CompletableFuture.failedFuture; + +public class IdsMultipartSender { + + private final Monitor monitor; + private final EdcHttpClient httpClient; + private final IdentityService identityService; + private final ObjectMapper objectMapper; + + public IdsMultipartSender(Monitor monitor, EdcHttpClient httpClient, + IdentityService identityService, + ObjectMapper objectMapper) { + this.monitor = monitor; + this.httpClient = httpClient; + this.identityService = identityService; + this.objectMapper = objectMapper; + } + + public CompletableFuture> send(M request, MultipartSenderDelegate senderDelegate) { + var remoteConnectorAddress = request.getCounterPartyAddress(); + + // Get Dynamic Attribute Token + var tokenResult = obtainDynamicAttributeToken(remoteConnectorAddress); + if (tokenResult.failed()) { + String message = "Failed to obtain token: " + String.join(",", tokenResult.getFailureMessages()); + monitor.severe(message); + return failedFuture(new EdcException(message)); + } + + var token = tokenResult.getContent(); + + // Get recipient address + var requestUrl = HttpUrl.parse(remoteConnectorAddress); + if (requestUrl == null) { + return failedFuture(new IllegalArgumentException("Connector address not specified")); + } + + // Build IDS message header + Message message; + try { + message = senderDelegate.buildMessageHeader(request, token); + } catch (Exception e) { + return failedFuture(e); + } + + // Build multipart header part + var headerPartHeaders = new Headers.Builder() + .add("Content-Disposition", "form-data; name=\"header\"") + .build(); + + RequestBody headerRequestBody; + try { + headerRequestBody = RequestBody.create( + objectMapper.writeValueAsString(message), + okhttp3.MediaType.get(MediaType.APPLICATION_JSON)); + } catch (IOException exception) { + return failedFuture(exception); + } + + var headerPart = MultipartBody.Part.create(headerPartHeaders, headerRequestBody); + + // Build IDS message payload + String payload; + try { + payload = senderDelegate.buildMessagePayload(request); + } catch (Exception e) { + return failedFuture(e); + } + + // Build multipart payload part + MultipartBody.Part payloadPart = null; + if (payload != null) { + var payloadRequestBody = RequestBody.create(payload, + okhttp3.MediaType.get(MediaType.APPLICATION_JSON)); + + var payloadPartHeaders = new Headers.Builder() + .add("Content-Disposition", "form-data; name=\"payload\"") + .build(); + + payloadPart = MultipartBody.Part.create(payloadPartHeaders, payloadRequestBody); + } + + // Build multipart body + var multipartBuilder = new MultipartBody.Builder() + .setType(okhttp3.MediaType.get(MediaType.MULTIPART_FORM_DATA)) + .addPart(headerPart); + + if (payloadPart != null) { + multipartBuilder.addPart(payloadPart); + } + + var multipartRequestBody = multipartBuilder.build(); + + // Build HTTP request + var httpRequest = new Request.Builder() + .url(requestUrl) + .addHeader("Content-Type", MediaType.MULTIPART_FORM_DATA) + .post(multipartRequestBody) + .build(); + + return httpClient.executeAsync(httpRequest, r -> { + monitor.debug("Response received from connector. Status " + r.code()); + if (r.isSuccessful()) { + try (var body = r.body()) { + if (body == null) { + throw new EdcException("Received an empty body response from connector"); + } else { + var parts = extractResponseParts(body); + var response = senderDelegate.getResponseContent(parts); + + checkResponseType(response, senderDelegate); + + return StatusResult.success(response.payload()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + throw new EdcException(format("Received an error from connector (%s): %s %s", requestUrl, r.code(), r.message())); + } + }); + } + + protected Result obtainDynamicAttributeToken(String recipientAddress) { + var tokenParameters = TokenParameters.Builder.newInstance() + .scope(IdsConstants.TOKEN_SCOPE) + .audience(recipientAddress) + .build(); + return identityService.obtainClientCredentials(tokenParameters) + .map(credentials -> new DynamicAttributeTokenBuilder() + ._tokenFormat_(TokenFormat.JWT) + ._tokenValue_(credentials.getToken()) + .build() + ); + } + + protected IdsMultipartParts extractResponseParts(ResponseBody body) throws Exception { + InputStream header = null; + InputStream payload = null; + try (var multipartReader = new MultipartReader(Objects.requireNonNull(body))) { + MultipartReader.Part part; + while ((part = multipartReader.nextPart()) != null) { + var httpHeaders = HttpHeaders.of( + part.headers().toMultimap(), + (a, b) -> a.equalsIgnoreCase("Content-Disposition") + ); + + var value = httpHeaders.firstValue("Content-Disposition").orElse(null); + if (value == null) { + continue; + } + + var contentDisposition = new ContentDisposition(value); + var multipartName = contentDisposition.getParameters().get("name"); + + if ("header".equalsIgnoreCase(multipartName)) { + header = new ByteArrayInputStream(part.body().readByteArray()); + } else if ("payload".equalsIgnoreCase(multipartName)) { + payload = new ByteArrayInputStream(part.body().readByteArray()); + } + } + } + + return IdsMultipartParts.Builder.newInstance() + .header(header) + .payload(payload) + .build(); + } + + public void checkResponseType(MultipartResponse response, MultipartSenderDelegate senderDelegate) { + var type = senderDelegate.getAllowedResponseTypes(); + if (!type.contains(response.header().getClass())) { + throw new EdcException(String.format("Received %s but expected %s.", response.header().getClass(), type)); + } + } + +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartResponse.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartResponse.java new file mode 100644 index 0000000..83b2ce5 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartResponse.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2021 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - Initial implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.multipart; + +import de.fraunhofer.iais.eis.Message; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public record MultipartResponse(Message header, @Nullable T payload) { + + public MultipartResponse(@NotNull Message header, @Nullable T payload) { + this.header = header; + this.payload = payload; + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartSenderDelegate.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartSenderDelegate.java new file mode 100644 index 0000000..b5e8bdb --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartSenderDelegate.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.multipart; + +import de.fraunhofer.iais.eis.DynamicAttributeToken; +import de.fraunhofer.iais.eis.Message; +import org.eclipse.edc.spi.types.domain.message.RemoteMessage; + +import java.util.List; + +public interface MultipartSenderDelegate { + + Message buildMessageHeader(M request, DynamicAttributeToken token) throws Exception; + + String buildMessagePayload(M request) throws Exception; + + MultipartResponse getResponseContent(IdsMultipartParts parts) throws Exception; + + List> getAllowedResponseTypes(); + + Class getMessageType(); +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/ResponseUtil.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/ResponseUtil.java new file mode 100644 index 0000000..493cc6e --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/ResponseUtil.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2020 - 2022 Fraunhofer Institute for Software and Systems Engineering + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Fraunhofer Institute for Software and Systems Engineering - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.ids.multipart; + +import com.fasterxml.jackson.databind.ObjectMapper; +import de.fraunhofer.iais.eis.Message; + +import java.io.IOException; + +public class ResponseUtil { + + public static MultipartResponse parseMultipartStringResponse(IdsMultipartParts parts, ObjectMapper objectMapper) throws IOException { + var header = objectMapper.readValue(parts.getHeader(), Message.class); + + String payload = null; + if (parts.getPayload() != null) { + payload = new String(parts.getPayload().readAllBytes()); + } + + return new MultipartResponse<>(header, payload); + } + +} From 8fd51fabbef154453071bb1e5092767b96dc360c Mon Sep 17 00:00:00 2001 From: dhommen Date: Sat, 3 Feb 2024 10:26:29 +0100 Subject: [PATCH 02/25] chore: bump version number --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index fcfd296..af34be6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.1.1" + version = "0.2.0" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") From e8e8f08c34da41652fef602ac8f4931e42b2b5e4 Mon Sep 17 00:00:00 2001 From: dhommen Date: Sat, 3 Feb 2024 10:28:44 +0100 Subject: [PATCH 03/25] chore: add placeholder README.md for lh client --- logging-house-client/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 logging-house-client/README.md diff --git a/logging-house-client/README.md b/logging-house-client/README.md new file mode 100644 index 0000000..aa0edcd --- /dev/null +++ b/logging-house-client/README.md @@ -0,0 +1 @@ +# Logging House Client From ac6665a80cd831398de1b9296b2a20607092ca78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Sat, 3 Feb 2024 11:50:02 +0100 Subject: [PATCH 04/25] fix: urls for logging and process creation --- build.gradle.kts | 2 +- .../ExtendedMessageProtocolClearing.java | 1 + .../client/IdsClearingHouseServiceImpl.java | 12 +++++++---- .../logginghouse/client/LogMessage.java | 1 + .../client/LoggingHouseClientExtension.java | 20 +++++++++---------- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index af34be6..6423ded 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.0" + version = "0.2.1" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java index 931fa9d..71ba1fc 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java @@ -9,6 +9,7 @@ * * Contributors: * sovity GmbH - initial API and implementation + * truzzt GmbH - adjusted for EDC 0.x * */ diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java index 8ce4ee7..308b87b 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java @@ -94,14 +94,18 @@ public void on(EventEnvelope event) { if (event.getPayload() instanceof ContractNegotiationFinalized contractNegotiationFinalized) { var contractAgreement = resolveContractAgreement(contractNegotiationFinalized); var pid = contractAgreement.getId(); - var extendedUrl = new URL(clearingHouseLogUrl + "/" + pid); - createProcess(contractAgreement, clearingHouseLogUrl); - logContractAgreement(contractAgreement, extendedUrl); + // Create Process + var extendedProcessUrl = new URL(clearingHouseLogUrl + "/process/" + pid); + createProcess(contractAgreement, extendedProcessUrl); + + // Log Contract Agreement + var extendedLogUrl = new URL(clearingHouseLogUrl + "/messages/log/" + pid); + logContractAgreement(contractAgreement, extendedLogUrl); } else if (event.getPayload() instanceof TransferProcessTerminated transferProcessTerminated) { var transferProcess = resolveTransferProcess(transferProcessTerminated); var pid = transferProcess.getContractId(); - var extendedUrl = new URL(clearingHouseLogUrl + "/" + pid); + var extendedUrl = new URL(clearingHouseLogUrl + "/messages/log/" + pid); logTransferProcess(transferProcess, extendedUrl); } } catch (Exception e) { diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java index 0b0e16a..e28d206 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java @@ -9,6 +9,7 @@ * * Contributors: * sovity GmbH - initial API and implementation + * truzzt GmbH - adjusted for EDC 0.x * */ diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index ea01dfa..d7400b1 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -64,12 +64,12 @@ public class LoggingHouseClientExtension implements ServiceExtension { "ids", "https://w3id.org/idsa/core/", "idsc", "https://w3id.org/idsa/code/"); @Setting - public static final String CLEARINGHOUSE_LOG_URL_SETTING = "edc.clearinghouse.log.url"; + public static final String LOGGINGHOUSE_LOG_URL_SETTING = "edc.logginghouse.extension.url"; @Setting - public static final String CLEARINGHOUSE_CLIENT_EXTENSION_ENABLED = "clearinghouse.client.extension.enabled"; + public static final String LOGGINGHOUSE_CLIENT_EXTENSION_ENABLED = "edc.logginghouse.extension.enabled"; - private URL clearingHouseLogUrl; + private URL loggingHouseLogUrl; public Monitor monitor; @@ -81,7 +81,7 @@ public String name() { @Override public void initialize(ServiceExtensionContext context) { monitor = context.getMonitor(); - var extensionEnabled = context.getSetting(CLEARINGHOUSE_CLIENT_EXTENSION_ENABLED, false); + var extensionEnabled = context.getSetting(LOGGINGHOUSE_CLIENT_EXTENSION_ENABLED, true); if (!extensionEnabled) { monitor.info("Logginghouse client extension is disabled."); @@ -89,22 +89,22 @@ public void initialize(ServiceExtensionContext context) { } monitor.info("Logginghouse client extension is enabled."); - clearingHouseLogUrl = readUrlFromSettings(context, CLEARINGHOUSE_LOG_URL_SETTING); + loggingHouseLogUrl = readUrlFromSettings(context); } - private URL readUrlFromSettings(ServiceExtensionContext context, String settingsPath) { + private URL readUrlFromSettings(ServiceExtensionContext context) { try { - var urlString = context.getSetting(settingsPath, null); + var urlString = context.getSetting(LoggingHouseClientExtension.LOGGINGHOUSE_LOG_URL_SETTING, null); if (urlString == null) { throw new EdcException(String.format("Could not initialize " + "LoggingHouseClientExtension: " + - "No url specified using setting %s", settingsPath)); + "No url specified using setting %s", LoggingHouseClientExtension.LOGGINGHOUSE_LOG_URL_SETTING)); } return new URL(urlString); } catch (MalformedURLException e) { throw new EdcException(String.format("Could not parse setting %s to Url", - settingsPath), e); + LoggingHouseClientExtension.LOGGINGHOUSE_LOG_URL_SETTING), e); } } @@ -112,7 +112,7 @@ private void registerEventSubscriber(ServiceExtensionContext context) { var eventSubscriber = new IdsClearingHouseServiceImpl( dispatcherRegistry, hostname, - clearingHouseLogUrl, + loggingHouseLogUrl, contractNegotiationStore, transferProcessStore, monitor); From 81740c9d48e3766a94b3ca5ba0bfeef5baa785c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Mon, 5 Feb 2024 10:45:49 +0100 Subject: [PATCH 05/25] fix: registering event subscriber --- build.gradle.kts | 2 +- .../client/LoggingHouseClientExtension.java | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 6423ded..b1d9b3e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.1" + version = "0.2.2" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index d7400b1..618a663 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -72,6 +72,8 @@ public class LoggingHouseClientExtension implements ServiceExtension { private URL loggingHouseLogUrl; public Monitor monitor; + private boolean enabled; + @Override public String name() { @@ -84,12 +86,19 @@ public void initialize(ServiceExtensionContext context) { var extensionEnabled = context.getSetting(LOGGINGHOUSE_CLIENT_EXTENSION_ENABLED, true); if (!extensionEnabled) { + enabled = false; monitor.info("Logginghouse client extension is disabled."); return; } + enabled = true; monitor.info("Logginghouse client extension is enabled."); loggingHouseLogUrl = readUrlFromSettings(context); + + registerSerializerClearingHouseMessages(context); + registerClearingHouseMessageSenders(context); + + registerEventSubscriber(context); } private URL readUrlFromSettings(ServiceExtensionContext context) { @@ -153,7 +162,11 @@ private void registerClearingHouseMessageSenders(ServiceExtensionContext context @Override public void start() { - monitor.info("Starting Logginghouse client extension."); + if (!enabled) { + monitor.info("Skipping start of Logginghouse client extension (disabled)."); + } else { + monitor.info("Starting Logginghouse client extension."); + } } @Override From 63f5f73d39f5b74cd69a5a93f586f36d4d996205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Mon, 5 Feb 2024 11:58:52 +0100 Subject: [PATCH 06/25] chore: add some logs --- build.gradle.kts | 2 +- .../client/LoggingHouseClientExtension.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index b1d9b3e..b05def5 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.2" + version = "0.2.3" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 618a663..f4d535b 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -118,6 +118,8 @@ private URL readUrlFromSettings(ServiceExtensionContext context) { } private void registerEventSubscriber(ServiceExtensionContext context) { + monitor.debug("Registering event subscriber for LoggingHouseClientExtension"); + var eventSubscriber = new IdsClearingHouseServiceImpl( dispatcherRegistry, hostname, @@ -129,21 +131,33 @@ private void registerEventSubscriber(ServiceExtensionContext context) { eventRouter.registerSync(ContractNegotiationFinalized.class, eventSubscriber); eventRouter.registerSync(TransferProcessTerminated.class, eventSubscriber); context.registerService(IdsClearingHouseServiceImpl.class, eventSubscriber); + + monitor.debug("Registered event subscriber for LoggingHouseClientExtension"); } private void registerSerializerClearingHouseMessages(ServiceExtensionContext context) { + monitor.debug("Registering serializers for LoggingHouseClientExtension"); + typeManager.registerContext(TYPE_MANAGER_SERIALIZER_KEY, JsonLd.getObjectMapper()); registerCommonTypes(typeManager); + + monitor.debug("Registered serializers for LoggingHouseClientExtension"); } private void registerCommonTypes(TypeManager typeManager) { + monitor.debug("Registering serializers for LoggingHouseClientExtension"); + typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, LogMessage.class, new MultiContextJsonLdSerializer<>(LogMessage.class, CONTEXT_MAP)); typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, CreateProcessMessage.class, new MultiContextJsonLdSerializer<>(CreateProcessMessage.class, CONTEXT_MAP)); + + monitor.debug("Registered serializers for LoggingHouseClientExtension"); } private void registerClearingHouseMessageSenders(ServiceExtensionContext context) { + monitor.debug("Registering message senders for LoggingHouseClientExtension"); + var httpClient = context.getService(EdcHttpClient.class); var monitor = context.getMonitor(); var objectMapper = typeManager.getMapper(TYPE_MANAGER_SERIALIZER_KEY); From 0f63780c5b086ebf24310c02cf31c31613137fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Mon, 5 Feb 2024 13:36:15 +0100 Subject: [PATCH 07/25] fix: serializer --- build.gradle.kts | 2 +- .../client/LoggingHouseClientExtension.java | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index b05def5..81bb4d0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.3" + version = "0.2.4" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index f4d535b..8fdae4f 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -15,8 +15,15 @@ import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartSender; +import de.fraunhofer.iais.eis.LogMessage; +import de.fraunhofer.iais.eis.RequestMessage; +import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAccepted; +import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAgreed; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessCompleted; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessInitiated; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessStarted; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessTerminated; import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; import org.eclipse.edc.runtime.metamodel.annotation.Inject; @@ -129,7 +136,12 @@ private void registerEventSubscriber(ServiceExtensionContext context) { monitor); eventRouter.registerSync(ContractNegotiationFinalized.class, eventSubscriber); + eventRouter.registerSync(ContractNegotiationAgreed.class, eventSubscriber); + eventRouter.registerSync(ContractNegotiationAccepted.class, eventSubscriber); + eventRouter.registerSync(TransferProcessCompleted.class, eventSubscriber); eventRouter.registerSync(TransferProcessTerminated.class, eventSubscriber); + eventRouter.registerSync(TransferProcessInitiated.class, eventSubscriber); + eventRouter.registerSync(TransferProcessStarted.class, eventSubscriber); context.registerService(IdsClearingHouseServiceImpl.class, eventSubscriber); monitor.debug("Registered event subscriber for LoggingHouseClientExtension"); @@ -149,8 +161,8 @@ private void registerCommonTypes(TypeManager typeManager) { typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, LogMessage.class, new MultiContextJsonLdSerializer<>(LogMessage.class, CONTEXT_MAP)); - typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, CreateProcessMessage.class, - new MultiContextJsonLdSerializer<>(CreateProcessMessage.class, CONTEXT_MAP)); + typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, RequestMessage.class, + new MultiContextJsonLdSerializer<>(RequestMessage.class, CONTEXT_MAP)); monitor.debug("Registered serializers for LoggingHouseClientExtension"); } From 5258ccb6e20f72b8068c52f3b3433a28a16b7e44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Wed, 14 Feb 2024 15:59:57 +0100 Subject: [PATCH 08/25] fix: handle more events for transfer --- build.gradle.kts | 2 +- .../client/IdsClearingHouseServiceImpl.java | 38 ++++++++++++++++--- .../client/LoggingHouseClientExtension.java | 13 +++---- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 81bb4d0..4b49795 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.4" + version = "0.2.5" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java index 308b87b..f5b18a8 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java @@ -17,7 +17,7 @@ import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessTerminated; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessEvent; import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; import org.eclipse.edc.spi.EdcException; @@ -28,6 +28,9 @@ import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.system.Hostname; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -73,7 +76,28 @@ public void createProcess(ContractAgreement contractAgreement, URL clearingHouse monitor.info("Creating Process in LoggingHouse"); var logMessage = new CreateProcessMessage(clearingHouseLogUrl, connectorBaseUrl, contractAgreement.getId(), processOwners); - dispatcherRegistry.dispatch(Object.class, logMessage); + + try { + dispatcherRegistry.dispatch(Object.class, logMessage); + } catch (EdcException e) { + if (e.getMessage().startsWith("No provider dispatcher registered for protocol")) { + throw e; + } else { + monitor.severe("Unhandled exception while creating process in LoggingHouse. " + e.getMessage()); + // Print stack trace + String errorStr; + try (StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw)) { + + e.printStackTrace(pw); + errorStr = sw.toString(); + + } catch (IOException ex) { + throw new RuntimeException("Error while converting the stacktrace"); + } + monitor.severe(errorStr); + } + } } public void logContractAgreement(ContractAgreement contractAgreement, URL clearingHouseLogUrl) { @@ -102,8 +126,10 @@ public void on(EventEnvelope event) { // Log Contract Agreement var extendedLogUrl = new URL(clearingHouseLogUrl + "/messages/log/" + pid); logContractAgreement(contractAgreement, extendedLogUrl); - } else if (event.getPayload() instanceof TransferProcessTerminated transferProcessTerminated) { - var transferProcess = resolveTransferProcess(transferProcessTerminated); + } else if (event.getPayload() instanceof TransferProcessEvent transferProcessEvent) { + monitor.debug("Logging transfer event with id " + event.getId()); + + var transferProcess = resolveTransferProcess(transferProcessEvent); var pid = transferProcess.getContractId(); var extendedUrl = new URL(clearingHouseLogUrl + "/messages/log/" + pid); logTransferProcess(transferProcess, extendedUrl); @@ -119,8 +145,8 @@ private ContractAgreement resolveContractAgreement(ContractNegotiationFinalized return Objects.requireNonNull(contractNegotiation).getContractAgreement(); } - private TransferProcess resolveTransferProcess(TransferProcessTerminated transferProcessTerminated) { - var transferProcessId = transferProcessTerminated.getTransferProcessId(); + private TransferProcess resolveTransferProcess(TransferProcessEvent transferProcessEvent) { + var transferProcessId = transferProcessEvent.getTransferProcessId(); return transferProcessStore.findById(transferProcessId); } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 8fdae4f..ebb780e 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -16,15 +16,11 @@ import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartSender; import de.fraunhofer.iais.eis.LogMessage; -import de.fraunhofer.iais.eis.RequestMessage; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAccepted; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAgreed; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessCompleted; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessInitiated; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessStarted; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessTerminated; +import org.eclipse.edc.connector.transfer.spi.event.*; import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Setting; @@ -139,9 +135,10 @@ private void registerEventSubscriber(ServiceExtensionContext context) { eventRouter.registerSync(ContractNegotiationAgreed.class, eventSubscriber); eventRouter.registerSync(ContractNegotiationAccepted.class, eventSubscriber); eventRouter.registerSync(TransferProcessCompleted.class, eventSubscriber); - eventRouter.registerSync(TransferProcessTerminated.class, eventSubscriber); eventRouter.registerSync(TransferProcessInitiated.class, eventSubscriber); eventRouter.registerSync(TransferProcessStarted.class, eventSubscriber); + eventRouter.registerSync(TransferProcessFailed.class, eventSubscriber); + eventRouter.registerSync(TransferProcessTerminated.class, eventSubscriber); context.registerService(IdsClearingHouseServiceImpl.class, eventSubscriber); monitor.debug("Registered event subscriber for LoggingHouseClientExtension"); @@ -161,8 +158,8 @@ private void registerCommonTypes(TypeManager typeManager) { typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, LogMessage.class, new MultiContextJsonLdSerializer<>(LogMessage.class, CONTEXT_MAP)); - typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, RequestMessage.class, - new MultiContextJsonLdSerializer<>(RequestMessage.class, CONTEXT_MAP)); + // typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, RequestMessage.class, + // new MultiContextJsonLdSerializer<>(RequestMessage.class, CONTEXT_MAP)); monitor.debug("Registered serializers for LoggingHouseClientExtension"); } From 5dc8dda55b3a2e83486b0462c31a3c3cbad610c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Wed, 14 Feb 2024 18:40:11 +0100 Subject: [PATCH 09/25] fix: message type for create process --- build.gradle.kts | 2 +- .../logginghouse/client/CreateProcessMessageSender.java | 4 ++-- .../logginghouse/client/IdsClearingHouseServiceImpl.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4b49795..78f9d9c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.5" + version = "0.2.6" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java index 58fa5ef..ee3c71d 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java @@ -22,9 +22,9 @@ import com.truzzt.extension.logginghouse.client.ids.multipart.MultipartSenderDelegate; import com.truzzt.extension.logginghouse.client.ids.multipart.ResponseUtil; import de.fraunhofer.iais.eis.DynamicAttributeToken; -import de.fraunhofer.iais.eis.LogMessageBuilder; import de.fraunhofer.iais.eis.Message; import de.fraunhofer.iais.eis.MessageProcessedNotificationMessageImpl; +import de.fraunhofer.iais.eis.RequestMessageBuilder; import org.json.JSONObject; import java.util.List; @@ -36,7 +36,7 @@ public CreateProcessMessageSender() { @Override public Message buildMessageHeader(CreateProcessMessage createProcessMessage, DynamicAttributeToken token) { - return new LogMessageBuilder() + return new RequestMessageBuilder() ._modelVersion_(IdsConstants.INFORMATION_MODEL_VERSION) ._issued_(CalendarUtil.gregorianNow()) ._securityToken_(token) diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java index f5b18a8..f57d24f 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java @@ -101,7 +101,7 @@ public void createProcess(ContractAgreement contractAgreement, URL clearingHouse } public void logContractAgreement(ContractAgreement contractAgreement, URL clearingHouseLogUrl) { - monitor.info("Logging contract agreement to LoggingHouse"); + monitor.info("Logging contract agreement to LoggingHouse with contract id: " + contractAgreement.getId()); var logMessage = new LogMessage(clearingHouseLogUrl, connectorBaseUrl, contractAgreement); dispatcherRegistry.dispatch(Object.class, logMessage); } From 4d0e1b74ed8000441627293ec0cc672e030ac14d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Wed, 14 Feb 2024 19:00:28 +0100 Subject: [PATCH 10/25] fix: add RequestMessage again to MultiContextJsonLdSerializer --- build.gradle.kts | 2 +- .../logginghouse/client/LoggingHouseClientExtension.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 78f9d9c..614e88f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.6" + version = "0.2.7" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index ebb780e..383fb1d 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -16,6 +16,7 @@ import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartSender; import de.fraunhofer.iais.eis.LogMessage; +import de.fraunhofer.iais.eis.RequestMessage; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAccepted; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAgreed; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; @@ -158,8 +159,8 @@ private void registerCommonTypes(TypeManager typeManager) { typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, LogMessage.class, new MultiContextJsonLdSerializer<>(LogMessage.class, CONTEXT_MAP)); - // typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, RequestMessage.class, - // new MultiContextJsonLdSerializer<>(RequestMessage.class, CONTEXT_MAP)); + typeManager.registerSerializer(TYPE_MANAGER_SERIALIZER_KEY, RequestMessage.class, + new MultiContextJsonLdSerializer<>(RequestMessage.class, CONTEXT_MAP)); monitor.debug("Registered serializers for LoggingHouseClientExtension"); } From e8d2e062e33981bf46cbaa35afcfdf5c2c9c0284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Wed, 14 Feb 2024 19:39:15 +0100 Subject: [PATCH 11/25] fix: add more TransferStates and add field for LogMessage of TransferState --- build.gradle.kts | 2 +- .../truzzt/extension/logginghouse/client/LogMessageSender.java | 2 ++ .../logginghouse/client/LoggingHouseClientExtension.java | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 614e88f..e368e93 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.7" + version = "0.2.8" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java index 3fbb1d5..9153478 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java @@ -89,6 +89,8 @@ private String buildContractAgreementPayload(ContractAgreement contractAgreement private String buildTransferProcessPayload(TransferProcess transferProcess) { var jo = new JSONObject(); jo.put("transferProcessId", transferProcess.getId()); + jo.put("transferState", transferProcess.stateAsString()); + var dataRequest = transferProcess.getDataRequest(); jo.put("contractId", dataRequest.getContractId()); jo.put("connectorId", dataRequest.getConnectorId()); diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 383fb1d..ee0f230 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -135,9 +135,10 @@ private void registerEventSubscriber(ServiceExtensionContext context) { eventRouter.registerSync(ContractNegotiationFinalized.class, eventSubscriber); eventRouter.registerSync(ContractNegotiationAgreed.class, eventSubscriber); eventRouter.registerSync(ContractNegotiationAccepted.class, eventSubscriber); - eventRouter.registerSync(TransferProcessCompleted.class, eventSubscriber); + eventRouter.registerSync(TransferProcessRequested.class, eventSubscriber); eventRouter.registerSync(TransferProcessInitiated.class, eventSubscriber); eventRouter.registerSync(TransferProcessStarted.class, eventSubscriber); + eventRouter.registerSync(TransferProcessCompleted.class, eventSubscriber); eventRouter.registerSync(TransferProcessFailed.class, eventSubscriber); eventRouter.registerSync(TransferProcessTerminated.class, eventSubscriber); context.registerService(IdsClearingHouseServiceImpl.class, eventSubscriber); From 2c2d257d4fe851711b8caafce52b5f5cf24cd7b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Thu, 14 Mar 2024 08:56:20 +0100 Subject: [PATCH 12/25] feat: mds requested properties --- build.gradle.kts | 2 +- .../client/IdsClearingHouseServiceImpl.java | 5 +- .../logginghouse/client/LogMessageSender.java | 58 +++++++++++++++---- .../client/LoggingHouseClientExtension.java | 30 ++++++---- .../LoggingHouseClientExtensionTest.java | 16 ++++- 5 files changed, 83 insertions(+), 28 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index e368e93..90caef7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.8" + version = "0.2.9" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java index f57d24f..98850c6 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java @@ -86,12 +86,9 @@ public void createProcess(ContractAgreement contractAgreement, URL clearingHouse monitor.severe("Unhandled exception while creating process in LoggingHouse. " + e.getMessage()); // Print stack trace String errorStr; - try (StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw)) { - + try (StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw)) { e.printStackTrace(pw); errorStr = sw.toString(); - } catch (IOException ex) { throw new RuntimeException("Error while converting the stacktrace"); } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java index 9153478..b5a9139 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java @@ -28,13 +28,22 @@ import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement; import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.asset.AssetIndex; +import org.eclipse.edc.spi.monitor.Monitor; import org.json.JSONObject; import java.util.List; public class LogMessageSender implements MultipartSenderDelegate { - public LogMessageSender() { + Monitor monitor; + String connectorId; + AssetIndex assetIndex; + + public LogMessageSender(Monitor monitor, AssetIndex assetIndex, String connectorId) { + this.monitor = monitor; + this.assetIndex = assetIndex; + this.connectorId = connectorId; } @Override @@ -76,24 +85,51 @@ public Class getMessageType() { } private String buildContractAgreementPayload(ContractAgreement contractAgreement) { + assert contractAgreement != null; + var jo = new JSONObject(); - jo.put("AgreementId", contractAgreement.getId()); - jo.put("ProviderId", contractAgreement.getProviderId()); - jo.put("ConsumerId", contractAgreement.getConsumerId()); + + jo.put("Timestamp", CalendarUtil.gregorianNow().toString()); + jo.put("ConnectorId", connectorId); + + // Check if connector is the provider + if (contractAgreement.getProviderId().equals(connectorId)) { + // In case of the provider, log asset information + var asset = assetIndex.findById(contractAgreement.getAssetId()); + + if (asset == null) { + monitor.warning("Asset with id " + contractAgreement.getAssetId() + " not found in asset index."); + } else { + jo.put("AssetId", asset.getId()); + jo.put("AssetName", asset.getName()); + jo.put("AssetDescription", asset.getDescription()); + jo.put("AssetVersion", asset.getVersion()); + jo.put("AssetContentType", asset.getContentType()); + jo.put("AssetProperties", asset.getProperties()); + } + } + + jo.put("ContractAgreementId", contractAgreement.getId()); + jo.put("ContractProviderId", contractAgreement.getProviderId()); + jo.put("ContractConsumerId", contractAgreement.getConsumerId()); jo.put("ContractSigningDate", contractAgreement.getContractSigningDate()); - jo.put("Policy", contractAgreement.getPolicy()); - jo.put("AssetId", contractAgreement.getAssetId()); + jo.put("ContractPolicy", contractAgreement.getPolicy()); return jo.toString(); } private String buildTransferProcessPayload(TransferProcess transferProcess) { var jo = new JSONObject(); - jo.put("transferProcessId", transferProcess.getId()); - jo.put("transferState", transferProcess.stateAsString()); - var dataRequest = transferProcess.getDataRequest(); - jo.put("contractId", dataRequest.getContractId()); - jo.put("connectorId", dataRequest.getConnectorId()); + jo.put("Timestamp", CalendarUtil.gregorianNow().toString()); + jo.put("ConnectorId", connectorId); + + jo.put("TransferProcessId", transferProcess.getId()); + jo.put("TransferState", transferProcess.stateAsString()); + jo.put("TransferProtocol", transferProcess.getProtocol()); + jo.put("TransferContractId", transferProcess.getContractId()); + jo.put("TransferConnectorId", transferProcess.getConnectorId()); + jo.put("TransferAssetId", transferProcess.getAssetId()); + return jo.toString(); } } \ No newline at end of file diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index ee0f230..cd0b8ba 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -20,12 +20,19 @@ import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAccepted; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAgreed; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; +import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationTerminated; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; -import org.eclipse.edc.connector.transfer.spi.event.*; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessCompleted; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessFailed; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessInitiated; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessRequested; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessStarted; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessTerminated; import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Setting; import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.asset.AssetIndex; import org.eclipse.edc.spi.event.EventRouter; import org.eclipse.edc.spi.http.EdcHttpClient; import org.eclipse.edc.spi.iam.IdentityService; @@ -62,6 +69,8 @@ public class LoggingHouseClientExtension implements ServiceExtension { private ContractNegotiationStore contractNegotiationStore; @Inject private TransferProcessStore transferProcessStore; + @Inject + private AssetIndex assetIndex; private static final Map CONTEXT_MAP = Map.of( "cat", "http://w3id.org/mds/data-categories#", @@ -75,7 +84,6 @@ public class LoggingHouseClientExtension implements ServiceExtension { private URL loggingHouseLogUrl; public Monitor monitor; - private boolean enabled; @@ -86,18 +94,18 @@ public String name() { @Override public void initialize(ServiceExtensionContext context) { - monitor = context.getMonitor(); - var extensionEnabled = context.getSetting(LOGGINGHOUSE_CLIENT_EXTENSION_ENABLED, true); + this.monitor = context.getMonitor(); + var extensionEnabled = context.getSetting(LOGGINGHOUSE_CLIENT_EXTENSION_ENABLED, true); if (!extensionEnabled) { - enabled = false; - monitor.info("Logginghouse client extension is disabled."); + this.enabled = false; + this.monitor.info("Logginghouse client extension is disabled."); return; } - enabled = true; - monitor.info("Logginghouse client extension is enabled."); + this.enabled = true; + this.monitor.info("Logginghouse client extension is enabled."); - loggingHouseLogUrl = readUrlFromSettings(context); + this.loggingHouseLogUrl = readUrlFromSettings(context); registerSerializerClearingHouseMessages(context); registerClearingHouseMessageSenders(context); @@ -135,6 +143,7 @@ private void registerEventSubscriber(ServiceExtensionContext context) { eventRouter.registerSync(ContractNegotiationFinalized.class, eventSubscriber); eventRouter.registerSync(ContractNegotiationAgreed.class, eventSubscriber); eventRouter.registerSync(ContractNegotiationAccepted.class, eventSubscriber); + eventRouter.registerSync(ContractNegotiationTerminated.class, eventSubscriber); // TODO: check pid eventRouter.registerSync(TransferProcessRequested.class, eventSubscriber); eventRouter.registerSync(TransferProcessInitiated.class, eventSubscriber); eventRouter.registerSync(TransferProcessStarted.class, eventSubscriber); @@ -170,10 +179,9 @@ private void registerClearingHouseMessageSenders(ServiceExtensionContext context monitor.debug("Registering message senders for LoggingHouseClientExtension"); var httpClient = context.getService(EdcHttpClient.class); - var monitor = context.getMonitor(); var objectMapper = typeManager.getMapper(TYPE_MANAGER_SERIALIZER_KEY); - var logMessageSender = new LogMessageSender(); + var logMessageSender = new LogMessageSender(monitor, assetIndex, context.getConnectorId()); var createProcessMessageSender = new CreateProcessMessageSender(); var idsMultipartSender = new IdsMultipartSender(monitor, httpClient, identityService, objectMapper); diff --git a/logging-house-client/src/test/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtensionTest.java b/logging-house-client/src/test/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtensionTest.java index 14c8cbe..90a5582 100644 --- a/logging-house-client/src/test/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtensionTest.java +++ b/logging-house-client/src/test/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtensionTest.java @@ -1,4 +1,18 @@ -import com.truzzt.extension.logginghouse.client.LoggingHouseClientExtension; +/* + * Copyright (c) 2024 truzzt GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * + */ + +package com.truzzt.extension.logginghouse.client; + import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; From c0215f3230de38b369b0b889bc6e87056c22865b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Sch=C3=B6nenberg?= Date: Thu, 14 Mar 2024 11:42:42 +0100 Subject: [PATCH 13/25] fix: restructure exception handling --- build.gradle.kts | 2 +- .../client/IdsClearingHouseServiceImpl.java | 25 ++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 90caef7..cc7df4b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.9" + version = "0.2.10" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java index 98850c6..9b6ccf2 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -98,13 +99,13 @@ public void createProcess(ContractAgreement contractAgreement, URL clearingHouse } public void logContractAgreement(ContractAgreement contractAgreement, URL clearingHouseLogUrl) { - monitor.info("Logging contract agreement to LoggingHouse with contract id: " + contractAgreement.getId()); + monitor.info("Logging ContractAgreement to LoggingHouse with contract id: " + contractAgreement.getId()); var logMessage = new LogMessage(clearingHouseLogUrl, connectorBaseUrl, contractAgreement); dispatcherRegistry.dispatch(Object.class, logMessage); } public void logTransferProcess(TransferProcess transferProcess, URL clearingHouseLogUrl) { - monitor.info("Logging transferprocess to LoggingHouse"); + monitor.info("Logging TransferProcess to LoggingHouse"); var logMessage = new LogMessage(clearingHouseLogUrl, connectorBaseUrl, transferProcess); dispatcherRegistry.dispatch(Object.class, logMessage); } @@ -118,20 +119,32 @@ public void on(EventEnvelope event) { // Create Process var extendedProcessUrl = new URL(clearingHouseLogUrl + "/process/" + pid); - createProcess(contractAgreement, extendedProcessUrl); + try { + createProcess(contractAgreement, extendedProcessUrl); + } catch (Exception e) { + monitor.warning("Could not create process in LoggingHouse: " + e.getMessage()); + } // Log Contract Agreement var extendedLogUrl = new URL(clearingHouseLogUrl + "/messages/log/" + pid); - logContractAgreement(contractAgreement, extendedLogUrl); + try { + logContractAgreement(contractAgreement, extendedLogUrl); + } catch (Exception e) { + monitor.warning("Could not log ContractNegotiation to LoggingHouse: " + e.getMessage()); + } } else if (event.getPayload() instanceof TransferProcessEvent transferProcessEvent) { monitor.debug("Logging transfer event with id " + event.getId()); var transferProcess = resolveTransferProcess(transferProcessEvent); var pid = transferProcess.getContractId(); var extendedUrl = new URL(clearingHouseLogUrl + "/messages/log/" + pid); - logTransferProcess(transferProcess, extendedUrl); + try { + logTransferProcess(transferProcess, extendedUrl); + } catch (Exception e) { + monitor.warning("Could not log TransferProcess to LoggingHouse: " + e.getMessage()); + } } - } catch (Exception e) { + } catch (MalformedURLException e) { throw new EdcException("Could not create extended clearinghouse url."); } } From 7f86700ee8c739c12fd1bcb99d48a373d4141bb2 Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Tue, 18 Jun 2024 19:31:27 -0300 Subject: [PATCH 14/25] feat: flyway migrations --- logging-house-client/build.gradle.kts | 5 + .../client/IdsClearingHouseServiceImpl.java | 2 + .../client/LoggingHouseClientExtension.java | 29 +++- .../client/flyway/FlywayService.java | 129 ++++++++++++++++++ .../connection/DatasourceProperties.java | 72 ++++++++++ .../DriverManagerConnectionFactory.java | 47 +++++++ .../migration/DatabaseMigrationManager.java | 40 ++++++ .../{ => messages}/CreateProcessMessage.java | 3 +- .../CreateProcessMessageSender.java | 2 +- .../client/{ => messages}/LogMessage.java | 3 +- .../{ => messages}/LogMessageSender.java | 2 +- .../ExtendedMessageProtocolClearing.java | 2 +- ...tipartClearingRemoteMessageDispatcher.java | 2 +- .../MultiContextJsonLdSerializer.java | 2 +- .../migration/V0_0_1__Create_Tables.sql | 6 + 15 files changed, 338 insertions(+), 8 deletions(-) create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DriverManagerConnectionFactory.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/migration/DatabaseMigrationManager.java rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => messages}/CreateProcessMessage.java (86%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => messages}/CreateProcessMessageSender.java (97%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => messages}/LogMessage.java (86%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => messages}/LogMessageSender.java (98%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ExtendedMessageProtocolClearing.java (92%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/IdsMultipartClearingRemoteMessageDispatcher.java (93%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/MultiContextJsonLdSerializer.java (98%) create mode 100644 logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql diff --git a/logging-house-client/build.gradle.kts b/logging-house-client/build.gradle.kts index 07c9610..d7b41a7 100644 --- a/logging-house-client/build.gradle.kts +++ b/logging-house-client/build.gradle.kts @@ -14,6 +14,8 @@ val jsonVersion: String by project dependencies { implementation("${edcGroup}:control-plane-core:${edcVersion}") implementation("${edcGroup}:http-spi:${edcVersion}") + implementation("${edcGroup}:sql-core:${edcVersion}") + implementation("${edcGroup}:transaction-datasource-spi:${edcVersion}") implementation("com.squareup.okhttp3:okhttp:${okHttpVersion}") implementation("org.json:json:${jsonVersion}") @@ -22,6 +24,9 @@ dependencies { implementation("de.fraunhofer.iais.eis.ids.infomodel:infomodel-java:1.0.2-basecamp") implementation("de.fraunhofer.iais.eis.ids.infomodel:infomodel-util:1.0.2-basecamp") + implementation("org.postgresql:postgresql:42.4.5") + implementation("org.flywaydb:flyway-core:9.0.1") + testImplementation("org.assertj:assertj-core:${assertj}") testImplementation("org.junit.jupiter:junit-jupiter-api:${jupiterVersion}") testImplementation("org.mockito:mockito-core:${mockitoVersion}") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java index 9b6ccf2..deec714 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java @@ -14,6 +14,8 @@ package com.truzzt.extension.logginghouse.client; +import com.truzzt.extension.logginghouse.client.messages.CreateProcessMessage; +import com.truzzt.extension.logginghouse.client.messages.LogMessage; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index cd0b8ba..c116f1f 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -13,8 +13,14 @@ package com.truzzt.extension.logginghouse.client; +import com.truzzt.extension.logginghouse.client.flyway.FlywayService; +import com.truzzt.extension.logginghouse.client.flyway.migration.DatabaseMigrationManager; import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartSender; +import com.truzzt.extension.logginghouse.client.messages.CreateProcessMessageSender; +import com.truzzt.extension.logginghouse.client.messages.LogMessageSender; +import com.truzzt.extension.logginghouse.client.multipart.IdsMultipartClearingRemoteMessageDispatcher; +import com.truzzt.extension.logginghouse.client.multipart.MultiContextJsonLdSerializer; import de.fraunhofer.iais.eis.LogMessage; import de.fraunhofer.iais.eis.RequestMessage; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAccepted; @@ -82,6 +88,15 @@ public class LoggingHouseClientExtension implements ServiceExtension { @Setting public static final String LOGGINGHOUSE_CLIENT_EXTENSION_ENABLED = "edc.logginghouse.extension.enabled"; + @Setting + private static final String EDC_DATASOURCE_REPAIR_SETTING = "edc.flyway.repair"; + + @Setting + private static final String FLYWAY_CLEAN_ENABLED = "edc.flyway.clean.enable"; + + @Setting + private static final String FLYWAY_CLEAN = "edc.flyway.clean"; + private URL loggingHouseLogUrl; public Monitor monitor; private boolean enabled; @@ -107,6 +122,8 @@ public void initialize(ServiceExtensionContext context) { this.loggingHouseLogUrl = readUrlFromSettings(context); + runFlywayMigrations(context); + registerSerializerClearingHouseMessages(context); registerClearingHouseMessageSenders(context); @@ -129,6 +146,17 @@ private URL readUrlFromSettings(ServiceExtensionContext context) { } } + public void runFlywayMigrations(ServiceExtensionContext context) { + var flywayService = new FlywayService( + context.getMonitor(), + context.getSetting(EDC_DATASOURCE_REPAIR_SETTING, false), + context.getSetting(FLYWAY_CLEAN_ENABLED, false), + context.getSetting(FLYWAY_CLEAN, false) + ); + var migrationManager = new DatabaseMigrationManager(context.getConfig(), context.getMonitor(), flywayService); + migrationManager.migrate(); + } + private void registerEventSubscriber(ServiceExtensionContext context) { monitor.debug("Registering event subscriber for LoggingHouseClientExtension"); @@ -192,7 +220,6 @@ private void registerClearingHouseMessageSenders(ServiceExtensionContext context dispatcherRegistry.register(dispatcher); } - @Override public void start() { if (!enabled) { diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java new file mode 100644 index 0000000..177554d --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package com.truzzt.extension.logginghouse.client.flyway; + +import com.truzzt.extension.logginghouse.client.flyway.connection.DatasourceProperties; +import com.truzzt.extension.logginghouse.client.flyway.connection.DriverManagerConnectionFactory; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.persistence.EdcPersistenceException; +import org.eclipse.edc.sql.datasource.ConnectionFactoryDataSource; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.FlywayException; +import org.flywaydb.core.api.MigrationVersion; +import org.flywaydb.core.api.output.MigrateResult; +import org.flywaydb.core.api.output.RepairResult; + +import java.util.List; +import javax.sql.DataSource; + +public class FlywayService { + + private static final String MIGRATION_LOCATION_BASE = "classpath:migration"; + private static final String MIGRATION_TABLE_NAME = "flyway_schema_history_logginghouse"; + + private final Monitor monitor; + private final boolean tryRepairOnFailedMigration; + private final boolean cleanEnabled; + private final boolean clean; + + public FlywayService(Monitor monitor, boolean tryRepairOnFailedMigration, boolean cleanEnabled, boolean clean) { + this.monitor = monitor; + this.tryRepairOnFailedMigration = tryRepairOnFailedMigration; + this.cleanEnabled = cleanEnabled; + this.clean = clean; + } + + public void cleanDatabase(DatasourceProperties datasourceProperties) { + if (clean) { + monitor.info("Running flyway clean."); + var flyway = setupFlyway(datasourceProperties); + flyway.clean(); + } + } + + public void migrateDatabase(DatasourceProperties datasourceProperties) { + var flyway = setupFlyway(datasourceProperties); + flyway.info().getInfoResult().migrations.stream() + .map(migration -> "Found migration: %s".formatted(migration.filepath)) + .forEach(monitor::info); + + try { + var migrateResult = flyway.migrate(); + handleFlywayMigrationResult(migrateResult); + } catch (FlywayException e) { + if (tryRepairOnFailedMigration) { + repairAndRetryMigration(flyway); + } else { + throw new EdcPersistenceException("Flyway migration failed", e); + } + } + } + + private void repairAndRetryMigration(Flyway flyway) { + try { + var repairResult = flyway.repair(); + handleFlywayRepairResult(repairResult); + var migrateResult = flyway.migrate(); + handleFlywayMigrationResult(migrateResult); + } catch (FlywayException e) { + throw new EdcPersistenceException("Flyway migration failed", e); + } + } + + private void handleFlywayRepairResult(RepairResult repairResult) { + if (!repairResult.repairActions.isEmpty()) { + var repairActions = String.join(", ", repairResult.repairActions); + monitor.info("Repair actions: %s".formatted(repairActions)); + } + + if (!repairResult.warnings.isEmpty()) { + var warnings = String.join(", ", repairResult.warnings); + throw new EdcPersistenceException("Repairing failed: %s".formatted(warnings)); + } + } + + private Flyway setupFlyway(DatasourceProperties datasourceProperties) { + var dataSource = getDataSource(datasourceProperties); + var migrationLocations = List.of(MIGRATION_LOCATION_BASE); + return Flyway.configure() + .baselineVersion(MigrationVersion.fromVersion("0.0.0")) + .baselineOnMigrate(true) + .failOnMissingLocations(true) + .dataSource(dataSource) + .table(MIGRATION_TABLE_NAME) + .locations(migrationLocations.toArray(new String[0])) + .cleanDisabled(!cleanEnabled) + .load(); + } + + private DataSource getDataSource(DatasourceProperties datasourceProperties) { + var connectionFactory = new DriverManagerConnectionFactory(datasourceProperties); + return new ConnectionFactoryDataSource(connectionFactory); + } + + private void handleFlywayMigrationResult(MigrateResult migrateResult) { + if (migrateResult.migrationsExecuted > 0) { + monitor.info(String.format( + "Successfully migrated database from version %s to version %s", + migrateResult.initialSchemaVersion, + migrateResult.targetSchemaVersion)); + } else { + monitor.info(String.format( + "No migration necessary. Current version is %s", + migrateResult.initialSchemaVersion)); + } + } + +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java new file mode 100644 index 0000000..c990604 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package com.truzzt.extension.logginghouse.client.flyway.connection; + +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.system.configuration.Config; + +import static org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry.DEFAULT_DATASOURCE; + +public class DatasourceProperties { + private static final String LOGGING_HOUSE_DATASOURCE = "logginghouse"; + + private static final String DATASOURCE_SETTING_NAME = "edc.datasource.%s.name"; + private static final String DATASOURCE_SETTING_JDBC_URL = "edc.datasource.%s.url"; + private static final String DATASOURCE_SETTING_USER = "edc.datasource.%s.user"; + private static final String DATASOURCE_SETTING_PASSWORD = "edc.datasource.%s.password"; + + private final String name; + private final String jdbcUrl; + private final String user; + private final String password; + + public DatasourceProperties(Config config) { + name = getSetting(config, String.format(DATASOURCE_SETTING_NAME, LOGGING_HOUSE_DATASOURCE), + String.format(DATASOURCE_SETTING_NAME, DEFAULT_DATASOURCE)); + + jdbcUrl = getSetting(config, String.format(DATASOURCE_SETTING_JDBC_URL, LOGGING_HOUSE_DATASOURCE), + String.format(DATASOURCE_SETTING_JDBC_URL, DEFAULT_DATASOURCE)); + + user = getSetting(config, String.format(DATASOURCE_SETTING_USER, LOGGING_HOUSE_DATASOURCE), + String.format(DATASOURCE_SETTING_USER, DEFAULT_DATASOURCE)); + + password = getSetting(config, String.format(DATASOURCE_SETTING_PASSWORD, LOGGING_HOUSE_DATASOURCE), + String.format(DATASOURCE_SETTING_PASSWORD, DEFAULT_DATASOURCE)); + } + + private String getSetting(Config config, String setting, String defaultSetting) { + try { + return config.getString(setting); + } catch (EdcException e) { + return config.getString(defaultSetting); + } + } + + public String getName() { + return name; + } + + public String getJdbcUrl() { + return jdbcUrl; + } + + public String getUser() { + return user; + } + + public String getPassword() { + return password; + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DriverManagerConnectionFactory.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DriverManagerConnectionFactory.java new file mode 100644 index 0000000..f1b1a53 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DriverManagerConnectionFactory.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package com.truzzt.extension.logginghouse.client.flyway.connection; + +import org.eclipse.edc.spi.persistence.EdcPersistenceException; +import org.eclipse.edc.sql.ConnectionFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +public class DriverManagerConnectionFactory implements ConnectionFactory { + + private static final String CONNECTION_PROPERTY_USER = "user"; + private static final String CONNECTION_PROPERTY_PASSWORD = "password"; + + private final DatasourceProperties datasourceProperties; + + public DriverManagerConnectionFactory(DatasourceProperties datasourceProperties) { + this.datasourceProperties = datasourceProperties; + } + + @Override + public Connection create() { + try { + var properties = new Properties(); + properties.setProperty(CONNECTION_PROPERTY_USER, datasourceProperties.getUser()); + properties.setProperty(CONNECTION_PROPERTY_PASSWORD, datasourceProperties.getPassword()); + return DriverManager.getConnection(datasourceProperties.getJdbcUrl(), properties); + } catch (SQLException e) { + throw new EdcPersistenceException(e); + } + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/migration/DatabaseMigrationManager.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/migration/DatabaseMigrationManager.java new file mode 100644 index 0000000..81aab02 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/migration/DatabaseMigrationManager.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + * + */ + +package com.truzzt.extension.logginghouse.client.flyway.migration; + +import com.truzzt.extension.logginghouse.client.flyway.FlywayService; +import com.truzzt.extension.logginghouse.client.flyway.connection.DatasourceProperties; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.configuration.Config; + +public class DatabaseMigrationManager { + private final Config config; + private final Monitor monitor; + private final FlywayService flywayService; + + public DatabaseMigrationManager(Config config, Monitor monitor, FlywayService flywayService) { + this.config = config; + this.monitor = monitor; + this.flywayService = flywayService; + } + + public void migrate() { + var datasourceProperties = new DatasourceProperties(config); + monitor.info("Using datasource %s to apply flyway migrations".formatted(datasourceProperties.getName())); + + flywayService.cleanDatabase(datasourceProperties); + flywayService.migrateDatabase(datasourceProperties); + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessage.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessage.java similarity index 86% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessage.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessage.java index 88854ba..bc1b298 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessage.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessage.java @@ -12,8 +12,9 @@ * */ -package com.truzzt.extension.logginghouse.client; +package com.truzzt.extension.logginghouse.client.messages; +import com.truzzt.extension.logginghouse.client.multipart.ExtendedMessageProtocolClearing; import org.eclipse.edc.spi.types.domain.message.RemoteMessage; import java.net.URI; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessageSender.java similarity index 97% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessageSender.java index ee3c71d..35ab185 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/CreateProcessMessageSender.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessageSender.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client; +package com.truzzt.extension.logginghouse.client.messages; import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; import com.truzzt.extension.logginghouse.client.ids.multipart.CalendarUtil; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessage.java similarity index 86% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessage.java index e28d206..7633f2a 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessage.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessage.java @@ -13,8 +13,9 @@ * */ -package com.truzzt.extension.logginghouse.client; +package com.truzzt.extension.logginghouse.client.messages; +import com.truzzt.extension.logginghouse.client.multipart.ExtendedMessageProtocolClearing; import org.eclipse.edc.spi.types.domain.message.RemoteMessage; import java.net.URI; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessageSender.java similarity index 98% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessageSender.java index b5a9139..9a014bf 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LogMessageSender.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessageSender.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client; +package com.truzzt.extension.logginghouse.client.messages; import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; import com.truzzt.extension.logginghouse.client.ids.multipart.CalendarUtil; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ExtendedMessageProtocolClearing.java similarity index 92% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ExtendedMessageProtocolClearing.java index 71ba1fc..e32cec8 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ExtendedMessageProtocolClearing.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ExtendedMessageProtocolClearing.java @@ -13,7 +13,7 @@ * */ -package com.truzzt.extension.logginghouse.client; +package com.truzzt.extension.logginghouse.client.multipart; public final class ExtendedMessageProtocolClearing { diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsMultipartClearingRemoteMessageDispatcher.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/IdsMultipartClearingRemoteMessageDispatcher.java similarity index 93% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsMultipartClearingRemoteMessageDispatcher.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/IdsMultipartClearingRemoteMessageDispatcher.java index 6f757cc..e08e965 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsMultipartClearingRemoteMessageDispatcher.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/IdsMultipartClearingRemoteMessageDispatcher.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client; +package com.truzzt.extension.logginghouse.client.multipart; import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartRemoteMessageDispatcher; import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartSender; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/MultiContextJsonLdSerializer.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/MultiContextJsonLdSerializer.java similarity index 98% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/MultiContextJsonLdSerializer.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/MultiContextJsonLdSerializer.java index 985dfb2..721cf55 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/MultiContextJsonLdSerializer.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/MultiContextJsonLdSerializer.java @@ -13,7 +13,7 @@ * */ -package com.truzzt.extension.logginghouse.client; +package com.truzzt.extension.logginghouse.client.multipart; import com.fasterxml.jackson.annotation.JsonTypeName; import com.fasterxml.jackson.core.JsonGenerator; diff --git a/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql b/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql new file mode 100644 index 0000000..f32e8fd --- /dev/null +++ b/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql @@ -0,0 +1,6 @@ +-- table: edc_logging_house_event +CREATE TABLE IF NOT EXISTS edc_logging_house_event +( + logging_house_event_id VARCHAR NOT NULL, + PRIMARY KEY (logging_house_event_id) +); From 3df5a4b700f1446ba7f5d9ad8d24a3fec804d6ea Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Fri, 28 Jun 2024 09:34:44 -0300 Subject: [PATCH 15/25] feat: logging house messages store --- .../client/IdsClearingHouseServiceImpl.java | 168 ------------------ .../client/LoggingHouseClientExtension.java | 42 +++-- .../client/LoggingHouseEventSubscriber.java | 104 +++++++++++ .../spi/store/LoggingHouseMessageStore.java | 22 +++ .../client/spi/types/LoggingHouseMessage.java | 72 ++++++++ .../sql/SqlLoggingHouseMessageStore.java | 69 +++++++ .../sql/schema/BaseSqlDialectStatements.java | 31 ++++ .../schema/LoggingHouseEventStatements.java | 40 +++++ .../postgres/PostgresDialectStatements.java | 27 +++ .../migration/V0_0_1__Create_Tables.sql | 25 ++- 10 files changed, 419 insertions(+), 181 deletions(-) delete mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseEventSubscriber.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/BaseSqlDialectStatements.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/LoggingHouseEventStatements.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/postgres/PostgresDialectStatements.java diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java deleted file mode 100644 index deec714..0000000 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/IdsClearingHouseServiceImpl.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2022 sovity GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * sovity GmbH - initial API and implementation - * - */ - -package com.truzzt.extension.logginghouse.client; - -import com.truzzt.extension.logginghouse.client.messages.CreateProcessMessage; -import com.truzzt.extension.logginghouse.client.messages.LogMessage; -import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; -import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; -import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessEvent; -import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; -import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; -import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.event.Event; -import org.eclipse.edc.spi.event.EventEnvelope; -import org.eclipse.edc.spi.event.EventSubscriber; -import org.eclipse.edc.spi.message.RemoteMessageDispatcherRegistry; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.system.Hostname; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -public class IdsClearingHouseServiceImpl implements EventSubscriber { - - private final RemoteMessageDispatcherRegistry dispatcherRegistry; - private final URI connectorBaseUrl; - private final URL clearingHouseLogUrl; - private final ContractNegotiationStore contractNegotiationStore; - private final TransferProcessStore transferProcessStore; - private final Monitor monitor; - - public IdsClearingHouseServiceImpl( - RemoteMessageDispatcherRegistry dispatcherRegistry, - Hostname hostname, - URL clearingHouseLogUrl, - ContractNegotiationStore contractNegotiationStore, - TransferProcessStore transferProcessStore, - Monitor monitor) { - this.dispatcherRegistry = dispatcherRegistry; - this.clearingHouseLogUrl = clearingHouseLogUrl; - this.contractNegotiationStore = contractNegotiationStore; - this.transferProcessStore = transferProcessStore; - this.monitor = monitor; - - try { - connectorBaseUrl = getConnectorBaseUrl(hostname); - } catch (URISyntaxException e) { - throw new EdcException("Could not create connectorBaseUrl. Hostname can be set using:" + - " edc.hostname", e); - } - } - - public void createProcess(ContractAgreement contractAgreement, URL clearingHouseLogUrl) { - // Create PID - List processOwners = new ArrayList<>(); - processOwners.add(contractAgreement.getConsumerId()); - processOwners.add(contractAgreement.getProviderId()); - - monitor.info("Creating Process in LoggingHouse"); - var logMessage = new CreateProcessMessage(clearingHouseLogUrl, connectorBaseUrl, contractAgreement.getId(), processOwners); - - try { - dispatcherRegistry.dispatch(Object.class, logMessage); - } catch (EdcException e) { - if (e.getMessage().startsWith("No provider dispatcher registered for protocol")) { - throw e; - } else { - monitor.severe("Unhandled exception while creating process in LoggingHouse. " + e.getMessage()); - // Print stack trace - String errorStr; - try (StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw)) { - e.printStackTrace(pw); - errorStr = sw.toString(); - } catch (IOException ex) { - throw new RuntimeException("Error while converting the stacktrace"); - } - monitor.severe(errorStr); - } - } - } - - public void logContractAgreement(ContractAgreement contractAgreement, URL clearingHouseLogUrl) { - monitor.info("Logging ContractAgreement to LoggingHouse with contract id: " + contractAgreement.getId()); - var logMessage = new LogMessage(clearingHouseLogUrl, connectorBaseUrl, contractAgreement); - dispatcherRegistry.dispatch(Object.class, logMessage); - } - - public void logTransferProcess(TransferProcess transferProcess, URL clearingHouseLogUrl) { - monitor.info("Logging TransferProcess to LoggingHouse"); - var logMessage = new LogMessage(clearingHouseLogUrl, connectorBaseUrl, transferProcess); - dispatcherRegistry.dispatch(Object.class, logMessage); - } - - @Override - public void on(EventEnvelope event) { - try { - if (event.getPayload() instanceof ContractNegotiationFinalized contractNegotiationFinalized) { - var contractAgreement = resolveContractAgreement(contractNegotiationFinalized); - var pid = contractAgreement.getId(); - - // Create Process - var extendedProcessUrl = new URL(clearingHouseLogUrl + "/process/" + pid); - try { - createProcess(contractAgreement, extendedProcessUrl); - } catch (Exception e) { - monitor.warning("Could not create process in LoggingHouse: " + e.getMessage()); - } - - // Log Contract Agreement - var extendedLogUrl = new URL(clearingHouseLogUrl + "/messages/log/" + pid); - try { - logContractAgreement(contractAgreement, extendedLogUrl); - } catch (Exception e) { - monitor.warning("Could not log ContractNegotiation to LoggingHouse: " + e.getMessage()); - } - } else if (event.getPayload() instanceof TransferProcessEvent transferProcessEvent) { - monitor.debug("Logging transfer event with id " + event.getId()); - - var transferProcess = resolveTransferProcess(transferProcessEvent); - var pid = transferProcess.getContractId(); - var extendedUrl = new URL(clearingHouseLogUrl + "/messages/log/" + pid); - try { - logTransferProcess(transferProcess, extendedUrl); - } catch (Exception e) { - monitor.warning("Could not log TransferProcess to LoggingHouse: " + e.getMessage()); - } - } - } catch (MalformedURLException e) { - throw new EdcException("Could not create extended clearinghouse url."); - } - } - - private ContractAgreement resolveContractAgreement(ContractNegotiationFinalized contractNegotiationFinalized) throws NullPointerException { - var contractNegotiationId = contractNegotiationFinalized.getContractNegotiationId(); - var contractNegotiation = contractNegotiationStore.findById(contractNegotiationId); - return Objects.requireNonNull(contractNegotiation).getContractAgreement(); - } - - private TransferProcess resolveTransferProcess(TransferProcessEvent transferProcessEvent) { - var transferProcessId = transferProcessEvent.getTransferProcessId(); - return transferProcessStore.findById(transferProcessId); - } - - private URI getConnectorBaseUrl(Hostname hostname) throws URISyntaxException { - return new URI(String.format("https://%s/", hostname.get())); - } -} \ No newline at end of file diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index c116f1f..714745b 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -8,6 +8,7 @@ * SPDX-License-Identifier: Apache-2.0 * * Contributors: + * truzzt GmbH - Initial implementation * */ @@ -21,6 +22,8 @@ import com.truzzt.extension.logginghouse.client.messages.LogMessageSender; import com.truzzt.extension.logginghouse.client.multipart.IdsMultipartClearingRemoteMessageDispatcher; import com.truzzt.extension.logginghouse.client.multipart.MultiContextJsonLdSerializer; +import com.truzzt.extension.logginghouse.client.store.sql.SqlLoggingHouseMessageStore; +import com.truzzt.extension.logginghouse.client.store.sql.schema.postgres.PostgresDialectStatements; import de.fraunhofer.iais.eis.LogMessage; import de.fraunhofer.iais.eis.RequestMessage; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAccepted; @@ -48,6 +51,9 @@ import org.eclipse.edc.spi.system.ServiceExtension; import org.eclipse.edc.spi.system.ServiceExtensionContext; import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.sql.QueryExecutor; +import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; +import org.eclipse.edc.transaction.spi.TransactionContext; import java.net.MalformedURLException; import java.net.URL; @@ -55,7 +61,6 @@ public class LoggingHouseClientExtension implements ServiceExtension { - public static final String LOGGINGHOUSE_CLIENT_EXTENSION = "LoggingHouseClientExtension"; private static final String TYPE_MANAGER_SERIALIZER_KEY = "ids-clearinghouse"; @@ -71,6 +76,13 @@ public class LoggingHouseClientExtension implements ServiceExtension { @Inject private Hostname hostname; + @Inject + private DataSourceRegistry dataSourceRegistry; + @Inject + private TransactionContext transactionContext; + @Inject + private QueryExecutor queryExecutor; + @Inject private ContractNegotiationStore contractNegotiationStore; @Inject @@ -101,7 +113,6 @@ public class LoggingHouseClientExtension implements ServiceExtension { public Monitor monitor; private boolean enabled; - @Override public String name() { return LOGGINGHOUSE_CLIENT_EXTENSION; @@ -127,7 +138,9 @@ public void initialize(ServiceExtensionContext context) { registerSerializerClearingHouseMessages(context); registerClearingHouseMessageSenders(context); - registerEventSubscriber(context); + var loggingHouseMessageStore = initializeLoggingHouseMessageStore(typeManager); + + registerEventSubscriber(context, loggingHouseMessageStore); } private URL readUrlFromSettings(ServiceExtensionContext context) { @@ -146,7 +159,7 @@ private URL readUrlFromSettings(ServiceExtensionContext context) { } } - public void runFlywayMigrations(ServiceExtensionContext context) { + private void runFlywayMigrations(ServiceExtensionContext context) { var flywayService = new FlywayService( context.getMonitor(), context.getSetting(EDC_DATASOURCE_REPAIR_SETTING, false), @@ -157,13 +170,22 @@ public void runFlywayMigrations(ServiceExtensionContext context) { migrationManager.migrate(); } - private void registerEventSubscriber(ServiceExtensionContext context) { + private SqlLoggingHouseMessageStore initializeLoggingHouseMessageStore(TypeManager typeManager) { + return new SqlLoggingHouseMessageStore( + dataSourceRegistry, + DataSourceRegistry.DEFAULT_DATASOURCE, + transactionContext, + typeManager.getMapper(), + new PostgresDialectStatements(), + queryExecutor + ); + } + + private void registerEventSubscriber(ServiceExtensionContext context, SqlLoggingHouseMessageStore loggingHouseMessageStore) { monitor.debug("Registering event subscriber for LoggingHouseClientExtension"); - var eventSubscriber = new IdsClearingHouseServiceImpl( - dispatcherRegistry, - hostname, - loggingHouseLogUrl, + var eventSubscriber = new LoggingHouseEventSubscriber( + loggingHouseMessageStore, contractNegotiationStore, transferProcessStore, monitor); @@ -178,7 +200,7 @@ private void registerEventSubscriber(ServiceExtensionContext context) { eventRouter.registerSync(TransferProcessCompleted.class, eventSubscriber); eventRouter.registerSync(TransferProcessFailed.class, eventSubscriber); eventRouter.registerSync(TransferProcessTerminated.class, eventSubscriber); - context.registerService(IdsClearingHouseServiceImpl.class, eventSubscriber); + context.registerService(LoggingHouseEventSubscriber.class, eventSubscriber); monitor.debug("Registered event subscriber for LoggingHouseClientExtension"); } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseEventSubscriber.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseEventSubscriber.java new file mode 100644 index 0000000..76d98ad --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseEventSubscriber.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2022 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client; + +import com.truzzt.extension.logginghouse.client.spi.store.LoggingHouseMessageStore; +import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessage; +import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; +import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; +import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessEvent; +import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; +import org.eclipse.edc.spi.event.Event; +import org.eclipse.edc.spi.event.EventEnvelope; +import org.eclipse.edc.spi.event.EventSubscriber; +import org.eclipse.edc.spi.monitor.Monitor; + +import java.time.ZonedDateTime; +import java.util.Objects; + +public class LoggingHouseEventSubscriber implements EventSubscriber { + + private final LoggingHouseMessageStore loggingHouseMessageStore; + private final ContractNegotiationStore contractNegotiationStore; + private final TransferProcessStore transferProcessStore; + private final Monitor monitor; + + public LoggingHouseEventSubscriber( + LoggingHouseMessageStore loggingHouseMessageStore, + ContractNegotiationStore contractNegotiationStore, + TransferProcessStore transferProcessStore, + Monitor monitor) { + this.loggingHouseMessageStore = loggingHouseMessageStore; + this.contractNegotiationStore = contractNegotiationStore; + this.transferProcessStore = transferProcessStore; + this.monitor = monitor; + } + + public void storeContractAgreement(ContractAgreement contractAgreement) { + monitor.info("Storing ContractAgreement to send to LoggingHouse"); + + var message = LoggingHouseMessage.Builder.newInstance() + .eventToLog(contractAgreement) + .createdAt(ZonedDateTime.now()) + .build(); + loggingHouseMessageStore.save(message); + } + + public void storeTransferProcess(TransferProcess transferProcess) { + monitor.info("Storing TransferProcess to send to LoggingHouse"); + + var message = LoggingHouseMessage.Builder.newInstance() + .eventToLog(transferProcess) + .createdAt(ZonedDateTime.now()) + .build(); + loggingHouseMessageStore.save(message); + } + + @Override + public void on(EventEnvelope event) { + if (event.getPayload() instanceof ContractNegotiationFinalized contractNegotiationFinalized) { + monitor.debug("Storing ContractNegotiationFinalized event with id " + event.getId()); + + var contractAgreement = resolveContractAgreement(contractNegotiationFinalized); + try { + storeContractAgreement(contractAgreement); + } catch (Exception e) { + monitor.warning("Could not store ContractNegotiation: " + e.getMessage()); + } + } else if (event.getPayload() instanceof TransferProcessEvent transferProcessEvent) { + monitor.debug("Storing TransferProcess event with id " + event.getId()); + + var transferProcess = resolveTransferProcess(transferProcessEvent); + try { + storeTransferProcess(transferProcess); + } catch (Exception e) { + monitor.warning("Could not store TransferProcess: " + e.getMessage()); + } + } + } + + private ContractAgreement resolveContractAgreement(ContractNegotiationFinalized contractNegotiationFinalized) throws NullPointerException { + var contractNegotiationId = contractNegotiationFinalized.getContractNegotiationId(); + var contractNegotiation = contractNegotiationStore.findById(contractNegotiationId); + return Objects.requireNonNull(contractNegotiation).getContractAgreement(); + } + + private TransferProcess resolveTransferProcess(TransferProcessEvent transferProcessEvent) { + var transferProcessId = transferProcessEvent.getTransferProcessId(); + return transferProcessStore.findById(transferProcessId); + } +} \ No newline at end of file diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java new file mode 100644 index 0000000..d04fa72 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024 truzzt GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * truzzt GmbH - Initial implementation + * + */ + +package com.truzzt.extension.logginghouse.client.spi.store; + +import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessage; + +public interface LoggingHouseMessageStore { + + void save(LoggingHouseMessage message); +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java new file mode 100644 index 0000000..7253f91 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2024 truzzt GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * truzzt GmbH - Initial implementation + * + */ + +package com.truzzt.extension.logginghouse.client.spi.types; + +import java.time.ZonedDateTime; +import java.util.Objects; + +public class LoggingHouseMessage { + private Long id; + private Object eventToLog; + private ZonedDateTime createdAt; + private ZonedDateTime sentAt; + + public Long getId() { + return id; + } + public Object getEventToLog() { + return eventToLog; + } + public ZonedDateTime getCreatedAt() { + return createdAt; + } + public ZonedDateTime getSentAt() { + return sentAt; + } + + public static final class Builder { + private final LoggingHouseMessage event = new LoggingHouseMessage(); + + private Builder() { + } + + public static LoggingHouseMessage.Builder newInstance() { + return new LoggingHouseMessage.Builder(); + } + + public LoggingHouseMessage.Builder id(Long id) { + this.event.id = id; + return this; + } + public LoggingHouseMessage.Builder eventToLog(Object eventToLog) { + this.event.eventToLog = eventToLog; + return this; + } + public LoggingHouseMessage.Builder createdAt(ZonedDateTime createdAt) { + this.event.createdAt = createdAt; + return this; + } + public LoggingHouseMessage.Builder sentAt(ZonedDateTime sentAt) { + this.event.sentAt = sentAt; + return this; + } + + public LoggingHouseMessage build() { + Objects.requireNonNull(this.event.eventToLog, "Message eventToLog must not be null"); + Objects.requireNonNull(this.event.createdAt, "Message createdAt must not be null"); + return this.event; + } + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java new file mode 100644 index 0000000..42e0c0b --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - Initial implementation + * truzzt GmbH - PostgreSQL implementation + * + */ + +package com.truzzt.extension.logginghouse.client.store.sql; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.truzzt.extension.logginghouse.client.spi.store.LoggingHouseMessageStore; +import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessage; +import com.truzzt.extension.logginghouse.client.store.sql.schema.LoggingHouseEventStatements; +import org.eclipse.edc.spi.persistence.EdcPersistenceException; +import org.eclipse.edc.sql.QueryExecutor; +import org.eclipse.edc.sql.store.AbstractSqlStore; +import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; +import org.eclipse.edc.transaction.spi.TransactionContext; + +import java.time.ZonedDateTime; +import java.util.Objects; + +public class SqlLoggingHouseMessageStore extends AbstractSqlStore implements LoggingHouseMessageStore { + + private final LoggingHouseEventStatements statements; + + public SqlLoggingHouseMessageStore(DataSourceRegistry dataSourceRegistry, + String dataSourceName, + TransactionContext transactionContext, + ObjectMapper objectMapper, + LoggingHouseEventStatements statements, + QueryExecutor queryExecutor) { + super(dataSourceRegistry, dataSourceName, transactionContext, objectMapper, queryExecutor); + this.statements = statements; + } + + @Override + public void save(LoggingHouseMessage event) { + + Objects.requireNonNull(event); + Objects.requireNonNull(event.getEventToLog()); + Objects.requireNonNull(event.getCreatedAt()); + + transactionContext.execute(() -> { + try (var connection = getConnection()) { + queryExecutor.execute(connection, statements.getInsertTemplate(), + toJson(event.getEventToLog()), + mapFromZonedDateTime(event.getCreatedAt()) + ); + + } catch (Exception e) { + throw new EdcPersistenceException(e.getMessage(), e); + } + }); + } + + private Long mapFromZonedDateTime(ZonedDateTime zonedDateTime) { + return zonedDateTime != null ? zonedDateTime.toEpochSecond() : null; + } + +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/BaseSqlDialectStatements.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/BaseSqlDialectStatements.java new file mode 100644 index 0000000..6017294 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/BaseSqlDialectStatements.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - Initial implementation + * truzzt GmbH - PostgreSQL implementation + * + */ + +package com.truzzt.extension.logginghouse.client.store.sql.schema; + +import static java.lang.String.format; + +public class BaseSqlDialectStatements implements LoggingHouseEventStatements { + + @Override + public String getInsertTemplate() { + return format("INSERT INTO %s (%s, %s) VALUES (?%s, ?)", + getLoggingHouseMessageTable(), + getEventToLogColumn(), + getCreatedAtColumn(), + getFormatAsJsonOperator() + ); + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/LoggingHouseEventStatements.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/LoggingHouseEventStatements.java new file mode 100644 index 0000000..5854cdb --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/LoggingHouseEventStatements.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - Initial implementation + * truzzt GmbH - PostgreSQL implementation + * + */ + +package com.truzzt.extension.logginghouse.client.store.sql.schema; + +import org.eclipse.edc.sql.dialect.PostgresDialect; + +public interface LoggingHouseEventStatements { + + default String getLoggingHouseMessageTable() { + return "edc_logging_house_message"; + } + + default String getEventToLogColumn() { + return "event_to_log"; + } + + default String getCreatedAtColumn() { + return "created_at"; + } + + String getInsertTemplate(); + + default String getFormatAsJsonOperator() { + return PostgresDialect.getJsonCastOperator(); + } + +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/postgres/PostgresDialectStatements.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/postgres/PostgresDialectStatements.java new file mode 100644 index 0000000..e48620d --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/postgres/PostgresDialectStatements.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - Initial implementation + * truzzt GmbH - PostgreSQL implementation + * + */ + +package com.truzzt.extension.logginghouse.client.store.sql.schema.postgres; + +import com.truzzt.extension.logginghouse.client.store.sql.schema.BaseSqlDialectStatements; +import org.eclipse.edc.sql.dialect.PostgresDialect; + +public class PostgresDialectStatements extends BaseSqlDialectStatements { + + @Override + public String getFormatAsJsonOperator() { + return PostgresDialect.getJsonCastOperator(); + } +} diff --git a/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql b/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql index f32e8fd..00c67b6 100644 --- a/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql +++ b/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql @@ -1,6 +1,25 @@ +-- +-- Copyright (c) 2024 Daimler TSS GmbH +-- +-- This program and the accompanying materials are made available under the +-- terms of the Apache License, Version 2.0 which is available at +-- https://www.apache.org/licenses/LICENSE-2.0 +-- +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Contributors: +-- Daimler TSS GmbH - Initial SQL Query +-- + +-- THIS SCHEMA HAS BEEN WRITTEN AND TESTED ONLY FOR POSTGRES + -- table: edc_logging_house_event -CREATE TABLE IF NOT EXISTS edc_logging_house_event +CREATE TABLE IF NOT EXISTS edc_logging_house_message ( - logging_house_event_id VARCHAR NOT NULL, - PRIMARY KEY (logging_house_event_id) + logging_house_message_id BIGSERIAL NOT NULL, + event_to_log JSON NOT NULL, + created_at TIMESTAMP NOT NULL, + sent_at TIMESTAMP NOT NULL, + PRIMARY KEY (logging_house_message_id) ); +COMMENT ON COLUMN edc_logging_house_message.event_to_log IS 'Event to log serialized as JSON'; From cb9127cec0fb1185be812b5b762a51e39ac4c721 Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Mon, 1 Jul 2024 16:45:13 -0300 Subject: [PATCH 16/25] feat: logging house messages store --- .../logginghouse/client/LoggingHouseClientExtension.java | 4 ---- .../extension/logginghouse/client/flyway/FlywayService.java | 6 ++---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 714745b..24f4dfe 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -103,9 +103,6 @@ public class LoggingHouseClientExtension implements ServiceExtension { @Setting private static final String EDC_DATASOURCE_REPAIR_SETTING = "edc.flyway.repair"; - @Setting - private static final String FLYWAY_CLEAN_ENABLED = "edc.flyway.clean.enable"; - @Setting private static final String FLYWAY_CLEAN = "edc.flyway.clean"; @@ -163,7 +160,6 @@ private void runFlywayMigrations(ServiceExtensionContext context) { var flywayService = new FlywayService( context.getMonitor(), context.getSetting(EDC_DATASOURCE_REPAIR_SETTING, false), - context.getSetting(FLYWAY_CLEAN_ENABLED, false), context.getSetting(FLYWAY_CLEAN, false) ); var migrationManager = new DatabaseMigrationManager(context.getConfig(), context.getMonitor(), flywayService); diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java index 177554d..4863dd9 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java @@ -35,13 +35,11 @@ public class FlywayService { private final Monitor monitor; private final boolean tryRepairOnFailedMigration; - private final boolean cleanEnabled; private final boolean clean; - public FlywayService(Monitor monitor, boolean tryRepairOnFailedMigration, boolean cleanEnabled, boolean clean) { + public FlywayService(Monitor monitor, boolean tryRepairOnFailedMigration, boolean clean) { this.monitor = monitor; this.tryRepairOnFailedMigration = tryRepairOnFailedMigration; - this.cleanEnabled = cleanEnabled; this.clean = clean; } @@ -104,7 +102,7 @@ private Flyway setupFlyway(DatasourceProperties datasourceProperties) { .dataSource(dataSource) .table(MIGRATION_TABLE_NAME) .locations(migrationLocations.toArray(new String[0])) - .cleanDisabled(!cleanEnabled) + .cleanDisabled(!clean) .load(); } From 2e3489e69b273f94192cb4991afc7150b4a897db Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Mon, 1 Jul 2024 20:06:23 -0300 Subject: [PATCH 17/25] feat: logging house sender worker --- .../logginghouse/client/ConfigConstants.java | 14 ++ .../client/LoggingHouseClientExtension.java | 79 ++++++----- .../spi/store/LoggingHouseMessageStore.java | 4 + .../client/spi/types/LoggingHouseMessage.java | 45 ++++++ .../sql/SqlLoggingHouseMessageStore.java | 7 + .../client/worker/MessageWorker.java | 110 +++++++++++++++ .../client/worker/WorkersManager.java | 132 ++++++++++++++++++ 7 files changed, 356 insertions(+), 35 deletions(-) create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersManager.java diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java new file mode 100644 index 0000000..dfd0ebc --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java @@ -0,0 +1,14 @@ +package com.truzzt.extension.logginghouse.client; + +public class ConfigConstants { + + static final String LOGGINGHOUSE_ENABLED = "edc.logginghouse.client.enabled"; + + static final String LOGGINGHOUSE_SERVER_URL_SETTING = "edc.logginghouse.client.server.url"; + + static final String LOGGINGHOUSE_FLYWAY_REPAIR_SETTING = "edc.logginghouse.client.flyway.repair"; + + static final String LOGGINGHOUSE_FLYWAY_CLEAN_SETTING = "edc.logginghouse.client.flyway.clean"; + + static final String LOGGINGHOUSE_EXTENSION_MAX_WORKERS = "edc.logginghouse.client.workers.max"; +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 24f4dfe..07260de 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -22,8 +22,10 @@ import com.truzzt.extension.logginghouse.client.messages.LogMessageSender; import com.truzzt.extension.logginghouse.client.multipart.IdsMultipartClearingRemoteMessageDispatcher; import com.truzzt.extension.logginghouse.client.multipart.MultiContextJsonLdSerializer; +import com.truzzt.extension.logginghouse.client.spi.store.LoggingHouseMessageStore; import com.truzzt.extension.logginghouse.client.store.sql.SqlLoggingHouseMessageStore; import com.truzzt.extension.logginghouse.client.store.sql.schema.postgres.PostgresDialectStatements; +import com.truzzt.extension.logginghouse.client.worker.WorkersManager; import de.fraunhofer.iais.eis.LogMessage; import de.fraunhofer.iais.eis.RequestMessage; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAccepted; @@ -31,15 +33,15 @@ import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationTerminated; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessCompleted; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessFailed; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessInitiated; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessRequested; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessStarted; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessTerminated; import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessStarted; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessCompleted; + import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.runtime.metamodel.annotation.Setting; import org.eclipse.edc.spi.EdcException; import org.eclipse.edc.spi.asset.AssetIndex; import org.eclipse.edc.spi.event.EventRouter; @@ -59,6 +61,12 @@ import java.net.URL; import java.util.Map; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_ENABLED; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_SERVER_URL_SETTING; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_FLYWAY_REPAIR_SETTING; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_FLYWAY_CLEAN_SETTING; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_EXTENSION_MAX_WORKERS; + public class LoggingHouseClientExtension implements ServiceExtension { public static final String LOGGINGHOUSE_CLIENT_EXTENSION = "LoggingHouseClientExtension"; @@ -94,21 +102,12 @@ public class LoggingHouseClientExtension implements ServiceExtension { "cat", "http://w3id.org/mds/data-categories#", "ids", "https://w3id.org/idsa/core/", "idsc", "https://w3id.org/idsa/code/"); - @Setting - public static final String LOGGINGHOUSE_LOG_URL_SETTING = "edc.logginghouse.extension.url"; - - @Setting - public static final String LOGGINGHOUSE_CLIENT_EXTENSION_ENABLED = "edc.logginghouse.extension.enabled"; - - @Setting - private static final String EDC_DATASOURCE_REPAIR_SETTING = "edc.flyway.repair"; - @Setting - private static final String FLYWAY_CLEAN = "edc.flyway.clean"; - - private URL loggingHouseLogUrl; public Monitor monitor; private boolean enabled; + private URL loggingHouseLogUrl; + + private WorkersManager workersManager; @Override public String name() { @@ -117,50 +116,50 @@ public String name() { @Override public void initialize(ServiceExtensionContext context) { - this.monitor = context.getMonitor(); + monitor = context.getMonitor(); - var extensionEnabled = context.getSetting(LOGGINGHOUSE_CLIENT_EXTENSION_ENABLED, true); + var extensionEnabled = context.getSetting(LOGGINGHOUSE_ENABLED, true); if (!extensionEnabled) { - this.enabled = false; - this.monitor.info("Logginghouse client extension is disabled."); + enabled = false; + monitor.info("Logginghouse client extension is disabled."); return; } - this.enabled = true; - this.monitor.info("Logginghouse client extension is enabled."); + enabled = true; + monitor.info("Logginghouse client extension is enabled."); - this.loggingHouseLogUrl = readUrlFromSettings(context); + loggingHouseLogUrl = readUrlFromSettings(context); runFlywayMigrations(context); registerSerializerClearingHouseMessages(context); - registerClearingHouseMessageSenders(context); - var loggingHouseMessageStore = initializeLoggingHouseMessageStore(typeManager); + var store = initializeLoggingHouseMessageStore(typeManager); + registerEventSubscriber(context, store); - registerEventSubscriber(context, loggingHouseMessageStore); + registerDispatcher(context); + workersManager = initializeWorkersManager(context, store); } private URL readUrlFromSettings(ServiceExtensionContext context) { try { - var urlString = context.getSetting(LoggingHouseClientExtension.LOGGINGHOUSE_LOG_URL_SETTING, null); + var urlString = context.getSetting(LOGGINGHOUSE_SERVER_URL_SETTING, null); if (urlString == null) { - throw new EdcException(String.format("Could not initialize " + - "LoggingHouseClientExtension: " + - "No url specified using setting %s", LoggingHouseClientExtension.LOGGINGHOUSE_LOG_URL_SETTING)); + throw new EdcException(String.format("Could not initialize LoggingHouseClientExtension: " + + "No url specified using setting %s", LOGGINGHOUSE_SERVER_URL_SETTING)); } return new URL(urlString); } catch (MalformedURLException e) { throw new EdcException(String.format("Could not parse setting %s to Url", - LoggingHouseClientExtension.LOGGINGHOUSE_LOG_URL_SETTING), e); + LOGGINGHOUSE_SERVER_URL_SETTING), e); } } private void runFlywayMigrations(ServiceExtensionContext context) { var flywayService = new FlywayService( context.getMonitor(), - context.getSetting(EDC_DATASOURCE_REPAIR_SETTING, false), - context.getSetting(FLYWAY_CLEAN, false) + context.getSetting(LOGGINGHOUSE_FLYWAY_REPAIR_SETTING, false), + context.getSetting(LOGGINGHOUSE_FLYWAY_CLEAN_SETTING, false) ); var migrationManager = new DatabaseMigrationManager(context.getConfig(), context.getMonitor(), flywayService); migrationManager.migrate(); @@ -177,7 +176,7 @@ private SqlLoggingHouseMessageStore initializeLoggingHouseMessageStore(TypeManag ); } - private void registerEventSubscriber(ServiceExtensionContext context, SqlLoggingHouseMessageStore loggingHouseMessageStore) { + private void registerEventSubscriber(ServiceExtensionContext context, LoggingHouseMessageStore loggingHouseMessageStore) { monitor.debug("Registering event subscriber for LoggingHouseClientExtension"); var eventSubscriber = new LoggingHouseEventSubscriber( @@ -221,8 +220,18 @@ private void registerCommonTypes(TypeManager typeManager) { monitor.debug("Registered serializers for LoggingHouseClientExtension"); } - private void registerClearingHouseMessageSenders(ServiceExtensionContext context) { - monitor.debug("Registering message senders for LoggingHouseClientExtension"); + private WorkersManager initializeWorkersManager(ServiceExtensionContext context, LoggingHouseMessageStore store) { + return new WorkersManager(this.monitor, + context.getSetting(LOGGINGHOUSE_EXTENSION_MAX_WORKERS, 1), + store, + dispatcherRegistry, + hostname, + loggingHouseLogUrl + ); + } + + private void registerDispatcher(ServiceExtensionContext context) { + monitor.debug("Registering IDS dispatch sender for LoggingHouseClientExtension"); var httpClient = context.getService(EdcHttpClient.class); var objectMapper = typeManager.getMapper(TYPE_MANAGER_SERIALIZER_KEY); diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java index d04fa72..6fbba22 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java @@ -16,7 +16,11 @@ import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessage; +import java.util.List; + public interface LoggingHouseMessageStore { void save(LoggingHouseMessage message); + + List listNotSent(); } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java index 7253f91..2272f1b 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java @@ -19,16 +19,36 @@ public class LoggingHouseMessage { private Long id; + private String eventType; + private String eventId; private Object eventToLog; + private String processId; + private String consumerId; + private String providerId; private ZonedDateTime createdAt; private ZonedDateTime sentAt; public Long getId() { return id; } + public String getEventType() { + return eventType; + } + public String getEventId() { + return eventId; + } public Object getEventToLog() { return eventToLog; } + public String getProcessId() { + return processId; + } + public String getConsumerId() { + return consumerId; + } + public String getProviderId() { + return providerId; + } public ZonedDateTime getCreatedAt() { return createdAt; } @@ -50,10 +70,30 @@ public LoggingHouseMessage.Builder id(Long id) { this.event.id = id; return this; } + public LoggingHouseMessage.Builder eventType(String eventType) { + this.event.eventType = eventType; + return this; + } + public LoggingHouseMessage.Builder eventId(String eventId) { + this.event.eventId = eventId; + return this; + } public LoggingHouseMessage.Builder eventToLog(Object eventToLog) { this.event.eventToLog = eventToLog; return this; } + public LoggingHouseMessage.Builder processId(String processId) { + this.event.processId = processId; + return this; + } + public LoggingHouseMessage.Builder consumerId(String consumerId) { + this.event.consumerId = consumerId; + return this; + } + public LoggingHouseMessage.Builder providerId(String providerId) { + this.event.providerId = providerId; + return this; + } public LoggingHouseMessage.Builder createdAt(ZonedDateTime createdAt) { this.event.createdAt = createdAt; return this; @@ -64,7 +104,12 @@ public LoggingHouseMessage.Builder sentAt(ZonedDateTime sentAt) { } public LoggingHouseMessage build() { + Objects.requireNonNull(this.event.eventType, "Message eventType must not be null"); + Objects.requireNonNull(this.event.eventId, "Message eventId must not be null"); Objects.requireNonNull(this.event.eventToLog, "Message eventToLog must not be null"); + Objects.requireNonNull(this.event.processId, "Message processId must not be null"); + Objects.requireNonNull(this.event.consumerId, "Message consumerId must not be null"); + Objects.requireNonNull(this.event.providerId, "Message providerId must not be null"); Objects.requireNonNull(this.event.createdAt, "Message createdAt must not be null"); return this.event; } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java index 42e0c0b..7ff745d 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java @@ -26,6 +26,8 @@ import org.eclipse.edc.transaction.spi.TransactionContext; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; public class SqlLoggingHouseMessageStore extends AbstractSqlStore implements LoggingHouseMessageStore { @@ -62,6 +64,11 @@ public void save(LoggingHouseMessage event) { }); } + @Override + public List listNotSent() { + return new ArrayList(); + } + private Long mapFromZonedDateTime(ZonedDateTime zonedDateTime) { return zonedDateTime != null ? zonedDateTime.toEpochSecond() : null; } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java new file mode 100644 index 0000000..d74bd82 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.worker; + +import com.truzzt.extension.logginghouse.client.messages.CreateProcessMessage; +import com.truzzt.extension.logginghouse.client.messages.LogMessage; +import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessage; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.message.RemoteMessageDispatcherRegistry; +import org.eclipse.edc.spi.monitor.Monitor; + +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +public class MessageWorker { + private final Monitor monitor; + private final RemoteMessageDispatcherRegistry dispatcherRegistry; + private final URI connectorBaseUrl; + private final URL loggingHouseUrl; + private final String workerId; + + public MessageWorker(Monitor monitor, RemoteMessageDispatcherRegistry dispatcherRegistry, URI connectorBaseUrl, URL loggingHouseUrl) { + this.monitor = monitor; + this.dispatcherRegistry = dispatcherRegistry; + this.connectorBaseUrl = connectorBaseUrl; + this.loggingHouseUrl = loggingHouseUrl; + + workerId = "Worker-" + UUID.randomUUID(); + } + + public String getId() { + return workerId; + } + + public CompletableFuture run(LoggingHouseMessage message) { + try { + monitor.debug("Worker " + workerId + " processing message with event of type " + message.getEventType() + " and id " + message.getEventId()); + process(message); + + return CompletableFuture.completedFuture(true); + + } catch (Exception e) { + monitor.severe(e.getMessage()); + return CompletableFuture.failedFuture(new EdcException(e)); + } + } + + public void process(LoggingHouseMessage message) { + try { + var pid = message.getProcessId(); + + // Create Process + var extendedProcessUrl = new URL(loggingHouseUrl + "/process/" + pid); + try { + createProcess(message, extendedProcessUrl); + } catch (Exception e) { + monitor.warning("Could not create process in LoggingHouse: " + e.getMessage()); + } + + // Log Contract Agreement + var extendedLogUrl = new URL(loggingHouseUrl + "/messages/log/" + pid); + try { + logMessage(message, extendedLogUrl); + } catch (Exception e) { + monitor.warning("Could not log message to LoggingHouse: " + e.getMessage()); + } + + } catch (MalformedURLException e) { + throw new EdcException("Could not create extended clearinghouse url."); + } + } + + public void createProcess(LoggingHouseMessage message, URL loggingHouseUrl) { + + List processOwners = new ArrayList<>(); + processOwners.add(message.getConsumerId()); + processOwners.add(message.getProviderId()); + + monitor.info("Creating process in LoggingHouse with id: " + message.getProcessId()); + var logMessage = new CreateProcessMessage(loggingHouseUrl, connectorBaseUrl, message.getProcessId(), processOwners); + + dispatcherRegistry.dispatch(Object.class, logMessage); + } + + public void logMessage(LoggingHouseMessage message, URL clearingHouseLogUrl) { + + monitor.info("Logging message to LoggingHouse with type " + message.getEventType() + " and id " + message.getEventId()); + var logMessage = new LogMessage(clearingHouseLogUrl, connectorBaseUrl, message.getEventToLog()); + + dispatcherRegistry.dispatch(Object.class, logMessage); + } + +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersManager.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersManager.java new file mode 100644 index 0000000..4b8b804 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersManager.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package com.truzzt.extension.logginghouse.client.worker; + +import com.truzzt.extension.logginghouse.client.spi.store.LoggingHouseMessageStore; +import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessage; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.message.RemoteMessageDispatcherRegistry; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.Hostname; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static java.lang.String.format; + +public class WorkersManager { + + private final Monitor monitor; + private final int maxWorkers; + private final LoggingHouseMessageStore store; + private final RemoteMessageDispatcherRegistry dispatcherRegistry; + private final URI connectorBaseUrl; + private final URL loggingHouseUrl; + + public WorkersManager(Monitor monitor, + int maxWorkers, + LoggingHouseMessageStore store, + RemoteMessageDispatcherRegistry dispatcherRegistry, + Hostname hostname, + URL loggingHouseUrl) { + this.monitor = monitor; + this.maxWorkers = maxWorkers; + this.store = store; + this.dispatcherRegistry = dispatcherRegistry; + this.loggingHouseUrl = loggingHouseUrl; + + try { + connectorBaseUrl = getConnectorBaseUrl(hostname); + } catch (URISyntaxException e) { + throw new EdcException("Could not create connectorBaseUrl. Hostname can be set using:" + hostname, e); + } + } + + public void run() { + List messages = store.listNotSent(); + if (messages.isEmpty()) { + monitor.warning("No Messages to send, aborting execution"); + return; + } + monitor.debug(log("Loaded " + messages.size() + " not sent messages from store")); + var allItems = new ArrayBlockingQueue<>(messages.size(), true, messages); + + monitor.debug(log("Instantiate workers...")); + + var actualNumWorkers = Math.min(allItems.size(), maxWorkers); + monitor.debug(format(log("Worker parallelism is %s, based on config and number of not sent messages"), actualNumWorkers)); + var availableWorkers = createWorkers(actualNumWorkers); + + while (!allItems.isEmpty()) { + var worker = nextAvailableWorker(availableWorkers); + if (worker == null) { + monitor.debug(log("No worker available, will retry later")); + continue; + } + + var item = allItems.poll(); + if (item == null) { + monitor.warning(log("WorkItem queue empty, abort execution")); + break; + } + + worker.run(item) + .whenComplete((updateResponse, throwable) -> { + if (throwable != null) { + monitor.severe(log(format("Unexpected exception happened during in worker %s", worker.getId())), throwable); + } else { + monitor.info(log(format("Worker [%s] is done", worker.getId()))); + } + availableWorkers.add(worker); + }); + } + } + + @Nullable + private MessageWorker nextAvailableWorker(ArrayBlockingQueue availableWorkers) { + MessageWorker worker = null; + try { + monitor.debug(log("Getting next available worker")); + worker = availableWorkers.poll(1, TimeUnit.SECONDS); + } catch (InterruptedException e) { + monitor.debug("interrupted while waiting for worker to become available"); + } + return worker; + } + + @NotNull + private ArrayBlockingQueue createWorkers(int numWorkers) { + + return new ArrayBlockingQueue<>(numWorkers, true, IntStream.range(0, numWorkers) + .mapToObj(i -> new MessageWorker(monitor, dispatcherRegistry, connectorBaseUrl, loggingHouseUrl)) + .collect(Collectors.toList())); + } + + private String log(String input) { + return "WorkersManager: " + input; + } + + private URI getConnectorBaseUrl(Hostname hostname) throws URISyntaxException { + return new URI(String.format("https://%s/", hostname.get())); + } +} From 817ef84dd2059e8c9dc4b936f3321978d9f51910 Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Tue, 2 Jul 2024 21:14:22 -0300 Subject: [PATCH 18/25] feat: logging house sender worker --- .../logginghouse/client/ConfigConstants.java | 4 + .../client/LoggingHouseClientExtension.java | 81 ++++++++------ .../LoggingHouseEventSubscriber.java | 57 ++++++---- .../messages/CreateProcessMessage.java | 2 +- .../messages/CreateProcessMessageSender.java | 16 +-- .../{ => events}/messages/LogMessage.java | 2 +- .../messages/LogMessageSender.java | 18 ++-- .../connection/DatasourceProperties.java | 36 ++----- ...tipartClearingRemoteMessageDispatcher.java | 4 +- .../{ => multipart}/ids/jsonld/JsonLd.java | 2 +- .../ids/jsonld/JsonLdModule.java | 2 +- .../ids/jsonld/UriDeserializer.java | 2 +- .../ids/jsonld/UriSerializer.java | 2 +- .../XmlGregorianCalendarDeserializer.java | 2 +- .../XmlGregorianCalendarSerializer.java | 2 +- .../ids/multipart/CalendarUtil.java | 2 +- .../ids/multipart/IdsConstants.java | 2 +- .../ids/multipart/IdsMultipartParts.java | 2 +- .../IdsMultipartRemoteMessageDispatcher.java | 2 +- .../ids/multipart/IdsMultipartSender.java | 4 +- .../ids/multipart/MultipartResponse.java | 2 +- .../multipart/MultipartSenderDelegate.java | 2 +- .../ids/multipart/ResponseUtil.java | 2 +- .../spi/store/LoggingHouseMessageStore.java | 4 +- .../client/spi/types/LoggingHouseMessage.java | 34 +++--- .../spi/types/LoggingHouseMessageStatus.java | 28 +++++ .../sql/SqlLoggingHouseMessageStore.java | 102 +++++++++++++++++- .../sql/schema/BaseSqlDialectStatements.java | 25 ++++- .../schema/LoggingHouseEventStatements.java | 40 +++++++ ...r.java => LoggingHouseWorkersManager.java} | 32 ++++-- .../client/worker/MessageWorker.java | 48 +++++---- .../client/worker/WorkersExecutor.java | 48 +++++++++ .../migration/V0_0_1__Create_Tables.sql | 13 ++- .../LoggingHouseClientExtensionTest.java | 2 +- 34 files changed, 459 insertions(+), 167 deletions(-) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => events}/LoggingHouseEventSubscriber.java (84%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => events}/messages/CreateProcessMessage.java (93%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => events}/messages/CreateProcessMessageSender.java (74%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => events}/messages/LogMessage.java (93%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => events}/messages/LogMessageSender.java (87%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/jsonld/JsonLd.java (95%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/jsonld/JsonLdModule.java (93%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/jsonld/UriDeserializer.java (95%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/jsonld/UriSerializer.java (94%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/jsonld/XmlGregorianCalendarDeserializer.java (95%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/jsonld/XmlGregorianCalendarSerializer.java (95%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/multipart/CalendarUtil.java (93%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/multipart/IdsConstants.java (89%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/multipart/IdsMultipartParts.java (95%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/multipart/IdsMultipartRemoteMessageDispatcher.java (97%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/multipart/IdsMultipartSender.java (97%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/multipart/MultipartResponse.java (91%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/multipart/MultipartSenderDelegate.java (93%) rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/{ => multipart}/ids/multipart/ResponseUtil.java (93%) create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessageStatus.java rename logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/{WorkersManager.java => LoggingHouseWorkersManager.java} (83%) create mode 100644 logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersExecutor.java diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java index dfd0ebc..56c5554 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java @@ -11,4 +11,8 @@ public class ConfigConstants { static final String LOGGINGHOUSE_FLYWAY_CLEAN_SETTING = "edc.logginghouse.client.flyway.clean"; static final String LOGGINGHOUSE_EXTENSION_MAX_WORKERS = "edc.logginghouse.client.workers.max"; + + static final String LOGGINGHOUSE_EXTENSION_WORKERS_DELAY = "edc.logginghouse.client.workers.delay"; + + static final String LOGGINGHOUSE_EXTENSION_WORKERS_PERIOD = "edc.logginghouse.client.workers.period"; } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 07260de..089c839 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -14,24 +14,24 @@ package com.truzzt.extension.logginghouse.client; +import com.truzzt.extension.logginghouse.client.events.LoggingHouseEventSubscriber; import com.truzzt.extension.logginghouse.client.flyway.FlywayService; +import com.truzzt.extension.logginghouse.client.flyway.connection.DatasourceProperties; import com.truzzt.extension.logginghouse.client.flyway.migration.DatabaseMigrationManager; -import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; -import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartSender; -import com.truzzt.extension.logginghouse.client.messages.CreateProcessMessageSender; -import com.truzzt.extension.logginghouse.client.messages.LogMessageSender; +import com.truzzt.extension.logginghouse.client.multipart.ids.jsonld.JsonLd; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.IdsMultipartSender; +import com.truzzt.extension.logginghouse.client.events.messages.CreateProcessMessageSender; +import com.truzzt.extension.logginghouse.client.events.messages.LogMessageSender; import com.truzzt.extension.logginghouse.client.multipart.IdsMultipartClearingRemoteMessageDispatcher; import com.truzzt.extension.logginghouse.client.multipart.MultiContextJsonLdSerializer; import com.truzzt.extension.logginghouse.client.spi.store.LoggingHouseMessageStore; import com.truzzt.extension.logginghouse.client.store.sql.SqlLoggingHouseMessageStore; import com.truzzt.extension.logginghouse.client.store.sql.schema.postgres.PostgresDialectStatements; -import com.truzzt.extension.logginghouse.client.worker.WorkersManager; +import com.truzzt.extension.logginghouse.client.worker.LoggingHouseWorkersManager; +import com.truzzt.extension.logginghouse.client.worker.WorkersExecutor; import de.fraunhofer.iais.eis.LogMessage; import de.fraunhofer.iais.eis.RequestMessage; -import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAccepted; -import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationAgreed; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; -import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationTerminated; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessFailed; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessInitiated; @@ -41,7 +41,9 @@ import org.eclipse.edc.connector.transfer.spi.event.TransferProcessStarted; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessCompleted; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Requires; import org.eclipse.edc.spi.EdcException; import org.eclipse.edc.spi.asset.AssetIndex; import org.eclipse.edc.spi.event.EventRouter; @@ -59,18 +61,39 @@ import java.net.MalformedURLException; import java.net.URL; +import java.time.Duration; import java.util.Map; -import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_ENABLED; -import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_SERVER_URL_SETTING; -import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_FLYWAY_REPAIR_SETTING; -import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_FLYWAY_CLEAN_SETTING; -import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_EXTENSION_MAX_WORKERS; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.*; +@Extension(value = LoggingHouseClientExtension.NAME) +@Requires(value = { + Hostname.class, + + TypeManager.class, + EventRouter.class, + IdentityService.class, + RemoteMessageDispatcherRegistry.class, + + DataSourceRegistry.class, + TransactionContext.class, + QueryExecutor.class, + + ContractNegotiationStore.class, + TransferProcessStore.class, + AssetIndex.class +}) public class LoggingHouseClientExtension implements ServiceExtension { - public static final String LOGGINGHOUSE_CLIENT_EXTENSION = "LoggingHouseClientExtension"; + public static final String NAME = "LoggingHouseClientExtension"; private static final String TYPE_MANAGER_SERIALIZER_KEY = "ids-clearinghouse"; + private static final Map CONTEXT_MAP = Map.of( + "cat", "http://w3id.org/mds/data-categories#", + "ids", "https://w3id.org/idsa/core/", + "idsc", "https://w3id.org/idsa/code/"); + + @Inject + private Hostname hostname; @Inject private TypeManager typeManager; @@ -81,9 +104,6 @@ public class LoggingHouseClientExtension implements ServiceExtension { @Inject private RemoteMessageDispatcherRegistry dispatcherRegistry; - @Inject - private Hostname hostname; - @Inject private DataSourceRegistry dataSourceRegistry; @Inject @@ -98,20 +118,14 @@ public class LoggingHouseClientExtension implements ServiceExtension { @Inject private AssetIndex assetIndex; - private static final Map CONTEXT_MAP = Map.of( - "cat", "http://w3id.org/mds/data-categories#", - "ids", "https://w3id.org/idsa/core/", - "idsc", "https://w3id.org/idsa/code/"); - public Monitor monitor; private boolean enabled; private URL loggingHouseLogUrl; - - private WorkersManager workersManager; + private LoggingHouseWorkersManager workersManager; @Override public String name() { - return LOGGINGHOUSE_CLIENT_EXTENSION; + return NAME; } @Override @@ -168,7 +182,7 @@ private void runFlywayMigrations(ServiceExtensionContext context) { private SqlLoggingHouseMessageStore initializeLoggingHouseMessageStore(TypeManager typeManager) { return new SqlLoggingHouseMessageStore( dataSourceRegistry, - DataSourceRegistry.DEFAULT_DATASOURCE, + DatasourceProperties.LOGGING_HOUSE_DATASOURCE, transactionContext, typeManager.getMapper(), new PostgresDialectStatements(), @@ -186,9 +200,7 @@ private void registerEventSubscriber(ServiceExtensionContext context, LoggingHou monitor); eventRouter.registerSync(ContractNegotiationFinalized.class, eventSubscriber); - eventRouter.registerSync(ContractNegotiationAgreed.class, eventSubscriber); - eventRouter.registerSync(ContractNegotiationAccepted.class, eventSubscriber); - eventRouter.registerSync(ContractNegotiationTerminated.class, eventSubscriber); // TODO: check pid + eventRouter.registerSync(TransferProcessRequested.class, eventSubscriber); eventRouter.registerSync(TransferProcessInitiated.class, eventSubscriber); eventRouter.registerSync(TransferProcessStarted.class, eventSubscriber); @@ -220,8 +232,13 @@ private void registerCommonTypes(TypeManager typeManager) { monitor.debug("Registered serializers for LoggingHouseClientExtension"); } - private WorkersManager initializeWorkersManager(ServiceExtensionContext context, LoggingHouseMessageStore store) { - return new WorkersManager(this.monitor, + private LoggingHouseWorkersManager initializeWorkersManager(ServiceExtensionContext context, LoggingHouseMessageStore store) { + var periodSeconds = context.getSetting(LOGGINGHOUSE_EXTENSION_WORKERS_DELAY, 30); + var initialDelaySeconds = context.getSetting(LOGGINGHOUSE_EXTENSION_WORKERS_PERIOD, 10); + var executor = new WorkersExecutor(Duration.ofSeconds(periodSeconds), Duration.ofSeconds(initialDelaySeconds), monitor); + + return new LoggingHouseWorkersManager(executor, + monitor, context.getSetting(LOGGINGHOUSE_EXTENSION_MAX_WORKERS, 1), store, dispatcherRegistry, @@ -254,6 +271,8 @@ public void start() { } else { monitor.info("Starting Logginghouse client extension."); } + + workersManager.execute(); } @Override diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseEventSubscriber.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/LoggingHouseEventSubscriber.java similarity index 84% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseEventSubscriber.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/LoggingHouseEventSubscriber.java index 76d98ad..25fe50e 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseEventSubscriber.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/LoggingHouseEventSubscriber.java @@ -12,10 +12,11 @@ * */ -package com.truzzt.extension.logginghouse.client; +package com.truzzt.extension.logginghouse.client.events; import com.truzzt.extension.logginghouse.client.spi.store.LoggingHouseMessageStore; import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessage; +import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessageStatus; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement; @@ -48,26 +49,6 @@ public LoggingHouseEventSubscriber( this.monitor = monitor; } - public void storeContractAgreement(ContractAgreement contractAgreement) { - monitor.info("Storing ContractAgreement to send to LoggingHouse"); - - var message = LoggingHouseMessage.Builder.newInstance() - .eventToLog(contractAgreement) - .createdAt(ZonedDateTime.now()) - .build(); - loggingHouseMessageStore.save(message); - } - - public void storeTransferProcess(TransferProcess transferProcess) { - monitor.info("Storing TransferProcess to send to LoggingHouse"); - - var message = LoggingHouseMessage.Builder.newInstance() - .eventToLog(transferProcess) - .createdAt(ZonedDateTime.now()) - .build(); - loggingHouseMessageStore.save(message); - } - @Override public void on(EventEnvelope event) { if (event.getPayload() instanceof ContractNegotiationFinalized contractNegotiationFinalized) { @@ -91,7 +72,7 @@ public void on(EventEnvelope event) { } } - private ContractAgreement resolveContractAgreement(ContractNegotiationFinalized contractNegotiationFinalized) throws NullPointerException { + private ContractAgreement resolveContractAgreement(ContractNegotiationFinalized contractNegotiationFinalized) { var contractNegotiationId = contractNegotiationFinalized.getContractNegotiationId(); var contractNegotiation = contractNegotiationStore.findById(contractNegotiationId); return Objects.requireNonNull(contractNegotiation).getContractAgreement(); @@ -101,4 +82,36 @@ private TransferProcess resolveTransferProcess(TransferProcessEvent transferProc var transferProcessId = transferProcessEvent.getTransferProcessId(); return transferProcessStore.findById(transferProcessId); } + + public void storeContractAgreement(ContractAgreement contractAgreement) { + monitor.info("Storing ContractAgreement to send to LoggingHouse"); + + var message = LoggingHouseMessage.Builder.newInstance() + .eventType(contractAgreement.getClass()) + .eventId(contractAgreement.getId()) + .eventToLog(contractAgreement) + .createProcess(true) + .processId(contractAgreement.getId()) + .consumerId(contractAgreement.getConsumerId()) + .providerId(contractAgreement.getProviderId()) + .status(LoggingHouseMessageStatus.PENDING) + .createdAt(ZonedDateTime.now()) + .build(); + loggingHouseMessageStore.save(message); + } + + public void storeTransferProcess(TransferProcess transferProcess) { + monitor.info("Storing TransferProcess to send to LoggingHouse"); + + var message = LoggingHouseMessage.Builder.newInstance() + .eventType(transferProcess.getClass()) + .eventId(transferProcess.getId()) + .eventToLog(transferProcess) + .createProcess(false) + .processId(transferProcess.getContractId()) + .status(LoggingHouseMessageStatus.PENDING) + .createdAt(ZonedDateTime.now()) + .build(); + loggingHouseMessageStore.save(message); + } } \ No newline at end of file diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessage.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/CreateProcessMessage.java similarity index 93% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessage.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/CreateProcessMessage.java index bc1b298..0bbf02c 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessage.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/CreateProcessMessage.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.messages; +package com.truzzt.extension.logginghouse.client.events.messages; import com.truzzt.extension.logginghouse.client.multipart.ExtendedMessageProtocolClearing; import org.eclipse.edc.spi.types.domain.message.RemoteMessage; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessageSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/CreateProcessMessageSender.java similarity index 74% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessageSender.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/CreateProcessMessageSender.java index 35ab185..c7040ad 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/CreateProcessMessageSender.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/CreateProcessMessageSender.java @@ -12,15 +12,15 @@ * */ -package com.truzzt.extension.logginghouse.client.messages; +package com.truzzt.extension.logginghouse.client.events.messages; -import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; -import com.truzzt.extension.logginghouse.client.ids.multipart.CalendarUtil; -import com.truzzt.extension.logginghouse.client.ids.multipart.IdsConstants; -import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartParts; -import com.truzzt.extension.logginghouse.client.ids.multipart.MultipartResponse; -import com.truzzt.extension.logginghouse.client.ids.multipart.MultipartSenderDelegate; -import com.truzzt.extension.logginghouse.client.ids.multipart.ResponseUtil; +import com.truzzt.extension.logginghouse.client.multipart.ids.jsonld.JsonLd; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.CalendarUtil; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.IdsConstants; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.IdsMultipartParts; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.MultipartResponse; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.MultipartSenderDelegate; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.ResponseUtil; import de.fraunhofer.iais.eis.DynamicAttributeToken; import de.fraunhofer.iais.eis.Message; import de.fraunhofer.iais.eis.MessageProcessedNotificationMessageImpl; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessage.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/LogMessage.java similarity index 93% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessage.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/LogMessage.java index 7633f2a..823597a 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessage.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/LogMessage.java @@ -13,7 +13,7 @@ * */ -package com.truzzt.extension.logginghouse.client.messages; +package com.truzzt.extension.logginghouse.client.events.messages; import com.truzzt.extension.logginghouse.client.multipart.ExtendedMessageProtocolClearing; import org.eclipse.edc.spi.types.domain.message.RemoteMessage; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessageSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/LogMessageSender.java similarity index 87% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessageSender.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/LogMessageSender.java index 9a014bf..8371ab1 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/messages/LogMessageSender.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/events/messages/LogMessageSender.java @@ -12,15 +12,15 @@ * */ -package com.truzzt.extension.logginghouse.client.messages; - -import com.truzzt.extension.logginghouse.client.ids.jsonld.JsonLd; -import com.truzzt.extension.logginghouse.client.ids.multipart.CalendarUtil; -import com.truzzt.extension.logginghouse.client.ids.multipart.IdsConstants; -import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartParts; -import com.truzzt.extension.logginghouse.client.ids.multipart.MultipartResponse; -import com.truzzt.extension.logginghouse.client.ids.multipart.MultipartSenderDelegate; -import com.truzzt.extension.logginghouse.client.ids.multipart.ResponseUtil; +package com.truzzt.extension.logginghouse.client.events.messages; + +import com.truzzt.extension.logginghouse.client.multipart.ids.jsonld.JsonLd; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.CalendarUtil; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.IdsConstants; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.IdsMultipartParts; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.MultipartResponse; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.MultipartSenderDelegate; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.ResponseUtil; import de.fraunhofer.iais.eis.DynamicAttributeToken; import de.fraunhofer.iais.eis.LogMessageBuilder; import de.fraunhofer.iais.eis.Message; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java index c990604..543eac3 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java @@ -14,18 +14,15 @@ package com.truzzt.extension.logginghouse.client.flyway.connection; -import org.eclipse.edc.spi.EdcException; import org.eclipse.edc.spi.system.configuration.Config; -import static org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry.DEFAULT_DATASOURCE; - public class DatasourceProperties { - private static final String LOGGING_HOUSE_DATASOURCE = "logginghouse"; + public static final String LOGGING_HOUSE_DATASOURCE = "logginghouse"; - private static final String DATASOURCE_SETTING_NAME = "edc.datasource.%s.name"; - private static final String DATASOURCE_SETTING_JDBC_URL = "edc.datasource.%s.url"; - private static final String DATASOURCE_SETTING_USER = "edc.datasource.%s.user"; - private static final String DATASOURCE_SETTING_PASSWORD = "edc.datasource.%s.password"; + private static final String DATASOURCE_SETTING_NAME = "edc.datasource.logginghouse.name"; + private static final String DATASOURCE_SETTING_JDBC_URL = "edc.datasource.logginghouse.url"; + private static final String DATASOURCE_SETTING_USER = "edc.datasource.logginghouse.user"; + private static final String DATASOURCE_SETTING_PASSWORD = "edc.datasource.logginghouse.password"; private final String name; private final String jdbcUrl; @@ -33,25 +30,10 @@ public class DatasourceProperties { private final String password; public DatasourceProperties(Config config) { - name = getSetting(config, String.format(DATASOURCE_SETTING_NAME, LOGGING_HOUSE_DATASOURCE), - String.format(DATASOURCE_SETTING_NAME, DEFAULT_DATASOURCE)); - - jdbcUrl = getSetting(config, String.format(DATASOURCE_SETTING_JDBC_URL, LOGGING_HOUSE_DATASOURCE), - String.format(DATASOURCE_SETTING_JDBC_URL, DEFAULT_DATASOURCE)); - - user = getSetting(config, String.format(DATASOURCE_SETTING_USER, LOGGING_HOUSE_DATASOURCE), - String.format(DATASOURCE_SETTING_USER, DEFAULT_DATASOURCE)); - - password = getSetting(config, String.format(DATASOURCE_SETTING_PASSWORD, LOGGING_HOUSE_DATASOURCE), - String.format(DATASOURCE_SETTING_PASSWORD, DEFAULT_DATASOURCE)); - } - - private String getSetting(Config config, String setting, String defaultSetting) { - try { - return config.getString(setting); - } catch (EdcException e) { - return config.getString(defaultSetting); - } + name = config.getString(DATASOURCE_SETTING_NAME); + jdbcUrl = config.getString(DATASOURCE_SETTING_JDBC_URL); + user = config.getString(DATASOURCE_SETTING_USER); + password = config.getString(DATASOURCE_SETTING_PASSWORD); } public String getName() { diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/IdsMultipartClearingRemoteMessageDispatcher.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/IdsMultipartClearingRemoteMessageDispatcher.java index e08e965..3e124ad 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/IdsMultipartClearingRemoteMessageDispatcher.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/IdsMultipartClearingRemoteMessageDispatcher.java @@ -14,8 +14,8 @@ package com.truzzt.extension.logginghouse.client.multipart; -import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartRemoteMessageDispatcher; -import com.truzzt.extension.logginghouse.client.ids.multipart.IdsMultipartSender; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.IdsMultipartRemoteMessageDispatcher; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.IdsMultipartSender; public class IdsMultipartClearingRemoteMessageDispatcher extends IdsMultipartRemoteMessageDispatcher { diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLd.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/JsonLd.java similarity index 95% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLd.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/JsonLd.java index c542ae0..5693a3e 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLd.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/JsonLd.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.jsonld; +package com.truzzt.extension.logginghouse.client.multipart.ids.jsonld; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLdModule.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/JsonLdModule.java similarity index 93% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLdModule.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/JsonLdModule.java index a5c4bfe..fa73508 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/JsonLdModule.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/JsonLdModule.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.jsonld; +package com.truzzt.extension.logginghouse.client.multipart.ids.jsonld; import com.fasterxml.jackson.databind.module.SimpleModule; import java.net.URI; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriDeserializer.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/UriDeserializer.java similarity index 95% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriDeserializer.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/UriDeserializer.java index cf684c0..e644d4f 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriDeserializer.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/UriDeserializer.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.jsonld; +package com.truzzt.extension.logginghouse.client.multipart.ids.jsonld; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriSerializer.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/UriSerializer.java similarity index 94% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriSerializer.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/UriSerializer.java index a2e94dc..3858ccf 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/UriSerializer.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/UriSerializer.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.jsonld; +package com.truzzt.extension.logginghouse.client.multipart.ids.jsonld; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarDeserializer.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/XmlGregorianCalendarDeserializer.java similarity index 95% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarDeserializer.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/XmlGregorianCalendarDeserializer.java index 116c11c..4279934 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarDeserializer.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/XmlGregorianCalendarDeserializer.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.jsonld; +package com.truzzt.extension.logginghouse.client.multipart.ids.jsonld; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarSerializer.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/XmlGregorianCalendarSerializer.java similarity index 95% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarSerializer.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/XmlGregorianCalendarSerializer.java index d42744f..9ad1cf6 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/jsonld/XmlGregorianCalendarSerializer.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/jsonld/XmlGregorianCalendarSerializer.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.jsonld; +package com.truzzt.extension.logginghouse.client.multipart.ids.jsonld; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/CalendarUtil.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/CalendarUtil.java similarity index 93% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/CalendarUtil.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/CalendarUtil.java index 7a7b8bf..b338de9 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/CalendarUtil.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/CalendarUtil.java @@ -13,7 +13,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.multipart; +package com.truzzt.extension.logginghouse.client.multipart.ids.multipart; import java.time.ZonedDateTime; import java.util.GregorianCalendar; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsConstants.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsConstants.java similarity index 89% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsConstants.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsConstants.java index 7a927e8..3950c21 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsConstants.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsConstants.java @@ -13,7 +13,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.multipart; +package com.truzzt.extension.logginghouse.client.multipart.ids.multipart; public final class IdsConstants { diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartParts.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsMultipartParts.java similarity index 95% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartParts.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsMultipartParts.java index 72bcfdf..5be1da3 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartParts.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsMultipartParts.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.multipart; +package com.truzzt.extension.logginghouse.client.multipart.ids.multipart; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartRemoteMessageDispatcher.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsMultipartRemoteMessageDispatcher.java similarity index 97% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartRemoteMessageDispatcher.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsMultipartRemoteMessageDispatcher.java index 7342e56..ea74ef6 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartRemoteMessageDispatcher.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsMultipartRemoteMessageDispatcher.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.multipart; +package com.truzzt.extension.logginghouse.client.multipart.ids.multipart; import org.eclipse.edc.connector.transfer.spi.types.protocol.TransferCompletionMessage; import org.eclipse.edc.connector.transfer.spi.types.protocol.TransferStartMessage; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartSender.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsMultipartSender.java similarity index 97% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartSender.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsMultipartSender.java index 3677494..a3ec49f 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/IdsMultipartSender.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/IdsMultipartSender.java @@ -14,7 +14,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.multipart; +package com.truzzt.extension.logginghouse.client.multipart.ids.multipart; import com.fasterxml.jackson.databind.ObjectMapper; import de.fraunhofer.iais.eis.DynamicAttributeToken; @@ -219,7 +219,7 @@ protected IdsMultipartParts extractResponseParts(ResponseBody body) throws Excep public void checkResponseType(MultipartResponse response, MultipartSenderDelegate senderDelegate) { var type = senderDelegate.getAllowedResponseTypes(); if (!type.contains(response.header().getClass())) { - throw new EdcException(String.format("Received %s but expected %s.", response.header().getClass(), type)); + throw new EdcException(format("Received %s but expected %s.", response.header().getClass(), type)); } } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartResponse.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/MultipartResponse.java similarity index 91% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartResponse.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/MultipartResponse.java index 83b2ce5..818c599 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartResponse.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/MultipartResponse.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.multipart; +package com.truzzt.extension.logginghouse.client.multipart.ids.multipart; import de.fraunhofer.iais.eis.Message; import org.jetbrains.annotations.NotNull; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartSenderDelegate.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/MultipartSenderDelegate.java similarity index 93% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartSenderDelegate.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/MultipartSenderDelegate.java index b5e8bdb..adfe1c3 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/MultipartSenderDelegate.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/MultipartSenderDelegate.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.multipart; +package com.truzzt.extension.logginghouse.client.multipart.ids.multipart; import de.fraunhofer.iais.eis.DynamicAttributeToken; import de.fraunhofer.iais.eis.Message; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/ResponseUtil.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/ResponseUtil.java similarity index 93% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/ResponseUtil.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/ResponseUtil.java index 493cc6e..63090ab 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ids/multipart/ResponseUtil.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/multipart/ids/multipart/ResponseUtil.java @@ -12,7 +12,7 @@ * */ -package com.truzzt.extension.logginghouse.client.ids.multipart; +package com.truzzt.extension.logginghouse.client.multipart.ids.multipart; import com.fasterxml.jackson.databind.ObjectMapper; import de.fraunhofer.iais.eis.Message; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java index 6fbba22..9351161 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/store/LoggingHouseMessageStore.java @@ -22,5 +22,7 @@ public interface LoggingHouseMessageStore { void save(LoggingHouseMessage message); - List listNotSent(); + List listPending(); + + void updateSent(long id); } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java index 2272f1b..e416e48 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessage.java @@ -19,19 +19,20 @@ public class LoggingHouseMessage { private Long id; - private String eventType; + private Class eventType; private String eventId; private Object eventToLog; + private boolean createProcess; private String processId; private String consumerId; private String providerId; + private LoggingHouseMessageStatus status; private ZonedDateTime createdAt; - private ZonedDateTime sentAt; public Long getId() { return id; } - public String getEventType() { + public Class getEventType() { return eventType; } public String getEventId() { @@ -40,6 +41,9 @@ public String getEventId() { public Object getEventToLog() { return eventToLog; } + public boolean getCreateProcess() { + return createProcess; + } public String getProcessId() { return processId; } @@ -49,12 +53,12 @@ public String getConsumerId() { public String getProviderId() { return providerId; } + public LoggingHouseMessageStatus getStatus() { + return status; + } public ZonedDateTime getCreatedAt() { return createdAt; } - public ZonedDateTime getSentAt() { - return sentAt; - } public static final class Builder { private final LoggingHouseMessage event = new LoggingHouseMessage(); @@ -70,7 +74,7 @@ public LoggingHouseMessage.Builder id(Long id) { this.event.id = id; return this; } - public LoggingHouseMessage.Builder eventType(String eventType) { + public LoggingHouseMessage.Builder eventType(Class eventType) { this.event.eventType = eventType; return this; } @@ -82,6 +86,10 @@ public LoggingHouseMessage.Builder eventToLog(Object eventToLog) { this.event.eventToLog = eventToLog; return this; } + public LoggingHouseMessage.Builder createProcess(boolean createProcess) { + this.event.createProcess = createProcess; + return this; + } public LoggingHouseMessage.Builder processId(String processId) { this.event.processId = processId; return this; @@ -94,12 +102,12 @@ public LoggingHouseMessage.Builder providerId(String providerId) { this.event.providerId = providerId; return this; } - public LoggingHouseMessage.Builder createdAt(ZonedDateTime createdAt) { - this.event.createdAt = createdAt; + public LoggingHouseMessage.Builder status(LoggingHouseMessageStatus status) { + this.event.status = status; return this; } - public LoggingHouseMessage.Builder sentAt(ZonedDateTime sentAt) { - this.event.sentAt = sentAt; + public LoggingHouseMessage.Builder createdAt(ZonedDateTime createdAt) { + this.event.createdAt = createdAt; return this; } @@ -107,9 +115,9 @@ public LoggingHouseMessage build() { Objects.requireNonNull(this.event.eventType, "Message eventType must not be null"); Objects.requireNonNull(this.event.eventId, "Message eventId must not be null"); Objects.requireNonNull(this.event.eventToLog, "Message eventToLog must not be null"); + Objects.requireNonNull(this.event.createProcess, "Message createProcess must not be null"); Objects.requireNonNull(this.event.processId, "Message processId must not be null"); - Objects.requireNonNull(this.event.consumerId, "Message consumerId must not be null"); - Objects.requireNonNull(this.event.providerId, "Message providerId must not be null"); + Objects.requireNonNull(this.event.status, "Message status must not be null"); Objects.requireNonNull(this.event.createdAt, "Message createdAt must not be null"); return this.event; } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessageStatus.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessageStatus.java new file mode 100644 index 0000000..5b3fdee --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/spi/types/LoggingHouseMessageStatus.java @@ -0,0 +1,28 @@ +package com.truzzt.extension.logginghouse.client.spi.types; + +import org.eclipse.edc.spi.EdcException; + +public enum LoggingHouseMessageStatus { + PENDING("P"), SENT("S"); + + private final String code; + + LoggingHouseMessageStatus(String code) { + this.code = code; + } + + public String getCode() { + return code; + } + + public static LoggingHouseMessageStatus codeOf(String code) { + switch(code) { + case "P": + return PENDING; + case "S": + return SENT; + default: + throw new EdcException("Invalid status code " + code); + } + } +} diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java index 7ff745d..be0bfcb 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java @@ -18,17 +18,25 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.truzzt.extension.logginghouse.client.spi.store.LoggingHouseMessageStore; import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessage; +import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessageStatus; import com.truzzt.extension.logginghouse.client.store.sql.schema.LoggingHouseEventStatements; +import org.eclipse.edc.connector.contract.spi.types.agreement.ContractAgreement; +import org.eclipse.edc.connector.transfer.spi.types.TransferProcess; +import org.eclipse.edc.spi.EdcException; import org.eclipse.edc.spi.persistence.EdcPersistenceException; import org.eclipse.edc.sql.QueryExecutor; import org.eclipse.edc.sql.store.AbstractSqlStore; import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; import org.eclipse.edc.transaction.spi.TransactionContext; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.time.ZoneId; import java.time.ZonedDateTime; -import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; public class SqlLoggingHouseMessageStore extends AbstractSqlStore implements LoggingHouseMessageStore { @@ -48,29 +56,115 @@ public SqlLoggingHouseMessageStore(DataSourceRegistry dataSourceRegistry, public void save(LoggingHouseMessage event) { Objects.requireNonNull(event); + Objects.requireNonNull(event.getEventType()); + Objects.requireNonNull(event.getEventId()); Objects.requireNonNull(event.getEventToLog()); + Objects.requireNonNull(event.getCreateProcess()); + Objects.requireNonNull(event.getProcessId()); + Objects.requireNonNull(event.getStatus()); Objects.requireNonNull(event.getCreatedAt()); transactionContext.execute(() -> { try (var connection = getConnection()) { queryExecutor.execute(connection, statements.getInsertTemplate(), + event.getEventType().getSimpleName(), + event.getEventId(), toJson(event.getEventToLog()), + event.getCreateProcess(), + event.getProcessId(), + event.getConsumerId(), + event.getProviderId(), + event.getStatus().getCode(), mapFromZonedDateTime(event.getCreatedAt()) ); } catch (Exception e) { - throw new EdcPersistenceException(e.getMessage(), e); + throw new EdcPersistenceException("Error executing INSERT statement", e); } }); } @Override - public List listNotSent() { - return new ArrayList(); + public List listPending() { + return transactionContext.execute(() -> { + try { + return queryExecutor.query(getConnection(), true, this::mapResultSet, + statements.getSelectPendingStatement(), + LoggingHouseMessageStatus.PENDING.getCode()) + .collect(Collectors.toList()); + } catch (SQLException e) { + throw new EdcPersistenceException("Error executing SELECT statement", e); + } + }); + } + + @Override + public void updateSent(long id) { + transactionContext.execute(() -> { + try { + queryExecutor.execute(getConnection(), + statements.getUpdateSentTemplate(), + LoggingHouseMessageStatus.SENT.getCode(), + mapFromZonedDateTime(ZonedDateTime.now()), + id); + } catch (SQLException e) { + throw new EdcPersistenceException("Error executing UPDATE statement", e); + } + }); } private Long mapFromZonedDateTime(ZonedDateTime zonedDateTime) { return zonedDateTime != null ? zonedDateTime.toEpochSecond() : null; } + private ZonedDateTime mapToZonedDateTime(ResultSet resultSet, String column) throws Exception { + return resultSet.getString(column) != null ? + Instant.ofEpochSecond(resultSet.getLong(column)).atZone(ZoneId.of("Z")) : + null; + } + + private LoggingHouseMessage mapResultSet(ResultSet resultSet) throws Exception { + + Class eventType = toClass(resultSet.getString(statements.getEventTypeColumn())); + + Object eventToLog; + try { + eventToLog = fromJson(resultSet.getString(statements.getEventToLogColumn()), eventType); + } catch (EdcPersistenceException e) { + throw new EdcPersistenceException("Error eventToLog JSON column", e); + } + + LoggingHouseMessageStatus status; + try { + status = LoggingHouseMessageStatus.codeOf(resultSet.getString(statements.getStatusColumn())); + } catch (EdcPersistenceException e) { + throw new EdcPersistenceException("Error eventToLog JSON column", e); + } + + return LoggingHouseMessage.Builder.newInstance() + .id(resultSet.getLong(statements.getIdColumn())) + .eventType(eventType) + .eventId(resultSet.getString(statements.getIdColumn())) + .eventToLog(eventToLog) + .createProcess(resultSet.getBoolean(statements.getCreateProcessColumn())) + .processId(resultSet.getString(statements.getIdColumn())) + .consumerId(resultSet.getString(statements.getIdColumn())) + .providerId(resultSet.getString(statements.getIdColumn())) + .status(status) + .createdAt(mapToZonedDateTime(resultSet, statements.getCreatedAtColumn())) + .build(); + } + + private Class toClass(String eventType) { + + switch (eventType) { + case "ContractAgreement": + return ContractAgreement.class; + case "TransferProcess": + return TransferProcess.class; + default: + throw new EdcException("Invalid eventType: " + eventType); + } + } + } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/BaseSqlDialectStatements.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/BaseSqlDialectStatements.java index 6017294..56a3010 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/BaseSqlDialectStatements.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/BaseSqlDialectStatements.java @@ -21,11 +21,34 @@ public class BaseSqlDialectStatements implements LoggingHouseEventStatements { @Override public String getInsertTemplate() { - return format("INSERT INTO %s (%s, %s) VALUES (?%s, ?)", + return format("INSERT INTO %s (%s, %s, %s, %s, %s, %s, %s, %s, %s) VALUES (?, ?, ?%s, ?, ?, ?, ?, ?, ?)", getLoggingHouseMessageTable(), + getEventTypeColumn(), + getEventIdColumn(), getEventToLogColumn(), + getCreateProcessColumn(), + getProcessIdColumn(), + getConsumerIdColumn(), + getProviderIdColumn(), + getStatusColumn(), getCreatedAtColumn(), getFormatAsJsonOperator() ); } + + @Override + public String getSelectPendingStatement() { + return format("SELECT * FROM %s WHERE %s = ?", + getLoggingHouseMessageTable(), + getStatusColumn()); + } + + @Override + public String getUpdateSentTemplate() { + return format("UPDATE %s SET %s = ?, %s = ? WHERE %s = ?", + getLoggingHouseMessageTable(), + getStatusColumn(), + getSentAtColumn(), + getIdColumn()); + } } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/LoggingHouseEventStatements.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/LoggingHouseEventStatements.java index 5854cdb..5dd8eea 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/LoggingHouseEventStatements.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/schema/LoggingHouseEventStatements.java @@ -22,17 +22,57 @@ public interface LoggingHouseEventStatements { default String getLoggingHouseMessageTable() { return "edc_logging_house_message"; } + + default String getIdColumn() { + return "logging_house_message_id"; + } + + default String getEventTypeColumn() { + return "event_type"; + } + + default String getEventIdColumn() { + return "event_id"; + } default String getEventToLogColumn() { return "event_to_log"; } + default String getCreateProcessColumn() { + return "create_process"; + } + + default String getProcessIdColumn() { + return "process_id"; + } + + default String getConsumerIdColumn() { + return "consumer_id"; + } + + default String getProviderIdColumn() { + return "provider_id"; + } + + default String getStatusColumn() { + return "status"; + } + default String getCreatedAtColumn() { return "created_at"; } + default String getSentAtColumn() { + return "sent_at"; + } + String getInsertTemplate(); + String getSelectPendingStatement(); + + String getUpdateSentTemplate(); + default String getFormatAsJsonOperator() { return PostgresDialect.getJsonCastOperator(); } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersManager.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/LoggingHouseWorkersManager.java similarity index 83% rename from logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersManager.java rename to logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/LoggingHouseWorkersManager.java index 4b8b804..d7265e8 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersManager.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/LoggingHouseWorkersManager.java @@ -34,8 +34,9 @@ import static java.lang.String.format; -public class WorkersManager { +public class LoggingHouseWorkersManager { + private final WorkersExecutor executor; private final Monitor monitor; private final int maxWorkers; private final LoggingHouseMessageStore store; @@ -43,12 +44,14 @@ public class WorkersManager { private final URI connectorBaseUrl; private final URL loggingHouseUrl; - public WorkersManager(Monitor monitor, - int maxWorkers, - LoggingHouseMessageStore store, - RemoteMessageDispatcherRegistry dispatcherRegistry, - Hostname hostname, - URL loggingHouseUrl) { + public LoggingHouseWorkersManager(WorkersExecutor executor, + Monitor monitor, + int maxWorkers, + LoggingHouseMessageStore store, + RemoteMessageDispatcherRegistry dispatcherRegistry, + Hostname hostname, + URL loggingHouseUrl) { + this.executor = executor; this.monitor = monitor; this.maxWorkers = maxWorkers; this.store = store; @@ -62,8 +65,15 @@ public WorkersManager(Monitor monitor, } } - public void run() { - List messages = store.listNotSent(); + public void execute() { + executor.run(() -> { + processPending(); + }); + + } + + private void processPending() { + List messages = store.listPending(); if (messages.isEmpty()) { monitor.warning("No Messages to send, aborting execution"); return; @@ -118,12 +128,12 @@ private MessageWorker nextAvailableWorker(ArrayBlockingQueue avai private ArrayBlockingQueue createWorkers(int numWorkers) { return new ArrayBlockingQueue<>(numWorkers, true, IntStream.range(0, numWorkers) - .mapToObj(i -> new MessageWorker(monitor, dispatcherRegistry, connectorBaseUrl, loggingHouseUrl)) + .mapToObj(i -> new MessageWorker(monitor, dispatcherRegistry, connectorBaseUrl, loggingHouseUrl, store)) .collect(Collectors.toList())); } private String log(String input) { - return "WorkersManager: " + input; + return "LoggingHouseWorkersManager: " + input; } private URI getConnectorBaseUrl(Hostname hostname) throws URISyntaxException { diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java index d74bd82..424a2d1 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java @@ -14,12 +14,14 @@ package com.truzzt.extension.logginghouse.client.worker; -import com.truzzt.extension.logginghouse.client.messages.CreateProcessMessage; -import com.truzzt.extension.logginghouse.client.messages.LogMessage; +import com.truzzt.extension.logginghouse.client.events.messages.CreateProcessMessage; +import com.truzzt.extension.logginghouse.client.events.messages.LogMessage; +import com.truzzt.extension.logginghouse.client.spi.store.LoggingHouseMessageStore; import com.truzzt.extension.logginghouse.client.spi.types.LoggingHouseMessage; import org.eclipse.edc.spi.EdcException; import org.eclipse.edc.spi.message.RemoteMessageDispatcherRegistry; import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.response.StatusResult; import java.net.MalformedURLException; import java.net.URI; @@ -34,13 +36,16 @@ public class MessageWorker { private final RemoteMessageDispatcherRegistry dispatcherRegistry; private final URI connectorBaseUrl; private final URL loggingHouseUrl; + private final LoggingHouseMessageStore store; private final String workerId; - public MessageWorker(Monitor monitor, RemoteMessageDispatcherRegistry dispatcherRegistry, URI connectorBaseUrl, URL loggingHouseUrl) { + public MessageWorker(Monitor monitor, RemoteMessageDispatcherRegistry dispatcherRegistry, URI connectorBaseUrl, URL loggingHouseUrl, + LoggingHouseMessageStore store) { this.monitor = monitor; this.dispatcherRegistry = dispatcherRegistry; this.connectorBaseUrl = connectorBaseUrl; this.loggingHouseUrl = loggingHouseUrl; + this.store = store; workerId = "Worker-" + UUID.randomUUID(); } @@ -64,30 +69,37 @@ public CompletableFuture run(LoggingHouseMessage message) { public void process(LoggingHouseMessage message) { try { - var pid = message.getProcessId(); + var pid = message.getProcessId(); - // Create Process + // Create Process + if (message.getCreateProcess()) { var extendedProcessUrl = new URL(loggingHouseUrl + "/process/" + pid); try { - createProcess(message, extendedProcessUrl); - } catch (Exception e) { - monitor.warning("Could not create process in LoggingHouse: " + e.getMessage()); - } + createProcess(message, extendedProcessUrl).join(); - // Log Contract Agreement - var extendedLogUrl = new URL(loggingHouseUrl + "/messages/log/" + pid); - try { - logMessage(message, extendedLogUrl); } catch (Exception e) { - monitor.warning("Could not log message to LoggingHouse: " + e.getMessage()); + throw new EdcException("Could not create process in LoggingHouse", e); } + } + + // Log Message + var extendedLogUrl = new URL(loggingHouseUrl + "/messages/log/" + pid); + try { + logMessage(message, extendedLogUrl).join(); + + } catch (Exception e) { + throw new EdcException("Could not log message to LoggingHouse", e); + } + + // Update Status + store.updateSent(message.getId()); } catch (MalformedURLException e) { throw new EdcException("Could not create extended clearinghouse url."); } } - public void createProcess(LoggingHouseMessage message, URL loggingHouseUrl) { + public CompletableFuture> createProcess(LoggingHouseMessage message, URL loggingHouseUrl) { List processOwners = new ArrayList<>(); processOwners.add(message.getConsumerId()); @@ -96,15 +108,15 @@ public void createProcess(LoggingHouseMessage message, URL loggingHouseUrl) { monitor.info("Creating process in LoggingHouse with id: " + message.getProcessId()); var logMessage = new CreateProcessMessage(loggingHouseUrl, connectorBaseUrl, message.getProcessId(), processOwners); - dispatcherRegistry.dispatch(Object.class, logMessage); + return dispatcherRegistry.dispatch(Object.class, logMessage); } - public void logMessage(LoggingHouseMessage message, URL clearingHouseLogUrl) { + public CompletableFuture> logMessage(LoggingHouseMessage message, URL clearingHouseLogUrl) { monitor.info("Logging message to LoggingHouse with type " + message.getEventType() + " and id " + message.getEventId()); var logMessage = new LogMessage(clearingHouseLogUrl, connectorBaseUrl, message.getEventToLog()); - dispatcherRegistry.dispatch(Object.class, logMessage); + return dispatcherRegistry.dispatch(Object.class, logMessage); } } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersExecutor.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersExecutor.java new file mode 100644 index 0000000..e917920 --- /dev/null +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/WorkersExecutor.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - Initial implementation + * + */ + +package com.truzzt.extension.logginghouse.client.worker; + +import org.eclipse.edc.spi.monitor.Monitor; + +import java.time.Duration; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +public class WorkersExecutor { + private final Duration schedule; + private final Duration withInitialDelay; + private final Monitor monitor; + + public WorkersExecutor(Duration schedule, Duration initialDelay, Monitor monitor) { + this.schedule = schedule; + withInitialDelay = initialDelay; + this.monitor = monitor; + } + + public void run(Runnable task) { + var ses = Executors.newSingleThreadScheduledExecutor(); + ses.scheduleAtFixedRate(catchExceptions(task), withInitialDelay.toMillis(), schedule.toMillis(), TimeUnit.MILLISECONDS); + } + + private Runnable catchExceptions(Runnable original) { + return () -> { + try { + original.run(); + } catch (Throwable thr) { + monitor.severe("Unexpected error during plan execution", thr); + } + }; + } +} diff --git a/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql b/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql index 00c67b6..7f4ec09 100644 --- a/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql +++ b/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql @@ -17,9 +17,18 @@ CREATE TABLE IF NOT EXISTS edc_logging_house_message ( logging_house_message_id BIGSERIAL NOT NULL, + event_type VARCHAR NOT NULL, + event_id VARCHAR NOT NULL, event_to_log JSON NOT NULL, - created_at TIMESTAMP NOT NULL, - sent_at TIMESTAMP NOT NULL, + create_process BOOLEAN NOT NULL, + process_id VARCHAR NOT NULL, + consumer_id VARCHAR NOT NULL, + provider_id VARCHAR NOT NULL, + status VARCHAR NOT NULL, + created_at BIGINT NOT NULL, + sent_at BIGINT, PRIMARY KEY (logging_house_message_id) ); COMMENT ON COLUMN edc_logging_house_message.event_to_log IS 'Event to log serialized as JSON'; + +CREATE INDEX IF NOT EXISTS idx_edc_logging_house_message_status ON edc_logging_house_message (status); diff --git a/logging-house-client/src/test/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtensionTest.java b/logging-house-client/src/test/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtensionTest.java index 90a5582..e77a091 100644 --- a/logging-house-client/src/test/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtensionTest.java +++ b/logging-house-client/src/test/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtensionTest.java @@ -36,6 +36,6 @@ void setUp() { @Test void name_shouldReturnCorrectName() { - assertEquals(LoggingHouseClientExtension.LOGGINGHOUSE_CLIENT_EXTENSION, extension.name()); + assertEquals(LoggingHouseClientExtension.NAME, extension.name()); } } From a76eee31923dc5f37ff72b1518b4df56b8e39bf0 Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Tue, 2 Jul 2024 21:20:32 -0300 Subject: [PATCH 19/25] fix: checkstyle issues --- .../client/LoggingHouseClientExtension.java | 16 +++++----------- .../client/worker/MessageWorker.java | 2 +- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 089c839..98b3b12 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -15,15 +15,15 @@ package com.truzzt.extension.logginghouse.client; import com.truzzt.extension.logginghouse.client.events.LoggingHouseEventSubscriber; +import com.truzzt.extension.logginghouse.client.events.messages.CreateProcessMessageSender; +import com.truzzt.extension.logginghouse.client.events.messages.LogMessageSender; import com.truzzt.extension.logginghouse.client.flyway.FlywayService; import com.truzzt.extension.logginghouse.client.flyway.connection.DatasourceProperties; import com.truzzt.extension.logginghouse.client.flyway.migration.DatabaseMigrationManager; -import com.truzzt.extension.logginghouse.client.multipart.ids.jsonld.JsonLd; -import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.IdsMultipartSender; -import com.truzzt.extension.logginghouse.client.events.messages.CreateProcessMessageSender; -import com.truzzt.extension.logginghouse.client.events.messages.LogMessageSender; import com.truzzt.extension.logginghouse.client.multipart.IdsMultipartClearingRemoteMessageDispatcher; import com.truzzt.extension.logginghouse.client.multipart.MultiContextJsonLdSerializer; +import com.truzzt.extension.logginghouse.client.multipart.ids.jsonld.JsonLd; +import com.truzzt.extension.logginghouse.client.multipart.ids.multipart.IdsMultipartSender; import com.truzzt.extension.logginghouse.client.spi.store.LoggingHouseMessageStore; import com.truzzt.extension.logginghouse.client.store.sql.SqlLoggingHouseMessageStore; import com.truzzt.extension.logginghouse.client.store.sql.schema.postgres.PostgresDialectStatements; @@ -33,14 +33,8 @@ import de.fraunhofer.iais.eis.RequestMessage; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessFailed; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessInitiated; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessRequested; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessTerminated; +import org.eclipse.edc.connector.transfer.spi.event.*; import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessStarted; -import org.eclipse.edc.connector.transfer.spi.event.TransferProcessCompleted; - import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Requires; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java index 424a2d1..659381e 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/worker/MessageWorker.java @@ -116,7 +116,7 @@ public CompletableFuture> logMessage(LoggingHouseMessage me monitor.info("Logging message to LoggingHouse with type " + message.getEventType() + " and id " + message.getEventId()); var logMessage = new LogMessage(clearingHouseLogUrl, connectorBaseUrl, message.getEventToLog()); - return dispatcherRegistry.dispatch(Object.class, logMessage); + return dispatcherRegistry.dispatch(Object.class, logMessage); } } From 840faa23735727d822da5c9d96e8d2e2fda609d1 Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Tue, 2 Jul 2024 21:25:01 -0300 Subject: [PATCH 20/25] fix: checkstyle issues --- .../logginghouse/client/LoggingHouseClientExtension.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 98b3b12..185c002 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -32,9 +32,14 @@ import de.fraunhofer.iais.eis.LogMessage; import de.fraunhofer.iais.eis.RequestMessage; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; -import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; -import org.eclipse.edc.connector.transfer.spi.event.*; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessCompleted; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessFailed; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessInitiated; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessRequested; import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessStarted; +import org.eclipse.edc.connector.transfer.spi.event.TransferProcessTerminated; +import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Requires; From 029eecb30efc2b22e52f5bc6bdffd83520c4148c Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Wed, 3 Jul 2024 07:34:05 -0300 Subject: [PATCH 21/25] fix: checkstyle issues --- .../logginghouse/client/ConfigConstants.java | 14 ++++++++++++++ .../client/LoggingHouseClientExtension.java | 12 +++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java index 56c5554..c2d4477 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2024 truzzt GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * truzzt GmbH - Initial implementation + * + */ + package com.truzzt.extension.logginghouse.client; public class ConfigConstants { diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 185c002..82efc7a 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -32,14 +32,14 @@ import de.fraunhofer.iais.eis.LogMessage; import de.fraunhofer.iais.eis.RequestMessage; import org.eclipse.edc.connector.contract.spi.event.contractnegotiation.ContractNegotiationFinalized; +import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessCompleted; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessFailed; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessInitiated; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessRequested; -import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessStarted; import org.eclipse.edc.connector.transfer.spi.event.TransferProcessTerminated; -import org.eclipse.edc.connector.contract.spi.negotiation.store.ContractNegotiationStore; +import org.eclipse.edc.connector.transfer.spi.store.TransferProcessStore; import org.eclipse.edc.runtime.metamodel.annotation.Extension; import org.eclipse.edc.runtime.metamodel.annotation.Inject; import org.eclipse.edc.runtime.metamodel.annotation.Requires; @@ -63,7 +63,13 @@ import java.time.Duration; import java.util.Map; -import static com.truzzt.extension.logginghouse.client.ConfigConstants.*; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_ENABLED; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_EXTENSION_MAX_WORKERS; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_EXTENSION_WORKERS_DELAY; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_EXTENSION_WORKERS_PERIOD; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_FLYWAY_CLEAN_SETTING; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_FLYWAY_REPAIR_SETTING; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_SERVER_URL_SETTING; @Extension(value = LoggingHouseClientExtension.NAME) @Requires(value = { From 25f9d8bffdcac353eb340d89af6eda90aeb77aed Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Wed, 3 Jul 2024 07:43:51 -0300 Subject: [PATCH 22/25] chore: upgrading version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index cc7df4b..3aae426 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.2.10" + version = "0.3.0" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") From 0c4c926d7f6930e7f5422c8175a814204fae965f Mon Sep 17 00:00:00 2001 From: "D.Hommen" <75446820+dhommen@users.noreply.github.com> Date: Wed, 3 Jul 2024 13:20:00 +0200 Subject: [PATCH 23/25] Update README.md --- logging-house-client/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/logging-house-client/README.md b/logging-house-client/README.md index aa0edcd..601f2cb 100644 --- a/logging-house-client/README.md +++ b/logging-house-client/README.md @@ -1 +1,3 @@ # Logging House Client + +## Notes From b66a38da47be67f372b6a6d5e236ecf27914ee94 Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Wed, 3 Jul 2024 11:50:01 -0300 Subject: [PATCH 24/25] fix: flyway migrations conflict --- build.gradle.kts | 2 +- .../logginghouse/client/ConfigConstants.java | 14 +++++++------- .../client/LoggingHouseClientExtension.java | 12 ++++++------ .../logginghouse/client/flyway/FlywayService.java | 2 +- .../flyway/connection/DatasourceProperties.java | 9 +-------- .../flyway/migration/DatabaseMigrationManager.java | 2 +- .../{ => logginghouse}/V0_0_1__Create_Tables.sql | 0 7 files changed, 17 insertions(+), 24 deletions(-) rename logging-house-client/src/main/resources/migration/{ => logginghouse}/V0_0_1__Create_Tables.sql (100%) diff --git a/build.gradle.kts b/build.gradle.kts index 3aae426..7be6075 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.3.0" + version = "0.3.1" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java index c2d4477..cac20f4 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/ConfigConstants.java @@ -16,17 +16,17 @@ public class ConfigConstants { - static final String LOGGINGHOUSE_ENABLED = "edc.logginghouse.client.enabled"; + static final String LOGGINGHOUSE_ENABLED_SETTING = "edc.logginghouse.extension.enabled"; - static final String LOGGINGHOUSE_SERVER_URL_SETTING = "edc.logginghouse.client.server.url"; + static final String LOGGINGHOUSE_URL_SETTING = "edc.logginghouse.extension.url"; - static final String LOGGINGHOUSE_FLYWAY_REPAIR_SETTING = "edc.logginghouse.client.flyway.repair"; + static final String LOGGINGHOUSE_FLYWAY_REPAIR_SETTING = "edc.logginghouse.extension.flyway.repair"; - static final String LOGGINGHOUSE_FLYWAY_CLEAN_SETTING = "edc.logginghouse.client.flyway.clean"; + static final String LOGGINGHOUSE_FLYWAY_CLEAN_SETTING = "edc.logginghouse.extension.flyway.clean"; - static final String LOGGINGHOUSE_EXTENSION_MAX_WORKERS = "edc.logginghouse.client.workers.max"; + static final String LOGGINGHOUSE_EXTENSION_MAX_WORKERS = "edc.logginghouse.extension.workers.max"; - static final String LOGGINGHOUSE_EXTENSION_WORKERS_DELAY = "edc.logginghouse.client.workers.delay"; + static final String LOGGINGHOUSE_EXTENSION_WORKERS_DELAY = "edc.logginghouse.extension.workers.delay"; - static final String LOGGINGHOUSE_EXTENSION_WORKERS_PERIOD = "edc.logginghouse.client.workers.period"; + static final String LOGGINGHOUSE_EXTENSION_WORKERS_PERIOD = "edc.logginghouse.extension.workers.period"; } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java index 82efc7a..b5c8e8b 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/LoggingHouseClientExtension.java @@ -63,13 +63,13 @@ import java.time.Duration; import java.util.Map; -import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_ENABLED; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_ENABLED_SETTING; import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_EXTENSION_MAX_WORKERS; import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_EXTENSION_WORKERS_DELAY; import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_EXTENSION_WORKERS_PERIOD; import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_FLYWAY_CLEAN_SETTING; import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_FLYWAY_REPAIR_SETTING; -import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_SERVER_URL_SETTING; +import static com.truzzt.extension.logginghouse.client.ConfigConstants.LOGGINGHOUSE_URL_SETTING; @Extension(value = LoggingHouseClientExtension.NAME) @Requires(value = { @@ -137,7 +137,7 @@ public String name() { public void initialize(ServiceExtensionContext context) { monitor = context.getMonitor(); - var extensionEnabled = context.getSetting(LOGGINGHOUSE_ENABLED, true); + var extensionEnabled = context.getSetting(LOGGINGHOUSE_ENABLED_SETTING, true); if (!extensionEnabled) { enabled = false; monitor.info("Logginghouse client extension is disabled."); @@ -161,16 +161,16 @@ public void initialize(ServiceExtensionContext context) { private URL readUrlFromSettings(ServiceExtensionContext context) { try { - var urlString = context.getSetting(LOGGINGHOUSE_SERVER_URL_SETTING, null); + var urlString = context.getSetting(LOGGINGHOUSE_URL_SETTING, null); if (urlString == null) { throw new EdcException(String.format("Could not initialize LoggingHouseClientExtension: " + - "No url specified using setting %s", LOGGINGHOUSE_SERVER_URL_SETTING)); + "No url specified using setting %s", LOGGINGHOUSE_URL_SETTING)); } return new URL(urlString); } catch (MalformedURLException e) { throw new EdcException(String.format("Could not parse setting %s to Url", - LOGGINGHOUSE_SERVER_URL_SETTING), e); + LOGGINGHOUSE_URL_SETTING), e); } } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java index 4863dd9..8a8f9c4 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/FlywayService.java @@ -30,7 +30,7 @@ public class FlywayService { - private static final String MIGRATION_LOCATION_BASE = "classpath:migration"; + private static final String MIGRATION_LOCATION_BASE = "classpath:migration/logginghouse"; private static final String MIGRATION_TABLE_NAME = "flyway_schema_history_logginghouse"; private final Monitor monitor; diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java index 543eac3..59c1ddc 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/connection/DatasourceProperties.java @@ -19,27 +19,20 @@ public class DatasourceProperties { public static final String LOGGING_HOUSE_DATASOURCE = "logginghouse"; - private static final String DATASOURCE_SETTING_NAME = "edc.datasource.logginghouse.name"; private static final String DATASOURCE_SETTING_JDBC_URL = "edc.datasource.logginghouse.url"; private static final String DATASOURCE_SETTING_USER = "edc.datasource.logginghouse.user"; private static final String DATASOURCE_SETTING_PASSWORD = "edc.datasource.logginghouse.password"; - - private final String name; + private final String jdbcUrl; private final String user; private final String password; public DatasourceProperties(Config config) { - name = config.getString(DATASOURCE_SETTING_NAME); jdbcUrl = config.getString(DATASOURCE_SETTING_JDBC_URL); user = config.getString(DATASOURCE_SETTING_USER); password = config.getString(DATASOURCE_SETTING_PASSWORD); } - public String getName() { - return name; - } - public String getJdbcUrl() { return jdbcUrl; } diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/migration/DatabaseMigrationManager.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/migration/DatabaseMigrationManager.java index 81aab02..416a082 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/migration/DatabaseMigrationManager.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/flyway/migration/DatabaseMigrationManager.java @@ -32,7 +32,7 @@ public DatabaseMigrationManager(Config config, Monitor monitor, FlywayService fl public void migrate() { var datasourceProperties = new DatasourceProperties(config); - monitor.info("Using datasource %s to apply flyway migrations".formatted(datasourceProperties.getName())); + monitor.info("Using datasource %s to apply flyway migrations".formatted(DatasourceProperties.LOGGING_HOUSE_DATASOURCE)); flywayService.cleanDatabase(datasourceProperties); flywayService.migrateDatabase(datasourceProperties); diff --git a/logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql b/logging-house-client/src/main/resources/migration/logginghouse/V0_0_1__Create_Tables.sql similarity index 100% rename from logging-house-client/src/main/resources/migration/V0_0_1__Create_Tables.sql rename to logging-house-client/src/main/resources/migration/logginghouse/V0_0_1__Create_Tables.sql From 8ad5b4ea675745748fe2ef99e50a36fa1cdb9e78 Mon Sep 17 00:00:00 2001 From: Glaucio Jannotti Date: Wed, 3 Jul 2024 13:25:09 -0300 Subject: [PATCH 25/25] fix: logging house process id --- build.gradle.kts | 2 +- .../client/store/sql/SqlLoggingHouseMessageStore.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 7be6075..c422dbe 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -62,7 +62,7 @@ subprojects { maven { name = "GitHubPackages" url = uri("https://maven.pkg.github.com/truzzt/mds-ap3") - version = "0.3.1" + version = "0.3.2" credentials { username = System.getenv("USERNAME") password = System.getenv("TOKEN") diff --git a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java index be0bfcb..dc8886a 100644 --- a/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java +++ b/logging-house-client/src/main/java/com/truzzt/extension/logginghouse/client/store/sql/SqlLoggingHouseMessageStore.java @@ -144,12 +144,12 @@ private LoggingHouseMessage mapResultSet(ResultSet resultSet) throws Exception { return LoggingHouseMessage.Builder.newInstance() .id(resultSet.getLong(statements.getIdColumn())) .eventType(eventType) - .eventId(resultSet.getString(statements.getIdColumn())) + .eventId(resultSet.getString(statements.getEventIdColumn())) .eventToLog(eventToLog) .createProcess(resultSet.getBoolean(statements.getCreateProcessColumn())) - .processId(resultSet.getString(statements.getIdColumn())) - .consumerId(resultSet.getString(statements.getIdColumn())) - .providerId(resultSet.getString(statements.getIdColumn())) + .processId(resultSet.getString(statements.getProcessIdColumn())) + .consumerId(resultSet.getString(statements.getConsumerIdColumn())) + .providerId(resultSet.getString(statements.getProviderIdColumn())) .status(status) .createdAt(mapToZonedDateTime(resultSet, statements.getCreatedAtColumn())) .build();