Skip to content

Commit

Permalink
chore: use new ldio features (#14)
Browse files Browse the repository at this point in the history
* chore: retire EventStreamFetcher

* feat: use new Content-Type of client status

* chore: use newest LDIO image in docker compose

* chore: update docker-compose file
  • Loading branch information
jobulcke authored Nov 14, 2024
1 parent b4bdfe3 commit 8f1caa0
Show file tree
Hide file tree
Showing 12 changed files with 22 additions and 174 deletions.
4 changes: 2 additions & 2 deletions docker/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ services:
- "7200:7200"

ldio-workbench:
image: ldes/ldi-orchestrator:2.9.0-SNAPSHOT
image: ldes/ldi-orchestrator:2.10.0-SNAPSHOT
container_name: ldio-workbench
healthcheck:
test: ["CMD", "wget", "-qO-", "http://ldio-workbench:8080/actuator/health"]
Expand All @@ -74,7 +74,7 @@ services:
- "8888:8080"

testbed-shacl-validator:
image: ghcr.io/informatievlaanderen/testbed-shacl-validator:20241104104943
image: ghcr.io/informatievlaanderen/testbed-shacl-validator:20241113094029
depends_on:
graphdb:
condition: service_started
Expand Down
Original file line number Diff line number Diff line change
@@ -1,38 +1,26 @@
package be.vlaanderen.informatievlaanderen.ldes.gitb.ldio;

import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.HttpResponse;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.RequestExecutor;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.requests.GetRequest;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.config.LdioConfigProperties;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.excpeptions.LdesClientStatusUnavailableException;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.valuebojects.ClientStatus;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.HttpResponse;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.RequestExecutor;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.requests.GetRequest;
import org.springframework.stereotype.Service;

import java.io.IOException;

@Service
public class LdesClientStatusManager {
private final RequestExecutor requestExecutor;
private final LdioConfigProperties ldioConfigProperties;
private final ObjectMapper objectMapper;

public LdesClientStatusManager(RequestExecutor requestExecutor, LdioConfigProperties ldioConfigProperties) {
this.requestExecutor = requestExecutor;
this.ldioConfigProperties = ldioConfigProperties;
objectMapper = new ObjectMapper();
}

public ClientStatus getClientStatus(String pipelineName) {
final String pollingUrl = ldioConfigProperties.getLdioLdesClientStatusUrlTemplate().formatted(pipelineName);
final HttpResponse response = requestExecutor.execute(new GetRequest(pollingUrl), 200, 404);
final String json = response.getBody().orElseThrow(LdesClientStatusUnavailableException::new);
try {
return objectMapper.readValue(json, ClientStatus.class);
} catch (IOException e) {
throw new IllegalStateException("Invalid client status received from %s".formatted(pollingUrl));
}
return response.getBody().map(ClientStatus::valueOf).orElseThrow(LdesClientStatusUnavailableException::new);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.RequestExecutor;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.requests.DeleteRequest;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.requests.PostRequest;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.ldes.EventStreamFetcher;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.ldes.EventStreamProperties;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.config.LdioConfigProperties;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.pipeline.ValidationPipelineSupplier;
import org.apache.http.entity.ContentType;
Expand All @@ -16,20 +14,17 @@
@Service
public class LdioPipelineManager {
private static final Logger log = LoggerFactory.getLogger(LdioPipelineManager.class);
private final EventStreamFetcher eventStreamFetcher;
private final RequestExecutor requestExecutor;
private final LdioConfigProperties ldioConfigProperties;

public LdioPipelineManager(EventStreamFetcher eventStreamFetcher, RequestExecutor requestExecutor, LdioConfigProperties ldioConfigProperties) {
this.eventStreamFetcher = eventStreamFetcher;
public LdioPipelineManager(RequestExecutor requestExecutor, LdioConfigProperties ldioConfigProperties) {
this.requestExecutor = requestExecutor;
this.ldioConfigProperties = ldioConfigProperties;
}

public void initPipeline(String serverUrl, String pipelineName) {
final String ldioAdminPipelineUrl = ldioConfigProperties.getLdioAdminPipelineUrl();
final EventStreamProperties eventStreamProperties = eventStreamFetcher.fetchProperties(serverUrl);
final String json = new ValidationPipelineSupplier(eventStreamProperties, ldioConfigProperties.getSparqlHost(), pipelineName).getValidationPipelineAsJson();
final String json = new ValidationPipelineSupplier(serverUrl, ldioConfigProperties.getSparqlHost(), pipelineName).getValidationPipelineAsJson();
requestExecutor.execute(new PostRequest(ldioAdminPipelineUrl, json, ContentType.APPLICATION_JSON), 201);
log.atInfo().log("LDIO pipeline created: {}", pipelineName);
}
Expand Down

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ public LdioLdesClientBuilder() {
super("Ldio:LdesClient",
new HashMap<>(Map.of(
"source-format", "text/turtle",
"state", "sqlite"
"state", "sqlite",
"materialisation", Map.of("enabled", true)
))
);
}
Expand All @@ -17,14 +18,4 @@ public LdioLdesClientBuilder withUrl(String url) {
setProperty("urls", url);
return this;
}

public LdioLdesClientBuilder withVersionOfProperty(String versionOfProperty) {
setProperty("materialisation", Map.of("enabled", true, "version-of-property", versionOfProperty));
return this;
}

public LdioLdesClientBuilder withTimestampProperty(String timestampProperty) {
setProperty("timestamp-path", timestampProperty);
return this;
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.pipeline;

import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.ldes.EventStreamProperties;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.valuebojects.LdioPipeline;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
Expand All @@ -9,12 +8,12 @@

public class ValidationPipelineSupplier {
private static final String PIPELINE_DESCRIPTION = "Pipeline that will only replicate an LDES for validation purposes";
private final EventStreamProperties eventStreamProperties;
private final String ldesUrl;
private final String sparqlHost;
private final String pipelineName;

public ValidationPipelineSupplier(EventStreamProperties eventStreamProperties, String sparqlHost, String pipelineName) {
this.eventStreamProperties = eventStreamProperties;
public ValidationPipelineSupplier(String ldesUrl, String sparqlHost, String pipelineName) {
this.ldesUrl = ldesUrl;
this.sparqlHost = sparqlHost;
this.pipelineName = pipelineName;
}
Expand All @@ -23,11 +22,7 @@ public LdioPipeline getValidationPipeline() {
return new LdioPipeline(
pipelineName,
PIPELINE_DESCRIPTION,
new LdioLdesClientBuilder()
.withUrl(eventStreamProperties.ldesServerUrl())
.withVersionOfProperty(eventStreamProperties.versionOfPath())
.withTimestampProperty(eventStreamProperties.timestampPath())
.build(),
new LdioLdesClientBuilder().withUrl(ldesUrl).build(),
List.of(new LdioRepositorySinkBuilder()
.withSparqlHost(sparqlHost)
.withRepositoryId(pipelineName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.RequestExecutor;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.requests.GetRequest;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
Expand All @@ -15,7 +14,8 @@

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
class LdesClientStatusManagerTest {
Expand Down Expand Up @@ -43,12 +43,8 @@ void test_GetClientStatus(ClientStatus status) {
assertThat(actual).isEqualTo(status);
}

private static HttpResponse createEmptyResponse() {
return new HttpResponse(404, null);
}

private static HttpResponse createResponse(ClientStatus status) {
final String content = '"' + status.toString() + '"';
final String content = status.toString();
return new HttpResponse(200, content);
}
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
package be.vlaanderen.informatievlaanderen.ldes.gitb.ldio;

import be.vlaanderen.informatievlaanderen.ldes.PostRequestAssert;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.config.LdioConfigProperties;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.RequestExecutor;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.requests.DeleteRequest;
import be.vlaanderen.informatievlaanderen.ldes.gitb.requestexecutor.requests.PostRequest;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.ldes.EventStreamFetcher;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.ldes.EventStreamProperties;
import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.config.LdioConfigProperties;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.http.entity.ContentType;
Expand All @@ -25,7 +23,6 @@
import static org.mockito.ArgumentMatchers.assertArg;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
class LdioPipelineManagerTest {
Expand All @@ -35,8 +32,6 @@ class LdioPipelineManagerTest {
private static final String PIPELINE_UUID = "test-pipeline-uuid";
private static final String PIPELINE_NAME = PIPELINE_NAME_TEMPLATE.formatted(PIPELINE_UUID);
@Mock
private EventStreamFetcher eventStreamFetcher;
@Mock
private RequestExecutor requestExecutor;
private LdioPipelineManager ldioPipelineManager;

Expand All @@ -45,17 +40,15 @@ void setUp() {
final LdioConfigProperties ldioConfigProperties = new LdioConfigProperties();
ldioConfigProperties.setHost(LDIO_HOST);
ldioConfigProperties.setSparqlHost(SPARQL_HOST);
ldioPipelineManager = new LdioPipelineManager(eventStreamFetcher, requestExecutor, ldioConfigProperties);
ldioPipelineManager = new LdioPipelineManager(requestExecutor, ldioConfigProperties);
}

@Test
void test_InitPipeline() throws IOException {
final JsonNode expectedJson = new ObjectMapper().readTree(ResourceUtils.getFile("classpath:ldio-pipeline.json"));
when(eventStreamFetcher.fetchProperties(LDES_SERVER_URL)).thenReturn(new EventStreamProperties(LDES_SERVER_URL, "http://purl.org/dc/terms/isVersionOf", "http://www.w3.org/ns/prov#generatedAtTime"));

ldioPipelineManager.initPipeline(LDES_SERVER_URL, PIPELINE_NAME);

verify(eventStreamFetcher).fetchProperties(LDES_SERVER_URL);
verify(requestExecutor).execute(
assertArg(actual -> assertThat(actual)
.asInstanceOf(new InstanceOfAssertFactory<>(PostRequest.class, PostRequestAssert::new))
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.pipeline;

import be.vlaanderen.informatievlaanderen.ldes.gitb.ldio.ldes.EventStreamProperties;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
Expand All @@ -20,14 +19,13 @@ class ValidationPipelineSupplierTest {

@Test
void test_createJson() throws IOException {
final ValidationPipelineSupplier factory = new ValidationPipelineSupplier(new EventStreamProperties(LDES_SERVER_URL, "http://purl.org/dc/terms/isVersionOf", "http://www.w3.org/ns/prov#generatedAtTime"), SPARQL_HOST, PIPELINE_NAME);
final ValidationPipelineSupplier factory = new ValidationPipelineSupplier(LDES_SERVER_URL, SPARQL_HOST, PIPELINE_NAME);
final JsonNode expectedJson = readJsonNode();

final String result = factory.getValidationPipelineAsJson();
final JsonNode actualJson = objectMapper.readTree(result);

assertThat(actualJson)
.isEqualTo(expectedJson);
assertThat(actualJson).isEqualTo(expectedJson);

}

Expand Down
4 changes: 1 addition & 3 deletions src/test/resources/ldio-pipeline.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,8 @@
"urls": "http://test-server/test-collection",
"source-format": "text/turtle",
"state": "sqlite",
"timestamp-path": "http://www.w3.org/ns/prov#generatedAtTime",
"materialisation": {
"enabled": true,
"version-of-property": "http://purl.org/dc/terms/isVersionOf"
"enabled": true
}
}
},
Expand Down

0 comments on commit 8f1caa0

Please sign in to comment.