diff --git a/docs/_config.yml b/docs/_config.yml index 468af2137..c52c05a6a 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -28,20 +28,18 @@ collections: ldi-nifi: permalink: "/:collection/:path" output: true - ldi-extensions: + ldi-standalones: permalink: "/:collection/:path" output: true just_the_docs: collections: - core: - name: Linked Data Interactions Core Building Blocks ldio: name: Linked Data Interactions Orchestrator ldi-nifi: name: Linked Data Interactions for Apache NiFi - ldi-extensions: - name: Linked Data Interactions Extensions + ldi-standalones: + name: Linked Data Interactions Standalones nav_external_links: - title: VSDS Technical Docs diff --git a/docs/_core/index.md b/docs/_core/index.md deleted file mode 100644 index c8594f86b..000000000 --- a/docs/_core/index.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: Introduction -layout: home -nav_order: 0 ---- - -# Building blocks - -As the LDI strives to be an easily reusable project, each of our building blocks are framework independent and is being maintained as part in our LDI Core. - -Each of the LDI Core Building Blocks falls under one of four categories: -* [LDI Input](ldi-inputs): A component that will receive data (not necessarily LD) to then feed the LDI pipeline. -* [LDI Adapter](ldi-adapters): To be used in conjunction with the LDI Input, the LDI Adapter will transform the provided content into and internal Linked Data model and sends it down the pipeline. -* [LDI Transformer](ldi-transformers): A component that takes in a Linked Data model, transforms/modifies it and then puts it back on the pipeline. -* [LDI Output](ldi-outputs): A component that will take in Linked Data and will export it to external sources. - -````mermaid -stateDiagram-v2 - direction LR - [*] --> LDI_Input - LDI_Input --> LDI_Transformer : LD - LDI_Transformer --> LDI_Output : LD - LDI_Output --> [*] - - state LDI_Input { - direction LR - [*] --> LDI_Adapter : Non LD - - state LDI_Adapter { - direction LR - [*] --> adapt - adapt --> [*] - } - - LDI_Adapter --> [*] : LD - } - - state LDI_Transformer { - direction LR - [*] --> transform - transform --> [*] - } - state LDI_Output { - direction LR - [*] --> [*] - } -```` \ No newline at end of file diff --git a/docs/_core/ldi-adapters/index.md b/docs/_core/ldi-adapters/index.md deleted file mode 100644 index 09ffa1e46..000000000 --- a/docs/_core/ldi-adapters/index.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -layout: default -title: LDI Adapters -has_children: true -has_toc: true -nav_order: 2 ---- - -# Linked Data Interactions Adapters - -The LDI Adapter is a component to be used in conjunction with the LDI Input, the LDI Adapter will transform the provided content into and internal Linked Data model and sends it down the pipeline. diff --git a/docs/_core/ldi-adapters/json-to-json-ld.md b/docs/_core/ldi-adapters/json-to-json-ld.md deleted file mode 100644 index 992ac7c9d..000000000 --- a/docs/_core/ldi-adapters/json-to-json-ld.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -layout: default -parent: LDI Adapters -title: Json to Json-LD Adapter ---- - -# Json to Json-LD Adapter - -The json-to-ld-adapter receives json messages and adds a linked data context to transform the messages to json-ld. \ No newline at end of file diff --git a/docs/_core/ldi-adapters/ngsiv2-to-ld.md b/docs/_core/ldi-adapters/ngsiv2-to-ld.md deleted file mode 100644 index 0e70e3d1e..000000000 --- a/docs/_core/ldi-adapters/ngsiv2-to-ld.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -layout: default -parent: LDI Adapters -title: NGSI V2 to LD Adapter ---- - -# NGSI v2 to LD Adapter - -This adapter will transform a [NGSI V2] input into [NGSI LD]. - -[Jackson] is used to first deserialize the input to java objects which can then be serialized to the LD format. - -## Notes - -The algorithm applies several deviations from the standard formats. These deviations are: - -1. The observedAt attribute is added to every property, - its value is determined by the dateObserved attribute of the input. -2. The timestamp attribute of a metadata property normally determines the observedAt property but is ignored in this algorithm. - -[NGSI V2]: https://fiware.github.io/specifications/ngsiv2/stable/ -[NGSI LD]: https://ngsi-ld-tutorials.readthedocs.io/en/latest/ -[Jackson]: https://github.com/FasterXML/jackson \ No newline at end of file diff --git a/docs/_core/ldi-adapters/rdf-adapter.md b/docs/_core/ldi-adapters/rdf-adapter.md deleted file mode 100644 index 909c1c91a..000000000 --- a/docs/_core/ldi-adapters/rdf-adapter.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -layout: default -parent: LDI Adapters -title: RDF Adapter ---- - -# RDF Adapter - -As the most basic Adapter of the LDI Core Building Blocks, the RDF Adapter will take in an RDF string and convert it -into an internal Linked Data model. - -## Notes - -This Adapter only supports valid RDF mime types \ No newline at end of file diff --git a/docs/_core/ldi-adapters/rml-adapter.md b/docs/_core/ldi-adapters/rml-adapter.md deleted file mode 100644 index 30701b9db..000000000 --- a/docs/_core/ldi-adapters/rml-adapter.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -layout: default -parent: LDI Adapters -title: RML Adapter ---- - -# RML Adapter - -The RML Adapter allows a user to transform a non-LD object (json/CSV/XML) to an RDF object. - -This is done by providing a [RML] mapping file. -However, as RML is written in RDF, it can be challenging for a new user to create a new mapping. - -That's where [YARRRML] comes into play. Along with the online editor [Matey], it's easy to build one's own mapping and then export it into RML. - -[RML]: https://rml.io/specs/rml/ -[YARRRML]: https://rml.io/yarrrml/spec/ \ No newline at end of file diff --git a/docs/_core/ldi-inputs/file-archiving.md b/docs/_core/ldi-inputs/file-archiving.md deleted file mode 100644 index 0f82daedb..000000000 --- a/docs/_core/ldi-inputs/file-archiving.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -layout: default -parent: LDI Inputs -title: File archiving ---- - -# Archive File In - -The Archive File In is used to read the models from a file archive created by the [Archive File Out component](../ldi-outputs/file-archiving.md). - -This component traverses all directories in the archive in lexical order and reads the members in lexical order as well. - -Example expected structure: -- archive - - 2022 - - 01 - - 01 - - member-1.nq - - member-2.nq - - ... - - 02 - - member-122.nq - - ... - -## Config - -| Property | Description | Required | Default | Example | Supported values | -|:-----------------|:----------------------------------|:---------|:----------------------------|:-----------------|:--------------------------------| -| archive-root-dir | The root directory of the archive | Yes | N/A | /parcels/archive | Linux (+ Mac) and Windows paths | -| source-format | The source format of the files | No | Deduced from file extension | text/turtle | Any Jena supported format | - -## Example -A complete demo of the archiving functionality with both LDIO and NiFi can be found in the [E2E repo](https://github.com/Informatievlaanderen/VSDS-LDES-E2E-testing/tree/main/tests/033.archiving) - -{: .note } -The traversal order is **lexical**, this means that 1, 2, 3, ..., 9 should have leading zeroes. -03 will be read before 10 but 3 will be read after 10. - -{: .note } -Not all file extensions can be deduced automatically. Extensions like `.ttl` and `.nq` work fine and don't need a source-format specified. -When using LD+JSON, you will need to specify the source-format. \ No newline at end of file diff --git a/docs/_core/ldi-inputs/index.md b/docs/_core/ldi-inputs/index.md deleted file mode 100644 index 175279c74..000000000 --- a/docs/_core/ldi-inputs/index.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -layout: default -title: LDI Inputs -has_children: true -has_toc: true -nav_order: 1 ---- - -# Linked Data Interactions Inputs - -The LDI Input is a component that will receive data (not necessarily LD) to then feed the LDI pipeline. diff --git a/docs/_core/ldi-inputs/ldes-client.md b/docs/_core/ldi-inputs/ldes-client.md deleted file mode 100644 index ea2c5e0a8..000000000 --- a/docs/_core/ldi-inputs/ldes-client.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -layout: default -parent: LDI Inputs -title: LDES Client ---- - -# Ldes Client - -The LDES Client contains the functionality to replicate and synchronise an LDES, and to persist its state for that process. More information on the functionalities can be found [here][VSDS Tech Docs]. - -This is achieved by configuring the processor with an initial fragment URL. When the processor is triggered, the fragment will be processed, and all relations will be added to the (non-persisted) queue. - -As long as the processor runs, a queue that accepts new fragments to process is maintained. The processor also keeps track of the mutable and immutable fragments already processed. - -It will be ignored when an attempt is made to queue a known immutable fragment. Fragments in the mutable fragment store will be queued when they're expired. Should a fragment be processed from a stream that does not set the max-age in the Cache-control header, a default expiration interval will be used to set an expiration date on the fragment. - -Processed members of mutable fragments are also kept in state. They are ignored if presented more than once. - -Within a fragment, members can be ordered based on a timestamp. The path to this timestamp has to be configured. If this path is missing, the members are ordered randomly. - -To allow the possibility to filter out already received members, the exactly-once-filter can be enabled. - -This causes the ids of all processed members to be kept in state. They are ignored if presented more than once. - -Enabling this filter may have a significant impact on performance. - -[VSDS Tech Docs]: https://informatievlaanderen.github.io/VSDS-Tech-Docs/introduction/LDES_client - -The client has different state persistence strategies: - -## MEMORY -In this case we persist the state in memory. - -| Advantages | Disadvantages | -|--------------------|--------------------------------------------------------------| -| Fastest processing | Not suitable for large datasets (500k +), heap will overflow | -| Easiest setup | State is lost when the client stops/restarts | - -## SQLITE -In this case an in memory SQLITE database is used to store the state. - -| Advantages | Disadvantages | -|--------------------------------|---------------------| -| Easy setup | Slowest processing* | -| State is not lost between runs | | - -* We use a transaction for every processed record. [SQLITE is limited by the cpu](https://www.sqlite.org/faq.html#q19) - -## PostgreSQL -In this case a PostreSQL database is used - -| Advantages | Disadvantages | -|----------------------------------------|--------------------| -| State is not lost between runs | Database is needed | -| Fastest processing for larger datasets | | - diff --git a/docs/_core/ldi-outputs/file-archiving.md b/docs/_core/ldi-outputs/file-archiving.md deleted file mode 100644 index 3720a2a63..000000000 --- a/docs/_core/ldi-outputs/file-archiving.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -layout: default -parent: LDI Outputs -title: File archiving ---- - -# Archive File Out - -The Archive File Out is used to write models to files based on a timestamp path property on the model. -Every file is written to NQuads with the extracted timestamp as name: 2023-11-21-05-05-00-000000000.nq -When two files have the same name, a sequence nr is added, for example: 2023-11-21-05-05-00-000000000-2.nq - -The files are ordered in directories based on the date. For every day, there is one directory. -For example: 2023-11-21-05-05-00-000000000.nq will be located at archive-root-dir/2023/11/21. - -## Config - -| Property | Description | Required | Default | Example | Supported values | -|:-----------------|:----------------------------------------------|:---------|:--------|:-------------------------------------------|:--------------------------------| -| archive-root-dir | The root directory where files are written to | Yes | N/A | /parcels/archive | Linux (+ Mac) and Windows paths | -| timestamp-path | The timestamp path used for naming the | Yes | N/A | http://www.w3.org/ns/prov#generatedAtTime | Any valid LD predicate | - -## Example -A complete demo of the archiving functionality with both LDIO and NiFi can be found in the [E2E repo](https://github.com/Informatievlaanderen/VSDS-LDES-E2E-testing/tree/main/tests/033.archiving) - -{: .note } -Only LD streams that contain a timestamp-path can be archived with these components \ No newline at end of file diff --git a/docs/_core/ldi-outputs/index.md b/docs/_core/ldi-outputs/index.md deleted file mode 100644 index 7f979c6f6..000000000 --- a/docs/_core/ldi-outputs/index.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -layout: default -title: LDI Outputs -has_children: true -has_toc: true -nav_order: 4 ---- - -# Linked Data Interactions Outputs - -The LDI Output is a component that will take in Linked Data and will export it to external sources. diff --git a/docs/_core/ldi-outputs/repository-materialiser.md b/docs/_core/ldi-outputs/repository-materialiser.md deleted file mode 100644 index 6b05464b5..000000000 --- a/docs/_core/ldi-outputs/repository-materialiser.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -layout: default -title: Repository Materialization -parent: LDI Outputs ---- - -# Repository Materialiser - -The repository materialiser is used to materialise an LDES stream into a triplestore. -Any triplestore that supports the RDF4J remote repository API can be used. diff --git a/docs/_core/ldi-transformers/geojson-to-wkt.md b/docs/_core/ldi-transformers/geojson-to-wkt.md deleted file mode 100644 index c1250c6bf..000000000 --- a/docs/_core/ldi-transformers/geojson-to-wkt.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -layout: default -parent: LDI Transformers -title: GeoJson to Wkt Transformer ---- - -# GeoJson to Wkt Transformer - -The GeoJson to Wkt Transformer will transform any [GeoJson] statements (with predicate https://purl.org/geojson/vocab#geometry) to a [wkt string][WKT]. - -For example: -```json -{ - "https://purl.org/geojson/vocab#geojson:geometry": { - "@type": "Point", - "https://purl.org/geojson/vocab#geojson:coordinates": [100.0, 0.0] - } -} -``` -becomes: -```json -{ - "http://www.w3.org/ns/locn#geometry": { - "@value": "POINT (100 0)", - "@type": "http://www.opengis.net/ont/geosparql#wktLiteral" - } -} -``` - -[GeoJson]: https://geojson.org/ -[WKT]: https://libgeos.org/specifications/wkt/ \ No newline at end of file diff --git a/docs/_core/ldi-transformers/index.md b/docs/_core/ldi-transformers/index.md deleted file mode 100644 index 37cf3fa48..000000000 --- a/docs/_core/ldi-transformers/index.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -layout: default -title: LDI Transformers -has_children: true -has_toc: true -nav_order: 3 ---- - -# Linked Data Interactions Transformers - -The LDI Transformer is a component that takes in a Linked Data model, transforms/modifies it and then puts it back on the pipeline. \ No newline at end of file diff --git a/docs/_core/ldi-transformers/sparql-construct.md b/docs/_core/ldi-transformers/sparql-construct.md deleted file mode 100644 index c5aec1053..000000000 --- a/docs/_core/ldi-transformers/sparql-construct.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -layout: default -parent: LDI Transformers -title: SPARQL Construct Transformer ---- - -# SPARQL Construct Transformer - -The SPARQL Construct Transformer will modify the model based on the given [SPARQL] Construct Query. - -SPARQL Construct is a query language used in semantic Web technologies to create RDF (Resource Description Framework) graphs from existing RDF data. It allows users to specify a pattern of data they wish to extract from the RDF data and construct a new graph based on that pattern. - -The SPARQL Construct query language provides a powerful way to create new RDF data by using existing data as the input. It can be used to transform RDF data into different formats, as well as to simplify the structure of RDF data by aggregating or filtering data. - -This SPARQL Construct Transfomer building block can be used to execute model transformations. - -[SPARQL]: https://www.w3.org/TR/rdf-sparql-query/ - -## Splitting models using SPARQL Construct - -This component can be used to split models into multiple models using graphs. -For example, the below query will create a dataset containing multiple models defined by 'GRAPH'. -The SPARQL construct component will extract all named models from the dataset and add all statements from the default model. -The component will then return a collection of models. - -```sparql -CONSTRUCT { - GRAPH ?s { - ?s ?p ?o - } -} -WHERE { ?s ?p ?o } -``` - -## SPARQL functions - -We support some additional geo functions that can call inside your SPARQL Construct query, - -with the following namespace: - -prefix geoc: - - - - -| Function | Description | Input | Output | -|:-----------------------|:------------------------------------------------------------|:-------------------------------------------------------|:-----------------------| -| geoc:lineAtIndex | get LineString from MultiLineString by index. | MultiLineString(wktLiteral) & index | LineString(wktLiteral) | -| geoc:firstCoordinate | get first Coordinate of LineString. | LineString(wktLiteral) | Coordinate(wktLiteral) | -| geoc:lastCoordinate | get last Coordinate of LineString. | LineString(wktLiteral) | Coordinate(wktLiteral) | -| geoc:lineLength | calculate total line length of LineString. | LineString(wktLiteral) | distance in meters | -| geoc:midPoint | calculate midpoint of LineString. | LineString(wktLiteral) | Coordinate(wktLiteral) | -| geoc:pointAtFromStart | calculate point on LineString by distance. | LineString(wktLiteral) & distance in meters | Coordinate(wktLiteral) | -| \ No newline at end of file diff --git a/docs/_core/ldi-transformers/version-materializer.md b/docs/_core/ldi-transformers/version-materializer.md deleted file mode 100644 index 52fd5df26..000000000 --- a/docs/_core/ldi-transformers/version-materializer.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -layout: default -parent: LDI Transformers -title: Version Materializer ---- - -# Version Materializer - -The Version Materializer will transform a Version Object to a State Object. \ No newline at end of file diff --git a/docs/_core/ldi-transformers/version-object-creator.md b/docs/_core/ldi-transformers/version-object-creator.md deleted file mode 100644 index 0acf8ae32..000000000 --- a/docs/_core/ldi-transformers/version-object-creator.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -layout: default -parent: LDI Transformers -title: Version Object Creator ---- - -# Version Object Creator - -The Version Object Creator will transform a State Object to a Version Object. \ No newline at end of file diff --git a/docs/_ldio/ldio-core/ldio-http-requester.md b/docs/_includes/ldio-core/http-requester.md similarity index 59% rename from docs/_ldio/ldio-core/ldio-http-requester.md rename to docs/_includes/ldio-core/http-requester.md index cc8fe3329..162ca2aaa 100644 --- a/docs/_ldio/ldio-core/ldio-http-requester.md +++ b/docs/_includes/ldio-core/http-requester.md @@ -1,30 +1,21 @@ ---- -layout: default -parent: LDIO Core -title: LDIO Http Requester ---- +### Additional LDIO Http Requester properties -# LDIO Http Requester - -Different LDIO components use the Http Requester to make HTTP requests. -This requester supports the below config: - -| Property | Description | Required | Default | Example | Supported values | -|:--------------------------|:----------------------------------------------------------------------------------------------------------------------------------|:---------|:----------|:----------------------------|:----------------------------------------------| -| auth.type | The type of authentication required by the LDES server | No | NO_AUTH | OAUTH2_CLIENT_CREDENTIALS | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS | -| auth.api-key | The api key when using auth.type 'API_KEY' | No | N/A | myKey | String | -| auth.api-key-header | The header for the api key when using auth.type 'API_KEY' | No | X-API-KEY | X-API-KEY | String | -| auth.client-id | The client identifier when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | myId | String | -| auth.client-secret | The client secret when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | mySecret | String | -| auth.token-endpoint | The token endpoint when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | http://localhost:8000/token | HTTP and HTTPS urls | -| auth.scope | The Oauth2 scope when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | http://localhost:8000/token | HTTP and HTTPS urls | -| retries.enabled | Indicates if the http client should retry http requests when the server cannot be reached. | No | true | true | true or false | -| retries.max | Max number of retries the http client should do when retries.enabled = true | No | 5 | 100 | Integer | -| retries.statuses-to-retry | Custom comma seperated list of http status codes that can trigger a retry in the http client. | No | N/A | 410,451 | Comma seperated list of Integers | -| rate-limit.enabled | Indicates if the http client should limit http requests when calling the server. | No | false | false | true or false | -| rate-limit.limit | Limit of requests per period, which is defined below, that the http client should do when `rate-limit.enabled = true` | No | 500 | 100 | Integer | -| rate-limit.period | Period in which the limit of requests, which is defined above, can be reached by the http client when `rate-limit.enabled = true` | No | PT1M | PT1H | ISO 8601 Duration | -| http.headers.[].key/value | A list of custom http headers can be added. A key and value has to be provided for every header. | No | N/A | role | String | +| Property | Description | Required | Default | Supported values | Example | +|:--------------------------|:----------------------------------------------------------------------------------------------------------------------------------|:---------|:----------|:----------------------------------------------|:----------------------------| +| auth.type | The type of authentication required by the LDES server | No | NO_AUTH | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS | OAUTH2_CLIENT_CREDENTIALS | +| auth.api-key | The api key when using auth.type 'API_KEY' | No | N/A | String | myKey | +| auth.api-key-header | The header for the api key when using auth.type 'API_KEY' | No | X-API-KEY | String | X-API-KEY | +| auth.client-id | The client identifier when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | String | myId | +| auth.client-secret | The client secret when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | String | mySecret | +| auth.token-endpoint | The token endpoint when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | HTTP and HTTPS urls | http://localhost:8000/token | +| auth.scope | The Oauth2 scope when using auth.type 'OAUTH2_CLIENT_CREDENTIALS' | No | N/A | HTTP and HTTPS urls | http://localhost:8000/token | +| retries.enabled | Indicates if the http client should retry http requests when the server cannot be reached. | No | true | true or false | true | +| retries.max | Max number of retries the http client should do when retries.enabled = true | No | 5 | Integer | 100 | +| retries.statuses-to-retry | Custom comma seperated list of http status codes that can trigger a retry in the http client. | No | N/A | Comma seperated list of Integers | 410,451 | +| rate-limit.enabled | Indicates if the http client should limit http requests when calling the server. | No | false | true or false | false | +| rate-limit.limit | Limit of requests per period, which is defined below, that the http client should do when `rate-limit.enabled = true` | No | 500 | Integer | 100 | +| rate-limit.period | Period in which the limit of requests, which is defined above, can be reached by the http client when `rate-limit.enabled = true` | No | PT1M | ISO 8601 Duration | PT1H | +| http.headers.[].key/value | A list of custom http headers can be added. A key and value has to be provided for every header. | No | N/A | String | role | ```yaml config: @@ -48,14 +39,14 @@ This requester supports the below config: limit: 1000 ``` -## Retry +### Retry When retries are enabled, the following statuses are always retried, regardless of the configured statuses-to-retry: - 5xx (500 and above) - 429 -## Rate limiter +### Rate limiter > **NOTE**: Since 1.14.0, rate-limiter.max-requests-per-minute is deprecated, as this was too restrictive for > configuring the rate limiter. Here is an example on how to convert the old config. diff --git a/docs/_includes/ldio-core/rdf-writer.md b/docs/_includes/ldio-core/rdf-writer.md new file mode 100644 index 000000000..84090a7fa --- /dev/null +++ b/docs/_includes/ldio-core/rdf-writer.md @@ -0,0 +1,35 @@ +### Additional RDF Writer Properties + +| Property | Description | Required | Default | Supported values | Example | +|:------------------------|:-----------------------------------------------------------------|:---------|:------------|:------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------| +| rdf-writer.content-type | Target content type. | No | text/turtle | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) | application/ld+json | +| rdf-writer.frame | Additional JSON-LD Frame to format the outputted JSON-LD Object. | No | N/A | Any valid JSON Object that describes a JSON-LD Frame | See https://www.w3.org/TR/json-ld11-framing/#sample-library-frame | + +### RDF Writer Example + +Format as N-Quads: + +```yaml + config: + rdf-writer: + content-type: application/n-quads +``` + +Format as JSON-LD with given frame: + +```yaml + config: + rdf-writer: + content-type: application/ld+json + frame: | + { + "@context": {"@vocab": "http://example.org/"}, + "@type": "Library", + "contains": { + "@type": "Book", + "contains": { + "@type": "Chapter" + } + } + } +``` \ No newline at end of file diff --git a/docs/_ldi-extensions/index.md b/docs/_ldi-extensions/index.md deleted file mode 100644 index 4acf4ccf8..000000000 --- a/docs/_ldi-extensions/index.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: Linked Data Interactions Extensions -layout: home -nav_order: 0 ---- - -# Linked Data Interactions Extensions - -LDI Extensions are more small applications that can run individually, aside from [LDIO](./../_ldio) or -the [LDI NiFi Processors](./../_ldi-nifi). - -This is an overview of the LDI Extensions - -* [LDI LDES Discoverer](./ldes-discoverer) \ No newline at end of file diff --git a/docs/_ldi-extensions/ldes-discoverer.md b/docs/_ldi-extensions/ldes-discoverer.md deleted file mode 100644 index 3ecf0340f..000000000 --- a/docs/_ldi-extensions/ldes-discoverer.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: Linked Data Interactions LDES Discoverer -layout: home -nav_order: 1 ---- - -# Linked Data Interactions LDES Discoverer - -A lightweight application that discovers the structure of an LDES or a view by retrieving all the tree node relations of -that LDES or view. - -A use case for this could be when you are only interested in a part of the event stream. To know what part you can -follow, the structure can be discovered first. - -## Config - -| Property | Description | Required | Default | Example | Supported values | -|:--------------|:-----------------------------------------------------------|:---------|:--------------------|:--------------------------|:------------------------------------------------------------------------------------------------------------------------| -| url | Url where from the discoverer needs to start | Yes | N/A | http://example.com/my-api | HTTP and HTTPS url | -| source-format | The 'Content-Type' that should be requested to the server. | No | application/n-quads | text/turtle | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) | - -## How to run - -This tutorial will show how to use the discoverer in Docker. -In this example, we will try to discover the structure of an event stream of Geomobility. - -For simplicity, we recommend passing the config as arguments, like shown below: - -```shell -docker run ldes/ldes-discoverer --url="http://ldes-server/observations" -``` - -> **NOTE**: when an url contains a `&` symbol, which will be picked up by the shell as an operator. -> For instance, running -> `docker run ldes/ldes-discoverer --url=http://ldes-server/observations?year=2023&month=05&day=11` will result in the -> shell trying to execute three different command, where the last two will be `month=05` and `day=11`, which will fail -> of course. -> -> To resolve this, make sure the url is encapsulated in quotation marks. - -### Example output - -In the logging of the application, both the total number of relations and the relations itself will be displayed. This -will be present as the last logging statement of the app and would look something like this: - -```text -2024-02-08T14:26:25.279+01:00 INFO 48176 --- [ main] b.v.i.l.l.d.common.LdesDiscovererExecutor : http://ldes-server/observations contains a total of 10 relations: -http://ldes-server/observations -+- http://ldes-server/observations/by-time -| +- http://ldes-server/observations/by-time?year=2022 -| | +- http://ldes-server/observations/by-time?year=2022&month=08 -| +- http://ldes-server/observations/by-time?year=2023 -| +- http://ldes-server/observations/by-time?year=2023&month=05 -| +- http://ldes-server/observations/by-time?year=2023&month=05&day=07 -| +- http://ldes-server/observations/by-time?year=2023&month=05&day=16 -| +- http://ldes-server/observations/by-time?year=2023&month=05&day=20 -+- http://ldes-server/observations/paged - +- http://ldes-server/observations/paged?pageNumber=1 -``` \ No newline at end of file diff --git a/docs/_ldi-nifi/index.md b/docs/_ldi-nifi/index.md index 05a400ccb..4460838f6 100644 --- a/docs/_ldi-nifi/index.md +++ b/docs/_ldi-nifi/index.md @@ -8,24 +8,35 @@ nav_order: 0 [Apache Nifi] is an easy to use, powerful, and reliable system to process and distribute data. -## Usage - -All the following processors can be found in the processor list when using the ldes/ldi-workbench-nifi docker image. - -These processors can be [added][Adding a processor in NiFi] by filtering on the ***be.vlaanderen.informatievlaanderen.ldes.ldi.nifi*** group or by filtering on the ***vsds*** tag - -- [Create Version Processor](../core/ldi-transformers/version-object-creator) -- [GeoJson to WKT Processor](../core/ldi-transformers/geojson-to-wkt) -- [Json to Json LD Processor](../core/ldi-adapters/json-to-json-ld) -- [Ngsi V2 to LD Processor](../core/ldi-adapters/ngsiv2-to-ld) -- [RDF4j Repository Materialization Processor](../core/ldi-outputs/repository-materialiser) -- [SPARQL Interactions Processor](./processors/sparql-interactions) -- [Version Materialization Processor](../core/ldi-transformers/version-materializer) -- [Archive File Out Processor](../core/ldi-outputs/file-archiving) -- [Archive File In Processor](../core/ldi-outputs/file-archiving) +## Set up NiFi instance with LDI processors + +The processors can be imported into a NiFi docker instance via volume binding: + +1. Create a `docker-compose.yml` file with the following content in a new directory + ````yaml + services: + nifi: + image: apache/nifi:2.0.0-M2 + environment: + SINGLE_USER_CREDENTIALS_USERNAME: admin + SINGLE_USER_CREDENTIALS_PASSWORD: ctsBtRBKHRAx69EqUghvvgEvjnaLjFEB + ports: + - 8443:8443 + volumes: + - ./nifi-ext:/opt/nifi/nifi-current/nar_extensions:rw + ```` +2. Create a directory `nifi-ext` in your current directory. +3. Download either the `...-nar-bundle.jar` and unpack this or download the individual required processors (.nar extension) from the [nexus repository]. + Next, place the required processors in the `nifi-ext` directory. +4. Finally, start your instance. + ````shell + docker compose up + ```` +5. Log in at `https://localhost:8443/nifi` with the credentials mentioned in step 1 +6. All downloaded extensions are available under the ``be.vlaanderen.informatievlaanderen.ldes.ldi.nifi`` group. {: .note } All documentation and notes about configuration are available in the NiFi component itself. [Apache NiFi]: https://nifi.apache.org/ -[Adding a processor in NiFi]: https://nifi.apache.org/docs/nifi-docs/html/getting-started.html#adding-a-processor \ No newline at end of file +[nexus repository]: https://s01.oss.sonatype.org/#nexus-search;quick~be.vlaanderen.informatievlaanderen.ldes.ldi.nifi \ No newline at end of file diff --git a/docs/_ldi-standalones/ldes-discoverer.md b/docs/_ldi-standalones/ldes-discoverer.md new file mode 100644 index 000000000..8fa5ff60b --- /dev/null +++ b/docs/_ldi-standalones/ldes-discoverer.md @@ -0,0 +1,96 @@ +--- +title: Linked Data Interactions LDES Discoverer +layout: home +nav_order: 0 +--- + +# Linked Data Interactions LDES Discoverer + +A lightweight application that discovers the structure of an LDES or a view by retrieving all the tree node relations of +that LDES or view. + +A use case for this could be when you are only interested in a part of the event stream. To know what part you can +follow, the structure can be discovered first. + +## Config + +### Base configuration + +| Property | Description | Required | Default | Example | Supported values | +|:--------------|:-----------------------------------------------------------|:---------|:------------|:--------------------------|:------------------------------------------------------------------------------------------------------------------------| +| url | Url where from the discoverer needs to start | true | N/A | http://example.com/my-api | HTTP and HTTPS url | +| source-format | The 'Content-Type' that should be requested to the server. | false | text/turtle | application/n-quads | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) | + +### Optional config + +#### Authentication + +| Property | Description | Default | Example | Supported values | +|:---------------|:-----------------------------------------------------------------------|:----------|:----------------------------|:----------------------------------------------| +| auth-type | The type of authentication required by the LDES server | NO_AUTH | OAUTH2_CLIENT_CREDENTIALS | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS | +| api-key | The api key when using auth-type 'API_KEY' | N/A | myKey | String | +| api-key-header | The header for the api key when using auth-type 'API_KEY' | X-API-KEY | X-API-KEY | String | +| client-id | The client identifier when using auth-type 'OAUTH2_CLIENT_CREDENTIALS' | N/A | myId | String | +| client-secret | The client secret when using auth-type 'OAUTH2_CLIENT_CREDENTIALS' | N/A | mySecret | String | +| token-endpoint | The token endpoint when using auth-type 'OAUTH2_CLIENT_CREDENTIALS' | N/A | http://localhost:8000/token | HTTP and HTTPS urls | +| scope | The Oauth2 scope when using auth-type 'OAUTH2_CLIENT_CREDENTIALS' | N/A | http://localhost:8000/token | HTTP and HTTPS urls | + +#### Further customization + +| Property | Description | Default | Example | Supported values | +|:------------------|:--------------------------------------------------------------------------------------------------------------------|:--------|:-----------------------|:---------------------------------| +| disable-retry | Boolean flag that disables retrying to send http requests when the server cannot be reached. (enabled when omitted) | N/A | N/A | N/A | +| retry-limit | Max number of retries the http client should do (only on absence of disable-retry) | 5 | 100 | Integer | +| retry-statuses | Custom comma seperated list of http status codes that can trigger a retry in the http client. | N/A | 410,451 | Comma seperated list of Integers | +| rate-limit | Limit of requests per period, which is defined below, that the http client should do | N/A | 500 | Integer | +| rate-limit-period | Period in which the limit of requests, which is defined above, can be reached by the http client | PT1M | PT1H | ISO 8601 Duration | +| header | Parameter for each individual header that is required | N/A | Connection: keep-alive | String | + +## How to run + +This tutorial will show how to use the discoverer in Docker. +In this example, we will try to discover the structure of an event stream called observations. + +For simplicity, we recommend passing the config as arguments. + +**Run the ldes-discoverer with minimal config** + +```shell +docker run ldes/ldes-discoverer --url="http://ldes-server/observations" +``` + +**Run the ldes-discoverer with rate-limit, authentication and two additional headers** + +```shell +docker run ldes/ldes-discoverer --url="http://ldes-server/observations" --retry-limit=3 --rate-limit=400 \ + --header="Connection: keep alive" --header="X-Source-App: ldes-discoverer" \ + --auth-type=API_KEY --api-key="my-secret-api-key" +``` + +> **NOTE**: when an url contains a `&` symbol, which will be picked up by the shell as an operator. +> For instance, running +> `docker run ldes/ldes-discoverer --url=http://ldes-server/observations?year=2023&month=05&day=11` will result in the +> shell trying to execute three different command, where the last two will be `month=05` and `day=11`, which will fail +> of course. +> +> To resolve this, make sure the url is encapsulated in quotation marks. + +### Example output + +In the logging of the application, both the total number of relations and the relations itself will be displayed. This +will be present as the last logging statement of the app and would look something like this: + +```text +2024-02-08T14:26:25.279+01:00 INFO 48176 --- [ main] b.v.i.l.l.d.common.LdesDiscovererExecutor : http://ldes-server/observations contains a total of 10 relations: +http://ldes-server/observations ++- http://ldes-server/observations/by-time +| +- http://ldes-server/observations/by-time?year=2022 +| | +- http://ldes-server/observations/by-time?year=2022&month=08 +| +- http://ldes-server/observations/by-time?year=2023 +| +- http://ldes-server/observations/by-time?year=2023&month=05 +| +- http://ldes-server/observations/by-time?year=2023&month=05&day=07 +| +- http://ldes-server/observations/by-time?year=2023&month=05&day=16 +| +- http://ldes-server/observations/by-time?year=2023&month=05&day=20 ++- http://ldes-server/observations/paged + +- http://ldes-server/observations/paged?pageNumber=1 +``` \ No newline at end of file diff --git a/docs/_ldio/examples/index.md b/docs/_ldio/examples/index.md index 12d0655c1..e439f4560 100644 --- a/docs/_ldio/examples/index.md +++ b/docs/_ldio/examples/index.md @@ -7,42 +7,5 @@ nav_order: 10 --- # Linked Data Interactions Orchestrator Examples -The easiest way to start working with the LDIO is by using Docker. -We'll go through the setup of the service here. - -***docker-compose.yml***: -````yaml -version: '3.3' -services: - ldio-workbench: - container_name: ldio-workbench - image: ldes/ldi-orchestrator:1.1.0 - volumes: - - ./ldio.config.yml:/ldio/application.yml:ro - ports: - - ":8080" -```` - Through the table of contents, you will find examples of the LDIO config that can be placed inside the ***ldio.config.yml*** -to get the desired results. - -{: .note } -> If any extra files are required for a processor (mapping/queries/...), you can add them in your volume binding pointing to the ``ldio`` folder as follows: -> -> Note that the name given for the file can be whatever, as long as it is unique. -> -> ``- ./file.extension:/ldio/file.extension:ro`` - -{: .note } -> If any custom processors have been created, you can add the jars in your volume binding pointing to the ``ldio/lib`` folder as follows: -> -> Note that the name given for the jar file can be whatever, as long as it is unique. -> -> ``- :/ldio/lib/custom-processor.jar:ro`` - -## Execution - -Once configured with the LDIO config, execute the command -````shell -docker compose up -```` \ No newline at end of file +to get the desired results. \ No newline at end of file diff --git a/docs/_ldio/index.md b/docs/_ldio/index.md index 033db7de1..3d925d6da 100644 --- a/docs/_ldio/index.md +++ b/docs/_ldio/index.md @@ -8,6 +8,47 @@ nav_order: 0 A lightweight application maintained by the LDI team. Its creation came when a more lightweight alternative for [Apache NiFi] was needed. +## Docker Compose + +The easiest way to start working with the LDIO is by using Docker. The image is located on the [Docker Hub](https://hub.docker.com/r/ldes/ldi-orchestrator/tags). + +To set up your environment, start by creating a new folder dedicated to your LDIO project. Within this folder, create two files: a `docker-compose.yml` and a YAML configuration file. +The YAML file can be named according to your preference and can be added to the volume bindings pointing to the `ldio/application.yml` file. + +To enable Swagger UI, debug logging, or monitoring, please follow the instructions provided below on how to incorporate them into the LDIO YAML configuration file. + +***docker-compose.yml***: +````yaml +version: '3.3' +services: + ldio-workbench: + container_name: ldio-workbench + image: ldes/ldi-orchestrator:2.4.0-SNAPSHOT + volumes: + - ./ldio.config.yml:/ldio/application.yml:ro + ports: + - ":8080" +```` + +Once configured with the LDIO config, execute the command +````shell +docker compose up +```` + +{: .note } +> If any extra files are required for a processor (mapping/queries/...), you can add them in your volume binding pointing to the ``ldio`` folder as follows: +> +> Note that the name given for the file can be whatever, as long as it is unique. +> +> ``- ./file.extension:/ldio/file.extension:ro`` + +{: .note } +> If any custom processors have been created, you can add the jars in your volume binding pointing to the ``ldio/lib`` folder as follows: +> +> Note that the name given for the jar file can be whatever, as long as it is unique. +> +> ``- :/ldio/lib/custom-processor.jar:ro`` + ## Enable swagger UI To use the swagger UI on your own LDIO deployment you can add the below config, diff --git a/docs/_ldio/ldio-adapters/index.md b/docs/_ldio/ldio-adapters/index.md index 7ac9edad2..9154e20ab 100644 --- a/docs/_ldio/ldio-adapters/index.md +++ b/docs/_ldio/ldio-adapters/index.md @@ -8,6 +8,13 @@ nav_order: 4 # Linked Data Orchestrator Adapters -The LDI Core module contains the components maintained by the VSDS team in order to accommodate the onboarding of LDES onboarders. +Adapters are be used in conjunction with the LDI Input. They will transform the provided content into and internal Linked Data model and sends it down the pipeline for further processing. -Each component can be wrapped in a desired implementation framework (LDI-orchestrator, NiFi, ...) to be used. +## Overview + +| Adapter | Description | Inputs | Advantages | Disadvantages | +| :------------------ | :--------------------------------------------------------------------------------------- | :----------------------------------------------- | :----------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------- | +| _JSON to JSON-LD_ | Receives JSON messages and adds linked data context to transform the messages to JSON-LD |
  • JSON
|
  • Easy to set up: plug in context
|
  • Only works with JSON as input
  • Slower performance when deserializing model
| +| _RDF_ | Takes in an RDF string and converts it into an internal linked data model |
  • RDF string
|
  • Easy to set up: no configuration needed
|
  • Only works with RDF as input
  • Only supports valid RDF MIME types
| +| _RML_ | Transform a non-linked data object (JSON/CSV/XML) to RDF object |
  • JSON
  • CSV
  • XML
|
  • Most powerful adapter
  • Can convert multiple input objects
|
  • RML knowledge needed to do mapping
| +| _NGSIv2 to NGSI-LD_ | Converts NGSIv2 to an NGSI-LD model |
  • NGSIv2 (JSON)
| |
  • Only works with NGSIv2 as input
  • Slower performance when deserializing model
| diff --git a/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md b/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md index 2eb886656..e30a60709 100644 --- a/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md +++ b/docs/_ldio/ldio-adapters/ldio-json-to-json-ld.md @@ -4,11 +4,10 @@ parent: LDIO Adapters title: Json To JsonLd Transformer --- -# LDIO Json To JsonLd Transformer - +# LDIO Json To JsonLd Adapter ***Ldio:JsonToLdAdapter*** -An LDIO wrapper component for the [LDI Json To JsonLd building block](../../core/ldi-adapters/json-to-json-ld) +The json-to-ld-adapter receives json messages and adds a linked data context to transform the messages to json-ld. ## Config diff --git a/docs/_ldio/ldio-adapters/ldio-ngsiv2-to-ld.md b/docs/_ldio/ldio-adapters/ldio-ngsiv2-to-ld.md index 062621964..c0f856c8b 100644 --- a/docs/_ldio/ldio-adapters/ldio-ngsiv2-to-ld.md +++ b/docs/_ldio/ldio-adapters/ldio-ngsiv2-to-ld.md @@ -8,7 +8,18 @@ title: NGSIv2 To LD Adapter ***Ldio:NgsiV2ToLdAdapter*** -An LDIO wrapper component for the [LDI NGSIv2 To LD building block](../../core/ldi-adapters/ngsiv2-to-ld) +This adapter will transform a [NGSI V2] input into [NGSI LD]. + +[Jackson] is used to first deserialize the input to java objects which can then be serialized to the LD format. + +## Notes + +The algorithm applies several deviations from the standard formats. These deviations are: + +1. The observedAt attribute is added to every property, + its value is determined by the dateObserved attribute of the input. +2. The timestamp attribute of a metadata property normally determines the observedAt property but is ignored in this + algorithm. ## Config @@ -16,4 +27,10 @@ An LDIO wrapper component for the [LDI NGSIv2 To LD building block](../../core/l |:----------------|:-------------------------------------------------|:---------|:--------|:--------------------------|:--------------------| | core-context | URI of a core json-ld context. | Yes | N/A | http://example.com/my-api | HTTP and HTTPS urls | | ld-context | URI of a custom json-ld context. | No | N/A | http://example.com/my-api | HTTP and HTTPS urls | -| data-identifier | Identifier that points to data in provided json. | Yes | N/A | data | String | \ No newline at end of file +| data-identifier | Identifier that points to data in provided json. | Yes | N/A | data | String | + +[NGSI V2]: https://fiware.github.io/specifications/ngsiv2/stable/ + +[NGSI LD]: https://ngsi-ld-tutorials.readthedocs.io/en/latest/ + +[Jackson]: https://github.com/FasterXML/jackson \ No newline at end of file diff --git a/docs/_ldio/ldio-adapters/ldio-rdf-adapter.md b/docs/_ldio/ldio-adapters/ldio-rdf-adapter.md index 1cf6390f4..3f6da4417 100644 --- a/docs/_ldio/ldio-adapters/ldio-rdf-adapter.md +++ b/docs/_ldio/ldio-adapters/ldio-rdf-adapter.md @@ -8,7 +8,12 @@ title: RDF Adapter ***Ldio:RdfAdapter*** -An LDIO wrapper component for the [LDI RDF Adapter building block](../../core/ldi-adapters/rdf-adapter) +As the most basic adapter, the RDF Adapter will take in an RDF string and convert it +into an internal Linked Data model based on the given content type. + +## Notes + +This Adapter only supports valid RDF mime types ## Config diff --git a/docs/_ldio/ldio-adapters/ldio-rml-adapter.md b/docs/_ldio/ldio-adapters/ldio-rml-adapter.md index d9e9f075a..1d9b33b7e 100644 --- a/docs/_ldio/ldio-adapters/ldio-rml-adapter.md +++ b/docs/_ldio/ldio-adapters/ldio-rml-adapter.md @@ -8,11 +8,16 @@ title: RML Adapter ***Ldio:RmlAdapter*** -An LDIO wrapper component for the [LDI RML Adapter building block](../../core/ldi-adapters/rml-adapter) +The RML Adapter allows a user to transform a non-LD object (json/CSV/XML) to an RDF object. + +This is done by providing a RML mapping file. For more details on how to form a correct RML mapping, visit +the [RML documentation]. ## Config | Property | Description | Required | Default | Example | Supported values | |:---------|:-----------------------------------------------|:---------|:--------|:----------------------------|:--------------------| -| mapping | Path to content of RML/content of RML mapping. | Yes | N/A | mapping.ttl | Path/String | \ No newline at end of file +| mapping | Path to content of RML/content of RML mapping. | Yes | N/A | mapping.ttl | Path/String | + +[RML documentation]: https://rml.io/specs/rml/ \ No newline at end of file diff --git a/docs/_ldio/ldio-core/index.md b/docs/_ldio/ldio-core/index.md deleted file mode 100644 index 06697a9cb..000000000 --- a/docs/_ldio/ldio-core/index.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -layout: default -title: LDIO Core -has_children: true -has_toc: true -nav_order: 2 ---- - -# Linked Data Orchestrator Core - -Contains common components used by multiple other LDIO components diff --git a/docs/_ldio/ldio-core/ldio-rdf-writer.md b/docs/_ldio/ldio-core/ldio-rdf-writer.md deleted file mode 100644 index 636ee031d..000000000 --- a/docs/_ldio/ldio-core/ldio-rdf-writer.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -layout: default -parent: LDIO Core -title: LDI RDF Writer ---- - -# LDI RDF Writer - -To easily output RDF in the correct format, a generic RDF Writer is introduced. -The RDF Writer supports the below config: - -| Property | Description | Required | Default | Example | Supported values | -|:-------------|:-----------------------------------------------------------------|:---------|:------------|:------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| -| content-type | Target content type. | No | text/turtle | application/ld+json | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) | -| frame | Additional JSON-LD Frame to format the outputted JSON-LD Object. | No | N/A | See https://www.w3.org/TR/json-ld11-framing/#sample-library-frame | Any valid JSON Object that describes a JSON-LD Frame | - -## Example - -Format as N-Quads: - -```yaml - config: - rdf-writer: - content-type: application/n-quads -``` - -Format as JSON-LD with given frame: - -```yaml - config: - rdf-writer: - content-type: application/ld+json - frame: | - { - "@context": {"@vocab": "http://example.org/"}, - "@type": "Library", - "contains": { - "@type": "Book", - "contains": { - "@type": "Chapter" - } - } - } -``` \ No newline at end of file diff --git a/docs/_ldio/ldio-inputs/ldio-archive-file-in.md b/docs/_ldio/ldio-inputs/ldio-archive-file-in.md deleted file mode 100644 index ee46f14b1..000000000 --- a/docs/_ldio/ldio-inputs/ldio-archive-file-in.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -layout: default -parent: LDIO Inputs -title: Archive File In ---- - -# LDIO File Out - -***Ldio:ArchiveFileIn*** - -The LDIO Archive File In is used to read models from files and feed them to the pipeline. -Please refer to the [core documentation](../../core/ldi-inputs/file-archiving) for more information. - -## Config - -| Property | Description | Required | Default | Example | Supported values | -|:-----------------|:----------------------------------|:---------|:----------------------------|:-----------------|:--------------------------------| -| archive-root-dir | The root directory of the archive | Yes | N/A | /parcels/archive | Linux (+ Mac) and Windows paths | -| source-format | The source format of the files | No | Deduced from file extension | text/turtle | Any Jena supported format | - -## Pausing - -When paused, this component will stop reading from the archive. -When resumed, it will pick up where it left of, ignoring any changes to the file structure. \ No newline at end of file diff --git a/docs/_ldio/ldio-inputs/ldio-ldes-client.md b/docs/_ldio/ldio-inputs/ldio-ldes-client.md index 182f287b0..e3311b72f 100644 --- a/docs/_ldio/ldio-inputs/ldio-ldes-client.md +++ b/docs/_ldio/ldio-inputs/ldio-ldes-client.md @@ -4,11 +4,51 @@ parent: LDIO Inputs title: LDES Client --- -# LDIO Ldes Client +# LDIO LDES Client -***Ldio:LdesClient*** +The LDES Client is a component which can be used by data consumers to replicate and synchronize an LDES. +When replication or synchronization is halted, the LDES Client is able to resume where it has stopped. More information on how consumption of an LDES works can be found [here](https://informatievlaanderen.github.io/VSDS-Tech-Docs/introduction/LDES_client). -An LDIO wrapper component for the [LDI LDES Client building block](../../core/ldi-inputs/ldes-client) +## Processing fragments + +One or more URLs need to be configured in the LDES Client. If more URLs are configured, they need to be part of the same LDES. +The configured fragments (URLs) will be processed and all relations will be added to the (non-persisted) queue. +As long as the LDES Client runs, new fragments that need to be processed can be added to the queue. +The LDES Client will keep track of the mutable and immutable fragments it did already process. When an immutable fragment that already has been processed is added to the queue, it will be ignored. + +Mutable fragments usually have a `max-age` set in the Cache-control header. If this isn't the case, a default expiration interval will be used to set an expiration date on the fragment. +When the `max-age` or default expiration interval of a fragment expires, the fragment will be put into the queue again so that the LDES Client fetches it again. + +### Processing members within fragments + +The LDES Client keeps track of the processed members of mutable fragments, to make sure members are only processed once within a fragment. +When the fragment is marked as immutable, and no members can be added anymore, the LDES Client will stop keeping track of members processed within that fragment. + +Members within a fragment can be processed in order of time based on a timestamp. The path to this timestamp needs to be configured. +If the patch is missing, members will be processed in random order. + +### Filtering + +To have the possibility to filter out already received members, the "exactly-once-filter" can be enabled in configuration. The filter will check whether a member was already processed in other fragments. +The IDs of all processed members will be remembered by the filter and when a duplicate member is processed, it will be filtered out before sending it to the output of the Client. +Other filters can be added in the future. + +### Persistence strategies + +The Client offers different ways to persist state of the processed members: + +| Strategy | Description | Advantages | Disadvantages | +|:------------|:------------------------------------------------------------|:---------------------------------------|:------------------------------------------------------------| +| _Memory_ | Store the state of members in the memory of the LDES Client | Fastest processing | Not suitable for large datasets (>500k), heap will overflow | +| | | Easiest setup | State is lost when the client stops/restarts | +| | | | | +| _SQLite_ | A SQLite database is used to store state of members | Easy setup | Slowest processing** | +| | | State is not lost between runs | | +| | | | | +| _PostgreSQL_| A PostgreSQL database is used to store state of the members | Fastest processing for larger datasets | Database is needed | +| | | State is not lost between runs | | + +** We use a transaction for every processed record and SQLite is limited by the CPU ([source](https://www.sqlite.org/faq.html#q19)). ## Config @@ -16,39 +56,44 @@ An LDIO wrapper component for the [LDI LDES Client building block](../../core/ld | Property | Description | Required | Default | Example | Supported values | |:----------------------------|:----------------------------------------------------------------------------------------|:---------|:-------------|:------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| -| urls | List of URLs of the LDES data sources | Yes | N/A | http://localhost:8080/my-ldes | HTTP and HTTPS urls | -| source-format | The 'Content-Type' that should be requested to the server. | No | text/turtle | application/n-quads | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) | -| state | 'sqlite', 'memory', 'file' or 'postgres' to indicate how the state should be persisted. | No | memory | sqlite | 'sqlite', 'files' or 'memory' | -| keep-state | Indicates if the state should be persisted on shutdown (n/a for in memory states) | No | false | false | true or false | -| timestamp-path | The property-path used to determine the timestamp on which the members will be ordered | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | A property path | -| enable-exactly-once | Indicates whether member must be send exactly once or at least once | No | true | true | true or false | - -> **_NOTE:_** The default `source-format` is `text/turtle`, as this rdf format supports relative uri's. However, if -> relative uri's are not used, `application/n-quads` or even the binary format `application/rdf+protobuf` are better +| _urls_ | List of URLs of the LDES data sources | Yes | N/A | http://localhost:8080/my-ldes | HTTP and HTTPS URLs | +| _source-format_ | The 'Content-Type' that should be requested to the server | No | text/turtle | application/n-quads | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) | +| _state_ | 'memory', 'sqlite' or 'postgres' to indicate how the state should be persisted | No | memory | sqlite | 'memory', 'sqlite' or 'postgres' | +| _keep-state_ | Indicates if the state should be persisted on shutdown (n/a for in memory states) | No | false | false | true or false | +| _timestamp-path_ | The property-path used to determine the timestamp on which the members will be ordered | No | N/A | http://www.w3.org/ns/prov#generatedAtTime | A property path | +| _enable-exactly-once_ | Indicates whether a member must be send exactly once or at least once | No | true | true | true or false | + +> **_NOTE:_** The default `source-format` is `text/turtle`, as this RDF format supports relative URIs. However, if +> relative URIs are not used, `application/n-quads` or even the binary format `application/rdf+protobuf` are better > options, as these formats are faster to parse. -### Postgres properties -| Property | Description | Required | Default | Example | Supported values | -|:------------------|:-----------------------------------------------|:---------|:--------|:---------------------------------------------------------------|:-----------------| -| postgres.url | JDBC url of the Postgres database. | No | N/A | jdbc:postgresql://test.postgres.database.azure.com:5432/sample | String | -| postgres.username | Username used to connect to Postgres database. | No | N/A | myUsername@test | String | -| postgres.password | Password used to connect to Postgres database. | No | N/A | myPassword | String | ### Version materialisation properties -| Property | Description | Required | Default | Example | Supported values | -|:------------------------------------|:---------------------------------------------------------------------------------------|:---------|:-------------------------------------|:---------------------------------------|:-----------------| -| materialisation.enabled | Indicates if the client should return state-objects (true) or version-objects (false). | No | false | true | true or false | -| materialisation.version-of-property | Property that points to the versionOfPath. | No | http://purl.org/dc/terms/isVersionOf | "http://purl.org/dc/terms/isVersionOf" | true or false | +| Property | Description | Required | Default | Example | Supported values | +|:-------------------------------------|:---------------------------------------------------------------------------------------|:---------|:-------------------------------------|:---------------------------------------|:-----------------| +| _materialisation.enabled_ | Indicates if the client should return state-objects (true) or version-objects (false) | No | false | true | true or false | +| _materialisation.version-of-property_| Property that points to the versionOfPath | No | http://purl.org/dc/terms/isVersionOf | "http://purl.org/dc/terms/isVersionOf" | true or false | + +This component uses the "LDIO Http Requester" to make the HTTP-request. +Refer to [LDIO Http Requester](../ldio-core/ldio-http-requester) for the configuration of that component. + +> **_NOTE:_** Setting the keep-state property to `true` makes it so that the state can not be deleted through the pipeline-management api + + + +### Postgres properties + +| Property | Description | Required | Default | Example | Supported values | +|:-------------------|:-----------------------------------------------|:---------|:--------|:---------------------------------------------------------------|:-----------------| +| _postgres.url_ | JDBC URL of the Postgres database | No | N/A | jdbc:postgresql://test.postgres.database.azure.com:5432/sample | String | +| _postgres.username_| Username used to connect to Postgres database | No | N/A | myUsername@test | String | +| _postgres.password_| Password used to connect to Postgres database | No | N/A | myPassword | String | -This component uses the "LDIO Http Requester" to make the HTTP request. -Refer to [LDIO Http Requester](../ldio-core) for the config. -> **_NOTE:_** Setting the keep-state property to true makes it so that the state can not be deleted through the -> pipeline-management api -## Examples +### Configuration Examples ```yaml input: @@ -84,7 +129,7 @@ Refer to [LDIO Http Requester](../ldio-core) for the config. password: myPassword ``` -## Pausing +## Pausing the LDES Client -When paused, this component will stop processing the current fragment and not make any calls to the server. -When resumed, it will continue with the fragment where it stopped and continue as normal. \ No newline at end of file +- When paused, the LDES Client will stop processing the current fragment and will not request new fragments from the server. +- When resumed, the LDES Client will continue processing the fragment where it has stopped and it will request new fragments form the server. diff --git a/docs/_ldio/ldio-outputs/ldio-amqp-out.md b/docs/_ldio/ldio-outputs/ldio-amqp-out.md index c1bac3295..f5671c9b2 100644 --- a/docs/_ldio/ldio-outputs/ldio-amqp-out.md +++ b/docs/_ldio/ldio-outputs/ldio-amqp-out.md @@ -9,18 +9,19 @@ title: AMQP Out ***Ldio:AmqpOut*** The LDIO AMQP Out sends messages to an [AMQP 1.0 queue](https://www.amqp.org/resources/specifications). -The content-type configured in the [rdf-writer](../ldio-core/ldio-rdf-writer.md) +The content-type configured in the rdf-writer.content-type is added as a header to the message with key "contentType". ## Config -| Property | Description | Required | Default | Example | Supported values | -|-------------|-------------------------------------|----------|----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| -| remote-url | URI to AMQP queue | Yes | N/A | amqp://server:61616 | In line with `amqp[s]://hostname:port[?option=value[&option2=value...]]` or `amqpws[s]://hostname:port[/path][?option=value[&option2=value...]]` | -| queue | Name of the queue | Yes | N/A | quickstart-events | String | -| username | Username used in authentication | Yes | N/A | client | String | -| password | Password used in the authentication | Yes | N/A | secret | String | -| rdf-writer | LDI RDF Writer Config | No | Empty Config | N/A | [LDI RDF Writer Config](../ldio-core/ldio-rdf-writer) | +| Property | Description | Required | Default | Supported values | Example | +|------------|-------------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------|---------------------| +| remote-url | URI to AMQP queue | Yes | N/A | In line with `amqp[s]://hostname:port[?option=value[&option2=value...]]` or `amqpws[s]://hostname:port[/path][?option=value[&option2=value...]]` | amqp://server:61616 | +| queue | Name of the queue | Yes | N/A | String | quickstart-events | +| username | Username used in authentication | Yes | N/A | String | client | +| password | Password used in the authentication | Yes | N/A | String | secret | + +{% include ldio-core/rdf-writer.md %} ## Example diff --git a/docs/_ldio/ldio-outputs/ldio-azure-blob-out.md b/docs/_ldio/ldio-outputs/ldio-azure-blob-out.md deleted file mode 100644 index 5dae9d8cd..000000000 --- a/docs/_ldio/ldio-outputs/ldio-azure-blob-out.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -layout: default -parent: LDIO Outputs -title: Azure Blob Out ---- - -# LDIO Kafka Out - -***Ldio:AzureBlobOut*** - -The LDIO Azure Blob Out writes out messages to an Azure Blob Container. -Messages can be written out in any format supported by Apache Jena or in json format. - -**IMPORTANT**: For the json format it is necessary to have a URI which holds the context. - -## Config - -| Property | Description | Required | Default | Example | Supported values | -|----------------------|-----------------------------------------------------------------------------------------------|----------|---------|----------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------| -| lang | **Json*** and any lang supported by Apache Jena. *json requires the json-context-uri property | No | n-quads | json | Any type supported by [Apache Jena](https://jena.apache.org/documentation/io/rdf-input.html#determining-the-rdf-syntax) | -| storage-account-name | Name of the Azure Storage Account | Yes | N/A | saldesclientdemo | String | -| connection-string | Value of the Connection String of the Azure Storage Account | Yes | N/A | DefaultEndpointsProtocol=https;AccountName=demopowerquery;AccountKey=...;EndpointSuffix=core.windows.net | [Azure Connection String](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string) | -| blob-container | Name of the Blob Container | Yes | N/A | data | String | -| json-context-uri | URI which holds the Json Context | No | "" | https://private-api.gipod.beta-vlaanderen.be/api/v1/context/gipod.jsonld | URI describing [context](https://www.w3.org/TR/json-ld11/#the-context) | - -## Example - -### JSON - -```yaml -outputs: - - name: "Ldio:AzureBlobOut" - config: - lang: "json" - storage-account-name: "storageaccount" - connection-string: "DefaultEndpointsProtocol=https;AccountName=demopowerquery;AccountKey=...;EndpointSuffix=core.windows.net" - blob-container: "blobcontainer" - json-context-uri: "https://essentialcomplexity.eu/gipod.jsonld" -``` - -### N-QUADS - -```yaml -outputs: - - name: "Ldio:AzureBlobOut" - config: - lang: "n-quads" - storage-account-name: "storageaccount" - connection-string: "DefaultEndpointsProtocol=https;AccountName=demopowerquery;AccountKey=...;EndpointSuffix=core.windows.net" - blob-container: "blobcontainer" -``` \ No newline at end of file diff --git a/docs/_ldio/ldio-outputs/ldio-console-out.md b/docs/_ldio/ldio-outputs/ldio-console-out.md index db56a11a6..e5036a826 100644 --- a/docs/_ldio/ldio-outputs/ldio-console-out.md +++ b/docs/_ldio/ldio-outputs/ldio-console-out.md @@ -12,6 +12,4 @@ The LDIO Console Out will output its given model to the console. ## Config -| Property | Description | Required | Default | Example | Supported values | -|:-----------|:-----------------------|:---------|:-------------|:--------|:------------------------------------------------------| -| rdf-writer | LDI RDF Writer Config | No | Empty Config | N/A | [LDI RDF Writer Config](../ldio-core/ldio-rdf-writer) | \ No newline at end of file +{% include ldio-core/rdf-writer.md %} \ No newline at end of file diff --git a/docs/_ldio/ldio-outputs/ldio-file-out.md b/docs/_ldio/ldio-outputs/ldio-file-out.md deleted file mode 100644 index 92e9b835d..000000000 --- a/docs/_ldio/ldio-outputs/ldio-file-out.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -layout: default -parent: LDIO Outputs -title: File Out ---- - -# LDIO File Out - -***Ldio:FileOut*** - -The LDIO File Out is used to write models to files based on a timestamp path property on the model. -Please refer to the [core documentation](../../core/ldi-outputs/file-archiving.md) for more information. - -## LDIO Config - -| Property | Description | Required | Default | Example | Supported values | -|:-----------------|:----------------------------------------------|:---------|:--------|:-------------------------------------------|:--------------------------------| -| archive-root-dir | The root directory where files are written to | Yes | N/A | /parcels/archive | Linux (+ Mac) and Windows paths | -| timestamp-path | The timestamp path used for naming the | Yes | N/A | http://www.w3.org/ns/prov#generatedAtTime | Any valid LD predicate | \ No newline at end of file diff --git a/docs/_ldio/ldio-outputs/ldio-http-out.md b/docs/_ldio/ldio-outputs/ldio-http-out.md index 106294f27..ecf9400da 100644 --- a/docs/_ldio/ldio-outputs/ldio-http-out.md +++ b/docs/_ldio/ldio-outputs/ldio-http-out.md @@ -15,4 +15,5 @@ The LDIO HTTP Out is a basic Http Client that will send the given Linked Data mo | Property | Description | Required | Default | Example | Supported values | |:-----------|:----------------------|:---------|:-------------|:----------------------------|:------------------------------------------------------| | endpoint | Target url. | Yes | N/A | http://example.com/endpoint | HTTP and HTTPS urls | -| rdf-writer | LDI RDF Writer Config | No | Empty Config | N/A | [LDI RDF Writer Config](../ldio-core/ldio-rdf-writer) | \ No newline at end of file + +{% include ldio-core/rdf-writer.md %} \ No newline at end of file diff --git a/docs/_ldio/ldio-outputs/ldio-kafka-out.md b/docs/_ldio/ldio-outputs/ldio-kafka-out.md index 6d01db3ec..ccde2c395 100644 --- a/docs/_ldio/ldio-outputs/ldio-kafka-out.md +++ b/docs/_ldio/ldio-outputs/ldio-kafka-out.md @@ -24,7 +24,8 @@ Two security protocols are supported: | sasl-jaas-user | Username used in the security protocol | No | null | client | String | | sasl-jaas-password | Password used in the security protocol | No | null | secret | String | | frame-type | RDF type of the objects that need to be included for JSON-LD framing. | No | N/A | http://purl.org/goodrelations/v1#Offering | Any RDF type | -| rdf-writer | LDI RDF Writer Config | No | Empty Config | N/A | [LDI RDF Writer Config](../ldio-core/ldio-rdf-writer) | + +{% include ldio-core/rdf-writer.md %} ## Example diff --git a/docs/_ldio/ldio-transformers/index.md b/docs/_ldio/ldio-transformers/index.md index 663022c6a..c52a15ac8 100644 --- a/docs/_ldio/ldio-transformers/index.md +++ b/docs/_ldio/ldio-transformers/index.md @@ -6,4 +6,4 @@ has_toc: true nav_order: 5 --- -# Linked Data Orchestrator Transformers \ No newline at end of file +# Linked Data Interactions Orchestrator Transformers \ No newline at end of file diff --git a/docs/_ldio/ldio-transformers/ldio-geojson-to-wkt.md b/docs/_ldio/ldio-transformers/ldio-geojson-to-wkt.md index e6a03247c..c9265c3fd 100644 --- a/docs/_ldio/ldio-transformers/ldio-geojson-to-wkt.md +++ b/docs/_ldio/ldio-transformers/ldio-geojson-to-wkt.md @@ -8,8 +8,35 @@ title: GeoJson To WKT Transformer ***Ldio:GeoJsonToWktTransformer*** -An LDIO wrapper component for the [LDI GeoJson to Wkt building block](../../core/ldi-transformers/geojson-to-wkt) +The GeoJson to Wkt Transformer will transform any [GeoJson] statements (with +predicate https://purl.org/geojson/vocab#geometry) to a [wkt string][WKT]. + +For example: + +```json +{ + "https://purl.org/geojson/vocab#geojson:geometry": { + "@type": "Point", + "https://purl.org/geojson/vocab#geojson:coordinates": [100.0, 0.0] + } +} +``` + +becomes: + +```json +{ + "http://www.w3.org/ns/locn#geometry": { + "@value": "POINT (100 0)", + "@type": "http://www.opengis.net/ont/geosparql#wktLiteral" + } +} +``` ## Config -This component has no required config \ No newline at end of file +This component has no required config + +[GeoJson]: https://geojson.org/ + +[WKT]: https://libgeos.org/specifications/wkt/ \ No newline at end of file diff --git a/docs/_ldio/ldio-transformers/ldio-http-enricher.md b/docs/_ldio/ldio-transformers/ldio-http-enricher.md index 79a9c6880..4cac31c3d 100644 --- a/docs/_ldio/ldio-transformers/ldio-http-enricher.md +++ b/docs/_ldio/ldio-transformers/ldio-http-enricher.md @@ -13,15 +13,16 @@ The response is converted to linked data and added to the incoming model. ## Config -| Property | Description | Required | Default | Example | Supported values | -|:------------------------------------|:------------------------------------------------------------------------------------------------------------|:---------|:--------|:----------------------------------------------------|:---------------------------------------------------------------| -| adapter.name | This transformer requires an [ldio-adapter](../ldio-adapters) to convert the responses to linked data. | Yes | N/A | Ldio:RdfAdapter | Paths of supported LDIO Adapters | -| adapter.config.xxx | Optional config that may be required by the adapter | No | N/A | Ldio:RdfAdapter | Paths of supported LDIO Adapters | -| url-property-path | Path defining the url that needs to be selected on the model for the http request. | Yes | N/A | | Valid property paths | -| header-property-path | Path defining the headers that needs to be selected on the model for the http request. | No | N/A | | Valid property paths | -| body-property-path | Path defining the body that needs to be selected on the model to be added when a POST http request is used. | No | N/A | / | Valid property paths | -| http-method-property-path | Path defining the http method that needs to be selected on the model for the http request. | No | GET | GET | GET or POST | -| auth.xxx retries.xxx rate-limit.xxx | LDIO Http Requester Config | No | N/A | N/A | [LDIO Http Requester Config](../ldio-core/ldio-http-requester) | +| Property | Description | Required | Default | Supported values | Example | +|:--------------------------|:------------------------------------------------------------------------------------------------------------|:---------|:--------|:---------------------------------|:----------------------------------------------------| +| adapter.name | This transformer requires an [ldio-adapter](../ldio-adapters) to convert the responses to linked data. | Yes | N/A | Paths of supported LDIO Adapters | Ldio:RdfAdapter | +| adapter.config.xxx | Optional config that may be required by the adapter | No | N/A | Paths of supported LDIO Adapters | Ldio:RdfAdapter | +| url-property-path | Path defining the url that needs to be selected on the model for the http request. | Yes | N/A | Valid property paths | | +| header-property-path | Path defining the headers that needs to be selected on the model for the http request. | No | N/A | Valid property paths | | +| body-property-path | Path defining the body that needs to be selected on the model to be added when a POST http request is used. | No | N/A | Valid property paths | / | +| http-method-property-path | Path defining the http method that needs to be selected on the model for the http request. | No | GET | GET or POST | GET | + +{% include ldio-core/http-requester.md %} Note that all adapters are supported. When the adapter requires additional config, this can be added as seen below in the example. diff --git a/docs/_ldio/ldio-transformers/ldio-sparql-construct.md b/docs/_ldio/ldio-transformers/ldio-sparql-construct.md index 2bb564351..545f89c86 100644 --- a/docs/_ldio/ldio-transformers/ldio-sparql-construct.md +++ b/docs/_ldio/ldio-transformers/ldio-sparql-construct.md @@ -8,7 +8,53 @@ title: SPARQL Construct ***Ldio:SparqlConstructTransformer*** -An LDIO wrapper component for the [LDI SPARQL Construct building block](../../core/ldi-transformers/sparql-construct) +The SPARQL Construct Transformer will modify the model based on the given [SPARQL] Construct Query. + +SPARQL Construct is a query language used in semantic Web technologies to create RDF (Resource Description Framework) +graphs from existing RDF data. It allows users to specify a pattern of data they wish to extract from the RDF data and +construct a new graph based on that pattern. + +The SPARQL Construct query language provides a powerful way to create new RDF data by using existing data as the input. +It can be used to transform RDF data into different formats, as well as to simplify the structure of RDF data by +aggregating or filtering data. + +This SPARQL Construct Transfomer building block can be used to execute model transformations. + +[SPARQL]: https://www.w3.org/TR/rdf-sparql-query/ + +## Splitting models using SPARQL Construct + +This component can be used to split models into multiple models using graphs. +For example, the below query will create a dataset containing multiple models defined by 'GRAPH'. +The SPARQL construct component will extract all named models from the dataset and add all statements from the default +model. +The component will then return a collection of models. + +```sparql +CONSTRUCT { + GRAPH ?s { + ?s ?p ?o + } +} +WHERE { ?s ?p ?o } +``` + +## SPARQL functions + +We support some additional geo functions that can call inside your SPARQL Construct query, + +with the following namespace: + +prefix geoc: + +| Function | Description | Input | Output | +|:----------------------|:----------------------------------------------|:--------------------------------------------|:-----------------------| +| geoc:lineAtIndex | get LineString from MultiLineString by index. | MultiLineString(wktLiteral) & index | LineString(wktLiteral) | +| geoc:firstCoordinate | get first Coordinate of LineString. | LineString(wktLiteral) | Coordinate(wktLiteral) | +| geoc:lastCoordinate | get last Coordinate of LineString. | LineString(wktLiteral) | Coordinate(wktLiteral) | +| geoc:lineLength | calculate total line length of LineString. | LineString(wktLiteral) | distance in meters | +| geoc:midPoint | calculate midpoint of LineString. | LineString(wktLiteral) | Coordinate(wktLiteral) | +| geoc:pointAtFromStart | calculate point on LineString by distance. | LineString(wktLiteral) & distance in meters | Coordinate(wktLiteral) | ## Config diff --git a/docs/_ldio/ldio-transformers/ldio-version-materializer.md b/docs/_ldio/ldio-transformers/ldio-version-materializer.md index 3ce5a0a51..4bbc72d67 100644 --- a/docs/_ldio/ldio-transformers/ldio-version-materializer.md +++ b/docs/_ldio/ldio-transformers/ldio-version-materializer.md @@ -8,7 +8,7 @@ title: Version Materializer ***Ldio:VersionMaterialiser*** -An LDIO wrapper component for the [LDI Version Materializer building block](../../core/ldi-transformers/version-materializer) +The Version Materializer will transform a Version Object to a State Object. ## Config diff --git a/docs/_ldio/ldio-transformers/ldio-version-object-creator.md b/docs/_ldio/ldio-transformers/ldio-version-object-creator.md index 51f6ce1a0..ff1ca798b 100644 --- a/docs/_ldio/ldio-transformers/ldio-version-object-creator.md +++ b/docs/_ldio/ldio-transformers/ldio-version-object-creator.md @@ -8,7 +8,7 @@ title: Version Object Creator ***Ldio:VersionObjectCreator*** -An LDIO wrapper component for the [LDI Version Object Creator building block](../../core/ldi-transformers/version-object-creator) +The Version Object Creator will transform a State Object to a Version Object. ## Config diff --git a/docs/_ldio/pipeline-management/index.md b/docs/_ldio/pipeline-management/index.md index 464e6cc30..6fdd07db8 100644 --- a/docs/_ldio/pipeline-management/index.md +++ b/docs/_ldio/pipeline-management/index.md @@ -36,6 +36,48 @@ A default pipeline looks as follows: - Note that one orchestrator can have multiple pipelines - Note that one pipeline can have multiple LDI Transformers and LDI Outputs +## Anatomy of a pipeline + +Each pipeline is built up of the following components: + +* [LDIO Input](ldi-inputs): A component that will receive data (not necessarily LD) to then feed the LDIO pipeline. +* [LDIO Adapter](ldi-adapters): To be used in conjunction with the LDIO Input, the LDIO Adapter will transform the + provided content into and internal Linked Data model and sends it down the pipeline. +* [LDIO Transformer](ldi-transformers): A component that takes in a Linked Data model, transforms/modifies it and then + puts it back on the pipeline. +* [LDIO Output](ldi-outputs): A component that will take in Linked Data and will export it to external sources. + +````mermaid +stateDiagram-v2 + direction LR + + LDI_Input --> LDI_Transformer : LD + LDI_Transformer --> LDI_Output : LD + + state LDI_Input { + direction LR + [*] --> LDI_Adapter : Non LD + + state LDI_Adapter { + direction LR + [*] --> adapt + adapt --> [*] + } + + LDI_Adapter --> [*] : LD + } + + state LDI_Transformer { + direction LR + [*] --> transform + transform --> [*] + } + state LDI_Output { + direction LR + [*] --> [*] + } +```` + ## Persistence of Pipelines By default, all pipelines defined after startup (via management API) will be lost on restart. diff --git a/docs/docker-compose.yml b/docs/docker-compose.yml index 52f9d00e6..3c10890fa 100644 --- a/docs/docker-compose.yml +++ b/docs/docker-compose.yml @@ -4,13 +4,4 @@ services: ports: - 4000:4000 volumes: - - .:/site - - postgres: - image: postgres:14-alpine - ports: - - 5432:5432 - environment: - - POSTGRES_PASSWORD=admin - - POSTGRES_USER=admin - - POSTGRES_DB=db \ No newline at end of file + - .:/site \ No newline at end of file diff --git a/ldi-api/pom.xml b/ldi-api/pom.xml index 83d6474d6..f629cde02 100644 --- a/ldi-api/pom.xml +++ b/ldi-api/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/file-archiving/pom.xml b/ldi-core/file-archiving/pom.xml index d8461781b..5db540c9d 100644 --- a/ldi-core/file-archiving/pom.xml +++ b/ldi-core/file-archiving/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT file-archiving diff --git a/ldi-core/geojson-to-wkt/pom.xml b/ldi-core/geojson-to-wkt/pom.xml index 67e492312..076666259 100644 --- a/ldi-core/geojson-to-wkt/pom.xml +++ b/ldi-core/geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT geojson-to-wkt diff --git a/ldi-core/json-to-ld-adapter/pom.xml b/ldi-core/json-to-ld-adapter/pom.xml index 500dde2a9..908cf368a 100644 --- a/ldi-core/json-to-ld-adapter/pom.xml +++ b/ldi-core/json-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT json-to-ld-adapter diff --git a/ldi-core/ldes-client/pom.xml b/ldi-core/ldes-client/pom.xml index ae20022db..7e6cb3785 100644 --- a/ldi-core/ldes-client/pom.xml +++ b/ldi-core/ldes-client/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/starting-node/pom.xml b/ldi-core/ldes-client/starting-node/pom.xml index 6e699a8a3..0256cc0c7 100644 --- a/ldi-core/ldes-client/starting-node/pom.xml +++ b/ldi-core/ldes-client/starting-node/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 starting-node diff --git a/ldi-core/ldes-client/tree-node-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-fetcher/pom.xml index 9e71cfc12..dd51d117f 100644 --- a/ldi-core/ldes-client/tree-node-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT diff --git a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml index 64e3ec1f7..1d12a23fe 100644 --- a/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml +++ b/ldi-core/ldes-client/tree-node-relations-fetcher/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/ldes-client/tree-node-supplier/pom.xml b/ldi-core/ldes-client/tree-node-supplier/pom.xml index 1f680fabc..21cddfccc 100644 --- a/ldi-core/ldes-client/tree-node-supplier/pom.xml +++ b/ldi-core/ldes-client/tree-node-supplier/pom.xml @@ -5,7 +5,7 @@ ldes-client be.vlaanderen.informatievlaanderen.ldes.client - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 tree-node-supplier diff --git a/ldi-core/ldi-common/pom.xml b/ldi-core/ldi-common/pom.xml index 5c97d2af2..fa87d2c22 100644 --- a/ldi-core/ldi-common/pom.xml +++ b/ldi-core/ldi-common/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldi-common diff --git a/ldi-core/ngsiv2-to-ld-adapter/pom.xml b/ldi-core/ngsiv2-to-ld-adapter/pom.xml index 10972eeb1..3f64a6c57 100644 --- a/ldi-core/ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-core/ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ngsiv2-to-ld-adapter diff --git a/ldi-core/pom.xml b/ldi-core/pom.xml index bc8cdbd86..ef24fbc72 100644 --- a/ldi-core/pom.xml +++ b/ldi-core/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/rdf-adapter/pom.xml b/ldi-core/rdf-adapter/pom.xml index 0295d1d0c..1d740042c 100644 --- a/ldi-core/rdf-adapter/pom.xml +++ b/ldi-core/rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RdfAdapter.java b/ldi-core/rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RdfAdapter.java index 05f37b036..e61a02f96 100644 --- a/ldi-core/rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RdfAdapter.java +++ b/ldi-core/rdf-adapter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/RdfAdapter.java @@ -5,6 +5,8 @@ import org.apache.jena.riot.RDFParser; import org.apache.jena.sparql.util.Context; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.util.stream.Stream; import static org.apache.jena.riot.RDFLanguages.nameToLang; @@ -21,7 +23,7 @@ public RdfAdapter(Context context) { public Stream apply(Content input) { return Stream.of( RDFParser - .fromString(input.content()) + .source(new ByteArrayInputStream(input.content().getBytes(StandardCharsets.UTF_8))) .context(context) .lang(nameToLang(input.mimeType())) .toModel() diff --git a/ldi-core/repository-materialiser/pom.xml b/ldi-core/repository-materialiser/pom.xml index 6c03010c6..0b11dbdc7 100644 --- a/ldi-core/repository-materialiser/pom.xml +++ b/ldi-core/repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT diff --git a/ldi-core/request-executor/pom.xml b/ldi-core/request-executor/pom.xml index f79dae4a4..dfa946b45 100644 --- a/ldi-core/request-executor/pom.xml +++ b/ldi-core/request-executor/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 request-executor diff --git a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/ratelimiter/RateLimiterConfig.java b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/ratelimiter/RateLimiterConfig.java index c7ed33f7c..24a9e1c77 100644 --- a/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/ratelimiter/RateLimiterConfig.java +++ b/ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/ratelimiter/RateLimiterConfig.java @@ -9,9 +9,9 @@ import java.time.format.DateTimeParseException; public class RateLimiterConfig { - private static final String RATE_LIMIT_PER_MINUTE_MIGRATION_WARNING = "'rate-limit.max-requests-per-minute' property is deprecated. Please consider migrating to the more generic properties 'rate-limit.limit' and 'rate-limit.period'"; - private static final String INVALID_PERIOD_ERROR = "Invalid config for the property 'rate-limiter.period': this must be a valid 8601 duration"; - private static final Logger log = LoggerFactory.getLogger(RateLimiterConfig.class); + private static final String RATE_LIMIT_PER_MINUTE_MIGRATION_WARNING = "'rate-limit.max-requests-per-minute' property is deprecated. Please consider migrating to the more generic properties 'rate-limit.limit' and 'rate-limit.period'"; + private static final String INVALID_PERIOD_ERROR = "Invalid config for the property 'rate-limiter.period': this must be a valid 8601 duration"; + private static final Logger log = LoggerFactory.getLogger(RateLimiterConfig.class); private final int limitForPeriod; private final Duration limitRefreshPeriod; @@ -23,28 +23,32 @@ private RateLimiterConfig(int limitForPeriod, Duration limitRefreshPeriod, Durat this.timeoutDuration = timeoutDuration; } - public static RateLimiterConfig limitPerMinute(int maxRequestsPerMinute) { - log.warn(RATE_LIMIT_PER_MINUTE_MIGRATION_WARNING); - return new RateLimiterConfig(maxRequestsPerMinute, Duration.ofMinutes(1), Duration.ofMinutes(1)); - } - - public static RateLimiterConfig limitForPeriod(int limit, String period) { - try { - return new RateLimiterConfig(limit, Duration.parse(period), Duration.parse(period)); - } catch (DateTimeParseException exception) { - throw new IllegalArgumentException(INVALID_PERIOD_ERROR); - } - } - - public RateLimiter getRateLimiter() { - return RateLimiterRegistry.of( - io.github.resilience4j.ratelimiter.RateLimiterConfig - .custom() - .limitForPeriod(limitForPeriod) - .limitRefreshPeriod(limitRefreshPeriod) - .timeoutDuration(timeoutDuration) - .build()) - .rateLimiter("rate-limit-http-requests"); - } + public static RateLimiterConfig limitPerMinute(int maxRequestsPerMinute) { + log.warn(RATE_LIMIT_PER_MINUTE_MIGRATION_WARNING); + return new RateLimiterConfig(maxRequestsPerMinute, Duration.ofMinutes(1), Duration.ofMinutes(1)); + } + + public static RateLimiterConfig limitForPeriod(int limit, String period) { + try { + return limitForPeriod(limit, Duration.parse(period)); + } catch (DateTimeParseException exception) { + throw new IllegalArgumentException(INVALID_PERIOD_ERROR); + } + } + + public static RateLimiterConfig limitForPeriod(int limit, Duration period) { + return new RateLimiterConfig(limit, period, period); + } + + public RateLimiter getRateLimiter() { + return RateLimiterRegistry.of( + io.github.resilience4j.ratelimiter.RateLimiterConfig + .custom() + .limitForPeriod(limitForPeriod) + .limitRefreshPeriod(limitRefreshPeriod) + .timeoutDuration(timeoutDuration) + .build()) + .rateLimiter("rate-limit-http-requests"); + } } diff --git a/ldi-core/rml-adapter/pom.xml b/ldi-core/rml-adapter/pom.xml index f843392fd..96cf5f383 100644 --- a/ldi-core/rml-adapter/pom.xml +++ b/ldi-core/rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/sparql-construct/pom.xml b/ldi-core/sparql-construct/pom.xml index daa3e5315..8704e4b7f 100644 --- a/ldi-core/sparql-construct/pom.xml +++ b/ldi-core/sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-core - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-materialiser/pom.xml b/ldi-core/version-materialiser/pom.xml index 58f59287f..bccebeb8a 100644 --- a/ldi-core/version-materialiser/pom.xml +++ b/ldi-core/version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-object-creator/pom.xml b/ldi-core/version-object-creator/pom.xml index 00afe2785..480b44dc6 100644 --- a/ldi-core/version-object-creator/pom.xml +++ b/ldi-core/version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldi-core be.vlaanderen.informatievlaanderen.ldes.ldi - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java b/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java index 42cc3c2a9..120aa3b1b 100644 --- a/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java +++ b/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java @@ -25,7 +25,11 @@ public class VersionObjectCreator implements LdiOneToOneTransformer { private static final String XMLSCHEMA_DATE_TIME = "http://www.w3.org/2001/XMLSchema#dateTime"; public static final Property SYNTAX_TYPE = initModel .createProperty("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"); - public static final String DATE_OBSERVED_PROPERTY_COULD_NOT_BE_FOUND = "Date observed property could not be found: {}"; + + public static final String DATE_OBSERVED_PROPERTY_COULD_NOT_BE_FOUND = "Date observed property could not be found, taking current datetime: {}"; + public static final String STATEMENT_NOT_FOUND = "Statement could not be found: {}"; + public static final String LINKED_DATA_MODEL_IS_EMPTY = "Received an empty data model"; + public static final String CREATED_VERSION = "Created version: {}"; private final PropertyExtractor dateObservedPropertyExtractor; private final Resource memberTypeResource; @@ -51,12 +55,16 @@ public Model transform(Model linkedDataModel) { if (memberInfo.isPresent()) return constructVersionObject(linkedDataModel, new MemberInfo(memberInfo.get(), dateObserved)); - LOGGER.warn(DATE_OBSERVED_PROPERTY_COULD_NOT_BE_FOUND, dateObserved); + if (linkedDataModel.isEmpty()) { + LOGGER.warn(LINKED_DATA_MODEL_IS_EMPTY); + } else { + LOGGER.warn(STATEMENT_NOT_FOUND, SYNTAX_TYPE.getNameSpace()); + } + return linkedDataModel; } private Optional extractMemberInfo(Model linkedDataModel) { - return linkedDataModel.listStatements(null, SYNTAX_TYPE, memberTypeResource) .nextOptional() .map(Statement::getSubject) @@ -73,7 +81,11 @@ private String getDateObserved(Model linkedDataModel) { .filter(literal -> literal.getDatatype().getURI().toLowerCase().contains("datetime")) .map(Literal::getString) .findFirst() - .orElse(LocalDateTime.now().format(formatter)); + .orElseGet(() -> { + String currentDate = LocalDateTime.now().format(formatter); + LOGGER.warn(DATE_OBSERVED_PROPERTY_COULD_NOT_BE_FOUND, currentDate); + return currentDate; + }); } protected Model constructVersionObject(Model inputModel, MemberInfo memberInfo) { @@ -81,7 +93,7 @@ protected Model constructVersionObject(Model inputModel, MemberInfo memberInfo) String versionObjectId = memberInfo.generateVersionObjectId(delimiter); Resource subject = versionObjectModel.createResource(versionObjectId); - LOGGER.info("Created version: {}", versionObjectId); + LOGGER.info(CREATED_VERSION, versionObjectId); List statementList = inputModel.listStatements().toList(); statementList.stream() diff --git a/ldi-core/version-object-creator/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreatorTest.java b/ldi-core/version-object-creator/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreatorTest.java index 705c3bb91..b69595439 100644 --- a/ldi-core/version-object-creator/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreatorTest.java +++ b/ldi-core/version-object-creator/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreatorTest.java @@ -4,7 +4,6 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.extractor.PropertyExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.extractor.PropertyPathExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldi.valueobjects.MemberInfo; -import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; @@ -31,7 +30,8 @@ import java.util.Objects; import java.util.stream.Stream; -import static be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator.SYNTAX_TYPE; +import static be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator.*; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; class VersionObjectCreatorTest { @@ -48,7 +48,7 @@ class VersionObjectCreatorTest { "urn:ngsi-v2:cot-imec-be:WaterQualityObserved:imec-iow-3orY3reQDK5n3TMpPnLVYR", "2022-04-19T11:40:42.000Z"); @Test - void when_constructVersionObject_ExpectLdesProperties() throws URISyntaxException, IOException { + void when_constructVersionObject_expectLdesProperties() throws URISyntaxException, IOException { Model model = RDFParserBuilder.create().fromString(getJsonString("outputformat/example-ldes.json")) .lang(Lang.JSONLD).toModel(); @@ -58,7 +58,7 @@ void when_constructVersionObject_ExpectLdesProperties() throws URISyntaxExceptio Model actualOutput = versionObjectCreator.constructVersionObject(model, memberInfo); assertFalse(actualOutput.listStatements(null, PROV_GENERATED_AT_TIME, - model.createTypedLiteral(memberInfo.getObservedAt(), "http://www.w3.org/2001/XMLSchema#dateTime")) + model.createTypedLiteral(memberInfo.getObservedAt(), "http://www.w3.org/2001/XMLSchema#dateTime")) .toList().isEmpty()); assertFalse(actualOutput.listStatements(null, SYNTAX_TYPE, model.createResource(WATER_QUALITY_OBSERVED)).toList().isEmpty()); @@ -77,15 +77,12 @@ void when_dateObservedPropertyIsNoDatetime_expectCurrentDatetime() { ex:foo "bar mitswa". """).lang(Lang.TTL).toModel(); - Resource memberType = inputModel.createResource("http://example.org/Something"); - PropertyExtractor dateObservedPropertyExtractor = PropertyPathExtractor.from(""); - Property generatedAtTimeProperty = inputModel.createProperty("http://www.w3.org/ns/prov#generatedAtTime"); - Property versionOfProperty = inputModel.createProperty("http://purl.org/dc/terms/isVersionOf"); - String expectedId = "http://example.org/member/"; - VersionObjectCreator versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberType, - DEFAULT_DELIMITER, generatedAtTimeProperty, versionOfProperty); + VersionObjectCreator versionObjectCreator = createVersionObjectCreator( + inputModel, + "" + ); Model versionObject = versionObjectCreator.transform(inputModel); @@ -115,13 +112,11 @@ void when_dateObservedPropertyIsNested_thenAPropertyPathCanBeProvided() { time:inXSDDateTimeStamp "2023-08-18T13:08:00+01:00"^^xsd:DateTime ] . """).lang(Lang.TTL).toModel(); - Resource memberType = inputModel.createResource("http://example.org/Something"); - PropertyExtractor dateObservedPropertyExtractor = PropertyPathExtractor.from( - "/"); - Property generatedAtTimeProperty = inputModel.createProperty("http://www.w3.org/ns/prov#generatedAtTime"); - Property versionOfProperty = inputModel.createProperty("http://purl.org/dc/terms/isVersionOf"); - VersionObjectCreator versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberType, - DEFAULT_DELIMITER, generatedAtTimeProperty, versionOfProperty); + + VersionObjectCreator versionObjectCreator = createVersionObjectCreator( + inputModel, + "/" + ); String result = versionObjectCreator .transform(inputModel) @@ -158,20 +153,69 @@ void shouldMatchCountOfObjects(String fileName, String expectedId, LocalDateTime @Test void when_memberInfoExtractionFails_warningMessageIsLogged() { - Logger vocLogger = (Logger) LoggerFactory.getLogger(VersionObjectCreator.class); - ListAppender listAppender = new ListAppender<>(); - listAppender.start(); - vocLogger.addAppender(listAppender); + ListAppender listAppender = createListAppender(); + Model inputModel = RDFParser.fromString(""" + @prefix ex: . + @prefix time: . + @prefix xsd: . + + ex:John + ex:hasAge "30" ; + ex:hasName "John Smith" ; + ex:hasFriend ex:Jane ; + ex:created [ + a time:Instant ; + time:inXSDDateTimeStamp "2023-08-18T13:08:00+01:00"^^xsd:DateTime + ] . + """).lang(Lang.TTL).toModel(); + + VersionObjectCreator versionObjectCreator = createVersionObjectCreator( + inputModel, + "/" + ); + + versionObjectCreator.transform(inputModel); + + assertExpectedLogs(listAppender.list, STATEMENT_NOT_FOUND); + } + + @Test + void when_dateObservedExtractionFails_warningMessageIsLogged() { + ListAppender listAppender = createListAppender(); + Model inputModel = RDFParser.fromString(""" + @prefix time: . + @prefix ex: . + @prefix xsd: . + + ex:member + a ex:Something . + """).lang(Lang.TTL).toModel(); + + VersionObjectCreator versionObjectCreator = createVersionObjectCreator( + inputModel, + "/" + ); + versionObjectCreator.transform(inputModel); + + assertExpectedLogs(listAppender.list, DATE_OBSERVED_PROPERTY_COULD_NOT_BE_FOUND, CREATED_VERSION); + } + + @Test + void when_modelIsEmpty_warningMessagesAreLogged() { + ListAppender listAppender = createListAppender(); VersionObjectCreator versionObjectCreator = new VersionObjectCreator(new EmptyPropertyExtractor(), null, DEFAULT_DELIMITER, null, null); - versionObjectCreator.transform(initModel); + versionObjectCreator.transform(ModelFactory.createDefaultModel()); - List logsList = listAppender.list; - assertTrue( - logsList.get(0).getMessage().contains(VersionObjectCreator.DATE_OBSERVED_PROPERTY_COULD_NOT_BE_FOUND)); - assertEquals(Level.WARN, logsList.get(0).getLevel()); + assertExpectedLogs(listAppender.list, LINKED_DATA_MODEL_IS_EMPTY, DATE_OBSERVED_PROPERTY_COULD_NOT_BE_FOUND); + } + + private static void assertExpectedLogs(List logsList, String... expectedMessages) { + assertThat(logsList) + .extracting(ILoggingEvent::getMessage) + .containsExactlyInAnyOrder(expectedMessages); } private String getPartOfLocalDateTime(LocalDateTime time) { @@ -184,6 +228,29 @@ private String getJsonString(String resource) throws URISyntaxException, IOExcep return Files.readString(file.toPath()); } + private VersionObjectCreator createVersionObjectCreator(Model inputModel, String dateObservedPath) { + Resource memberType = inputModel.createResource("http://example.org/Something"); + PropertyExtractor dateObservedPropertyExtractor = PropertyPathExtractor.from(dateObservedPath); + Property generatedAtTimeProperty = inputModel.createProperty("http://www.w3.org/ns/prov#generatedAtTime"); + Property versionOfProperty = inputModel.createProperty("http://purl.org/dc/terms/isVersionOf"); + + return new VersionObjectCreator( + dateObservedPropertyExtractor, + memberType, + DEFAULT_DELIMITER, + generatedAtTimeProperty, + versionOfProperty + ); + } + + private ListAppender createListAppender() { + Logger vocLogger = (Logger) LoggerFactory.getLogger(VersionObjectCreator.class); + ListAppender listAppender = new ListAppender<>(); + listAppender.start(); + vocLogger.addAppender(listAppender); + return listAppender; + } + static class JsonLDFileArgumentsProvider implements ArgumentsProvider { @Override diff --git a/ldi-extensions/ldes-discoverer/pom.xml b/ldi-extensions/ldes-discoverer/pom.xml index da524ef79..7d36008d3 100644 --- a/ldi-extensions/ldes-discoverer/pom.xml +++ b/ldi-extensions/ldes-discoverer/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-extensions - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldes-discoverer @@ -34,6 +34,10 @@ tree-node-relations-fetcher + + org.springframework.boot + spring-boot-test + com.github.tomakehurst wiremock-jre8-standalone @@ -42,6 +46,14 @@ org.junit.jupiter junit-jupiter-api + + org.mockito + mockito-core + + + org.mockito + mockito-junit-jupiter + diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java index 1eb04cc3c..5f5a523ff 100644 --- a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutor.java @@ -1,8 +1,9 @@ package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.common; import be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.config.LdesDiscovererConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.config.RequestExecutorProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.services.DiscovererRequestExecutorSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import ldes.client.treenoderelationsfetcher.LdesStructureDiscoverer; import ldes.client.treenoderelationsfetcher.domain.valueobjects.LdesStructure; import org.slf4j.Logger; @@ -16,9 +17,9 @@ public class LdesDiscovererExecutor implements CommandLineRunner { private final LdesDiscovererConfig config; private final LdesStructureDiscoverer ldesStructureDiscoverer; - public LdesDiscovererExecutor(LdesDiscovererConfig config) { + public LdesDiscovererExecutor(LdesDiscovererConfig config, RequestExecutorProperties requestExecutorProperties) { this.config = config; - final RequestExecutor requestExecutor = new RequestExecutorFactory(false).createNoAuthExecutor(); + final RequestExecutor requestExecutor = new DiscovererRequestExecutorSupplier(requestExecutorProperties).createRequestExecutor(); ldesStructureDiscoverer = new LdesStructureDiscoverer(config.getUrl(), config.getSourceFormatAsLang(), requestExecutor); } diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java index a9aface78..73fdec98e 100644 --- a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/LdesDiscovererConfig.java @@ -12,7 +12,6 @@ public class LdesDiscovererConfig { private String url; private String sourceFormat; private String outputFormat; - public String getUrl() { if (url == null) { throw new IllegalArgumentException("Missing value for 'url'"); diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/RequestExecutorProperties.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/RequestExecutorProperties.java new file mode 100644 index 000000000..a29da63d1 --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/RequestExecutorProperties.java @@ -0,0 +1,81 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects.AuthenticationProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects.Headers; +import be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects.RateLimitProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects.RetryProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy; +import org.apache.http.Header; +import org.springframework.boot.ApplicationArguments; +import org.springframework.context.annotation.Configuration; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; + +@Configuration +public class RequestExecutorProperties { + private final AuthenticationProperties authProperties; + private final RateLimitProperties rateLimitProperties; + private final RetryProperties retryProperties; + private final Headers headers; + + public RequestExecutorProperties(ApplicationArguments arguments) { + this.authProperties = new AuthenticationProperties(arguments); + this.rateLimitProperties = new RateLimitProperties(arguments); + this.retryProperties = new RetryProperties(arguments); + this.headers = new Headers(arguments); + } + + public AuthStrategy getAuthStrategy() { + return authProperties.getAuthStrategy(); + } + + public String getApiKeyHeader() { + return authProperties.getApiKeyHeader(); + } + + public String getApiKey() { + return authProperties.getApiKey(); + } + + public String getClientId() { + return authProperties.getClientId(); + } + + public String getClientSecret() { + return authProperties.getClientSecret(); + } + + public String getTokenEndpoint() { + return authProperties.getTokenEndpoint(); + } + + public Optional getAuthScope() { + return authProperties.getAuthScope(); + } + + public boolean isRetryingDisabled() { + return retryProperties.isRetryingDisabled(); + } + + public int getRetryLimit() { + return retryProperties.getRetryLimit(); + } + + public List getRetryStatuses() { + return retryProperties.getRetryStatuses(); + } + + public Optional getRateLimit() { + return rateLimitProperties.getRateLimit(); + } + + public Duration getRateLimitPeriod() { + return rateLimitProperties.getRateLimitPeriod(); + } + + public List
getHeaders() { + return headers.getHeaders(); + } +} diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/services/DiscovererRequestExecutorSupplier.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/services/DiscovererRequestExecutorSupplier.java new file mode 100644 index 000000000..8c1012a2a --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/services/DiscovererRequestExecutorSupplier.java @@ -0,0 +1,70 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.services; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.config.RequestExecutorProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.ratelimiter.RateLimiterConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.retry.RetryConfig; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorDecorator; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; +import io.github.resilience4j.ratelimiter.RateLimiter; +import io.github.resilience4j.retry.Retry; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +public class DiscovererRequestExecutorSupplier { + private final RequestExecutorProperties properties; + private final RequestExecutorFactory requestExecutorFactory; + + public DiscovererRequestExecutorSupplier(RequestExecutorProperties properties) { + this(properties, new RequestExecutorFactory(false)); + } + + public DiscovererRequestExecutorSupplier(RequestExecutorProperties properties, RequestExecutorFactory requestExecutorFactory) { + this.properties = properties; + this.requestExecutorFactory = requestExecutorFactory; + } + + public RequestExecutor createRequestExecutor() { + final RequestExecutorDecorator decorator = RequestExecutorDecorator.decorate(getBaseRequestExecutor()); + getRateLimiter().ifPresent(decorator::with); + getRetry().ifPresent(decorator::with); + return decorator.get(); + } + + private Optional getRateLimiter() { + return properties.getRateLimit() + .map(rateLimit -> RateLimiterConfig.limitForPeriod(rateLimit, properties.getRateLimitPeriod())) + .map(RateLimiterConfig::getRateLimiter); + } + + private Optional getRetry() { + if (properties.isRetryingDisabled()) { + return Optional.empty(); + } + return Optional.of(RetryConfig.of(properties.getRetryLimit(), properties.getRetryStatuses()).getRetry()); + } + + private RequestExecutor getBaseRequestExecutor() { + final Collection
headers = properties.getHeaders(); + return switch (properties.getAuthStrategy()) { + case NO_AUTH -> requestExecutorFactory.createNoAuthExecutor(headers); + case API_KEY -> { + List
headersWithApiKey = new ArrayList<>(headers); + headersWithApiKey.add(new BasicHeader(properties.getApiKeyHeader(), properties.getApiKey())); + yield requestExecutorFactory.createNoAuthExecutor(headersWithApiKey); + } + case OAUTH2_CLIENT_CREDENTIALS -> requestExecutorFactory.createClientCredentialsExecutor( + headers, + properties.getClientId(), + properties.getClientSecret(), + properties.getTokenEndpoint(), + properties.getAuthScope().orElse(null)); + }; + } + +} diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/Arguments.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/Arguments.java new file mode 100644 index 000000000..5ecd75f02 --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/Arguments.java @@ -0,0 +1,41 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects; + +import org.springframework.boot.ApplicationArguments; + +import java.util.List; +import java.util.Optional; + +public class Arguments { + private final ApplicationArguments applicationArguments; + + public Arguments(ApplicationArguments applicationArguments) { + this.applicationArguments = applicationArguments; + } + + public boolean containsFlag(String key) { + return applicationArguments.containsOption(key); + } + + public List getArgumentValues(String key) { + final List values = applicationArguments.getOptionValues(key); + return values == null ? List.of() : values; + } + + public Optional getValue(String key) { + return getArgumentValues(key).stream().findFirst(); + } + + public String getRequiredValue(String key) { + return getValue(key).orElseThrow(() -> new IllegalArgumentException("Missing configuration for %s".formatted(key))); + } + + public Optional getInteger(String key) { + try { + return getArgumentValues(key).stream() + .findFirst() + .map(Integer::parseInt); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid configuration for %s: unable to parse number %s".formatted(key, e.getMessage()), e); + } + } +} diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/AuthenticationProperties.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/AuthenticationProperties.java new file mode 100644 index 000000000..7c4438e55 --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/AuthenticationProperties.java @@ -0,0 +1,55 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy; +import org.springframework.boot.ApplicationArguments; + +import java.util.Optional; + +public class AuthenticationProperties { + private static final String AUTH_TYPE = "auth-type"; + private static final String API_KEY = "api-key"; + private static final String API_KEY_HEADER = "api-key-header"; + private static final String DEFAULT_API_KEY_HEADER = "X-API-KEY"; + private static final String CLIENT_ID = "client-id"; + private static final String CLIENT_SECRET = "client-secret"; + private static final String TOKEN_ENDPOINT = "token-endpoint"; + private static final String SCOPE = "scope"; + private final Arguments arguments; + + public AuthenticationProperties(ApplicationArguments arguments) { + this.arguments = new Arguments(arguments); + } + + + public AuthStrategy getAuthStrategy() { + return arguments.getValue(AUTH_TYPE) + .map(AuthStrategy::from) + .map(authStrategy -> authStrategy.orElseThrow(() -> new UnsupportedOperationException("Requested authentication not available."))) + .orElse(AuthStrategy.NO_AUTH); + } + + public String getApiKeyHeader() { + return arguments.getValue(API_KEY_HEADER).orElse(DEFAULT_API_KEY_HEADER); + } + + public String getApiKey() { + return arguments.getRequiredValue(API_KEY); + } + + public String getClientId() { + return arguments.getRequiredValue(CLIENT_ID); + } + + public String getClientSecret() { + return arguments.getRequiredValue(CLIENT_SECRET); + } + + public String getTokenEndpoint() { + return arguments.getRequiredValue(TOKEN_ENDPOINT); + } + + public Optional getAuthScope() { + return arguments.getValue(SCOPE); + } + +} diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/Headers.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/Headers.java new file mode 100644 index 000000000..0a9f2fc29 --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/Headers.java @@ -0,0 +1,29 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects; + +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; +import org.springframework.boot.ApplicationArguments; + +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class Headers { + private static final String HEADER = "header"; + private static final Pattern headerPattern = Pattern.compile("([^:]+):\\s*(\\S.*?)"); + + private final Arguments arguments; + + public Headers(ApplicationArguments arguments) { + this.arguments = new Arguments(arguments); + } + + public List
getHeaders() { + return arguments.getArgumentValues(HEADER).stream() + .map(headerPattern::matcher) + .filter(Matcher::matches) + .map(matcher -> new BasicHeader(matcher.group(1), matcher.group(2))) + .collect(Collectors.toUnmodifiableList()); + } +} diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RateLimitProperties.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RateLimitProperties.java new file mode 100644 index 000000000..856dbe64a --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RateLimitProperties.java @@ -0,0 +1,32 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects; + +import org.springframework.boot.ApplicationArguments; + +import java.time.Duration; +import java.time.format.DateTimeParseException; +import java.util.Optional; + +public class RateLimitProperties { + private static final String RATE_LIMIT = "rate-limit"; + private static final String RATE_LIMIT_PERIOD = "rate-limit-period"; + private static final String DEFAULT_PERIOD = "PT1M"; + private final Arguments arguments; + + public RateLimitProperties(ApplicationArguments arguments) { + this.arguments = new Arguments(arguments); + } + + public Optional getRateLimit() { + return arguments.getInteger(RATE_LIMIT); + } + + public Duration getRateLimitPeriod() { + final String periodStringValue = arguments.getValue(RATE_LIMIT_PERIOD).orElse(DEFAULT_PERIOD); + try { + return Duration.parse(periodStringValue); + } catch (DateTimeParseException e) { + throw new IllegalArgumentException("Illegal value for %s:%s".formatted(RATE_LIMIT_PERIOD, periodStringValue), e); + } + } + +} diff --git a/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RetryProperties.java b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RetryProperties.java new file mode 100644 index 000000000..02023b3b8 --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RetryProperties.java @@ -0,0 +1,38 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects; + +import org.apache.commons.lang3.stream.Streams; +import org.springframework.boot.ApplicationArguments; + +import java.util.List; +import java.util.stream.Stream; + +public class RetryProperties { + private static final String DISABLE_RETRY = "disable-retry"; + private static final String RETRY_LIMIT = "retry-limit"; + private static final int DEFAULT_RETRY_LIMIT = 5; + private static final String RETRY_STATUSES = "retry-statuses"; + + private final Arguments arguments; + + public RetryProperties(ApplicationArguments arguments) { + this.arguments = new Arguments(arguments); + } + + public boolean isRetryingDisabled() { + return arguments.containsFlag(DISABLE_RETRY); + } + + public int getRetryLimit() { + return arguments.getInteger(RETRY_LIMIT).orElse(DEFAULT_RETRY_LIMIT); + } + + public List getRetryStatuses() { + return arguments.getValue(RETRY_STATUSES) + .map(string -> string.split(",")) + .map(Streams::of) + .orElseGet(Stream::of) + .map(String::trim) + .map(Integer::parseInt) + .toList(); + } +} diff --git a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/HeadersMatcher.java b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/HeadersMatcher.java new file mode 100644 index 000000000..cdc7e6647 --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/HeadersMatcher.java @@ -0,0 +1,52 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer; + +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; +import org.mockito.ArgumentMatcher; + +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +public class HeadersMatcher implements ArgumentMatcher> { + private final Collection expectedHeaders; + + public HeadersMatcher(Collection
expectedHeaders) { + this.expectedHeaders = expectedHeaders.stream() + .map(header -> new CustomHeader(header.getName(), header.getValue())) + .toList(); + } + + public static HeadersMatcher containsAllHeaders(Collection
elements) { + return new HeadersMatcher(elements); + } + + @Override + public boolean matches(List
headers) { + final var actualHeaders = headers.stream().map(header -> new CustomHeader(header.getName(), header.getValue())).toList(); + return expectedHeaders.containsAll(actualHeaders); + } + + public static class CustomHeader extends BasicHeader { + + public CustomHeader(String name, String value) { + super(name, value); + } + + public static CustomHeader fromHeader(Header header) { + return new CustomHeader(header.getName(), header.getValue()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Header header)) return false; + return Objects.equals(getName(), header.getName()) && Objects.equals(getValue(), header.getValue()); + } + + @Override + public int hashCode() { + return Objects.hash(getName(), getValue()); + } + } +} diff --git a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutorTest.java b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutorTest.java new file mode 100644 index 000000000..0f00a8cc0 --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/common/LdesDiscovererExecutorTest.java @@ -0,0 +1,134 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.common; + +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.github.tomakehurst.wiremock.matching.ContainsPattern; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.util.ResourceUtils; +import org.springframework.util.StopWatch; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.assertj.core.api.Assertions.assertThat; + +class LdesDiscovererExecutorTest { + private static final int WIREMOCK_PORT = 10101; + private static final String ENDPOINT = "/observations"; + + + @Nested + @WireMockTest(httpPort = WIREMOCK_PORT) + @SpringBootTest(args = {"--url=http://localhost:10101/observations"}) + class NoAuth { + @Autowired + private LdesDiscovererExecutor executor; + + @Test + void test_discoverWithoutAuth() throws IOException { + stubFor(get(ENDPOINT).willReturn(okForContentType("text/turtle", readDataModel()))); + + executor.run(); + + verify(getRequestedFor(urlEqualTo(ENDPOINT)).withoutHeader("X-API-KEY").withoutHeader("Authorization")); + } + + @Test + void test_discoverWithRetry() { + stubFor(get(ENDPOINT).willReturn(serverError())); + + executor.run(); + + verify(5, getRequestedFor(urlEqualTo(ENDPOINT))); + } + } + + @Nested + @WireMockTest(httpPort = WIREMOCK_PORT) + @SpringBootTest(args = {"--url=http://localhost:10101/observations", "--auth-type=API_KEY", "--api-key=my-secret-api-key"}) + class BasicAuth { + @Autowired + private LdesDiscovererExecutor executor; + + @Test + void name() throws IOException { + stubFor(get(ENDPOINT).willReturn(okForContentType("text/turtle", readDataModel()))); + + executor.run(); + + verify(getRequestedFor(urlEqualTo(ENDPOINT)).withHeader("X-API-KEY", new ContainsPattern("my-secret-api-key"))); + } + } + + @Nested + @WireMockTest(httpPort = WIREMOCK_PORT) + @SpringBootTest(args = {"--url=http://localhost:10101/observations", "--retry-limit=3", "--rate-limit=1", "--rate-limit-period=PT5S"}) + class RateLimit { + @Autowired + private LdesDiscovererExecutor executor; + + @Test + void test_discoverWithRateLimit() { + final StopWatch stopWatch = new StopWatch(); + stubFor(get(ENDPOINT).willReturn(serverError())); + + stopWatch.start(); + executor.run(); + stopWatch.stop(); + + assertThat(stopWatch.getTotalTimeSeconds()).isGreaterThanOrEqualTo(10); + verify(3, getRequestedFor(urlEqualTo(ENDPOINT))); + } + } + + @Nested + @WireMockTest(httpPort = WIREMOCK_PORT) + @SpringBootTest(args = {"--url=http://localhost:10101/observations", "--disable-retry"}) + class RetryDisabled { + @Autowired + private LdesDiscovererExecutor executor; + + @Test + void test_discoverWithoutRetry() { + stubFor(get(ENDPOINT).willReturn(serverError())); + + executor.run(); + + verify(1, getRequestedFor(urlEqualTo(ENDPOINT))); + } + } + + @Nested + @WireMockTest(httpPort = WIREMOCK_PORT) + @SpringBootTest(args = {"--url=http://localhost:10101/observations", "--source-format=application/n-quads", "--header=Connection: keep-alive", "--header=Cache-Control:no-cache", "--header=role: developer"}) + class HeadersProvided { + @Autowired + private LdesDiscovererExecutor executor; + + @Test + void test_discoverWithoutRetry() { + stubFor(get(ENDPOINT).willReturn(okForContentType("application/n-quads", ""))); + + executor.run(); + + verify( + getRequestedFor(urlEqualTo(ENDPOINT)) + .withHeader("Accept", new ContainsPattern("application/n-quads")) + .withHeader("Connection", new ContainsPattern("keep-alive")) + .withHeader("Cache-Control", new ContainsPattern("no-cache")) + .withHeader("role", new ContainsPattern("developer")) + ); + } + } + + + private String readDataModel() throws IOException { + File file = ResourceUtils.getFile("classpath:tree-relations/relation-1.ttl"); + return FileUtils.readFileToString(file, StandardCharsets.UTF_8); + } +} \ No newline at end of file diff --git a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/RequestExecutorPropertiesTest.java b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/RequestExecutorPropertiesTest.java new file mode 100644 index 000000000..ddc800b4e --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/config/RequestExecutorPropertiesTest.java @@ -0,0 +1,44 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.config; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.boot.ApplicationArguments; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class RequestExecutorPropertiesTest { + @Mock + private ApplicationArguments arguments; + @InjectMocks + private RequestExecutorProperties executorProperties; + + @Nested + class AuthProperties { + @Test + void given_NoAuthType_when_GetAuthStrategy_then_ReturnDefault() { + final AuthStrategy actual = executorProperties.getAuthStrategy(); + + assertThat(actual).isEqualTo(AuthStrategy.NO_AUTH); + } + + @Test + void given_ApiKeyAuthConfig_when_GetAuthStrategy_then_ReturnOAuth2Strategy() { + when(arguments.getOptionValues("auth-type")).thenReturn(List.of(AuthStrategy.API_KEY.name())); + + final AuthStrategy actual = executorProperties.getAuthStrategy(); + + assertThat(actual).isEqualTo(AuthStrategy.API_KEY); + } + + + } +} diff --git a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/services/DiscovererRequestExecutorSupplierTest.java b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/services/DiscovererRequestExecutorSupplierTest.java new file mode 100644 index 000000000..0014c1255 --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/services/DiscovererRequestExecutorSupplierTest.java @@ -0,0 +1,107 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.services; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.config.RequestExecutorProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.clientcredentials.ClientCredentialsRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.clientcredentials.ClientCredentialsRequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.noauth.DefaultRequestExecutor; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.AuthStrategy; +import org.apache.http.message.BasicHeader; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.boot.ApplicationArguments; + +import java.util.Collection; +import java.util.List; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.HeadersMatcher.containsAllHeaders; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class DiscovererRequestExecutorSupplierTest { + @Mock + private ApplicationArguments arguments; + @InjectMocks + private RequestExecutorProperties properties; + @Spy + private RequestExecutorFactory requestExecutorFactory = new RequestExecutorFactory(false); + private DiscovererRequestExecutorSupplier requestExecutorSupplier; + + @BeforeEach + void setUp() { + requestExecutorSupplier = new DiscovererRequestExecutorSupplier(properties, requestExecutorFactory); + } + + @Test + void given_DefaultConfig_when_CreateExecutor_then_ReturnDefaultExecutorWithAdditionalHeaders() { + when(arguments.getOptionValues(anyString())).thenReturn(null); + + final RequestExecutor actual = requestExecutorSupplier.createRequestExecutor(); + + assertThat(actual) + .isInstanceOf(RequestExecutor.class) + .isNotInstanceOfAny(DefaultRequestExecutor.class, ClientCredentialsRequest.class); + verify(requestExecutorFactory).createNoAuthExecutor(argThat(containsAllHeaders(List.of()))); + } + + @Test + void given_ApiKeyAuthConfig_when_CreateExecutor_then_ReturnDefaultExecutorWithAdditionalHeaders() { + when(arguments.getOptionValues(anyString())).thenReturn(null); + when(arguments.getOptionValues("auth-type")).thenReturn(List.of(AuthStrategy.API_KEY.name())); + when(arguments.getOptionValues("api-key")).thenReturn(List.of("my-secret-api-key")); + + final RequestExecutor actual = requestExecutorSupplier.createRequestExecutor(); + + assertThat(actual) + .isInstanceOf(RequestExecutor.class) + .isNotInstanceOfAny(DefaultRequestExecutor.class, ClientCredentialsRequest.class) ; + verify(arguments).getOptionValues("auth-type"); + verify(requestExecutorFactory).createNoAuthExecutor(argThat(containsAllHeaders(List.of(new BasicHeader("X-API-KEY", "my-secret-api-key"))))); + } + + @Test + void given_NoDecorationConfigIsProvided_when_CreateExecutor_then_ReturnBaseExecutor() { + when(arguments.containsOption("disable-retry")).thenReturn(true); + + final RequestExecutor actual = requestExecutorSupplier.createRequestExecutor(); + + assertThat(actual).isInstanceOf(DefaultRequestExecutor.class); + verify(requestExecutorFactory).createNoAuthExecutor(argThat(containsAllHeaders(List.of()))); + } + + @Test + void given_ClientCredentialsAuthConfig_when_CreateExecutor_then_ReturnDefaultExecutorWithAdditionalHeaders() { + when(arguments.containsOption("disable-retry")).thenReturn(true); + when(arguments.getOptionValues("header")).thenReturn(null); + when(arguments.getOptionValues("auth-type")).thenReturn(List.of(AuthStrategy.OAUTH2_CLIENT_CREDENTIALS.name())); + when(arguments.getOptionValues("client-id")).thenReturn(List.of("my-client-id")); + when(arguments.getOptionValues("client-secret")).thenReturn(List.of("my-client-secret")); + when(arguments.getOptionValues("token-endpoint")).thenReturn(List.of("my-token-endpoint")); + + final RequestExecutor actual = requestExecutorSupplier.createRequestExecutor(); + + assertThat(actual).isInstanceOf(ClientCredentialsRequestExecutor.class); + verify(requestExecutorFactory).createClientCredentialsExecutor(argThat(Collection::isEmpty), anyString(), anyString(), anyString(), isNull()); + } + + @Test + void given_RateLimitConfig_when_CreateExecutor_then_ReturnLambda() { + when(arguments.getOptionValues(anyString())).thenReturn(null); + when(arguments.getOptionValues("rate-limit")).thenReturn(List.of("500")); + + final RequestExecutor actual = requestExecutorSupplier.createRequestExecutor(); + + assertThat(actual) + .isInstanceOf(RequestExecutor.class) + .isNotInstanceOfAny(DefaultRequestExecutor.class, ClientCredentialsRequest.class) ; + verify(arguments).getOptionValues("rate-limit"); + } +} \ No newline at end of file diff --git a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/ArgumentsTest.java b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/ArgumentsTest.java new file mode 100644 index 000000000..9b0ef32e7 --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/ArgumentsTest.java @@ -0,0 +1,121 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.boot.ApplicationArguments; +import java.util.List; +import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ArgumentsTest { + @Mock + private ApplicationArguments applicationArguments; + @InjectMocks + private Arguments arguments; + + + @Test + void containsFlag_WhenFlagExists_ReturnsTrue() { + final String flagKey = "enable-flag"; + when(applicationArguments.containsOption(flagKey)).thenReturn(true); + + boolean containsFlag = arguments.containsFlag(flagKey); + + assertThat(containsFlag).isTrue(); + } + + @Test + void containsFlag_WhenFlagDoesNotExist_ReturnsFalse() { + boolean containsFlag = arguments.containsFlag("test"); + + assertThat(containsFlag).isFalse(); + } + + @Test + void getArgumentValues_WhenValuesExist_ReturnsValuesList() { + final String headerKey = "header"; + final List headers = List.of("Content-Type: application/json", "Accept: application/json"); + when(applicationArguments.getOptionValues(headerKey)).thenReturn(headers); + + List argumentValues = arguments.getArgumentValues(headerKey); + + assertThat(argumentValues).containsExactlyInAnyOrderElementsOf(headers); + } + + @Test + void getArgumentValues_WhenValuesDoNotExist_ReturnsEmptyList() { + List argumentValues = arguments.getArgumentValues("not-provided"); + + assertThat(argumentValues).isEmpty(); + } + + @Test + void getValue_WhenValueExists_ReturnsOptionalWithValue() { + final String key = "key"; + final String value = "value"; + when(applicationArguments.getOptionValues(key)).thenReturn(List.of("value")); + + Optional actual = arguments.getValue(key); + + assertThat(actual).isPresent().contains(value); + } + + @Test + void getValue_WhenValueDoesNotExist_ReturnsEmptyOptional() { + Optional value = arguments.getValue("empty-value"); + + assertThat(value).isEmpty(); + } + + @Test + void getRequiredValue_WhenValueExists_ReturnsValue() { + final String key = "required-key"; + final String value = "value"; + when(applicationArguments.getOptionValues(key)).thenReturn(List.of(value)); + + String requiredValue = arguments.getRequiredValue(key); + + assertThat(requiredValue).isEqualTo(value); + } + + @Test + void getRequiredValue_WhenValueDoesNotExist_ThrowsIllegalArgumentException() { + final String key = "my-absent-key"; + assertThatThrownBy(() -> arguments.getRequiredValue(key)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing configuration for %s", key); + } + + @Test + void getInteger_WhenValueExists_ReturnsOptionalWithValue() { + final String key = "max-age"; + when(applicationArguments.getOptionValues(key)).thenReturn(List.of("123")); + + Optional integerValue = arguments.getInteger(key); + + assertThat(integerValue).contains(123); + } + + @Test + void getInteger_WhenValueDoesNotExist_ReturnsEmptyOptional() { + Optional integerValue = arguments.getInteger("max-age"); + + assertThat(integerValue).isEmpty(); + } + + @Test + void getInteger_WhenValueIsInvalid_ThrowException() { + final String key = "invalid-max-age"; + when(applicationArguments.getOptionValues(key)).thenReturn(List.of("abc")); + + assertThatThrownBy(() -> arguments.getInteger(key)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid configuration for invalid-max-age: unable to parse number For input string: \"abc\""); + } +} diff --git a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/HeadersTest.java b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/HeadersTest.java new file mode 100644 index 000000000..ac477823c --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/HeadersTest.java @@ -0,0 +1,72 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.HeadersMatcher; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.boot.ApplicationArguments; + +import java.util.Collection; +import java.util.List; + +import static be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.HeadersMatcher.containsAllHeaders; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class HeadersTest { + private static final String HEADER_KEY = "header"; + @Mock + private ApplicationArguments arguments; + @InjectMocks + private Headers headers; + + @Test + void given_OneHeaderProvided_when_GetHeaders_then_ReturnSingletonList() { + final Collection expectedHeaders = List.of(new HeadersMatcher.CustomHeader("Content-Type", "application/json")); + when(arguments.getOptionValues(HEADER_KEY)).thenReturn(List.of("Content-Type: application/json")); + + final List
actual = headers.getHeaders(); + + assertThat(actual) + .map(HeadersMatcher.CustomHeader::fromHeader) + .containsExactlyElementsOf(expectedHeaders); + } + + @Test + void given_TwoHeadersProvided_when_GetHeaders_then_ReturnListOfTwoHeaders() { + final Collection expectedHeaders = List.of( + new HeadersMatcher.CustomHeader("Content-Type", "application/json"), + new HeadersMatcher.CustomHeader("Accept", "application/rdf+protobuf") + ); + when(arguments.getOptionValues(HEADER_KEY)).thenReturn(List.of("Content-Type: application/json", "Accept: application/rdf+protobuf")); + + final List
actual = headers.getHeaders(); + + assertThat(actual) + .map(HeadersMatcher.CustomHeader::fromHeader) + .containsExactlyElementsOf(expectedHeaders); + } + + @Test + void given_NullHeaders_when_GetHeaders_then_ReturnEmptyList() { + final List
actual = headers.getHeaders(); + + assertThat(actual).isEmpty(); + } + + @Test + void given_InvalidHeaders_when_GetHeaders_then_ReturnEmptyList() { + when(arguments.getOptionValues(HEADER_KEY)) + .thenReturn(List.of("", "Content-Type=text/turtle", ":", "Header-Key:", ":Header-Value")); + + final List
actual = headers.getHeaders(); + + assertThat(actual).isEmpty(); + } +} \ No newline at end of file diff --git a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RateLimitPropertiesTest.java b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RateLimitPropertiesTest.java new file mode 100644 index 000000000..e19ff6b5c --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RateLimitPropertiesTest.java @@ -0,0 +1,50 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.boot.ApplicationArguments; + +import java.time.Duration; +import java.time.format.DateTimeParseException; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class RateLimitPropertiesTest { + @Mock + private ApplicationArguments arguments; + @InjectMocks + private RateLimitProperties properties; + + @Test + void test_Defaults() { + assertThat(properties.getRateLimit()).isEmpty(); + assertThat(properties.getRateLimitPeriod()).isEqualTo(Duration.ofMinutes(1)); + } + + @Test + void test_CustomConfig() { + when(arguments.getOptionValues("rate-limit")).thenReturn(List.of("250")); + when(arguments.getOptionValues("rate-limit-period")).thenReturn(List.of("PT1S")); + + assertThat(properties.getRateLimit()).contains(250); + assertThat(properties.getRateLimitPeriod()).isEqualTo(Duration.ofSeconds(1)); + } + + @Test + void test_InvalidPeriod() { + when(arguments.getOptionValues("rate-limit-period")).thenReturn(List.of("P1S")); + + assertThatThrownBy(() -> properties.getRateLimitPeriod()) + .isInstanceOf(IllegalArgumentException.class) + .hasCauseInstanceOf(DateTimeParseException.class) + .hasMessage("Illegal value for rate-limit-period:P1S"); + + } +} \ No newline at end of file diff --git a/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RetryPropertiesTest.java b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RetryPropertiesTest.java new file mode 100644 index 000000000..2c50cd2fd --- /dev/null +++ b/ldi-extensions/ldes-discoverer/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/discoverer/valueobjects/RetryPropertiesTest.java @@ -0,0 +1,39 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldi.discoverer.valueobjects; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.boot.ApplicationArguments; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class RetryPropertiesTest { + @Mock + private ApplicationArguments arguments; + @InjectMocks + private RetryProperties properties; + + @Test + void test_Defaults() { + assertThat(properties.isRetryingDisabled()).isFalse(); + assertThat(properties.getRetryLimit()).isEqualTo(5); + assertThat(properties.getRetryStatuses()).isEmpty(); + } + + @Test + void test_CustomConfig() { + when(arguments.containsOption("disable-retry")).thenReturn(true); + when(arguments.getOptionValues("retry-limit")).thenReturn(List.of("10")); + when(arguments.getOptionValues("retry-statuses")).thenReturn(List.of("400,404,500")); + + assertThat(properties.isRetryingDisabled()).isTrue(); + assertThat(properties.getRetryLimit()).isEqualTo(10); + assertThat(properties.getRetryStatuses()).isEqualTo(List.of(400, 404, 500)); + } +} \ No newline at end of file diff --git a/ldi-extensions/pom.xml b/ldi-extensions/pom.xml index 148ffb99f..efc8b5a19 100644 --- a/ldi-extensions/pom.xml +++ b/ldi-extensions/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT be.vlaanderen.informatievlaanderen.ldes.ldi @@ -34,6 +34,11 @@ ${spring-test.version} test + + org.springframework.boot + spring-boot-test + ${spring-boot.version} + diff --git a/ldi-nifi/ldi-nifi-common/pom.xml b/ldi-nifi/ldi-nifi-common/pom.xml index 1d6890948..de7a3aa1a 100644 --- a/ldi-nifi/ldi-nifi-common/pom.xml +++ b/ldi-nifi/ldi-nifi-common/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml index 667b56b7e..4f64f90b2 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT archive-file-in diff --git a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml index c7bd29879..2486dd0fd 100644 --- a/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/archive-file-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT archive-file-out diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml index a6f8cfcfa..c223601d7 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml index 46f7626d2..7146432d8 100644 --- a/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/geojson-to-wkt-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT geojson-to-wkt-processor diff --git a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml index 68fac0e64..9e30e1080 100644 --- a/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/json-to-ld-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT json-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml index 91b3e3f23..6c839f4c1 100644 --- a/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ldes-client-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml new file mode 100644 index 000000000..3e8621de1 --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/pom.xml @@ -0,0 +1,91 @@ + + + 4.0.0 + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + ldi-nifi-processors + 2.5.0-SNAPSHOT + + + ldi-processors-bundle + jar + + + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + create-version-object-processor + ${project.version} + nar + + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + geojson-to-wkt-processor + ${project.version} + nar + + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + json-to-ld-processor + ${project.version} + nar + + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + ldes-client-processor + ${project.version} + nar + + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + ngsiv2-to-ld-processor + ${project.version} + nar + + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + rdf4j-repository-materialisation-processor + ${project.version} + nar + + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + sparql-interactions-processor + ${project.version} + nar + + + be.vlaanderen.informatievlaanderen.ldes.ldi.nifi + version-materialisation-processor + ${project.version} + nar + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven-assembly-plugin.version} + + + src/main/resources/assembly.xml + + + + + make-assembly + package + + single + + + + + + + + + \ No newline at end of file diff --git a/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/src/main/resources/assembly.xml b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/src/main/resources/assembly.xml new file mode 100644 index 000000000..def21e2fe --- /dev/null +++ b/ldi-nifi/ldi-nifi-processors/ldi-processors-bundle/src/main/resources/assembly.xml @@ -0,0 +1,22 @@ + + + nar-bundle + + + jar + + + + + / + + *:nar + + false + + + + diff --git a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml index 37aacee46..ef83bdd23 100644 --- a/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/ngsiv2-to-ld-processor/pom.xml @@ -7,7 +7,7 @@ ldi-nifi-processors be.vlaanderen.informatievlaanderen.ldes.ldi.nifi - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ngsiv2-to-ld-processor diff --git a/ldi-nifi/ldi-nifi-processors/pom.xml b/ldi-nifi/ldi-nifi-processors/pom.xml index 60da9d89e..69b151d03 100644 --- a/ldi-nifi/ldi-nifi-processors/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-nifi - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 @@ -22,6 +22,7 @@ json-to-ld-processor archive-file-out archive-file-in + ldi-processors-bundle diff --git a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml index 61339c851..3e96fa47a 100644 --- a/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/rdf4j-repository-materialisation-processor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT rdf4j-repository-materialisation-processor diff --git a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml index f7e0651f3..5eda381d7 100644 --- a/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/sparql-interactions-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml index d33a379ca..867781f6c 100644 --- a/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml +++ b/ldi-nifi/ldi-nifi-processors/version-materialisation-processor/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi.nifi ldi-nifi-processors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-nifi/pom.xml b/ldi-nifi/pom.xml index 9a7c496fb..9e0b00cc5 100644 --- a/ldi-nifi/pom.xml +++ b/ldi-nifi/pom.xml @@ -3,7 +3,7 @@ linked-data-interactions be.vlaanderen.informatievlaanderen.ldes - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 pom diff --git a/ldi-orchestrator/ldio-application/pom.xml b/ldi-orchestrator/ldio-application/pom.xml index ae6ef7502..36e152eb3 100644 --- a/ldi-orchestrator/ldio-application/pom.xml +++ b/ldi-orchestrator/ldio-application/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java index a882fa3be..0d14d6b7a 100644 --- a/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java +++ b/ldi-orchestrator/ldio-application/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/modules/DummyIn.java @@ -3,6 +3,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import org.springframework.context.ApplicationEventPublisher; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.START; @@ -11,7 +12,7 @@ public class DummyIn extends LdioInput { private int counter = 0; public DummyIn(ComponentExecutor executor, LdiAdapter adapter, ApplicationEventPublisher applicationEventPublisher) { - super("DummyIn", "test", executor, adapter, null, applicationEventPublisher); + super(executor, adapter, LdioObserver.register("DummyIn", "test", null), applicationEventPublisher); this.updateStatus(START); } diff --git a/ldi-orchestrator/ldio-common/pom.xml b/ldi-orchestrator/ldio-common/pom.xml index c3de76ca7..1b4076862 100644 --- a/ldi-orchestrator/ldio-common/pom.xml +++ b/ldi-orchestrator/ldio-common/pom.xml @@ -5,7 +5,7 @@ ldi-orchestrator be.vlaanderen.informatievlaanderen.ldes.ldi - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 @@ -28,8 +28,7 @@ io.micrometer micrometer-core - 1.11.5 - compile + ${micrometer-prometheus.version} org.springframework.boot @@ -39,6 +38,10 @@ org.assertj assertj-core + + org.mockito + mockito-core + \ No newline at end of file diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java index 1bb205529..92f590eb4 100644 --- a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioInput.java @@ -3,19 +3,16 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiComponent; -import be.vlaanderen.informatievlaanderen.ldes.ldio.config.ObserveConfiguration; import be.vlaanderen.informatievlaanderen.ldes.ldio.events.PipelineStatusEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger; -import io.micrometer.core.instrument.Metrics; -import io.micrometer.observation.Observation; -import io.micrometer.observation.ObservationRegistry; import org.apache.jena.rdf.model.Model; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; +import java.util.function.Supplier; + import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatus.*; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.PipelineStatusTrigger.START; import static be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.StatusChangeSource.MANUAL; @@ -28,17 +25,12 @@ * onto the LDIO pipeline */ public abstract class LdioInput implements LdiComponent { - - protected final String componentName; - protected final String pipelineName; private final ComponentExecutor executor; private final LdiAdapter adapter; + private final LdioObserver ldioObserver; private final ApplicationEventPublisher applicationEventPublisher; private final Logger log = LoggerFactory.getLogger(this.getClass()); - private final ObservationRegistry observationRegistry; - private static final String LDIO_DATA_IN = "ldio_data_in"; - private static final String LDIO_COMPONENT_NAME = "ldio_type"; private PipelineStatus pipelineStatus; /** @@ -49,16 +41,12 @@ public abstract class LdioInput implements LdiComponent { * @param adapter Instance of the LDI Adapter. Facilitates transforming the input * data to a linked data model (RDF). */ - protected LdioInput(String componentName, String pipelineName, ComponentExecutor executor, LdiAdapter adapter, - ObservationRegistry observationRegistry, ApplicationEventPublisher applicationEventPublisher) { - this.componentName = componentName; - this.pipelineName = pipelineName; + protected LdioInput(ComponentExecutor executor, LdiAdapter adapter, LdioObserver ldioObserver, ApplicationEventPublisher applicationEventPublisher) { this.executor = executor; this.adapter = adapter; - this.observationRegistry = observationRegistry; + this.ldioObserver = ldioObserver; this.applicationEventPublisher = applicationEventPublisher; this.pipelineStatus = INIT; - Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).increment(0); } public void processInput(String content, String contentType) { @@ -66,31 +54,13 @@ public void processInput(String content, String contentType) { } public void processInput(LdiAdapter.Content content) { - Observation.createNotStarted(this.componentName, observationRegistry) - .observe(() -> { - try { - adapter.apply(content).forEach(this::processModel); - } catch (Exception e) { - final var errorLocation = this.pipelineName + ":processInput"; - log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, errorLocation, e.getMessage()); - log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, errorLocation, - "Processing below message.%n%n###mime###%n%s%n###content###%n%s".formatted(content.mimeType(), content.content())); - throw e; - } - }); + final Supplier failedContentLogSupplier = () -> "Processing below message.%n%n###mime###%n%s%n###content###%n%s".formatted(content.mimeType(), content.content()); + ldioObserver.observe(() -> adapter.apply(content).forEach(this::processModel), "processInput", failedContentLogSupplier); } protected void processModel(Model model) { - Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).increment(); - Observation.createNotStarted(this.componentName, observationRegistry) - .observe(() -> { - try { - executor.transformLinkedData(model); - } catch (Exception e) { - log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, this.pipelineName + ":processModel", e.getMessage()); - throw e; - } - }); + ldioObserver.increment(); + ldioObserver.observe(() -> executor.transformLinkedData(model), "processModel"); } public abstract void shutdown(); @@ -109,10 +79,10 @@ public PipelineStatus updateStatus(PipelineStatusTrigger trigger) { } } case STOP -> this.pipelineStatus = STOPPED; - default -> log.warn("Unhandled status update on pipeline: {} for status: {}", pipelineName, pipelineStatus); + default -> log.warn("Unhandled status update on pipeline: {} for status: {}", ldioObserver.getPipelineName(), pipelineStatus); } - applicationEventPublisher.publishEvent(new PipelineStatusEvent(pipelineName, this.pipelineStatus, MANUAL)); + applicationEventPublisher.publishEvent(new PipelineStatusEvent(ldioObserver.getPipelineName(), this.pipelineStatus, MANUAL)); return this.pipelineStatus; } diff --git a/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioObserver.java b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioObserver.java new file mode 100644 index 000000000..d90252910 --- /dev/null +++ b/ldi-orchestrator/ldio-common/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioObserver.java @@ -0,0 +1,61 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.types; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.ObserveConfiguration; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.function.Supplier; + +import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PipelineConfig.PIPELINE_NAME; + +public class LdioObserver { + private static final String LDIO_DATA_IN = "ldio_data_in"; + private static final String LDIO_COMPONENT_NAME = "ldio_type"; + private static final Logger log = LoggerFactory.getLogger(LdioObserver.class); + private final String componentName; + private final String pipelineName; + private final ObservationRegistry observationRegistry; + + private LdioObserver(String componentName, String pipelineName, ObservationRegistry observationRegistry) { + this.componentName = componentName; + this.pipelineName = pipelineName; + this.observationRegistry = observationRegistry; + } + + @SafeVarargs + public final void observe(Runnable observable, String location, Supplier... additionalLoggingContent) { + final String errorLocation = pipelineName + ":" + location; + Observation.createNotStarted(this.componentName, observationRegistry) + .observe(() -> { + try { + observable.run(); + } catch (Exception e) { + log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, errorLocation, e.getMessage()); + Arrays.stream(additionalLoggingContent) + .forEach(content -> + log.atError().log(ObserveConfiguration.ERROR_TEMPLATE, errorLocation, content.get()) + ); + throw e; + } + }); + } + + public void increment() { + Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).increment(); + } + + public static LdioObserver register(String componentName, String pipelineName, ObservationRegistry observationRegistry) { + Metrics.counter(LDIO_DATA_IN, PIPELINE_NAME, pipelineName, LDIO_COMPONENT_NAME, componentName).increment(0); + return new LdioObserver(componentName, pipelineName, observationRegistry); + } + + public String getPipelineName() { + return pipelineName; + } + + +} diff --git a/ldi-orchestrator/ldio-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioObserverTest.java b/ldi-orchestrator/ldio-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioObserverTest.java new file mode 100644 index 000000000..485a9e889 --- /dev/null +++ b/ldi-orchestrator/ldio-common/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/types/LdioObserverTest.java @@ -0,0 +1,43 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.types; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.function.Supplier; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.*; + +class LdioObserverTest { + private static final String COMPONENT_NAME = "Ldio:ComponentName"; + private static final String PIPELINE_NAME = "pipeline-name"; + private LdioObserver ldioObserver; + + @BeforeEach + void setUp() { + ldioObserver = LdioObserver.register(COMPONENT_NAME, PIPELINE_NAME, null); + } + + @Test + void when_ObservableDoesNotThrowException_then_DoNotLog() { + final Supplier additionalLogSupplier = mock(); + final Runnable observable = () -> {}; + + ldioObserver.observe(observable, "test", additionalLogSupplier); + + verifyNoInteractions(additionalLogSupplier); + } + + @Test + void when_ObservableDoesThrowsException_then_Log() { + final Supplier additionalLogSupplier = mock(); + final Runnable observable = () -> { + throw new RuntimeException(); + }; + + assertThatThrownBy(() -> ldioObserver.observe(observable, "test", additionalLogSupplier)) + .isInstanceOf(RuntimeException.class); + + verify(additionalLogSupplier).get(); + } +} \ No newline at end of file diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml index 04bf8506a..8f9a33a52 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldio-amqp diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java index 1ce02e87b..403c09ac6 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioAmqpIn.java @@ -2,11 +2,11 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; -import be.vlaanderen.informatievlaanderen.ldes.ldio.config.JmsConfig; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioAmqpInRegistrator; import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.InvalidAmqpMessageException; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; -import io.micrometer.observation.ObservationRegistry; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.LdioAmpqInProperties; import jakarta.jms.JMSException; import jakarta.jms.Message; import jakarta.jms.MessageListener; @@ -21,30 +21,29 @@ public class LdioAmqpIn extends LdioInput implements MessageListener { public static final String NAME = "Ldio:AmqpIn"; private static final Logger log = LoggerFactory.getLogger(LdioAmqpIn.class); private final LdioAmqpInRegistrator ldioAmqpInRegistrator; + private final LdioAmpqInProperties properties; private final String listenerId; - private final String defaultContentType; /** * Creates a LdiInput with its Component Executor and LDI Adapter * - * @param pipelineName Unique identifier for the pipeline. * @param executor Instance of the Component Executor. Allows the LDI Input to pass * data on the pipeline * @param adapter Instance of the LDI Adapter. Facilitates transforming the input * data to a linked data model (RDF). - * @param jmsConfig Configuration class containing the necessary info to spin up a listener + * @param ldioObserver Instance of the LDIO Observer, for observation, logging and monitoring reaons * @param jmsInRegistrator Global service to maintain JMS listeners. - * @param observationRegistry + * @param properties Configuration properties */ - public LdioAmqpIn(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, - String defaultContentType, JmsConfig jmsConfig, LdioAmqpInRegistrator jmsInRegistrator, - ObservationRegistry observationRegistry, ApplicationEventPublisher applicationEventPublisher) { - super(NAME, pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); - this.defaultContentType = defaultContentType; - SimpleJmsListenerEndpoint endpoint = listenerEndpoint(jmsConfig.queue()); + public LdioAmqpIn(ComponentExecutor executor, LdiAdapter adapter, LdioObserver ldioObserver, + LdioAmqpInRegistrator jmsInRegistrator, LdioAmpqInProperties properties, + ApplicationEventPublisher applicationEventPublisher) { + super(executor, adapter, ldioObserver, applicationEventPublisher); + this.properties = properties; + SimpleJmsListenerEndpoint endpoint = listenerEndpoint(properties.jmsConfig().queue()); ldioAmqpInRegistrator = jmsInRegistrator; listenerId = endpoint.getId(); - jmsInRegistrator.registerListener(jmsConfig, endpoint); + jmsInRegistrator.registerListener(properties.jmsConfig(), endpoint); this.start(); } @@ -53,7 +52,7 @@ public void onMessage(Message message) { final LdiAdapter.Content content; try { String contentTypeProperty = message.getStringProperty(CONTENT_TYPE_HEADER); - String contentType = contentTypeProperty != null ? contentTypeProperty : defaultContentType; + String contentType = contentTypeProperty != null ? contentTypeProperty : properties.defaultContentType(); content = LdiAdapter.Content.of(message.getBody(String.class), contentType); } catch (JMSException e) { throw new InvalidAmqpMessageException(e); @@ -64,7 +63,7 @@ public void onMessage(Message message) { private SimpleJmsListenerEndpoint listenerEndpoint(String queue) { SimpleJmsListenerEndpoint endpoint = new SimpleJmsListenerEndpoint(); - endpoint.setId(pipelineName); + endpoint.setId(properties.pipelineName()); endpoint.setDestination(queue); endpoint.setMessageListener(this); return endpoint; diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfig.java index febcc959b..b979fc3a9 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioAmqpInAutoConfig.java @@ -5,7 +5,9 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioAmqpIn; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.LdioAmpqInProperties; import io.micrometer.observation.ObservationRegistry; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFLanguages; @@ -38,14 +40,13 @@ public LdioJmsInConfigurator(LdioAmqpInRegistrator ldioAmqpInRegistrator, Observ @Override public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, ApplicationEventPublisher applicationEventPublisher, ComponentProperties config) { - String pipelineName = config.getPipelineName(); - - String remoteUrl = new RemoteUrlExtractor(config).getRemoteUrl(); - JmsConfig jmsConfig = new JmsConfig(config.getProperty(USERNAME), config.getProperty(PASSWORD), + final String pipelineName = config.getPipelineName(); + final String remoteUrl = new RemoteUrlExtractor(config).getRemoteUrl(); + final JmsConfig jmsConfig = new JmsConfig(config.getProperty(USERNAME), config.getProperty(PASSWORD), remoteUrl, config.getProperty(QUEUE)); - - return new LdioAmqpIn(pipelineName, executor, adapter, getContentType(config), jmsConfig, ldioAmqpInRegistrator, - observationRegistry, applicationEventPublisher); + final LdioAmpqInProperties properties = new LdioAmpqInProperties(pipelineName, getContentType(config), jmsConfig); + final LdioObserver ldioObserver = LdioObserver.register(LdioAmqpIn.NAME, pipelineName, observationRegistry); + return new LdioAmqpIn(executor, adapter, ldioObserver, ldioAmqpInRegistrator, properties, applicationEventPublisher); } @Override diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/LdioAmpqInProperties.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/LdioAmpqInProperties.java new file mode 100644 index 000000000..4f9fd19d3 --- /dev/null +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/valueobjects/LdioAmpqInProperties.java @@ -0,0 +1,6 @@ +package be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects; + +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.JmsConfig; + +public record LdioAmpqInProperties(String pipelineName, String defaultContentType, JmsConfig jmsConfig) { +} diff --git a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java index 05c2e3d02..bc64a92d5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java +++ b/ldi-orchestrator/ldio-connectors/ldio-amqp/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/AmqpInIntegrationTestSteps.java @@ -8,6 +8,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.config.OrchestratorConfig; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; +import io.cucumber.java.After; import io.cucumber.java.ParameterType; import io.cucumber.java.en.And; import io.cucumber.java.en.Then; @@ -50,6 +51,13 @@ public class AmqpInIntegrationTestSteps extends AmqpIntegrationTest { private final TestContext testContext = TestContextContainer.getTestContext(); private MessageProducer producer; + @After + public void tearDown() { + if(ldioInput != null) { + ldioInput.shutdown(); + } + } + @And("I create a message producer") public void iCreateAMessageProducer() throws JMSException { producer = testContext.session.createProducer(testContext.queue); @@ -105,13 +113,13 @@ public void theListenerWillWaitForTheMessage(int i) { } @Then("Wait for a grace period") - public void waitForGracePeriod() throws InterruptedException { + public void waitForGracePeriod() { Awaitility.waitAtMost(Duration.of(500, ChronoUnit.MILLIS)); } @And("The result value will contain the model") public void theResultValueWillContainTheModel() { - Model resultModel = RDFParser.fromString(adapterResult.get(0).content()).lang(contentTypeToLang(contentType)) + Model resultModel = RDFParser.fromString(adapterResult.getFirst().content()).lang(contentTypeToLang(contentType)) .build().toModel(); assertTrue(resultModel.isIsomorphicWith(inputModel)); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml index aef2490e5..a1d64231c 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldio-archive-file-in diff --git a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java index 6ce6e259d..67c9e3cc7 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-archive-file-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioArchiveFileIn.java @@ -2,6 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import io.micrometer.observation.ObservationRegistry; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; @@ -18,7 +19,7 @@ public class LdioArchiveFileIn extends LdioInput { private boolean paused = false; public LdioArchiveFileIn(String pipelineName, ComponentExecutor executor, ObservationRegistry observationRegistry, ApplicationEventPublisher applicationEventPublisher, ArchiveFileCrawler crawler, Lang source) { - super(NAME, pipelineName, executor, null, observationRegistry, applicationEventPublisher); + super(executor, null, LdioObserver.register(NAME, pipelineName, observationRegistry), applicationEventPublisher); this.archiveFileCrawler = crawler; this.sourceFormat = source; start(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml index 62c774e50..023614753 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT diff --git a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml index 8b858844a..ed35cd6c5 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-console-out/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml index 50d954b7e..0c2a65a7e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-file-out/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldio-file-out diff --git a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml index f7745cf8c..179e510db 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-geojson-to-wkt/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldio-geojson-to-wkt diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml index 35b8af2ed..d60df9905 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-enricher/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml index 5d7d3f716..b10559ba9 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldio-http-in-poller diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java index 5555f8889..a9daf1740 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPoller.java @@ -1,17 +1,15 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.RequestExecutor; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.GetRequest; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Request; -import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeaders; import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.Response; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; -import be.vlaanderen.informatievlaanderen.ldes.ldio.config.PollingInterval; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInputPollerProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.MissingHeaderException; import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.UnsuccesfulPollingException; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; -import io.micrometer.observation.ObservationRegistry; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; @@ -19,41 +17,40 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import java.time.Instant; -import java.util.List; import java.util.concurrent.ScheduledFuture; import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.PollingInterval.TYPE.CRON; public class LdioHttpInputPoller extends LdioInput implements Runnable { public static final String NAME = "Ldio:HttpInPoller"; + private static final String CONTENT_TYPE = "Content-Type"; + private static final Logger log = LoggerFactory.getLogger(LdioHttpInputPoller.class); private final ThreadPoolTaskScheduler scheduler; private final RequestExecutor requestExecutor; - private final List requests; - private final boolean continueOnFail; - private static final Logger log = LoggerFactory.getLogger(LdioHttpInputPoller.class); - private static final String CONTENT_TYPE = "Content-Type"; - private PollingInterval pollingInterval; + private final LdioHttpInputPollerProperties properties; private ScheduledFuture scheduledPoll; - public LdioHttpInputPoller(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry, List endpoints, - boolean continueOnFail, RequestExecutor requestExecutor, ApplicationEventPublisher applicationEventPublisher) { - super(NAME, pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); + public LdioHttpInputPoller(ComponentExecutor executor, LdiAdapter adapter, LdioObserver ldioObserver, + RequestExecutor requestExecutor, LdioHttpInputPollerProperties properties, + ApplicationEventPublisher applicationEventPublisher) { + super(executor, adapter, ldioObserver, applicationEventPublisher); this.requestExecutor = requestExecutor; - this.requests = endpoints.stream().map(endpoint -> new GetRequest(endpoint, RequestHeaders.empty())).toList(); - this.continueOnFail = continueOnFail; + this.properties = properties; this.scheduler = new ThreadPoolTaskScheduler(); this.scheduler.setWaitForTasksToCompleteOnShutdown(false); this.scheduler.setErrorHandler(t -> log.error(t.getMessage())); } - public void schedulePoller(PollingInterval pollingInterval) { - this.pollingInterval = pollingInterval; - scheduler.initialize(); - schedule(pollingInterval); + @Override + public void start() { + super.start(); + startScheduler(); } - private void schedule(PollingInterval pollingInterval) { + private void startScheduler() { + scheduler.initialize(); + final var pollingInterval = properties.getPollingInterval(); if (pollingInterval.getType() == CRON) { scheduledPoll = scheduler.schedule(this, pollingInterval.getCronTrigger()); } else { @@ -63,11 +60,11 @@ private void schedule(PollingInterval pollingInterval) { @Override public void run() { - requests.forEach(request -> { + properties.getRequests().forEach(request -> { try { executeRequest(request); } catch (Exception e) { - if (!continueOnFail) { + if (!properties.isContinueOnFailEnabled()) { throw e; } } @@ -99,8 +96,8 @@ private void executeRequest(Request request) { @Override protected synchronized void resume() { - if (pollingInterval != null) { - schedule(pollingInterval); + if (properties.getPollingInterval() != null) { + startScheduler(); } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java index e8836a576..5a5b85cc6 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerAutoConfig.java @@ -4,19 +4,15 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInputPoller; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; -import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.ConfigPropertyMissingException; import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import java.util.List; -import java.util.Optional; - import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInputPoller.NAME; -import static be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInputPollerProperties.*; @Configuration public class LdioHttpInputPollerAutoConfig { @@ -38,21 +34,10 @@ public HttpInputPollerConfigurator(ObservationRegistry observationRegistry) { @Override public LdioHttpInputPoller configure(LdiAdapter adapter, ComponentExecutor executor, ApplicationEventPublisher applicationEventPublisher, ComponentProperties properties) { - String pipelineName = properties.getPipelineName(); - List endpoints = properties.getPropertyList(URL); - - if(endpoints.isEmpty()) { - throw new ConfigPropertyMissingException(pipelineName, properties.getComponentName(), URL); - } - - boolean continueOnFail = properties.getOptionalBoolean(CONTINUE_ON_FAIL).orElse(true); - - var requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties); - - var httpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, observationRegistry, endpoints, continueOnFail, requestExecutor, applicationEventPublisher); - - httpInputPoller.schedulePoller(getPollingInterval(properties)); - + final var ldioObserver = LdioObserver.register(NAME, properties.getPipelineName(), observationRegistry); + final var requestExecutor = ldioRequestExecutorSupplier.getRequestExecutor(properties); + final var ldioHttpInPollerProperties = LdioHttpInputPollerProperties.fromComponentProperties(properties); + final var httpInputPoller = new LdioHttpInputPoller(executor, adapter, ldioObserver, requestExecutor, ldioHttpInPollerProperties, applicationEventPublisher); httpInputPoller.start(); return httpInputPoller; } @@ -61,13 +46,6 @@ public LdioHttpInputPoller configure(LdiAdapter adapter, ComponentExecutor execu public boolean isAdapterRequired() { return true; } - - private PollingInterval getPollingInterval(ComponentProperties properties) { - Optional expression = properties.getOptionalProperty(CRON); - - return expression.map(PollingInterval::withCron) - .orElseGet(() -> PollingInterval.withInterval(properties.getProperty(INTERVAL))); - } } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerProperties.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerProperties.java index 00833bba4..789717c10 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerProperties.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInputPollerProperties.java @@ -1,10 +1,13 @@ package be.vlaanderen.informatievlaanderen.ldes.ldio.config; -public class LdioHttpInputPollerProperties { +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.GetRequest; +import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.valueobjects.RequestHeaders; +import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.ConfigPropertyMissingException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; - private LdioHttpInputPollerProperties() { - } +import java.util.List; +public final class LdioHttpInputPollerProperties { public static final String URL = "url"; public static final String INTERVAL = "interval"; public static final String CRON = "cron"; @@ -12,5 +15,43 @@ private LdioHttpInputPollerProperties() { public static final String INVALID_PROPERTY = "Invalid config for the ldio http in poller: %s cannot have following value: %s"; public static final String INTERVAL_MIGRATION_WARNING = "'interval' property is deprecated. Please consider migrating to 'cron' property"; + private final List requests; + private final PollingInterval pollingInterval; + private final boolean continueOnFail; + + public LdioHttpInputPollerProperties(List endpoints, PollingInterval pollingInterval, boolean continueOnFail) { + this.requests = endpoints.stream().map(endpoint -> new GetRequest(endpoint, RequestHeaders.empty())).toList(); + this.pollingInterval = pollingInterval; + this.continueOnFail = continueOnFail; + } + + public static LdioHttpInputPollerProperties fromComponentProperties(ComponentProperties properties) { + final List endpoints = extractEndpoints(properties); + final boolean continueOnFail = properties.getOptionalBoolean(CONTINUE_ON_FAIL).orElse(true); + final PollingInterval pollingInterval = properties.getOptionalProperty(CRON) + .map(PollingInterval::withCron) + .orElseGet(() -> PollingInterval.withInterval(properties.getProperty(INTERVAL))); + return new LdioHttpInputPollerProperties(endpoints, pollingInterval, continueOnFail); + } + + private static List extractEndpoints(ComponentProperties properties) { + final List endpoints = properties.getPropertyList(URL); + if (endpoints.isEmpty()) { + throw new ConfigPropertyMissingException(properties.getPipelineName(), properties.getComponentName(), URL); + } + return endpoints; + } + + public List getRequests() { + return requests; + } + + public PollingInterval getPollingInterval() { + return pollingInterval; + } + + public boolean isContinueOnFailEnabled() { + return continueOnFail; + } } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java index 40065d9f6..7aa6ec067 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInputPollerTest.java @@ -4,8 +4,13 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.services.RequestExecutorFactory; import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; +import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInputPollerProperties; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.PollingInterval; import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.MissingHeaderException; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; +import com.github.tomakehurst.wiremock.client.CountMatchingStrategy; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -17,7 +22,6 @@ import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; import org.springframework.scheduling.support.CronTrigger; @@ -27,27 +31,22 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; -import com.github.tomakehurst.wiremock.client.CountMatchingStrategy; -import com.github.tomakehurst.wiremock.client.WireMock; -import com.github.tomakehurst.wiremock.junit5.WireMockTest; - import static com.github.tomakehurst.wiremock.client.CountMatchingStrategy.GREATER_THAN_OR_EQUAL; import static com.github.tomakehurst.wiremock.client.CountMatchingStrategy.LESS_THAN_OR_EQUAL; -import static com.github.tomakehurst.wiremock.client.WireMock.*; import static com.github.tomakehurst.wiremock.client.WireMock.reset; +import static com.github.tomakehurst.wiremock.client.WireMock.*; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; import static org.mockito.Mockito.after; +import static org.mockito.Mockito.*; @WireMockTest(httpPort = LdioHttpInputPollerTest.WIREMOCK_PORT) class LdioHttpInputPollerTest { public static final int WIREMOCK_PORT = 12321; private final LdiAdapter adapter = mock(LdiAdapter.class); private final ComponentExecutor executor = mock(ComponentExecutor.class); - @Autowired - ApplicationEventPublisher applicationEventPublisher; + private final ApplicationEventPublisher applicationEventPublisher = mock(ApplicationEventPublisher.class); private final static String pipelineName = "pipeline"; private static final String BASE_URL = "http://localhost:%d".formatted(WIREMOCK_PORT); private static final String ENDPOINT = "/resource"; @@ -57,16 +56,10 @@ class LdioHttpInputPollerTest { private static final RequestExecutor noAuthExecutor = new RequestExecutorFactory().createNoAuthExecutor(); @BeforeEach - void setUp() { + void setUp() { reset(); - - when(adapter.apply(any())) - .thenReturn(Stream.of()) - .thenReturn(Stream.of()) - .thenReturn(Stream.of()) - .thenReturn(Stream.of()); - - } + when(adapter.apply(any())).thenReturn(Stream.of()); + } @AfterEach void tearDown() { @@ -75,16 +68,12 @@ void tearDown() { @Nested class DefaultConfig { - @BeforeEach - void setUp() { - ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), true, noAuthExecutor, applicationEventPublisher); - } - @Test void testClientPolling() { stubFor(get(ENDPOINT).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); + initPollerWithInterval(null); ldioHttpInputPoller.run(); WireMock.verify(getRequestedFor(urlEqualTo(ENDPOINT))); } @@ -94,7 +83,8 @@ void testClientPolling() { void whenPeriodicPolling_thenReturnTwoTimesTheSameResponse(PollingInterval pollingInterval) { stubFor(get(ENDPOINT).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); - ldioHttpInputPoller.schedulePoller(pollingInterval); + initPollerWithInterval(pollingInterval); + ldioHttpInputPoller.start(); Mockito.verify(adapter, timeout(4000).times(2)).apply(LdiAdapter.Content.of(CONTENT, CONTENT_TYPE)); WireMock.verify(new CountMatchingStrategy(GREATER_THAN_OR_EQUAL, 2), getRequestedFor(urlEqualTo(ENDPOINT))); @@ -105,7 +95,8 @@ void whenPeriodicPolling_thenReturnTwoTimesTheSameResponse(PollingInterval polli void when_PausePeriodicPolling_then_DontPoll(PollingInterval pollingInterval) { stubFor(get(ENDPOINT).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); - ldioHttpInputPoller.schedulePoller(pollingInterval); + initPollerWithInterval(pollingInterval); + ldioHttpInputPoller.start(); ldioHttpInputPoller.pause(); await().atLeast(2, TimeUnit.SECONDS); @@ -122,7 +113,8 @@ void when_PausePeriodicPolling_then_DontPoll(PollingInterval pollingInterval) { void when_OnContinueIsTrueAndPeriodPollingReturnsNot2xx_thenKeepPolling(PollingInterval pollingInterval) { stubFor(get(ENDPOINT).willReturn(forbidden())); - ldioHttpInputPoller.schedulePoller(pollingInterval); + initPollerWithInterval(pollingInterval); + ldioHttpInputPoller.start(); Mockito.verify(adapter, after(4000).never()).apply(any()); WireMock.verify(new CountMatchingStrategy(GREATER_THAN_OR_EQUAL, 2), @@ -132,9 +124,7 @@ void when_OnContinueIsTrueAndPeriodPollingReturnsNot2xx_thenKeepPolling(PollingI @Test void when_EndpointDoesNotExist_Then_NoDataIsSent() { String wrongEndpoint = "/non-existing-resource"; - ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + wrongEndpoint), true, - noAuthExecutor, applicationEventPublisher); - + initPoller(List.of(BASE_URL + wrongEndpoint), null, true); ldioHttpInputPoller.run(); WireMock.verify(getRequestedFor(urlEqualTo(wrongEndpoint))); @@ -145,18 +135,23 @@ void when_EndpointDoesNotExist_Then_NoDataIsSent() { void when_ResponseIsNot200_Then_NoDataIsSent() { stubFor(get(ENDPOINT).willReturn(forbidden())); + initPollerWithInterval(null); ldioHttpInputPoller.run(); WireMock.verify(getRequestedFor(urlEqualTo(ENDPOINT))); Mockito.verifyNoInteractions(adapter); } + + private void initPollerWithInterval(PollingInterval pollingInterval) { + initPoller(List.of(BASE_URL + ENDPOINT), pollingInterval, true); + } } @Test void whenPolling_andMissesHeader() { stubFor(get(ENDPOINT).willReturn(ok().withBody(CONTENT))); - ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor, applicationEventPublisher); + initPoller(List.of(BASE_URL + ENDPOINT), null, false); Executable polling = () -> ldioHttpInputPoller.run(); assertThrows(MissingHeaderException.class, polling); @@ -170,8 +165,7 @@ void whenPollMultipleEndpoints_andOneEndpointFails_thenTheOtherEndpointShouldSti stubFor(get(ENDPOINT).willReturn(serverError().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); String otherEndpoint = "/other-resource"; stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); - ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT, BASE_URL + otherEndpoint), - true, noAuthExecutor, applicationEventPublisher); + initPoller(List.of(BASE_URL + ENDPOINT, BASE_URL + otherEndpoint), null, true); ldioHttpInputPoller.run(); @@ -186,10 +180,9 @@ void whenPeriodicPollingMultipleEndpoints_thenReturnTwoTimesTheSameResponse(Poll stubFor(get(endpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); String otherEndpoint = "/other-endpoint"; stubFor(get(otherEndpoint).willReturn(ok().withHeader("Content-Type", CONTENT_TYPE).withBody(CONTENT))); - ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + endpoint, BASE_URL + otherEndpoint), - true, noAuthExecutor, applicationEventPublisher); + initPoller(List.of(BASE_URL + endpoint, BASE_URL + otherEndpoint), pollingInterval, true); - ldioHttpInputPoller.schedulePoller(pollingInterval); + ldioHttpInputPoller.start(); Mockito.verify(adapter, timeout(6000).times(4)).apply(LdiAdapter.Content.of(CONTENT, CONTENT_TYPE)); WireMock.verify(new CountMatchingStrategy(GREATER_THAN_OR_EQUAL, 2), getRequestedFor(urlEqualTo(endpoint))); @@ -197,14 +190,13 @@ void whenPeriodicPollingMultipleEndpoints_thenReturnTwoTimesTheSameResponse(Poll } - @ParameterizedTest @ArgumentsSource(PollingIntervalArgumentsProvider.class) void when_OnContinueIsFalseAndPeriodPollingReturnsNot2xx_thenStopPolling(PollingInterval pollingInterval) { stubFor(get(ENDPOINT).willReturn(forbidden())); - ldioHttpInputPoller = new LdioHttpInputPoller(pipelineName, executor, adapter, null, List.of(BASE_URL + ENDPOINT), false, noAuthExecutor, applicationEventPublisher); - ldioHttpInputPoller.schedulePoller(pollingInterval); + initPoller(List.of(BASE_URL + ENDPOINT), pollingInterval, false); + ldioHttpInputPoller.start(); Mockito.verify(adapter, after(4000).never()).apply(any()); WireMock.verify(new CountMatchingStrategy(GREATER_THAN_OR_EQUAL, 1), getRequestedFor(urlEqualTo(ENDPOINT))); @@ -219,4 +211,9 @@ public Stream provideArguments(ExtensionContext extensionCo } } + private void initPoller(List endpoints, PollingInterval pollingInterval, boolean continueOnFail) { + final LdioHttpInputPollerProperties properties = new LdioHttpInputPollerProperties(endpoints, pollingInterval, continueOnFail); + ldioHttpInputPoller = new LdioHttpInputPoller(executor, adapter, LdioObserver.register("Ldio:HttpInPoller", pipelineName, null), noAuthExecutor, properties, applicationEventPublisher); + } + } diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java index 0ba7ebbe3..29fd10c9b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in-poller/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdioHttpInputPollerAutoConfigTest.java @@ -6,7 +6,6 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.PollingIntervalTest.InvalidIntervalArgumentsProvider; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInputPollerAutoConfig; import be.vlaanderen.informatievlaanderen.ldes.ldio.config.LdioHttpInputPollerProperties; -import be.vlaanderen.informatievlaanderen.ldes.ldio.config.PollingInterval; import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.ConfigPropertyMissingException; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import org.assertj.core.api.ThrowableAssert; @@ -17,12 +16,8 @@ import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; import org.mockito.MockedConstruction; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; -import org.springframework.scheduling.support.CronTrigger; -import java.time.Duration; -import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; @@ -31,9 +26,7 @@ import static org.mockito.Mockito.*; class LdioLdioHttpInputPollerAutoConfigTest { - - @Autowired - ApplicationEventPublisher applicationEventPublisher; + private final ApplicationEventPublisher applicationEventPublisher = mock(ApplicationEventPublisher.class); private final LdiAdapter adapter = mock(LdiAdapter.class); private final ComponentExecutor executor = mock(ComponentExecutor.class); private static final String BASE_URL = "http://localhost:10101"; @@ -65,7 +58,7 @@ void when_ValidIntervalConfig() { LdioHttpInputPoller poller = new LdioHttpInputPollerAutoConfig() .httpInputPollerConfigurator(null) .configure(adapter, executor, applicationEventPublisher, createDefaultISOTestConfig()); - verify(poller, times(1)).schedulePoller(new PollingInterval(Duration.of(1, ChronoUnit.SECONDS))); + verify(poller, times(1)).start(); } } @@ -75,7 +68,7 @@ void when_ValidCronConfig() { LdioHttpInputPoller poller = new LdioHttpInputPollerAutoConfig() .httpInputPollerConfigurator(null) .configure(adapter, executor, applicationEventPublisher, createDefaultCronTestConfig()); - verify(poller, times(1)).schedulePoller(new PollingInterval(new CronTrigger("* * * * * *"))); + verify(poller, times(1)).start(); } } @@ -108,7 +101,7 @@ void when_MissingUrlConfigProvided_then_ThrowException(String urlKey) { LdioHttpInputPollerProperties.INTERVAL, "PT1S", LdioHttpInputPollerProperties.CONTINUE_ON_FAIL, "false")); - if(urlKey != null) { + if (urlKey != null) { properties.put(urlKey, "http://some-server.com/ldes"); } @@ -124,7 +117,7 @@ void when_MissingUrlConfigProvided_then_ThrowException(String urlKey) { static class InvalidUrlArgumentsProvider implements ArgumentsProvider { @Override - public Stream provideArguments(ExtensionContext extensionContext) throws Exception { + public Stream provideArguments(ExtensionContext extensionContext) { return Stream.of( Arguments.of("endpoint"), Arguments.of((Object) null), diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml index 8592214eb..5bd42b087 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/pom.xml @@ -4,7 +4,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java index f3c65de42..e143f1f48 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioHttpInProcess.java @@ -3,16 +3,16 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiAdapter; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; -import io.micrometer.observation.ObservationRegistry; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import org.springframework.context.ApplicationEventPublisher; public class LdioHttpInProcess extends LdioInput { public static final String NAME = "Ldio:HttpIn"; private boolean isPaused = false; - public LdioHttpInProcess(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, - ObservationRegistry observationRegistry, ApplicationEventPublisher applicationEventPublisher) { - super(NAME, pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); + public LdioHttpInProcess(ComponentExecutor executor, LdiAdapter adapter, + LdioObserver ldioObserver, ApplicationEventPublisher applicationEventPublisher) { + super(executor, adapter, ldioObserver, applicationEventPublisher); } @Override diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java index 7e77bd331..9bd68a784 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-http-in/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioHttpInAutoConfig.java @@ -6,6 +6,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.event.HttpInPipelineCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import org.springframework.context.ApplicationEventPublisher; @@ -41,7 +42,8 @@ public LdioInput configure(LdiAdapter adapter, ComponentProperties config) { String pipelineName = config.getPipelineName(); - LdioHttpInProcess ldioHttpIn = new LdioHttpInProcess(pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); + LdioObserver ldioObserver = LdioObserver.register(NAME, pipelineName, observationRegistry); + LdioHttpInProcess ldioHttpIn = new LdioHttpInProcess(executor, adapter, ldioObserver, applicationEventPublisher); eventPublisher.publishEvent(new HttpInPipelineCreatedEvent(pipelineName, ldioHttpIn)); ldioHttpIn.start(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml index 31dac60e2..cdf18276b 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-http-out/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml index 557c3c519..ba93d41f0 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-json-to-ld-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml index 6fa14f59b..6b79f479e 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java index 17f8c4e82..7981ebf64 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioKafkaIn.java @@ -7,8 +7,8 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.exceptions.SecurityProtocolNotSupportedException; import be.vlaanderen.informatievlaanderen.ldes.ldio.listener.LdioKafkaInListener; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; -import io.micrometer.observation.ObservationRegistry; import org.apache.jena.riot.Lang; import org.apache.jena.riot.RDFLanguages; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -27,27 +27,23 @@ public class LdioKafkaIn extends LdioInput { public static final String NAME = "Ldio:KafkaIn"; - @SuppressWarnings("java:S3740") - private final KafkaMessageListenerContainer container; - private final LdioKafkaInListener listener; - /** + private final KafkaMessageListenerContainer container; + + /** * Creates a LdiInput with its Component Executor and LDI Adapter * - * @param pipelineName * @param executor Instance of the Component Executor. Allows the LDI Input to pass * data on the pipeline * @param adapter Instance of the LDI Adapter. Facilitates transforming the input * data to a linked data model (RDF). - * @param observationRegistry - * @param applicationEventPublisher - * @param config + * @param ldioObserver Instance of the LDIO Observer, for observing and monitoring reasons */ - public LdioKafkaIn(String pipelineName, ComponentExecutor executor, LdiAdapter adapter, ObservationRegistry observationRegistry, + public LdioKafkaIn(ComponentExecutor executor, LdiAdapter adapter, LdioObserver ldioObserver, ApplicationEventPublisher applicationEventPublisher, ComponentProperties config) { - super(NAME, pipelineName, executor, adapter, observationRegistry, applicationEventPublisher); - this.listener = new LdioKafkaInListener(getContentType(config), this::processInput); - var consumerFactory = new DefaultKafkaConsumerFactory<>(getConsumerConfig(config)); - ContainerProperties containerProps = new ContainerProperties(config.getProperty(TOPICS).split(",")); + super(executor, adapter, ldioObserver, applicationEventPublisher); + final LdioKafkaInListener listener = new LdioKafkaInListener(getContentType(config), this::processInput); + final var consumerFactory = new DefaultKafkaConsumerFactory<>(getConsumerConfig(config)); + final ContainerProperties containerProps = new ContainerProperties(config.getProperty(TOPICS).split(",")); containerProps.setMessageListener(listener); this.container = new KafkaMessageListenerContainer<>(consumerFactory, containerProps); container.start(); diff --git a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java index 6874550fa..7c561aa17 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-kafka/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioKafkaInAutoConfig.java @@ -5,6 +5,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioKafkaIn; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import org.springframework.context.ApplicationEventPublisher; @@ -31,8 +32,9 @@ public LdioKafkaInConfigurator(ObservationRegistry observationRegistry) { @Override public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, ApplicationEventPublisher applicationEventPublisher, ComponentProperties config) { - String pipelineName = config.getPipelineName(); - return new LdioKafkaIn(pipelineName, executor, adapter, observationRegistry, applicationEventPublisher, config); + final String pipelineName = config.getPipelineName(); + final LdioObserver ldioObserver = LdioObserver.register(NAME, pipelineName, observationRegistry); + return new LdioKafkaIn(executor, adapter, ldioObserver, applicationEventPublisher, config); } @Override diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml index 40cfe4b93..2cff4613f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/pom.xml @@ -5,7 +5,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java index b6e57899c..93da70878 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientConnectorAutoConfig.java @@ -13,6 +13,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.event.LdesClientConnectorApiCreatedEvent; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import ldes.client.treenodesupplier.MemberSupplier; @@ -62,7 +63,8 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor executor, Appli urlProxy); final MemberSupplier memberSupplier = new MemberSupplierFactory(properties, edcRequestExecutor).getMemberSupplier(); final boolean keepState = properties.getOptionalBoolean(KEEP_STATE).orElse(false); - var ldesClient = new LdioLdesClient(pipelineName, executor, observationRegistry, memberSupplier, + final LdioObserver ldioObserver = LdioObserver.register(NAME, pipelineName, observationRegistry); + final var ldesClient = new LdioLdesClient(executor, ldioObserver, memberSupplier, applicationEventPublisher, keepState); eventPublisher.publishEvent(new LdesClientConnectorApiCreatedEvent(pipelineName, new LdioLdesClientConnectorApi(transferService, tokenService, ldesClient))); diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml index 3269b8375..ade33f2c6 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java index 3e84d42e5..c136173e3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioLdesClient.java @@ -2,7 +2,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.services.ComponentExecutor; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; -import io.micrometer.observation.ObservationRegistry; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import ldes.client.treenodesupplier.MemberSupplier; import ldes.client.treenodesupplier.domain.valueobject.EndOfLdesException; import org.slf4j.Logger; @@ -24,13 +24,12 @@ public class LdioLdesClient extends LdioInput { private boolean paused = false; private final boolean keepState; - public LdioLdesClient(String pipelineName, - ComponentExecutor componentExecutor, - ObservationRegistry observationRegistry, + public LdioLdesClient(ComponentExecutor componentExecutor, + LdioObserver ldioObserver, MemberSupplier memberSupplier, ApplicationEventPublisher applicationEventPublisher, boolean keepState) { - super(NAME, pipelineName, componentExecutor, null, observationRegistry, applicationEventPublisher); + super(componentExecutor, null, ldioObserver, applicationEventPublisher); this.memberSupplier = memberSupplier; this.keepState = keepState; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java index 52ac39679..e07bcf0bf 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-ldes-client/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioLdesClientAutoConfig.java @@ -8,6 +8,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioInputConfigurator; import be.vlaanderen.informatievlaanderen.ldes.ldio.requestexecutor.LdioRequestExecutorSupplier; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioInput; +import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioObserver; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import io.micrometer.observation.ObservationRegistry; import ldes.client.treenodesupplier.MemberSupplier; @@ -42,8 +43,8 @@ public LdioInput configure(LdiAdapter adapter, ComponentExecutor componentExecut final var requestExecutor = new LdioRequestExecutorSupplier(requestExecutorFactory).getRequestExecutor(properties); final MemberSupplier memberSupplier = new MemberSupplierFactory(properties, requestExecutor).getMemberSupplier(); final boolean keepState = properties.getOptionalBoolean(KEEP_STATE).orElse(false); - var ldesClient = new LdioLdesClient(pipelineName, componentExecutor, observationRegistry, memberSupplier, - applicationEventPublisher, keepState); + final LdioObserver ldioObserver = LdioObserver.register(LdioLdesClient.NAME, pipelineName, observationRegistry); + final var ldesClient = new LdioLdesClient(componentExecutor, ldioObserver, memberSupplier, applicationEventPublisher, keepState); ldesClient.start(); return ldesClient; } diff --git a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml index efcd37f04..aa2e22e1f 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/pom.xml @@ -6,7 +6,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldio-ngsiv2-to-ld-adapter @@ -15,7 +15,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ngsiv2-to-ld-adapter - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT compile diff --git a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml index 9ef3d5073..14996bd80 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-noop-out/pom.xml @@ -4,7 +4,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml index df3ec9738..977f283a3 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml index 629338db6..388227f2d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldio-repository-materialiser diff --git a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml index 8d4458479..e6db3090a 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-request-executor/pom.xml @@ -6,7 +6,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldio-request-executor diff --git a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml index 2c108101b..65f7cc6a2 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-rml-adapter/pom.xml @@ -5,7 +5,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml index ba2f75903..315c0af21 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-sparql-construct/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldio ldio-connectors - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml index eab79311b..833a7b83d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-materialiser/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml index 2c994abff..ad9e5e08d 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/pom.xml @@ -3,7 +3,7 @@ ldio-connectors be.vlaanderen.informatievlaanderen.ldes.ldio - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/ldio-connectors/pom.xml b/ldi-orchestrator/ldio-connectors/pom.xml index e4a658be0..44efb301c 100644 --- a/ldi-orchestrator/ldio-connectors/pom.xml +++ b/ldi-orchestrator/ldio-connectors/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes.ldi ldi-orchestrator - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldi-orchestrator/pom.xml b/ldi-orchestrator/pom.xml index 5c74d3382..ba0c4341d 100644 --- a/ldi-orchestrator/pom.xml +++ b/ldi-orchestrator/pom.xml @@ -3,7 +3,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT 4.0.0 diff --git a/ldio.Dockerfile b/ldio.Dockerfile index c144efd2a..0bd566e6c 100644 --- a/ldio.Dockerfile +++ b/ldio.Dockerfile @@ -27,7 +27,6 @@ COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-ldes-client/target/l COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-ldes-client-connector/target/ldio-ldes-client-connector-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-kafka/target/ldio-kafka-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-amqp/target/ldio-amqp-jar-with-dependencies.jar ./lib/ -COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-archive-file-in/target/ldio-archive-file-in-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-rdf-adapter/target/ldio-rdf-adapter-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-ngsiv2-to-ld-adapter/target/ldio-ngsiv2-to-ld-adapter-jar-with-dependencies.jar ./lib/ @@ -44,8 +43,6 @@ COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-http-enricher/target COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-console-out/target/ldio-console-out-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-http-out/target/ldio-http-out-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-noop-out/target/ldio-noop-out-jar-with-dependencies.jar ./lib/ -COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-azure-blob-out/target/ldio-azure-blob-out-jar-with-dependencies.jar ./lib/ -COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-file-out/target/ldio-file-out-jar-with-dependencies.jar ./lib/ COPY --from=app-stage ldi-orchestrator/ldio-connectors/ldio-repository-materialiser/target/ldio-repository-materialiser-jar-with-dependencies.jar ./lib/ diff --git a/pom.xml b/pom.xml index 1482f800d..d37a94549 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ be.vlaanderen.informatievlaanderen.ldes linked-data-interactions pom - 2.4.0-SNAPSHOT + 2.5.0-SNAPSHOT ldi-api