ci: finish 1.8.0 sprint (#379)

Informatievlaanderen · Oct 24, 2023 · 442598a · 442598a
2 parents 8865ec5 + 35493f7
commit 442598a
Show file tree

Hide file tree

Showing 276 changed files with 2,278 additions and 1,954 deletions.
diff --git a/docs/Gemfile b/docs/Gemfile
@@ -1,13 +1,14 @@
 source 'https://rubygems.org'
 
-gem "jekyll", "~> 4.3.2" # installed by `gem jekyll`
-# gem "webrick"        # required when using Ruby >= 3 and Jekyll <= 4.2.2
+gem "jekyll", "~> 3.9.3" # installed by `gem jekyll`
+gem "webrick"        # required when using Ruby >= 3 and Jekyll <= 4.2.2
 
 gem "just-the-docs", "0.5.3" # pinned to the current release
 # gem "just-the-docs"        # always download the latest release
 
 # If you have any plugins, put them here!
 group :jekyll_plugins do
     gem "kramdown"
+    gem "kramdown-parser-gfm"
     gem "jekyll-remote-theme"
 end
diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
@@ -9,36 +9,32 @@ GEM
       eventmachine (>= 0.12.9)
       http_parser.rb (~> 0)
     eventmachine (1.2.7)
-    ffi (1.15.5)
+    ffi (1.16.3)
+    ffi (1.16.3-x64-mingw-ucrt)
     forwardable-extended (2.6.0)
-    google-protobuf (3.23.2-x64-mingw-ucrt)
-    google-protobuf (3.23.2-x86_64-linux)
     http_parser.rb (0.8.0)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    jekyll (4.3.2)
+    jekyll (3.9.3)
       addressable (~> 2.4)
       colorator (~> 1.0)
       em-websocket (~> 0.5)
-      i18n (~> 1.0)
-      jekyll-sass-converter (>= 2.0, < 4.0)
+      i18n (>= 0.7, < 2)
+      jekyll-sass-converter (~> 1.0)
       jekyll-watch (~> 2.0)
-      kramdown (~> 2.3, >= 2.3.1)
-      kramdown-parser-gfm (~> 1.0)
+      kramdown (>= 1.17, < 3)
       liquid (~> 4.0)
-      mercenary (>= 0.3.6, < 0.5)
+      mercenary (~> 0.3.3)
       pathutil (~> 0.9)
-      rouge (>= 3.0, < 5.0)
+      rouge (>= 1.7, < 4)
       safe_yaml (~> 1.0)
-      terminal-table (>= 1.8, < 4.0)
-      webrick (~> 1.7)
     jekyll-remote-theme (0.4.3)
       addressable (~> 2.0)
       jekyll (>= 3.5, < 5.0)
       jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0)
       rubyzip (>= 1.3.0, < 3.0)
-    jekyll-sass-converter (3.0.0)
-      sass-embedded (~> 1.54)
+    jekyll-sass-converter (1.5.2)
+      sass (~> 3.4)
     jekyll-seo-tag (2.8.0)
       jekyll (>= 3.8, < 5.0)
     jekyll-watch (2.2.1)
@@ -55,7 +51,7 @@ GEM
     listen (3.8.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
-    mercenary (0.4.0)
+    mercenary (0.3.6)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (5.0.1)
@@ -64,27 +60,27 @@ GEM
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     rexml (3.2.5)
-    rouge (4.1.2)
+    rouge (3.30.0)
     rubyzip (2.3.2)
     safe_yaml (1.0.5)
-    sass-embedded (1.62.1-x64-mingw-ucrt)
-      google-protobuf (~> 3.21)
-    sass-embedded (1.62.1-x86_64-linux-gnu)
-      google-protobuf (~> 3.21)
-    terminal-table (3.0.2)
-      unicode-display_width (>= 1.1.1, < 3)
-    unicode-display_width (2.4.2)
+    sass (3.7.4)
+      sass-listen (~> 4.0.0)
+    sass-listen (4.0.0)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
     webrick (1.8.1)
 
 PLATFORMS
   x64-mingw-ucrt
   x86_64-linux
 
 DEPENDENCIES
-  jekyll (~> 4.3.2)
+  jekyll (~> 3.9.3)
   jekyll-remote-theme
   just-the-docs (= 0.5.3)
   kramdown
+  kramdown-parser-gfm
+  webrick
 
 BUNDLED WITH
    2.3.26
diff --git a/docs/_config.yml b/docs/_config.yml
@@ -2,8 +2,6 @@ title: Linked Data Interactions
 description: Linked Data Interactions
 remote_theme: just-the-docs/[email protected]
 
-url: https://just-the-docs.github.io
-
 favicon_ico: "/assets/images/favicon.ico"
 
 callouts:
@@ -19,9 +17,6 @@ aux_links:
 ga_tracking: G-9F4LJXRX6T
 ga_tracking_anonymize_ip: true # Use GDPR compliant Google Analytics settings (true/nil by default)
 
-
-
-
 # Define Jekyll collections
 collections:
   core:

diff --git a/docs/_includes/head_custom.html b/docs/_includes/head_custom.html
@@ -1,11 +1,11 @@
 <div class="test-header" data-v-df634f57="" data-v-7a7a37b1="">
-    <div class="global-header" data-v-df634f57=""><img src="/VSDS-Linked-Data-Interactions/images/LOGO_Vlaanderen_NIV2.svg"
+    <div class="global-header" data-v-df634f57=""><img src="{{ '/assets/images/LOGO_Vlaanderen_NIV2.svg' | relative_url }}"
             alt="Vlaanderen" data-v-df634f57=""><img class="small-global-header-divider"
-            src="/VSDS-Linked-Data-Interactions/images/divider_globalheader.svg" alt="divider" data-v-df634f57=""><span
+            src="{{ '/assets/images/divider_globalheader.svg' | relative_url }}" alt="divider" data-v-df634f57=""><span
             class="header-title header-font" data-v-df634f57="">Vlaamse Smart Data Space</span><span class="spacer"
             data-v-df634f57=""></span>
         <div id="help-needed" data-v-df634f57=""><span class="header-font" data-v-df634f57="">Hulp nodig</span><img
-                class="question-menu-item" src="/VSDS-Linked-Data-Interactions/images/question-mark.svg" alt="?"
+                class="question-menu-item" src="{{ '/assets/images/question-mark.svg' | relative_url }}" alt="?"
                 data-v-df634f57=""></div>
     </div>
     <hr class="divider" data-v-df634f57="">
@@ -19,7 +19,7 @@
     display: flex;
     align-items: center;
     gap: 5px;
-    background-image: url(/VSDS-Linked-Data-Interactions/images/hulp_nodig.svg);
+    background-image: url(/assets/images/hulp_nodig.svg | relative_url);
     width: 144px;
     height: 44px;
 }

diff --git a/docs/_includes/swagger.html b/docs/_includes/swagger.html
diff --git a/docs/_includes/swagger2.html b/docs/_includes/swagger2.html
diff --git a/docs/_ldio/index.md b/docs/_ldio/index.md
@@ -56,24 +56,92 @@ Make sure you
     ````
 * Add the ```debug: true``` property to your transformer or output config.
 
-## LDIO Process Flow 
+## Complete Workflow
 
 ````mermaid
-sequenceDiagram
-    LDI Input->>+LDI Adapter: Received Content.
-    loop For every Linked Data Model
-        LDI Adapter->>-LDI Input: Returns Model
-
+flowchart LR
+    subgraph LDIO Input
+        HttpIn(Http In)
+        HttpPoller(Http Poller)
+        LdesClient(LDES Client)
+        ArchiveFileIn(Archive File In)
+        KafkaIn(Kafka In)
     end
-    LDI Input->>ComponentExecutor: Process Model
-    loop For every LDI Transformer in pipeline
-        ComponentExecutor->>+LDI Transformer: Start Transformation
-        LDI Transformer->>-ComponentExecutor: Return Transformed Model
+
+    HttpIn--Non Linked Data -->Adapter
+    HttpPoller--Non Linked Data -->Adapter
+    LdesClient--Non Linked Data -->Adapter
+    ArchiveFileIn--Non Linked Data -->Adapter
+    KafkaIn--Non Linked Data -->Adapter
+
+    subgraph LDIO Adapter
+        Adapter{Adapter}
+
+        Adapter-->RdfAdapter
+        Adapter-->RmlAdapter
+        Adapter-->JsonToJsonLdAdapter
+        Adapter-->NgsiV2Adapter
+
+        RdfAdapter(Rdf Adapter)
+        RmlAdapter(Rml Adapter)
+        JsonToJsonLdAdapter(JSON to JSON LD Adapter)
+        NgsiV2Adapter(NGSI v2 to LD Adapter)
     end
 
-    loop For every LDI Output in pipeline
-        ComponentExecutor->>+LDI Output: Start Output
-        Note right of LDI Output: Model exported!
+    RdfAdapter--Linked Data-->TransformerIn
+    RmlAdapter--Linked Data-->TransformerIn
+    JsonToJsonLdAdapter--Linked Data-->TransformerIn
+    NgsiV2Adapter--Linked Data-->TransformerIn
+
+    subgraph LDIO Transformers
+        TransformerIn{Transformer}
+        TransformerOut{Transformer}
+
+        TransformerIn --> HttpEnricherWrapper
+        TransformerIn --> GeoJsonToWkt
+        TransformerIn --> ModelSplitter
+        TransformerIn --> SPARQLConstruct
+        TransformerIn --> VersionMaterialiser
+        TransformerIn --> VersionObjectCreator
+
+        GeoJsonToWkt(GeoJSON to WKT Transformer)
+        ModelSplitter(Model Split Transformer)
+        SPARQLConstruct(SPARQL Construct Transformer)
+        VersionMaterialiser(Version Materialiser)
+        VersionObjectCreator(Version Object Creator)
+
+        subgraph HttpEnricherWrapper[Http Enricher]
+            HttpEnricher(Http Enricher)
+            HttpEnrichAdapter{Adapter}
+            HttpEnrichAdapter --> HttpEnricher
+        end
+
+        HttpEnricherWrapper -- Model --> TransformerOut
+        GeoJsonToWkt -- Model --> TransformerOut
+        ModelSplitter -- Model[] --> TransformerOut
+        SPARQLConstruct -- Model --> TransformerOut
+        VersionMaterialiser -- Model --> TransformerOut
+        VersionObjectCreator -- Model --> TransformerOut
+
+        TransformerOut -- Model --> TransformerIn
+    end
+
+    TransformerOut -- Model --> Output
+
+    subgraph LDIO Out
+        Output{Output}
+
+        Output --> HttpOut
+        Output --> KafkaOut
+        Output --> ConsoleOut
+        Output --> ArchiveOut
+        Output --> AzureBlobOut
+
+        HttpOut
+        KafkaOut
+        ConsoleOut
+        ArchiveOut
+        AzureBlobOut
     end
 ````
 

diff --git a/docs/_ldio/ldio-adapters/index.md b/docs/_ldio/ldio-adapters/index.md
@@ -3,7 +3,7 @@ layout: default
 title: LDIO Adapters
 has_children: true
 has_toc: true
-nav_order: 2
+nav_order: 3
 ---
 
 # Linked Data Adapters

diff --git a/docs/_ldio/ldio-core/index.md b/docs/_ldio/ldio-core/index.md
@@ -0,0 +1,11 @@
+---
+layout: default
+title: LDIO Core
+has_children: true
+has_toc: true
+nav_order: 1
+---
+
+# LDIO Core
+
+Contains common components used by multiple other LDIO components
diff --git a/docs/_ldio/ldio-core/ldio-http-requester.md b/docs/_ldio/ldio-core/ldio-http-requester.md
@@ -0,0 +1,40 @@
+---
+layout: default
+parent: LDIO Core
+title: LDIO Http Requester
+---
+
+# LDIO Http Requester
+
+Different LDIO components use the Http Requester to make HTTP requests. 
+This requester supports the below config:
+
+
+| Property                            | Description                                                                                   | Required | Default   | Example                     | Supported values                              |
+|:------------------------------------|:----------------------------------------------------------------------------------------------|:---------|:----------|:----------------------------|:----------------------------------------------|
+| auth.type                           | The type of authentication required by the LDES server                                        | No       | NO_AUTH   | OAUTH2_CLIENT_CREDENTIALS   | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS |
+| auth.api-key                        | The api key when using auth.type 'API_KEY'                                                    | No       | N/A       | myKey                       | String                                        |
+| auth.api-key-header                 | The header for the api key when using auth.type 'API_KEY'                                     | No       | X-API-KEY | X-API-KEY                   | String                                        |
+| auth.client-id                      | The client identifier when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                        | No       | N/A       | myId                        | String                                        |
+| auth.client-secret                  | The client secret when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                            | No       | N/A       | mySecret                    | String                                        |
+| auth.token-endpoint                 | The token endpoint when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                           | No       | N/A       | http://localhost:8000/token | HTTP and HTTPS urls                           |
+| retries.enabled                     | Indicates if the http client should retry http requests when the server cannot be reached.    | No       | true      | true                        | true or false                                 |
+| retries.max                         | Max number of retries the http client should do when retries.enabled = true                   | No       | 5         | 100                         | Integer                                       |
+| retries.statuses-to-retry           | Custom comma seperated list of http status codes that can trigger a retry in the http client. | No       | N/A       | 410,451                     | Comma seperated list of Integers              |
+| rate-limit.enabled                  | Indicates if the http client should limit http requests when calling the server.              | No       | false     | false                       | true or false                                 |
+| rate-limit.max-requests-per-minute  | Max number of requests per minute the http client should do when rate-limit.enabled = true    | No       | 500       | 500                         | Integer                                       |
+
+```yaml
+      config:
+        auth:
+          type: API_KEY
+          api-key: my-secret
+          api-key-header: x-api-key
+        retries:
+          enabled: true
+          max: 10
+          statuses-to-retry: 410,451
+        rate-limit:
+          enabled: true
+          max-requests-per-minute: 500
+```
diff --git a/docs/_ldio/ldio-core/ldio-rdf-writer.md b/docs/_ldio/ldio-core/ldio-rdf-writer.md
@@ -0,0 +1,24 @@
+---
+layout: default
+parent: LDIO Core
+title: LDI RDF Writer
+---
+
+# LDI RDF Writer
+
+To easily output RDF in the correct format, a generic RDF Writer is introduced.  
+The RDF Writer supports the below config:
+
+
+| Property     | Description                                                           | Required | Default     | Example                                   | Supported values                                              |
+|:-------------|:----------------------------------------------------------------------|:---------|:------------|:------------------------------------------|:--------------------------------------------------------------|
+| content-type | Target content type.                                                  | No       | text/turtle | application/ld+json                       | Any type supported by [Apache Jena](https://jena.apache.org/) |
+| frame-type   | RDF type of the objects that need to be included for JSON-LD framing. | No       | N/A         | http://purl.org/goodrelations/v1#Offering | Any RDF type                                                  |
+
+## Example
+```yaml
+      config:
+        rdf-writer:
+          content-type: application/n-quads
+          frame-type: http://purl.org/goodrelations/v1#Offering
+```
diff --git a/docs/_ldio/ldio-inputs/index.md b/docs/_ldio/ldio-inputs/index.md
@@ -3,7 +3,7 @@ layout: default
 title: LDIO Inputs
 has_children: true
 has_toc: true
-nav_order: 1
+nav_order: 2
 ---
 
 # LDIO Inputs

diff --git a/docs/_ldio/ldio-inputs/ldio-http-in-poller.md b/docs/_ldio/ldio-inputs/ldio-http-in-poller.md
@@ -11,20 +11,14 @@ The LDIO Http In Poller is a basic Http Poller that will poll a target URL on a
 
 ## Config
 
-| Property                  | Description                                                                                   | Required | Default     | Example                                                  | Supported values                              |
-|:--------------------------|:----------------------------------------------------------------------------------------------|:---------|:------------|:---------------------------------------------------------|:----------------------------------------------|
-| url                       | Target URL to poll from.                                                                      | Yes      | N/A         | http://example.com/my-api                                | HTTP and HTTPS urls (lists are supported)     |
-| interval                  | Polling interval declared in ISO 8601 format.                                                 | Yes      | N/A         | PT1S                                                     | ISO 8601 formatted String                     |
-| continueOnFail            | Indicated if continue if polling results in failure                                           | No       | true        | true                                                     | true or false                                 |
-| auth.type                 | The type of authentication required by the LDES server                                        | No       | NO_AUTH     | OAUTH2_CLIENT_CREDENTIALS                                | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS |
-| auth.api-key              | The api key when using auth.type 'API_KEY'                                                    | No       | N/A         | myKey                                                    | String                                        |
-| auth.api-key-header       | The header for the api key when using auth.type 'API_KEY'                                     | No       | X-API-KEY   | X-API-KEY                                                | String                                        |
-| auth.client-id            | The client identifier when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                        | No       | N/A         | myId                                                     | String                                        |
-| auth.client-secret        | The client secret when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                            | No       | N/A         | mySecret                                                 | String                                        |
-| auth.token-endpoint       | The token endpoint when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                           | No       | N/A         | http://localhost:8000/token                              | HTTP and HTTPS urls                           |
-| retries.enabled           | Indicates if the http client should retry http requests when the server cannot be reached.    | No       | true        | true                                                     | true or false                                 |
-| retries.max               | Max number of retries the http client should do when retries.enabled = true                   | No       | 5           | 100                                                      | Integer                                       |
-| retries.statuses-to-retry | Custom comma seperated list of http status codes that can trigger a retry in the http client. | No       | N/A         | 410,451                                                  | Comma seperated list of Integers              |
+| Property                            | Description                                                                                   | Required | Default   | Example                     | Supported values                              |
+|:------------------------------------|:----------------------------------------------------------------------------------------------|:---------|:----------|:----------------------------|:----------------------------------------------|
+| url                                 | Target URL to poll from.                                                                      | Yes      | N/A       | http://example.com/my-api   | HTTP and HTTPS urls (lists are supported)     |
+| interval                            | Polling interval declared in ISO 8601 format.                                                 | Yes      | N/A       | PT1S                        | ISO 8601 formatted String                     |
+| continueOnFail                      | Indicated if continue if polling results in failure                                           | No       | true      | true                        | true or false                                 |
+
+This component uses the "LDIO Http Requester" to make the HTTP request.
+Refer to [LDIO Http Requester](../ldio-core) for the config.
 
 The Http In Poller supports polling multiple endpoints. Example configuration: