chore: finalise 1.14.0 (#469)

docs/_ldio/ldio-core/ldio-http-requester.md

@@ -6,24 +6,24 @@ title: LDIO Http Requester
 
 # LDIO Http Requester
 
-Different LDIO components use the Http Requester to make HTTP requests. 
+Different LDIO components use the Http Requester to make HTTP requests.
 This requester supports the below config:
 
-
-| Property                           | Description                                                                                      | Required | Default   | Example                     | Supported values                              |
-|:-----------------------------------|:-------------------------------------------------------------------------------------------------|:---------|:----------|:----------------------------|:----------------------------------------------|
-| auth.type                          | The type of authentication required by the LDES server                                           | No       | NO_AUTH   | OAUTH2_CLIENT_CREDENTIALS   | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS |
-| auth.api-key                       | The api key when using auth.type 'API_KEY'                                                       | No       | N/A       | myKey                       | String                                        |
-| auth.api-key-header                | The header for the api key when using auth.type 'API_KEY'                                        | No       | X-API-KEY | X-API-KEY                   | String                                        |
-| auth.client-id                     | The client identifier when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                           | No       | N/A       | myId                        | String                                        |
-| auth.client-secret                 | The client secret when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                               | No       | N/A       | mySecret                    | String                                        |
-| auth.token-endpoint                | The token endpoint when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                              | No       | N/A       | http://localhost:8000/token | HTTP and HTTPS urls                           |
-| retries.enabled                    | Indicates if the http client should retry http requests when the server cannot be reached.       | No       | true      | true                        | true or false                                 |
-| retries.max                        | Max number of retries the http client should do when retries.enabled = true                      | No       | 5         | 100                         | Integer                                       |
-| retries.statuses-to-retry          | Custom comma seperated list of http status codes that can trigger a retry in the http client.    | No       | N/A       | 410,451                     | Comma seperated list of Integers              |
-| rate-limit.enabled                 | Indicates if the http client should limit http requests when calling the server.                 | No       | false     | false                       | true or false                                 |
-| rate-limit.max-requests-per-minute | Max number of requests per minute the http client should do when rate-limit.enabled = true       | No       | 500       | 500                         | Integer                                       |
-| http.headers.[].key/value          | A list of custom http headers can be added. A key and value has to be provided for every header. | No       | N/A       | role                        | String                                        |
+| Property                  | Description                                                                                                                       | Required | Default   | Example                     | Supported values                              |
+|:--------------------------|:----------------------------------------------------------------------------------------------------------------------------------|:---------|:----------|:----------------------------|:----------------------------------------------|
+| auth.type                 | The type of authentication required by the LDES server                                                                            | No       | NO_AUTH   | OAUTH2_CLIENT_CREDENTIALS   | NO_AUTH, API_KEY or OAUTH2_CLIENT_CREDENTIALS |
+| auth.api-key              | The api key when using auth.type 'API_KEY'                                                                                        | No       | N/A       | myKey                       | String                                        |
+| auth.api-key-header       | The header for the api key when using auth.type 'API_KEY'                                                                         | No       | X-API-KEY | X-API-KEY                   | String                                        |
+| auth.client-id            | The client identifier when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                                                            | No       | N/A       | myId                        | String                                        |
+| auth.client-secret        | The client secret when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                                                                | No       | N/A       | mySecret                    | String                                        |
+| auth.token-endpoint       | The token endpoint when using auth.type 'OAUTH2_CLIENT_CREDENTIALS'                                                               | No       | N/A       | http://localhost:8000/token | HTTP and HTTPS urls                           |
+| retries.enabled           | Indicates if the http client should retry http requests when the server cannot be reached.                                        | No       | true      | true                        | true or false                                 |
+| retries.max               | Max number of retries the http client should do when retries.enabled = true                                                       | No       | 5         | 100                         | Integer                                       |
+| retries.statuses-to-retry | Custom comma seperated list of http status codes that can trigger a retry in the http client.                                     | No       | N/A       | 410,451                     | Comma seperated list of Integers              |
+| rate-limit.enabled        | Indicates if the http client should limit http requests when calling the server.                                                  | No       | false     | false                       | true or false                                 |
+| rate-limit.limit          | Limit of requests per period, which is defined below, that the http client should do when `rate-limit.enabled = true`             | No       | 500       | 100                         | Integer                                       |
+| rate-limit.period         | Period in which the limit of requests, which is defined above, can be reached by the http client when `rate-limit.enabled = true` | No       | PT1M      | PT1H                        | ISO 8601 Duration                             |
+| http.headers.[].key/value | A list of custom http headers can be added. A key and value has to be provided for every header.                                  | No       | N/A       | role                        | String                                        |
 
 ```yaml
       config:
@@ -43,10 +43,35 @@ This requester supports the below config:
           statuses-to-retry: 410,451
         rate-limit:
           enabled: true
-          max-requests-per-minute: 500
+          period: P1D
+          limit: 1000
 ```
 
 ## Retry
+
 When retries are enabled, the following statuses are always retried, regardless of the configured statuses-to-retry:
+
 - 5xx (500 and above)
-- 429
+- 429
+
+## Rate limiter
+
+> **NOTE**: Since 1.14.0, rate-limiter.max-requests-per-minute is deprecated, as this was too restrictive for
+> configuring the rate limiter. Here is an example on how to convert the old config.
+> 
+> *Old config:*
+> ```yaml
+>       config:
+>         rate-limit:
+>           enabled: true
+>           max-requests-per-minute: 500
+> ```
+> 
+> *New config:*
+> ```yaml
+>       config:
+>         rate-limit:
+>           enabled: true
+>           limit: 500
+>           period: PT1M
+> ```

ldi-api/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes</groupId>
         <artifactId>linked-data-interactions</artifactId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

ldi-core/file-archiving/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>file-archiving</artifactId>

ldi-core/geojson-to-wkt/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>geojson-to-wkt</artifactId>

ldi-core/json-to-ld-adapter/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>json-to-ld-adapter</artifactId>

ldi-core/ldes-client/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

ldi-core/ldes-client/starting-node/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>ldes-client</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.client</groupId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>starting-node</artifactId>

ldi-core/ldes-client/tree-node-fetcher/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>ldes-client</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.client</groupId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>tree-node-fetcher</artifactId>

ldi-core/ldes-client/tree-node-supplier/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>ldes-client</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.client</groupId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>tree-node-supplier</artifactId>

ldi-core/ldi-common/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>ldi-common</artifactId>

ldi-core/ngsiv2-to-ld-adapter/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>ngsiv2-to-ld-adapter</artifactId>

ldi-core/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>linked-data-interactions</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.ldes</groupId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

ldi-core/rdf-adapter/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>ldi-core</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

ldi-core/repository-materialiser/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
 
     <properties>

ldi-core/request-executor/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>ldi-core</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>request-executor</artifactId>

ldi-core/request-executor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/ratelimiter/RateLimiterConfig.java

@@ -2,34 +2,49 @@
 
 import io.github.resilience4j.ratelimiter.RateLimiter;
 import io.github.resilience4j.ratelimiter.RateLimiterRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.time.Duration;
+import java.time.format.DateTimeParseException;
 
 public class RateLimiterConfig {
+    private static final String RATE_LIMIT_PER_MINUTE_MIGRATION_WARNING = "'rate-limit.max-requests-per-minute' property is deprecated. Please consider migrating to the more generic properties 'rate-limit.limit' and 'rate-limit.period'";
+    private static final String INVALID_PERIOD_ERROR = "Invalid config for the property 'rate-limiter.period': this must be a valid 8601 duration";
+    private static final Logger log = LoggerFactory.getLogger(RateLimiterConfig.class);
 
 	private final int limitForPeriod;
 	private final Duration limitRefreshPeriod;
 	private final Duration timeoutDuration;
 
-	public RateLimiterConfig(int limitForPeriod, Duration limitRefreshPeriod, Duration timeoutDuration) {
+	private RateLimiterConfig(int limitForPeriod, Duration limitRefreshPeriod, Duration timeoutDuration) {
 		this.limitForPeriod = limitForPeriod;
 		this.limitRefreshPeriod = limitRefreshPeriod;
 		this.timeoutDuration = timeoutDuration;
 	}
 
-	public static RateLimiterConfig limitPerMinute(int maxRequestsPerMinute) {
-		return new RateLimiterConfig(maxRequestsPerMinute, Duration.ofMinutes(1), Duration.ofMinutes(1));
-	}
-
-	public RateLimiter getRateLimiter() {
-		return RateLimiterRegistry.of(
-				io.github.resilience4j.ratelimiter.RateLimiterConfig
-						.custom()
-						.limitForPeriod(limitForPeriod)
-						.limitRefreshPeriod(limitRefreshPeriod)
-						.timeoutDuration(timeoutDuration)
-						.build())
-				.rateLimiter("rate-limit-http-requests");
-	}
+    public static RateLimiterConfig limitPerMinute(int maxRequestsPerMinute) {
+        log.warn(RATE_LIMIT_PER_MINUTE_MIGRATION_WARNING);
+        return new RateLimiterConfig(maxRequestsPerMinute, Duration.ofMinutes(1), Duration.ofMinutes(1));
+    }
+
+    public static RateLimiterConfig limitForPeriod(int limit, String period) {
+        try {
+            return new RateLimiterConfig(limit, Duration.parse(period), Duration.parse(period));
+        } catch (DateTimeParseException exception) {
+            throw new IllegalArgumentException(INVALID_PERIOD_ERROR);
+        }
+    }
+
+    public RateLimiter getRateLimiter() {
+        return RateLimiterRegistry.of(
+                        io.github.resilience4j.ratelimiter.RateLimiterConfig
+                                .custom()
+                                .limitForPeriod(limitForPeriod)
+                                .limitRefreshPeriod(limitRefreshPeriod)
+                                .timeoutDuration(timeoutDuration)
+                                .build())
+                .rateLimiter("rate-limit-http-requests");
+    }
 
 }

ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/RequestExecutorSteps.java

@@ -130,8 +130,7 @@ public void iMockToFailTheFirstTimeAndSucceedTheSecondTime(String url) {
 
 	@Given("I have a requestExecutor which limits the requests to 1 per second")
 	public void iHaveARequestExecutorWithRateLimiter() {
-		Duration waitTime = Duration.ofSeconds(1);
-		RateLimiter rateLimiter = new RateLimiterConfig(1, waitTime, waitTime).getRateLimiter();
+		RateLimiter rateLimiter = RateLimiterConfig.limitForPeriod(1, "PT1S").getRateLimiter();
 		requestExecutor = RequestExecutorDecorator.decorate(factory.createNoAuthExecutor()).with(rateLimiter).get();
 	}
 
@@ -150,8 +149,7 @@ public void itTakesSecondsToExecuteTheRequestTimes(int ms, int requestCount) {
 	@Given("I have a requestExecutor which does {int} retries with custom http status code {int} and limits requests")
 	public void iHaveARequestExecutorWhichDoesRetriesWithCustomHttpStatusCodeAndLimitsRequests(int retryCount,
 			int httpStatus) {
-		Duration waitTime = Duration.ofSeconds(1);
-		RateLimiter rateLimiter = new RateLimiterConfig(1, waitTime, waitTime).getRateLimiter();
+		RateLimiter rateLimiter = RateLimiterConfig.limitForPeriod(1, "PT1S").getRateLimiter();
 		Retry retry = RetryConfig.of(retryCount, List.of(httpStatus)).getRetry();
 		requestExecutor = RequestExecutorDecorator.decorate(factory.createNoAuthExecutor()).with(retry)
 				.with(rateLimiter).get();

ldi-core/request-executor/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/requestexecutor/executor/ratelimiter/RateLimiterConfigTest.java

@@ -0,0 +1,40 @@
+package be.vlaanderen.informatievlaanderen.ldes.ldi.requestexecutor.executor.ratelimiter;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class RateLimiterConfigTest {
+    private static final int LIMIT = 500;
+
+    @Test
+    void test_LimitPerMinute() {
+        final io.github.resilience4j.ratelimiter.RateLimiterConfig rateLimiterConfig = RateLimiterConfig.limitPerMinute(LIMIT).getRateLimiter().getRateLimiterConfig();
+
+        assertThat(rateLimiterConfig.getLimitForPeriod()).isEqualTo(LIMIT);
+        assertThat(rateLimiterConfig.getLimitRefreshPeriod()).isEqualTo(Duration.ofMinutes(1));
+    }
+
+    @Test
+    void given_ValidPeriodAndLimit_when_LimitForPeriod_then_ReturnRateLimiterConfig() {
+        final String periodString = "PT1M";
+
+        final io.github.resilience4j.ratelimiter.RateLimiterConfig rateLimiterConfig = RateLimiterConfig.limitForPeriod(LIMIT, periodString).getRateLimiter().getRateLimiterConfig();
+
+        assertThat(rateLimiterConfig.getLimitForPeriod()).isEqualTo(LIMIT);
+        assertThat(rateLimiterConfig.getLimitRefreshPeriod()).isEqualTo(Duration.parse(periodString));
+        assertThat(rateLimiterConfig.getTimeoutDuration()).isEqualTo(Duration.parse(periodString));
+    }
+
+    @Test
+    void given_InvalidPeriod_when_LimitForPeriod_then_ThrowException() {
+        String periodString = "TP1J";
+
+        assertThatThrownBy(() -> RateLimiterConfig.limitForPeriod(LIMIT, periodString))
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessage("Invalid config for the property 'rate-limiter.period': this must be a valid 8601 duration");
+    }
+}

ldi-core/rml-adapter/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>ldi-core</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

ldi-core/sparql-construct/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-core</artifactId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

ldi-core/version-materialiser/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>ldi-core</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

ldi-core/version-object-creator/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>ldi-core</artifactId>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

ldi-nifi/ldi-nifi-common/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
         <artifactId>ldi-nifi</artifactId>
-        <version>1.13.0</version>
+        <version>1.14.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>